blob: 27d031ef59c19472c800eec6f73ff4e3e81b7b22 [file] [log] [blame]
Simon Glass0d24de92012-01-14 15:12:45 +00001# Copyright (c) 2011 The Chromium OS Authors.
2#
Wolfgang Denk1a459662013-07-08 09:37:19 +02003# SPDX-License-Identifier: GPL-2.0+
Simon Glass0d24de92012-01-14 15:12:45 +00004#
5
Wu, Josh35ce2dc2015-04-03 10:51:17 +08006import math
Simon Glass0d24de92012-01-14 15:12:45 +00007import os
8import re
9import shutil
10import tempfile
11
12import command
13import commit
14import gitutil
15from series import Series
16
17# Tags that we detect and remove
Simon Glass619dd5d2012-11-02 06:38:43 +000018re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:'
Simon Glass3fefd5e2013-04-03 11:01:39 +000019 '|Reviewed-on:|Commit-\w*:')
Simon Glass0d24de92012-01-14 15:12:45 +000020
21# Lines which are allowed after a TEST= line
22re_allowed_after_test = re.compile('^Signed-off-by:')
23
Ilya Yanok05e5b732012-08-06 23:46:05 +000024# Signoffs
Simon Glass102061b2014-04-20 10:50:14 -060025re_signoff = re.compile('^Signed-off-by: *(.*)')
Ilya Yanok05e5b732012-08-06 23:46:05 +000026
Simon Glass0d24de92012-01-14 15:12:45 +000027# The start of the cover letter
28re_cover = re.compile('^Cover-letter:')
29
Simon Glassfe2f8d92013-03-20 16:43:00 +000030# A cover letter Cc
31re_cover_cc = re.compile('^Cover-letter-cc: *(.*)')
32
Simon Glass0d24de92012-01-14 15:12:45 +000033# Patch series tag
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +010034re_series_tag = re.compile('^Series-([a-z-]*): *(.*)')
35
36# Commit series tag
37re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)')
Simon Glass0d24de92012-01-14 15:12:45 +000038
39# Commit tags that we want to collect and keep
Simon Glass659c89d2014-02-16 08:23:47 -070040re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)')
Simon Glass0d24de92012-01-14 15:12:45 +000041
42# The start of a new commit in the git log
Doug Anderson68618282013-03-01 11:11:07 +000043re_commit = re.compile('^commit ([0-9a-f]*)$')
Simon Glass0d24de92012-01-14 15:12:45 +000044
45# We detect these since checkpatch doesn't always do it
46re_space_before_tab = re.compile('^[+].* \t')
47
48# States we can be in - can we use range() and still have comments?
49STATE_MSG_HEADER = 0 # Still in the message header
50STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit)
51STATE_PATCH_HEADER = 2 # In patch header (after the subject)
52STATE_DIFFS = 3 # In the diff part (past --- line)
53
54class PatchStream:
55 """Class for detecting/injecting tags in a patch or series of patches
56
57 We support processing the output of 'git log' to read out the tags we
58 are interested in. We can also process a patch file in order to remove
59 unwanted tags or inject additional ones. These correspond to the two
60 phases of processing.
61 """
62 def __init__(self, series, name=None, is_log=False):
63 self.skip_blank = False # True to skip a single blank line
64 self.found_test = False # Found a TEST= line
65 self.lines_after_test = 0 # MNumber of lines found after TEST=
66 self.warn = [] # List of warnings we have collected
67 self.linenum = 1 # Output line number we are up to
68 self.in_section = None # Name of start...END section we are in
69 self.notes = [] # Series notes
70 self.section = [] # The current section...END section
71 self.series = series # Info about the patch series
72 self.is_log = is_log # True if indent like git log
73 self.in_change = 0 # Non-zero if we are in a change list
74 self.blank_count = 0 # Number of blank lines stored up
75 self.state = STATE_MSG_HEADER # What state are we in?
Simon Glass0d24de92012-01-14 15:12:45 +000076 self.signoff = [] # Contents of signoff line
77 self.commit = None # Current commit
78
79 def AddToSeries(self, line, name, value):
80 """Add a new Series-xxx tag.
81
82 When a Series-xxx tag is detected, we come here to record it, if we
83 are scanning a 'git log'.
84
85 Args:
86 line: Source line containing tag (useful for debug/error messages)
87 name: Tag name (part after 'Series-')
88 value: Tag value (part after 'Series-xxx: ')
89 """
90 if name == 'notes':
91 self.in_section = name
92 self.skip_blank = False
93 if self.is_log:
94 self.series.AddTag(self.commit, line, name, value)
95
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +010096 def AddToCommit(self, line, name, value):
97 """Add a new Commit-xxx tag.
98
99 When a Commit-xxx tag is detected, we come here to record it.
100
101 Args:
102 line: Source line containing tag (useful for debug/error messages)
103 name: Tag name (part after 'Commit-')
104 value: Tag value (part after 'Commit-xxx: ')
105 """
106 if name == 'notes':
107 self.in_section = 'commit-' + name
108 self.skip_blank = False
109
Simon Glass0d24de92012-01-14 15:12:45 +0000110 def CloseCommit(self):
111 """Save the current commit into our commit list, and reset our state"""
112 if self.commit and self.is_log:
113 self.series.AddCommit(self.commit)
114 self.commit = None
115
Simon Glass0d24de92012-01-14 15:12:45 +0000116 def ProcessLine(self, line):
117 """Process a single line of a patch file or commit log
118
119 This process a line and returns a list of lines to output. The list
120 may be empty or may contain multiple output lines.
121
122 This is where all the complicated logic is located. The class's
123 state is used to move between different states and detect things
124 properly.
125
126 We can be in one of two modes:
127 self.is_log == True: This is 'git log' mode, where most output is
128 indented by 4 characters and we are scanning for tags
129
130 self.is_log == False: This is 'patch' mode, where we already have
131 all the tags, and are processing patches to remove junk we
132 don't want, and add things we think are required.
133
134 Args:
135 line: text line to process
136
137 Returns:
138 list of output lines, or [] if nothing should be output
139 """
140 # Initially we have no output. Prepare the input line string
141 out = []
142 line = line.rstrip('\n')
Scott Wood4b89b812014-09-25 14:30:46 -0500143
144 commit_match = re_commit.match(line) if self.is_log else None
145
Simon Glass0d24de92012-01-14 15:12:45 +0000146 if self.is_log:
147 if line[:4] == ' ':
148 line = line[4:]
149
150 # Handle state transition and skipping blank lines
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100151 series_tag_match = re_series_tag.match(line)
152 commit_tag_match = re_commit_tag.match(line)
Simon Glassfe2f8d92013-03-20 16:43:00 +0000153 cover_cc_match = re_cover_cc.match(line)
Simon Glass102061b2014-04-20 10:50:14 -0600154 signoff_match = re_signoff.match(line)
Simon Glass0d24de92012-01-14 15:12:45 +0000155 tag_match = None
156 if self.state == STATE_PATCH_HEADER:
157 tag_match = re_tag.match(line)
158 is_blank = not line.strip()
159 if is_blank:
160 if (self.state == STATE_MSG_HEADER
161 or self.state == STATE_PATCH_SUBJECT):
162 self.state += 1
163
164 # We don't have a subject in the text stream of patch files
165 # It has its own line with a Subject: tag
166 if not self.is_log and self.state == STATE_PATCH_SUBJECT:
167 self.state += 1
168 elif commit_match:
169 self.state = STATE_MSG_HEADER
170
171 # If we are in a section, keep collecting lines until we see END
172 if self.in_section:
173 if line == 'END':
174 if self.in_section == 'cover':
175 self.series.cover = self.section
176 elif self.in_section == 'notes':
177 if self.is_log:
178 self.series.notes += self.section
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100179 elif self.in_section == 'commit-notes':
180 if self.is_log:
181 self.commit.notes += self.section
Simon Glass0d24de92012-01-14 15:12:45 +0000182 else:
183 self.warn.append("Unknown section '%s'" % self.in_section)
184 self.in_section = None
185 self.skip_blank = True
186 self.section = []
187 else:
188 self.section.append(line)
189
190 # Detect the commit subject
191 elif not is_blank and self.state == STATE_PATCH_SUBJECT:
192 self.commit.subject = line
193
194 # Detect the tags we want to remove, and skip blank lines
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100195 elif re_remove.match(line) and not commit_tag_match:
Simon Glass0d24de92012-01-14 15:12:45 +0000196 self.skip_blank = True
197
198 # TEST= should be the last thing in the commit, so remove
199 # everything after it
200 if line.startswith('TEST='):
201 self.found_test = True
202 elif self.skip_blank and is_blank:
203 self.skip_blank = False
204
205 # Detect the start of a cover letter section
206 elif re_cover.match(line):
207 self.in_section = 'cover'
208 self.skip_blank = False
209
Simon Glassfe2f8d92013-03-20 16:43:00 +0000210 elif cover_cc_match:
211 value = cover_cc_match.group(1)
212 self.AddToSeries(line, 'cover-cc', value)
213
Simon Glass0d24de92012-01-14 15:12:45 +0000214 # If we are in a change list, key collected lines until a blank one
215 elif self.in_change:
216 if is_blank:
217 # Blank line ends this change list
218 self.in_change = 0
Simon Glass102061b2014-04-20 10:50:14 -0600219 elif line == '---':
Ilya Yanok05e5b732012-08-06 23:46:05 +0000220 self.in_change = 0
221 out = self.ProcessLine(line)
Simon Glass0d24de92012-01-14 15:12:45 +0000222 else:
Ilya Yanoka8840cb2012-08-06 23:46:06 +0000223 if self.is_log:
224 self.series.AddChange(self.in_change, self.commit, line)
Simon Glass0d24de92012-01-14 15:12:45 +0000225 self.skip_blank = False
226
227 # Detect Series-xxx tags
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100228 elif series_tag_match:
229 name = series_tag_match.group(1)
230 value = series_tag_match.group(2)
Simon Glass0d24de92012-01-14 15:12:45 +0000231 if name == 'changes':
232 # value is the version number: e.g. 1, or 2
233 try:
234 value = int(value)
235 except ValueError as str:
236 raise ValueError("%s: Cannot decode version info '%s'" %
237 (self.commit.hash, line))
238 self.in_change = int(value)
239 else:
240 self.AddToSeries(line, name, value)
241 self.skip_blank = True
242
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100243 # Detect Commit-xxx tags
244 elif commit_tag_match:
245 name = commit_tag_match.group(1)
246 value = commit_tag_match.group(2)
247 if name == 'notes':
248 self.AddToCommit(line, name, value)
249 self.skip_blank = True
250
Simon Glass0d24de92012-01-14 15:12:45 +0000251 # Detect the start of a new commit
252 elif commit_match:
253 self.CloseCommit()
Simon Glass0b5b4092014-10-15 02:27:00 -0600254 self.commit = commit.Commit(commit_match.group(1))
Simon Glass0d24de92012-01-14 15:12:45 +0000255
256 # Detect tags in the commit message
257 elif tag_match:
Simon Glass0d24de92012-01-14 15:12:45 +0000258 # Remove Tested-by self, since few will take much notice
Ilya Yanokc7379142012-08-06 23:46:08 +0000259 if (tag_match.group(1) == 'Tested-by' and
Simon Glass0d24de92012-01-14 15:12:45 +0000260 tag_match.group(2).find(os.getenv('USER') + '@') != -1):
261 self.warn.append("Ignoring %s" % line)
Simon Glass659c89d2014-02-16 08:23:47 -0700262 elif tag_match.group(1) == 'Patch-cc':
Simon Glass0d24de92012-01-14 15:12:45 +0000263 self.commit.AddCc(tag_match.group(2).split(','))
264 else:
Simon Glassd0c57192014-08-28 09:43:38 -0600265 out = [line]
Simon Glass0d24de92012-01-14 15:12:45 +0000266
Simon Glass102061b2014-04-20 10:50:14 -0600267 # Suppress duplicate signoffs
268 elif signoff_match:
Simon Glasse752edc2014-08-28 09:43:35 -0600269 if (self.is_log or not self.commit or
Simon Glass6be6b6b2014-05-13 12:14:02 -0600270 self.commit.CheckDuplicateSignoff(signoff_match.group(1))):
Simon Glass102061b2014-04-20 10:50:14 -0600271 out = [line]
272
Simon Glass0d24de92012-01-14 15:12:45 +0000273 # Well that means this is an ordinary line
274 else:
275 pos = 1
276 # Look for ugly ASCII characters
277 for ch in line:
278 # TODO: Would be nicer to report source filename and line
279 if ord(ch) > 0x80:
280 self.warn.append("Line %d/%d ('%s') has funny ascii char" %
281 (self.linenum, pos, line))
282 pos += 1
283
284 # Look for space before tab
285 m = re_space_before_tab.match(line)
286 if m:
287 self.warn.append('Line %d/%d has space before tab' %
288 (self.linenum, m.start()))
289
290 # OK, we have a valid non-blank line
291 out = [line]
292 self.linenum += 1
293 self.skip_blank = False
294 if self.state == STATE_DIFFS:
295 pass
296
297 # If this is the start of the diffs section, emit our tags and
298 # change log
299 elif line == '---':
300 self.state = STATE_DIFFS
301
302 # Output the tags (signeoff first), then change list
303 out = []
Simon Glass0d24de92012-01-14 15:12:45 +0000304 log = self.series.MakeChangeLog(self.commit)
Simon Glasse752edc2014-08-28 09:43:35 -0600305 out += [line]
306 if self.commit:
307 out += self.commit.notes
308 out += [''] + log
Simon Glass0d24de92012-01-14 15:12:45 +0000309 elif self.found_test:
310 if not re_allowed_after_test.match(line):
311 self.lines_after_test += 1
312
313 return out
314
315 def Finalize(self):
316 """Close out processing of this patch stream"""
317 self.CloseCommit()
318 if self.lines_after_test:
319 self.warn.append('Found %d lines after TEST=' %
320 self.lines_after_test)
321
322 def ProcessStream(self, infd, outfd):
323 """Copy a stream from infd to outfd, filtering out unwanting things.
324
325 This is used to process patch files one at a time.
326
327 Args:
328 infd: Input stream file object
329 outfd: Output stream file object
330 """
331 # Extract the filename from each diff, for nice warnings
332 fname = None
333 last_fname = None
334 re_fname = re.compile('diff --git a/(.*) b/.*')
335 while True:
336 line = infd.readline()
337 if not line:
338 break
339 out = self.ProcessLine(line)
340
341 # Try to detect blank lines at EOF
342 for line in out:
343 match = re_fname.match(line)
344 if match:
345 last_fname = fname
346 fname = match.group(1)
347 if line == '+':
348 self.blank_count += 1
349 else:
350 if self.blank_count and (line == '-- ' or match):
351 self.warn.append("Found possible blank line(s) at "
352 "end of file '%s'" % last_fname)
353 outfd.write('+\n' * self.blank_count)
354 outfd.write(line + '\n')
355 self.blank_count = 0
356 self.Finalize()
357
358
Simon Glasse62f9052012-12-15 10:42:06 +0000359def GetMetaDataForList(commit_range, git_dir=None, count=None,
Simon Glass950a2312014-09-05 19:00:23 -0600360 series = None, allow_overwrite=False):
Simon Glasse62f9052012-12-15 10:42:06 +0000361 """Reads out patch series metadata from the commits
362
363 This does a 'git log' on the relevant commits and pulls out the tags we
364 are interested in.
365
366 Args:
367 commit_range: Range of commits to count (e.g. 'HEAD..base')
368 git_dir: Path to git repositiory (None to use default)
369 count: Number of commits to list, or None for no limit
370 series: Series object to add information into. By default a new series
371 is started.
Simon Glass950a2312014-09-05 19:00:23 -0600372 allow_overwrite: Allow tags to overwrite an existing tag
Simon Glasse62f9052012-12-15 10:42:06 +0000373 Returns:
374 A Series object containing information about the commits.
375 """
Simon Glass891b7a02014-09-05 19:00:19 -0600376 if not series:
377 series = Series()
Simon Glass950a2312014-09-05 19:00:23 -0600378 series.allow_overwrite = allow_overwrite
Simon Glass9ad96982016-03-06 19:45:33 -0700379 params = gitutil.LogCmd(commit_range, reverse=True, count=count,
Simon Glasscda2a612014-08-09 15:33:10 -0600380 git_dir=git_dir)
381 stdout = command.RunPipe([params], capture=True).stdout
Simon Glasse62f9052012-12-15 10:42:06 +0000382 ps = PatchStream(series, is_log=True)
383 for line in stdout.splitlines():
384 ps.ProcessLine(line)
385 ps.Finalize()
386 return series
387
Simon Glass0d24de92012-01-14 15:12:45 +0000388def GetMetaData(start, count):
389 """Reads out patch series metadata from the commits
390
391 This does a 'git log' on the relevant commits and pulls out the tags we
392 are interested in.
393
394 Args:
395 start: Commit to start from: 0=HEAD, 1=next one, etc.
396 count: Number of commits to list
397 """
Simon Glasse62f9052012-12-15 10:42:06 +0000398 return GetMetaDataForList('HEAD~%d' % start, None, count)
Simon Glass0d24de92012-01-14 15:12:45 +0000399
400def FixPatch(backup_dir, fname, series, commit):
401 """Fix up a patch file, by adding/removing as required.
402
403 We remove our tags from the patch file, insert changes lists, etc.
404 The patch file is processed in place, and overwritten.
405
406 A backup file is put into backup_dir (if not None).
407
408 Args:
409 fname: Filename to patch file to process
410 series: Series information about this patch set
411 commit: Commit object for this patch file
412 Return:
413 A list of errors, or [] if all ok.
414 """
415 handle, tmpname = tempfile.mkstemp()
416 outfd = os.fdopen(handle, 'w')
417 infd = open(fname, 'r')
418 ps = PatchStream(series)
419 ps.commit = commit
420 ps.ProcessStream(infd, outfd)
421 infd.close()
422 outfd.close()
423
424 # Create a backup file if required
425 if backup_dir:
426 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
427 shutil.move(tmpname, fname)
428 return ps.warn
429
430def FixPatches(series, fnames):
431 """Fix up a list of patches identified by filenames
432
433 The patch files are processed in place, and overwritten.
434
435 Args:
436 series: The series object
437 fnames: List of patch files to process
438 """
439 # Current workflow creates patches, so we shouldn't need a backup
440 backup_dir = None #tempfile.mkdtemp('clean-patch')
441 count = 0
442 for fname in fnames:
443 commit = series.commits[count]
444 commit.patch = fname
445 result = FixPatch(backup_dir, fname, series, commit)
446 if result:
447 print '%d warnings for %s:' % (len(result), fname)
448 for warn in result:
449 print '\t', warn
450 print
451 count += 1
452 print 'Cleaned %d patches' % count
453 return series
454
455def InsertCoverLetter(fname, series, count):
456 """Inserts a cover letter with the required info into patch 0
457
458 Args:
459 fname: Input / output filename of the cover letter file
460 series: Series object
461 count: Number of patches in the series
462 """
463 fd = open(fname, 'r')
464 lines = fd.readlines()
465 fd.close()
466
467 fd = open(fname, 'w')
468 text = series.cover
469 prefix = series.GetPatchPrefix()
470 for line in lines:
471 if line.startswith('Subject:'):
Wu, Josh35ce2dc2015-04-03 10:51:17 +0800472 # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc
473 zero_repeat = int(math.log10(count)) + 1
474 zero = '0' * zero_repeat
475 line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0])
Simon Glass0d24de92012-01-14 15:12:45 +0000476
477 # Insert our cover letter
478 elif line.startswith('*** BLURB HERE ***'):
479 # First the blurb test
480 line = '\n'.join(text[1:]) + '\n'
481 if series.get('notes'):
482 line += '\n'.join(series.notes) + '\n'
483
484 # Now the change list
485 out = series.MakeChangeLog(None)
486 line += '\n' + '\n'.join(out)
487 fd.write(line)
488 fd.close()