blob: 1b9136aa5c2eaff60f20c5c425b87353ae998efc [file] [log] [blame]
Simon Glass0d24de92012-01-14 15:12:45 +00001# Copyright (c) 2011 The Chromium OS Authors.
2#
Wolfgang Denk1a459662013-07-08 09:37:19 +02003# SPDX-License-Identifier: GPL-2.0+
Simon Glass0d24de92012-01-14 15:12:45 +00004#
5
Wu, Josh35ce2dc2015-04-03 10:51:17 +08006import math
Simon Glass0d24de92012-01-14 15:12:45 +00007import os
8import re
9import shutil
10import tempfile
11
12import command
13import commit
14import gitutil
15from series import Series
16
17# Tags that we detect and remove
Simon Glass619dd5d2012-11-02 06:38:43 +000018re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:'
Simon Glass3fefd5e2013-04-03 11:01:39 +000019 '|Reviewed-on:|Commit-\w*:')
Simon Glass0d24de92012-01-14 15:12:45 +000020
21# Lines which are allowed after a TEST= line
22re_allowed_after_test = re.compile('^Signed-off-by:')
23
Ilya Yanok05e5b732012-08-06 23:46:05 +000024# Signoffs
Simon Glass102061b2014-04-20 10:50:14 -060025re_signoff = re.compile('^Signed-off-by: *(.*)')
Ilya Yanok05e5b732012-08-06 23:46:05 +000026
Simon Glass0d24de92012-01-14 15:12:45 +000027# The start of the cover letter
28re_cover = re.compile('^Cover-letter:')
29
Simon Glassfe2f8d92013-03-20 16:43:00 +000030# A cover letter Cc
31re_cover_cc = re.compile('^Cover-letter-cc: *(.*)')
32
Simon Glass0d24de92012-01-14 15:12:45 +000033# Patch series tag
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +010034re_series_tag = re.compile('^Series-([a-z-]*): *(.*)')
35
36# Commit series tag
37re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)')
Simon Glass0d24de92012-01-14 15:12:45 +000038
39# Commit tags that we want to collect and keep
Simon Glass659c89d2014-02-16 08:23:47 -070040re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)')
Simon Glass0d24de92012-01-14 15:12:45 +000041
42# The start of a new commit in the git log
Doug Anderson68618282013-03-01 11:11:07 +000043re_commit = re.compile('^commit ([0-9a-f]*)$')
Simon Glass0d24de92012-01-14 15:12:45 +000044
45# We detect these since checkpatch doesn't always do it
46re_space_before_tab = re.compile('^[+].* \t')
47
48# States we can be in - can we use range() and still have comments?
49STATE_MSG_HEADER = 0 # Still in the message header
50STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit)
51STATE_PATCH_HEADER = 2 # In patch header (after the subject)
52STATE_DIFFS = 3 # In the diff part (past --- line)
53
54class PatchStream:
55 """Class for detecting/injecting tags in a patch or series of patches
56
57 We support processing the output of 'git log' to read out the tags we
58 are interested in. We can also process a patch file in order to remove
59 unwanted tags or inject additional ones. These correspond to the two
60 phases of processing.
61 """
62 def __init__(self, series, name=None, is_log=False):
63 self.skip_blank = False # True to skip a single blank line
64 self.found_test = False # Found a TEST= line
65 self.lines_after_test = 0 # MNumber of lines found after TEST=
66 self.warn = [] # List of warnings we have collected
67 self.linenum = 1 # Output line number we are up to
68 self.in_section = None # Name of start...END section we are in
69 self.notes = [] # Series notes
70 self.section = [] # The current section...END section
71 self.series = series # Info about the patch series
72 self.is_log = is_log # True if indent like git log
73 self.in_change = 0 # Non-zero if we are in a change list
74 self.blank_count = 0 # Number of blank lines stored up
75 self.state = STATE_MSG_HEADER # What state are we in?
Simon Glass0d24de92012-01-14 15:12:45 +000076 self.signoff = [] # Contents of signoff line
77 self.commit = None # Current commit
78
79 def AddToSeries(self, line, name, value):
80 """Add a new Series-xxx tag.
81
82 When a Series-xxx tag is detected, we come here to record it, if we
83 are scanning a 'git log'.
84
85 Args:
86 line: Source line containing tag (useful for debug/error messages)
87 name: Tag name (part after 'Series-')
88 value: Tag value (part after 'Series-xxx: ')
89 """
90 if name == 'notes':
91 self.in_section = name
92 self.skip_blank = False
93 if self.is_log:
94 self.series.AddTag(self.commit, line, name, value)
95
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +010096 def AddToCommit(self, line, name, value):
97 """Add a new Commit-xxx tag.
98
99 When a Commit-xxx tag is detected, we come here to record it.
100
101 Args:
102 line: Source line containing tag (useful for debug/error messages)
103 name: Tag name (part after 'Commit-')
104 value: Tag value (part after 'Commit-xxx: ')
105 """
106 if name == 'notes':
107 self.in_section = 'commit-' + name
108 self.skip_blank = False
109
Simon Glass0d24de92012-01-14 15:12:45 +0000110 def CloseCommit(self):
111 """Save the current commit into our commit list, and reset our state"""
112 if self.commit and self.is_log:
113 self.series.AddCommit(self.commit)
114 self.commit = None
Bin Meng0d577182016-06-26 23:24:30 -0700115 # If 'END' is missing in a 'Cover-letter' section, and that section
116 # happens to show up at the very end of the commit message, this is
117 # the chance for us to fix it up.
118 if self.in_section == 'cover' and self.is_log:
119 self.series.cover = self.section
120 self.in_section = None
121 self.skip_blank = True
122 self.section = []
Simon Glass0d24de92012-01-14 15:12:45 +0000123
Simon Glass0d24de92012-01-14 15:12:45 +0000124 def ProcessLine(self, line):
125 """Process a single line of a patch file or commit log
126
127 This process a line and returns a list of lines to output. The list
128 may be empty or may contain multiple output lines.
129
130 This is where all the complicated logic is located. The class's
131 state is used to move between different states and detect things
132 properly.
133
134 We can be in one of two modes:
135 self.is_log == True: This is 'git log' mode, where most output is
136 indented by 4 characters and we are scanning for tags
137
138 self.is_log == False: This is 'patch' mode, where we already have
139 all the tags, and are processing patches to remove junk we
140 don't want, and add things we think are required.
141
142 Args:
143 line: text line to process
144
145 Returns:
146 list of output lines, or [] if nothing should be output
147 """
148 # Initially we have no output. Prepare the input line string
149 out = []
150 line = line.rstrip('\n')
Scott Wood4b89b812014-09-25 14:30:46 -0500151
152 commit_match = re_commit.match(line) if self.is_log else None
153
Simon Glass0d24de92012-01-14 15:12:45 +0000154 if self.is_log:
155 if line[:4] == ' ':
156 line = line[4:]
157
158 # Handle state transition and skipping blank lines
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100159 series_tag_match = re_series_tag.match(line)
160 commit_tag_match = re_commit_tag.match(line)
Bin Menge7df2182016-06-26 23:24:28 -0700161 cover_match = re_cover.match(line)
Simon Glassfe2f8d92013-03-20 16:43:00 +0000162 cover_cc_match = re_cover_cc.match(line)
Simon Glass102061b2014-04-20 10:50:14 -0600163 signoff_match = re_signoff.match(line)
Simon Glass0d24de92012-01-14 15:12:45 +0000164 tag_match = None
165 if self.state == STATE_PATCH_HEADER:
166 tag_match = re_tag.match(line)
167 is_blank = not line.strip()
168 if is_blank:
169 if (self.state == STATE_MSG_HEADER
170 or self.state == STATE_PATCH_SUBJECT):
171 self.state += 1
172
173 # We don't have a subject in the text stream of patch files
174 # It has its own line with a Subject: tag
175 if not self.is_log and self.state == STATE_PATCH_SUBJECT:
176 self.state += 1
177 elif commit_match:
178 self.state = STATE_MSG_HEADER
179
Bin Meng94fbd3e2016-06-26 23:24:32 -0700180 # If a tag is detected, or a new commit starts
Bin Meng13b98d92016-06-26 23:24:29 -0700181 if series_tag_match or commit_tag_match or \
Bin Meng94fbd3e2016-06-26 23:24:32 -0700182 cover_match or cover_cc_match or signoff_match or \
183 self.state == STATE_MSG_HEADER:
Bin Meng57b6b192016-06-26 23:24:31 -0700184 # but we are already in a section, this means 'END' is missing
185 # for that section, fix it up.
Bin Meng13b98d92016-06-26 23:24:29 -0700186 if self.in_section:
187 self.warn.append("Missing 'END' in section '%s'" % self.in_section)
188 if self.in_section == 'cover':
189 self.series.cover = self.section
190 elif self.in_section == 'notes':
191 if self.is_log:
192 self.series.notes += self.section
193 elif self.in_section == 'commit-notes':
194 if self.is_log:
195 self.commit.notes += self.section
196 else:
197 self.warn.append("Unknown section '%s'" % self.in_section)
198 self.in_section = None
199 self.skip_blank = True
200 self.section = []
Bin Meng57b6b192016-06-26 23:24:31 -0700201 # but we are already in a change list, that means a blank line
202 # is missing, fix it up.
203 if self.in_change:
204 self.warn.append("Missing 'blank line' in section 'Series-changes'")
205 self.in_change = 0
Bin Meng13b98d92016-06-26 23:24:29 -0700206
Simon Glass0d24de92012-01-14 15:12:45 +0000207 # If we are in a section, keep collecting lines until we see END
208 if self.in_section:
209 if line == 'END':
210 if self.in_section == 'cover':
211 self.series.cover = self.section
212 elif self.in_section == 'notes':
213 if self.is_log:
214 self.series.notes += self.section
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100215 elif self.in_section == 'commit-notes':
216 if self.is_log:
217 self.commit.notes += self.section
Simon Glass0d24de92012-01-14 15:12:45 +0000218 else:
219 self.warn.append("Unknown section '%s'" % self.in_section)
220 self.in_section = None
221 self.skip_blank = True
222 self.section = []
223 else:
224 self.section.append(line)
225
226 # Detect the commit subject
227 elif not is_blank and self.state == STATE_PATCH_SUBJECT:
228 self.commit.subject = line
229
230 # Detect the tags we want to remove, and skip blank lines
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100231 elif re_remove.match(line) and not commit_tag_match:
Simon Glass0d24de92012-01-14 15:12:45 +0000232 self.skip_blank = True
233
234 # TEST= should be the last thing in the commit, so remove
235 # everything after it
236 if line.startswith('TEST='):
237 self.found_test = True
238 elif self.skip_blank and is_blank:
239 self.skip_blank = False
240
241 # Detect the start of a cover letter section
Bin Menge7df2182016-06-26 23:24:28 -0700242 elif cover_match:
Simon Glass0d24de92012-01-14 15:12:45 +0000243 self.in_section = 'cover'
244 self.skip_blank = False
245
Simon Glassfe2f8d92013-03-20 16:43:00 +0000246 elif cover_cc_match:
247 value = cover_cc_match.group(1)
248 self.AddToSeries(line, 'cover-cc', value)
249
Simon Glass0d24de92012-01-14 15:12:45 +0000250 # If we are in a change list, key collected lines until a blank one
251 elif self.in_change:
252 if is_blank:
253 # Blank line ends this change list
254 self.in_change = 0
Simon Glass102061b2014-04-20 10:50:14 -0600255 elif line == '---':
Ilya Yanok05e5b732012-08-06 23:46:05 +0000256 self.in_change = 0
257 out = self.ProcessLine(line)
Simon Glass0d24de92012-01-14 15:12:45 +0000258 else:
Ilya Yanoka8840cb2012-08-06 23:46:06 +0000259 if self.is_log:
260 self.series.AddChange(self.in_change, self.commit, line)
Simon Glass0d24de92012-01-14 15:12:45 +0000261 self.skip_blank = False
262
263 # Detect Series-xxx tags
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100264 elif series_tag_match:
265 name = series_tag_match.group(1)
266 value = series_tag_match.group(2)
Simon Glass0d24de92012-01-14 15:12:45 +0000267 if name == 'changes':
268 # value is the version number: e.g. 1, or 2
269 try:
270 value = int(value)
271 except ValueError as str:
272 raise ValueError("%s: Cannot decode version info '%s'" %
273 (self.commit.hash, line))
274 self.in_change = int(value)
275 else:
276 self.AddToSeries(line, name, value)
277 self.skip_blank = True
278
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100279 # Detect Commit-xxx tags
280 elif commit_tag_match:
281 name = commit_tag_match.group(1)
282 value = commit_tag_match.group(2)
283 if name == 'notes':
284 self.AddToCommit(line, name, value)
285 self.skip_blank = True
286
Simon Glass0d24de92012-01-14 15:12:45 +0000287 # Detect the start of a new commit
288 elif commit_match:
289 self.CloseCommit()
Simon Glass0b5b4092014-10-15 02:27:00 -0600290 self.commit = commit.Commit(commit_match.group(1))
Simon Glass0d24de92012-01-14 15:12:45 +0000291
292 # Detect tags in the commit message
293 elif tag_match:
Simon Glass0d24de92012-01-14 15:12:45 +0000294 # Remove Tested-by self, since few will take much notice
Ilya Yanokc7379142012-08-06 23:46:08 +0000295 if (tag_match.group(1) == 'Tested-by' and
Simon Glass0d24de92012-01-14 15:12:45 +0000296 tag_match.group(2).find(os.getenv('USER') + '@') != -1):
297 self.warn.append("Ignoring %s" % line)
Simon Glass659c89d2014-02-16 08:23:47 -0700298 elif tag_match.group(1) == 'Patch-cc':
Simon Glass0d24de92012-01-14 15:12:45 +0000299 self.commit.AddCc(tag_match.group(2).split(','))
300 else:
Simon Glassd0c57192014-08-28 09:43:38 -0600301 out = [line]
Simon Glass0d24de92012-01-14 15:12:45 +0000302
Simon Glass102061b2014-04-20 10:50:14 -0600303 # Suppress duplicate signoffs
304 elif signoff_match:
Simon Glasse752edc2014-08-28 09:43:35 -0600305 if (self.is_log or not self.commit or
Simon Glass6be6b6b2014-05-13 12:14:02 -0600306 self.commit.CheckDuplicateSignoff(signoff_match.group(1))):
Simon Glass102061b2014-04-20 10:50:14 -0600307 out = [line]
308
Simon Glass0d24de92012-01-14 15:12:45 +0000309 # Well that means this is an ordinary line
310 else:
Simon Glass0d24de92012-01-14 15:12:45 +0000311 # Look for space before tab
312 m = re_space_before_tab.match(line)
313 if m:
314 self.warn.append('Line %d/%d has space before tab' %
315 (self.linenum, m.start()))
316
317 # OK, we have a valid non-blank line
318 out = [line]
319 self.linenum += 1
320 self.skip_blank = False
321 if self.state == STATE_DIFFS:
322 pass
323
324 # If this is the start of the diffs section, emit our tags and
325 # change log
326 elif line == '---':
327 self.state = STATE_DIFFS
328
329 # Output the tags (signeoff first), then change list
330 out = []
Simon Glass0d24de92012-01-14 15:12:45 +0000331 log = self.series.MakeChangeLog(self.commit)
Simon Glasse752edc2014-08-28 09:43:35 -0600332 out += [line]
333 if self.commit:
334 out += self.commit.notes
335 out += [''] + log
Simon Glass0d24de92012-01-14 15:12:45 +0000336 elif self.found_test:
337 if not re_allowed_after_test.match(line):
338 self.lines_after_test += 1
339
340 return out
341
342 def Finalize(self):
343 """Close out processing of this patch stream"""
344 self.CloseCommit()
345 if self.lines_after_test:
346 self.warn.append('Found %d lines after TEST=' %
347 self.lines_after_test)
348
349 def ProcessStream(self, infd, outfd):
350 """Copy a stream from infd to outfd, filtering out unwanting things.
351
352 This is used to process patch files one at a time.
353
354 Args:
355 infd: Input stream file object
356 outfd: Output stream file object
357 """
358 # Extract the filename from each diff, for nice warnings
359 fname = None
360 last_fname = None
361 re_fname = re.compile('diff --git a/(.*) b/.*')
362 while True:
363 line = infd.readline()
364 if not line:
365 break
366 out = self.ProcessLine(line)
367
368 # Try to detect blank lines at EOF
369 for line in out:
370 match = re_fname.match(line)
371 if match:
372 last_fname = fname
373 fname = match.group(1)
374 if line == '+':
375 self.blank_count += 1
376 else:
377 if self.blank_count and (line == '-- ' or match):
378 self.warn.append("Found possible blank line(s) at "
379 "end of file '%s'" % last_fname)
380 outfd.write('+\n' * self.blank_count)
381 outfd.write(line + '\n')
382 self.blank_count = 0
383 self.Finalize()
384
385
Simon Glasse62f9052012-12-15 10:42:06 +0000386def GetMetaDataForList(commit_range, git_dir=None, count=None,
Simon Glass950a2312014-09-05 19:00:23 -0600387 series = None, allow_overwrite=False):
Simon Glasse62f9052012-12-15 10:42:06 +0000388 """Reads out patch series metadata from the commits
389
390 This does a 'git log' on the relevant commits and pulls out the tags we
391 are interested in.
392
393 Args:
394 commit_range: Range of commits to count (e.g. 'HEAD..base')
395 git_dir: Path to git repositiory (None to use default)
396 count: Number of commits to list, or None for no limit
397 series: Series object to add information into. By default a new series
398 is started.
Simon Glass950a2312014-09-05 19:00:23 -0600399 allow_overwrite: Allow tags to overwrite an existing tag
Simon Glasse62f9052012-12-15 10:42:06 +0000400 Returns:
401 A Series object containing information about the commits.
402 """
Simon Glass891b7a02014-09-05 19:00:19 -0600403 if not series:
404 series = Series()
Simon Glass950a2312014-09-05 19:00:23 -0600405 series.allow_overwrite = allow_overwrite
Simon Glass9ad96982016-03-06 19:45:33 -0700406 params = gitutil.LogCmd(commit_range, reverse=True, count=count,
Simon Glasscda2a612014-08-09 15:33:10 -0600407 git_dir=git_dir)
408 stdout = command.RunPipe([params], capture=True).stdout
Simon Glasse62f9052012-12-15 10:42:06 +0000409 ps = PatchStream(series, is_log=True)
410 for line in stdout.splitlines():
411 ps.ProcessLine(line)
412 ps.Finalize()
413 return series
414
Simon Glass0d24de92012-01-14 15:12:45 +0000415def GetMetaData(start, count):
416 """Reads out patch series metadata from the commits
417
418 This does a 'git log' on the relevant commits and pulls out the tags we
419 are interested in.
420
421 Args:
422 start: Commit to start from: 0=HEAD, 1=next one, etc.
423 count: Number of commits to list
424 """
Simon Glasse62f9052012-12-15 10:42:06 +0000425 return GetMetaDataForList('HEAD~%d' % start, None, count)
Simon Glass0d24de92012-01-14 15:12:45 +0000426
Simon Glass6e87ae12017-05-29 15:31:31 -0600427def GetMetaDataForTest(text):
428 """Process metadata from a file containing a git log. Used for tests
429
430 Args:
431 text:
432 """
433 series = Series()
434 ps = PatchStream(series, is_log=True)
435 for line in text.splitlines():
436 ps.ProcessLine(line)
437 ps.Finalize()
438 return series
439
Simon Glass0d24de92012-01-14 15:12:45 +0000440def FixPatch(backup_dir, fname, series, commit):
441 """Fix up a patch file, by adding/removing as required.
442
443 We remove our tags from the patch file, insert changes lists, etc.
444 The patch file is processed in place, and overwritten.
445
446 A backup file is put into backup_dir (if not None).
447
448 Args:
449 fname: Filename to patch file to process
450 series: Series information about this patch set
451 commit: Commit object for this patch file
452 Return:
453 A list of errors, or [] if all ok.
454 """
455 handle, tmpname = tempfile.mkstemp()
456 outfd = os.fdopen(handle, 'w')
457 infd = open(fname, 'r')
458 ps = PatchStream(series)
459 ps.commit = commit
460 ps.ProcessStream(infd, outfd)
461 infd.close()
462 outfd.close()
463
464 # Create a backup file if required
465 if backup_dir:
466 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
467 shutil.move(tmpname, fname)
468 return ps.warn
469
470def FixPatches(series, fnames):
471 """Fix up a list of patches identified by filenames
472
473 The patch files are processed in place, and overwritten.
474
475 Args:
476 series: The series object
477 fnames: List of patch files to process
478 """
479 # Current workflow creates patches, so we shouldn't need a backup
480 backup_dir = None #tempfile.mkdtemp('clean-patch')
481 count = 0
482 for fname in fnames:
483 commit = series.commits[count]
484 commit.patch = fname
485 result = FixPatch(backup_dir, fname, series, commit)
486 if result:
Paul Burtona920a172016-09-27 16:03:50 +0100487 print('%d warnings for %s:' % (len(result), fname))
Simon Glass0d24de92012-01-14 15:12:45 +0000488 for warn in result:
Paul Burtona920a172016-09-27 16:03:50 +0100489 print('\t', warn)
Simon Glass0d24de92012-01-14 15:12:45 +0000490 print
491 count += 1
Paul Burtona920a172016-09-27 16:03:50 +0100492 print('Cleaned %d patches' % count)
Simon Glass0d24de92012-01-14 15:12:45 +0000493
494def InsertCoverLetter(fname, series, count):
495 """Inserts a cover letter with the required info into patch 0
496
497 Args:
498 fname: Input / output filename of the cover letter file
499 series: Series object
500 count: Number of patches in the series
501 """
502 fd = open(fname, 'r')
503 lines = fd.readlines()
504 fd.close()
505
506 fd = open(fname, 'w')
507 text = series.cover
508 prefix = series.GetPatchPrefix()
509 for line in lines:
510 if line.startswith('Subject:'):
Wu, Josh35ce2dc2015-04-03 10:51:17 +0800511 # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc
512 zero_repeat = int(math.log10(count)) + 1
513 zero = '0' * zero_repeat
514 line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0])
Simon Glass0d24de92012-01-14 15:12:45 +0000515
516 # Insert our cover letter
517 elif line.startswith('*** BLURB HERE ***'):
518 # First the blurb test
519 line = '\n'.join(text[1:]) + '\n'
520 if series.get('notes'):
521 line += '\n'.join(series.notes) + '\n'
522
523 # Now the change list
524 out = series.MakeChangeLog(None)
525 line += '\n' + '\n'.join(out)
526 fd.write(line)
527 fd.close()