blob: 322871920772aeab0682883bdddd185c7b321e73 [file] [log] [blame]
Simon Glass0d24de92012-01-14 15:12:45 +00001# Copyright (c) 2011 The Chromium OS Authors.
2#
Wolfgang Denk1a459662013-07-08 09:37:19 +02003# SPDX-License-Identifier: GPL-2.0+
Simon Glass0d24de92012-01-14 15:12:45 +00004#
5
6import os
7import re
8import shutil
9import tempfile
10
11import command
12import commit
13import gitutil
14from series import Series
15
16# Tags that we detect and remove
Simon Glass619dd5d2012-11-02 06:38:43 +000017re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:'
Simon Glass3fefd5e2013-04-03 11:01:39 +000018 '|Reviewed-on:|Commit-\w*:')
Simon Glass0d24de92012-01-14 15:12:45 +000019
20# Lines which are allowed after a TEST= line
21re_allowed_after_test = re.compile('^Signed-off-by:')
22
Ilya Yanok05e5b732012-08-06 23:46:05 +000023# Signoffs
Simon Glass102061b2014-04-20 10:50:14 -060024re_signoff = re.compile('^Signed-off-by: *(.*)')
Ilya Yanok05e5b732012-08-06 23:46:05 +000025
Simon Glass0d24de92012-01-14 15:12:45 +000026# The start of the cover letter
27re_cover = re.compile('^Cover-letter:')
28
Simon Glassfe2f8d92013-03-20 16:43:00 +000029# A cover letter Cc
30re_cover_cc = re.compile('^Cover-letter-cc: *(.*)')
31
Simon Glass0d24de92012-01-14 15:12:45 +000032# Patch series tag
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +010033re_series_tag = re.compile('^Series-([a-z-]*): *(.*)')
34
35# Commit series tag
36re_commit_tag = re.compile('^Commit-([a-z-]*): *(.*)')
Simon Glass0d24de92012-01-14 15:12:45 +000037
38# Commit tags that we want to collect and keep
Simon Glass659c89d2014-02-16 08:23:47 -070039re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc): (.*)')
Simon Glass0d24de92012-01-14 15:12:45 +000040
41# The start of a new commit in the git log
Doug Anderson68618282013-03-01 11:11:07 +000042re_commit = re.compile('^commit ([0-9a-f]*)$')
Simon Glass0d24de92012-01-14 15:12:45 +000043
44# We detect these since checkpatch doesn't always do it
45re_space_before_tab = re.compile('^[+].* \t')
46
47# States we can be in - can we use range() and still have comments?
48STATE_MSG_HEADER = 0 # Still in the message header
49STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit)
50STATE_PATCH_HEADER = 2 # In patch header (after the subject)
51STATE_DIFFS = 3 # In the diff part (past --- line)
52
53class PatchStream:
54 """Class for detecting/injecting tags in a patch or series of patches
55
56 We support processing the output of 'git log' to read out the tags we
57 are interested in. We can also process a patch file in order to remove
58 unwanted tags or inject additional ones. These correspond to the two
59 phases of processing.
60 """
61 def __init__(self, series, name=None, is_log=False):
62 self.skip_blank = False # True to skip a single blank line
63 self.found_test = False # Found a TEST= line
64 self.lines_after_test = 0 # MNumber of lines found after TEST=
65 self.warn = [] # List of warnings we have collected
66 self.linenum = 1 # Output line number we are up to
67 self.in_section = None # Name of start...END section we are in
68 self.notes = [] # Series notes
69 self.section = [] # The current section...END section
70 self.series = series # Info about the patch series
71 self.is_log = is_log # True if indent like git log
72 self.in_change = 0 # Non-zero if we are in a change list
73 self.blank_count = 0 # Number of blank lines stored up
74 self.state = STATE_MSG_HEADER # What state are we in?
75 self.tags = [] # Tags collected, like Tested-by...
76 self.signoff = [] # Contents of signoff line
77 self.commit = None # Current commit
78
79 def AddToSeries(self, line, name, value):
80 """Add a new Series-xxx tag.
81
82 When a Series-xxx tag is detected, we come here to record it, if we
83 are scanning a 'git log'.
84
85 Args:
86 line: Source line containing tag (useful for debug/error messages)
87 name: Tag name (part after 'Series-')
88 value: Tag value (part after 'Series-xxx: ')
89 """
90 if name == 'notes':
91 self.in_section = name
92 self.skip_blank = False
93 if self.is_log:
94 self.series.AddTag(self.commit, line, name, value)
95
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +010096 def AddToCommit(self, line, name, value):
97 """Add a new Commit-xxx tag.
98
99 When a Commit-xxx tag is detected, we come here to record it.
100
101 Args:
102 line: Source line containing tag (useful for debug/error messages)
103 name: Tag name (part after 'Commit-')
104 value: Tag value (part after 'Commit-xxx: ')
105 """
106 if name == 'notes':
107 self.in_section = 'commit-' + name
108 self.skip_blank = False
109
Simon Glass0d24de92012-01-14 15:12:45 +0000110 def CloseCommit(self):
111 """Save the current commit into our commit list, and reset our state"""
112 if self.commit and self.is_log:
113 self.series.AddCommit(self.commit)
114 self.commit = None
115
116 def FormatTags(self, tags):
117 out_list = []
118 for tag in sorted(tags):
119 if tag.startswith('Cc:'):
120 tag_list = tag[4:].split(',')
121 out_list += gitutil.BuildEmailList(tag_list, 'Cc:')
122 else:
123 out_list.append(tag)
124 return out_list
125
126 def ProcessLine(self, line):
127 """Process a single line of a patch file or commit log
128
129 This process a line and returns a list of lines to output. The list
130 may be empty or may contain multiple output lines.
131
132 This is where all the complicated logic is located. The class's
133 state is used to move between different states and detect things
134 properly.
135
136 We can be in one of two modes:
137 self.is_log == True: This is 'git log' mode, where most output is
138 indented by 4 characters and we are scanning for tags
139
140 self.is_log == False: This is 'patch' mode, where we already have
141 all the tags, and are processing patches to remove junk we
142 don't want, and add things we think are required.
143
144 Args:
145 line: text line to process
146
147 Returns:
148 list of output lines, or [] if nothing should be output
149 """
150 # Initially we have no output. Prepare the input line string
151 out = []
152 line = line.rstrip('\n')
153 if self.is_log:
154 if line[:4] == ' ':
155 line = line[4:]
156
157 # Handle state transition and skipping blank lines
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100158 series_tag_match = re_series_tag.match(line)
159 commit_tag_match = re_commit_tag.match(line)
Simon Glass0d24de92012-01-14 15:12:45 +0000160 commit_match = re_commit.match(line) if self.is_log else None
Simon Glassfe2f8d92013-03-20 16:43:00 +0000161 cover_cc_match = re_cover_cc.match(line)
Simon Glass102061b2014-04-20 10:50:14 -0600162 signoff_match = re_signoff.match(line)
Simon Glass0d24de92012-01-14 15:12:45 +0000163 tag_match = None
164 if self.state == STATE_PATCH_HEADER:
165 tag_match = re_tag.match(line)
166 is_blank = not line.strip()
167 if is_blank:
168 if (self.state == STATE_MSG_HEADER
169 or self.state == STATE_PATCH_SUBJECT):
170 self.state += 1
171
172 # We don't have a subject in the text stream of patch files
173 # It has its own line with a Subject: tag
174 if not self.is_log and self.state == STATE_PATCH_SUBJECT:
175 self.state += 1
176 elif commit_match:
177 self.state = STATE_MSG_HEADER
178
179 # If we are in a section, keep collecting lines until we see END
180 if self.in_section:
181 if line == 'END':
182 if self.in_section == 'cover':
183 self.series.cover = self.section
184 elif self.in_section == 'notes':
185 if self.is_log:
186 self.series.notes += self.section
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100187 elif self.in_section == 'commit-notes':
188 if self.is_log:
189 self.commit.notes += self.section
Simon Glass0d24de92012-01-14 15:12:45 +0000190 else:
191 self.warn.append("Unknown section '%s'" % self.in_section)
192 self.in_section = None
193 self.skip_blank = True
194 self.section = []
195 else:
196 self.section.append(line)
197
198 # Detect the commit subject
199 elif not is_blank and self.state == STATE_PATCH_SUBJECT:
200 self.commit.subject = line
201
202 # Detect the tags we want to remove, and skip blank lines
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100203 elif re_remove.match(line) and not commit_tag_match:
Simon Glass0d24de92012-01-14 15:12:45 +0000204 self.skip_blank = True
205
206 # TEST= should be the last thing in the commit, so remove
207 # everything after it
208 if line.startswith('TEST='):
209 self.found_test = True
210 elif self.skip_blank and is_blank:
211 self.skip_blank = False
212
213 # Detect the start of a cover letter section
214 elif re_cover.match(line):
215 self.in_section = 'cover'
216 self.skip_blank = False
217
Simon Glassfe2f8d92013-03-20 16:43:00 +0000218 elif cover_cc_match:
219 value = cover_cc_match.group(1)
220 self.AddToSeries(line, 'cover-cc', value)
221
Simon Glass0d24de92012-01-14 15:12:45 +0000222 # If we are in a change list, key collected lines until a blank one
223 elif self.in_change:
224 if is_blank:
225 # Blank line ends this change list
226 self.in_change = 0
Simon Glass102061b2014-04-20 10:50:14 -0600227 elif line == '---':
Ilya Yanok05e5b732012-08-06 23:46:05 +0000228 self.in_change = 0
229 out = self.ProcessLine(line)
Simon Glass0d24de92012-01-14 15:12:45 +0000230 else:
Ilya Yanoka8840cb2012-08-06 23:46:06 +0000231 if self.is_log:
232 self.series.AddChange(self.in_change, self.commit, line)
Simon Glass0d24de92012-01-14 15:12:45 +0000233 self.skip_blank = False
234
235 # Detect Series-xxx tags
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100236 elif series_tag_match:
237 name = series_tag_match.group(1)
238 value = series_tag_match.group(2)
Simon Glass0d24de92012-01-14 15:12:45 +0000239 if name == 'changes':
240 # value is the version number: e.g. 1, or 2
241 try:
242 value = int(value)
243 except ValueError as str:
244 raise ValueError("%s: Cannot decode version info '%s'" %
245 (self.commit.hash, line))
246 self.in_change = int(value)
247 else:
248 self.AddToSeries(line, name, value)
249 self.skip_blank = True
250
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100251 # Detect Commit-xxx tags
252 elif commit_tag_match:
253 name = commit_tag_match.group(1)
254 value = commit_tag_match.group(2)
255 if name == 'notes':
256 self.AddToCommit(line, name, value)
257 self.skip_blank = True
258
Simon Glass0d24de92012-01-14 15:12:45 +0000259 # Detect the start of a new commit
260 elif commit_match:
261 self.CloseCommit()
Simon Glasse62f9052012-12-15 10:42:06 +0000262 # TODO: We should store the whole hash, and just display a subset
263 self.commit = commit.Commit(commit_match.group(1)[:8])
Simon Glass0d24de92012-01-14 15:12:45 +0000264
265 # Detect tags in the commit message
266 elif tag_match:
Simon Glass0d24de92012-01-14 15:12:45 +0000267 # Remove Tested-by self, since few will take much notice
Ilya Yanokc7379142012-08-06 23:46:08 +0000268 if (tag_match.group(1) == 'Tested-by' and
Simon Glass0d24de92012-01-14 15:12:45 +0000269 tag_match.group(2).find(os.getenv('USER') + '@') != -1):
270 self.warn.append("Ignoring %s" % line)
Simon Glass659c89d2014-02-16 08:23:47 -0700271 elif tag_match.group(1) == 'Patch-cc':
Simon Glass0d24de92012-01-14 15:12:45 +0000272 self.commit.AddCc(tag_match.group(2).split(','))
273 else:
274 self.tags.append(line);
275
Simon Glass102061b2014-04-20 10:50:14 -0600276 # Suppress duplicate signoffs
277 elif signoff_match:
Simon Glass6be6b6b2014-05-13 12:14:02 -0600278 if (self.is_log or
279 self.commit.CheckDuplicateSignoff(signoff_match.group(1))):
Simon Glass102061b2014-04-20 10:50:14 -0600280 out = [line]
281
Simon Glass0d24de92012-01-14 15:12:45 +0000282 # Well that means this is an ordinary line
283 else:
284 pos = 1
285 # Look for ugly ASCII characters
286 for ch in line:
287 # TODO: Would be nicer to report source filename and line
288 if ord(ch) > 0x80:
289 self.warn.append("Line %d/%d ('%s') has funny ascii char" %
290 (self.linenum, pos, line))
291 pos += 1
292
293 # Look for space before tab
294 m = re_space_before_tab.match(line)
295 if m:
296 self.warn.append('Line %d/%d has space before tab' %
297 (self.linenum, m.start()))
298
299 # OK, we have a valid non-blank line
300 out = [line]
301 self.linenum += 1
302 self.skip_blank = False
303 if self.state == STATE_DIFFS:
304 pass
305
306 # If this is the start of the diffs section, emit our tags and
307 # change log
308 elif line == '---':
309 self.state = STATE_DIFFS
310
311 # Output the tags (signeoff first), then change list
312 out = []
Simon Glass0d24de92012-01-14 15:12:45 +0000313 log = self.series.MakeChangeLog(self.commit)
314 out += self.FormatTags(self.tags)
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100315 out += [line] + self.commit.notes + [''] + log
Simon Glass0d24de92012-01-14 15:12:45 +0000316 elif self.found_test:
317 if not re_allowed_after_test.match(line):
318 self.lines_after_test += 1
319
320 return out
321
322 def Finalize(self):
323 """Close out processing of this patch stream"""
324 self.CloseCommit()
325 if self.lines_after_test:
326 self.warn.append('Found %d lines after TEST=' %
327 self.lines_after_test)
328
329 def ProcessStream(self, infd, outfd):
330 """Copy a stream from infd to outfd, filtering out unwanting things.
331
332 This is used to process patch files one at a time.
333
334 Args:
335 infd: Input stream file object
336 outfd: Output stream file object
337 """
338 # Extract the filename from each diff, for nice warnings
339 fname = None
340 last_fname = None
341 re_fname = re.compile('diff --git a/(.*) b/.*')
342 while True:
343 line = infd.readline()
344 if not line:
345 break
346 out = self.ProcessLine(line)
347
348 # Try to detect blank lines at EOF
349 for line in out:
350 match = re_fname.match(line)
351 if match:
352 last_fname = fname
353 fname = match.group(1)
354 if line == '+':
355 self.blank_count += 1
356 else:
357 if self.blank_count and (line == '-- ' or match):
358 self.warn.append("Found possible blank line(s) at "
359 "end of file '%s'" % last_fname)
360 outfd.write('+\n' * self.blank_count)
361 outfd.write(line + '\n')
362 self.blank_count = 0
363 self.Finalize()
364
365
Simon Glasse62f9052012-12-15 10:42:06 +0000366def GetMetaDataForList(commit_range, git_dir=None, count=None,
367 series = Series()):
368 """Reads out patch series metadata from the commits
369
370 This does a 'git log' on the relevant commits and pulls out the tags we
371 are interested in.
372
373 Args:
374 commit_range: Range of commits to count (e.g. 'HEAD..base')
375 git_dir: Path to git repositiory (None to use default)
376 count: Number of commits to list, or None for no limit
377 series: Series object to add information into. By default a new series
378 is started.
379 Returns:
380 A Series object containing information about the commits.
381 """
Andreas Bießmann23860602013-04-15 23:52:18 +0000382 params = ['git', 'log', '--no-color', '--reverse', '--no-decorate',
383 commit_range]
Simon Glasse62f9052012-12-15 10:42:06 +0000384 if count is not None:
385 params[2:2] = ['-n%d' % count]
386 if git_dir:
387 params[1:1] = ['--git-dir', git_dir]
388 pipe = [params]
389 stdout = command.RunPipe(pipe, capture=True).stdout
390 ps = PatchStream(series, is_log=True)
391 for line in stdout.splitlines():
392 ps.ProcessLine(line)
393 ps.Finalize()
394 return series
395
Simon Glass0d24de92012-01-14 15:12:45 +0000396def GetMetaData(start, count):
397 """Reads out patch series metadata from the commits
398
399 This does a 'git log' on the relevant commits and pulls out the tags we
400 are interested in.
401
402 Args:
403 start: Commit to start from: 0=HEAD, 1=next one, etc.
404 count: Number of commits to list
405 """
Simon Glasse62f9052012-12-15 10:42:06 +0000406 return GetMetaDataForList('HEAD~%d' % start, None, count)
Simon Glass0d24de92012-01-14 15:12:45 +0000407
408def FixPatch(backup_dir, fname, series, commit):
409 """Fix up a patch file, by adding/removing as required.
410
411 We remove our tags from the patch file, insert changes lists, etc.
412 The patch file is processed in place, and overwritten.
413
414 A backup file is put into backup_dir (if not None).
415
416 Args:
417 fname: Filename to patch file to process
418 series: Series information about this patch set
419 commit: Commit object for this patch file
420 Return:
421 A list of errors, or [] if all ok.
422 """
423 handle, tmpname = tempfile.mkstemp()
424 outfd = os.fdopen(handle, 'w')
425 infd = open(fname, 'r')
426 ps = PatchStream(series)
427 ps.commit = commit
428 ps.ProcessStream(infd, outfd)
429 infd.close()
430 outfd.close()
431
432 # Create a backup file if required
433 if backup_dir:
434 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
435 shutil.move(tmpname, fname)
436 return ps.warn
437
438def FixPatches(series, fnames):
439 """Fix up a list of patches identified by filenames
440
441 The patch files are processed in place, and overwritten.
442
443 Args:
444 series: The series object
445 fnames: List of patch files to process
446 """
447 # Current workflow creates patches, so we shouldn't need a backup
448 backup_dir = None #tempfile.mkdtemp('clean-patch')
449 count = 0
450 for fname in fnames:
451 commit = series.commits[count]
452 commit.patch = fname
453 result = FixPatch(backup_dir, fname, series, commit)
454 if result:
455 print '%d warnings for %s:' % (len(result), fname)
456 for warn in result:
457 print '\t', warn
458 print
459 count += 1
460 print 'Cleaned %d patches' % count
461 return series
462
463def InsertCoverLetter(fname, series, count):
464 """Inserts a cover letter with the required info into patch 0
465
466 Args:
467 fname: Input / output filename of the cover letter file
468 series: Series object
469 count: Number of patches in the series
470 """
471 fd = open(fname, 'r')
472 lines = fd.readlines()
473 fd.close()
474
475 fd = open(fname, 'w')
476 text = series.cover
477 prefix = series.GetPatchPrefix()
478 for line in lines:
479 if line.startswith('Subject:'):
480 # TODO: if more than 10 patches this should save 00/xx, not 0/xx
481 line = 'Subject: [%s 0/%d] %s\n' % (prefix, count, text[0])
482
483 # Insert our cover letter
484 elif line.startswith('*** BLURB HERE ***'):
485 # First the blurb test
486 line = '\n'.join(text[1:]) + '\n'
487 if series.get('notes'):
488 line += '\n'.join(series.notes) + '\n'
489
490 # Now the change list
491 out = series.MakeChangeLog(None)
492 line += '\n' + '\n'.join(out)
493 fd.write(line)
494 fd.close()