blob: 5c2d3bc9b0cef923b8377320971e82c2e7d7c13e [file] [log] [blame]
Simon Glass0d24de92012-01-14 15:12:45 +00001# Copyright (c) 2011 The Chromium OS Authors.
2#
3# See file CREDITS for list of people who contributed to this
4# project.
5#
6# This program is free software; you can redistribute it and/or
7# modify it under the terms of the GNU General Public License as
8# published by the Free Software Foundation; either version 2 of
9# the License, or (at your option) any later version.
10#
11# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14# GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License
17# along with this program; if not, write to the Free Software
18# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
19# MA 02111-1307 USA
20#
21
22import os
23import re
24import shutil
25import tempfile
26
27import command
28import commit
29import gitutil
30from series import Series
31
32# Tags that we detect and remove
Simon Glass619dd5d2012-11-02 06:38:43 +000033re_remove = re.compile('^BUG=|^TEST=|^BRANCH=|^Change-Id:|^Review URL:'
Doug Anderson28b35942013-03-15 13:24:05 +000034 '|Reviewed-on:|Commit-Ready:')
Simon Glass0d24de92012-01-14 15:12:45 +000035
36# Lines which are allowed after a TEST= line
37re_allowed_after_test = re.compile('^Signed-off-by:')
38
Ilya Yanok05e5b732012-08-06 23:46:05 +000039# Signoffs
40re_signoff = re.compile('^Signed-off-by:')
41
Simon Glass0d24de92012-01-14 15:12:45 +000042# The start of the cover letter
43re_cover = re.compile('^Cover-letter:')
44
Simon Glassfe2f8d92013-03-20 16:43:00 +000045# A cover letter Cc
46re_cover_cc = re.compile('^Cover-letter-cc: *(.*)')
47
Simon Glass0d24de92012-01-14 15:12:45 +000048# Patch series tag
49re_series = re.compile('^Series-(\w*): *(.*)')
50
51# Commit tags that we want to collect and keep
Doug Anderson28b35942013-03-15 13:24:05 +000052re_tag = re.compile('^(Tested-by|Acked-by|Reviewed-by|Cc): (.*)')
Simon Glass0d24de92012-01-14 15:12:45 +000053
54# The start of a new commit in the git log
55re_commit = re.compile('^commit (.*)')
56
57# We detect these since checkpatch doesn't always do it
58re_space_before_tab = re.compile('^[+].* \t')
59
60# States we can be in - can we use range() and still have comments?
61STATE_MSG_HEADER = 0 # Still in the message header
62STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit)
63STATE_PATCH_HEADER = 2 # In patch header (after the subject)
64STATE_DIFFS = 3 # In the diff part (past --- line)
65
66class PatchStream:
67 """Class for detecting/injecting tags in a patch or series of patches
68
69 We support processing the output of 'git log' to read out the tags we
70 are interested in. We can also process a patch file in order to remove
71 unwanted tags or inject additional ones. These correspond to the two
72 phases of processing.
73 """
74 def __init__(self, series, name=None, is_log=False):
75 self.skip_blank = False # True to skip a single blank line
76 self.found_test = False # Found a TEST= line
77 self.lines_after_test = 0 # MNumber of lines found after TEST=
78 self.warn = [] # List of warnings we have collected
79 self.linenum = 1 # Output line number we are up to
80 self.in_section = None # Name of start...END section we are in
81 self.notes = [] # Series notes
82 self.section = [] # The current section...END section
83 self.series = series # Info about the patch series
84 self.is_log = is_log # True if indent like git log
85 self.in_change = 0 # Non-zero if we are in a change list
86 self.blank_count = 0 # Number of blank lines stored up
87 self.state = STATE_MSG_HEADER # What state are we in?
88 self.tags = [] # Tags collected, like Tested-by...
89 self.signoff = [] # Contents of signoff line
90 self.commit = None # Current commit
91
92 def AddToSeries(self, line, name, value):
93 """Add a new Series-xxx tag.
94
95 When a Series-xxx tag is detected, we come here to record it, if we
96 are scanning a 'git log'.
97
98 Args:
99 line: Source line containing tag (useful for debug/error messages)
100 name: Tag name (part after 'Series-')
101 value: Tag value (part after 'Series-xxx: ')
102 """
103 if name == 'notes':
104 self.in_section = name
105 self.skip_blank = False
106 if self.is_log:
107 self.series.AddTag(self.commit, line, name, value)
108
109 def CloseCommit(self):
110 """Save the current commit into our commit list, and reset our state"""
111 if self.commit and self.is_log:
112 self.series.AddCommit(self.commit)
113 self.commit = None
114
115 def FormatTags(self, tags):
116 out_list = []
117 for tag in sorted(tags):
118 if tag.startswith('Cc:'):
119 tag_list = tag[4:].split(',')
120 out_list += gitutil.BuildEmailList(tag_list, 'Cc:')
121 else:
122 out_list.append(tag)
123 return out_list
124
125 def ProcessLine(self, line):
126 """Process a single line of a patch file or commit log
127
128 This process a line and returns a list of lines to output. The list
129 may be empty or may contain multiple output lines.
130
131 This is where all the complicated logic is located. The class's
132 state is used to move between different states and detect things
133 properly.
134
135 We can be in one of two modes:
136 self.is_log == True: This is 'git log' mode, where most output is
137 indented by 4 characters and we are scanning for tags
138
139 self.is_log == False: This is 'patch' mode, where we already have
140 all the tags, and are processing patches to remove junk we
141 don't want, and add things we think are required.
142
143 Args:
144 line: text line to process
145
146 Returns:
147 list of output lines, or [] if nothing should be output
148 """
149 # Initially we have no output. Prepare the input line string
150 out = []
151 line = line.rstrip('\n')
152 if self.is_log:
153 if line[:4] == ' ':
154 line = line[4:]
155
156 # Handle state transition and skipping blank lines
157 series_match = re_series.match(line)
158 commit_match = re_commit.match(line) if self.is_log else None
Simon Glassfe2f8d92013-03-20 16:43:00 +0000159 cover_cc_match = re_cover_cc.match(line)
Simon Glass0d24de92012-01-14 15:12:45 +0000160 tag_match = None
161 if self.state == STATE_PATCH_HEADER:
162 tag_match = re_tag.match(line)
163 is_blank = not line.strip()
164 if is_blank:
165 if (self.state == STATE_MSG_HEADER
166 or self.state == STATE_PATCH_SUBJECT):
167 self.state += 1
168
169 # We don't have a subject in the text stream of patch files
170 # It has its own line with a Subject: tag
171 if not self.is_log and self.state == STATE_PATCH_SUBJECT:
172 self.state += 1
173 elif commit_match:
174 self.state = STATE_MSG_HEADER
175
176 # If we are in a section, keep collecting lines until we see END
177 if self.in_section:
178 if line == 'END':
179 if self.in_section == 'cover':
180 self.series.cover = self.section
181 elif self.in_section == 'notes':
182 if self.is_log:
183 self.series.notes += self.section
184 else:
185 self.warn.append("Unknown section '%s'" % self.in_section)
186 self.in_section = None
187 self.skip_blank = True
188 self.section = []
189 else:
190 self.section.append(line)
191
192 # Detect the commit subject
193 elif not is_blank and self.state == STATE_PATCH_SUBJECT:
194 self.commit.subject = line
195
196 # Detect the tags we want to remove, and skip blank lines
197 elif re_remove.match(line):
198 self.skip_blank = True
199
200 # TEST= should be the last thing in the commit, so remove
201 # everything after it
202 if line.startswith('TEST='):
203 self.found_test = True
204 elif self.skip_blank and is_blank:
205 self.skip_blank = False
206
207 # Detect the start of a cover letter section
208 elif re_cover.match(line):
209 self.in_section = 'cover'
210 self.skip_blank = False
211
Simon Glassfe2f8d92013-03-20 16:43:00 +0000212 elif cover_cc_match:
213 value = cover_cc_match.group(1)
214 self.AddToSeries(line, 'cover-cc', value)
215
Simon Glass0d24de92012-01-14 15:12:45 +0000216 # If we are in a change list, key collected lines until a blank one
217 elif self.in_change:
218 if is_blank:
219 # Blank line ends this change list
220 self.in_change = 0
Ilya Yanok05e5b732012-08-06 23:46:05 +0000221 elif line == '---' or re_signoff.match(line):
222 self.in_change = 0
223 out = self.ProcessLine(line)
Simon Glass0d24de92012-01-14 15:12:45 +0000224 else:
Ilya Yanoka8840cb2012-08-06 23:46:06 +0000225 if self.is_log:
226 self.series.AddChange(self.in_change, self.commit, line)
Simon Glass0d24de92012-01-14 15:12:45 +0000227 self.skip_blank = False
228
229 # Detect Series-xxx tags
230 elif series_match:
231 name = series_match.group(1)
232 value = series_match.group(2)
233 if name == 'changes':
234 # value is the version number: e.g. 1, or 2
235 try:
236 value = int(value)
237 except ValueError as str:
238 raise ValueError("%s: Cannot decode version info '%s'" %
239 (self.commit.hash, line))
240 self.in_change = int(value)
241 else:
242 self.AddToSeries(line, name, value)
243 self.skip_blank = True
244
245 # Detect the start of a new commit
246 elif commit_match:
247 self.CloseCommit()
Simon Glasse62f9052012-12-15 10:42:06 +0000248 # TODO: We should store the whole hash, and just display a subset
249 self.commit = commit.Commit(commit_match.group(1)[:8])
Simon Glass0d24de92012-01-14 15:12:45 +0000250
251 # Detect tags in the commit message
252 elif tag_match:
Simon Glass0d24de92012-01-14 15:12:45 +0000253 # Remove Tested-by self, since few will take much notice
Ilya Yanokc7379142012-08-06 23:46:08 +0000254 if (tag_match.group(1) == 'Tested-by' and
Simon Glass0d24de92012-01-14 15:12:45 +0000255 tag_match.group(2).find(os.getenv('USER') + '@') != -1):
256 self.warn.append("Ignoring %s" % line)
257 elif tag_match.group(1) == 'Cc':
258 self.commit.AddCc(tag_match.group(2).split(','))
259 else:
260 self.tags.append(line);
261
262 # Well that means this is an ordinary line
263 else:
264 pos = 1
265 # Look for ugly ASCII characters
266 for ch in line:
267 # TODO: Would be nicer to report source filename and line
268 if ord(ch) > 0x80:
269 self.warn.append("Line %d/%d ('%s') has funny ascii char" %
270 (self.linenum, pos, line))
271 pos += 1
272
273 # Look for space before tab
274 m = re_space_before_tab.match(line)
275 if m:
276 self.warn.append('Line %d/%d has space before tab' %
277 (self.linenum, m.start()))
278
279 # OK, we have a valid non-blank line
280 out = [line]
281 self.linenum += 1
282 self.skip_blank = False
283 if self.state == STATE_DIFFS:
284 pass
285
286 # If this is the start of the diffs section, emit our tags and
287 # change log
288 elif line == '---':
289 self.state = STATE_DIFFS
290
291 # Output the tags (signeoff first), then change list
292 out = []
Simon Glass0d24de92012-01-14 15:12:45 +0000293 log = self.series.MakeChangeLog(self.commit)
294 out += self.FormatTags(self.tags)
295 out += [line] + log
296 elif self.found_test:
297 if not re_allowed_after_test.match(line):
298 self.lines_after_test += 1
299
300 return out
301
302 def Finalize(self):
303 """Close out processing of this patch stream"""
304 self.CloseCommit()
305 if self.lines_after_test:
306 self.warn.append('Found %d lines after TEST=' %
307 self.lines_after_test)
308
309 def ProcessStream(self, infd, outfd):
310 """Copy a stream from infd to outfd, filtering out unwanting things.
311
312 This is used to process patch files one at a time.
313
314 Args:
315 infd: Input stream file object
316 outfd: Output stream file object
317 """
318 # Extract the filename from each diff, for nice warnings
319 fname = None
320 last_fname = None
321 re_fname = re.compile('diff --git a/(.*) b/.*')
322 while True:
323 line = infd.readline()
324 if not line:
325 break
326 out = self.ProcessLine(line)
327
328 # Try to detect blank lines at EOF
329 for line in out:
330 match = re_fname.match(line)
331 if match:
332 last_fname = fname
333 fname = match.group(1)
334 if line == '+':
335 self.blank_count += 1
336 else:
337 if self.blank_count and (line == '-- ' or match):
338 self.warn.append("Found possible blank line(s) at "
339 "end of file '%s'" % last_fname)
340 outfd.write('+\n' * self.blank_count)
341 outfd.write(line + '\n')
342 self.blank_count = 0
343 self.Finalize()
344
345
Simon Glasse62f9052012-12-15 10:42:06 +0000346def GetMetaDataForList(commit_range, git_dir=None, count=None,
347 series = Series()):
348 """Reads out patch series metadata from the commits
349
350 This does a 'git log' on the relevant commits and pulls out the tags we
351 are interested in.
352
353 Args:
354 commit_range: Range of commits to count (e.g. 'HEAD..base')
355 git_dir: Path to git repositiory (None to use default)
356 count: Number of commits to list, or None for no limit
357 series: Series object to add information into. By default a new series
358 is started.
359 Returns:
360 A Series object containing information about the commits.
361 """
362 params = ['git', 'log', '--no-color', '--reverse', commit_range]
363 if count is not None:
364 params[2:2] = ['-n%d' % count]
365 if git_dir:
366 params[1:1] = ['--git-dir', git_dir]
367 pipe = [params]
368 stdout = command.RunPipe(pipe, capture=True).stdout
369 ps = PatchStream(series, is_log=True)
370 for line in stdout.splitlines():
371 ps.ProcessLine(line)
372 ps.Finalize()
373 return series
374
Simon Glass0d24de92012-01-14 15:12:45 +0000375def GetMetaData(start, count):
376 """Reads out patch series metadata from the commits
377
378 This does a 'git log' on the relevant commits and pulls out the tags we
379 are interested in.
380
381 Args:
382 start: Commit to start from: 0=HEAD, 1=next one, etc.
383 count: Number of commits to list
384 """
Simon Glasse62f9052012-12-15 10:42:06 +0000385 return GetMetaDataForList('HEAD~%d' % start, None, count)
Simon Glass0d24de92012-01-14 15:12:45 +0000386
387def FixPatch(backup_dir, fname, series, commit):
388 """Fix up a patch file, by adding/removing as required.
389
390 We remove our tags from the patch file, insert changes lists, etc.
391 The patch file is processed in place, and overwritten.
392
393 A backup file is put into backup_dir (if not None).
394
395 Args:
396 fname: Filename to patch file to process
397 series: Series information about this patch set
398 commit: Commit object for this patch file
399 Return:
400 A list of errors, or [] if all ok.
401 """
402 handle, tmpname = tempfile.mkstemp()
403 outfd = os.fdopen(handle, 'w')
404 infd = open(fname, 'r')
405 ps = PatchStream(series)
406 ps.commit = commit
407 ps.ProcessStream(infd, outfd)
408 infd.close()
409 outfd.close()
410
411 # Create a backup file if required
412 if backup_dir:
413 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
414 shutil.move(tmpname, fname)
415 return ps.warn
416
417def FixPatches(series, fnames):
418 """Fix up a list of patches identified by filenames
419
420 The patch files are processed in place, and overwritten.
421
422 Args:
423 series: The series object
424 fnames: List of patch files to process
425 """
426 # Current workflow creates patches, so we shouldn't need a backup
427 backup_dir = None #tempfile.mkdtemp('clean-patch')
428 count = 0
429 for fname in fnames:
430 commit = series.commits[count]
431 commit.patch = fname
432 result = FixPatch(backup_dir, fname, series, commit)
433 if result:
434 print '%d warnings for %s:' % (len(result), fname)
435 for warn in result:
436 print '\t', warn
437 print
438 count += 1
439 print 'Cleaned %d patches' % count
440 return series
441
442def InsertCoverLetter(fname, series, count):
443 """Inserts a cover letter with the required info into patch 0
444
445 Args:
446 fname: Input / output filename of the cover letter file
447 series: Series object
448 count: Number of patches in the series
449 """
450 fd = open(fname, 'r')
451 lines = fd.readlines()
452 fd.close()
453
454 fd = open(fname, 'w')
455 text = series.cover
456 prefix = series.GetPatchPrefix()
457 for line in lines:
458 if line.startswith('Subject:'):
459 # TODO: if more than 10 patches this should save 00/xx, not 0/xx
460 line = 'Subject: [%s 0/%d] %s\n' % (prefix, count, text[0])
461
462 # Insert our cover letter
463 elif line.startswith('*** BLURB HERE ***'):
464 # First the blurb test
465 line = '\n'.join(text[1:]) + '\n'
466 if series.get('notes'):
467 line += '\n'.join(series.notes) + '\n'
468
469 # Now the change list
470 out = series.MakeChangeLog(None)
471 line += '\n' + '\n'.join(out)
472 fd.write(line)
473 fd.close()