blob: 1da9d53b650e0c3a498e67ba3381f8afb201c506 [file] [log] [blame]
Tom Rini83d290c2018-05-06 17:58:06 -04001# SPDX-License-Identifier: GPL-2.0+
Simon Glass0d24de92012-01-14 15:12:45 +00002# Copyright (c) 2011 The Chromium OS Authors.
3#
Simon Glass0d24de92012-01-14 15:12:45 +00004
Simon Glassd06e55a2020-10-29 21:46:17 -06005"""Handles parsing a stream of commits/emails from 'git log' or other source"""
6
Simon Glass6b3252e2020-10-29 21:46:37 -06007import collections
Douglas Anderson833e4192019-09-27 09:23:56 -07008import datetime
Simon Glass74570512020-10-29 21:46:27 -06009import io
Wu, Josh35ce2dc2015-04-03 10:51:17 +080010import math
Simon Glass0d24de92012-01-14 15:12:45 +000011import os
12import re
Simon Glass6b3252e2020-10-29 21:46:37 -060013import queue
Simon Glass0d24de92012-01-14 15:12:45 +000014import shutil
15import tempfile
16
Simon Glassbf776672020-04-17 18:09:04 -060017from patman import command
18from patman import commit
19from patman import gitutil
20from patman.series import Series
Simon Glass0d24de92012-01-14 15:12:45 +000021
22# Tags that we detect and remove
Simon Glass57699042020-10-29 21:46:18 -060023RE_REMOVE = re.compile(r'^BUG=|^TEST=|^BRANCH=|^Review URL:'
Simon Glassd06e55a2020-10-29 21:46:17 -060024 r'|Reviewed-on:|Commit-\w*:')
Simon Glass0d24de92012-01-14 15:12:45 +000025
26# Lines which are allowed after a TEST= line
Simon Glass57699042020-10-29 21:46:18 -060027RE_ALLOWED_AFTER_TEST = re.compile('^Signed-off-by:')
Simon Glass0d24de92012-01-14 15:12:45 +000028
Ilya Yanok05e5b732012-08-06 23:46:05 +000029# Signoffs
Simon Glass57699042020-10-29 21:46:18 -060030RE_SIGNOFF = re.compile('^Signed-off-by: *(.*)')
Ilya Yanok05e5b732012-08-06 23:46:05 +000031
Sean Anderson6949f702020-05-04 16:28:34 -040032# Cover letter tag
Simon Glass57699042020-10-29 21:46:18 -060033RE_COVER = re.compile('^Cover-([a-z-]*): *(.*)')
Simon Glassfe2f8d92013-03-20 16:43:00 +000034
Simon Glass0d24de92012-01-14 15:12:45 +000035# Patch series tag
Simon Glass57699042020-10-29 21:46:18 -060036RE_SERIES_TAG = re.compile('^Series-([a-z-]*): *(.*)')
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +010037
Douglas Anderson833e4192019-09-27 09:23:56 -070038# Change-Id will be used to generate the Message-Id and then be stripped
Simon Glass57699042020-10-29 21:46:18 -060039RE_CHANGE_ID = re.compile('^Change-Id: *(.*)')
Douglas Anderson833e4192019-09-27 09:23:56 -070040
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +010041# Commit series tag
Simon Glass57699042020-10-29 21:46:18 -060042RE_COMMIT_TAG = re.compile('^Commit-([a-z-]*): *(.*)')
Simon Glass0d24de92012-01-14 15:12:45 +000043
44# Commit tags that we want to collect and keep
Simon Glass57699042020-10-29 21:46:18 -060045RE_TAG = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc|Fixes): (.*)')
Simon Glass0d24de92012-01-14 15:12:45 +000046
47# The start of a new commit in the git log
Simon Glass57699042020-10-29 21:46:18 -060048RE_COMMIT = re.compile('^commit ([0-9a-f]*)$')
Simon Glass0d24de92012-01-14 15:12:45 +000049
50# We detect these since checkpatch doesn't always do it
Simon Glass57699042020-10-29 21:46:18 -060051RE_SPACE_BEFORE_TAB = re.compile('^[+].* \t')
Simon Glass0d24de92012-01-14 15:12:45 +000052
Sean Anderson0411fff2020-05-04 16:28:35 -040053# Match indented lines for changes
Simon Glass57699042020-10-29 21:46:18 -060054RE_LEADING_WHITESPACE = re.compile(r'^\s')
Sean Anderson0411fff2020-05-04 16:28:35 -040055
Simon Glass6b3252e2020-10-29 21:46:37 -060056# Detect a 'diff' line
57RE_DIFF = re.compile(r'^>.*diff --git a/(.*) b/(.*)$')
58
59# Detect a context line, like '> @@ -153,8 +153,13 @@ CheckPatch
60RE_LINE = re.compile(r'>.*@@ \-(\d+),\d+ \+(\d+),\d+ @@ *(.*)')
61
Patrick Delaunaya6123332021-07-22 16:51:42 +020062# Detect line with invalid TAG
63RE_INV_TAG = re.compile('^Serie-([a-z-]*): *(.*)')
64
Simon Glass0d24de92012-01-14 15:12:45 +000065# States we can be in - can we use range() and still have comments?
66STATE_MSG_HEADER = 0 # Still in the message header
67STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit)
68STATE_PATCH_HEADER = 2 # In patch header (after the subject)
69STATE_DIFFS = 3 # In the diff part (past --- line)
70
71class PatchStream:
72 """Class for detecting/injecting tags in a patch or series of patches
73
74 We support processing the output of 'git log' to read out the tags we
75 are interested in. We can also process a patch file in order to remove
76 unwanted tags or inject additional ones. These correspond to the two
77 phases of processing.
78 """
Simon Glasse3a816b2020-10-29 21:46:21 -060079 def __init__(self, series, is_log=False):
Simon Glass0d24de92012-01-14 15:12:45 +000080 self.skip_blank = False # True to skip a single blank line
81 self.found_test = False # Found a TEST= line
Sean Anderson6949f702020-05-04 16:28:34 -040082 self.lines_after_test = 0 # Number of lines found after TEST=
Simon Glass0d24de92012-01-14 15:12:45 +000083 self.linenum = 1 # Output line number we are up to
84 self.in_section = None # Name of start...END section we are in
85 self.notes = [] # Series notes
86 self.section = [] # The current section...END section
87 self.series = series # Info about the patch series
88 self.is_log = is_log # True if indent like git log
Sean Anderson6949f702020-05-04 16:28:34 -040089 self.in_change = None # Name of the change list we are in
90 self.change_version = 0 # Non-zero if we are in a change list
Sean Anderson0411fff2020-05-04 16:28:35 -040091 self.change_lines = [] # Lines of the current change
Simon Glass0d24de92012-01-14 15:12:45 +000092 self.blank_count = 0 # Number of blank lines stored up
93 self.state = STATE_MSG_HEADER # What state are we in?
Simon Glass0d24de92012-01-14 15:12:45 +000094 self.commit = None # Current commit
Simon Glassdc4b2a92020-10-29 21:46:38 -060095 # List of unquoted test blocks, each a list of str lines
96 self.snippets = []
Simon Glass6b3252e2020-10-29 21:46:37 -060097 self.cur_diff = None # Last 'diff' line seen (str)
98 self.cur_line = None # Last context (@@) line seen (str)
Simon Glassdc4b2a92020-10-29 21:46:38 -060099 self.recent_diff = None # 'diff' line for current snippet (str)
100 self.recent_line = None # '@@' line for current snippet (str)
Simon Glass6b3252e2020-10-29 21:46:37 -0600101 self.recent_quoted = collections.deque([], 5)
102 self.recent_unquoted = queue.Queue()
103 self.was_quoted = None
Simon Glass0d24de92012-01-14 15:12:45 +0000104
Simon Glass74570512020-10-29 21:46:27 -0600105 @staticmethod
106 def process_text(text, is_comment=False):
107 """Process some text through this class using a default Commit/Series
108
109 Args:
110 text (str): Text to parse
111 is_comment (bool): True if this is a comment rather than a patch.
112 If True, PatchStream doesn't expect a patch subject at the
113 start, but jumps straight into the body
114
115 Returns:
116 PatchStream: object with results
117 """
118 pstrm = PatchStream(Series())
119 pstrm.commit = commit.Commit(None)
120 infd = io.StringIO(text)
121 outfd = io.StringIO()
122 if is_comment:
123 pstrm.state = STATE_PATCH_HEADER
124 pstrm.process_stream(infd, outfd)
125 return pstrm
126
Simon Glassb5cc3992020-10-29 21:46:23 -0600127 def _add_warn(self, warn):
Simon Glass313ef5f2020-10-29 21:46:24 -0600128 """Add a new warning to report to the user about the current commit
129
130 The new warning is added to the current commit if not already present.
Simon Glassb5cc3992020-10-29 21:46:23 -0600131
132 Args:
133 warn (str): Warning to report
Simon Glass313ef5f2020-10-29 21:46:24 -0600134
135 Raises:
136 ValueError: Warning is generated with no commit associated
Simon Glassb5cc3992020-10-29 21:46:23 -0600137 """
Simon Glass313ef5f2020-10-29 21:46:24 -0600138 if not self.commit:
Simon Glass42bc1562021-03-22 18:01:42 +1300139 print('Warning outside commit: %s' % warn)
140 elif warn not in self.commit.warn:
Simon Glass313ef5f2020-10-29 21:46:24 -0600141 self.commit.warn.append(warn)
Simon Glassb5cc3992020-10-29 21:46:23 -0600142
Simon Glassd93720e2020-10-29 21:46:19 -0600143 def _add_to_series(self, line, name, value):
Simon Glass0d24de92012-01-14 15:12:45 +0000144 """Add a new Series-xxx tag.
145
146 When a Series-xxx tag is detected, we come here to record it, if we
147 are scanning a 'git log'.
148
149 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600150 line (str): Source line containing tag (useful for debug/error
151 messages)
152 name (str): Tag name (part after 'Series-')
153 value (str): Tag value (part after 'Series-xxx: ')
Simon Glass0d24de92012-01-14 15:12:45 +0000154 """
155 if name == 'notes':
156 self.in_section = name
157 self.skip_blank = False
158 if self.is_log:
Simon Glassdffa42c2020-10-29 21:46:25 -0600159 warn = self.series.AddTag(self.commit, line, name, value)
160 if warn:
161 self.commit.warn.append(warn)
Simon Glass0d24de92012-01-14 15:12:45 +0000162
Simon Glasse3a816b2020-10-29 21:46:21 -0600163 def _add_to_commit(self, name):
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100164 """Add a new Commit-xxx tag.
165
166 When a Commit-xxx tag is detected, we come here to record it.
167
168 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600169 name (str): Tag name (part after 'Commit-')
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100170 """
171 if name == 'notes':
172 self.in_section = 'commit-' + name
173 self.skip_blank = False
174
Simon Glassd93720e2020-10-29 21:46:19 -0600175 def _add_commit_rtag(self, rtag_type, who):
Simon Glass7207e2b2020-07-05 21:41:57 -0600176 """Add a response tag to the current commit
177
178 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600179 rtag_type (str): rtag type (e.g. 'Reviewed-by')
180 who (str): Person who gave that rtag, e.g.
181 'Fred Bloggs <fred@bloggs.org>'
Simon Glass7207e2b2020-07-05 21:41:57 -0600182 """
183 self.commit.AddRtag(rtag_type, who)
184
Simon Glassd93720e2020-10-29 21:46:19 -0600185 def _close_commit(self):
Simon Glass0d24de92012-01-14 15:12:45 +0000186 """Save the current commit into our commit list, and reset our state"""
187 if self.commit and self.is_log:
188 self.series.AddCommit(self.commit)
189 self.commit = None
Bin Meng0d577182016-06-26 23:24:30 -0700190 # If 'END' is missing in a 'Cover-letter' section, and that section
191 # happens to show up at the very end of the commit message, this is
192 # the chance for us to fix it up.
193 if self.in_section == 'cover' and self.is_log:
194 self.series.cover = self.section
195 self.in_section = None
196 self.skip_blank = True
197 self.section = []
Simon Glass0d24de92012-01-14 15:12:45 +0000198
Simon Glass6b3252e2020-10-29 21:46:37 -0600199 self.cur_diff = None
200 self.recent_diff = None
201 self.recent_line = None
202
Simon Glassd93720e2020-10-29 21:46:19 -0600203 def _parse_version(self, value, line):
Sean Anderson6949f702020-05-04 16:28:34 -0400204 """Parse a version from a *-changes tag
205
206 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600207 value (str): Tag value (part after 'xxx-changes: '
208 line (str): Source line containing tag
Sean Anderson6949f702020-05-04 16:28:34 -0400209
210 Returns:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600211 int: The version as an integer
212
213 Raises:
214 ValueError: the value cannot be converted
Sean Anderson6949f702020-05-04 16:28:34 -0400215 """
216 try:
217 return int(value)
Simon Glassdd147ed2020-10-29 21:46:20 -0600218 except ValueError:
Sean Anderson6949f702020-05-04 16:28:34 -0400219 raise ValueError("%s: Cannot decode version info '%s'" %
Simon Glassd06e55a2020-10-29 21:46:17 -0600220 (self.commit.hash, line))
Sean Anderson6949f702020-05-04 16:28:34 -0400221
Simon Glassd93720e2020-10-29 21:46:19 -0600222 def _finalise_change(self):
223 """_finalise a (multi-line) change and add it to the series or commit"""
Sean Anderson0411fff2020-05-04 16:28:35 -0400224 if not self.change_lines:
225 return
226 change = '\n'.join(self.change_lines)
227
228 if self.in_change == 'Series':
229 self.series.AddChange(self.change_version, self.commit, change)
230 elif self.in_change == 'Cover':
231 self.series.AddChange(self.change_version, None, change)
232 elif self.in_change == 'Commit':
233 self.commit.AddChange(self.change_version, change)
234 self.change_lines = []
235
Simon Glass6b3252e2020-10-29 21:46:37 -0600236 def _finalise_snippet(self):
237 """Finish off a snippet and add it to the list
238
239 This is called when we get to the end of a snippet, i.e. the we enter
240 the next block of quoted text:
241
242 This is a comment from someone.
243
244 Something else
245
246 > Now we have some code <----- end of snippet
247 > more code
248
249 Now a comment about the above code
250
251 This adds the snippet to our list
252 """
253 quoted_lines = []
254 while self.recent_quoted:
255 quoted_lines.append(self.recent_quoted.popleft())
256 unquoted_lines = []
257 valid = False
258 while not self.recent_unquoted.empty():
259 text = self.recent_unquoted.get()
260 if not (text.startswith('On ') and text.endswith('wrote:')):
261 unquoted_lines.append(text)
262 if text:
263 valid = True
264 if valid:
265 lines = []
266 if self.recent_diff:
267 lines.append('> File: %s' % self.recent_diff)
268 if self.recent_line:
269 out = '> Line: %s / %s' % self.recent_line[:2]
270 if self.recent_line[2]:
271 out += ': %s' % self.recent_line[2]
272 lines.append(out)
273 lines += quoted_lines + unquoted_lines
274 if lines:
275 self.snippets.append(lines)
276
Simon Glassd93720e2020-10-29 21:46:19 -0600277 def process_line(self, line):
Simon Glass0d24de92012-01-14 15:12:45 +0000278 """Process a single line of a patch file or commit log
279
280 This process a line and returns a list of lines to output. The list
281 may be empty or may contain multiple output lines.
282
283 This is where all the complicated logic is located. The class's
284 state is used to move between different states and detect things
285 properly.
286
287 We can be in one of two modes:
288 self.is_log == True: This is 'git log' mode, where most output is
289 indented by 4 characters and we are scanning for tags
290
291 self.is_log == False: This is 'patch' mode, where we already have
292 all the tags, and are processing patches to remove junk we
293 don't want, and add things we think are required.
294
295 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600296 line (str): text line to process
Simon Glass0d24de92012-01-14 15:12:45 +0000297
298 Returns:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600299 list: list of output lines, or [] if nothing should be output
300
301 Raises:
302 ValueError: a fatal error occurred while parsing, e.g. an END
303 without a starting tag, or two commits with two change IDs
Simon Glass0d24de92012-01-14 15:12:45 +0000304 """
305 # Initially we have no output. Prepare the input line string
306 out = []
307 line = line.rstrip('\n')
Scott Wood4b89b812014-09-25 14:30:46 -0500308
Simon Glass57699042020-10-29 21:46:18 -0600309 commit_match = RE_COMMIT.match(line) if self.is_log else None
Scott Wood4b89b812014-09-25 14:30:46 -0500310
Simon Glass0d24de92012-01-14 15:12:45 +0000311 if self.is_log:
312 if line[:4] == ' ':
313 line = line[4:]
314
315 # Handle state transition and skipping blank lines
Simon Glass57699042020-10-29 21:46:18 -0600316 series_tag_match = RE_SERIES_TAG.match(line)
317 change_id_match = RE_CHANGE_ID.match(line)
318 commit_tag_match = RE_COMMIT_TAG.match(line)
319 cover_match = RE_COVER.match(line)
320 signoff_match = RE_SIGNOFF.match(line)
321 leading_whitespace_match = RE_LEADING_WHITESPACE.match(line)
Simon Glass6b3252e2020-10-29 21:46:37 -0600322 diff_match = RE_DIFF.match(line)
323 line_match = RE_LINE.match(line)
Patrick Delaunaya6123332021-07-22 16:51:42 +0200324 invalid_match = RE_INV_TAG.match(line)
Simon Glass0d24de92012-01-14 15:12:45 +0000325 tag_match = None
326 if self.state == STATE_PATCH_HEADER:
Simon Glass57699042020-10-29 21:46:18 -0600327 tag_match = RE_TAG.match(line)
Simon Glass0d24de92012-01-14 15:12:45 +0000328 is_blank = not line.strip()
329 if is_blank:
330 if (self.state == STATE_MSG_HEADER
331 or self.state == STATE_PATCH_SUBJECT):
332 self.state += 1
333
334 # We don't have a subject in the text stream of patch files
335 # It has its own line with a Subject: tag
336 if not self.is_log and self.state == STATE_PATCH_SUBJECT:
337 self.state += 1
338 elif commit_match:
339 self.state = STATE_MSG_HEADER
340
Bin Meng94fbd3e2016-06-26 23:24:32 -0700341 # If a tag is detected, or a new commit starts
Douglas Anderson833e4192019-09-27 09:23:56 -0700342 if series_tag_match or commit_tag_match or change_id_match or \
Sean Anderson6949f702020-05-04 16:28:34 -0400343 cover_match or signoff_match or self.state == STATE_MSG_HEADER:
Bin Meng57b6b192016-06-26 23:24:31 -0700344 # but we are already in a section, this means 'END' is missing
345 # for that section, fix it up.
Bin Meng13b98d92016-06-26 23:24:29 -0700346 if self.in_section:
Simon Glassb5cc3992020-10-29 21:46:23 -0600347 self._add_warn("Missing 'END' in section '%s'" % self.in_section)
Bin Meng13b98d92016-06-26 23:24:29 -0700348 if self.in_section == 'cover':
349 self.series.cover = self.section
350 elif self.in_section == 'notes':
351 if self.is_log:
352 self.series.notes += self.section
353 elif self.in_section == 'commit-notes':
354 if self.is_log:
355 self.commit.notes += self.section
356 else:
Simon Glassb5cc3992020-10-29 21:46:23 -0600357 # This should not happen
358 raise ValueError("Unknown section '%s'" % self.in_section)
Bin Meng13b98d92016-06-26 23:24:29 -0700359 self.in_section = None
360 self.skip_blank = True
361 self.section = []
Bin Meng57b6b192016-06-26 23:24:31 -0700362 # but we are already in a change list, that means a blank line
363 # is missing, fix it up.
364 if self.in_change:
Simon Glassb5cc3992020-10-29 21:46:23 -0600365 self._add_warn("Missing 'blank line' in section '%s-changes'" %
366 self.in_change)
Simon Glassd93720e2020-10-29 21:46:19 -0600367 self._finalise_change()
Sean Anderson6949f702020-05-04 16:28:34 -0400368 self.in_change = None
369 self.change_version = 0
Bin Meng13b98d92016-06-26 23:24:29 -0700370
Simon Glass0d24de92012-01-14 15:12:45 +0000371 # If we are in a section, keep collecting lines until we see END
372 if self.in_section:
373 if line == 'END':
374 if self.in_section == 'cover':
375 self.series.cover = self.section
376 elif self.in_section == 'notes':
377 if self.is_log:
378 self.series.notes += self.section
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100379 elif self.in_section == 'commit-notes':
380 if self.is_log:
381 self.commit.notes += self.section
Simon Glass0d24de92012-01-14 15:12:45 +0000382 else:
Simon Glassb5cc3992020-10-29 21:46:23 -0600383 # This should not happen
384 raise ValueError("Unknown section '%s'" % self.in_section)
Simon Glass0d24de92012-01-14 15:12:45 +0000385 self.in_section = None
386 self.skip_blank = True
387 self.section = []
388 else:
389 self.section.append(line)
390
Patrick Delaunay7058dd02020-07-02 19:08:24 +0200391 # If we are not in a section, it is an unexpected END
392 elif line == 'END':
Simon Glassd06e55a2020-10-29 21:46:17 -0600393 raise ValueError("'END' wihout section")
Patrick Delaunay7058dd02020-07-02 19:08:24 +0200394
Simon Glass0d24de92012-01-14 15:12:45 +0000395 # Detect the commit subject
396 elif not is_blank and self.state == STATE_PATCH_SUBJECT:
397 self.commit.subject = line
398
399 # Detect the tags we want to remove, and skip blank lines
Simon Glass57699042020-10-29 21:46:18 -0600400 elif RE_REMOVE.match(line) and not commit_tag_match:
Simon Glass0d24de92012-01-14 15:12:45 +0000401 self.skip_blank = True
402
403 # TEST= should be the last thing in the commit, so remove
404 # everything after it
405 if line.startswith('TEST='):
406 self.found_test = True
407 elif self.skip_blank and is_blank:
408 self.skip_blank = False
409
Sean Anderson6949f702020-05-04 16:28:34 -0400410 # Detect Cover-xxx tags
Bin Menge7df2182016-06-26 23:24:28 -0700411 elif cover_match:
Sean Anderson6949f702020-05-04 16:28:34 -0400412 name = cover_match.group(1)
413 value = cover_match.group(2)
414 if name == 'letter':
415 self.in_section = 'cover'
416 self.skip_blank = False
417 elif name == 'letter-cc':
Simon Glassd93720e2020-10-29 21:46:19 -0600418 self._add_to_series(line, 'cover-cc', value)
Sean Anderson6949f702020-05-04 16:28:34 -0400419 elif name == 'changes':
420 self.in_change = 'Cover'
Simon Glassd93720e2020-10-29 21:46:19 -0600421 self.change_version = self._parse_version(value, line)
Simon Glassfe2f8d92013-03-20 16:43:00 +0000422
Simon Glass0d24de92012-01-14 15:12:45 +0000423 # If we are in a change list, key collected lines until a blank one
424 elif self.in_change:
425 if is_blank:
426 # Blank line ends this change list
Simon Glassd93720e2020-10-29 21:46:19 -0600427 self._finalise_change()
Sean Anderson6949f702020-05-04 16:28:34 -0400428 self.in_change = None
429 self.change_version = 0
Simon Glass102061b2014-04-20 10:50:14 -0600430 elif line == '---':
Simon Glassd93720e2020-10-29 21:46:19 -0600431 self._finalise_change()
Sean Anderson6949f702020-05-04 16:28:34 -0400432 self.in_change = None
433 self.change_version = 0
Simon Glassd93720e2020-10-29 21:46:19 -0600434 out = self.process_line(line)
Sean Anderson0411fff2020-05-04 16:28:35 -0400435 elif self.is_log:
436 if not leading_whitespace_match:
Simon Glassd93720e2020-10-29 21:46:19 -0600437 self._finalise_change()
Sean Anderson0411fff2020-05-04 16:28:35 -0400438 self.change_lines.append(line)
Simon Glass0d24de92012-01-14 15:12:45 +0000439 self.skip_blank = False
440
441 # Detect Series-xxx tags
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100442 elif series_tag_match:
443 name = series_tag_match.group(1)
444 value = series_tag_match.group(2)
Simon Glass0d24de92012-01-14 15:12:45 +0000445 if name == 'changes':
446 # value is the version number: e.g. 1, or 2
Sean Anderson6949f702020-05-04 16:28:34 -0400447 self.in_change = 'Series'
Simon Glassd93720e2020-10-29 21:46:19 -0600448 self.change_version = self._parse_version(value, line)
Simon Glass0d24de92012-01-14 15:12:45 +0000449 else:
Simon Glassd93720e2020-10-29 21:46:19 -0600450 self._add_to_series(line, name, value)
Simon Glass0d24de92012-01-14 15:12:45 +0000451 self.skip_blank = True
452
Douglas Anderson833e4192019-09-27 09:23:56 -0700453 # Detect Change-Id tags
454 elif change_id_match:
455 value = change_id_match.group(1)
456 if self.is_log:
457 if self.commit.change_id:
Simon Glassd06e55a2020-10-29 21:46:17 -0600458 raise ValueError(
Simon Glass53336e62020-11-03 13:54:11 -0700459 "%s: Two Change-Ids: '%s' vs. '%s'" %
460 (self.commit.hash, self.commit.change_id, value))
Douglas Anderson833e4192019-09-27 09:23:56 -0700461 self.commit.change_id = value
462 self.skip_blank = True
463
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100464 # Detect Commit-xxx tags
465 elif commit_tag_match:
466 name = commit_tag_match.group(1)
467 value = commit_tag_match.group(2)
468 if name == 'notes':
Simon Glasse3a816b2020-10-29 21:46:21 -0600469 self._add_to_commit(name)
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100470 self.skip_blank = True
Sean Anderson6949f702020-05-04 16:28:34 -0400471 elif name == 'changes':
472 self.in_change = 'Commit'
Simon Glassd93720e2020-10-29 21:46:19 -0600473 self.change_version = self._parse_version(value, line)
Patrick Delaunaye5ff9ab2020-07-02 19:52:54 +0200474 else:
Simon Glassb5cc3992020-10-29 21:46:23 -0600475 self._add_warn('Line %d: Ignoring Commit-%s' %
476 (self.linenum, name))
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100477
Patrick Delaunaya6123332021-07-22 16:51:42 +0200478 # Detect invalid tags
479 elif invalid_match:
480 raise ValueError("Line %d: Invalid tag = '%s'" %
481 (self.linenum, line))
482
Simon Glass0d24de92012-01-14 15:12:45 +0000483 # Detect the start of a new commit
484 elif commit_match:
Simon Glassd93720e2020-10-29 21:46:19 -0600485 self._close_commit()
Simon Glass0b5b4092014-10-15 02:27:00 -0600486 self.commit = commit.Commit(commit_match.group(1))
Simon Glass0d24de92012-01-14 15:12:45 +0000487
488 # Detect tags in the commit message
489 elif tag_match:
Simon Glass7207e2b2020-07-05 21:41:57 -0600490 rtag_type, who = tag_match.groups()
Simon Glassd93720e2020-10-29 21:46:19 -0600491 self._add_commit_rtag(rtag_type, who)
Simon Glass0d24de92012-01-14 15:12:45 +0000492 # Remove Tested-by self, since few will take much notice
Simon Glass7207e2b2020-07-05 21:41:57 -0600493 if (rtag_type == 'Tested-by' and
494 who.find(os.getenv('USER') + '@') != -1):
Simon Glass4af99872020-10-29 21:46:28 -0600495 self._add_warn("Ignoring '%s'" % line)
Simon Glass7207e2b2020-07-05 21:41:57 -0600496 elif rtag_type == 'Patch-cc':
497 self.commit.AddCc(who.split(','))
Simon Glass0d24de92012-01-14 15:12:45 +0000498 else:
Simon Glassd0c57192014-08-28 09:43:38 -0600499 out = [line]
Simon Glass0d24de92012-01-14 15:12:45 +0000500
Simon Glass102061b2014-04-20 10:50:14 -0600501 # Suppress duplicate signoffs
502 elif signoff_match:
Simon Glasse752edc2014-08-28 09:43:35 -0600503 if (self.is_log or not self.commit or
Simon Glassd06e55a2020-10-29 21:46:17 -0600504 self.commit.CheckDuplicateSignoff(signoff_match.group(1))):
Simon Glass102061b2014-04-20 10:50:14 -0600505 out = [line]
506
Simon Glass0d24de92012-01-14 15:12:45 +0000507 # Well that means this is an ordinary line
508 else:
Simon Glass0d24de92012-01-14 15:12:45 +0000509 # Look for space before tab
Simon Glassdd147ed2020-10-29 21:46:20 -0600510 mat = RE_SPACE_BEFORE_TAB.match(line)
511 if mat:
Simon Glassb5cc3992020-10-29 21:46:23 -0600512 self._add_warn('Line %d/%d has space before tab' %
513 (self.linenum, mat.start()))
Simon Glass0d24de92012-01-14 15:12:45 +0000514
515 # OK, we have a valid non-blank line
516 out = [line]
517 self.linenum += 1
518 self.skip_blank = False
Simon Glass6b3252e2020-10-29 21:46:37 -0600519
520 if diff_match:
521 self.cur_diff = diff_match.group(1)
522
523 # If this is quoted, keep recent lines
524 if not diff_match and self.linenum > 1 and line:
525 if line.startswith('>'):
526 if not self.was_quoted:
527 self._finalise_snippet()
528 self.recent_line = None
529 if not line_match:
530 self.recent_quoted.append(line)
531 self.was_quoted = True
532 self.recent_diff = self.cur_diff
533 else:
534 self.recent_unquoted.put(line)
535 self.was_quoted = False
536
537 if line_match:
538 self.recent_line = line_match.groups()
539
Simon Glass0d24de92012-01-14 15:12:45 +0000540 if self.state == STATE_DIFFS:
541 pass
542
543 # If this is the start of the diffs section, emit our tags and
544 # change log
545 elif line == '---':
546 self.state = STATE_DIFFS
547
Sean Anderson6949f702020-05-04 16:28:34 -0400548 # Output the tags (signoff first), then change list
Simon Glass0d24de92012-01-14 15:12:45 +0000549 out = []
Simon Glass0d24de92012-01-14 15:12:45 +0000550 log = self.series.MakeChangeLog(self.commit)
Simon Glasse752edc2014-08-28 09:43:35 -0600551 out += [line]
552 if self.commit:
553 out += self.commit.notes
554 out += [''] + log
Simon Glass0d24de92012-01-14 15:12:45 +0000555 elif self.found_test:
Simon Glass57699042020-10-29 21:46:18 -0600556 if not RE_ALLOWED_AFTER_TEST.match(line):
Simon Glass0d24de92012-01-14 15:12:45 +0000557 self.lines_after_test += 1
558
559 return out
560
Simon Glassd93720e2020-10-29 21:46:19 -0600561 def finalise(self):
Simon Glass0d24de92012-01-14 15:12:45 +0000562 """Close out processing of this patch stream"""
Simon Glass6b3252e2020-10-29 21:46:37 -0600563 self._finalise_snippet()
Simon Glassd93720e2020-10-29 21:46:19 -0600564 self._finalise_change()
565 self._close_commit()
Simon Glass0d24de92012-01-14 15:12:45 +0000566 if self.lines_after_test:
Simon Glassb5cc3992020-10-29 21:46:23 -0600567 self._add_warn('Found %d lines after TEST=' % self.lines_after_test)
Simon Glass0d24de92012-01-14 15:12:45 +0000568
Simon Glassd93720e2020-10-29 21:46:19 -0600569 def _write_message_id(self, outfd):
Douglas Anderson833e4192019-09-27 09:23:56 -0700570 """Write the Message-Id into the output.
571
572 This is based on the Change-Id in the original patch, the version,
573 and the prefix.
574
575 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600576 outfd (io.IOBase): Output stream file object
Douglas Anderson833e4192019-09-27 09:23:56 -0700577 """
578 if not self.commit.change_id:
579 return
580
581 # If the count is -1 we're testing, so use a fixed time
582 if self.commit.count == -1:
583 time_now = datetime.datetime(1999, 12, 31, 23, 59, 59)
584 else:
585 time_now = datetime.datetime.now()
586
587 # In theory there is email.utils.make_msgid() which would be nice
588 # to use, but it already produces something way too long and thus
589 # will produce ugly commit lines if someone throws this into
590 # a "Link:" tag in the final commit. So (sigh) roll our own.
591
592 # Start with the time; presumably we wouldn't send the same series
593 # with the same Change-Id at the exact same second.
594 parts = [time_now.strftime("%Y%m%d%H%M%S")]
595
596 # These seem like they would be nice to include.
597 if 'prefix' in self.series:
598 parts.append(self.series['prefix'])
Sean Anderson082c1192021-10-22 19:07:04 -0400599 if 'postfix' in self.series:
600 parts.append(self.serties['postfix'])
Douglas Anderson833e4192019-09-27 09:23:56 -0700601 if 'version' in self.series:
602 parts.append("v%s" % self.series['version'])
603
604 parts.append(str(self.commit.count + 1))
605
606 # The Change-Id must be last, right before the @
607 parts.append(self.commit.change_id)
608
609 # Join parts together with "." and write it out.
610 outfd.write('Message-Id: <%s@changeid>\n' % '.'.join(parts))
611
Simon Glassd93720e2020-10-29 21:46:19 -0600612 def process_stream(self, infd, outfd):
Simon Glass0d24de92012-01-14 15:12:45 +0000613 """Copy a stream from infd to outfd, filtering out unwanting things.
614
615 This is used to process patch files one at a time.
616
617 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600618 infd (io.IOBase): Input stream file object
619 outfd (io.IOBase): Output stream file object
Simon Glass0d24de92012-01-14 15:12:45 +0000620 """
621 # Extract the filename from each diff, for nice warnings
622 fname = None
623 last_fname = None
624 re_fname = re.compile('diff --git a/(.*) b/.*')
Douglas Anderson833e4192019-09-27 09:23:56 -0700625
Simon Glassd93720e2020-10-29 21:46:19 -0600626 self._write_message_id(outfd)
Douglas Anderson833e4192019-09-27 09:23:56 -0700627
Simon Glass0d24de92012-01-14 15:12:45 +0000628 while True:
629 line = infd.readline()
630 if not line:
631 break
Simon Glassd93720e2020-10-29 21:46:19 -0600632 out = self.process_line(line)
Simon Glass0d24de92012-01-14 15:12:45 +0000633
634 # Try to detect blank lines at EOF
635 for line in out:
636 match = re_fname.match(line)
637 if match:
638 last_fname = fname
639 fname = match.group(1)
640 if line == '+':
641 self.blank_count += 1
642 else:
643 if self.blank_count and (line == '-- ' or match):
Simon Glassb5cc3992020-10-29 21:46:23 -0600644 self._add_warn("Found possible blank line(s) at end of file '%s'" %
645 last_fname)
Simon Glass0d24de92012-01-14 15:12:45 +0000646 outfd.write('+\n' * self.blank_count)
647 outfd.write(line + '\n')
648 self.blank_count = 0
Simon Glassd93720e2020-10-29 21:46:19 -0600649 self.finalise()
Simon Glass0d24de92012-01-14 15:12:45 +0000650
Simon Glass8f9ba3a2020-10-29 21:46:36 -0600651def insert_tags(msg, tags_to_emit):
652 """Add extra tags to a commit message
653
654 The tags are added after an existing block of tags if found, otherwise at
655 the end.
656
657 Args:
658 msg (str): Commit message
659 tags_to_emit (list): List of tags to emit, each a str
660
661 Returns:
662 (str) new message
663 """
664 out = []
665 done = False
666 emit_tags = False
Simon Glass59747182021-08-01 16:02:39 -0600667 emit_blank = False
Simon Glass8f9ba3a2020-10-29 21:46:36 -0600668 for line in msg.splitlines():
669 if not done:
670 signoff_match = RE_SIGNOFF.match(line)
671 tag_match = RE_TAG.match(line)
672 if tag_match or signoff_match:
673 emit_tags = True
674 if emit_tags and not tag_match and not signoff_match:
675 out += tags_to_emit
676 emit_tags = False
677 done = True
Simon Glass59747182021-08-01 16:02:39 -0600678 emit_blank = not (signoff_match or tag_match)
679 else:
680 emit_blank = line
Simon Glass8f9ba3a2020-10-29 21:46:36 -0600681 out.append(line)
682 if not done:
Simon Glass59747182021-08-01 16:02:39 -0600683 if emit_blank:
684 out.append('')
Simon Glass8f9ba3a2020-10-29 21:46:36 -0600685 out += tags_to_emit
686 return '\n'.join(out)
687
688def get_list(commit_range, git_dir=None, count=None):
689 """Get a log of a list of comments
690
691 This returns the output of 'git log' for the selected commits
692
693 Args:
694 commit_range (str): Range of commits to count (e.g. 'HEAD..base')
695 git_dir (str): Path to git repositiory (None to use default)
696 count (int): Number of commits to list, or None for no limit
697
698 Returns
699 str: String containing the contents of the git log
700 """
701 params = gitutil.LogCmd(commit_range, reverse=True, count=count,
702 git_dir=git_dir)
703 return command.RunPipe([params], capture=True).stdout
Simon Glass0d24de92012-01-14 15:12:45 +0000704
Simon Glassd93720e2020-10-29 21:46:19 -0600705def get_metadata_for_list(commit_range, git_dir=None, count=None,
706 series=None, allow_overwrite=False):
Simon Glasse62f9052012-12-15 10:42:06 +0000707 """Reads out patch series metadata from the commits
708
709 This does a 'git log' on the relevant commits and pulls out the tags we
710 are interested in.
711
712 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600713 commit_range (str): Range of commits to count (e.g. 'HEAD..base')
714 git_dir (str): Path to git repositiory (None to use default)
715 count (int): Number of commits to list, or None for no limit
716 series (Series): Object to add information into. By default a new series
Simon Glasse62f9052012-12-15 10:42:06 +0000717 is started.
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600718 allow_overwrite (bool): Allow tags to overwrite an existing tag
719
Simon Glasse62f9052012-12-15 10:42:06 +0000720 Returns:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600721 Series: Object containing information about the commits.
Simon Glasse62f9052012-12-15 10:42:06 +0000722 """
Simon Glass891b7a02014-09-05 19:00:19 -0600723 if not series:
724 series = Series()
Simon Glass950a2312014-09-05 19:00:23 -0600725 series.allow_overwrite = allow_overwrite
Simon Glass8f9ba3a2020-10-29 21:46:36 -0600726 stdout = get_list(commit_range, git_dir, count)
Simon Glassdd147ed2020-10-29 21:46:20 -0600727 pst = PatchStream(series, is_log=True)
Simon Glasse62f9052012-12-15 10:42:06 +0000728 for line in stdout.splitlines():
Simon Glassdd147ed2020-10-29 21:46:20 -0600729 pst.process_line(line)
730 pst.finalise()
Simon Glasse62f9052012-12-15 10:42:06 +0000731 return series
732
Simon Glassd93720e2020-10-29 21:46:19 -0600733def get_metadata(branch, start, count):
Simon Glass0d24de92012-01-14 15:12:45 +0000734 """Reads out patch series metadata from the commits
735
736 This does a 'git log' on the relevant commits and pulls out the tags we
737 are interested in.
738
739 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600740 branch (str): Branch to use (None for current branch)
741 start (int): Commit to start from: 0=branch HEAD, 1=next one, etc.
742 count (int): Number of commits to list
743
744 Returns:
745 Series: Object containing information about the commits.
Simon Glass0d24de92012-01-14 15:12:45 +0000746 """
Simon Glassd93720e2020-10-29 21:46:19 -0600747 return get_metadata_for_list(
748 '%s~%d' % (branch if branch else 'HEAD', start), None, count)
Simon Glass0d24de92012-01-14 15:12:45 +0000749
Simon Glassd93720e2020-10-29 21:46:19 -0600750def get_metadata_for_test(text):
Simon Glass6e87ae12017-05-29 15:31:31 -0600751 """Process metadata from a file containing a git log. Used for tests
752
753 Args:
754 text:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600755
756 Returns:
757 Series: Object containing information about the commits.
Simon Glass6e87ae12017-05-29 15:31:31 -0600758 """
759 series = Series()
Simon Glassdd147ed2020-10-29 21:46:20 -0600760 pst = PatchStream(series, is_log=True)
Simon Glass6e87ae12017-05-29 15:31:31 -0600761 for line in text.splitlines():
Simon Glassdd147ed2020-10-29 21:46:20 -0600762 pst.process_line(line)
763 pst.finalise()
Simon Glass6e87ae12017-05-29 15:31:31 -0600764 return series
765
Simon Glassdd147ed2020-10-29 21:46:20 -0600766def fix_patch(backup_dir, fname, series, cmt):
Simon Glass0d24de92012-01-14 15:12:45 +0000767 """Fix up a patch file, by adding/removing as required.
768
769 We remove our tags from the patch file, insert changes lists, etc.
770 The patch file is processed in place, and overwritten.
771
772 A backup file is put into backup_dir (if not None).
773
774 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600775 backup_dir (str): Path to directory to use to backup the file
776 fname (str): Filename to patch file to process
777 series (Series): Series information about this patch set
778 cmt (Commit): Commit object for this patch file
779
Simon Glass0d24de92012-01-14 15:12:45 +0000780 Return:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600781 list: A list of errors, each str, or [] if all ok.
Simon Glass0d24de92012-01-14 15:12:45 +0000782 """
783 handle, tmpname = tempfile.mkstemp()
Simon Glass272cd852019-10-31 07:42:51 -0600784 outfd = os.fdopen(handle, 'w', encoding='utf-8')
785 infd = open(fname, 'r', encoding='utf-8')
Simon Glassdd147ed2020-10-29 21:46:20 -0600786 pst = PatchStream(series)
787 pst.commit = cmt
788 pst.process_stream(infd, outfd)
Simon Glass0d24de92012-01-14 15:12:45 +0000789 infd.close()
790 outfd.close()
791
792 # Create a backup file if required
793 if backup_dir:
794 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
795 shutil.move(tmpname, fname)
Simon Glass313ef5f2020-10-29 21:46:24 -0600796 return cmt.warn
Simon Glass0d24de92012-01-14 15:12:45 +0000797
Simon Glassd93720e2020-10-29 21:46:19 -0600798def fix_patches(series, fnames):
Simon Glass0d24de92012-01-14 15:12:45 +0000799 """Fix up a list of patches identified by filenames
800
801 The patch files are processed in place, and overwritten.
802
803 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600804 series (Series): The Series object
805 fnames (:type: list of str): List of patch files to process
Simon Glass0d24de92012-01-14 15:12:45 +0000806 """
807 # Current workflow creates patches, so we shouldn't need a backup
808 backup_dir = None #tempfile.mkdtemp('clean-patch')
809 count = 0
810 for fname in fnames:
Simon Glassdd147ed2020-10-29 21:46:20 -0600811 cmt = series.commits[count]
812 cmt.patch = fname
813 cmt.count = count
814 result = fix_patch(backup_dir, fname, series, cmt)
Simon Glass0d24de92012-01-14 15:12:45 +0000815 if result:
Simon Glass9994baa2020-10-29 21:46:30 -0600816 print('%d warning%s for %s:' %
817 (len(result), 's' if len(result) > 1 else '', fname))
Simon Glass0d24de92012-01-14 15:12:45 +0000818 for warn in result:
Simon Glass9994baa2020-10-29 21:46:30 -0600819 print('\t%s' % warn)
820 print()
Simon Glass0d24de92012-01-14 15:12:45 +0000821 count += 1
Simon Glass9994baa2020-10-29 21:46:30 -0600822 print('Cleaned %d patch%s' % (count, 'es' if count > 1 else ''))
Simon Glass0d24de92012-01-14 15:12:45 +0000823
Simon Glassd93720e2020-10-29 21:46:19 -0600824def insert_cover_letter(fname, series, count):
Simon Glass0d24de92012-01-14 15:12:45 +0000825 """Inserts a cover letter with the required info into patch 0
826
827 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600828 fname (str): Input / output filename of the cover letter file
829 series (Series): Series object
830 count (int): Number of patches in the series
Simon Glass0d24de92012-01-14 15:12:45 +0000831 """
Simon Glassdd147ed2020-10-29 21:46:20 -0600832 fil = open(fname, 'r')
833 lines = fil.readlines()
834 fil.close()
Simon Glass0d24de92012-01-14 15:12:45 +0000835
Simon Glassdd147ed2020-10-29 21:46:20 -0600836 fil = open(fname, 'w')
Simon Glass0d24de92012-01-14 15:12:45 +0000837 text = series.cover
838 prefix = series.GetPatchPrefix()
839 for line in lines:
840 if line.startswith('Subject:'):
Wu, Josh35ce2dc2015-04-03 10:51:17 +0800841 # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc
842 zero_repeat = int(math.log10(count)) + 1
843 zero = '0' * zero_repeat
844 line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0])
Simon Glass0d24de92012-01-14 15:12:45 +0000845
846 # Insert our cover letter
847 elif line.startswith('*** BLURB HERE ***'):
848 # First the blurb test
849 line = '\n'.join(text[1:]) + '\n'
850 if series.get('notes'):
851 line += '\n'.join(series.notes) + '\n'
852
853 # Now the change list
854 out = series.MakeChangeLog(None)
855 line += '\n' + '\n'.join(out)
Simon Glassdd147ed2020-10-29 21:46:20 -0600856 fil.write(line)
857 fil.close()