blob: cdcd50a8499eca031412abdf4a18ae668e04e75e [file] [log] [blame]
Tom Rini83d290c2018-05-06 17:58:06 -04001# SPDX-License-Identifier: GPL-2.0+
Simon Glass0d24de92012-01-14 15:12:45 +00002# Copyright (c) 2011 The Chromium OS Authors.
3#
Simon Glass0d24de92012-01-14 15:12:45 +00004
Simon Glassd06e55a2020-10-29 21:46:17 -06005"""Handles parsing a stream of commits/emails from 'git log' or other source"""
6
Simon Glass6b3252e2020-10-29 21:46:37 -06007import collections
Douglas Anderson833e4192019-09-27 09:23:56 -07008import datetime
Simon Glass74570512020-10-29 21:46:27 -06009import io
Wu, Josh35ce2dc2015-04-03 10:51:17 +080010import math
Simon Glass0d24de92012-01-14 15:12:45 +000011import os
12import re
Simon Glass6b3252e2020-10-29 21:46:37 -060013import queue
Simon Glass0d24de92012-01-14 15:12:45 +000014import shutil
15import tempfile
16
Simon Glassbf776672020-04-17 18:09:04 -060017from patman import command
18from patman import commit
19from patman import gitutil
20from patman.series import Series
Simon Glass0d24de92012-01-14 15:12:45 +000021
22# Tags that we detect and remove
Simon Glass57699042020-10-29 21:46:18 -060023RE_REMOVE = re.compile(r'^BUG=|^TEST=|^BRANCH=|^Review URL:'
Simon Glassd06e55a2020-10-29 21:46:17 -060024 r'|Reviewed-on:|Commit-\w*:')
Simon Glass0d24de92012-01-14 15:12:45 +000025
26# Lines which are allowed after a TEST= line
Simon Glass57699042020-10-29 21:46:18 -060027RE_ALLOWED_AFTER_TEST = re.compile('^Signed-off-by:')
Simon Glass0d24de92012-01-14 15:12:45 +000028
Ilya Yanok05e5b732012-08-06 23:46:05 +000029# Signoffs
Simon Glass57699042020-10-29 21:46:18 -060030RE_SIGNOFF = re.compile('^Signed-off-by: *(.*)')
Ilya Yanok05e5b732012-08-06 23:46:05 +000031
Sean Anderson6949f702020-05-04 16:28:34 -040032# Cover letter tag
Simon Glass57699042020-10-29 21:46:18 -060033RE_COVER = re.compile('^Cover-([a-z-]*): *(.*)')
Simon Glassfe2f8d92013-03-20 16:43:00 +000034
Simon Glass0d24de92012-01-14 15:12:45 +000035# Patch series tag
Simon Glass57699042020-10-29 21:46:18 -060036RE_SERIES_TAG = re.compile('^Series-([a-z-]*): *(.*)')
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +010037
Douglas Anderson833e4192019-09-27 09:23:56 -070038# Change-Id will be used to generate the Message-Id and then be stripped
Simon Glass57699042020-10-29 21:46:18 -060039RE_CHANGE_ID = re.compile('^Change-Id: *(.*)')
Douglas Anderson833e4192019-09-27 09:23:56 -070040
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +010041# Commit series tag
Simon Glass57699042020-10-29 21:46:18 -060042RE_COMMIT_TAG = re.compile('^Commit-([a-z-]*): *(.*)')
Simon Glass0d24de92012-01-14 15:12:45 +000043
44# Commit tags that we want to collect and keep
Simon Glass57699042020-10-29 21:46:18 -060045RE_TAG = re.compile('^(Tested-by|Acked-by|Reviewed-by|Patch-cc|Fixes): (.*)')
Simon Glass0d24de92012-01-14 15:12:45 +000046
47# The start of a new commit in the git log
Simon Glass57699042020-10-29 21:46:18 -060048RE_COMMIT = re.compile('^commit ([0-9a-f]*)$')
Simon Glass0d24de92012-01-14 15:12:45 +000049
50# We detect these since checkpatch doesn't always do it
Simon Glass57699042020-10-29 21:46:18 -060051RE_SPACE_BEFORE_TAB = re.compile('^[+].* \t')
Simon Glass0d24de92012-01-14 15:12:45 +000052
Sean Anderson0411fff2020-05-04 16:28:35 -040053# Match indented lines for changes
Simon Glass57699042020-10-29 21:46:18 -060054RE_LEADING_WHITESPACE = re.compile(r'^\s')
Sean Anderson0411fff2020-05-04 16:28:35 -040055
Simon Glass6b3252e2020-10-29 21:46:37 -060056# Detect a 'diff' line
57RE_DIFF = re.compile(r'^>.*diff --git a/(.*) b/(.*)$')
58
59# Detect a context line, like '> @@ -153,8 +153,13 @@ CheckPatch
60RE_LINE = re.compile(r'>.*@@ \-(\d+),\d+ \+(\d+),\d+ @@ *(.*)')
61
Simon Glass0d24de92012-01-14 15:12:45 +000062# States we can be in - can we use range() and still have comments?
63STATE_MSG_HEADER = 0 # Still in the message header
64STATE_PATCH_SUBJECT = 1 # In patch subject (first line of log for a commit)
65STATE_PATCH_HEADER = 2 # In patch header (after the subject)
66STATE_DIFFS = 3 # In the diff part (past --- line)
67
68class PatchStream:
69 """Class for detecting/injecting tags in a patch or series of patches
70
71 We support processing the output of 'git log' to read out the tags we
72 are interested in. We can also process a patch file in order to remove
73 unwanted tags or inject additional ones. These correspond to the two
74 phases of processing.
75 """
Simon Glasse3a816b2020-10-29 21:46:21 -060076 def __init__(self, series, is_log=False):
Simon Glass0d24de92012-01-14 15:12:45 +000077 self.skip_blank = False # True to skip a single blank line
78 self.found_test = False # Found a TEST= line
Sean Anderson6949f702020-05-04 16:28:34 -040079 self.lines_after_test = 0 # Number of lines found after TEST=
Simon Glass0d24de92012-01-14 15:12:45 +000080 self.linenum = 1 # Output line number we are up to
81 self.in_section = None # Name of start...END section we are in
82 self.notes = [] # Series notes
83 self.section = [] # The current section...END section
84 self.series = series # Info about the patch series
85 self.is_log = is_log # True if indent like git log
Sean Anderson6949f702020-05-04 16:28:34 -040086 self.in_change = None # Name of the change list we are in
87 self.change_version = 0 # Non-zero if we are in a change list
Sean Anderson0411fff2020-05-04 16:28:35 -040088 self.change_lines = [] # Lines of the current change
Simon Glass0d24de92012-01-14 15:12:45 +000089 self.blank_count = 0 # Number of blank lines stored up
90 self.state = STATE_MSG_HEADER # What state are we in?
Simon Glass0d24de92012-01-14 15:12:45 +000091 self.commit = None # Current commit
Simon Glassdc4b2a92020-10-29 21:46:38 -060092 # List of unquoted test blocks, each a list of str lines
93 self.snippets = []
Simon Glass6b3252e2020-10-29 21:46:37 -060094 self.cur_diff = None # Last 'diff' line seen (str)
95 self.cur_line = None # Last context (@@) line seen (str)
Simon Glassdc4b2a92020-10-29 21:46:38 -060096 self.recent_diff = None # 'diff' line for current snippet (str)
97 self.recent_line = None # '@@' line for current snippet (str)
Simon Glass6b3252e2020-10-29 21:46:37 -060098 self.recent_quoted = collections.deque([], 5)
99 self.recent_unquoted = queue.Queue()
100 self.was_quoted = None
Simon Glass0d24de92012-01-14 15:12:45 +0000101
Simon Glass74570512020-10-29 21:46:27 -0600102 @staticmethod
103 def process_text(text, is_comment=False):
104 """Process some text through this class using a default Commit/Series
105
106 Args:
107 text (str): Text to parse
108 is_comment (bool): True if this is a comment rather than a patch.
109 If True, PatchStream doesn't expect a patch subject at the
110 start, but jumps straight into the body
111
112 Returns:
113 PatchStream: object with results
114 """
115 pstrm = PatchStream(Series())
116 pstrm.commit = commit.Commit(None)
117 infd = io.StringIO(text)
118 outfd = io.StringIO()
119 if is_comment:
120 pstrm.state = STATE_PATCH_HEADER
121 pstrm.process_stream(infd, outfd)
122 return pstrm
123
Simon Glassb5cc3992020-10-29 21:46:23 -0600124 def _add_warn(self, warn):
Simon Glass313ef5f2020-10-29 21:46:24 -0600125 """Add a new warning to report to the user about the current commit
126
127 The new warning is added to the current commit if not already present.
Simon Glassb5cc3992020-10-29 21:46:23 -0600128
129 Args:
130 warn (str): Warning to report
Simon Glass313ef5f2020-10-29 21:46:24 -0600131
132 Raises:
133 ValueError: Warning is generated with no commit associated
Simon Glassb5cc3992020-10-29 21:46:23 -0600134 """
Simon Glass313ef5f2020-10-29 21:46:24 -0600135 if not self.commit:
136 raise ValueError('Warning outside commit: %s' % warn)
137 if warn not in self.commit.warn:
138 self.commit.warn.append(warn)
Simon Glassb5cc3992020-10-29 21:46:23 -0600139
Simon Glassd93720e2020-10-29 21:46:19 -0600140 def _add_to_series(self, line, name, value):
Simon Glass0d24de92012-01-14 15:12:45 +0000141 """Add a new Series-xxx tag.
142
143 When a Series-xxx tag is detected, we come here to record it, if we
144 are scanning a 'git log'.
145
146 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600147 line (str): Source line containing tag (useful for debug/error
148 messages)
149 name (str): Tag name (part after 'Series-')
150 value (str): Tag value (part after 'Series-xxx: ')
Simon Glass0d24de92012-01-14 15:12:45 +0000151 """
152 if name == 'notes':
153 self.in_section = name
154 self.skip_blank = False
155 if self.is_log:
Simon Glassdffa42c2020-10-29 21:46:25 -0600156 warn = self.series.AddTag(self.commit, line, name, value)
157 if warn:
158 self.commit.warn.append(warn)
Simon Glass0d24de92012-01-14 15:12:45 +0000159
Simon Glasse3a816b2020-10-29 21:46:21 -0600160 def _add_to_commit(self, name):
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100161 """Add a new Commit-xxx tag.
162
163 When a Commit-xxx tag is detected, we come here to record it.
164
165 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600166 name (str): Tag name (part after 'Commit-')
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100167 """
168 if name == 'notes':
169 self.in_section = 'commit-' + name
170 self.skip_blank = False
171
Simon Glassd93720e2020-10-29 21:46:19 -0600172 def _add_commit_rtag(self, rtag_type, who):
Simon Glass7207e2b2020-07-05 21:41:57 -0600173 """Add a response tag to the current commit
174
175 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600176 rtag_type (str): rtag type (e.g. 'Reviewed-by')
177 who (str): Person who gave that rtag, e.g.
178 'Fred Bloggs <fred@bloggs.org>'
Simon Glass7207e2b2020-07-05 21:41:57 -0600179 """
180 self.commit.AddRtag(rtag_type, who)
181
Simon Glassd93720e2020-10-29 21:46:19 -0600182 def _close_commit(self):
Simon Glass0d24de92012-01-14 15:12:45 +0000183 """Save the current commit into our commit list, and reset our state"""
184 if self.commit and self.is_log:
185 self.series.AddCommit(self.commit)
186 self.commit = None
Bin Meng0d577182016-06-26 23:24:30 -0700187 # If 'END' is missing in a 'Cover-letter' section, and that section
188 # happens to show up at the very end of the commit message, this is
189 # the chance for us to fix it up.
190 if self.in_section == 'cover' and self.is_log:
191 self.series.cover = self.section
192 self.in_section = None
193 self.skip_blank = True
194 self.section = []
Simon Glass0d24de92012-01-14 15:12:45 +0000195
Simon Glass6b3252e2020-10-29 21:46:37 -0600196 self.cur_diff = None
197 self.recent_diff = None
198 self.recent_line = None
199
Simon Glassd93720e2020-10-29 21:46:19 -0600200 def _parse_version(self, value, line):
Sean Anderson6949f702020-05-04 16:28:34 -0400201 """Parse a version from a *-changes tag
202
203 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600204 value (str): Tag value (part after 'xxx-changes: '
205 line (str): Source line containing tag
Sean Anderson6949f702020-05-04 16:28:34 -0400206
207 Returns:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600208 int: The version as an integer
209
210 Raises:
211 ValueError: the value cannot be converted
Sean Anderson6949f702020-05-04 16:28:34 -0400212 """
213 try:
214 return int(value)
Simon Glassdd147ed2020-10-29 21:46:20 -0600215 except ValueError:
Sean Anderson6949f702020-05-04 16:28:34 -0400216 raise ValueError("%s: Cannot decode version info '%s'" %
Simon Glassd06e55a2020-10-29 21:46:17 -0600217 (self.commit.hash, line))
Sean Anderson6949f702020-05-04 16:28:34 -0400218
Simon Glassd93720e2020-10-29 21:46:19 -0600219 def _finalise_change(self):
220 """_finalise a (multi-line) change and add it to the series or commit"""
Sean Anderson0411fff2020-05-04 16:28:35 -0400221 if not self.change_lines:
222 return
223 change = '\n'.join(self.change_lines)
224
225 if self.in_change == 'Series':
226 self.series.AddChange(self.change_version, self.commit, change)
227 elif self.in_change == 'Cover':
228 self.series.AddChange(self.change_version, None, change)
229 elif self.in_change == 'Commit':
230 self.commit.AddChange(self.change_version, change)
231 self.change_lines = []
232
Simon Glass6b3252e2020-10-29 21:46:37 -0600233 def _finalise_snippet(self):
234 """Finish off a snippet and add it to the list
235
236 This is called when we get to the end of a snippet, i.e. the we enter
237 the next block of quoted text:
238
239 This is a comment from someone.
240
241 Something else
242
243 > Now we have some code <----- end of snippet
244 > more code
245
246 Now a comment about the above code
247
248 This adds the snippet to our list
249 """
250 quoted_lines = []
251 while self.recent_quoted:
252 quoted_lines.append(self.recent_quoted.popleft())
253 unquoted_lines = []
254 valid = False
255 while not self.recent_unquoted.empty():
256 text = self.recent_unquoted.get()
257 if not (text.startswith('On ') and text.endswith('wrote:')):
258 unquoted_lines.append(text)
259 if text:
260 valid = True
261 if valid:
262 lines = []
263 if self.recent_diff:
264 lines.append('> File: %s' % self.recent_diff)
265 if self.recent_line:
266 out = '> Line: %s / %s' % self.recent_line[:2]
267 if self.recent_line[2]:
268 out += ': %s' % self.recent_line[2]
269 lines.append(out)
270 lines += quoted_lines + unquoted_lines
271 if lines:
272 self.snippets.append(lines)
273
Simon Glassd93720e2020-10-29 21:46:19 -0600274 def process_line(self, line):
Simon Glass0d24de92012-01-14 15:12:45 +0000275 """Process a single line of a patch file or commit log
276
277 This process a line and returns a list of lines to output. The list
278 may be empty or may contain multiple output lines.
279
280 This is where all the complicated logic is located. The class's
281 state is used to move between different states and detect things
282 properly.
283
284 We can be in one of two modes:
285 self.is_log == True: This is 'git log' mode, where most output is
286 indented by 4 characters and we are scanning for tags
287
288 self.is_log == False: This is 'patch' mode, where we already have
289 all the tags, and are processing patches to remove junk we
290 don't want, and add things we think are required.
291
292 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600293 line (str): text line to process
Simon Glass0d24de92012-01-14 15:12:45 +0000294
295 Returns:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600296 list: list of output lines, or [] if nothing should be output
297
298 Raises:
299 ValueError: a fatal error occurred while parsing, e.g. an END
300 without a starting tag, or two commits with two change IDs
Simon Glass0d24de92012-01-14 15:12:45 +0000301 """
302 # Initially we have no output. Prepare the input line string
303 out = []
304 line = line.rstrip('\n')
Scott Wood4b89b812014-09-25 14:30:46 -0500305
Simon Glass57699042020-10-29 21:46:18 -0600306 commit_match = RE_COMMIT.match(line) if self.is_log else None
Scott Wood4b89b812014-09-25 14:30:46 -0500307
Simon Glass0d24de92012-01-14 15:12:45 +0000308 if self.is_log:
309 if line[:4] == ' ':
310 line = line[4:]
311
312 # Handle state transition and skipping blank lines
Simon Glass57699042020-10-29 21:46:18 -0600313 series_tag_match = RE_SERIES_TAG.match(line)
314 change_id_match = RE_CHANGE_ID.match(line)
315 commit_tag_match = RE_COMMIT_TAG.match(line)
316 cover_match = RE_COVER.match(line)
317 signoff_match = RE_SIGNOFF.match(line)
318 leading_whitespace_match = RE_LEADING_WHITESPACE.match(line)
Simon Glass6b3252e2020-10-29 21:46:37 -0600319 diff_match = RE_DIFF.match(line)
320 line_match = RE_LINE.match(line)
Simon Glass0d24de92012-01-14 15:12:45 +0000321 tag_match = None
322 if self.state == STATE_PATCH_HEADER:
Simon Glass57699042020-10-29 21:46:18 -0600323 tag_match = RE_TAG.match(line)
Simon Glass0d24de92012-01-14 15:12:45 +0000324 is_blank = not line.strip()
325 if is_blank:
326 if (self.state == STATE_MSG_HEADER
327 or self.state == STATE_PATCH_SUBJECT):
328 self.state += 1
329
330 # We don't have a subject in the text stream of patch files
331 # It has its own line with a Subject: tag
332 if not self.is_log and self.state == STATE_PATCH_SUBJECT:
333 self.state += 1
334 elif commit_match:
335 self.state = STATE_MSG_HEADER
336
Bin Meng94fbd3e2016-06-26 23:24:32 -0700337 # If a tag is detected, or a new commit starts
Douglas Anderson833e4192019-09-27 09:23:56 -0700338 if series_tag_match or commit_tag_match or change_id_match or \
Sean Anderson6949f702020-05-04 16:28:34 -0400339 cover_match or signoff_match or self.state == STATE_MSG_HEADER:
Bin Meng57b6b192016-06-26 23:24:31 -0700340 # but we are already in a section, this means 'END' is missing
341 # for that section, fix it up.
Bin Meng13b98d92016-06-26 23:24:29 -0700342 if self.in_section:
Simon Glassb5cc3992020-10-29 21:46:23 -0600343 self._add_warn("Missing 'END' in section '%s'" % self.in_section)
Bin Meng13b98d92016-06-26 23:24:29 -0700344 if self.in_section == 'cover':
345 self.series.cover = self.section
346 elif self.in_section == 'notes':
347 if self.is_log:
348 self.series.notes += self.section
349 elif self.in_section == 'commit-notes':
350 if self.is_log:
351 self.commit.notes += self.section
352 else:
Simon Glassb5cc3992020-10-29 21:46:23 -0600353 # This should not happen
354 raise ValueError("Unknown section '%s'" % self.in_section)
Bin Meng13b98d92016-06-26 23:24:29 -0700355 self.in_section = None
356 self.skip_blank = True
357 self.section = []
Bin Meng57b6b192016-06-26 23:24:31 -0700358 # but we are already in a change list, that means a blank line
359 # is missing, fix it up.
360 if self.in_change:
Simon Glassb5cc3992020-10-29 21:46:23 -0600361 self._add_warn("Missing 'blank line' in section '%s-changes'" %
362 self.in_change)
Simon Glassd93720e2020-10-29 21:46:19 -0600363 self._finalise_change()
Sean Anderson6949f702020-05-04 16:28:34 -0400364 self.in_change = None
365 self.change_version = 0
Bin Meng13b98d92016-06-26 23:24:29 -0700366
Simon Glass0d24de92012-01-14 15:12:45 +0000367 # If we are in a section, keep collecting lines until we see END
368 if self.in_section:
369 if line == 'END':
370 if self.in_section == 'cover':
371 self.series.cover = self.section
372 elif self.in_section == 'notes':
373 if self.is_log:
374 self.series.notes += self.section
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100375 elif self.in_section == 'commit-notes':
376 if self.is_log:
377 self.commit.notes += self.section
Simon Glass0d24de92012-01-14 15:12:45 +0000378 else:
Simon Glassb5cc3992020-10-29 21:46:23 -0600379 # This should not happen
380 raise ValueError("Unknown section '%s'" % self.in_section)
Simon Glass0d24de92012-01-14 15:12:45 +0000381 self.in_section = None
382 self.skip_blank = True
383 self.section = []
384 else:
385 self.section.append(line)
386
Patrick Delaunay7058dd02020-07-02 19:08:24 +0200387 # If we are not in a section, it is an unexpected END
388 elif line == 'END':
Simon Glassd06e55a2020-10-29 21:46:17 -0600389 raise ValueError("'END' wihout section")
Patrick Delaunay7058dd02020-07-02 19:08:24 +0200390
Simon Glass0d24de92012-01-14 15:12:45 +0000391 # Detect the commit subject
392 elif not is_blank and self.state == STATE_PATCH_SUBJECT:
393 self.commit.subject = line
394
395 # Detect the tags we want to remove, and skip blank lines
Simon Glass57699042020-10-29 21:46:18 -0600396 elif RE_REMOVE.match(line) and not commit_tag_match:
Simon Glass0d24de92012-01-14 15:12:45 +0000397 self.skip_blank = True
398
399 # TEST= should be the last thing in the commit, so remove
400 # everything after it
401 if line.startswith('TEST='):
402 self.found_test = True
403 elif self.skip_blank and is_blank:
404 self.skip_blank = False
405
Sean Anderson6949f702020-05-04 16:28:34 -0400406 # Detect Cover-xxx tags
Bin Menge7df2182016-06-26 23:24:28 -0700407 elif cover_match:
Sean Anderson6949f702020-05-04 16:28:34 -0400408 name = cover_match.group(1)
409 value = cover_match.group(2)
410 if name == 'letter':
411 self.in_section = 'cover'
412 self.skip_blank = False
413 elif name == 'letter-cc':
Simon Glassd93720e2020-10-29 21:46:19 -0600414 self._add_to_series(line, 'cover-cc', value)
Sean Anderson6949f702020-05-04 16:28:34 -0400415 elif name == 'changes':
416 self.in_change = 'Cover'
Simon Glassd93720e2020-10-29 21:46:19 -0600417 self.change_version = self._parse_version(value, line)
Simon Glassfe2f8d92013-03-20 16:43:00 +0000418
Simon Glass0d24de92012-01-14 15:12:45 +0000419 # If we are in a change list, key collected lines until a blank one
420 elif self.in_change:
421 if is_blank:
422 # Blank line ends this change list
Simon Glassd93720e2020-10-29 21:46:19 -0600423 self._finalise_change()
Sean Anderson6949f702020-05-04 16:28:34 -0400424 self.in_change = None
425 self.change_version = 0
Simon Glass102061b2014-04-20 10:50:14 -0600426 elif line == '---':
Simon Glassd93720e2020-10-29 21:46:19 -0600427 self._finalise_change()
Sean Anderson6949f702020-05-04 16:28:34 -0400428 self.in_change = None
429 self.change_version = 0
Simon Glassd93720e2020-10-29 21:46:19 -0600430 out = self.process_line(line)
Sean Anderson0411fff2020-05-04 16:28:35 -0400431 elif self.is_log:
432 if not leading_whitespace_match:
Simon Glassd93720e2020-10-29 21:46:19 -0600433 self._finalise_change()
Sean Anderson0411fff2020-05-04 16:28:35 -0400434 self.change_lines.append(line)
Simon Glass0d24de92012-01-14 15:12:45 +0000435 self.skip_blank = False
436
437 # Detect Series-xxx tags
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100438 elif series_tag_match:
439 name = series_tag_match.group(1)
440 value = series_tag_match.group(2)
Simon Glass0d24de92012-01-14 15:12:45 +0000441 if name == 'changes':
442 # value is the version number: e.g. 1, or 2
Sean Anderson6949f702020-05-04 16:28:34 -0400443 self.in_change = 'Series'
Simon Glassd93720e2020-10-29 21:46:19 -0600444 self.change_version = self._parse_version(value, line)
Simon Glass0d24de92012-01-14 15:12:45 +0000445 else:
Simon Glassd93720e2020-10-29 21:46:19 -0600446 self._add_to_series(line, name, value)
Simon Glass0d24de92012-01-14 15:12:45 +0000447 self.skip_blank = True
448
Douglas Anderson833e4192019-09-27 09:23:56 -0700449 # Detect Change-Id tags
450 elif change_id_match:
451 value = change_id_match.group(1)
452 if self.is_log:
453 if self.commit.change_id:
Simon Glassd06e55a2020-10-29 21:46:17 -0600454 raise ValueError(
Simon Glass53336e62020-11-03 13:54:11 -0700455 "%s: Two Change-Ids: '%s' vs. '%s'" %
456 (self.commit.hash, self.commit.change_id, value))
Douglas Anderson833e4192019-09-27 09:23:56 -0700457 self.commit.change_id = value
458 self.skip_blank = True
459
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100460 # Detect Commit-xxx tags
461 elif commit_tag_match:
462 name = commit_tag_match.group(1)
463 value = commit_tag_match.group(2)
464 if name == 'notes':
Simon Glasse3a816b2020-10-29 21:46:21 -0600465 self._add_to_commit(name)
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100466 self.skip_blank = True
Sean Anderson6949f702020-05-04 16:28:34 -0400467 elif name == 'changes':
468 self.in_change = 'Commit'
Simon Glassd93720e2020-10-29 21:46:19 -0600469 self.change_version = self._parse_version(value, line)
Patrick Delaunaye5ff9ab2020-07-02 19:52:54 +0200470 else:
Simon Glassb5cc3992020-10-29 21:46:23 -0600471 self._add_warn('Line %d: Ignoring Commit-%s' %
472 (self.linenum, name))
Albert ARIBAUD5c8fdd92013-11-12 11:14:41 +0100473
Simon Glass0d24de92012-01-14 15:12:45 +0000474 # Detect the start of a new commit
475 elif commit_match:
Simon Glassd93720e2020-10-29 21:46:19 -0600476 self._close_commit()
Simon Glass0b5b4092014-10-15 02:27:00 -0600477 self.commit = commit.Commit(commit_match.group(1))
Simon Glass0d24de92012-01-14 15:12:45 +0000478
479 # Detect tags in the commit message
480 elif tag_match:
Simon Glass7207e2b2020-07-05 21:41:57 -0600481 rtag_type, who = tag_match.groups()
Simon Glassd93720e2020-10-29 21:46:19 -0600482 self._add_commit_rtag(rtag_type, who)
Simon Glass0d24de92012-01-14 15:12:45 +0000483 # Remove Tested-by self, since few will take much notice
Simon Glass7207e2b2020-07-05 21:41:57 -0600484 if (rtag_type == 'Tested-by' and
485 who.find(os.getenv('USER') + '@') != -1):
Simon Glass4af99872020-10-29 21:46:28 -0600486 self._add_warn("Ignoring '%s'" % line)
Simon Glass7207e2b2020-07-05 21:41:57 -0600487 elif rtag_type == 'Patch-cc':
488 self.commit.AddCc(who.split(','))
Simon Glass0d24de92012-01-14 15:12:45 +0000489 else:
Simon Glassd0c57192014-08-28 09:43:38 -0600490 out = [line]
Simon Glass0d24de92012-01-14 15:12:45 +0000491
Simon Glass102061b2014-04-20 10:50:14 -0600492 # Suppress duplicate signoffs
493 elif signoff_match:
Simon Glasse752edc2014-08-28 09:43:35 -0600494 if (self.is_log or not self.commit or
Simon Glassd06e55a2020-10-29 21:46:17 -0600495 self.commit.CheckDuplicateSignoff(signoff_match.group(1))):
Simon Glass102061b2014-04-20 10:50:14 -0600496 out = [line]
497
Simon Glass0d24de92012-01-14 15:12:45 +0000498 # Well that means this is an ordinary line
499 else:
Simon Glass0d24de92012-01-14 15:12:45 +0000500 # Look for space before tab
Simon Glassdd147ed2020-10-29 21:46:20 -0600501 mat = RE_SPACE_BEFORE_TAB.match(line)
502 if mat:
Simon Glassb5cc3992020-10-29 21:46:23 -0600503 self._add_warn('Line %d/%d has space before tab' %
504 (self.linenum, mat.start()))
Simon Glass0d24de92012-01-14 15:12:45 +0000505
506 # OK, we have a valid non-blank line
507 out = [line]
508 self.linenum += 1
509 self.skip_blank = False
Simon Glass6b3252e2020-10-29 21:46:37 -0600510
511 if diff_match:
512 self.cur_diff = diff_match.group(1)
513
514 # If this is quoted, keep recent lines
515 if not diff_match and self.linenum > 1 and line:
516 if line.startswith('>'):
517 if not self.was_quoted:
518 self._finalise_snippet()
519 self.recent_line = None
520 if not line_match:
521 self.recent_quoted.append(line)
522 self.was_quoted = True
523 self.recent_diff = self.cur_diff
524 else:
525 self.recent_unquoted.put(line)
526 self.was_quoted = False
527
528 if line_match:
529 self.recent_line = line_match.groups()
530
Simon Glass0d24de92012-01-14 15:12:45 +0000531 if self.state == STATE_DIFFS:
532 pass
533
534 # If this is the start of the diffs section, emit our tags and
535 # change log
536 elif line == '---':
537 self.state = STATE_DIFFS
538
Sean Anderson6949f702020-05-04 16:28:34 -0400539 # Output the tags (signoff first), then change list
Simon Glass0d24de92012-01-14 15:12:45 +0000540 out = []
Simon Glass0d24de92012-01-14 15:12:45 +0000541 log = self.series.MakeChangeLog(self.commit)
Simon Glasse752edc2014-08-28 09:43:35 -0600542 out += [line]
543 if self.commit:
544 out += self.commit.notes
545 out += [''] + log
Simon Glass0d24de92012-01-14 15:12:45 +0000546 elif self.found_test:
Simon Glass57699042020-10-29 21:46:18 -0600547 if not RE_ALLOWED_AFTER_TEST.match(line):
Simon Glass0d24de92012-01-14 15:12:45 +0000548 self.lines_after_test += 1
549
550 return out
551
Simon Glassd93720e2020-10-29 21:46:19 -0600552 def finalise(self):
Simon Glass0d24de92012-01-14 15:12:45 +0000553 """Close out processing of this patch stream"""
Simon Glass6b3252e2020-10-29 21:46:37 -0600554 self._finalise_snippet()
Simon Glassd93720e2020-10-29 21:46:19 -0600555 self._finalise_change()
556 self._close_commit()
Simon Glass0d24de92012-01-14 15:12:45 +0000557 if self.lines_after_test:
Simon Glassb5cc3992020-10-29 21:46:23 -0600558 self._add_warn('Found %d lines after TEST=' % self.lines_after_test)
Simon Glass0d24de92012-01-14 15:12:45 +0000559
Simon Glassd93720e2020-10-29 21:46:19 -0600560 def _write_message_id(self, outfd):
Douglas Anderson833e4192019-09-27 09:23:56 -0700561 """Write the Message-Id into the output.
562
563 This is based on the Change-Id in the original patch, the version,
564 and the prefix.
565
566 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600567 outfd (io.IOBase): Output stream file object
Douglas Anderson833e4192019-09-27 09:23:56 -0700568 """
569 if not self.commit.change_id:
570 return
571
572 # If the count is -1 we're testing, so use a fixed time
573 if self.commit.count == -1:
574 time_now = datetime.datetime(1999, 12, 31, 23, 59, 59)
575 else:
576 time_now = datetime.datetime.now()
577
578 # In theory there is email.utils.make_msgid() which would be nice
579 # to use, but it already produces something way too long and thus
580 # will produce ugly commit lines if someone throws this into
581 # a "Link:" tag in the final commit. So (sigh) roll our own.
582
583 # Start with the time; presumably we wouldn't send the same series
584 # with the same Change-Id at the exact same second.
585 parts = [time_now.strftime("%Y%m%d%H%M%S")]
586
587 # These seem like they would be nice to include.
588 if 'prefix' in self.series:
589 parts.append(self.series['prefix'])
590 if 'version' in self.series:
591 parts.append("v%s" % self.series['version'])
592
593 parts.append(str(self.commit.count + 1))
594
595 # The Change-Id must be last, right before the @
596 parts.append(self.commit.change_id)
597
598 # Join parts together with "." and write it out.
599 outfd.write('Message-Id: <%s@changeid>\n' % '.'.join(parts))
600
Simon Glassd93720e2020-10-29 21:46:19 -0600601 def process_stream(self, infd, outfd):
Simon Glass0d24de92012-01-14 15:12:45 +0000602 """Copy a stream from infd to outfd, filtering out unwanting things.
603
604 This is used to process patch files one at a time.
605
606 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600607 infd (io.IOBase): Input stream file object
608 outfd (io.IOBase): Output stream file object
Simon Glass0d24de92012-01-14 15:12:45 +0000609 """
610 # Extract the filename from each diff, for nice warnings
611 fname = None
612 last_fname = None
613 re_fname = re.compile('diff --git a/(.*) b/.*')
Douglas Anderson833e4192019-09-27 09:23:56 -0700614
Simon Glassd93720e2020-10-29 21:46:19 -0600615 self._write_message_id(outfd)
Douglas Anderson833e4192019-09-27 09:23:56 -0700616
Simon Glass0d24de92012-01-14 15:12:45 +0000617 while True:
618 line = infd.readline()
619 if not line:
620 break
Simon Glassd93720e2020-10-29 21:46:19 -0600621 out = self.process_line(line)
Simon Glass0d24de92012-01-14 15:12:45 +0000622
623 # Try to detect blank lines at EOF
624 for line in out:
625 match = re_fname.match(line)
626 if match:
627 last_fname = fname
628 fname = match.group(1)
629 if line == '+':
630 self.blank_count += 1
631 else:
632 if self.blank_count and (line == '-- ' or match):
Simon Glassb5cc3992020-10-29 21:46:23 -0600633 self._add_warn("Found possible blank line(s) at end of file '%s'" %
634 last_fname)
Simon Glass0d24de92012-01-14 15:12:45 +0000635 outfd.write('+\n' * self.blank_count)
636 outfd.write(line + '\n')
637 self.blank_count = 0
Simon Glassd93720e2020-10-29 21:46:19 -0600638 self.finalise()
Simon Glass0d24de92012-01-14 15:12:45 +0000639
Simon Glass8f9ba3a2020-10-29 21:46:36 -0600640def insert_tags(msg, tags_to_emit):
641 """Add extra tags to a commit message
642
643 The tags are added after an existing block of tags if found, otherwise at
644 the end.
645
646 Args:
647 msg (str): Commit message
648 tags_to_emit (list): List of tags to emit, each a str
649
650 Returns:
651 (str) new message
652 """
653 out = []
654 done = False
655 emit_tags = False
656 for line in msg.splitlines():
657 if not done:
658 signoff_match = RE_SIGNOFF.match(line)
659 tag_match = RE_TAG.match(line)
660 if tag_match or signoff_match:
661 emit_tags = True
662 if emit_tags and not tag_match and not signoff_match:
663 out += tags_to_emit
664 emit_tags = False
665 done = True
666 out.append(line)
667 if not done:
668 out.append('')
669 out += tags_to_emit
670 return '\n'.join(out)
671
672def get_list(commit_range, git_dir=None, count=None):
673 """Get a log of a list of comments
674
675 This returns the output of 'git log' for the selected commits
676
677 Args:
678 commit_range (str): Range of commits to count (e.g. 'HEAD..base')
679 git_dir (str): Path to git repositiory (None to use default)
680 count (int): Number of commits to list, or None for no limit
681
682 Returns
683 str: String containing the contents of the git log
684 """
685 params = gitutil.LogCmd(commit_range, reverse=True, count=count,
686 git_dir=git_dir)
687 return command.RunPipe([params], capture=True).stdout
Simon Glass0d24de92012-01-14 15:12:45 +0000688
Simon Glassd93720e2020-10-29 21:46:19 -0600689def get_metadata_for_list(commit_range, git_dir=None, count=None,
690 series=None, allow_overwrite=False):
Simon Glasse62f9052012-12-15 10:42:06 +0000691 """Reads out patch series metadata from the commits
692
693 This does a 'git log' on the relevant commits and pulls out the tags we
694 are interested in.
695
696 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600697 commit_range (str): Range of commits to count (e.g. 'HEAD..base')
698 git_dir (str): Path to git repositiory (None to use default)
699 count (int): Number of commits to list, or None for no limit
700 series (Series): Object to add information into. By default a new series
Simon Glasse62f9052012-12-15 10:42:06 +0000701 is started.
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600702 allow_overwrite (bool): Allow tags to overwrite an existing tag
703
Simon Glasse62f9052012-12-15 10:42:06 +0000704 Returns:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600705 Series: Object containing information about the commits.
Simon Glasse62f9052012-12-15 10:42:06 +0000706 """
Simon Glass891b7a02014-09-05 19:00:19 -0600707 if not series:
708 series = Series()
Simon Glass950a2312014-09-05 19:00:23 -0600709 series.allow_overwrite = allow_overwrite
Simon Glass8f9ba3a2020-10-29 21:46:36 -0600710 stdout = get_list(commit_range, git_dir, count)
Simon Glassdd147ed2020-10-29 21:46:20 -0600711 pst = PatchStream(series, is_log=True)
Simon Glasse62f9052012-12-15 10:42:06 +0000712 for line in stdout.splitlines():
Simon Glassdd147ed2020-10-29 21:46:20 -0600713 pst.process_line(line)
714 pst.finalise()
Simon Glasse62f9052012-12-15 10:42:06 +0000715 return series
716
Simon Glassd93720e2020-10-29 21:46:19 -0600717def get_metadata(branch, start, count):
Simon Glass0d24de92012-01-14 15:12:45 +0000718 """Reads out patch series metadata from the commits
719
720 This does a 'git log' on the relevant commits and pulls out the tags we
721 are interested in.
722
723 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600724 branch (str): Branch to use (None for current branch)
725 start (int): Commit to start from: 0=branch HEAD, 1=next one, etc.
726 count (int): Number of commits to list
727
728 Returns:
729 Series: Object containing information about the commits.
Simon Glass0d24de92012-01-14 15:12:45 +0000730 """
Simon Glassd93720e2020-10-29 21:46:19 -0600731 return get_metadata_for_list(
732 '%s~%d' % (branch if branch else 'HEAD', start), None, count)
Simon Glass0d24de92012-01-14 15:12:45 +0000733
Simon Glassd93720e2020-10-29 21:46:19 -0600734def get_metadata_for_test(text):
Simon Glass6e87ae12017-05-29 15:31:31 -0600735 """Process metadata from a file containing a git log. Used for tests
736
737 Args:
738 text:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600739
740 Returns:
741 Series: Object containing information about the commits.
Simon Glass6e87ae12017-05-29 15:31:31 -0600742 """
743 series = Series()
Simon Glassdd147ed2020-10-29 21:46:20 -0600744 pst = PatchStream(series, is_log=True)
Simon Glass6e87ae12017-05-29 15:31:31 -0600745 for line in text.splitlines():
Simon Glassdd147ed2020-10-29 21:46:20 -0600746 pst.process_line(line)
747 pst.finalise()
Simon Glass6e87ae12017-05-29 15:31:31 -0600748 return series
749
Simon Glassdd147ed2020-10-29 21:46:20 -0600750def fix_patch(backup_dir, fname, series, cmt):
Simon Glass0d24de92012-01-14 15:12:45 +0000751 """Fix up a patch file, by adding/removing as required.
752
753 We remove our tags from the patch file, insert changes lists, etc.
754 The patch file is processed in place, and overwritten.
755
756 A backup file is put into backup_dir (if not None).
757
758 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600759 backup_dir (str): Path to directory to use to backup the file
760 fname (str): Filename to patch file to process
761 series (Series): Series information about this patch set
762 cmt (Commit): Commit object for this patch file
763
Simon Glass0d24de92012-01-14 15:12:45 +0000764 Return:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600765 list: A list of errors, each str, or [] if all ok.
Simon Glass0d24de92012-01-14 15:12:45 +0000766 """
767 handle, tmpname = tempfile.mkstemp()
Simon Glass272cd852019-10-31 07:42:51 -0600768 outfd = os.fdopen(handle, 'w', encoding='utf-8')
769 infd = open(fname, 'r', encoding='utf-8')
Simon Glassdd147ed2020-10-29 21:46:20 -0600770 pst = PatchStream(series)
771 pst.commit = cmt
772 pst.process_stream(infd, outfd)
Simon Glass0d24de92012-01-14 15:12:45 +0000773 infd.close()
774 outfd.close()
775
776 # Create a backup file if required
777 if backup_dir:
778 shutil.copy(fname, os.path.join(backup_dir, os.path.basename(fname)))
779 shutil.move(tmpname, fname)
Simon Glass313ef5f2020-10-29 21:46:24 -0600780 return cmt.warn
Simon Glass0d24de92012-01-14 15:12:45 +0000781
Simon Glassd93720e2020-10-29 21:46:19 -0600782def fix_patches(series, fnames):
Simon Glass0d24de92012-01-14 15:12:45 +0000783 """Fix up a list of patches identified by filenames
784
785 The patch files are processed in place, and overwritten.
786
787 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600788 series (Series): The Series object
789 fnames (:type: list of str): List of patch files to process
Simon Glass0d24de92012-01-14 15:12:45 +0000790 """
791 # Current workflow creates patches, so we shouldn't need a backup
792 backup_dir = None #tempfile.mkdtemp('clean-patch')
793 count = 0
794 for fname in fnames:
Simon Glassdd147ed2020-10-29 21:46:20 -0600795 cmt = series.commits[count]
796 cmt.patch = fname
797 cmt.count = count
798 result = fix_patch(backup_dir, fname, series, cmt)
Simon Glass0d24de92012-01-14 15:12:45 +0000799 if result:
Simon Glass9994baa2020-10-29 21:46:30 -0600800 print('%d warning%s for %s:' %
801 (len(result), 's' if len(result) > 1 else '', fname))
Simon Glass0d24de92012-01-14 15:12:45 +0000802 for warn in result:
Simon Glass9994baa2020-10-29 21:46:30 -0600803 print('\t%s' % warn)
804 print()
Simon Glass0d24de92012-01-14 15:12:45 +0000805 count += 1
Simon Glass9994baa2020-10-29 21:46:30 -0600806 print('Cleaned %d patch%s' % (count, 'es' if count > 1 else ''))
Simon Glass0d24de92012-01-14 15:12:45 +0000807
Simon Glassd93720e2020-10-29 21:46:19 -0600808def insert_cover_letter(fname, series, count):
Simon Glass0d24de92012-01-14 15:12:45 +0000809 """Inserts a cover letter with the required info into patch 0
810
811 Args:
Simon Glass1cb1c0f2020-10-29 21:46:22 -0600812 fname (str): Input / output filename of the cover letter file
813 series (Series): Series object
814 count (int): Number of patches in the series
Simon Glass0d24de92012-01-14 15:12:45 +0000815 """
Simon Glassdd147ed2020-10-29 21:46:20 -0600816 fil = open(fname, 'r')
817 lines = fil.readlines()
818 fil.close()
Simon Glass0d24de92012-01-14 15:12:45 +0000819
Simon Glassdd147ed2020-10-29 21:46:20 -0600820 fil = open(fname, 'w')
Simon Glass0d24de92012-01-14 15:12:45 +0000821 text = series.cover
822 prefix = series.GetPatchPrefix()
823 for line in lines:
824 if line.startswith('Subject:'):
Wu, Josh35ce2dc2015-04-03 10:51:17 +0800825 # if more than 10 or 100 patches, it should say 00/xx, 000/xxx, etc
826 zero_repeat = int(math.log10(count)) + 1
827 zero = '0' * zero_repeat
828 line = 'Subject: [%s %s/%d] %s\n' % (prefix, zero, count, text[0])
Simon Glass0d24de92012-01-14 15:12:45 +0000829
830 # Insert our cover letter
831 elif line.startswith('*** BLURB HERE ***'):
832 # First the blurb test
833 line = '\n'.join(text[1:]) + '\n'
834 if series.get('notes'):
835 line += '\n'.join(series.notes) + '\n'
836
837 # Now the change list
838 out = series.MakeChangeLog(None)
839 line += '\n' + '\n'.join(out)
Simon Glassdd147ed2020-10-29 21:46:20 -0600840 fil.write(line)
841 fil.close()