blob: 7cbb1a6f628c87fcc95f830480e60953e350f2a9 [file] [log] [blame]
Tom Rini83d290c2018-05-06 17:58:06 -04001# SPDX-License-Identifier: GPL-2.0+
Simon Glassfc3fe1c2013-04-03 11:07:16 +00002# Copyright (c) 2013 The Chromium OS Authors.
3#
4# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
5#
Simon Glassfc3fe1c2013-04-03 11:07:16 +00006
7import collections
Simon Glassfc3fe1c2013-04-03 11:07:16 +00008from datetime import datetime, timedelta
9import glob
10import os
11import re
Simon Glassc05aa032019-10-31 07:42:53 -060012import queue
Simon Glassfc3fe1c2013-04-03 11:07:16 +000013import shutil
Simon Glass2f256642016-09-18 16:48:37 -060014import signal
Simon Glassfc3fe1c2013-04-03 11:07:16 +000015import string
16import sys
Simon Glassd436e382016-09-18 16:48:35 -060017import threading
Simon Glassfc3fe1c2013-04-03 11:07:16 +000018import time
19
Simon Glass190064b2014-08-09 15:33:00 -060020import builderthread
Simon Glassfc3fe1c2013-04-03 11:07:16 +000021import command
22import gitutil
23import terminal
Simon Glass4653a882014-09-05 19:00:07 -060024from terminal import Print
Simon Glassfc3fe1c2013-04-03 11:07:16 +000025import toolchain
26
Simon Glassfc3fe1c2013-04-03 11:07:16 +000027"""
28Theory of Operation
29
30Please see README for user documentation, and you should be familiar with
31that before trying to make sense of this.
32
33Buildman works by keeping the machine as busy as possible, building different
34commits for different boards on multiple CPUs at once.
35
36The source repo (self.git_dir) contains all the commits to be built. Each
37thread works on a single board at a time. It checks out the first commit,
38configures it for that board, then builds it. Then it checks out the next
39commit and builds it (typically without re-configuring). When it runs out
40of commits, it gets another job from the builder and starts again with that
41board.
42
43Clearly the builder threads could work either way - they could check out a
44commit and then built it for all boards. Using separate directories for each
45commit/board pair they could leave their build product around afterwards
46also.
47
48The intent behind building a single board for multiple commits, is to make
49use of incremental builds. Since each commit is built incrementally from
50the previous one, builds are faster. Reconfiguring for a different board
51removes all intermediate object files.
52
53Many threads can be working at once, but each has its own working directory.
54When a thread finishes a build, it puts the output files into a result
55directory.
56
57The base directory used by buildman is normally '../<branch>', i.e.
58a directory higher than the source repository and named after the branch
59being built.
60
61Within the base directory, we have one subdirectory for each commit. Within
62that is one subdirectory for each board. Within that is the build output for
63that commit/board combination.
64
65Buildman also create working directories for each thread, in a .bm-work/
66subdirectory in the base dir.
67
68As an example, say we are building branch 'us-net' for boards 'sandbox' and
69'seaboard', and say that us-net has two commits. We will have directories
70like this:
71
72us-net/ base directory
73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/
74 sandbox/
75 u-boot.bin
76 seaboard/
77 u-boot.bin
78 02_of_02_g4ed4ebc_net--Check-tftp-comp/
79 sandbox/
80 u-boot.bin
81 seaboard/
82 u-boot.bin
83 .bm-work/
84 00/ working directory for thread 0 (contains source checkout)
85 build/ build output
86 01/ working directory for thread 1
87 build/ build output
88 ...
89u-boot/ source directory
90 .git/ repository
91"""
92
Simon Glass35d696d2020-04-09 15:08:36 -060093"""Holds information about a particular error line we are outputing
94
95 char: Character representation: '+': error, '-': fixed error, 'w+': warning,
96 'w-' = fixed warning
97 boards: List of Board objects which have line in the error/warning output
98 errline: The text of the error line
99"""
100ErrLine = collections.namedtuple('ErrLine', 'char,boards,errline')
101
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000102# Possible build outcomes
Simon Glassc05aa032019-10-31 07:42:53 -0600103OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000104
Simon Glass9a6d2e22017-04-12 18:23:26 -0600105# Translate a commit subject into a valid filename (and handle unicode)
Simon Glassc05aa032019-10-31 07:42:53 -0600106trans_valid_chars = str.maketrans('/: ', '---')
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000107
Simon Glassb464f8e2016-11-13 14:25:53 -0700108BASE_CONFIG_FILENAMES = [
109 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
110]
111
112EXTRA_CONFIG_FILENAMES = [
Simon Glass843312d2015-02-05 22:06:15 -0700113 '.config', '.config-spl', '.config-tpl',
114 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
115 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
Simon Glass843312d2015-02-05 22:06:15 -0700116]
117
Simon Glass8270e3c2015-08-25 21:52:14 -0600118class Config:
119 """Holds information about configuration settings for a board."""
Simon Glassb464f8e2016-11-13 14:25:53 -0700120 def __init__(self, config_filename, target):
Simon Glass8270e3c2015-08-25 21:52:14 -0600121 self.target = target
122 self.config = {}
Simon Glassb464f8e2016-11-13 14:25:53 -0700123 for fname in config_filename:
Simon Glass8270e3c2015-08-25 21:52:14 -0600124 self.config[fname] = {}
125
126 def Add(self, fname, key, value):
127 self.config[fname][key] = value
128
129 def __hash__(self):
130 val = 0
131 for fname in self.config:
Simon Glassc05aa032019-10-31 07:42:53 -0600132 for key, value in self.config[fname].items():
133 print(key, value)
Simon Glass8270e3c2015-08-25 21:52:14 -0600134 val = val ^ hash(key) & hash(value)
135 return val
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000136
Alex Kiernan48ae4122018-05-31 04:48:34 +0000137class Environment:
138 """Holds information about environment variables for a board."""
139 def __init__(self, target):
140 self.target = target
141 self.environment = {}
142
143 def Add(self, key, value):
144 self.environment[key] = value
145
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000146class Builder:
147 """Class for building U-Boot for a particular commit.
148
149 Public members: (many should ->private)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000150 already_done: Number of builds already completed
151 base_dir: Base directory to use for builder
152 checkout: True to check out source, False to skip that step.
153 This is used for testing.
154 col: terminal.Color() object
155 count: Number of commits to build
156 do_make: Method to call to invoke Make
157 fail: Number of builds that failed due to error
158 force_build: Force building even if a build already exists
159 force_config_on_failure: If a commit fails for a board, disable
160 incremental building for the next commit we build for that
161 board, so that we will see all warnings/errors again.
Simon Glass4266dc22014-07-13 12:22:31 -0600162 force_build_failures: If a previously-built build (i.e. built on
163 a previous run of buildman) is marked as failed, rebuild it.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000164 git_dir: Git directory containing source repository
165 last_line_len: Length of the last line we printed (used for erasing
166 it with new progress information)
167 num_jobs: Number of jobs to run at once (passed to make as -j)
168 num_threads: Number of builder threads to run
169 out_queue: Queue of results to process
170 re_make_err: Compiled regular expression for ignore_lines
171 queue: Queue of jobs to run
172 threads: List of active threads
173 toolchains: Toolchains object to use for building
174 upto: Current commit number we are building (0.count-1)
175 warned: Number of builds that produced at least one warning
Simon Glass97e91522014-07-14 17:51:02 -0600176 force_reconfig: Reconfigure U-Boot on each comiit. This disables
177 incremental building, where buildman reconfigures on the first
178 commit for a baord, and then just does an incremental build for
179 the following commits. In fact buildman will reconfigure and
180 retry for any failing commits, so generally the only effect of
181 this option is to slow things down.
Simon Glass189a4962014-07-14 17:51:03 -0600182 in_tree: Build U-Boot in-tree instead of specifying an output
183 directory separate from the source code. This option is really
184 only useful for testing in-tree builds.
Simon Glassd829f122020-03-18 09:42:42 -0600185 work_in_output: Use the output directory as the work directory and
186 don't write to a separate output directory.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000187
188 Private members:
189 _base_board_dict: Last-summarised Dict of boards
190 _base_err_lines: Last-summarised list of errors
Simon Glasse30965d2014-08-28 09:43:44 -0600191 _base_warn_lines: Last-summarised list of warnings
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000192 _build_period_us: Time taken for a single build (float object).
193 _complete_delay: Expected delay until completion (timedelta)
194 _next_delay_update: Next time we plan to display a progress update
195 (datatime)
196 _show_unknown: Show unknown boards (those not built) in summary
197 _timestamps: List of timestamps for the completion of the last
198 last _timestamp_count builds. Each is a datetime object.
199 _timestamp_count: Number of timestamps to keep in our list.
200 _working_dir: Base working directory containing all threads
201 """
202 class Outcome:
203 """Records a build outcome for a single make invocation
204
205 Public Members:
206 rc: Outcome value (OUTCOME_...)
207 err_lines: List of error lines or [] if none
208 sizes: Dictionary of image size information, keyed by filename
209 - Each value is itself a dictionary containing
210 values for 'text', 'data' and 'bss', being the integer
211 size in bytes of each section.
212 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
213 value is itself a dictionary:
214 key: function name
215 value: Size of function in bytes
Simon Glass843312d2015-02-05 22:06:15 -0700216 config: Dictionary keyed by filename - e.g. '.config'. Each
217 value is itself a dictionary:
218 key: config name
219 value: config value
Alex Kiernan48ae4122018-05-31 04:48:34 +0000220 environment: Dictionary keyed by environment variable, Each
221 value is the value of environment variable.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000222 """
Alex Kiernan48ae4122018-05-31 04:48:34 +0000223 def __init__(self, rc, err_lines, sizes, func_sizes, config,
224 environment):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000225 self.rc = rc
226 self.err_lines = err_lines
227 self.sizes = sizes
228 self.func_sizes = func_sizes
Simon Glass843312d2015-02-05 22:06:15 -0700229 self.config = config
Alex Kiernan48ae4122018-05-31 04:48:34 +0000230 self.environment = environment
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000231
232 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
Simon Glass5971ab52014-12-01 17:33:55 -0700233 gnu_make='make', checkout=True, show_unknown=True, step=1,
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600234 no_subdirs=False, full_path=False, verbose_build=False,
Simon Glassb50113f2016-11-13 14:25:51 -0700235 incremental=False, per_board_out_dir=False,
Daniel Schwierzeck2371d1b2018-01-26 16:31:05 +0100236 config_only=False, squash_config_y=False,
Simon Glassd829f122020-03-18 09:42:42 -0600237 warnings_as_errors=False, work_in_output=False):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000238 """Create a new Builder object
239
240 Args:
241 toolchains: Toolchains object to use for building
242 base_dir: Base directory to use for builder
243 git_dir: Git directory containing source repository
244 num_threads: Number of builder threads to run
245 num_jobs: Number of jobs to run at once (passed to make as -j)
Masahiro Yamada99796922014-07-22 11:19:09 +0900246 gnu_make: the command name of GNU Make.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000247 checkout: True to check out source, False to skip that step.
248 This is used for testing.
249 show_unknown: Show unknown boards (those not built) in summary
250 step: 1 to process every commit, n to process every nth commit
Simon Glassbb1501f2014-12-01 17:34:00 -0700251 no_subdirs: Don't create subdirectories when building current
252 source for a single board
253 full_path: Return the full path in CROSS_COMPILE and don't set
254 PATH
Simon Glassd2ce6582014-12-01 17:34:07 -0700255 verbose_build: Run build with V=1 and don't use 'make -s'
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600256 incremental: Always perform incremental builds; don't run make
257 mrproper when configuring
258 per_board_out_dir: Build in a separate persistent directory per
259 board rather than a thread-specific directory
Simon Glassb50113f2016-11-13 14:25:51 -0700260 config_only: Only configure each build, don't build it
Simon Glassb464f8e2016-11-13 14:25:53 -0700261 squash_config_y: Convert CONFIG options with the value 'y' to '1'
Daniel Schwierzeck2371d1b2018-01-26 16:31:05 +0100262 warnings_as_errors: Treat all compiler warnings as errors
Simon Glassd829f122020-03-18 09:42:42 -0600263 work_in_output: Use the output directory as the work directory and
264 don't write to a separate output directory.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000265 """
266 self.toolchains = toolchains
267 self.base_dir = base_dir
Simon Glassd829f122020-03-18 09:42:42 -0600268 if work_in_output:
269 self._working_dir = base_dir
270 else:
271 self._working_dir = os.path.join(base_dir, '.bm-work')
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000272 self.threads = []
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000273 self.do_make = self.Make
Masahiro Yamada99796922014-07-22 11:19:09 +0900274 self.gnu_make = gnu_make
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000275 self.checkout = checkout
276 self.num_threads = num_threads
277 self.num_jobs = num_jobs
278 self.already_done = 0
279 self.force_build = False
280 self.git_dir = git_dir
281 self._show_unknown = show_unknown
282 self._timestamp_count = 10
283 self._build_period_us = None
284 self._complete_delay = None
285 self._next_delay_update = datetime.now()
286 self.force_config_on_failure = True
Simon Glass4266dc22014-07-13 12:22:31 -0600287 self.force_build_failures = False
Simon Glass97e91522014-07-14 17:51:02 -0600288 self.force_reconfig = False
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000289 self._step = step
Simon Glass189a4962014-07-14 17:51:03 -0600290 self.in_tree = False
Simon Glass28370c12014-08-09 15:33:06 -0600291 self._error_lines = 0
Simon Glass5971ab52014-12-01 17:33:55 -0700292 self.no_subdirs = no_subdirs
Simon Glassbb1501f2014-12-01 17:34:00 -0700293 self.full_path = full_path
Simon Glassd2ce6582014-12-01 17:34:07 -0700294 self.verbose_build = verbose_build
Simon Glassb50113f2016-11-13 14:25:51 -0700295 self.config_only = config_only
Simon Glassb464f8e2016-11-13 14:25:53 -0700296 self.squash_config_y = squash_config_y
297 self.config_filenames = BASE_CONFIG_FILENAMES
Simon Glassd829f122020-03-18 09:42:42 -0600298 self.work_in_output = work_in_output
Simon Glassb464f8e2016-11-13 14:25:53 -0700299 if not self.squash_config_y:
300 self.config_filenames += EXTRA_CONFIG_FILENAMES
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000301
Daniel Schwierzeck2371d1b2018-01-26 16:31:05 +0100302 self.warnings_as_errors = warnings_as_errors
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000303 self.col = terminal.Color()
304
Simon Glasse30965d2014-08-28 09:43:44 -0600305 self._re_function = re.compile('(.*): In function.*')
306 self._re_files = re.compile('In file included from.*')
307 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
Simon Glass2d483332018-11-06 16:02:11 -0700308 self._re_dtb_warning = re.compile('(.*): Warning .*')
Simon Glasse30965d2014-08-28 09:43:44 -0600309 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
310
Simon Glassc05aa032019-10-31 07:42:53 -0600311 self.queue = queue.Queue()
312 self.out_queue = queue.Queue()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000313 for i in range(self.num_threads):
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600314 t = builderthread.BuilderThread(self, i, incremental,
315 per_board_out_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000316 t.setDaemon(True)
317 t.start()
318 self.threads.append(t)
319
320 self.last_line_len = 0
Simon Glass190064b2014-08-09 15:33:00 -0600321 t = builderthread.ResultThread(self)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000322 t.setDaemon(True)
323 t.start()
324 self.threads.append(t)
325
326 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
327 self.re_make_err = re.compile('|'.join(ignore_lines))
328
Simon Glass2f256642016-09-18 16:48:37 -0600329 # Handle existing graceful with SIGINT / Ctrl-C
330 signal.signal(signal.SIGINT, self.signal_handler)
331
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000332 def __del__(self):
333 """Get rid of all threads created by the builder"""
334 for t in self.threads:
335 del t
336
Simon Glass2f256642016-09-18 16:48:37 -0600337 def signal_handler(self, signal, frame):
338 sys.exit(1)
339
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600340 def SetDisplayOptions(self, show_errors=False, show_sizes=False,
Simon Glassed966652014-08-28 09:43:43 -0600341 show_detail=False, show_bloat=False,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000342 list_error_boards=False, show_config=False,
343 show_environment=False):
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600344 """Setup display options for the builder.
345
346 show_errors: True to show summarised error/warning info
347 show_sizes: Show size deltas
Simon Glassf9c094b2020-03-18 09:42:43 -0600348 show_detail: Show size delta detail for each board if show_sizes
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600349 show_bloat: Show detail for each function
Simon Glassed966652014-08-28 09:43:43 -0600350 list_error_boards: Show the boards which caused each error/warning
Simon Glass843312d2015-02-05 22:06:15 -0700351 show_config: Show config deltas
Alex Kiernan48ae4122018-05-31 04:48:34 +0000352 show_environment: Show environment deltas
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600353 """
354 self._show_errors = show_errors
355 self._show_sizes = show_sizes
356 self._show_detail = show_detail
357 self._show_bloat = show_bloat
Simon Glassed966652014-08-28 09:43:43 -0600358 self._list_error_boards = list_error_boards
Simon Glass843312d2015-02-05 22:06:15 -0700359 self._show_config = show_config
Alex Kiernan48ae4122018-05-31 04:48:34 +0000360 self._show_environment = show_environment
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600361
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000362 def _AddTimestamp(self):
363 """Add a new timestamp to the list and record the build period.
364
365 The build period is the length of time taken to perform a single
366 build (one board, one commit).
367 """
368 now = datetime.now()
369 self._timestamps.append(now)
370 count = len(self._timestamps)
371 delta = self._timestamps[-1] - self._timestamps[0]
372 seconds = delta.total_seconds()
373
374 # If we have enough data, estimate build period (time taken for a
375 # single build) and therefore completion time.
376 if count > 1 and self._next_delay_update < now:
377 self._next_delay_update = now + timedelta(seconds=2)
378 if seconds > 0:
379 self._build_period = float(seconds) / count
380 todo = self.count - self.upto
381 self._complete_delay = timedelta(microseconds=
382 self._build_period * todo * 1000000)
383 # Round it
384 self._complete_delay -= timedelta(
385 microseconds=self._complete_delay.microseconds)
386
387 if seconds > 60:
388 self._timestamps.popleft()
389 count -= 1
390
391 def ClearLine(self, length):
392 """Clear any characters on the current line
393
394 Make way for a new line of length 'length', by outputting enough
395 spaces to clear out the old line. Then remember the new length for
396 next time.
397
398 Args:
399 length: Length of new line, in characters
400 """
401 if length < self.last_line_len:
Simon Glass4653a882014-09-05 19:00:07 -0600402 Print(' ' * (self.last_line_len - length), newline=False)
403 Print('\r', newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000404 self.last_line_len = length
405 sys.stdout.flush()
406
407 def SelectCommit(self, commit, checkout=True):
408 """Checkout the selected commit for this build
409 """
410 self.commit = commit
411 if checkout and self.checkout:
412 gitutil.Checkout(commit.hash)
413
414 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
415 """Run make
416
417 Args:
418 commit: Commit object that is being built
419 brd: Board object that is being built
Roger Meierfd18a892014-08-20 22:10:29 +0200420 stage: Stage that we are at (mrproper, config, build)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000421 cwd: Directory where make should be run
422 args: Arguments to pass to make
423 kwargs: Arguments to pass to command.RunPipe()
424 """
Masahiro Yamada99796922014-07-22 11:19:09 +0900425 cmd = [self.gnu_make] + list(args)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000426 result = command.RunPipe([cmd], capture=True, capture_stderr=True,
Simon Glasse62a24c2018-09-17 23:55:42 -0600427 cwd=cwd, raise_on_error=False, infile='/dev/null', **kwargs)
Simon Glass40f11fc2015-02-05 22:06:12 -0700428 if self.verbose_build:
429 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
430 result.combined = '%s\n' % (' '.join(cmd)) + result.combined
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000431 return result
432
433 def ProcessResult(self, result):
434 """Process the result of a build, showing progress information
435
436 Args:
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600437 result: A CommandResult object, which indicates the result for
438 a single build
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000439 """
440 col = terminal.Color()
441 if result:
442 target = result.brd.target
443
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000444 self.upto += 1
445 if result.return_code != 0:
446 self.fail += 1
447 elif result.stderr:
448 self.warned += 1
449 if result.already_done:
450 self.already_done += 1
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600451 if self._verbose:
Simon Glass4653a882014-09-05 19:00:07 -0600452 Print('\r', newline=False)
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600453 self.ClearLine(0)
454 boards_selected = {target : result.brd}
455 self.ResetResultSummary(boards_selected)
456 self.ProduceResultSummary(result.commit_upto, self.commits,
457 boards_selected)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000458 else:
459 target = '(starting)'
460
461 # Display separate counts for ok, warned and fail
462 ok = self.upto - self.warned - self.fail
463 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
464 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
465 line += self.col.Color(self.col.RED, '%5d' % self.fail)
466
467 name = ' /%-5d ' % self.count
468
469 # Add our current completion time estimate
470 self._AddTimestamp()
471 if self._complete_delay:
472 name += '%s : ' % self._complete_delay
473 # When building all boards for a commit, we can print a commit
474 # progress message.
475 if result and result.commit_upto is None:
476 name += 'commit %2d/%-3d' % (self.commit_upto + 1,
477 self.commit_count)
478
479 name += target
Simon Glass4653a882014-09-05 19:00:07 -0600480 Print(line + name, newline=False)
Simon Glass960421e2016-11-15 15:32:59 -0700481 length = 16 + len(name)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000482 self.ClearLine(length)
483
484 def _GetOutputDir(self, commit_upto):
485 """Get the name of the output directory for a commit number
486
487 The output directory is typically .../<branch>/<commit>.
488
489 Args:
490 commit_upto: Commit number to use (0..self.count-1)
491 """
Simon Glass5971ab52014-12-01 17:33:55 -0700492 commit_dir = None
Simon Glassfea58582014-08-09 15:32:59 -0600493 if self.commits:
494 commit = self.commits[commit_upto]
495 subject = commit.subject.translate(trans_valid_chars)
Simon Glass925f6ad2020-03-18 09:42:45 -0600496 # See _GetOutputSpaceRemovals() which parses this name
Simon Glassfea58582014-08-09 15:32:59 -0600497 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
498 self.commit_count, commit.hash, subject[:20]))
Simon Glass5971ab52014-12-01 17:33:55 -0700499 elif not self.no_subdirs:
Simon Glassfea58582014-08-09 15:32:59 -0600500 commit_dir = 'current'
Simon Glass5971ab52014-12-01 17:33:55 -0700501 if not commit_dir:
502 return self.base_dir
503 return os.path.join(self.base_dir, commit_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000504
505 def GetBuildDir(self, commit_upto, target):
506 """Get the name of the build directory for a commit number
507
508 The build directory is typically .../<branch>/<commit>/<target>.
509
510 Args:
511 commit_upto: Commit number to use (0..self.count-1)
512 target: Target name
513 """
514 output_dir = self._GetOutputDir(commit_upto)
515 return os.path.join(output_dir, target)
516
517 def GetDoneFile(self, commit_upto, target):
518 """Get the name of the done file for a commit number
519
520 Args:
521 commit_upto: Commit number to use (0..self.count-1)
522 target: Target name
523 """
524 return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
525
526 def GetSizesFile(self, commit_upto, target):
527 """Get the name of the sizes file for a commit number
528
529 Args:
530 commit_upto: Commit number to use (0..self.count-1)
531 target: Target name
532 """
533 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
534
535 def GetFuncSizesFile(self, commit_upto, target, elf_fname):
536 """Get the name of the funcsizes file for a commit number and ELF file
537
538 Args:
539 commit_upto: Commit number to use (0..self.count-1)
540 target: Target name
541 elf_fname: Filename of elf image
542 """
543 return os.path.join(self.GetBuildDir(commit_upto, target),
544 '%s.sizes' % elf_fname.replace('/', '-'))
545
546 def GetObjdumpFile(self, commit_upto, target, elf_fname):
547 """Get the name of the objdump file for a commit number and ELF file
548
549 Args:
550 commit_upto: Commit number to use (0..self.count-1)
551 target: Target name
552 elf_fname: Filename of elf image
553 """
554 return os.path.join(self.GetBuildDir(commit_upto, target),
555 '%s.objdump' % elf_fname.replace('/', '-'))
556
557 def GetErrFile(self, commit_upto, target):
558 """Get the name of the err file for a commit number
559
560 Args:
561 commit_upto: Commit number to use (0..self.count-1)
562 target: Target name
563 """
564 output_dir = self.GetBuildDir(commit_upto, target)
565 return os.path.join(output_dir, 'err')
566
567 def FilterErrors(self, lines):
568 """Filter out errors in which we have no interest
569
570 We should probably use map().
571
572 Args:
573 lines: List of error lines, each a string
574 Returns:
575 New list with only interesting lines included
576 """
577 out_lines = []
578 for line in lines:
579 if not self.re_make_err.search(line):
580 out_lines.append(line)
581 return out_lines
582
583 def ReadFuncSizes(self, fname, fd):
584 """Read function sizes from the output of 'nm'
585
586 Args:
587 fd: File containing data to read
588 fname: Filename we are reading from (just for errors)
589
590 Returns:
591 Dictionary containing size of each function in bytes, indexed by
592 function name.
593 """
594 sym = {}
595 for line in fd.readlines():
596 try:
Tom Rinid08c38c2019-12-06 15:31:31 -0500597 if line.strip():
598 size, type, name = line[:-1].split()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000599 except:
Simon Glass4653a882014-09-05 19:00:07 -0600600 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000601 continue
602 if type in 'tTdDbB':
603 # function names begin with '.' on 64-bit powerpc
604 if '.' in name[1:]:
605 name = 'static.' + name.split('.')[0]
606 sym[name] = sym.get(name, 0) + int(size, 16)
607 return sym
608
Simon Glass843312d2015-02-05 22:06:15 -0700609 def _ProcessConfig(self, fname):
610 """Read in a .config, autoconf.mk or autoconf.h file
611
612 This function handles all config file types. It ignores comments and
613 any #defines which don't start with CONFIG_.
614
615 Args:
616 fname: Filename to read
617
618 Returns:
619 Dictionary:
620 key: Config name (e.g. CONFIG_DM)
621 value: Config value (e.g. 1)
622 """
623 config = {}
624 if os.path.exists(fname):
625 with open(fname) as fd:
626 for line in fd:
627 line = line.strip()
628 if line.startswith('#define'):
629 values = line[8:].split(' ', 1)
630 if len(values) > 1:
631 key, value = values
632 else:
633 key = values[0]
Simon Glassb464f8e2016-11-13 14:25:53 -0700634 value = '1' if self.squash_config_y else ''
Simon Glass843312d2015-02-05 22:06:15 -0700635 if not key.startswith('CONFIG_'):
636 continue
637 elif not line or line[0] in ['#', '*', '/']:
638 continue
639 else:
640 key, value = line.split('=', 1)
Simon Glassb464f8e2016-11-13 14:25:53 -0700641 if self.squash_config_y and value == 'y':
642 value = '1'
Simon Glass843312d2015-02-05 22:06:15 -0700643 config[key] = value
644 return config
645
Alex Kiernan48ae4122018-05-31 04:48:34 +0000646 def _ProcessEnvironment(self, fname):
647 """Read in a uboot.env file
648
649 This function reads in environment variables from a file.
650
651 Args:
652 fname: Filename to read
653
654 Returns:
655 Dictionary:
656 key: environment variable (e.g. bootlimit)
657 value: value of environment variable (e.g. 1)
658 """
659 environment = {}
660 if os.path.exists(fname):
661 with open(fname) as fd:
662 for line in fd.read().split('\0'):
663 try:
664 key, value = line.split('=', 1)
665 environment[key] = value
666 except ValueError:
667 # ignore lines we can't parse
668 pass
669 return environment
670
Simon Glass843312d2015-02-05 22:06:15 -0700671 def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000672 read_config, read_environment):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000673 """Work out the outcome of a build.
674
675 Args:
676 commit_upto: Commit number to check (0..n-1)
677 target: Target board to check
678 read_func_sizes: True to read function size information
Simon Glass843312d2015-02-05 22:06:15 -0700679 read_config: True to read .config and autoconf.h files
Alex Kiernan48ae4122018-05-31 04:48:34 +0000680 read_environment: True to read uboot.env files
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000681
682 Returns:
683 Outcome object
684 """
685 done_file = self.GetDoneFile(commit_upto, target)
686 sizes_file = self.GetSizesFile(commit_upto, target)
687 sizes = {}
688 func_sizes = {}
Simon Glass843312d2015-02-05 22:06:15 -0700689 config = {}
Alex Kiernan48ae4122018-05-31 04:48:34 +0000690 environment = {}
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000691 if os.path.exists(done_file):
692 with open(done_file, 'r') as fd:
Simon Glass347ea0b2019-04-26 19:02:23 -0600693 try:
694 return_code = int(fd.readline())
695 except ValueError:
696 # The file may be empty due to running out of disk space.
697 # Try a rebuild
698 return_code = 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000699 err_lines = []
700 err_file = self.GetErrFile(commit_upto, target)
701 if os.path.exists(err_file):
702 with open(err_file, 'r') as fd:
703 err_lines = self.FilterErrors(fd.readlines())
704
705 # Decide whether the build was ok, failed or created warnings
706 if return_code:
707 rc = OUTCOME_ERROR
708 elif len(err_lines):
709 rc = OUTCOME_WARNING
710 else:
711 rc = OUTCOME_OK
712
713 # Convert size information to our simple format
714 if os.path.exists(sizes_file):
715 with open(sizes_file, 'r') as fd:
716 for line in fd.readlines():
717 values = line.split()
718 rodata = 0
719 if len(values) > 6:
720 rodata = int(values[6], 16)
721 size_dict = {
722 'all' : int(values[0]) + int(values[1]) +
723 int(values[2]),
724 'text' : int(values[0]) - rodata,
725 'data' : int(values[1]),
726 'bss' : int(values[2]),
727 'rodata' : rodata,
728 }
729 sizes[values[5]] = size_dict
730
731 if read_func_sizes:
732 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
733 for fname in glob.glob(pattern):
734 with open(fname, 'r') as fd:
735 dict_name = os.path.basename(fname).replace('.sizes',
736 '')
737 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
738
Simon Glass843312d2015-02-05 22:06:15 -0700739 if read_config:
740 output_dir = self.GetBuildDir(commit_upto, target)
Simon Glassb464f8e2016-11-13 14:25:53 -0700741 for name in self.config_filenames:
Simon Glass843312d2015-02-05 22:06:15 -0700742 fname = os.path.join(output_dir, name)
743 config[name] = self._ProcessConfig(fname)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000744
Alex Kiernan48ae4122018-05-31 04:48:34 +0000745 if read_environment:
746 output_dir = self.GetBuildDir(commit_upto, target)
747 fname = os.path.join(output_dir, 'uboot.env')
748 environment = self._ProcessEnvironment(fname)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000749
Alex Kiernan48ae4122018-05-31 04:48:34 +0000750 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config,
751 environment)
752
753 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {})
Simon Glass843312d2015-02-05 22:06:15 -0700754
755 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000756 read_config, read_environment):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000757 """Calculate a summary of the results of building a commit.
758
759 Args:
760 board_selected: Dict containing boards to summarise
761 commit_upto: Commit number to summarize (0..self.count-1)
762 read_func_sizes: True to read function size information
Simon Glass843312d2015-02-05 22:06:15 -0700763 read_config: True to read .config and autoconf.h files
Alex Kiernan48ae4122018-05-31 04:48:34 +0000764 read_environment: True to read uboot.env files
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000765
766 Returns:
767 Tuple:
768 Dict containing boards which passed building this commit.
769 keyed by board.target
Simon Glasse30965d2014-08-28 09:43:44 -0600770 List containing a summary of error lines
Simon Glassed966652014-08-28 09:43:43 -0600771 Dict keyed by error line, containing a list of the Board
772 objects with that error
Simon Glasse30965d2014-08-28 09:43:44 -0600773 List containing a summary of warning lines
774 Dict keyed by error line, containing a list of the Board
775 objects with that warning
Simon Glass8270e3c2015-08-25 21:52:14 -0600776 Dictionary keyed by board.target. Each value is a dictionary:
777 key: filename - e.g. '.config'
Simon Glass843312d2015-02-05 22:06:15 -0700778 value is itself a dictionary:
779 key: config name
780 value: config value
Alex Kiernan48ae4122018-05-31 04:48:34 +0000781 Dictionary keyed by board.target. Each value is a dictionary:
782 key: environment variable
783 value: value of environment variable
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000784 """
Simon Glasse30965d2014-08-28 09:43:44 -0600785 def AddLine(lines_summary, lines_boards, line, board):
786 line = line.rstrip()
787 if line in lines_boards:
788 lines_boards[line].append(board)
789 else:
790 lines_boards[line] = [board]
791 lines_summary.append(line)
792
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000793 board_dict = {}
794 err_lines_summary = []
Simon Glassed966652014-08-28 09:43:43 -0600795 err_lines_boards = {}
Simon Glasse30965d2014-08-28 09:43:44 -0600796 warn_lines_summary = []
797 warn_lines_boards = {}
Simon Glass843312d2015-02-05 22:06:15 -0700798 config = {}
Alex Kiernan48ae4122018-05-31 04:48:34 +0000799 environment = {}
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000800
Simon Glassc05aa032019-10-31 07:42:53 -0600801 for board in boards_selected.values():
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000802 outcome = self.GetBuildOutcome(commit_upto, board.target,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000803 read_func_sizes, read_config,
804 read_environment)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000805 board_dict[board.target] = outcome
Simon Glasse30965d2014-08-28 09:43:44 -0600806 last_func = None
807 last_was_warning = False
808 for line in outcome.err_lines:
809 if line:
810 if (self._re_function.match(line) or
811 self._re_files.match(line)):
812 last_func = line
Simon Glassed966652014-08-28 09:43:43 -0600813 else:
Simon Glass2d483332018-11-06 16:02:11 -0700814 is_warning = (self._re_warning.match(line) or
815 self._re_dtb_warning.match(line))
Simon Glasse30965d2014-08-28 09:43:44 -0600816 is_note = self._re_note.match(line)
817 if is_warning or (last_was_warning and is_note):
818 if last_func:
819 AddLine(warn_lines_summary, warn_lines_boards,
820 last_func, board)
821 AddLine(warn_lines_summary, warn_lines_boards,
822 line, board)
823 else:
824 if last_func:
825 AddLine(err_lines_summary, err_lines_boards,
826 last_func, board)
827 AddLine(err_lines_summary, err_lines_boards,
828 line, board)
829 last_was_warning = is_warning
830 last_func = None
Simon Glassb464f8e2016-11-13 14:25:53 -0700831 tconfig = Config(self.config_filenames, board.target)
832 for fname in self.config_filenames:
Simon Glass843312d2015-02-05 22:06:15 -0700833 if outcome.config:
Simon Glassc05aa032019-10-31 07:42:53 -0600834 for key, value in outcome.config[fname].items():
Simon Glass8270e3c2015-08-25 21:52:14 -0600835 tconfig.Add(fname, key, value)
836 config[board.target] = tconfig
Simon Glass843312d2015-02-05 22:06:15 -0700837
Alex Kiernan48ae4122018-05-31 04:48:34 +0000838 tenvironment = Environment(board.target)
839 if outcome.environment:
Simon Glassc05aa032019-10-31 07:42:53 -0600840 for key, value in outcome.environment.items():
Alex Kiernan48ae4122018-05-31 04:48:34 +0000841 tenvironment.Add(key, value)
842 environment[board.target] = tenvironment
843
Simon Glasse30965d2014-08-28 09:43:44 -0600844 return (board_dict, err_lines_summary, err_lines_boards,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000845 warn_lines_summary, warn_lines_boards, config, environment)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000846
847 def AddOutcome(self, board_dict, arch_list, changes, char, color):
848 """Add an output to our list of outcomes for each architecture
849
850 This simple function adds failing boards (changes) to the
851 relevant architecture string, so we can print the results out
852 sorted by architecture.
853
854 Args:
855 board_dict: Dict containing all boards
856 arch_list: Dict keyed by arch name. Value is a string containing
857 a list of board names which failed for that arch.
858 changes: List of boards to add to arch_list
859 color: terminal.Colour object
860 """
861 done_arch = {}
862 for target in changes:
863 if target in board_dict:
864 arch = board_dict[target].arch
865 else:
866 arch = 'unknown'
867 str = self.col.Color(color, ' ' + target)
868 if not arch in done_arch:
Simon Glass63c619e2015-02-05 22:06:11 -0700869 str = ' %s %s' % (self.col.Color(color, char), str)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000870 done_arch[arch] = True
871 if not arch in arch_list:
872 arch_list[arch] = str
873 else:
874 arch_list[arch] += str
875
876
877 def ColourNum(self, num):
878 color = self.col.RED if num > 0 else self.col.GREEN
879 if num == 0:
880 return '0'
881 return self.col.Color(color, str(num))
882
883 def ResetResultSummary(self, board_selected):
884 """Reset the results summary ready for use.
885
886 Set up the base board list to be all those selected, and set the
887 error lines to empty.
888
889 Following this, calls to PrintResultSummary() will use this
890 information to work out what has changed.
891
892 Args:
893 board_selected: Dict containing boards to summarise, keyed by
894 board.target
895 """
896 self._base_board_dict = {}
897 for board in board_selected:
Alex Kiernan48ae4122018-05-31 04:48:34 +0000898 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {},
899 {})
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000900 self._base_err_lines = []
Simon Glasse30965d2014-08-28 09:43:44 -0600901 self._base_warn_lines = []
902 self._base_err_line_boards = {}
903 self._base_warn_line_boards = {}
Simon Glass8270e3c2015-08-25 21:52:14 -0600904 self._base_config = None
Alex Kiernan48ae4122018-05-31 04:48:34 +0000905 self._base_environment = None
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000906
907 def PrintFuncSizeDetail(self, fname, old, new):
908 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
909 delta, common = [], {}
910
911 for a in old:
912 if a in new:
913 common[a] = 1
914
915 for name in old:
916 if name not in common:
917 remove += 1
918 down += old[name]
919 delta.append([-old[name], name])
920
921 for name in new:
922 if name not in common:
923 add += 1
924 up += new[name]
925 delta.append([new[name], name])
926
927 for name in common:
928 diff = new.get(name, 0) - old.get(name, 0)
929 if diff > 0:
930 grow, up = grow + 1, up + diff
931 elif diff < 0:
932 shrink, down = shrink + 1, down - diff
933 delta.append([diff, name])
934
935 delta.sort()
936 delta.reverse()
937
938 args = [add, -remove, grow, -shrink, up, -down, up - down]
Tom Rinid5686a62017-05-22 13:48:52 -0400939 if max(args) == 0 and min(args) == 0:
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000940 return
941 args = [self.ColourNum(x) for x in args]
942 indent = ' ' * 15
Simon Glass4653a882014-09-05 19:00:07 -0600943 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
944 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
945 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
946 'delta'))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000947 for diff, name in delta:
948 if diff:
949 color = self.col.RED if diff > 0 else self.col.GREEN
950 msg = '%s %-38s %7s %7s %+7d' % (indent, name,
951 old.get(name, '-'), new.get(name,'-'), diff)
Simon Glass4653a882014-09-05 19:00:07 -0600952 Print(msg, colour=color)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000953
954
955 def PrintSizeDetail(self, target_list, show_bloat):
956 """Show details size information for each board
957
958 Args:
959 target_list: List of targets, each a dict containing:
960 'target': Target name
961 'total_diff': Total difference in bytes across all areas
962 <part_name>: Difference for that part
963 show_bloat: Show detail for each function
964 """
965 targets_by_diff = sorted(target_list, reverse=True,
966 key=lambda x: x['_total_diff'])
967 for result in targets_by_diff:
968 printed_target = False
969 for name in sorted(result):
970 diff = result[name]
971 if name.startswith('_'):
972 continue
973 if diff != 0:
974 color = self.col.RED if diff > 0 else self.col.GREEN
975 msg = ' %s %+d' % (name, diff)
976 if not printed_target:
Simon Glass4653a882014-09-05 19:00:07 -0600977 Print('%10s %-15s:' % ('', result['_target']),
978 newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000979 printed_target = True
Simon Glass4653a882014-09-05 19:00:07 -0600980 Print(msg, colour=color, newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000981 if printed_target:
Simon Glass4653a882014-09-05 19:00:07 -0600982 Print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000983 if show_bloat:
984 target = result['_target']
985 outcome = result['_outcome']
986 base_outcome = self._base_board_dict[target]
987 for fname in outcome.func_sizes:
988 self.PrintFuncSizeDetail(fname,
989 base_outcome.func_sizes[fname],
990 outcome.func_sizes[fname])
991
992
993 def PrintSizeSummary(self, board_selected, board_dict, show_detail,
994 show_bloat):
995 """Print a summary of image sizes broken down by section.
996
997 The summary takes the form of one line per architecture. The
998 line contains deltas for each of the sections (+ means the section
Flavio Suligoi9de5c392020-01-29 09:56:05 +0100999 got bigger, - means smaller). The numbers are the average number
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001000 of bytes that a board in this section increased by.
1001
1002 For example:
1003 powerpc: (622 boards) text -0.0
1004 arm: (285 boards) text -0.0
1005 nds32: (3 boards) text -8.0
1006
1007 Args:
1008 board_selected: Dict containing boards to summarise, keyed by
1009 board.target
1010 board_dict: Dict containing boards for which we built this
1011 commit, keyed by board.target. The value is an Outcome object.
Simon Glassf9c094b2020-03-18 09:42:43 -06001012 show_detail: Show size delta detail for each board
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001013 show_bloat: Show detail for each function
1014 """
1015 arch_list = {}
1016 arch_count = {}
1017
1018 # Calculate changes in size for different image parts
1019 # The previous sizes are in Board.sizes, for each board
1020 for target in board_dict:
1021 if target not in board_selected:
1022 continue
1023 base_sizes = self._base_board_dict[target].sizes
1024 outcome = board_dict[target]
1025 sizes = outcome.sizes
1026
1027 # Loop through the list of images, creating a dict of size
1028 # changes for each image/part. We end up with something like
1029 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1030 # which means that U-Boot data increased by 5 bytes and SPL
1031 # text decreased by 4.
1032 err = {'_target' : target}
1033 for image in sizes:
1034 if image in base_sizes:
1035 base_image = base_sizes[image]
1036 # Loop through the text, data, bss parts
1037 for part in sorted(sizes[image]):
1038 diff = sizes[image][part] - base_image[part]
1039 col = None
1040 if diff:
1041 if image == 'u-boot':
1042 name = part
1043 else:
1044 name = image + ':' + part
1045 err[name] = diff
1046 arch = board_selected[target].arch
1047 if not arch in arch_count:
1048 arch_count[arch] = 1
1049 else:
1050 arch_count[arch] += 1
1051 if not sizes:
1052 pass # Only add to our list when we have some stats
1053 elif not arch in arch_list:
1054 arch_list[arch] = [err]
1055 else:
1056 arch_list[arch].append(err)
1057
1058 # We now have a list of image size changes sorted by arch
1059 # Print out a summary of these
Simon Glassc05aa032019-10-31 07:42:53 -06001060 for arch, target_list in arch_list.items():
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001061 # Get total difference for each type
1062 totals = {}
1063 for result in target_list:
1064 total = 0
Simon Glassc05aa032019-10-31 07:42:53 -06001065 for name, diff in result.items():
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001066 if name.startswith('_'):
1067 continue
1068 total += diff
1069 if name in totals:
1070 totals[name] += diff
1071 else:
1072 totals[name] = diff
1073 result['_total_diff'] = total
1074 result['_outcome'] = board_dict[result['_target']]
1075
1076 count = len(target_list)
1077 printed_arch = False
1078 for name in sorted(totals):
1079 diff = totals[name]
1080 if diff:
1081 # Display the average difference in this name for this
1082 # architecture
1083 avg_diff = float(diff) / count
1084 color = self.col.RED if avg_diff > 0 else self.col.GREEN
1085 msg = ' %s %+1.1f' % (name, avg_diff)
1086 if not printed_arch:
Simon Glass4653a882014-09-05 19:00:07 -06001087 Print('%10s: (for %d/%d boards)' % (arch, count,
1088 arch_count[arch]), newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001089 printed_arch = True
Simon Glass4653a882014-09-05 19:00:07 -06001090 Print(msg, colour=color, newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001091
1092 if printed_arch:
Simon Glass4653a882014-09-05 19:00:07 -06001093 Print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001094 if show_detail:
1095 self.PrintSizeDetail(target_list, show_bloat)
1096
1097
1098 def PrintResultSummary(self, board_selected, board_dict, err_lines,
Simon Glasse30965d2014-08-28 09:43:44 -06001099 err_line_boards, warn_lines, warn_line_boards,
Alex Kiernan48ae4122018-05-31 04:48:34 +00001100 config, environment, show_sizes, show_detail,
1101 show_bloat, show_config, show_environment):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001102 """Compare results with the base results and display delta.
1103
1104 Only boards mentioned in board_selected will be considered. This
1105 function is intended to be called repeatedly with the results of
1106 each commit. It therefore shows a 'diff' between what it saw in
1107 the last call and what it sees now.
1108
1109 Args:
1110 board_selected: Dict containing boards to summarise, keyed by
1111 board.target
1112 board_dict: Dict containing boards for which we built this
1113 commit, keyed by board.target. The value is an Outcome object.
1114 err_lines: A list of errors for this commit, or [] if there is
1115 none, or we don't want to print errors
Simon Glassed966652014-08-28 09:43:43 -06001116 err_line_boards: Dict keyed by error line, containing a list of
1117 the Board objects with that error
Simon Glasse30965d2014-08-28 09:43:44 -06001118 warn_lines: A list of warnings for this commit, or [] if there is
1119 none, or we don't want to print errors
1120 warn_line_boards: Dict keyed by warning line, containing a list of
1121 the Board objects with that warning
Simon Glass843312d2015-02-05 22:06:15 -07001122 config: Dictionary keyed by filename - e.g. '.config'. Each
1123 value is itself a dictionary:
1124 key: config name
1125 value: config value
Alex Kiernan48ae4122018-05-31 04:48:34 +00001126 environment: Dictionary keyed by environment variable, Each
1127 value is the value of environment variable.
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001128 show_sizes: Show image size deltas
Simon Glassf9c094b2020-03-18 09:42:43 -06001129 show_detail: Show size delta detail for each board if show_sizes
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001130 show_bloat: Show detail for each function
Simon Glass843312d2015-02-05 22:06:15 -07001131 show_config: Show config changes
Alex Kiernan48ae4122018-05-31 04:48:34 +00001132 show_environment: Show environment changes
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001133 """
Simon Glasse30965d2014-08-28 09:43:44 -06001134 def _BoardList(line, line_boards):
Simon Glassed966652014-08-28 09:43:43 -06001135 """Helper function to get a line of boards containing a line
1136
1137 Args:
1138 line: Error line to search for
Simon Glass35d696d2020-04-09 15:08:36 -06001139 line_boards: boards to search, each a Board
Simon Glassed966652014-08-28 09:43:43 -06001140 Return:
Simon Glass35d696d2020-04-09 15:08:36 -06001141 List of boards with that error line, or [] if the user has not
1142 requested such a list
Simon Glassed966652014-08-28 09:43:43 -06001143 """
Simon Glass35d696d2020-04-09 15:08:36 -06001144 boards = []
1145 board_set = set()
Simon Glassed966652014-08-28 09:43:43 -06001146 if self._list_error_boards:
Simon Glasse30965d2014-08-28 09:43:44 -06001147 for board in line_boards[line]:
Simon Glass35d696d2020-04-09 15:08:36 -06001148 if not board in board_set:
1149 boards.append(board)
1150 board_set.add(board)
1151 return boards
Simon Glassed966652014-08-28 09:43:43 -06001152
Simon Glasse30965d2014-08-28 09:43:44 -06001153 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1154 char):
Simon Glass35d696d2020-04-09 15:08:36 -06001155 """Calculate the required output based on changes in errors
1156
1157 Args:
1158 base_lines: List of errors/warnings for previous commit
1159 base_line_boards: Dict keyed by error line, containing a list
1160 of the Board objects with that error in the previous commit
1161 lines: List of errors/warning for this commit, each a str
1162 line_boards: Dict keyed by error line, containing a list
1163 of the Board objects with that error in this commit
1164 char: Character representing error ('') or warning ('w'). The
1165 broken ('+') or fixed ('-') characters are added in this
1166 function
1167
1168 Returns:
1169 Tuple
1170 List of ErrLine objects for 'better' lines
1171 List of ErrLine objects for 'worse' lines
1172 """
Simon Glasse30965d2014-08-28 09:43:44 -06001173 better_lines = []
1174 worse_lines = []
1175 for line in lines:
1176 if line not in base_lines:
Simon Glass35d696d2020-04-09 15:08:36 -06001177 errline = ErrLine(char + '+', _BoardList(line, line_boards),
1178 line)
1179 worse_lines.append(errline)
Simon Glasse30965d2014-08-28 09:43:44 -06001180 for line in base_lines:
1181 if line not in lines:
Simon Glass35d696d2020-04-09 15:08:36 -06001182 errline = ErrLine(char + '-',
1183 _BoardList(line, base_line_boards), line)
1184 better_lines.append(errline)
Simon Glasse30965d2014-08-28 09:43:44 -06001185 return better_lines, worse_lines
1186
Simon Glass843312d2015-02-05 22:06:15 -07001187 def _CalcConfig(delta, name, config):
1188 """Calculate configuration changes
1189
1190 Args:
1191 delta: Type of the delta, e.g. '+'
1192 name: name of the file which changed (e.g. .config)
1193 config: configuration change dictionary
1194 key: config name
1195 value: config value
1196 Returns:
1197 String containing the configuration changes which can be
1198 printed
1199 """
1200 out = ''
1201 for key in sorted(config.keys()):
1202 out += '%s=%s ' % (key, config[key])
Simon Glass8270e3c2015-08-25 21:52:14 -06001203 return '%s %s: %s' % (delta, name, out)
Simon Glass843312d2015-02-05 22:06:15 -07001204
Simon Glass8270e3c2015-08-25 21:52:14 -06001205 def _AddConfig(lines, name, config_plus, config_minus, config_change):
1206 """Add changes in configuration to a list
Simon Glass843312d2015-02-05 22:06:15 -07001207
1208 Args:
Simon Glass8270e3c2015-08-25 21:52:14 -06001209 lines: list to add to
1210 name: config file name
Simon Glass843312d2015-02-05 22:06:15 -07001211 config_plus: configurations added, dictionary
1212 key: config name
1213 value: config value
1214 config_minus: configurations removed, dictionary
1215 key: config name
1216 value: config value
1217 config_change: configurations changed, dictionary
1218 key: config name
1219 value: config value
1220 """
1221 if config_plus:
Simon Glass8270e3c2015-08-25 21:52:14 -06001222 lines.append(_CalcConfig('+', name, config_plus))
Simon Glass843312d2015-02-05 22:06:15 -07001223 if config_minus:
Simon Glass8270e3c2015-08-25 21:52:14 -06001224 lines.append(_CalcConfig('-', name, config_minus))
Simon Glass843312d2015-02-05 22:06:15 -07001225 if config_change:
Simon Glass8270e3c2015-08-25 21:52:14 -06001226 lines.append(_CalcConfig('c', name, config_change))
1227
1228 def _OutputConfigInfo(lines):
1229 for line in lines:
1230 if not line:
1231 continue
1232 if line[0] == '+':
1233 col = self.col.GREEN
1234 elif line[0] == '-':
1235 col = self.col.RED
1236 elif line[0] == 'c':
1237 col = self.col.YELLOW
1238 Print(' ' + line, newline=True, colour=col)
1239
Simon Glassb206d872020-04-09 15:08:28 -06001240 def _OutputErrLines(err_lines, colour):
1241 """Output the line of error/warning lines, if not empty
1242
1243 Also increments self._error_lines if err_lines not empty
1244
1245 Args:
Simon Glass35d696d2020-04-09 15:08:36 -06001246 err_lines: List of ErrLine objects, each an error or warning
1247 line, possibly including a list of boards with that
1248 error/warning
Simon Glassb206d872020-04-09 15:08:28 -06001249 colour: Colour to use for output
1250 """
1251 if err_lines:
Simon Glass8c9a2672020-04-09 15:08:37 -06001252 out_list = []
Simon Glass35d696d2020-04-09 15:08:36 -06001253 for line in err_lines:
1254 boards = ''
1255 names = [board.target for board in line.boards]
Simon Glass8c9a2672020-04-09 15:08:37 -06001256 board_str = ','.join(names) if names else ''
1257 if board_str:
1258 out = self.col.Color(colour, line.char + '(')
1259 out += self.col.Color(self.col.MAGENTA, board_str,
1260 bright=False)
1261 out += self.col.Color(colour, ') %s' % line.errline)
1262 else:
1263 out = self.col.Color(colour, line.char + line.errline)
1264 out_list.append(out)
1265 Print('\n'.join(out_list))
Simon Glassb206d872020-04-09 15:08:28 -06001266 self._error_lines += 1
1267
Simon Glass843312d2015-02-05 22:06:15 -07001268
Simon Glass4cf2b222018-11-06 16:02:12 -07001269 ok_boards = [] # List of boards fixed since last commit
Simon Glass6af71012018-11-06 16:02:13 -07001270 warn_boards = [] # List of boards with warnings since last commit
Simon Glass4cf2b222018-11-06 16:02:12 -07001271 err_boards = [] # List of new broken boards since last commit
1272 new_boards = [] # List of boards that didn't exist last time
1273 unknown_boards = [] # List of boards that were not built
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001274
1275 for target in board_dict:
1276 if target not in board_selected:
1277 continue
1278
1279 # If the board was built last time, add its outcome to a list
1280 if target in self._base_board_dict:
1281 base_outcome = self._base_board_dict[target].rc
1282 outcome = board_dict[target]
1283 if outcome.rc == OUTCOME_UNKNOWN:
Simon Glass4cf2b222018-11-06 16:02:12 -07001284 unknown_boards.append(target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001285 elif outcome.rc < base_outcome:
Simon Glass6af71012018-11-06 16:02:13 -07001286 if outcome.rc == OUTCOME_WARNING:
1287 warn_boards.append(target)
1288 else:
1289 ok_boards.append(target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001290 elif outcome.rc > base_outcome:
Simon Glass6af71012018-11-06 16:02:13 -07001291 if outcome.rc == OUTCOME_WARNING:
1292 warn_boards.append(target)
1293 else:
1294 err_boards.append(target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001295 else:
Simon Glass4cf2b222018-11-06 16:02:12 -07001296 new_boards.append(target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001297
Simon Glassb206d872020-04-09 15:08:28 -06001298 # Get a list of errors and warnings that have appeared, and disappeared
Simon Glasse30965d2014-08-28 09:43:44 -06001299 better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1300 self._base_err_line_boards, err_lines, err_line_boards, '')
1301 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1302 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001303
1304 # Display results by arch
Simon Glass6af71012018-11-06 16:02:13 -07001305 if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards,
1306 worse_err, better_err, worse_warn, better_warn)):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001307 arch_list = {}
Simon Glass4cf2b222018-11-06 16:02:12 -07001308 self.AddOutcome(board_selected, arch_list, ok_boards, '',
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001309 self.col.GREEN)
Simon Glass6af71012018-11-06 16:02:13 -07001310 self.AddOutcome(board_selected, arch_list, warn_boards, 'w+',
1311 self.col.YELLOW)
Simon Glass4cf2b222018-11-06 16:02:12 -07001312 self.AddOutcome(board_selected, arch_list, err_boards, '+',
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001313 self.col.RED)
Simon Glass4cf2b222018-11-06 16:02:12 -07001314 self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001315 if self._show_unknown:
Simon Glass4cf2b222018-11-06 16:02:12 -07001316 self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001317 self.col.MAGENTA)
Simon Glassc05aa032019-10-31 07:42:53 -06001318 for arch, target_list in arch_list.items():
Simon Glass4653a882014-09-05 19:00:07 -06001319 Print('%10s: %s' % (arch, target_list))
Simon Glass28370c12014-08-09 15:33:06 -06001320 self._error_lines += 1
Simon Glassb206d872020-04-09 15:08:28 -06001321 _OutputErrLines(better_err, colour=self.col.GREEN)
1322 _OutputErrLines(worse_err, colour=self.col.RED)
1323 _OutputErrLines(better_warn, colour=self.col.CYAN)
Simon Glass5627bd92020-04-09 15:08:35 -06001324 _OutputErrLines(worse_warn, colour=self.col.YELLOW)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001325
1326 if show_sizes:
1327 self.PrintSizeSummary(board_selected, board_dict, show_detail,
1328 show_bloat)
1329
Alex Kiernan48ae4122018-05-31 04:48:34 +00001330 if show_environment and self._base_environment:
1331 lines = []
1332
1333 for target in board_dict:
1334 if target not in board_selected:
1335 continue
1336
1337 tbase = self._base_environment[target]
1338 tenvironment = environment[target]
1339 environment_plus = {}
1340 environment_minus = {}
1341 environment_change = {}
1342 base = tbase.environment
Simon Glassc05aa032019-10-31 07:42:53 -06001343 for key, value in tenvironment.environment.items():
Alex Kiernan48ae4122018-05-31 04:48:34 +00001344 if key not in base:
1345 environment_plus[key] = value
Simon Glassc05aa032019-10-31 07:42:53 -06001346 for key, value in base.items():
Alex Kiernan48ae4122018-05-31 04:48:34 +00001347 if key not in tenvironment.environment:
1348 environment_minus[key] = value
Simon Glassc05aa032019-10-31 07:42:53 -06001349 for key, value in base.items():
Alex Kiernan48ae4122018-05-31 04:48:34 +00001350 new_value = tenvironment.environment.get(key)
1351 if new_value and value != new_value:
1352 desc = '%s -> %s' % (value, new_value)
1353 environment_change[key] = desc
1354
1355 _AddConfig(lines, target, environment_plus, environment_minus,
1356 environment_change)
1357
1358 _OutputConfigInfo(lines)
1359
Simon Glass8270e3c2015-08-25 21:52:14 -06001360 if show_config and self._base_config:
1361 summary = {}
1362 arch_config_plus = {}
1363 arch_config_minus = {}
1364 arch_config_change = {}
1365 arch_list = []
1366
1367 for target in board_dict:
1368 if target not in board_selected:
Simon Glass843312d2015-02-05 22:06:15 -07001369 continue
Simon Glass8270e3c2015-08-25 21:52:14 -06001370 arch = board_selected[target].arch
1371 if arch not in arch_list:
1372 arch_list.append(arch)
1373
1374 for arch in arch_list:
1375 arch_config_plus[arch] = {}
1376 arch_config_minus[arch] = {}
1377 arch_config_change[arch] = {}
Simon Glassb464f8e2016-11-13 14:25:53 -07001378 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001379 arch_config_plus[arch][name] = {}
1380 arch_config_minus[arch][name] = {}
1381 arch_config_change[arch][name] = {}
1382
1383 for target in board_dict:
1384 if target not in board_selected:
1385 continue
1386
1387 arch = board_selected[target].arch
1388
1389 all_config_plus = {}
1390 all_config_minus = {}
1391 all_config_change = {}
1392 tbase = self._base_config[target]
1393 tconfig = config[target]
1394 lines = []
Simon Glassb464f8e2016-11-13 14:25:53 -07001395 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001396 if not tconfig.config[name]:
1397 continue
1398 config_plus = {}
1399 config_minus = {}
1400 config_change = {}
1401 base = tbase.config[name]
Simon Glassc05aa032019-10-31 07:42:53 -06001402 for key, value in tconfig.config[name].items():
Simon Glass8270e3c2015-08-25 21:52:14 -06001403 if key not in base:
1404 config_plus[key] = value
1405 all_config_plus[key] = value
Simon Glassc05aa032019-10-31 07:42:53 -06001406 for key, value in base.items():
Simon Glass8270e3c2015-08-25 21:52:14 -06001407 if key not in tconfig.config[name]:
1408 config_minus[key] = value
1409 all_config_minus[key] = value
Simon Glassc05aa032019-10-31 07:42:53 -06001410 for key, value in base.items():
Simon Glass8270e3c2015-08-25 21:52:14 -06001411 new_value = tconfig.config.get(key)
1412 if new_value and value != new_value:
1413 desc = '%s -> %s' % (value, new_value)
1414 config_change[key] = desc
1415 all_config_change[key] = desc
1416
1417 arch_config_plus[arch][name].update(config_plus)
1418 arch_config_minus[arch][name].update(config_minus)
1419 arch_config_change[arch][name].update(config_change)
1420
1421 _AddConfig(lines, name, config_plus, config_minus,
1422 config_change)
1423 _AddConfig(lines, 'all', all_config_plus, all_config_minus,
1424 all_config_change)
1425 summary[target] = '\n'.join(lines)
1426
1427 lines_by_target = {}
Simon Glassc05aa032019-10-31 07:42:53 -06001428 for target, lines in summary.items():
Simon Glass8270e3c2015-08-25 21:52:14 -06001429 if lines in lines_by_target:
1430 lines_by_target[lines].append(target)
1431 else:
1432 lines_by_target[lines] = [target]
1433
1434 for arch in arch_list:
1435 lines = []
1436 all_plus = {}
1437 all_minus = {}
1438 all_change = {}
Simon Glassb464f8e2016-11-13 14:25:53 -07001439 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001440 all_plus.update(arch_config_plus[arch][name])
1441 all_minus.update(arch_config_minus[arch][name])
1442 all_change.update(arch_config_change[arch][name])
1443 _AddConfig(lines, name, arch_config_plus[arch][name],
1444 arch_config_minus[arch][name],
1445 arch_config_change[arch][name])
1446 _AddConfig(lines, 'all', all_plus, all_minus, all_change)
1447 #arch_summary[target] = '\n'.join(lines)
1448 if lines:
1449 Print('%s:' % arch)
1450 _OutputConfigInfo(lines)
1451
Simon Glassc05aa032019-10-31 07:42:53 -06001452 for lines, targets in lines_by_target.items():
Simon Glass8270e3c2015-08-25 21:52:14 -06001453 if not lines:
1454 continue
1455 Print('%s :' % ' '.join(sorted(targets)))
1456 _OutputConfigInfo(lines.split('\n'))
1457
Simon Glass843312d2015-02-05 22:06:15 -07001458
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001459 # Save our updated information for the next call to this function
1460 self._base_board_dict = board_dict
1461 self._base_err_lines = err_lines
Simon Glasse30965d2014-08-28 09:43:44 -06001462 self._base_warn_lines = warn_lines
1463 self._base_err_line_boards = err_line_boards
1464 self._base_warn_line_boards = warn_line_boards
Simon Glass843312d2015-02-05 22:06:15 -07001465 self._base_config = config
Alex Kiernan48ae4122018-05-31 04:48:34 +00001466 self._base_environment = environment
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001467
1468 # Get a list of boards that did not get built, if needed
1469 not_built = []
1470 for board in board_selected:
1471 if not board in board_dict:
1472 not_built.append(board)
1473 if not_built:
Simon Glass4653a882014-09-05 19:00:07 -06001474 Print("Boards not built (%d): %s" % (len(not_built),
1475 ', '.join(not_built)))
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001476
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001477 def ProduceResultSummary(self, commit_upto, commits, board_selected):
Simon Glasse30965d2014-08-28 09:43:44 -06001478 (board_dict, err_lines, err_line_boards, warn_lines,
Alex Kiernan48ae4122018-05-31 04:48:34 +00001479 warn_line_boards, config, environment) = self.GetResultSummary(
Simon Glassed966652014-08-28 09:43:43 -06001480 board_selected, commit_upto,
Simon Glass843312d2015-02-05 22:06:15 -07001481 read_func_sizes=self._show_bloat,
Alex Kiernan48ae4122018-05-31 04:48:34 +00001482 read_config=self._show_config,
1483 read_environment=self._show_environment)
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001484 if commits:
1485 msg = '%02d: %s' % (commit_upto + 1,
1486 commits[commit_upto].subject)
Simon Glass4653a882014-09-05 19:00:07 -06001487 Print(msg, colour=self.col.BLUE)
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001488 self.PrintResultSummary(board_selected, board_dict,
Simon Glassed966652014-08-28 09:43:43 -06001489 err_lines if self._show_errors else [], err_line_boards,
Simon Glasse30965d2014-08-28 09:43:44 -06001490 warn_lines if self._show_errors else [], warn_line_boards,
Alex Kiernan48ae4122018-05-31 04:48:34 +00001491 config, environment, self._show_sizes, self._show_detail,
1492 self._show_bloat, self._show_config, self._show_environment)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001493
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001494 def ShowSummary(self, commits, board_selected):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001495 """Show a build summary for U-Boot for a given board list.
1496
1497 Reset the result summary, then repeatedly call GetResultSummary on
1498 each commit's results, then display the differences we see.
1499
1500 Args:
1501 commit: Commit objects to summarise
1502 board_selected: Dict containing boards to summarise
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001503 """
Simon Glassfea58582014-08-09 15:32:59 -06001504 self.commit_count = len(commits) if commits else 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001505 self.commits = commits
1506 self.ResetResultSummary(board_selected)
Simon Glass28370c12014-08-09 15:33:06 -06001507 self._error_lines = 0
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001508
1509 for commit_upto in range(0, self.commit_count, self._step):
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001510 self.ProduceResultSummary(commit_upto, commits, board_selected)
Simon Glass28370c12014-08-09 15:33:06 -06001511 if not self._error_lines:
Simon Glass4653a882014-09-05 19:00:07 -06001512 Print('(no errors to report)', colour=self.col.GREEN)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001513
1514
1515 def SetupBuild(self, board_selected, commits):
1516 """Set up ready to start a build.
1517
1518 Args:
1519 board_selected: Selected boards to build
1520 commits: Selected commits to build
1521 """
1522 # First work out how many commits we will build
Simon Glassc05aa032019-10-31 07:42:53 -06001523 count = (self.commit_count + self._step - 1) // self._step
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001524 self.count = len(board_selected) * count
1525 self.upto = self.warned = self.fail = 0
1526 self._timestamps = collections.deque()
1527
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001528 def GetThreadDir(self, thread_num):
1529 """Get the directory path to the working dir for a thread.
1530
1531 Args:
1532 thread_num: Number of thread to check.
1533 """
Simon Glassd829f122020-03-18 09:42:42 -06001534 if self.work_in_output:
1535 return self._working_dir
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001536 return os.path.join(self._working_dir, '%02d' % thread_num)
1537
Simon Glassfea58582014-08-09 15:32:59 -06001538 def _PrepareThread(self, thread_num, setup_git):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001539 """Prepare the working directory for a thread.
1540
1541 This clones or fetches the repo into the thread's work directory.
1542
1543 Args:
1544 thread_num: Thread number (0, 1, ...)
Simon Glassfea58582014-08-09 15:32:59 -06001545 setup_git: True to set up a git repo clone
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001546 """
1547 thread_dir = self.GetThreadDir(thread_num)
Simon Glass190064b2014-08-09 15:33:00 -06001548 builderthread.Mkdir(thread_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001549 git_dir = os.path.join(thread_dir, '.git')
1550
1551 # Clone the repo if it doesn't already exist
1552 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1553 # we have a private index but uses the origin repo's contents?
Simon Glassfea58582014-08-09 15:32:59 -06001554 if setup_git and self.git_dir:
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001555 src_dir = os.path.abspath(self.git_dir)
1556 if os.path.exists(git_dir):
1557 gitutil.Fetch(git_dir, thread_dir)
1558 else:
Simon Glass21f0eb32016-09-18 16:48:31 -06001559 Print('\rCloning repo for thread %d' % thread_num,
1560 newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001561 gitutil.Clone(src_dir, thread_dir)
Simon Glass21f0eb32016-09-18 16:48:31 -06001562 Print('\r%s\r' % (' ' * 30), newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001563
Simon Glassfea58582014-08-09 15:32:59 -06001564 def _PrepareWorkingSpace(self, max_threads, setup_git):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001565 """Prepare the working directory for use.
1566
1567 Set up the git repo for each thread.
1568
1569 Args:
1570 max_threads: Maximum number of threads we expect to need.
Simon Glassfea58582014-08-09 15:32:59 -06001571 setup_git: True to set up a git repo clone
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001572 """
Simon Glass190064b2014-08-09 15:33:00 -06001573 builderthread.Mkdir(self._working_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001574 for thread in range(max_threads):
Simon Glassfea58582014-08-09 15:32:59 -06001575 self._PrepareThread(thread, setup_git)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001576
Simon Glass925f6ad2020-03-18 09:42:45 -06001577 def _GetOutputSpaceRemovals(self):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001578 """Get the output directories ready to receive files.
1579
Simon Glass925f6ad2020-03-18 09:42:45 -06001580 Figure out what needs to be deleted in the output directory before it
1581 can be used. We only delete old buildman directories which have the
1582 expected name pattern. See _GetOutputDir().
1583
1584 Returns:
1585 List of full paths of directories to remove
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001586 """
Simon Glass1a915672014-12-01 17:33:53 -07001587 if not self.commits:
1588 return
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001589 dir_list = []
1590 for commit_upto in range(self.commit_count):
1591 dir_list.append(self._GetOutputDir(commit_upto))
1592
Simon Glassb222abe2016-09-18 16:48:32 -06001593 to_remove = []
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001594 for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1595 if dirname not in dir_list:
Simon Glass925f6ad2020-03-18 09:42:45 -06001596 leaf = dirname[len(self.base_dir) + 1:]
1597 m = re.match('[0-9]+_of_[0-9]+_g[0-9a-f]+_.*', leaf)
1598 if m:
1599 to_remove.append(dirname)
1600 return to_remove
1601
1602 def _PrepareOutputSpace(self):
1603 """Get the output directories ready to receive files.
1604
1605 We delete any output directories which look like ones we need to
1606 create. Having left over directories is confusing when the user wants
1607 to check the output manually.
1608 """
1609 to_remove = self._GetOutputSpaceRemovals()
Simon Glassb222abe2016-09-18 16:48:32 -06001610 if to_remove:
Simon Glassb2d89bc2020-03-18 09:42:46 -06001611 Print('Removing %d old build directories...' % len(to_remove),
Simon Glassb222abe2016-09-18 16:48:32 -06001612 newline=False)
1613 for dirname in to_remove:
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001614 shutil.rmtree(dirname)
Simon Glassb2d89bc2020-03-18 09:42:46 -06001615 Print('done')
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001616
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001617 def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001618 """Build all commits for a list of boards
1619
1620 Args:
1621 commits: List of commits to be build, each a Commit object
1622 boards_selected: Dict of selected boards, key is target name,
1623 value is Board object
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001624 keep_outputs: True to save build output files
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001625 verbose: Display build results as they are completed
Simon Glass2c3deb92014-08-28 09:43:39 -06001626 Returns:
1627 Tuple containing:
1628 - number of boards that failed to build
1629 - number of boards that issued warnings
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001630 """
Simon Glassfea58582014-08-09 15:32:59 -06001631 self.commit_count = len(commits) if commits else 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001632 self.commits = commits
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001633 self._verbose = verbose
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001634
1635 self.ResetResultSummary(board_selected)
Thierry Redingf3d015c2014-08-19 10:22:39 +02001636 builderthread.Mkdir(self.base_dir, parents = True)
Simon Glassfea58582014-08-09 15:32:59 -06001637 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1638 commits is not None)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001639 self._PrepareOutputSpace()
Simon Glass745b3952016-09-18 16:48:33 -06001640 Print('\rStarting build...', newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001641 self.SetupBuild(board_selected, commits)
1642 self.ProcessResult(None)
1643
1644 # Create jobs to build all commits for each board
Simon Glassc05aa032019-10-31 07:42:53 -06001645 for brd in board_selected.values():
Simon Glass190064b2014-08-09 15:33:00 -06001646 job = builderthread.BuilderJob()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001647 job.board = brd
1648 job.commits = commits
1649 job.keep_outputs = keep_outputs
Simon Glassd829f122020-03-18 09:42:42 -06001650 job.work_in_output = self.work_in_output
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001651 job.step = self._step
1652 self.queue.put(job)
1653
Simon Glassd436e382016-09-18 16:48:35 -06001654 term = threading.Thread(target=self.queue.join)
1655 term.setDaemon(True)
1656 term.start()
1657 while term.isAlive():
1658 term.join(100)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001659
1660 # Wait until we have processed all output
1661 self.out_queue.join()
Simon Glass4653a882014-09-05 19:00:07 -06001662 Print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001663 self.ClearLine(0)
Simon Glass2c3deb92014-08-28 09:43:39 -06001664 return (self.fail, self.warned)