blob: 1b61e3a837ef1082d56a0a5e770b45239424e503 [file] [log] [blame]
Tom Rini83d290c2018-05-06 17:58:06 -04001# SPDX-License-Identifier: GPL-2.0+
Simon Glassfc3fe1c2013-04-03 11:07:16 +00002# Copyright (c) 2013 The Chromium OS Authors.
3#
4# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
5#
Simon Glassfc3fe1c2013-04-03 11:07:16 +00006
7import collections
Simon Glassfc3fe1c2013-04-03 11:07:16 +00008from datetime import datetime, timedelta
9import glob
10import os
11import re
Simon Glassc05aa032019-10-31 07:42:53 -060012import queue
Simon Glassfc3fe1c2013-04-03 11:07:16 +000013import shutil
Simon Glass2f256642016-09-18 16:48:37 -060014import signal
Simon Glassfc3fe1c2013-04-03 11:07:16 +000015import string
16import sys
Simon Glassd436e382016-09-18 16:48:35 -060017import threading
Simon Glassfc3fe1c2013-04-03 11:07:16 +000018import time
19
Simon Glass190064b2014-08-09 15:33:00 -060020import builderthread
Simon Glassfc3fe1c2013-04-03 11:07:16 +000021import command
22import gitutil
23import terminal
Simon Glass4653a882014-09-05 19:00:07 -060024from terminal import Print
Simon Glassfc3fe1c2013-04-03 11:07:16 +000025import toolchain
26
Simon Glassfc3fe1c2013-04-03 11:07:16 +000027"""
28Theory of Operation
29
30Please see README for user documentation, and you should be familiar with
31that before trying to make sense of this.
32
33Buildman works by keeping the machine as busy as possible, building different
34commits for different boards on multiple CPUs at once.
35
36The source repo (self.git_dir) contains all the commits to be built. Each
37thread works on a single board at a time. It checks out the first commit,
38configures it for that board, then builds it. Then it checks out the next
39commit and builds it (typically without re-configuring). When it runs out
40of commits, it gets another job from the builder and starts again with that
41board.
42
43Clearly the builder threads could work either way - they could check out a
44commit and then built it for all boards. Using separate directories for each
45commit/board pair they could leave their build product around afterwards
46also.
47
48The intent behind building a single board for multiple commits, is to make
49use of incremental builds. Since each commit is built incrementally from
50the previous one, builds are faster. Reconfiguring for a different board
51removes all intermediate object files.
52
53Many threads can be working at once, but each has its own working directory.
54When a thread finishes a build, it puts the output files into a result
55directory.
56
57The base directory used by buildman is normally '../<branch>', i.e.
58a directory higher than the source repository and named after the branch
59being built.
60
61Within the base directory, we have one subdirectory for each commit. Within
62that is one subdirectory for each board. Within that is the build output for
63that commit/board combination.
64
65Buildman also create working directories for each thread, in a .bm-work/
66subdirectory in the base dir.
67
68As an example, say we are building branch 'us-net' for boards 'sandbox' and
69'seaboard', and say that us-net has two commits. We will have directories
70like this:
71
72us-net/ base directory
73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/
74 sandbox/
75 u-boot.bin
76 seaboard/
77 u-boot.bin
78 02_of_02_g4ed4ebc_net--Check-tftp-comp/
79 sandbox/
80 u-boot.bin
81 seaboard/
82 u-boot.bin
83 .bm-work/
84 00/ working directory for thread 0 (contains source checkout)
85 build/ build output
86 01/ working directory for thread 1
87 build/ build output
88 ...
89u-boot/ source directory
90 .git/ repository
91"""
92
Simon Glass35d696d2020-04-09 15:08:36 -060093"""Holds information about a particular error line we are outputing
94
95 char: Character representation: '+': error, '-': fixed error, 'w+': warning,
96 'w-' = fixed warning
97 boards: List of Board objects which have line in the error/warning output
98 errline: The text of the error line
99"""
100ErrLine = collections.namedtuple('ErrLine', 'char,boards,errline')
101
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000102# Possible build outcomes
Simon Glassc05aa032019-10-31 07:42:53 -0600103OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000104
Simon Glass9a6d2e22017-04-12 18:23:26 -0600105# Translate a commit subject into a valid filename (and handle unicode)
Simon Glassc05aa032019-10-31 07:42:53 -0600106trans_valid_chars = str.maketrans('/: ', '---')
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000107
Simon Glassb464f8e2016-11-13 14:25:53 -0700108BASE_CONFIG_FILENAMES = [
109 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
110]
111
112EXTRA_CONFIG_FILENAMES = [
Simon Glass843312d2015-02-05 22:06:15 -0700113 '.config', '.config-spl', '.config-tpl',
114 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
115 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
Simon Glass843312d2015-02-05 22:06:15 -0700116]
117
Simon Glass8270e3c2015-08-25 21:52:14 -0600118class Config:
119 """Holds information about configuration settings for a board."""
Simon Glassb464f8e2016-11-13 14:25:53 -0700120 def __init__(self, config_filename, target):
Simon Glass8270e3c2015-08-25 21:52:14 -0600121 self.target = target
122 self.config = {}
Simon Glassb464f8e2016-11-13 14:25:53 -0700123 for fname in config_filename:
Simon Glass8270e3c2015-08-25 21:52:14 -0600124 self.config[fname] = {}
125
126 def Add(self, fname, key, value):
127 self.config[fname][key] = value
128
129 def __hash__(self):
130 val = 0
131 for fname in self.config:
Simon Glassc05aa032019-10-31 07:42:53 -0600132 for key, value in self.config[fname].items():
133 print(key, value)
Simon Glass8270e3c2015-08-25 21:52:14 -0600134 val = val ^ hash(key) & hash(value)
135 return val
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000136
Alex Kiernan48ae4122018-05-31 04:48:34 +0000137class Environment:
138 """Holds information about environment variables for a board."""
139 def __init__(self, target):
140 self.target = target
141 self.environment = {}
142
143 def Add(self, key, value):
144 self.environment[key] = value
145
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000146class Builder:
147 """Class for building U-Boot for a particular commit.
148
149 Public members: (many should ->private)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000150 already_done: Number of builds already completed
151 base_dir: Base directory to use for builder
152 checkout: True to check out source, False to skip that step.
153 This is used for testing.
154 col: terminal.Color() object
155 count: Number of commits to build
156 do_make: Method to call to invoke Make
157 fail: Number of builds that failed due to error
158 force_build: Force building even if a build already exists
159 force_config_on_failure: If a commit fails for a board, disable
160 incremental building for the next commit we build for that
161 board, so that we will see all warnings/errors again.
Simon Glass4266dc22014-07-13 12:22:31 -0600162 force_build_failures: If a previously-built build (i.e. built on
163 a previous run of buildman) is marked as failed, rebuild it.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000164 git_dir: Git directory containing source repository
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000165 num_jobs: Number of jobs to run at once (passed to make as -j)
166 num_threads: Number of builder threads to run
167 out_queue: Queue of results to process
168 re_make_err: Compiled regular expression for ignore_lines
169 queue: Queue of jobs to run
170 threads: List of active threads
171 toolchains: Toolchains object to use for building
172 upto: Current commit number we are building (0.count-1)
173 warned: Number of builds that produced at least one warning
Simon Glass97e91522014-07-14 17:51:02 -0600174 force_reconfig: Reconfigure U-Boot on each comiit. This disables
175 incremental building, where buildman reconfigures on the first
176 commit for a baord, and then just does an incremental build for
177 the following commits. In fact buildman will reconfigure and
178 retry for any failing commits, so generally the only effect of
179 this option is to slow things down.
Simon Glass189a4962014-07-14 17:51:03 -0600180 in_tree: Build U-Boot in-tree instead of specifying an output
181 directory separate from the source code. This option is really
182 only useful for testing in-tree builds.
Simon Glassd829f122020-03-18 09:42:42 -0600183 work_in_output: Use the output directory as the work directory and
184 don't write to a separate output directory.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000185
186 Private members:
187 _base_board_dict: Last-summarised Dict of boards
188 _base_err_lines: Last-summarised list of errors
Simon Glasse30965d2014-08-28 09:43:44 -0600189 _base_warn_lines: Last-summarised list of warnings
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000190 _build_period_us: Time taken for a single build (float object).
191 _complete_delay: Expected delay until completion (timedelta)
192 _next_delay_update: Next time we plan to display a progress update
193 (datatime)
194 _show_unknown: Show unknown boards (those not built) in summary
Simon Glass7b33f212020-04-09 15:08:47 -0600195 _start_time: Start time for the build
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000196 _timestamps: List of timestamps for the completion of the last
197 last _timestamp_count builds. Each is a datetime object.
198 _timestamp_count: Number of timestamps to keep in our list.
199 _working_dir: Base working directory containing all threads
200 """
201 class Outcome:
202 """Records a build outcome for a single make invocation
203
204 Public Members:
205 rc: Outcome value (OUTCOME_...)
206 err_lines: List of error lines or [] if none
207 sizes: Dictionary of image size information, keyed by filename
208 - Each value is itself a dictionary containing
209 values for 'text', 'data' and 'bss', being the integer
210 size in bytes of each section.
211 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
212 value is itself a dictionary:
213 key: function name
214 value: Size of function in bytes
Simon Glass843312d2015-02-05 22:06:15 -0700215 config: Dictionary keyed by filename - e.g. '.config'. Each
216 value is itself a dictionary:
217 key: config name
218 value: config value
Alex Kiernan48ae4122018-05-31 04:48:34 +0000219 environment: Dictionary keyed by environment variable, Each
220 value is the value of environment variable.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000221 """
Alex Kiernan48ae4122018-05-31 04:48:34 +0000222 def __init__(self, rc, err_lines, sizes, func_sizes, config,
223 environment):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000224 self.rc = rc
225 self.err_lines = err_lines
226 self.sizes = sizes
227 self.func_sizes = func_sizes
Simon Glass843312d2015-02-05 22:06:15 -0700228 self.config = config
Alex Kiernan48ae4122018-05-31 04:48:34 +0000229 self.environment = environment
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000230
231 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
Simon Glass5971ab52014-12-01 17:33:55 -0700232 gnu_make='make', checkout=True, show_unknown=True, step=1,
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600233 no_subdirs=False, full_path=False, verbose_build=False,
Simon Glasseb70a2c2020-04-09 15:08:51 -0600234 mrproper=False, per_board_out_dir=False,
Daniel Schwierzeck2371d1b2018-01-26 16:31:05 +0100235 config_only=False, squash_config_y=False,
Simon Glassd829f122020-03-18 09:42:42 -0600236 warnings_as_errors=False, work_in_output=False):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000237 """Create a new Builder object
238
239 Args:
240 toolchains: Toolchains object to use for building
241 base_dir: Base directory to use for builder
242 git_dir: Git directory containing source repository
243 num_threads: Number of builder threads to run
244 num_jobs: Number of jobs to run at once (passed to make as -j)
Masahiro Yamada99796922014-07-22 11:19:09 +0900245 gnu_make: the command name of GNU Make.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000246 checkout: True to check out source, False to skip that step.
247 This is used for testing.
248 show_unknown: Show unknown boards (those not built) in summary
249 step: 1 to process every commit, n to process every nth commit
Simon Glassbb1501f2014-12-01 17:34:00 -0700250 no_subdirs: Don't create subdirectories when building current
251 source for a single board
252 full_path: Return the full path in CROSS_COMPILE and don't set
253 PATH
Simon Glassd2ce6582014-12-01 17:34:07 -0700254 verbose_build: Run build with V=1 and don't use 'make -s'
Simon Glasseb70a2c2020-04-09 15:08:51 -0600255 mrproper: Always run 'make mrproper' when configuring
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600256 per_board_out_dir: Build in a separate persistent directory per
257 board rather than a thread-specific directory
Simon Glassb50113f2016-11-13 14:25:51 -0700258 config_only: Only configure each build, don't build it
Simon Glassb464f8e2016-11-13 14:25:53 -0700259 squash_config_y: Convert CONFIG options with the value 'y' to '1'
Daniel Schwierzeck2371d1b2018-01-26 16:31:05 +0100260 warnings_as_errors: Treat all compiler warnings as errors
Simon Glassd829f122020-03-18 09:42:42 -0600261 work_in_output: Use the output directory as the work directory and
262 don't write to a separate output directory.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000263 """
264 self.toolchains = toolchains
265 self.base_dir = base_dir
Simon Glassd829f122020-03-18 09:42:42 -0600266 if work_in_output:
267 self._working_dir = base_dir
268 else:
269 self._working_dir = os.path.join(base_dir, '.bm-work')
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000270 self.threads = []
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000271 self.do_make = self.Make
Masahiro Yamada99796922014-07-22 11:19:09 +0900272 self.gnu_make = gnu_make
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000273 self.checkout = checkout
274 self.num_threads = num_threads
275 self.num_jobs = num_jobs
276 self.already_done = 0
277 self.force_build = False
278 self.git_dir = git_dir
279 self._show_unknown = show_unknown
280 self._timestamp_count = 10
281 self._build_period_us = None
282 self._complete_delay = None
283 self._next_delay_update = datetime.now()
Simon Glass7b33f212020-04-09 15:08:47 -0600284 self._start_time = datetime.now()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000285 self.force_config_on_failure = True
Simon Glass4266dc22014-07-13 12:22:31 -0600286 self.force_build_failures = False
Simon Glass97e91522014-07-14 17:51:02 -0600287 self.force_reconfig = False
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000288 self._step = step
Simon Glass189a4962014-07-14 17:51:03 -0600289 self.in_tree = False
Simon Glass28370c12014-08-09 15:33:06 -0600290 self._error_lines = 0
Simon Glass5971ab52014-12-01 17:33:55 -0700291 self.no_subdirs = no_subdirs
Simon Glassbb1501f2014-12-01 17:34:00 -0700292 self.full_path = full_path
Simon Glassd2ce6582014-12-01 17:34:07 -0700293 self.verbose_build = verbose_build
Simon Glassb50113f2016-11-13 14:25:51 -0700294 self.config_only = config_only
Simon Glassb464f8e2016-11-13 14:25:53 -0700295 self.squash_config_y = squash_config_y
296 self.config_filenames = BASE_CONFIG_FILENAMES
Simon Glassd829f122020-03-18 09:42:42 -0600297 self.work_in_output = work_in_output
Simon Glassb464f8e2016-11-13 14:25:53 -0700298 if not self.squash_config_y:
299 self.config_filenames += EXTRA_CONFIG_FILENAMES
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000300
Daniel Schwierzeck2371d1b2018-01-26 16:31:05 +0100301 self.warnings_as_errors = warnings_as_errors
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000302 self.col = terminal.Color()
303
Simon Glasse30965d2014-08-28 09:43:44 -0600304 self._re_function = re.compile('(.*): In function.*')
305 self._re_files = re.compile('In file included from.*')
306 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
Simon Glass2d483332018-11-06 16:02:11 -0700307 self._re_dtb_warning = re.compile('(.*): Warning .*')
Simon Glasse30965d2014-08-28 09:43:44 -0600308 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
Simon Glass113a8a52020-04-09 15:08:53 -0600309 self._re_migration_warning = re.compile(r'^={21} WARNING ={22}\n.*\n=+\n',
310 re.MULTILINE | re.DOTALL)
Simon Glasse30965d2014-08-28 09:43:44 -0600311
Simon Glassc05aa032019-10-31 07:42:53 -0600312 self.queue = queue.Queue()
313 self.out_queue = queue.Queue()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000314 for i in range(self.num_threads):
Simon Glasseb70a2c2020-04-09 15:08:51 -0600315 t = builderthread.BuilderThread(self, i, mrproper,
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600316 per_board_out_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000317 t.setDaemon(True)
318 t.start()
319 self.threads.append(t)
320
Simon Glass190064b2014-08-09 15:33:00 -0600321 t = builderthread.ResultThread(self)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000322 t.setDaemon(True)
323 t.start()
324 self.threads.append(t)
325
326 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
327 self.re_make_err = re.compile('|'.join(ignore_lines))
328
Simon Glass2f256642016-09-18 16:48:37 -0600329 # Handle existing graceful with SIGINT / Ctrl-C
330 signal.signal(signal.SIGINT, self.signal_handler)
331
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000332 def __del__(self):
333 """Get rid of all threads created by the builder"""
334 for t in self.threads:
335 del t
336
Simon Glass2f256642016-09-18 16:48:37 -0600337 def signal_handler(self, signal, frame):
338 sys.exit(1)
339
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600340 def SetDisplayOptions(self, show_errors=False, show_sizes=False,
Simon Glassed966652014-08-28 09:43:43 -0600341 show_detail=False, show_bloat=False,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000342 list_error_boards=False, show_config=False,
Simon Glass113a8a52020-04-09 15:08:53 -0600343 show_environment=False, filter_dtb_warnings=False,
344 filter_migration_warnings=False):
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600345 """Setup display options for the builder.
346
Simon Glass174592b2020-04-09 15:08:52 -0600347 Args:
348 show_errors: True to show summarised error/warning info
349 show_sizes: Show size deltas
350 show_detail: Show size delta detail for each board if show_sizes
351 show_bloat: Show detail for each function
352 list_error_boards: Show the boards which caused each error/warning
353 show_config: Show config deltas
354 show_environment: Show environment deltas
355 filter_dtb_warnings: Filter out any warnings from the device-tree
356 compiler
Simon Glass113a8a52020-04-09 15:08:53 -0600357 filter_migration_warnings: Filter out any warnings about migrating
358 a board to driver model
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600359 """
360 self._show_errors = show_errors
361 self._show_sizes = show_sizes
362 self._show_detail = show_detail
363 self._show_bloat = show_bloat
Simon Glassed966652014-08-28 09:43:43 -0600364 self._list_error_boards = list_error_boards
Simon Glass843312d2015-02-05 22:06:15 -0700365 self._show_config = show_config
Alex Kiernan48ae4122018-05-31 04:48:34 +0000366 self._show_environment = show_environment
Simon Glass174592b2020-04-09 15:08:52 -0600367 self._filter_dtb_warnings = filter_dtb_warnings
Simon Glass113a8a52020-04-09 15:08:53 -0600368 self._filter_migration_warnings = filter_migration_warnings
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600369
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000370 def _AddTimestamp(self):
371 """Add a new timestamp to the list and record the build period.
372
373 The build period is the length of time taken to perform a single
374 build (one board, one commit).
375 """
376 now = datetime.now()
377 self._timestamps.append(now)
378 count = len(self._timestamps)
379 delta = self._timestamps[-1] - self._timestamps[0]
380 seconds = delta.total_seconds()
381
382 # If we have enough data, estimate build period (time taken for a
383 # single build) and therefore completion time.
384 if count > 1 and self._next_delay_update < now:
385 self._next_delay_update = now + timedelta(seconds=2)
386 if seconds > 0:
387 self._build_period = float(seconds) / count
388 todo = self.count - self.upto
389 self._complete_delay = timedelta(microseconds=
390 self._build_period * todo * 1000000)
391 # Round it
392 self._complete_delay -= timedelta(
393 microseconds=self._complete_delay.microseconds)
394
395 if seconds > 60:
396 self._timestamps.popleft()
397 count -= 1
398
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000399 def SelectCommit(self, commit, checkout=True):
400 """Checkout the selected commit for this build
401 """
402 self.commit = commit
403 if checkout and self.checkout:
404 gitutil.Checkout(commit.hash)
405
406 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
407 """Run make
408
409 Args:
410 commit: Commit object that is being built
411 brd: Board object that is being built
Roger Meierfd18a892014-08-20 22:10:29 +0200412 stage: Stage that we are at (mrproper, config, build)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000413 cwd: Directory where make should be run
414 args: Arguments to pass to make
415 kwargs: Arguments to pass to command.RunPipe()
416 """
Masahiro Yamada99796922014-07-22 11:19:09 +0900417 cmd = [self.gnu_make] + list(args)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000418 result = command.RunPipe([cmd], capture=True, capture_stderr=True,
Simon Glasse62a24c2018-09-17 23:55:42 -0600419 cwd=cwd, raise_on_error=False, infile='/dev/null', **kwargs)
Simon Glass40f11fc2015-02-05 22:06:12 -0700420 if self.verbose_build:
421 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
422 result.combined = '%s\n' % (' '.join(cmd)) + result.combined
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000423 return result
424
425 def ProcessResult(self, result):
426 """Process the result of a build, showing progress information
427
428 Args:
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600429 result: A CommandResult object, which indicates the result for
430 a single build
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000431 """
432 col = terminal.Color()
433 if result:
434 target = result.brd.target
435
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000436 self.upto += 1
437 if result.return_code != 0:
438 self.fail += 1
439 elif result.stderr:
440 self.warned += 1
441 if result.already_done:
442 self.already_done += 1
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600443 if self._verbose:
Simon Glass102969b2020-04-09 15:08:42 -0600444 terminal.PrintClear()
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600445 boards_selected = {target : result.brd}
446 self.ResetResultSummary(boards_selected)
447 self.ProduceResultSummary(result.commit_upto, self.commits,
448 boards_selected)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000449 else:
450 target = '(starting)'
451
452 # Display separate counts for ok, warned and fail
453 ok = self.upto - self.warned - self.fail
454 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
455 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
456 line += self.col.Color(self.col.RED, '%5d' % self.fail)
457
Simon Glass6eb76ca2020-04-09 15:08:45 -0600458 line += ' /%-5d ' % self.count
459 remaining = self.count - self.upto
460 if remaining:
461 line += self.col.Color(self.col.MAGENTA, ' -%-5d ' % remaining)
462 else:
463 line += ' ' * 8
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000464
465 # Add our current completion time estimate
466 self._AddTimestamp()
467 if self._complete_delay:
Simon Glass6eb76ca2020-04-09 15:08:45 -0600468 line += '%s : ' % self._complete_delay
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000469
Simon Glass6eb76ca2020-04-09 15:08:45 -0600470 line += target
Simon Glass102969b2020-04-09 15:08:42 -0600471 terminal.PrintClear()
Simon Glass95ed0a22020-04-09 15:08:46 -0600472 Print(line, newline=False, limit_to_line=True)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000473
474 def _GetOutputDir(self, commit_upto):
475 """Get the name of the output directory for a commit number
476
477 The output directory is typically .../<branch>/<commit>.
478
479 Args:
480 commit_upto: Commit number to use (0..self.count-1)
481 """
Simon Glass60b285f2020-04-17 17:51:34 -0600482 if self.work_in_output:
483 return self._working_dir
484
Simon Glass5971ab52014-12-01 17:33:55 -0700485 commit_dir = None
Simon Glassfea58582014-08-09 15:32:59 -0600486 if self.commits:
487 commit = self.commits[commit_upto]
488 subject = commit.subject.translate(trans_valid_chars)
Simon Glass925f6ad2020-03-18 09:42:45 -0600489 # See _GetOutputSpaceRemovals() which parses this name
Simon Glassfea58582014-08-09 15:32:59 -0600490 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
491 self.commit_count, commit.hash, subject[:20]))
Simon Glass5971ab52014-12-01 17:33:55 -0700492 elif not self.no_subdirs:
Simon Glassfea58582014-08-09 15:32:59 -0600493 commit_dir = 'current'
Simon Glass5971ab52014-12-01 17:33:55 -0700494 if not commit_dir:
495 return self.base_dir
496 return os.path.join(self.base_dir, commit_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000497
498 def GetBuildDir(self, commit_upto, target):
499 """Get the name of the build directory for a commit number
500
501 The build directory is typically .../<branch>/<commit>/<target>.
502
503 Args:
504 commit_upto: Commit number to use (0..self.count-1)
505 target: Target name
506 """
507 output_dir = self._GetOutputDir(commit_upto)
Simon Glass60b285f2020-04-17 17:51:34 -0600508 if self.work_in_output:
509 return output_dir
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000510 return os.path.join(output_dir, target)
511
512 def GetDoneFile(self, commit_upto, target):
513 """Get the name of the done file for a commit number
514
515 Args:
516 commit_upto: Commit number to use (0..self.count-1)
517 target: Target name
518 """
519 return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
520
521 def GetSizesFile(self, commit_upto, target):
522 """Get the name of the sizes file for a commit number
523
524 Args:
525 commit_upto: Commit number to use (0..self.count-1)
526 target: Target name
527 """
528 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
529
530 def GetFuncSizesFile(self, commit_upto, target, elf_fname):
531 """Get the name of the funcsizes file for a commit number and ELF file
532
533 Args:
534 commit_upto: Commit number to use (0..self.count-1)
535 target: Target name
536 elf_fname: Filename of elf image
537 """
538 return os.path.join(self.GetBuildDir(commit_upto, target),
539 '%s.sizes' % elf_fname.replace('/', '-'))
540
541 def GetObjdumpFile(self, commit_upto, target, elf_fname):
542 """Get the name of the objdump file for a commit number and ELF file
543
544 Args:
545 commit_upto: Commit number to use (0..self.count-1)
546 target: Target name
547 elf_fname: Filename of elf image
548 """
549 return os.path.join(self.GetBuildDir(commit_upto, target),
550 '%s.objdump' % elf_fname.replace('/', '-'))
551
552 def GetErrFile(self, commit_upto, target):
553 """Get the name of the err file for a commit number
554
555 Args:
556 commit_upto: Commit number to use (0..self.count-1)
557 target: Target name
558 """
559 output_dir = self.GetBuildDir(commit_upto, target)
560 return os.path.join(output_dir, 'err')
561
562 def FilterErrors(self, lines):
563 """Filter out errors in which we have no interest
564
565 We should probably use map().
566
567 Args:
568 lines: List of error lines, each a string
569 Returns:
570 New list with only interesting lines included
571 """
572 out_lines = []
Simon Glass113a8a52020-04-09 15:08:53 -0600573 if self._filter_migration_warnings:
574 text = '\n'.join(lines)
575 text = self._re_migration_warning.sub('', text)
576 lines = text.splitlines()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000577 for line in lines:
Simon Glass174592b2020-04-09 15:08:52 -0600578 if self.re_make_err.search(line):
579 continue
580 if self._filter_dtb_warnings and self._re_dtb_warning.search(line):
581 continue
582 out_lines.append(line)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000583 return out_lines
584
585 def ReadFuncSizes(self, fname, fd):
586 """Read function sizes from the output of 'nm'
587
588 Args:
589 fd: File containing data to read
590 fname: Filename we are reading from (just for errors)
591
592 Returns:
593 Dictionary containing size of each function in bytes, indexed by
594 function name.
595 """
596 sym = {}
597 for line in fd.readlines():
598 try:
Tom Rinid08c38c2019-12-06 15:31:31 -0500599 if line.strip():
600 size, type, name = line[:-1].split()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000601 except:
Simon Glass4653a882014-09-05 19:00:07 -0600602 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000603 continue
604 if type in 'tTdDbB':
605 # function names begin with '.' on 64-bit powerpc
606 if '.' in name[1:]:
607 name = 'static.' + name.split('.')[0]
608 sym[name] = sym.get(name, 0) + int(size, 16)
609 return sym
610
Simon Glass843312d2015-02-05 22:06:15 -0700611 def _ProcessConfig(self, fname):
612 """Read in a .config, autoconf.mk or autoconf.h file
613
614 This function handles all config file types. It ignores comments and
615 any #defines which don't start with CONFIG_.
616
617 Args:
618 fname: Filename to read
619
620 Returns:
621 Dictionary:
622 key: Config name (e.g. CONFIG_DM)
623 value: Config value (e.g. 1)
624 """
625 config = {}
626 if os.path.exists(fname):
627 with open(fname) as fd:
628 for line in fd:
629 line = line.strip()
630 if line.startswith('#define'):
631 values = line[8:].split(' ', 1)
632 if len(values) > 1:
633 key, value = values
634 else:
635 key = values[0]
Simon Glassb464f8e2016-11-13 14:25:53 -0700636 value = '1' if self.squash_config_y else ''
Simon Glass843312d2015-02-05 22:06:15 -0700637 if not key.startswith('CONFIG_'):
638 continue
639 elif not line or line[0] in ['#', '*', '/']:
640 continue
641 else:
642 key, value = line.split('=', 1)
Simon Glassb464f8e2016-11-13 14:25:53 -0700643 if self.squash_config_y and value == 'y':
644 value = '1'
Simon Glass843312d2015-02-05 22:06:15 -0700645 config[key] = value
646 return config
647
Alex Kiernan48ae4122018-05-31 04:48:34 +0000648 def _ProcessEnvironment(self, fname):
649 """Read in a uboot.env file
650
651 This function reads in environment variables from a file.
652
653 Args:
654 fname: Filename to read
655
656 Returns:
657 Dictionary:
658 key: environment variable (e.g. bootlimit)
659 value: value of environment variable (e.g. 1)
660 """
661 environment = {}
662 if os.path.exists(fname):
663 with open(fname) as fd:
664 for line in fd.read().split('\0'):
665 try:
666 key, value = line.split('=', 1)
667 environment[key] = value
668 except ValueError:
669 # ignore lines we can't parse
670 pass
671 return environment
672
Simon Glass843312d2015-02-05 22:06:15 -0700673 def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000674 read_config, read_environment):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000675 """Work out the outcome of a build.
676
677 Args:
678 commit_upto: Commit number to check (0..n-1)
679 target: Target board to check
680 read_func_sizes: True to read function size information
Simon Glass843312d2015-02-05 22:06:15 -0700681 read_config: True to read .config and autoconf.h files
Alex Kiernan48ae4122018-05-31 04:48:34 +0000682 read_environment: True to read uboot.env files
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000683
684 Returns:
685 Outcome object
686 """
687 done_file = self.GetDoneFile(commit_upto, target)
688 sizes_file = self.GetSizesFile(commit_upto, target)
689 sizes = {}
690 func_sizes = {}
Simon Glass843312d2015-02-05 22:06:15 -0700691 config = {}
Alex Kiernan48ae4122018-05-31 04:48:34 +0000692 environment = {}
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000693 if os.path.exists(done_file):
694 with open(done_file, 'r') as fd:
Simon Glass347ea0b2019-04-26 19:02:23 -0600695 try:
696 return_code = int(fd.readline())
697 except ValueError:
698 # The file may be empty due to running out of disk space.
699 # Try a rebuild
700 return_code = 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000701 err_lines = []
702 err_file = self.GetErrFile(commit_upto, target)
703 if os.path.exists(err_file):
704 with open(err_file, 'r') as fd:
705 err_lines = self.FilterErrors(fd.readlines())
706
707 # Decide whether the build was ok, failed or created warnings
708 if return_code:
709 rc = OUTCOME_ERROR
710 elif len(err_lines):
711 rc = OUTCOME_WARNING
712 else:
713 rc = OUTCOME_OK
714
715 # Convert size information to our simple format
716 if os.path.exists(sizes_file):
717 with open(sizes_file, 'r') as fd:
718 for line in fd.readlines():
719 values = line.split()
720 rodata = 0
721 if len(values) > 6:
722 rodata = int(values[6], 16)
723 size_dict = {
724 'all' : int(values[0]) + int(values[1]) +
725 int(values[2]),
726 'text' : int(values[0]) - rodata,
727 'data' : int(values[1]),
728 'bss' : int(values[2]),
729 'rodata' : rodata,
730 }
731 sizes[values[5]] = size_dict
732
733 if read_func_sizes:
734 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
735 for fname in glob.glob(pattern):
736 with open(fname, 'r') as fd:
737 dict_name = os.path.basename(fname).replace('.sizes',
738 '')
739 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
740
Simon Glass843312d2015-02-05 22:06:15 -0700741 if read_config:
742 output_dir = self.GetBuildDir(commit_upto, target)
Simon Glassb464f8e2016-11-13 14:25:53 -0700743 for name in self.config_filenames:
Simon Glass843312d2015-02-05 22:06:15 -0700744 fname = os.path.join(output_dir, name)
745 config[name] = self._ProcessConfig(fname)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000746
Alex Kiernan48ae4122018-05-31 04:48:34 +0000747 if read_environment:
748 output_dir = self.GetBuildDir(commit_upto, target)
749 fname = os.path.join(output_dir, 'uboot.env')
750 environment = self._ProcessEnvironment(fname)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000751
Alex Kiernan48ae4122018-05-31 04:48:34 +0000752 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config,
753 environment)
754
755 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {})
Simon Glass843312d2015-02-05 22:06:15 -0700756
757 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000758 read_config, read_environment):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000759 """Calculate a summary of the results of building a commit.
760
761 Args:
762 board_selected: Dict containing boards to summarise
763 commit_upto: Commit number to summarize (0..self.count-1)
764 read_func_sizes: True to read function size information
Simon Glass843312d2015-02-05 22:06:15 -0700765 read_config: True to read .config and autoconf.h files
Alex Kiernan48ae4122018-05-31 04:48:34 +0000766 read_environment: True to read uboot.env files
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000767
768 Returns:
769 Tuple:
770 Dict containing boards which passed building this commit.
771 keyed by board.target
Simon Glasse30965d2014-08-28 09:43:44 -0600772 List containing a summary of error lines
Simon Glassed966652014-08-28 09:43:43 -0600773 Dict keyed by error line, containing a list of the Board
774 objects with that error
Simon Glasse30965d2014-08-28 09:43:44 -0600775 List containing a summary of warning lines
776 Dict keyed by error line, containing a list of the Board
777 objects with that warning
Simon Glass8270e3c2015-08-25 21:52:14 -0600778 Dictionary keyed by board.target. Each value is a dictionary:
779 key: filename - e.g. '.config'
Simon Glass843312d2015-02-05 22:06:15 -0700780 value is itself a dictionary:
781 key: config name
782 value: config value
Alex Kiernan48ae4122018-05-31 04:48:34 +0000783 Dictionary keyed by board.target. Each value is a dictionary:
784 key: environment variable
785 value: value of environment variable
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000786 """
Simon Glasse30965d2014-08-28 09:43:44 -0600787 def AddLine(lines_summary, lines_boards, line, board):
788 line = line.rstrip()
789 if line in lines_boards:
790 lines_boards[line].append(board)
791 else:
792 lines_boards[line] = [board]
793 lines_summary.append(line)
794
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000795 board_dict = {}
796 err_lines_summary = []
Simon Glassed966652014-08-28 09:43:43 -0600797 err_lines_boards = {}
Simon Glasse30965d2014-08-28 09:43:44 -0600798 warn_lines_summary = []
799 warn_lines_boards = {}
Simon Glass843312d2015-02-05 22:06:15 -0700800 config = {}
Alex Kiernan48ae4122018-05-31 04:48:34 +0000801 environment = {}
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000802
Simon Glassc05aa032019-10-31 07:42:53 -0600803 for board in boards_selected.values():
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000804 outcome = self.GetBuildOutcome(commit_upto, board.target,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000805 read_func_sizes, read_config,
806 read_environment)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000807 board_dict[board.target] = outcome
Simon Glasse30965d2014-08-28 09:43:44 -0600808 last_func = None
809 last_was_warning = False
810 for line in outcome.err_lines:
811 if line:
812 if (self._re_function.match(line) or
813 self._re_files.match(line)):
814 last_func = line
Simon Glassed966652014-08-28 09:43:43 -0600815 else:
Simon Glass2d483332018-11-06 16:02:11 -0700816 is_warning = (self._re_warning.match(line) or
817 self._re_dtb_warning.match(line))
Simon Glasse30965d2014-08-28 09:43:44 -0600818 is_note = self._re_note.match(line)
819 if is_warning or (last_was_warning and is_note):
820 if last_func:
821 AddLine(warn_lines_summary, warn_lines_boards,
822 last_func, board)
823 AddLine(warn_lines_summary, warn_lines_boards,
824 line, board)
825 else:
826 if last_func:
827 AddLine(err_lines_summary, err_lines_boards,
828 last_func, board)
829 AddLine(err_lines_summary, err_lines_boards,
830 line, board)
831 last_was_warning = is_warning
832 last_func = None
Simon Glassb464f8e2016-11-13 14:25:53 -0700833 tconfig = Config(self.config_filenames, board.target)
834 for fname in self.config_filenames:
Simon Glass843312d2015-02-05 22:06:15 -0700835 if outcome.config:
Simon Glassc05aa032019-10-31 07:42:53 -0600836 for key, value in outcome.config[fname].items():
Simon Glass8270e3c2015-08-25 21:52:14 -0600837 tconfig.Add(fname, key, value)
838 config[board.target] = tconfig
Simon Glass843312d2015-02-05 22:06:15 -0700839
Alex Kiernan48ae4122018-05-31 04:48:34 +0000840 tenvironment = Environment(board.target)
841 if outcome.environment:
Simon Glassc05aa032019-10-31 07:42:53 -0600842 for key, value in outcome.environment.items():
Alex Kiernan48ae4122018-05-31 04:48:34 +0000843 tenvironment.Add(key, value)
844 environment[board.target] = tenvironment
845
Simon Glasse30965d2014-08-28 09:43:44 -0600846 return (board_dict, err_lines_summary, err_lines_boards,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000847 warn_lines_summary, warn_lines_boards, config, environment)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000848
849 def AddOutcome(self, board_dict, arch_list, changes, char, color):
850 """Add an output to our list of outcomes for each architecture
851
852 This simple function adds failing boards (changes) to the
853 relevant architecture string, so we can print the results out
854 sorted by architecture.
855
856 Args:
857 board_dict: Dict containing all boards
858 arch_list: Dict keyed by arch name. Value is a string containing
859 a list of board names which failed for that arch.
860 changes: List of boards to add to arch_list
861 color: terminal.Colour object
862 """
863 done_arch = {}
864 for target in changes:
865 if target in board_dict:
866 arch = board_dict[target].arch
867 else:
868 arch = 'unknown'
869 str = self.col.Color(color, ' ' + target)
870 if not arch in done_arch:
Simon Glass63c619e2015-02-05 22:06:11 -0700871 str = ' %s %s' % (self.col.Color(color, char), str)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000872 done_arch[arch] = True
873 if not arch in arch_list:
874 arch_list[arch] = str
875 else:
876 arch_list[arch] += str
877
878
879 def ColourNum(self, num):
880 color = self.col.RED if num > 0 else self.col.GREEN
881 if num == 0:
882 return '0'
883 return self.col.Color(color, str(num))
884
885 def ResetResultSummary(self, board_selected):
886 """Reset the results summary ready for use.
887
888 Set up the base board list to be all those selected, and set the
889 error lines to empty.
890
891 Following this, calls to PrintResultSummary() will use this
892 information to work out what has changed.
893
894 Args:
895 board_selected: Dict containing boards to summarise, keyed by
896 board.target
897 """
898 self._base_board_dict = {}
899 for board in board_selected:
Alex Kiernan48ae4122018-05-31 04:48:34 +0000900 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {},
901 {})
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000902 self._base_err_lines = []
Simon Glasse30965d2014-08-28 09:43:44 -0600903 self._base_warn_lines = []
904 self._base_err_line_boards = {}
905 self._base_warn_line_boards = {}
Simon Glass8270e3c2015-08-25 21:52:14 -0600906 self._base_config = None
Alex Kiernan48ae4122018-05-31 04:48:34 +0000907 self._base_environment = None
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000908
909 def PrintFuncSizeDetail(self, fname, old, new):
910 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
911 delta, common = [], {}
912
913 for a in old:
914 if a in new:
915 common[a] = 1
916
917 for name in old:
918 if name not in common:
919 remove += 1
920 down += old[name]
921 delta.append([-old[name], name])
922
923 for name in new:
924 if name not in common:
925 add += 1
926 up += new[name]
927 delta.append([new[name], name])
928
929 for name in common:
930 diff = new.get(name, 0) - old.get(name, 0)
931 if diff > 0:
932 grow, up = grow + 1, up + diff
933 elif diff < 0:
934 shrink, down = shrink + 1, down - diff
935 delta.append([diff, name])
936
937 delta.sort()
938 delta.reverse()
939
940 args = [add, -remove, grow, -shrink, up, -down, up - down]
Tom Rinid5686a62017-05-22 13:48:52 -0400941 if max(args) == 0 and min(args) == 0:
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000942 return
943 args = [self.ColourNum(x) for x in args]
944 indent = ' ' * 15
Simon Glass4653a882014-09-05 19:00:07 -0600945 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
946 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
947 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
948 'delta'))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000949 for diff, name in delta:
950 if diff:
951 color = self.col.RED if diff > 0 else self.col.GREEN
952 msg = '%s %-38s %7s %7s %+7d' % (indent, name,
953 old.get(name, '-'), new.get(name,'-'), diff)
Simon Glass4653a882014-09-05 19:00:07 -0600954 Print(msg, colour=color)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000955
956
957 def PrintSizeDetail(self, target_list, show_bloat):
958 """Show details size information for each board
959
960 Args:
961 target_list: List of targets, each a dict containing:
962 'target': Target name
963 'total_diff': Total difference in bytes across all areas
964 <part_name>: Difference for that part
965 show_bloat: Show detail for each function
966 """
967 targets_by_diff = sorted(target_list, reverse=True,
968 key=lambda x: x['_total_diff'])
969 for result in targets_by_diff:
970 printed_target = False
971 for name in sorted(result):
972 diff = result[name]
973 if name.startswith('_'):
974 continue
975 if diff != 0:
976 color = self.col.RED if diff > 0 else self.col.GREEN
977 msg = ' %s %+d' % (name, diff)
978 if not printed_target:
Simon Glass4653a882014-09-05 19:00:07 -0600979 Print('%10s %-15s:' % ('', result['_target']),
980 newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000981 printed_target = True
Simon Glass4653a882014-09-05 19:00:07 -0600982 Print(msg, colour=color, newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000983 if printed_target:
Simon Glass4653a882014-09-05 19:00:07 -0600984 Print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000985 if show_bloat:
986 target = result['_target']
987 outcome = result['_outcome']
988 base_outcome = self._base_board_dict[target]
989 for fname in outcome.func_sizes:
990 self.PrintFuncSizeDetail(fname,
991 base_outcome.func_sizes[fname],
992 outcome.func_sizes[fname])
993
994
995 def PrintSizeSummary(self, board_selected, board_dict, show_detail,
996 show_bloat):
997 """Print a summary of image sizes broken down by section.
998
999 The summary takes the form of one line per architecture. The
1000 line contains deltas for each of the sections (+ means the section
Flavio Suligoi9de5c392020-01-29 09:56:05 +01001001 got bigger, - means smaller). The numbers are the average number
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001002 of bytes that a board in this section increased by.
1003
1004 For example:
1005 powerpc: (622 boards) text -0.0
1006 arm: (285 boards) text -0.0
1007 nds32: (3 boards) text -8.0
1008
1009 Args:
1010 board_selected: Dict containing boards to summarise, keyed by
1011 board.target
1012 board_dict: Dict containing boards for which we built this
1013 commit, keyed by board.target. The value is an Outcome object.
Simon Glassf9c094b2020-03-18 09:42:43 -06001014 show_detail: Show size delta detail for each board
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001015 show_bloat: Show detail for each function
1016 """
1017 arch_list = {}
1018 arch_count = {}
1019
1020 # Calculate changes in size for different image parts
1021 # The previous sizes are in Board.sizes, for each board
1022 for target in board_dict:
1023 if target not in board_selected:
1024 continue
1025 base_sizes = self._base_board_dict[target].sizes
1026 outcome = board_dict[target]
1027 sizes = outcome.sizes
1028
1029 # Loop through the list of images, creating a dict of size
1030 # changes for each image/part. We end up with something like
1031 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1032 # which means that U-Boot data increased by 5 bytes and SPL
1033 # text decreased by 4.
1034 err = {'_target' : target}
1035 for image in sizes:
1036 if image in base_sizes:
1037 base_image = base_sizes[image]
1038 # Loop through the text, data, bss parts
1039 for part in sorted(sizes[image]):
1040 diff = sizes[image][part] - base_image[part]
1041 col = None
1042 if diff:
1043 if image == 'u-boot':
1044 name = part
1045 else:
1046 name = image + ':' + part
1047 err[name] = diff
1048 arch = board_selected[target].arch
1049 if not arch in arch_count:
1050 arch_count[arch] = 1
1051 else:
1052 arch_count[arch] += 1
1053 if not sizes:
1054 pass # Only add to our list when we have some stats
1055 elif not arch in arch_list:
1056 arch_list[arch] = [err]
1057 else:
1058 arch_list[arch].append(err)
1059
1060 # We now have a list of image size changes sorted by arch
1061 # Print out a summary of these
Simon Glassc05aa032019-10-31 07:42:53 -06001062 for arch, target_list in arch_list.items():
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001063 # Get total difference for each type
1064 totals = {}
1065 for result in target_list:
1066 total = 0
Simon Glassc05aa032019-10-31 07:42:53 -06001067 for name, diff in result.items():
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001068 if name.startswith('_'):
1069 continue
1070 total += diff
1071 if name in totals:
1072 totals[name] += diff
1073 else:
1074 totals[name] = diff
1075 result['_total_diff'] = total
1076 result['_outcome'] = board_dict[result['_target']]
1077
1078 count = len(target_list)
1079 printed_arch = False
1080 for name in sorted(totals):
1081 diff = totals[name]
1082 if diff:
1083 # Display the average difference in this name for this
1084 # architecture
1085 avg_diff = float(diff) / count
1086 color = self.col.RED if avg_diff > 0 else self.col.GREEN
1087 msg = ' %s %+1.1f' % (name, avg_diff)
1088 if not printed_arch:
Simon Glass4653a882014-09-05 19:00:07 -06001089 Print('%10s: (for %d/%d boards)' % (arch, count,
1090 arch_count[arch]), newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001091 printed_arch = True
Simon Glass4653a882014-09-05 19:00:07 -06001092 Print(msg, colour=color, newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001093
1094 if printed_arch:
Simon Glass4653a882014-09-05 19:00:07 -06001095 Print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001096 if show_detail:
1097 self.PrintSizeDetail(target_list, show_bloat)
1098
1099
1100 def PrintResultSummary(self, board_selected, board_dict, err_lines,
Simon Glasse30965d2014-08-28 09:43:44 -06001101 err_line_boards, warn_lines, warn_line_boards,
Alex Kiernan48ae4122018-05-31 04:48:34 +00001102 config, environment, show_sizes, show_detail,
1103 show_bloat, show_config, show_environment):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001104 """Compare results with the base results and display delta.
1105
1106 Only boards mentioned in board_selected will be considered. This
1107 function is intended to be called repeatedly with the results of
1108 each commit. It therefore shows a 'diff' between what it saw in
1109 the last call and what it sees now.
1110
1111 Args:
1112 board_selected: Dict containing boards to summarise, keyed by
1113 board.target
1114 board_dict: Dict containing boards for which we built this
1115 commit, keyed by board.target. The value is an Outcome object.
1116 err_lines: A list of errors for this commit, or [] if there is
1117 none, or we don't want to print errors
Simon Glassed966652014-08-28 09:43:43 -06001118 err_line_boards: Dict keyed by error line, containing a list of
1119 the Board objects with that error
Simon Glasse30965d2014-08-28 09:43:44 -06001120 warn_lines: A list of warnings for this commit, or [] if there is
1121 none, or we don't want to print errors
1122 warn_line_boards: Dict keyed by warning line, containing a list of
1123 the Board objects with that warning
Simon Glass843312d2015-02-05 22:06:15 -07001124 config: Dictionary keyed by filename - e.g. '.config'. Each
1125 value is itself a dictionary:
1126 key: config name
1127 value: config value
Alex Kiernan48ae4122018-05-31 04:48:34 +00001128 environment: Dictionary keyed by environment variable, Each
1129 value is the value of environment variable.
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001130 show_sizes: Show image size deltas
Simon Glassf9c094b2020-03-18 09:42:43 -06001131 show_detail: Show size delta detail for each board if show_sizes
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001132 show_bloat: Show detail for each function
Simon Glass843312d2015-02-05 22:06:15 -07001133 show_config: Show config changes
Alex Kiernan48ae4122018-05-31 04:48:34 +00001134 show_environment: Show environment changes
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001135 """
Simon Glasse30965d2014-08-28 09:43:44 -06001136 def _BoardList(line, line_boards):
Simon Glassed966652014-08-28 09:43:43 -06001137 """Helper function to get a line of boards containing a line
1138
1139 Args:
1140 line: Error line to search for
Simon Glass35d696d2020-04-09 15:08:36 -06001141 line_boards: boards to search, each a Board
Simon Glassed966652014-08-28 09:43:43 -06001142 Return:
Simon Glass35d696d2020-04-09 15:08:36 -06001143 List of boards with that error line, or [] if the user has not
1144 requested such a list
Simon Glassed966652014-08-28 09:43:43 -06001145 """
Simon Glass35d696d2020-04-09 15:08:36 -06001146 boards = []
1147 board_set = set()
Simon Glassed966652014-08-28 09:43:43 -06001148 if self._list_error_boards:
Simon Glasse30965d2014-08-28 09:43:44 -06001149 for board in line_boards[line]:
Simon Glass35d696d2020-04-09 15:08:36 -06001150 if not board in board_set:
1151 boards.append(board)
1152 board_set.add(board)
1153 return boards
Simon Glassed966652014-08-28 09:43:43 -06001154
Simon Glasse30965d2014-08-28 09:43:44 -06001155 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1156 char):
Simon Glass35d696d2020-04-09 15:08:36 -06001157 """Calculate the required output based on changes in errors
1158
1159 Args:
1160 base_lines: List of errors/warnings for previous commit
1161 base_line_boards: Dict keyed by error line, containing a list
1162 of the Board objects with that error in the previous commit
1163 lines: List of errors/warning for this commit, each a str
1164 line_boards: Dict keyed by error line, containing a list
1165 of the Board objects with that error in this commit
1166 char: Character representing error ('') or warning ('w'). The
1167 broken ('+') or fixed ('-') characters are added in this
1168 function
1169
1170 Returns:
1171 Tuple
1172 List of ErrLine objects for 'better' lines
1173 List of ErrLine objects for 'worse' lines
1174 """
Simon Glasse30965d2014-08-28 09:43:44 -06001175 better_lines = []
1176 worse_lines = []
1177 for line in lines:
1178 if line not in base_lines:
Simon Glass35d696d2020-04-09 15:08:36 -06001179 errline = ErrLine(char + '+', _BoardList(line, line_boards),
1180 line)
1181 worse_lines.append(errline)
Simon Glasse30965d2014-08-28 09:43:44 -06001182 for line in base_lines:
1183 if line not in lines:
Simon Glass35d696d2020-04-09 15:08:36 -06001184 errline = ErrLine(char + '-',
1185 _BoardList(line, base_line_boards), line)
1186 better_lines.append(errline)
Simon Glasse30965d2014-08-28 09:43:44 -06001187 return better_lines, worse_lines
1188
Simon Glass843312d2015-02-05 22:06:15 -07001189 def _CalcConfig(delta, name, config):
1190 """Calculate configuration changes
1191
1192 Args:
1193 delta: Type of the delta, e.g. '+'
1194 name: name of the file which changed (e.g. .config)
1195 config: configuration change dictionary
1196 key: config name
1197 value: config value
1198 Returns:
1199 String containing the configuration changes which can be
1200 printed
1201 """
1202 out = ''
1203 for key in sorted(config.keys()):
1204 out += '%s=%s ' % (key, config[key])
Simon Glass8270e3c2015-08-25 21:52:14 -06001205 return '%s %s: %s' % (delta, name, out)
Simon Glass843312d2015-02-05 22:06:15 -07001206
Simon Glass8270e3c2015-08-25 21:52:14 -06001207 def _AddConfig(lines, name, config_plus, config_minus, config_change):
1208 """Add changes in configuration to a list
Simon Glass843312d2015-02-05 22:06:15 -07001209
1210 Args:
Simon Glass8270e3c2015-08-25 21:52:14 -06001211 lines: list to add to
1212 name: config file name
Simon Glass843312d2015-02-05 22:06:15 -07001213 config_plus: configurations added, dictionary
1214 key: config name
1215 value: config value
1216 config_minus: configurations removed, dictionary
1217 key: config name
1218 value: config value
1219 config_change: configurations changed, dictionary
1220 key: config name
1221 value: config value
1222 """
1223 if config_plus:
Simon Glass8270e3c2015-08-25 21:52:14 -06001224 lines.append(_CalcConfig('+', name, config_plus))
Simon Glass843312d2015-02-05 22:06:15 -07001225 if config_minus:
Simon Glass8270e3c2015-08-25 21:52:14 -06001226 lines.append(_CalcConfig('-', name, config_minus))
Simon Glass843312d2015-02-05 22:06:15 -07001227 if config_change:
Simon Glass8270e3c2015-08-25 21:52:14 -06001228 lines.append(_CalcConfig('c', name, config_change))
1229
1230 def _OutputConfigInfo(lines):
1231 for line in lines:
1232 if not line:
1233 continue
1234 if line[0] == '+':
1235 col = self.col.GREEN
1236 elif line[0] == '-':
1237 col = self.col.RED
1238 elif line[0] == 'c':
1239 col = self.col.YELLOW
1240 Print(' ' + line, newline=True, colour=col)
1241
Simon Glassb206d872020-04-09 15:08:28 -06001242 def _OutputErrLines(err_lines, colour):
1243 """Output the line of error/warning lines, if not empty
1244
1245 Also increments self._error_lines if err_lines not empty
1246
1247 Args:
Simon Glass35d696d2020-04-09 15:08:36 -06001248 err_lines: List of ErrLine objects, each an error or warning
1249 line, possibly including a list of boards with that
1250 error/warning
Simon Glassb206d872020-04-09 15:08:28 -06001251 colour: Colour to use for output
1252 """
1253 if err_lines:
Simon Glass8c9a2672020-04-09 15:08:37 -06001254 out_list = []
Simon Glass35d696d2020-04-09 15:08:36 -06001255 for line in err_lines:
1256 boards = ''
1257 names = [board.target for board in line.boards]
Simon Glass9ef0ceb2020-04-09 15:08:38 -06001258 board_str = ' '.join(names) if names else ''
Simon Glass8c9a2672020-04-09 15:08:37 -06001259 if board_str:
1260 out = self.col.Color(colour, line.char + '(')
1261 out += self.col.Color(self.col.MAGENTA, board_str,
1262 bright=False)
1263 out += self.col.Color(colour, ') %s' % line.errline)
1264 else:
1265 out = self.col.Color(colour, line.char + line.errline)
1266 out_list.append(out)
1267 Print('\n'.join(out_list))
Simon Glassb206d872020-04-09 15:08:28 -06001268 self._error_lines += 1
1269
Simon Glass843312d2015-02-05 22:06:15 -07001270
Simon Glass4cf2b222018-11-06 16:02:12 -07001271 ok_boards = [] # List of boards fixed since last commit
Simon Glass6af71012018-11-06 16:02:13 -07001272 warn_boards = [] # List of boards with warnings since last commit
Simon Glass4cf2b222018-11-06 16:02:12 -07001273 err_boards = [] # List of new broken boards since last commit
1274 new_boards = [] # List of boards that didn't exist last time
1275 unknown_boards = [] # List of boards that were not built
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001276
1277 for target in board_dict:
1278 if target not in board_selected:
1279 continue
1280
1281 # If the board was built last time, add its outcome to a list
1282 if target in self._base_board_dict:
1283 base_outcome = self._base_board_dict[target].rc
1284 outcome = board_dict[target]
1285 if outcome.rc == OUTCOME_UNKNOWN:
Simon Glass4cf2b222018-11-06 16:02:12 -07001286 unknown_boards.append(target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001287 elif outcome.rc < base_outcome:
Simon Glass6af71012018-11-06 16:02:13 -07001288 if outcome.rc == OUTCOME_WARNING:
1289 warn_boards.append(target)
1290 else:
1291 ok_boards.append(target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001292 elif outcome.rc > base_outcome:
Simon Glass6af71012018-11-06 16:02:13 -07001293 if outcome.rc == OUTCOME_WARNING:
1294 warn_boards.append(target)
1295 else:
1296 err_boards.append(target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001297 else:
Simon Glass4cf2b222018-11-06 16:02:12 -07001298 new_boards.append(target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001299
Simon Glassb206d872020-04-09 15:08:28 -06001300 # Get a list of errors and warnings that have appeared, and disappeared
Simon Glasse30965d2014-08-28 09:43:44 -06001301 better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1302 self._base_err_line_boards, err_lines, err_line_boards, '')
1303 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1304 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001305
1306 # Display results by arch
Simon Glass6af71012018-11-06 16:02:13 -07001307 if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards,
1308 worse_err, better_err, worse_warn, better_warn)):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001309 arch_list = {}
Simon Glass4cf2b222018-11-06 16:02:12 -07001310 self.AddOutcome(board_selected, arch_list, ok_boards, '',
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001311 self.col.GREEN)
Simon Glass6af71012018-11-06 16:02:13 -07001312 self.AddOutcome(board_selected, arch_list, warn_boards, 'w+',
1313 self.col.YELLOW)
Simon Glass4cf2b222018-11-06 16:02:12 -07001314 self.AddOutcome(board_selected, arch_list, err_boards, '+',
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001315 self.col.RED)
Simon Glass4cf2b222018-11-06 16:02:12 -07001316 self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001317 if self._show_unknown:
Simon Glass4cf2b222018-11-06 16:02:12 -07001318 self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001319 self.col.MAGENTA)
Simon Glassc05aa032019-10-31 07:42:53 -06001320 for arch, target_list in arch_list.items():
Simon Glass4653a882014-09-05 19:00:07 -06001321 Print('%10s: %s' % (arch, target_list))
Simon Glass28370c12014-08-09 15:33:06 -06001322 self._error_lines += 1
Simon Glassb206d872020-04-09 15:08:28 -06001323 _OutputErrLines(better_err, colour=self.col.GREEN)
1324 _OutputErrLines(worse_err, colour=self.col.RED)
1325 _OutputErrLines(better_warn, colour=self.col.CYAN)
Simon Glass5627bd92020-04-09 15:08:35 -06001326 _OutputErrLines(worse_warn, colour=self.col.YELLOW)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001327
1328 if show_sizes:
1329 self.PrintSizeSummary(board_selected, board_dict, show_detail,
1330 show_bloat)
1331
Alex Kiernan48ae4122018-05-31 04:48:34 +00001332 if show_environment and self._base_environment:
1333 lines = []
1334
1335 for target in board_dict:
1336 if target not in board_selected:
1337 continue
1338
1339 tbase = self._base_environment[target]
1340 tenvironment = environment[target]
1341 environment_plus = {}
1342 environment_minus = {}
1343 environment_change = {}
1344 base = tbase.environment
Simon Glassc05aa032019-10-31 07:42:53 -06001345 for key, value in tenvironment.environment.items():
Alex Kiernan48ae4122018-05-31 04:48:34 +00001346 if key not in base:
1347 environment_plus[key] = value
Simon Glassc05aa032019-10-31 07:42:53 -06001348 for key, value in base.items():
Alex Kiernan48ae4122018-05-31 04:48:34 +00001349 if key not in tenvironment.environment:
1350 environment_minus[key] = value
Simon Glassc05aa032019-10-31 07:42:53 -06001351 for key, value in base.items():
Alex Kiernan48ae4122018-05-31 04:48:34 +00001352 new_value = tenvironment.environment.get(key)
1353 if new_value and value != new_value:
1354 desc = '%s -> %s' % (value, new_value)
1355 environment_change[key] = desc
1356
1357 _AddConfig(lines, target, environment_plus, environment_minus,
1358 environment_change)
1359
1360 _OutputConfigInfo(lines)
1361
Simon Glass8270e3c2015-08-25 21:52:14 -06001362 if show_config and self._base_config:
1363 summary = {}
1364 arch_config_plus = {}
1365 arch_config_minus = {}
1366 arch_config_change = {}
1367 arch_list = []
1368
1369 for target in board_dict:
1370 if target not in board_selected:
Simon Glass843312d2015-02-05 22:06:15 -07001371 continue
Simon Glass8270e3c2015-08-25 21:52:14 -06001372 arch = board_selected[target].arch
1373 if arch not in arch_list:
1374 arch_list.append(arch)
1375
1376 for arch in arch_list:
1377 arch_config_plus[arch] = {}
1378 arch_config_minus[arch] = {}
1379 arch_config_change[arch] = {}
Simon Glassb464f8e2016-11-13 14:25:53 -07001380 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001381 arch_config_plus[arch][name] = {}
1382 arch_config_minus[arch][name] = {}
1383 arch_config_change[arch][name] = {}
1384
1385 for target in board_dict:
1386 if target not in board_selected:
1387 continue
1388
1389 arch = board_selected[target].arch
1390
1391 all_config_plus = {}
1392 all_config_minus = {}
1393 all_config_change = {}
1394 tbase = self._base_config[target]
1395 tconfig = config[target]
1396 lines = []
Simon Glassb464f8e2016-11-13 14:25:53 -07001397 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001398 if not tconfig.config[name]:
1399 continue
1400 config_plus = {}
1401 config_minus = {}
1402 config_change = {}
1403 base = tbase.config[name]
Simon Glassc05aa032019-10-31 07:42:53 -06001404 for key, value in tconfig.config[name].items():
Simon Glass8270e3c2015-08-25 21:52:14 -06001405 if key not in base:
1406 config_plus[key] = value
1407 all_config_plus[key] = value
Simon Glassc05aa032019-10-31 07:42:53 -06001408 for key, value in base.items():
Simon Glass8270e3c2015-08-25 21:52:14 -06001409 if key not in tconfig.config[name]:
1410 config_minus[key] = value
1411 all_config_minus[key] = value
Simon Glassc05aa032019-10-31 07:42:53 -06001412 for key, value in base.items():
Simon Glass8270e3c2015-08-25 21:52:14 -06001413 new_value = tconfig.config.get(key)
1414 if new_value and value != new_value:
1415 desc = '%s -> %s' % (value, new_value)
1416 config_change[key] = desc
1417 all_config_change[key] = desc
1418
1419 arch_config_plus[arch][name].update(config_plus)
1420 arch_config_minus[arch][name].update(config_minus)
1421 arch_config_change[arch][name].update(config_change)
1422
1423 _AddConfig(lines, name, config_plus, config_minus,
1424 config_change)
1425 _AddConfig(lines, 'all', all_config_plus, all_config_minus,
1426 all_config_change)
1427 summary[target] = '\n'.join(lines)
1428
1429 lines_by_target = {}
Simon Glassc05aa032019-10-31 07:42:53 -06001430 for target, lines in summary.items():
Simon Glass8270e3c2015-08-25 21:52:14 -06001431 if lines in lines_by_target:
1432 lines_by_target[lines].append(target)
1433 else:
1434 lines_by_target[lines] = [target]
1435
1436 for arch in arch_list:
1437 lines = []
1438 all_plus = {}
1439 all_minus = {}
1440 all_change = {}
Simon Glassb464f8e2016-11-13 14:25:53 -07001441 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001442 all_plus.update(arch_config_plus[arch][name])
1443 all_minus.update(arch_config_minus[arch][name])
1444 all_change.update(arch_config_change[arch][name])
1445 _AddConfig(lines, name, arch_config_plus[arch][name],
1446 arch_config_minus[arch][name],
1447 arch_config_change[arch][name])
1448 _AddConfig(lines, 'all', all_plus, all_minus, all_change)
1449 #arch_summary[target] = '\n'.join(lines)
1450 if lines:
1451 Print('%s:' % arch)
1452 _OutputConfigInfo(lines)
1453
Simon Glassc05aa032019-10-31 07:42:53 -06001454 for lines, targets in lines_by_target.items():
Simon Glass8270e3c2015-08-25 21:52:14 -06001455 if not lines:
1456 continue
1457 Print('%s :' % ' '.join(sorted(targets)))
1458 _OutputConfigInfo(lines.split('\n'))
1459
Simon Glass843312d2015-02-05 22:06:15 -07001460
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001461 # Save our updated information for the next call to this function
1462 self._base_board_dict = board_dict
1463 self._base_err_lines = err_lines
Simon Glasse30965d2014-08-28 09:43:44 -06001464 self._base_warn_lines = warn_lines
1465 self._base_err_line_boards = err_line_boards
1466 self._base_warn_line_boards = warn_line_boards
Simon Glass843312d2015-02-05 22:06:15 -07001467 self._base_config = config
Alex Kiernan48ae4122018-05-31 04:48:34 +00001468 self._base_environment = environment
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001469
1470 # Get a list of boards that did not get built, if needed
1471 not_built = []
1472 for board in board_selected:
1473 if not board in board_dict:
1474 not_built.append(board)
1475 if not_built:
Simon Glass4653a882014-09-05 19:00:07 -06001476 Print("Boards not built (%d): %s" % (len(not_built),
1477 ', '.join(not_built)))
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001478
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001479 def ProduceResultSummary(self, commit_upto, commits, board_selected):
Simon Glasse30965d2014-08-28 09:43:44 -06001480 (board_dict, err_lines, err_line_boards, warn_lines,
Alex Kiernan48ae4122018-05-31 04:48:34 +00001481 warn_line_boards, config, environment) = self.GetResultSummary(
Simon Glassed966652014-08-28 09:43:43 -06001482 board_selected, commit_upto,
Simon Glass843312d2015-02-05 22:06:15 -07001483 read_func_sizes=self._show_bloat,
Alex Kiernan48ae4122018-05-31 04:48:34 +00001484 read_config=self._show_config,
1485 read_environment=self._show_environment)
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001486 if commits:
1487 msg = '%02d: %s' % (commit_upto + 1,
1488 commits[commit_upto].subject)
Simon Glass4653a882014-09-05 19:00:07 -06001489 Print(msg, colour=self.col.BLUE)
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001490 self.PrintResultSummary(board_selected, board_dict,
Simon Glassed966652014-08-28 09:43:43 -06001491 err_lines if self._show_errors else [], err_line_boards,
Simon Glasse30965d2014-08-28 09:43:44 -06001492 warn_lines if self._show_errors else [], warn_line_boards,
Alex Kiernan48ae4122018-05-31 04:48:34 +00001493 config, environment, self._show_sizes, self._show_detail,
1494 self._show_bloat, self._show_config, self._show_environment)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001495
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001496 def ShowSummary(self, commits, board_selected):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001497 """Show a build summary for U-Boot for a given board list.
1498
1499 Reset the result summary, then repeatedly call GetResultSummary on
1500 each commit's results, then display the differences we see.
1501
1502 Args:
1503 commit: Commit objects to summarise
1504 board_selected: Dict containing boards to summarise
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001505 """
Simon Glassfea58582014-08-09 15:32:59 -06001506 self.commit_count = len(commits) if commits else 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001507 self.commits = commits
1508 self.ResetResultSummary(board_selected)
Simon Glass28370c12014-08-09 15:33:06 -06001509 self._error_lines = 0
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001510
1511 for commit_upto in range(0, self.commit_count, self._step):
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001512 self.ProduceResultSummary(commit_upto, commits, board_selected)
Simon Glass28370c12014-08-09 15:33:06 -06001513 if not self._error_lines:
Simon Glass4653a882014-09-05 19:00:07 -06001514 Print('(no errors to report)', colour=self.col.GREEN)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001515
1516
1517 def SetupBuild(self, board_selected, commits):
1518 """Set up ready to start a build.
1519
1520 Args:
1521 board_selected: Selected boards to build
1522 commits: Selected commits to build
1523 """
1524 # First work out how many commits we will build
Simon Glassc05aa032019-10-31 07:42:53 -06001525 count = (self.commit_count + self._step - 1) // self._step
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001526 self.count = len(board_selected) * count
1527 self.upto = self.warned = self.fail = 0
1528 self._timestamps = collections.deque()
1529
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001530 def GetThreadDir(self, thread_num):
1531 """Get the directory path to the working dir for a thread.
1532
1533 Args:
1534 thread_num: Number of thread to check.
1535 """
Simon Glassd829f122020-03-18 09:42:42 -06001536 if self.work_in_output:
1537 return self._working_dir
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001538 return os.path.join(self._working_dir, '%02d' % thread_num)
1539
Simon Glassfea58582014-08-09 15:32:59 -06001540 def _PrepareThread(self, thread_num, setup_git):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001541 """Prepare the working directory for a thread.
1542
1543 This clones or fetches the repo into the thread's work directory.
1544
1545 Args:
1546 thread_num: Thread number (0, 1, ...)
Simon Glassfea58582014-08-09 15:32:59 -06001547 setup_git: True to set up a git repo clone
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001548 """
1549 thread_dir = self.GetThreadDir(thread_num)
Simon Glass190064b2014-08-09 15:33:00 -06001550 builderthread.Mkdir(thread_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001551 git_dir = os.path.join(thread_dir, '.git')
1552
1553 # Clone the repo if it doesn't already exist
1554 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1555 # we have a private index but uses the origin repo's contents?
Simon Glassfea58582014-08-09 15:32:59 -06001556 if setup_git and self.git_dir:
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001557 src_dir = os.path.abspath(self.git_dir)
1558 if os.path.exists(git_dir):
Simon Glass212c0b82020-04-09 15:08:43 -06001559 Print('\rFetching repo for thread %d' % thread_num,
1560 newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001561 gitutil.Fetch(git_dir, thread_dir)
Simon Glass212c0b82020-04-09 15:08:43 -06001562 terminal.PrintClear()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001563 else:
Simon Glass21f0eb32016-09-18 16:48:31 -06001564 Print('\rCloning repo for thread %d' % thread_num,
1565 newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001566 gitutil.Clone(src_dir, thread_dir)
Simon Glass102969b2020-04-09 15:08:42 -06001567 terminal.PrintClear()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001568
Simon Glassfea58582014-08-09 15:32:59 -06001569 def _PrepareWorkingSpace(self, max_threads, setup_git):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001570 """Prepare the working directory for use.
1571
1572 Set up the git repo for each thread.
1573
1574 Args:
1575 max_threads: Maximum number of threads we expect to need.
Simon Glassfea58582014-08-09 15:32:59 -06001576 setup_git: True to set up a git repo clone
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001577 """
Simon Glass190064b2014-08-09 15:33:00 -06001578 builderthread.Mkdir(self._working_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001579 for thread in range(max_threads):
Simon Glassfea58582014-08-09 15:32:59 -06001580 self._PrepareThread(thread, setup_git)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001581
Simon Glass925f6ad2020-03-18 09:42:45 -06001582 def _GetOutputSpaceRemovals(self):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001583 """Get the output directories ready to receive files.
1584
Simon Glass925f6ad2020-03-18 09:42:45 -06001585 Figure out what needs to be deleted in the output directory before it
1586 can be used. We only delete old buildman directories which have the
1587 expected name pattern. See _GetOutputDir().
1588
1589 Returns:
1590 List of full paths of directories to remove
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001591 """
Simon Glass1a915672014-12-01 17:33:53 -07001592 if not self.commits:
1593 return
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001594 dir_list = []
1595 for commit_upto in range(self.commit_count):
1596 dir_list.append(self._GetOutputDir(commit_upto))
1597
Simon Glassb222abe2016-09-18 16:48:32 -06001598 to_remove = []
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001599 for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1600 if dirname not in dir_list:
Simon Glass925f6ad2020-03-18 09:42:45 -06001601 leaf = dirname[len(self.base_dir) + 1:]
1602 m = re.match('[0-9]+_of_[0-9]+_g[0-9a-f]+_.*', leaf)
1603 if m:
1604 to_remove.append(dirname)
1605 return to_remove
1606
1607 def _PrepareOutputSpace(self):
1608 """Get the output directories ready to receive files.
1609
1610 We delete any output directories which look like ones we need to
1611 create. Having left over directories is confusing when the user wants
1612 to check the output manually.
1613 """
1614 to_remove = self._GetOutputSpaceRemovals()
Simon Glassb222abe2016-09-18 16:48:32 -06001615 if to_remove:
Simon Glassb2d89bc2020-03-18 09:42:46 -06001616 Print('Removing %d old build directories...' % len(to_remove),
Simon Glassb222abe2016-09-18 16:48:32 -06001617 newline=False)
1618 for dirname in to_remove:
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001619 shutil.rmtree(dirname)
Simon Glass212c0b82020-04-09 15:08:43 -06001620 terminal.PrintClear()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001621
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001622 def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001623 """Build all commits for a list of boards
1624
1625 Args:
1626 commits: List of commits to be build, each a Commit object
1627 boards_selected: Dict of selected boards, key is target name,
1628 value is Board object
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001629 keep_outputs: True to save build output files
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001630 verbose: Display build results as they are completed
Simon Glass2c3deb92014-08-28 09:43:39 -06001631 Returns:
1632 Tuple containing:
1633 - number of boards that failed to build
1634 - number of boards that issued warnings
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001635 """
Simon Glassfea58582014-08-09 15:32:59 -06001636 self.commit_count = len(commits) if commits else 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001637 self.commits = commits
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001638 self._verbose = verbose
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001639
1640 self.ResetResultSummary(board_selected)
Thierry Redingf3d015c2014-08-19 10:22:39 +02001641 builderthread.Mkdir(self.base_dir, parents = True)
Simon Glassfea58582014-08-09 15:32:59 -06001642 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1643 commits is not None)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001644 self._PrepareOutputSpace()
Simon Glass745b3952016-09-18 16:48:33 -06001645 Print('\rStarting build...', newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001646 self.SetupBuild(board_selected, commits)
1647 self.ProcessResult(None)
1648
1649 # Create jobs to build all commits for each board
Simon Glassc05aa032019-10-31 07:42:53 -06001650 for brd in board_selected.values():
Simon Glass190064b2014-08-09 15:33:00 -06001651 job = builderthread.BuilderJob()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001652 job.board = brd
1653 job.commits = commits
1654 job.keep_outputs = keep_outputs
Simon Glassd829f122020-03-18 09:42:42 -06001655 job.work_in_output = self.work_in_output
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001656 job.step = self._step
1657 self.queue.put(job)
1658
Simon Glassd436e382016-09-18 16:48:35 -06001659 term = threading.Thread(target=self.queue.join)
1660 term.setDaemon(True)
1661 term.start()
1662 while term.isAlive():
1663 term.join(100)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001664
1665 # Wait until we have processed all output
1666 self.out_queue.join()
Simon Glass4653a882014-09-05 19:00:07 -06001667 Print()
Simon Glass7b33f212020-04-09 15:08:47 -06001668
1669 msg = 'Completed: %d total built' % self.count
1670 if self.already_done:
1671 msg += ' (%d previously' % self.already_done
1672 if self.already_done != self.count:
1673 msg += ', %d newly' % (self.count - self.already_done)
1674 msg += ')'
1675 duration = datetime.now() - self._start_time
1676 if duration > timedelta(microseconds=1000000):
1677 if duration.microseconds >= 500000:
1678 duration = duration + timedelta(seconds=1)
1679 duration = duration - timedelta(microseconds=duration.microseconds)
1680 msg += ', duration %s' % duration
1681 Print(msg)
1682
Simon Glass2c3deb92014-08-28 09:43:39 -06001683 return (self.fail, self.warned)