blob: 3e42c987d1cd14fdcdb00dc16734f0878afbcac0 [file] [log] [blame]
Tom Rini83d290c2018-05-06 17:58:06 -04001# SPDX-License-Identifier: GPL-2.0+
Simon Glassfc3fe1c2013-04-03 11:07:16 +00002# Copyright (c) 2013 The Chromium OS Authors.
3#
4# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
5#
Simon Glassfc3fe1c2013-04-03 11:07:16 +00006
7import collections
Simon Glassfc3fe1c2013-04-03 11:07:16 +00008from datetime import datetime, timedelta
9import glob
10import os
11import re
Simon Glassc05aa032019-10-31 07:42:53 -060012import queue
Simon Glassfc3fe1c2013-04-03 11:07:16 +000013import shutil
Simon Glass2f256642016-09-18 16:48:37 -060014import signal
Simon Glassfc3fe1c2013-04-03 11:07:16 +000015import string
16import sys
Simon Glassd436e382016-09-18 16:48:35 -060017import threading
Simon Glassfc3fe1c2013-04-03 11:07:16 +000018import time
19
Simon Glass0ede00f2020-04-17 18:09:02 -060020from buildman import builderthread
21from buildman import toolchain
Simon Glassbf776672020-04-17 18:09:04 -060022from patman import gitutil
Simon Glass4583c002023-02-23 18:18:04 -070023from u_boot_pylib import command
24from u_boot_pylib import terminal
25from u_boot_pylib.terminal import tprint
Simon Glassfc3fe1c2013-04-03 11:07:16 +000026
Simon Glass7bf83a52021-10-19 21:43:24 -060027# This indicates an new int or hex Kconfig property with no default
28# It hangs the build since the 'conf' tool cannot proceed without valid input.
29#
30# We get a repeat sequence of something like this:
31# >>
32# Break things (BREAK_ME) [] (NEW)
33# Error in reading or end of file.
34# <<
35# which indicates that BREAK_ME has an empty default
36RE_NO_DEFAULT = re.compile(b'\((\w+)\) \[] \(NEW\)')
37
Simon Glassad8dbab2023-10-23 00:52:43 -070038# Symbol types which appear in the bloat feature (-B). Others are silently
39# dropped when reading in the 'nm' output
40NM_SYMBOL_TYPES = 'tTdDbBr'
41
Simon Glassfc3fe1c2013-04-03 11:07:16 +000042"""
43Theory of Operation
44
45Please see README for user documentation, and you should be familiar with
46that before trying to make sense of this.
47
48Buildman works by keeping the machine as busy as possible, building different
49commits for different boards on multiple CPUs at once.
50
51The source repo (self.git_dir) contains all the commits to be built. Each
52thread works on a single board at a time. It checks out the first commit,
53configures it for that board, then builds it. Then it checks out the next
54commit and builds it (typically without re-configuring). When it runs out
55of commits, it gets another job from the builder and starts again with that
56board.
57
58Clearly the builder threads could work either way - they could check out a
59commit and then built it for all boards. Using separate directories for each
60commit/board pair they could leave their build product around afterwards
61also.
62
63The intent behind building a single board for multiple commits, is to make
64use of incremental builds. Since each commit is built incrementally from
65the previous one, builds are faster. Reconfiguring for a different board
66removes all intermediate object files.
67
68Many threads can be working at once, but each has its own working directory.
69When a thread finishes a build, it puts the output files into a result
70directory.
71
72The base directory used by buildman is normally '../<branch>', i.e.
73a directory higher than the source repository and named after the branch
74being built.
75
76Within the base directory, we have one subdirectory for each commit. Within
77that is one subdirectory for each board. Within that is the build output for
78that commit/board combination.
79
80Buildman also create working directories for each thread, in a .bm-work/
81subdirectory in the base dir.
82
83As an example, say we are building branch 'us-net' for boards 'sandbox' and
84'seaboard', and say that us-net has two commits. We will have directories
85like this:
86
87us-net/ base directory
Ovidiu Panait7664b032020-05-15 09:30:12 +030088 01_g4ed4ebc_net--Add-tftp-speed-/
Simon Glassfc3fe1c2013-04-03 11:07:16 +000089 sandbox/
90 u-boot.bin
91 seaboard/
92 u-boot.bin
Ovidiu Panait7664b032020-05-15 09:30:12 +030093 02_g4ed4ebc_net--Check-tftp-comp/
Simon Glassfc3fe1c2013-04-03 11:07:16 +000094 sandbox/
95 u-boot.bin
96 seaboard/
97 u-boot.bin
98 .bm-work/
99 00/ working directory for thread 0 (contains source checkout)
100 build/ build output
101 01/ working directory for thread 1
102 build/ build output
103 ...
104u-boot/ source directory
105 .git/ repository
106"""
107
Simon Glass35d696d2020-04-09 15:08:36 -0600108"""Holds information about a particular error line we are outputing
109
110 char: Character representation: '+': error, '-': fixed error, 'w+': warning,
111 'w-' = fixed warning
112 boards: List of Board objects which have line in the error/warning output
113 errline: The text of the error line
114"""
Simon Glasscc2c0d12022-07-11 19:04:00 -0600115ErrLine = collections.namedtuple('ErrLine', 'char,brds,errline')
Simon Glass35d696d2020-04-09 15:08:36 -0600116
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000117# Possible build outcomes
Simon Glassc05aa032019-10-31 07:42:53 -0600118OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000119
Simon Glass9a6d2e22017-04-12 18:23:26 -0600120# Translate a commit subject into a valid filename (and handle unicode)
Simon Glassc05aa032019-10-31 07:42:53 -0600121trans_valid_chars = str.maketrans('/: ', '---')
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000122
Simon Glassb464f8e2016-11-13 14:25:53 -0700123BASE_CONFIG_FILENAMES = [
124 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
125]
126
127EXTRA_CONFIG_FILENAMES = [
Simon Glass843312d2015-02-05 22:06:15 -0700128 '.config', '.config-spl', '.config-tpl',
129 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
130 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
Simon Glass843312d2015-02-05 22:06:15 -0700131]
132
Simon Glass8270e3c2015-08-25 21:52:14 -0600133class Config:
134 """Holds information about configuration settings for a board."""
Simon Glassb464f8e2016-11-13 14:25:53 -0700135 def __init__(self, config_filename, target):
Simon Glass8270e3c2015-08-25 21:52:14 -0600136 self.target = target
137 self.config = {}
Simon Glassb464f8e2016-11-13 14:25:53 -0700138 for fname in config_filename:
Simon Glass8270e3c2015-08-25 21:52:14 -0600139 self.config[fname] = {}
140
Simon Glass37edf5f2023-07-19 17:49:06 -0600141 def add(self, fname, key, value):
Simon Glass8270e3c2015-08-25 21:52:14 -0600142 self.config[fname][key] = value
143
144 def __hash__(self):
145 val = 0
146 for fname in self.config:
Simon Glassc05aa032019-10-31 07:42:53 -0600147 for key, value in self.config[fname].items():
148 print(key, value)
Simon Glass8270e3c2015-08-25 21:52:14 -0600149 val = val ^ hash(key) & hash(value)
150 return val
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000151
Alex Kiernan48ae4122018-05-31 04:48:34 +0000152class Environment:
153 """Holds information about environment variables for a board."""
154 def __init__(self, target):
155 self.target = target
156 self.environment = {}
157
Simon Glass37edf5f2023-07-19 17:49:06 -0600158 def add(self, key, value):
Alex Kiernan48ae4122018-05-31 04:48:34 +0000159 self.environment[key] = value
160
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000161class Builder:
162 """Class for building U-Boot for a particular commit.
163
164 Public members: (many should ->private)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000165 already_done: Number of builds already completed
166 base_dir: Base directory to use for builder
167 checkout: True to check out source, False to skip that step.
168 This is used for testing.
169 col: terminal.Color() object
Simon Glass6a0c7b42023-07-19 17:49:03 -0600170 count: Total number of commits to build, which is the number of commits
171 multiplied by the number of boards
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000172 do_make: Method to call to invoke Make
173 fail: Number of builds that failed due to error
174 force_build: Force building even if a build already exists
175 force_config_on_failure: If a commit fails for a board, disable
176 incremental building for the next commit we build for that
177 board, so that we will see all warnings/errors again.
Simon Glass4266dc22014-07-13 12:22:31 -0600178 force_build_failures: If a previously-built build (i.e. built on
179 a previous run of buildman) is marked as failed, rebuild it.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000180 git_dir: Git directory containing source repository
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000181 num_jobs: Number of jobs to run at once (passed to make as -j)
182 num_threads: Number of builder threads to run
183 out_queue: Queue of results to process
184 re_make_err: Compiled regular expression for ignore_lines
185 queue: Queue of jobs to run
186 threads: List of active threads
187 toolchains: Toolchains object to use for building
188 upto: Current commit number we are building (0.count-1)
189 warned: Number of builds that produced at least one warning
Simon Glass97e91522014-07-14 17:51:02 -0600190 force_reconfig: Reconfigure U-Boot on each comiit. This disables
191 incremental building, where buildman reconfigures on the first
192 commit for a baord, and then just does an incremental build for
193 the following commits. In fact buildman will reconfigure and
194 retry for any failing commits, so generally the only effect of
195 this option is to slow things down.
Simon Glass189a4962014-07-14 17:51:03 -0600196 in_tree: Build U-Boot in-tree instead of specifying an output
197 directory separate from the source code. This option is really
198 only useful for testing in-tree builds.
Simon Glassd829f122020-03-18 09:42:42 -0600199 work_in_output: Use the output directory as the work directory and
200 don't write to a separate output directory.
Simon Glass8116c782021-04-11 16:27:27 +1200201 thread_exceptions: List of exceptions raised by thread jobs
Simon Glass93202d72023-02-21 12:40:28 -0700202 no_lto (bool): True to set the NO_LTO flag when building
Simon Glassbfb708a2023-02-21 12:40:29 -0700203 reproducible_builds (bool): True to set SOURCE_DATE_EPOCH=0 for builds
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000204
205 Private members:
206 _base_board_dict: Last-summarised Dict of boards
207 _base_err_lines: Last-summarised list of errors
Simon Glasse30965d2014-08-28 09:43:44 -0600208 _base_warn_lines: Last-summarised list of warnings
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000209 _build_period_us: Time taken for a single build (float object).
210 _complete_delay: Expected delay until completion (timedelta)
211 _next_delay_update: Next time we plan to display a progress update
212 (datatime)
213 _show_unknown: Show unknown boards (those not built) in summary
Simon Glass7b33f212020-04-09 15:08:47 -0600214 _start_time: Start time for the build
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000215 _timestamps: List of timestamps for the completion of the last
216 last _timestamp_count builds. Each is a datetime object.
217 _timestamp_count: Number of timestamps to keep in our list.
218 _working_dir: Base working directory containing all threads
Simon Glassb82492b2021-01-30 22:17:46 -0700219 _single_builder: BuilderThread object for the singer builder, if
220 threading is not being used
Simon Glass7bf83a52021-10-19 21:43:24 -0600221 _terminated: Thread was terminated due to an error
222 _restarting_config: True if 'Restart config' is detected in output
Simon Glassae1a09f2022-07-11 19:03:56 -0600223 _ide: Produce output suitable for an Integrated Development Environment,
224 i.e. dont emit progress information and put errors/warnings on stderr
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000225 """
226 class Outcome:
227 """Records a build outcome for a single make invocation
228
229 Public Members:
230 rc: Outcome value (OUTCOME_...)
231 err_lines: List of error lines or [] if none
232 sizes: Dictionary of image size information, keyed by filename
233 - Each value is itself a dictionary containing
234 values for 'text', 'data' and 'bss', being the integer
235 size in bytes of each section.
236 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
237 value is itself a dictionary:
238 key: function name
239 value: Size of function in bytes
Simon Glass843312d2015-02-05 22:06:15 -0700240 config: Dictionary keyed by filename - e.g. '.config'. Each
241 value is itself a dictionary:
242 key: config name
243 value: config value
Alex Kiernan48ae4122018-05-31 04:48:34 +0000244 environment: Dictionary keyed by environment variable, Each
245 value is the value of environment variable.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000246 """
Alex Kiernan48ae4122018-05-31 04:48:34 +0000247 def __init__(self, rc, err_lines, sizes, func_sizes, config,
248 environment):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000249 self.rc = rc
250 self.err_lines = err_lines
251 self.sizes = sizes
252 self.func_sizes = func_sizes
Simon Glass843312d2015-02-05 22:06:15 -0700253 self.config = config
Alex Kiernan48ae4122018-05-31 04:48:34 +0000254 self.environment = environment
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000255
256 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
Simon Glass5971ab52014-12-01 17:33:55 -0700257 gnu_make='make', checkout=True, show_unknown=True, step=1,
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600258 no_subdirs=False, full_path=False, verbose_build=False,
Simon Glasseb70a2c2020-04-09 15:08:51 -0600259 mrproper=False, per_board_out_dir=False,
Daniel Schwierzeck2371d1b2018-01-26 16:31:05 +0100260 config_only=False, squash_config_y=False,
Simon Glass8116c782021-04-11 16:27:27 +1200261 warnings_as_errors=False, work_in_output=False,
Tom Rinid7713ad2022-11-09 19:14:53 -0700262 test_thread_exceptions=False, adjust_cfg=None,
Simon Glassffd06d32023-07-19 17:48:52 -0600263 allow_missing=False, no_lto=False, reproducible_builds=False,
264 force_build=False, force_build_failures=False,
265 force_reconfig=False, in_tree=False,
266 force_config_on_failure=False, make_func=None):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000267 """Create a new Builder object
268
269 Args:
270 toolchains: Toolchains object to use for building
271 base_dir: Base directory to use for builder
272 git_dir: Git directory containing source repository
273 num_threads: Number of builder threads to run
274 num_jobs: Number of jobs to run at once (passed to make as -j)
Masahiro Yamada99796922014-07-22 11:19:09 +0900275 gnu_make: the command name of GNU Make.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000276 checkout: True to check out source, False to skip that step.
277 This is used for testing.
278 show_unknown: Show unknown boards (those not built) in summary
279 step: 1 to process every commit, n to process every nth commit
Simon Glassbb1501f2014-12-01 17:34:00 -0700280 no_subdirs: Don't create subdirectories when building current
281 source for a single board
282 full_path: Return the full path in CROSS_COMPILE and don't set
283 PATH
Simon Glassd2ce6582014-12-01 17:34:07 -0700284 verbose_build: Run build with V=1 and don't use 'make -s'
Simon Glasseb70a2c2020-04-09 15:08:51 -0600285 mrproper: Always run 'make mrproper' when configuring
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600286 per_board_out_dir: Build in a separate persistent directory per
287 board rather than a thread-specific directory
Simon Glassb50113f2016-11-13 14:25:51 -0700288 config_only: Only configure each build, don't build it
Simon Glassb464f8e2016-11-13 14:25:53 -0700289 squash_config_y: Convert CONFIG options with the value 'y' to '1'
Daniel Schwierzeck2371d1b2018-01-26 16:31:05 +0100290 warnings_as_errors: Treat all compiler warnings as errors
Simon Glassd829f122020-03-18 09:42:42 -0600291 work_in_output: Use the output directory as the work directory and
292 don't write to a separate output directory.
Simon Glass8116c782021-04-11 16:27:27 +1200293 test_thread_exceptions: Uses for tests only, True to make the
294 threads raise an exception instead of reporting their result.
295 This simulates a failure in the code somewhere
Simon Glass2b4806e2022-01-22 05:07:33 -0700296 adjust_cfg_list (list of str): List of changes to make to .config
297 file before building. Each is one of (where C is the config
298 option with or without the CONFIG_ prefix)
299
300 C to enable C
301 ~C to disable C
302 C=val to set the value of C (val must have quotes if C is
303 a string Kconfig
Tom Rinid7713ad2022-11-09 19:14:53 -0700304 allow_missing: Run build with BINMAN_ALLOW_MISSING=1
Simon Glass93202d72023-02-21 12:40:28 -0700305 no_lto (bool): True to set the NO_LTO flag when building
Simon Glassffd06d32023-07-19 17:48:52 -0600306 force_build (bool): Rebuild even commits that are already built
307 force_build_failures (bool): Rebuild commits that have not been
308 built, or failed to build
309 force_reconfig (bool): Reconfigure on each commit
310 in_tree (bool): Bulid in tree instead of out-of-tree
311 force_config_on_failure (bool): Reconfigure the build before
312 retrying a failed build
313 make_func (function): Function to call to run 'make'
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000314 """
315 self.toolchains = toolchains
316 self.base_dir = base_dir
Simon Glassd829f122020-03-18 09:42:42 -0600317 if work_in_output:
318 self._working_dir = base_dir
319 else:
320 self._working_dir = os.path.join(base_dir, '.bm-work')
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000321 self.threads = []
Simon Glass37edf5f2023-07-19 17:49:06 -0600322 self.do_make = make_func or self.make
Masahiro Yamada99796922014-07-22 11:19:09 +0900323 self.gnu_make = gnu_make
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000324 self.checkout = checkout
325 self.num_threads = num_threads
326 self.num_jobs = num_jobs
327 self.already_done = 0
328 self.force_build = False
329 self.git_dir = git_dir
330 self._show_unknown = show_unknown
331 self._timestamp_count = 10
332 self._build_period_us = None
333 self._complete_delay = None
334 self._next_delay_update = datetime.now()
Simon Glass2ce06f52023-09-07 10:00:19 -0600335 self._start_time = None
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000336 self._step = step
Simon Glass28370c12014-08-09 15:33:06 -0600337 self._error_lines = 0
Simon Glass5971ab52014-12-01 17:33:55 -0700338 self.no_subdirs = no_subdirs
Simon Glassbb1501f2014-12-01 17:34:00 -0700339 self.full_path = full_path
Simon Glassd2ce6582014-12-01 17:34:07 -0700340 self.verbose_build = verbose_build
Simon Glassb50113f2016-11-13 14:25:51 -0700341 self.config_only = config_only
Simon Glassb464f8e2016-11-13 14:25:53 -0700342 self.squash_config_y = squash_config_y
343 self.config_filenames = BASE_CONFIG_FILENAMES
Simon Glassd829f122020-03-18 09:42:42 -0600344 self.work_in_output = work_in_output
Simon Glass2b4806e2022-01-22 05:07:33 -0700345 self.adjust_cfg = adjust_cfg
Tom Rinid7713ad2022-11-09 19:14:53 -0700346 self.allow_missing = allow_missing
Simon Glassae1a09f2022-07-11 19:03:56 -0600347 self._ide = False
Simon Glass93202d72023-02-21 12:40:28 -0700348 self.no_lto = no_lto
Simon Glassbfb708a2023-02-21 12:40:29 -0700349 self.reproducible_builds = reproducible_builds
Simon Glassffd06d32023-07-19 17:48:52 -0600350 self.force_build = force_build
351 self.force_build_failures = force_build_failures
352 self.force_reconfig = force_reconfig
353 self.in_tree = in_tree
354 self.force_config_on_failure = force_config_on_failure
Simon Glass2b4806e2022-01-22 05:07:33 -0700355
Simon Glassb464f8e2016-11-13 14:25:53 -0700356 if not self.squash_config_y:
357 self.config_filenames += EXTRA_CONFIG_FILENAMES
Simon Glass7bf83a52021-10-19 21:43:24 -0600358 self._terminated = False
359 self._restarting_config = False
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000360
Daniel Schwierzeck2371d1b2018-01-26 16:31:05 +0100361 self.warnings_as_errors = warnings_as_errors
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000362 self.col = terminal.Color()
363
Simon Glasse30965d2014-08-28 09:43:44 -0600364 self._re_function = re.compile('(.*): In function.*')
365 self._re_files = re.compile('In file included from.*')
366 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
Simon Glass2d483332018-11-06 16:02:11 -0700367 self._re_dtb_warning = re.compile('(.*): Warning .*')
Simon Glasse30965d2014-08-28 09:43:44 -0600368 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
Simon Glass113a8a52020-04-09 15:08:53 -0600369 self._re_migration_warning = re.compile(r'^={21} WARNING ={22}\n.*\n=+\n',
370 re.MULTILINE | re.DOTALL)
Simon Glasse30965d2014-08-28 09:43:44 -0600371
Simon Glass8116c782021-04-11 16:27:27 +1200372 self.thread_exceptions = []
373 self.test_thread_exceptions = test_thread_exceptions
Simon Glassb82492b2021-01-30 22:17:46 -0700374 if self.num_threads:
375 self._single_builder = None
376 self.queue = queue.Queue()
377 self.out_queue = queue.Queue()
378 for i in range(self.num_threads):
Simon Glass8116c782021-04-11 16:27:27 +1200379 t = builderthread.BuilderThread(
380 self, i, mrproper, per_board_out_dir,
381 test_exception=test_thread_exceptions)
Simon Glassb82492b2021-01-30 22:17:46 -0700382 t.setDaemon(True)
383 t.start()
384 self.threads.append(t)
385
386 t = builderthread.ResultThread(self)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000387 t.setDaemon(True)
388 t.start()
389 self.threads.append(t)
Simon Glassb82492b2021-01-30 22:17:46 -0700390 else:
391 self._single_builder = builderthread.BuilderThread(
392 self, -1, mrproper, per_board_out_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000393
394 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
395 self.re_make_err = re.compile('|'.join(ignore_lines))
396
Simon Glass2f256642016-09-18 16:48:37 -0600397 # Handle existing graceful with SIGINT / Ctrl-C
398 signal.signal(signal.SIGINT, self.signal_handler)
399
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000400 def __del__(self):
401 """Get rid of all threads created by the builder"""
402 for t in self.threads:
403 del t
404
Simon Glass2f256642016-09-18 16:48:37 -0600405 def signal_handler(self, signal, frame):
406 sys.exit(1)
407
Simon Glass37edf5f2023-07-19 17:49:06 -0600408 def set_display_options(self, show_errors=False, show_sizes=False,
Simon Glassed966652014-08-28 09:43:43 -0600409 show_detail=False, show_bloat=False,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000410 list_error_boards=False, show_config=False,
Simon Glass113a8a52020-04-09 15:08:53 -0600411 show_environment=False, filter_dtb_warnings=False,
Simon Glassae1a09f2022-07-11 19:03:56 -0600412 filter_migration_warnings=False, ide=False):
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600413 """Setup display options for the builder.
414
Simon Glass174592b2020-04-09 15:08:52 -0600415 Args:
416 show_errors: True to show summarised error/warning info
417 show_sizes: Show size deltas
418 show_detail: Show size delta detail for each board if show_sizes
419 show_bloat: Show detail for each function
420 list_error_boards: Show the boards which caused each error/warning
421 show_config: Show config deltas
422 show_environment: Show environment deltas
423 filter_dtb_warnings: Filter out any warnings from the device-tree
424 compiler
Simon Glass113a8a52020-04-09 15:08:53 -0600425 filter_migration_warnings: Filter out any warnings about migrating
426 a board to driver model
Simon Glassae1a09f2022-07-11 19:03:56 -0600427 ide: Create output that can be parsed by an IDE. There is no '+' prefix on
428 error lines and output on stderr stays on stderr.
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600429 """
430 self._show_errors = show_errors
431 self._show_sizes = show_sizes
432 self._show_detail = show_detail
433 self._show_bloat = show_bloat
Simon Glassed966652014-08-28 09:43:43 -0600434 self._list_error_boards = list_error_boards
Simon Glass843312d2015-02-05 22:06:15 -0700435 self._show_config = show_config
Alex Kiernan48ae4122018-05-31 04:48:34 +0000436 self._show_environment = show_environment
Simon Glass174592b2020-04-09 15:08:52 -0600437 self._filter_dtb_warnings = filter_dtb_warnings
Simon Glass113a8a52020-04-09 15:08:53 -0600438 self._filter_migration_warnings = filter_migration_warnings
Simon Glassae1a09f2022-07-11 19:03:56 -0600439 self._ide = ide
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600440
Simon Glass37edf5f2023-07-19 17:49:06 -0600441 def _add_timestamp(self):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000442 """Add a new timestamp to the list and record the build period.
443
444 The build period is the length of time taken to perform a single
445 build (one board, one commit).
446 """
447 now = datetime.now()
448 self._timestamps.append(now)
449 count = len(self._timestamps)
450 delta = self._timestamps[-1] - self._timestamps[0]
451 seconds = delta.total_seconds()
452
453 # If we have enough data, estimate build period (time taken for a
454 # single build) and therefore completion time.
455 if count > 1 and self._next_delay_update < now:
456 self._next_delay_update = now + timedelta(seconds=2)
457 if seconds > 0:
458 self._build_period = float(seconds) / count
459 todo = self.count - self.upto
460 self._complete_delay = timedelta(microseconds=
461 self._build_period * todo * 1000000)
462 # Round it
463 self._complete_delay -= timedelta(
464 microseconds=self._complete_delay.microseconds)
465
466 if seconds > 60:
467 self._timestamps.popleft()
468 count -= 1
469
Simon Glass37edf5f2023-07-19 17:49:06 -0600470 def select_commit(self, commit, checkout=True):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000471 """Checkout the selected commit for this build
472 """
473 self.commit = commit
474 if checkout and self.checkout:
Simon Glass0157b182022-01-29 14:14:11 -0700475 gitutil.checkout(commit.hash)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000476
Simon Glass37edf5f2023-07-19 17:49:06 -0600477 def make(self, commit, brd, stage, cwd, *args, **kwargs):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000478 """Run make
479
480 Args:
481 commit: Commit object that is being built
482 brd: Board object that is being built
Roger Meierfd18a892014-08-20 22:10:29 +0200483 stage: Stage that we are at (mrproper, config, build)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000484 cwd: Directory where make should be run
485 args: Arguments to pass to make
Simon Glassd9800692022-01-29 14:14:05 -0700486 kwargs: Arguments to pass to command.run_pipe()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000487 """
Simon Glass7bf83a52021-10-19 21:43:24 -0600488
489 def check_output(stream, data):
490 if b'Restart config' in data:
491 self._restarting_config = True
492
493 # If we see 'Restart config' following by multiple errors
494 if self._restarting_config:
495 m = RE_NO_DEFAULT.findall(data)
496
497 # Number of occurences of each Kconfig item
498 multiple = [m.count(val) for val in set(m)]
499
500 # If any of them occur more than once, we have a loop
501 if [val for val in multiple if val > 1]:
502 self._terminated = True
503 return True
504 return False
505
506 self._restarting_config = False
507 self._terminated = False
Masahiro Yamada99796922014-07-22 11:19:09 +0900508 cmd = [self.gnu_make] + list(args)
Simon Glassd9800692022-01-29 14:14:05 -0700509 result = command.run_pipe([cmd], capture=True, capture_stderr=True,
Simon Glass7bf83a52021-10-19 21:43:24 -0600510 cwd=cwd, raise_on_error=False, infile='/dev/null',
511 output_func=check_output, **kwargs)
512
513 if self._terminated:
514 # Try to be helpful
515 result.stderr += '(** did you define an int/hex Kconfig with no default? **)'
516
Simon Glass40f11fc2015-02-05 22:06:12 -0700517 if self.verbose_build:
518 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
519 result.combined = '%s\n' % (' '.join(cmd)) + result.combined
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000520 return result
521
Simon Glass37edf5f2023-07-19 17:49:06 -0600522 def process_result(self, result):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000523 """Process the result of a build, showing progress information
524
525 Args:
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600526 result: A CommandResult object, which indicates the result for
527 a single build
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000528 """
529 col = terminal.Color()
530 if result:
531 target = result.brd.target
532
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000533 self.upto += 1
534 if result.return_code != 0:
535 self.fail += 1
536 elif result.stderr:
537 self.warned += 1
538 if result.already_done:
539 self.already_done += 1
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600540 if self._verbose:
Simon Glass098b10f2022-01-29 14:14:18 -0700541 terminal.print_clear()
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600542 boards_selected = {target : result.brd}
Simon Glass37edf5f2023-07-19 17:49:06 -0600543 self.reset_result_summary(boards_selected)
544 self.produce_result_summary(result.commit_upto, self.commits,
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600545 boards_selected)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000546 else:
547 target = '(starting)'
548
549 # Display separate counts for ok, warned and fail
550 ok = self.upto - self.warned - self.fail
Simon Glass252ac582022-01-29 14:14:17 -0700551 line = '\r' + self.col.build(self.col.GREEN, '%5d' % ok)
552 line += self.col.build(self.col.YELLOW, '%5d' % self.warned)
553 line += self.col.build(self.col.RED, '%5d' % self.fail)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000554
Simon Glass6eb76ca2020-04-09 15:08:45 -0600555 line += ' /%-5d ' % self.count
556 remaining = self.count - self.upto
557 if remaining:
Simon Glass252ac582022-01-29 14:14:17 -0700558 line += self.col.build(self.col.MAGENTA, ' -%-5d ' % remaining)
Simon Glass6eb76ca2020-04-09 15:08:45 -0600559 else:
560 line += ' ' * 8
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000561
562 # Add our current completion time estimate
Simon Glass37edf5f2023-07-19 17:49:06 -0600563 self._add_timestamp()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000564 if self._complete_delay:
Simon Glass6eb76ca2020-04-09 15:08:45 -0600565 line += '%s : ' % self._complete_delay
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000566
Simon Glass6eb76ca2020-04-09 15:08:45 -0600567 line += target
Simon Glassae1a09f2022-07-11 19:03:56 -0600568 if not self._ide:
569 terminal.print_clear()
570 tprint(line, newline=False, limit_to_line=True)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000571
Simon Glass4a7419b2023-07-19 17:49:10 -0600572 def get_output_dir(self, commit_upto):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000573 """Get the name of the output directory for a commit number
574
575 The output directory is typically .../<branch>/<commit>.
576
577 Args:
578 commit_upto: Commit number to use (0..self.count-1)
579 """
Simon Glass60b285f2020-04-17 17:51:34 -0600580 if self.work_in_output:
581 return self._working_dir
582
Simon Glass5971ab52014-12-01 17:33:55 -0700583 commit_dir = None
Simon Glassfea58582014-08-09 15:32:59 -0600584 if self.commits:
585 commit = self.commits[commit_upto]
586 subject = commit.subject.translate(trans_valid_chars)
Simon Glass37edf5f2023-07-19 17:49:06 -0600587 # See _get_output_space_removals() which parses this name
Ovidiu Panait7664b032020-05-15 09:30:12 +0300588 commit_dir = ('%02d_g%s_%s' % (commit_upto + 1,
589 commit.hash, subject[:20]))
Simon Glass5971ab52014-12-01 17:33:55 -0700590 elif not self.no_subdirs:
Simon Glassfea58582014-08-09 15:32:59 -0600591 commit_dir = 'current'
Simon Glass5971ab52014-12-01 17:33:55 -0700592 if not commit_dir:
593 return self.base_dir
594 return os.path.join(self.base_dir, commit_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000595
Simon Glass37edf5f2023-07-19 17:49:06 -0600596 def get_build_dir(self, commit_upto, target):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000597 """Get the name of the build directory for a commit number
598
599 The build directory is typically .../<branch>/<commit>/<target>.
600
601 Args:
602 commit_upto: Commit number to use (0..self.count-1)
603 target: Target name
604 """
Simon Glass4a7419b2023-07-19 17:49:10 -0600605 output_dir = self.get_output_dir(commit_upto)
Simon Glass60b285f2020-04-17 17:51:34 -0600606 if self.work_in_output:
607 return output_dir
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000608 return os.path.join(output_dir, target)
609
Simon Glass37edf5f2023-07-19 17:49:06 -0600610 def get_done_file(self, commit_upto, target):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000611 """Get the name of the done file for a commit number
612
613 Args:
614 commit_upto: Commit number to use (0..self.count-1)
615 target: Target name
616 """
Simon Glass37edf5f2023-07-19 17:49:06 -0600617 return os.path.join(self.get_build_dir(commit_upto, target), 'done')
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000618
Simon Glass37edf5f2023-07-19 17:49:06 -0600619 def get_sizes_file(self, commit_upto, target):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000620 """Get the name of the sizes file for a commit number
621
622 Args:
623 commit_upto: Commit number to use (0..self.count-1)
624 target: Target name
625 """
Simon Glass37edf5f2023-07-19 17:49:06 -0600626 return os.path.join(self.get_build_dir(commit_upto, target), 'sizes')
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000627
Simon Glass37edf5f2023-07-19 17:49:06 -0600628 def get_func_sizes_file(self, commit_upto, target, elf_fname):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000629 """Get the name of the funcsizes file for a commit number and ELF file
630
631 Args:
632 commit_upto: Commit number to use (0..self.count-1)
633 target: Target name
634 elf_fname: Filename of elf image
635 """
Simon Glass37edf5f2023-07-19 17:49:06 -0600636 return os.path.join(self.get_build_dir(commit_upto, target),
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000637 '%s.sizes' % elf_fname.replace('/', '-'))
638
Simon Glass37edf5f2023-07-19 17:49:06 -0600639 def get_objdump_file(self, commit_upto, target, elf_fname):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000640 """Get the name of the objdump file for a commit number and ELF file
641
642 Args:
643 commit_upto: Commit number to use (0..self.count-1)
644 target: Target name
645 elf_fname: Filename of elf image
646 """
Simon Glass37edf5f2023-07-19 17:49:06 -0600647 return os.path.join(self.get_build_dir(commit_upto, target),
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000648 '%s.objdump' % elf_fname.replace('/', '-'))
649
Simon Glass37edf5f2023-07-19 17:49:06 -0600650 def get_err_file(self, commit_upto, target):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000651 """Get the name of the err file for a commit number
652
653 Args:
654 commit_upto: Commit number to use (0..self.count-1)
655 target: Target name
656 """
Simon Glass37edf5f2023-07-19 17:49:06 -0600657 output_dir = self.get_build_dir(commit_upto, target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000658 return os.path.join(output_dir, 'err')
659
Simon Glass37edf5f2023-07-19 17:49:06 -0600660 def filter_errors(self, lines):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000661 """Filter out errors in which we have no interest
662
663 We should probably use map().
664
665 Args:
666 lines: List of error lines, each a string
667 Returns:
668 New list with only interesting lines included
669 """
670 out_lines = []
Simon Glass113a8a52020-04-09 15:08:53 -0600671 if self._filter_migration_warnings:
672 text = '\n'.join(lines)
673 text = self._re_migration_warning.sub('', text)
674 lines = text.splitlines()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000675 for line in lines:
Simon Glass174592b2020-04-09 15:08:52 -0600676 if self.re_make_err.search(line):
677 continue
678 if self._filter_dtb_warnings and self._re_dtb_warning.search(line):
679 continue
680 out_lines.append(line)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000681 return out_lines
682
Simon Glass37edf5f2023-07-19 17:49:06 -0600683 def read_func_sizes(self, fname, fd):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000684 """Read function sizes from the output of 'nm'
685
686 Args:
687 fd: File containing data to read
688 fname: Filename we are reading from (just for errors)
689
690 Returns:
691 Dictionary containing size of each function in bytes, indexed by
692 function name.
693 """
694 sym = {}
695 for line in fd.readlines():
Simon Glassf2e67752022-07-11 19:04:11 -0600696 line = line.strip()
697 parts = line.split()
698 if line and len(parts) == 3:
699 size, type, name = line.split()
Simon Glassad8dbab2023-10-23 00:52:43 -0700700 if type in NM_SYMBOL_TYPES:
Simon Glassf2e67752022-07-11 19:04:11 -0600701 # function names begin with '.' on 64-bit powerpc
702 if '.' in name[1:]:
703 name = 'static.' + name.split('.')[0]
704 sym[name] = sym.get(name, 0) + int(size, 16)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000705 return sym
706
Simon Glass37edf5f2023-07-19 17:49:06 -0600707 def _process_config(self, fname):
Simon Glass843312d2015-02-05 22:06:15 -0700708 """Read in a .config, autoconf.mk or autoconf.h file
709
710 This function handles all config file types. It ignores comments and
711 any #defines which don't start with CONFIG_.
712
713 Args:
714 fname: Filename to read
715
716 Returns:
717 Dictionary:
718 key: Config name (e.g. CONFIG_DM)
719 value: Config value (e.g. 1)
720 """
721 config = {}
722 if os.path.exists(fname):
723 with open(fname) as fd:
724 for line in fd:
725 line = line.strip()
726 if line.startswith('#define'):
727 values = line[8:].split(' ', 1)
728 if len(values) > 1:
729 key, value = values
730 else:
731 key = values[0]
Simon Glassb464f8e2016-11-13 14:25:53 -0700732 value = '1' if self.squash_config_y else ''
Simon Glass843312d2015-02-05 22:06:15 -0700733 if not key.startswith('CONFIG_'):
734 continue
735 elif not line or line[0] in ['#', '*', '/']:
736 continue
737 else:
738 key, value = line.split('=', 1)
Simon Glassb464f8e2016-11-13 14:25:53 -0700739 if self.squash_config_y and value == 'y':
740 value = '1'
Simon Glass843312d2015-02-05 22:06:15 -0700741 config[key] = value
742 return config
743
Simon Glass37edf5f2023-07-19 17:49:06 -0600744 def _process_environment(self, fname):
Alex Kiernan48ae4122018-05-31 04:48:34 +0000745 """Read in a uboot.env file
746
747 This function reads in environment variables from a file.
748
749 Args:
750 fname: Filename to read
751
752 Returns:
753 Dictionary:
754 key: environment variable (e.g. bootlimit)
755 value: value of environment variable (e.g. 1)
756 """
757 environment = {}
758 if os.path.exists(fname):
759 with open(fname) as fd:
760 for line in fd.read().split('\0'):
761 try:
762 key, value = line.split('=', 1)
763 environment[key] = value
764 except ValueError:
765 # ignore lines we can't parse
766 pass
767 return environment
768
Simon Glass37edf5f2023-07-19 17:49:06 -0600769 def get_build_outcome(self, commit_upto, target, read_func_sizes,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000770 read_config, read_environment):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000771 """Work out the outcome of a build.
772
773 Args:
774 commit_upto: Commit number to check (0..n-1)
775 target: Target board to check
776 read_func_sizes: True to read function size information
Simon Glass843312d2015-02-05 22:06:15 -0700777 read_config: True to read .config and autoconf.h files
Alex Kiernan48ae4122018-05-31 04:48:34 +0000778 read_environment: True to read uboot.env files
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000779
780 Returns:
781 Outcome object
782 """
Simon Glass37edf5f2023-07-19 17:49:06 -0600783 done_file = self.get_done_file(commit_upto, target)
784 sizes_file = self.get_sizes_file(commit_upto, target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000785 sizes = {}
786 func_sizes = {}
Simon Glass843312d2015-02-05 22:06:15 -0700787 config = {}
Alex Kiernan48ae4122018-05-31 04:48:34 +0000788 environment = {}
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000789 if os.path.exists(done_file):
790 with open(done_file, 'r') as fd:
Simon Glass347ea0b2019-04-26 19:02:23 -0600791 try:
792 return_code = int(fd.readline())
793 except ValueError:
794 # The file may be empty due to running out of disk space.
795 # Try a rebuild
796 return_code = 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000797 err_lines = []
Simon Glass37edf5f2023-07-19 17:49:06 -0600798 err_file = self.get_err_file(commit_upto, target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000799 if os.path.exists(err_file):
800 with open(err_file, 'r') as fd:
Simon Glass37edf5f2023-07-19 17:49:06 -0600801 err_lines = self.filter_errors(fd.readlines())
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000802
803 # Decide whether the build was ok, failed or created warnings
804 if return_code:
805 rc = OUTCOME_ERROR
806 elif len(err_lines):
807 rc = OUTCOME_WARNING
808 else:
809 rc = OUTCOME_OK
810
811 # Convert size information to our simple format
812 if os.path.exists(sizes_file):
813 with open(sizes_file, 'r') as fd:
814 for line in fd.readlines():
815 values = line.split()
816 rodata = 0
817 if len(values) > 6:
818 rodata = int(values[6], 16)
819 size_dict = {
820 'all' : int(values[0]) + int(values[1]) +
821 int(values[2]),
822 'text' : int(values[0]) - rodata,
823 'data' : int(values[1]),
824 'bss' : int(values[2]),
825 'rodata' : rodata,
826 }
827 sizes[values[5]] = size_dict
828
829 if read_func_sizes:
Simon Glass37edf5f2023-07-19 17:49:06 -0600830 pattern = self.get_func_sizes_file(commit_upto, target, '*')
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000831 for fname in glob.glob(pattern):
832 with open(fname, 'r') as fd:
833 dict_name = os.path.basename(fname).replace('.sizes',
834 '')
Simon Glass37edf5f2023-07-19 17:49:06 -0600835 func_sizes[dict_name] = self.read_func_sizes(fname, fd)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000836
Simon Glass843312d2015-02-05 22:06:15 -0700837 if read_config:
Simon Glass37edf5f2023-07-19 17:49:06 -0600838 output_dir = self.get_build_dir(commit_upto, target)
Simon Glassb464f8e2016-11-13 14:25:53 -0700839 for name in self.config_filenames:
Simon Glass843312d2015-02-05 22:06:15 -0700840 fname = os.path.join(output_dir, name)
Simon Glass37edf5f2023-07-19 17:49:06 -0600841 config[name] = self._process_config(fname)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000842
Alex Kiernan48ae4122018-05-31 04:48:34 +0000843 if read_environment:
Simon Glass37edf5f2023-07-19 17:49:06 -0600844 output_dir = self.get_build_dir(commit_upto, target)
Alex Kiernan48ae4122018-05-31 04:48:34 +0000845 fname = os.path.join(output_dir, 'uboot.env')
Simon Glass37edf5f2023-07-19 17:49:06 -0600846 environment = self._process_environment(fname)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000847
Alex Kiernan48ae4122018-05-31 04:48:34 +0000848 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config,
849 environment)
850
851 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {})
Simon Glass843312d2015-02-05 22:06:15 -0700852
Simon Glass37edf5f2023-07-19 17:49:06 -0600853 def get_result_summary(self, boards_selected, commit_upto, read_func_sizes,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000854 read_config, read_environment):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000855 """Calculate a summary of the results of building a commit.
856
857 Args:
858 board_selected: Dict containing boards to summarise
859 commit_upto: Commit number to summarize (0..self.count-1)
860 read_func_sizes: True to read function size information
Simon Glass843312d2015-02-05 22:06:15 -0700861 read_config: True to read .config and autoconf.h files
Alex Kiernan48ae4122018-05-31 04:48:34 +0000862 read_environment: True to read uboot.env files
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000863
864 Returns:
865 Tuple:
Simon Glassae1a09f2022-07-11 19:03:56 -0600866 Dict containing boards which built this commit:
867 key: board.target
868 value: Builder.Outcome object
Simon Glasse30965d2014-08-28 09:43:44 -0600869 List containing a summary of error lines
Simon Glassed966652014-08-28 09:43:43 -0600870 Dict keyed by error line, containing a list of the Board
871 objects with that error
Simon Glasse30965d2014-08-28 09:43:44 -0600872 List containing a summary of warning lines
873 Dict keyed by error line, containing a list of the Board
874 objects with that warning
Simon Glass8270e3c2015-08-25 21:52:14 -0600875 Dictionary keyed by board.target. Each value is a dictionary:
876 key: filename - e.g. '.config'
Simon Glass843312d2015-02-05 22:06:15 -0700877 value is itself a dictionary:
878 key: config name
879 value: config value
Alex Kiernan48ae4122018-05-31 04:48:34 +0000880 Dictionary keyed by board.target. Each value is a dictionary:
881 key: environment variable
882 value: value of environment variable
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000883 """
Simon Glass37edf5f2023-07-19 17:49:06 -0600884 def add_line(lines_summary, lines_boards, line, board):
Simon Glasse30965d2014-08-28 09:43:44 -0600885 line = line.rstrip()
886 if line in lines_boards:
887 lines_boards[line].append(board)
888 else:
889 lines_boards[line] = [board]
890 lines_summary.append(line)
891
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000892 board_dict = {}
893 err_lines_summary = []
Simon Glassed966652014-08-28 09:43:43 -0600894 err_lines_boards = {}
Simon Glasse30965d2014-08-28 09:43:44 -0600895 warn_lines_summary = []
896 warn_lines_boards = {}
Simon Glass843312d2015-02-05 22:06:15 -0700897 config = {}
Alex Kiernan48ae4122018-05-31 04:48:34 +0000898 environment = {}
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000899
Simon Glassf4ed4702022-07-11 19:03:57 -0600900 for brd in boards_selected.values():
Simon Glass37edf5f2023-07-19 17:49:06 -0600901 outcome = self.get_build_outcome(commit_upto, brd.target,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000902 read_func_sizes, read_config,
903 read_environment)
Simon Glassf4ed4702022-07-11 19:03:57 -0600904 board_dict[brd.target] = outcome
Simon Glasse30965d2014-08-28 09:43:44 -0600905 last_func = None
906 last_was_warning = False
907 for line in outcome.err_lines:
908 if line:
909 if (self._re_function.match(line) or
910 self._re_files.match(line)):
911 last_func = line
Simon Glassed966652014-08-28 09:43:43 -0600912 else:
Simon Glass2d483332018-11-06 16:02:11 -0700913 is_warning = (self._re_warning.match(line) or
914 self._re_dtb_warning.match(line))
Simon Glasse30965d2014-08-28 09:43:44 -0600915 is_note = self._re_note.match(line)
916 if is_warning or (last_was_warning and is_note):
917 if last_func:
Simon Glass37edf5f2023-07-19 17:49:06 -0600918 add_line(warn_lines_summary, warn_lines_boards,
Simon Glassf4ed4702022-07-11 19:03:57 -0600919 last_func, brd)
Simon Glass37edf5f2023-07-19 17:49:06 -0600920 add_line(warn_lines_summary, warn_lines_boards,
Simon Glassf4ed4702022-07-11 19:03:57 -0600921 line, brd)
Simon Glasse30965d2014-08-28 09:43:44 -0600922 else:
923 if last_func:
Simon Glass37edf5f2023-07-19 17:49:06 -0600924 add_line(err_lines_summary, err_lines_boards,
Simon Glassf4ed4702022-07-11 19:03:57 -0600925 last_func, brd)
Simon Glass37edf5f2023-07-19 17:49:06 -0600926 add_line(err_lines_summary, err_lines_boards,
Simon Glassf4ed4702022-07-11 19:03:57 -0600927 line, brd)
Simon Glasse30965d2014-08-28 09:43:44 -0600928 last_was_warning = is_warning
929 last_func = None
Simon Glassf4ed4702022-07-11 19:03:57 -0600930 tconfig = Config(self.config_filenames, brd.target)
Simon Glassb464f8e2016-11-13 14:25:53 -0700931 for fname in self.config_filenames:
Simon Glass843312d2015-02-05 22:06:15 -0700932 if outcome.config:
Simon Glassc05aa032019-10-31 07:42:53 -0600933 for key, value in outcome.config[fname].items():
Simon Glass37edf5f2023-07-19 17:49:06 -0600934 tconfig.add(fname, key, value)
Simon Glassf4ed4702022-07-11 19:03:57 -0600935 config[brd.target] = tconfig
Simon Glass843312d2015-02-05 22:06:15 -0700936
Simon Glassf4ed4702022-07-11 19:03:57 -0600937 tenvironment = Environment(brd.target)
Alex Kiernan48ae4122018-05-31 04:48:34 +0000938 if outcome.environment:
Simon Glassc05aa032019-10-31 07:42:53 -0600939 for key, value in outcome.environment.items():
Simon Glass37edf5f2023-07-19 17:49:06 -0600940 tenvironment.add(key, value)
Simon Glassf4ed4702022-07-11 19:03:57 -0600941 environment[brd.target] = tenvironment
Alex Kiernan48ae4122018-05-31 04:48:34 +0000942
Simon Glasse30965d2014-08-28 09:43:44 -0600943 return (board_dict, err_lines_summary, err_lines_boards,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000944 warn_lines_summary, warn_lines_boards, config, environment)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000945
Simon Glass37edf5f2023-07-19 17:49:06 -0600946 def add_outcome(self, board_dict, arch_list, changes, char, color):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000947 """Add an output to our list of outcomes for each architecture
948
949 This simple function adds failing boards (changes) to the
950 relevant architecture string, so we can print the results out
951 sorted by architecture.
952
953 Args:
954 board_dict: Dict containing all boards
955 arch_list: Dict keyed by arch name. Value is a string containing
956 a list of board names which failed for that arch.
957 changes: List of boards to add to arch_list
958 color: terminal.Colour object
959 """
960 done_arch = {}
961 for target in changes:
962 if target in board_dict:
963 arch = board_dict[target].arch
964 else:
965 arch = 'unknown'
Simon Glass252ac582022-01-29 14:14:17 -0700966 str = self.col.build(color, ' ' + target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000967 if not arch in done_arch:
Simon Glass252ac582022-01-29 14:14:17 -0700968 str = ' %s %s' % (self.col.build(color, char), str)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000969 done_arch[arch] = True
970 if not arch in arch_list:
971 arch_list[arch] = str
972 else:
973 arch_list[arch] += str
974
975
Simon Glass37edf5f2023-07-19 17:49:06 -0600976 def colour_num(self, num):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000977 color = self.col.RED if num > 0 else self.col.GREEN
978 if num == 0:
979 return '0'
Simon Glass252ac582022-01-29 14:14:17 -0700980 return self.col.build(color, str(num))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000981
Simon Glass37edf5f2023-07-19 17:49:06 -0600982 def reset_result_summary(self, board_selected):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000983 """Reset the results summary ready for use.
984
985 Set up the base board list to be all those selected, and set the
986 error lines to empty.
987
Simon Glass37edf5f2023-07-19 17:49:06 -0600988 Following this, calls to print_result_summary() will use this
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000989 information to work out what has changed.
990
991 Args:
992 board_selected: Dict containing boards to summarise, keyed by
993 board.target
994 """
995 self._base_board_dict = {}
Simon Glassf4ed4702022-07-11 19:03:57 -0600996 for brd in board_selected:
997 self._base_board_dict[brd] = Builder.Outcome(0, [], [], {}, {}, {})
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000998 self._base_err_lines = []
Simon Glasse30965d2014-08-28 09:43:44 -0600999 self._base_warn_lines = []
1000 self._base_err_line_boards = {}
1001 self._base_warn_line_boards = {}
Simon Glass8270e3c2015-08-25 21:52:14 -06001002 self._base_config = None
Alex Kiernan48ae4122018-05-31 04:48:34 +00001003 self._base_environment = None
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001004
Simon Glass37edf5f2023-07-19 17:49:06 -06001005 def print_func_size_detail(self, fname, old, new):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001006 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
1007 delta, common = [], {}
1008
1009 for a in old:
1010 if a in new:
1011 common[a] = 1
1012
1013 for name in old:
1014 if name not in common:
1015 remove += 1
1016 down += old[name]
1017 delta.append([-old[name], name])
1018
1019 for name in new:
1020 if name not in common:
1021 add += 1
1022 up += new[name]
1023 delta.append([new[name], name])
1024
1025 for name in common:
1026 diff = new.get(name, 0) - old.get(name, 0)
1027 if diff > 0:
1028 grow, up = grow + 1, up + diff
1029 elif diff < 0:
1030 shrink, down = shrink + 1, down - diff
1031 delta.append([diff, name])
1032
1033 delta.sort()
1034 delta.reverse()
1035
1036 args = [add, -remove, grow, -shrink, up, -down, up - down]
Tom Rinid5686a62017-05-22 13:48:52 -04001037 if max(args) == 0 and min(args) == 0:
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001038 return
Simon Glass37edf5f2023-07-19 17:49:06 -06001039 args = [self.colour_num(x) for x in args]
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001040 indent = ' ' * 15
Simon Glass098b10f2022-01-29 14:14:18 -07001041 tprint('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
Simon Glass252ac582022-01-29 14:14:17 -07001042 tuple([indent, self.col.build(self.col.YELLOW, fname)] + args))
Simon Glass098b10f2022-01-29 14:14:18 -07001043 tprint('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
Simon Glass4653a882014-09-05 19:00:07 -06001044 'delta'))
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001045 for diff, name in delta:
1046 if diff:
1047 color = self.col.RED if diff > 0 else self.col.GREEN
1048 msg = '%s %-38s %7s %7s %+7d' % (indent, name,
1049 old.get(name, '-'), new.get(name,'-'), diff)
Simon Glass098b10f2022-01-29 14:14:18 -07001050 tprint(msg, colour=color)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001051
1052
Simon Glass37edf5f2023-07-19 17:49:06 -06001053 def print_size_detail(self, target_list, show_bloat):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001054 """Show details size information for each board
1055
1056 Args:
1057 target_list: List of targets, each a dict containing:
1058 'target': Target name
1059 'total_diff': Total difference in bytes across all areas
1060 <part_name>: Difference for that part
1061 show_bloat: Show detail for each function
1062 """
1063 targets_by_diff = sorted(target_list, reverse=True,
1064 key=lambda x: x['_total_diff'])
1065 for result in targets_by_diff:
1066 printed_target = False
1067 for name in sorted(result):
1068 diff = result[name]
1069 if name.startswith('_'):
1070 continue
1071 if diff != 0:
1072 color = self.col.RED if diff > 0 else self.col.GREEN
1073 msg = ' %s %+d' % (name, diff)
1074 if not printed_target:
Simon Glass098b10f2022-01-29 14:14:18 -07001075 tprint('%10s %-15s:' % ('', result['_target']),
Simon Glass4653a882014-09-05 19:00:07 -06001076 newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001077 printed_target = True
Simon Glass098b10f2022-01-29 14:14:18 -07001078 tprint(msg, colour=color, newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001079 if printed_target:
Simon Glass098b10f2022-01-29 14:14:18 -07001080 tprint()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001081 if show_bloat:
1082 target = result['_target']
1083 outcome = result['_outcome']
1084 base_outcome = self._base_board_dict[target]
1085 for fname in outcome.func_sizes:
Simon Glass37edf5f2023-07-19 17:49:06 -06001086 self.print_func_size_detail(fname,
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001087 base_outcome.func_sizes[fname],
1088 outcome.func_sizes[fname])
1089
1090
Simon Glass37edf5f2023-07-19 17:49:06 -06001091 def print_size_summary(self, board_selected, board_dict, show_detail,
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001092 show_bloat):
1093 """Print a summary of image sizes broken down by section.
1094
1095 The summary takes the form of one line per architecture. The
1096 line contains deltas for each of the sections (+ means the section
Flavio Suligoi9de5c392020-01-29 09:56:05 +01001097 got bigger, - means smaller). The numbers are the average number
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001098 of bytes that a board in this section increased by.
1099
1100 For example:
1101 powerpc: (622 boards) text -0.0
1102 arm: (285 boards) text -0.0
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001103
1104 Args:
1105 board_selected: Dict containing boards to summarise, keyed by
1106 board.target
1107 board_dict: Dict containing boards for which we built this
1108 commit, keyed by board.target. The value is an Outcome object.
Simon Glassf9c094b2020-03-18 09:42:43 -06001109 show_detail: Show size delta detail for each board
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001110 show_bloat: Show detail for each function
1111 """
1112 arch_list = {}
1113 arch_count = {}
1114
1115 # Calculate changes in size for different image parts
1116 # The previous sizes are in Board.sizes, for each board
1117 for target in board_dict:
1118 if target not in board_selected:
1119 continue
1120 base_sizes = self._base_board_dict[target].sizes
1121 outcome = board_dict[target]
1122 sizes = outcome.sizes
1123
1124 # Loop through the list of images, creating a dict of size
1125 # changes for each image/part. We end up with something like
1126 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1127 # which means that U-Boot data increased by 5 bytes and SPL
1128 # text decreased by 4.
1129 err = {'_target' : target}
1130 for image in sizes:
1131 if image in base_sizes:
1132 base_image = base_sizes[image]
1133 # Loop through the text, data, bss parts
1134 for part in sorted(sizes[image]):
1135 diff = sizes[image][part] - base_image[part]
1136 col = None
1137 if diff:
1138 if image == 'u-boot':
1139 name = part
1140 else:
1141 name = image + ':' + part
1142 err[name] = diff
1143 arch = board_selected[target].arch
1144 if not arch in arch_count:
1145 arch_count[arch] = 1
1146 else:
1147 arch_count[arch] += 1
1148 if not sizes:
1149 pass # Only add to our list when we have some stats
1150 elif not arch in arch_list:
1151 arch_list[arch] = [err]
1152 else:
1153 arch_list[arch].append(err)
1154
1155 # We now have a list of image size changes sorted by arch
1156 # Print out a summary of these
Simon Glassc05aa032019-10-31 07:42:53 -06001157 for arch, target_list in arch_list.items():
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001158 # Get total difference for each type
1159 totals = {}
1160 for result in target_list:
1161 total = 0
Simon Glassc05aa032019-10-31 07:42:53 -06001162 for name, diff in result.items():
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001163 if name.startswith('_'):
1164 continue
1165 total += diff
1166 if name in totals:
1167 totals[name] += diff
1168 else:
1169 totals[name] = diff
1170 result['_total_diff'] = total
1171 result['_outcome'] = board_dict[result['_target']]
1172
1173 count = len(target_list)
1174 printed_arch = False
1175 for name in sorted(totals):
1176 diff = totals[name]
1177 if diff:
1178 # Display the average difference in this name for this
1179 # architecture
1180 avg_diff = float(diff) / count
1181 color = self.col.RED if avg_diff > 0 else self.col.GREEN
1182 msg = ' %s %+1.1f' % (name, avg_diff)
1183 if not printed_arch:
Simon Glass098b10f2022-01-29 14:14:18 -07001184 tprint('%10s: (for %d/%d boards)' % (arch, count,
Simon Glass4653a882014-09-05 19:00:07 -06001185 arch_count[arch]), newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001186 printed_arch = True
Simon Glass098b10f2022-01-29 14:14:18 -07001187 tprint(msg, colour=color, newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001188
1189 if printed_arch:
Simon Glass098b10f2022-01-29 14:14:18 -07001190 tprint()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001191 if show_detail:
Simon Glass37edf5f2023-07-19 17:49:06 -06001192 self.print_size_detail(target_list, show_bloat)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001193
1194
Simon Glass37edf5f2023-07-19 17:49:06 -06001195 def print_result_summary(self, board_selected, board_dict, err_lines,
Simon Glasse30965d2014-08-28 09:43:44 -06001196 err_line_boards, warn_lines, warn_line_boards,
Alex Kiernan48ae4122018-05-31 04:48:34 +00001197 config, environment, show_sizes, show_detail,
1198 show_bloat, show_config, show_environment):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001199 """Compare results with the base results and display delta.
1200
1201 Only boards mentioned in board_selected will be considered. This
1202 function is intended to be called repeatedly with the results of
1203 each commit. It therefore shows a 'diff' between what it saw in
1204 the last call and what it sees now.
1205
1206 Args:
1207 board_selected: Dict containing boards to summarise, keyed by
1208 board.target
1209 board_dict: Dict containing boards for which we built this
1210 commit, keyed by board.target. The value is an Outcome object.
1211 err_lines: A list of errors for this commit, or [] if there is
1212 none, or we don't want to print errors
Simon Glassed966652014-08-28 09:43:43 -06001213 err_line_boards: Dict keyed by error line, containing a list of
1214 the Board objects with that error
Simon Glasse30965d2014-08-28 09:43:44 -06001215 warn_lines: A list of warnings for this commit, or [] if there is
1216 none, or we don't want to print errors
1217 warn_line_boards: Dict keyed by warning line, containing a list of
1218 the Board objects with that warning
Simon Glass843312d2015-02-05 22:06:15 -07001219 config: Dictionary keyed by filename - e.g. '.config'. Each
1220 value is itself a dictionary:
1221 key: config name
1222 value: config value
Alex Kiernan48ae4122018-05-31 04:48:34 +00001223 environment: Dictionary keyed by environment variable, Each
1224 value is the value of environment variable.
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001225 show_sizes: Show image size deltas
Simon Glassf9c094b2020-03-18 09:42:43 -06001226 show_detail: Show size delta detail for each board if show_sizes
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001227 show_bloat: Show detail for each function
Simon Glass843312d2015-02-05 22:06:15 -07001228 show_config: Show config changes
Alex Kiernan48ae4122018-05-31 04:48:34 +00001229 show_environment: Show environment changes
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001230 """
Simon Glass37edf5f2023-07-19 17:49:06 -06001231 def _board_list(line, line_boards):
Simon Glassed966652014-08-28 09:43:43 -06001232 """Helper function to get a line of boards containing a line
1233
1234 Args:
1235 line: Error line to search for
Simon Glass35d696d2020-04-09 15:08:36 -06001236 line_boards: boards to search, each a Board
Simon Glassed966652014-08-28 09:43:43 -06001237 Return:
Simon Glass35d696d2020-04-09 15:08:36 -06001238 List of boards with that error line, or [] if the user has not
1239 requested such a list
Simon Glassed966652014-08-28 09:43:43 -06001240 """
Simon Glasscc2c0d12022-07-11 19:04:00 -06001241 brds = []
Simon Glass35d696d2020-04-09 15:08:36 -06001242 board_set = set()
Simon Glassed966652014-08-28 09:43:43 -06001243 if self._list_error_boards:
Simon Glassf4ed4702022-07-11 19:03:57 -06001244 for brd in line_boards[line]:
1245 if not brd in board_set:
Simon Glasscc2c0d12022-07-11 19:04:00 -06001246 brds.append(brd)
Simon Glassf4ed4702022-07-11 19:03:57 -06001247 board_set.add(brd)
Simon Glasscc2c0d12022-07-11 19:04:00 -06001248 return brds
Simon Glassed966652014-08-28 09:43:43 -06001249
Simon Glass37edf5f2023-07-19 17:49:06 -06001250 def _calc_error_delta(base_lines, base_line_boards, lines, line_boards,
Simon Glasse30965d2014-08-28 09:43:44 -06001251 char):
Simon Glass35d696d2020-04-09 15:08:36 -06001252 """Calculate the required output based on changes in errors
1253
1254 Args:
1255 base_lines: List of errors/warnings for previous commit
1256 base_line_boards: Dict keyed by error line, containing a list
1257 of the Board objects with that error in the previous commit
1258 lines: List of errors/warning for this commit, each a str
1259 line_boards: Dict keyed by error line, containing a list
1260 of the Board objects with that error in this commit
1261 char: Character representing error ('') or warning ('w'). The
1262 broken ('+') or fixed ('-') characters are added in this
1263 function
1264
1265 Returns:
1266 Tuple
1267 List of ErrLine objects for 'better' lines
1268 List of ErrLine objects for 'worse' lines
1269 """
Simon Glasse30965d2014-08-28 09:43:44 -06001270 better_lines = []
1271 worse_lines = []
1272 for line in lines:
1273 if line not in base_lines:
Simon Glass37edf5f2023-07-19 17:49:06 -06001274 errline = ErrLine(char + '+', _board_list(line, line_boards),
Simon Glass35d696d2020-04-09 15:08:36 -06001275 line)
1276 worse_lines.append(errline)
Simon Glasse30965d2014-08-28 09:43:44 -06001277 for line in base_lines:
1278 if line not in lines:
Simon Glass35d696d2020-04-09 15:08:36 -06001279 errline = ErrLine(char + '-',
Simon Glass37edf5f2023-07-19 17:49:06 -06001280 _board_list(line, base_line_boards), line)
Simon Glass35d696d2020-04-09 15:08:36 -06001281 better_lines.append(errline)
Simon Glasse30965d2014-08-28 09:43:44 -06001282 return better_lines, worse_lines
1283
Simon Glass37edf5f2023-07-19 17:49:06 -06001284 def _calc_config(delta, name, config):
Simon Glass843312d2015-02-05 22:06:15 -07001285 """Calculate configuration changes
1286
1287 Args:
1288 delta: Type of the delta, e.g. '+'
1289 name: name of the file which changed (e.g. .config)
1290 config: configuration change dictionary
1291 key: config name
1292 value: config value
1293 Returns:
1294 String containing the configuration changes which can be
1295 printed
1296 """
1297 out = ''
1298 for key in sorted(config.keys()):
1299 out += '%s=%s ' % (key, config[key])
Simon Glass8270e3c2015-08-25 21:52:14 -06001300 return '%s %s: %s' % (delta, name, out)
Simon Glass843312d2015-02-05 22:06:15 -07001301
Simon Glass37edf5f2023-07-19 17:49:06 -06001302 def _add_config(lines, name, config_plus, config_minus, config_change):
Simon Glass8270e3c2015-08-25 21:52:14 -06001303 """Add changes in configuration to a list
Simon Glass843312d2015-02-05 22:06:15 -07001304
1305 Args:
Simon Glass8270e3c2015-08-25 21:52:14 -06001306 lines: list to add to
1307 name: config file name
Simon Glass843312d2015-02-05 22:06:15 -07001308 config_plus: configurations added, dictionary
1309 key: config name
1310 value: config value
1311 config_minus: configurations removed, dictionary
1312 key: config name
1313 value: config value
1314 config_change: configurations changed, dictionary
1315 key: config name
1316 value: config value
1317 """
1318 if config_plus:
Simon Glass37edf5f2023-07-19 17:49:06 -06001319 lines.append(_calc_config('+', name, config_plus))
Simon Glass843312d2015-02-05 22:06:15 -07001320 if config_minus:
Simon Glass37edf5f2023-07-19 17:49:06 -06001321 lines.append(_calc_config('-', name, config_minus))
Simon Glass843312d2015-02-05 22:06:15 -07001322 if config_change:
Simon Glass37edf5f2023-07-19 17:49:06 -06001323 lines.append(_calc_config('c', name, config_change))
Simon Glass8270e3c2015-08-25 21:52:14 -06001324
Simon Glass37edf5f2023-07-19 17:49:06 -06001325 def _output_config_info(lines):
Simon Glass8270e3c2015-08-25 21:52:14 -06001326 for line in lines:
1327 if not line:
1328 continue
1329 if line[0] == '+':
1330 col = self.col.GREEN
1331 elif line[0] == '-':
1332 col = self.col.RED
1333 elif line[0] == 'c':
1334 col = self.col.YELLOW
Simon Glass098b10f2022-01-29 14:14:18 -07001335 tprint(' ' + line, newline=True, colour=col)
Simon Glass8270e3c2015-08-25 21:52:14 -06001336
Simon Glass37edf5f2023-07-19 17:49:06 -06001337 def _output_err_lines(err_lines, colour):
Simon Glassb206d872020-04-09 15:08:28 -06001338 """Output the line of error/warning lines, if not empty
1339
1340 Also increments self._error_lines if err_lines not empty
1341
1342 Args:
Simon Glass35d696d2020-04-09 15:08:36 -06001343 err_lines: List of ErrLine objects, each an error or warning
1344 line, possibly including a list of boards with that
1345 error/warning
Simon Glassb206d872020-04-09 15:08:28 -06001346 colour: Colour to use for output
1347 """
1348 if err_lines:
Simon Glass8c9a2672020-04-09 15:08:37 -06001349 out_list = []
Simon Glass35d696d2020-04-09 15:08:36 -06001350 for line in err_lines:
Simon Glasscc2c0d12022-07-11 19:04:00 -06001351 names = [brd.target for brd in line.brds]
Simon Glass9ef0ceb2020-04-09 15:08:38 -06001352 board_str = ' '.join(names) if names else ''
Simon Glass8c9a2672020-04-09 15:08:37 -06001353 if board_str:
Simon Glass252ac582022-01-29 14:14:17 -07001354 out = self.col.build(colour, line.char + '(')
1355 out += self.col.build(self.col.MAGENTA, board_str,
Simon Glass8c9a2672020-04-09 15:08:37 -06001356 bright=False)
Simon Glass252ac582022-01-29 14:14:17 -07001357 out += self.col.build(colour, ') %s' % line.errline)
Simon Glass8c9a2672020-04-09 15:08:37 -06001358 else:
Simon Glass252ac582022-01-29 14:14:17 -07001359 out = self.col.build(colour, line.char + line.errline)
Simon Glass8c9a2672020-04-09 15:08:37 -06001360 out_list.append(out)
Simon Glass098b10f2022-01-29 14:14:18 -07001361 tprint('\n'.join(out_list))
Simon Glassb206d872020-04-09 15:08:28 -06001362 self._error_lines += 1
1363
Simon Glass843312d2015-02-05 22:06:15 -07001364
Simon Glass4cf2b222018-11-06 16:02:12 -07001365 ok_boards = [] # List of boards fixed since last commit
Simon Glass6af71012018-11-06 16:02:13 -07001366 warn_boards = [] # List of boards with warnings since last commit
Simon Glass4cf2b222018-11-06 16:02:12 -07001367 err_boards = [] # List of new broken boards since last commit
1368 new_boards = [] # List of boards that didn't exist last time
1369 unknown_boards = [] # List of boards that were not built
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001370
1371 for target in board_dict:
1372 if target not in board_selected:
1373 continue
1374
1375 # If the board was built last time, add its outcome to a list
1376 if target in self._base_board_dict:
1377 base_outcome = self._base_board_dict[target].rc
1378 outcome = board_dict[target]
1379 if outcome.rc == OUTCOME_UNKNOWN:
Simon Glass4cf2b222018-11-06 16:02:12 -07001380 unknown_boards.append(target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001381 elif outcome.rc < base_outcome:
Simon Glass6af71012018-11-06 16:02:13 -07001382 if outcome.rc == OUTCOME_WARNING:
1383 warn_boards.append(target)
1384 else:
1385 ok_boards.append(target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001386 elif outcome.rc > base_outcome:
Simon Glass6af71012018-11-06 16:02:13 -07001387 if outcome.rc == OUTCOME_WARNING:
1388 warn_boards.append(target)
1389 else:
1390 err_boards.append(target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001391 else:
Simon Glass4cf2b222018-11-06 16:02:12 -07001392 new_boards.append(target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001393
Simon Glassb206d872020-04-09 15:08:28 -06001394 # Get a list of errors and warnings that have appeared, and disappeared
Simon Glass37edf5f2023-07-19 17:49:06 -06001395 better_err, worse_err = _calc_error_delta(self._base_err_lines,
Simon Glasse30965d2014-08-28 09:43:44 -06001396 self._base_err_line_boards, err_lines, err_line_boards, '')
Simon Glass37edf5f2023-07-19 17:49:06 -06001397 better_warn, worse_warn = _calc_error_delta(self._base_warn_lines,
Simon Glasse30965d2014-08-28 09:43:44 -06001398 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001399
Simon Glassae1a09f2022-07-11 19:03:56 -06001400 # For the IDE mode, print out all the output
1401 if self._ide:
1402 outcome = board_dict[target]
1403 for line in outcome.err_lines:
1404 sys.stderr.write(line)
1405
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001406 # Display results by arch
Simon Glassae1a09f2022-07-11 19:03:56 -06001407 elif any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards,
Simon Glass6af71012018-11-06 16:02:13 -07001408 worse_err, better_err, worse_warn, better_warn)):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001409 arch_list = {}
Simon Glass37edf5f2023-07-19 17:49:06 -06001410 self.add_outcome(board_selected, arch_list, ok_boards, '',
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001411 self.col.GREEN)
Simon Glass37edf5f2023-07-19 17:49:06 -06001412 self.add_outcome(board_selected, arch_list, warn_boards, 'w+',
Simon Glass6af71012018-11-06 16:02:13 -07001413 self.col.YELLOW)
Simon Glass37edf5f2023-07-19 17:49:06 -06001414 self.add_outcome(board_selected, arch_list, err_boards, '+',
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001415 self.col.RED)
Simon Glass37edf5f2023-07-19 17:49:06 -06001416 self.add_outcome(board_selected, arch_list, new_boards, '*', self.col.BLUE)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001417 if self._show_unknown:
Simon Glass37edf5f2023-07-19 17:49:06 -06001418 self.add_outcome(board_selected, arch_list, unknown_boards, '?',
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001419 self.col.MAGENTA)
Simon Glassc05aa032019-10-31 07:42:53 -06001420 for arch, target_list in arch_list.items():
Simon Glass098b10f2022-01-29 14:14:18 -07001421 tprint('%10s: %s' % (arch, target_list))
Simon Glass28370c12014-08-09 15:33:06 -06001422 self._error_lines += 1
Simon Glass37edf5f2023-07-19 17:49:06 -06001423 _output_err_lines(better_err, colour=self.col.GREEN)
1424 _output_err_lines(worse_err, colour=self.col.RED)
1425 _output_err_lines(better_warn, colour=self.col.CYAN)
1426 _output_err_lines(worse_warn, colour=self.col.YELLOW)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001427
1428 if show_sizes:
Simon Glass37edf5f2023-07-19 17:49:06 -06001429 self.print_size_summary(board_selected, board_dict, show_detail,
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001430 show_bloat)
1431
Alex Kiernan48ae4122018-05-31 04:48:34 +00001432 if show_environment and self._base_environment:
1433 lines = []
1434
1435 for target in board_dict:
1436 if target not in board_selected:
1437 continue
1438
1439 tbase = self._base_environment[target]
1440 tenvironment = environment[target]
1441 environment_plus = {}
1442 environment_minus = {}
1443 environment_change = {}
1444 base = tbase.environment
Simon Glassc05aa032019-10-31 07:42:53 -06001445 for key, value in tenvironment.environment.items():
Alex Kiernan48ae4122018-05-31 04:48:34 +00001446 if key not in base:
1447 environment_plus[key] = value
Simon Glassc05aa032019-10-31 07:42:53 -06001448 for key, value in base.items():
Alex Kiernan48ae4122018-05-31 04:48:34 +00001449 if key not in tenvironment.environment:
1450 environment_minus[key] = value
Simon Glassc05aa032019-10-31 07:42:53 -06001451 for key, value in base.items():
Alex Kiernan48ae4122018-05-31 04:48:34 +00001452 new_value = tenvironment.environment.get(key)
1453 if new_value and value != new_value:
1454 desc = '%s -> %s' % (value, new_value)
1455 environment_change[key] = desc
1456
Simon Glass37edf5f2023-07-19 17:49:06 -06001457 _add_config(lines, target, environment_plus, environment_minus,
Alex Kiernan48ae4122018-05-31 04:48:34 +00001458 environment_change)
1459
Simon Glass37edf5f2023-07-19 17:49:06 -06001460 _output_config_info(lines)
Alex Kiernan48ae4122018-05-31 04:48:34 +00001461
Simon Glass8270e3c2015-08-25 21:52:14 -06001462 if show_config and self._base_config:
1463 summary = {}
1464 arch_config_plus = {}
1465 arch_config_minus = {}
1466 arch_config_change = {}
1467 arch_list = []
1468
1469 for target in board_dict:
1470 if target not in board_selected:
Simon Glass843312d2015-02-05 22:06:15 -07001471 continue
Simon Glass8270e3c2015-08-25 21:52:14 -06001472 arch = board_selected[target].arch
1473 if arch not in arch_list:
1474 arch_list.append(arch)
1475
1476 for arch in arch_list:
1477 arch_config_plus[arch] = {}
1478 arch_config_minus[arch] = {}
1479 arch_config_change[arch] = {}
Simon Glassb464f8e2016-11-13 14:25:53 -07001480 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001481 arch_config_plus[arch][name] = {}
1482 arch_config_minus[arch][name] = {}
1483 arch_config_change[arch][name] = {}
1484
1485 for target in board_dict:
1486 if target not in board_selected:
1487 continue
1488
1489 arch = board_selected[target].arch
1490
1491 all_config_plus = {}
1492 all_config_minus = {}
1493 all_config_change = {}
1494 tbase = self._base_config[target]
1495 tconfig = config[target]
1496 lines = []
Simon Glassb464f8e2016-11-13 14:25:53 -07001497 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001498 if not tconfig.config[name]:
1499 continue
1500 config_plus = {}
1501 config_minus = {}
1502 config_change = {}
1503 base = tbase.config[name]
Simon Glassc05aa032019-10-31 07:42:53 -06001504 for key, value in tconfig.config[name].items():
Simon Glass8270e3c2015-08-25 21:52:14 -06001505 if key not in base:
1506 config_plus[key] = value
1507 all_config_plus[key] = value
Simon Glassc05aa032019-10-31 07:42:53 -06001508 for key, value in base.items():
Simon Glass8270e3c2015-08-25 21:52:14 -06001509 if key not in tconfig.config[name]:
1510 config_minus[key] = value
1511 all_config_minus[key] = value
Simon Glassc05aa032019-10-31 07:42:53 -06001512 for key, value in base.items():
Simon Glass8270e3c2015-08-25 21:52:14 -06001513 new_value = tconfig.config.get(key)
1514 if new_value and value != new_value:
1515 desc = '%s -> %s' % (value, new_value)
1516 config_change[key] = desc
1517 all_config_change[key] = desc
1518
1519 arch_config_plus[arch][name].update(config_plus)
1520 arch_config_minus[arch][name].update(config_minus)
1521 arch_config_change[arch][name].update(config_change)
1522
Simon Glass37edf5f2023-07-19 17:49:06 -06001523 _add_config(lines, name, config_plus, config_minus,
Simon Glass8270e3c2015-08-25 21:52:14 -06001524 config_change)
Simon Glass37edf5f2023-07-19 17:49:06 -06001525 _add_config(lines, 'all', all_config_plus, all_config_minus,
Simon Glass8270e3c2015-08-25 21:52:14 -06001526 all_config_change)
1527 summary[target] = '\n'.join(lines)
1528
1529 lines_by_target = {}
Simon Glassc05aa032019-10-31 07:42:53 -06001530 for target, lines in summary.items():
Simon Glass8270e3c2015-08-25 21:52:14 -06001531 if lines in lines_by_target:
1532 lines_by_target[lines].append(target)
1533 else:
1534 lines_by_target[lines] = [target]
1535
1536 for arch in arch_list:
1537 lines = []
1538 all_plus = {}
1539 all_minus = {}
1540 all_change = {}
Simon Glassb464f8e2016-11-13 14:25:53 -07001541 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001542 all_plus.update(arch_config_plus[arch][name])
1543 all_minus.update(arch_config_minus[arch][name])
1544 all_change.update(arch_config_change[arch][name])
Simon Glass37edf5f2023-07-19 17:49:06 -06001545 _add_config(lines, name, arch_config_plus[arch][name],
Simon Glass8270e3c2015-08-25 21:52:14 -06001546 arch_config_minus[arch][name],
1547 arch_config_change[arch][name])
Simon Glass37edf5f2023-07-19 17:49:06 -06001548 _add_config(lines, 'all', all_plus, all_minus, all_change)
Simon Glass8270e3c2015-08-25 21:52:14 -06001549 #arch_summary[target] = '\n'.join(lines)
1550 if lines:
Simon Glass098b10f2022-01-29 14:14:18 -07001551 tprint('%s:' % arch)
Simon Glass37edf5f2023-07-19 17:49:06 -06001552 _output_config_info(lines)
Simon Glass8270e3c2015-08-25 21:52:14 -06001553
Simon Glassc05aa032019-10-31 07:42:53 -06001554 for lines, targets in lines_by_target.items():
Simon Glass8270e3c2015-08-25 21:52:14 -06001555 if not lines:
1556 continue
Simon Glass098b10f2022-01-29 14:14:18 -07001557 tprint('%s :' % ' '.join(sorted(targets)))
Simon Glass37edf5f2023-07-19 17:49:06 -06001558 _output_config_info(lines.split('\n'))
Simon Glass8270e3c2015-08-25 21:52:14 -06001559
Simon Glass843312d2015-02-05 22:06:15 -07001560
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001561 # Save our updated information for the next call to this function
1562 self._base_board_dict = board_dict
1563 self._base_err_lines = err_lines
Simon Glasse30965d2014-08-28 09:43:44 -06001564 self._base_warn_lines = warn_lines
1565 self._base_err_line_boards = err_line_boards
1566 self._base_warn_line_boards = warn_line_boards
Simon Glass843312d2015-02-05 22:06:15 -07001567 self._base_config = config
Alex Kiernan48ae4122018-05-31 04:48:34 +00001568 self._base_environment = environment
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001569
1570 # Get a list of boards that did not get built, if needed
1571 not_built = []
Simon Glassf4ed4702022-07-11 19:03:57 -06001572 for brd in board_selected:
1573 if not brd in board_dict:
1574 not_built.append(brd)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001575 if not_built:
Simon Glass098b10f2022-01-29 14:14:18 -07001576 tprint("Boards not built (%d): %s" % (len(not_built),
Simon Glass4653a882014-09-05 19:00:07 -06001577 ', '.join(not_built)))
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001578
Simon Glass37edf5f2023-07-19 17:49:06 -06001579 def produce_result_summary(self, commit_upto, commits, board_selected):
Simon Glasse30965d2014-08-28 09:43:44 -06001580 (board_dict, err_lines, err_line_boards, warn_lines,
Simon Glass37edf5f2023-07-19 17:49:06 -06001581 warn_line_boards, config, environment) = self.get_result_summary(
Simon Glassed966652014-08-28 09:43:43 -06001582 board_selected, commit_upto,
Simon Glass843312d2015-02-05 22:06:15 -07001583 read_func_sizes=self._show_bloat,
Alex Kiernan48ae4122018-05-31 04:48:34 +00001584 read_config=self._show_config,
1585 read_environment=self._show_environment)
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001586 if commits:
1587 msg = '%02d: %s' % (commit_upto + 1,
1588 commits[commit_upto].subject)
Simon Glass098b10f2022-01-29 14:14:18 -07001589 tprint(msg, colour=self.col.BLUE)
Simon Glass37edf5f2023-07-19 17:49:06 -06001590 self.print_result_summary(board_selected, board_dict,
Simon Glassed966652014-08-28 09:43:43 -06001591 err_lines if self._show_errors else [], err_line_boards,
Simon Glasse30965d2014-08-28 09:43:44 -06001592 warn_lines if self._show_errors else [], warn_line_boards,
Alex Kiernan48ae4122018-05-31 04:48:34 +00001593 config, environment, self._show_sizes, self._show_detail,
1594 self._show_bloat, self._show_config, self._show_environment)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001595
Simon Glass37edf5f2023-07-19 17:49:06 -06001596 def show_summary(self, commits, board_selected):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001597 """Show a build summary for U-Boot for a given board list.
1598
1599 Reset the result summary, then repeatedly call GetResultSummary on
1600 each commit's results, then display the differences we see.
1601
1602 Args:
1603 commit: Commit objects to summarise
1604 board_selected: Dict containing boards to summarise
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001605 """
Simon Glassfea58582014-08-09 15:32:59 -06001606 self.commit_count = len(commits) if commits else 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001607 self.commits = commits
Simon Glass37edf5f2023-07-19 17:49:06 -06001608 self.reset_result_summary(board_selected)
Simon Glass28370c12014-08-09 15:33:06 -06001609 self._error_lines = 0
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001610
1611 for commit_upto in range(0, self.commit_count, self._step):
Simon Glass37edf5f2023-07-19 17:49:06 -06001612 self.produce_result_summary(commit_upto, commits, board_selected)
Simon Glass28370c12014-08-09 15:33:06 -06001613 if not self._error_lines:
Simon Glass098b10f2022-01-29 14:14:18 -07001614 tprint('(no errors to report)', colour=self.col.GREEN)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001615
1616
Simon Glass37edf5f2023-07-19 17:49:06 -06001617 def setup_build(self, board_selected, commits):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001618 """Set up ready to start a build.
1619
1620 Args:
1621 board_selected: Selected boards to build
1622 commits: Selected commits to build
1623 """
1624 # First work out how many commits we will build
Simon Glassc05aa032019-10-31 07:42:53 -06001625 count = (self.commit_count + self._step - 1) // self._step
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001626 self.count = len(board_selected) * count
1627 self.upto = self.warned = self.fail = 0
1628 self._timestamps = collections.deque()
1629
Simon Glass37edf5f2023-07-19 17:49:06 -06001630 def get_thread_dir(self, thread_num):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001631 """Get the directory path to the working dir for a thread.
1632
1633 Args:
Simon Glassb82492b2021-01-30 22:17:46 -07001634 thread_num: Number of thread to check (-1 for main process, which
1635 is treated as 0)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001636 """
Simon Glassd829f122020-03-18 09:42:42 -06001637 if self.work_in_output:
1638 return self._working_dir
Simon Glassb82492b2021-01-30 22:17:46 -07001639 return os.path.join(self._working_dir, '%02d' % max(thread_num, 0))
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001640
Simon Glass37edf5f2023-07-19 17:49:06 -06001641 def _prepare_thread(self, thread_num, setup_git):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001642 """Prepare the working directory for a thread.
1643
1644 This clones or fetches the repo into the thread's work directory.
Alper Nebi Yasak76de29f2020-09-03 15:51:03 +03001645 Optionally, it can create a linked working tree of the repo in the
1646 thread's work directory instead.
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001647
1648 Args:
1649 thread_num: Thread number (0, 1, ...)
Alper Nebi Yasak76de29f2020-09-03 15:51:03 +03001650 setup_git:
1651 'clone' to set up a git clone
1652 'worktree' to set up a git worktree
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001653 """
Simon Glass37edf5f2023-07-19 17:49:06 -06001654 thread_dir = self.get_thread_dir(thread_num)
Simon Glassf06d3332023-07-19 17:49:08 -06001655 builderthread.mkdir(thread_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001656 git_dir = os.path.join(thread_dir, '.git')
1657
Alper Nebi Yasak76de29f2020-09-03 15:51:03 +03001658 # Create a worktree or a git repo clone for this thread if it
1659 # doesn't already exist
Simon Glassfea58582014-08-09 15:32:59 -06001660 if setup_git and self.git_dir:
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001661 src_dir = os.path.abspath(self.git_dir)
Alper Nebi Yasak76de29f2020-09-03 15:51:03 +03001662 if os.path.isdir(git_dir):
1663 # This is a clone of the src_dir repo, we can keep using
1664 # it but need to fetch from src_dir.
Simon Glass098b10f2022-01-29 14:14:18 -07001665 tprint('\rFetching repo for thread %d' % thread_num,
Simon Glass212c0b82020-04-09 15:08:43 -06001666 newline=False)
Simon Glass0157b182022-01-29 14:14:11 -07001667 gitutil.fetch(git_dir, thread_dir)
Simon Glass098b10f2022-01-29 14:14:18 -07001668 terminal.print_clear()
Alper Nebi Yasak76de29f2020-09-03 15:51:03 +03001669 elif os.path.isfile(git_dir):
1670 # This is a worktree of the src_dir repo, we don't need to
1671 # create it again or update it in any way.
1672 pass
1673 elif os.path.exists(git_dir):
1674 # Don't know what could trigger this, but we probably
1675 # can't create a git worktree/clone here.
1676 raise ValueError('Git dir %s exists, but is not a file '
1677 'or a directory.' % git_dir)
1678 elif setup_git == 'worktree':
Simon Glass098b10f2022-01-29 14:14:18 -07001679 tprint('\rChecking out worktree for thread %d' % thread_num,
Alper Nebi Yasak76de29f2020-09-03 15:51:03 +03001680 newline=False)
Simon Glass0157b182022-01-29 14:14:11 -07001681 gitutil.add_worktree(src_dir, thread_dir)
Simon Glass098b10f2022-01-29 14:14:18 -07001682 terminal.print_clear()
Alper Nebi Yasak76de29f2020-09-03 15:51:03 +03001683 elif setup_git == 'clone' or setup_git == True:
Simon Glass098b10f2022-01-29 14:14:18 -07001684 tprint('\rCloning repo for thread %d' % thread_num,
Simon Glass21f0eb32016-09-18 16:48:31 -06001685 newline=False)
Simon Glass0157b182022-01-29 14:14:11 -07001686 gitutil.clone(src_dir, thread_dir)
Simon Glass098b10f2022-01-29 14:14:18 -07001687 terminal.print_clear()
Alper Nebi Yasak76de29f2020-09-03 15:51:03 +03001688 else:
1689 raise ValueError("Can't setup git repo with %s." % setup_git)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001690
Simon Glass37edf5f2023-07-19 17:49:06 -06001691 def _prepare_working_space(self, max_threads, setup_git):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001692 """Prepare the working directory for use.
1693
Alper Nebi Yasak76de29f2020-09-03 15:51:03 +03001694 Set up the git repo for each thread. Creates a linked working tree
1695 if git-worktree is available, or clones the repo if it isn't.
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001696
1697 Args:
Simon Glassb82492b2021-01-30 22:17:46 -07001698 max_threads: Maximum number of threads we expect to need. If 0 then
1699 1 is set up, since the main process still needs somewhere to
1700 work
Alper Nebi Yasak76de29f2020-09-03 15:51:03 +03001701 setup_git: True to set up a git worktree or a git clone
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001702 """
Simon Glassf06d3332023-07-19 17:49:08 -06001703 builderthread.mkdir(self._working_dir)
Alper Nebi Yasak76de29f2020-09-03 15:51:03 +03001704 if setup_git and self.git_dir:
1705 src_dir = os.path.abspath(self.git_dir)
Simon Glass0157b182022-01-29 14:14:11 -07001706 if gitutil.check_worktree_is_available(src_dir):
Alper Nebi Yasak76de29f2020-09-03 15:51:03 +03001707 setup_git = 'worktree'
1708 # If we previously added a worktree but the directory for it
1709 # got deleted, we need to prune its files from the repo so
1710 # that we can check out another in its place.
Simon Glass0157b182022-01-29 14:14:11 -07001711 gitutil.prune_worktrees(src_dir)
Alper Nebi Yasak76de29f2020-09-03 15:51:03 +03001712 else:
1713 setup_git = 'clone'
Simon Glassb82492b2021-01-30 22:17:46 -07001714
1715 # Always do at least one thread
1716 for thread in range(max(max_threads, 1)):
Simon Glass37edf5f2023-07-19 17:49:06 -06001717 self._prepare_thread(thread, setup_git)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001718
Simon Glass37edf5f2023-07-19 17:49:06 -06001719 def _get_output_space_removals(self):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001720 """Get the output directories ready to receive files.
1721
Simon Glass925f6ad2020-03-18 09:42:45 -06001722 Figure out what needs to be deleted in the output directory before it
1723 can be used. We only delete old buildman directories which have the
Simon Glass4a7419b2023-07-19 17:49:10 -06001724 expected name pattern. See get_output_dir().
Simon Glass925f6ad2020-03-18 09:42:45 -06001725
1726 Returns:
1727 List of full paths of directories to remove
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001728 """
Simon Glass1a915672014-12-01 17:33:53 -07001729 if not self.commits:
1730 return
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001731 dir_list = []
1732 for commit_upto in range(self.commit_count):
Simon Glass4a7419b2023-07-19 17:49:10 -06001733 dir_list.append(self.get_output_dir(commit_upto))
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001734
Simon Glassb222abe2016-09-18 16:48:32 -06001735 to_remove = []
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001736 for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1737 if dirname not in dir_list:
Simon Glass925f6ad2020-03-18 09:42:45 -06001738 leaf = dirname[len(self.base_dir) + 1:]
Ovidiu Panait7664b032020-05-15 09:30:12 +03001739 m = re.match('[0-9]+_g[0-9a-f]+_.*', leaf)
Simon Glass925f6ad2020-03-18 09:42:45 -06001740 if m:
1741 to_remove.append(dirname)
1742 return to_remove
1743
Simon Glass37edf5f2023-07-19 17:49:06 -06001744 def _prepare_output_space(self):
Simon Glass925f6ad2020-03-18 09:42:45 -06001745 """Get the output directories ready to receive files.
1746
1747 We delete any output directories which look like ones we need to
1748 create. Having left over directories is confusing when the user wants
1749 to check the output manually.
1750 """
Simon Glass37edf5f2023-07-19 17:49:06 -06001751 to_remove = self._get_output_space_removals()
Simon Glassb222abe2016-09-18 16:48:32 -06001752 if to_remove:
Simon Glass098b10f2022-01-29 14:14:18 -07001753 tprint('Removing %d old build directories...' % len(to_remove),
Simon Glassb222abe2016-09-18 16:48:32 -06001754 newline=False)
1755 for dirname in to_remove:
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001756 shutil.rmtree(dirname)
Simon Glass098b10f2022-01-29 14:14:18 -07001757 terminal.print_clear()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001758
Simon Glass37edf5f2023-07-19 17:49:06 -06001759 def build_boards(self, commits, board_selected, keep_outputs, verbose):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001760 """Build all commits for a list of boards
1761
1762 Args:
1763 commits: List of commits to be build, each a Commit object
1764 boards_selected: Dict of selected boards, key is target name,
1765 value is Board object
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001766 keep_outputs: True to save build output files
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001767 verbose: Display build results as they are completed
Simon Glass2c3deb92014-08-28 09:43:39 -06001768 Returns:
1769 Tuple containing:
1770 - number of boards that failed to build
1771 - number of boards that issued warnings
Simon Glass8116c782021-04-11 16:27:27 +12001772 - list of thread exceptions raised
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001773 """
Simon Glassfea58582014-08-09 15:32:59 -06001774 self.commit_count = len(commits) if commits else 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001775 self.commits = commits
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001776 self._verbose = verbose
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001777
Simon Glass37edf5f2023-07-19 17:49:06 -06001778 self.reset_result_summary(board_selected)
Simon Glassf06d3332023-07-19 17:49:08 -06001779 builderthread.mkdir(self.base_dir, parents = True)
Simon Glass37edf5f2023-07-19 17:49:06 -06001780 self._prepare_working_space(min(self.num_threads, len(board_selected)),
Simon Glassfea58582014-08-09 15:32:59 -06001781 commits is not None)
Simon Glass37edf5f2023-07-19 17:49:06 -06001782 self._prepare_output_space()
Simon Glassae1a09f2022-07-11 19:03:56 -06001783 if not self._ide:
1784 tprint('\rStarting build...', newline=False)
Simon Glass2ce06f52023-09-07 10:00:19 -06001785 self._start_time = datetime.now()
Simon Glass37edf5f2023-07-19 17:49:06 -06001786 self.setup_build(board_selected, commits)
1787 self.process_result(None)
Simon Glass8116c782021-04-11 16:27:27 +12001788 self.thread_exceptions = []
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001789 # Create jobs to build all commits for each board
Simon Glassc05aa032019-10-31 07:42:53 -06001790 for brd in board_selected.values():
Simon Glass190064b2014-08-09 15:33:00 -06001791 job = builderthread.BuilderJob()
Simon Glassf4ed4702022-07-11 19:03:57 -06001792 job.brd = brd
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001793 job.commits = commits
1794 job.keep_outputs = keep_outputs
Simon Glassd829f122020-03-18 09:42:42 -06001795 job.work_in_output = self.work_in_output
Simon Glass2b4806e2022-01-22 05:07:33 -07001796 job.adjust_cfg = self.adjust_cfg
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001797 job.step = self._step
Simon Glassb82492b2021-01-30 22:17:46 -07001798 if self.num_threads:
1799 self.queue.put(job)
1800 else:
Simon Glassf06d3332023-07-19 17:49:08 -06001801 self._single_builder.run_job(job)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001802
Simon Glassb82492b2021-01-30 22:17:46 -07001803 if self.num_threads:
1804 term = threading.Thread(target=self.queue.join)
1805 term.setDaemon(True)
1806 term.start()
1807 while term.is_alive():
1808 term.join(100)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001809
Simon Glassb82492b2021-01-30 22:17:46 -07001810 # Wait until we have processed all output
1811 self.out_queue.join()
Simon Glassae1a09f2022-07-11 19:03:56 -06001812 if not self._ide:
1813 tprint()
Simon Glass7b33f212020-04-09 15:08:47 -06001814
Simon Glassae1a09f2022-07-11 19:03:56 -06001815 msg = 'Completed: %d total built' % self.count
1816 if self.already_done:
1817 msg += ' (%d previously' % self.already_done
1818 if self.already_done != self.count:
1819 msg += ', %d newly' % (self.count - self.already_done)
1820 msg += ')'
1821 duration = datetime.now() - self._start_time
1822 if duration > timedelta(microseconds=1000000):
1823 if duration.microseconds >= 500000:
1824 duration = duration + timedelta(seconds=1)
1825 duration = duration - timedelta(microseconds=duration.microseconds)
1826 rate = float(self.count) / duration.total_seconds()
1827 msg += ', duration %s, rate %1.2f' % (duration, rate)
1828 tprint(msg)
1829 if self.thread_exceptions:
1830 tprint('Failed: %d thread exceptions' % len(self.thread_exceptions),
1831 colour=self.col.RED)
Simon Glass7b33f212020-04-09 15:08:47 -06001832
Simon Glass8116c782021-04-11 16:27:27 +12001833 return (self.fail, self.warned, self.thread_exceptions)