Tom Rini | 83d290c | 2018-05-06 17:58:06 -0400 | [diff] [blame] | 1 | # SPDX-License-Identifier: GPL-2.0+ |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 2 | # Copyright (c) 2013 The Chromium OS Authors. |
| 3 | # |
| 4 | # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> |
| 5 | # |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 6 | |
| 7 | import collections |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 8 | from datetime import datetime, timedelta |
| 9 | import glob |
| 10 | import os |
| 11 | import re |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 12 | import queue |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 13 | import shutil |
Simon Glass | 2f25664 | 2016-09-18 16:48:37 -0600 | [diff] [blame] | 14 | import signal |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 15 | import string |
| 16 | import sys |
Simon Glass | d436e38 | 2016-09-18 16:48:35 -0600 | [diff] [blame] | 17 | import threading |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 18 | import time |
| 19 | |
Simon Glass | 0ede00f | 2020-04-17 18:09:02 -0600 | [diff] [blame] | 20 | from buildman import builderthread |
| 21 | from buildman import toolchain |
Simon Glass | bf77667 | 2020-04-17 18:09:04 -0600 | [diff] [blame] | 22 | from patman import gitutil |
Simon Glass | 4583c00 | 2023-02-23 18:18:04 -0700 | [diff] [blame] | 23 | from u_boot_pylib import command |
| 24 | from u_boot_pylib import terminal |
| 25 | from u_boot_pylib.terminal import tprint |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 26 | |
Simon Glass | 7bf83a5 | 2021-10-19 21:43:24 -0600 | [diff] [blame] | 27 | # This indicates an new int or hex Kconfig property with no default |
| 28 | # It hangs the build since the 'conf' tool cannot proceed without valid input. |
| 29 | # |
| 30 | # We get a repeat sequence of something like this: |
| 31 | # >> |
| 32 | # Break things (BREAK_ME) [] (NEW) |
| 33 | # Error in reading or end of file. |
| 34 | # << |
| 35 | # which indicates that BREAK_ME has an empty default |
| 36 | RE_NO_DEFAULT = re.compile(b'\((\w+)\) \[] \(NEW\)') |
| 37 | |
Simon Glass | ad8dbab | 2023-10-23 00:52:43 -0700 | [diff] [blame] | 38 | # Symbol types which appear in the bloat feature (-B). Others are silently |
| 39 | # dropped when reading in the 'nm' output |
| 40 | NM_SYMBOL_TYPES = 'tTdDbBr' |
| 41 | |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 42 | """ |
| 43 | Theory of Operation |
| 44 | |
| 45 | Please see README for user documentation, and you should be familiar with |
| 46 | that before trying to make sense of this. |
| 47 | |
| 48 | Buildman works by keeping the machine as busy as possible, building different |
| 49 | commits for different boards on multiple CPUs at once. |
| 50 | |
| 51 | The source repo (self.git_dir) contains all the commits to be built. Each |
| 52 | thread works on a single board at a time. It checks out the first commit, |
| 53 | configures it for that board, then builds it. Then it checks out the next |
| 54 | commit and builds it (typically without re-configuring). When it runs out |
| 55 | of commits, it gets another job from the builder and starts again with that |
| 56 | board. |
| 57 | |
| 58 | Clearly the builder threads could work either way - they could check out a |
| 59 | commit and then built it for all boards. Using separate directories for each |
| 60 | commit/board pair they could leave their build product around afterwards |
| 61 | also. |
| 62 | |
| 63 | The intent behind building a single board for multiple commits, is to make |
| 64 | use of incremental builds. Since each commit is built incrementally from |
| 65 | the previous one, builds are faster. Reconfiguring for a different board |
| 66 | removes all intermediate object files. |
| 67 | |
| 68 | Many threads can be working at once, but each has its own working directory. |
| 69 | When a thread finishes a build, it puts the output files into a result |
| 70 | directory. |
| 71 | |
| 72 | The base directory used by buildman is normally '../<branch>', i.e. |
| 73 | a directory higher than the source repository and named after the branch |
| 74 | being built. |
| 75 | |
| 76 | Within the base directory, we have one subdirectory for each commit. Within |
| 77 | that is one subdirectory for each board. Within that is the build output for |
| 78 | that commit/board combination. |
| 79 | |
| 80 | Buildman also create working directories for each thread, in a .bm-work/ |
| 81 | subdirectory in the base dir. |
| 82 | |
| 83 | As an example, say we are building branch 'us-net' for boards 'sandbox' and |
| 84 | 'seaboard', and say that us-net has two commits. We will have directories |
| 85 | like this: |
| 86 | |
| 87 | us-net/ base directory |
Ovidiu Panait | 7664b03 | 2020-05-15 09:30:12 +0300 | [diff] [blame] | 88 | 01_g4ed4ebc_net--Add-tftp-speed-/ |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 89 | sandbox/ |
| 90 | u-boot.bin |
| 91 | seaboard/ |
| 92 | u-boot.bin |
Ovidiu Panait | 7664b03 | 2020-05-15 09:30:12 +0300 | [diff] [blame] | 93 | 02_g4ed4ebc_net--Check-tftp-comp/ |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 94 | sandbox/ |
| 95 | u-boot.bin |
| 96 | seaboard/ |
| 97 | u-boot.bin |
| 98 | .bm-work/ |
| 99 | 00/ working directory for thread 0 (contains source checkout) |
| 100 | build/ build output |
| 101 | 01/ working directory for thread 1 |
| 102 | build/ build output |
| 103 | ... |
| 104 | u-boot/ source directory |
| 105 | .git/ repository |
| 106 | """ |
| 107 | |
Simon Glass | 35d696d | 2020-04-09 15:08:36 -0600 | [diff] [blame] | 108 | """Holds information about a particular error line we are outputing |
| 109 | |
| 110 | char: Character representation: '+': error, '-': fixed error, 'w+': warning, |
| 111 | 'w-' = fixed warning |
| 112 | boards: List of Board objects which have line in the error/warning output |
| 113 | errline: The text of the error line |
| 114 | """ |
Simon Glass | cc2c0d1 | 2022-07-11 19:04:00 -0600 | [diff] [blame] | 115 | ErrLine = collections.namedtuple('ErrLine', 'char,brds,errline') |
Simon Glass | 35d696d | 2020-04-09 15:08:36 -0600 | [diff] [blame] | 116 | |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 117 | # Possible build outcomes |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 118 | OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4)) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 119 | |
Simon Glass | 9a6d2e2 | 2017-04-12 18:23:26 -0600 | [diff] [blame] | 120 | # Translate a commit subject into a valid filename (and handle unicode) |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 121 | trans_valid_chars = str.maketrans('/: ', '---') |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 122 | |
Simon Glass | b464f8e | 2016-11-13 14:25:53 -0700 | [diff] [blame] | 123 | BASE_CONFIG_FILENAMES = [ |
| 124 | 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg' |
| 125 | ] |
| 126 | |
| 127 | EXTRA_CONFIG_FILENAMES = [ |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 128 | '.config', '.config-spl', '.config-tpl', |
| 129 | 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk', |
| 130 | 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h', |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 131 | ] |
| 132 | |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 133 | class Config: |
| 134 | """Holds information about configuration settings for a board.""" |
Simon Glass | b464f8e | 2016-11-13 14:25:53 -0700 | [diff] [blame] | 135 | def __init__(self, config_filename, target): |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 136 | self.target = target |
| 137 | self.config = {} |
Simon Glass | b464f8e | 2016-11-13 14:25:53 -0700 | [diff] [blame] | 138 | for fname in config_filename: |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 139 | self.config[fname] = {} |
| 140 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 141 | def add(self, fname, key, value): |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 142 | self.config[fname][key] = value |
| 143 | |
| 144 | def __hash__(self): |
| 145 | val = 0 |
| 146 | for fname in self.config: |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 147 | for key, value in self.config[fname].items(): |
| 148 | print(key, value) |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 149 | val = val ^ hash(key) & hash(value) |
| 150 | return val |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 151 | |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 152 | class Environment: |
| 153 | """Holds information about environment variables for a board.""" |
| 154 | def __init__(self, target): |
| 155 | self.target = target |
| 156 | self.environment = {} |
| 157 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 158 | def add(self, key, value): |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 159 | self.environment[key] = value |
| 160 | |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 161 | class Builder: |
| 162 | """Class for building U-Boot for a particular commit. |
| 163 | |
| 164 | Public members: (many should ->private) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 165 | already_done: Number of builds already completed |
| 166 | base_dir: Base directory to use for builder |
| 167 | checkout: True to check out source, False to skip that step. |
| 168 | This is used for testing. |
| 169 | col: terminal.Color() object |
Simon Glass | 6a0c7b4 | 2023-07-19 17:49:03 -0600 | [diff] [blame] | 170 | count: Total number of commits to build, which is the number of commits |
| 171 | multiplied by the number of boards |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 172 | do_make: Method to call to invoke Make |
| 173 | fail: Number of builds that failed due to error |
| 174 | force_build: Force building even if a build already exists |
| 175 | force_config_on_failure: If a commit fails for a board, disable |
| 176 | incremental building for the next commit we build for that |
| 177 | board, so that we will see all warnings/errors again. |
Simon Glass | 4266dc2 | 2014-07-13 12:22:31 -0600 | [diff] [blame] | 178 | force_build_failures: If a previously-built build (i.e. built on |
| 179 | a previous run of buildman) is marked as failed, rebuild it. |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 180 | git_dir: Git directory containing source repository |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 181 | num_jobs: Number of jobs to run at once (passed to make as -j) |
| 182 | num_threads: Number of builder threads to run |
| 183 | out_queue: Queue of results to process |
| 184 | re_make_err: Compiled regular expression for ignore_lines |
| 185 | queue: Queue of jobs to run |
| 186 | threads: List of active threads |
| 187 | toolchains: Toolchains object to use for building |
| 188 | upto: Current commit number we are building (0.count-1) |
| 189 | warned: Number of builds that produced at least one warning |
Simon Glass | 97e9152 | 2014-07-14 17:51:02 -0600 | [diff] [blame] | 190 | force_reconfig: Reconfigure U-Boot on each comiit. This disables |
| 191 | incremental building, where buildman reconfigures on the first |
| 192 | commit for a baord, and then just does an incremental build for |
| 193 | the following commits. In fact buildman will reconfigure and |
| 194 | retry for any failing commits, so generally the only effect of |
| 195 | this option is to slow things down. |
Simon Glass | 189a496 | 2014-07-14 17:51:03 -0600 | [diff] [blame] | 196 | in_tree: Build U-Boot in-tree instead of specifying an output |
| 197 | directory separate from the source code. This option is really |
| 198 | only useful for testing in-tree builds. |
Simon Glass | d829f12 | 2020-03-18 09:42:42 -0600 | [diff] [blame] | 199 | work_in_output: Use the output directory as the work directory and |
| 200 | don't write to a separate output directory. |
Simon Glass | 8116c78 | 2021-04-11 16:27:27 +1200 | [diff] [blame] | 201 | thread_exceptions: List of exceptions raised by thread jobs |
Simon Glass | 93202d7 | 2023-02-21 12:40:28 -0700 | [diff] [blame] | 202 | no_lto (bool): True to set the NO_LTO flag when building |
Simon Glass | bfb708a | 2023-02-21 12:40:29 -0700 | [diff] [blame] | 203 | reproducible_builds (bool): True to set SOURCE_DATE_EPOCH=0 for builds |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 204 | |
| 205 | Private members: |
| 206 | _base_board_dict: Last-summarised Dict of boards |
| 207 | _base_err_lines: Last-summarised list of errors |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 208 | _base_warn_lines: Last-summarised list of warnings |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 209 | _build_period_us: Time taken for a single build (float object). |
| 210 | _complete_delay: Expected delay until completion (timedelta) |
| 211 | _next_delay_update: Next time we plan to display a progress update |
| 212 | (datatime) |
| 213 | _show_unknown: Show unknown boards (those not built) in summary |
Simon Glass | 7b33f21 | 2020-04-09 15:08:47 -0600 | [diff] [blame] | 214 | _start_time: Start time for the build |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 215 | _timestamps: List of timestamps for the completion of the last |
| 216 | last _timestamp_count builds. Each is a datetime object. |
| 217 | _timestamp_count: Number of timestamps to keep in our list. |
| 218 | _working_dir: Base working directory containing all threads |
Simon Glass | b82492b | 2021-01-30 22:17:46 -0700 | [diff] [blame] | 219 | _single_builder: BuilderThread object for the singer builder, if |
| 220 | threading is not being used |
Simon Glass | 7bf83a5 | 2021-10-19 21:43:24 -0600 | [diff] [blame] | 221 | _terminated: Thread was terminated due to an error |
| 222 | _restarting_config: True if 'Restart config' is detected in output |
Simon Glass | ae1a09f | 2022-07-11 19:03:56 -0600 | [diff] [blame] | 223 | _ide: Produce output suitable for an Integrated Development Environment, |
| 224 | i.e. dont emit progress information and put errors/warnings on stderr |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 225 | """ |
| 226 | class Outcome: |
| 227 | """Records a build outcome for a single make invocation |
| 228 | |
| 229 | Public Members: |
| 230 | rc: Outcome value (OUTCOME_...) |
| 231 | err_lines: List of error lines or [] if none |
| 232 | sizes: Dictionary of image size information, keyed by filename |
| 233 | - Each value is itself a dictionary containing |
| 234 | values for 'text', 'data' and 'bss', being the integer |
| 235 | size in bytes of each section. |
| 236 | func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each |
| 237 | value is itself a dictionary: |
| 238 | key: function name |
| 239 | value: Size of function in bytes |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 240 | config: Dictionary keyed by filename - e.g. '.config'. Each |
| 241 | value is itself a dictionary: |
| 242 | key: config name |
| 243 | value: config value |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 244 | environment: Dictionary keyed by environment variable, Each |
| 245 | value is the value of environment variable. |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 246 | """ |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 247 | def __init__(self, rc, err_lines, sizes, func_sizes, config, |
| 248 | environment): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 249 | self.rc = rc |
| 250 | self.err_lines = err_lines |
| 251 | self.sizes = sizes |
| 252 | self.func_sizes = func_sizes |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 253 | self.config = config |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 254 | self.environment = environment |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 255 | |
| 256 | def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, |
Simon Glass | 5971ab5 | 2014-12-01 17:33:55 -0700 | [diff] [blame] | 257 | gnu_make='make', checkout=True, show_unknown=True, step=1, |
Stephen Warren | f79f1e0 | 2016-04-11 10:48:44 -0600 | [diff] [blame] | 258 | no_subdirs=False, full_path=False, verbose_build=False, |
Simon Glass | eb70a2c | 2020-04-09 15:08:51 -0600 | [diff] [blame] | 259 | mrproper=False, per_board_out_dir=False, |
Daniel Schwierzeck | 2371d1b | 2018-01-26 16:31:05 +0100 | [diff] [blame] | 260 | config_only=False, squash_config_y=False, |
Simon Glass | 8116c78 | 2021-04-11 16:27:27 +1200 | [diff] [blame] | 261 | warnings_as_errors=False, work_in_output=False, |
Tom Rini | d7713ad | 2022-11-09 19:14:53 -0700 | [diff] [blame] | 262 | test_thread_exceptions=False, adjust_cfg=None, |
Simon Glass | ffd06d3 | 2023-07-19 17:48:52 -0600 | [diff] [blame] | 263 | allow_missing=False, no_lto=False, reproducible_builds=False, |
| 264 | force_build=False, force_build_failures=False, |
| 265 | force_reconfig=False, in_tree=False, |
| 266 | force_config_on_failure=False, make_func=None): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 267 | """Create a new Builder object |
| 268 | |
| 269 | Args: |
| 270 | toolchains: Toolchains object to use for building |
| 271 | base_dir: Base directory to use for builder |
| 272 | git_dir: Git directory containing source repository |
| 273 | num_threads: Number of builder threads to run |
| 274 | num_jobs: Number of jobs to run at once (passed to make as -j) |
Masahiro Yamada | 9979692 | 2014-07-22 11:19:09 +0900 | [diff] [blame] | 275 | gnu_make: the command name of GNU Make. |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 276 | checkout: True to check out source, False to skip that step. |
| 277 | This is used for testing. |
| 278 | show_unknown: Show unknown boards (those not built) in summary |
| 279 | step: 1 to process every commit, n to process every nth commit |
Simon Glass | bb1501f | 2014-12-01 17:34:00 -0700 | [diff] [blame] | 280 | no_subdirs: Don't create subdirectories when building current |
| 281 | source for a single board |
| 282 | full_path: Return the full path in CROSS_COMPILE and don't set |
| 283 | PATH |
Simon Glass | d2ce658 | 2014-12-01 17:34:07 -0700 | [diff] [blame] | 284 | verbose_build: Run build with V=1 and don't use 'make -s' |
Simon Glass | eb70a2c | 2020-04-09 15:08:51 -0600 | [diff] [blame] | 285 | mrproper: Always run 'make mrproper' when configuring |
Stephen Warren | f79f1e0 | 2016-04-11 10:48:44 -0600 | [diff] [blame] | 286 | per_board_out_dir: Build in a separate persistent directory per |
| 287 | board rather than a thread-specific directory |
Simon Glass | b50113f | 2016-11-13 14:25:51 -0700 | [diff] [blame] | 288 | config_only: Only configure each build, don't build it |
Simon Glass | b464f8e | 2016-11-13 14:25:53 -0700 | [diff] [blame] | 289 | squash_config_y: Convert CONFIG options with the value 'y' to '1' |
Daniel Schwierzeck | 2371d1b | 2018-01-26 16:31:05 +0100 | [diff] [blame] | 290 | warnings_as_errors: Treat all compiler warnings as errors |
Simon Glass | d829f12 | 2020-03-18 09:42:42 -0600 | [diff] [blame] | 291 | work_in_output: Use the output directory as the work directory and |
| 292 | don't write to a separate output directory. |
Simon Glass | 8116c78 | 2021-04-11 16:27:27 +1200 | [diff] [blame] | 293 | test_thread_exceptions: Uses for tests only, True to make the |
| 294 | threads raise an exception instead of reporting their result. |
| 295 | This simulates a failure in the code somewhere |
Simon Glass | 2b4806e | 2022-01-22 05:07:33 -0700 | [diff] [blame] | 296 | adjust_cfg_list (list of str): List of changes to make to .config |
| 297 | file before building. Each is one of (where C is the config |
| 298 | option with or without the CONFIG_ prefix) |
| 299 | |
| 300 | C to enable C |
| 301 | ~C to disable C |
| 302 | C=val to set the value of C (val must have quotes if C is |
| 303 | a string Kconfig |
Tom Rini | d7713ad | 2022-11-09 19:14:53 -0700 | [diff] [blame] | 304 | allow_missing: Run build with BINMAN_ALLOW_MISSING=1 |
Simon Glass | 93202d7 | 2023-02-21 12:40:28 -0700 | [diff] [blame] | 305 | no_lto (bool): True to set the NO_LTO flag when building |
Simon Glass | ffd06d3 | 2023-07-19 17:48:52 -0600 | [diff] [blame] | 306 | force_build (bool): Rebuild even commits that are already built |
| 307 | force_build_failures (bool): Rebuild commits that have not been |
| 308 | built, or failed to build |
| 309 | force_reconfig (bool): Reconfigure on each commit |
| 310 | in_tree (bool): Bulid in tree instead of out-of-tree |
| 311 | force_config_on_failure (bool): Reconfigure the build before |
| 312 | retrying a failed build |
| 313 | make_func (function): Function to call to run 'make' |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 314 | """ |
| 315 | self.toolchains = toolchains |
| 316 | self.base_dir = base_dir |
Simon Glass | d829f12 | 2020-03-18 09:42:42 -0600 | [diff] [blame] | 317 | if work_in_output: |
| 318 | self._working_dir = base_dir |
| 319 | else: |
| 320 | self._working_dir = os.path.join(base_dir, '.bm-work') |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 321 | self.threads = [] |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 322 | self.do_make = make_func or self.make |
Masahiro Yamada | 9979692 | 2014-07-22 11:19:09 +0900 | [diff] [blame] | 323 | self.gnu_make = gnu_make |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 324 | self.checkout = checkout |
| 325 | self.num_threads = num_threads |
| 326 | self.num_jobs = num_jobs |
| 327 | self.already_done = 0 |
| 328 | self.force_build = False |
| 329 | self.git_dir = git_dir |
| 330 | self._show_unknown = show_unknown |
| 331 | self._timestamp_count = 10 |
| 332 | self._build_period_us = None |
| 333 | self._complete_delay = None |
| 334 | self._next_delay_update = datetime.now() |
Simon Glass | 2ce06f5 | 2023-09-07 10:00:19 -0600 | [diff] [blame] | 335 | self._start_time = None |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 336 | self._step = step |
Simon Glass | 28370c1 | 2014-08-09 15:33:06 -0600 | [diff] [blame] | 337 | self._error_lines = 0 |
Simon Glass | 5971ab5 | 2014-12-01 17:33:55 -0700 | [diff] [blame] | 338 | self.no_subdirs = no_subdirs |
Simon Glass | bb1501f | 2014-12-01 17:34:00 -0700 | [diff] [blame] | 339 | self.full_path = full_path |
Simon Glass | d2ce658 | 2014-12-01 17:34:07 -0700 | [diff] [blame] | 340 | self.verbose_build = verbose_build |
Simon Glass | b50113f | 2016-11-13 14:25:51 -0700 | [diff] [blame] | 341 | self.config_only = config_only |
Simon Glass | b464f8e | 2016-11-13 14:25:53 -0700 | [diff] [blame] | 342 | self.squash_config_y = squash_config_y |
| 343 | self.config_filenames = BASE_CONFIG_FILENAMES |
Simon Glass | d829f12 | 2020-03-18 09:42:42 -0600 | [diff] [blame] | 344 | self.work_in_output = work_in_output |
Simon Glass | 2b4806e | 2022-01-22 05:07:33 -0700 | [diff] [blame] | 345 | self.adjust_cfg = adjust_cfg |
Tom Rini | d7713ad | 2022-11-09 19:14:53 -0700 | [diff] [blame] | 346 | self.allow_missing = allow_missing |
Simon Glass | ae1a09f | 2022-07-11 19:03:56 -0600 | [diff] [blame] | 347 | self._ide = False |
Simon Glass | 93202d7 | 2023-02-21 12:40:28 -0700 | [diff] [blame] | 348 | self.no_lto = no_lto |
Simon Glass | bfb708a | 2023-02-21 12:40:29 -0700 | [diff] [blame] | 349 | self.reproducible_builds = reproducible_builds |
Simon Glass | ffd06d3 | 2023-07-19 17:48:52 -0600 | [diff] [blame] | 350 | self.force_build = force_build |
| 351 | self.force_build_failures = force_build_failures |
| 352 | self.force_reconfig = force_reconfig |
| 353 | self.in_tree = in_tree |
| 354 | self.force_config_on_failure = force_config_on_failure |
Simon Glass | 2b4806e | 2022-01-22 05:07:33 -0700 | [diff] [blame] | 355 | |
Simon Glass | b464f8e | 2016-11-13 14:25:53 -0700 | [diff] [blame] | 356 | if not self.squash_config_y: |
| 357 | self.config_filenames += EXTRA_CONFIG_FILENAMES |
Simon Glass | 7bf83a5 | 2021-10-19 21:43:24 -0600 | [diff] [blame] | 358 | self._terminated = False |
| 359 | self._restarting_config = False |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 360 | |
Daniel Schwierzeck | 2371d1b | 2018-01-26 16:31:05 +0100 | [diff] [blame] | 361 | self.warnings_as_errors = warnings_as_errors |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 362 | self.col = terminal.Color() |
| 363 | |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 364 | self._re_function = re.compile('(.*): In function.*') |
| 365 | self._re_files = re.compile('In file included from.*') |
| 366 | self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') |
Simon Glass | 2d48333 | 2018-11-06 16:02:11 -0700 | [diff] [blame] | 367 | self._re_dtb_warning = re.compile('(.*): Warning .*') |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 368 | self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') |
Simon Glass | 113a8a5 | 2020-04-09 15:08:53 -0600 | [diff] [blame] | 369 | self._re_migration_warning = re.compile(r'^={21} WARNING ={22}\n.*\n=+\n', |
| 370 | re.MULTILINE | re.DOTALL) |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 371 | |
Simon Glass | 8116c78 | 2021-04-11 16:27:27 +1200 | [diff] [blame] | 372 | self.thread_exceptions = [] |
| 373 | self.test_thread_exceptions = test_thread_exceptions |
Simon Glass | b82492b | 2021-01-30 22:17:46 -0700 | [diff] [blame] | 374 | if self.num_threads: |
| 375 | self._single_builder = None |
| 376 | self.queue = queue.Queue() |
| 377 | self.out_queue = queue.Queue() |
| 378 | for i in range(self.num_threads): |
Simon Glass | 8116c78 | 2021-04-11 16:27:27 +1200 | [diff] [blame] | 379 | t = builderthread.BuilderThread( |
| 380 | self, i, mrproper, per_board_out_dir, |
| 381 | test_exception=test_thread_exceptions) |
Simon Glass | b82492b | 2021-01-30 22:17:46 -0700 | [diff] [blame] | 382 | t.setDaemon(True) |
| 383 | t.start() |
| 384 | self.threads.append(t) |
| 385 | |
| 386 | t = builderthread.ResultThread(self) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 387 | t.setDaemon(True) |
| 388 | t.start() |
| 389 | self.threads.append(t) |
Simon Glass | b82492b | 2021-01-30 22:17:46 -0700 | [diff] [blame] | 390 | else: |
| 391 | self._single_builder = builderthread.BuilderThread( |
| 392 | self, -1, mrproper, per_board_out_dir) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 393 | |
| 394 | ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] |
| 395 | self.re_make_err = re.compile('|'.join(ignore_lines)) |
| 396 | |
Simon Glass | 2f25664 | 2016-09-18 16:48:37 -0600 | [diff] [blame] | 397 | # Handle existing graceful with SIGINT / Ctrl-C |
| 398 | signal.signal(signal.SIGINT, self.signal_handler) |
| 399 | |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 400 | def __del__(self): |
| 401 | """Get rid of all threads created by the builder""" |
| 402 | for t in self.threads: |
| 403 | del t |
| 404 | |
Simon Glass | 2f25664 | 2016-09-18 16:48:37 -0600 | [diff] [blame] | 405 | def signal_handler(self, signal, frame): |
| 406 | sys.exit(1) |
| 407 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 408 | def set_display_options(self, show_errors=False, show_sizes=False, |
Simon Glass | ed96665 | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 409 | show_detail=False, show_bloat=False, |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 410 | list_error_boards=False, show_config=False, |
Simon Glass | 113a8a5 | 2020-04-09 15:08:53 -0600 | [diff] [blame] | 411 | show_environment=False, filter_dtb_warnings=False, |
Simon Glass | ae1a09f | 2022-07-11 19:03:56 -0600 | [diff] [blame] | 412 | filter_migration_warnings=False, ide=False): |
Simon Glass | b2ea7ab | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 413 | """Setup display options for the builder. |
| 414 | |
Simon Glass | 174592b | 2020-04-09 15:08:52 -0600 | [diff] [blame] | 415 | Args: |
| 416 | show_errors: True to show summarised error/warning info |
| 417 | show_sizes: Show size deltas |
| 418 | show_detail: Show size delta detail for each board if show_sizes |
| 419 | show_bloat: Show detail for each function |
| 420 | list_error_boards: Show the boards which caused each error/warning |
| 421 | show_config: Show config deltas |
| 422 | show_environment: Show environment deltas |
| 423 | filter_dtb_warnings: Filter out any warnings from the device-tree |
| 424 | compiler |
Simon Glass | 113a8a5 | 2020-04-09 15:08:53 -0600 | [diff] [blame] | 425 | filter_migration_warnings: Filter out any warnings about migrating |
| 426 | a board to driver model |
Simon Glass | ae1a09f | 2022-07-11 19:03:56 -0600 | [diff] [blame] | 427 | ide: Create output that can be parsed by an IDE. There is no '+' prefix on |
| 428 | error lines and output on stderr stays on stderr. |
Simon Glass | b2ea7ab | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 429 | """ |
| 430 | self._show_errors = show_errors |
| 431 | self._show_sizes = show_sizes |
| 432 | self._show_detail = show_detail |
| 433 | self._show_bloat = show_bloat |
Simon Glass | ed96665 | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 434 | self._list_error_boards = list_error_boards |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 435 | self._show_config = show_config |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 436 | self._show_environment = show_environment |
Simon Glass | 174592b | 2020-04-09 15:08:52 -0600 | [diff] [blame] | 437 | self._filter_dtb_warnings = filter_dtb_warnings |
Simon Glass | 113a8a5 | 2020-04-09 15:08:53 -0600 | [diff] [blame] | 438 | self._filter_migration_warnings = filter_migration_warnings |
Simon Glass | ae1a09f | 2022-07-11 19:03:56 -0600 | [diff] [blame] | 439 | self._ide = ide |
Simon Glass | b2ea7ab | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 440 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 441 | def _add_timestamp(self): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 442 | """Add a new timestamp to the list and record the build period. |
| 443 | |
| 444 | The build period is the length of time taken to perform a single |
| 445 | build (one board, one commit). |
| 446 | """ |
| 447 | now = datetime.now() |
| 448 | self._timestamps.append(now) |
| 449 | count = len(self._timestamps) |
| 450 | delta = self._timestamps[-1] - self._timestamps[0] |
| 451 | seconds = delta.total_seconds() |
| 452 | |
| 453 | # If we have enough data, estimate build period (time taken for a |
| 454 | # single build) and therefore completion time. |
| 455 | if count > 1 and self._next_delay_update < now: |
| 456 | self._next_delay_update = now + timedelta(seconds=2) |
| 457 | if seconds > 0: |
| 458 | self._build_period = float(seconds) / count |
| 459 | todo = self.count - self.upto |
| 460 | self._complete_delay = timedelta(microseconds= |
| 461 | self._build_period * todo * 1000000) |
| 462 | # Round it |
| 463 | self._complete_delay -= timedelta( |
| 464 | microseconds=self._complete_delay.microseconds) |
| 465 | |
| 466 | if seconds > 60: |
| 467 | self._timestamps.popleft() |
| 468 | count -= 1 |
| 469 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 470 | def select_commit(self, commit, checkout=True): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 471 | """Checkout the selected commit for this build |
| 472 | """ |
| 473 | self.commit = commit |
| 474 | if checkout and self.checkout: |
Simon Glass | 0157b18 | 2022-01-29 14:14:11 -0700 | [diff] [blame] | 475 | gitutil.checkout(commit.hash) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 476 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 477 | def make(self, commit, brd, stage, cwd, *args, **kwargs): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 478 | """Run make |
| 479 | |
| 480 | Args: |
| 481 | commit: Commit object that is being built |
| 482 | brd: Board object that is being built |
Roger Meier | fd18a89 | 2014-08-20 22:10:29 +0200 | [diff] [blame] | 483 | stage: Stage that we are at (mrproper, config, build) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 484 | cwd: Directory where make should be run |
| 485 | args: Arguments to pass to make |
Simon Glass | d980069 | 2022-01-29 14:14:05 -0700 | [diff] [blame] | 486 | kwargs: Arguments to pass to command.run_pipe() |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 487 | """ |
Simon Glass | 7bf83a5 | 2021-10-19 21:43:24 -0600 | [diff] [blame] | 488 | |
| 489 | def check_output(stream, data): |
| 490 | if b'Restart config' in data: |
| 491 | self._restarting_config = True |
| 492 | |
| 493 | # If we see 'Restart config' following by multiple errors |
| 494 | if self._restarting_config: |
| 495 | m = RE_NO_DEFAULT.findall(data) |
| 496 | |
| 497 | # Number of occurences of each Kconfig item |
| 498 | multiple = [m.count(val) for val in set(m)] |
| 499 | |
| 500 | # If any of them occur more than once, we have a loop |
| 501 | if [val for val in multiple if val > 1]: |
| 502 | self._terminated = True |
| 503 | return True |
| 504 | return False |
| 505 | |
| 506 | self._restarting_config = False |
| 507 | self._terminated = False |
Masahiro Yamada | 9979692 | 2014-07-22 11:19:09 +0900 | [diff] [blame] | 508 | cmd = [self.gnu_make] + list(args) |
Simon Glass | d980069 | 2022-01-29 14:14:05 -0700 | [diff] [blame] | 509 | result = command.run_pipe([cmd], capture=True, capture_stderr=True, |
Simon Glass | 7bf83a5 | 2021-10-19 21:43:24 -0600 | [diff] [blame] | 510 | cwd=cwd, raise_on_error=False, infile='/dev/null', |
| 511 | output_func=check_output, **kwargs) |
| 512 | |
| 513 | if self._terminated: |
| 514 | # Try to be helpful |
| 515 | result.stderr += '(** did you define an int/hex Kconfig with no default? **)' |
| 516 | |
Simon Glass | 40f11fc | 2015-02-05 22:06:12 -0700 | [diff] [blame] | 517 | if self.verbose_build: |
| 518 | result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout |
| 519 | result.combined = '%s\n' % (' '.join(cmd)) + result.combined |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 520 | return result |
| 521 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 522 | def process_result(self, result): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 523 | """Process the result of a build, showing progress information |
| 524 | |
| 525 | Args: |
Simon Glass | e5a0e5d | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 526 | result: A CommandResult object, which indicates the result for |
| 527 | a single build |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 528 | """ |
| 529 | col = terminal.Color() |
| 530 | if result: |
| 531 | target = result.brd.target |
| 532 | |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 533 | self.upto += 1 |
| 534 | if result.return_code != 0: |
| 535 | self.fail += 1 |
| 536 | elif result.stderr: |
| 537 | self.warned += 1 |
| 538 | if result.already_done: |
| 539 | self.already_done += 1 |
Simon Glass | e5a0e5d | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 540 | if self._verbose: |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 541 | terminal.print_clear() |
Simon Glass | e5a0e5d | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 542 | boards_selected = {target : result.brd} |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 543 | self.reset_result_summary(boards_selected) |
| 544 | self.produce_result_summary(result.commit_upto, self.commits, |
Simon Glass | e5a0e5d | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 545 | boards_selected) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 546 | else: |
| 547 | target = '(starting)' |
| 548 | |
| 549 | # Display separate counts for ok, warned and fail |
| 550 | ok = self.upto - self.warned - self.fail |
Simon Glass | 252ac58 | 2022-01-29 14:14:17 -0700 | [diff] [blame] | 551 | line = '\r' + self.col.build(self.col.GREEN, '%5d' % ok) |
| 552 | line += self.col.build(self.col.YELLOW, '%5d' % self.warned) |
| 553 | line += self.col.build(self.col.RED, '%5d' % self.fail) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 554 | |
Simon Glass | 6eb76ca | 2020-04-09 15:08:45 -0600 | [diff] [blame] | 555 | line += ' /%-5d ' % self.count |
| 556 | remaining = self.count - self.upto |
| 557 | if remaining: |
Simon Glass | 252ac58 | 2022-01-29 14:14:17 -0700 | [diff] [blame] | 558 | line += self.col.build(self.col.MAGENTA, ' -%-5d ' % remaining) |
Simon Glass | 6eb76ca | 2020-04-09 15:08:45 -0600 | [diff] [blame] | 559 | else: |
| 560 | line += ' ' * 8 |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 561 | |
| 562 | # Add our current completion time estimate |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 563 | self._add_timestamp() |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 564 | if self._complete_delay: |
Simon Glass | 6eb76ca | 2020-04-09 15:08:45 -0600 | [diff] [blame] | 565 | line += '%s : ' % self._complete_delay |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 566 | |
Simon Glass | 6eb76ca | 2020-04-09 15:08:45 -0600 | [diff] [blame] | 567 | line += target |
Simon Glass | ae1a09f | 2022-07-11 19:03:56 -0600 | [diff] [blame] | 568 | if not self._ide: |
| 569 | terminal.print_clear() |
| 570 | tprint(line, newline=False, limit_to_line=True) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 571 | |
Simon Glass | 4a7419b | 2023-07-19 17:49:10 -0600 | [diff] [blame] | 572 | def get_output_dir(self, commit_upto): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 573 | """Get the name of the output directory for a commit number |
| 574 | |
| 575 | The output directory is typically .../<branch>/<commit>. |
| 576 | |
| 577 | Args: |
| 578 | commit_upto: Commit number to use (0..self.count-1) |
| 579 | """ |
Simon Glass | 60b285f | 2020-04-17 17:51:34 -0600 | [diff] [blame] | 580 | if self.work_in_output: |
| 581 | return self._working_dir |
| 582 | |
Simon Glass | 5971ab5 | 2014-12-01 17:33:55 -0700 | [diff] [blame] | 583 | commit_dir = None |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 584 | if self.commits: |
| 585 | commit = self.commits[commit_upto] |
| 586 | subject = commit.subject.translate(trans_valid_chars) |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 587 | # See _get_output_space_removals() which parses this name |
Ovidiu Panait | 7664b03 | 2020-05-15 09:30:12 +0300 | [diff] [blame] | 588 | commit_dir = ('%02d_g%s_%s' % (commit_upto + 1, |
| 589 | commit.hash, subject[:20])) |
Simon Glass | 5971ab5 | 2014-12-01 17:33:55 -0700 | [diff] [blame] | 590 | elif not self.no_subdirs: |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 591 | commit_dir = 'current' |
Simon Glass | 5971ab5 | 2014-12-01 17:33:55 -0700 | [diff] [blame] | 592 | if not commit_dir: |
| 593 | return self.base_dir |
| 594 | return os.path.join(self.base_dir, commit_dir) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 595 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 596 | def get_build_dir(self, commit_upto, target): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 597 | """Get the name of the build directory for a commit number |
| 598 | |
| 599 | The build directory is typically .../<branch>/<commit>/<target>. |
| 600 | |
| 601 | Args: |
| 602 | commit_upto: Commit number to use (0..self.count-1) |
| 603 | target: Target name |
| 604 | """ |
Simon Glass | 4a7419b | 2023-07-19 17:49:10 -0600 | [diff] [blame] | 605 | output_dir = self.get_output_dir(commit_upto) |
Simon Glass | 60b285f | 2020-04-17 17:51:34 -0600 | [diff] [blame] | 606 | if self.work_in_output: |
| 607 | return output_dir |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 608 | return os.path.join(output_dir, target) |
| 609 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 610 | def get_done_file(self, commit_upto, target): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 611 | """Get the name of the done file for a commit number |
| 612 | |
| 613 | Args: |
| 614 | commit_upto: Commit number to use (0..self.count-1) |
| 615 | target: Target name |
| 616 | """ |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 617 | return os.path.join(self.get_build_dir(commit_upto, target), 'done') |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 618 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 619 | def get_sizes_file(self, commit_upto, target): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 620 | """Get the name of the sizes file for a commit number |
| 621 | |
| 622 | Args: |
| 623 | commit_upto: Commit number to use (0..self.count-1) |
| 624 | target: Target name |
| 625 | """ |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 626 | return os.path.join(self.get_build_dir(commit_upto, target), 'sizes') |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 627 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 628 | def get_func_sizes_file(self, commit_upto, target, elf_fname): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 629 | """Get the name of the funcsizes file for a commit number and ELF file |
| 630 | |
| 631 | Args: |
| 632 | commit_upto: Commit number to use (0..self.count-1) |
| 633 | target: Target name |
| 634 | elf_fname: Filename of elf image |
| 635 | """ |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 636 | return os.path.join(self.get_build_dir(commit_upto, target), |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 637 | '%s.sizes' % elf_fname.replace('/', '-')) |
| 638 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 639 | def get_objdump_file(self, commit_upto, target, elf_fname): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 640 | """Get the name of the objdump file for a commit number and ELF file |
| 641 | |
| 642 | Args: |
| 643 | commit_upto: Commit number to use (0..self.count-1) |
| 644 | target: Target name |
| 645 | elf_fname: Filename of elf image |
| 646 | """ |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 647 | return os.path.join(self.get_build_dir(commit_upto, target), |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 648 | '%s.objdump' % elf_fname.replace('/', '-')) |
| 649 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 650 | def get_err_file(self, commit_upto, target): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 651 | """Get the name of the err file for a commit number |
| 652 | |
| 653 | Args: |
| 654 | commit_upto: Commit number to use (0..self.count-1) |
| 655 | target: Target name |
| 656 | """ |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 657 | output_dir = self.get_build_dir(commit_upto, target) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 658 | return os.path.join(output_dir, 'err') |
| 659 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 660 | def filter_errors(self, lines): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 661 | """Filter out errors in which we have no interest |
| 662 | |
| 663 | We should probably use map(). |
| 664 | |
| 665 | Args: |
| 666 | lines: List of error lines, each a string |
| 667 | Returns: |
| 668 | New list with only interesting lines included |
| 669 | """ |
| 670 | out_lines = [] |
Simon Glass | 113a8a5 | 2020-04-09 15:08:53 -0600 | [diff] [blame] | 671 | if self._filter_migration_warnings: |
| 672 | text = '\n'.join(lines) |
| 673 | text = self._re_migration_warning.sub('', text) |
| 674 | lines = text.splitlines() |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 675 | for line in lines: |
Simon Glass | 174592b | 2020-04-09 15:08:52 -0600 | [diff] [blame] | 676 | if self.re_make_err.search(line): |
| 677 | continue |
| 678 | if self._filter_dtb_warnings and self._re_dtb_warning.search(line): |
| 679 | continue |
| 680 | out_lines.append(line) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 681 | return out_lines |
| 682 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 683 | def read_func_sizes(self, fname, fd): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 684 | """Read function sizes from the output of 'nm' |
| 685 | |
| 686 | Args: |
| 687 | fd: File containing data to read |
| 688 | fname: Filename we are reading from (just for errors) |
| 689 | |
| 690 | Returns: |
| 691 | Dictionary containing size of each function in bytes, indexed by |
| 692 | function name. |
| 693 | """ |
| 694 | sym = {} |
| 695 | for line in fd.readlines(): |
Simon Glass | f2e6775 | 2022-07-11 19:04:11 -0600 | [diff] [blame] | 696 | line = line.strip() |
| 697 | parts = line.split() |
| 698 | if line and len(parts) == 3: |
| 699 | size, type, name = line.split() |
Simon Glass | ad8dbab | 2023-10-23 00:52:43 -0700 | [diff] [blame] | 700 | if type in NM_SYMBOL_TYPES: |
Simon Glass | f2e6775 | 2022-07-11 19:04:11 -0600 | [diff] [blame] | 701 | # function names begin with '.' on 64-bit powerpc |
| 702 | if '.' in name[1:]: |
| 703 | name = 'static.' + name.split('.')[0] |
| 704 | sym[name] = sym.get(name, 0) + int(size, 16) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 705 | return sym |
| 706 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 707 | def _process_config(self, fname): |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 708 | """Read in a .config, autoconf.mk or autoconf.h file |
| 709 | |
| 710 | This function handles all config file types. It ignores comments and |
| 711 | any #defines which don't start with CONFIG_. |
| 712 | |
| 713 | Args: |
| 714 | fname: Filename to read |
| 715 | |
| 716 | Returns: |
| 717 | Dictionary: |
| 718 | key: Config name (e.g. CONFIG_DM) |
| 719 | value: Config value (e.g. 1) |
| 720 | """ |
| 721 | config = {} |
| 722 | if os.path.exists(fname): |
| 723 | with open(fname) as fd: |
| 724 | for line in fd: |
| 725 | line = line.strip() |
| 726 | if line.startswith('#define'): |
| 727 | values = line[8:].split(' ', 1) |
| 728 | if len(values) > 1: |
| 729 | key, value = values |
| 730 | else: |
| 731 | key = values[0] |
Simon Glass | b464f8e | 2016-11-13 14:25:53 -0700 | [diff] [blame] | 732 | value = '1' if self.squash_config_y else '' |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 733 | if not key.startswith('CONFIG_'): |
| 734 | continue |
| 735 | elif not line or line[0] in ['#', '*', '/']: |
| 736 | continue |
| 737 | else: |
| 738 | key, value = line.split('=', 1) |
Simon Glass | b464f8e | 2016-11-13 14:25:53 -0700 | [diff] [blame] | 739 | if self.squash_config_y and value == 'y': |
| 740 | value = '1' |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 741 | config[key] = value |
| 742 | return config |
| 743 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 744 | def _process_environment(self, fname): |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 745 | """Read in a uboot.env file |
| 746 | |
| 747 | This function reads in environment variables from a file. |
| 748 | |
| 749 | Args: |
| 750 | fname: Filename to read |
| 751 | |
| 752 | Returns: |
| 753 | Dictionary: |
| 754 | key: environment variable (e.g. bootlimit) |
| 755 | value: value of environment variable (e.g. 1) |
| 756 | """ |
| 757 | environment = {} |
| 758 | if os.path.exists(fname): |
| 759 | with open(fname) as fd: |
| 760 | for line in fd.read().split('\0'): |
| 761 | try: |
| 762 | key, value = line.split('=', 1) |
| 763 | environment[key] = value |
| 764 | except ValueError: |
| 765 | # ignore lines we can't parse |
| 766 | pass |
| 767 | return environment |
| 768 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 769 | def get_build_outcome(self, commit_upto, target, read_func_sizes, |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 770 | read_config, read_environment): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 771 | """Work out the outcome of a build. |
| 772 | |
| 773 | Args: |
| 774 | commit_upto: Commit number to check (0..n-1) |
| 775 | target: Target board to check |
| 776 | read_func_sizes: True to read function size information |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 777 | read_config: True to read .config and autoconf.h files |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 778 | read_environment: True to read uboot.env files |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 779 | |
| 780 | Returns: |
| 781 | Outcome object |
| 782 | """ |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 783 | done_file = self.get_done_file(commit_upto, target) |
| 784 | sizes_file = self.get_sizes_file(commit_upto, target) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 785 | sizes = {} |
| 786 | func_sizes = {} |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 787 | config = {} |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 788 | environment = {} |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 789 | if os.path.exists(done_file): |
| 790 | with open(done_file, 'r') as fd: |
Simon Glass | 347ea0b | 2019-04-26 19:02:23 -0600 | [diff] [blame] | 791 | try: |
| 792 | return_code = int(fd.readline()) |
| 793 | except ValueError: |
| 794 | # The file may be empty due to running out of disk space. |
| 795 | # Try a rebuild |
| 796 | return_code = 1 |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 797 | err_lines = [] |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 798 | err_file = self.get_err_file(commit_upto, target) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 799 | if os.path.exists(err_file): |
| 800 | with open(err_file, 'r') as fd: |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 801 | err_lines = self.filter_errors(fd.readlines()) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 802 | |
| 803 | # Decide whether the build was ok, failed or created warnings |
| 804 | if return_code: |
| 805 | rc = OUTCOME_ERROR |
| 806 | elif len(err_lines): |
| 807 | rc = OUTCOME_WARNING |
| 808 | else: |
| 809 | rc = OUTCOME_OK |
| 810 | |
| 811 | # Convert size information to our simple format |
| 812 | if os.path.exists(sizes_file): |
| 813 | with open(sizes_file, 'r') as fd: |
| 814 | for line in fd.readlines(): |
| 815 | values = line.split() |
| 816 | rodata = 0 |
| 817 | if len(values) > 6: |
| 818 | rodata = int(values[6], 16) |
| 819 | size_dict = { |
| 820 | 'all' : int(values[0]) + int(values[1]) + |
| 821 | int(values[2]), |
| 822 | 'text' : int(values[0]) - rodata, |
| 823 | 'data' : int(values[1]), |
| 824 | 'bss' : int(values[2]), |
| 825 | 'rodata' : rodata, |
| 826 | } |
| 827 | sizes[values[5]] = size_dict |
| 828 | |
| 829 | if read_func_sizes: |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 830 | pattern = self.get_func_sizes_file(commit_upto, target, '*') |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 831 | for fname in glob.glob(pattern): |
| 832 | with open(fname, 'r') as fd: |
| 833 | dict_name = os.path.basename(fname).replace('.sizes', |
| 834 | '') |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 835 | func_sizes[dict_name] = self.read_func_sizes(fname, fd) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 836 | |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 837 | if read_config: |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 838 | output_dir = self.get_build_dir(commit_upto, target) |
Simon Glass | b464f8e | 2016-11-13 14:25:53 -0700 | [diff] [blame] | 839 | for name in self.config_filenames: |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 840 | fname = os.path.join(output_dir, name) |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 841 | config[name] = self._process_config(fname) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 842 | |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 843 | if read_environment: |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 844 | output_dir = self.get_build_dir(commit_upto, target) |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 845 | fname = os.path.join(output_dir, 'uboot.env') |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 846 | environment = self._process_environment(fname) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 847 | |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 848 | return Builder.Outcome(rc, err_lines, sizes, func_sizes, config, |
| 849 | environment) |
| 850 | |
| 851 | return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {}) |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 852 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 853 | def get_result_summary(self, boards_selected, commit_upto, read_func_sizes, |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 854 | read_config, read_environment): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 855 | """Calculate a summary of the results of building a commit. |
| 856 | |
| 857 | Args: |
| 858 | board_selected: Dict containing boards to summarise |
| 859 | commit_upto: Commit number to summarize (0..self.count-1) |
| 860 | read_func_sizes: True to read function size information |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 861 | read_config: True to read .config and autoconf.h files |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 862 | read_environment: True to read uboot.env files |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 863 | |
| 864 | Returns: |
| 865 | Tuple: |
Simon Glass | ae1a09f | 2022-07-11 19:03:56 -0600 | [diff] [blame] | 866 | Dict containing boards which built this commit: |
| 867 | key: board.target |
| 868 | value: Builder.Outcome object |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 869 | List containing a summary of error lines |
Simon Glass | ed96665 | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 870 | Dict keyed by error line, containing a list of the Board |
| 871 | objects with that error |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 872 | List containing a summary of warning lines |
| 873 | Dict keyed by error line, containing a list of the Board |
| 874 | objects with that warning |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 875 | Dictionary keyed by board.target. Each value is a dictionary: |
| 876 | key: filename - e.g. '.config' |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 877 | value is itself a dictionary: |
| 878 | key: config name |
| 879 | value: config value |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 880 | Dictionary keyed by board.target. Each value is a dictionary: |
| 881 | key: environment variable |
| 882 | value: value of environment variable |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 883 | """ |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 884 | def add_line(lines_summary, lines_boards, line, board): |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 885 | line = line.rstrip() |
| 886 | if line in lines_boards: |
| 887 | lines_boards[line].append(board) |
| 888 | else: |
| 889 | lines_boards[line] = [board] |
| 890 | lines_summary.append(line) |
| 891 | |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 892 | board_dict = {} |
| 893 | err_lines_summary = [] |
Simon Glass | ed96665 | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 894 | err_lines_boards = {} |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 895 | warn_lines_summary = [] |
| 896 | warn_lines_boards = {} |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 897 | config = {} |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 898 | environment = {} |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 899 | |
Simon Glass | f4ed470 | 2022-07-11 19:03:57 -0600 | [diff] [blame] | 900 | for brd in boards_selected.values(): |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 901 | outcome = self.get_build_outcome(commit_upto, brd.target, |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 902 | read_func_sizes, read_config, |
| 903 | read_environment) |
Simon Glass | f4ed470 | 2022-07-11 19:03:57 -0600 | [diff] [blame] | 904 | board_dict[brd.target] = outcome |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 905 | last_func = None |
| 906 | last_was_warning = False |
| 907 | for line in outcome.err_lines: |
| 908 | if line: |
| 909 | if (self._re_function.match(line) or |
| 910 | self._re_files.match(line)): |
| 911 | last_func = line |
Simon Glass | ed96665 | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 912 | else: |
Simon Glass | 2d48333 | 2018-11-06 16:02:11 -0700 | [diff] [blame] | 913 | is_warning = (self._re_warning.match(line) or |
| 914 | self._re_dtb_warning.match(line)) |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 915 | is_note = self._re_note.match(line) |
| 916 | if is_warning or (last_was_warning and is_note): |
| 917 | if last_func: |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 918 | add_line(warn_lines_summary, warn_lines_boards, |
Simon Glass | f4ed470 | 2022-07-11 19:03:57 -0600 | [diff] [blame] | 919 | last_func, brd) |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 920 | add_line(warn_lines_summary, warn_lines_boards, |
Simon Glass | f4ed470 | 2022-07-11 19:03:57 -0600 | [diff] [blame] | 921 | line, brd) |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 922 | else: |
| 923 | if last_func: |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 924 | add_line(err_lines_summary, err_lines_boards, |
Simon Glass | f4ed470 | 2022-07-11 19:03:57 -0600 | [diff] [blame] | 925 | last_func, brd) |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 926 | add_line(err_lines_summary, err_lines_boards, |
Simon Glass | f4ed470 | 2022-07-11 19:03:57 -0600 | [diff] [blame] | 927 | line, brd) |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 928 | last_was_warning = is_warning |
| 929 | last_func = None |
Simon Glass | f4ed470 | 2022-07-11 19:03:57 -0600 | [diff] [blame] | 930 | tconfig = Config(self.config_filenames, brd.target) |
Simon Glass | b464f8e | 2016-11-13 14:25:53 -0700 | [diff] [blame] | 931 | for fname in self.config_filenames: |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 932 | if outcome.config: |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 933 | for key, value in outcome.config[fname].items(): |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 934 | tconfig.add(fname, key, value) |
Simon Glass | f4ed470 | 2022-07-11 19:03:57 -0600 | [diff] [blame] | 935 | config[brd.target] = tconfig |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 936 | |
Simon Glass | f4ed470 | 2022-07-11 19:03:57 -0600 | [diff] [blame] | 937 | tenvironment = Environment(brd.target) |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 938 | if outcome.environment: |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 939 | for key, value in outcome.environment.items(): |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 940 | tenvironment.add(key, value) |
Simon Glass | f4ed470 | 2022-07-11 19:03:57 -0600 | [diff] [blame] | 941 | environment[brd.target] = tenvironment |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 942 | |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 943 | return (board_dict, err_lines_summary, err_lines_boards, |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 944 | warn_lines_summary, warn_lines_boards, config, environment) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 945 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 946 | def add_outcome(self, board_dict, arch_list, changes, char, color): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 947 | """Add an output to our list of outcomes for each architecture |
| 948 | |
| 949 | This simple function adds failing boards (changes) to the |
| 950 | relevant architecture string, so we can print the results out |
| 951 | sorted by architecture. |
| 952 | |
| 953 | Args: |
| 954 | board_dict: Dict containing all boards |
| 955 | arch_list: Dict keyed by arch name. Value is a string containing |
| 956 | a list of board names which failed for that arch. |
| 957 | changes: List of boards to add to arch_list |
| 958 | color: terminal.Colour object |
| 959 | """ |
| 960 | done_arch = {} |
| 961 | for target in changes: |
| 962 | if target in board_dict: |
| 963 | arch = board_dict[target].arch |
| 964 | else: |
| 965 | arch = 'unknown' |
Simon Glass | 252ac58 | 2022-01-29 14:14:17 -0700 | [diff] [blame] | 966 | str = self.col.build(color, ' ' + target) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 967 | if not arch in done_arch: |
Simon Glass | 252ac58 | 2022-01-29 14:14:17 -0700 | [diff] [blame] | 968 | str = ' %s %s' % (self.col.build(color, char), str) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 969 | done_arch[arch] = True |
| 970 | if not arch in arch_list: |
| 971 | arch_list[arch] = str |
| 972 | else: |
| 973 | arch_list[arch] += str |
| 974 | |
| 975 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 976 | def colour_num(self, num): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 977 | color = self.col.RED if num > 0 else self.col.GREEN |
| 978 | if num == 0: |
| 979 | return '0' |
Simon Glass | 252ac58 | 2022-01-29 14:14:17 -0700 | [diff] [blame] | 980 | return self.col.build(color, str(num)) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 981 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 982 | def reset_result_summary(self, board_selected): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 983 | """Reset the results summary ready for use. |
| 984 | |
| 985 | Set up the base board list to be all those selected, and set the |
| 986 | error lines to empty. |
| 987 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 988 | Following this, calls to print_result_summary() will use this |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 989 | information to work out what has changed. |
| 990 | |
| 991 | Args: |
| 992 | board_selected: Dict containing boards to summarise, keyed by |
| 993 | board.target |
| 994 | """ |
| 995 | self._base_board_dict = {} |
Simon Glass | f4ed470 | 2022-07-11 19:03:57 -0600 | [diff] [blame] | 996 | for brd in board_selected: |
| 997 | self._base_board_dict[brd] = Builder.Outcome(0, [], [], {}, {}, {}) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 998 | self._base_err_lines = [] |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 999 | self._base_warn_lines = [] |
| 1000 | self._base_err_line_boards = {} |
| 1001 | self._base_warn_line_boards = {} |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1002 | self._base_config = None |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 1003 | self._base_environment = None |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1004 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1005 | def print_func_size_detail(self, fname, old, new): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1006 | grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 |
| 1007 | delta, common = [], {} |
| 1008 | |
| 1009 | for a in old: |
| 1010 | if a in new: |
| 1011 | common[a] = 1 |
| 1012 | |
| 1013 | for name in old: |
| 1014 | if name not in common: |
| 1015 | remove += 1 |
| 1016 | down += old[name] |
| 1017 | delta.append([-old[name], name]) |
| 1018 | |
| 1019 | for name in new: |
| 1020 | if name not in common: |
| 1021 | add += 1 |
| 1022 | up += new[name] |
| 1023 | delta.append([new[name], name]) |
| 1024 | |
| 1025 | for name in common: |
| 1026 | diff = new.get(name, 0) - old.get(name, 0) |
| 1027 | if diff > 0: |
| 1028 | grow, up = grow + 1, up + diff |
| 1029 | elif diff < 0: |
| 1030 | shrink, down = shrink + 1, down - diff |
| 1031 | delta.append([diff, name]) |
| 1032 | |
| 1033 | delta.sort() |
| 1034 | delta.reverse() |
| 1035 | |
| 1036 | args = [add, -remove, grow, -shrink, up, -down, up - down] |
Tom Rini | d5686a6 | 2017-05-22 13:48:52 -0400 | [diff] [blame] | 1037 | if max(args) == 0 and min(args) == 0: |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1038 | return |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1039 | args = [self.colour_num(x) for x in args] |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1040 | indent = ' ' * 15 |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1041 | tprint('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % |
Simon Glass | 252ac58 | 2022-01-29 14:14:17 -0700 | [diff] [blame] | 1042 | tuple([indent, self.col.build(self.col.YELLOW, fname)] + args)) |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1043 | tprint('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', |
Simon Glass | 4653a88 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 1044 | 'delta')) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1045 | for diff, name in delta: |
| 1046 | if diff: |
| 1047 | color = self.col.RED if diff > 0 else self.col.GREEN |
| 1048 | msg = '%s %-38s %7s %7s %+7d' % (indent, name, |
| 1049 | old.get(name, '-'), new.get(name,'-'), diff) |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1050 | tprint(msg, colour=color) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1051 | |
| 1052 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1053 | def print_size_detail(self, target_list, show_bloat): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1054 | """Show details size information for each board |
| 1055 | |
| 1056 | Args: |
| 1057 | target_list: List of targets, each a dict containing: |
| 1058 | 'target': Target name |
| 1059 | 'total_diff': Total difference in bytes across all areas |
| 1060 | <part_name>: Difference for that part |
| 1061 | show_bloat: Show detail for each function |
| 1062 | """ |
| 1063 | targets_by_diff = sorted(target_list, reverse=True, |
| 1064 | key=lambda x: x['_total_diff']) |
| 1065 | for result in targets_by_diff: |
| 1066 | printed_target = False |
| 1067 | for name in sorted(result): |
| 1068 | diff = result[name] |
| 1069 | if name.startswith('_'): |
| 1070 | continue |
| 1071 | if diff != 0: |
| 1072 | color = self.col.RED if diff > 0 else self.col.GREEN |
| 1073 | msg = ' %s %+d' % (name, diff) |
| 1074 | if not printed_target: |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1075 | tprint('%10s %-15s:' % ('', result['_target']), |
Simon Glass | 4653a88 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 1076 | newline=False) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1077 | printed_target = True |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1078 | tprint(msg, colour=color, newline=False) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1079 | if printed_target: |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1080 | tprint() |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1081 | if show_bloat: |
| 1082 | target = result['_target'] |
| 1083 | outcome = result['_outcome'] |
| 1084 | base_outcome = self._base_board_dict[target] |
| 1085 | for fname in outcome.func_sizes: |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1086 | self.print_func_size_detail(fname, |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1087 | base_outcome.func_sizes[fname], |
| 1088 | outcome.func_sizes[fname]) |
| 1089 | |
| 1090 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1091 | def print_size_summary(self, board_selected, board_dict, show_detail, |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1092 | show_bloat): |
| 1093 | """Print a summary of image sizes broken down by section. |
| 1094 | |
| 1095 | The summary takes the form of one line per architecture. The |
| 1096 | line contains deltas for each of the sections (+ means the section |
Flavio Suligoi | 9de5c39 | 2020-01-29 09:56:05 +0100 | [diff] [blame] | 1097 | got bigger, - means smaller). The numbers are the average number |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1098 | of bytes that a board in this section increased by. |
| 1099 | |
| 1100 | For example: |
| 1101 | powerpc: (622 boards) text -0.0 |
| 1102 | arm: (285 boards) text -0.0 |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1103 | |
| 1104 | Args: |
| 1105 | board_selected: Dict containing boards to summarise, keyed by |
| 1106 | board.target |
| 1107 | board_dict: Dict containing boards for which we built this |
| 1108 | commit, keyed by board.target. The value is an Outcome object. |
Simon Glass | f9c094b | 2020-03-18 09:42:43 -0600 | [diff] [blame] | 1109 | show_detail: Show size delta detail for each board |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1110 | show_bloat: Show detail for each function |
| 1111 | """ |
| 1112 | arch_list = {} |
| 1113 | arch_count = {} |
| 1114 | |
| 1115 | # Calculate changes in size for different image parts |
| 1116 | # The previous sizes are in Board.sizes, for each board |
| 1117 | for target in board_dict: |
| 1118 | if target not in board_selected: |
| 1119 | continue |
| 1120 | base_sizes = self._base_board_dict[target].sizes |
| 1121 | outcome = board_dict[target] |
| 1122 | sizes = outcome.sizes |
| 1123 | |
| 1124 | # Loop through the list of images, creating a dict of size |
| 1125 | # changes for each image/part. We end up with something like |
| 1126 | # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} |
| 1127 | # which means that U-Boot data increased by 5 bytes and SPL |
| 1128 | # text decreased by 4. |
| 1129 | err = {'_target' : target} |
| 1130 | for image in sizes: |
| 1131 | if image in base_sizes: |
| 1132 | base_image = base_sizes[image] |
| 1133 | # Loop through the text, data, bss parts |
| 1134 | for part in sorted(sizes[image]): |
| 1135 | diff = sizes[image][part] - base_image[part] |
| 1136 | col = None |
| 1137 | if diff: |
| 1138 | if image == 'u-boot': |
| 1139 | name = part |
| 1140 | else: |
| 1141 | name = image + ':' + part |
| 1142 | err[name] = diff |
| 1143 | arch = board_selected[target].arch |
| 1144 | if not arch in arch_count: |
| 1145 | arch_count[arch] = 1 |
| 1146 | else: |
| 1147 | arch_count[arch] += 1 |
| 1148 | if not sizes: |
| 1149 | pass # Only add to our list when we have some stats |
| 1150 | elif not arch in arch_list: |
| 1151 | arch_list[arch] = [err] |
| 1152 | else: |
| 1153 | arch_list[arch].append(err) |
| 1154 | |
| 1155 | # We now have a list of image size changes sorted by arch |
| 1156 | # Print out a summary of these |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 1157 | for arch, target_list in arch_list.items(): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1158 | # Get total difference for each type |
| 1159 | totals = {} |
| 1160 | for result in target_list: |
| 1161 | total = 0 |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 1162 | for name, diff in result.items(): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1163 | if name.startswith('_'): |
| 1164 | continue |
| 1165 | total += diff |
| 1166 | if name in totals: |
| 1167 | totals[name] += diff |
| 1168 | else: |
| 1169 | totals[name] = diff |
| 1170 | result['_total_diff'] = total |
| 1171 | result['_outcome'] = board_dict[result['_target']] |
| 1172 | |
| 1173 | count = len(target_list) |
| 1174 | printed_arch = False |
| 1175 | for name in sorted(totals): |
| 1176 | diff = totals[name] |
| 1177 | if diff: |
| 1178 | # Display the average difference in this name for this |
| 1179 | # architecture |
| 1180 | avg_diff = float(diff) / count |
| 1181 | color = self.col.RED if avg_diff > 0 else self.col.GREEN |
| 1182 | msg = ' %s %+1.1f' % (name, avg_diff) |
| 1183 | if not printed_arch: |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1184 | tprint('%10s: (for %d/%d boards)' % (arch, count, |
Simon Glass | 4653a88 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 1185 | arch_count[arch]), newline=False) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1186 | printed_arch = True |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1187 | tprint(msg, colour=color, newline=False) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1188 | |
| 1189 | if printed_arch: |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1190 | tprint() |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1191 | if show_detail: |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1192 | self.print_size_detail(target_list, show_bloat) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1193 | |
| 1194 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1195 | def print_result_summary(self, board_selected, board_dict, err_lines, |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1196 | err_line_boards, warn_lines, warn_line_boards, |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 1197 | config, environment, show_sizes, show_detail, |
| 1198 | show_bloat, show_config, show_environment): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1199 | """Compare results with the base results and display delta. |
| 1200 | |
| 1201 | Only boards mentioned in board_selected will be considered. This |
| 1202 | function is intended to be called repeatedly with the results of |
| 1203 | each commit. It therefore shows a 'diff' between what it saw in |
| 1204 | the last call and what it sees now. |
| 1205 | |
| 1206 | Args: |
| 1207 | board_selected: Dict containing boards to summarise, keyed by |
| 1208 | board.target |
| 1209 | board_dict: Dict containing boards for which we built this |
| 1210 | commit, keyed by board.target. The value is an Outcome object. |
| 1211 | err_lines: A list of errors for this commit, or [] if there is |
| 1212 | none, or we don't want to print errors |
Simon Glass | ed96665 | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 1213 | err_line_boards: Dict keyed by error line, containing a list of |
| 1214 | the Board objects with that error |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1215 | warn_lines: A list of warnings for this commit, or [] if there is |
| 1216 | none, or we don't want to print errors |
| 1217 | warn_line_boards: Dict keyed by warning line, containing a list of |
| 1218 | the Board objects with that warning |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 1219 | config: Dictionary keyed by filename - e.g. '.config'. Each |
| 1220 | value is itself a dictionary: |
| 1221 | key: config name |
| 1222 | value: config value |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 1223 | environment: Dictionary keyed by environment variable, Each |
| 1224 | value is the value of environment variable. |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1225 | show_sizes: Show image size deltas |
Simon Glass | f9c094b | 2020-03-18 09:42:43 -0600 | [diff] [blame] | 1226 | show_detail: Show size delta detail for each board if show_sizes |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1227 | show_bloat: Show detail for each function |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 1228 | show_config: Show config changes |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 1229 | show_environment: Show environment changes |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1230 | """ |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1231 | def _board_list(line, line_boards): |
Simon Glass | ed96665 | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 1232 | """Helper function to get a line of boards containing a line |
| 1233 | |
| 1234 | Args: |
| 1235 | line: Error line to search for |
Simon Glass | 35d696d | 2020-04-09 15:08:36 -0600 | [diff] [blame] | 1236 | line_boards: boards to search, each a Board |
Simon Glass | ed96665 | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 1237 | Return: |
Simon Glass | 35d696d | 2020-04-09 15:08:36 -0600 | [diff] [blame] | 1238 | List of boards with that error line, or [] if the user has not |
| 1239 | requested such a list |
Simon Glass | ed96665 | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 1240 | """ |
Simon Glass | cc2c0d1 | 2022-07-11 19:04:00 -0600 | [diff] [blame] | 1241 | brds = [] |
Simon Glass | 35d696d | 2020-04-09 15:08:36 -0600 | [diff] [blame] | 1242 | board_set = set() |
Simon Glass | ed96665 | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 1243 | if self._list_error_boards: |
Simon Glass | f4ed470 | 2022-07-11 19:03:57 -0600 | [diff] [blame] | 1244 | for brd in line_boards[line]: |
| 1245 | if not brd in board_set: |
Simon Glass | cc2c0d1 | 2022-07-11 19:04:00 -0600 | [diff] [blame] | 1246 | brds.append(brd) |
Simon Glass | f4ed470 | 2022-07-11 19:03:57 -0600 | [diff] [blame] | 1247 | board_set.add(brd) |
Simon Glass | cc2c0d1 | 2022-07-11 19:04:00 -0600 | [diff] [blame] | 1248 | return brds |
Simon Glass | ed96665 | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 1249 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1250 | def _calc_error_delta(base_lines, base_line_boards, lines, line_boards, |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1251 | char): |
Simon Glass | 35d696d | 2020-04-09 15:08:36 -0600 | [diff] [blame] | 1252 | """Calculate the required output based on changes in errors |
| 1253 | |
| 1254 | Args: |
| 1255 | base_lines: List of errors/warnings for previous commit |
| 1256 | base_line_boards: Dict keyed by error line, containing a list |
| 1257 | of the Board objects with that error in the previous commit |
| 1258 | lines: List of errors/warning for this commit, each a str |
| 1259 | line_boards: Dict keyed by error line, containing a list |
| 1260 | of the Board objects with that error in this commit |
| 1261 | char: Character representing error ('') or warning ('w'). The |
| 1262 | broken ('+') or fixed ('-') characters are added in this |
| 1263 | function |
| 1264 | |
| 1265 | Returns: |
| 1266 | Tuple |
| 1267 | List of ErrLine objects for 'better' lines |
| 1268 | List of ErrLine objects for 'worse' lines |
| 1269 | """ |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1270 | better_lines = [] |
| 1271 | worse_lines = [] |
| 1272 | for line in lines: |
| 1273 | if line not in base_lines: |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1274 | errline = ErrLine(char + '+', _board_list(line, line_boards), |
Simon Glass | 35d696d | 2020-04-09 15:08:36 -0600 | [diff] [blame] | 1275 | line) |
| 1276 | worse_lines.append(errline) |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1277 | for line in base_lines: |
| 1278 | if line not in lines: |
Simon Glass | 35d696d | 2020-04-09 15:08:36 -0600 | [diff] [blame] | 1279 | errline = ErrLine(char + '-', |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1280 | _board_list(line, base_line_boards), line) |
Simon Glass | 35d696d | 2020-04-09 15:08:36 -0600 | [diff] [blame] | 1281 | better_lines.append(errline) |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1282 | return better_lines, worse_lines |
| 1283 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1284 | def _calc_config(delta, name, config): |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 1285 | """Calculate configuration changes |
| 1286 | |
| 1287 | Args: |
| 1288 | delta: Type of the delta, e.g. '+' |
| 1289 | name: name of the file which changed (e.g. .config) |
| 1290 | config: configuration change dictionary |
| 1291 | key: config name |
| 1292 | value: config value |
| 1293 | Returns: |
| 1294 | String containing the configuration changes which can be |
| 1295 | printed |
| 1296 | """ |
| 1297 | out = '' |
| 1298 | for key in sorted(config.keys()): |
| 1299 | out += '%s=%s ' % (key, config[key]) |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1300 | return '%s %s: %s' % (delta, name, out) |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 1301 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1302 | def _add_config(lines, name, config_plus, config_minus, config_change): |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1303 | """Add changes in configuration to a list |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 1304 | |
| 1305 | Args: |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1306 | lines: list to add to |
| 1307 | name: config file name |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 1308 | config_plus: configurations added, dictionary |
| 1309 | key: config name |
| 1310 | value: config value |
| 1311 | config_minus: configurations removed, dictionary |
| 1312 | key: config name |
| 1313 | value: config value |
| 1314 | config_change: configurations changed, dictionary |
| 1315 | key: config name |
| 1316 | value: config value |
| 1317 | """ |
| 1318 | if config_plus: |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1319 | lines.append(_calc_config('+', name, config_plus)) |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 1320 | if config_minus: |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1321 | lines.append(_calc_config('-', name, config_minus)) |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 1322 | if config_change: |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1323 | lines.append(_calc_config('c', name, config_change)) |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1324 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1325 | def _output_config_info(lines): |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1326 | for line in lines: |
| 1327 | if not line: |
| 1328 | continue |
| 1329 | if line[0] == '+': |
| 1330 | col = self.col.GREEN |
| 1331 | elif line[0] == '-': |
| 1332 | col = self.col.RED |
| 1333 | elif line[0] == 'c': |
| 1334 | col = self.col.YELLOW |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1335 | tprint(' ' + line, newline=True, colour=col) |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1336 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1337 | def _output_err_lines(err_lines, colour): |
Simon Glass | b206d87 | 2020-04-09 15:08:28 -0600 | [diff] [blame] | 1338 | """Output the line of error/warning lines, if not empty |
| 1339 | |
| 1340 | Also increments self._error_lines if err_lines not empty |
| 1341 | |
| 1342 | Args: |
Simon Glass | 35d696d | 2020-04-09 15:08:36 -0600 | [diff] [blame] | 1343 | err_lines: List of ErrLine objects, each an error or warning |
| 1344 | line, possibly including a list of boards with that |
| 1345 | error/warning |
Simon Glass | b206d87 | 2020-04-09 15:08:28 -0600 | [diff] [blame] | 1346 | colour: Colour to use for output |
| 1347 | """ |
| 1348 | if err_lines: |
Simon Glass | 8c9a267 | 2020-04-09 15:08:37 -0600 | [diff] [blame] | 1349 | out_list = [] |
Simon Glass | 35d696d | 2020-04-09 15:08:36 -0600 | [diff] [blame] | 1350 | for line in err_lines: |
Simon Glass | cc2c0d1 | 2022-07-11 19:04:00 -0600 | [diff] [blame] | 1351 | names = [brd.target for brd in line.brds] |
Simon Glass | 9ef0ceb | 2020-04-09 15:08:38 -0600 | [diff] [blame] | 1352 | board_str = ' '.join(names) if names else '' |
Simon Glass | 8c9a267 | 2020-04-09 15:08:37 -0600 | [diff] [blame] | 1353 | if board_str: |
Simon Glass | 252ac58 | 2022-01-29 14:14:17 -0700 | [diff] [blame] | 1354 | out = self.col.build(colour, line.char + '(') |
| 1355 | out += self.col.build(self.col.MAGENTA, board_str, |
Simon Glass | 8c9a267 | 2020-04-09 15:08:37 -0600 | [diff] [blame] | 1356 | bright=False) |
Simon Glass | 252ac58 | 2022-01-29 14:14:17 -0700 | [diff] [blame] | 1357 | out += self.col.build(colour, ') %s' % line.errline) |
Simon Glass | 8c9a267 | 2020-04-09 15:08:37 -0600 | [diff] [blame] | 1358 | else: |
Simon Glass | 252ac58 | 2022-01-29 14:14:17 -0700 | [diff] [blame] | 1359 | out = self.col.build(colour, line.char + line.errline) |
Simon Glass | 8c9a267 | 2020-04-09 15:08:37 -0600 | [diff] [blame] | 1360 | out_list.append(out) |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1361 | tprint('\n'.join(out_list)) |
Simon Glass | b206d87 | 2020-04-09 15:08:28 -0600 | [diff] [blame] | 1362 | self._error_lines += 1 |
| 1363 | |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 1364 | |
Simon Glass | 4cf2b22 | 2018-11-06 16:02:12 -0700 | [diff] [blame] | 1365 | ok_boards = [] # List of boards fixed since last commit |
Simon Glass | 6af7101 | 2018-11-06 16:02:13 -0700 | [diff] [blame] | 1366 | warn_boards = [] # List of boards with warnings since last commit |
Simon Glass | 4cf2b22 | 2018-11-06 16:02:12 -0700 | [diff] [blame] | 1367 | err_boards = [] # List of new broken boards since last commit |
| 1368 | new_boards = [] # List of boards that didn't exist last time |
| 1369 | unknown_boards = [] # List of boards that were not built |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1370 | |
| 1371 | for target in board_dict: |
| 1372 | if target not in board_selected: |
| 1373 | continue |
| 1374 | |
| 1375 | # If the board was built last time, add its outcome to a list |
| 1376 | if target in self._base_board_dict: |
| 1377 | base_outcome = self._base_board_dict[target].rc |
| 1378 | outcome = board_dict[target] |
| 1379 | if outcome.rc == OUTCOME_UNKNOWN: |
Simon Glass | 4cf2b22 | 2018-11-06 16:02:12 -0700 | [diff] [blame] | 1380 | unknown_boards.append(target) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1381 | elif outcome.rc < base_outcome: |
Simon Glass | 6af7101 | 2018-11-06 16:02:13 -0700 | [diff] [blame] | 1382 | if outcome.rc == OUTCOME_WARNING: |
| 1383 | warn_boards.append(target) |
| 1384 | else: |
| 1385 | ok_boards.append(target) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1386 | elif outcome.rc > base_outcome: |
Simon Glass | 6af7101 | 2018-11-06 16:02:13 -0700 | [diff] [blame] | 1387 | if outcome.rc == OUTCOME_WARNING: |
| 1388 | warn_boards.append(target) |
| 1389 | else: |
| 1390 | err_boards.append(target) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1391 | else: |
Simon Glass | 4cf2b22 | 2018-11-06 16:02:12 -0700 | [diff] [blame] | 1392 | new_boards.append(target) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1393 | |
Simon Glass | b206d87 | 2020-04-09 15:08:28 -0600 | [diff] [blame] | 1394 | # Get a list of errors and warnings that have appeared, and disappeared |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1395 | better_err, worse_err = _calc_error_delta(self._base_err_lines, |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1396 | self._base_err_line_boards, err_lines, err_line_boards, '') |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1397 | better_warn, worse_warn = _calc_error_delta(self._base_warn_lines, |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1398 | self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1399 | |
Simon Glass | ae1a09f | 2022-07-11 19:03:56 -0600 | [diff] [blame] | 1400 | # For the IDE mode, print out all the output |
| 1401 | if self._ide: |
| 1402 | outcome = board_dict[target] |
| 1403 | for line in outcome.err_lines: |
| 1404 | sys.stderr.write(line) |
| 1405 | |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1406 | # Display results by arch |
Simon Glass | ae1a09f | 2022-07-11 19:03:56 -0600 | [diff] [blame] | 1407 | elif any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards, |
Simon Glass | 6af7101 | 2018-11-06 16:02:13 -0700 | [diff] [blame] | 1408 | worse_err, better_err, worse_warn, better_warn)): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1409 | arch_list = {} |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1410 | self.add_outcome(board_selected, arch_list, ok_boards, '', |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1411 | self.col.GREEN) |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1412 | self.add_outcome(board_selected, arch_list, warn_boards, 'w+', |
Simon Glass | 6af7101 | 2018-11-06 16:02:13 -0700 | [diff] [blame] | 1413 | self.col.YELLOW) |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1414 | self.add_outcome(board_selected, arch_list, err_boards, '+', |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1415 | self.col.RED) |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1416 | self.add_outcome(board_selected, arch_list, new_boards, '*', self.col.BLUE) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1417 | if self._show_unknown: |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1418 | self.add_outcome(board_selected, arch_list, unknown_boards, '?', |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1419 | self.col.MAGENTA) |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 1420 | for arch, target_list in arch_list.items(): |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1421 | tprint('%10s: %s' % (arch, target_list)) |
Simon Glass | 28370c1 | 2014-08-09 15:33:06 -0600 | [diff] [blame] | 1422 | self._error_lines += 1 |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1423 | _output_err_lines(better_err, colour=self.col.GREEN) |
| 1424 | _output_err_lines(worse_err, colour=self.col.RED) |
| 1425 | _output_err_lines(better_warn, colour=self.col.CYAN) |
| 1426 | _output_err_lines(worse_warn, colour=self.col.YELLOW) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1427 | |
| 1428 | if show_sizes: |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1429 | self.print_size_summary(board_selected, board_dict, show_detail, |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1430 | show_bloat) |
| 1431 | |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 1432 | if show_environment and self._base_environment: |
| 1433 | lines = [] |
| 1434 | |
| 1435 | for target in board_dict: |
| 1436 | if target not in board_selected: |
| 1437 | continue |
| 1438 | |
| 1439 | tbase = self._base_environment[target] |
| 1440 | tenvironment = environment[target] |
| 1441 | environment_plus = {} |
| 1442 | environment_minus = {} |
| 1443 | environment_change = {} |
| 1444 | base = tbase.environment |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 1445 | for key, value in tenvironment.environment.items(): |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 1446 | if key not in base: |
| 1447 | environment_plus[key] = value |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 1448 | for key, value in base.items(): |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 1449 | if key not in tenvironment.environment: |
| 1450 | environment_minus[key] = value |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 1451 | for key, value in base.items(): |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 1452 | new_value = tenvironment.environment.get(key) |
| 1453 | if new_value and value != new_value: |
| 1454 | desc = '%s -> %s' % (value, new_value) |
| 1455 | environment_change[key] = desc |
| 1456 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1457 | _add_config(lines, target, environment_plus, environment_minus, |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 1458 | environment_change) |
| 1459 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1460 | _output_config_info(lines) |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 1461 | |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1462 | if show_config and self._base_config: |
| 1463 | summary = {} |
| 1464 | arch_config_plus = {} |
| 1465 | arch_config_minus = {} |
| 1466 | arch_config_change = {} |
| 1467 | arch_list = [] |
| 1468 | |
| 1469 | for target in board_dict: |
| 1470 | if target not in board_selected: |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 1471 | continue |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1472 | arch = board_selected[target].arch |
| 1473 | if arch not in arch_list: |
| 1474 | arch_list.append(arch) |
| 1475 | |
| 1476 | for arch in arch_list: |
| 1477 | arch_config_plus[arch] = {} |
| 1478 | arch_config_minus[arch] = {} |
| 1479 | arch_config_change[arch] = {} |
Simon Glass | b464f8e | 2016-11-13 14:25:53 -0700 | [diff] [blame] | 1480 | for name in self.config_filenames: |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1481 | arch_config_plus[arch][name] = {} |
| 1482 | arch_config_minus[arch][name] = {} |
| 1483 | arch_config_change[arch][name] = {} |
| 1484 | |
| 1485 | for target in board_dict: |
| 1486 | if target not in board_selected: |
| 1487 | continue |
| 1488 | |
| 1489 | arch = board_selected[target].arch |
| 1490 | |
| 1491 | all_config_plus = {} |
| 1492 | all_config_minus = {} |
| 1493 | all_config_change = {} |
| 1494 | tbase = self._base_config[target] |
| 1495 | tconfig = config[target] |
| 1496 | lines = [] |
Simon Glass | b464f8e | 2016-11-13 14:25:53 -0700 | [diff] [blame] | 1497 | for name in self.config_filenames: |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1498 | if not tconfig.config[name]: |
| 1499 | continue |
| 1500 | config_plus = {} |
| 1501 | config_minus = {} |
| 1502 | config_change = {} |
| 1503 | base = tbase.config[name] |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 1504 | for key, value in tconfig.config[name].items(): |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1505 | if key not in base: |
| 1506 | config_plus[key] = value |
| 1507 | all_config_plus[key] = value |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 1508 | for key, value in base.items(): |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1509 | if key not in tconfig.config[name]: |
| 1510 | config_minus[key] = value |
| 1511 | all_config_minus[key] = value |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 1512 | for key, value in base.items(): |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1513 | new_value = tconfig.config.get(key) |
| 1514 | if new_value and value != new_value: |
| 1515 | desc = '%s -> %s' % (value, new_value) |
| 1516 | config_change[key] = desc |
| 1517 | all_config_change[key] = desc |
| 1518 | |
| 1519 | arch_config_plus[arch][name].update(config_plus) |
| 1520 | arch_config_minus[arch][name].update(config_minus) |
| 1521 | arch_config_change[arch][name].update(config_change) |
| 1522 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1523 | _add_config(lines, name, config_plus, config_minus, |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1524 | config_change) |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1525 | _add_config(lines, 'all', all_config_plus, all_config_minus, |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1526 | all_config_change) |
| 1527 | summary[target] = '\n'.join(lines) |
| 1528 | |
| 1529 | lines_by_target = {} |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 1530 | for target, lines in summary.items(): |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1531 | if lines in lines_by_target: |
| 1532 | lines_by_target[lines].append(target) |
| 1533 | else: |
| 1534 | lines_by_target[lines] = [target] |
| 1535 | |
| 1536 | for arch in arch_list: |
| 1537 | lines = [] |
| 1538 | all_plus = {} |
| 1539 | all_minus = {} |
| 1540 | all_change = {} |
Simon Glass | b464f8e | 2016-11-13 14:25:53 -0700 | [diff] [blame] | 1541 | for name in self.config_filenames: |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1542 | all_plus.update(arch_config_plus[arch][name]) |
| 1543 | all_minus.update(arch_config_minus[arch][name]) |
| 1544 | all_change.update(arch_config_change[arch][name]) |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1545 | _add_config(lines, name, arch_config_plus[arch][name], |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1546 | arch_config_minus[arch][name], |
| 1547 | arch_config_change[arch][name]) |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1548 | _add_config(lines, 'all', all_plus, all_minus, all_change) |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1549 | #arch_summary[target] = '\n'.join(lines) |
| 1550 | if lines: |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1551 | tprint('%s:' % arch) |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1552 | _output_config_info(lines) |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1553 | |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 1554 | for lines, targets in lines_by_target.items(): |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1555 | if not lines: |
| 1556 | continue |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1557 | tprint('%s :' % ' '.join(sorted(targets))) |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1558 | _output_config_info(lines.split('\n')) |
Simon Glass | 8270e3c | 2015-08-25 21:52:14 -0600 | [diff] [blame] | 1559 | |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 1560 | |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1561 | # Save our updated information for the next call to this function |
| 1562 | self._base_board_dict = board_dict |
| 1563 | self._base_err_lines = err_lines |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1564 | self._base_warn_lines = warn_lines |
| 1565 | self._base_err_line_boards = err_line_boards |
| 1566 | self._base_warn_line_boards = warn_line_boards |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 1567 | self._base_config = config |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 1568 | self._base_environment = environment |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1569 | |
| 1570 | # Get a list of boards that did not get built, if needed |
| 1571 | not_built = [] |
Simon Glass | f4ed470 | 2022-07-11 19:03:57 -0600 | [diff] [blame] | 1572 | for brd in board_selected: |
| 1573 | if not brd in board_dict: |
| 1574 | not_built.append(brd) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1575 | if not_built: |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1576 | tprint("Boards not built (%d): %s" % (len(not_built), |
Simon Glass | 4653a88 | 2014-09-05 19:00:07 -0600 | [diff] [blame] | 1577 | ', '.join(not_built))) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1578 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1579 | def produce_result_summary(self, commit_upto, commits, board_selected): |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1580 | (board_dict, err_lines, err_line_boards, warn_lines, |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1581 | warn_line_boards, config, environment) = self.get_result_summary( |
Simon Glass | ed96665 | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 1582 | board_selected, commit_upto, |
Simon Glass | 843312d | 2015-02-05 22:06:15 -0700 | [diff] [blame] | 1583 | read_func_sizes=self._show_bloat, |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 1584 | read_config=self._show_config, |
| 1585 | read_environment=self._show_environment) |
Simon Glass | b2ea7ab | 2014-08-09 15:33:02 -0600 | [diff] [blame] | 1586 | if commits: |
| 1587 | msg = '%02d: %s' % (commit_upto + 1, |
| 1588 | commits[commit_upto].subject) |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1589 | tprint(msg, colour=self.col.BLUE) |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1590 | self.print_result_summary(board_selected, board_dict, |
Simon Glass | ed96665 | 2014-08-28 09:43:43 -0600 | [diff] [blame] | 1591 | err_lines if self._show_errors else [], err_line_boards, |
Simon Glass | e30965d | 2014-08-28 09:43:44 -0600 | [diff] [blame] | 1592 | warn_lines if self._show_errors else [], warn_line_boards, |
Alex Kiernan | 48ae412 | 2018-05-31 04:48:34 +0000 | [diff] [blame] | 1593 | config, environment, self._show_sizes, self._show_detail, |
| 1594 | self._show_bloat, self._show_config, self._show_environment) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1595 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1596 | def show_summary(self, commits, board_selected): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1597 | """Show a build summary for U-Boot for a given board list. |
| 1598 | |
| 1599 | Reset the result summary, then repeatedly call GetResultSummary on |
| 1600 | each commit's results, then display the differences we see. |
| 1601 | |
| 1602 | Args: |
| 1603 | commit: Commit objects to summarise |
| 1604 | board_selected: Dict containing boards to summarise |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1605 | """ |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1606 | self.commit_count = len(commits) if commits else 1 |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1607 | self.commits = commits |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1608 | self.reset_result_summary(board_selected) |
Simon Glass | 28370c1 | 2014-08-09 15:33:06 -0600 | [diff] [blame] | 1609 | self._error_lines = 0 |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1610 | |
| 1611 | for commit_upto in range(0, self.commit_count, self._step): |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1612 | self.produce_result_summary(commit_upto, commits, board_selected) |
Simon Glass | 28370c1 | 2014-08-09 15:33:06 -0600 | [diff] [blame] | 1613 | if not self._error_lines: |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1614 | tprint('(no errors to report)', colour=self.col.GREEN) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1615 | |
| 1616 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1617 | def setup_build(self, board_selected, commits): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1618 | """Set up ready to start a build. |
| 1619 | |
| 1620 | Args: |
| 1621 | board_selected: Selected boards to build |
| 1622 | commits: Selected commits to build |
| 1623 | """ |
| 1624 | # First work out how many commits we will build |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 1625 | count = (self.commit_count + self._step - 1) // self._step |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1626 | self.count = len(board_selected) * count |
| 1627 | self.upto = self.warned = self.fail = 0 |
| 1628 | self._timestamps = collections.deque() |
| 1629 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1630 | def get_thread_dir(self, thread_num): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1631 | """Get the directory path to the working dir for a thread. |
| 1632 | |
| 1633 | Args: |
Simon Glass | b82492b | 2021-01-30 22:17:46 -0700 | [diff] [blame] | 1634 | thread_num: Number of thread to check (-1 for main process, which |
| 1635 | is treated as 0) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1636 | """ |
Simon Glass | d829f12 | 2020-03-18 09:42:42 -0600 | [diff] [blame] | 1637 | if self.work_in_output: |
| 1638 | return self._working_dir |
Simon Glass | b82492b | 2021-01-30 22:17:46 -0700 | [diff] [blame] | 1639 | return os.path.join(self._working_dir, '%02d' % max(thread_num, 0)) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1640 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1641 | def _prepare_thread(self, thread_num, setup_git): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1642 | """Prepare the working directory for a thread. |
| 1643 | |
| 1644 | This clones or fetches the repo into the thread's work directory. |
Alper Nebi Yasak | 76de29f | 2020-09-03 15:51:03 +0300 | [diff] [blame] | 1645 | Optionally, it can create a linked working tree of the repo in the |
| 1646 | thread's work directory instead. |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1647 | |
| 1648 | Args: |
| 1649 | thread_num: Thread number (0, 1, ...) |
Alper Nebi Yasak | 76de29f | 2020-09-03 15:51:03 +0300 | [diff] [blame] | 1650 | setup_git: |
| 1651 | 'clone' to set up a git clone |
| 1652 | 'worktree' to set up a git worktree |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1653 | """ |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1654 | thread_dir = self.get_thread_dir(thread_num) |
Simon Glass | f06d333 | 2023-07-19 17:49:08 -0600 | [diff] [blame] | 1655 | builderthread.mkdir(thread_dir) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1656 | git_dir = os.path.join(thread_dir, '.git') |
| 1657 | |
Alper Nebi Yasak | 76de29f | 2020-09-03 15:51:03 +0300 | [diff] [blame] | 1658 | # Create a worktree or a git repo clone for this thread if it |
| 1659 | # doesn't already exist |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1660 | if setup_git and self.git_dir: |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1661 | src_dir = os.path.abspath(self.git_dir) |
Alper Nebi Yasak | 76de29f | 2020-09-03 15:51:03 +0300 | [diff] [blame] | 1662 | if os.path.isdir(git_dir): |
| 1663 | # This is a clone of the src_dir repo, we can keep using |
| 1664 | # it but need to fetch from src_dir. |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1665 | tprint('\rFetching repo for thread %d' % thread_num, |
Simon Glass | 212c0b8 | 2020-04-09 15:08:43 -0600 | [diff] [blame] | 1666 | newline=False) |
Simon Glass | 0157b18 | 2022-01-29 14:14:11 -0700 | [diff] [blame] | 1667 | gitutil.fetch(git_dir, thread_dir) |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1668 | terminal.print_clear() |
Alper Nebi Yasak | 76de29f | 2020-09-03 15:51:03 +0300 | [diff] [blame] | 1669 | elif os.path.isfile(git_dir): |
| 1670 | # This is a worktree of the src_dir repo, we don't need to |
| 1671 | # create it again or update it in any way. |
| 1672 | pass |
| 1673 | elif os.path.exists(git_dir): |
| 1674 | # Don't know what could trigger this, but we probably |
| 1675 | # can't create a git worktree/clone here. |
| 1676 | raise ValueError('Git dir %s exists, but is not a file ' |
| 1677 | 'or a directory.' % git_dir) |
| 1678 | elif setup_git == 'worktree': |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1679 | tprint('\rChecking out worktree for thread %d' % thread_num, |
Alper Nebi Yasak | 76de29f | 2020-09-03 15:51:03 +0300 | [diff] [blame] | 1680 | newline=False) |
Simon Glass | 0157b18 | 2022-01-29 14:14:11 -0700 | [diff] [blame] | 1681 | gitutil.add_worktree(src_dir, thread_dir) |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1682 | terminal.print_clear() |
Alper Nebi Yasak | 76de29f | 2020-09-03 15:51:03 +0300 | [diff] [blame] | 1683 | elif setup_git == 'clone' or setup_git == True: |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1684 | tprint('\rCloning repo for thread %d' % thread_num, |
Simon Glass | 21f0eb3 | 2016-09-18 16:48:31 -0600 | [diff] [blame] | 1685 | newline=False) |
Simon Glass | 0157b18 | 2022-01-29 14:14:11 -0700 | [diff] [blame] | 1686 | gitutil.clone(src_dir, thread_dir) |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1687 | terminal.print_clear() |
Alper Nebi Yasak | 76de29f | 2020-09-03 15:51:03 +0300 | [diff] [blame] | 1688 | else: |
| 1689 | raise ValueError("Can't setup git repo with %s." % setup_git) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1690 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1691 | def _prepare_working_space(self, max_threads, setup_git): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1692 | """Prepare the working directory for use. |
| 1693 | |
Alper Nebi Yasak | 76de29f | 2020-09-03 15:51:03 +0300 | [diff] [blame] | 1694 | Set up the git repo for each thread. Creates a linked working tree |
| 1695 | if git-worktree is available, or clones the repo if it isn't. |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1696 | |
| 1697 | Args: |
Simon Glass | b82492b | 2021-01-30 22:17:46 -0700 | [diff] [blame] | 1698 | max_threads: Maximum number of threads we expect to need. If 0 then |
| 1699 | 1 is set up, since the main process still needs somewhere to |
| 1700 | work |
Alper Nebi Yasak | 76de29f | 2020-09-03 15:51:03 +0300 | [diff] [blame] | 1701 | setup_git: True to set up a git worktree or a git clone |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1702 | """ |
Simon Glass | f06d333 | 2023-07-19 17:49:08 -0600 | [diff] [blame] | 1703 | builderthread.mkdir(self._working_dir) |
Alper Nebi Yasak | 76de29f | 2020-09-03 15:51:03 +0300 | [diff] [blame] | 1704 | if setup_git and self.git_dir: |
| 1705 | src_dir = os.path.abspath(self.git_dir) |
Simon Glass | 0157b18 | 2022-01-29 14:14:11 -0700 | [diff] [blame] | 1706 | if gitutil.check_worktree_is_available(src_dir): |
Alper Nebi Yasak | 76de29f | 2020-09-03 15:51:03 +0300 | [diff] [blame] | 1707 | setup_git = 'worktree' |
| 1708 | # If we previously added a worktree but the directory for it |
| 1709 | # got deleted, we need to prune its files from the repo so |
| 1710 | # that we can check out another in its place. |
Simon Glass | 0157b18 | 2022-01-29 14:14:11 -0700 | [diff] [blame] | 1711 | gitutil.prune_worktrees(src_dir) |
Alper Nebi Yasak | 76de29f | 2020-09-03 15:51:03 +0300 | [diff] [blame] | 1712 | else: |
| 1713 | setup_git = 'clone' |
Simon Glass | b82492b | 2021-01-30 22:17:46 -0700 | [diff] [blame] | 1714 | |
| 1715 | # Always do at least one thread |
| 1716 | for thread in range(max(max_threads, 1)): |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1717 | self._prepare_thread(thread, setup_git) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1718 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1719 | def _get_output_space_removals(self): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1720 | """Get the output directories ready to receive files. |
| 1721 | |
Simon Glass | 925f6ad | 2020-03-18 09:42:45 -0600 | [diff] [blame] | 1722 | Figure out what needs to be deleted in the output directory before it |
| 1723 | can be used. We only delete old buildman directories which have the |
Simon Glass | 4a7419b | 2023-07-19 17:49:10 -0600 | [diff] [blame] | 1724 | expected name pattern. See get_output_dir(). |
Simon Glass | 925f6ad | 2020-03-18 09:42:45 -0600 | [diff] [blame] | 1725 | |
| 1726 | Returns: |
| 1727 | List of full paths of directories to remove |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1728 | """ |
Simon Glass | 1a91567 | 2014-12-01 17:33:53 -0700 | [diff] [blame] | 1729 | if not self.commits: |
| 1730 | return |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1731 | dir_list = [] |
| 1732 | for commit_upto in range(self.commit_count): |
Simon Glass | 4a7419b | 2023-07-19 17:49:10 -0600 | [diff] [blame] | 1733 | dir_list.append(self.get_output_dir(commit_upto)) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1734 | |
Simon Glass | b222abe | 2016-09-18 16:48:32 -0600 | [diff] [blame] | 1735 | to_remove = [] |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1736 | for dirname in glob.glob(os.path.join(self.base_dir, '*')): |
| 1737 | if dirname not in dir_list: |
Simon Glass | 925f6ad | 2020-03-18 09:42:45 -0600 | [diff] [blame] | 1738 | leaf = dirname[len(self.base_dir) + 1:] |
Ovidiu Panait | 7664b03 | 2020-05-15 09:30:12 +0300 | [diff] [blame] | 1739 | m = re.match('[0-9]+_g[0-9a-f]+_.*', leaf) |
Simon Glass | 925f6ad | 2020-03-18 09:42:45 -0600 | [diff] [blame] | 1740 | if m: |
| 1741 | to_remove.append(dirname) |
| 1742 | return to_remove |
| 1743 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1744 | def _prepare_output_space(self): |
Simon Glass | 925f6ad | 2020-03-18 09:42:45 -0600 | [diff] [blame] | 1745 | """Get the output directories ready to receive files. |
| 1746 | |
| 1747 | We delete any output directories which look like ones we need to |
| 1748 | create. Having left over directories is confusing when the user wants |
| 1749 | to check the output manually. |
| 1750 | """ |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1751 | to_remove = self._get_output_space_removals() |
Simon Glass | b222abe | 2016-09-18 16:48:32 -0600 | [diff] [blame] | 1752 | if to_remove: |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1753 | tprint('Removing %d old build directories...' % len(to_remove), |
Simon Glass | b222abe | 2016-09-18 16:48:32 -0600 | [diff] [blame] | 1754 | newline=False) |
| 1755 | for dirname in to_remove: |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1756 | shutil.rmtree(dirname) |
Simon Glass | 098b10f | 2022-01-29 14:14:18 -0700 | [diff] [blame] | 1757 | terminal.print_clear() |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1758 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1759 | def build_boards(self, commits, board_selected, keep_outputs, verbose): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1760 | """Build all commits for a list of boards |
| 1761 | |
| 1762 | Args: |
| 1763 | commits: List of commits to be build, each a Commit object |
| 1764 | boards_selected: Dict of selected boards, key is target name, |
| 1765 | value is Board object |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1766 | keep_outputs: True to save build output files |
Simon Glass | e5a0e5d | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 1767 | verbose: Display build results as they are completed |
Simon Glass | 2c3deb9 | 2014-08-28 09:43:39 -0600 | [diff] [blame] | 1768 | Returns: |
| 1769 | Tuple containing: |
| 1770 | - number of boards that failed to build |
| 1771 | - number of boards that issued warnings |
Simon Glass | 8116c78 | 2021-04-11 16:27:27 +1200 | [diff] [blame] | 1772 | - list of thread exceptions raised |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1773 | """ |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1774 | self.commit_count = len(commits) if commits else 1 |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1775 | self.commits = commits |
Simon Glass | e5a0e5d | 2014-08-09 15:33:03 -0600 | [diff] [blame] | 1776 | self._verbose = verbose |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1777 | |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1778 | self.reset_result_summary(board_selected) |
Simon Glass | f06d333 | 2023-07-19 17:49:08 -0600 | [diff] [blame] | 1779 | builderthread.mkdir(self.base_dir, parents = True) |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1780 | self._prepare_working_space(min(self.num_threads, len(board_selected)), |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 1781 | commits is not None) |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1782 | self._prepare_output_space() |
Simon Glass | ae1a09f | 2022-07-11 19:03:56 -0600 | [diff] [blame] | 1783 | if not self._ide: |
| 1784 | tprint('\rStarting build...', newline=False) |
Simon Glass | 2ce06f5 | 2023-09-07 10:00:19 -0600 | [diff] [blame] | 1785 | self._start_time = datetime.now() |
Simon Glass | 37edf5f | 2023-07-19 17:49:06 -0600 | [diff] [blame] | 1786 | self.setup_build(board_selected, commits) |
| 1787 | self.process_result(None) |
Simon Glass | 8116c78 | 2021-04-11 16:27:27 +1200 | [diff] [blame] | 1788 | self.thread_exceptions = [] |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1789 | # Create jobs to build all commits for each board |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 1790 | for brd in board_selected.values(): |
Simon Glass | 190064b | 2014-08-09 15:33:00 -0600 | [diff] [blame] | 1791 | job = builderthread.BuilderJob() |
Simon Glass | f4ed470 | 2022-07-11 19:03:57 -0600 | [diff] [blame] | 1792 | job.brd = brd |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1793 | job.commits = commits |
| 1794 | job.keep_outputs = keep_outputs |
Simon Glass | d829f12 | 2020-03-18 09:42:42 -0600 | [diff] [blame] | 1795 | job.work_in_output = self.work_in_output |
Simon Glass | 2b4806e | 2022-01-22 05:07:33 -0700 | [diff] [blame] | 1796 | job.adjust_cfg = self.adjust_cfg |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1797 | job.step = self._step |
Simon Glass | b82492b | 2021-01-30 22:17:46 -0700 | [diff] [blame] | 1798 | if self.num_threads: |
| 1799 | self.queue.put(job) |
| 1800 | else: |
Simon Glass | f06d333 | 2023-07-19 17:49:08 -0600 | [diff] [blame] | 1801 | self._single_builder.run_job(job) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1802 | |
Simon Glass | b82492b | 2021-01-30 22:17:46 -0700 | [diff] [blame] | 1803 | if self.num_threads: |
| 1804 | term = threading.Thread(target=self.queue.join) |
| 1805 | term.setDaemon(True) |
| 1806 | term.start() |
| 1807 | while term.is_alive(): |
| 1808 | term.join(100) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1809 | |
Simon Glass | b82492b | 2021-01-30 22:17:46 -0700 | [diff] [blame] | 1810 | # Wait until we have processed all output |
| 1811 | self.out_queue.join() |
Simon Glass | ae1a09f | 2022-07-11 19:03:56 -0600 | [diff] [blame] | 1812 | if not self._ide: |
| 1813 | tprint() |
Simon Glass | 7b33f21 | 2020-04-09 15:08:47 -0600 | [diff] [blame] | 1814 | |
Simon Glass | ae1a09f | 2022-07-11 19:03:56 -0600 | [diff] [blame] | 1815 | msg = 'Completed: %d total built' % self.count |
| 1816 | if self.already_done: |
| 1817 | msg += ' (%d previously' % self.already_done |
| 1818 | if self.already_done != self.count: |
| 1819 | msg += ', %d newly' % (self.count - self.already_done) |
| 1820 | msg += ')' |
| 1821 | duration = datetime.now() - self._start_time |
| 1822 | if duration > timedelta(microseconds=1000000): |
| 1823 | if duration.microseconds >= 500000: |
| 1824 | duration = duration + timedelta(seconds=1) |
| 1825 | duration = duration - timedelta(microseconds=duration.microseconds) |
| 1826 | rate = float(self.count) / duration.total_seconds() |
| 1827 | msg += ', duration %s, rate %1.2f' % (duration, rate) |
| 1828 | tprint(msg) |
| 1829 | if self.thread_exceptions: |
| 1830 | tprint('Failed: %d thread exceptions' % len(self.thread_exceptions), |
| 1831 | colour=self.col.RED) |
Simon Glass | 7b33f21 | 2020-04-09 15:08:47 -0600 | [diff] [blame] | 1832 | |
Simon Glass | 8116c78 | 2021-04-11 16:27:27 +1200 | [diff] [blame] | 1833 | return (self.fail, self.warned, self.thread_exceptions) |