Tom Rini | 83d290c | 2018-05-06 17:58:06 -0400 | [diff] [blame] | 1 | # SPDX-License-Identifier: GPL-2.0+ |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 2 | # Copyright (c) 2013 The Chromium OS Authors. |
| 3 | # |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 4 | |
Simon Glass | 9ef05b9 | 2023-07-19 17:48:30 -0600 | [diff] [blame] | 5 | """Control module for buildman |
| 6 | |
| 7 | This holds the main control logic for buildman, when not running tests. |
| 8 | """ |
| 9 | |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 10 | import multiprocessing |
| 11 | import os |
Simon Glass | 883a321 | 2014-09-05 19:00:18 -0600 | [diff] [blame] | 12 | import shutil |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 13 | import sys |
| 14 | |
Simon Glass | c52bd22 | 2022-07-11 19:04:03 -0600 | [diff] [blame] | 15 | from buildman import boards |
Simon Glass | 0ede00f | 2020-04-17 18:09:02 -0600 | [diff] [blame] | 16 | from buildman import bsettings |
Simon Glass | 2b4806e | 2022-01-22 05:07:33 -0700 | [diff] [blame] | 17 | from buildman import cfgutil |
Simon Glass | 0ede00f | 2020-04-17 18:09:02 -0600 | [diff] [blame] | 18 | from buildman import toolchain |
| 19 | from buildman.builder import Builder |
Simon Glass | bf77667 | 2020-04-17 18:09:04 -0600 | [diff] [blame] | 20 | from patman import gitutil |
| 21 | from patman import patchstream |
Simon Glass | 4583c00 | 2023-02-23 18:18:04 -0700 | [diff] [blame] | 22 | from u_boot_pylib import command |
| 23 | from u_boot_pylib import terminal |
Simon Glass | 4583c00 | 2023-02-23 18:18:04 -0700 | [diff] [blame] | 24 | from u_boot_pylib.terminal import tprint |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 25 | |
Simon Glass | b8be2bd | 2023-07-19 17:48:31 -0600 | [diff] [blame] | 26 | TEST_BUILDER = None |
| 27 | |
Simon Glass | 9ef05b9 | 2023-07-19 17:48:30 -0600 | [diff] [blame] | 28 | def get_plural(count): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 29 | """Returns a plural 's' if count is not 1""" |
| 30 | return 's' if count != 1 else '' |
| 31 | |
Simon Glass | 1d3a5a5 | 2023-07-19 17:48:45 -0600 | [diff] [blame] | 32 | def get_action_summary(is_summary, commits, selected, step, threads, jobs): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 33 | """Return a string summarising the intended action. |
| 34 | |
Simon Glass | 1d3a5a5 | 2023-07-19 17:48:45 -0600 | [diff] [blame] | 35 | Args: |
| 36 | is_summary (bool): True if this is a summary (otherwise it is building) |
| 37 | commits (list): List of commits being built |
| 38 | selected (list of Board): List of Board objects that are marked |
| 39 | step (int): Step increment through commits |
| 40 | threads (int): Number of processor threads being used |
| 41 | jobs (int): Number of jobs to build at once |
| 42 | |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 43 | Returns: |
| 44 | Summary string. |
| 45 | """ |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 46 | if commits: |
| 47 | count = len(commits) |
Simon Glass | 1d3a5a5 | 2023-07-19 17:48:45 -0600 | [diff] [blame] | 48 | count = (count + step - 1) // step |
Simon Glass | b8be2bd | 2023-07-19 17:48:31 -0600 | [diff] [blame] | 49 | commit_str = f'{count} commit{get_plural(count)}' |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 50 | else: |
| 51 | commit_str = 'current source' |
Simon Glass | b8be2bd | 2023-07-19 17:48:31 -0600 | [diff] [blame] | 52 | msg = (f"{'Summary of' if is_summary else 'Building'} " |
| 53 | f'{commit_str} for {len(selected)} boards') |
Simon Glass | 1d3a5a5 | 2023-07-19 17:48:45 -0600 | [diff] [blame] | 54 | msg += (f' ({threads} thread{get_plural(threads)}, ' |
| 55 | f'{jobs} job{get_plural(jobs)} per thread)') |
Simon Glass | b8be2bd | 2023-07-19 17:48:31 -0600 | [diff] [blame] | 56 | return msg |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 57 | |
Simon Glass | b8be2bd | 2023-07-19 17:48:31 -0600 | [diff] [blame] | 58 | # pylint: disable=R0913 |
Simon Glass | 1b820ee | 2023-07-19 17:48:46 -0600 | [diff] [blame] | 59 | def show_actions(series, why_selected, boards_selected, output_dir, |
| 60 | board_warnings, step, threads, jobs, verbose): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 61 | """Display a list of actions that we would take, if not a dry run. |
| 62 | |
| 63 | Args: |
| 64 | series: Series object |
| 65 | why_selected: Dictionary where each key is a buildman argument |
Simon Glass | 8d7523c | 2017-01-23 05:38:56 -0700 | [diff] [blame] | 66 | provided by the user, and the value is the list of boards |
| 67 | brought in by that argument. For example, 'arm' might bring |
| 68 | in 400 boards, so in this case the key would be 'arm' and |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 69 | the value would be a list of board names. |
| 70 | boards_selected: Dict of selected boards, key is target name, |
| 71 | value is Board object |
Simon Glass | d233dfb | 2023-07-19 17:48:36 -0600 | [diff] [blame] | 72 | output_dir (str): Output directory for builder |
Simon Glass | 0689036 | 2018-06-11 23:26:46 -0600 | [diff] [blame] | 73 | board_warnings: List of warnings obtained from board selected |
Simon Glass | 1b820ee | 2023-07-19 17:48:46 -0600 | [diff] [blame] | 74 | step (int): Step increment through commits |
| 75 | threads (int): Number of processor threads being used |
| 76 | jobs (int): Number of jobs to build at once |
| 77 | verbose (bool): True to indicate why each board was selected |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 78 | """ |
| 79 | col = terminal.Color() |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 80 | print('Dry run, so not doing much. But I would do this:') |
| 81 | print() |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 82 | if series: |
| 83 | commits = series.commits |
| 84 | else: |
| 85 | commits = None |
Simon Glass | 1b820ee | 2023-07-19 17:48:46 -0600 | [diff] [blame] | 86 | print(get_action_summary(False, commits, boards_selected, step, threads, |
| 87 | jobs)) |
Simon Glass | d233dfb | 2023-07-19 17:48:36 -0600 | [diff] [blame] | 88 | print(f'Build directory: {output_dir}') |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 89 | if commits: |
Simon Glass | 1b820ee | 2023-07-19 17:48:46 -0600 | [diff] [blame] | 90 | for upto in range(0, len(series.commits), step): |
Simon Glass | fea5858 | 2014-08-09 15:32:59 -0600 | [diff] [blame] | 91 | commit = series.commits[upto] |
Simon Glass | 252ac58 | 2022-01-29 14:14:17 -0700 | [diff] [blame] | 92 | print(' ', col.build(col.YELLOW, commit.hash[:8], bright=False), end=' ') |
Simon Glass | c05aa03 | 2019-10-31 07:42:53 -0600 | [diff] [blame] | 93 | print(commit.subject) |
| 94 | print() |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 95 | for arg in why_selected: |
| 96 | if arg != 'all': |
Simon Glass | b8be2bd | 2023-07-19 17:48:31 -0600 | [diff] [blame] | 97 | print(arg, f': {len(why_selected[arg])} boards') |
Simon Glass | 1b820ee | 2023-07-19 17:48:46 -0600 | [diff] [blame] | 98 | if verbose: |
Simon Glass | b8be2bd | 2023-07-19 17:48:31 -0600 | [diff] [blame] | 99 | print(f" {' '.join(why_selected[arg])}") |
| 100 | print('Total boards to build for each ' |
| 101 | f"commit: {len(why_selected['all'])}\n") |
Simon Glass | 0689036 | 2018-06-11 23:26:46 -0600 | [diff] [blame] | 102 | if board_warnings: |
| 103 | for warning in board_warnings: |
Simon Glass | 252ac58 | 2022-01-29 14:14:17 -0700 | [diff] [blame] | 104 | print(col.build(col.YELLOW, warning)) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 105 | |
Simon Glass | 9ef05b9 | 2023-07-19 17:48:30 -0600 | [diff] [blame] | 106 | def show_toolchain_prefix(brds, toolchains): |
Simon Glass | 57cb9d5 | 2019-12-05 15:59:14 -0700 | [diff] [blame] | 107 | """Show information about a the tool chain used by one or more boards |
| 108 | |
Simon Glass | 4e9162d | 2020-03-18 09:42:47 -0600 | [diff] [blame] | 109 | The function checks that all boards use the same toolchain, then prints |
| 110 | the correct value for CROSS_COMPILE. |
Simon Glass | 57cb9d5 | 2019-12-05 15:59:14 -0700 | [diff] [blame] | 111 | |
| 112 | Args: |
| 113 | boards: Boards object containing selected boards |
| 114 | toolchains: Toolchains object containing available toolchains |
Simon Glass | 57cb9d5 | 2019-12-05 15:59:14 -0700 | [diff] [blame] | 115 | |
| 116 | Return: |
| 117 | None on success, string error message otherwise |
| 118 | """ |
Simon Glass | 6014db6 | 2022-07-11 19:04:02 -0600 | [diff] [blame] | 119 | board_selected = brds.get_selected_dict() |
Simon Glass | 57cb9d5 | 2019-12-05 15:59:14 -0700 | [diff] [blame] | 120 | tc_set = set() |
Simon Glass | cc2c0d1 | 2022-07-11 19:04:00 -0600 | [diff] [blame] | 121 | for brd in board_selected.values(): |
Simon Glass | 57cb9d5 | 2019-12-05 15:59:14 -0700 | [diff] [blame] | 122 | tc_set.add(toolchains.Select(brd.arch)) |
| 123 | if len(tc_set) != 1: |
Simon Glass | ea78233 | 2023-07-19 17:48:56 -0600 | [diff] [blame] | 124 | sys.exit('Supplied boards must share one toolchain') |
Simon Glass | b8be2bd | 2023-07-19 17:48:31 -0600 | [diff] [blame] | 125 | tchain = tc_set.pop() |
| 126 | print(tchain.GetEnvArgs(toolchain.VAR_CROSS_COMPILE)) |
Simon Glass | 57cb9d5 | 2019-12-05 15:59:14 -0700 | [diff] [blame] | 127 | |
Tom Rini | d7713ad | 2022-11-09 19:14:53 -0700 | [diff] [blame] | 128 | def get_allow_missing(opt_allow, opt_no_allow, num_selected, has_branch): |
Simon Glass | b8be2bd | 2023-07-19 17:48:31 -0600 | [diff] [blame] | 129 | """Figure out whether to allow external blobs |
| 130 | |
| 131 | Uses the allow-missing setting and the provided arguments to decide whether |
| 132 | missing external blobs should be allowed |
| 133 | |
| 134 | Args: |
| 135 | opt_allow (bool): True if --allow-missing flag is set |
| 136 | opt_no_allow (bool): True if --no-allow-missing flag is set |
| 137 | num_selected (int): Number of selected board |
| 138 | has_branch (bool): True if a git branch (to build) has been provided |
| 139 | |
| 140 | Returns: |
| 141 | bool: True to allow missing external blobs, False to produce an error if |
| 142 | external blobs are used |
| 143 | """ |
Tom Rini | d7713ad | 2022-11-09 19:14:53 -0700 | [diff] [blame] | 144 | allow_missing = False |
| 145 | am_setting = bsettings.GetGlobalItemValue('allow-missing') |
| 146 | if am_setting: |
| 147 | if am_setting == 'always': |
| 148 | allow_missing = True |
| 149 | if 'multiple' in am_setting and num_selected > 1: |
| 150 | allow_missing = True |
| 151 | if 'branch' in am_setting and has_branch: |
| 152 | allow_missing = True |
| 153 | |
| 154 | if opt_allow: |
| 155 | allow_missing = True |
| 156 | if opt_no_allow: |
| 157 | allow_missing = False |
| 158 | return allow_missing |
| 159 | |
Simon Glass | d230c01 | 2023-07-19 17:48:33 -0600 | [diff] [blame] | 160 | |
Simon Glass | aeb2381 | 2023-07-19 17:48:48 -0600 | [diff] [blame] | 161 | def count_commits(branch, count, col, git_dir): |
| 162 | """Could the number of commits in the branch/ranch being built |
| 163 | |
| 164 | Args: |
| 165 | branch (str): Name of branch to build, or None if none |
| 166 | count (int): Number of commits to build, or -1 for all |
| 167 | col (Terminal.Color): Color object to use |
| 168 | git_dir (str): Git directory to use, e.g. './.git' |
| 169 | |
| 170 | Returns: |
| 171 | tuple: |
| 172 | Number of commits being built |
| 173 | True if the 'branch' string contains a range rather than a simple |
| 174 | name |
| 175 | """ |
| 176 | has_range = branch and '..' in branch |
| 177 | if count == -1: |
| 178 | if not branch: |
| 179 | count = 1 |
| 180 | else: |
| 181 | if has_range: |
| 182 | count, msg = gitutil.count_commits_in_range(git_dir, branch) |
| 183 | else: |
| 184 | count, msg = gitutil.count_commits_in_branch(git_dir, branch) |
| 185 | if count is None: |
| 186 | sys.exit(col.build(col.RED, msg)) |
| 187 | elif count == 0: |
| 188 | sys.exit(col.build(col.RED, |
| 189 | f"Range '{branch}' has no commits")) |
| 190 | if msg: |
| 191 | print(col.build(col.YELLOW, msg)) |
| 192 | count += 1 # Build upstream commit also |
| 193 | |
| 194 | if not count: |
| 195 | msg = (f"No commits found to process in branch '{branch}': " |
| 196 | "set branch's upstream or use -c flag") |
| 197 | sys.exit(col.build(col.RED, msg)) |
| 198 | return count, has_range |
| 199 | |
| 200 | |
Simon Glass | 9df59e4 | 2023-07-19 17:48:40 -0600 | [diff] [blame] | 201 | def determine_series(selected, col, git_dir, count, branch, work_in_output): |
Simon Glass | d230c01 | 2023-07-19 17:48:33 -0600 | [diff] [blame] | 202 | """Determine the series which is to be built, if any |
| 203 | |
Simon Glass | 6378bad | 2023-07-19 17:48:50 -0600 | [diff] [blame] | 204 | If there is a series, the commits in that series are numbered by setting |
| 205 | their sequence value (starting from 0). This is used by tests. |
| 206 | |
Simon Glass | d230c01 | 2023-07-19 17:48:33 -0600 | [diff] [blame] | 207 | Args: |
Simon Glass | 1d3a5a5 | 2023-07-19 17:48:45 -0600 | [diff] [blame] | 208 | selected (list of Board): List of Board objects that are marked |
Simon Glass | 9df59e4 | 2023-07-19 17:48:40 -0600 | [diff] [blame] | 209 | selected |
| 210 | col (Terminal.Color): Color object to use |
Simon Glass | d230c01 | 2023-07-19 17:48:33 -0600 | [diff] [blame] | 211 | git_dir (str): Git directory to use, e.g. './.git' |
Simon Glass | 9df59e4 | 2023-07-19 17:48:40 -0600 | [diff] [blame] | 212 | count (int): Number of commits in branch |
| 213 | branch (str): Name of branch to build, or None if none |
| 214 | work_in_output (bool): True to work in the output directory |
Simon Glass | d230c01 | 2023-07-19 17:48:33 -0600 | [diff] [blame] | 215 | |
| 216 | Returns: |
| 217 | Series: Series to build, or None for none |
| 218 | |
| 219 | Read the metadata from the commits. First look at the upstream commit, |
| 220 | then the ones in the branch. We would like to do something like |
| 221 | upstream/master~..branch but that isn't possible if upstream/master is |
| 222 | a merge commit (it will list all the commits that form part of the |
| 223 | merge) |
| 224 | |
| 225 | Conflicting tags are not a problem for buildman, since it does not use |
| 226 | them. For example, Series-version is not useful for buildman. On the |
| 227 | other hand conflicting tags will cause an error. So allow later tags |
| 228 | to overwrite earlier ones by setting allow_overwrite=True |
| 229 | """ |
Simon Glass | 9df59e4 | 2023-07-19 17:48:40 -0600 | [diff] [blame] | 230 | |
| 231 | # Work out how many commits to build. We want to build everything on the |
| 232 | # branch. We also build the upstream commit as a control so we can see |
| 233 | # problems introduced by the first commit on the branch. |
Simon Glass | aeb2381 | 2023-07-19 17:48:48 -0600 | [diff] [blame] | 234 | count, has_range = count_commits(branch, count, col, git_dir) |
Simon Glass | 9df59e4 | 2023-07-19 17:48:40 -0600 | [diff] [blame] | 235 | if work_in_output: |
| 236 | if len(selected) != 1: |
| 237 | sys.exit(col.build(col.RED, |
| 238 | '-w can only be used with a single board')) |
| 239 | if count != 1: |
| 240 | sys.exit(col.build(col.RED, |
| 241 | '-w can only be used with a single commit')) |
| 242 | |
Simon Glass | d230c01 | 2023-07-19 17:48:33 -0600 | [diff] [blame] | 243 | if branch: |
| 244 | if count == -1: |
| 245 | if has_range: |
| 246 | range_expr = branch |
| 247 | else: |
| 248 | range_expr = gitutil.get_range_in_branch(git_dir, branch) |
| 249 | upstream_commit = gitutil.get_upstream(git_dir, branch) |
| 250 | series = patchstream.get_metadata_for_list(upstream_commit, |
| 251 | git_dir, 1, series=None, allow_overwrite=True) |
| 252 | |
| 253 | series = patchstream.get_metadata_for_list(range_expr, |
| 254 | git_dir, None, series, allow_overwrite=True) |
| 255 | else: |
| 256 | # Honour the count |
| 257 | series = patchstream.get_metadata_for_list(branch, |
| 258 | git_dir, count, series=None, allow_overwrite=True) |
Simon Glass | 6378bad | 2023-07-19 17:48:50 -0600 | [diff] [blame] | 259 | |
| 260 | # Number the commits for test purposes |
| 261 | for i, commit in enumerate(series.commits): |
| 262 | commit.sequence = i |
Simon Glass | d230c01 | 2023-07-19 17:48:33 -0600 | [diff] [blame] | 263 | else: |
| 264 | series = None |
| 265 | return series |
| 266 | |
| 267 | |
Simon Glass | f7a36d5 | 2023-07-19 17:48:34 -0600 | [diff] [blame] | 268 | def do_fetch_arch(toolchains, col, fetch_arch): |
| 269 | """Handle the --fetch-arch option |
| 270 | |
| 271 | Args: |
| 272 | toolchains (Toolchains): Tool chains to use |
| 273 | col (terminal.Color): Color object to build |
| 274 | fetch_arch (str): Argument passed to the --fetch-arch option |
| 275 | |
| 276 | Returns: |
| 277 | int: Return code for buildman |
| 278 | """ |
| 279 | if fetch_arch == 'list': |
| 280 | sorted_list = toolchains.ListArchs() |
| 281 | print(col.build( |
| 282 | col.BLUE, |
| 283 | f"Available architectures: {' '.join(sorted_list)}\n")) |
| 284 | return 0 |
| 285 | |
| 286 | if fetch_arch == 'all': |
| 287 | fetch_arch = ','.join(toolchains.ListArchs()) |
| 288 | print(col.build(col.CYAN, |
| 289 | f'\nDownloading toolchains: {fetch_arch}')) |
| 290 | for arch in fetch_arch.split(','): |
| 291 | print() |
| 292 | ret = toolchains.FetchAndInstall(arch) |
| 293 | if ret: |
| 294 | return ret |
| 295 | return 0 |
| 296 | |
| 297 | |
Simon Glass | b868064 | 2023-07-19 17:48:42 -0600 | [diff] [blame] | 298 | def get_toolchains(toolchains, col, override_toolchain, fetch_arch, |
| 299 | list_tool_chains, verbose): |
| 300 | """Get toolchains object to use |
| 301 | |
| 302 | Args: |
| 303 | toolchains (Toolchains or None): Toolchains to use. If None, then a |
| 304 | Toolchains object will be created and scanned |
| 305 | col (Terminal.Color): Color object |
| 306 | override_toolchain (str or None): Override value for toolchain, or None |
| 307 | fetch_arch (bool): True to fetch the toolchain for the architectures |
| 308 | list_tool_chains (bool): True to list all tool chains |
| 309 | verbose (bool): True for verbose output when listing toolchains |
| 310 | |
| 311 | Returns: |
| 312 | Either: |
| 313 | int: Operation completed and buildman should exit with exit code |
| 314 | Toolchains: Toolchains object to use |
| 315 | """ |
| 316 | no_toolchains = toolchains is None |
| 317 | if no_toolchains: |
| 318 | toolchains = toolchain.Toolchains(override_toolchain) |
| 319 | |
| 320 | if fetch_arch: |
| 321 | return do_fetch_arch(toolchains, col, fetch_arch) |
| 322 | |
| 323 | if no_toolchains: |
| 324 | toolchains.GetSettings() |
| 325 | toolchains.Scan(list_tool_chains and verbose) |
| 326 | if list_tool_chains: |
| 327 | toolchains.List() |
| 328 | print() |
| 329 | return 0 |
| 330 | return toolchains |
| 331 | |
| 332 | |
Simon Glass | 180c718 | 2023-07-19 17:48:41 -0600 | [diff] [blame] | 333 | def get_boards_obj(output_dir, regen_board_list, maintainer_check, threads, |
| 334 | verbose): |
| 335 | """Object the Boards object to use |
| 336 | |
| 337 | Creates the output directory and ensures there is a boards.cfg file, then |
| 338 | read it in. |
| 339 | |
| 340 | Args: |
| 341 | output_dir (str): Output directory to use |
| 342 | regen_board_list (bool): True to just regenerate the board list |
| 343 | maintainer_check (bool): True to just run a maintainer check |
| 344 | threads (int or None): Number of threads to use to create boards file |
| 345 | verbose (bool): False to suppress output from boards-file generation |
| 346 | |
| 347 | Returns: |
| 348 | Either: |
| 349 | int: Operation completed and buildman should exit with exit code |
| 350 | Boards: Boards object to use |
| 351 | """ |
| 352 | brds = boards.Boards() |
| 353 | nr_cpus = threads or multiprocessing.cpu_count() |
| 354 | if maintainer_check: |
| 355 | warnings = brds.build_board_list(jobs=nr_cpus)[1] |
| 356 | if warnings: |
| 357 | for warn in warnings: |
| 358 | print(warn, file=sys.stderr) |
| 359 | return 2 |
| 360 | return 0 |
| 361 | |
| 362 | if not os.path.exists(output_dir): |
| 363 | os.makedirs(output_dir) |
| 364 | board_file = os.path.join(output_dir, 'boards.cfg') |
| 365 | if regen_board_list and regen_board_list != '-': |
| 366 | board_file = regen_board_list |
| 367 | |
| 368 | okay = brds.ensure_board_list(board_file, nr_cpus, force=regen_board_list, |
| 369 | quiet=not verbose) |
| 370 | if regen_board_list: |
| 371 | return 0 if okay else 2 |
| 372 | brds.read_boards(board_file) |
| 373 | return brds |
| 374 | |
| 375 | |
Simon Glass | 0d4874f | 2023-07-19 17:48:39 -0600 | [diff] [blame] | 376 | def determine_boards(brds, args, col, opt_boards, exclude_list): |
| 377 | """Determine which boards to build |
| 378 | |
| 379 | Each element of args and exclude can refer to a board name, arch or SoC |
| 380 | |
| 381 | Args: |
| 382 | brds (Boards): Boards object |
| 383 | args (list of str): Arguments describing boards to build |
| 384 | col (Terminal.Color): Color object |
| 385 | opt_boards (list of str): Specific boards to build, or None for all |
| 386 | exclude_list (list of str): Arguments describing boards to exclude |
| 387 | |
| 388 | Returns: |
| 389 | tuple: |
| 390 | list of Board: List of Board objects that are marked selected |
| 391 | why_selected: Dictionary where each key is a buildman argument |
| 392 | provided by the user, and the value is the list of boards |
| 393 | brought in by that argument. For example, 'arm' might bring |
| 394 | in 400 boards, so in this case the key would be 'arm' and |
| 395 | the value would be a list of board names. |
| 396 | board_warnings: List of warnings obtained from board selected |
| 397 | """ |
| 398 | exclude = [] |
| 399 | if exclude_list: |
| 400 | for arg in exclude_list: |
| 401 | exclude += arg.split(',') |
| 402 | |
| 403 | if opt_boards: |
| 404 | requested_boards = [] |
| 405 | for brd in opt_boards: |
| 406 | requested_boards += brd.split(',') |
| 407 | else: |
| 408 | requested_boards = None |
| 409 | why_selected, board_warnings = brds.select_boards(args, exclude, |
| 410 | requested_boards) |
| 411 | selected = brds.get_selected() |
| 412 | if not selected: |
| 413 | sys.exit(col.build(col.RED, 'No matching boards found')) |
| 414 | return selected, why_selected, board_warnings |
| 415 | |
| 416 | |
Simon Glass | 168d792 | 2023-07-19 17:48:47 -0600 | [diff] [blame] | 417 | def adjust_options(options, series, selected): |
| 418 | """Adjust options according to various constraints |
| 419 | |
| 420 | Updates verbose, show_errors, threads, jobs and step |
| 421 | |
| 422 | Args: |
| 423 | options (Options): Options object to adjust |
| 424 | series (Series): Series being built / summarised |
| 425 | selected (list of Board): List of Board objects that are marked |
| 426 | """ |
| 427 | if not series and not options.dry_run: |
| 428 | options.verbose = True |
| 429 | if not options.summary: |
| 430 | options.show_errors = True |
| 431 | |
| 432 | # By default we have one thread per CPU. But if there are not enough jobs |
| 433 | # we can have fewer threads and use a high '-j' value for make. |
| 434 | if options.threads is None: |
| 435 | options.threads = min(multiprocessing.cpu_count(), len(selected)) |
| 436 | if not options.jobs: |
| 437 | options.jobs = max(1, (multiprocessing.cpu_count() + |
| 438 | len(selected) - 1) // len(selected)) |
| 439 | |
| 440 | if not options.step: |
| 441 | options.step = len(series.commits) - 1 |
| 442 | |
Simon Glass | 4ec7682 | 2023-07-19 17:48:53 -0600 | [diff] [blame] | 443 | # We can't show function sizes without board details at present |
| 444 | if options.show_bloat: |
| 445 | options.show_detail = True |
| 446 | |
Simon Glass | e48b946 | 2023-07-19 17:48:49 -0600 | [diff] [blame] | 447 | |
| 448 | def setup_output_dir(output_dir, work_in_output, branch, no_subdirs, col, |
| 449 | clean_dir): |
| 450 | """Set up the output directory |
| 451 | |
| 452 | Args: |
| 453 | output_dir (str): Output directory provided by the user, or None if none |
| 454 | work_in_output (bool): True to work in the output directory |
| 455 | branch (str): Name of branch to build, or None if none |
| 456 | no_subdirs (bool): True to put the output in the top-level output dir |
| 457 | clean_dir: Used for tests only, indicates that the existing output_dir |
| 458 | should be removed before starting the build |
| 459 | |
| 460 | Returns: |
| 461 | str: Updated output directory pathname |
| 462 | """ |
| 463 | if not output_dir: |
| 464 | if work_in_output: |
| 465 | sys.exit(col.build(col.RED, '-w requires that you specify -o')) |
| 466 | output_dir = '..' |
| 467 | if branch and not no_subdirs: |
| 468 | # As a special case allow the board directory to be placed in the |
| 469 | # output directory itself rather than any subdirectory. |
| 470 | dirname = branch.replace('/', '_') |
| 471 | output_dir = os.path.join(output_dir, dirname) |
| 472 | if clean_dir and os.path.exists(output_dir): |
| 473 | shutil.rmtree(output_dir) |
| 474 | return output_dir |
| 475 | |
Simon Glass | a659b8d | 2023-07-19 17:48:57 -0600 | [diff] [blame] | 476 | |
Simon Glass | 68f917c | 2023-07-19 17:48:54 -0600 | [diff] [blame] | 477 | def run_builder(builder, commits, board_selected, options): |
| 478 | """Run the builder or show the summary |
| 479 | |
| 480 | Args: |
| 481 | commits (list of Commit): List of commits being built, None if no branch |
| 482 | boards_selected (dict): Dict of selected boards: |
| 483 | key: target name |
| 484 | value: Board object |
| 485 | options (Options): Options to use |
| 486 | |
| 487 | Returns: |
| 488 | int: Return code for buildman |
| 489 | """ |
Simon Glass | a659b8d | 2023-07-19 17:48:57 -0600 | [diff] [blame] | 490 | gnu_make = command.output(os.path.join(options.git, |
| 491 | 'scripts/show-gnu-make'), raise_on_error=False).rstrip() |
| 492 | if not gnu_make: |
| 493 | sys.exit('GNU Make not found') |
| 494 | builder.gnu_make = gnu_make |
| 495 | |
Simon Glass | 68f917c | 2023-07-19 17:48:54 -0600 | [diff] [blame] | 496 | if not options.ide: |
| 497 | tprint(get_action_summary(options.summary, commits, board_selected, |
| 498 | options.step, options.threads, options.jobs)) |
| 499 | |
| 500 | builder.SetDisplayOptions( |
| 501 | options.show_errors, options.show_sizes, options.show_detail, |
| 502 | options.show_bloat, options.list_error_boards, options.show_config, |
| 503 | options.show_environment, options.filter_dtb_warnings, |
| 504 | options.filter_migration_warnings, options.ide) |
| 505 | if options.summary: |
| 506 | builder.ShowSummary(commits, board_selected) |
| 507 | else: |
| 508 | fail, warned, excs = builder.BuildBoards( |
| 509 | commits, board_selected, options.keep_outputs, options.verbose) |
| 510 | if excs: |
| 511 | return 102 |
| 512 | if fail: |
| 513 | return 100 |
| 514 | if warned and not options.ignore_warnings: |
| 515 | return 101 |
| 516 | return 0 |
Simon Glass | e48b946 | 2023-07-19 17:48:49 -0600 | [diff] [blame] | 517 | |
Simon Glass | 75584e1 | 2023-07-19 17:48:58 -0600 | [diff] [blame] | 518 | |
| 519 | def calc_adjust_cfg(adjust_cfg, reproducible_builds): |
| 520 | """Calculate the value to use for adjust_cfg |
| 521 | |
| 522 | Args: |
| 523 | adjust_cfg (list of str): List of configuration changes. See cfgutil for |
| 524 | details |
| 525 | reproducible_builds (bool): True to adjust the configuration to get |
| 526 | reproduceable builds |
| 527 | |
| 528 | Returns: |
| 529 | adjust_cfg (list of str): List of configuration changes |
| 530 | """ |
| 531 | adjust_cfg = cfgutil.convert_list_to_dict(adjust_cfg) |
| 532 | |
| 533 | # Drop LOCALVERSION_AUTO since it changes the version string on every commit |
| 534 | if reproducible_builds: |
| 535 | # If these are mentioned, leave the local version alone |
| 536 | if 'LOCALVERSION' in adjust_cfg or 'LOCALVERSION_AUTO' in adjust_cfg: |
| 537 | print('Not dropping LOCALVERSION_AUTO for reproducible build') |
| 538 | else: |
| 539 | adjust_cfg['LOCALVERSION_AUTO'] = '~' |
| 540 | return adjust_cfg |
| 541 | |
| 542 | |
Simon Glass | 9ef05b9 | 2023-07-19 17:48:30 -0600 | [diff] [blame] | 543 | def do_buildman(options, args, toolchains=None, make_func=None, brds=None, |
| 544 | clean_dir=False, test_thread_exceptions=False): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 545 | """The main control code for buildman |
| 546 | |
| 547 | Args: |
| 548 | options: Command line options object |
| 549 | args: Command line arguments (list of strings) |
Simon Glass | d4144e4 | 2014-09-05 19:00:13 -0600 | [diff] [blame] | 550 | toolchains: Toolchains to use - this should be a Toolchains() |
| 551 | object. If None, then it will be created and scanned |
| 552 | make_func: Make function to use for the builder. This is called |
| 553 | to execute 'make'. If this is None, the normal function |
| 554 | will be used, which calls the 'make' tool with suitable |
| 555 | arguments. This setting is useful for tests. |
Simon Glass | cc2c0d1 | 2022-07-11 19:04:00 -0600 | [diff] [blame] | 556 | brds: Boards() object to use, containing a list of available |
Simon Glass | 823e60b | 2014-09-05 19:00:16 -0600 | [diff] [blame] | 557 | boards. If this is None it will be created and scanned. |
Simon Glass | 2499331 | 2021-04-11 16:27:25 +1200 | [diff] [blame] | 558 | clean_dir: Used for tests only, indicates that the existing output_dir |
| 559 | should be removed before starting the build |
Simon Glass | 8116c78 | 2021-04-11 16:27:27 +1200 | [diff] [blame] | 560 | test_thread_exceptions: Uses for tests only, True to make the threads |
| 561 | raise an exception instead of reporting their result. This simulates |
| 562 | a failure in the code somewhere |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 563 | """ |
Simon Glass | b8be2bd | 2023-07-19 17:48:31 -0600 | [diff] [blame] | 564 | # Used so testing can obtain the builder: pylint: disable=W0603 |
| 565 | global TEST_BUILDER |
Simon Glass | 883a321 | 2014-09-05 19:00:18 -0600 | [diff] [blame] | 566 | |
Simon Glass | 0157b18 | 2022-01-29 14:14:11 -0700 | [diff] [blame] | 567 | gitutil.setup() |
Simon Glass | 713bea3 | 2016-07-27 20:33:02 -0600 | [diff] [blame] | 568 | col = terminal.Color() |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 569 | |
Simon Glass | d230c01 | 2023-07-19 17:48:33 -0600 | [diff] [blame] | 570 | git_dir = os.path.join(options.git, '.git') |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 571 | |
Simon Glass | b868064 | 2023-07-19 17:48:42 -0600 | [diff] [blame] | 572 | toolchains = get_toolchains(toolchains, col, options.override_toolchain, |
| 573 | options.fetch_arch, options.list_tool_chains, |
| 574 | options.verbose) |
Simon Glass | e48b946 | 2023-07-19 17:48:49 -0600 | [diff] [blame] | 575 | output_dir = setup_output_dir( |
| 576 | options.output_dir, options.work_in_output, options.branch, |
| 577 | options.no_subdirs, col, clean_dir) |
Simon Glass | eb70a2c | 2020-04-09 15:08:51 -0600 | [diff] [blame] | 578 | |
Simon Glass | 7c66ead | 2019-12-05 15:59:13 -0700 | [diff] [blame] | 579 | # Work out what subset of the boards we are building |
Simon Glass | cc2c0d1 | 2022-07-11 19:04:00 -0600 | [diff] [blame] | 580 | if not brds: |
Simon Glass | 372b445 | 2023-07-19 17:48:43 -0600 | [diff] [blame] | 581 | brds = get_boards_obj(output_dir, options.regen_board_list, |
Simon Glass | 180c718 | 2023-07-19 17:48:41 -0600 | [diff] [blame] | 582 | options.maintainer_check, options.threads, |
| 583 | options.verbose) |
| 584 | if isinstance(brds, int): |
| 585 | return brds |
Simon Glass | 7c66ead | 2019-12-05 15:59:13 -0700 | [diff] [blame] | 586 | |
Simon Glass | 0d4874f | 2023-07-19 17:48:39 -0600 | [diff] [blame] | 587 | selected, why_selected, board_warnings = determine_boards( |
| 588 | brds, args, col, options.boards, options.exclude) |
Simon Glass | 7c66ead | 2019-12-05 15:59:13 -0700 | [diff] [blame] | 589 | |
Simon Glass | 4e9162d | 2020-03-18 09:42:47 -0600 | [diff] [blame] | 590 | if options.print_prefix: |
Simon Glass | ea78233 | 2023-07-19 17:48:56 -0600 | [diff] [blame] | 591 | show_toolchain_prefix(brds, toolchains) |
Simon Glass | 57cb9d5 | 2019-12-05 15:59:14 -0700 | [diff] [blame] | 592 | return 0 |
| 593 | |
Simon Glass | 9df59e4 | 2023-07-19 17:48:40 -0600 | [diff] [blame] | 594 | series = determine_series(selected, col, git_dir, options.count, |
| 595 | options.branch, options.work_in_output) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 596 | |
Simon Glass | 168d792 | 2023-07-19 17:48:47 -0600 | [diff] [blame] | 597 | adjust_options(options, series, selected) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 598 | |
Simon Glass | 168d792 | 2023-07-19 17:48:47 -0600 | [diff] [blame] | 599 | # For a dry run, just show our actions as a sanity check |
| 600 | if options.dry_run: |
| 601 | show_actions(series, why_selected, selected, output_dir, board_warnings, |
| 602 | options.step, options.threads, options.jobs, |
| 603 | options.verbose) |
| 604 | return 0 |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 605 | |
Simon Glass | ffd06d3 | 2023-07-19 17:48:52 -0600 | [diff] [blame] | 606 | # Create a new builder with the selected options |
Simon Glass | d230c01 | 2023-07-19 17:48:33 -0600 | [diff] [blame] | 607 | builder = Builder(toolchains, output_dir, git_dir, |
Simon Glass | a659b8d | 2023-07-19 17:48:57 -0600 | [diff] [blame] | 608 | options.threads, options.jobs, checkout=True, |
Simon Glass | 5971ab5 | 2014-12-01 17:33:55 -0700 | [diff] [blame] | 609 | show_unknown=options.show_unknown, step=options.step, |
Simon Glass | d2ce658 | 2014-12-01 17:34:07 -0700 | [diff] [blame] | 610 | no_subdirs=options.no_subdirs, full_path=options.full_path, |
Stephen Warren | f79f1e0 | 2016-04-11 10:48:44 -0600 | [diff] [blame] | 611 | verbose_build=options.verbose_build, |
Simon Glass | eb70a2c | 2020-04-09 15:08:51 -0600 | [diff] [blame] | 612 | mrproper=options.mrproper, |
Simon Glass | b50113f | 2016-11-13 14:25:51 -0700 | [diff] [blame] | 613 | per_board_out_dir=options.per_board_out_dir, |
Simon Glass | b464f8e | 2016-11-13 14:25:53 -0700 | [diff] [blame] | 614 | config_only=options.config_only, |
Daniel Schwierzeck | 2371d1b | 2018-01-26 16:31:05 +0100 | [diff] [blame] | 615 | squash_config_y=not options.preserve_config_y, |
Simon Glass | d829f12 | 2020-03-18 09:42:42 -0600 | [diff] [blame] | 616 | warnings_as_errors=options.warnings_as_errors, |
Simon Glass | 8116c78 | 2021-04-11 16:27:27 +1200 | [diff] [blame] | 617 | work_in_output=options.work_in_output, |
Simon Glass | 2b4806e | 2022-01-22 05:07:33 -0700 | [diff] [blame] | 618 | test_thread_exceptions=test_thread_exceptions, |
Simon Glass | 75584e1 | 2023-07-19 17:48:58 -0600 | [diff] [blame] | 619 | adjust_cfg=calc_adjust_cfg(options.adjust_cfg, |
| 620 | options.reproducible_builds), |
Simon Glass | f6df5ed | 2023-07-19 17:48:59 -0600 | [diff] [blame^] | 621 | allow_missing=get_allow_missing(options.allow_missing, |
| 622 | options.no_allow_missing, |
| 623 | len(selected), options.branch), |
| 624 | no_lto=options.no_lto, |
Simon Glass | ffd06d3 | 2023-07-19 17:48:52 -0600 | [diff] [blame] | 625 | reproducible_builds=options.reproducible_builds, |
| 626 | force_build = options.force_build, |
| 627 | force_build_failures = options.force_build_failures, |
| 628 | force_reconfig = options.force_reconfig, in_tree = options.in_tree, |
| 629 | force_config_on_failure=not options.quick, make_func=make_func) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 630 | |
Simon Glass | ffd06d3 | 2023-07-19 17:48:52 -0600 | [diff] [blame] | 631 | TEST_BUILDER = builder |
Simon Glass | f0207d7 | 2023-07-19 17:48:37 -0600 | [diff] [blame] | 632 | |
Simon Glass | 985d7ae | 2023-07-19 17:48:55 -0600 | [diff] [blame] | 633 | return run_builder(builder, series.commits if series else None, |
| 634 | brds.get_selected_dict(), options) |