blob: 45d9ab73ceb19b9bcb6aeaba3cf0a5617515fca9 [file] [log] [blame]
Tom Rini83d290c2018-05-06 17:58:06 -04001# SPDX-License-Identifier: GPL-2.0+
Simon Glassfc3fe1c2013-04-03 11:07:16 +00002# Copyright (c) 2013 The Chromium OS Authors.
3#
Simon Glassfc3fe1c2013-04-03 11:07:16 +00004
5import multiprocessing
6import os
Simon Glass883a3212014-09-05 19:00:18 -06007import shutil
Simon Glassfc3fe1c2013-04-03 11:07:16 +00008import sys
9
10import board
11import bsettings
12from builder import Builder
13import gitutil
14import patchstream
15import terminal
Simon Glassd4144e42014-09-05 19:00:13 -060016from terminal import Print
Simon Glassfc3fe1c2013-04-03 11:07:16 +000017import toolchain
Masahiro Yamada99796922014-07-22 11:19:09 +090018import command
Masahiro Yamada73f30b92014-07-30 14:08:22 +090019import subprocess
Simon Glassfc3fe1c2013-04-03 11:07:16 +000020
21def GetPlural(count):
22 """Returns a plural 's' if count is not 1"""
23 return 's' if count != 1 else ''
24
Simon Glassfea58582014-08-09 15:32:59 -060025def GetActionSummary(is_summary, commits, selected, options):
Simon Glassfc3fe1c2013-04-03 11:07:16 +000026 """Return a string summarising the intended action.
27
28 Returns:
29 Summary string.
30 """
Simon Glassfea58582014-08-09 15:32:59 -060031 if commits:
32 count = len(commits)
Simon Glassc05aa032019-10-31 07:42:53 -060033 count = (count + options.step - 1) // options.step
Simon Glassfea58582014-08-09 15:32:59 -060034 commit_str = '%d commit%s' % (count, GetPlural(count))
35 else:
36 commit_str = 'current source'
37 str = '%s %s for %d boards' % (
38 'Summary of' if is_summary else 'Building', commit_str,
Simon Glassfc3fe1c2013-04-03 11:07:16 +000039 len(selected))
40 str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
41 GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
42 return str
43
Simon Glass06890362018-06-11 23:26:46 -060044def ShowActions(series, why_selected, boards_selected, builder, options,
45 board_warnings):
Simon Glassfc3fe1c2013-04-03 11:07:16 +000046 """Display a list of actions that we would take, if not a dry run.
47
48 Args:
49 series: Series object
50 why_selected: Dictionary where each key is a buildman argument
Simon Glass8d7523c2017-01-23 05:38:56 -070051 provided by the user, and the value is the list of boards
52 brought in by that argument. For example, 'arm' might bring
53 in 400 boards, so in this case the key would be 'arm' and
Simon Glassfc3fe1c2013-04-03 11:07:16 +000054 the value would be a list of board names.
55 boards_selected: Dict of selected boards, key is target name,
56 value is Board object
57 builder: The builder that will be used to build the commits
58 options: Command line options object
Simon Glass06890362018-06-11 23:26:46 -060059 board_warnings: List of warnings obtained from board selected
Simon Glassfc3fe1c2013-04-03 11:07:16 +000060 """
61 col = terminal.Color()
Simon Glassc05aa032019-10-31 07:42:53 -060062 print('Dry run, so not doing much. But I would do this:')
63 print()
Simon Glassfea58582014-08-09 15:32:59 -060064 if series:
65 commits = series.commits
66 else:
67 commits = None
Simon Glassc05aa032019-10-31 07:42:53 -060068 print(GetActionSummary(False, commits, boards_selected,
69 options))
70 print('Build directory: %s' % builder.base_dir)
Simon Glassfea58582014-08-09 15:32:59 -060071 if commits:
72 for upto in range(0, len(series.commits), options.step):
73 commit = series.commits[upto]
Simon Glassc05aa032019-10-31 07:42:53 -060074 print(' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
75 print(commit.subject)
76 print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +000077 for arg in why_selected:
78 if arg != 'all':
Simon Glassc05aa032019-10-31 07:42:53 -060079 print(arg, ': %d boards' % len(why_selected[arg]))
Simon Glass8d7523c2017-01-23 05:38:56 -070080 if options.verbose:
Simon Glassc05aa032019-10-31 07:42:53 -060081 print(' %s' % ' '.join(why_selected[arg]))
82 print(('Total boards to build for each commit: %d\n' %
83 len(why_selected['all'])))
Simon Glass06890362018-06-11 23:26:46 -060084 if board_warnings:
85 for warning in board_warnings:
Simon Glassc05aa032019-10-31 07:42:53 -060086 print(col.Color(col.YELLOW, warning))
Simon Glassfc3fe1c2013-04-03 11:07:16 +000087
Simon Glass4e9162d2020-03-18 09:42:47 -060088def ShowToolchainPrefix(boards, toolchains):
Simon Glass57cb9d52019-12-05 15:59:14 -070089 """Show information about a the tool chain used by one or more boards
90
Simon Glass4e9162d2020-03-18 09:42:47 -060091 The function checks that all boards use the same toolchain, then prints
92 the correct value for CROSS_COMPILE.
Simon Glass57cb9d52019-12-05 15:59:14 -070093
94 Args:
95 boards: Boards object containing selected boards
96 toolchains: Toolchains object containing available toolchains
Simon Glass57cb9d52019-12-05 15:59:14 -070097
98 Return:
99 None on success, string error message otherwise
100 """
101 boards = boards.GetSelectedDict()
102 tc_set = set()
103 for brd in boards.values():
104 tc_set.add(toolchains.Select(brd.arch))
105 if len(tc_set) != 1:
106 return 'Supplied boards must share one toolchain'
107 return False
108 tc = tc_set.pop()
Simon Glass4e9162d2020-03-18 09:42:47 -0600109 print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
Simon Glass57cb9d52019-12-05 15:59:14 -0700110 return None
111
Simon Glass883a3212014-09-05 19:00:18 -0600112def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
113 clean_dir=False):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000114 """The main control code for buildman
115
116 Args:
117 options: Command line options object
118 args: Command line arguments (list of strings)
Simon Glassd4144e42014-09-05 19:00:13 -0600119 toolchains: Toolchains to use - this should be a Toolchains()
120 object. If None, then it will be created and scanned
121 make_func: Make function to use for the builder. This is called
122 to execute 'make'. If this is None, the normal function
123 will be used, which calls the 'make' tool with suitable
124 arguments. This setting is useful for tests.
Simon Glass823e60b2014-09-05 19:00:16 -0600125 board: Boards() object to use, containing a list of available
126 boards. If this is None it will be created and scanned.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000127 """
Simon Glass883a3212014-09-05 19:00:18 -0600128 global builder
129
Simon Glass48ba5852014-09-05 19:00:11 -0600130 if options.full_help:
131 pager = os.getenv('PAGER')
132 if not pager:
133 pager = 'more'
Simon Glass2bdeade2016-03-06 19:45:34 -0700134 fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
135 'README')
Simon Glass48ba5852014-09-05 19:00:11 -0600136 command.Run(pager, fname)
137 return 0
138
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000139 gitutil.Setup()
Simon Glass713bea32016-07-27 20:33:02 -0600140 col = terminal.Color()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000141
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000142 options.git_dir = os.path.join(options.git, '.git')
143
Simon Glass7e92e462016-07-27 20:33:04 -0600144 no_toolchains = toolchains is None
145 if no_toolchains:
Simon Glass00beb242019-01-07 16:44:20 -0700146 toolchains = toolchain.Toolchains(options.override_toolchain)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000147
Simon Glass827e37b2014-12-01 17:34:06 -0700148 if options.fetch_arch:
149 if options.fetch_arch == 'list':
150 sorted_list = toolchains.ListArchs()
Simon Glassc05aa032019-10-31 07:42:53 -0600151 print(col.Color(col.BLUE, 'Available architectures: %s\n' %
152 ' '.join(sorted_list)))
Simon Glass827e37b2014-12-01 17:34:06 -0700153 return 0
154 else:
155 fetch_arch = options.fetch_arch
156 if fetch_arch == 'all':
157 fetch_arch = ','.join(toolchains.ListArchs())
Simon Glassc05aa032019-10-31 07:42:53 -0600158 print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
159 fetch_arch))
Simon Glass827e37b2014-12-01 17:34:06 -0700160 for arch in fetch_arch.split(','):
Simon Glassc05aa032019-10-31 07:42:53 -0600161 print()
Simon Glass827e37b2014-12-01 17:34:06 -0700162 ret = toolchains.FetchAndInstall(arch)
163 if ret:
164 return ret
165 return 0
166
Simon Glass7e92e462016-07-27 20:33:04 -0600167 if no_toolchains:
168 toolchains.GetSettings()
Simon Glass40232c92018-11-06 16:02:10 -0700169 toolchains.Scan(options.list_tool_chains and options.verbose)
Simon Glass7e92e462016-07-27 20:33:04 -0600170 if options.list_tool_chains:
171 toolchains.List()
Simon Glassc05aa032019-10-31 07:42:53 -0600172 print()
Simon Glass7e92e462016-07-27 20:33:04 -0600173 return 0
174
Simon Glasseb70a2c2020-04-09 15:08:51 -0600175 if options.incremental:
176 print(col.Color(col.RED,
177 'Warning: -I has been removed. See documentation'))
178
Simon Glass7c66ead2019-12-05 15:59:13 -0700179 # Work out what subset of the boards we are building
180 if not boards:
181 if not os.path.exists(options.output_dir):
182 os.makedirs(options.output_dir)
183 board_file = os.path.join(options.output_dir, 'boards.cfg')
184 genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
185 status = subprocess.call([genboardscfg, '-q', '-o', board_file])
186 if status != 0:
187 sys.exit("Failed to generate boards.cfg")
188
189 boards = board.Boards()
190 boards.ReadBoards(board_file)
191
192 exclude = []
193 if options.exclude:
194 for arg in options.exclude:
195 exclude += arg.split(',')
196
197 if options.boards:
198 requested_boards = []
199 for b in options.boards:
200 requested_boards += b.split(',')
201 else:
202 requested_boards = None
203 why_selected, board_warnings = boards.SelectBoards(args, exclude,
204 requested_boards)
205 selected = boards.GetSelected()
206 if not len(selected):
207 sys.exit(col.Color(col.RED, 'No matching boards found'))
208
Simon Glass4e9162d2020-03-18 09:42:47 -0600209 if options.print_prefix:
210 err = ShowToolchainInfo(boards, toolchains)
Simon Glass57cb9d52019-12-05 15:59:14 -0700211 if err:
212 sys.exit(col.Color(col.RED, err))
213 return 0
214
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000215 # Work out how many commits to build. We want to build everything on the
216 # branch. We also build the upstream commit as a control so we can see
217 # problems introduced by the first commit on the branch.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000218 count = options.count
Simon Glass5abab202014-12-01 17:33:57 -0700219 has_range = options.branch and '..' in options.branch
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000220 if count == -1:
221 if not options.branch:
Simon Glassfea58582014-08-09 15:32:59 -0600222 count = 1
223 else:
Simon Glass5abab202014-12-01 17:33:57 -0700224 if has_range:
225 count, msg = gitutil.CountCommitsInRange(options.git_dir,
226 options.branch)
227 else:
228 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
229 options.branch)
Simon Glassfea58582014-08-09 15:32:59 -0600230 if count is None:
Simon Glass2a9e2c62014-12-01 17:33:54 -0700231 sys.exit(col.Color(col.RED, msg))
Simon Glass5abab202014-12-01 17:33:57 -0700232 elif count == 0:
233 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
234 options.branch))
Simon Glass2a9e2c62014-12-01 17:33:54 -0700235 if msg:
Simon Glassc05aa032019-10-31 07:42:53 -0600236 print(col.Color(col.YELLOW, msg))
Simon Glassfea58582014-08-09 15:32:59 -0600237 count += 1 # Build upstream commit also
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000238
239 if not count:
240 str = ("No commits found to process in branch '%s': "
241 "set branch's upstream or use -c flag" % options.branch)
Masahiro Yamada31e21412014-08-16 00:59:26 +0900242 sys.exit(col.Color(col.RED, str))
Simon Glassd829f122020-03-18 09:42:42 -0600243 if options.work_in_output:
244 if len(selected) != 1:
245 sys.exit(col.Color(col.RED,
246 '-w can only be used with a single board'))
247 if count != 1:
248 sys.exit(col.Color(col.RED,
249 '-w can only be used with a single commit'))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000250
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000251 # Read the metadata from the commits. First look at the upstream commit,
252 # then the ones in the branch. We would like to do something like
253 # upstream/master~..branch but that isn't possible if upstream/master is
254 # a merge commit (it will list all the commits that form part of the
255 # merge)
Simon Glass950a2312014-09-05 19:00:23 -0600256 # Conflicting tags are not a problem for buildman, since it does not use
257 # them. For example, Series-version is not useful for buildman. On the
258 # other hand conflicting tags will cause an error. So allow later tags
259 # to overwrite earlier ones by setting allow_overwrite=True
Simon Glassfea58582014-08-09 15:32:59 -0600260 if options.branch:
Simon Glass3b74ba52014-08-09 15:33:09 -0600261 if count == -1:
Simon Glass5abab202014-12-01 17:33:57 -0700262 if has_range:
263 range_expr = options.branch
264 else:
265 range_expr = gitutil.GetRangeInBranch(options.git_dir,
266 options.branch)
Simon Glass3b74ba52014-08-09 15:33:09 -0600267 upstream_commit = gitutil.GetUpstream(options.git_dir,
268 options.branch)
269 series = patchstream.GetMetaDataForList(upstream_commit,
Simon Glass950a2312014-09-05 19:00:23 -0600270 options.git_dir, 1, series=None, allow_overwrite=True)
Simon Glassfea58582014-08-09 15:32:59 -0600271
Simon Glass3b74ba52014-08-09 15:33:09 -0600272 series = patchstream.GetMetaDataForList(range_expr,
Simon Glass950a2312014-09-05 19:00:23 -0600273 options.git_dir, None, series, allow_overwrite=True)
Simon Glass3b74ba52014-08-09 15:33:09 -0600274 else:
275 # Honour the count
276 series = patchstream.GetMetaDataForList(options.branch,
Simon Glass950a2312014-09-05 19:00:23 -0600277 options.git_dir, count, series=None, allow_overwrite=True)
Simon Glassfea58582014-08-09 15:32:59 -0600278 else:
279 series = None
Simon Glass8d7523c2017-01-23 05:38:56 -0700280 if not options.dry_run:
281 options.verbose = True
282 if not options.summary:
283 options.show_errors = True
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000284
285 # By default we have one thread per CPU. But if there are not enough jobs
286 # we can have fewer threads and use a high '-j' value for make.
287 if not options.threads:
288 options.threads = min(multiprocessing.cpu_count(), len(selected))
289 if not options.jobs:
290 options.jobs = max(1, (multiprocessing.cpu_count() +
Simon Glassc05aa032019-10-31 07:42:53 -0600291 len(selected) - 1) // len(selected))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000292
293 if not options.step:
294 options.step = len(series.commits) - 1
295
Masahiro Yamada99796922014-07-22 11:19:09 +0900296 gnu_make = command.Output(os.path.join(options.git,
Simon Glass785f1542016-07-25 18:59:00 -0600297 'scripts/show-gnu-make'), raise_on_error=False).rstrip()
Masahiro Yamada99796922014-07-22 11:19:09 +0900298 if not gnu_make:
Masahiro Yamada31e21412014-08-16 00:59:26 +0900299 sys.exit('GNU Make not found')
Masahiro Yamada99796922014-07-22 11:19:09 +0900300
Simon Glass05c96b12014-12-01 17:33:52 -0700301 # Create a new builder with the selected options.
302 output_dir = options.output_dir
Simon Glassfea58582014-08-09 15:32:59 -0600303 if options.branch:
Simon Glassf7582ce2014-09-05 19:00:22 -0600304 dirname = options.branch.replace('/', '_')
Simon Glass5971ab52014-12-01 17:33:55 -0700305 # As a special case allow the board directory to be placed in the
306 # output directory itself rather than any subdirectory.
307 if not options.no_subdirs:
308 output_dir = os.path.join(options.output_dir, dirname)
Lothar Waßmann409fc022018-04-08 05:14:11 -0600309 if clean_dir and os.path.exists(output_dir):
310 shutil.rmtree(output_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000311 builder = Builder(toolchains, output_dir, options.git_dir,
Masahiro Yamada99796922014-07-22 11:19:09 +0900312 options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
Simon Glass5971ab52014-12-01 17:33:55 -0700313 show_unknown=options.show_unknown, step=options.step,
Simon Glassd2ce6582014-12-01 17:34:07 -0700314 no_subdirs=options.no_subdirs, full_path=options.full_path,
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600315 verbose_build=options.verbose_build,
Simon Glasseb70a2c2020-04-09 15:08:51 -0600316 mrproper=options.mrproper,
Simon Glassb50113f2016-11-13 14:25:51 -0700317 per_board_out_dir=options.per_board_out_dir,
Simon Glassb464f8e2016-11-13 14:25:53 -0700318 config_only=options.config_only,
Daniel Schwierzeck2371d1b2018-01-26 16:31:05 +0100319 squash_config_y=not options.preserve_config_y,
Simon Glassd829f122020-03-18 09:42:42 -0600320 warnings_as_errors=options.warnings_as_errors,
321 work_in_output=options.work_in_output)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000322 builder.force_config_on_failure = not options.quick
Simon Glassd4144e42014-09-05 19:00:13 -0600323 if make_func:
324 builder.do_make = make_func
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000325
326 # For a dry run, just show our actions as a sanity check
327 if options.dry_run:
Simon Glass06890362018-06-11 23:26:46 -0600328 ShowActions(series, why_selected, selected, builder, options,
329 board_warnings)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000330 else:
331 builder.force_build = options.force_build
Simon Glass4266dc22014-07-13 12:22:31 -0600332 builder.force_build_failures = options.force_build_failures
Simon Glass97e91522014-07-14 17:51:02 -0600333 builder.force_reconfig = options.force_reconfig
Simon Glass189a4962014-07-14 17:51:03 -0600334 builder.in_tree = options.in_tree
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000335
336 # Work out which boards to build
337 board_selected = boards.GetSelectedDict()
338
Simon Glassfea58582014-08-09 15:32:59 -0600339 if series:
340 commits = series.commits
Simon Glass883a3212014-09-05 19:00:18 -0600341 # Number the commits for test purposes
342 for commit in range(len(commits)):
343 commits[commit].sequence = commit
Simon Glassfea58582014-08-09 15:32:59 -0600344 else:
345 commits = None
346
Simon Glassd4144e42014-09-05 19:00:13 -0600347 Print(GetActionSummary(options.summary, commits, board_selected,
Simon Glass174592b2020-04-09 15:08:52 -0600348 options))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000349
Simon Glass7798e222014-09-14 20:23:16 -0600350 # We can't show function sizes without board details at present
351 if options.show_bloat:
352 options.show_detail = True
Simon Glass174592b2020-04-09 15:08:52 -0600353 builder.SetDisplayOptions(
354 options.show_errors, options.show_sizes, options.show_detail,
355 options.show_bloat, options.list_error_boards, options.show_config,
356 options.show_environment, options.filter_dtb_warnings)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000357 if options.summary:
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600358 builder.ShowSummary(commits, board_selected)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000359 else:
Simon Glass2c3deb92014-08-28 09:43:39 -0600360 fail, warned = builder.BuildBoards(commits, board_selected,
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600361 options.keep_outputs, options.verbose)
Simon Glass2c3deb92014-08-28 09:43:39 -0600362 if fail:
363 return 128
Simon Glass7beb43c2020-03-18 09:42:44 -0600364 elif warned and not options.ignore_warnings:
Simon Glass2c3deb92014-08-28 09:43:39 -0600365 return 129
366 return 0