blob: 8b3cd30c00119aa990cc0dcc22aa627aa567b400 [file] [log] [blame]
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001# Copyright (c) 2013 The Chromium OS Authors.
2#
Wolfgang Denk1a459662013-07-08 09:37:19 +02003# SPDX-License-Identifier: GPL-2.0+
Simon Glassfc3fe1c2013-04-03 11:07:16 +00004#
5
6import multiprocessing
7import os
Simon Glass883a3212014-09-05 19:00:18 -06008import shutil
Simon Glassfc3fe1c2013-04-03 11:07:16 +00009import sys
10
11import board
12import bsettings
13from builder import Builder
14import gitutil
15import patchstream
16import terminal
Simon Glassd4144e42014-09-05 19:00:13 -060017from terminal import Print
Simon Glassfc3fe1c2013-04-03 11:07:16 +000018import toolchain
Masahiro Yamada99796922014-07-22 11:19:09 +090019import command
Masahiro Yamada73f30b92014-07-30 14:08:22 +090020import subprocess
Simon Glassfc3fe1c2013-04-03 11:07:16 +000021
22def GetPlural(count):
23 """Returns a plural 's' if count is not 1"""
24 return 's' if count != 1 else ''
25
Simon Glassfea58582014-08-09 15:32:59 -060026def GetActionSummary(is_summary, commits, selected, options):
Simon Glassfc3fe1c2013-04-03 11:07:16 +000027 """Return a string summarising the intended action.
28
29 Returns:
30 Summary string.
31 """
Simon Glassfea58582014-08-09 15:32:59 -060032 if commits:
33 count = len(commits)
34 count = (count + options.step - 1) / options.step
35 commit_str = '%d commit%s' % (count, GetPlural(count))
36 else:
37 commit_str = 'current source'
38 str = '%s %s for %d boards' % (
39 'Summary of' if is_summary else 'Building', commit_str,
Simon Glassfc3fe1c2013-04-03 11:07:16 +000040 len(selected))
41 str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
42 GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
43 return str
44
45def ShowActions(series, why_selected, boards_selected, builder, options):
46 """Display a list of actions that we would take, if not a dry run.
47
48 Args:
49 series: Series object
50 why_selected: Dictionary where each key is a buildman argument
51 provided by the user, and the value is the boards brought
52 in by that argument. For example, 'arm' might bring in
53 400 boards, so in this case the key would be 'arm' and
54 the value would be a list of board names.
55 boards_selected: Dict of selected boards, key is target name,
56 value is Board object
57 builder: The builder that will be used to build the commits
58 options: Command line options object
59 """
60 col = terminal.Color()
61 print 'Dry run, so not doing much. But I would do this:'
62 print
Simon Glassfea58582014-08-09 15:32:59 -060063 if series:
64 commits = series.commits
65 else:
66 commits = None
67 print GetActionSummary(False, commits, boards_selected,
Simon Glassfc3fe1c2013-04-03 11:07:16 +000068 options)
69 print 'Build directory: %s' % builder.base_dir
Simon Glassfea58582014-08-09 15:32:59 -060070 if commits:
71 for upto in range(0, len(series.commits), options.step):
72 commit = series.commits[upto]
Simon Glass1ddda1b2014-10-15 02:27:00 -060073 print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False),
Simon Glassfea58582014-08-09 15:32:59 -060074 print commit.subject
Simon Glassfc3fe1c2013-04-03 11:07:16 +000075 print
76 for arg in why_selected:
77 if arg != 'all':
78 print arg, ': %d boards' % why_selected[arg]
79 print ('Total boards to build for each commit: %d\n' %
80 why_selected['all'])
81
Simon Glass883a3212014-09-05 19:00:18 -060082def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
83 clean_dir=False):
Simon Glassfc3fe1c2013-04-03 11:07:16 +000084 """The main control code for buildman
85
86 Args:
87 options: Command line options object
88 args: Command line arguments (list of strings)
Simon Glassd4144e42014-09-05 19:00:13 -060089 toolchains: Toolchains to use - this should be a Toolchains()
90 object. If None, then it will be created and scanned
91 make_func: Make function to use for the builder. This is called
92 to execute 'make'. If this is None, the normal function
93 will be used, which calls the 'make' tool with suitable
94 arguments. This setting is useful for tests.
Simon Glass823e60b2014-09-05 19:00:16 -060095 board: Boards() object to use, containing a list of available
96 boards. If this is None it will be created and scanned.
Simon Glassfc3fe1c2013-04-03 11:07:16 +000097 """
Simon Glass883a3212014-09-05 19:00:18 -060098 global builder
99
Simon Glass48ba5852014-09-05 19:00:11 -0600100 if options.full_help:
101 pager = os.getenv('PAGER')
102 if not pager:
103 pager = 'more'
104 fname = os.path.join(os.path.dirname(sys.argv[0]), 'README')
105 command.Run(pager, fname)
106 return 0
107
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000108 gitutil.Setup()
109
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000110 options.git_dir = os.path.join(options.git, '.git')
111
Simon Glassd4144e42014-09-05 19:00:13 -0600112 if not toolchains:
113 toolchains = toolchain.Toolchains()
114 toolchains.GetSettings()
115 toolchains.Scan(options.list_tool_chains)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000116 if options.list_tool_chains:
117 toolchains.List()
118 print
Simon Glass2c3deb92014-08-28 09:43:39 -0600119 return 0
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000120
Simon Glass827e37b2014-12-01 17:34:06 -0700121 if options.fetch_arch:
122 if options.fetch_arch == 'list':
123 sorted_list = toolchains.ListArchs()
124 print 'Available architectures: %s\n' % ' '.join(sorted_list)
125 return 0
126 else:
127 fetch_arch = options.fetch_arch
128 if fetch_arch == 'all':
129 fetch_arch = ','.join(toolchains.ListArchs())
130 print 'Downloading toolchains: %s\n' % fetch_arch
131 for arch in fetch_arch.split(','):
132 ret = toolchains.FetchAndInstall(arch)
133 if ret:
134 return ret
135 return 0
136
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000137 # Work out how many commits to build. We want to build everything on the
138 # branch. We also build the upstream commit as a control so we can see
139 # problems introduced by the first commit on the branch.
140 col = terminal.Color()
141 count = options.count
Simon Glass5abab202014-12-01 17:33:57 -0700142 has_range = options.branch and '..' in options.branch
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000143 if count == -1:
144 if not options.branch:
Simon Glassfea58582014-08-09 15:32:59 -0600145 count = 1
146 else:
Simon Glass5abab202014-12-01 17:33:57 -0700147 if has_range:
148 count, msg = gitutil.CountCommitsInRange(options.git_dir,
149 options.branch)
150 else:
151 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
152 options.branch)
Simon Glassfea58582014-08-09 15:32:59 -0600153 if count is None:
Simon Glass2a9e2c62014-12-01 17:33:54 -0700154 sys.exit(col.Color(col.RED, msg))
Simon Glass5abab202014-12-01 17:33:57 -0700155 elif count == 0:
156 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
157 options.branch))
Simon Glass2a9e2c62014-12-01 17:33:54 -0700158 if msg:
159 print col.Color(col.YELLOW, msg)
Simon Glassfea58582014-08-09 15:32:59 -0600160 count += 1 # Build upstream commit also
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000161
162 if not count:
163 str = ("No commits found to process in branch '%s': "
164 "set branch's upstream or use -c flag" % options.branch)
Masahiro Yamada31e21412014-08-16 00:59:26 +0900165 sys.exit(col.Color(col.RED, str))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000166
167 # Work out what subset of the boards we are building
Simon Glass823e60b2014-09-05 19:00:16 -0600168 if not boards:
169 board_file = os.path.join(options.git, 'boards.cfg')
170 status = subprocess.call([os.path.join(options.git,
171 'tools/genboardscfg.py')])
172 if status != 0:
173 sys.exit("Failed to generate boards.cfg")
Masahiro Yamada73f30b92014-07-30 14:08:22 +0900174
Simon Glass823e60b2014-09-05 19:00:16 -0600175 boards = board.Boards()
176 boards.ReadBoards(os.path.join(options.git, 'boards.cfg'))
Simon Glass3cf4ae62014-08-28 09:43:41 -0600177
178 exclude = []
179 if options.exclude:
180 for arg in options.exclude:
181 exclude += arg.split(',')
182
183 why_selected = boards.SelectBoards(args, exclude)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000184 selected = boards.GetSelected()
185 if not len(selected):
Masahiro Yamada31e21412014-08-16 00:59:26 +0900186 sys.exit(col.Color(col.RED, 'No matching boards found'))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000187
188 # Read the metadata from the commits. First look at the upstream commit,
189 # then the ones in the branch. We would like to do something like
190 # upstream/master~..branch but that isn't possible if upstream/master is
191 # a merge commit (it will list all the commits that form part of the
192 # merge)
Simon Glass950a2312014-09-05 19:00:23 -0600193 # Conflicting tags are not a problem for buildman, since it does not use
194 # them. For example, Series-version is not useful for buildman. On the
195 # other hand conflicting tags will cause an error. So allow later tags
196 # to overwrite earlier ones by setting allow_overwrite=True
Simon Glassfea58582014-08-09 15:32:59 -0600197 if options.branch:
Simon Glass3b74ba52014-08-09 15:33:09 -0600198 if count == -1:
Simon Glass5abab202014-12-01 17:33:57 -0700199 if has_range:
200 range_expr = options.branch
201 else:
202 range_expr = gitutil.GetRangeInBranch(options.git_dir,
203 options.branch)
Simon Glass3b74ba52014-08-09 15:33:09 -0600204 upstream_commit = gitutil.GetUpstream(options.git_dir,
205 options.branch)
206 series = patchstream.GetMetaDataForList(upstream_commit,
Simon Glass950a2312014-09-05 19:00:23 -0600207 options.git_dir, 1, series=None, allow_overwrite=True)
Simon Glassfea58582014-08-09 15:32:59 -0600208
Simon Glass3b74ba52014-08-09 15:33:09 -0600209 series = patchstream.GetMetaDataForList(range_expr,
Simon Glass950a2312014-09-05 19:00:23 -0600210 options.git_dir, None, series, allow_overwrite=True)
Simon Glass3b74ba52014-08-09 15:33:09 -0600211 else:
212 # Honour the count
213 series = patchstream.GetMetaDataForList(options.branch,
Simon Glass950a2312014-09-05 19:00:23 -0600214 options.git_dir, count, series=None, allow_overwrite=True)
Simon Glassfea58582014-08-09 15:32:59 -0600215 else:
216 series = None
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600217 options.verbose = True
Simon Glass58d818f2014-10-15 14:37:25 +0200218 if not options.summary:
219 options.show_errors = True
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000220
221 # By default we have one thread per CPU. But if there are not enough jobs
222 # we can have fewer threads and use a high '-j' value for make.
223 if not options.threads:
224 options.threads = min(multiprocessing.cpu_count(), len(selected))
225 if not options.jobs:
226 options.jobs = max(1, (multiprocessing.cpu_count() +
227 len(selected) - 1) / len(selected))
228
229 if not options.step:
230 options.step = len(series.commits) - 1
231
Masahiro Yamada99796922014-07-22 11:19:09 +0900232 gnu_make = command.Output(os.path.join(options.git,
233 'scripts/show-gnu-make')).rstrip()
234 if not gnu_make:
Masahiro Yamada31e21412014-08-16 00:59:26 +0900235 sys.exit('GNU Make not found')
Masahiro Yamada99796922014-07-22 11:19:09 +0900236
Simon Glass05c96b12014-12-01 17:33:52 -0700237 # Create a new builder with the selected options.
238 output_dir = options.output_dir
Simon Glassfea58582014-08-09 15:32:59 -0600239 if options.branch:
Simon Glassf7582ce2014-09-05 19:00:22 -0600240 dirname = options.branch.replace('/', '_')
Simon Glass5971ab52014-12-01 17:33:55 -0700241 # As a special case allow the board directory to be placed in the
242 # output directory itself rather than any subdirectory.
243 if not options.no_subdirs:
244 output_dir = os.path.join(options.output_dir, dirname)
Simon Glass07401272014-12-01 17:33:56 -0700245 if (clean_dir and output_dir != options.output_dir and
246 os.path.exists(output_dir)):
Simon Glass883a3212014-09-05 19:00:18 -0600247 shutil.rmtree(output_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000248 builder = Builder(toolchains, output_dir, options.git_dir,
Masahiro Yamada99796922014-07-22 11:19:09 +0900249 options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
Simon Glass5971ab52014-12-01 17:33:55 -0700250 show_unknown=options.show_unknown, step=options.step,
Simon Glassd2ce6582014-12-01 17:34:07 -0700251 no_subdirs=options.no_subdirs, full_path=options.full_path,
252 verbose_build=options.verbose_build)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000253 builder.force_config_on_failure = not options.quick
Simon Glassd4144e42014-09-05 19:00:13 -0600254 if make_func:
255 builder.do_make = make_func
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000256
257 # For a dry run, just show our actions as a sanity check
258 if options.dry_run:
259 ShowActions(series, why_selected, selected, builder, options)
260 else:
261 builder.force_build = options.force_build
Simon Glass4266dc22014-07-13 12:22:31 -0600262 builder.force_build_failures = options.force_build_failures
Simon Glass97e91522014-07-14 17:51:02 -0600263 builder.force_reconfig = options.force_reconfig
Simon Glass189a4962014-07-14 17:51:03 -0600264 builder.in_tree = options.in_tree
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000265
266 # Work out which boards to build
267 board_selected = boards.GetSelectedDict()
268
Simon Glassfea58582014-08-09 15:32:59 -0600269 if series:
270 commits = series.commits
Simon Glass883a3212014-09-05 19:00:18 -0600271 # Number the commits for test purposes
272 for commit in range(len(commits)):
273 commits[commit].sequence = commit
Simon Glassfea58582014-08-09 15:32:59 -0600274 else:
275 commits = None
276
Simon Glassd4144e42014-09-05 19:00:13 -0600277 Print(GetActionSummary(options.summary, commits, board_selected,
278 options))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000279
Simon Glass7798e222014-09-14 20:23:16 -0600280 # We can't show function sizes without board details at present
281 if options.show_bloat:
282 options.show_detail = True
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600283 builder.SetDisplayOptions(options.show_errors, options.show_sizes,
Simon Glassed966652014-08-28 09:43:43 -0600284 options.show_detail, options.show_bloat,
Simon Glass843312d2015-02-05 22:06:15 -0700285 options.list_error_boards,
286 options.show_config)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000287 if options.summary:
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600288 builder.ShowSummary(commits, board_selected)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000289 else:
Simon Glass2c3deb92014-08-28 09:43:39 -0600290 fail, warned = builder.BuildBoards(commits, board_selected,
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600291 options.keep_outputs, options.verbose)
Simon Glass2c3deb92014-08-28 09:43:39 -0600292 if fail:
293 return 128
294 elif warned:
295 return 129
296 return 0