blob: 9e8ca80c5b535b10f9f88414c01d7da5930ad3e9 [file] [log] [blame]
Simon Glass190064b2014-08-09 15:33:00 -06001# Copyright (c) 2014 Google, Inc
2#
3# SPDX-License-Identifier: GPL-2.0+
4#
5
6import errno
7import glob
8import os
9import shutil
10import threading
11
12import command
13import gitutil
14
Simon Glass88c8dcf2015-02-05 22:06:13 -070015RETURN_CODE_RETRY = -1
16
Thierry Redingf3d015c2014-08-19 10:22:39 +020017def Mkdir(dirname, parents = False):
Simon Glass190064b2014-08-09 15:33:00 -060018 """Make a directory if it doesn't already exist.
19
20 Args:
21 dirname: Directory to create
22 """
23 try:
Thierry Redingf3d015c2014-08-19 10:22:39 +020024 if parents:
25 os.makedirs(dirname)
26 else:
27 os.mkdir(dirname)
Simon Glass190064b2014-08-09 15:33:00 -060028 except OSError as err:
29 if err.errno == errno.EEXIST:
30 pass
31 else:
32 raise
33
34class BuilderJob:
35 """Holds information about a job to be performed by a thread
36
37 Members:
38 board: Board object to build
39 commits: List of commit options to build.
40 """
41 def __init__(self):
42 self.board = None
43 self.commits = []
44
45
46class ResultThread(threading.Thread):
47 """This thread processes results from builder threads.
48
49 It simply passes the results on to the builder. There is only one
50 result thread, and this helps to serialise the build output.
51 """
52 def __init__(self, builder):
53 """Set up a new result thread
54
55 Args:
56 builder: Builder which will be sent each result
57 """
58 threading.Thread.__init__(self)
59 self.builder = builder
60
61 def run(self):
62 """Called to start up the result thread.
63
64 We collect the next result job and pass it on to the build.
65 """
66 while True:
67 result = self.builder.out_queue.get()
68 self.builder.ProcessResult(result)
69 self.builder.out_queue.task_done()
70
71
72class BuilderThread(threading.Thread):
73 """This thread builds U-Boot for a particular board.
74
75 An input queue provides each new job. We run 'make' to build U-Boot
76 and then pass the results on to the output queue.
77
78 Members:
79 builder: The builder which contains information we might need
80 thread_num: Our thread number (0-n-1), used to decide on a
81 temporary directory
82 """
Stephen Warrenf79f1e02016-04-11 10:48:44 -060083 def __init__(self, builder, thread_num, incremental, per_board_out_dir):
Simon Glass190064b2014-08-09 15:33:00 -060084 """Set up a new builder thread"""
85 threading.Thread.__init__(self)
86 self.builder = builder
87 self.thread_num = thread_num
Stephen Warrenf79f1e02016-04-11 10:48:44 -060088 self.incremental = incremental
89 self.per_board_out_dir = per_board_out_dir
Simon Glass190064b2014-08-09 15:33:00 -060090
91 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
92 """Run 'make' on a particular commit and board.
93
94 The source code will already be checked out, so the 'commit'
95 argument is only for information.
96
97 Args:
98 commit: Commit object that is being built
99 brd: Board object that is being built
100 stage: Stage of the build. Valid stages are:
Roger Meierfd18a892014-08-20 22:10:29 +0200101 mrproper - can be called to clean source
Simon Glass190064b2014-08-09 15:33:00 -0600102 config - called to configure for a board
103 build - the main make invocation - it does the build
104 args: A list of arguments to pass to 'make'
105 kwargs: A list of keyword arguments to pass to command.RunPipe()
106
107 Returns:
108 CommandResult object
109 """
110 return self.builder.do_make(commit, brd, stage, cwd, *args,
111 **kwargs)
112
Simon Glassa9401b22016-11-16 14:09:25 -0700113 def RunCommit(self, commit_upto, brd, work_dir, do_config, config_only,
Simon Glassb50113f2016-11-13 14:25:51 -0700114 force_build, force_build_failures):
Simon Glass190064b2014-08-09 15:33:00 -0600115 """Build a particular commit.
116
117 If the build is already done, and we are not forcing a build, we skip
118 the build and just return the previously-saved results.
119
120 Args:
121 commit_upto: Commit number to build (0...n-1)
122 brd: Board object to build
123 work_dir: Directory to which the source will be checked out
124 do_config: True to run a make <board>_defconfig on the source
Simon Glassa9401b22016-11-16 14:09:25 -0700125 config_only: Only configure the source, do not build it
Simon Glass190064b2014-08-09 15:33:00 -0600126 force_build: Force a build even if one was previously done
127 force_build_failures: Force a bulid if the previous result showed
128 failure
129
130 Returns:
131 tuple containing:
132 - CommandResult object containing the results of the build
133 - boolean indicating whether 'make config' is still needed
134 """
135 # Create a default result - it will be overwritte by the call to
136 # self.Make() below, in the event that we do a build.
137 result = command.CommandResult()
138 result.return_code = 0
139 if self.builder.in_tree:
140 out_dir = work_dir
141 else:
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600142 if self.per_board_out_dir:
143 out_rel_dir = os.path.join('..', brd.target)
144 else:
145 out_rel_dir = 'build'
146 out_dir = os.path.join(work_dir, out_rel_dir)
Simon Glass190064b2014-08-09 15:33:00 -0600147
148 # Check if the job was already completed last time
149 done_file = self.builder.GetDoneFile(commit_upto, brd.target)
150 result.already_done = os.path.exists(done_file)
151 will_build = (force_build or force_build_failures or
152 not result.already_done)
Simon Glassfb3954f2014-09-05 19:00:17 -0600153 if result.already_done:
Simon Glass190064b2014-08-09 15:33:00 -0600154 # Get the return code from that build and use it
155 with open(done_file, 'r') as fd:
156 result.return_code = int(fd.readline())
Simon Glass88c8dcf2015-02-05 22:06:13 -0700157
158 # Check the signal that the build needs to be retried
159 if result.return_code == RETURN_CODE_RETRY:
160 will_build = True
161 elif will_build:
Simon Glassfb3954f2014-09-05 19:00:17 -0600162 err_file = self.builder.GetErrFile(commit_upto, brd.target)
163 if os.path.exists(err_file) and os.stat(err_file).st_size:
164 result.stderr = 'bad'
165 elif not force_build:
166 # The build passed, so no need to build it again
167 will_build = False
Simon Glass190064b2014-08-09 15:33:00 -0600168
169 if will_build:
170 # We are going to have to build it. First, get a toolchain
171 if not self.toolchain:
172 try:
173 self.toolchain = self.builder.toolchains.Select(brd.arch)
174 except ValueError as err:
175 result.return_code = 10
176 result.stdout = ''
177 result.stderr = str(err)
178 # TODO(sjg@chromium.org): This gets swallowed, but needs
179 # to be reported.
180
181 if self.toolchain:
182 # Checkout the right commit
183 if self.builder.commits:
184 commit = self.builder.commits[commit_upto]
185 if self.builder.checkout:
186 git_dir = os.path.join(work_dir, '.git')
187 gitutil.Checkout(commit.hash, git_dir, work_dir,
188 force=True)
189 else:
190 commit = 'current'
191
192 # Set up the environment and command line
Simon Glassbb1501f2014-12-01 17:34:00 -0700193 env = self.toolchain.MakeEnvironment(self.builder.full_path)
Simon Glass190064b2014-08-09 15:33:00 -0600194 Mkdir(out_dir)
195 args = []
196 cwd = work_dir
Simon Glass48c1b6a2014-08-28 09:43:42 -0600197 src_dir = os.path.realpath(work_dir)
Simon Glass190064b2014-08-09 15:33:00 -0600198 if not self.builder.in_tree:
199 if commit_upto is None:
200 # In this case we are building in the original source
201 # directory (i.e. the current directory where buildman
202 # is invoked. The output directory is set to this
203 # thread's selected work directory.
204 #
205 # Symlinks can confuse U-Boot's Makefile since
206 # we may use '..' in our path, so remove them.
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600207 out_dir = os.path.realpath(out_dir)
208 args.append('O=%s' % out_dir)
Simon Glass190064b2014-08-09 15:33:00 -0600209 cwd = None
Simon Glass48c1b6a2014-08-28 09:43:42 -0600210 src_dir = os.getcwd()
Simon Glass190064b2014-08-09 15:33:00 -0600211 else:
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600212 args.append('O=%s' % out_rel_dir)
Tom Rinif5e5ece2015-04-01 07:47:41 -0400213 if self.builder.verbose_build:
214 args.append('V=1')
215 else:
Simon Glassd2ce6582014-12-01 17:34:07 -0700216 args.append('-s')
Simon Glass190064b2014-08-09 15:33:00 -0600217 if self.builder.num_jobs is not None:
218 args.extend(['-j', str(self.builder.num_jobs)])
219 config_args = ['%s_defconfig' % brd.target]
220 config_out = ''
221 args.extend(self.builder.toolchains.GetMakeArguments(brd))
222
223 # If we need to reconfigure, do that now
224 if do_config:
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600225 config_out = ''
226 if not self.incremental:
227 result = self.Make(commit, brd, 'mrproper', cwd,
228 'mrproper', *args, env=env)
229 config_out += result.combined
Simon Glass190064b2014-08-09 15:33:00 -0600230 result = self.Make(commit, brd, 'config', cwd,
231 *(args + config_args), env=env)
Simon Glass40f11fc2015-02-05 22:06:12 -0700232 config_out += result.combined
Simon Glass190064b2014-08-09 15:33:00 -0600233 do_config = False # No need to configure next time
234 if result.return_code == 0:
Simon Glassa9401b22016-11-16 14:09:25 -0700235 if config_only:
Simon Glassb50113f2016-11-13 14:25:51 -0700236 args.append('cfg')
Simon Glass190064b2014-08-09 15:33:00 -0600237 result = self.Make(commit, brd, 'build', cwd, *args,
238 env=env)
Simon Glass48c1b6a2014-08-28 09:43:42 -0600239 result.stderr = result.stderr.replace(src_dir + '/', '')
Simon Glass40f11fc2015-02-05 22:06:12 -0700240 if self.builder.verbose_build:
241 result.stdout = config_out + result.stdout
Simon Glass190064b2014-08-09 15:33:00 -0600242 else:
243 result.return_code = 1
244 result.stderr = 'No tool chain for %s\n' % brd.arch
245 result.already_done = False
246
247 result.toolchain = self.toolchain
248 result.brd = brd
249 result.commit_upto = commit_upto
250 result.out_dir = out_dir
251 return result, do_config
252
253 def _WriteResult(self, result, keep_outputs):
254 """Write a built result to the output directory.
255
256 Args:
257 result: CommandResult object containing result to write
258 keep_outputs: True to store the output binaries, False
259 to delete them
260 """
261 # Fatal error
262 if result.return_code < 0:
263 return
264
Simon Glass88c8dcf2015-02-05 22:06:13 -0700265 # If we think this might have been aborted with Ctrl-C, record the
266 # failure but not that we are 'done' with this board. A retry may fix
267 # it.
268 maybe_aborted = result.stderr and 'No child processes' in result.stderr
Simon Glass190064b2014-08-09 15:33:00 -0600269
270 if result.already_done:
271 return
272
273 # Write the output and stderr
274 output_dir = self.builder._GetOutputDir(result.commit_upto)
275 Mkdir(output_dir)
276 build_dir = self.builder.GetBuildDir(result.commit_upto,
277 result.brd.target)
278 Mkdir(build_dir)
279
280 outfile = os.path.join(build_dir, 'log')
281 with open(outfile, 'w') as fd:
282 if result.stdout:
Daniel Schwierzeckaafbe822017-06-08 03:07:09 +0200283 # We don't want unicode characters in log files
284 fd.write(result.stdout.decode('UTF-8').encode('ASCII', 'replace'))
Simon Glass190064b2014-08-09 15:33:00 -0600285
286 errfile = self.builder.GetErrFile(result.commit_upto,
287 result.brd.target)
288 if result.stderr:
289 with open(errfile, 'w') as fd:
Daniel Schwierzeckaafbe822017-06-08 03:07:09 +0200290 # We don't want unicode characters in log files
291 fd.write(result.stderr.decode('UTF-8').encode('ASCII', 'replace'))
Simon Glass190064b2014-08-09 15:33:00 -0600292 elif os.path.exists(errfile):
293 os.remove(errfile)
294
295 if result.toolchain:
296 # Write the build result and toolchain information.
297 done_file = self.builder.GetDoneFile(result.commit_upto,
298 result.brd.target)
299 with open(done_file, 'w') as fd:
Simon Glass88c8dcf2015-02-05 22:06:13 -0700300 if maybe_aborted:
301 # Special code to indicate we need to retry
302 fd.write('%s' % RETURN_CODE_RETRY)
303 else:
304 fd.write('%s' % result.return_code)
Simon Glass190064b2014-08-09 15:33:00 -0600305 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
306 print >>fd, 'gcc', result.toolchain.gcc
307 print >>fd, 'path', result.toolchain.path
308 print >>fd, 'cross', result.toolchain.cross
309 print >>fd, 'arch', result.toolchain.arch
310 fd.write('%s' % result.return_code)
311
Simon Glass190064b2014-08-09 15:33:00 -0600312 # Write out the image and function size information and an objdump
Simon Glassbb1501f2014-12-01 17:34:00 -0700313 env = result.toolchain.MakeEnvironment(self.builder.full_path)
Simon Glass190064b2014-08-09 15:33:00 -0600314 lines = []
315 for fname in ['u-boot', 'spl/u-boot-spl']:
316 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
317 nm_result = command.RunPipe([cmd], capture=True,
318 capture_stderr=True, cwd=result.out_dir,
319 raise_on_error=False, env=env)
320 if nm_result.stdout:
321 nm = self.builder.GetFuncSizesFile(result.commit_upto,
322 result.brd.target, fname)
323 with open(nm, 'w') as fd:
324 print >>fd, nm_result.stdout,
325
326 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
327 dump_result = command.RunPipe([cmd], capture=True,
328 capture_stderr=True, cwd=result.out_dir,
329 raise_on_error=False, env=env)
330 rodata_size = ''
331 if dump_result.stdout:
332 objdump = self.builder.GetObjdumpFile(result.commit_upto,
333 result.brd.target, fname)
334 with open(objdump, 'w') as fd:
335 print >>fd, dump_result.stdout,
336 for line in dump_result.stdout.splitlines():
337 fields = line.split()
338 if len(fields) > 5 and fields[1] == '.rodata':
339 rodata_size = fields[2]
340
341 cmd = ['%ssize' % self.toolchain.cross, fname]
342 size_result = command.RunPipe([cmd], capture=True,
343 capture_stderr=True, cwd=result.out_dir,
344 raise_on_error=False, env=env)
345 if size_result.stdout:
346 lines.append(size_result.stdout.splitlines()[1] + ' ' +
347 rodata_size)
348
349 # Write out the image sizes file. This is similar to the output
350 # of binutil's 'size' utility, but it omits the header line and
351 # adds an additional hex value at the end of each line for the
352 # rodata size
353 if len(lines):
354 sizes = self.builder.GetSizesFile(result.commit_upto,
355 result.brd.target)
356 with open(sizes, 'w') as fd:
357 print >>fd, '\n'.join(lines)
358
Simon Glass970f9322015-02-05 22:06:14 -0700359 # Write out the configuration files, with a special case for SPL
360 for dirname in ['', 'spl', 'tpl']:
361 self.CopyFiles(result.out_dir, build_dir, dirname, ['u-boot.cfg',
362 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config',
363 'include/autoconf.mk', 'include/generated/autoconf.h'])
364
Simon Glass190064b2014-08-09 15:33:00 -0600365 # Now write the actual build output
366 if keep_outputs:
Tom Rini0eb4c042015-03-20 10:50:38 -0400367 self.CopyFiles(result.out_dir, build_dir, '', ['u-boot*', '*.bin',
Tom Rinidd592112015-04-27 11:34:38 -0400368 '*.map', '*.img', 'MLO', 'SPL', 'include/autoconf.mk',
Tom Rini0eb4c042015-03-20 10:50:38 -0400369 'spl/u-boot-spl*'])
Simon Glass190064b2014-08-09 15:33:00 -0600370
Simon Glass970f9322015-02-05 22:06:14 -0700371 def CopyFiles(self, out_dir, build_dir, dirname, patterns):
372 """Copy files from the build directory to the output.
373
374 Args:
375 out_dir: Path to output directory containing the files
376 build_dir: Place to copy the files
377 dirname: Source directory, '' for normal U-Boot, 'spl' for SPL
378 patterns: A list of filenames (strings) to copy, each relative
379 to the build directory
380 """
381 for pattern in patterns:
382 file_list = glob.glob(os.path.join(out_dir, dirname, pattern))
383 for fname in file_list:
384 target = os.path.basename(fname)
385 if dirname:
386 base, ext = os.path.splitext(target)
387 if ext:
388 target = '%s-%s%s' % (base, dirname, ext)
389 shutil.copy(fname, os.path.join(build_dir, target))
Simon Glass190064b2014-08-09 15:33:00 -0600390
391 def RunJob(self, job):
392 """Run a single job
393
394 A job consists of a building a list of commits for a particular board.
395
396 Args:
397 job: Job to build
398 """
399 brd = job.board
400 work_dir = self.builder.GetThreadDir(self.thread_num)
401 self.toolchain = None
402 if job.commits:
403 # Run 'make board_defconfig' on the first commit
404 do_config = True
405 commit_upto = 0
406 force_build = False
407 for commit_upto in range(0, len(job.commits), job.step):
408 result, request_config = self.RunCommit(commit_upto, brd,
Simon Glassa9401b22016-11-16 14:09:25 -0700409 work_dir, do_config, self.builder.config_only,
Simon Glass190064b2014-08-09 15:33:00 -0600410 force_build or self.builder.force_build,
411 self.builder.force_build_failures)
412 failed = result.return_code or result.stderr
413 did_config = do_config
414 if failed and not do_config:
415 # If our incremental build failed, try building again
416 # with a reconfig.
417 if self.builder.force_config_on_failure:
418 result, request_config = self.RunCommit(commit_upto,
Simon Glassb50113f2016-11-13 14:25:51 -0700419 brd, work_dir, True, False, True, False)
Simon Glass190064b2014-08-09 15:33:00 -0600420 did_config = True
421 if not self.builder.force_reconfig:
422 do_config = request_config
423
424 # If we built that commit, then config is done. But if we got
425 # an warning, reconfig next time to force it to build the same
426 # files that created warnings this time. Otherwise an
427 # incremental build may not build the same file, and we will
428 # think that the warning has gone away.
429 # We could avoid this by using -Werror everywhere...
430 # For errors, the problem doesn't happen, since presumably
431 # the build stopped and didn't generate output, so will retry
432 # that file next time. So we could detect warnings and deal
433 # with them specially here. For now, we just reconfigure if
434 # anything goes work.
435 # Of course this is substantially slower if there are build
436 # errors/warnings (e.g. 2-3x slower even if only 10% of builds
437 # have problems).
438 if (failed and not result.already_done and not did_config and
439 self.builder.force_config_on_failure):
440 # If this build failed, try the next one with a
441 # reconfigure.
442 # Sometimes if the board_config.h file changes it can mess
443 # with dependencies, and we get:
444 # make: *** No rule to make target `include/autoconf.mk',
445 # needed by `depend'.
446 do_config = True
447 force_build = True
448 else:
449 force_build = False
450 if self.builder.force_config_on_failure:
451 if failed:
452 do_config = True
453 result.commit_upto = commit_upto
454 if result.return_code < 0:
455 raise ValueError('Interrupt')
456
457 # We have the build results, so output the result
458 self._WriteResult(result, job.keep_outputs)
459 self.builder.out_queue.put(result)
460 else:
461 # Just build the currently checked-out build
462 result, request_config = self.RunCommit(None, brd, work_dir, True,
Simon Glassa9401b22016-11-16 14:09:25 -0700463 self.builder.config_only, True,
Simon Glassb50113f2016-11-13 14:25:51 -0700464 self.builder.force_build_failures)
Simon Glass190064b2014-08-09 15:33:00 -0600465 result.commit_upto = 0
466 self._WriteResult(result, job.keep_outputs)
467 self.builder.out_queue.put(result)
468
469 def run(self):
470 """Our thread's run function
471
472 This thread picks a job from the queue, runs it, and then goes to the
473 next job.
474 """
Simon Glass190064b2014-08-09 15:33:00 -0600475 while True:
476 job = self.builder.queue.get()
Simon Glass2880e6b2016-09-18 16:48:38 -0600477 self.RunJob(job)
Simon Glass190064b2014-08-09 15:33:00 -0600478 self.builder.queue.task_done()