blob: 6a6c83bf3368dfb48fa3a2a846d11002dcb45158 [file] [log] [blame]
Tom Rini83d290c2018-05-06 17:58:06 -04001# SPDX-License-Identifier: GPL-2.0+
Simon Glassfc3fe1c2013-04-03 11:07:16 +00002# Copyright (c) 2013 The Chromium OS Authors.
3#
4# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
5#
Simon Glassfc3fe1c2013-04-03 11:07:16 +00006
7import collections
Simon Glassfc3fe1c2013-04-03 11:07:16 +00008from datetime import datetime, timedelta
9import glob
10import os
11import re
12import Queue
13import shutil
Simon Glass2f256642016-09-18 16:48:37 -060014import signal
Simon Glassfc3fe1c2013-04-03 11:07:16 +000015import string
16import sys
Simon Glassd436e382016-09-18 16:48:35 -060017import threading
Simon Glassfc3fe1c2013-04-03 11:07:16 +000018import time
19
Simon Glass190064b2014-08-09 15:33:00 -060020import builderthread
Simon Glassfc3fe1c2013-04-03 11:07:16 +000021import command
22import gitutil
23import terminal
Simon Glass4653a882014-09-05 19:00:07 -060024from terminal import Print
Simon Glassfc3fe1c2013-04-03 11:07:16 +000025import toolchain
26
27
28"""
29Theory of Operation
30
31Please see README for user documentation, and you should be familiar with
32that before trying to make sense of this.
33
34Buildman works by keeping the machine as busy as possible, building different
35commits for different boards on multiple CPUs at once.
36
37The source repo (self.git_dir) contains all the commits to be built. Each
38thread works on a single board at a time. It checks out the first commit,
39configures it for that board, then builds it. Then it checks out the next
40commit and builds it (typically without re-configuring). When it runs out
41of commits, it gets another job from the builder and starts again with that
42board.
43
44Clearly the builder threads could work either way - they could check out a
45commit and then built it for all boards. Using separate directories for each
46commit/board pair they could leave their build product around afterwards
47also.
48
49The intent behind building a single board for multiple commits, is to make
50use of incremental builds. Since each commit is built incrementally from
51the previous one, builds are faster. Reconfiguring for a different board
52removes all intermediate object files.
53
54Many threads can be working at once, but each has its own working directory.
55When a thread finishes a build, it puts the output files into a result
56directory.
57
58The base directory used by buildman is normally '../<branch>', i.e.
59a directory higher than the source repository and named after the branch
60being built.
61
62Within the base directory, we have one subdirectory for each commit. Within
63that is one subdirectory for each board. Within that is the build output for
64that commit/board combination.
65
66Buildman also create working directories for each thread, in a .bm-work/
67subdirectory in the base dir.
68
69As an example, say we are building branch 'us-net' for boards 'sandbox' and
70'seaboard', and say that us-net has two commits. We will have directories
71like this:
72
73us-net/ base directory
74 01_of_02_g4ed4ebc_net--Add-tftp-speed-/
75 sandbox/
76 u-boot.bin
77 seaboard/
78 u-boot.bin
79 02_of_02_g4ed4ebc_net--Check-tftp-comp/
80 sandbox/
81 u-boot.bin
82 seaboard/
83 u-boot.bin
84 .bm-work/
85 00/ working directory for thread 0 (contains source checkout)
86 build/ build output
87 01/ working directory for thread 1
88 build/ build output
89 ...
90u-boot/ source directory
91 .git/ repository
92"""
93
94# Possible build outcomes
95OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4)
96
Simon Glass9a6d2e22017-04-12 18:23:26 -060097# Translate a commit subject into a valid filename (and handle unicode)
98trans_valid_chars = string.maketrans('/: ', '---')
99trans_valid_chars = trans_valid_chars.decode('latin-1')
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000100
Simon Glassb464f8e2016-11-13 14:25:53 -0700101BASE_CONFIG_FILENAMES = [
102 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
103]
104
105EXTRA_CONFIG_FILENAMES = [
Simon Glass843312d2015-02-05 22:06:15 -0700106 '.config', '.config-spl', '.config-tpl',
107 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
108 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
Simon Glass843312d2015-02-05 22:06:15 -0700109]
110
Simon Glass8270e3c2015-08-25 21:52:14 -0600111class Config:
112 """Holds information about configuration settings for a board."""
Simon Glassb464f8e2016-11-13 14:25:53 -0700113 def __init__(self, config_filename, target):
Simon Glass8270e3c2015-08-25 21:52:14 -0600114 self.target = target
115 self.config = {}
Simon Glassb464f8e2016-11-13 14:25:53 -0700116 for fname in config_filename:
Simon Glass8270e3c2015-08-25 21:52:14 -0600117 self.config[fname] = {}
118
119 def Add(self, fname, key, value):
120 self.config[fname][key] = value
121
122 def __hash__(self):
123 val = 0
124 for fname in self.config:
125 for key, value in self.config[fname].iteritems():
126 print key, value
127 val = val ^ hash(key) & hash(value)
128 return val
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000129
Alex Kiernan48ae4122018-05-31 04:48:34 +0000130class Environment:
131 """Holds information about environment variables for a board."""
132 def __init__(self, target):
133 self.target = target
134 self.environment = {}
135
136 def Add(self, key, value):
137 self.environment[key] = value
138
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000139class Builder:
140 """Class for building U-Boot for a particular commit.
141
142 Public members: (many should ->private)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000143 already_done: Number of builds already completed
144 base_dir: Base directory to use for builder
145 checkout: True to check out source, False to skip that step.
146 This is used for testing.
147 col: terminal.Color() object
148 count: Number of commits to build
149 do_make: Method to call to invoke Make
150 fail: Number of builds that failed due to error
151 force_build: Force building even if a build already exists
152 force_config_on_failure: If a commit fails for a board, disable
153 incremental building for the next commit we build for that
154 board, so that we will see all warnings/errors again.
Simon Glass4266dc22014-07-13 12:22:31 -0600155 force_build_failures: If a previously-built build (i.e. built on
156 a previous run of buildman) is marked as failed, rebuild it.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000157 git_dir: Git directory containing source repository
158 last_line_len: Length of the last line we printed (used for erasing
159 it with new progress information)
160 num_jobs: Number of jobs to run at once (passed to make as -j)
161 num_threads: Number of builder threads to run
162 out_queue: Queue of results to process
163 re_make_err: Compiled regular expression for ignore_lines
164 queue: Queue of jobs to run
165 threads: List of active threads
166 toolchains: Toolchains object to use for building
167 upto: Current commit number we are building (0.count-1)
168 warned: Number of builds that produced at least one warning
Simon Glass97e91522014-07-14 17:51:02 -0600169 force_reconfig: Reconfigure U-Boot on each comiit. This disables
170 incremental building, where buildman reconfigures on the first
171 commit for a baord, and then just does an incremental build for
172 the following commits. In fact buildman will reconfigure and
173 retry for any failing commits, so generally the only effect of
174 this option is to slow things down.
Simon Glass189a4962014-07-14 17:51:03 -0600175 in_tree: Build U-Boot in-tree instead of specifying an output
176 directory separate from the source code. This option is really
177 only useful for testing in-tree builds.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000178
179 Private members:
180 _base_board_dict: Last-summarised Dict of boards
181 _base_err_lines: Last-summarised list of errors
Simon Glasse30965d2014-08-28 09:43:44 -0600182 _base_warn_lines: Last-summarised list of warnings
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000183 _build_period_us: Time taken for a single build (float object).
184 _complete_delay: Expected delay until completion (timedelta)
185 _next_delay_update: Next time we plan to display a progress update
186 (datatime)
187 _show_unknown: Show unknown boards (those not built) in summary
188 _timestamps: List of timestamps for the completion of the last
189 last _timestamp_count builds. Each is a datetime object.
190 _timestamp_count: Number of timestamps to keep in our list.
191 _working_dir: Base working directory containing all threads
192 """
193 class Outcome:
194 """Records a build outcome for a single make invocation
195
196 Public Members:
197 rc: Outcome value (OUTCOME_...)
198 err_lines: List of error lines or [] if none
199 sizes: Dictionary of image size information, keyed by filename
200 - Each value is itself a dictionary containing
201 values for 'text', 'data' and 'bss', being the integer
202 size in bytes of each section.
203 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
204 value is itself a dictionary:
205 key: function name
206 value: Size of function in bytes
Simon Glass843312d2015-02-05 22:06:15 -0700207 config: Dictionary keyed by filename - e.g. '.config'. Each
208 value is itself a dictionary:
209 key: config name
210 value: config value
Alex Kiernan48ae4122018-05-31 04:48:34 +0000211 environment: Dictionary keyed by environment variable, Each
212 value is the value of environment variable.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000213 """
Alex Kiernan48ae4122018-05-31 04:48:34 +0000214 def __init__(self, rc, err_lines, sizes, func_sizes, config,
215 environment):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000216 self.rc = rc
217 self.err_lines = err_lines
218 self.sizes = sizes
219 self.func_sizes = func_sizes
Simon Glass843312d2015-02-05 22:06:15 -0700220 self.config = config
Alex Kiernan48ae4122018-05-31 04:48:34 +0000221 self.environment = environment
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000222
223 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
Simon Glass5971ab52014-12-01 17:33:55 -0700224 gnu_make='make', checkout=True, show_unknown=True, step=1,
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600225 no_subdirs=False, full_path=False, verbose_build=False,
Simon Glassb50113f2016-11-13 14:25:51 -0700226 incremental=False, per_board_out_dir=False,
Daniel Schwierzeck2371d1b2018-01-26 16:31:05 +0100227 config_only=False, squash_config_y=False,
228 warnings_as_errors=False):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000229 """Create a new Builder object
230
231 Args:
232 toolchains: Toolchains object to use for building
233 base_dir: Base directory to use for builder
234 git_dir: Git directory containing source repository
235 num_threads: Number of builder threads to run
236 num_jobs: Number of jobs to run at once (passed to make as -j)
Masahiro Yamada99796922014-07-22 11:19:09 +0900237 gnu_make: the command name of GNU Make.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000238 checkout: True to check out source, False to skip that step.
239 This is used for testing.
240 show_unknown: Show unknown boards (those not built) in summary
241 step: 1 to process every commit, n to process every nth commit
Simon Glassbb1501f2014-12-01 17:34:00 -0700242 no_subdirs: Don't create subdirectories when building current
243 source for a single board
244 full_path: Return the full path in CROSS_COMPILE and don't set
245 PATH
Simon Glassd2ce6582014-12-01 17:34:07 -0700246 verbose_build: Run build with V=1 and don't use 'make -s'
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600247 incremental: Always perform incremental builds; don't run make
248 mrproper when configuring
249 per_board_out_dir: Build in a separate persistent directory per
250 board rather than a thread-specific directory
Simon Glassb50113f2016-11-13 14:25:51 -0700251 config_only: Only configure each build, don't build it
Simon Glassb464f8e2016-11-13 14:25:53 -0700252 squash_config_y: Convert CONFIG options with the value 'y' to '1'
Daniel Schwierzeck2371d1b2018-01-26 16:31:05 +0100253 warnings_as_errors: Treat all compiler warnings as errors
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000254 """
255 self.toolchains = toolchains
256 self.base_dir = base_dir
257 self._working_dir = os.path.join(base_dir, '.bm-work')
258 self.threads = []
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000259 self.do_make = self.Make
Masahiro Yamada99796922014-07-22 11:19:09 +0900260 self.gnu_make = gnu_make
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000261 self.checkout = checkout
262 self.num_threads = num_threads
263 self.num_jobs = num_jobs
264 self.already_done = 0
265 self.force_build = False
266 self.git_dir = git_dir
267 self._show_unknown = show_unknown
268 self._timestamp_count = 10
269 self._build_period_us = None
270 self._complete_delay = None
271 self._next_delay_update = datetime.now()
272 self.force_config_on_failure = True
Simon Glass4266dc22014-07-13 12:22:31 -0600273 self.force_build_failures = False
Simon Glass97e91522014-07-14 17:51:02 -0600274 self.force_reconfig = False
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000275 self._step = step
Simon Glass189a4962014-07-14 17:51:03 -0600276 self.in_tree = False
Simon Glass28370c12014-08-09 15:33:06 -0600277 self._error_lines = 0
Simon Glass5971ab52014-12-01 17:33:55 -0700278 self.no_subdirs = no_subdirs
Simon Glassbb1501f2014-12-01 17:34:00 -0700279 self.full_path = full_path
Simon Glassd2ce6582014-12-01 17:34:07 -0700280 self.verbose_build = verbose_build
Simon Glassb50113f2016-11-13 14:25:51 -0700281 self.config_only = config_only
Simon Glassb464f8e2016-11-13 14:25:53 -0700282 self.squash_config_y = squash_config_y
283 self.config_filenames = BASE_CONFIG_FILENAMES
284 if not self.squash_config_y:
285 self.config_filenames += EXTRA_CONFIG_FILENAMES
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000286
Daniel Schwierzeck2371d1b2018-01-26 16:31:05 +0100287 self.warnings_as_errors = warnings_as_errors
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000288 self.col = terminal.Color()
289
Simon Glasse30965d2014-08-28 09:43:44 -0600290 self._re_function = re.compile('(.*): In function.*')
291 self._re_files = re.compile('In file included from.*')
292 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
Simon Glass2d483332018-11-06 16:02:11 -0700293 self._re_dtb_warning = re.compile('(.*): Warning .*')
Simon Glasse30965d2014-08-28 09:43:44 -0600294 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
295
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000296 self.queue = Queue.Queue()
297 self.out_queue = Queue.Queue()
298 for i in range(self.num_threads):
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600299 t = builderthread.BuilderThread(self, i, incremental,
300 per_board_out_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000301 t.setDaemon(True)
302 t.start()
303 self.threads.append(t)
304
305 self.last_line_len = 0
Simon Glass190064b2014-08-09 15:33:00 -0600306 t = builderthread.ResultThread(self)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000307 t.setDaemon(True)
308 t.start()
309 self.threads.append(t)
310
311 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
312 self.re_make_err = re.compile('|'.join(ignore_lines))
313
Simon Glass2f256642016-09-18 16:48:37 -0600314 # Handle existing graceful with SIGINT / Ctrl-C
315 signal.signal(signal.SIGINT, self.signal_handler)
316
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000317 def __del__(self):
318 """Get rid of all threads created by the builder"""
319 for t in self.threads:
320 del t
321
Simon Glass2f256642016-09-18 16:48:37 -0600322 def signal_handler(self, signal, frame):
323 sys.exit(1)
324
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600325 def SetDisplayOptions(self, show_errors=False, show_sizes=False,
Simon Glassed966652014-08-28 09:43:43 -0600326 show_detail=False, show_bloat=False,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000327 list_error_boards=False, show_config=False,
328 show_environment=False):
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600329 """Setup display options for the builder.
330
331 show_errors: True to show summarised error/warning info
332 show_sizes: Show size deltas
333 show_detail: Show detail for each board
334 show_bloat: Show detail for each function
Simon Glassed966652014-08-28 09:43:43 -0600335 list_error_boards: Show the boards which caused each error/warning
Simon Glass843312d2015-02-05 22:06:15 -0700336 show_config: Show config deltas
Alex Kiernan48ae4122018-05-31 04:48:34 +0000337 show_environment: Show environment deltas
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600338 """
339 self._show_errors = show_errors
340 self._show_sizes = show_sizes
341 self._show_detail = show_detail
342 self._show_bloat = show_bloat
Simon Glassed966652014-08-28 09:43:43 -0600343 self._list_error_boards = list_error_boards
Simon Glass843312d2015-02-05 22:06:15 -0700344 self._show_config = show_config
Alex Kiernan48ae4122018-05-31 04:48:34 +0000345 self._show_environment = show_environment
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600346
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000347 def _AddTimestamp(self):
348 """Add a new timestamp to the list and record the build period.
349
350 The build period is the length of time taken to perform a single
351 build (one board, one commit).
352 """
353 now = datetime.now()
354 self._timestamps.append(now)
355 count = len(self._timestamps)
356 delta = self._timestamps[-1] - self._timestamps[0]
357 seconds = delta.total_seconds()
358
359 # If we have enough data, estimate build period (time taken for a
360 # single build) and therefore completion time.
361 if count > 1 and self._next_delay_update < now:
362 self._next_delay_update = now + timedelta(seconds=2)
363 if seconds > 0:
364 self._build_period = float(seconds) / count
365 todo = self.count - self.upto
366 self._complete_delay = timedelta(microseconds=
367 self._build_period * todo * 1000000)
368 # Round it
369 self._complete_delay -= timedelta(
370 microseconds=self._complete_delay.microseconds)
371
372 if seconds > 60:
373 self._timestamps.popleft()
374 count -= 1
375
376 def ClearLine(self, length):
377 """Clear any characters on the current line
378
379 Make way for a new line of length 'length', by outputting enough
380 spaces to clear out the old line. Then remember the new length for
381 next time.
382
383 Args:
384 length: Length of new line, in characters
385 """
386 if length < self.last_line_len:
Simon Glass4653a882014-09-05 19:00:07 -0600387 Print(' ' * (self.last_line_len - length), newline=False)
388 Print('\r', newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000389 self.last_line_len = length
390 sys.stdout.flush()
391
392 def SelectCommit(self, commit, checkout=True):
393 """Checkout the selected commit for this build
394 """
395 self.commit = commit
396 if checkout and self.checkout:
397 gitutil.Checkout(commit.hash)
398
399 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
400 """Run make
401
402 Args:
403 commit: Commit object that is being built
404 brd: Board object that is being built
Roger Meierfd18a892014-08-20 22:10:29 +0200405 stage: Stage that we are at (mrproper, config, build)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000406 cwd: Directory where make should be run
407 args: Arguments to pass to make
408 kwargs: Arguments to pass to command.RunPipe()
409 """
Masahiro Yamada99796922014-07-22 11:19:09 +0900410 cmd = [self.gnu_make] + list(args)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000411 result = command.RunPipe([cmd], capture=True, capture_stderr=True,
Simon Glasse62a24c2018-09-17 23:55:42 -0600412 cwd=cwd, raise_on_error=False, infile='/dev/null', **kwargs)
Simon Glass40f11fc2015-02-05 22:06:12 -0700413 if self.verbose_build:
414 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
415 result.combined = '%s\n' % (' '.join(cmd)) + result.combined
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000416 return result
417
418 def ProcessResult(self, result):
419 """Process the result of a build, showing progress information
420
421 Args:
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600422 result: A CommandResult object, which indicates the result for
423 a single build
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000424 """
425 col = terminal.Color()
426 if result:
427 target = result.brd.target
428
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000429 self.upto += 1
430 if result.return_code != 0:
431 self.fail += 1
432 elif result.stderr:
433 self.warned += 1
434 if result.already_done:
435 self.already_done += 1
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600436 if self._verbose:
Simon Glass4653a882014-09-05 19:00:07 -0600437 Print('\r', newline=False)
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600438 self.ClearLine(0)
439 boards_selected = {target : result.brd}
440 self.ResetResultSummary(boards_selected)
441 self.ProduceResultSummary(result.commit_upto, self.commits,
442 boards_selected)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000443 else:
444 target = '(starting)'
445
446 # Display separate counts for ok, warned and fail
447 ok = self.upto - self.warned - self.fail
448 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
449 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
450 line += self.col.Color(self.col.RED, '%5d' % self.fail)
451
452 name = ' /%-5d ' % self.count
453
454 # Add our current completion time estimate
455 self._AddTimestamp()
456 if self._complete_delay:
457 name += '%s : ' % self._complete_delay
458 # When building all boards for a commit, we can print a commit
459 # progress message.
460 if result and result.commit_upto is None:
461 name += 'commit %2d/%-3d' % (self.commit_upto + 1,
462 self.commit_count)
463
464 name += target
Simon Glass4653a882014-09-05 19:00:07 -0600465 Print(line + name, newline=False)
Simon Glass960421e2016-11-15 15:32:59 -0700466 length = 16 + len(name)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000467 self.ClearLine(length)
468
469 def _GetOutputDir(self, commit_upto):
470 """Get the name of the output directory for a commit number
471
472 The output directory is typically .../<branch>/<commit>.
473
474 Args:
475 commit_upto: Commit number to use (0..self.count-1)
476 """
Simon Glass5971ab52014-12-01 17:33:55 -0700477 commit_dir = None
Simon Glassfea58582014-08-09 15:32:59 -0600478 if self.commits:
479 commit = self.commits[commit_upto]
480 subject = commit.subject.translate(trans_valid_chars)
481 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
482 self.commit_count, commit.hash, subject[:20]))
Simon Glass5971ab52014-12-01 17:33:55 -0700483 elif not self.no_subdirs:
Simon Glassfea58582014-08-09 15:32:59 -0600484 commit_dir = 'current'
Simon Glass5971ab52014-12-01 17:33:55 -0700485 if not commit_dir:
486 return self.base_dir
487 return os.path.join(self.base_dir, commit_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000488
489 def GetBuildDir(self, commit_upto, target):
490 """Get the name of the build directory for a commit number
491
492 The build directory is typically .../<branch>/<commit>/<target>.
493
494 Args:
495 commit_upto: Commit number to use (0..self.count-1)
496 target: Target name
497 """
498 output_dir = self._GetOutputDir(commit_upto)
499 return os.path.join(output_dir, target)
500
501 def GetDoneFile(self, commit_upto, target):
502 """Get the name of the done file for a commit number
503
504 Args:
505 commit_upto: Commit number to use (0..self.count-1)
506 target: Target name
507 """
508 return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
509
510 def GetSizesFile(self, commit_upto, target):
511 """Get the name of the sizes file for a commit number
512
513 Args:
514 commit_upto: Commit number to use (0..self.count-1)
515 target: Target name
516 """
517 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
518
519 def GetFuncSizesFile(self, commit_upto, target, elf_fname):
520 """Get the name of the funcsizes file for a commit number and ELF file
521
522 Args:
523 commit_upto: Commit number to use (0..self.count-1)
524 target: Target name
525 elf_fname: Filename of elf image
526 """
527 return os.path.join(self.GetBuildDir(commit_upto, target),
528 '%s.sizes' % elf_fname.replace('/', '-'))
529
530 def GetObjdumpFile(self, commit_upto, target, elf_fname):
531 """Get the name of the objdump file for a commit number and ELF file
532
533 Args:
534 commit_upto: Commit number to use (0..self.count-1)
535 target: Target name
536 elf_fname: Filename of elf image
537 """
538 return os.path.join(self.GetBuildDir(commit_upto, target),
539 '%s.objdump' % elf_fname.replace('/', '-'))
540
541 def GetErrFile(self, commit_upto, target):
542 """Get the name of the err file for a commit number
543
544 Args:
545 commit_upto: Commit number to use (0..self.count-1)
546 target: Target name
547 """
548 output_dir = self.GetBuildDir(commit_upto, target)
549 return os.path.join(output_dir, 'err')
550
551 def FilterErrors(self, lines):
552 """Filter out errors in which we have no interest
553
554 We should probably use map().
555
556 Args:
557 lines: List of error lines, each a string
558 Returns:
559 New list with only interesting lines included
560 """
561 out_lines = []
562 for line in lines:
563 if not self.re_make_err.search(line):
564 out_lines.append(line)
565 return out_lines
566
567 def ReadFuncSizes(self, fname, fd):
568 """Read function sizes from the output of 'nm'
569
570 Args:
571 fd: File containing data to read
572 fname: Filename we are reading from (just for errors)
573
574 Returns:
575 Dictionary containing size of each function in bytes, indexed by
576 function name.
577 """
578 sym = {}
579 for line in fd.readlines():
580 try:
581 size, type, name = line[:-1].split()
582 except:
Simon Glass4653a882014-09-05 19:00:07 -0600583 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000584 continue
585 if type in 'tTdDbB':
586 # function names begin with '.' on 64-bit powerpc
587 if '.' in name[1:]:
588 name = 'static.' + name.split('.')[0]
589 sym[name] = sym.get(name, 0) + int(size, 16)
590 return sym
591
Simon Glass843312d2015-02-05 22:06:15 -0700592 def _ProcessConfig(self, fname):
593 """Read in a .config, autoconf.mk or autoconf.h file
594
595 This function handles all config file types. It ignores comments and
596 any #defines which don't start with CONFIG_.
597
598 Args:
599 fname: Filename to read
600
601 Returns:
602 Dictionary:
603 key: Config name (e.g. CONFIG_DM)
604 value: Config value (e.g. 1)
605 """
606 config = {}
607 if os.path.exists(fname):
608 with open(fname) as fd:
609 for line in fd:
610 line = line.strip()
611 if line.startswith('#define'):
612 values = line[8:].split(' ', 1)
613 if len(values) > 1:
614 key, value = values
615 else:
616 key = values[0]
Simon Glassb464f8e2016-11-13 14:25:53 -0700617 value = '1' if self.squash_config_y else ''
Simon Glass843312d2015-02-05 22:06:15 -0700618 if not key.startswith('CONFIG_'):
619 continue
620 elif not line or line[0] in ['#', '*', '/']:
621 continue
622 else:
623 key, value = line.split('=', 1)
Simon Glassb464f8e2016-11-13 14:25:53 -0700624 if self.squash_config_y and value == 'y':
625 value = '1'
Simon Glass843312d2015-02-05 22:06:15 -0700626 config[key] = value
627 return config
628
Alex Kiernan48ae4122018-05-31 04:48:34 +0000629 def _ProcessEnvironment(self, fname):
630 """Read in a uboot.env file
631
632 This function reads in environment variables from a file.
633
634 Args:
635 fname: Filename to read
636
637 Returns:
638 Dictionary:
639 key: environment variable (e.g. bootlimit)
640 value: value of environment variable (e.g. 1)
641 """
642 environment = {}
643 if os.path.exists(fname):
644 with open(fname) as fd:
645 for line in fd.read().split('\0'):
646 try:
647 key, value = line.split('=', 1)
648 environment[key] = value
649 except ValueError:
650 # ignore lines we can't parse
651 pass
652 return environment
653
Simon Glass843312d2015-02-05 22:06:15 -0700654 def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000655 read_config, read_environment):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000656 """Work out the outcome of a build.
657
658 Args:
659 commit_upto: Commit number to check (0..n-1)
660 target: Target board to check
661 read_func_sizes: True to read function size information
Simon Glass843312d2015-02-05 22:06:15 -0700662 read_config: True to read .config and autoconf.h files
Alex Kiernan48ae4122018-05-31 04:48:34 +0000663 read_environment: True to read uboot.env files
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000664
665 Returns:
666 Outcome object
667 """
668 done_file = self.GetDoneFile(commit_upto, target)
669 sizes_file = self.GetSizesFile(commit_upto, target)
670 sizes = {}
671 func_sizes = {}
Simon Glass843312d2015-02-05 22:06:15 -0700672 config = {}
Alex Kiernan48ae4122018-05-31 04:48:34 +0000673 environment = {}
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000674 if os.path.exists(done_file):
675 with open(done_file, 'r') as fd:
676 return_code = int(fd.readline())
677 err_lines = []
678 err_file = self.GetErrFile(commit_upto, target)
679 if os.path.exists(err_file):
680 with open(err_file, 'r') as fd:
681 err_lines = self.FilterErrors(fd.readlines())
682
683 # Decide whether the build was ok, failed or created warnings
684 if return_code:
685 rc = OUTCOME_ERROR
686 elif len(err_lines):
687 rc = OUTCOME_WARNING
688 else:
689 rc = OUTCOME_OK
690
691 # Convert size information to our simple format
692 if os.path.exists(sizes_file):
693 with open(sizes_file, 'r') as fd:
694 for line in fd.readlines():
695 values = line.split()
696 rodata = 0
697 if len(values) > 6:
698 rodata = int(values[6], 16)
699 size_dict = {
700 'all' : int(values[0]) + int(values[1]) +
701 int(values[2]),
702 'text' : int(values[0]) - rodata,
703 'data' : int(values[1]),
704 'bss' : int(values[2]),
705 'rodata' : rodata,
706 }
707 sizes[values[5]] = size_dict
708
709 if read_func_sizes:
710 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
711 for fname in glob.glob(pattern):
712 with open(fname, 'r') as fd:
713 dict_name = os.path.basename(fname).replace('.sizes',
714 '')
715 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
716
Simon Glass843312d2015-02-05 22:06:15 -0700717 if read_config:
718 output_dir = self.GetBuildDir(commit_upto, target)
Simon Glassb464f8e2016-11-13 14:25:53 -0700719 for name in self.config_filenames:
Simon Glass843312d2015-02-05 22:06:15 -0700720 fname = os.path.join(output_dir, name)
721 config[name] = self._ProcessConfig(fname)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000722
Alex Kiernan48ae4122018-05-31 04:48:34 +0000723 if read_environment:
724 output_dir = self.GetBuildDir(commit_upto, target)
725 fname = os.path.join(output_dir, 'uboot.env')
726 environment = self._ProcessEnvironment(fname)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000727
Alex Kiernan48ae4122018-05-31 04:48:34 +0000728 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config,
729 environment)
730
731 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {})
Simon Glass843312d2015-02-05 22:06:15 -0700732
733 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000734 read_config, read_environment):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000735 """Calculate a summary of the results of building a commit.
736
737 Args:
738 board_selected: Dict containing boards to summarise
739 commit_upto: Commit number to summarize (0..self.count-1)
740 read_func_sizes: True to read function size information
Simon Glass843312d2015-02-05 22:06:15 -0700741 read_config: True to read .config and autoconf.h files
Alex Kiernan48ae4122018-05-31 04:48:34 +0000742 read_environment: True to read uboot.env files
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000743
744 Returns:
745 Tuple:
746 Dict containing boards which passed building this commit.
747 keyed by board.target
Simon Glasse30965d2014-08-28 09:43:44 -0600748 List containing a summary of error lines
Simon Glassed966652014-08-28 09:43:43 -0600749 Dict keyed by error line, containing a list of the Board
750 objects with that error
Simon Glasse30965d2014-08-28 09:43:44 -0600751 List containing a summary of warning lines
752 Dict keyed by error line, containing a list of the Board
753 objects with that warning
Simon Glass8270e3c2015-08-25 21:52:14 -0600754 Dictionary keyed by board.target. Each value is a dictionary:
755 key: filename - e.g. '.config'
Simon Glass843312d2015-02-05 22:06:15 -0700756 value is itself a dictionary:
757 key: config name
758 value: config value
Alex Kiernan48ae4122018-05-31 04:48:34 +0000759 Dictionary keyed by board.target. Each value is a dictionary:
760 key: environment variable
761 value: value of environment variable
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000762 """
Simon Glasse30965d2014-08-28 09:43:44 -0600763 def AddLine(lines_summary, lines_boards, line, board):
764 line = line.rstrip()
765 if line in lines_boards:
766 lines_boards[line].append(board)
767 else:
768 lines_boards[line] = [board]
769 lines_summary.append(line)
770
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000771 board_dict = {}
772 err_lines_summary = []
Simon Glassed966652014-08-28 09:43:43 -0600773 err_lines_boards = {}
Simon Glasse30965d2014-08-28 09:43:44 -0600774 warn_lines_summary = []
775 warn_lines_boards = {}
Simon Glass843312d2015-02-05 22:06:15 -0700776 config = {}
Alex Kiernan48ae4122018-05-31 04:48:34 +0000777 environment = {}
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000778
779 for board in boards_selected.itervalues():
780 outcome = self.GetBuildOutcome(commit_upto, board.target,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000781 read_func_sizes, read_config,
782 read_environment)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000783 board_dict[board.target] = outcome
Simon Glasse30965d2014-08-28 09:43:44 -0600784 last_func = None
785 last_was_warning = False
786 for line in outcome.err_lines:
787 if line:
788 if (self._re_function.match(line) or
789 self._re_files.match(line)):
790 last_func = line
Simon Glassed966652014-08-28 09:43:43 -0600791 else:
Simon Glass2d483332018-11-06 16:02:11 -0700792 is_warning = (self._re_warning.match(line) or
793 self._re_dtb_warning.match(line))
Simon Glasse30965d2014-08-28 09:43:44 -0600794 is_note = self._re_note.match(line)
795 if is_warning or (last_was_warning and is_note):
796 if last_func:
797 AddLine(warn_lines_summary, warn_lines_boards,
798 last_func, board)
799 AddLine(warn_lines_summary, warn_lines_boards,
800 line, board)
801 else:
802 if last_func:
803 AddLine(err_lines_summary, err_lines_boards,
804 last_func, board)
805 AddLine(err_lines_summary, err_lines_boards,
806 line, board)
807 last_was_warning = is_warning
808 last_func = None
Simon Glassb464f8e2016-11-13 14:25:53 -0700809 tconfig = Config(self.config_filenames, board.target)
810 for fname in self.config_filenames:
Simon Glass843312d2015-02-05 22:06:15 -0700811 if outcome.config:
812 for key, value in outcome.config[fname].iteritems():
Simon Glass8270e3c2015-08-25 21:52:14 -0600813 tconfig.Add(fname, key, value)
814 config[board.target] = tconfig
Simon Glass843312d2015-02-05 22:06:15 -0700815
Alex Kiernan48ae4122018-05-31 04:48:34 +0000816 tenvironment = Environment(board.target)
817 if outcome.environment:
818 for key, value in outcome.environment.iteritems():
819 tenvironment.Add(key, value)
820 environment[board.target] = tenvironment
821
Simon Glasse30965d2014-08-28 09:43:44 -0600822 return (board_dict, err_lines_summary, err_lines_boards,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000823 warn_lines_summary, warn_lines_boards, config, environment)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000824
825 def AddOutcome(self, board_dict, arch_list, changes, char, color):
826 """Add an output to our list of outcomes for each architecture
827
828 This simple function adds failing boards (changes) to the
829 relevant architecture string, so we can print the results out
830 sorted by architecture.
831
832 Args:
833 board_dict: Dict containing all boards
834 arch_list: Dict keyed by arch name. Value is a string containing
835 a list of board names which failed for that arch.
836 changes: List of boards to add to arch_list
837 color: terminal.Colour object
838 """
839 done_arch = {}
840 for target in changes:
841 if target in board_dict:
842 arch = board_dict[target].arch
843 else:
844 arch = 'unknown'
845 str = self.col.Color(color, ' ' + target)
846 if not arch in done_arch:
Simon Glass63c619e2015-02-05 22:06:11 -0700847 str = ' %s %s' % (self.col.Color(color, char), str)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000848 done_arch[arch] = True
849 if not arch in arch_list:
850 arch_list[arch] = str
851 else:
852 arch_list[arch] += str
853
854
855 def ColourNum(self, num):
856 color = self.col.RED if num > 0 else self.col.GREEN
857 if num == 0:
858 return '0'
859 return self.col.Color(color, str(num))
860
861 def ResetResultSummary(self, board_selected):
862 """Reset the results summary ready for use.
863
864 Set up the base board list to be all those selected, and set the
865 error lines to empty.
866
867 Following this, calls to PrintResultSummary() will use this
868 information to work out what has changed.
869
870 Args:
871 board_selected: Dict containing boards to summarise, keyed by
872 board.target
873 """
874 self._base_board_dict = {}
875 for board in board_selected:
Alex Kiernan48ae4122018-05-31 04:48:34 +0000876 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {},
877 {})
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000878 self._base_err_lines = []
Simon Glasse30965d2014-08-28 09:43:44 -0600879 self._base_warn_lines = []
880 self._base_err_line_boards = {}
881 self._base_warn_line_boards = {}
Simon Glass8270e3c2015-08-25 21:52:14 -0600882 self._base_config = None
Alex Kiernan48ae4122018-05-31 04:48:34 +0000883 self._base_environment = None
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000884
885 def PrintFuncSizeDetail(self, fname, old, new):
886 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
887 delta, common = [], {}
888
889 for a in old:
890 if a in new:
891 common[a] = 1
892
893 for name in old:
894 if name not in common:
895 remove += 1
896 down += old[name]
897 delta.append([-old[name], name])
898
899 for name in new:
900 if name not in common:
901 add += 1
902 up += new[name]
903 delta.append([new[name], name])
904
905 for name in common:
906 diff = new.get(name, 0) - old.get(name, 0)
907 if diff > 0:
908 grow, up = grow + 1, up + diff
909 elif diff < 0:
910 shrink, down = shrink + 1, down - diff
911 delta.append([diff, name])
912
913 delta.sort()
914 delta.reverse()
915
916 args = [add, -remove, grow, -shrink, up, -down, up - down]
Tom Rinid5686a62017-05-22 13:48:52 -0400917 if max(args) == 0 and min(args) == 0:
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000918 return
919 args = [self.ColourNum(x) for x in args]
920 indent = ' ' * 15
Simon Glass4653a882014-09-05 19:00:07 -0600921 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
922 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
923 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
924 'delta'))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000925 for diff, name in delta:
926 if diff:
927 color = self.col.RED if diff > 0 else self.col.GREEN
928 msg = '%s %-38s %7s %7s %+7d' % (indent, name,
929 old.get(name, '-'), new.get(name,'-'), diff)
Simon Glass4653a882014-09-05 19:00:07 -0600930 Print(msg, colour=color)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000931
932
933 def PrintSizeDetail(self, target_list, show_bloat):
934 """Show details size information for each board
935
936 Args:
937 target_list: List of targets, each a dict containing:
938 'target': Target name
939 'total_diff': Total difference in bytes across all areas
940 <part_name>: Difference for that part
941 show_bloat: Show detail for each function
942 """
943 targets_by_diff = sorted(target_list, reverse=True,
944 key=lambda x: x['_total_diff'])
945 for result in targets_by_diff:
946 printed_target = False
947 for name in sorted(result):
948 diff = result[name]
949 if name.startswith('_'):
950 continue
951 if diff != 0:
952 color = self.col.RED if diff > 0 else self.col.GREEN
953 msg = ' %s %+d' % (name, diff)
954 if not printed_target:
Simon Glass4653a882014-09-05 19:00:07 -0600955 Print('%10s %-15s:' % ('', result['_target']),
956 newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000957 printed_target = True
Simon Glass4653a882014-09-05 19:00:07 -0600958 Print(msg, colour=color, newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000959 if printed_target:
Simon Glass4653a882014-09-05 19:00:07 -0600960 Print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000961 if show_bloat:
962 target = result['_target']
963 outcome = result['_outcome']
964 base_outcome = self._base_board_dict[target]
965 for fname in outcome.func_sizes:
966 self.PrintFuncSizeDetail(fname,
967 base_outcome.func_sizes[fname],
968 outcome.func_sizes[fname])
969
970
971 def PrintSizeSummary(self, board_selected, board_dict, show_detail,
972 show_bloat):
973 """Print a summary of image sizes broken down by section.
974
975 The summary takes the form of one line per architecture. The
976 line contains deltas for each of the sections (+ means the section
977 got bigger, - means smaller). The nunmbers are the average number
978 of bytes that a board in this section increased by.
979
980 For example:
981 powerpc: (622 boards) text -0.0
982 arm: (285 boards) text -0.0
983 nds32: (3 boards) text -8.0
984
985 Args:
986 board_selected: Dict containing boards to summarise, keyed by
987 board.target
988 board_dict: Dict containing boards for which we built this
989 commit, keyed by board.target. The value is an Outcome object.
990 show_detail: Show detail for each board
991 show_bloat: Show detail for each function
992 """
993 arch_list = {}
994 arch_count = {}
995
996 # Calculate changes in size for different image parts
997 # The previous sizes are in Board.sizes, for each board
998 for target in board_dict:
999 if target not in board_selected:
1000 continue
1001 base_sizes = self._base_board_dict[target].sizes
1002 outcome = board_dict[target]
1003 sizes = outcome.sizes
1004
1005 # Loop through the list of images, creating a dict of size
1006 # changes for each image/part. We end up with something like
1007 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1008 # which means that U-Boot data increased by 5 bytes and SPL
1009 # text decreased by 4.
1010 err = {'_target' : target}
1011 for image in sizes:
1012 if image in base_sizes:
1013 base_image = base_sizes[image]
1014 # Loop through the text, data, bss parts
1015 for part in sorted(sizes[image]):
1016 diff = sizes[image][part] - base_image[part]
1017 col = None
1018 if diff:
1019 if image == 'u-boot':
1020 name = part
1021 else:
1022 name = image + ':' + part
1023 err[name] = diff
1024 arch = board_selected[target].arch
1025 if not arch in arch_count:
1026 arch_count[arch] = 1
1027 else:
1028 arch_count[arch] += 1
1029 if not sizes:
1030 pass # Only add to our list when we have some stats
1031 elif not arch in arch_list:
1032 arch_list[arch] = [err]
1033 else:
1034 arch_list[arch].append(err)
1035
1036 # We now have a list of image size changes sorted by arch
1037 # Print out a summary of these
1038 for arch, target_list in arch_list.iteritems():
1039 # Get total difference for each type
1040 totals = {}
1041 for result in target_list:
1042 total = 0
1043 for name, diff in result.iteritems():
1044 if name.startswith('_'):
1045 continue
1046 total += diff
1047 if name in totals:
1048 totals[name] += diff
1049 else:
1050 totals[name] = diff
1051 result['_total_diff'] = total
1052 result['_outcome'] = board_dict[result['_target']]
1053
1054 count = len(target_list)
1055 printed_arch = False
1056 for name in sorted(totals):
1057 diff = totals[name]
1058 if diff:
1059 # Display the average difference in this name for this
1060 # architecture
1061 avg_diff = float(diff) / count
1062 color = self.col.RED if avg_diff > 0 else self.col.GREEN
1063 msg = ' %s %+1.1f' % (name, avg_diff)
1064 if not printed_arch:
Simon Glass4653a882014-09-05 19:00:07 -06001065 Print('%10s: (for %d/%d boards)' % (arch, count,
1066 arch_count[arch]), newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001067 printed_arch = True
Simon Glass4653a882014-09-05 19:00:07 -06001068 Print(msg, colour=color, newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001069
1070 if printed_arch:
Simon Glass4653a882014-09-05 19:00:07 -06001071 Print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001072 if show_detail:
1073 self.PrintSizeDetail(target_list, show_bloat)
1074
1075
1076 def PrintResultSummary(self, board_selected, board_dict, err_lines,
Simon Glasse30965d2014-08-28 09:43:44 -06001077 err_line_boards, warn_lines, warn_line_boards,
Alex Kiernan48ae4122018-05-31 04:48:34 +00001078 config, environment, show_sizes, show_detail,
1079 show_bloat, show_config, show_environment):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001080 """Compare results with the base results and display delta.
1081
1082 Only boards mentioned in board_selected will be considered. This
1083 function is intended to be called repeatedly with the results of
1084 each commit. It therefore shows a 'diff' between what it saw in
1085 the last call and what it sees now.
1086
1087 Args:
1088 board_selected: Dict containing boards to summarise, keyed by
1089 board.target
1090 board_dict: Dict containing boards for which we built this
1091 commit, keyed by board.target. The value is an Outcome object.
1092 err_lines: A list of errors for this commit, or [] if there is
1093 none, or we don't want to print errors
Simon Glassed966652014-08-28 09:43:43 -06001094 err_line_boards: Dict keyed by error line, containing a list of
1095 the Board objects with that error
Simon Glasse30965d2014-08-28 09:43:44 -06001096 warn_lines: A list of warnings for this commit, or [] if there is
1097 none, or we don't want to print errors
1098 warn_line_boards: Dict keyed by warning line, containing a list of
1099 the Board objects with that warning
Simon Glass843312d2015-02-05 22:06:15 -07001100 config: Dictionary keyed by filename - e.g. '.config'. Each
1101 value is itself a dictionary:
1102 key: config name
1103 value: config value
Alex Kiernan48ae4122018-05-31 04:48:34 +00001104 environment: Dictionary keyed by environment variable, Each
1105 value is the value of environment variable.
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001106 show_sizes: Show image size deltas
1107 show_detail: Show detail for each board
1108 show_bloat: Show detail for each function
Simon Glass843312d2015-02-05 22:06:15 -07001109 show_config: Show config changes
Alex Kiernan48ae4122018-05-31 04:48:34 +00001110 show_environment: Show environment changes
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001111 """
Simon Glasse30965d2014-08-28 09:43:44 -06001112 def _BoardList(line, line_boards):
Simon Glassed966652014-08-28 09:43:43 -06001113 """Helper function to get a line of boards containing a line
1114
1115 Args:
1116 line: Error line to search for
1117 Return:
1118 String containing a list of boards with that error line, or
1119 '' if the user has not requested such a list
1120 """
1121 if self._list_error_boards:
1122 names = []
Simon Glasse30965d2014-08-28 09:43:44 -06001123 for board in line_boards[line]:
Simon Glassf66153b2014-10-16 01:05:55 -06001124 if not board.target in names:
1125 names.append(board.target)
Simon Glassed966652014-08-28 09:43:43 -06001126 names_str = '(%s) ' % ','.join(names)
1127 else:
1128 names_str = ''
1129 return names_str
1130
Simon Glasse30965d2014-08-28 09:43:44 -06001131 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1132 char):
1133 better_lines = []
1134 worse_lines = []
1135 for line in lines:
1136 if line not in base_lines:
1137 worse_lines.append(char + '+' +
1138 _BoardList(line, line_boards) + line)
1139 for line in base_lines:
1140 if line not in lines:
1141 better_lines.append(char + '-' +
1142 _BoardList(line, base_line_boards) + line)
1143 return better_lines, worse_lines
1144
Simon Glass843312d2015-02-05 22:06:15 -07001145 def _CalcConfig(delta, name, config):
1146 """Calculate configuration changes
1147
1148 Args:
1149 delta: Type of the delta, e.g. '+'
1150 name: name of the file which changed (e.g. .config)
1151 config: configuration change dictionary
1152 key: config name
1153 value: config value
1154 Returns:
1155 String containing the configuration changes which can be
1156 printed
1157 """
1158 out = ''
1159 for key in sorted(config.keys()):
1160 out += '%s=%s ' % (key, config[key])
Simon Glass8270e3c2015-08-25 21:52:14 -06001161 return '%s %s: %s' % (delta, name, out)
Simon Glass843312d2015-02-05 22:06:15 -07001162
Simon Glass8270e3c2015-08-25 21:52:14 -06001163 def _AddConfig(lines, name, config_plus, config_minus, config_change):
1164 """Add changes in configuration to a list
Simon Glass843312d2015-02-05 22:06:15 -07001165
1166 Args:
Simon Glass8270e3c2015-08-25 21:52:14 -06001167 lines: list to add to
1168 name: config file name
Simon Glass843312d2015-02-05 22:06:15 -07001169 config_plus: configurations added, dictionary
1170 key: config name
1171 value: config value
1172 config_minus: configurations removed, dictionary
1173 key: config name
1174 value: config value
1175 config_change: configurations changed, dictionary
1176 key: config name
1177 value: config value
1178 """
1179 if config_plus:
Simon Glass8270e3c2015-08-25 21:52:14 -06001180 lines.append(_CalcConfig('+', name, config_plus))
Simon Glass843312d2015-02-05 22:06:15 -07001181 if config_minus:
Simon Glass8270e3c2015-08-25 21:52:14 -06001182 lines.append(_CalcConfig('-', name, config_minus))
Simon Glass843312d2015-02-05 22:06:15 -07001183 if config_change:
Simon Glass8270e3c2015-08-25 21:52:14 -06001184 lines.append(_CalcConfig('c', name, config_change))
1185
1186 def _OutputConfigInfo(lines):
1187 for line in lines:
1188 if not line:
1189 continue
1190 if line[0] == '+':
1191 col = self.col.GREEN
1192 elif line[0] == '-':
1193 col = self.col.RED
1194 elif line[0] == 'c':
1195 col = self.col.YELLOW
1196 Print(' ' + line, newline=True, colour=col)
1197
Simon Glass843312d2015-02-05 22:06:15 -07001198
Simon Glass4cf2b222018-11-06 16:02:12 -07001199 ok_boards = [] # List of boards fixed since last commit
Simon Glass6af71012018-11-06 16:02:13 -07001200 warn_boards = [] # List of boards with warnings since last commit
Simon Glass4cf2b222018-11-06 16:02:12 -07001201 err_boards = [] # List of new broken boards since last commit
1202 new_boards = [] # List of boards that didn't exist last time
1203 unknown_boards = [] # List of boards that were not built
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001204
1205 for target in board_dict:
1206 if target not in board_selected:
1207 continue
1208
1209 # If the board was built last time, add its outcome to a list
1210 if target in self._base_board_dict:
1211 base_outcome = self._base_board_dict[target].rc
1212 outcome = board_dict[target]
1213 if outcome.rc == OUTCOME_UNKNOWN:
Simon Glass4cf2b222018-11-06 16:02:12 -07001214 unknown_boards.append(target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001215 elif outcome.rc < base_outcome:
Simon Glass6af71012018-11-06 16:02:13 -07001216 if outcome.rc == OUTCOME_WARNING:
1217 warn_boards.append(target)
1218 else:
1219 ok_boards.append(target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001220 elif outcome.rc > base_outcome:
Simon Glass6af71012018-11-06 16:02:13 -07001221 if outcome.rc == OUTCOME_WARNING:
1222 warn_boards.append(target)
1223 else:
1224 err_boards.append(target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001225 else:
Simon Glass4cf2b222018-11-06 16:02:12 -07001226 new_boards.append(target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001227
1228 # Get a list of errors that have appeared, and disappeared
Simon Glasse30965d2014-08-28 09:43:44 -06001229 better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1230 self._base_err_line_boards, err_lines, err_line_boards, '')
1231 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1232 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001233
1234 # Display results by arch
Simon Glass6af71012018-11-06 16:02:13 -07001235 if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards,
1236 worse_err, better_err, worse_warn, better_warn)):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001237 arch_list = {}
Simon Glass4cf2b222018-11-06 16:02:12 -07001238 self.AddOutcome(board_selected, arch_list, ok_boards, '',
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001239 self.col.GREEN)
Simon Glass6af71012018-11-06 16:02:13 -07001240 self.AddOutcome(board_selected, arch_list, warn_boards, 'w+',
1241 self.col.YELLOW)
Simon Glass4cf2b222018-11-06 16:02:12 -07001242 self.AddOutcome(board_selected, arch_list, err_boards, '+',
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001243 self.col.RED)
Simon Glass4cf2b222018-11-06 16:02:12 -07001244 self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001245 if self._show_unknown:
Simon Glass4cf2b222018-11-06 16:02:12 -07001246 self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001247 self.col.MAGENTA)
1248 for arch, target_list in arch_list.iteritems():
Simon Glass4653a882014-09-05 19:00:07 -06001249 Print('%10s: %s' % (arch, target_list))
Simon Glass28370c12014-08-09 15:33:06 -06001250 self._error_lines += 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001251 if better_err:
Simon Glass4653a882014-09-05 19:00:07 -06001252 Print('\n'.join(better_err), colour=self.col.GREEN)
Simon Glass28370c12014-08-09 15:33:06 -06001253 self._error_lines += 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001254 if worse_err:
Simon Glass4653a882014-09-05 19:00:07 -06001255 Print('\n'.join(worse_err), colour=self.col.RED)
Simon Glass28370c12014-08-09 15:33:06 -06001256 self._error_lines += 1
Simon Glasse30965d2014-08-28 09:43:44 -06001257 if better_warn:
Simon Glass4653a882014-09-05 19:00:07 -06001258 Print('\n'.join(better_warn), colour=self.col.CYAN)
Simon Glasse30965d2014-08-28 09:43:44 -06001259 self._error_lines += 1
1260 if worse_warn:
Simon Glass4653a882014-09-05 19:00:07 -06001261 Print('\n'.join(worse_warn), colour=self.col.MAGENTA)
Simon Glasse30965d2014-08-28 09:43:44 -06001262 self._error_lines += 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001263
1264 if show_sizes:
1265 self.PrintSizeSummary(board_selected, board_dict, show_detail,
1266 show_bloat)
1267
Alex Kiernan48ae4122018-05-31 04:48:34 +00001268 if show_environment and self._base_environment:
1269 lines = []
1270
1271 for target in board_dict:
1272 if target not in board_selected:
1273 continue
1274
1275 tbase = self._base_environment[target]
1276 tenvironment = environment[target]
1277 environment_plus = {}
1278 environment_minus = {}
1279 environment_change = {}
1280 base = tbase.environment
1281 for key, value in tenvironment.environment.iteritems():
1282 if key not in base:
1283 environment_plus[key] = value
1284 for key, value in base.iteritems():
1285 if key not in tenvironment.environment:
1286 environment_minus[key] = value
1287 for key, value in base.iteritems():
1288 new_value = tenvironment.environment.get(key)
1289 if new_value and value != new_value:
1290 desc = '%s -> %s' % (value, new_value)
1291 environment_change[key] = desc
1292
1293 _AddConfig(lines, target, environment_plus, environment_minus,
1294 environment_change)
1295
1296 _OutputConfigInfo(lines)
1297
Simon Glass8270e3c2015-08-25 21:52:14 -06001298 if show_config and self._base_config:
1299 summary = {}
1300 arch_config_plus = {}
1301 arch_config_minus = {}
1302 arch_config_change = {}
1303 arch_list = []
1304
1305 for target in board_dict:
1306 if target not in board_selected:
Simon Glass843312d2015-02-05 22:06:15 -07001307 continue
Simon Glass8270e3c2015-08-25 21:52:14 -06001308 arch = board_selected[target].arch
1309 if arch not in arch_list:
1310 arch_list.append(arch)
1311
1312 for arch in arch_list:
1313 arch_config_plus[arch] = {}
1314 arch_config_minus[arch] = {}
1315 arch_config_change[arch] = {}
Simon Glassb464f8e2016-11-13 14:25:53 -07001316 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001317 arch_config_plus[arch][name] = {}
1318 arch_config_minus[arch][name] = {}
1319 arch_config_change[arch][name] = {}
1320
1321 for target in board_dict:
1322 if target not in board_selected:
1323 continue
1324
1325 arch = board_selected[target].arch
1326
1327 all_config_plus = {}
1328 all_config_minus = {}
1329 all_config_change = {}
1330 tbase = self._base_config[target]
1331 tconfig = config[target]
1332 lines = []
Simon Glassb464f8e2016-11-13 14:25:53 -07001333 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001334 if not tconfig.config[name]:
1335 continue
1336 config_plus = {}
1337 config_minus = {}
1338 config_change = {}
1339 base = tbase.config[name]
1340 for key, value in tconfig.config[name].iteritems():
1341 if key not in base:
1342 config_plus[key] = value
1343 all_config_plus[key] = value
1344 for key, value in base.iteritems():
1345 if key not in tconfig.config[name]:
1346 config_minus[key] = value
1347 all_config_minus[key] = value
1348 for key, value in base.iteritems():
1349 new_value = tconfig.config.get(key)
1350 if new_value and value != new_value:
1351 desc = '%s -> %s' % (value, new_value)
1352 config_change[key] = desc
1353 all_config_change[key] = desc
1354
1355 arch_config_plus[arch][name].update(config_plus)
1356 arch_config_minus[arch][name].update(config_minus)
1357 arch_config_change[arch][name].update(config_change)
1358
1359 _AddConfig(lines, name, config_plus, config_minus,
1360 config_change)
1361 _AddConfig(lines, 'all', all_config_plus, all_config_minus,
1362 all_config_change)
1363 summary[target] = '\n'.join(lines)
1364
1365 lines_by_target = {}
1366 for target, lines in summary.iteritems():
1367 if lines in lines_by_target:
1368 lines_by_target[lines].append(target)
1369 else:
1370 lines_by_target[lines] = [target]
1371
1372 for arch in arch_list:
1373 lines = []
1374 all_plus = {}
1375 all_minus = {}
1376 all_change = {}
Simon Glassb464f8e2016-11-13 14:25:53 -07001377 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001378 all_plus.update(arch_config_plus[arch][name])
1379 all_minus.update(arch_config_minus[arch][name])
1380 all_change.update(arch_config_change[arch][name])
1381 _AddConfig(lines, name, arch_config_plus[arch][name],
1382 arch_config_minus[arch][name],
1383 arch_config_change[arch][name])
1384 _AddConfig(lines, 'all', all_plus, all_minus, all_change)
1385 #arch_summary[target] = '\n'.join(lines)
1386 if lines:
1387 Print('%s:' % arch)
1388 _OutputConfigInfo(lines)
1389
1390 for lines, targets in lines_by_target.iteritems():
1391 if not lines:
1392 continue
1393 Print('%s :' % ' '.join(sorted(targets)))
1394 _OutputConfigInfo(lines.split('\n'))
1395
Simon Glass843312d2015-02-05 22:06:15 -07001396
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001397 # Save our updated information for the next call to this function
1398 self._base_board_dict = board_dict
1399 self._base_err_lines = err_lines
Simon Glasse30965d2014-08-28 09:43:44 -06001400 self._base_warn_lines = warn_lines
1401 self._base_err_line_boards = err_line_boards
1402 self._base_warn_line_boards = warn_line_boards
Simon Glass843312d2015-02-05 22:06:15 -07001403 self._base_config = config
Alex Kiernan48ae4122018-05-31 04:48:34 +00001404 self._base_environment = environment
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001405
1406 # Get a list of boards that did not get built, if needed
1407 not_built = []
1408 for board in board_selected:
1409 if not board in board_dict:
1410 not_built.append(board)
1411 if not_built:
Simon Glass4653a882014-09-05 19:00:07 -06001412 Print("Boards not built (%d): %s" % (len(not_built),
1413 ', '.join(not_built)))
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001414
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001415 def ProduceResultSummary(self, commit_upto, commits, board_selected):
Simon Glasse30965d2014-08-28 09:43:44 -06001416 (board_dict, err_lines, err_line_boards, warn_lines,
Alex Kiernan48ae4122018-05-31 04:48:34 +00001417 warn_line_boards, config, environment) = self.GetResultSummary(
Simon Glassed966652014-08-28 09:43:43 -06001418 board_selected, commit_upto,
Simon Glass843312d2015-02-05 22:06:15 -07001419 read_func_sizes=self._show_bloat,
Alex Kiernan48ae4122018-05-31 04:48:34 +00001420 read_config=self._show_config,
1421 read_environment=self._show_environment)
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001422 if commits:
1423 msg = '%02d: %s' % (commit_upto + 1,
1424 commits[commit_upto].subject)
Simon Glass4653a882014-09-05 19:00:07 -06001425 Print(msg, colour=self.col.BLUE)
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001426 self.PrintResultSummary(board_selected, board_dict,
Simon Glassed966652014-08-28 09:43:43 -06001427 err_lines if self._show_errors else [], err_line_boards,
Simon Glasse30965d2014-08-28 09:43:44 -06001428 warn_lines if self._show_errors else [], warn_line_boards,
Alex Kiernan48ae4122018-05-31 04:48:34 +00001429 config, environment, self._show_sizes, self._show_detail,
1430 self._show_bloat, self._show_config, self._show_environment)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001431
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001432 def ShowSummary(self, commits, board_selected):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001433 """Show a build summary for U-Boot for a given board list.
1434
1435 Reset the result summary, then repeatedly call GetResultSummary on
1436 each commit's results, then display the differences we see.
1437
1438 Args:
1439 commit: Commit objects to summarise
1440 board_selected: Dict containing boards to summarise
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001441 """
Simon Glassfea58582014-08-09 15:32:59 -06001442 self.commit_count = len(commits) if commits else 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001443 self.commits = commits
1444 self.ResetResultSummary(board_selected)
Simon Glass28370c12014-08-09 15:33:06 -06001445 self._error_lines = 0
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001446
1447 for commit_upto in range(0, self.commit_count, self._step):
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001448 self.ProduceResultSummary(commit_upto, commits, board_selected)
Simon Glass28370c12014-08-09 15:33:06 -06001449 if not self._error_lines:
Simon Glass4653a882014-09-05 19:00:07 -06001450 Print('(no errors to report)', colour=self.col.GREEN)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001451
1452
1453 def SetupBuild(self, board_selected, commits):
1454 """Set up ready to start a build.
1455
1456 Args:
1457 board_selected: Selected boards to build
1458 commits: Selected commits to build
1459 """
1460 # First work out how many commits we will build
Simon Glassfea58582014-08-09 15:32:59 -06001461 count = (self.commit_count + self._step - 1) / self._step
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001462 self.count = len(board_selected) * count
1463 self.upto = self.warned = self.fail = 0
1464 self._timestamps = collections.deque()
1465
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001466 def GetThreadDir(self, thread_num):
1467 """Get the directory path to the working dir for a thread.
1468
1469 Args:
1470 thread_num: Number of thread to check.
1471 """
1472 return os.path.join(self._working_dir, '%02d' % thread_num)
1473
Simon Glassfea58582014-08-09 15:32:59 -06001474 def _PrepareThread(self, thread_num, setup_git):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001475 """Prepare the working directory for a thread.
1476
1477 This clones or fetches the repo into the thread's work directory.
1478
1479 Args:
1480 thread_num: Thread number (0, 1, ...)
Simon Glassfea58582014-08-09 15:32:59 -06001481 setup_git: True to set up a git repo clone
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001482 """
1483 thread_dir = self.GetThreadDir(thread_num)
Simon Glass190064b2014-08-09 15:33:00 -06001484 builderthread.Mkdir(thread_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001485 git_dir = os.path.join(thread_dir, '.git')
1486
1487 # Clone the repo if it doesn't already exist
1488 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1489 # we have a private index but uses the origin repo's contents?
Simon Glassfea58582014-08-09 15:32:59 -06001490 if setup_git and self.git_dir:
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001491 src_dir = os.path.abspath(self.git_dir)
1492 if os.path.exists(git_dir):
1493 gitutil.Fetch(git_dir, thread_dir)
1494 else:
Simon Glass21f0eb32016-09-18 16:48:31 -06001495 Print('\rCloning repo for thread %d' % thread_num,
1496 newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001497 gitutil.Clone(src_dir, thread_dir)
Simon Glass21f0eb32016-09-18 16:48:31 -06001498 Print('\r%s\r' % (' ' * 30), newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001499
Simon Glassfea58582014-08-09 15:32:59 -06001500 def _PrepareWorkingSpace(self, max_threads, setup_git):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001501 """Prepare the working directory for use.
1502
1503 Set up the git repo for each thread.
1504
1505 Args:
1506 max_threads: Maximum number of threads we expect to need.
Simon Glassfea58582014-08-09 15:32:59 -06001507 setup_git: True to set up a git repo clone
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001508 """
Simon Glass190064b2014-08-09 15:33:00 -06001509 builderthread.Mkdir(self._working_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001510 for thread in range(max_threads):
Simon Glassfea58582014-08-09 15:32:59 -06001511 self._PrepareThread(thread, setup_git)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001512
1513 def _PrepareOutputSpace(self):
1514 """Get the output directories ready to receive files.
1515
1516 We delete any output directories which look like ones we need to
1517 create. Having left over directories is confusing when the user wants
1518 to check the output manually.
1519 """
Simon Glass1a915672014-12-01 17:33:53 -07001520 if not self.commits:
1521 return
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001522 dir_list = []
1523 for commit_upto in range(self.commit_count):
1524 dir_list.append(self._GetOutputDir(commit_upto))
1525
Simon Glassb222abe2016-09-18 16:48:32 -06001526 to_remove = []
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001527 for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1528 if dirname not in dir_list:
Simon Glassb222abe2016-09-18 16:48:32 -06001529 to_remove.append(dirname)
1530 if to_remove:
1531 Print('Removing %d old build directories' % len(to_remove),
1532 newline=False)
1533 for dirname in to_remove:
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001534 shutil.rmtree(dirname)
1535
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001536 def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001537 """Build all commits for a list of boards
1538
1539 Args:
1540 commits: List of commits to be build, each a Commit object
1541 boards_selected: Dict of selected boards, key is target name,
1542 value is Board object
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001543 keep_outputs: True to save build output files
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001544 verbose: Display build results as they are completed
Simon Glass2c3deb92014-08-28 09:43:39 -06001545 Returns:
1546 Tuple containing:
1547 - number of boards that failed to build
1548 - number of boards that issued warnings
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001549 """
Simon Glassfea58582014-08-09 15:32:59 -06001550 self.commit_count = len(commits) if commits else 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001551 self.commits = commits
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001552 self._verbose = verbose
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001553
1554 self.ResetResultSummary(board_selected)
Thierry Redingf3d015c2014-08-19 10:22:39 +02001555 builderthread.Mkdir(self.base_dir, parents = True)
Simon Glassfea58582014-08-09 15:32:59 -06001556 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1557 commits is not None)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001558 self._PrepareOutputSpace()
Simon Glass745b3952016-09-18 16:48:33 -06001559 Print('\rStarting build...', newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001560 self.SetupBuild(board_selected, commits)
1561 self.ProcessResult(None)
1562
1563 # Create jobs to build all commits for each board
1564 for brd in board_selected.itervalues():
Simon Glass190064b2014-08-09 15:33:00 -06001565 job = builderthread.BuilderJob()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001566 job.board = brd
1567 job.commits = commits
1568 job.keep_outputs = keep_outputs
1569 job.step = self._step
1570 self.queue.put(job)
1571
Simon Glassd436e382016-09-18 16:48:35 -06001572 term = threading.Thread(target=self.queue.join)
1573 term.setDaemon(True)
1574 term.start()
1575 while term.isAlive():
1576 term.join(100)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001577
1578 # Wait until we have processed all output
1579 self.out_queue.join()
Simon Glass4653a882014-09-05 19:00:07 -06001580 Print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001581 self.ClearLine(0)
Simon Glass2c3deb92014-08-28 09:43:39 -06001582 return (self.fail, self.warned)