blob: 784c64122ba67285f2552926527126b52ae6b1b8 [file] [log] [blame]
Tom Rini83d290c2018-05-06 17:58:06 -04001# SPDX-License-Identifier: GPL-2.0+
Simon Glassfc3fe1c2013-04-03 11:07:16 +00002# Copyright (c) 2013 The Chromium OS Authors.
3#
4# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
5#
Simon Glassfc3fe1c2013-04-03 11:07:16 +00006
7import collections
Simon Glassfc3fe1c2013-04-03 11:07:16 +00008from datetime import datetime, timedelta
9import glob
10import os
11import re
Simon Glassc05aa032019-10-31 07:42:53 -060012import queue
Simon Glassfc3fe1c2013-04-03 11:07:16 +000013import shutil
Simon Glass2f256642016-09-18 16:48:37 -060014import signal
Simon Glassfc3fe1c2013-04-03 11:07:16 +000015import string
16import sys
Simon Glassd436e382016-09-18 16:48:35 -060017import threading
Simon Glassfc3fe1c2013-04-03 11:07:16 +000018import time
19
Simon Glass190064b2014-08-09 15:33:00 -060020import builderthread
Simon Glassfc3fe1c2013-04-03 11:07:16 +000021import command
22import gitutil
23import terminal
Simon Glass4653a882014-09-05 19:00:07 -060024from terminal import Print
Simon Glassfc3fe1c2013-04-03 11:07:16 +000025import toolchain
26
27
28"""
29Theory of Operation
30
31Please see README for user documentation, and you should be familiar with
32that before trying to make sense of this.
33
34Buildman works by keeping the machine as busy as possible, building different
35commits for different boards on multiple CPUs at once.
36
37The source repo (self.git_dir) contains all the commits to be built. Each
38thread works on a single board at a time. It checks out the first commit,
39configures it for that board, then builds it. Then it checks out the next
40commit and builds it (typically without re-configuring). When it runs out
41of commits, it gets another job from the builder and starts again with that
42board.
43
44Clearly the builder threads could work either way - they could check out a
45commit and then built it for all boards. Using separate directories for each
46commit/board pair they could leave their build product around afterwards
47also.
48
49The intent behind building a single board for multiple commits, is to make
50use of incremental builds. Since each commit is built incrementally from
51the previous one, builds are faster. Reconfiguring for a different board
52removes all intermediate object files.
53
54Many threads can be working at once, but each has its own working directory.
55When a thread finishes a build, it puts the output files into a result
56directory.
57
58The base directory used by buildman is normally '../<branch>', i.e.
59a directory higher than the source repository and named after the branch
60being built.
61
62Within the base directory, we have one subdirectory for each commit. Within
63that is one subdirectory for each board. Within that is the build output for
64that commit/board combination.
65
66Buildman also create working directories for each thread, in a .bm-work/
67subdirectory in the base dir.
68
69As an example, say we are building branch 'us-net' for boards 'sandbox' and
70'seaboard', and say that us-net has two commits. We will have directories
71like this:
72
73us-net/ base directory
74 01_of_02_g4ed4ebc_net--Add-tftp-speed-/
75 sandbox/
76 u-boot.bin
77 seaboard/
78 u-boot.bin
79 02_of_02_g4ed4ebc_net--Check-tftp-comp/
80 sandbox/
81 u-boot.bin
82 seaboard/
83 u-boot.bin
84 .bm-work/
85 00/ working directory for thread 0 (contains source checkout)
86 build/ build output
87 01/ working directory for thread 1
88 build/ build output
89 ...
90u-boot/ source directory
91 .git/ repository
92"""
93
94# Possible build outcomes
Simon Glassc05aa032019-10-31 07:42:53 -060095OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
Simon Glassfc3fe1c2013-04-03 11:07:16 +000096
Simon Glass9a6d2e22017-04-12 18:23:26 -060097# Translate a commit subject into a valid filename (and handle unicode)
Simon Glassc05aa032019-10-31 07:42:53 -060098trans_valid_chars = str.maketrans('/: ', '---')
Simon Glassfc3fe1c2013-04-03 11:07:16 +000099
Simon Glassb464f8e2016-11-13 14:25:53 -0700100BASE_CONFIG_FILENAMES = [
101 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
102]
103
104EXTRA_CONFIG_FILENAMES = [
Simon Glass843312d2015-02-05 22:06:15 -0700105 '.config', '.config-spl', '.config-tpl',
106 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
107 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
Simon Glass843312d2015-02-05 22:06:15 -0700108]
109
Simon Glass8270e3c2015-08-25 21:52:14 -0600110class Config:
111 """Holds information about configuration settings for a board."""
Simon Glassb464f8e2016-11-13 14:25:53 -0700112 def __init__(self, config_filename, target):
Simon Glass8270e3c2015-08-25 21:52:14 -0600113 self.target = target
114 self.config = {}
Simon Glassb464f8e2016-11-13 14:25:53 -0700115 for fname in config_filename:
Simon Glass8270e3c2015-08-25 21:52:14 -0600116 self.config[fname] = {}
117
118 def Add(self, fname, key, value):
119 self.config[fname][key] = value
120
121 def __hash__(self):
122 val = 0
123 for fname in self.config:
Simon Glassc05aa032019-10-31 07:42:53 -0600124 for key, value in self.config[fname].items():
125 print(key, value)
Simon Glass8270e3c2015-08-25 21:52:14 -0600126 val = val ^ hash(key) & hash(value)
127 return val
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000128
Alex Kiernan48ae4122018-05-31 04:48:34 +0000129class Environment:
130 """Holds information about environment variables for a board."""
131 def __init__(self, target):
132 self.target = target
133 self.environment = {}
134
135 def Add(self, key, value):
136 self.environment[key] = value
137
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000138class Builder:
139 """Class for building U-Boot for a particular commit.
140
141 Public members: (many should ->private)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000142 already_done: Number of builds already completed
143 base_dir: Base directory to use for builder
144 checkout: True to check out source, False to skip that step.
145 This is used for testing.
146 col: terminal.Color() object
147 count: Number of commits to build
148 do_make: Method to call to invoke Make
149 fail: Number of builds that failed due to error
150 force_build: Force building even if a build already exists
151 force_config_on_failure: If a commit fails for a board, disable
152 incremental building for the next commit we build for that
153 board, so that we will see all warnings/errors again.
Simon Glass4266dc22014-07-13 12:22:31 -0600154 force_build_failures: If a previously-built build (i.e. built on
155 a previous run of buildman) is marked as failed, rebuild it.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000156 git_dir: Git directory containing source repository
157 last_line_len: Length of the last line we printed (used for erasing
158 it with new progress information)
159 num_jobs: Number of jobs to run at once (passed to make as -j)
160 num_threads: Number of builder threads to run
161 out_queue: Queue of results to process
162 re_make_err: Compiled regular expression for ignore_lines
163 queue: Queue of jobs to run
164 threads: List of active threads
165 toolchains: Toolchains object to use for building
166 upto: Current commit number we are building (0.count-1)
167 warned: Number of builds that produced at least one warning
Simon Glass97e91522014-07-14 17:51:02 -0600168 force_reconfig: Reconfigure U-Boot on each comiit. This disables
169 incremental building, where buildman reconfigures on the first
170 commit for a baord, and then just does an incremental build for
171 the following commits. In fact buildman will reconfigure and
172 retry for any failing commits, so generally the only effect of
173 this option is to slow things down.
Simon Glass189a4962014-07-14 17:51:03 -0600174 in_tree: Build U-Boot in-tree instead of specifying an output
175 directory separate from the source code. This option is really
176 only useful for testing in-tree builds.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000177
178 Private members:
179 _base_board_dict: Last-summarised Dict of boards
180 _base_err_lines: Last-summarised list of errors
Simon Glasse30965d2014-08-28 09:43:44 -0600181 _base_warn_lines: Last-summarised list of warnings
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000182 _build_period_us: Time taken for a single build (float object).
183 _complete_delay: Expected delay until completion (timedelta)
184 _next_delay_update: Next time we plan to display a progress update
185 (datatime)
186 _show_unknown: Show unknown boards (those not built) in summary
187 _timestamps: List of timestamps for the completion of the last
188 last _timestamp_count builds. Each is a datetime object.
189 _timestamp_count: Number of timestamps to keep in our list.
190 _working_dir: Base working directory containing all threads
191 """
192 class Outcome:
193 """Records a build outcome for a single make invocation
194
195 Public Members:
196 rc: Outcome value (OUTCOME_...)
197 err_lines: List of error lines or [] if none
198 sizes: Dictionary of image size information, keyed by filename
199 - Each value is itself a dictionary containing
200 values for 'text', 'data' and 'bss', being the integer
201 size in bytes of each section.
202 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
203 value is itself a dictionary:
204 key: function name
205 value: Size of function in bytes
Simon Glass843312d2015-02-05 22:06:15 -0700206 config: Dictionary keyed by filename - e.g. '.config'. Each
207 value is itself a dictionary:
208 key: config name
209 value: config value
Alex Kiernan48ae4122018-05-31 04:48:34 +0000210 environment: Dictionary keyed by environment variable, Each
211 value is the value of environment variable.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000212 """
Alex Kiernan48ae4122018-05-31 04:48:34 +0000213 def __init__(self, rc, err_lines, sizes, func_sizes, config,
214 environment):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000215 self.rc = rc
216 self.err_lines = err_lines
217 self.sizes = sizes
218 self.func_sizes = func_sizes
Simon Glass843312d2015-02-05 22:06:15 -0700219 self.config = config
Alex Kiernan48ae4122018-05-31 04:48:34 +0000220 self.environment = environment
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000221
222 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
Simon Glass5971ab52014-12-01 17:33:55 -0700223 gnu_make='make', checkout=True, show_unknown=True, step=1,
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600224 no_subdirs=False, full_path=False, verbose_build=False,
Simon Glassb50113f2016-11-13 14:25:51 -0700225 incremental=False, per_board_out_dir=False,
Daniel Schwierzeck2371d1b2018-01-26 16:31:05 +0100226 config_only=False, squash_config_y=False,
227 warnings_as_errors=False):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000228 """Create a new Builder object
229
230 Args:
231 toolchains: Toolchains object to use for building
232 base_dir: Base directory to use for builder
233 git_dir: Git directory containing source repository
234 num_threads: Number of builder threads to run
235 num_jobs: Number of jobs to run at once (passed to make as -j)
Masahiro Yamada99796922014-07-22 11:19:09 +0900236 gnu_make: the command name of GNU Make.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000237 checkout: True to check out source, False to skip that step.
238 This is used for testing.
239 show_unknown: Show unknown boards (those not built) in summary
240 step: 1 to process every commit, n to process every nth commit
Simon Glassbb1501f2014-12-01 17:34:00 -0700241 no_subdirs: Don't create subdirectories when building current
242 source for a single board
243 full_path: Return the full path in CROSS_COMPILE and don't set
244 PATH
Simon Glassd2ce6582014-12-01 17:34:07 -0700245 verbose_build: Run build with V=1 and don't use 'make -s'
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600246 incremental: Always perform incremental builds; don't run make
247 mrproper when configuring
248 per_board_out_dir: Build in a separate persistent directory per
249 board rather than a thread-specific directory
Simon Glassb50113f2016-11-13 14:25:51 -0700250 config_only: Only configure each build, don't build it
Simon Glassb464f8e2016-11-13 14:25:53 -0700251 squash_config_y: Convert CONFIG options with the value 'y' to '1'
Daniel Schwierzeck2371d1b2018-01-26 16:31:05 +0100252 warnings_as_errors: Treat all compiler warnings as errors
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000253 """
254 self.toolchains = toolchains
255 self.base_dir = base_dir
256 self._working_dir = os.path.join(base_dir, '.bm-work')
257 self.threads = []
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000258 self.do_make = self.Make
Masahiro Yamada99796922014-07-22 11:19:09 +0900259 self.gnu_make = gnu_make
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000260 self.checkout = checkout
261 self.num_threads = num_threads
262 self.num_jobs = num_jobs
263 self.already_done = 0
264 self.force_build = False
265 self.git_dir = git_dir
266 self._show_unknown = show_unknown
267 self._timestamp_count = 10
268 self._build_period_us = None
269 self._complete_delay = None
270 self._next_delay_update = datetime.now()
271 self.force_config_on_failure = True
Simon Glass4266dc22014-07-13 12:22:31 -0600272 self.force_build_failures = False
Simon Glass97e91522014-07-14 17:51:02 -0600273 self.force_reconfig = False
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000274 self._step = step
Simon Glass189a4962014-07-14 17:51:03 -0600275 self.in_tree = False
Simon Glass28370c12014-08-09 15:33:06 -0600276 self._error_lines = 0
Simon Glass5971ab52014-12-01 17:33:55 -0700277 self.no_subdirs = no_subdirs
Simon Glassbb1501f2014-12-01 17:34:00 -0700278 self.full_path = full_path
Simon Glassd2ce6582014-12-01 17:34:07 -0700279 self.verbose_build = verbose_build
Simon Glassb50113f2016-11-13 14:25:51 -0700280 self.config_only = config_only
Simon Glassb464f8e2016-11-13 14:25:53 -0700281 self.squash_config_y = squash_config_y
282 self.config_filenames = BASE_CONFIG_FILENAMES
283 if not self.squash_config_y:
284 self.config_filenames += EXTRA_CONFIG_FILENAMES
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000285
Daniel Schwierzeck2371d1b2018-01-26 16:31:05 +0100286 self.warnings_as_errors = warnings_as_errors
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000287 self.col = terminal.Color()
288
Simon Glasse30965d2014-08-28 09:43:44 -0600289 self._re_function = re.compile('(.*): In function.*')
290 self._re_files = re.compile('In file included from.*')
291 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
Simon Glass2d483332018-11-06 16:02:11 -0700292 self._re_dtb_warning = re.compile('(.*): Warning .*')
Simon Glasse30965d2014-08-28 09:43:44 -0600293 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
294
Simon Glassc05aa032019-10-31 07:42:53 -0600295 self.queue = queue.Queue()
296 self.out_queue = queue.Queue()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000297 for i in range(self.num_threads):
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600298 t = builderthread.BuilderThread(self, i, incremental,
299 per_board_out_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000300 t.setDaemon(True)
301 t.start()
302 self.threads.append(t)
303
304 self.last_line_len = 0
Simon Glass190064b2014-08-09 15:33:00 -0600305 t = builderthread.ResultThread(self)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000306 t.setDaemon(True)
307 t.start()
308 self.threads.append(t)
309
310 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
311 self.re_make_err = re.compile('|'.join(ignore_lines))
312
Simon Glass2f256642016-09-18 16:48:37 -0600313 # Handle existing graceful with SIGINT / Ctrl-C
314 signal.signal(signal.SIGINT, self.signal_handler)
315
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000316 def __del__(self):
317 """Get rid of all threads created by the builder"""
318 for t in self.threads:
319 del t
320
Simon Glass2f256642016-09-18 16:48:37 -0600321 def signal_handler(self, signal, frame):
322 sys.exit(1)
323
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600324 def SetDisplayOptions(self, show_errors=False, show_sizes=False,
Simon Glassed966652014-08-28 09:43:43 -0600325 show_detail=False, show_bloat=False,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000326 list_error_boards=False, show_config=False,
327 show_environment=False):
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600328 """Setup display options for the builder.
329
330 show_errors: True to show summarised error/warning info
331 show_sizes: Show size deltas
332 show_detail: Show detail for each board
333 show_bloat: Show detail for each function
Simon Glassed966652014-08-28 09:43:43 -0600334 list_error_boards: Show the boards which caused each error/warning
Simon Glass843312d2015-02-05 22:06:15 -0700335 show_config: Show config deltas
Alex Kiernan48ae4122018-05-31 04:48:34 +0000336 show_environment: Show environment deltas
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600337 """
338 self._show_errors = show_errors
339 self._show_sizes = show_sizes
340 self._show_detail = show_detail
341 self._show_bloat = show_bloat
Simon Glassed966652014-08-28 09:43:43 -0600342 self._list_error_boards = list_error_boards
Simon Glass843312d2015-02-05 22:06:15 -0700343 self._show_config = show_config
Alex Kiernan48ae4122018-05-31 04:48:34 +0000344 self._show_environment = show_environment
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600345
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000346 def _AddTimestamp(self):
347 """Add a new timestamp to the list and record the build period.
348
349 The build period is the length of time taken to perform a single
350 build (one board, one commit).
351 """
352 now = datetime.now()
353 self._timestamps.append(now)
354 count = len(self._timestamps)
355 delta = self._timestamps[-1] - self._timestamps[0]
356 seconds = delta.total_seconds()
357
358 # If we have enough data, estimate build period (time taken for a
359 # single build) and therefore completion time.
360 if count > 1 and self._next_delay_update < now:
361 self._next_delay_update = now + timedelta(seconds=2)
362 if seconds > 0:
363 self._build_period = float(seconds) / count
364 todo = self.count - self.upto
365 self._complete_delay = timedelta(microseconds=
366 self._build_period * todo * 1000000)
367 # Round it
368 self._complete_delay -= timedelta(
369 microseconds=self._complete_delay.microseconds)
370
371 if seconds > 60:
372 self._timestamps.popleft()
373 count -= 1
374
375 def ClearLine(self, length):
376 """Clear any characters on the current line
377
378 Make way for a new line of length 'length', by outputting enough
379 spaces to clear out the old line. Then remember the new length for
380 next time.
381
382 Args:
383 length: Length of new line, in characters
384 """
385 if length < self.last_line_len:
Simon Glass4653a882014-09-05 19:00:07 -0600386 Print(' ' * (self.last_line_len - length), newline=False)
387 Print('\r', newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000388 self.last_line_len = length
389 sys.stdout.flush()
390
391 def SelectCommit(self, commit, checkout=True):
392 """Checkout the selected commit for this build
393 """
394 self.commit = commit
395 if checkout and self.checkout:
396 gitutil.Checkout(commit.hash)
397
398 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
399 """Run make
400
401 Args:
402 commit: Commit object that is being built
403 brd: Board object that is being built
Roger Meierfd18a892014-08-20 22:10:29 +0200404 stage: Stage that we are at (mrproper, config, build)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000405 cwd: Directory where make should be run
406 args: Arguments to pass to make
407 kwargs: Arguments to pass to command.RunPipe()
408 """
Masahiro Yamada99796922014-07-22 11:19:09 +0900409 cmd = [self.gnu_make] + list(args)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000410 result = command.RunPipe([cmd], capture=True, capture_stderr=True,
Simon Glasse62a24c2018-09-17 23:55:42 -0600411 cwd=cwd, raise_on_error=False, infile='/dev/null', **kwargs)
Simon Glass40f11fc2015-02-05 22:06:12 -0700412 if self.verbose_build:
413 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
414 result.combined = '%s\n' % (' '.join(cmd)) + result.combined
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000415 return result
416
417 def ProcessResult(self, result):
418 """Process the result of a build, showing progress information
419
420 Args:
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600421 result: A CommandResult object, which indicates the result for
422 a single build
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000423 """
424 col = terminal.Color()
425 if result:
426 target = result.brd.target
427
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000428 self.upto += 1
429 if result.return_code != 0:
430 self.fail += 1
431 elif result.stderr:
432 self.warned += 1
433 if result.already_done:
434 self.already_done += 1
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600435 if self._verbose:
Simon Glass4653a882014-09-05 19:00:07 -0600436 Print('\r', newline=False)
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600437 self.ClearLine(0)
438 boards_selected = {target : result.brd}
439 self.ResetResultSummary(boards_selected)
440 self.ProduceResultSummary(result.commit_upto, self.commits,
441 boards_selected)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000442 else:
443 target = '(starting)'
444
445 # Display separate counts for ok, warned and fail
446 ok = self.upto - self.warned - self.fail
447 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
448 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
449 line += self.col.Color(self.col.RED, '%5d' % self.fail)
450
451 name = ' /%-5d ' % self.count
452
453 # Add our current completion time estimate
454 self._AddTimestamp()
455 if self._complete_delay:
456 name += '%s : ' % self._complete_delay
457 # When building all boards for a commit, we can print a commit
458 # progress message.
459 if result and result.commit_upto is None:
460 name += 'commit %2d/%-3d' % (self.commit_upto + 1,
461 self.commit_count)
462
463 name += target
Simon Glass4653a882014-09-05 19:00:07 -0600464 Print(line + name, newline=False)
Simon Glass960421e2016-11-15 15:32:59 -0700465 length = 16 + len(name)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000466 self.ClearLine(length)
467
468 def _GetOutputDir(self, commit_upto):
469 """Get the name of the output directory for a commit number
470
471 The output directory is typically .../<branch>/<commit>.
472
473 Args:
474 commit_upto: Commit number to use (0..self.count-1)
475 """
Simon Glass5971ab52014-12-01 17:33:55 -0700476 commit_dir = None
Simon Glassfea58582014-08-09 15:32:59 -0600477 if self.commits:
478 commit = self.commits[commit_upto]
479 subject = commit.subject.translate(trans_valid_chars)
480 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
481 self.commit_count, commit.hash, subject[:20]))
Simon Glass5971ab52014-12-01 17:33:55 -0700482 elif not self.no_subdirs:
Simon Glassfea58582014-08-09 15:32:59 -0600483 commit_dir = 'current'
Simon Glass5971ab52014-12-01 17:33:55 -0700484 if not commit_dir:
485 return self.base_dir
486 return os.path.join(self.base_dir, commit_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000487
488 def GetBuildDir(self, commit_upto, target):
489 """Get the name of the build directory for a commit number
490
491 The build directory is typically .../<branch>/<commit>/<target>.
492
493 Args:
494 commit_upto: Commit number to use (0..self.count-1)
495 target: Target name
496 """
497 output_dir = self._GetOutputDir(commit_upto)
498 return os.path.join(output_dir, target)
499
500 def GetDoneFile(self, commit_upto, target):
501 """Get the name of the done file for a commit number
502
503 Args:
504 commit_upto: Commit number to use (0..self.count-1)
505 target: Target name
506 """
507 return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
508
509 def GetSizesFile(self, commit_upto, target):
510 """Get the name of the sizes file for a commit number
511
512 Args:
513 commit_upto: Commit number to use (0..self.count-1)
514 target: Target name
515 """
516 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
517
518 def GetFuncSizesFile(self, commit_upto, target, elf_fname):
519 """Get the name of the funcsizes file for a commit number and ELF file
520
521 Args:
522 commit_upto: Commit number to use (0..self.count-1)
523 target: Target name
524 elf_fname: Filename of elf image
525 """
526 return os.path.join(self.GetBuildDir(commit_upto, target),
527 '%s.sizes' % elf_fname.replace('/', '-'))
528
529 def GetObjdumpFile(self, commit_upto, target, elf_fname):
530 """Get the name of the objdump file for a commit number and ELF file
531
532 Args:
533 commit_upto: Commit number to use (0..self.count-1)
534 target: Target name
535 elf_fname: Filename of elf image
536 """
537 return os.path.join(self.GetBuildDir(commit_upto, target),
538 '%s.objdump' % elf_fname.replace('/', '-'))
539
540 def GetErrFile(self, commit_upto, target):
541 """Get the name of the err file for a commit number
542
543 Args:
544 commit_upto: Commit number to use (0..self.count-1)
545 target: Target name
546 """
547 output_dir = self.GetBuildDir(commit_upto, target)
548 return os.path.join(output_dir, 'err')
549
550 def FilterErrors(self, lines):
551 """Filter out errors in which we have no interest
552
553 We should probably use map().
554
555 Args:
556 lines: List of error lines, each a string
557 Returns:
558 New list with only interesting lines included
559 """
560 out_lines = []
561 for line in lines:
562 if not self.re_make_err.search(line):
563 out_lines.append(line)
564 return out_lines
565
566 def ReadFuncSizes(self, fname, fd):
567 """Read function sizes from the output of 'nm'
568
569 Args:
570 fd: File containing data to read
571 fname: Filename we are reading from (just for errors)
572
573 Returns:
574 Dictionary containing size of each function in bytes, indexed by
575 function name.
576 """
577 sym = {}
578 for line in fd.readlines():
579 try:
Tom Rinid08c38c2019-12-06 15:31:31 -0500580 if line.strip():
581 size, type, name = line[:-1].split()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000582 except:
Simon Glass4653a882014-09-05 19:00:07 -0600583 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000584 continue
585 if type in 'tTdDbB':
586 # function names begin with '.' on 64-bit powerpc
587 if '.' in name[1:]:
588 name = 'static.' + name.split('.')[0]
589 sym[name] = sym.get(name, 0) + int(size, 16)
590 return sym
591
Simon Glass843312d2015-02-05 22:06:15 -0700592 def _ProcessConfig(self, fname):
593 """Read in a .config, autoconf.mk or autoconf.h file
594
595 This function handles all config file types. It ignores comments and
596 any #defines which don't start with CONFIG_.
597
598 Args:
599 fname: Filename to read
600
601 Returns:
602 Dictionary:
603 key: Config name (e.g. CONFIG_DM)
604 value: Config value (e.g. 1)
605 """
606 config = {}
607 if os.path.exists(fname):
608 with open(fname) as fd:
609 for line in fd:
610 line = line.strip()
611 if line.startswith('#define'):
612 values = line[8:].split(' ', 1)
613 if len(values) > 1:
614 key, value = values
615 else:
616 key = values[0]
Simon Glassb464f8e2016-11-13 14:25:53 -0700617 value = '1' if self.squash_config_y else ''
Simon Glass843312d2015-02-05 22:06:15 -0700618 if not key.startswith('CONFIG_'):
619 continue
620 elif not line or line[0] in ['#', '*', '/']:
621 continue
622 else:
623 key, value = line.split('=', 1)
Simon Glassb464f8e2016-11-13 14:25:53 -0700624 if self.squash_config_y and value == 'y':
625 value = '1'
Simon Glass843312d2015-02-05 22:06:15 -0700626 config[key] = value
627 return config
628
Alex Kiernan48ae4122018-05-31 04:48:34 +0000629 def _ProcessEnvironment(self, fname):
630 """Read in a uboot.env file
631
632 This function reads in environment variables from a file.
633
634 Args:
635 fname: Filename to read
636
637 Returns:
638 Dictionary:
639 key: environment variable (e.g. bootlimit)
640 value: value of environment variable (e.g. 1)
641 """
642 environment = {}
643 if os.path.exists(fname):
644 with open(fname) as fd:
645 for line in fd.read().split('\0'):
646 try:
647 key, value = line.split('=', 1)
648 environment[key] = value
649 except ValueError:
650 # ignore lines we can't parse
651 pass
652 return environment
653
Simon Glass843312d2015-02-05 22:06:15 -0700654 def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000655 read_config, read_environment):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000656 """Work out the outcome of a build.
657
658 Args:
659 commit_upto: Commit number to check (0..n-1)
660 target: Target board to check
661 read_func_sizes: True to read function size information
Simon Glass843312d2015-02-05 22:06:15 -0700662 read_config: True to read .config and autoconf.h files
Alex Kiernan48ae4122018-05-31 04:48:34 +0000663 read_environment: True to read uboot.env files
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000664
665 Returns:
666 Outcome object
667 """
668 done_file = self.GetDoneFile(commit_upto, target)
669 sizes_file = self.GetSizesFile(commit_upto, target)
670 sizes = {}
671 func_sizes = {}
Simon Glass843312d2015-02-05 22:06:15 -0700672 config = {}
Alex Kiernan48ae4122018-05-31 04:48:34 +0000673 environment = {}
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000674 if os.path.exists(done_file):
675 with open(done_file, 'r') as fd:
Simon Glass347ea0b2019-04-26 19:02:23 -0600676 try:
677 return_code = int(fd.readline())
678 except ValueError:
679 # The file may be empty due to running out of disk space.
680 # Try a rebuild
681 return_code = 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000682 err_lines = []
683 err_file = self.GetErrFile(commit_upto, target)
684 if os.path.exists(err_file):
685 with open(err_file, 'r') as fd:
686 err_lines = self.FilterErrors(fd.readlines())
687
688 # Decide whether the build was ok, failed or created warnings
689 if return_code:
690 rc = OUTCOME_ERROR
691 elif len(err_lines):
692 rc = OUTCOME_WARNING
693 else:
694 rc = OUTCOME_OK
695
696 # Convert size information to our simple format
697 if os.path.exists(sizes_file):
698 with open(sizes_file, 'r') as fd:
699 for line in fd.readlines():
700 values = line.split()
701 rodata = 0
702 if len(values) > 6:
703 rodata = int(values[6], 16)
704 size_dict = {
705 'all' : int(values[0]) + int(values[1]) +
706 int(values[2]),
707 'text' : int(values[0]) - rodata,
708 'data' : int(values[1]),
709 'bss' : int(values[2]),
710 'rodata' : rodata,
711 }
712 sizes[values[5]] = size_dict
713
714 if read_func_sizes:
715 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
716 for fname in glob.glob(pattern):
717 with open(fname, 'r') as fd:
718 dict_name = os.path.basename(fname).replace('.sizes',
719 '')
720 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
721
Simon Glass843312d2015-02-05 22:06:15 -0700722 if read_config:
723 output_dir = self.GetBuildDir(commit_upto, target)
Simon Glassb464f8e2016-11-13 14:25:53 -0700724 for name in self.config_filenames:
Simon Glass843312d2015-02-05 22:06:15 -0700725 fname = os.path.join(output_dir, name)
726 config[name] = self._ProcessConfig(fname)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000727
Alex Kiernan48ae4122018-05-31 04:48:34 +0000728 if read_environment:
729 output_dir = self.GetBuildDir(commit_upto, target)
730 fname = os.path.join(output_dir, 'uboot.env')
731 environment = self._ProcessEnvironment(fname)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000732
Alex Kiernan48ae4122018-05-31 04:48:34 +0000733 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config,
734 environment)
735
736 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {})
Simon Glass843312d2015-02-05 22:06:15 -0700737
738 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000739 read_config, read_environment):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000740 """Calculate a summary of the results of building a commit.
741
742 Args:
743 board_selected: Dict containing boards to summarise
744 commit_upto: Commit number to summarize (0..self.count-1)
745 read_func_sizes: True to read function size information
Simon Glass843312d2015-02-05 22:06:15 -0700746 read_config: True to read .config and autoconf.h files
Alex Kiernan48ae4122018-05-31 04:48:34 +0000747 read_environment: True to read uboot.env files
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000748
749 Returns:
750 Tuple:
751 Dict containing boards which passed building this commit.
752 keyed by board.target
Simon Glasse30965d2014-08-28 09:43:44 -0600753 List containing a summary of error lines
Simon Glassed966652014-08-28 09:43:43 -0600754 Dict keyed by error line, containing a list of the Board
755 objects with that error
Simon Glasse30965d2014-08-28 09:43:44 -0600756 List containing a summary of warning lines
757 Dict keyed by error line, containing a list of the Board
758 objects with that warning
Simon Glass8270e3c2015-08-25 21:52:14 -0600759 Dictionary keyed by board.target. Each value is a dictionary:
760 key: filename - e.g. '.config'
Simon Glass843312d2015-02-05 22:06:15 -0700761 value is itself a dictionary:
762 key: config name
763 value: config value
Alex Kiernan48ae4122018-05-31 04:48:34 +0000764 Dictionary keyed by board.target. Each value is a dictionary:
765 key: environment variable
766 value: value of environment variable
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000767 """
Simon Glasse30965d2014-08-28 09:43:44 -0600768 def AddLine(lines_summary, lines_boards, line, board):
769 line = line.rstrip()
770 if line in lines_boards:
771 lines_boards[line].append(board)
772 else:
773 lines_boards[line] = [board]
774 lines_summary.append(line)
775
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000776 board_dict = {}
777 err_lines_summary = []
Simon Glassed966652014-08-28 09:43:43 -0600778 err_lines_boards = {}
Simon Glasse30965d2014-08-28 09:43:44 -0600779 warn_lines_summary = []
780 warn_lines_boards = {}
Simon Glass843312d2015-02-05 22:06:15 -0700781 config = {}
Alex Kiernan48ae4122018-05-31 04:48:34 +0000782 environment = {}
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000783
Simon Glassc05aa032019-10-31 07:42:53 -0600784 for board in boards_selected.values():
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000785 outcome = self.GetBuildOutcome(commit_upto, board.target,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000786 read_func_sizes, read_config,
787 read_environment)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000788 board_dict[board.target] = outcome
Simon Glasse30965d2014-08-28 09:43:44 -0600789 last_func = None
790 last_was_warning = False
791 for line in outcome.err_lines:
792 if line:
793 if (self._re_function.match(line) or
794 self._re_files.match(line)):
795 last_func = line
Simon Glassed966652014-08-28 09:43:43 -0600796 else:
Simon Glass2d483332018-11-06 16:02:11 -0700797 is_warning = (self._re_warning.match(line) or
798 self._re_dtb_warning.match(line))
Simon Glasse30965d2014-08-28 09:43:44 -0600799 is_note = self._re_note.match(line)
800 if is_warning or (last_was_warning and is_note):
801 if last_func:
802 AddLine(warn_lines_summary, warn_lines_boards,
803 last_func, board)
804 AddLine(warn_lines_summary, warn_lines_boards,
805 line, board)
806 else:
807 if last_func:
808 AddLine(err_lines_summary, err_lines_boards,
809 last_func, board)
810 AddLine(err_lines_summary, err_lines_boards,
811 line, board)
812 last_was_warning = is_warning
813 last_func = None
Simon Glassb464f8e2016-11-13 14:25:53 -0700814 tconfig = Config(self.config_filenames, board.target)
815 for fname in self.config_filenames:
Simon Glass843312d2015-02-05 22:06:15 -0700816 if outcome.config:
Simon Glassc05aa032019-10-31 07:42:53 -0600817 for key, value in outcome.config[fname].items():
Simon Glass8270e3c2015-08-25 21:52:14 -0600818 tconfig.Add(fname, key, value)
819 config[board.target] = tconfig
Simon Glass843312d2015-02-05 22:06:15 -0700820
Alex Kiernan48ae4122018-05-31 04:48:34 +0000821 tenvironment = Environment(board.target)
822 if outcome.environment:
Simon Glassc05aa032019-10-31 07:42:53 -0600823 for key, value in outcome.environment.items():
Alex Kiernan48ae4122018-05-31 04:48:34 +0000824 tenvironment.Add(key, value)
825 environment[board.target] = tenvironment
826
Simon Glasse30965d2014-08-28 09:43:44 -0600827 return (board_dict, err_lines_summary, err_lines_boards,
Alex Kiernan48ae4122018-05-31 04:48:34 +0000828 warn_lines_summary, warn_lines_boards, config, environment)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000829
830 def AddOutcome(self, board_dict, arch_list, changes, char, color):
831 """Add an output to our list of outcomes for each architecture
832
833 This simple function adds failing boards (changes) to the
834 relevant architecture string, so we can print the results out
835 sorted by architecture.
836
837 Args:
838 board_dict: Dict containing all boards
839 arch_list: Dict keyed by arch name. Value is a string containing
840 a list of board names which failed for that arch.
841 changes: List of boards to add to arch_list
842 color: terminal.Colour object
843 """
844 done_arch = {}
845 for target in changes:
846 if target in board_dict:
847 arch = board_dict[target].arch
848 else:
849 arch = 'unknown'
850 str = self.col.Color(color, ' ' + target)
851 if not arch in done_arch:
Simon Glass63c619e2015-02-05 22:06:11 -0700852 str = ' %s %s' % (self.col.Color(color, char), str)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000853 done_arch[arch] = True
854 if not arch in arch_list:
855 arch_list[arch] = str
856 else:
857 arch_list[arch] += str
858
859
860 def ColourNum(self, num):
861 color = self.col.RED if num > 0 else self.col.GREEN
862 if num == 0:
863 return '0'
864 return self.col.Color(color, str(num))
865
866 def ResetResultSummary(self, board_selected):
867 """Reset the results summary ready for use.
868
869 Set up the base board list to be all those selected, and set the
870 error lines to empty.
871
872 Following this, calls to PrintResultSummary() will use this
873 information to work out what has changed.
874
875 Args:
876 board_selected: Dict containing boards to summarise, keyed by
877 board.target
878 """
879 self._base_board_dict = {}
880 for board in board_selected:
Alex Kiernan48ae4122018-05-31 04:48:34 +0000881 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {},
882 {})
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000883 self._base_err_lines = []
Simon Glasse30965d2014-08-28 09:43:44 -0600884 self._base_warn_lines = []
885 self._base_err_line_boards = {}
886 self._base_warn_line_boards = {}
Simon Glass8270e3c2015-08-25 21:52:14 -0600887 self._base_config = None
Alex Kiernan48ae4122018-05-31 04:48:34 +0000888 self._base_environment = None
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000889
890 def PrintFuncSizeDetail(self, fname, old, new):
891 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
892 delta, common = [], {}
893
894 for a in old:
895 if a in new:
896 common[a] = 1
897
898 for name in old:
899 if name not in common:
900 remove += 1
901 down += old[name]
902 delta.append([-old[name], name])
903
904 for name in new:
905 if name not in common:
906 add += 1
907 up += new[name]
908 delta.append([new[name], name])
909
910 for name in common:
911 diff = new.get(name, 0) - old.get(name, 0)
912 if diff > 0:
913 grow, up = grow + 1, up + diff
914 elif diff < 0:
915 shrink, down = shrink + 1, down - diff
916 delta.append([diff, name])
917
918 delta.sort()
919 delta.reverse()
920
921 args = [add, -remove, grow, -shrink, up, -down, up - down]
Tom Rinid5686a62017-05-22 13:48:52 -0400922 if max(args) == 0 and min(args) == 0:
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000923 return
924 args = [self.ColourNum(x) for x in args]
925 indent = ' ' * 15
Simon Glass4653a882014-09-05 19:00:07 -0600926 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
927 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
928 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
929 'delta'))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000930 for diff, name in delta:
931 if diff:
932 color = self.col.RED if diff > 0 else self.col.GREEN
933 msg = '%s %-38s %7s %7s %+7d' % (indent, name,
934 old.get(name, '-'), new.get(name,'-'), diff)
Simon Glass4653a882014-09-05 19:00:07 -0600935 Print(msg, colour=color)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000936
937
938 def PrintSizeDetail(self, target_list, show_bloat):
939 """Show details size information for each board
940
941 Args:
942 target_list: List of targets, each a dict containing:
943 'target': Target name
944 'total_diff': Total difference in bytes across all areas
945 <part_name>: Difference for that part
946 show_bloat: Show detail for each function
947 """
948 targets_by_diff = sorted(target_list, reverse=True,
949 key=lambda x: x['_total_diff'])
950 for result in targets_by_diff:
951 printed_target = False
952 for name in sorted(result):
953 diff = result[name]
954 if name.startswith('_'):
955 continue
956 if diff != 0:
957 color = self.col.RED if diff > 0 else self.col.GREEN
958 msg = ' %s %+d' % (name, diff)
959 if not printed_target:
Simon Glass4653a882014-09-05 19:00:07 -0600960 Print('%10s %-15s:' % ('', result['_target']),
961 newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000962 printed_target = True
Simon Glass4653a882014-09-05 19:00:07 -0600963 Print(msg, colour=color, newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000964 if printed_target:
Simon Glass4653a882014-09-05 19:00:07 -0600965 Print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000966 if show_bloat:
967 target = result['_target']
968 outcome = result['_outcome']
969 base_outcome = self._base_board_dict[target]
970 for fname in outcome.func_sizes:
971 self.PrintFuncSizeDetail(fname,
972 base_outcome.func_sizes[fname],
973 outcome.func_sizes[fname])
974
975
976 def PrintSizeSummary(self, board_selected, board_dict, show_detail,
977 show_bloat):
978 """Print a summary of image sizes broken down by section.
979
980 The summary takes the form of one line per architecture. The
981 line contains deltas for each of the sections (+ means the section
982 got bigger, - means smaller). The nunmbers are the average number
983 of bytes that a board in this section increased by.
984
985 For example:
986 powerpc: (622 boards) text -0.0
987 arm: (285 boards) text -0.0
988 nds32: (3 boards) text -8.0
989
990 Args:
991 board_selected: Dict containing boards to summarise, keyed by
992 board.target
993 board_dict: Dict containing boards for which we built this
994 commit, keyed by board.target. The value is an Outcome object.
995 show_detail: Show detail for each board
996 show_bloat: Show detail for each function
997 """
998 arch_list = {}
999 arch_count = {}
1000
1001 # Calculate changes in size for different image parts
1002 # The previous sizes are in Board.sizes, for each board
1003 for target in board_dict:
1004 if target not in board_selected:
1005 continue
1006 base_sizes = self._base_board_dict[target].sizes
1007 outcome = board_dict[target]
1008 sizes = outcome.sizes
1009
1010 # Loop through the list of images, creating a dict of size
1011 # changes for each image/part. We end up with something like
1012 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1013 # which means that U-Boot data increased by 5 bytes and SPL
1014 # text decreased by 4.
1015 err = {'_target' : target}
1016 for image in sizes:
1017 if image in base_sizes:
1018 base_image = base_sizes[image]
1019 # Loop through the text, data, bss parts
1020 for part in sorted(sizes[image]):
1021 diff = sizes[image][part] - base_image[part]
1022 col = None
1023 if diff:
1024 if image == 'u-boot':
1025 name = part
1026 else:
1027 name = image + ':' + part
1028 err[name] = diff
1029 arch = board_selected[target].arch
1030 if not arch in arch_count:
1031 arch_count[arch] = 1
1032 else:
1033 arch_count[arch] += 1
1034 if not sizes:
1035 pass # Only add to our list when we have some stats
1036 elif not arch in arch_list:
1037 arch_list[arch] = [err]
1038 else:
1039 arch_list[arch].append(err)
1040
1041 # We now have a list of image size changes sorted by arch
1042 # Print out a summary of these
Simon Glassc05aa032019-10-31 07:42:53 -06001043 for arch, target_list in arch_list.items():
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001044 # Get total difference for each type
1045 totals = {}
1046 for result in target_list:
1047 total = 0
Simon Glassc05aa032019-10-31 07:42:53 -06001048 for name, diff in result.items():
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001049 if name.startswith('_'):
1050 continue
1051 total += diff
1052 if name in totals:
1053 totals[name] += diff
1054 else:
1055 totals[name] = diff
1056 result['_total_diff'] = total
1057 result['_outcome'] = board_dict[result['_target']]
1058
1059 count = len(target_list)
1060 printed_arch = False
1061 for name in sorted(totals):
1062 diff = totals[name]
1063 if diff:
1064 # Display the average difference in this name for this
1065 # architecture
1066 avg_diff = float(diff) / count
1067 color = self.col.RED if avg_diff > 0 else self.col.GREEN
1068 msg = ' %s %+1.1f' % (name, avg_diff)
1069 if not printed_arch:
Simon Glass4653a882014-09-05 19:00:07 -06001070 Print('%10s: (for %d/%d boards)' % (arch, count,
1071 arch_count[arch]), newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001072 printed_arch = True
Simon Glass4653a882014-09-05 19:00:07 -06001073 Print(msg, colour=color, newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001074
1075 if printed_arch:
Simon Glass4653a882014-09-05 19:00:07 -06001076 Print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001077 if show_detail:
1078 self.PrintSizeDetail(target_list, show_bloat)
1079
1080
1081 def PrintResultSummary(self, board_selected, board_dict, err_lines,
Simon Glasse30965d2014-08-28 09:43:44 -06001082 err_line_boards, warn_lines, warn_line_boards,
Alex Kiernan48ae4122018-05-31 04:48:34 +00001083 config, environment, show_sizes, show_detail,
1084 show_bloat, show_config, show_environment):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001085 """Compare results with the base results and display delta.
1086
1087 Only boards mentioned in board_selected will be considered. This
1088 function is intended to be called repeatedly with the results of
1089 each commit. It therefore shows a 'diff' between what it saw in
1090 the last call and what it sees now.
1091
1092 Args:
1093 board_selected: Dict containing boards to summarise, keyed by
1094 board.target
1095 board_dict: Dict containing boards for which we built this
1096 commit, keyed by board.target. The value is an Outcome object.
1097 err_lines: A list of errors for this commit, or [] if there is
1098 none, or we don't want to print errors
Simon Glassed966652014-08-28 09:43:43 -06001099 err_line_boards: Dict keyed by error line, containing a list of
1100 the Board objects with that error
Simon Glasse30965d2014-08-28 09:43:44 -06001101 warn_lines: A list of warnings for this commit, or [] if there is
1102 none, or we don't want to print errors
1103 warn_line_boards: Dict keyed by warning line, containing a list of
1104 the Board objects with that warning
Simon Glass843312d2015-02-05 22:06:15 -07001105 config: Dictionary keyed by filename - e.g. '.config'. Each
1106 value is itself a dictionary:
1107 key: config name
1108 value: config value
Alex Kiernan48ae4122018-05-31 04:48:34 +00001109 environment: Dictionary keyed by environment variable, Each
1110 value is the value of environment variable.
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001111 show_sizes: Show image size deltas
1112 show_detail: Show detail for each board
1113 show_bloat: Show detail for each function
Simon Glass843312d2015-02-05 22:06:15 -07001114 show_config: Show config changes
Alex Kiernan48ae4122018-05-31 04:48:34 +00001115 show_environment: Show environment changes
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001116 """
Simon Glasse30965d2014-08-28 09:43:44 -06001117 def _BoardList(line, line_boards):
Simon Glassed966652014-08-28 09:43:43 -06001118 """Helper function to get a line of boards containing a line
1119
1120 Args:
1121 line: Error line to search for
1122 Return:
1123 String containing a list of boards with that error line, or
1124 '' if the user has not requested such a list
1125 """
1126 if self._list_error_boards:
1127 names = []
Simon Glasse30965d2014-08-28 09:43:44 -06001128 for board in line_boards[line]:
Simon Glassf66153b2014-10-16 01:05:55 -06001129 if not board.target in names:
1130 names.append(board.target)
Simon Glassed966652014-08-28 09:43:43 -06001131 names_str = '(%s) ' % ','.join(names)
1132 else:
1133 names_str = ''
1134 return names_str
1135
Simon Glasse30965d2014-08-28 09:43:44 -06001136 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1137 char):
1138 better_lines = []
1139 worse_lines = []
1140 for line in lines:
1141 if line not in base_lines:
1142 worse_lines.append(char + '+' +
1143 _BoardList(line, line_boards) + line)
1144 for line in base_lines:
1145 if line not in lines:
1146 better_lines.append(char + '-' +
1147 _BoardList(line, base_line_boards) + line)
1148 return better_lines, worse_lines
1149
Simon Glass843312d2015-02-05 22:06:15 -07001150 def _CalcConfig(delta, name, config):
1151 """Calculate configuration changes
1152
1153 Args:
1154 delta: Type of the delta, e.g. '+'
1155 name: name of the file which changed (e.g. .config)
1156 config: configuration change dictionary
1157 key: config name
1158 value: config value
1159 Returns:
1160 String containing the configuration changes which can be
1161 printed
1162 """
1163 out = ''
1164 for key in sorted(config.keys()):
1165 out += '%s=%s ' % (key, config[key])
Simon Glass8270e3c2015-08-25 21:52:14 -06001166 return '%s %s: %s' % (delta, name, out)
Simon Glass843312d2015-02-05 22:06:15 -07001167
Simon Glass8270e3c2015-08-25 21:52:14 -06001168 def _AddConfig(lines, name, config_plus, config_minus, config_change):
1169 """Add changes in configuration to a list
Simon Glass843312d2015-02-05 22:06:15 -07001170
1171 Args:
Simon Glass8270e3c2015-08-25 21:52:14 -06001172 lines: list to add to
1173 name: config file name
Simon Glass843312d2015-02-05 22:06:15 -07001174 config_plus: configurations added, dictionary
1175 key: config name
1176 value: config value
1177 config_minus: configurations removed, dictionary
1178 key: config name
1179 value: config value
1180 config_change: configurations changed, dictionary
1181 key: config name
1182 value: config value
1183 """
1184 if config_plus:
Simon Glass8270e3c2015-08-25 21:52:14 -06001185 lines.append(_CalcConfig('+', name, config_plus))
Simon Glass843312d2015-02-05 22:06:15 -07001186 if config_minus:
Simon Glass8270e3c2015-08-25 21:52:14 -06001187 lines.append(_CalcConfig('-', name, config_minus))
Simon Glass843312d2015-02-05 22:06:15 -07001188 if config_change:
Simon Glass8270e3c2015-08-25 21:52:14 -06001189 lines.append(_CalcConfig('c', name, config_change))
1190
1191 def _OutputConfigInfo(lines):
1192 for line in lines:
1193 if not line:
1194 continue
1195 if line[0] == '+':
1196 col = self.col.GREEN
1197 elif line[0] == '-':
1198 col = self.col.RED
1199 elif line[0] == 'c':
1200 col = self.col.YELLOW
1201 Print(' ' + line, newline=True, colour=col)
1202
Simon Glass843312d2015-02-05 22:06:15 -07001203
Simon Glass4cf2b222018-11-06 16:02:12 -07001204 ok_boards = [] # List of boards fixed since last commit
Simon Glass6af71012018-11-06 16:02:13 -07001205 warn_boards = [] # List of boards with warnings since last commit
Simon Glass4cf2b222018-11-06 16:02:12 -07001206 err_boards = [] # List of new broken boards since last commit
1207 new_boards = [] # List of boards that didn't exist last time
1208 unknown_boards = [] # List of boards that were not built
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001209
1210 for target in board_dict:
1211 if target not in board_selected:
1212 continue
1213
1214 # If the board was built last time, add its outcome to a list
1215 if target in self._base_board_dict:
1216 base_outcome = self._base_board_dict[target].rc
1217 outcome = board_dict[target]
1218 if outcome.rc == OUTCOME_UNKNOWN:
Simon Glass4cf2b222018-11-06 16:02:12 -07001219 unknown_boards.append(target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001220 elif outcome.rc < base_outcome:
Simon Glass6af71012018-11-06 16:02:13 -07001221 if outcome.rc == OUTCOME_WARNING:
1222 warn_boards.append(target)
1223 else:
1224 ok_boards.append(target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001225 elif outcome.rc > base_outcome:
Simon Glass6af71012018-11-06 16:02:13 -07001226 if outcome.rc == OUTCOME_WARNING:
1227 warn_boards.append(target)
1228 else:
1229 err_boards.append(target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001230 else:
Simon Glass4cf2b222018-11-06 16:02:12 -07001231 new_boards.append(target)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001232
1233 # Get a list of errors that have appeared, and disappeared
Simon Glasse30965d2014-08-28 09:43:44 -06001234 better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1235 self._base_err_line_boards, err_lines, err_line_boards, '')
1236 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1237 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001238
1239 # Display results by arch
Simon Glass6af71012018-11-06 16:02:13 -07001240 if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards,
1241 worse_err, better_err, worse_warn, better_warn)):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001242 arch_list = {}
Simon Glass4cf2b222018-11-06 16:02:12 -07001243 self.AddOutcome(board_selected, arch_list, ok_boards, '',
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001244 self.col.GREEN)
Simon Glass6af71012018-11-06 16:02:13 -07001245 self.AddOutcome(board_selected, arch_list, warn_boards, 'w+',
1246 self.col.YELLOW)
Simon Glass4cf2b222018-11-06 16:02:12 -07001247 self.AddOutcome(board_selected, arch_list, err_boards, '+',
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001248 self.col.RED)
Simon Glass4cf2b222018-11-06 16:02:12 -07001249 self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001250 if self._show_unknown:
Simon Glass4cf2b222018-11-06 16:02:12 -07001251 self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001252 self.col.MAGENTA)
Simon Glassc05aa032019-10-31 07:42:53 -06001253 for arch, target_list in arch_list.items():
Simon Glass4653a882014-09-05 19:00:07 -06001254 Print('%10s: %s' % (arch, target_list))
Simon Glass28370c12014-08-09 15:33:06 -06001255 self._error_lines += 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001256 if better_err:
Simon Glass4653a882014-09-05 19:00:07 -06001257 Print('\n'.join(better_err), colour=self.col.GREEN)
Simon Glass28370c12014-08-09 15:33:06 -06001258 self._error_lines += 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001259 if worse_err:
Simon Glass4653a882014-09-05 19:00:07 -06001260 Print('\n'.join(worse_err), colour=self.col.RED)
Simon Glass28370c12014-08-09 15:33:06 -06001261 self._error_lines += 1
Simon Glasse30965d2014-08-28 09:43:44 -06001262 if better_warn:
Simon Glass4653a882014-09-05 19:00:07 -06001263 Print('\n'.join(better_warn), colour=self.col.CYAN)
Simon Glasse30965d2014-08-28 09:43:44 -06001264 self._error_lines += 1
1265 if worse_warn:
Simon Glass4653a882014-09-05 19:00:07 -06001266 Print('\n'.join(worse_warn), colour=self.col.MAGENTA)
Simon Glasse30965d2014-08-28 09:43:44 -06001267 self._error_lines += 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001268
1269 if show_sizes:
1270 self.PrintSizeSummary(board_selected, board_dict, show_detail,
1271 show_bloat)
1272
Alex Kiernan48ae4122018-05-31 04:48:34 +00001273 if show_environment and self._base_environment:
1274 lines = []
1275
1276 for target in board_dict:
1277 if target not in board_selected:
1278 continue
1279
1280 tbase = self._base_environment[target]
1281 tenvironment = environment[target]
1282 environment_plus = {}
1283 environment_minus = {}
1284 environment_change = {}
1285 base = tbase.environment
Simon Glassc05aa032019-10-31 07:42:53 -06001286 for key, value in tenvironment.environment.items():
Alex Kiernan48ae4122018-05-31 04:48:34 +00001287 if key not in base:
1288 environment_plus[key] = value
Simon Glassc05aa032019-10-31 07:42:53 -06001289 for key, value in base.items():
Alex Kiernan48ae4122018-05-31 04:48:34 +00001290 if key not in tenvironment.environment:
1291 environment_minus[key] = value
Simon Glassc05aa032019-10-31 07:42:53 -06001292 for key, value in base.items():
Alex Kiernan48ae4122018-05-31 04:48:34 +00001293 new_value = tenvironment.environment.get(key)
1294 if new_value and value != new_value:
1295 desc = '%s -> %s' % (value, new_value)
1296 environment_change[key] = desc
1297
1298 _AddConfig(lines, target, environment_plus, environment_minus,
1299 environment_change)
1300
1301 _OutputConfigInfo(lines)
1302
Simon Glass8270e3c2015-08-25 21:52:14 -06001303 if show_config and self._base_config:
1304 summary = {}
1305 arch_config_plus = {}
1306 arch_config_minus = {}
1307 arch_config_change = {}
1308 arch_list = []
1309
1310 for target in board_dict:
1311 if target not in board_selected:
Simon Glass843312d2015-02-05 22:06:15 -07001312 continue
Simon Glass8270e3c2015-08-25 21:52:14 -06001313 arch = board_selected[target].arch
1314 if arch not in arch_list:
1315 arch_list.append(arch)
1316
1317 for arch in arch_list:
1318 arch_config_plus[arch] = {}
1319 arch_config_minus[arch] = {}
1320 arch_config_change[arch] = {}
Simon Glassb464f8e2016-11-13 14:25:53 -07001321 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001322 arch_config_plus[arch][name] = {}
1323 arch_config_minus[arch][name] = {}
1324 arch_config_change[arch][name] = {}
1325
1326 for target in board_dict:
1327 if target not in board_selected:
1328 continue
1329
1330 arch = board_selected[target].arch
1331
1332 all_config_plus = {}
1333 all_config_minus = {}
1334 all_config_change = {}
1335 tbase = self._base_config[target]
1336 tconfig = config[target]
1337 lines = []
Simon Glassb464f8e2016-11-13 14:25:53 -07001338 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001339 if not tconfig.config[name]:
1340 continue
1341 config_plus = {}
1342 config_minus = {}
1343 config_change = {}
1344 base = tbase.config[name]
Simon Glassc05aa032019-10-31 07:42:53 -06001345 for key, value in tconfig.config[name].items():
Simon Glass8270e3c2015-08-25 21:52:14 -06001346 if key not in base:
1347 config_plus[key] = value
1348 all_config_plus[key] = value
Simon Glassc05aa032019-10-31 07:42:53 -06001349 for key, value in base.items():
Simon Glass8270e3c2015-08-25 21:52:14 -06001350 if key not in tconfig.config[name]:
1351 config_minus[key] = value
1352 all_config_minus[key] = value
Simon Glassc05aa032019-10-31 07:42:53 -06001353 for key, value in base.items():
Simon Glass8270e3c2015-08-25 21:52:14 -06001354 new_value = tconfig.config.get(key)
1355 if new_value and value != new_value:
1356 desc = '%s -> %s' % (value, new_value)
1357 config_change[key] = desc
1358 all_config_change[key] = desc
1359
1360 arch_config_plus[arch][name].update(config_plus)
1361 arch_config_minus[arch][name].update(config_minus)
1362 arch_config_change[arch][name].update(config_change)
1363
1364 _AddConfig(lines, name, config_plus, config_minus,
1365 config_change)
1366 _AddConfig(lines, 'all', all_config_plus, all_config_minus,
1367 all_config_change)
1368 summary[target] = '\n'.join(lines)
1369
1370 lines_by_target = {}
Simon Glassc05aa032019-10-31 07:42:53 -06001371 for target, lines in summary.items():
Simon Glass8270e3c2015-08-25 21:52:14 -06001372 if lines in lines_by_target:
1373 lines_by_target[lines].append(target)
1374 else:
1375 lines_by_target[lines] = [target]
1376
1377 for arch in arch_list:
1378 lines = []
1379 all_plus = {}
1380 all_minus = {}
1381 all_change = {}
Simon Glassb464f8e2016-11-13 14:25:53 -07001382 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001383 all_plus.update(arch_config_plus[arch][name])
1384 all_minus.update(arch_config_minus[arch][name])
1385 all_change.update(arch_config_change[arch][name])
1386 _AddConfig(lines, name, arch_config_plus[arch][name],
1387 arch_config_minus[arch][name],
1388 arch_config_change[arch][name])
1389 _AddConfig(lines, 'all', all_plus, all_minus, all_change)
1390 #arch_summary[target] = '\n'.join(lines)
1391 if lines:
1392 Print('%s:' % arch)
1393 _OutputConfigInfo(lines)
1394
Simon Glassc05aa032019-10-31 07:42:53 -06001395 for lines, targets in lines_by_target.items():
Simon Glass8270e3c2015-08-25 21:52:14 -06001396 if not lines:
1397 continue
1398 Print('%s :' % ' '.join(sorted(targets)))
1399 _OutputConfigInfo(lines.split('\n'))
1400
Simon Glass843312d2015-02-05 22:06:15 -07001401
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001402 # Save our updated information for the next call to this function
1403 self._base_board_dict = board_dict
1404 self._base_err_lines = err_lines
Simon Glasse30965d2014-08-28 09:43:44 -06001405 self._base_warn_lines = warn_lines
1406 self._base_err_line_boards = err_line_boards
1407 self._base_warn_line_boards = warn_line_boards
Simon Glass843312d2015-02-05 22:06:15 -07001408 self._base_config = config
Alex Kiernan48ae4122018-05-31 04:48:34 +00001409 self._base_environment = environment
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001410
1411 # Get a list of boards that did not get built, if needed
1412 not_built = []
1413 for board in board_selected:
1414 if not board in board_dict:
1415 not_built.append(board)
1416 if not_built:
Simon Glass4653a882014-09-05 19:00:07 -06001417 Print("Boards not built (%d): %s" % (len(not_built),
1418 ', '.join(not_built)))
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001419
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001420 def ProduceResultSummary(self, commit_upto, commits, board_selected):
Simon Glasse30965d2014-08-28 09:43:44 -06001421 (board_dict, err_lines, err_line_boards, warn_lines,
Alex Kiernan48ae4122018-05-31 04:48:34 +00001422 warn_line_boards, config, environment) = self.GetResultSummary(
Simon Glassed966652014-08-28 09:43:43 -06001423 board_selected, commit_upto,
Simon Glass843312d2015-02-05 22:06:15 -07001424 read_func_sizes=self._show_bloat,
Alex Kiernan48ae4122018-05-31 04:48:34 +00001425 read_config=self._show_config,
1426 read_environment=self._show_environment)
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001427 if commits:
1428 msg = '%02d: %s' % (commit_upto + 1,
1429 commits[commit_upto].subject)
Simon Glass4653a882014-09-05 19:00:07 -06001430 Print(msg, colour=self.col.BLUE)
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001431 self.PrintResultSummary(board_selected, board_dict,
Simon Glassed966652014-08-28 09:43:43 -06001432 err_lines if self._show_errors else [], err_line_boards,
Simon Glasse30965d2014-08-28 09:43:44 -06001433 warn_lines if self._show_errors else [], warn_line_boards,
Alex Kiernan48ae4122018-05-31 04:48:34 +00001434 config, environment, self._show_sizes, self._show_detail,
1435 self._show_bloat, self._show_config, self._show_environment)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001436
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001437 def ShowSummary(self, commits, board_selected):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001438 """Show a build summary for U-Boot for a given board list.
1439
1440 Reset the result summary, then repeatedly call GetResultSummary on
1441 each commit's results, then display the differences we see.
1442
1443 Args:
1444 commit: Commit objects to summarise
1445 board_selected: Dict containing boards to summarise
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001446 """
Simon Glassfea58582014-08-09 15:32:59 -06001447 self.commit_count = len(commits) if commits else 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001448 self.commits = commits
1449 self.ResetResultSummary(board_selected)
Simon Glass28370c12014-08-09 15:33:06 -06001450 self._error_lines = 0
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001451
1452 for commit_upto in range(0, self.commit_count, self._step):
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001453 self.ProduceResultSummary(commit_upto, commits, board_selected)
Simon Glass28370c12014-08-09 15:33:06 -06001454 if not self._error_lines:
Simon Glass4653a882014-09-05 19:00:07 -06001455 Print('(no errors to report)', colour=self.col.GREEN)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001456
1457
1458 def SetupBuild(self, board_selected, commits):
1459 """Set up ready to start a build.
1460
1461 Args:
1462 board_selected: Selected boards to build
1463 commits: Selected commits to build
1464 """
1465 # First work out how many commits we will build
Simon Glassc05aa032019-10-31 07:42:53 -06001466 count = (self.commit_count + self._step - 1) // self._step
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001467 self.count = len(board_selected) * count
1468 self.upto = self.warned = self.fail = 0
1469 self._timestamps = collections.deque()
1470
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001471 def GetThreadDir(self, thread_num):
1472 """Get the directory path to the working dir for a thread.
1473
1474 Args:
1475 thread_num: Number of thread to check.
1476 """
1477 return os.path.join(self._working_dir, '%02d' % thread_num)
1478
Simon Glassfea58582014-08-09 15:32:59 -06001479 def _PrepareThread(self, thread_num, setup_git):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001480 """Prepare the working directory for a thread.
1481
1482 This clones or fetches the repo into the thread's work directory.
1483
1484 Args:
1485 thread_num: Thread number (0, 1, ...)
Simon Glassfea58582014-08-09 15:32:59 -06001486 setup_git: True to set up a git repo clone
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001487 """
1488 thread_dir = self.GetThreadDir(thread_num)
Simon Glass190064b2014-08-09 15:33:00 -06001489 builderthread.Mkdir(thread_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001490 git_dir = os.path.join(thread_dir, '.git')
1491
1492 # Clone the repo if it doesn't already exist
1493 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1494 # we have a private index but uses the origin repo's contents?
Simon Glassfea58582014-08-09 15:32:59 -06001495 if setup_git and self.git_dir:
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001496 src_dir = os.path.abspath(self.git_dir)
1497 if os.path.exists(git_dir):
1498 gitutil.Fetch(git_dir, thread_dir)
1499 else:
Simon Glass21f0eb32016-09-18 16:48:31 -06001500 Print('\rCloning repo for thread %d' % thread_num,
1501 newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001502 gitutil.Clone(src_dir, thread_dir)
Simon Glass21f0eb32016-09-18 16:48:31 -06001503 Print('\r%s\r' % (' ' * 30), newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001504
Simon Glassfea58582014-08-09 15:32:59 -06001505 def _PrepareWorkingSpace(self, max_threads, setup_git):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001506 """Prepare the working directory for use.
1507
1508 Set up the git repo for each thread.
1509
1510 Args:
1511 max_threads: Maximum number of threads we expect to need.
Simon Glassfea58582014-08-09 15:32:59 -06001512 setup_git: True to set up a git repo clone
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001513 """
Simon Glass190064b2014-08-09 15:33:00 -06001514 builderthread.Mkdir(self._working_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001515 for thread in range(max_threads):
Simon Glassfea58582014-08-09 15:32:59 -06001516 self._PrepareThread(thread, setup_git)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001517
1518 def _PrepareOutputSpace(self):
1519 """Get the output directories ready to receive files.
1520
1521 We delete any output directories which look like ones we need to
1522 create. Having left over directories is confusing when the user wants
1523 to check the output manually.
1524 """
Simon Glass1a915672014-12-01 17:33:53 -07001525 if not self.commits:
1526 return
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001527 dir_list = []
1528 for commit_upto in range(self.commit_count):
1529 dir_list.append(self._GetOutputDir(commit_upto))
1530
Simon Glassb222abe2016-09-18 16:48:32 -06001531 to_remove = []
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001532 for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1533 if dirname not in dir_list:
Simon Glassb222abe2016-09-18 16:48:32 -06001534 to_remove.append(dirname)
1535 if to_remove:
1536 Print('Removing %d old build directories' % len(to_remove),
1537 newline=False)
1538 for dirname in to_remove:
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001539 shutil.rmtree(dirname)
1540
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001541 def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001542 """Build all commits for a list of boards
1543
1544 Args:
1545 commits: List of commits to be build, each a Commit object
1546 boards_selected: Dict of selected boards, key is target name,
1547 value is Board object
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001548 keep_outputs: True to save build output files
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001549 verbose: Display build results as they are completed
Simon Glass2c3deb92014-08-28 09:43:39 -06001550 Returns:
1551 Tuple containing:
1552 - number of boards that failed to build
1553 - number of boards that issued warnings
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001554 """
Simon Glassfea58582014-08-09 15:32:59 -06001555 self.commit_count = len(commits) if commits else 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001556 self.commits = commits
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001557 self._verbose = verbose
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001558
1559 self.ResetResultSummary(board_selected)
Thierry Redingf3d015c2014-08-19 10:22:39 +02001560 builderthread.Mkdir(self.base_dir, parents = True)
Simon Glassfea58582014-08-09 15:32:59 -06001561 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1562 commits is not None)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001563 self._PrepareOutputSpace()
Simon Glass745b3952016-09-18 16:48:33 -06001564 Print('\rStarting build...', newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001565 self.SetupBuild(board_selected, commits)
1566 self.ProcessResult(None)
1567
1568 # Create jobs to build all commits for each board
Simon Glassc05aa032019-10-31 07:42:53 -06001569 for brd in board_selected.values():
Simon Glass190064b2014-08-09 15:33:00 -06001570 job = builderthread.BuilderJob()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001571 job.board = brd
1572 job.commits = commits
1573 job.keep_outputs = keep_outputs
1574 job.step = self._step
1575 self.queue.put(job)
1576
Simon Glassd436e382016-09-18 16:48:35 -06001577 term = threading.Thread(target=self.queue.join)
1578 term.setDaemon(True)
1579 term.start()
1580 while term.isAlive():
1581 term.join(100)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001582
1583 # Wait until we have processed all output
1584 self.out_queue.join()
Simon Glass4653a882014-09-05 19:00:07 -06001585 Print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001586 self.ClearLine(0)
Simon Glass2c3deb92014-08-28 09:43:39 -06001587 return (self.fail, self.warned)