blob: 4e72b7d60dfa7e1122e1b8a04ae6623bac8e1959 [file] [log] [blame]
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001# Copyright (c) 2013 The Chromium OS Authors.
2#
3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
4#
Wolfgang Denk1a459662013-07-08 09:37:19 +02005# SPDX-License-Identifier: GPL-2.0+
Simon Glassfc3fe1c2013-04-03 11:07:16 +00006#
7
8import collections
Simon Glassfc3fe1c2013-04-03 11:07:16 +00009from datetime import datetime, timedelta
10import glob
11import os
12import re
13import Queue
14import shutil
Simon Glass2f256642016-09-18 16:48:37 -060015import signal
Simon Glassfc3fe1c2013-04-03 11:07:16 +000016import string
17import sys
Simon Glassd436e382016-09-18 16:48:35 -060018import threading
Simon Glassfc3fe1c2013-04-03 11:07:16 +000019import time
20
Simon Glass190064b2014-08-09 15:33:00 -060021import builderthread
Simon Glassfc3fe1c2013-04-03 11:07:16 +000022import command
23import gitutil
24import terminal
Simon Glass4653a882014-09-05 19:00:07 -060025from terminal import Print
Simon Glassfc3fe1c2013-04-03 11:07:16 +000026import toolchain
27
28
29"""
30Theory of Operation
31
32Please see README for user documentation, and you should be familiar with
33that before trying to make sense of this.
34
35Buildman works by keeping the machine as busy as possible, building different
36commits for different boards on multiple CPUs at once.
37
38The source repo (self.git_dir) contains all the commits to be built. Each
39thread works on a single board at a time. It checks out the first commit,
40configures it for that board, then builds it. Then it checks out the next
41commit and builds it (typically without re-configuring). When it runs out
42of commits, it gets another job from the builder and starts again with that
43board.
44
45Clearly the builder threads could work either way - they could check out a
46commit and then built it for all boards. Using separate directories for each
47commit/board pair they could leave their build product around afterwards
48also.
49
50The intent behind building a single board for multiple commits, is to make
51use of incremental builds. Since each commit is built incrementally from
52the previous one, builds are faster. Reconfiguring for a different board
53removes all intermediate object files.
54
55Many threads can be working at once, but each has its own working directory.
56When a thread finishes a build, it puts the output files into a result
57directory.
58
59The base directory used by buildman is normally '../<branch>', i.e.
60a directory higher than the source repository and named after the branch
61being built.
62
63Within the base directory, we have one subdirectory for each commit. Within
64that is one subdirectory for each board. Within that is the build output for
65that commit/board combination.
66
67Buildman also create working directories for each thread, in a .bm-work/
68subdirectory in the base dir.
69
70As an example, say we are building branch 'us-net' for boards 'sandbox' and
71'seaboard', and say that us-net has two commits. We will have directories
72like this:
73
74us-net/ base directory
75 01_of_02_g4ed4ebc_net--Add-tftp-speed-/
76 sandbox/
77 u-boot.bin
78 seaboard/
79 u-boot.bin
80 02_of_02_g4ed4ebc_net--Check-tftp-comp/
81 sandbox/
82 u-boot.bin
83 seaboard/
84 u-boot.bin
85 .bm-work/
86 00/ working directory for thread 0 (contains source checkout)
87 build/ build output
88 01/ working directory for thread 1
89 build/ build output
90 ...
91u-boot/ source directory
92 .git/ repository
93"""
94
95# Possible build outcomes
96OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4)
97
Simon Glass9a6d2e22017-04-12 18:23:26 -060098# Translate a commit subject into a valid filename (and handle unicode)
99trans_valid_chars = string.maketrans('/: ', '---')
100trans_valid_chars = trans_valid_chars.decode('latin-1')
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000101
Simon Glassb464f8e2016-11-13 14:25:53 -0700102BASE_CONFIG_FILENAMES = [
103 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
104]
105
106EXTRA_CONFIG_FILENAMES = [
Simon Glass843312d2015-02-05 22:06:15 -0700107 '.config', '.config-spl', '.config-tpl',
108 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
109 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
Simon Glass843312d2015-02-05 22:06:15 -0700110]
111
Simon Glass8270e3c2015-08-25 21:52:14 -0600112class Config:
113 """Holds information about configuration settings for a board."""
Simon Glassb464f8e2016-11-13 14:25:53 -0700114 def __init__(self, config_filename, target):
Simon Glass8270e3c2015-08-25 21:52:14 -0600115 self.target = target
116 self.config = {}
Simon Glassb464f8e2016-11-13 14:25:53 -0700117 for fname in config_filename:
Simon Glass8270e3c2015-08-25 21:52:14 -0600118 self.config[fname] = {}
119
120 def Add(self, fname, key, value):
121 self.config[fname][key] = value
122
123 def __hash__(self):
124 val = 0
125 for fname in self.config:
126 for key, value in self.config[fname].iteritems():
127 print key, value
128 val = val ^ hash(key) & hash(value)
129 return val
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000130
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000131class Builder:
132 """Class for building U-Boot for a particular commit.
133
134 Public members: (many should ->private)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000135 already_done: Number of builds already completed
136 base_dir: Base directory to use for builder
137 checkout: True to check out source, False to skip that step.
138 This is used for testing.
139 col: terminal.Color() object
140 count: Number of commits to build
141 do_make: Method to call to invoke Make
142 fail: Number of builds that failed due to error
143 force_build: Force building even if a build already exists
144 force_config_on_failure: If a commit fails for a board, disable
145 incremental building for the next commit we build for that
146 board, so that we will see all warnings/errors again.
Simon Glass4266dc22014-07-13 12:22:31 -0600147 force_build_failures: If a previously-built build (i.e. built on
148 a previous run of buildman) is marked as failed, rebuild it.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000149 git_dir: Git directory containing source repository
150 last_line_len: Length of the last line we printed (used for erasing
151 it with new progress information)
152 num_jobs: Number of jobs to run at once (passed to make as -j)
153 num_threads: Number of builder threads to run
154 out_queue: Queue of results to process
155 re_make_err: Compiled regular expression for ignore_lines
156 queue: Queue of jobs to run
157 threads: List of active threads
158 toolchains: Toolchains object to use for building
159 upto: Current commit number we are building (0.count-1)
160 warned: Number of builds that produced at least one warning
Simon Glass97e91522014-07-14 17:51:02 -0600161 force_reconfig: Reconfigure U-Boot on each comiit. This disables
162 incremental building, where buildman reconfigures on the first
163 commit for a baord, and then just does an incremental build for
164 the following commits. In fact buildman will reconfigure and
165 retry for any failing commits, so generally the only effect of
166 this option is to slow things down.
Simon Glass189a4962014-07-14 17:51:03 -0600167 in_tree: Build U-Boot in-tree instead of specifying an output
168 directory separate from the source code. This option is really
169 only useful for testing in-tree builds.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000170
171 Private members:
172 _base_board_dict: Last-summarised Dict of boards
173 _base_err_lines: Last-summarised list of errors
Simon Glasse30965d2014-08-28 09:43:44 -0600174 _base_warn_lines: Last-summarised list of warnings
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000175 _build_period_us: Time taken for a single build (float object).
176 _complete_delay: Expected delay until completion (timedelta)
177 _next_delay_update: Next time we plan to display a progress update
178 (datatime)
179 _show_unknown: Show unknown boards (those not built) in summary
180 _timestamps: List of timestamps for the completion of the last
181 last _timestamp_count builds. Each is a datetime object.
182 _timestamp_count: Number of timestamps to keep in our list.
183 _working_dir: Base working directory containing all threads
184 """
185 class Outcome:
186 """Records a build outcome for a single make invocation
187
188 Public Members:
189 rc: Outcome value (OUTCOME_...)
190 err_lines: List of error lines or [] if none
191 sizes: Dictionary of image size information, keyed by filename
192 - Each value is itself a dictionary containing
193 values for 'text', 'data' and 'bss', being the integer
194 size in bytes of each section.
195 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
196 value is itself a dictionary:
197 key: function name
198 value: Size of function in bytes
Simon Glass843312d2015-02-05 22:06:15 -0700199 config: Dictionary keyed by filename - e.g. '.config'. Each
200 value is itself a dictionary:
201 key: config name
202 value: config value
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000203 """
Simon Glass843312d2015-02-05 22:06:15 -0700204 def __init__(self, rc, err_lines, sizes, func_sizes, config):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000205 self.rc = rc
206 self.err_lines = err_lines
207 self.sizes = sizes
208 self.func_sizes = func_sizes
Simon Glass843312d2015-02-05 22:06:15 -0700209 self.config = config
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000210
211 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
Simon Glass5971ab52014-12-01 17:33:55 -0700212 gnu_make='make', checkout=True, show_unknown=True, step=1,
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600213 no_subdirs=False, full_path=False, verbose_build=False,
Simon Glassb50113f2016-11-13 14:25:51 -0700214 incremental=False, per_board_out_dir=False,
Daniel Schwierzeck2371d1b2018-01-26 16:31:05 +0100215 config_only=False, squash_config_y=False,
216 warnings_as_errors=False):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000217 """Create a new Builder object
218
219 Args:
220 toolchains: Toolchains object to use for building
221 base_dir: Base directory to use for builder
222 git_dir: Git directory containing source repository
223 num_threads: Number of builder threads to run
224 num_jobs: Number of jobs to run at once (passed to make as -j)
Masahiro Yamada99796922014-07-22 11:19:09 +0900225 gnu_make: the command name of GNU Make.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000226 checkout: True to check out source, False to skip that step.
227 This is used for testing.
228 show_unknown: Show unknown boards (those not built) in summary
229 step: 1 to process every commit, n to process every nth commit
Simon Glassbb1501f2014-12-01 17:34:00 -0700230 no_subdirs: Don't create subdirectories when building current
231 source for a single board
232 full_path: Return the full path in CROSS_COMPILE and don't set
233 PATH
Simon Glassd2ce6582014-12-01 17:34:07 -0700234 verbose_build: Run build with V=1 and don't use 'make -s'
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600235 incremental: Always perform incremental builds; don't run make
236 mrproper when configuring
237 per_board_out_dir: Build in a separate persistent directory per
238 board rather than a thread-specific directory
Simon Glassb50113f2016-11-13 14:25:51 -0700239 config_only: Only configure each build, don't build it
Simon Glassb464f8e2016-11-13 14:25:53 -0700240 squash_config_y: Convert CONFIG options with the value 'y' to '1'
Daniel Schwierzeck2371d1b2018-01-26 16:31:05 +0100241 warnings_as_errors: Treat all compiler warnings as errors
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000242 """
243 self.toolchains = toolchains
244 self.base_dir = base_dir
245 self._working_dir = os.path.join(base_dir, '.bm-work')
246 self.threads = []
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000247 self.do_make = self.Make
Masahiro Yamada99796922014-07-22 11:19:09 +0900248 self.gnu_make = gnu_make
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000249 self.checkout = checkout
250 self.num_threads = num_threads
251 self.num_jobs = num_jobs
252 self.already_done = 0
253 self.force_build = False
254 self.git_dir = git_dir
255 self._show_unknown = show_unknown
256 self._timestamp_count = 10
257 self._build_period_us = None
258 self._complete_delay = None
259 self._next_delay_update = datetime.now()
260 self.force_config_on_failure = True
Simon Glass4266dc22014-07-13 12:22:31 -0600261 self.force_build_failures = False
Simon Glass97e91522014-07-14 17:51:02 -0600262 self.force_reconfig = False
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000263 self._step = step
Simon Glass189a4962014-07-14 17:51:03 -0600264 self.in_tree = False
Simon Glass28370c12014-08-09 15:33:06 -0600265 self._error_lines = 0
Simon Glass5971ab52014-12-01 17:33:55 -0700266 self.no_subdirs = no_subdirs
Simon Glassbb1501f2014-12-01 17:34:00 -0700267 self.full_path = full_path
Simon Glassd2ce6582014-12-01 17:34:07 -0700268 self.verbose_build = verbose_build
Simon Glassb50113f2016-11-13 14:25:51 -0700269 self.config_only = config_only
Simon Glassb464f8e2016-11-13 14:25:53 -0700270 self.squash_config_y = squash_config_y
271 self.config_filenames = BASE_CONFIG_FILENAMES
272 if not self.squash_config_y:
273 self.config_filenames += EXTRA_CONFIG_FILENAMES
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000274
Daniel Schwierzeck2371d1b2018-01-26 16:31:05 +0100275 self.warnings_as_errors = warnings_as_errors
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000276 self.col = terminal.Color()
277
Simon Glasse30965d2014-08-28 09:43:44 -0600278 self._re_function = re.compile('(.*): In function.*')
279 self._re_files = re.compile('In file included from.*')
280 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
281 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
282
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000283 self.queue = Queue.Queue()
284 self.out_queue = Queue.Queue()
285 for i in range(self.num_threads):
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600286 t = builderthread.BuilderThread(self, i, incremental,
287 per_board_out_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000288 t.setDaemon(True)
289 t.start()
290 self.threads.append(t)
291
292 self.last_line_len = 0
Simon Glass190064b2014-08-09 15:33:00 -0600293 t = builderthread.ResultThread(self)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000294 t.setDaemon(True)
295 t.start()
296 self.threads.append(t)
297
298 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
299 self.re_make_err = re.compile('|'.join(ignore_lines))
300
Simon Glass2f256642016-09-18 16:48:37 -0600301 # Handle existing graceful with SIGINT / Ctrl-C
302 signal.signal(signal.SIGINT, self.signal_handler)
303
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000304 def __del__(self):
305 """Get rid of all threads created by the builder"""
306 for t in self.threads:
307 del t
308
Simon Glass2f256642016-09-18 16:48:37 -0600309 def signal_handler(self, signal, frame):
310 sys.exit(1)
311
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600312 def SetDisplayOptions(self, show_errors=False, show_sizes=False,
Simon Glassed966652014-08-28 09:43:43 -0600313 show_detail=False, show_bloat=False,
Simon Glass843312d2015-02-05 22:06:15 -0700314 list_error_boards=False, show_config=False):
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600315 """Setup display options for the builder.
316
317 show_errors: True to show summarised error/warning info
318 show_sizes: Show size deltas
319 show_detail: Show detail for each board
320 show_bloat: Show detail for each function
Simon Glassed966652014-08-28 09:43:43 -0600321 list_error_boards: Show the boards which caused each error/warning
Simon Glass843312d2015-02-05 22:06:15 -0700322 show_config: Show config deltas
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600323 """
324 self._show_errors = show_errors
325 self._show_sizes = show_sizes
326 self._show_detail = show_detail
327 self._show_bloat = show_bloat
Simon Glassed966652014-08-28 09:43:43 -0600328 self._list_error_boards = list_error_boards
Simon Glass843312d2015-02-05 22:06:15 -0700329 self._show_config = show_config
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600330
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000331 def _AddTimestamp(self):
332 """Add a new timestamp to the list and record the build period.
333
334 The build period is the length of time taken to perform a single
335 build (one board, one commit).
336 """
337 now = datetime.now()
338 self._timestamps.append(now)
339 count = len(self._timestamps)
340 delta = self._timestamps[-1] - self._timestamps[0]
341 seconds = delta.total_seconds()
342
343 # If we have enough data, estimate build period (time taken for a
344 # single build) and therefore completion time.
345 if count > 1 and self._next_delay_update < now:
346 self._next_delay_update = now + timedelta(seconds=2)
347 if seconds > 0:
348 self._build_period = float(seconds) / count
349 todo = self.count - self.upto
350 self._complete_delay = timedelta(microseconds=
351 self._build_period * todo * 1000000)
352 # Round it
353 self._complete_delay -= timedelta(
354 microseconds=self._complete_delay.microseconds)
355
356 if seconds > 60:
357 self._timestamps.popleft()
358 count -= 1
359
360 def ClearLine(self, length):
361 """Clear any characters on the current line
362
363 Make way for a new line of length 'length', by outputting enough
364 spaces to clear out the old line. Then remember the new length for
365 next time.
366
367 Args:
368 length: Length of new line, in characters
369 """
370 if length < self.last_line_len:
Simon Glass4653a882014-09-05 19:00:07 -0600371 Print(' ' * (self.last_line_len - length), newline=False)
372 Print('\r', newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000373 self.last_line_len = length
374 sys.stdout.flush()
375
376 def SelectCommit(self, commit, checkout=True):
377 """Checkout the selected commit for this build
378 """
379 self.commit = commit
380 if checkout and self.checkout:
381 gitutil.Checkout(commit.hash)
382
383 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
384 """Run make
385
386 Args:
387 commit: Commit object that is being built
388 brd: Board object that is being built
Roger Meierfd18a892014-08-20 22:10:29 +0200389 stage: Stage that we are at (mrproper, config, build)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000390 cwd: Directory where make should be run
391 args: Arguments to pass to make
392 kwargs: Arguments to pass to command.RunPipe()
393 """
Masahiro Yamada99796922014-07-22 11:19:09 +0900394 cmd = [self.gnu_make] + list(args)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000395 result = command.RunPipe([cmd], capture=True, capture_stderr=True,
396 cwd=cwd, raise_on_error=False, **kwargs)
Simon Glass40f11fc2015-02-05 22:06:12 -0700397 if self.verbose_build:
398 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
399 result.combined = '%s\n' % (' '.join(cmd)) + result.combined
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000400 return result
401
402 def ProcessResult(self, result):
403 """Process the result of a build, showing progress information
404
405 Args:
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600406 result: A CommandResult object, which indicates the result for
407 a single build
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000408 """
409 col = terminal.Color()
410 if result:
411 target = result.brd.target
412
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000413 self.upto += 1
414 if result.return_code != 0:
415 self.fail += 1
416 elif result.stderr:
417 self.warned += 1
418 if result.already_done:
419 self.already_done += 1
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600420 if self._verbose:
Simon Glass4653a882014-09-05 19:00:07 -0600421 Print('\r', newline=False)
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600422 self.ClearLine(0)
423 boards_selected = {target : result.brd}
424 self.ResetResultSummary(boards_selected)
425 self.ProduceResultSummary(result.commit_upto, self.commits,
426 boards_selected)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000427 else:
428 target = '(starting)'
429
430 # Display separate counts for ok, warned and fail
431 ok = self.upto - self.warned - self.fail
432 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
433 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
434 line += self.col.Color(self.col.RED, '%5d' % self.fail)
435
436 name = ' /%-5d ' % self.count
437
438 # Add our current completion time estimate
439 self._AddTimestamp()
440 if self._complete_delay:
441 name += '%s : ' % self._complete_delay
442 # When building all boards for a commit, we can print a commit
443 # progress message.
444 if result and result.commit_upto is None:
445 name += 'commit %2d/%-3d' % (self.commit_upto + 1,
446 self.commit_count)
447
448 name += target
Simon Glass4653a882014-09-05 19:00:07 -0600449 Print(line + name, newline=False)
Simon Glass960421e2016-11-15 15:32:59 -0700450 length = 16 + len(name)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000451 self.ClearLine(length)
452
453 def _GetOutputDir(self, commit_upto):
454 """Get the name of the output directory for a commit number
455
456 The output directory is typically .../<branch>/<commit>.
457
458 Args:
459 commit_upto: Commit number to use (0..self.count-1)
460 """
Simon Glass5971ab52014-12-01 17:33:55 -0700461 commit_dir = None
Simon Glassfea58582014-08-09 15:32:59 -0600462 if self.commits:
463 commit = self.commits[commit_upto]
464 subject = commit.subject.translate(trans_valid_chars)
465 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
466 self.commit_count, commit.hash, subject[:20]))
Simon Glass5971ab52014-12-01 17:33:55 -0700467 elif not self.no_subdirs:
Simon Glassfea58582014-08-09 15:32:59 -0600468 commit_dir = 'current'
Simon Glass5971ab52014-12-01 17:33:55 -0700469 if not commit_dir:
470 return self.base_dir
471 return os.path.join(self.base_dir, commit_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000472
473 def GetBuildDir(self, commit_upto, target):
474 """Get the name of the build directory for a commit number
475
476 The build directory is typically .../<branch>/<commit>/<target>.
477
478 Args:
479 commit_upto: Commit number to use (0..self.count-1)
480 target: Target name
481 """
482 output_dir = self._GetOutputDir(commit_upto)
483 return os.path.join(output_dir, target)
484
485 def GetDoneFile(self, commit_upto, target):
486 """Get the name of the done file for a commit number
487
488 Args:
489 commit_upto: Commit number to use (0..self.count-1)
490 target: Target name
491 """
492 return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
493
494 def GetSizesFile(self, commit_upto, target):
495 """Get the name of the sizes file for a commit number
496
497 Args:
498 commit_upto: Commit number to use (0..self.count-1)
499 target: Target name
500 """
501 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
502
503 def GetFuncSizesFile(self, commit_upto, target, elf_fname):
504 """Get the name of the funcsizes file for a commit number and ELF file
505
506 Args:
507 commit_upto: Commit number to use (0..self.count-1)
508 target: Target name
509 elf_fname: Filename of elf image
510 """
511 return os.path.join(self.GetBuildDir(commit_upto, target),
512 '%s.sizes' % elf_fname.replace('/', '-'))
513
514 def GetObjdumpFile(self, commit_upto, target, elf_fname):
515 """Get the name of the objdump file for a commit number and ELF file
516
517 Args:
518 commit_upto: Commit number to use (0..self.count-1)
519 target: Target name
520 elf_fname: Filename of elf image
521 """
522 return os.path.join(self.GetBuildDir(commit_upto, target),
523 '%s.objdump' % elf_fname.replace('/', '-'))
524
525 def GetErrFile(self, commit_upto, target):
526 """Get the name of the err file for a commit number
527
528 Args:
529 commit_upto: Commit number to use (0..self.count-1)
530 target: Target name
531 """
532 output_dir = self.GetBuildDir(commit_upto, target)
533 return os.path.join(output_dir, 'err')
534
535 def FilterErrors(self, lines):
536 """Filter out errors in which we have no interest
537
538 We should probably use map().
539
540 Args:
541 lines: List of error lines, each a string
542 Returns:
543 New list with only interesting lines included
544 """
545 out_lines = []
546 for line in lines:
547 if not self.re_make_err.search(line):
548 out_lines.append(line)
549 return out_lines
550
551 def ReadFuncSizes(self, fname, fd):
552 """Read function sizes from the output of 'nm'
553
554 Args:
555 fd: File containing data to read
556 fname: Filename we are reading from (just for errors)
557
558 Returns:
559 Dictionary containing size of each function in bytes, indexed by
560 function name.
561 """
562 sym = {}
563 for line in fd.readlines():
564 try:
565 size, type, name = line[:-1].split()
566 except:
Simon Glass4653a882014-09-05 19:00:07 -0600567 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000568 continue
569 if type in 'tTdDbB':
570 # function names begin with '.' on 64-bit powerpc
571 if '.' in name[1:]:
572 name = 'static.' + name.split('.')[0]
573 sym[name] = sym.get(name, 0) + int(size, 16)
574 return sym
575
Simon Glass843312d2015-02-05 22:06:15 -0700576 def _ProcessConfig(self, fname):
577 """Read in a .config, autoconf.mk or autoconf.h file
578
579 This function handles all config file types. It ignores comments and
580 any #defines which don't start with CONFIG_.
581
582 Args:
583 fname: Filename to read
584
585 Returns:
586 Dictionary:
587 key: Config name (e.g. CONFIG_DM)
588 value: Config value (e.g. 1)
589 """
590 config = {}
591 if os.path.exists(fname):
592 with open(fname) as fd:
593 for line in fd:
594 line = line.strip()
595 if line.startswith('#define'):
596 values = line[8:].split(' ', 1)
597 if len(values) > 1:
598 key, value = values
599 else:
600 key = values[0]
Simon Glassb464f8e2016-11-13 14:25:53 -0700601 value = '1' if self.squash_config_y else ''
Simon Glass843312d2015-02-05 22:06:15 -0700602 if not key.startswith('CONFIG_'):
603 continue
604 elif not line or line[0] in ['#', '*', '/']:
605 continue
606 else:
607 key, value = line.split('=', 1)
Simon Glassb464f8e2016-11-13 14:25:53 -0700608 if self.squash_config_y and value == 'y':
609 value = '1'
Simon Glass843312d2015-02-05 22:06:15 -0700610 config[key] = value
611 return config
612
613 def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
614 read_config):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000615 """Work out the outcome of a build.
616
617 Args:
618 commit_upto: Commit number to check (0..n-1)
619 target: Target board to check
620 read_func_sizes: True to read function size information
Simon Glass843312d2015-02-05 22:06:15 -0700621 read_config: True to read .config and autoconf.h files
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000622
623 Returns:
624 Outcome object
625 """
626 done_file = self.GetDoneFile(commit_upto, target)
627 sizes_file = self.GetSizesFile(commit_upto, target)
628 sizes = {}
629 func_sizes = {}
Simon Glass843312d2015-02-05 22:06:15 -0700630 config = {}
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000631 if os.path.exists(done_file):
632 with open(done_file, 'r') as fd:
633 return_code = int(fd.readline())
634 err_lines = []
635 err_file = self.GetErrFile(commit_upto, target)
636 if os.path.exists(err_file):
637 with open(err_file, 'r') as fd:
638 err_lines = self.FilterErrors(fd.readlines())
639
640 # Decide whether the build was ok, failed or created warnings
641 if return_code:
642 rc = OUTCOME_ERROR
643 elif len(err_lines):
644 rc = OUTCOME_WARNING
645 else:
646 rc = OUTCOME_OK
647
648 # Convert size information to our simple format
649 if os.path.exists(sizes_file):
650 with open(sizes_file, 'r') as fd:
651 for line in fd.readlines():
652 values = line.split()
653 rodata = 0
654 if len(values) > 6:
655 rodata = int(values[6], 16)
656 size_dict = {
657 'all' : int(values[0]) + int(values[1]) +
658 int(values[2]),
659 'text' : int(values[0]) - rodata,
660 'data' : int(values[1]),
661 'bss' : int(values[2]),
662 'rodata' : rodata,
663 }
664 sizes[values[5]] = size_dict
665
666 if read_func_sizes:
667 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
668 for fname in glob.glob(pattern):
669 with open(fname, 'r') as fd:
670 dict_name = os.path.basename(fname).replace('.sizes',
671 '')
672 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
673
Simon Glass843312d2015-02-05 22:06:15 -0700674 if read_config:
675 output_dir = self.GetBuildDir(commit_upto, target)
Simon Glassb464f8e2016-11-13 14:25:53 -0700676 for name in self.config_filenames:
Simon Glass843312d2015-02-05 22:06:15 -0700677 fname = os.path.join(output_dir, name)
678 config[name] = self._ProcessConfig(fname)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000679
Simon Glass843312d2015-02-05 22:06:15 -0700680 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000681
Simon Glass843312d2015-02-05 22:06:15 -0700682 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {})
683
684 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
685 read_config):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000686 """Calculate a summary of the results of building a commit.
687
688 Args:
689 board_selected: Dict containing boards to summarise
690 commit_upto: Commit number to summarize (0..self.count-1)
691 read_func_sizes: True to read function size information
Simon Glass843312d2015-02-05 22:06:15 -0700692 read_config: True to read .config and autoconf.h files
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000693
694 Returns:
695 Tuple:
696 Dict containing boards which passed building this commit.
697 keyed by board.target
Simon Glasse30965d2014-08-28 09:43:44 -0600698 List containing a summary of error lines
Simon Glassed966652014-08-28 09:43:43 -0600699 Dict keyed by error line, containing a list of the Board
700 objects with that error
Simon Glasse30965d2014-08-28 09:43:44 -0600701 List containing a summary of warning lines
702 Dict keyed by error line, containing a list of the Board
703 objects with that warning
Simon Glass8270e3c2015-08-25 21:52:14 -0600704 Dictionary keyed by board.target. Each value is a dictionary:
705 key: filename - e.g. '.config'
Simon Glass843312d2015-02-05 22:06:15 -0700706 value is itself a dictionary:
707 key: config name
708 value: config value
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000709 """
Simon Glasse30965d2014-08-28 09:43:44 -0600710 def AddLine(lines_summary, lines_boards, line, board):
711 line = line.rstrip()
712 if line in lines_boards:
713 lines_boards[line].append(board)
714 else:
715 lines_boards[line] = [board]
716 lines_summary.append(line)
717
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000718 board_dict = {}
719 err_lines_summary = []
Simon Glassed966652014-08-28 09:43:43 -0600720 err_lines_boards = {}
Simon Glasse30965d2014-08-28 09:43:44 -0600721 warn_lines_summary = []
722 warn_lines_boards = {}
Simon Glass843312d2015-02-05 22:06:15 -0700723 config = {}
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000724
725 for board in boards_selected.itervalues():
726 outcome = self.GetBuildOutcome(commit_upto, board.target,
Simon Glass843312d2015-02-05 22:06:15 -0700727 read_func_sizes, read_config)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000728 board_dict[board.target] = outcome
Simon Glasse30965d2014-08-28 09:43:44 -0600729 last_func = None
730 last_was_warning = False
731 for line in outcome.err_lines:
732 if line:
733 if (self._re_function.match(line) or
734 self._re_files.match(line)):
735 last_func = line
Simon Glassed966652014-08-28 09:43:43 -0600736 else:
Simon Glasse30965d2014-08-28 09:43:44 -0600737 is_warning = self._re_warning.match(line)
738 is_note = self._re_note.match(line)
739 if is_warning or (last_was_warning and is_note):
740 if last_func:
741 AddLine(warn_lines_summary, warn_lines_boards,
742 last_func, board)
743 AddLine(warn_lines_summary, warn_lines_boards,
744 line, board)
745 else:
746 if last_func:
747 AddLine(err_lines_summary, err_lines_boards,
748 last_func, board)
749 AddLine(err_lines_summary, err_lines_boards,
750 line, board)
751 last_was_warning = is_warning
752 last_func = None
Simon Glassb464f8e2016-11-13 14:25:53 -0700753 tconfig = Config(self.config_filenames, board.target)
754 for fname in self.config_filenames:
Simon Glass843312d2015-02-05 22:06:15 -0700755 if outcome.config:
756 for key, value in outcome.config[fname].iteritems():
Simon Glass8270e3c2015-08-25 21:52:14 -0600757 tconfig.Add(fname, key, value)
758 config[board.target] = tconfig
Simon Glass843312d2015-02-05 22:06:15 -0700759
Simon Glasse30965d2014-08-28 09:43:44 -0600760 return (board_dict, err_lines_summary, err_lines_boards,
Simon Glass843312d2015-02-05 22:06:15 -0700761 warn_lines_summary, warn_lines_boards, config)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000762
763 def AddOutcome(self, board_dict, arch_list, changes, char, color):
764 """Add an output to our list of outcomes for each architecture
765
766 This simple function adds failing boards (changes) to the
767 relevant architecture string, so we can print the results out
768 sorted by architecture.
769
770 Args:
771 board_dict: Dict containing all boards
772 arch_list: Dict keyed by arch name. Value is a string containing
773 a list of board names which failed for that arch.
774 changes: List of boards to add to arch_list
775 color: terminal.Colour object
776 """
777 done_arch = {}
778 for target in changes:
779 if target in board_dict:
780 arch = board_dict[target].arch
781 else:
782 arch = 'unknown'
783 str = self.col.Color(color, ' ' + target)
784 if not arch in done_arch:
Simon Glass63c619e2015-02-05 22:06:11 -0700785 str = ' %s %s' % (self.col.Color(color, char), str)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000786 done_arch[arch] = True
787 if not arch in arch_list:
788 arch_list[arch] = str
789 else:
790 arch_list[arch] += str
791
792
793 def ColourNum(self, num):
794 color = self.col.RED if num > 0 else self.col.GREEN
795 if num == 0:
796 return '0'
797 return self.col.Color(color, str(num))
798
799 def ResetResultSummary(self, board_selected):
800 """Reset the results summary ready for use.
801
802 Set up the base board list to be all those selected, and set the
803 error lines to empty.
804
805 Following this, calls to PrintResultSummary() will use this
806 information to work out what has changed.
807
808 Args:
809 board_selected: Dict containing boards to summarise, keyed by
810 board.target
811 """
812 self._base_board_dict = {}
813 for board in board_selected:
Simon Glass843312d2015-02-05 22:06:15 -0700814 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {})
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000815 self._base_err_lines = []
Simon Glasse30965d2014-08-28 09:43:44 -0600816 self._base_warn_lines = []
817 self._base_err_line_boards = {}
818 self._base_warn_line_boards = {}
Simon Glass8270e3c2015-08-25 21:52:14 -0600819 self._base_config = None
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000820
821 def PrintFuncSizeDetail(self, fname, old, new):
822 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
823 delta, common = [], {}
824
825 for a in old:
826 if a in new:
827 common[a] = 1
828
829 for name in old:
830 if name not in common:
831 remove += 1
832 down += old[name]
833 delta.append([-old[name], name])
834
835 for name in new:
836 if name not in common:
837 add += 1
838 up += new[name]
839 delta.append([new[name], name])
840
841 for name in common:
842 diff = new.get(name, 0) - old.get(name, 0)
843 if diff > 0:
844 grow, up = grow + 1, up + diff
845 elif diff < 0:
846 shrink, down = shrink + 1, down - diff
847 delta.append([diff, name])
848
849 delta.sort()
850 delta.reverse()
851
852 args = [add, -remove, grow, -shrink, up, -down, up - down]
Tom Rinid5686a62017-05-22 13:48:52 -0400853 if max(args) == 0 and min(args) == 0:
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000854 return
855 args = [self.ColourNum(x) for x in args]
856 indent = ' ' * 15
Simon Glass4653a882014-09-05 19:00:07 -0600857 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
858 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
859 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
860 'delta'))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000861 for diff, name in delta:
862 if diff:
863 color = self.col.RED if diff > 0 else self.col.GREEN
864 msg = '%s %-38s %7s %7s %+7d' % (indent, name,
865 old.get(name, '-'), new.get(name,'-'), diff)
Simon Glass4653a882014-09-05 19:00:07 -0600866 Print(msg, colour=color)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000867
868
869 def PrintSizeDetail(self, target_list, show_bloat):
870 """Show details size information for each board
871
872 Args:
873 target_list: List of targets, each a dict containing:
874 'target': Target name
875 'total_diff': Total difference in bytes across all areas
876 <part_name>: Difference for that part
877 show_bloat: Show detail for each function
878 """
879 targets_by_diff = sorted(target_list, reverse=True,
880 key=lambda x: x['_total_diff'])
881 for result in targets_by_diff:
882 printed_target = False
883 for name in sorted(result):
884 diff = result[name]
885 if name.startswith('_'):
886 continue
887 if diff != 0:
888 color = self.col.RED if diff > 0 else self.col.GREEN
889 msg = ' %s %+d' % (name, diff)
890 if not printed_target:
Simon Glass4653a882014-09-05 19:00:07 -0600891 Print('%10s %-15s:' % ('', result['_target']),
892 newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000893 printed_target = True
Simon Glass4653a882014-09-05 19:00:07 -0600894 Print(msg, colour=color, newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000895 if printed_target:
Simon Glass4653a882014-09-05 19:00:07 -0600896 Print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000897 if show_bloat:
898 target = result['_target']
899 outcome = result['_outcome']
900 base_outcome = self._base_board_dict[target]
901 for fname in outcome.func_sizes:
902 self.PrintFuncSizeDetail(fname,
903 base_outcome.func_sizes[fname],
904 outcome.func_sizes[fname])
905
906
907 def PrintSizeSummary(self, board_selected, board_dict, show_detail,
908 show_bloat):
909 """Print a summary of image sizes broken down by section.
910
911 The summary takes the form of one line per architecture. The
912 line contains deltas for each of the sections (+ means the section
913 got bigger, - means smaller). The nunmbers are the average number
914 of bytes that a board in this section increased by.
915
916 For example:
917 powerpc: (622 boards) text -0.0
918 arm: (285 boards) text -0.0
919 nds32: (3 boards) text -8.0
920
921 Args:
922 board_selected: Dict containing boards to summarise, keyed by
923 board.target
924 board_dict: Dict containing boards for which we built this
925 commit, keyed by board.target. The value is an Outcome object.
926 show_detail: Show detail for each board
927 show_bloat: Show detail for each function
928 """
929 arch_list = {}
930 arch_count = {}
931
932 # Calculate changes in size for different image parts
933 # The previous sizes are in Board.sizes, for each board
934 for target in board_dict:
935 if target not in board_selected:
936 continue
937 base_sizes = self._base_board_dict[target].sizes
938 outcome = board_dict[target]
939 sizes = outcome.sizes
940
941 # Loop through the list of images, creating a dict of size
942 # changes for each image/part. We end up with something like
943 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
944 # which means that U-Boot data increased by 5 bytes and SPL
945 # text decreased by 4.
946 err = {'_target' : target}
947 for image in sizes:
948 if image in base_sizes:
949 base_image = base_sizes[image]
950 # Loop through the text, data, bss parts
951 for part in sorted(sizes[image]):
952 diff = sizes[image][part] - base_image[part]
953 col = None
954 if diff:
955 if image == 'u-boot':
956 name = part
957 else:
958 name = image + ':' + part
959 err[name] = diff
960 arch = board_selected[target].arch
961 if not arch in arch_count:
962 arch_count[arch] = 1
963 else:
964 arch_count[arch] += 1
965 if not sizes:
966 pass # Only add to our list when we have some stats
967 elif not arch in arch_list:
968 arch_list[arch] = [err]
969 else:
970 arch_list[arch].append(err)
971
972 # We now have a list of image size changes sorted by arch
973 # Print out a summary of these
974 for arch, target_list in arch_list.iteritems():
975 # Get total difference for each type
976 totals = {}
977 for result in target_list:
978 total = 0
979 for name, diff in result.iteritems():
980 if name.startswith('_'):
981 continue
982 total += diff
983 if name in totals:
984 totals[name] += diff
985 else:
986 totals[name] = diff
987 result['_total_diff'] = total
988 result['_outcome'] = board_dict[result['_target']]
989
990 count = len(target_list)
991 printed_arch = False
992 for name in sorted(totals):
993 diff = totals[name]
994 if diff:
995 # Display the average difference in this name for this
996 # architecture
997 avg_diff = float(diff) / count
998 color = self.col.RED if avg_diff > 0 else self.col.GREEN
999 msg = ' %s %+1.1f' % (name, avg_diff)
1000 if not printed_arch:
Simon Glass4653a882014-09-05 19:00:07 -06001001 Print('%10s: (for %d/%d boards)' % (arch, count,
1002 arch_count[arch]), newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001003 printed_arch = True
Simon Glass4653a882014-09-05 19:00:07 -06001004 Print(msg, colour=color, newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001005
1006 if printed_arch:
Simon Glass4653a882014-09-05 19:00:07 -06001007 Print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001008 if show_detail:
1009 self.PrintSizeDetail(target_list, show_bloat)
1010
1011
1012 def PrintResultSummary(self, board_selected, board_dict, err_lines,
Simon Glasse30965d2014-08-28 09:43:44 -06001013 err_line_boards, warn_lines, warn_line_boards,
Simon Glass843312d2015-02-05 22:06:15 -07001014 config, show_sizes, show_detail, show_bloat,
1015 show_config):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001016 """Compare results with the base results and display delta.
1017
1018 Only boards mentioned in board_selected will be considered. This
1019 function is intended to be called repeatedly with the results of
1020 each commit. It therefore shows a 'diff' between what it saw in
1021 the last call and what it sees now.
1022
1023 Args:
1024 board_selected: Dict containing boards to summarise, keyed by
1025 board.target
1026 board_dict: Dict containing boards for which we built this
1027 commit, keyed by board.target. The value is an Outcome object.
1028 err_lines: A list of errors for this commit, or [] if there is
1029 none, or we don't want to print errors
Simon Glassed966652014-08-28 09:43:43 -06001030 err_line_boards: Dict keyed by error line, containing a list of
1031 the Board objects with that error
Simon Glasse30965d2014-08-28 09:43:44 -06001032 warn_lines: A list of warnings for this commit, or [] if there is
1033 none, or we don't want to print errors
1034 warn_line_boards: Dict keyed by warning line, containing a list of
1035 the Board objects with that warning
Simon Glass843312d2015-02-05 22:06:15 -07001036 config: Dictionary keyed by filename - e.g. '.config'. Each
1037 value is itself a dictionary:
1038 key: config name
1039 value: config value
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001040 show_sizes: Show image size deltas
1041 show_detail: Show detail for each board
1042 show_bloat: Show detail for each function
Simon Glass843312d2015-02-05 22:06:15 -07001043 show_config: Show config changes
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001044 """
Simon Glasse30965d2014-08-28 09:43:44 -06001045 def _BoardList(line, line_boards):
Simon Glassed966652014-08-28 09:43:43 -06001046 """Helper function to get a line of boards containing a line
1047
1048 Args:
1049 line: Error line to search for
1050 Return:
1051 String containing a list of boards with that error line, or
1052 '' if the user has not requested such a list
1053 """
1054 if self._list_error_boards:
1055 names = []
Simon Glasse30965d2014-08-28 09:43:44 -06001056 for board in line_boards[line]:
Simon Glassf66153b2014-10-16 01:05:55 -06001057 if not board.target in names:
1058 names.append(board.target)
Simon Glassed966652014-08-28 09:43:43 -06001059 names_str = '(%s) ' % ','.join(names)
1060 else:
1061 names_str = ''
1062 return names_str
1063
Simon Glasse30965d2014-08-28 09:43:44 -06001064 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1065 char):
1066 better_lines = []
1067 worse_lines = []
1068 for line in lines:
1069 if line not in base_lines:
1070 worse_lines.append(char + '+' +
1071 _BoardList(line, line_boards) + line)
1072 for line in base_lines:
1073 if line not in lines:
1074 better_lines.append(char + '-' +
1075 _BoardList(line, base_line_boards) + line)
1076 return better_lines, worse_lines
1077
Simon Glass843312d2015-02-05 22:06:15 -07001078 def _CalcConfig(delta, name, config):
1079 """Calculate configuration changes
1080
1081 Args:
1082 delta: Type of the delta, e.g. '+'
1083 name: name of the file which changed (e.g. .config)
1084 config: configuration change dictionary
1085 key: config name
1086 value: config value
1087 Returns:
1088 String containing the configuration changes which can be
1089 printed
1090 """
1091 out = ''
1092 for key in sorted(config.keys()):
1093 out += '%s=%s ' % (key, config[key])
Simon Glass8270e3c2015-08-25 21:52:14 -06001094 return '%s %s: %s' % (delta, name, out)
Simon Glass843312d2015-02-05 22:06:15 -07001095
Simon Glass8270e3c2015-08-25 21:52:14 -06001096 def _AddConfig(lines, name, config_plus, config_minus, config_change):
1097 """Add changes in configuration to a list
Simon Glass843312d2015-02-05 22:06:15 -07001098
1099 Args:
Simon Glass8270e3c2015-08-25 21:52:14 -06001100 lines: list to add to
1101 name: config file name
Simon Glass843312d2015-02-05 22:06:15 -07001102 config_plus: configurations added, dictionary
1103 key: config name
1104 value: config value
1105 config_minus: configurations removed, dictionary
1106 key: config name
1107 value: config value
1108 config_change: configurations changed, dictionary
1109 key: config name
1110 value: config value
1111 """
1112 if config_plus:
Simon Glass8270e3c2015-08-25 21:52:14 -06001113 lines.append(_CalcConfig('+', name, config_plus))
Simon Glass843312d2015-02-05 22:06:15 -07001114 if config_minus:
Simon Glass8270e3c2015-08-25 21:52:14 -06001115 lines.append(_CalcConfig('-', name, config_minus))
Simon Glass843312d2015-02-05 22:06:15 -07001116 if config_change:
Simon Glass8270e3c2015-08-25 21:52:14 -06001117 lines.append(_CalcConfig('c', name, config_change))
1118
1119 def _OutputConfigInfo(lines):
1120 for line in lines:
1121 if not line:
1122 continue
1123 if line[0] == '+':
1124 col = self.col.GREEN
1125 elif line[0] == '-':
1126 col = self.col.RED
1127 elif line[0] == 'c':
1128 col = self.col.YELLOW
1129 Print(' ' + line, newline=True, colour=col)
1130
Simon Glass843312d2015-02-05 22:06:15 -07001131
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001132 better = [] # List of boards fixed since last commit
1133 worse = [] # List of new broken boards since last commit
1134 new = [] # List of boards that didn't exist last time
1135 unknown = [] # List of boards that were not built
1136
1137 for target in board_dict:
1138 if target not in board_selected:
1139 continue
1140
1141 # If the board was built last time, add its outcome to a list
1142 if target in self._base_board_dict:
1143 base_outcome = self._base_board_dict[target].rc
1144 outcome = board_dict[target]
1145 if outcome.rc == OUTCOME_UNKNOWN:
1146 unknown.append(target)
1147 elif outcome.rc < base_outcome:
1148 better.append(target)
1149 elif outcome.rc > base_outcome:
1150 worse.append(target)
1151 else:
1152 new.append(target)
1153
1154 # Get a list of errors that have appeared, and disappeared
Simon Glasse30965d2014-08-28 09:43:44 -06001155 better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1156 self._base_err_line_boards, err_lines, err_line_boards, '')
1157 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1158 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001159
1160 # Display results by arch
Simon Glasse30965d2014-08-28 09:43:44 -06001161 if (better or worse or unknown or new or worse_err or better_err
1162 or worse_warn or better_warn):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001163 arch_list = {}
1164 self.AddOutcome(board_selected, arch_list, better, '',
1165 self.col.GREEN)
1166 self.AddOutcome(board_selected, arch_list, worse, '+',
1167 self.col.RED)
1168 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE)
1169 if self._show_unknown:
1170 self.AddOutcome(board_selected, arch_list, unknown, '?',
1171 self.col.MAGENTA)
1172 for arch, target_list in arch_list.iteritems():
Simon Glass4653a882014-09-05 19:00:07 -06001173 Print('%10s: %s' % (arch, target_list))
Simon Glass28370c12014-08-09 15:33:06 -06001174 self._error_lines += 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001175 if better_err:
Simon Glass4653a882014-09-05 19:00:07 -06001176 Print('\n'.join(better_err), colour=self.col.GREEN)
Simon Glass28370c12014-08-09 15:33:06 -06001177 self._error_lines += 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001178 if worse_err:
Simon Glass4653a882014-09-05 19:00:07 -06001179 Print('\n'.join(worse_err), colour=self.col.RED)
Simon Glass28370c12014-08-09 15:33:06 -06001180 self._error_lines += 1
Simon Glasse30965d2014-08-28 09:43:44 -06001181 if better_warn:
Simon Glass4653a882014-09-05 19:00:07 -06001182 Print('\n'.join(better_warn), colour=self.col.CYAN)
Simon Glasse30965d2014-08-28 09:43:44 -06001183 self._error_lines += 1
1184 if worse_warn:
Simon Glass4653a882014-09-05 19:00:07 -06001185 Print('\n'.join(worse_warn), colour=self.col.MAGENTA)
Simon Glasse30965d2014-08-28 09:43:44 -06001186 self._error_lines += 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001187
1188 if show_sizes:
1189 self.PrintSizeSummary(board_selected, board_dict, show_detail,
1190 show_bloat)
1191
Simon Glass8270e3c2015-08-25 21:52:14 -06001192 if show_config and self._base_config:
1193 summary = {}
1194 arch_config_plus = {}
1195 arch_config_minus = {}
1196 arch_config_change = {}
1197 arch_list = []
1198
1199 for target in board_dict:
1200 if target not in board_selected:
Simon Glass843312d2015-02-05 22:06:15 -07001201 continue
Simon Glass8270e3c2015-08-25 21:52:14 -06001202 arch = board_selected[target].arch
1203 if arch not in arch_list:
1204 arch_list.append(arch)
1205
1206 for arch in arch_list:
1207 arch_config_plus[arch] = {}
1208 arch_config_minus[arch] = {}
1209 arch_config_change[arch] = {}
Simon Glassb464f8e2016-11-13 14:25:53 -07001210 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001211 arch_config_plus[arch][name] = {}
1212 arch_config_minus[arch][name] = {}
1213 arch_config_change[arch][name] = {}
1214
1215 for target in board_dict:
1216 if target not in board_selected:
1217 continue
1218
1219 arch = board_selected[target].arch
1220
1221 all_config_plus = {}
1222 all_config_minus = {}
1223 all_config_change = {}
1224 tbase = self._base_config[target]
1225 tconfig = config[target]
1226 lines = []
Simon Glassb464f8e2016-11-13 14:25:53 -07001227 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001228 if not tconfig.config[name]:
1229 continue
1230 config_plus = {}
1231 config_minus = {}
1232 config_change = {}
1233 base = tbase.config[name]
1234 for key, value in tconfig.config[name].iteritems():
1235 if key not in base:
1236 config_plus[key] = value
1237 all_config_plus[key] = value
1238 for key, value in base.iteritems():
1239 if key not in tconfig.config[name]:
1240 config_minus[key] = value
1241 all_config_minus[key] = value
1242 for key, value in base.iteritems():
1243 new_value = tconfig.config.get(key)
1244 if new_value and value != new_value:
1245 desc = '%s -> %s' % (value, new_value)
1246 config_change[key] = desc
1247 all_config_change[key] = desc
1248
1249 arch_config_plus[arch][name].update(config_plus)
1250 arch_config_minus[arch][name].update(config_minus)
1251 arch_config_change[arch][name].update(config_change)
1252
1253 _AddConfig(lines, name, config_plus, config_minus,
1254 config_change)
1255 _AddConfig(lines, 'all', all_config_plus, all_config_minus,
1256 all_config_change)
1257 summary[target] = '\n'.join(lines)
1258
1259 lines_by_target = {}
1260 for target, lines in summary.iteritems():
1261 if lines in lines_by_target:
1262 lines_by_target[lines].append(target)
1263 else:
1264 lines_by_target[lines] = [target]
1265
1266 for arch in arch_list:
1267 lines = []
1268 all_plus = {}
1269 all_minus = {}
1270 all_change = {}
Simon Glassb464f8e2016-11-13 14:25:53 -07001271 for name in self.config_filenames:
Simon Glass8270e3c2015-08-25 21:52:14 -06001272 all_plus.update(arch_config_plus[arch][name])
1273 all_minus.update(arch_config_minus[arch][name])
1274 all_change.update(arch_config_change[arch][name])
1275 _AddConfig(lines, name, arch_config_plus[arch][name],
1276 arch_config_minus[arch][name],
1277 arch_config_change[arch][name])
1278 _AddConfig(lines, 'all', all_plus, all_minus, all_change)
1279 #arch_summary[target] = '\n'.join(lines)
1280 if lines:
1281 Print('%s:' % arch)
1282 _OutputConfigInfo(lines)
1283
1284 for lines, targets in lines_by_target.iteritems():
1285 if not lines:
1286 continue
1287 Print('%s :' % ' '.join(sorted(targets)))
1288 _OutputConfigInfo(lines.split('\n'))
1289
Simon Glass843312d2015-02-05 22:06:15 -07001290
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001291 # Save our updated information for the next call to this function
1292 self._base_board_dict = board_dict
1293 self._base_err_lines = err_lines
Simon Glasse30965d2014-08-28 09:43:44 -06001294 self._base_warn_lines = warn_lines
1295 self._base_err_line_boards = err_line_boards
1296 self._base_warn_line_boards = warn_line_boards
Simon Glass843312d2015-02-05 22:06:15 -07001297 self._base_config = config
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001298
1299 # Get a list of boards that did not get built, if needed
1300 not_built = []
1301 for board in board_selected:
1302 if not board in board_dict:
1303 not_built.append(board)
1304 if not_built:
Simon Glass4653a882014-09-05 19:00:07 -06001305 Print("Boards not built (%d): %s" % (len(not_built),
1306 ', '.join(not_built)))
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001307
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001308 def ProduceResultSummary(self, commit_upto, commits, board_selected):
Simon Glasse30965d2014-08-28 09:43:44 -06001309 (board_dict, err_lines, err_line_boards, warn_lines,
Simon Glass843312d2015-02-05 22:06:15 -07001310 warn_line_boards, config) = self.GetResultSummary(
Simon Glassed966652014-08-28 09:43:43 -06001311 board_selected, commit_upto,
Simon Glass843312d2015-02-05 22:06:15 -07001312 read_func_sizes=self._show_bloat,
1313 read_config=self._show_config)
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001314 if commits:
1315 msg = '%02d: %s' % (commit_upto + 1,
1316 commits[commit_upto].subject)
Simon Glass4653a882014-09-05 19:00:07 -06001317 Print(msg, colour=self.col.BLUE)
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001318 self.PrintResultSummary(board_selected, board_dict,
Simon Glassed966652014-08-28 09:43:43 -06001319 err_lines if self._show_errors else [], err_line_boards,
Simon Glasse30965d2014-08-28 09:43:44 -06001320 warn_lines if self._show_errors else [], warn_line_boards,
Simon Glass843312d2015-02-05 22:06:15 -07001321 config, self._show_sizes, self._show_detail,
1322 self._show_bloat, self._show_config)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001323
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001324 def ShowSummary(self, commits, board_selected):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001325 """Show a build summary for U-Boot for a given board list.
1326
1327 Reset the result summary, then repeatedly call GetResultSummary on
1328 each commit's results, then display the differences we see.
1329
1330 Args:
1331 commit: Commit objects to summarise
1332 board_selected: Dict containing boards to summarise
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001333 """
Simon Glassfea58582014-08-09 15:32:59 -06001334 self.commit_count = len(commits) if commits else 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001335 self.commits = commits
1336 self.ResetResultSummary(board_selected)
Simon Glass28370c12014-08-09 15:33:06 -06001337 self._error_lines = 0
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001338
1339 for commit_upto in range(0, self.commit_count, self._step):
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001340 self.ProduceResultSummary(commit_upto, commits, board_selected)
Simon Glass28370c12014-08-09 15:33:06 -06001341 if not self._error_lines:
Simon Glass4653a882014-09-05 19:00:07 -06001342 Print('(no errors to report)', colour=self.col.GREEN)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001343
1344
1345 def SetupBuild(self, board_selected, commits):
1346 """Set up ready to start a build.
1347
1348 Args:
1349 board_selected: Selected boards to build
1350 commits: Selected commits to build
1351 """
1352 # First work out how many commits we will build
Simon Glassfea58582014-08-09 15:32:59 -06001353 count = (self.commit_count + self._step - 1) / self._step
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001354 self.count = len(board_selected) * count
1355 self.upto = self.warned = self.fail = 0
1356 self._timestamps = collections.deque()
1357
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001358 def GetThreadDir(self, thread_num):
1359 """Get the directory path to the working dir for a thread.
1360
1361 Args:
1362 thread_num: Number of thread to check.
1363 """
1364 return os.path.join(self._working_dir, '%02d' % thread_num)
1365
Simon Glassfea58582014-08-09 15:32:59 -06001366 def _PrepareThread(self, thread_num, setup_git):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001367 """Prepare the working directory for a thread.
1368
1369 This clones or fetches the repo into the thread's work directory.
1370
1371 Args:
1372 thread_num: Thread number (0, 1, ...)
Simon Glassfea58582014-08-09 15:32:59 -06001373 setup_git: True to set up a git repo clone
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001374 """
1375 thread_dir = self.GetThreadDir(thread_num)
Simon Glass190064b2014-08-09 15:33:00 -06001376 builderthread.Mkdir(thread_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001377 git_dir = os.path.join(thread_dir, '.git')
1378
1379 # Clone the repo if it doesn't already exist
1380 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1381 # we have a private index but uses the origin repo's contents?
Simon Glassfea58582014-08-09 15:32:59 -06001382 if setup_git and self.git_dir:
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001383 src_dir = os.path.abspath(self.git_dir)
1384 if os.path.exists(git_dir):
1385 gitutil.Fetch(git_dir, thread_dir)
1386 else:
Simon Glass21f0eb32016-09-18 16:48:31 -06001387 Print('\rCloning repo for thread %d' % thread_num,
1388 newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001389 gitutil.Clone(src_dir, thread_dir)
Simon Glass21f0eb32016-09-18 16:48:31 -06001390 Print('\r%s\r' % (' ' * 30), newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001391
Simon Glassfea58582014-08-09 15:32:59 -06001392 def _PrepareWorkingSpace(self, max_threads, setup_git):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001393 """Prepare the working directory for use.
1394
1395 Set up the git repo for each thread.
1396
1397 Args:
1398 max_threads: Maximum number of threads we expect to need.
Simon Glassfea58582014-08-09 15:32:59 -06001399 setup_git: True to set up a git repo clone
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001400 """
Simon Glass190064b2014-08-09 15:33:00 -06001401 builderthread.Mkdir(self._working_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001402 for thread in range(max_threads):
Simon Glassfea58582014-08-09 15:32:59 -06001403 self._PrepareThread(thread, setup_git)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001404
1405 def _PrepareOutputSpace(self):
1406 """Get the output directories ready to receive files.
1407
1408 We delete any output directories which look like ones we need to
1409 create. Having left over directories is confusing when the user wants
1410 to check the output manually.
1411 """
Simon Glass1a915672014-12-01 17:33:53 -07001412 if not self.commits:
1413 return
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001414 dir_list = []
1415 for commit_upto in range(self.commit_count):
1416 dir_list.append(self._GetOutputDir(commit_upto))
1417
Simon Glassb222abe2016-09-18 16:48:32 -06001418 to_remove = []
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001419 for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1420 if dirname not in dir_list:
Simon Glassb222abe2016-09-18 16:48:32 -06001421 to_remove.append(dirname)
1422 if to_remove:
1423 Print('Removing %d old build directories' % len(to_remove),
1424 newline=False)
1425 for dirname in to_remove:
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001426 shutil.rmtree(dirname)
1427
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001428 def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001429 """Build all commits for a list of boards
1430
1431 Args:
1432 commits: List of commits to be build, each a Commit object
1433 boards_selected: Dict of selected boards, key is target name,
1434 value is Board object
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001435 keep_outputs: True to save build output files
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001436 verbose: Display build results as they are completed
Simon Glass2c3deb92014-08-28 09:43:39 -06001437 Returns:
1438 Tuple containing:
1439 - number of boards that failed to build
1440 - number of boards that issued warnings
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001441 """
Simon Glassfea58582014-08-09 15:32:59 -06001442 self.commit_count = len(commits) if commits else 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001443 self.commits = commits
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001444 self._verbose = verbose
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001445
1446 self.ResetResultSummary(board_selected)
Thierry Redingf3d015c2014-08-19 10:22:39 +02001447 builderthread.Mkdir(self.base_dir, parents = True)
Simon Glassfea58582014-08-09 15:32:59 -06001448 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1449 commits is not None)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001450 self._PrepareOutputSpace()
Simon Glass745b3952016-09-18 16:48:33 -06001451 Print('\rStarting build...', newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001452 self.SetupBuild(board_selected, commits)
1453 self.ProcessResult(None)
1454
1455 # Create jobs to build all commits for each board
1456 for brd in board_selected.itervalues():
Simon Glass190064b2014-08-09 15:33:00 -06001457 job = builderthread.BuilderJob()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001458 job.board = brd
1459 job.commits = commits
1460 job.keep_outputs = keep_outputs
1461 job.step = self._step
1462 self.queue.put(job)
1463
Simon Glassd436e382016-09-18 16:48:35 -06001464 term = threading.Thread(target=self.queue.join)
1465 term.setDaemon(True)
1466 term.start()
1467 while term.isAlive():
1468 term.join(100)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001469
1470 # Wait until we have processed all output
1471 self.out_queue.join()
Simon Glass4653a882014-09-05 19:00:07 -06001472 Print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001473 self.ClearLine(0)
Simon Glass2c3deb92014-08-28 09:43:39 -06001474 return (self.fail, self.warned)