blob: 384f053015e06f95778cda8c2a53a0f442d62349 [file] [log] [blame]
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001# Copyright (c) 2013 The Chromium OS Authors.
2#
3# Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
4#
Wolfgang Denk1a459662013-07-08 09:37:19 +02005# SPDX-License-Identifier: GPL-2.0+
Simon Glassfc3fe1c2013-04-03 11:07:16 +00006#
7
8import collections
Simon Glassfc3fe1c2013-04-03 11:07:16 +00009from datetime import datetime, timedelta
10import glob
11import os
12import re
13import Queue
14import shutil
15import string
16import sys
Simon Glassfc3fe1c2013-04-03 11:07:16 +000017import time
18
Simon Glass190064b2014-08-09 15:33:00 -060019import builderthread
Simon Glassfc3fe1c2013-04-03 11:07:16 +000020import command
21import gitutil
22import terminal
Simon Glass4653a882014-09-05 19:00:07 -060023from terminal import Print
Simon Glassfc3fe1c2013-04-03 11:07:16 +000024import toolchain
25
26
27"""
28Theory of Operation
29
30Please see README for user documentation, and you should be familiar with
31that before trying to make sense of this.
32
33Buildman works by keeping the machine as busy as possible, building different
34commits for different boards on multiple CPUs at once.
35
36The source repo (self.git_dir) contains all the commits to be built. Each
37thread works on a single board at a time. It checks out the first commit,
38configures it for that board, then builds it. Then it checks out the next
39commit and builds it (typically without re-configuring). When it runs out
40of commits, it gets another job from the builder and starts again with that
41board.
42
43Clearly the builder threads could work either way - they could check out a
44commit and then built it for all boards. Using separate directories for each
45commit/board pair they could leave their build product around afterwards
46also.
47
48The intent behind building a single board for multiple commits, is to make
49use of incremental builds. Since each commit is built incrementally from
50the previous one, builds are faster. Reconfiguring for a different board
51removes all intermediate object files.
52
53Many threads can be working at once, but each has its own working directory.
54When a thread finishes a build, it puts the output files into a result
55directory.
56
57The base directory used by buildman is normally '../<branch>', i.e.
58a directory higher than the source repository and named after the branch
59being built.
60
61Within the base directory, we have one subdirectory for each commit. Within
62that is one subdirectory for each board. Within that is the build output for
63that commit/board combination.
64
65Buildman also create working directories for each thread, in a .bm-work/
66subdirectory in the base dir.
67
68As an example, say we are building branch 'us-net' for boards 'sandbox' and
69'seaboard', and say that us-net has two commits. We will have directories
70like this:
71
72us-net/ base directory
73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/
74 sandbox/
75 u-boot.bin
76 seaboard/
77 u-boot.bin
78 02_of_02_g4ed4ebc_net--Check-tftp-comp/
79 sandbox/
80 u-boot.bin
81 seaboard/
82 u-boot.bin
83 .bm-work/
84 00/ working directory for thread 0 (contains source checkout)
85 build/ build output
86 01/ working directory for thread 1
87 build/ build output
88 ...
89u-boot/ source directory
90 .git/ repository
91"""
92
93# Possible build outcomes
94OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4)
95
96# Translate a commit subject into a valid filename
97trans_valid_chars = string.maketrans("/: ", "---")
98
Simon Glass843312d2015-02-05 22:06:15 -070099CONFIG_FILENAMES = [
100 '.config', '.config-spl', '.config-tpl',
101 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
102 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
103 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
104]
105
Simon Glass8270e3c2015-08-25 21:52:14 -0600106class Config:
107 """Holds information about configuration settings for a board."""
108 def __init__(self, target):
109 self.target = target
110 self.config = {}
111 for fname in CONFIG_FILENAMES:
112 self.config[fname] = {}
113
114 def Add(self, fname, key, value):
115 self.config[fname][key] = value
116
117 def __hash__(self):
118 val = 0
119 for fname in self.config:
120 for key, value in self.config[fname].iteritems():
121 print key, value
122 val = val ^ hash(key) & hash(value)
123 return val
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000124
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000125class Builder:
126 """Class for building U-Boot for a particular commit.
127
128 Public members: (many should ->private)
129 active: True if the builder is active and has not been stopped
130 already_done: Number of builds already completed
131 base_dir: Base directory to use for builder
132 checkout: True to check out source, False to skip that step.
133 This is used for testing.
134 col: terminal.Color() object
135 count: Number of commits to build
136 do_make: Method to call to invoke Make
137 fail: Number of builds that failed due to error
138 force_build: Force building even if a build already exists
139 force_config_on_failure: If a commit fails for a board, disable
140 incremental building for the next commit we build for that
141 board, so that we will see all warnings/errors again.
Simon Glass4266dc22014-07-13 12:22:31 -0600142 force_build_failures: If a previously-built build (i.e. built on
143 a previous run of buildman) is marked as failed, rebuild it.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000144 git_dir: Git directory containing source repository
145 last_line_len: Length of the last line we printed (used for erasing
146 it with new progress information)
147 num_jobs: Number of jobs to run at once (passed to make as -j)
148 num_threads: Number of builder threads to run
149 out_queue: Queue of results to process
150 re_make_err: Compiled regular expression for ignore_lines
151 queue: Queue of jobs to run
152 threads: List of active threads
153 toolchains: Toolchains object to use for building
154 upto: Current commit number we are building (0.count-1)
155 warned: Number of builds that produced at least one warning
Simon Glass97e91522014-07-14 17:51:02 -0600156 force_reconfig: Reconfigure U-Boot on each comiit. This disables
157 incremental building, where buildman reconfigures on the first
158 commit for a baord, and then just does an incremental build for
159 the following commits. In fact buildman will reconfigure and
160 retry for any failing commits, so generally the only effect of
161 this option is to slow things down.
Simon Glass189a4962014-07-14 17:51:03 -0600162 in_tree: Build U-Boot in-tree instead of specifying an output
163 directory separate from the source code. This option is really
164 only useful for testing in-tree builds.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000165
166 Private members:
167 _base_board_dict: Last-summarised Dict of boards
168 _base_err_lines: Last-summarised list of errors
Simon Glasse30965d2014-08-28 09:43:44 -0600169 _base_warn_lines: Last-summarised list of warnings
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000170 _build_period_us: Time taken for a single build (float object).
171 _complete_delay: Expected delay until completion (timedelta)
172 _next_delay_update: Next time we plan to display a progress update
173 (datatime)
174 _show_unknown: Show unknown boards (those not built) in summary
175 _timestamps: List of timestamps for the completion of the last
176 last _timestamp_count builds. Each is a datetime object.
177 _timestamp_count: Number of timestamps to keep in our list.
178 _working_dir: Base working directory containing all threads
179 """
180 class Outcome:
181 """Records a build outcome for a single make invocation
182
183 Public Members:
184 rc: Outcome value (OUTCOME_...)
185 err_lines: List of error lines or [] if none
186 sizes: Dictionary of image size information, keyed by filename
187 - Each value is itself a dictionary containing
188 values for 'text', 'data' and 'bss', being the integer
189 size in bytes of each section.
190 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
191 value is itself a dictionary:
192 key: function name
193 value: Size of function in bytes
Simon Glass843312d2015-02-05 22:06:15 -0700194 config: Dictionary keyed by filename - e.g. '.config'. Each
195 value is itself a dictionary:
196 key: config name
197 value: config value
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000198 """
Simon Glass843312d2015-02-05 22:06:15 -0700199 def __init__(self, rc, err_lines, sizes, func_sizes, config):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000200 self.rc = rc
201 self.err_lines = err_lines
202 self.sizes = sizes
203 self.func_sizes = func_sizes
Simon Glass843312d2015-02-05 22:06:15 -0700204 self.config = config
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000205
206 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
Simon Glass5971ab52014-12-01 17:33:55 -0700207 gnu_make='make', checkout=True, show_unknown=True, step=1,
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600208 no_subdirs=False, full_path=False, verbose_build=False,
209 incremental=False, per_board_out_dir=False):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000210 """Create a new Builder object
211
212 Args:
213 toolchains: Toolchains object to use for building
214 base_dir: Base directory to use for builder
215 git_dir: Git directory containing source repository
216 num_threads: Number of builder threads to run
217 num_jobs: Number of jobs to run at once (passed to make as -j)
Masahiro Yamada99796922014-07-22 11:19:09 +0900218 gnu_make: the command name of GNU Make.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000219 checkout: True to check out source, False to skip that step.
220 This is used for testing.
221 show_unknown: Show unknown boards (those not built) in summary
222 step: 1 to process every commit, n to process every nth commit
Simon Glassbb1501f2014-12-01 17:34:00 -0700223 no_subdirs: Don't create subdirectories when building current
224 source for a single board
225 full_path: Return the full path in CROSS_COMPILE and don't set
226 PATH
Simon Glassd2ce6582014-12-01 17:34:07 -0700227 verbose_build: Run build with V=1 and don't use 'make -s'
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600228 incremental: Always perform incremental builds; don't run make
229 mrproper when configuring
230 per_board_out_dir: Build in a separate persistent directory per
231 board rather than a thread-specific directory
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000232 """
233 self.toolchains = toolchains
234 self.base_dir = base_dir
235 self._working_dir = os.path.join(base_dir, '.bm-work')
236 self.threads = []
237 self.active = True
238 self.do_make = self.Make
Masahiro Yamada99796922014-07-22 11:19:09 +0900239 self.gnu_make = gnu_make
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000240 self.checkout = checkout
241 self.num_threads = num_threads
242 self.num_jobs = num_jobs
243 self.already_done = 0
244 self.force_build = False
245 self.git_dir = git_dir
246 self._show_unknown = show_unknown
247 self._timestamp_count = 10
248 self._build_period_us = None
249 self._complete_delay = None
250 self._next_delay_update = datetime.now()
251 self.force_config_on_failure = True
Simon Glass4266dc22014-07-13 12:22:31 -0600252 self.force_build_failures = False
Simon Glass97e91522014-07-14 17:51:02 -0600253 self.force_reconfig = False
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000254 self._step = step
Simon Glass189a4962014-07-14 17:51:03 -0600255 self.in_tree = False
Simon Glass28370c12014-08-09 15:33:06 -0600256 self._error_lines = 0
Simon Glass5971ab52014-12-01 17:33:55 -0700257 self.no_subdirs = no_subdirs
Simon Glassbb1501f2014-12-01 17:34:00 -0700258 self.full_path = full_path
Simon Glassd2ce6582014-12-01 17:34:07 -0700259 self.verbose_build = verbose_build
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000260
261 self.col = terminal.Color()
262
Simon Glasse30965d2014-08-28 09:43:44 -0600263 self._re_function = re.compile('(.*): In function.*')
264 self._re_files = re.compile('In file included from.*')
265 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
266 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
267
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000268 self.queue = Queue.Queue()
269 self.out_queue = Queue.Queue()
270 for i in range(self.num_threads):
Stephen Warrenf79f1e02016-04-11 10:48:44 -0600271 t = builderthread.BuilderThread(self, i, incremental,
272 per_board_out_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000273 t.setDaemon(True)
274 t.start()
275 self.threads.append(t)
276
277 self.last_line_len = 0
Simon Glass190064b2014-08-09 15:33:00 -0600278 t = builderthread.ResultThread(self)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000279 t.setDaemon(True)
280 t.start()
281 self.threads.append(t)
282
283 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
284 self.re_make_err = re.compile('|'.join(ignore_lines))
285
286 def __del__(self):
287 """Get rid of all threads created by the builder"""
288 for t in self.threads:
289 del t
290
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600291 def SetDisplayOptions(self, show_errors=False, show_sizes=False,
Simon Glassed966652014-08-28 09:43:43 -0600292 show_detail=False, show_bloat=False,
Simon Glass843312d2015-02-05 22:06:15 -0700293 list_error_boards=False, show_config=False):
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600294 """Setup display options for the builder.
295
296 show_errors: True to show summarised error/warning info
297 show_sizes: Show size deltas
298 show_detail: Show detail for each board
299 show_bloat: Show detail for each function
Simon Glassed966652014-08-28 09:43:43 -0600300 list_error_boards: Show the boards which caused each error/warning
Simon Glass843312d2015-02-05 22:06:15 -0700301 show_config: Show config deltas
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600302 """
303 self._show_errors = show_errors
304 self._show_sizes = show_sizes
305 self._show_detail = show_detail
306 self._show_bloat = show_bloat
Simon Glassed966652014-08-28 09:43:43 -0600307 self._list_error_boards = list_error_boards
Simon Glass843312d2015-02-05 22:06:15 -0700308 self._show_config = show_config
Simon Glassb2ea7ab2014-08-09 15:33:02 -0600309
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000310 def _AddTimestamp(self):
311 """Add a new timestamp to the list and record the build period.
312
313 The build period is the length of time taken to perform a single
314 build (one board, one commit).
315 """
316 now = datetime.now()
317 self._timestamps.append(now)
318 count = len(self._timestamps)
319 delta = self._timestamps[-1] - self._timestamps[0]
320 seconds = delta.total_seconds()
321
322 # If we have enough data, estimate build period (time taken for a
323 # single build) and therefore completion time.
324 if count > 1 and self._next_delay_update < now:
325 self._next_delay_update = now + timedelta(seconds=2)
326 if seconds > 0:
327 self._build_period = float(seconds) / count
328 todo = self.count - self.upto
329 self._complete_delay = timedelta(microseconds=
330 self._build_period * todo * 1000000)
331 # Round it
332 self._complete_delay -= timedelta(
333 microseconds=self._complete_delay.microseconds)
334
335 if seconds > 60:
336 self._timestamps.popleft()
337 count -= 1
338
339 def ClearLine(self, length):
340 """Clear any characters on the current line
341
342 Make way for a new line of length 'length', by outputting enough
343 spaces to clear out the old line. Then remember the new length for
344 next time.
345
346 Args:
347 length: Length of new line, in characters
348 """
349 if length < self.last_line_len:
Simon Glass4653a882014-09-05 19:00:07 -0600350 Print(' ' * (self.last_line_len - length), newline=False)
351 Print('\r', newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000352 self.last_line_len = length
353 sys.stdout.flush()
354
355 def SelectCommit(self, commit, checkout=True):
356 """Checkout the selected commit for this build
357 """
358 self.commit = commit
359 if checkout and self.checkout:
360 gitutil.Checkout(commit.hash)
361
362 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
363 """Run make
364
365 Args:
366 commit: Commit object that is being built
367 brd: Board object that is being built
Roger Meierfd18a892014-08-20 22:10:29 +0200368 stage: Stage that we are at (mrproper, config, build)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000369 cwd: Directory where make should be run
370 args: Arguments to pass to make
371 kwargs: Arguments to pass to command.RunPipe()
372 """
Masahiro Yamada99796922014-07-22 11:19:09 +0900373 cmd = [self.gnu_make] + list(args)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000374 result = command.RunPipe([cmd], capture=True, capture_stderr=True,
375 cwd=cwd, raise_on_error=False, **kwargs)
Simon Glass40f11fc2015-02-05 22:06:12 -0700376 if self.verbose_build:
377 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
378 result.combined = '%s\n' % (' '.join(cmd)) + result.combined
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000379 return result
380
381 def ProcessResult(self, result):
382 """Process the result of a build, showing progress information
383
384 Args:
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600385 result: A CommandResult object, which indicates the result for
386 a single build
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000387 """
388 col = terminal.Color()
389 if result:
390 target = result.brd.target
391
392 if result.return_code < 0:
393 self.active = False
394 command.StopAll()
395 return
396
397 self.upto += 1
398 if result.return_code != 0:
399 self.fail += 1
400 elif result.stderr:
401 self.warned += 1
402 if result.already_done:
403 self.already_done += 1
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600404 if self._verbose:
Simon Glass4653a882014-09-05 19:00:07 -0600405 Print('\r', newline=False)
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600406 self.ClearLine(0)
407 boards_selected = {target : result.brd}
408 self.ResetResultSummary(boards_selected)
409 self.ProduceResultSummary(result.commit_upto, self.commits,
410 boards_selected)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000411 else:
412 target = '(starting)'
413
414 # Display separate counts for ok, warned and fail
415 ok = self.upto - self.warned - self.fail
416 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
417 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
418 line += self.col.Color(self.col.RED, '%5d' % self.fail)
419
420 name = ' /%-5d ' % self.count
421
422 # Add our current completion time estimate
423 self._AddTimestamp()
424 if self._complete_delay:
425 name += '%s : ' % self._complete_delay
426 # When building all boards for a commit, we can print a commit
427 # progress message.
428 if result and result.commit_upto is None:
429 name += 'commit %2d/%-3d' % (self.commit_upto + 1,
430 self.commit_count)
431
432 name += target
Simon Glass4653a882014-09-05 19:00:07 -0600433 Print(line + name, newline=False)
Simon Glasse5a0e5d2014-08-09 15:33:03 -0600434 length = 14 + len(name)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000435 self.ClearLine(length)
436
437 def _GetOutputDir(self, commit_upto):
438 """Get the name of the output directory for a commit number
439
440 The output directory is typically .../<branch>/<commit>.
441
442 Args:
443 commit_upto: Commit number to use (0..self.count-1)
444 """
Simon Glass5971ab52014-12-01 17:33:55 -0700445 commit_dir = None
Simon Glassfea58582014-08-09 15:32:59 -0600446 if self.commits:
447 commit = self.commits[commit_upto]
448 subject = commit.subject.translate(trans_valid_chars)
449 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
450 self.commit_count, commit.hash, subject[:20]))
Simon Glass5971ab52014-12-01 17:33:55 -0700451 elif not self.no_subdirs:
Simon Glassfea58582014-08-09 15:32:59 -0600452 commit_dir = 'current'
Simon Glass5971ab52014-12-01 17:33:55 -0700453 if not commit_dir:
454 return self.base_dir
455 return os.path.join(self.base_dir, commit_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000456
457 def GetBuildDir(self, commit_upto, target):
458 """Get the name of the build directory for a commit number
459
460 The build directory is typically .../<branch>/<commit>/<target>.
461
462 Args:
463 commit_upto: Commit number to use (0..self.count-1)
464 target: Target name
465 """
466 output_dir = self._GetOutputDir(commit_upto)
467 return os.path.join(output_dir, target)
468
469 def GetDoneFile(self, commit_upto, target):
470 """Get the name of the done file for a commit number
471
472 Args:
473 commit_upto: Commit number to use (0..self.count-1)
474 target: Target name
475 """
476 return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
477
478 def GetSizesFile(self, commit_upto, target):
479 """Get the name of the sizes file for a commit number
480
481 Args:
482 commit_upto: Commit number to use (0..self.count-1)
483 target: Target name
484 """
485 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
486
487 def GetFuncSizesFile(self, commit_upto, target, elf_fname):
488 """Get the name of the funcsizes file for a commit number and ELF file
489
490 Args:
491 commit_upto: Commit number to use (0..self.count-1)
492 target: Target name
493 elf_fname: Filename of elf image
494 """
495 return os.path.join(self.GetBuildDir(commit_upto, target),
496 '%s.sizes' % elf_fname.replace('/', '-'))
497
498 def GetObjdumpFile(self, commit_upto, target, elf_fname):
499 """Get the name of the objdump file for a commit number and ELF file
500
501 Args:
502 commit_upto: Commit number to use (0..self.count-1)
503 target: Target name
504 elf_fname: Filename of elf image
505 """
506 return os.path.join(self.GetBuildDir(commit_upto, target),
507 '%s.objdump' % elf_fname.replace('/', '-'))
508
509 def GetErrFile(self, commit_upto, target):
510 """Get the name of the err file for a commit number
511
512 Args:
513 commit_upto: Commit number to use (0..self.count-1)
514 target: Target name
515 """
516 output_dir = self.GetBuildDir(commit_upto, target)
517 return os.path.join(output_dir, 'err')
518
519 def FilterErrors(self, lines):
520 """Filter out errors in which we have no interest
521
522 We should probably use map().
523
524 Args:
525 lines: List of error lines, each a string
526 Returns:
527 New list with only interesting lines included
528 """
529 out_lines = []
530 for line in lines:
531 if not self.re_make_err.search(line):
532 out_lines.append(line)
533 return out_lines
534
535 def ReadFuncSizes(self, fname, fd):
536 """Read function sizes from the output of 'nm'
537
538 Args:
539 fd: File containing data to read
540 fname: Filename we are reading from (just for errors)
541
542 Returns:
543 Dictionary containing size of each function in bytes, indexed by
544 function name.
545 """
546 sym = {}
547 for line in fd.readlines():
548 try:
549 size, type, name = line[:-1].split()
550 except:
Simon Glass4653a882014-09-05 19:00:07 -0600551 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000552 continue
553 if type in 'tTdDbB':
554 # function names begin with '.' on 64-bit powerpc
555 if '.' in name[1:]:
556 name = 'static.' + name.split('.')[0]
557 sym[name] = sym.get(name, 0) + int(size, 16)
558 return sym
559
Simon Glass843312d2015-02-05 22:06:15 -0700560 def _ProcessConfig(self, fname):
561 """Read in a .config, autoconf.mk or autoconf.h file
562
563 This function handles all config file types. It ignores comments and
564 any #defines which don't start with CONFIG_.
565
566 Args:
567 fname: Filename to read
568
569 Returns:
570 Dictionary:
571 key: Config name (e.g. CONFIG_DM)
572 value: Config value (e.g. 1)
573 """
574 config = {}
575 if os.path.exists(fname):
576 with open(fname) as fd:
577 for line in fd:
578 line = line.strip()
579 if line.startswith('#define'):
580 values = line[8:].split(' ', 1)
581 if len(values) > 1:
582 key, value = values
583 else:
584 key = values[0]
585 value = ''
586 if not key.startswith('CONFIG_'):
587 continue
588 elif not line or line[0] in ['#', '*', '/']:
589 continue
590 else:
591 key, value = line.split('=', 1)
592 config[key] = value
593 return config
594
595 def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
596 read_config):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000597 """Work out the outcome of a build.
598
599 Args:
600 commit_upto: Commit number to check (0..n-1)
601 target: Target board to check
602 read_func_sizes: True to read function size information
Simon Glass843312d2015-02-05 22:06:15 -0700603 read_config: True to read .config and autoconf.h files
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000604
605 Returns:
606 Outcome object
607 """
608 done_file = self.GetDoneFile(commit_upto, target)
609 sizes_file = self.GetSizesFile(commit_upto, target)
610 sizes = {}
611 func_sizes = {}
Simon Glass843312d2015-02-05 22:06:15 -0700612 config = {}
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000613 if os.path.exists(done_file):
614 with open(done_file, 'r') as fd:
615 return_code = int(fd.readline())
616 err_lines = []
617 err_file = self.GetErrFile(commit_upto, target)
618 if os.path.exists(err_file):
619 with open(err_file, 'r') as fd:
620 err_lines = self.FilterErrors(fd.readlines())
621
622 # Decide whether the build was ok, failed or created warnings
623 if return_code:
624 rc = OUTCOME_ERROR
625 elif len(err_lines):
626 rc = OUTCOME_WARNING
627 else:
628 rc = OUTCOME_OK
629
630 # Convert size information to our simple format
631 if os.path.exists(sizes_file):
632 with open(sizes_file, 'r') as fd:
633 for line in fd.readlines():
634 values = line.split()
635 rodata = 0
636 if len(values) > 6:
637 rodata = int(values[6], 16)
638 size_dict = {
639 'all' : int(values[0]) + int(values[1]) +
640 int(values[2]),
641 'text' : int(values[0]) - rodata,
642 'data' : int(values[1]),
643 'bss' : int(values[2]),
644 'rodata' : rodata,
645 }
646 sizes[values[5]] = size_dict
647
648 if read_func_sizes:
649 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
650 for fname in glob.glob(pattern):
651 with open(fname, 'r') as fd:
652 dict_name = os.path.basename(fname).replace('.sizes',
653 '')
654 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
655
Simon Glass843312d2015-02-05 22:06:15 -0700656 if read_config:
657 output_dir = self.GetBuildDir(commit_upto, target)
658 for name in CONFIG_FILENAMES:
659 fname = os.path.join(output_dir, name)
660 config[name] = self._ProcessConfig(fname)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000661
Simon Glass843312d2015-02-05 22:06:15 -0700662 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000663
Simon Glass843312d2015-02-05 22:06:15 -0700664 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {})
665
666 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
667 read_config):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000668 """Calculate a summary of the results of building a commit.
669
670 Args:
671 board_selected: Dict containing boards to summarise
672 commit_upto: Commit number to summarize (0..self.count-1)
673 read_func_sizes: True to read function size information
Simon Glass843312d2015-02-05 22:06:15 -0700674 read_config: True to read .config and autoconf.h files
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000675
676 Returns:
677 Tuple:
678 Dict containing boards which passed building this commit.
679 keyed by board.target
Simon Glasse30965d2014-08-28 09:43:44 -0600680 List containing a summary of error lines
Simon Glassed966652014-08-28 09:43:43 -0600681 Dict keyed by error line, containing a list of the Board
682 objects with that error
Simon Glasse30965d2014-08-28 09:43:44 -0600683 List containing a summary of warning lines
684 Dict keyed by error line, containing a list of the Board
685 objects with that warning
Simon Glass8270e3c2015-08-25 21:52:14 -0600686 Dictionary keyed by board.target. Each value is a dictionary:
687 key: filename - e.g. '.config'
Simon Glass843312d2015-02-05 22:06:15 -0700688 value is itself a dictionary:
689 key: config name
690 value: config value
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000691 """
Simon Glasse30965d2014-08-28 09:43:44 -0600692 def AddLine(lines_summary, lines_boards, line, board):
693 line = line.rstrip()
694 if line in lines_boards:
695 lines_boards[line].append(board)
696 else:
697 lines_boards[line] = [board]
698 lines_summary.append(line)
699
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000700 board_dict = {}
701 err_lines_summary = []
Simon Glassed966652014-08-28 09:43:43 -0600702 err_lines_boards = {}
Simon Glasse30965d2014-08-28 09:43:44 -0600703 warn_lines_summary = []
704 warn_lines_boards = {}
Simon Glass843312d2015-02-05 22:06:15 -0700705 config = {}
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000706
707 for board in boards_selected.itervalues():
708 outcome = self.GetBuildOutcome(commit_upto, board.target,
Simon Glass843312d2015-02-05 22:06:15 -0700709 read_func_sizes, read_config)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000710 board_dict[board.target] = outcome
Simon Glasse30965d2014-08-28 09:43:44 -0600711 last_func = None
712 last_was_warning = False
713 for line in outcome.err_lines:
714 if line:
715 if (self._re_function.match(line) or
716 self._re_files.match(line)):
717 last_func = line
Simon Glassed966652014-08-28 09:43:43 -0600718 else:
Simon Glasse30965d2014-08-28 09:43:44 -0600719 is_warning = self._re_warning.match(line)
720 is_note = self._re_note.match(line)
721 if is_warning or (last_was_warning and is_note):
722 if last_func:
723 AddLine(warn_lines_summary, warn_lines_boards,
724 last_func, board)
725 AddLine(warn_lines_summary, warn_lines_boards,
726 line, board)
727 else:
728 if last_func:
729 AddLine(err_lines_summary, err_lines_boards,
730 last_func, board)
731 AddLine(err_lines_summary, err_lines_boards,
732 line, board)
733 last_was_warning = is_warning
734 last_func = None
Simon Glass8270e3c2015-08-25 21:52:14 -0600735 tconfig = Config(board.target)
Simon Glass843312d2015-02-05 22:06:15 -0700736 for fname in CONFIG_FILENAMES:
Simon Glass843312d2015-02-05 22:06:15 -0700737 if outcome.config:
738 for key, value in outcome.config[fname].iteritems():
Simon Glass8270e3c2015-08-25 21:52:14 -0600739 tconfig.Add(fname, key, value)
740 config[board.target] = tconfig
Simon Glass843312d2015-02-05 22:06:15 -0700741
Simon Glasse30965d2014-08-28 09:43:44 -0600742 return (board_dict, err_lines_summary, err_lines_boards,
Simon Glass843312d2015-02-05 22:06:15 -0700743 warn_lines_summary, warn_lines_boards, config)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000744
745 def AddOutcome(self, board_dict, arch_list, changes, char, color):
746 """Add an output to our list of outcomes for each architecture
747
748 This simple function adds failing boards (changes) to the
749 relevant architecture string, so we can print the results out
750 sorted by architecture.
751
752 Args:
753 board_dict: Dict containing all boards
754 arch_list: Dict keyed by arch name. Value is a string containing
755 a list of board names which failed for that arch.
756 changes: List of boards to add to arch_list
757 color: terminal.Colour object
758 """
759 done_arch = {}
760 for target in changes:
761 if target in board_dict:
762 arch = board_dict[target].arch
763 else:
764 arch = 'unknown'
765 str = self.col.Color(color, ' ' + target)
766 if not arch in done_arch:
Simon Glass63c619e2015-02-05 22:06:11 -0700767 str = ' %s %s' % (self.col.Color(color, char), str)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000768 done_arch[arch] = True
769 if not arch in arch_list:
770 arch_list[arch] = str
771 else:
772 arch_list[arch] += str
773
774
775 def ColourNum(self, num):
776 color = self.col.RED if num > 0 else self.col.GREEN
777 if num == 0:
778 return '0'
779 return self.col.Color(color, str(num))
780
781 def ResetResultSummary(self, board_selected):
782 """Reset the results summary ready for use.
783
784 Set up the base board list to be all those selected, and set the
785 error lines to empty.
786
787 Following this, calls to PrintResultSummary() will use this
788 information to work out what has changed.
789
790 Args:
791 board_selected: Dict containing boards to summarise, keyed by
792 board.target
793 """
794 self._base_board_dict = {}
795 for board in board_selected:
Simon Glass843312d2015-02-05 22:06:15 -0700796 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {})
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000797 self._base_err_lines = []
Simon Glasse30965d2014-08-28 09:43:44 -0600798 self._base_warn_lines = []
799 self._base_err_line_boards = {}
800 self._base_warn_line_boards = {}
Simon Glass8270e3c2015-08-25 21:52:14 -0600801 self._base_config = None
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000802
803 def PrintFuncSizeDetail(self, fname, old, new):
804 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
805 delta, common = [], {}
806
807 for a in old:
808 if a in new:
809 common[a] = 1
810
811 for name in old:
812 if name not in common:
813 remove += 1
814 down += old[name]
815 delta.append([-old[name], name])
816
817 for name in new:
818 if name not in common:
819 add += 1
820 up += new[name]
821 delta.append([new[name], name])
822
823 for name in common:
824 diff = new.get(name, 0) - old.get(name, 0)
825 if diff > 0:
826 grow, up = grow + 1, up + diff
827 elif diff < 0:
828 shrink, down = shrink + 1, down - diff
829 delta.append([diff, name])
830
831 delta.sort()
832 delta.reverse()
833
834 args = [add, -remove, grow, -shrink, up, -down, up - down]
835 if max(args) == 0:
836 return
837 args = [self.ColourNum(x) for x in args]
838 indent = ' ' * 15
Simon Glass4653a882014-09-05 19:00:07 -0600839 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
840 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
841 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
842 'delta'))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000843 for diff, name in delta:
844 if diff:
845 color = self.col.RED if diff > 0 else self.col.GREEN
846 msg = '%s %-38s %7s %7s %+7d' % (indent, name,
847 old.get(name, '-'), new.get(name,'-'), diff)
Simon Glass4653a882014-09-05 19:00:07 -0600848 Print(msg, colour=color)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000849
850
851 def PrintSizeDetail(self, target_list, show_bloat):
852 """Show details size information for each board
853
854 Args:
855 target_list: List of targets, each a dict containing:
856 'target': Target name
857 'total_diff': Total difference in bytes across all areas
858 <part_name>: Difference for that part
859 show_bloat: Show detail for each function
860 """
861 targets_by_diff = sorted(target_list, reverse=True,
862 key=lambda x: x['_total_diff'])
863 for result in targets_by_diff:
864 printed_target = False
865 for name in sorted(result):
866 diff = result[name]
867 if name.startswith('_'):
868 continue
869 if diff != 0:
870 color = self.col.RED if diff > 0 else self.col.GREEN
871 msg = ' %s %+d' % (name, diff)
872 if not printed_target:
Simon Glass4653a882014-09-05 19:00:07 -0600873 Print('%10s %-15s:' % ('', result['_target']),
874 newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000875 printed_target = True
Simon Glass4653a882014-09-05 19:00:07 -0600876 Print(msg, colour=color, newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000877 if printed_target:
Simon Glass4653a882014-09-05 19:00:07 -0600878 Print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000879 if show_bloat:
880 target = result['_target']
881 outcome = result['_outcome']
882 base_outcome = self._base_board_dict[target]
883 for fname in outcome.func_sizes:
884 self.PrintFuncSizeDetail(fname,
885 base_outcome.func_sizes[fname],
886 outcome.func_sizes[fname])
887
888
889 def PrintSizeSummary(self, board_selected, board_dict, show_detail,
890 show_bloat):
891 """Print a summary of image sizes broken down by section.
892
893 The summary takes the form of one line per architecture. The
894 line contains deltas for each of the sections (+ means the section
895 got bigger, - means smaller). The nunmbers are the average number
896 of bytes that a board in this section increased by.
897
898 For example:
899 powerpc: (622 boards) text -0.0
900 arm: (285 boards) text -0.0
901 nds32: (3 boards) text -8.0
902
903 Args:
904 board_selected: Dict containing boards to summarise, keyed by
905 board.target
906 board_dict: Dict containing boards for which we built this
907 commit, keyed by board.target. The value is an Outcome object.
908 show_detail: Show detail for each board
909 show_bloat: Show detail for each function
910 """
911 arch_list = {}
912 arch_count = {}
913
914 # Calculate changes in size for different image parts
915 # The previous sizes are in Board.sizes, for each board
916 for target in board_dict:
917 if target not in board_selected:
918 continue
919 base_sizes = self._base_board_dict[target].sizes
920 outcome = board_dict[target]
921 sizes = outcome.sizes
922
923 # Loop through the list of images, creating a dict of size
924 # changes for each image/part. We end up with something like
925 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
926 # which means that U-Boot data increased by 5 bytes and SPL
927 # text decreased by 4.
928 err = {'_target' : target}
929 for image in sizes:
930 if image in base_sizes:
931 base_image = base_sizes[image]
932 # Loop through the text, data, bss parts
933 for part in sorted(sizes[image]):
934 diff = sizes[image][part] - base_image[part]
935 col = None
936 if diff:
937 if image == 'u-boot':
938 name = part
939 else:
940 name = image + ':' + part
941 err[name] = diff
942 arch = board_selected[target].arch
943 if not arch in arch_count:
944 arch_count[arch] = 1
945 else:
946 arch_count[arch] += 1
947 if not sizes:
948 pass # Only add to our list when we have some stats
949 elif not arch in arch_list:
950 arch_list[arch] = [err]
951 else:
952 arch_list[arch].append(err)
953
954 # We now have a list of image size changes sorted by arch
955 # Print out a summary of these
956 for arch, target_list in arch_list.iteritems():
957 # Get total difference for each type
958 totals = {}
959 for result in target_list:
960 total = 0
961 for name, diff in result.iteritems():
962 if name.startswith('_'):
963 continue
964 total += diff
965 if name in totals:
966 totals[name] += diff
967 else:
968 totals[name] = diff
969 result['_total_diff'] = total
970 result['_outcome'] = board_dict[result['_target']]
971
972 count = len(target_list)
973 printed_arch = False
974 for name in sorted(totals):
975 diff = totals[name]
976 if diff:
977 # Display the average difference in this name for this
978 # architecture
979 avg_diff = float(diff) / count
980 color = self.col.RED if avg_diff > 0 else self.col.GREEN
981 msg = ' %s %+1.1f' % (name, avg_diff)
982 if not printed_arch:
Simon Glass4653a882014-09-05 19:00:07 -0600983 Print('%10s: (for %d/%d boards)' % (arch, count,
984 arch_count[arch]), newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000985 printed_arch = True
Simon Glass4653a882014-09-05 19:00:07 -0600986 Print(msg, colour=color, newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000987
988 if printed_arch:
Simon Glass4653a882014-09-05 19:00:07 -0600989 Print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000990 if show_detail:
991 self.PrintSizeDetail(target_list, show_bloat)
992
993
994 def PrintResultSummary(self, board_selected, board_dict, err_lines,
Simon Glasse30965d2014-08-28 09:43:44 -0600995 err_line_boards, warn_lines, warn_line_boards,
Simon Glass843312d2015-02-05 22:06:15 -0700996 config, show_sizes, show_detail, show_bloat,
997 show_config):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000998 """Compare results with the base results and display delta.
999
1000 Only boards mentioned in board_selected will be considered. This
1001 function is intended to be called repeatedly with the results of
1002 each commit. It therefore shows a 'diff' between what it saw in
1003 the last call and what it sees now.
1004
1005 Args:
1006 board_selected: Dict containing boards to summarise, keyed by
1007 board.target
1008 board_dict: Dict containing boards for which we built this
1009 commit, keyed by board.target. The value is an Outcome object.
1010 err_lines: A list of errors for this commit, or [] if there is
1011 none, or we don't want to print errors
Simon Glassed966652014-08-28 09:43:43 -06001012 err_line_boards: Dict keyed by error line, containing a list of
1013 the Board objects with that error
Simon Glasse30965d2014-08-28 09:43:44 -06001014 warn_lines: A list of warnings for this commit, or [] if there is
1015 none, or we don't want to print errors
1016 warn_line_boards: Dict keyed by warning line, containing a list of
1017 the Board objects with that warning
Simon Glass843312d2015-02-05 22:06:15 -07001018 config: Dictionary keyed by filename - e.g. '.config'. Each
1019 value is itself a dictionary:
1020 key: config name
1021 value: config value
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001022 show_sizes: Show image size deltas
1023 show_detail: Show detail for each board
1024 show_bloat: Show detail for each function
Simon Glass843312d2015-02-05 22:06:15 -07001025 show_config: Show config changes
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001026 """
Simon Glasse30965d2014-08-28 09:43:44 -06001027 def _BoardList(line, line_boards):
Simon Glassed966652014-08-28 09:43:43 -06001028 """Helper function to get a line of boards containing a line
1029
1030 Args:
1031 line: Error line to search for
1032 Return:
1033 String containing a list of boards with that error line, or
1034 '' if the user has not requested such a list
1035 """
1036 if self._list_error_boards:
1037 names = []
Simon Glasse30965d2014-08-28 09:43:44 -06001038 for board in line_boards[line]:
Simon Glassf66153b2014-10-16 01:05:55 -06001039 if not board.target in names:
1040 names.append(board.target)
Simon Glassed966652014-08-28 09:43:43 -06001041 names_str = '(%s) ' % ','.join(names)
1042 else:
1043 names_str = ''
1044 return names_str
1045
Simon Glasse30965d2014-08-28 09:43:44 -06001046 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1047 char):
1048 better_lines = []
1049 worse_lines = []
1050 for line in lines:
1051 if line not in base_lines:
1052 worse_lines.append(char + '+' +
1053 _BoardList(line, line_boards) + line)
1054 for line in base_lines:
1055 if line not in lines:
1056 better_lines.append(char + '-' +
1057 _BoardList(line, base_line_boards) + line)
1058 return better_lines, worse_lines
1059
Simon Glass843312d2015-02-05 22:06:15 -07001060 def _CalcConfig(delta, name, config):
1061 """Calculate configuration changes
1062
1063 Args:
1064 delta: Type of the delta, e.g. '+'
1065 name: name of the file which changed (e.g. .config)
1066 config: configuration change dictionary
1067 key: config name
1068 value: config value
1069 Returns:
1070 String containing the configuration changes which can be
1071 printed
1072 """
1073 out = ''
1074 for key in sorted(config.keys()):
1075 out += '%s=%s ' % (key, config[key])
Simon Glass8270e3c2015-08-25 21:52:14 -06001076 return '%s %s: %s' % (delta, name, out)
Simon Glass843312d2015-02-05 22:06:15 -07001077
Simon Glass8270e3c2015-08-25 21:52:14 -06001078 def _AddConfig(lines, name, config_plus, config_minus, config_change):
1079 """Add changes in configuration to a list
Simon Glass843312d2015-02-05 22:06:15 -07001080
1081 Args:
Simon Glass8270e3c2015-08-25 21:52:14 -06001082 lines: list to add to
1083 name: config file name
Simon Glass843312d2015-02-05 22:06:15 -07001084 config_plus: configurations added, dictionary
1085 key: config name
1086 value: config value
1087 config_minus: configurations removed, dictionary
1088 key: config name
1089 value: config value
1090 config_change: configurations changed, dictionary
1091 key: config name
1092 value: config value
1093 """
1094 if config_plus:
Simon Glass8270e3c2015-08-25 21:52:14 -06001095 lines.append(_CalcConfig('+', name, config_plus))
Simon Glass843312d2015-02-05 22:06:15 -07001096 if config_minus:
Simon Glass8270e3c2015-08-25 21:52:14 -06001097 lines.append(_CalcConfig('-', name, config_minus))
Simon Glass843312d2015-02-05 22:06:15 -07001098 if config_change:
Simon Glass8270e3c2015-08-25 21:52:14 -06001099 lines.append(_CalcConfig('c', name, config_change))
1100
1101 def _OutputConfigInfo(lines):
1102 for line in lines:
1103 if not line:
1104 continue
1105 if line[0] == '+':
1106 col = self.col.GREEN
1107 elif line[0] == '-':
1108 col = self.col.RED
1109 elif line[0] == 'c':
1110 col = self.col.YELLOW
1111 Print(' ' + line, newline=True, colour=col)
1112
Simon Glass843312d2015-02-05 22:06:15 -07001113
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001114 better = [] # List of boards fixed since last commit
1115 worse = [] # List of new broken boards since last commit
1116 new = [] # List of boards that didn't exist last time
1117 unknown = [] # List of boards that were not built
1118
1119 for target in board_dict:
1120 if target not in board_selected:
1121 continue
1122
1123 # If the board was built last time, add its outcome to a list
1124 if target in self._base_board_dict:
1125 base_outcome = self._base_board_dict[target].rc
1126 outcome = board_dict[target]
1127 if outcome.rc == OUTCOME_UNKNOWN:
1128 unknown.append(target)
1129 elif outcome.rc < base_outcome:
1130 better.append(target)
1131 elif outcome.rc > base_outcome:
1132 worse.append(target)
1133 else:
1134 new.append(target)
1135
1136 # Get a list of errors that have appeared, and disappeared
Simon Glasse30965d2014-08-28 09:43:44 -06001137 better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1138 self._base_err_line_boards, err_lines, err_line_boards, '')
1139 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1140 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001141
1142 # Display results by arch
Simon Glasse30965d2014-08-28 09:43:44 -06001143 if (better or worse or unknown or new or worse_err or better_err
1144 or worse_warn or better_warn):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001145 arch_list = {}
1146 self.AddOutcome(board_selected, arch_list, better, '',
1147 self.col.GREEN)
1148 self.AddOutcome(board_selected, arch_list, worse, '+',
1149 self.col.RED)
1150 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE)
1151 if self._show_unknown:
1152 self.AddOutcome(board_selected, arch_list, unknown, '?',
1153 self.col.MAGENTA)
1154 for arch, target_list in arch_list.iteritems():
Simon Glass4653a882014-09-05 19:00:07 -06001155 Print('%10s: %s' % (arch, target_list))
Simon Glass28370c12014-08-09 15:33:06 -06001156 self._error_lines += 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001157 if better_err:
Simon Glass4653a882014-09-05 19:00:07 -06001158 Print('\n'.join(better_err), colour=self.col.GREEN)
Simon Glass28370c12014-08-09 15:33:06 -06001159 self._error_lines += 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001160 if worse_err:
Simon Glass4653a882014-09-05 19:00:07 -06001161 Print('\n'.join(worse_err), colour=self.col.RED)
Simon Glass28370c12014-08-09 15:33:06 -06001162 self._error_lines += 1
Simon Glasse30965d2014-08-28 09:43:44 -06001163 if better_warn:
Simon Glass4653a882014-09-05 19:00:07 -06001164 Print('\n'.join(better_warn), colour=self.col.CYAN)
Simon Glasse30965d2014-08-28 09:43:44 -06001165 self._error_lines += 1
1166 if worse_warn:
Simon Glass4653a882014-09-05 19:00:07 -06001167 Print('\n'.join(worse_warn), colour=self.col.MAGENTA)
Simon Glasse30965d2014-08-28 09:43:44 -06001168 self._error_lines += 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001169
1170 if show_sizes:
1171 self.PrintSizeSummary(board_selected, board_dict, show_detail,
1172 show_bloat)
1173
Simon Glass8270e3c2015-08-25 21:52:14 -06001174 if show_config and self._base_config:
1175 summary = {}
1176 arch_config_plus = {}
1177 arch_config_minus = {}
1178 arch_config_change = {}
1179 arch_list = []
1180
1181 for target in board_dict:
1182 if target not in board_selected:
Simon Glass843312d2015-02-05 22:06:15 -07001183 continue
Simon Glass8270e3c2015-08-25 21:52:14 -06001184 arch = board_selected[target].arch
1185 if arch not in arch_list:
1186 arch_list.append(arch)
1187
1188 for arch in arch_list:
1189 arch_config_plus[arch] = {}
1190 arch_config_minus[arch] = {}
1191 arch_config_change[arch] = {}
1192 for name in CONFIG_FILENAMES:
1193 arch_config_plus[arch][name] = {}
1194 arch_config_minus[arch][name] = {}
1195 arch_config_change[arch][name] = {}
1196
1197 for target in board_dict:
1198 if target not in board_selected:
1199 continue
1200
1201 arch = board_selected[target].arch
1202
1203 all_config_plus = {}
1204 all_config_minus = {}
1205 all_config_change = {}
1206 tbase = self._base_config[target]
1207 tconfig = config[target]
1208 lines = []
1209 for name in CONFIG_FILENAMES:
1210 if not tconfig.config[name]:
1211 continue
1212 config_plus = {}
1213 config_minus = {}
1214 config_change = {}
1215 base = tbase.config[name]
1216 for key, value in tconfig.config[name].iteritems():
1217 if key not in base:
1218 config_plus[key] = value
1219 all_config_plus[key] = value
1220 for key, value in base.iteritems():
1221 if key not in tconfig.config[name]:
1222 config_minus[key] = value
1223 all_config_minus[key] = value
1224 for key, value in base.iteritems():
1225 new_value = tconfig.config.get(key)
1226 if new_value and value != new_value:
1227 desc = '%s -> %s' % (value, new_value)
1228 config_change[key] = desc
1229 all_config_change[key] = desc
1230
1231 arch_config_plus[arch][name].update(config_plus)
1232 arch_config_minus[arch][name].update(config_minus)
1233 arch_config_change[arch][name].update(config_change)
1234
1235 _AddConfig(lines, name, config_plus, config_minus,
1236 config_change)
1237 _AddConfig(lines, 'all', all_config_plus, all_config_minus,
1238 all_config_change)
1239 summary[target] = '\n'.join(lines)
1240
1241 lines_by_target = {}
1242 for target, lines in summary.iteritems():
1243 if lines in lines_by_target:
1244 lines_by_target[lines].append(target)
1245 else:
1246 lines_by_target[lines] = [target]
1247
1248 for arch in arch_list:
1249 lines = []
1250 all_plus = {}
1251 all_minus = {}
1252 all_change = {}
1253 for name in CONFIG_FILENAMES:
1254 all_plus.update(arch_config_plus[arch][name])
1255 all_minus.update(arch_config_minus[arch][name])
1256 all_change.update(arch_config_change[arch][name])
1257 _AddConfig(lines, name, arch_config_plus[arch][name],
1258 arch_config_minus[arch][name],
1259 arch_config_change[arch][name])
1260 _AddConfig(lines, 'all', all_plus, all_minus, all_change)
1261 #arch_summary[target] = '\n'.join(lines)
1262 if lines:
1263 Print('%s:' % arch)
1264 _OutputConfigInfo(lines)
1265
1266 for lines, targets in lines_by_target.iteritems():
1267 if not lines:
1268 continue
1269 Print('%s :' % ' '.join(sorted(targets)))
1270 _OutputConfigInfo(lines.split('\n'))
1271
Simon Glass843312d2015-02-05 22:06:15 -07001272
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001273 # Save our updated information for the next call to this function
1274 self._base_board_dict = board_dict
1275 self._base_err_lines = err_lines
Simon Glasse30965d2014-08-28 09:43:44 -06001276 self._base_warn_lines = warn_lines
1277 self._base_err_line_boards = err_line_boards
1278 self._base_warn_line_boards = warn_line_boards
Simon Glass843312d2015-02-05 22:06:15 -07001279 self._base_config = config
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001280
1281 # Get a list of boards that did not get built, if needed
1282 not_built = []
1283 for board in board_selected:
1284 if not board in board_dict:
1285 not_built.append(board)
1286 if not_built:
Simon Glass4653a882014-09-05 19:00:07 -06001287 Print("Boards not built (%d): %s" % (len(not_built),
1288 ', '.join(not_built)))
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001289
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001290 def ProduceResultSummary(self, commit_upto, commits, board_selected):
Simon Glasse30965d2014-08-28 09:43:44 -06001291 (board_dict, err_lines, err_line_boards, warn_lines,
Simon Glass843312d2015-02-05 22:06:15 -07001292 warn_line_boards, config) = self.GetResultSummary(
Simon Glassed966652014-08-28 09:43:43 -06001293 board_selected, commit_upto,
Simon Glass843312d2015-02-05 22:06:15 -07001294 read_func_sizes=self._show_bloat,
1295 read_config=self._show_config)
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001296 if commits:
1297 msg = '%02d: %s' % (commit_upto + 1,
1298 commits[commit_upto].subject)
Simon Glass4653a882014-09-05 19:00:07 -06001299 Print(msg, colour=self.col.BLUE)
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001300 self.PrintResultSummary(board_selected, board_dict,
Simon Glassed966652014-08-28 09:43:43 -06001301 err_lines if self._show_errors else [], err_line_boards,
Simon Glasse30965d2014-08-28 09:43:44 -06001302 warn_lines if self._show_errors else [], warn_line_boards,
Simon Glass843312d2015-02-05 22:06:15 -07001303 config, self._show_sizes, self._show_detail,
1304 self._show_bloat, self._show_config)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001305
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001306 def ShowSummary(self, commits, board_selected):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001307 """Show a build summary for U-Boot for a given board list.
1308
1309 Reset the result summary, then repeatedly call GetResultSummary on
1310 each commit's results, then display the differences we see.
1311
1312 Args:
1313 commit: Commit objects to summarise
1314 board_selected: Dict containing boards to summarise
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001315 """
Simon Glassfea58582014-08-09 15:32:59 -06001316 self.commit_count = len(commits) if commits else 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001317 self.commits = commits
1318 self.ResetResultSummary(board_selected)
Simon Glass28370c12014-08-09 15:33:06 -06001319 self._error_lines = 0
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001320
1321 for commit_upto in range(0, self.commit_count, self._step):
Simon Glassb2ea7ab2014-08-09 15:33:02 -06001322 self.ProduceResultSummary(commit_upto, commits, board_selected)
Simon Glass28370c12014-08-09 15:33:06 -06001323 if not self._error_lines:
Simon Glass4653a882014-09-05 19:00:07 -06001324 Print('(no errors to report)', colour=self.col.GREEN)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001325
1326
1327 def SetupBuild(self, board_selected, commits):
1328 """Set up ready to start a build.
1329
1330 Args:
1331 board_selected: Selected boards to build
1332 commits: Selected commits to build
1333 """
1334 # First work out how many commits we will build
Simon Glassfea58582014-08-09 15:32:59 -06001335 count = (self.commit_count + self._step - 1) / self._step
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001336 self.count = len(board_selected) * count
1337 self.upto = self.warned = self.fail = 0
1338 self._timestamps = collections.deque()
1339
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001340 def GetThreadDir(self, thread_num):
1341 """Get the directory path to the working dir for a thread.
1342
1343 Args:
1344 thread_num: Number of thread to check.
1345 """
1346 return os.path.join(self._working_dir, '%02d' % thread_num)
1347
Simon Glassfea58582014-08-09 15:32:59 -06001348 def _PrepareThread(self, thread_num, setup_git):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001349 """Prepare the working directory for a thread.
1350
1351 This clones or fetches the repo into the thread's work directory.
1352
1353 Args:
1354 thread_num: Thread number (0, 1, ...)
Simon Glassfea58582014-08-09 15:32:59 -06001355 setup_git: True to set up a git repo clone
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001356 """
1357 thread_dir = self.GetThreadDir(thread_num)
Simon Glass190064b2014-08-09 15:33:00 -06001358 builderthread.Mkdir(thread_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001359 git_dir = os.path.join(thread_dir, '.git')
1360
1361 # Clone the repo if it doesn't already exist
1362 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1363 # we have a private index but uses the origin repo's contents?
Simon Glassfea58582014-08-09 15:32:59 -06001364 if setup_git and self.git_dir:
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001365 src_dir = os.path.abspath(self.git_dir)
1366 if os.path.exists(git_dir):
1367 gitutil.Fetch(git_dir, thread_dir)
1368 else:
Simon Glass21f0eb32016-09-18 16:48:31 -06001369 Print('\rCloning repo for thread %d' % thread_num,
1370 newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001371 gitutil.Clone(src_dir, thread_dir)
Simon Glass21f0eb32016-09-18 16:48:31 -06001372 Print('\r%s\r' % (' ' * 30), newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001373
Simon Glassfea58582014-08-09 15:32:59 -06001374 def _PrepareWorkingSpace(self, max_threads, setup_git):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001375 """Prepare the working directory for use.
1376
1377 Set up the git repo for each thread.
1378
1379 Args:
1380 max_threads: Maximum number of threads we expect to need.
Simon Glassfea58582014-08-09 15:32:59 -06001381 setup_git: True to set up a git repo clone
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001382 """
Simon Glass190064b2014-08-09 15:33:00 -06001383 builderthread.Mkdir(self._working_dir)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001384 for thread in range(max_threads):
Simon Glassfea58582014-08-09 15:32:59 -06001385 self._PrepareThread(thread, setup_git)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001386
1387 def _PrepareOutputSpace(self):
1388 """Get the output directories ready to receive files.
1389
1390 We delete any output directories which look like ones we need to
1391 create. Having left over directories is confusing when the user wants
1392 to check the output manually.
1393 """
Simon Glass1a915672014-12-01 17:33:53 -07001394 if not self.commits:
1395 return
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001396 dir_list = []
1397 for commit_upto in range(self.commit_count):
1398 dir_list.append(self._GetOutputDir(commit_upto))
1399
Simon Glassb222abe2016-09-18 16:48:32 -06001400 to_remove = []
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001401 for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1402 if dirname not in dir_list:
Simon Glassb222abe2016-09-18 16:48:32 -06001403 to_remove.append(dirname)
1404 if to_remove:
1405 Print('Removing %d old build directories' % len(to_remove),
1406 newline=False)
1407 for dirname in to_remove:
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001408 shutil.rmtree(dirname)
1409
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001410 def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001411 """Build all commits for a list of boards
1412
1413 Args:
1414 commits: List of commits to be build, each a Commit object
1415 boards_selected: Dict of selected boards, key is target name,
1416 value is Board object
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001417 keep_outputs: True to save build output files
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001418 verbose: Display build results as they are completed
Simon Glass2c3deb92014-08-28 09:43:39 -06001419 Returns:
1420 Tuple containing:
1421 - number of boards that failed to build
1422 - number of boards that issued warnings
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001423 """
Simon Glassfea58582014-08-09 15:32:59 -06001424 self.commit_count = len(commits) if commits else 1
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001425 self.commits = commits
Simon Glasse5a0e5d2014-08-09 15:33:03 -06001426 self._verbose = verbose
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001427
1428 self.ResetResultSummary(board_selected)
Thierry Redingf3d015c2014-08-19 10:22:39 +02001429 builderthread.Mkdir(self.base_dir, parents = True)
Simon Glassfea58582014-08-09 15:32:59 -06001430 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1431 commits is not None)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001432 self._PrepareOutputSpace()
Simon Glass745b3952016-09-18 16:48:33 -06001433 Print('\rStarting build...', newline=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001434 self.SetupBuild(board_selected, commits)
1435 self.ProcessResult(None)
1436
1437 # Create jobs to build all commits for each board
1438 for brd in board_selected.itervalues():
Simon Glass190064b2014-08-09 15:33:00 -06001439 job = builderthread.BuilderJob()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001440 job.board = brd
1441 job.commits = commits
1442 job.keep_outputs = keep_outputs
1443 job.step = self._step
1444 self.queue.put(job)
1445
1446 # Wait until all jobs are started
1447 self.queue.join()
1448
1449 # Wait until we have processed all output
1450 self.out_queue.join()
Simon Glass4653a882014-09-05 19:00:07 -06001451 Print()
Simon Glassfc3fe1c2013-04-03 11:07:16 +00001452 self.ClearLine(0)
Simon Glass2c3deb92014-08-28 09:43:39 -06001453 return (self.fail, self.warned)