| # Copyright (c) 2013 The Chromium OS Authors. |
| # |
| # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> |
| # |
| # SPDX-License-Identifier: GPL-2.0+ |
| # |
| |
| import collections |
| from datetime import datetime, timedelta |
| import glob |
| import os |
| import re |
| import Queue |
| import shutil |
| import signal |
| import string |
| import sys |
| import threading |
| import time |
| |
| import builderthread |
| import command |
| import gitutil |
| import terminal |
| from terminal import Print |
| import toolchain |
| |
| |
| """ |
| Theory of Operation |
| |
| Please see README for user documentation, and you should be familiar with |
| that before trying to make sense of this. |
| |
| Buildman works by keeping the machine as busy as possible, building different |
| commits for different boards on multiple CPUs at once. |
| |
| The source repo (self.git_dir) contains all the commits to be built. Each |
| thread works on a single board at a time. It checks out the first commit, |
| configures it for that board, then builds it. Then it checks out the next |
| commit and builds it (typically without re-configuring). When it runs out |
| of commits, it gets another job from the builder and starts again with that |
| board. |
| |
| Clearly the builder threads could work either way - they could check out a |
| commit and then built it for all boards. Using separate directories for each |
| commit/board pair they could leave their build product around afterwards |
| also. |
| |
| The intent behind building a single board for multiple commits, is to make |
| use of incremental builds. Since each commit is built incrementally from |
| the previous one, builds are faster. Reconfiguring for a different board |
| removes all intermediate object files. |
| |
| Many threads can be working at once, but each has its own working directory. |
| When a thread finishes a build, it puts the output files into a result |
| directory. |
| |
| The base directory used by buildman is normally '../<branch>', i.e. |
| a directory higher than the source repository and named after the branch |
| being built. |
| |
| Within the base directory, we have one subdirectory for each commit. Within |
| that is one subdirectory for each board. Within that is the build output for |
| that commit/board combination. |
| |
| Buildman also create working directories for each thread, in a .bm-work/ |
| subdirectory in the base dir. |
| |
| As an example, say we are building branch 'us-net' for boards 'sandbox' and |
| 'seaboard', and say that us-net has two commits. We will have directories |
| like this: |
| |
| us-net/ base directory |
| 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ |
| sandbox/ |
| u-boot.bin |
| seaboard/ |
| u-boot.bin |
| 02_of_02_g4ed4ebc_net--Check-tftp-comp/ |
| sandbox/ |
| u-boot.bin |
| seaboard/ |
| u-boot.bin |
| .bm-work/ |
| 00/ working directory for thread 0 (contains source checkout) |
| build/ build output |
| 01/ working directory for thread 1 |
| build/ build output |
| ... |
| u-boot/ source directory |
| .git/ repository |
| """ |
| |
| # Possible build outcomes |
| OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) |
| |
| # Translate a commit subject into a valid filename (and handle unicode) |
| trans_valid_chars = string.maketrans('/: ', '---') |
| trans_valid_chars = trans_valid_chars.decode('latin-1') |
| |
| BASE_CONFIG_FILENAMES = [ |
| 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg' |
| ] |
| |
| EXTRA_CONFIG_FILENAMES = [ |
| '.config', '.config-spl', '.config-tpl', |
| 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk', |
| 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h', |
| ] |
| |
| class Config: |
| """Holds information about configuration settings for a board.""" |
| def __init__(self, config_filename, target): |
| self.target = target |
| self.config = {} |
| for fname in config_filename: |
| self.config[fname] = {} |
| |
| def Add(self, fname, key, value): |
| self.config[fname][key] = value |
| |
| def __hash__(self): |
| val = 0 |
| for fname in self.config: |
| for key, value in self.config[fname].iteritems(): |
| print key, value |
| val = val ^ hash(key) & hash(value) |
| return val |
| |
| class Builder: |
| """Class for building U-Boot for a particular commit. |
| |
| Public members: (many should ->private) |
| already_done: Number of builds already completed |
| base_dir: Base directory to use for builder |
| checkout: True to check out source, False to skip that step. |
| This is used for testing. |
| col: terminal.Color() object |
| count: Number of commits to build |
| do_make: Method to call to invoke Make |
| fail: Number of builds that failed due to error |
| force_build: Force building even if a build already exists |
| force_config_on_failure: If a commit fails for a board, disable |
| incremental building for the next commit we build for that |
| board, so that we will see all warnings/errors again. |
| force_build_failures: If a previously-built build (i.e. built on |
| a previous run of buildman) is marked as failed, rebuild it. |
| git_dir: Git directory containing source repository |
| last_line_len: Length of the last line we printed (used for erasing |
| it with new progress information) |
| num_jobs: Number of jobs to run at once (passed to make as -j) |
| num_threads: Number of builder threads to run |
| out_queue: Queue of results to process |
| re_make_err: Compiled regular expression for ignore_lines |
| queue: Queue of jobs to run |
| threads: List of active threads |
| toolchains: Toolchains object to use for building |
| upto: Current commit number we are building (0.count-1) |
| warned: Number of builds that produced at least one warning |
| force_reconfig: Reconfigure U-Boot on each comiit. This disables |
| incremental building, where buildman reconfigures on the first |
| commit for a baord, and then just does an incremental build for |
| the following commits. In fact buildman will reconfigure and |
| retry for any failing commits, so generally the only effect of |
| this option is to slow things down. |
| in_tree: Build U-Boot in-tree instead of specifying an output |
| directory separate from the source code. This option is really |
| only useful for testing in-tree builds. |
| |
| Private members: |
| _base_board_dict: Last-summarised Dict of boards |
| _base_err_lines: Last-summarised list of errors |
| _base_warn_lines: Last-summarised list of warnings |
| _build_period_us: Time taken for a single build (float object). |
| _complete_delay: Expected delay until completion (timedelta) |
| _next_delay_update: Next time we plan to display a progress update |
| (datatime) |
| _show_unknown: Show unknown boards (those not built) in summary |
| _timestamps: List of timestamps for the completion of the last |
| last _timestamp_count builds. Each is a datetime object. |
| _timestamp_count: Number of timestamps to keep in our list. |
| _working_dir: Base working directory containing all threads |
| """ |
| class Outcome: |
| """Records a build outcome for a single make invocation |
| |
| Public Members: |
| rc: Outcome value (OUTCOME_...) |
| err_lines: List of error lines or [] if none |
| sizes: Dictionary of image size information, keyed by filename |
| - Each value is itself a dictionary containing |
| values for 'text', 'data' and 'bss', being the integer |
| size in bytes of each section. |
| func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each |
| value is itself a dictionary: |
| key: function name |
| value: Size of function in bytes |
| config: Dictionary keyed by filename - e.g. '.config'. Each |
| value is itself a dictionary: |
| key: config name |
| value: config value |
| """ |
| def __init__(self, rc, err_lines, sizes, func_sizes, config): |
| self.rc = rc |
| self.err_lines = err_lines |
| self.sizes = sizes |
| self.func_sizes = func_sizes |
| self.config = config |
| |
| def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, |
| gnu_make='make', checkout=True, show_unknown=True, step=1, |
| no_subdirs=False, full_path=False, verbose_build=False, |
| incremental=False, per_board_out_dir=False, |
| config_only=False, squash_config_y=False): |
| """Create a new Builder object |
| |
| Args: |
| toolchains: Toolchains object to use for building |
| base_dir: Base directory to use for builder |
| git_dir: Git directory containing source repository |
| num_threads: Number of builder threads to run |
| num_jobs: Number of jobs to run at once (passed to make as -j) |
| gnu_make: the command name of GNU Make. |
| checkout: True to check out source, False to skip that step. |
| This is used for testing. |
| show_unknown: Show unknown boards (those not built) in summary |
| step: 1 to process every commit, n to process every nth commit |
| no_subdirs: Don't create subdirectories when building current |
| source for a single board |
| full_path: Return the full path in CROSS_COMPILE and don't set |
| PATH |
| verbose_build: Run build with V=1 and don't use 'make -s' |
| incremental: Always perform incremental builds; don't run make |
| mrproper when configuring |
| per_board_out_dir: Build in a separate persistent directory per |
| board rather than a thread-specific directory |
| config_only: Only configure each build, don't build it |
| squash_config_y: Convert CONFIG options with the value 'y' to '1' |
| """ |
| self.toolchains = toolchains |
| self.base_dir = base_dir |
| self._working_dir = os.path.join(base_dir, '.bm-work') |
| self.threads = [] |
| self.do_make = self.Make |
| self.gnu_make = gnu_make |
| self.checkout = checkout |
| self.num_threads = num_threads |
| self.num_jobs = num_jobs |
| self.already_done = 0 |
| self.force_build = False |
| self.git_dir = git_dir |
| self._show_unknown = show_unknown |
| self._timestamp_count = 10 |
| self._build_period_us = None |
| self._complete_delay = None |
| self._next_delay_update = datetime.now() |
| self.force_config_on_failure = True |
| self.force_build_failures = False |
| self.force_reconfig = False |
| self._step = step |
| self.in_tree = False |
| self._error_lines = 0 |
| self.no_subdirs = no_subdirs |
| self.full_path = full_path |
| self.verbose_build = verbose_build |
| self.config_only = config_only |
| self.squash_config_y = squash_config_y |
| self.config_filenames = BASE_CONFIG_FILENAMES |
| if not self.squash_config_y: |
| self.config_filenames += EXTRA_CONFIG_FILENAMES |
| |
| self.col = terminal.Color() |
| |
| self._re_function = re.compile('(.*): In function.*') |
| self._re_files = re.compile('In file included from.*') |
| self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') |
| self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') |
| |
| self.queue = Queue.Queue() |
| self.out_queue = Queue.Queue() |
| for i in range(self.num_threads): |
| t = builderthread.BuilderThread(self, i, incremental, |
| per_board_out_dir) |
| t.setDaemon(True) |
| t.start() |
| self.threads.append(t) |
| |
| self.last_line_len = 0 |
| t = builderthread.ResultThread(self) |
| t.setDaemon(True) |
| t.start() |
| self.threads.append(t) |
| |
| ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] |
| self.re_make_err = re.compile('|'.join(ignore_lines)) |
| |
| # Handle existing graceful with SIGINT / Ctrl-C |
| signal.signal(signal.SIGINT, self.signal_handler) |
| |
| def __del__(self): |
| """Get rid of all threads created by the builder""" |
| for t in self.threads: |
| del t |
| |
| def signal_handler(self, signal, frame): |
| sys.exit(1) |
| |
| def SetDisplayOptions(self, show_errors=False, show_sizes=False, |
| show_detail=False, show_bloat=False, |
| list_error_boards=False, show_config=False): |
| """Setup display options for the builder. |
| |
| show_errors: True to show summarised error/warning info |
| show_sizes: Show size deltas |
| show_detail: Show detail for each board |
| show_bloat: Show detail for each function |
| list_error_boards: Show the boards which caused each error/warning |
| show_config: Show config deltas |
| """ |
| self._show_errors = show_errors |
| self._show_sizes = show_sizes |
| self._show_detail = show_detail |
| self._show_bloat = show_bloat |
| self._list_error_boards = list_error_boards |
| self._show_config = show_config |
| |
| def _AddTimestamp(self): |
| """Add a new timestamp to the list and record the build period. |
| |
| The build period is the length of time taken to perform a single |
| build (one board, one commit). |
| """ |
| now = datetime.now() |
| self._timestamps.append(now) |
| count = len(self._timestamps) |
| delta = self._timestamps[-1] - self._timestamps[0] |
| seconds = delta.total_seconds() |
| |
| # If we have enough data, estimate build period (time taken for a |
| # single build) and therefore completion time. |
| if count > 1 and self._next_delay_update < now: |
| self._next_delay_update = now + timedelta(seconds=2) |
| if seconds > 0: |
| self._build_period = float(seconds) / count |
| todo = self.count - self.upto |
| self._complete_delay = timedelta(microseconds= |
| self._build_period * todo * 1000000) |
| # Round it |
| self._complete_delay -= timedelta( |
| microseconds=self._complete_delay.microseconds) |
| |
| if seconds > 60: |
| self._timestamps.popleft() |
| count -= 1 |
| |
| def ClearLine(self, length): |
| """Clear any characters on the current line |
| |
| Make way for a new line of length 'length', by outputting enough |
| spaces to clear out the old line. Then remember the new length for |
| next time. |
| |
| Args: |
| length: Length of new line, in characters |
| """ |
| if length < self.last_line_len: |
| Print(' ' * (self.last_line_len - length), newline=False) |
| Print('\r', newline=False) |
| self.last_line_len = length |
| sys.stdout.flush() |
| |
| def SelectCommit(self, commit, checkout=True): |
| """Checkout the selected commit for this build |
| """ |
| self.commit = commit |
| if checkout and self.checkout: |
| gitutil.Checkout(commit.hash) |
| |
| def Make(self, commit, brd, stage, cwd, *args, **kwargs): |
| """Run make |
| |
| Args: |
| commit: Commit object that is being built |
| brd: Board object that is being built |
| stage: Stage that we are at (mrproper, config, build) |
| cwd: Directory where make should be run |
| args: Arguments to pass to make |
| kwargs: Arguments to pass to command.RunPipe() |
| """ |
| cmd = [self.gnu_make] + list(args) |
| result = command.RunPipe([cmd], capture=True, capture_stderr=True, |
| cwd=cwd, raise_on_error=False, **kwargs) |
| if self.verbose_build: |
| result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout |
| result.combined = '%s\n' % (' '.join(cmd)) + result.combined |
| return result |
| |
| def ProcessResult(self, result): |
| """Process the result of a build, showing progress information |
| |
| Args: |
| result: A CommandResult object, which indicates the result for |
| a single build |
| """ |
| col = terminal.Color() |
| if result: |
| target = result.brd.target |
| |
| self.upto += 1 |
| if result.return_code != 0: |
| self.fail += 1 |
| elif result.stderr: |
| self.warned += 1 |
| if result.already_done: |
| self.already_done += 1 |
| if self._verbose: |
| Print('\r', newline=False) |
| self.ClearLine(0) |
| boards_selected = {target : result.brd} |
| self.ResetResultSummary(boards_selected) |
| self.ProduceResultSummary(result.commit_upto, self.commits, |
| boards_selected) |
| else: |
| target = '(starting)' |
| |
| # Display separate counts for ok, warned and fail |
| ok = self.upto - self.warned - self.fail |
| line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) |
| line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) |
| line += self.col.Color(self.col.RED, '%5d' % self.fail) |
| |
| name = ' /%-5d ' % self.count |
| |
| # Add our current completion time estimate |
| self._AddTimestamp() |
| if self._complete_delay: |
| name += '%s : ' % self._complete_delay |
| # When building all boards for a commit, we can print a commit |
| # progress message. |
| if result and result.commit_upto is None: |
| name += 'commit %2d/%-3d' % (self.commit_upto + 1, |
| self.commit_count) |
| |
| name += target |
| Print(line + name, newline=False) |
| length = 16 + len(name) |
| self.ClearLine(length) |
| |
| def _GetOutputDir(self, commit_upto): |
| """Get the name of the output directory for a commit number |
| |
| The output directory is typically .../<branch>/<commit>. |
| |
| Args: |
| commit_upto: Commit number to use (0..self.count-1) |
| """ |
| commit_dir = None |
| if self.commits: |
| commit = self.commits[commit_upto] |
| subject = commit.subject.translate(trans_valid_chars) |
| commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, |
| self.commit_count, commit.hash, subject[:20])) |
| elif not self.no_subdirs: |
| commit_dir = 'current' |
| if not commit_dir: |
| return self.base_dir |
| return os.path.join(self.base_dir, commit_dir) |
| |
| def GetBuildDir(self, commit_upto, target): |
| """Get the name of the build directory for a commit number |
| |
| The build directory is typically .../<branch>/<commit>/<target>. |
| |
| Args: |
| commit_upto: Commit number to use (0..self.count-1) |
| target: Target name |
| """ |
| output_dir = self._GetOutputDir(commit_upto) |
| return os.path.join(output_dir, target) |
| |
| def GetDoneFile(self, commit_upto, target): |
| """Get the name of the done file for a commit number |
| |
| Args: |
| commit_upto: Commit number to use (0..self.count-1) |
| target: Target name |
| """ |
| return os.path.join(self.GetBuildDir(commit_upto, target), 'done') |
| |
| def GetSizesFile(self, commit_upto, target): |
| """Get the name of the sizes file for a commit number |
| |
| Args: |
| commit_upto: Commit number to use (0..self.count-1) |
| target: Target name |
| """ |
| return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') |
| |
| def GetFuncSizesFile(self, commit_upto, target, elf_fname): |
| """Get the name of the funcsizes file for a commit number and ELF file |
| |
| Args: |
| commit_upto: Commit number to use (0..self.count-1) |
| target: Target name |
| elf_fname: Filename of elf image |
| """ |
| return os.path.join(self.GetBuildDir(commit_upto, target), |
| '%s.sizes' % elf_fname.replace('/', '-')) |
| |
| def GetObjdumpFile(self, commit_upto, target, elf_fname): |
| """Get the name of the objdump file for a commit number and ELF file |
| |
| Args: |
| commit_upto: Commit number to use (0..self.count-1) |
| target: Target name |
| elf_fname: Filename of elf image |
| """ |
| return os.path.join(self.GetBuildDir(commit_upto, target), |
| '%s.objdump' % elf_fname.replace('/', '-')) |
| |
| def GetErrFile(self, commit_upto, target): |
| """Get the name of the err file for a commit number |
| |
| Args: |
| commit_upto: Commit number to use (0..self.count-1) |
| target: Target name |
| """ |
| output_dir = self.GetBuildDir(commit_upto, target) |
| return os.path.join(output_dir, 'err') |
| |
| def FilterErrors(self, lines): |
| """Filter out errors in which we have no interest |
| |
| We should probably use map(). |
| |
| Args: |
| lines: List of error lines, each a string |
| Returns: |
| New list with only interesting lines included |
| """ |
| out_lines = [] |
| for line in lines: |
| if not self.re_make_err.search(line): |
| out_lines.append(line) |
| return out_lines |
| |
| def ReadFuncSizes(self, fname, fd): |
| """Read function sizes from the output of 'nm' |
| |
| Args: |
| fd: File containing data to read |
| fname: Filename we are reading from (just for errors) |
| |
| Returns: |
| Dictionary containing size of each function in bytes, indexed by |
| function name. |
| """ |
| sym = {} |
| for line in fd.readlines(): |
| try: |
| size, type, name = line[:-1].split() |
| except: |
| Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) |
| continue |
| if type in 'tTdDbB': |
| # function names begin with '.' on 64-bit powerpc |
| if '.' in name[1:]: |
| name = 'static.' + name.split('.')[0] |
| sym[name] = sym.get(name, 0) + int(size, 16) |
| return sym |
| |
| def _ProcessConfig(self, fname): |
| """Read in a .config, autoconf.mk or autoconf.h file |
| |
| This function handles all config file types. It ignores comments and |
| any #defines which don't start with CONFIG_. |
| |
| Args: |
| fname: Filename to read |
| |
| Returns: |
| Dictionary: |
| key: Config name (e.g. CONFIG_DM) |
| value: Config value (e.g. 1) |
| """ |
| config = {} |
| if os.path.exists(fname): |
| with open(fname) as fd: |
| for line in fd: |
| line = line.strip() |
| if line.startswith('#define'): |
| values = line[8:].split(' ', 1) |
| if len(values) > 1: |
| key, value = values |
| else: |
| key = values[0] |
| value = '1' if self.squash_config_y else '' |
| if not key.startswith('CONFIG_'): |
| continue |
| elif not line or line[0] in ['#', '*', '/']: |
| continue |
| else: |
| key, value = line.split('=', 1) |
| if self.squash_config_y and value == 'y': |
| value = '1' |
| config[key] = value |
| return config |
| |
| def GetBuildOutcome(self, commit_upto, target, read_func_sizes, |
| read_config): |
| """Work out the outcome of a build. |
| |
| Args: |
| commit_upto: Commit number to check (0..n-1) |
| target: Target board to check |
| read_func_sizes: True to read function size information |
| read_config: True to read .config and autoconf.h files |
| |
| Returns: |
| Outcome object |
| """ |
| done_file = self.GetDoneFile(commit_upto, target) |
| sizes_file = self.GetSizesFile(commit_upto, target) |
| sizes = {} |
| func_sizes = {} |
| config = {} |
| if os.path.exists(done_file): |
| with open(done_file, 'r') as fd: |
| return_code = int(fd.readline()) |
| err_lines = [] |
| err_file = self.GetErrFile(commit_upto, target) |
| if os.path.exists(err_file): |
| with open(err_file, 'r') as fd: |
| err_lines = self.FilterErrors(fd.readlines()) |
| |
| # Decide whether the build was ok, failed or created warnings |
| if return_code: |
| rc = OUTCOME_ERROR |
| elif len(err_lines): |
| rc = OUTCOME_WARNING |
| else: |
| rc = OUTCOME_OK |
| |
| # Convert size information to our simple format |
| if os.path.exists(sizes_file): |
| with open(sizes_file, 'r') as fd: |
| for line in fd.readlines(): |
| values = line.split() |
| rodata = 0 |
| if len(values) > 6: |
| rodata = int(values[6], 16) |
| size_dict = { |
| 'all' : int(values[0]) + int(values[1]) + |
| int(values[2]), |
| 'text' : int(values[0]) - rodata, |
| 'data' : int(values[1]), |
| 'bss' : int(values[2]), |
| 'rodata' : rodata, |
| } |
| sizes[values[5]] = size_dict |
| |
| if read_func_sizes: |
| pattern = self.GetFuncSizesFile(commit_upto, target, '*') |
| for fname in glob.glob(pattern): |
| with open(fname, 'r') as fd: |
| dict_name = os.path.basename(fname).replace('.sizes', |
| '') |
| func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) |
| |
| if read_config: |
| output_dir = self.GetBuildDir(commit_upto, target) |
| for name in self.config_filenames: |
| fname = os.path.join(output_dir, name) |
| config[name] = self._ProcessConfig(fname) |
| |
| return Builder.Outcome(rc, err_lines, sizes, func_sizes, config) |
| |
| return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}) |
| |
| def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes, |
| read_config): |
| """Calculate a summary of the results of building a commit. |
| |
| Args: |
| board_selected: Dict containing boards to summarise |
| commit_upto: Commit number to summarize (0..self.count-1) |
| read_func_sizes: True to read function size information |
| read_config: True to read .config and autoconf.h files |
| |
| Returns: |
| Tuple: |
| Dict containing boards which passed building this commit. |
| keyed by board.target |
| List containing a summary of error lines |
| Dict keyed by error line, containing a list of the Board |
| objects with that error |
| List containing a summary of warning lines |
| Dict keyed by error line, containing a list of the Board |
| objects with that warning |
| Dictionary keyed by board.target. Each value is a dictionary: |
| key: filename - e.g. '.config' |
| value is itself a dictionary: |
| key: config name |
| value: config value |
| """ |
| def AddLine(lines_summary, lines_boards, line, board): |
| line = line.rstrip() |
| if line in lines_boards: |
| lines_boards[line].append(board) |
| else: |
| lines_boards[line] = [board] |
| lines_summary.append(line) |
| |
| board_dict = {} |
| err_lines_summary = [] |
| err_lines_boards = {} |
| warn_lines_summary = [] |
| warn_lines_boards = {} |
| config = {} |
| |
| for board in boards_selected.itervalues(): |
| outcome = self.GetBuildOutcome(commit_upto, board.target, |
| read_func_sizes, read_config) |
| board_dict[board.target] = outcome |
| last_func = None |
| last_was_warning = False |
| for line in outcome.err_lines: |
| if line: |
| if (self._re_function.match(line) or |
| self._re_files.match(line)): |
| last_func = line |
| else: |
| is_warning = self._re_warning.match(line) |
| is_note = self._re_note.match(line) |
| if is_warning or (last_was_warning and is_note): |
| if last_func: |
| AddLine(warn_lines_summary, warn_lines_boards, |
| last_func, board) |
| AddLine(warn_lines_summary, warn_lines_boards, |
| line, board) |
| else: |
| if last_func: |
| AddLine(err_lines_summary, err_lines_boards, |
| last_func, board) |
| AddLine(err_lines_summary, err_lines_boards, |
| line, board) |
| last_was_warning = is_warning |
| last_func = None |
| tconfig = Config(self.config_filenames, board.target) |
| for fname in self.config_filenames: |
| if outcome.config: |
| for key, value in outcome.config[fname].iteritems(): |
| tconfig.Add(fname, key, value) |
| config[board.target] = tconfig |
| |
| return (board_dict, err_lines_summary, err_lines_boards, |
| warn_lines_summary, warn_lines_boards, config) |
| |
| def AddOutcome(self, board_dict, arch_list, changes, char, color): |
| """Add an output to our list of outcomes for each architecture |
| |
| This simple function adds failing boards (changes) to the |
| relevant architecture string, so we can print the results out |
| sorted by architecture. |
| |
| Args: |
| board_dict: Dict containing all boards |
| arch_list: Dict keyed by arch name. Value is a string containing |
| a list of board names which failed for that arch. |
| changes: List of boards to add to arch_list |
| color: terminal.Colour object |
| """ |
| done_arch = {} |
| for target in changes: |
| if target in board_dict: |
| arch = board_dict[target].arch |
| else: |
| arch = 'unknown' |
| str = self.col.Color(color, ' ' + target) |
| if not arch in done_arch: |
| str = ' %s %s' % (self.col.Color(color, char), str) |
| done_arch[arch] = True |
| if not arch in arch_list: |
| arch_list[arch] = str |
| else: |
| arch_list[arch] += str |
| |
| |
| def ColourNum(self, num): |
| color = self.col.RED if num > 0 else self.col.GREEN |
| if num == 0: |
| return '0' |
| return self.col.Color(color, str(num)) |
| |
| def ResetResultSummary(self, board_selected): |
| """Reset the results summary ready for use. |
| |
| Set up the base board list to be all those selected, and set the |
| error lines to empty. |
| |
| Following this, calls to PrintResultSummary() will use this |
| information to work out what has changed. |
| |
| Args: |
| board_selected: Dict containing boards to summarise, keyed by |
| board.target |
| """ |
| self._base_board_dict = {} |
| for board in board_selected: |
| self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {}) |
| self._base_err_lines = [] |
| self._base_warn_lines = [] |
| self._base_err_line_boards = {} |
| self._base_warn_line_boards = {} |
| self._base_config = None |
| |
| def PrintFuncSizeDetail(self, fname, old, new): |
| grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 |
| delta, common = [], {} |
| |
| for a in old: |
| if a in new: |
| common[a] = 1 |
| |
| for name in old: |
| if name not in common: |
| remove += 1 |
| down += old[name] |
| delta.append([-old[name], name]) |
| |
| for name in new: |
| if name not in common: |
| add += 1 |
| up += new[name] |
| delta.append([new[name], name]) |
| |
| for name in common: |
| diff = new.get(name, 0) - old.get(name, 0) |
| if diff > 0: |
| grow, up = grow + 1, up + diff |
| elif diff < 0: |
| shrink, down = shrink + 1, down - diff |
| delta.append([diff, name]) |
| |
| delta.sort() |
| delta.reverse() |
| |
| args = [add, -remove, grow, -shrink, up, -down, up - down] |
| if max(args) == 0: |
| return |
| args = [self.ColourNum(x) for x in args] |
| indent = ' ' * 15 |
| Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % |
| tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) |
| Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', |
| 'delta')) |
| for diff, name in delta: |
| if diff: |
| color = self.col.RED if diff > 0 else self.col.GREEN |
| msg = '%s %-38s %7s %7s %+7d' % (indent, name, |
| old.get(name, '-'), new.get(name,'-'), diff) |
| Print(msg, colour=color) |
| |
| |
| def PrintSizeDetail(self, target_list, show_bloat): |
| """Show details size information for each board |
| |
| Args: |
| target_list: List of targets, each a dict containing: |
| 'target': Target name |
| 'total_diff': Total difference in bytes across all areas |
| <part_name>: Difference for that part |
| show_bloat: Show detail for each function |
| """ |
| targets_by_diff = sorted(target_list, reverse=True, |
| key=lambda x: x['_total_diff']) |
| for result in targets_by_diff: |
| printed_target = False |
| for name in sorted(result): |
| diff = result[name] |
| if name.startswith('_'): |
| continue |
| if diff != 0: |
| color = self.col.RED if diff > 0 else self.col.GREEN |
| msg = ' %s %+d' % (name, diff) |
| if not printed_target: |
| Print('%10s %-15s:' % ('', result['_target']), |
| newline=False) |
| printed_target = True |
| Print(msg, colour=color, newline=False) |
| if printed_target: |
| Print() |
| if show_bloat: |
| target = result['_target'] |
| outcome = result['_outcome'] |
| base_outcome = self._base_board_dict[target] |
| for fname in outcome.func_sizes: |
| self.PrintFuncSizeDetail(fname, |
| base_outcome.func_sizes[fname], |
| outcome.func_sizes[fname]) |
| |
| |
| def PrintSizeSummary(self, board_selected, board_dict, show_detail, |
| show_bloat): |
| """Print a summary of image sizes broken down by section. |
| |
| The summary takes the form of one line per architecture. The |
| line contains deltas for each of the sections (+ means the section |
| got bigger, - means smaller). The nunmbers are the average number |
| of bytes that a board in this section increased by. |
| |
| For example: |
| powerpc: (622 boards) text -0.0 |
| arm: (285 boards) text -0.0 |
| nds32: (3 boards) text -8.0 |
| |
| Args: |
| board_selected: Dict containing boards to summarise, keyed by |
| board.target |
| board_dict: Dict containing boards for which we built this |
| commit, keyed by board.target. The value is an Outcome object. |
| show_detail: Show detail for each board |
| show_bloat: Show detail for each function |
| """ |
| arch_list = {} |
| arch_count = {} |
| |
| # Calculate changes in size for different image parts |
| # The previous sizes are in Board.sizes, for each board |
| for target in board_dict: |
| if target not in board_selected: |
| continue |
| base_sizes = self._base_board_dict[target].sizes |
| outcome = board_dict[target] |
| sizes = outcome.sizes |
| |
| # Loop through the list of images, creating a dict of size |
| # changes for each image/part. We end up with something like |
| # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} |
| # which means that U-Boot data increased by 5 bytes and SPL |
| # text decreased by 4. |
| err = {'_target' : target} |
| for image in sizes: |
| if image in base_sizes: |
| base_image = base_sizes[image] |
| # Loop through the text, data, bss parts |
| for part in sorted(sizes[image]): |
| diff = sizes[image][part] - base_image[part] |
| col = None |
| if diff: |
| if image == 'u-boot': |
| name = part |
| else: |
| name = image + ':' + part |
| err[name] = diff |
| arch = board_selected[target].arch |
| if not arch in arch_count: |
| arch_count[arch] = 1 |
| else: |
| arch_count[arch] += 1 |
| if not sizes: |
| pass # Only add to our list when we have some stats |
| elif not arch in arch_list: |
| arch_list[arch] = [err] |
| else: |
| arch_list[arch].append(err) |
| |
| # We now have a list of image size changes sorted by arch |
| # Print out a summary of these |
| for arch, target_list in arch_list.iteritems(): |
| # Get total difference for each type |
| totals = {} |
| for result in target_list: |
| total = 0 |
| for name, diff in result.iteritems(): |
| if name.startswith('_'): |
| continue |
| total += diff |
| if name in totals: |
| totals[name] += diff |
| else: |
| totals[name] = diff |
| result['_total_diff'] = total |
| result['_outcome'] = board_dict[result['_target']] |
| |
| count = len(target_list) |
| printed_arch = False |
| for name in sorted(totals): |
| diff = totals[name] |
| if diff: |
| # Display the average difference in this name for this |
| # architecture |
| avg_diff = float(diff) / count |
| color = self.col.RED if avg_diff > 0 else self.col.GREEN |
| msg = ' %s %+1.1f' % (name, avg_diff) |
| if not printed_arch: |
| Print('%10s: (for %d/%d boards)' % (arch, count, |
| arch_count[arch]), newline=False) |
| printed_arch = True |
| Print(msg, colour=color, newline=False) |
| |
| if printed_arch: |
| Print() |
| if show_detail: |
| self.PrintSizeDetail(target_list, show_bloat) |
| |
| |
| def PrintResultSummary(self, board_selected, board_dict, err_lines, |
| err_line_boards, warn_lines, warn_line_boards, |
| config, show_sizes, show_detail, show_bloat, |
| show_config): |
| """Compare results with the base results and display delta. |
| |
| Only boards mentioned in board_selected will be considered. This |
| function is intended to be called repeatedly with the results of |
| each commit. It therefore shows a 'diff' between what it saw in |
| the last call and what it sees now. |
| |
| Args: |
| board_selected: Dict containing boards to summarise, keyed by |
| board.target |
| board_dict: Dict containing boards for which we built this |
| commit, keyed by board.target. The value is an Outcome object. |
| err_lines: A list of errors for this commit, or [] if there is |
| none, or we don't want to print errors |
| err_line_boards: Dict keyed by error line, containing a list of |
| the Board objects with that error |
| warn_lines: A list of warnings for this commit, or [] if there is |
| none, or we don't want to print errors |
| warn_line_boards: Dict keyed by warning line, containing a list of |
| the Board objects with that warning |
| config: Dictionary keyed by filename - e.g. '.config'. Each |
| value is itself a dictionary: |
| key: config name |
| value: config value |
| show_sizes: Show image size deltas |
| show_detail: Show detail for each board |
| show_bloat: Show detail for each function |
| show_config: Show config changes |
| """ |
| def _BoardList(line, line_boards): |
| """Helper function to get a line of boards containing a line |
| |
| Args: |
| line: Error line to search for |
| Return: |
| String containing a list of boards with that error line, or |
| '' if the user has not requested such a list |
| """ |
| if self._list_error_boards: |
| names = [] |
| for board in line_boards[line]: |
| if not board.target in names: |
| names.append(board.target) |
| names_str = '(%s) ' % ','.join(names) |
| else: |
| names_str = '' |
| return names_str |
| |
| def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, |
| char): |
| better_lines = [] |
| worse_lines = [] |
| for line in lines: |
| if line not in base_lines: |
| worse_lines.append(char + '+' + |
| _BoardList(line, line_boards) + line) |
| for line in base_lines: |
| if line not in lines: |
| better_lines.append(char + '-' + |
| _BoardList(line, base_line_boards) + line) |
| return better_lines, worse_lines |
| |
| def _CalcConfig(delta, name, config): |
| """Calculate configuration changes |
| |
| Args: |
| delta: Type of the delta, e.g. '+' |
| name: name of the file which changed (e.g. .config) |
| config: configuration change dictionary |
| key: config name |
| value: config value |
| Returns: |
| String containing the configuration changes which can be |
| printed |
| """ |
| out = '' |
| for key in sorted(config.keys()): |
| out += '%s=%s ' % (key, config[key]) |
| return '%s %s: %s' % (delta, name, out) |
| |
| def _AddConfig(lines, name, config_plus, config_minus, config_change): |
| """Add changes in configuration to a list |
| |
| Args: |
| lines: list to add to |
| name: config file name |
| config_plus: configurations added, dictionary |
| key: config name |
| value: config value |
| config_minus: configurations removed, dictionary |
| key: config name |
| value: config value |
| config_change: configurations changed, dictionary |
| key: config name |
| value: config value |
| """ |
| if config_plus: |
| lines.append(_CalcConfig('+', name, config_plus)) |
| if config_minus: |
| lines.append(_CalcConfig('-', name, config_minus)) |
| if config_change: |
| lines.append(_CalcConfig('c', name, config_change)) |
| |
| def _OutputConfigInfo(lines): |
| for line in lines: |
| if not line: |
| continue |
| if line[0] == '+': |
| col = self.col.GREEN |
| elif line[0] == '-': |
| col = self.col.RED |
| elif line[0] == 'c': |
| col = self.col.YELLOW |
| Print(' ' + line, newline=True, colour=col) |
| |
| |
| better = [] # List of boards fixed since last commit |
| worse = [] # List of new broken boards since last commit |
| new = [] # List of boards that didn't exist last time |
| unknown = [] # List of boards that were not built |
| |
| for target in board_dict: |
| if target not in board_selected: |
| continue |
| |
| # If the board was built last time, add its outcome to a list |
| if target in self._base_board_dict: |
| base_outcome = self._base_board_dict[target].rc |
| outcome = board_dict[target] |
| if outcome.rc == OUTCOME_UNKNOWN: |
| unknown.append(target) |
| elif outcome.rc < base_outcome: |
| better.append(target) |
| elif outcome.rc > base_outcome: |
| worse.append(target) |
| else: |
| new.append(target) |
| |
| # Get a list of errors that have appeared, and disappeared |
| better_err, worse_err = _CalcErrorDelta(self._base_err_lines, |
| self._base_err_line_boards, err_lines, err_line_boards, '') |
| better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, |
| self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') |
| |
| # Display results by arch |
| if (better or worse or unknown or new or worse_err or better_err |
| or worse_warn or better_warn): |
| arch_list = {} |
| self.AddOutcome(board_selected, arch_list, better, '', |
| self.col.GREEN) |
| self.AddOutcome(board_selected, arch_list, worse, '+', |
| self.col.RED) |
| self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) |
| if self._show_unknown: |
| self.AddOutcome(board_selected, arch_list, unknown, '?', |
| self.col.MAGENTA) |
| for arch, target_list in arch_list.iteritems(): |
| Print('%10s: %s' % (arch, target_list)) |
| self._error_lines += 1 |
| if better_err: |
| Print('\n'.join(better_err), colour=self.col.GREEN) |
| self._error_lines += 1 |
| if worse_err: |
| Print('\n'.join(worse_err), colour=self.col.RED) |
| self._error_lines += 1 |
| if better_warn: |
| Print('\n'.join(better_warn), colour=self.col.CYAN) |
| self._error_lines += 1 |
| if worse_warn: |
| Print('\n'.join(worse_warn), colour=self.col.MAGENTA) |
| self._error_lines += 1 |
| |
| if show_sizes: |
| self.PrintSizeSummary(board_selected, board_dict, show_detail, |
| show_bloat) |
| |
| if show_config and self._base_config: |
| summary = {} |
| arch_config_plus = {} |
| arch_config_minus = {} |
| arch_config_change = {} |
| arch_list = [] |
| |
| for target in board_dict: |
| if target not in board_selected: |
| continue |
| arch = board_selected[target].arch |
| if arch not in arch_list: |
| arch_list.append(arch) |
| |
| for arch in arch_list: |
| arch_config_plus[arch] = {} |
| arch_config_minus[arch] = {} |
| arch_config_change[arch] = {} |
| for name in self.config_filenames: |
| arch_config_plus[arch][name] = {} |
| arch_config_minus[arch][name] = {} |
| arch_config_change[arch][name] = {} |
| |
| for target in board_dict: |
| if target not in board_selected: |
| continue |
| |
| arch = board_selected[target].arch |
| |
| all_config_plus = {} |
| all_config_minus = {} |
| all_config_change = {} |
| tbase = self._base_config[target] |
| tconfig = config[target] |
| lines = [] |
| for name in self.config_filenames: |
| if not tconfig.config[name]: |
| continue |
| config_plus = {} |
| config_minus = {} |
| config_change = {} |
| base = tbase.config[name] |
| for key, value in tconfig.config[name].iteritems(): |
| if key not in base: |
| config_plus[key] = value |
| all_config_plus[key] = value |
| for key, value in base.iteritems(): |
| if key not in tconfig.config[name]: |
| config_minus[key] = value |
| all_config_minus[key] = value |
| for key, value in base.iteritems(): |
| new_value = tconfig.config.get(key) |
| if new_value and value != new_value: |
| desc = '%s -> %s' % (value, new_value) |
| config_change[key] = desc |
| all_config_change[key] = desc |
| |
| arch_config_plus[arch][name].update(config_plus) |
| arch_config_minus[arch][name].update(config_minus) |
| arch_config_change[arch][name].update(config_change) |
| |
| _AddConfig(lines, name, config_plus, config_minus, |
| config_change) |
| _AddConfig(lines, 'all', all_config_plus, all_config_minus, |
| all_config_change) |
| summary[target] = '\n'.join(lines) |
| |
| lines_by_target = {} |
| for target, lines in summary.iteritems(): |
| if lines in lines_by_target: |
| lines_by_target[lines].append(target) |
| else: |
| lines_by_target[lines] = [target] |
| |
| for arch in arch_list: |
| lines = [] |
| all_plus = {} |
| all_minus = {} |
| all_change = {} |
| for name in self.config_filenames: |
| all_plus.update(arch_config_plus[arch][name]) |
| all_minus.update(arch_config_minus[arch][name]) |
| all_change.update(arch_config_change[arch][name]) |
| _AddConfig(lines, name, arch_config_plus[arch][name], |
| arch_config_minus[arch][name], |
| arch_config_change[arch][name]) |
| _AddConfig(lines, 'all', all_plus, all_minus, all_change) |
| #arch_summary[target] = '\n'.join(lines) |
| if lines: |
| Print('%s:' % arch) |
| _OutputConfigInfo(lines) |
| |
| for lines, targets in lines_by_target.iteritems(): |
| if not lines: |
| continue |
| Print('%s :' % ' '.join(sorted(targets))) |
| _OutputConfigInfo(lines.split('\n')) |
| |
| |
| # Save our updated information for the next call to this function |
| self._base_board_dict = board_dict |
| self._base_err_lines = err_lines |
| self._base_warn_lines = warn_lines |
| self._base_err_line_boards = err_line_boards |
| self._base_warn_line_boards = warn_line_boards |
| self._base_config = config |
| |
| # Get a list of boards that did not get built, if needed |
| not_built = [] |
| for board in board_selected: |
| if not board in board_dict: |
| not_built.append(board) |
| if not_built: |
| Print("Boards not built (%d): %s" % (len(not_built), |
| ', '.join(not_built))) |
| |
| def ProduceResultSummary(self, commit_upto, commits, board_selected): |
| (board_dict, err_lines, err_line_boards, warn_lines, |
| warn_line_boards, config) = self.GetResultSummary( |
| board_selected, commit_upto, |
| read_func_sizes=self._show_bloat, |
| read_config=self._show_config) |
| if commits: |
| msg = '%02d: %s' % (commit_upto + 1, |
| commits[commit_upto].subject) |
| Print(msg, colour=self.col.BLUE) |
| self.PrintResultSummary(board_selected, board_dict, |
| err_lines if self._show_errors else [], err_line_boards, |
| warn_lines if self._show_errors else [], warn_line_boards, |
| config, self._show_sizes, self._show_detail, |
| self._show_bloat, self._show_config) |
| |
| def ShowSummary(self, commits, board_selected): |
| """Show a build summary for U-Boot for a given board list. |
| |
| Reset the result summary, then repeatedly call GetResultSummary on |
| each commit's results, then display the differences we see. |
| |
| Args: |
| commit: Commit objects to summarise |
| board_selected: Dict containing boards to summarise |
| """ |
| self.commit_count = len(commits) if commits else 1 |
| self.commits = commits |
| self.ResetResultSummary(board_selected) |
| self._error_lines = 0 |
| |
| for commit_upto in range(0, self.commit_count, self._step): |
| self.ProduceResultSummary(commit_upto, commits, board_selected) |
| if not self._error_lines: |
| Print('(no errors to report)', colour=self.col.GREEN) |
| |
| |
| def SetupBuild(self, board_selected, commits): |
| """Set up ready to start a build. |
| |
| Args: |
| board_selected: Selected boards to build |
| commits: Selected commits to build |
| """ |
| # First work out how many commits we will build |
| count = (self.commit_count + self._step - 1) / self._step |
| self.count = len(board_selected) * count |
| self.upto = self.warned = self.fail = 0 |
| self._timestamps = collections.deque() |
| |
| def GetThreadDir(self, thread_num): |
| """Get the directory path to the working dir for a thread. |
| |
| Args: |
| thread_num: Number of thread to check. |
| """ |
| return os.path.join(self._working_dir, '%02d' % thread_num) |
| |
| def _PrepareThread(self, thread_num, setup_git): |
| """Prepare the working directory for a thread. |
| |
| This clones or fetches the repo into the thread's work directory. |
| |
| Args: |
| thread_num: Thread number (0, 1, ...) |
| setup_git: True to set up a git repo clone |
| """ |
| thread_dir = self.GetThreadDir(thread_num) |
| builderthread.Mkdir(thread_dir) |
| git_dir = os.path.join(thread_dir, '.git') |
| |
| # Clone the repo if it doesn't already exist |
| # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so |
| # we have a private index but uses the origin repo's contents? |
| if setup_git and self.git_dir: |
| src_dir = os.path.abspath(self.git_dir) |
| if os.path.exists(git_dir): |
| gitutil.Fetch(git_dir, thread_dir) |
| else: |
| Print('\rCloning repo for thread %d' % thread_num, |
| newline=False) |
| gitutil.Clone(src_dir, thread_dir) |
| Print('\r%s\r' % (' ' * 30), newline=False) |
| |
| def _PrepareWorkingSpace(self, max_threads, setup_git): |
| """Prepare the working directory for use. |
| |
| Set up the git repo for each thread. |
| |
| Args: |
| max_threads: Maximum number of threads we expect to need. |
| setup_git: True to set up a git repo clone |
| """ |
| builderthread.Mkdir(self._working_dir) |
| for thread in range(max_threads): |
| self._PrepareThread(thread, setup_git) |
| |
| def _PrepareOutputSpace(self): |
| """Get the output directories ready to receive files. |
| |
| We delete any output directories which look like ones we need to |
| create. Having left over directories is confusing when the user wants |
| to check the output manually. |
| """ |
| if not self.commits: |
| return |
| dir_list = [] |
| for commit_upto in range(self.commit_count): |
| dir_list.append(self._GetOutputDir(commit_upto)) |
| |
| to_remove = [] |
| for dirname in glob.glob(os.path.join(self.base_dir, '*')): |
| if dirname not in dir_list: |
| to_remove.append(dirname) |
| if to_remove: |
| Print('Removing %d old build directories' % len(to_remove), |
| newline=False) |
| for dirname in to_remove: |
| shutil.rmtree(dirname) |
| |
| def BuildBoards(self, commits, board_selected, keep_outputs, verbose): |
| """Build all commits for a list of boards |
| |
| Args: |
| commits: List of commits to be build, each a Commit object |
| boards_selected: Dict of selected boards, key is target name, |
| value is Board object |
| keep_outputs: True to save build output files |
| verbose: Display build results as they are completed |
| Returns: |
| Tuple containing: |
| - number of boards that failed to build |
| - number of boards that issued warnings |
| """ |
| self.commit_count = len(commits) if commits else 1 |
| self.commits = commits |
| self._verbose = verbose |
| |
| self.ResetResultSummary(board_selected) |
| builderthread.Mkdir(self.base_dir, parents = True) |
| self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), |
| commits is not None) |
| self._PrepareOutputSpace() |
| Print('\rStarting build...', newline=False) |
| self.SetupBuild(board_selected, commits) |
| self.ProcessResult(None) |
| |
| # Create jobs to build all commits for each board |
| for brd in board_selected.itervalues(): |
| job = builderthread.BuilderJob() |
| job.board = brd |
| job.commits = commits |
| job.keep_outputs = keep_outputs |
| job.step = self._step |
| self.queue.put(job) |
| |
| term = threading.Thread(target=self.queue.join) |
| term.setDaemon(True) |
| term.start() |
| while term.isAlive(): |
| term.join(100) |
| |
| # Wait until we have processed all output |
| self.out_queue.join() |
| Print() |
| self.ClearLine(0) |
| return (self.fail, self.warned) |