Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 1 | # Copyright (c) 2013 The Chromium OS Authors. |
| 2 | # |
| 3 | # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com> |
| 4 | # |
Wolfgang Denk | 1a45966 | 2013-07-08 09:37:19 +0200 | [diff] [blame] | 5 | # SPDX-License-Identifier: GPL-2.0+ |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 6 | # |
| 7 | |
| 8 | import collections |
| 9 | import errno |
| 10 | from datetime import datetime, timedelta |
| 11 | import glob |
| 12 | import os |
| 13 | import re |
| 14 | import Queue |
| 15 | import shutil |
| 16 | import string |
| 17 | import sys |
| 18 | import threading |
| 19 | import time |
| 20 | |
| 21 | import command |
| 22 | import gitutil |
| 23 | import terminal |
| 24 | import toolchain |
| 25 | |
| 26 | |
| 27 | """ |
| 28 | Theory of Operation |
| 29 | |
| 30 | Please see README for user documentation, and you should be familiar with |
| 31 | that before trying to make sense of this. |
| 32 | |
| 33 | Buildman works by keeping the machine as busy as possible, building different |
| 34 | commits for different boards on multiple CPUs at once. |
| 35 | |
| 36 | The source repo (self.git_dir) contains all the commits to be built. Each |
| 37 | thread works on a single board at a time. It checks out the first commit, |
| 38 | configures it for that board, then builds it. Then it checks out the next |
| 39 | commit and builds it (typically without re-configuring). When it runs out |
| 40 | of commits, it gets another job from the builder and starts again with that |
| 41 | board. |
| 42 | |
| 43 | Clearly the builder threads could work either way - they could check out a |
| 44 | commit and then built it for all boards. Using separate directories for each |
| 45 | commit/board pair they could leave their build product around afterwards |
| 46 | also. |
| 47 | |
| 48 | The intent behind building a single board for multiple commits, is to make |
| 49 | use of incremental builds. Since each commit is built incrementally from |
| 50 | the previous one, builds are faster. Reconfiguring for a different board |
| 51 | removes all intermediate object files. |
| 52 | |
| 53 | Many threads can be working at once, but each has its own working directory. |
| 54 | When a thread finishes a build, it puts the output files into a result |
| 55 | directory. |
| 56 | |
| 57 | The base directory used by buildman is normally '../<branch>', i.e. |
| 58 | a directory higher than the source repository and named after the branch |
| 59 | being built. |
| 60 | |
| 61 | Within the base directory, we have one subdirectory for each commit. Within |
| 62 | that is one subdirectory for each board. Within that is the build output for |
| 63 | that commit/board combination. |
| 64 | |
| 65 | Buildman also create working directories for each thread, in a .bm-work/ |
| 66 | subdirectory in the base dir. |
| 67 | |
| 68 | As an example, say we are building branch 'us-net' for boards 'sandbox' and |
| 69 | 'seaboard', and say that us-net has two commits. We will have directories |
| 70 | like this: |
| 71 | |
| 72 | us-net/ base directory |
| 73 | 01_of_02_g4ed4ebc_net--Add-tftp-speed-/ |
| 74 | sandbox/ |
| 75 | u-boot.bin |
| 76 | seaboard/ |
| 77 | u-boot.bin |
| 78 | 02_of_02_g4ed4ebc_net--Check-tftp-comp/ |
| 79 | sandbox/ |
| 80 | u-boot.bin |
| 81 | seaboard/ |
| 82 | u-boot.bin |
| 83 | .bm-work/ |
| 84 | 00/ working directory for thread 0 (contains source checkout) |
| 85 | build/ build output |
| 86 | 01/ working directory for thread 1 |
| 87 | build/ build output |
| 88 | ... |
| 89 | u-boot/ source directory |
| 90 | .git/ repository |
| 91 | """ |
| 92 | |
| 93 | # Possible build outcomes |
| 94 | OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) |
| 95 | |
| 96 | # Translate a commit subject into a valid filename |
| 97 | trans_valid_chars = string.maketrans("/: ", "---") |
| 98 | |
| 99 | |
| 100 | def Mkdir(dirname): |
| 101 | """Make a directory if it doesn't already exist. |
| 102 | |
| 103 | Args: |
| 104 | dirname: Directory to create |
| 105 | """ |
| 106 | try: |
| 107 | os.mkdir(dirname) |
| 108 | except OSError as err: |
| 109 | if err.errno == errno.EEXIST: |
| 110 | pass |
| 111 | else: |
| 112 | raise |
| 113 | |
| 114 | class BuilderJob: |
| 115 | """Holds information about a job to be performed by a thread |
| 116 | |
| 117 | Members: |
| 118 | board: Board object to build |
| 119 | commits: List of commit options to build. |
| 120 | """ |
| 121 | def __init__(self): |
| 122 | self.board = None |
| 123 | self.commits = [] |
| 124 | |
| 125 | |
| 126 | class ResultThread(threading.Thread): |
| 127 | """This thread processes results from builder threads. |
| 128 | |
| 129 | It simply passes the results on to the builder. There is only one |
| 130 | result thread, and this helps to serialise the build output. |
| 131 | """ |
| 132 | def __init__(self, builder): |
| 133 | """Set up a new result thread |
| 134 | |
| 135 | Args: |
| 136 | builder: Builder which will be sent each result |
| 137 | """ |
| 138 | threading.Thread.__init__(self) |
| 139 | self.builder = builder |
| 140 | |
| 141 | def run(self): |
| 142 | """Called to start up the result thread. |
| 143 | |
| 144 | We collect the next result job and pass it on to the build. |
| 145 | """ |
| 146 | while True: |
| 147 | result = self.builder.out_queue.get() |
| 148 | self.builder.ProcessResult(result) |
| 149 | self.builder.out_queue.task_done() |
| 150 | |
| 151 | |
| 152 | class BuilderThread(threading.Thread): |
| 153 | """This thread builds U-Boot for a particular board. |
| 154 | |
| 155 | An input queue provides each new job. We run 'make' to build U-Boot |
| 156 | and then pass the results on to the output queue. |
| 157 | |
| 158 | Members: |
| 159 | builder: The builder which contains information we might need |
| 160 | thread_num: Our thread number (0-n-1), used to decide on a |
| 161 | temporary directory |
| 162 | """ |
| 163 | def __init__(self, builder, thread_num): |
| 164 | """Set up a new builder thread""" |
| 165 | threading.Thread.__init__(self) |
| 166 | self.builder = builder |
| 167 | self.thread_num = thread_num |
| 168 | |
| 169 | def Make(self, commit, brd, stage, cwd, *args, **kwargs): |
| 170 | """Run 'make' on a particular commit and board. |
| 171 | |
| 172 | The source code will already be checked out, so the 'commit' |
| 173 | argument is only for information. |
| 174 | |
| 175 | Args: |
| 176 | commit: Commit object that is being built |
| 177 | brd: Board object that is being built |
| 178 | stage: Stage of the build. Valid stages are: |
| 179 | distclean - can be called to clean source |
| 180 | config - called to configure for a board |
| 181 | build - the main make invocation - it does the build |
| 182 | args: A list of arguments to pass to 'make' |
| 183 | kwargs: A list of keyword arguments to pass to command.RunPipe() |
| 184 | |
| 185 | Returns: |
| 186 | CommandResult object |
| 187 | """ |
| 188 | return self.builder.do_make(commit, brd, stage, cwd, *args, |
| 189 | **kwargs) |
| 190 | |
Simon Glass | 4266dc2 | 2014-07-13 12:22:31 -0600 | [diff] [blame] | 191 | def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build, |
| 192 | force_build_failures): |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 193 | """Build a particular commit. |
| 194 | |
| 195 | If the build is already done, and we are not forcing a build, we skip |
| 196 | the build and just return the previously-saved results. |
| 197 | |
| 198 | Args: |
| 199 | commit_upto: Commit number to build (0...n-1) |
| 200 | brd: Board object to build |
| 201 | work_dir: Directory to which the source will be checked out |
| 202 | do_config: True to run a make <board>_config on the source |
| 203 | force_build: Force a build even if one was previously done |
Simon Glass | 4266dc2 | 2014-07-13 12:22:31 -0600 | [diff] [blame] | 204 | force_build_failures: Force a bulid if the previous result showed |
| 205 | failure |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 206 | |
| 207 | Returns: |
| 208 | tuple containing: |
| 209 | - CommandResult object containing the results of the build |
| 210 | - boolean indicating whether 'make config' is still needed |
| 211 | """ |
| 212 | # Create a default result - it will be overwritte by the call to |
| 213 | # self.Make() below, in the event that we do a build. |
| 214 | result = command.CommandResult() |
| 215 | result.return_code = 0 |
| 216 | out_dir = os.path.join(work_dir, 'build') |
| 217 | |
| 218 | # Check if the job was already completed last time |
| 219 | done_file = self.builder.GetDoneFile(commit_upto, brd.target) |
| 220 | result.already_done = os.path.exists(done_file) |
Simon Glass | 4266dc2 | 2014-07-13 12:22:31 -0600 | [diff] [blame] | 221 | will_build = (force_build or force_build_failures or |
| 222 | not result.already_done) |
| 223 | if result.already_done and will_build: |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 224 | # Get the return code from that build and use it |
| 225 | with open(done_file, 'r') as fd: |
| 226 | result.return_code = int(fd.readline()) |
| 227 | err_file = self.builder.GetErrFile(commit_upto, brd.target) |
| 228 | if os.path.exists(err_file) and os.stat(err_file).st_size: |
| 229 | result.stderr = 'bad' |
Simon Glass | 4266dc2 | 2014-07-13 12:22:31 -0600 | [diff] [blame] | 230 | elif not force_build: |
| 231 | # The build passed, so no need to build it again |
| 232 | will_build = False |
| 233 | |
| 234 | if will_build: |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 235 | # We are going to have to build it. First, get a toolchain |
| 236 | if not self.toolchain: |
| 237 | try: |
| 238 | self.toolchain = self.builder.toolchains.Select(brd.arch) |
| 239 | except ValueError as err: |
| 240 | result.return_code = 10 |
| 241 | result.stdout = '' |
| 242 | result.stderr = str(err) |
| 243 | # TODO(sjg@chromium.org): This gets swallowed, but needs |
| 244 | # to be reported. |
| 245 | |
| 246 | if self.toolchain: |
| 247 | # Checkout the right commit |
| 248 | if commit_upto is not None: |
| 249 | commit = self.builder.commits[commit_upto] |
| 250 | if self.builder.checkout: |
| 251 | git_dir = os.path.join(work_dir, '.git') |
| 252 | gitutil.Checkout(commit.hash, git_dir, work_dir, |
| 253 | force=True) |
| 254 | else: |
| 255 | commit = self.builder.commit # Ick, fix this for BuildCommits() |
| 256 | |
| 257 | # Set up the environment and command line |
| 258 | env = self.toolchain.MakeEnvironment() |
| 259 | Mkdir(out_dir) |
| 260 | args = ['O=build', '-s'] |
| 261 | if self.builder.num_jobs is not None: |
| 262 | args.extend(['-j', str(self.builder.num_jobs)]) |
| 263 | config_args = ['%s_config' % brd.target] |
| 264 | config_out = '' |
Simon Glass | 4281ad8 | 2013-09-23 17:35:17 -0600 | [diff] [blame] | 265 | args.extend(self.builder.toolchains.GetMakeArguments(brd)) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 266 | |
| 267 | # If we need to reconfigure, do that now |
| 268 | if do_config: |
| 269 | result = self.Make(commit, brd, 'distclean', work_dir, |
| 270 | 'distclean', *args, env=env) |
| 271 | result = self.Make(commit, brd, 'config', work_dir, |
| 272 | *(args + config_args), env=env) |
| 273 | config_out = result.combined |
| 274 | do_config = False # No need to configure next time |
| 275 | if result.return_code == 0: |
| 276 | result = self.Make(commit, brd, 'build', work_dir, *args, |
| 277 | env=env) |
| 278 | result.stdout = config_out + result.stdout |
| 279 | else: |
| 280 | result.return_code = 1 |
| 281 | result.stderr = 'No tool chain for %s\n' % brd.arch |
| 282 | result.already_done = False |
| 283 | |
| 284 | result.toolchain = self.toolchain |
| 285 | result.brd = brd |
| 286 | result.commit_upto = commit_upto |
| 287 | result.out_dir = out_dir |
| 288 | return result, do_config |
| 289 | |
| 290 | def _WriteResult(self, result, keep_outputs): |
| 291 | """Write a built result to the output directory. |
| 292 | |
| 293 | Args: |
| 294 | result: CommandResult object containing result to write |
| 295 | keep_outputs: True to store the output binaries, False |
| 296 | to delete them |
| 297 | """ |
| 298 | # Fatal error |
| 299 | if result.return_code < 0: |
| 300 | return |
| 301 | |
| 302 | # Aborted? |
| 303 | if result.stderr and 'No child processes' in result.stderr: |
| 304 | return |
| 305 | |
| 306 | if result.already_done: |
| 307 | return |
| 308 | |
| 309 | # Write the output and stderr |
| 310 | output_dir = self.builder._GetOutputDir(result.commit_upto) |
| 311 | Mkdir(output_dir) |
| 312 | build_dir = self.builder.GetBuildDir(result.commit_upto, |
| 313 | result.brd.target) |
| 314 | Mkdir(build_dir) |
| 315 | |
| 316 | outfile = os.path.join(build_dir, 'log') |
| 317 | with open(outfile, 'w') as fd: |
| 318 | if result.stdout: |
| 319 | fd.write(result.stdout) |
| 320 | |
| 321 | errfile = self.builder.GetErrFile(result.commit_upto, |
| 322 | result.brd.target) |
| 323 | if result.stderr: |
| 324 | with open(errfile, 'w') as fd: |
| 325 | fd.write(result.stderr) |
| 326 | elif os.path.exists(errfile): |
| 327 | os.remove(errfile) |
| 328 | |
| 329 | if result.toolchain: |
| 330 | # Write the build result and toolchain information. |
| 331 | done_file = self.builder.GetDoneFile(result.commit_upto, |
| 332 | result.brd.target) |
| 333 | with open(done_file, 'w') as fd: |
| 334 | fd.write('%s' % result.return_code) |
| 335 | with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: |
| 336 | print >>fd, 'gcc', result.toolchain.gcc |
| 337 | print >>fd, 'path', result.toolchain.path |
| 338 | print >>fd, 'cross', result.toolchain.cross |
| 339 | print >>fd, 'arch', result.toolchain.arch |
| 340 | fd.write('%s' % result.return_code) |
| 341 | |
| 342 | with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: |
| 343 | print >>fd, 'gcc', result.toolchain.gcc |
| 344 | print >>fd, 'path', result.toolchain.path |
| 345 | |
| 346 | # Write out the image and function size information and an objdump |
| 347 | env = result.toolchain.MakeEnvironment() |
| 348 | lines = [] |
| 349 | for fname in ['u-boot', 'spl/u-boot-spl']: |
| 350 | cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] |
| 351 | nm_result = command.RunPipe([cmd], capture=True, |
| 352 | capture_stderr=True, cwd=result.out_dir, |
| 353 | raise_on_error=False, env=env) |
| 354 | if nm_result.stdout: |
| 355 | nm = self.builder.GetFuncSizesFile(result.commit_upto, |
| 356 | result.brd.target, fname) |
| 357 | with open(nm, 'w') as fd: |
| 358 | print >>fd, nm_result.stdout, |
| 359 | |
| 360 | cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] |
| 361 | dump_result = command.RunPipe([cmd], capture=True, |
| 362 | capture_stderr=True, cwd=result.out_dir, |
| 363 | raise_on_error=False, env=env) |
| 364 | rodata_size = '' |
| 365 | if dump_result.stdout: |
| 366 | objdump = self.builder.GetObjdumpFile(result.commit_upto, |
| 367 | result.brd.target, fname) |
| 368 | with open(objdump, 'w') as fd: |
| 369 | print >>fd, dump_result.stdout, |
| 370 | for line in dump_result.stdout.splitlines(): |
| 371 | fields = line.split() |
| 372 | if len(fields) > 5 and fields[1] == '.rodata': |
| 373 | rodata_size = fields[2] |
| 374 | |
| 375 | cmd = ['%ssize' % self.toolchain.cross, fname] |
| 376 | size_result = command.RunPipe([cmd], capture=True, |
| 377 | capture_stderr=True, cwd=result.out_dir, |
| 378 | raise_on_error=False, env=env) |
| 379 | if size_result.stdout: |
| 380 | lines.append(size_result.stdout.splitlines()[1] + ' ' + |
| 381 | rodata_size) |
| 382 | |
| 383 | # Write out the image sizes file. This is similar to the output |
| 384 | # of binutil's 'size' utility, but it omits the header line and |
| 385 | # adds an additional hex value at the end of each line for the |
| 386 | # rodata size |
| 387 | if len(lines): |
| 388 | sizes = self.builder.GetSizesFile(result.commit_upto, |
| 389 | result.brd.target) |
| 390 | with open(sizes, 'w') as fd: |
| 391 | print >>fd, '\n'.join(lines) |
| 392 | |
| 393 | # Now write the actual build output |
| 394 | if keep_outputs: |
| 395 | patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map', |
| 396 | 'include/autoconf.mk', 'spl/u-boot-spl', |
| 397 | 'spl/u-boot-spl.bin'] |
| 398 | for pattern in patterns: |
| 399 | file_list = glob.glob(os.path.join(result.out_dir, pattern)) |
| 400 | for fname in file_list: |
| 401 | shutil.copy(fname, build_dir) |
| 402 | |
| 403 | |
| 404 | def RunJob(self, job): |
| 405 | """Run a single job |
| 406 | |
| 407 | A job consists of a building a list of commits for a particular board. |
| 408 | |
| 409 | Args: |
| 410 | job: Job to build |
| 411 | """ |
| 412 | brd = job.board |
| 413 | work_dir = self.builder.GetThreadDir(self.thread_num) |
| 414 | self.toolchain = None |
| 415 | if job.commits: |
| 416 | # Run 'make board_config' on the first commit |
| 417 | do_config = True |
| 418 | commit_upto = 0 |
| 419 | force_build = False |
| 420 | for commit_upto in range(0, len(job.commits), job.step): |
| 421 | result, request_config = self.RunCommit(commit_upto, brd, |
| 422 | work_dir, do_config, |
Simon Glass | 4266dc2 | 2014-07-13 12:22:31 -0600 | [diff] [blame] | 423 | force_build or self.builder.force_build, |
| 424 | self.builder.force_build_failures) |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 425 | failed = result.return_code or result.stderr |
Simon Glass | 21fe8ec | 2014-07-13 14:03:41 -0600 | [diff] [blame] | 426 | did_config = do_config |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 427 | if failed and not do_config: |
| 428 | # If our incremental build failed, try building again |
| 429 | # with a reconfig. |
| 430 | if self.builder.force_config_on_failure: |
| 431 | result, request_config = self.RunCommit(commit_upto, |
Simon Glass | 4266dc2 | 2014-07-13 12:22:31 -0600 | [diff] [blame] | 432 | brd, work_dir, True, True, False) |
Simon Glass | 21fe8ec | 2014-07-13 14:03:41 -0600 | [diff] [blame] | 433 | did_config = True |
Simon Glass | 97e9152 | 2014-07-14 17:51:02 -0600 | [diff] [blame^] | 434 | if not self.builder.force_reconfig: |
| 435 | do_config = request_config |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 436 | |
| 437 | # If we built that commit, then config is done. But if we got |
| 438 | # an warning, reconfig next time to force it to build the same |
| 439 | # files that created warnings this time. Otherwise an |
| 440 | # incremental build may not build the same file, and we will |
| 441 | # think that the warning has gone away. |
| 442 | # We could avoid this by using -Werror everywhere... |
| 443 | # For errors, the problem doesn't happen, since presumably |
| 444 | # the build stopped and didn't generate output, so will retry |
| 445 | # that file next time. So we could detect warnings and deal |
| 446 | # with them specially here. For now, we just reconfigure if |
| 447 | # anything goes work. |
| 448 | # Of course this is substantially slower if there are build |
| 449 | # errors/warnings (e.g. 2-3x slower even if only 10% of builds |
| 450 | # have problems). |
Simon Glass | 21fe8ec | 2014-07-13 14:03:41 -0600 | [diff] [blame] | 451 | if (failed and not result.already_done and not did_config and |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 452 | self.builder.force_config_on_failure): |
| 453 | # If this build failed, try the next one with a |
| 454 | # reconfigure. |
| 455 | # Sometimes if the board_config.h file changes it can mess |
| 456 | # with dependencies, and we get: |
| 457 | # make: *** No rule to make target `include/autoconf.mk', |
| 458 | # needed by `depend'. |
| 459 | do_config = True |
| 460 | force_build = True |
| 461 | else: |
| 462 | force_build = False |
| 463 | if self.builder.force_config_on_failure: |
| 464 | if failed: |
| 465 | do_config = True |
| 466 | result.commit_upto = commit_upto |
| 467 | if result.return_code < 0: |
| 468 | raise ValueError('Interrupt') |
| 469 | |
| 470 | # We have the build results, so output the result |
| 471 | self._WriteResult(result, job.keep_outputs) |
| 472 | self.builder.out_queue.put(result) |
| 473 | else: |
| 474 | # Just build the currently checked-out build |
| 475 | result = self.RunCommit(None, True) |
| 476 | result.commit_upto = self.builder.upto |
| 477 | self.builder.out_queue.put(result) |
| 478 | |
| 479 | def run(self): |
| 480 | """Our thread's run function |
| 481 | |
| 482 | This thread picks a job from the queue, runs it, and then goes to the |
| 483 | next job. |
| 484 | """ |
| 485 | alive = True |
| 486 | while True: |
| 487 | job = self.builder.queue.get() |
| 488 | try: |
| 489 | if self.builder.active and alive: |
| 490 | self.RunJob(job) |
| 491 | except Exception as err: |
| 492 | alive = False |
| 493 | print err |
| 494 | self.builder.queue.task_done() |
| 495 | |
| 496 | |
| 497 | class Builder: |
| 498 | """Class for building U-Boot for a particular commit. |
| 499 | |
| 500 | Public members: (many should ->private) |
| 501 | active: True if the builder is active and has not been stopped |
| 502 | already_done: Number of builds already completed |
| 503 | base_dir: Base directory to use for builder |
| 504 | checkout: True to check out source, False to skip that step. |
| 505 | This is used for testing. |
| 506 | col: terminal.Color() object |
| 507 | count: Number of commits to build |
| 508 | do_make: Method to call to invoke Make |
| 509 | fail: Number of builds that failed due to error |
| 510 | force_build: Force building even if a build already exists |
| 511 | force_config_on_failure: If a commit fails for a board, disable |
| 512 | incremental building for the next commit we build for that |
| 513 | board, so that we will see all warnings/errors again. |
Simon Glass | 4266dc2 | 2014-07-13 12:22:31 -0600 | [diff] [blame] | 514 | force_build_failures: If a previously-built build (i.e. built on |
| 515 | a previous run of buildman) is marked as failed, rebuild it. |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 516 | git_dir: Git directory containing source repository |
| 517 | last_line_len: Length of the last line we printed (used for erasing |
| 518 | it with new progress information) |
| 519 | num_jobs: Number of jobs to run at once (passed to make as -j) |
| 520 | num_threads: Number of builder threads to run |
| 521 | out_queue: Queue of results to process |
| 522 | re_make_err: Compiled regular expression for ignore_lines |
| 523 | queue: Queue of jobs to run |
| 524 | threads: List of active threads |
| 525 | toolchains: Toolchains object to use for building |
| 526 | upto: Current commit number we are building (0.count-1) |
| 527 | warned: Number of builds that produced at least one warning |
Simon Glass | 97e9152 | 2014-07-14 17:51:02 -0600 | [diff] [blame^] | 528 | force_reconfig: Reconfigure U-Boot on each comiit. This disables |
| 529 | incremental building, where buildman reconfigures on the first |
| 530 | commit for a baord, and then just does an incremental build for |
| 531 | the following commits. In fact buildman will reconfigure and |
| 532 | retry for any failing commits, so generally the only effect of |
| 533 | this option is to slow things down. |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 534 | |
| 535 | Private members: |
| 536 | _base_board_dict: Last-summarised Dict of boards |
| 537 | _base_err_lines: Last-summarised list of errors |
| 538 | _build_period_us: Time taken for a single build (float object). |
| 539 | _complete_delay: Expected delay until completion (timedelta) |
| 540 | _next_delay_update: Next time we plan to display a progress update |
| 541 | (datatime) |
| 542 | _show_unknown: Show unknown boards (those not built) in summary |
| 543 | _timestamps: List of timestamps for the completion of the last |
| 544 | last _timestamp_count builds. Each is a datetime object. |
| 545 | _timestamp_count: Number of timestamps to keep in our list. |
| 546 | _working_dir: Base working directory containing all threads |
| 547 | """ |
| 548 | class Outcome: |
| 549 | """Records a build outcome for a single make invocation |
| 550 | |
| 551 | Public Members: |
| 552 | rc: Outcome value (OUTCOME_...) |
| 553 | err_lines: List of error lines or [] if none |
| 554 | sizes: Dictionary of image size information, keyed by filename |
| 555 | - Each value is itself a dictionary containing |
| 556 | values for 'text', 'data' and 'bss', being the integer |
| 557 | size in bytes of each section. |
| 558 | func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each |
| 559 | value is itself a dictionary: |
| 560 | key: function name |
| 561 | value: Size of function in bytes |
| 562 | """ |
| 563 | def __init__(self, rc, err_lines, sizes, func_sizes): |
| 564 | self.rc = rc |
| 565 | self.err_lines = err_lines |
| 566 | self.sizes = sizes |
| 567 | self.func_sizes = func_sizes |
| 568 | |
| 569 | def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, |
| 570 | checkout=True, show_unknown=True, step=1): |
| 571 | """Create a new Builder object |
| 572 | |
| 573 | Args: |
| 574 | toolchains: Toolchains object to use for building |
| 575 | base_dir: Base directory to use for builder |
| 576 | git_dir: Git directory containing source repository |
| 577 | num_threads: Number of builder threads to run |
| 578 | num_jobs: Number of jobs to run at once (passed to make as -j) |
| 579 | checkout: True to check out source, False to skip that step. |
| 580 | This is used for testing. |
| 581 | show_unknown: Show unknown boards (those not built) in summary |
| 582 | step: 1 to process every commit, n to process every nth commit |
| 583 | """ |
| 584 | self.toolchains = toolchains |
| 585 | self.base_dir = base_dir |
| 586 | self._working_dir = os.path.join(base_dir, '.bm-work') |
| 587 | self.threads = [] |
| 588 | self.active = True |
| 589 | self.do_make = self.Make |
| 590 | self.checkout = checkout |
| 591 | self.num_threads = num_threads |
| 592 | self.num_jobs = num_jobs |
| 593 | self.already_done = 0 |
| 594 | self.force_build = False |
| 595 | self.git_dir = git_dir |
| 596 | self._show_unknown = show_unknown |
| 597 | self._timestamp_count = 10 |
| 598 | self._build_period_us = None |
| 599 | self._complete_delay = None |
| 600 | self._next_delay_update = datetime.now() |
| 601 | self.force_config_on_failure = True |
Simon Glass | 4266dc2 | 2014-07-13 12:22:31 -0600 | [diff] [blame] | 602 | self.force_build_failures = False |
Simon Glass | 97e9152 | 2014-07-14 17:51:02 -0600 | [diff] [blame^] | 603 | self.force_reconfig = False |
Simon Glass | fc3fe1c | 2013-04-03 11:07:16 +0000 | [diff] [blame] | 604 | self._step = step |
| 605 | |
| 606 | self.col = terminal.Color() |
| 607 | |
| 608 | self.queue = Queue.Queue() |
| 609 | self.out_queue = Queue.Queue() |
| 610 | for i in range(self.num_threads): |
| 611 | t = BuilderThread(self, i) |
| 612 | t.setDaemon(True) |
| 613 | t.start() |
| 614 | self.threads.append(t) |
| 615 | |
| 616 | self.last_line_len = 0 |
| 617 | t = ResultThread(self) |
| 618 | t.setDaemon(True) |
| 619 | t.start() |
| 620 | self.threads.append(t) |
| 621 | |
| 622 | ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)'] |
| 623 | self.re_make_err = re.compile('|'.join(ignore_lines)) |
| 624 | |
| 625 | def __del__(self): |
| 626 | """Get rid of all threads created by the builder""" |
| 627 | for t in self.threads: |
| 628 | del t |
| 629 | |
| 630 | def _AddTimestamp(self): |
| 631 | """Add a new timestamp to the list and record the build period. |
| 632 | |
| 633 | The build period is the length of time taken to perform a single |
| 634 | build (one board, one commit). |
| 635 | """ |
| 636 | now = datetime.now() |
| 637 | self._timestamps.append(now) |
| 638 | count = len(self._timestamps) |
| 639 | delta = self._timestamps[-1] - self._timestamps[0] |
| 640 | seconds = delta.total_seconds() |
| 641 | |
| 642 | # If we have enough data, estimate build period (time taken for a |
| 643 | # single build) and therefore completion time. |
| 644 | if count > 1 and self._next_delay_update < now: |
| 645 | self._next_delay_update = now + timedelta(seconds=2) |
| 646 | if seconds > 0: |
| 647 | self._build_period = float(seconds) / count |
| 648 | todo = self.count - self.upto |
| 649 | self._complete_delay = timedelta(microseconds= |
| 650 | self._build_period * todo * 1000000) |
| 651 | # Round it |
| 652 | self._complete_delay -= timedelta( |
| 653 | microseconds=self._complete_delay.microseconds) |
| 654 | |
| 655 | if seconds > 60: |
| 656 | self._timestamps.popleft() |
| 657 | count -= 1 |
| 658 | |
| 659 | def ClearLine(self, length): |
| 660 | """Clear any characters on the current line |
| 661 | |
| 662 | Make way for a new line of length 'length', by outputting enough |
| 663 | spaces to clear out the old line. Then remember the new length for |
| 664 | next time. |
| 665 | |
| 666 | Args: |
| 667 | length: Length of new line, in characters |
| 668 | """ |
| 669 | if length < self.last_line_len: |
| 670 | print ' ' * (self.last_line_len - length), |
| 671 | print '\r', |
| 672 | self.last_line_len = length |
| 673 | sys.stdout.flush() |
| 674 | |
| 675 | def SelectCommit(self, commit, checkout=True): |
| 676 | """Checkout the selected commit for this build |
| 677 | """ |
| 678 | self.commit = commit |
| 679 | if checkout and self.checkout: |
| 680 | gitutil.Checkout(commit.hash) |
| 681 | |
| 682 | def Make(self, commit, brd, stage, cwd, *args, **kwargs): |
| 683 | """Run make |
| 684 | |
| 685 | Args: |
| 686 | commit: Commit object that is being built |
| 687 | brd: Board object that is being built |
| 688 | stage: Stage that we are at (distclean, config, build) |
| 689 | cwd: Directory where make should be run |
| 690 | args: Arguments to pass to make |
| 691 | kwargs: Arguments to pass to command.RunPipe() |
| 692 | """ |
| 693 | cmd = ['make'] + list(args) |
| 694 | result = command.RunPipe([cmd], capture=True, capture_stderr=True, |
| 695 | cwd=cwd, raise_on_error=False, **kwargs) |
| 696 | return result |
| 697 | |
| 698 | def ProcessResult(self, result): |
| 699 | """Process the result of a build, showing progress information |
| 700 | |
| 701 | Args: |
| 702 | result: A CommandResult object |
| 703 | """ |
| 704 | col = terminal.Color() |
| 705 | if result: |
| 706 | target = result.brd.target |
| 707 | |
| 708 | if result.return_code < 0: |
| 709 | self.active = False |
| 710 | command.StopAll() |
| 711 | return |
| 712 | |
| 713 | self.upto += 1 |
| 714 | if result.return_code != 0: |
| 715 | self.fail += 1 |
| 716 | elif result.stderr: |
| 717 | self.warned += 1 |
| 718 | if result.already_done: |
| 719 | self.already_done += 1 |
| 720 | else: |
| 721 | target = '(starting)' |
| 722 | |
| 723 | # Display separate counts for ok, warned and fail |
| 724 | ok = self.upto - self.warned - self.fail |
| 725 | line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok) |
| 726 | line += self.col.Color(self.col.YELLOW, '%5d' % self.warned) |
| 727 | line += self.col.Color(self.col.RED, '%5d' % self.fail) |
| 728 | |
| 729 | name = ' /%-5d ' % self.count |
| 730 | |
| 731 | # Add our current completion time estimate |
| 732 | self._AddTimestamp() |
| 733 | if self._complete_delay: |
| 734 | name += '%s : ' % self._complete_delay |
| 735 | # When building all boards for a commit, we can print a commit |
| 736 | # progress message. |
| 737 | if result and result.commit_upto is None: |
| 738 | name += 'commit %2d/%-3d' % (self.commit_upto + 1, |
| 739 | self.commit_count) |
| 740 | |
| 741 | name += target |
| 742 | print line + name, |
| 743 | length = 13 + len(name) |
| 744 | self.ClearLine(length) |
| 745 | |
| 746 | def _GetOutputDir(self, commit_upto): |
| 747 | """Get the name of the output directory for a commit number |
| 748 | |
| 749 | The output directory is typically .../<branch>/<commit>. |
| 750 | |
| 751 | Args: |
| 752 | commit_upto: Commit number to use (0..self.count-1) |
| 753 | """ |
| 754 | commit = self.commits[commit_upto] |
| 755 | subject = commit.subject.translate(trans_valid_chars) |
| 756 | commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, |
| 757 | self.commit_count, commit.hash, subject[:20])) |
| 758 | output_dir = os.path.join(self.base_dir, commit_dir) |
| 759 | return output_dir |
| 760 | |
| 761 | def GetBuildDir(self, commit_upto, target): |
| 762 | """Get the name of the build directory for a commit number |
| 763 | |
| 764 | The build directory is typically .../<branch>/<commit>/<target>. |
| 765 | |
| 766 | Args: |
| 767 | commit_upto: Commit number to use (0..self.count-1) |
| 768 | target: Target name |
| 769 | """ |
| 770 | output_dir = self._GetOutputDir(commit_upto) |
| 771 | return os.path.join(output_dir, target) |
| 772 | |
| 773 | def GetDoneFile(self, commit_upto, target): |
| 774 | """Get the name of the done file for a commit number |
| 775 | |
| 776 | Args: |
| 777 | commit_upto: Commit number to use (0..self.count-1) |
| 778 | target: Target name |
| 779 | """ |
| 780 | return os.path.join(self.GetBuildDir(commit_upto, target), 'done') |
| 781 | |
| 782 | def GetSizesFile(self, commit_upto, target): |
| 783 | """Get the name of the sizes file for a commit number |
| 784 | |
| 785 | Args: |
| 786 | commit_upto: Commit number to use (0..self.count-1) |
| 787 | target: Target name |
| 788 | """ |
| 789 | return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes') |
| 790 | |
| 791 | def GetFuncSizesFile(self, commit_upto, target, elf_fname): |
| 792 | """Get the name of the funcsizes file for a commit number and ELF file |
| 793 | |
| 794 | Args: |
| 795 | commit_upto: Commit number to use (0..self.count-1) |
| 796 | target: Target name |
| 797 | elf_fname: Filename of elf image |
| 798 | """ |
| 799 | return os.path.join(self.GetBuildDir(commit_upto, target), |
| 800 | '%s.sizes' % elf_fname.replace('/', '-')) |
| 801 | |
| 802 | def GetObjdumpFile(self, commit_upto, target, elf_fname): |
| 803 | """Get the name of the objdump file for a commit number and ELF file |
| 804 | |
| 805 | Args: |
| 806 | commit_upto: Commit number to use (0..self.count-1) |
| 807 | target: Target name |
| 808 | elf_fname: Filename of elf image |
| 809 | """ |
| 810 | return os.path.join(self.GetBuildDir(commit_upto, target), |
| 811 | '%s.objdump' % elf_fname.replace('/', '-')) |
| 812 | |
| 813 | def GetErrFile(self, commit_upto, target): |
| 814 | """Get the name of the err file for a commit number |
| 815 | |
| 816 | Args: |
| 817 | commit_upto: Commit number to use (0..self.count-1) |
| 818 | target: Target name |
| 819 | """ |
| 820 | output_dir = self.GetBuildDir(commit_upto, target) |
| 821 | return os.path.join(output_dir, 'err') |
| 822 | |
| 823 | def FilterErrors(self, lines): |
| 824 | """Filter out errors in which we have no interest |
| 825 | |
| 826 | We should probably use map(). |
| 827 | |
| 828 | Args: |
| 829 | lines: List of error lines, each a string |
| 830 | Returns: |
| 831 | New list with only interesting lines included |
| 832 | """ |
| 833 | out_lines = [] |
| 834 | for line in lines: |
| 835 | if not self.re_make_err.search(line): |
| 836 | out_lines.append(line) |
| 837 | return out_lines |
| 838 | |
| 839 | def ReadFuncSizes(self, fname, fd): |
| 840 | """Read function sizes from the output of 'nm' |
| 841 | |
| 842 | Args: |
| 843 | fd: File containing data to read |
| 844 | fname: Filename we are reading from (just for errors) |
| 845 | |
| 846 | Returns: |
| 847 | Dictionary containing size of each function in bytes, indexed by |
| 848 | function name. |
| 849 | """ |
| 850 | sym = {} |
| 851 | for line in fd.readlines(): |
| 852 | try: |
| 853 | size, type, name = line[:-1].split() |
| 854 | except: |
| 855 | print "Invalid line in file '%s': '%s'" % (fname, line[:-1]) |
| 856 | continue |
| 857 | if type in 'tTdDbB': |
| 858 | # function names begin with '.' on 64-bit powerpc |
| 859 | if '.' in name[1:]: |
| 860 | name = 'static.' + name.split('.')[0] |
| 861 | sym[name] = sym.get(name, 0) + int(size, 16) |
| 862 | return sym |
| 863 | |
| 864 | def GetBuildOutcome(self, commit_upto, target, read_func_sizes): |
| 865 | """Work out the outcome of a build. |
| 866 | |
| 867 | Args: |
| 868 | commit_upto: Commit number to check (0..n-1) |
| 869 | target: Target board to check |
| 870 | read_func_sizes: True to read function size information |
| 871 | |
| 872 | Returns: |
| 873 | Outcome object |
| 874 | """ |
| 875 | done_file = self.GetDoneFile(commit_upto, target) |
| 876 | sizes_file = self.GetSizesFile(commit_upto, target) |
| 877 | sizes = {} |
| 878 | func_sizes = {} |
| 879 | if os.path.exists(done_file): |
| 880 | with open(done_file, 'r') as fd: |
| 881 | return_code = int(fd.readline()) |
| 882 | err_lines = [] |
| 883 | err_file = self.GetErrFile(commit_upto, target) |
| 884 | if os.path.exists(err_file): |
| 885 | with open(err_file, 'r') as fd: |
| 886 | err_lines = self.FilterErrors(fd.readlines()) |
| 887 | |
| 888 | # Decide whether the build was ok, failed or created warnings |
| 889 | if return_code: |
| 890 | rc = OUTCOME_ERROR |
| 891 | elif len(err_lines): |
| 892 | rc = OUTCOME_WARNING |
| 893 | else: |
| 894 | rc = OUTCOME_OK |
| 895 | |
| 896 | # Convert size information to our simple format |
| 897 | if os.path.exists(sizes_file): |
| 898 | with open(sizes_file, 'r') as fd: |
| 899 | for line in fd.readlines(): |
| 900 | values = line.split() |
| 901 | rodata = 0 |
| 902 | if len(values) > 6: |
| 903 | rodata = int(values[6], 16) |
| 904 | size_dict = { |
| 905 | 'all' : int(values[0]) + int(values[1]) + |
| 906 | int(values[2]), |
| 907 | 'text' : int(values[0]) - rodata, |
| 908 | 'data' : int(values[1]), |
| 909 | 'bss' : int(values[2]), |
| 910 | 'rodata' : rodata, |
| 911 | } |
| 912 | sizes[values[5]] = size_dict |
| 913 | |
| 914 | if read_func_sizes: |
| 915 | pattern = self.GetFuncSizesFile(commit_upto, target, '*') |
| 916 | for fname in glob.glob(pattern): |
| 917 | with open(fname, 'r') as fd: |
| 918 | dict_name = os.path.basename(fname).replace('.sizes', |
| 919 | '') |
| 920 | func_sizes[dict_name] = self.ReadFuncSizes(fname, fd) |
| 921 | |
| 922 | return Builder.Outcome(rc, err_lines, sizes, func_sizes) |
| 923 | |
| 924 | return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}) |
| 925 | |
| 926 | def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes): |
| 927 | """Calculate a summary of the results of building a commit. |
| 928 | |
| 929 | Args: |
| 930 | board_selected: Dict containing boards to summarise |
| 931 | commit_upto: Commit number to summarize (0..self.count-1) |
| 932 | read_func_sizes: True to read function size information |
| 933 | |
| 934 | Returns: |
| 935 | Tuple: |
| 936 | Dict containing boards which passed building this commit. |
| 937 | keyed by board.target |
| 938 | List containing a summary of error/warning lines |
| 939 | """ |
| 940 | board_dict = {} |
| 941 | err_lines_summary = [] |
| 942 | |
| 943 | for board in boards_selected.itervalues(): |
| 944 | outcome = self.GetBuildOutcome(commit_upto, board.target, |
| 945 | read_func_sizes) |
| 946 | board_dict[board.target] = outcome |
| 947 | for err in outcome.err_lines: |
| 948 | if err and not err.rstrip() in err_lines_summary: |
| 949 | err_lines_summary.append(err.rstrip()) |
| 950 | return board_dict, err_lines_summary |
| 951 | |
| 952 | def AddOutcome(self, board_dict, arch_list, changes, char, color): |
| 953 | """Add an output to our list of outcomes for each architecture |
| 954 | |
| 955 | This simple function adds failing boards (changes) to the |
| 956 | relevant architecture string, so we can print the results out |
| 957 | sorted by architecture. |
| 958 | |
| 959 | Args: |
| 960 | board_dict: Dict containing all boards |
| 961 | arch_list: Dict keyed by arch name. Value is a string containing |
| 962 | a list of board names which failed for that arch. |
| 963 | changes: List of boards to add to arch_list |
| 964 | color: terminal.Colour object |
| 965 | """ |
| 966 | done_arch = {} |
| 967 | for target in changes: |
| 968 | if target in board_dict: |
| 969 | arch = board_dict[target].arch |
| 970 | else: |
| 971 | arch = 'unknown' |
| 972 | str = self.col.Color(color, ' ' + target) |
| 973 | if not arch in done_arch: |
| 974 | str = self.col.Color(color, char) + ' ' + str |
| 975 | done_arch[arch] = True |
| 976 | if not arch in arch_list: |
| 977 | arch_list[arch] = str |
| 978 | else: |
| 979 | arch_list[arch] += str |
| 980 | |
| 981 | |
| 982 | def ColourNum(self, num): |
| 983 | color = self.col.RED if num > 0 else self.col.GREEN |
| 984 | if num == 0: |
| 985 | return '0' |
| 986 | return self.col.Color(color, str(num)) |
| 987 | |
| 988 | def ResetResultSummary(self, board_selected): |
| 989 | """Reset the results summary ready for use. |
| 990 | |
| 991 | Set up the base board list to be all those selected, and set the |
| 992 | error lines to empty. |
| 993 | |
| 994 | Following this, calls to PrintResultSummary() will use this |
| 995 | information to work out what has changed. |
| 996 | |
| 997 | Args: |
| 998 | board_selected: Dict containing boards to summarise, keyed by |
| 999 | board.target |
| 1000 | """ |
| 1001 | self._base_board_dict = {} |
| 1002 | for board in board_selected: |
| 1003 | self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) |
| 1004 | self._base_err_lines = [] |
| 1005 | |
| 1006 | def PrintFuncSizeDetail(self, fname, old, new): |
| 1007 | grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 |
| 1008 | delta, common = [], {} |
| 1009 | |
| 1010 | for a in old: |
| 1011 | if a in new: |
| 1012 | common[a] = 1 |
| 1013 | |
| 1014 | for name in old: |
| 1015 | if name not in common: |
| 1016 | remove += 1 |
| 1017 | down += old[name] |
| 1018 | delta.append([-old[name], name]) |
| 1019 | |
| 1020 | for name in new: |
| 1021 | if name not in common: |
| 1022 | add += 1 |
| 1023 | up += new[name] |
| 1024 | delta.append([new[name], name]) |
| 1025 | |
| 1026 | for name in common: |
| 1027 | diff = new.get(name, 0) - old.get(name, 0) |
| 1028 | if diff > 0: |
| 1029 | grow, up = grow + 1, up + diff |
| 1030 | elif diff < 0: |
| 1031 | shrink, down = shrink + 1, down - diff |
| 1032 | delta.append([diff, name]) |
| 1033 | |
| 1034 | delta.sort() |
| 1035 | delta.reverse() |
| 1036 | |
| 1037 | args = [add, -remove, grow, -shrink, up, -down, up - down] |
| 1038 | if max(args) == 0: |
| 1039 | return |
| 1040 | args = [self.ColourNum(x) for x in args] |
| 1041 | indent = ' ' * 15 |
| 1042 | print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % |
| 1043 | tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) |
| 1044 | print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', |
| 1045 | 'delta') |
| 1046 | for diff, name in delta: |
| 1047 | if diff: |
| 1048 | color = self.col.RED if diff > 0 else self.col.GREEN |
| 1049 | msg = '%s %-38s %7s %7s %+7d' % (indent, name, |
| 1050 | old.get(name, '-'), new.get(name,'-'), diff) |
| 1051 | print self.col.Color(color, msg) |
| 1052 | |
| 1053 | |
| 1054 | def PrintSizeDetail(self, target_list, show_bloat): |
| 1055 | """Show details size information for each board |
| 1056 | |
| 1057 | Args: |
| 1058 | target_list: List of targets, each a dict containing: |
| 1059 | 'target': Target name |
| 1060 | 'total_diff': Total difference in bytes across all areas |
| 1061 | <part_name>: Difference for that part |
| 1062 | show_bloat: Show detail for each function |
| 1063 | """ |
| 1064 | targets_by_diff = sorted(target_list, reverse=True, |
| 1065 | key=lambda x: x['_total_diff']) |
| 1066 | for result in targets_by_diff: |
| 1067 | printed_target = False |
| 1068 | for name in sorted(result): |
| 1069 | diff = result[name] |
| 1070 | if name.startswith('_'): |
| 1071 | continue |
| 1072 | if diff != 0: |
| 1073 | color = self.col.RED if diff > 0 else self.col.GREEN |
| 1074 | msg = ' %s %+d' % (name, diff) |
| 1075 | if not printed_target: |
| 1076 | print '%10s %-15s:' % ('', result['_target']), |
| 1077 | printed_target = True |
| 1078 | print self.col.Color(color, msg), |
| 1079 | if printed_target: |
| 1080 | print |
| 1081 | if show_bloat: |
| 1082 | target = result['_target'] |
| 1083 | outcome = result['_outcome'] |
| 1084 | base_outcome = self._base_board_dict[target] |
| 1085 | for fname in outcome.func_sizes: |
| 1086 | self.PrintFuncSizeDetail(fname, |
| 1087 | base_outcome.func_sizes[fname], |
| 1088 | outcome.func_sizes[fname]) |
| 1089 | |
| 1090 | |
| 1091 | def PrintSizeSummary(self, board_selected, board_dict, show_detail, |
| 1092 | show_bloat): |
| 1093 | """Print a summary of image sizes broken down by section. |
| 1094 | |
| 1095 | The summary takes the form of one line per architecture. The |
| 1096 | line contains deltas for each of the sections (+ means the section |
| 1097 | got bigger, - means smaller). The nunmbers are the average number |
| 1098 | of bytes that a board in this section increased by. |
| 1099 | |
| 1100 | For example: |
| 1101 | powerpc: (622 boards) text -0.0 |
| 1102 | arm: (285 boards) text -0.0 |
| 1103 | nds32: (3 boards) text -8.0 |
| 1104 | |
| 1105 | Args: |
| 1106 | board_selected: Dict containing boards to summarise, keyed by |
| 1107 | board.target |
| 1108 | board_dict: Dict containing boards for which we built this |
| 1109 | commit, keyed by board.target. The value is an Outcome object. |
| 1110 | show_detail: Show detail for each board |
| 1111 | show_bloat: Show detail for each function |
| 1112 | """ |
| 1113 | arch_list = {} |
| 1114 | arch_count = {} |
| 1115 | |
| 1116 | # Calculate changes in size for different image parts |
| 1117 | # The previous sizes are in Board.sizes, for each board |
| 1118 | for target in board_dict: |
| 1119 | if target not in board_selected: |
| 1120 | continue |
| 1121 | base_sizes = self._base_board_dict[target].sizes |
| 1122 | outcome = board_dict[target] |
| 1123 | sizes = outcome.sizes |
| 1124 | |
| 1125 | # Loop through the list of images, creating a dict of size |
| 1126 | # changes for each image/part. We end up with something like |
| 1127 | # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4} |
| 1128 | # which means that U-Boot data increased by 5 bytes and SPL |
| 1129 | # text decreased by 4. |
| 1130 | err = {'_target' : target} |
| 1131 | for image in sizes: |
| 1132 | if image in base_sizes: |
| 1133 | base_image = base_sizes[image] |
| 1134 | # Loop through the text, data, bss parts |
| 1135 | for part in sorted(sizes[image]): |
| 1136 | diff = sizes[image][part] - base_image[part] |
| 1137 | col = None |
| 1138 | if diff: |
| 1139 | if image == 'u-boot': |
| 1140 | name = part |
| 1141 | else: |
| 1142 | name = image + ':' + part |
| 1143 | err[name] = diff |
| 1144 | arch = board_selected[target].arch |
| 1145 | if not arch in arch_count: |
| 1146 | arch_count[arch] = 1 |
| 1147 | else: |
| 1148 | arch_count[arch] += 1 |
| 1149 | if not sizes: |
| 1150 | pass # Only add to our list when we have some stats |
| 1151 | elif not arch in arch_list: |
| 1152 | arch_list[arch] = [err] |
| 1153 | else: |
| 1154 | arch_list[arch].append(err) |
| 1155 | |
| 1156 | # We now have a list of image size changes sorted by arch |
| 1157 | # Print out a summary of these |
| 1158 | for arch, target_list in arch_list.iteritems(): |
| 1159 | # Get total difference for each type |
| 1160 | totals = {} |
| 1161 | for result in target_list: |
| 1162 | total = 0 |
| 1163 | for name, diff in result.iteritems(): |
| 1164 | if name.startswith('_'): |
| 1165 | continue |
| 1166 | total += diff |
| 1167 | if name in totals: |
| 1168 | totals[name] += diff |
| 1169 | else: |
| 1170 | totals[name] = diff |
| 1171 | result['_total_diff'] = total |
| 1172 | result['_outcome'] = board_dict[result['_target']] |
| 1173 | |
| 1174 | count = len(target_list) |
| 1175 | printed_arch = False |
| 1176 | for name in sorted(totals): |
| 1177 | diff = totals[name] |
| 1178 | if diff: |
| 1179 | # Display the average difference in this name for this |
| 1180 | # architecture |
| 1181 | avg_diff = float(diff) / count |
| 1182 | color = self.col.RED if avg_diff > 0 else self.col.GREEN |
| 1183 | msg = ' %s %+1.1f' % (name, avg_diff) |
| 1184 | if not printed_arch: |
| 1185 | print '%10s: (for %d/%d boards)' % (arch, count, |
| 1186 | arch_count[arch]), |
| 1187 | printed_arch = True |
| 1188 | print self.col.Color(color, msg), |
| 1189 | |
| 1190 | if printed_arch: |
| 1191 | print |
| 1192 | if show_detail: |
| 1193 | self.PrintSizeDetail(target_list, show_bloat) |
| 1194 | |
| 1195 | |
| 1196 | def PrintResultSummary(self, board_selected, board_dict, err_lines, |
| 1197 | show_sizes, show_detail, show_bloat): |
| 1198 | """Compare results with the base results and display delta. |
| 1199 | |
| 1200 | Only boards mentioned in board_selected will be considered. This |
| 1201 | function is intended to be called repeatedly with the results of |
| 1202 | each commit. It therefore shows a 'diff' between what it saw in |
| 1203 | the last call and what it sees now. |
| 1204 | |
| 1205 | Args: |
| 1206 | board_selected: Dict containing boards to summarise, keyed by |
| 1207 | board.target |
| 1208 | board_dict: Dict containing boards for which we built this |
| 1209 | commit, keyed by board.target. The value is an Outcome object. |
| 1210 | err_lines: A list of errors for this commit, or [] if there is |
| 1211 | none, or we don't want to print errors |
| 1212 | show_sizes: Show image size deltas |
| 1213 | show_detail: Show detail for each board |
| 1214 | show_bloat: Show detail for each function |
| 1215 | """ |
| 1216 | better = [] # List of boards fixed since last commit |
| 1217 | worse = [] # List of new broken boards since last commit |
| 1218 | new = [] # List of boards that didn't exist last time |
| 1219 | unknown = [] # List of boards that were not built |
| 1220 | |
| 1221 | for target in board_dict: |
| 1222 | if target not in board_selected: |
| 1223 | continue |
| 1224 | |
| 1225 | # If the board was built last time, add its outcome to a list |
| 1226 | if target in self._base_board_dict: |
| 1227 | base_outcome = self._base_board_dict[target].rc |
| 1228 | outcome = board_dict[target] |
| 1229 | if outcome.rc == OUTCOME_UNKNOWN: |
| 1230 | unknown.append(target) |
| 1231 | elif outcome.rc < base_outcome: |
| 1232 | better.append(target) |
| 1233 | elif outcome.rc > base_outcome: |
| 1234 | worse.append(target) |
| 1235 | else: |
| 1236 | new.append(target) |
| 1237 | |
| 1238 | # Get a list of errors that have appeared, and disappeared |
| 1239 | better_err = [] |
| 1240 | worse_err = [] |
| 1241 | for line in err_lines: |
| 1242 | if line not in self._base_err_lines: |
| 1243 | worse_err.append('+' + line) |
| 1244 | for line in self._base_err_lines: |
| 1245 | if line not in err_lines: |
| 1246 | better_err.append('-' + line) |
| 1247 | |
| 1248 | # Display results by arch |
| 1249 | if better or worse or unknown or new or worse_err or better_err: |
| 1250 | arch_list = {} |
| 1251 | self.AddOutcome(board_selected, arch_list, better, '', |
| 1252 | self.col.GREEN) |
| 1253 | self.AddOutcome(board_selected, arch_list, worse, '+', |
| 1254 | self.col.RED) |
| 1255 | self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE) |
| 1256 | if self._show_unknown: |
| 1257 | self.AddOutcome(board_selected, arch_list, unknown, '?', |
| 1258 | self.col.MAGENTA) |
| 1259 | for arch, target_list in arch_list.iteritems(): |
| 1260 | print '%10s: %s' % (arch, target_list) |
| 1261 | if better_err: |
| 1262 | print self.col.Color(self.col.GREEN, '\n'.join(better_err)) |
| 1263 | if worse_err: |
| 1264 | print self.col.Color(self.col.RED, '\n'.join(worse_err)) |
| 1265 | |
| 1266 | if show_sizes: |
| 1267 | self.PrintSizeSummary(board_selected, board_dict, show_detail, |
| 1268 | show_bloat) |
| 1269 | |
| 1270 | # Save our updated information for the next call to this function |
| 1271 | self._base_board_dict = board_dict |
| 1272 | self._base_err_lines = err_lines |
| 1273 | |
| 1274 | # Get a list of boards that did not get built, if needed |
| 1275 | not_built = [] |
| 1276 | for board in board_selected: |
| 1277 | if not board in board_dict: |
| 1278 | not_built.append(board) |
| 1279 | if not_built: |
| 1280 | print "Boards not built (%d): %s" % (len(not_built), |
| 1281 | ', '.join(not_built)) |
| 1282 | |
| 1283 | |
| 1284 | def ShowSummary(self, commits, board_selected, show_errors, show_sizes, |
| 1285 | show_detail, show_bloat): |
| 1286 | """Show a build summary for U-Boot for a given board list. |
| 1287 | |
| 1288 | Reset the result summary, then repeatedly call GetResultSummary on |
| 1289 | each commit's results, then display the differences we see. |
| 1290 | |
| 1291 | Args: |
| 1292 | commit: Commit objects to summarise |
| 1293 | board_selected: Dict containing boards to summarise |
| 1294 | show_errors: Show errors that occured |
| 1295 | show_sizes: Show size deltas |
| 1296 | show_detail: Show detail for each board |
| 1297 | show_bloat: Show detail for each function |
| 1298 | """ |
| 1299 | self.commit_count = len(commits) |
| 1300 | self.commits = commits |
| 1301 | self.ResetResultSummary(board_selected) |
| 1302 | |
| 1303 | for commit_upto in range(0, self.commit_count, self._step): |
| 1304 | board_dict, err_lines = self.GetResultSummary(board_selected, |
| 1305 | commit_upto, read_func_sizes=show_bloat) |
| 1306 | msg = '%02d: %s' % (commit_upto + 1, commits[commit_upto].subject) |
| 1307 | print self.col.Color(self.col.BLUE, msg) |
| 1308 | self.PrintResultSummary(board_selected, board_dict, |
| 1309 | err_lines if show_errors else [], show_sizes, show_detail, |
| 1310 | show_bloat) |
| 1311 | |
| 1312 | |
| 1313 | def SetupBuild(self, board_selected, commits): |
| 1314 | """Set up ready to start a build. |
| 1315 | |
| 1316 | Args: |
| 1317 | board_selected: Selected boards to build |
| 1318 | commits: Selected commits to build |
| 1319 | """ |
| 1320 | # First work out how many commits we will build |
| 1321 | count = (len(commits) + self._step - 1) / self._step |
| 1322 | self.count = len(board_selected) * count |
| 1323 | self.upto = self.warned = self.fail = 0 |
| 1324 | self._timestamps = collections.deque() |
| 1325 | |
| 1326 | def BuildBoardsForCommit(self, board_selected, keep_outputs): |
| 1327 | """Build all boards for a single commit""" |
| 1328 | self.SetupBuild(board_selected) |
| 1329 | self.count = len(board_selected) |
| 1330 | for brd in board_selected.itervalues(): |
| 1331 | job = BuilderJob() |
| 1332 | job.board = brd |
| 1333 | job.commits = None |
| 1334 | job.keep_outputs = keep_outputs |
| 1335 | self.queue.put(brd) |
| 1336 | |
| 1337 | self.queue.join() |
| 1338 | self.out_queue.join() |
| 1339 | print |
| 1340 | self.ClearLine(0) |
| 1341 | |
| 1342 | def BuildCommits(self, commits, board_selected, show_errors, keep_outputs): |
| 1343 | """Build all boards for all commits (non-incremental)""" |
| 1344 | self.commit_count = len(commits) |
| 1345 | |
| 1346 | self.ResetResultSummary(board_selected) |
| 1347 | for self.commit_upto in range(self.commit_count): |
| 1348 | self.SelectCommit(commits[self.commit_upto]) |
| 1349 | self.SelectOutputDir() |
| 1350 | Mkdir(self.output_dir) |
| 1351 | |
| 1352 | self.BuildBoardsForCommit(board_selected, keep_outputs) |
| 1353 | board_dict, err_lines = self.GetResultSummary() |
| 1354 | self.PrintResultSummary(board_selected, board_dict, |
| 1355 | err_lines if show_errors else []) |
| 1356 | |
| 1357 | if self.already_done: |
| 1358 | print '%d builds already done' % self.already_done |
| 1359 | |
| 1360 | def GetThreadDir(self, thread_num): |
| 1361 | """Get the directory path to the working dir for a thread. |
| 1362 | |
| 1363 | Args: |
| 1364 | thread_num: Number of thread to check. |
| 1365 | """ |
| 1366 | return os.path.join(self._working_dir, '%02d' % thread_num) |
| 1367 | |
| 1368 | def _PrepareThread(self, thread_num): |
| 1369 | """Prepare the working directory for a thread. |
| 1370 | |
| 1371 | This clones or fetches the repo into the thread's work directory. |
| 1372 | |
| 1373 | Args: |
| 1374 | thread_num: Thread number (0, 1, ...) |
| 1375 | """ |
| 1376 | thread_dir = self.GetThreadDir(thread_num) |
| 1377 | Mkdir(thread_dir) |
| 1378 | git_dir = os.path.join(thread_dir, '.git') |
| 1379 | |
| 1380 | # Clone the repo if it doesn't already exist |
| 1381 | # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so |
| 1382 | # we have a private index but uses the origin repo's contents? |
| 1383 | if self.git_dir: |
| 1384 | src_dir = os.path.abspath(self.git_dir) |
| 1385 | if os.path.exists(git_dir): |
| 1386 | gitutil.Fetch(git_dir, thread_dir) |
| 1387 | else: |
| 1388 | print 'Cloning repo for thread %d' % thread_num |
| 1389 | gitutil.Clone(src_dir, thread_dir) |
| 1390 | |
| 1391 | def _PrepareWorkingSpace(self, max_threads): |
| 1392 | """Prepare the working directory for use. |
| 1393 | |
| 1394 | Set up the git repo for each thread. |
| 1395 | |
| 1396 | Args: |
| 1397 | max_threads: Maximum number of threads we expect to need. |
| 1398 | """ |
| 1399 | Mkdir(self._working_dir) |
| 1400 | for thread in range(max_threads): |
| 1401 | self._PrepareThread(thread) |
| 1402 | |
| 1403 | def _PrepareOutputSpace(self): |
| 1404 | """Get the output directories ready to receive files. |
| 1405 | |
| 1406 | We delete any output directories which look like ones we need to |
| 1407 | create. Having left over directories is confusing when the user wants |
| 1408 | to check the output manually. |
| 1409 | """ |
| 1410 | dir_list = [] |
| 1411 | for commit_upto in range(self.commit_count): |
| 1412 | dir_list.append(self._GetOutputDir(commit_upto)) |
| 1413 | |
| 1414 | for dirname in glob.glob(os.path.join(self.base_dir, '*')): |
| 1415 | if dirname not in dir_list: |
| 1416 | shutil.rmtree(dirname) |
| 1417 | |
| 1418 | def BuildBoards(self, commits, board_selected, show_errors, keep_outputs): |
| 1419 | """Build all commits for a list of boards |
| 1420 | |
| 1421 | Args: |
| 1422 | commits: List of commits to be build, each a Commit object |
| 1423 | boards_selected: Dict of selected boards, key is target name, |
| 1424 | value is Board object |
| 1425 | show_errors: True to show summarised error/warning info |
| 1426 | keep_outputs: True to save build output files |
| 1427 | """ |
| 1428 | self.commit_count = len(commits) |
| 1429 | self.commits = commits |
| 1430 | |
| 1431 | self.ResetResultSummary(board_selected) |
| 1432 | Mkdir(self.base_dir) |
| 1433 | self._PrepareWorkingSpace(min(self.num_threads, len(board_selected))) |
| 1434 | self._PrepareOutputSpace() |
| 1435 | self.SetupBuild(board_selected, commits) |
| 1436 | self.ProcessResult(None) |
| 1437 | |
| 1438 | # Create jobs to build all commits for each board |
| 1439 | for brd in board_selected.itervalues(): |
| 1440 | job = BuilderJob() |
| 1441 | job.board = brd |
| 1442 | job.commits = commits |
| 1443 | job.keep_outputs = keep_outputs |
| 1444 | job.step = self._step |
| 1445 | self.queue.put(job) |
| 1446 | |
| 1447 | # Wait until all jobs are started |
| 1448 | self.queue.join() |
| 1449 | |
| 1450 | # Wait until we have processed all output |
| 1451 | self.out_queue.join() |
| 1452 | print |
| 1453 | self.ClearLine(0) |