Tom Rini | 83d290c | 2018-05-06 17:58:06 -0400 | [diff] [blame] | 1 | # SPDX-License-Identifier: GPL-2.0+ |
Simon Glass | 1f1864b | 2016-07-25 18:59:08 -0600 | [diff] [blame] | 2 | # |
| 3 | # Copyright (c) 2016 Google, Inc |
| 4 | # |
Simon Glass | 1f1864b | 2016-07-25 18:59:08 -0600 | [diff] [blame] | 5 | |
Simon Glass | 0a98b28 | 2018-09-14 04:57:28 -0600 | [diff] [blame] | 6 | import glob |
Simon Glass | 1f1864b | 2016-07-25 18:59:08 -0600 | [diff] [blame] | 7 | import os |
Paul Barker | 0d60e5d | 2021-09-08 12:38:02 +0100 | [diff] [blame] | 8 | import shlex |
Simon Glass | 1f1864b | 2016-07-25 18:59:08 -0600 | [diff] [blame] | 9 | import shutil |
Simon Glass | eb0f4a4 | 2019-07-20 12:24:06 -0600 | [diff] [blame] | 10 | import struct |
Simon Glass | e6d85ff | 2019-05-14 15:53:47 -0600 | [diff] [blame] | 11 | import sys |
Simon Glass | 1f1864b | 2016-07-25 18:59:08 -0600 | [diff] [blame] | 12 | import tempfile |
Simon Glass | 8ea6d23 | 2022-01-09 20:13:41 -0700 | [diff] [blame] | 13 | import urllib.request |
Simon Glass | 1f1864b | 2016-07-25 18:59:08 -0600 | [diff] [blame] | 14 | |
Simon Glass | bf77667 | 2020-04-17 18:09:04 -0600 | [diff] [blame] | 15 | from patman import command |
| 16 | from patman import tout |
Simon Glass | 1f1864b | 2016-07-25 18:59:08 -0600 | [diff] [blame] | 17 | |
Simon Glass | aeffc5e | 2018-07-17 13:25:43 -0600 | [diff] [blame] | 18 | # Output directly (generally this is temporary) |
Simon Glass | 1f1864b | 2016-07-25 18:59:08 -0600 | [diff] [blame] | 19 | outdir = None |
Simon Glass | aeffc5e | 2018-07-17 13:25:43 -0600 | [diff] [blame] | 20 | |
| 21 | # True to keep the output directory around after exiting |
Simon Glass | 1f1864b | 2016-07-25 18:59:08 -0600 | [diff] [blame] | 22 | preserve_outdir = False |
| 23 | |
Simon Glass | aeffc5e | 2018-07-17 13:25:43 -0600 | [diff] [blame] | 24 | # Path to the Chrome OS chroot, if we know it |
| 25 | chroot_path = None |
| 26 | |
| 27 | # Search paths to use for Filename(), used to find files |
| 28 | search_paths = [] |
| 29 | |
Simon Glass | c22b8cf | 2019-07-08 13:18:27 -0600 | [diff] [blame] | 30 | tool_search_paths = [] |
| 31 | |
Simon Glass | 04187a8 | 2018-09-14 04:57:25 -0600 | [diff] [blame] | 32 | # Tools and the packages that contain them, on debian |
| 33 | packages = { |
| 34 | 'lz4': 'liblz4-tool', |
| 35 | } |
Simon Glass | aeffc5e | 2018-07-17 13:25:43 -0600 | [diff] [blame] | 36 | |
Simon Glass | 1fda182 | 2018-10-01 21:12:44 -0600 | [diff] [blame] | 37 | # List of paths to use when looking for an input file |
| 38 | indir = [] |
| 39 | |
Simon Glass | 1f1864b | 2016-07-25 18:59:08 -0600 | [diff] [blame] | 40 | def PrepareOutputDir(dirname, preserve=False): |
| 41 | """Select an output directory, ensuring it exists. |
| 42 | |
| 43 | This either creates a temporary directory or checks that the one supplied |
| 44 | by the user is valid. For a temporary directory, it makes a note to |
| 45 | remove it later if required. |
| 46 | |
| 47 | Args: |
| 48 | dirname: a string, name of the output directory to use to store |
| 49 | intermediate and output files. If is None - create a temporary |
| 50 | directory. |
| 51 | preserve: a Boolean. If outdir above is None and preserve is False, the |
| 52 | created temporary directory will be destroyed on exit. |
| 53 | |
| 54 | Raises: |
| 55 | OSError: If it cannot create the output directory. |
| 56 | """ |
| 57 | global outdir, preserve_outdir |
| 58 | |
| 59 | preserve_outdir = dirname or preserve |
| 60 | if dirname: |
| 61 | outdir = dirname |
| 62 | if not os.path.isdir(outdir): |
| 63 | try: |
| 64 | os.makedirs(outdir) |
| 65 | except OSError as err: |
| 66 | raise CmdError("Cannot make output directory '%s': '%s'" % |
| 67 | (outdir, err.strerror)) |
| 68 | tout.Debug("Using output directory '%s'" % outdir) |
| 69 | else: |
| 70 | outdir = tempfile.mkdtemp(prefix='binman.') |
| 71 | tout.Debug("Using temporary directory '%s'" % outdir) |
| 72 | |
| 73 | def _RemoveOutputDir(): |
| 74 | global outdir |
| 75 | |
| 76 | shutil.rmtree(outdir) |
| 77 | tout.Debug("Deleted temporary directory '%s'" % outdir) |
| 78 | outdir = None |
| 79 | |
| 80 | def FinaliseOutputDir(): |
| 81 | global outdir, preserve_outdir |
| 82 | |
| 83 | """Tidy up: delete output directory if temporary and not preserved.""" |
| 84 | if outdir and not preserve_outdir: |
| 85 | _RemoveOutputDir() |
Simon Glass | 3135330 | 2019-07-20 12:24:07 -0600 | [diff] [blame] | 86 | outdir = None |
Simon Glass | 1f1864b | 2016-07-25 18:59:08 -0600 | [diff] [blame] | 87 | |
| 88 | def GetOutputFilename(fname): |
| 89 | """Return a filename within the output directory. |
| 90 | |
| 91 | Args: |
| 92 | fname: Filename to use for new file |
| 93 | |
| 94 | Returns: |
| 95 | The full path of the filename, within the output directory |
| 96 | """ |
| 97 | return os.path.join(outdir, fname) |
| 98 | |
Simon Glass | 10cbd3b | 2020-12-28 20:34:52 -0700 | [diff] [blame] | 99 | def GetOutputDir(): |
| 100 | """Return the current output directory |
| 101 | |
| 102 | Returns: |
| 103 | str: The output directory |
| 104 | """ |
| 105 | return outdir |
| 106 | |
Simon Glass | 1f1864b | 2016-07-25 18:59:08 -0600 | [diff] [blame] | 107 | def _FinaliseForTest(): |
| 108 | """Remove the output directory (for use by tests)""" |
| 109 | global outdir |
| 110 | |
| 111 | if outdir: |
| 112 | _RemoveOutputDir() |
Simon Glass | 3135330 | 2019-07-20 12:24:07 -0600 | [diff] [blame] | 113 | outdir = None |
Simon Glass | 1f1864b | 2016-07-25 18:59:08 -0600 | [diff] [blame] | 114 | |
| 115 | def SetInputDirs(dirname): |
| 116 | """Add a list of input directories, where input files are kept. |
| 117 | |
| 118 | Args: |
| 119 | dirname: a list of paths to input directories to use for obtaining |
| 120 | files needed by binman to place in the image. |
| 121 | """ |
| 122 | global indir |
| 123 | |
| 124 | indir = dirname |
| 125 | tout.Debug("Using input directories %s" % indir) |
| 126 | |
Simon Glass | 4f9f105 | 2020-07-09 18:39:38 -0600 | [diff] [blame] | 127 | def GetInputFilename(fname, allow_missing=False): |
Simon Glass | 1f1864b | 2016-07-25 18:59:08 -0600 | [diff] [blame] | 128 | """Return a filename for use as input. |
| 129 | |
| 130 | Args: |
| 131 | fname: Filename to use for new file |
Simon Glass | 4f9f105 | 2020-07-09 18:39:38 -0600 | [diff] [blame] | 132 | allow_missing: True if the filename can be missing |
Simon Glass | 1f1864b | 2016-07-25 18:59:08 -0600 | [diff] [blame] | 133 | |
| 134 | Returns: |
Simon Glass | 5187b80 | 2021-03-18 20:25:03 +1300 | [diff] [blame] | 135 | fname, if indir is None; |
| 136 | full path of the filename, within the input directory; |
| 137 | None, if file is missing and allow_missing is True |
| 138 | |
| 139 | Raises: |
| 140 | ValueError if file is missing and allow_missing is False |
Simon Glass | 1f1864b | 2016-07-25 18:59:08 -0600 | [diff] [blame] | 141 | """ |
Simon Glass | f514d8f | 2019-08-24 07:22:54 -0600 | [diff] [blame] | 142 | if not indir or fname[:1] == '/': |
Simon Glass | 1f1864b | 2016-07-25 18:59:08 -0600 | [diff] [blame] | 143 | return fname |
| 144 | for dirname in indir: |
| 145 | pathname = os.path.join(dirname, fname) |
| 146 | if os.path.exists(pathname): |
| 147 | return pathname |
| 148 | |
Simon Glass | 4f9f105 | 2020-07-09 18:39:38 -0600 | [diff] [blame] | 149 | if allow_missing: |
| 150 | return None |
Simon Glass | 4f5dea4 | 2018-07-17 13:25:45 -0600 | [diff] [blame] | 151 | raise ValueError("Filename '%s' not found in input path (%s) (cwd='%s')" % |
| 152 | (fname, ','.join(indir), os.getcwd())) |
Simon Glass | 1f1864b | 2016-07-25 18:59:08 -0600 | [diff] [blame] | 153 | |
Simon Glass | 0a98b28 | 2018-09-14 04:57:28 -0600 | [diff] [blame] | 154 | def GetInputFilenameGlob(pattern): |
| 155 | """Return a list of filenames for use as input. |
| 156 | |
| 157 | Args: |
| 158 | pattern: Filename pattern to search for |
| 159 | |
| 160 | Returns: |
| 161 | A list of matching files in all input directories |
| 162 | """ |
| 163 | if not indir: |
| 164 | return glob.glob(fname) |
| 165 | files = [] |
| 166 | for dirname in indir: |
| 167 | pathname = os.path.join(dirname, pattern) |
| 168 | files += glob.glob(pathname) |
| 169 | return sorted(files) |
| 170 | |
Simon Glass | 1f1864b | 2016-07-25 18:59:08 -0600 | [diff] [blame] | 171 | def Align(pos, align): |
| 172 | if align: |
| 173 | mask = align - 1 |
| 174 | pos = (pos + mask) & ~mask |
| 175 | return pos |
| 176 | |
| 177 | def NotPowerOfTwo(num): |
| 178 | return num and (num & (num - 1)) |
Simon Glass | aeffc5e | 2018-07-17 13:25:43 -0600 | [diff] [blame] | 179 | |
Simon Glass | c22b8cf | 2019-07-08 13:18:27 -0600 | [diff] [blame] | 180 | def SetToolPaths(toolpaths): |
| 181 | """Set the path to search for tools |
| 182 | |
| 183 | Args: |
| 184 | toolpaths: List of paths to search for tools executed by Run() |
| 185 | """ |
| 186 | global tool_search_paths |
| 187 | |
| 188 | tool_search_paths = toolpaths |
| 189 | |
| 190 | def PathHasFile(path_spec, fname): |
Simon Glass | 04187a8 | 2018-09-14 04:57:25 -0600 | [diff] [blame] | 191 | """Check if a given filename is in the PATH |
| 192 | |
| 193 | Args: |
Simon Glass | c22b8cf | 2019-07-08 13:18:27 -0600 | [diff] [blame] | 194 | path_spec: Value of PATH variable to check |
Simon Glass | 04187a8 | 2018-09-14 04:57:25 -0600 | [diff] [blame] | 195 | fname: Filename to check |
| 196 | |
| 197 | Returns: |
| 198 | True if found, False if not |
| 199 | """ |
Simon Glass | c22b8cf | 2019-07-08 13:18:27 -0600 | [diff] [blame] | 200 | for dir in path_spec.split(':'): |
Simon Glass | 04187a8 | 2018-09-14 04:57:25 -0600 | [diff] [blame] | 201 | if os.path.exists(os.path.join(dir, fname)): |
| 202 | return True |
| 203 | return False |
| 204 | |
Alper Nebi Yasak | 29cc091 | 2020-09-06 14:46:06 +0300 | [diff] [blame] | 205 | def GetHostCompileTool(name): |
| 206 | """Get the host-specific version for a compile tool |
| 207 | |
| 208 | This checks the environment variables that specify which version of |
| 209 | the tool should be used (e.g. ${HOSTCC}). |
| 210 | |
| 211 | The following table lists the host-specific versions of the tools |
| 212 | this function resolves to: |
| 213 | |
| 214 | Compile Tool | Host version |
| 215 | --------------+---------------- |
| 216 | as | ${HOSTAS} |
| 217 | ld | ${HOSTLD} |
| 218 | cc | ${HOSTCC} |
| 219 | cpp | ${HOSTCPP} |
| 220 | c++ | ${HOSTCXX} |
| 221 | ar | ${HOSTAR} |
| 222 | nm | ${HOSTNM} |
| 223 | ldr | ${HOSTLDR} |
| 224 | strip | ${HOSTSTRIP} |
| 225 | objcopy | ${HOSTOBJCOPY} |
| 226 | objdump | ${HOSTOBJDUMP} |
| 227 | dtc | ${HOSTDTC} |
| 228 | |
| 229 | Args: |
| 230 | name: Command name to run |
| 231 | |
| 232 | Returns: |
| 233 | host_name: Exact command name to run instead |
| 234 | extra_args: List of extra arguments to pass |
| 235 | """ |
| 236 | host_name = None |
| 237 | extra_args = [] |
| 238 | if name in ('as', 'ld', 'cc', 'cpp', 'ar', 'nm', 'ldr', 'strip', |
| 239 | 'objcopy', 'objdump', 'dtc'): |
| 240 | host_name, *host_args = env.get('HOST' + name.upper(), '').split(' ') |
| 241 | elif name == 'c++': |
| 242 | host_name, *host_args = env.get('HOSTCXX', '').split(' ') |
| 243 | |
| 244 | if host_name: |
| 245 | return host_name, extra_args |
| 246 | return name, [] |
| 247 | |
Alper Nebi Yasak | 1e4687a | 2020-09-06 14:46:05 +0300 | [diff] [blame] | 248 | def GetTargetCompileTool(name, cross_compile=None): |
| 249 | """Get the target-specific version for a compile tool |
| 250 | |
| 251 | This first checks the environment variables that specify which |
| 252 | version of the tool should be used (e.g. ${CC}). If those aren't |
| 253 | specified, it checks the CROSS_COMPILE variable as a prefix for the |
| 254 | tool with some substitutions (e.g. "${CROSS_COMPILE}gcc" for cc). |
| 255 | |
| 256 | The following table lists the target-specific versions of the tools |
| 257 | this function resolves to: |
| 258 | |
| 259 | Compile Tool | First choice | Second choice |
| 260 | --------------+----------------+---------------------------- |
| 261 | as | ${AS} | ${CROSS_COMPILE}as |
| 262 | ld | ${LD} | ${CROSS_COMPILE}ld.bfd |
| 263 | | | or ${CROSS_COMPILE}ld |
| 264 | cc | ${CC} | ${CROSS_COMPILE}gcc |
| 265 | cpp | ${CPP} | ${CROSS_COMPILE}gcc -E |
| 266 | c++ | ${CXX} | ${CROSS_COMPILE}g++ |
| 267 | ar | ${AR} | ${CROSS_COMPILE}ar |
| 268 | nm | ${NM} | ${CROSS_COMPILE}nm |
| 269 | ldr | ${LDR} | ${CROSS_COMPILE}ldr |
| 270 | strip | ${STRIP} | ${CROSS_COMPILE}strip |
| 271 | objcopy | ${OBJCOPY} | ${CROSS_COMPILE}objcopy |
| 272 | objdump | ${OBJDUMP} | ${CROSS_COMPILE}objdump |
| 273 | dtc | ${DTC} | (no CROSS_COMPILE version) |
| 274 | |
| 275 | Args: |
| 276 | name: Command name to run |
| 277 | |
| 278 | Returns: |
| 279 | target_name: Exact command name to run instead |
| 280 | extra_args: List of extra arguments to pass |
| 281 | """ |
| 282 | env = dict(os.environ) |
| 283 | |
| 284 | target_name = None |
| 285 | extra_args = [] |
| 286 | if name in ('as', 'ld', 'cc', 'cpp', 'ar', 'nm', 'ldr', 'strip', |
| 287 | 'objcopy', 'objdump', 'dtc'): |
| 288 | target_name, *extra_args = env.get(name.upper(), '').split(' ') |
| 289 | elif name == 'c++': |
| 290 | target_name, *extra_args = env.get('CXX', '').split(' ') |
| 291 | |
| 292 | if target_name: |
| 293 | return target_name, extra_args |
| 294 | |
| 295 | if cross_compile is None: |
| 296 | cross_compile = env.get('CROSS_COMPILE', '') |
Alper Nebi Yasak | 1e4687a | 2020-09-06 14:46:05 +0300 | [diff] [blame] | 297 | |
| 298 | if name in ('as', 'ar', 'nm', 'ldr', 'strip', 'objcopy', 'objdump'): |
| 299 | target_name = cross_compile + name |
| 300 | elif name == 'ld': |
| 301 | try: |
| 302 | if Run(cross_compile + 'ld.bfd', '-v'): |
| 303 | target_name = cross_compile + 'ld.bfd' |
| 304 | except: |
| 305 | target_name = cross_compile + 'ld' |
| 306 | elif name == 'cc': |
| 307 | target_name = cross_compile + 'gcc' |
| 308 | elif name == 'cpp': |
| 309 | target_name = cross_compile + 'gcc' |
| 310 | extra_args = ['-E'] |
| 311 | elif name == 'c++': |
| 312 | target_name = cross_compile + 'g++' |
| 313 | else: |
| 314 | target_name = name |
| 315 | return target_name, extra_args |
| 316 | |
Simon Glass | ade5327 | 2022-01-09 20:13:40 -0700 | [diff] [blame] | 317 | def get_env_with_path(): |
| 318 | """Get an updated environment with the PATH variable set correctly |
| 319 | |
| 320 | If there are any search paths set, these need to come first in the PATH so |
| 321 | that these override any other version of the tools. |
| 322 | |
| 323 | Returns: |
| 324 | dict: New environment with PATH updated, or None if there are not search |
| 325 | paths |
| 326 | """ |
| 327 | if tool_search_paths: |
| 328 | env = dict(os.environ) |
| 329 | env['PATH'] = ':'.join(tool_search_paths) + ':' + env['PATH'] |
| 330 | return env |
| 331 | |
| 332 | def run_result(name, *args, **kwargs): |
| 333 | """Run a tool with some arguments |
| 334 | |
| 335 | This runs a 'tool', which is a program used by binman to process files and |
| 336 | perhaps produce some output. Tools can be located on the PATH or in a |
| 337 | search path. |
| 338 | |
| 339 | Args: |
| 340 | name: Command name to run |
| 341 | args: Arguments to the tool |
| 342 | for_host: True to resolve the command to the version for the host |
| 343 | for_target: False to run the command as-is, without resolving it |
| 344 | to the version for the compile target |
| 345 | raise_on_error: Raise an error if the command fails (True by default) |
| 346 | |
| 347 | Returns: |
| 348 | CommandResult object |
| 349 | """ |
| 350 | try: |
| 351 | binary = kwargs.get('binary') |
| 352 | for_host = kwargs.get('for_host', False) |
| 353 | for_target = kwargs.get('for_target', not for_host) |
| 354 | raise_on_error = kwargs.get('raise_on_error', True) |
| 355 | env = get_env_with_path() |
| 356 | if for_target: |
| 357 | name, extra_args = GetTargetCompileTool(name) |
| 358 | args = tuple(extra_args) + args |
| 359 | elif for_host: |
| 360 | name, extra_args = GetHostCompileTool(name) |
| 361 | args = tuple(extra_args) + args |
| 362 | name = os.path.expanduser(name) # Expand paths containing ~ |
| 363 | all_args = (name,) + args |
| 364 | result = command.RunPipe([all_args], capture=True, capture_stderr=True, |
| 365 | env=env, raise_on_error=False, binary=binary) |
| 366 | if result.return_code: |
| 367 | if raise_on_error: |
| 368 | raise ValueError("Error %d running '%s': %s" % |
| 369 | (result.return_code,' '.join(all_args), |
| 370 | result.stderr or result.stdout)) |
| 371 | return result |
| 372 | except ValueError: |
| 373 | if env and not PathHasFile(env['PATH'], name): |
| 374 | msg = "Please install tool '%s'" % name |
| 375 | package = packages.get(name) |
| 376 | if package: |
| 377 | msg += " (e.g. from package '%s')" % package |
| 378 | raise ValueError(msg) |
| 379 | raise |
| 380 | |
Simon Glass | 3b3e3c0 | 2019-10-31 07:42:50 -0600 | [diff] [blame] | 381 | def Run(name, *args, **kwargs): |
Simon Glass | c22b8cf | 2019-07-08 13:18:27 -0600 | [diff] [blame] | 382 | """Run a tool with some arguments |
| 383 | |
| 384 | This runs a 'tool', which is a program used by binman to process files and |
| 385 | perhaps produce some output. Tools can be located on the PATH or in a |
| 386 | search path. |
| 387 | |
| 388 | Args: |
| 389 | name: Command name to run |
| 390 | args: Arguments to the tool |
Alper Nebi Yasak | 29cc091 | 2020-09-06 14:46:06 +0300 | [diff] [blame] | 391 | for_host: True to resolve the command to the version for the host |
Alper Nebi Yasak | 1e4687a | 2020-09-06 14:46:05 +0300 | [diff] [blame] | 392 | for_target: False to run the command as-is, without resolving it |
| 393 | to the version for the compile target |
Simon Glass | c22b8cf | 2019-07-08 13:18:27 -0600 | [diff] [blame] | 394 | |
| 395 | Returns: |
| 396 | CommandResult object |
| 397 | """ |
Simon Glass | ade5327 | 2022-01-09 20:13:40 -0700 | [diff] [blame] | 398 | result = run_result(name, *args, **kwargs) |
| 399 | if result is not None: |
Simon Glass | 6eace39 | 2019-08-24 07:22:42 -0600 | [diff] [blame] | 400 | return result.stdout |
Simon Glass | aeffc5e | 2018-07-17 13:25:43 -0600 | [diff] [blame] | 401 | |
| 402 | def Filename(fname): |
| 403 | """Resolve a file path to an absolute path. |
| 404 | |
| 405 | If fname starts with ##/ and chroot is available, ##/ gets replaced with |
| 406 | the chroot path. If chroot is not available, this file name can not be |
| 407 | resolved, `None' is returned. |
| 408 | |
| 409 | If fname is not prepended with the above prefix, and is not an existing |
| 410 | file, the actual file name is retrieved from the passed in string and the |
| 411 | search_paths directories (if any) are searched to for the file. If found - |
| 412 | the path to the found file is returned, `None' is returned otherwise. |
| 413 | |
| 414 | Args: |
| 415 | fname: a string, the path to resolve. |
| 416 | |
| 417 | Returns: |
| 418 | Absolute path to the file or None if not found. |
| 419 | """ |
| 420 | if fname.startswith('##/'): |
| 421 | if chroot_path: |
| 422 | fname = os.path.join(chroot_path, fname[3:]) |
| 423 | else: |
| 424 | return None |
| 425 | |
| 426 | # Search for a pathname that exists, and return it if found |
| 427 | if fname and not os.path.exists(fname): |
| 428 | for path in search_paths: |
| 429 | pathname = os.path.join(path, os.path.basename(fname)) |
| 430 | if os.path.exists(pathname): |
| 431 | return pathname |
| 432 | |
| 433 | # If not found, just return the standard, unchanged path |
| 434 | return fname |
| 435 | |
Simon Glass | 3c47e41 | 2019-05-17 22:00:44 -0600 | [diff] [blame] | 436 | def ReadFile(fname, binary=True): |
Simon Glass | aeffc5e | 2018-07-17 13:25:43 -0600 | [diff] [blame] | 437 | """Read and return the contents of a file. |
| 438 | |
| 439 | Args: |
| 440 | fname: path to filename to read, where ## signifiies the chroot. |
| 441 | |
| 442 | Returns: |
| 443 | data read from file, as a string. |
| 444 | """ |
Simon Glass | 3c47e41 | 2019-05-17 22:00:44 -0600 | [diff] [blame] | 445 | with open(Filename(fname), binary and 'rb' or 'r') as fd: |
Simon Glass | aeffc5e | 2018-07-17 13:25:43 -0600 | [diff] [blame] | 446 | data = fd.read() |
| 447 | #self._out.Info("Read file '%s' size %d (%#0x)" % |
| 448 | #(fname, len(data), len(data))) |
| 449 | return data |
| 450 | |
Simon Glass | fd70986 | 2020-07-05 21:41:50 -0600 | [diff] [blame] | 451 | def WriteFile(fname, data, binary=True): |
Simon Glass | aeffc5e | 2018-07-17 13:25:43 -0600 | [diff] [blame] | 452 | """Write data into a file. |
| 453 | |
| 454 | Args: |
| 455 | fname: path to filename to write |
| 456 | data: data to write to file, as a string |
| 457 | """ |
| 458 | #self._out.Info("Write file '%s' size %d (%#0x)" % |
| 459 | #(fname, len(data), len(data))) |
Simon Glass | fd70986 | 2020-07-05 21:41:50 -0600 | [diff] [blame] | 460 | with open(Filename(fname), binary and 'wb' or 'w') as fd: |
Simon Glass | aeffc5e | 2018-07-17 13:25:43 -0600 | [diff] [blame] | 461 | fd.write(data) |
Simon Glass | e6d85ff | 2019-05-14 15:53:47 -0600 | [diff] [blame] | 462 | |
| 463 | def GetBytes(byte, size): |
| 464 | """Get a string of bytes of a given size |
| 465 | |
Simon Glass | e6d85ff | 2019-05-14 15:53:47 -0600 | [diff] [blame] | 466 | Args: |
| 467 | byte: Numeric byte value to use |
| 468 | size: Size of bytes/string to return |
| 469 | |
| 470 | Returns: |
| 471 | A bytes type with 'byte' repeated 'size' times |
| 472 | """ |
Simon Glass | fc0056e | 2020-11-08 20:36:18 -0700 | [diff] [blame] | 473 | return bytes([byte]) * size |
Simon Glass | 2b6ed5e | 2019-05-17 22:00:35 -0600 | [diff] [blame] | 474 | |
Simon Glass | f6b6481 | 2019-05-17 22:00:36 -0600 | [diff] [blame] | 475 | def ToBytes(string): |
| 476 | """Convert a str type into a bytes type |
| 477 | |
| 478 | Args: |
Simon Glass | 3b3e3c0 | 2019-10-31 07:42:50 -0600 | [diff] [blame] | 479 | string: string to convert |
Simon Glass | f6b6481 | 2019-05-17 22:00:36 -0600 | [diff] [blame] | 480 | |
| 481 | Returns: |
Simon Glass | fc0056e | 2020-11-08 20:36:18 -0700 | [diff] [blame] | 482 | A bytes type |
Simon Glass | f6b6481 | 2019-05-17 22:00:36 -0600 | [diff] [blame] | 483 | """ |
Simon Glass | fc0056e | 2020-11-08 20:36:18 -0700 | [diff] [blame] | 484 | return string.encode('utf-8') |
Simon Glass | 07d9e70 | 2019-07-08 13:18:41 -0600 | [diff] [blame] | 485 | |
Simon Glass | 3b3e3c0 | 2019-10-31 07:42:50 -0600 | [diff] [blame] | 486 | def ToString(bval): |
| 487 | """Convert a bytes type into a str type |
| 488 | |
| 489 | Args: |
| 490 | bval: bytes value to convert |
| 491 | |
| 492 | Returns: |
| 493 | Python 3: A bytes type |
| 494 | Python 2: A string type |
| 495 | """ |
| 496 | return bval.decode('utf-8') |
| 497 | |
Simon Glass | eb0f4a4 | 2019-07-20 12:24:06 -0600 | [diff] [blame] | 498 | def Compress(indata, algo, with_header=True): |
Simon Glass | 07d9e70 | 2019-07-08 13:18:41 -0600 | [diff] [blame] | 499 | """Compress some data using a given algorithm |
| 500 | |
| 501 | Note that for lzma this uses an old version of the algorithm, not that |
| 502 | provided by xz. |
| 503 | |
| 504 | This requires 'lz4' and 'lzma_alone' tools. It also requires an output |
| 505 | directory to be previously set up, by calling PrepareOutputDir(). |
| 506 | |
Simon Glass | 650ead1 | 2021-07-06 10:36:36 -0600 | [diff] [blame] | 507 | Care is taken to use unique temporary files so that this function can be |
| 508 | called from multiple threads. |
| 509 | |
Simon Glass | 07d9e70 | 2019-07-08 13:18:41 -0600 | [diff] [blame] | 510 | Args: |
| 511 | indata: Input data to compress |
| 512 | algo: Algorithm to use ('none', 'gzip', 'lz4' or 'lzma') |
| 513 | |
| 514 | Returns: |
| 515 | Compressed data |
| 516 | """ |
| 517 | if algo == 'none': |
| 518 | return indata |
Simon Glass | 650ead1 | 2021-07-06 10:36:36 -0600 | [diff] [blame] | 519 | fname = tempfile.NamedTemporaryFile(prefix='%s.comp.tmp' % algo, |
| 520 | dir=outdir).name |
Simon Glass | 07d9e70 | 2019-07-08 13:18:41 -0600 | [diff] [blame] | 521 | WriteFile(fname, indata) |
| 522 | if algo == 'lz4': |
Simon Glass | 6deff87 | 2021-01-06 21:35:11 -0700 | [diff] [blame] | 523 | data = Run('lz4', '--no-frame-crc', '-B4', '-5', '-c', fname, |
| 524 | binary=True) |
Simon Glass | 07d9e70 | 2019-07-08 13:18:41 -0600 | [diff] [blame] | 525 | # cbfstool uses a very old version of lzma |
| 526 | elif algo == 'lzma': |
Simon Glass | 650ead1 | 2021-07-06 10:36:36 -0600 | [diff] [blame] | 527 | outfname = tempfile.NamedTemporaryFile(prefix='%s.comp.otmp' % algo, |
| 528 | dir=outdir).name |
Simon Glass | 07d9e70 | 2019-07-08 13:18:41 -0600 | [diff] [blame] | 529 | Run('lzma_alone', 'e', fname, outfname, '-lc1', '-lp0', '-pb0', '-d8') |
| 530 | data = ReadFile(outfname) |
| 531 | elif algo == 'gzip': |
Simon Glass | 3b3e3c0 | 2019-10-31 07:42:50 -0600 | [diff] [blame] | 532 | data = Run('gzip', '-c', fname, binary=True) |
Simon Glass | 07d9e70 | 2019-07-08 13:18:41 -0600 | [diff] [blame] | 533 | else: |
| 534 | raise ValueError("Unknown algorithm '%s'" % algo) |
Simon Glass | eb0f4a4 | 2019-07-20 12:24:06 -0600 | [diff] [blame] | 535 | if with_header: |
| 536 | hdr = struct.pack('<I', len(data)) |
| 537 | data = hdr + data |
Simon Glass | 07d9e70 | 2019-07-08 13:18:41 -0600 | [diff] [blame] | 538 | return data |
| 539 | |
Simon Glass | eb0f4a4 | 2019-07-20 12:24:06 -0600 | [diff] [blame] | 540 | def Decompress(indata, algo, with_header=True): |
Simon Glass | 07d9e70 | 2019-07-08 13:18:41 -0600 | [diff] [blame] | 541 | """Decompress some data using a given algorithm |
| 542 | |
| 543 | Note that for lzma this uses an old version of the algorithm, not that |
| 544 | provided by xz. |
| 545 | |
| 546 | This requires 'lz4' and 'lzma_alone' tools. It also requires an output |
| 547 | directory to be previously set up, by calling PrepareOutputDir(). |
| 548 | |
| 549 | Args: |
| 550 | indata: Input data to decompress |
| 551 | algo: Algorithm to use ('none', 'gzip', 'lz4' or 'lzma') |
| 552 | |
| 553 | Returns: |
| 554 | Compressed data |
| 555 | """ |
| 556 | if algo == 'none': |
| 557 | return indata |
Simon Glass | eb0f4a4 | 2019-07-20 12:24:06 -0600 | [diff] [blame] | 558 | if with_header: |
| 559 | data_len = struct.unpack('<I', indata[:4])[0] |
| 560 | indata = indata[4:4 + data_len] |
Simon Glass | 07d9e70 | 2019-07-08 13:18:41 -0600 | [diff] [blame] | 561 | fname = GetOutputFilename('%s.decomp.tmp' % algo) |
| 562 | with open(fname, 'wb') as fd: |
| 563 | fd.write(indata) |
| 564 | if algo == 'lz4': |
Simon Glass | 3b3e3c0 | 2019-10-31 07:42:50 -0600 | [diff] [blame] | 565 | data = Run('lz4', '-dc', fname, binary=True) |
Simon Glass | 07d9e70 | 2019-07-08 13:18:41 -0600 | [diff] [blame] | 566 | elif algo == 'lzma': |
| 567 | outfname = GetOutputFilename('%s.decomp.otmp' % algo) |
| 568 | Run('lzma_alone', 'd', fname, outfname) |
Simon Glass | 3b3e3c0 | 2019-10-31 07:42:50 -0600 | [diff] [blame] | 569 | data = ReadFile(outfname, binary=True) |
Simon Glass | 07d9e70 | 2019-07-08 13:18:41 -0600 | [diff] [blame] | 570 | elif algo == 'gzip': |
Simon Glass | 3b3e3c0 | 2019-10-31 07:42:50 -0600 | [diff] [blame] | 571 | data = Run('gzip', '-cd', fname, binary=True) |
Simon Glass | 07d9e70 | 2019-07-08 13:18:41 -0600 | [diff] [blame] | 572 | else: |
| 573 | raise ValueError("Unknown algorithm '%s'" % algo) |
| 574 | return data |
Simon Glass | 1cfdfc0 | 2019-07-08 13:18:51 -0600 | [diff] [blame] | 575 | |
| 576 | CMD_CREATE, CMD_DELETE, CMD_ADD, CMD_REPLACE, CMD_EXTRACT = range(5) |
| 577 | |
| 578 | IFWITOOL_CMDS = { |
| 579 | CMD_CREATE: 'create', |
| 580 | CMD_DELETE: 'delete', |
| 581 | CMD_ADD: 'add', |
| 582 | CMD_REPLACE: 'replace', |
| 583 | CMD_EXTRACT: 'extract', |
| 584 | } |
| 585 | |
| 586 | def RunIfwiTool(ifwi_file, cmd, fname=None, subpart=None, entry_name=None): |
| 587 | """Run ifwitool with the given arguments: |
| 588 | |
| 589 | Args: |
| 590 | ifwi_file: IFWI file to operation on |
| 591 | cmd: Command to execute (CMD_...) |
| 592 | fname: Filename of file to add/replace/extract/create (None for |
| 593 | CMD_DELETE) |
| 594 | subpart: Name of sub-partition to operation on (None for CMD_CREATE) |
| 595 | entry_name: Name of directory entry to operate on, or None if none |
| 596 | """ |
| 597 | args = ['ifwitool', ifwi_file] |
| 598 | args.append(IFWITOOL_CMDS[cmd]) |
| 599 | if fname: |
| 600 | args += ['-f', fname] |
| 601 | if subpart: |
| 602 | args += ['-n', subpart] |
| 603 | if entry_name: |
| 604 | args += ['-d', '-e', entry_name] |
| 605 | Run(*args) |
Simon Glass | 9f297b0 | 2019-07-20 12:23:36 -0600 | [diff] [blame] | 606 | |
| 607 | def ToHex(val): |
| 608 | """Convert an integer value (or None) to a string |
| 609 | |
| 610 | Returns: |
| 611 | hex value, or 'None' if the value is None |
| 612 | """ |
| 613 | return 'None' if val is None else '%#x' % val |
| 614 | |
| 615 | def ToHexSize(val): |
| 616 | """Return the size of an object in hex |
| 617 | |
| 618 | Returns: |
| 619 | hex value of size, or 'None' if the value is None |
| 620 | """ |
| 621 | return 'None' if val is None else '%#x' % len(val) |
Paul Barker | 5fe50f9 | 2021-09-08 12:38:01 +0100 | [diff] [blame] | 622 | |
| 623 | def PrintFullHelp(fname): |
| 624 | """Print the full help message for a tool using an appropriate pager. |
| 625 | |
| 626 | Args: |
| 627 | fname: Path to a file containing the full help message |
| 628 | """ |
Paul Barker | 0d60e5d | 2021-09-08 12:38:02 +0100 | [diff] [blame] | 629 | pager = shlex.split(os.getenv('PAGER', '')) |
Paul Barker | 5fe50f9 | 2021-09-08 12:38:01 +0100 | [diff] [blame] | 630 | if not pager: |
Paul Barker | 0d60e5d | 2021-09-08 12:38:02 +0100 | [diff] [blame] | 631 | lesspath = shutil.which('less') |
| 632 | pager = [lesspath] if lesspath else None |
Paul Barker | 5fe50f9 | 2021-09-08 12:38:01 +0100 | [diff] [blame] | 633 | if not pager: |
Paul Barker | 0d60e5d | 2021-09-08 12:38:02 +0100 | [diff] [blame] | 634 | pager = ['more'] |
| 635 | command.Run(*pager, fname) |
Simon Glass | 8ea6d23 | 2022-01-09 20:13:41 -0700 | [diff] [blame] | 636 | |
Simon Glass | 5b79686 | 2022-01-09 20:13:42 -0700 | [diff] [blame^] | 637 | def Download(url, tmpdir_pattern='.patman'): |
Simon Glass | 8ea6d23 | 2022-01-09 20:13:41 -0700 | [diff] [blame] | 638 | """Download a file to a temporary directory |
| 639 | |
| 640 | Args: |
Simon Glass | 5b79686 | 2022-01-09 20:13:42 -0700 | [diff] [blame^] | 641 | url (str): URL to download |
| 642 | tmpdir_pattern (str): pattern to use for the temporary directory |
| 643 | |
Simon Glass | 8ea6d23 | 2022-01-09 20:13:41 -0700 | [diff] [blame] | 644 | Returns: |
| 645 | Tuple: |
Simon Glass | 8ea6d23 | 2022-01-09 20:13:41 -0700 | [diff] [blame] | 646 | Full path to the downloaded archive file in that directory, |
| 647 | or None if there was an error while downloading |
Simon Glass | 5b79686 | 2022-01-09 20:13:42 -0700 | [diff] [blame^] | 648 | Temporary directory name |
Simon Glass | 8ea6d23 | 2022-01-09 20:13:41 -0700 | [diff] [blame] | 649 | """ |
Simon Glass | 5b79686 | 2022-01-09 20:13:42 -0700 | [diff] [blame^] | 650 | print('- downloading: %s' % url) |
Simon Glass | 8ea6d23 | 2022-01-09 20:13:41 -0700 | [diff] [blame] | 651 | leaf = url.split('/')[-1] |
Simon Glass | 5b79686 | 2022-01-09 20:13:42 -0700 | [diff] [blame^] | 652 | tmpdir = tempfile.mkdtemp(tmpdir_pattern) |
Simon Glass | 8ea6d23 | 2022-01-09 20:13:41 -0700 | [diff] [blame] | 653 | response = urllib.request.urlopen(url) |
| 654 | fname = os.path.join(tmpdir, leaf) |
| 655 | fd = open(fname, 'wb') |
| 656 | meta = response.info() |
| 657 | size = int(meta.get('Content-Length')) |
| 658 | done = 0 |
| 659 | block_size = 1 << 16 |
| 660 | status = '' |
| 661 | |
| 662 | # Read the file in chunks and show progress as we go |
| 663 | while True: |
| 664 | buffer = response.read(block_size) |
| 665 | if not buffer: |
| 666 | print(chr(8) * (len(status) + 1), '\r', end=' ') |
| 667 | break |
| 668 | |
| 669 | done += len(buffer) |
| 670 | fd.write(buffer) |
| 671 | status = r'%10d MiB [%3d%%]' % (done // 1024 // 1024, |
| 672 | done * 100 // size) |
| 673 | status = status + chr(8) * (len(status) + 1) |
| 674 | print(status, end=' ') |
| 675 | sys.stdout.flush() |
Simon Glass | 5b79686 | 2022-01-09 20:13:42 -0700 | [diff] [blame^] | 676 | print('\r', end='') |
| 677 | sys.stdout.flush() |
Simon Glass | 8ea6d23 | 2022-01-09 20:13:41 -0700 | [diff] [blame] | 678 | fd.close() |
| 679 | if done != size: |
| 680 | print('Error, failed to download') |
| 681 | os.remove(fname) |
| 682 | fname = None |
Simon Glass | 5b79686 | 2022-01-09 20:13:42 -0700 | [diff] [blame^] | 683 | return fname, tmpdir |