blob: 8fda13ff012fe5b1aa6f36d9762d5c80b9246bfb [file] [log] [blame]
Simon Glass252de6b2022-01-09 20:13:49 -07001# SPDX-License-Identifier: GPL-2.0+
2# Copyright 2022 Google LLC
Stefan Herbrechtsmeier867eed12022-08-19 16:25:33 +02003# Copyright (C) 2022 Weidmüller Interface GmbH & Co. KG
4# Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com>
Simon Glass252de6b2022-01-09 20:13:49 -07005#
6"""Base class for all bintools
7
8This defines the common functionality for all bintools, including running
9the tool, checking its version and fetching it if needed.
10"""
11
12import collections
13import glob
14import importlib
15import multiprocessing
16import os
17import shutil
18import tempfile
19import urllib.error
20
21from patman import command
22from patman import terminal
23from patman import tools
24from patman import tout
25
26BINMAN_DIR = os.path.dirname(os.path.realpath(__file__))
27
28# Format string for listing bintools, see also the header in list_all()
29FORMAT = '%-16.16s %-12.12s %-26.26s %s'
30
31# List of known modules, to avoid importing the module multiple times
32modules = {}
33
34# Possible ways of fetching a tool (FETCH_COUNT is number of ways)
35FETCH_ANY, FETCH_BIN, FETCH_BUILD, FETCH_COUNT = range(4)
36
37FETCH_NAMES = {
38 FETCH_ANY: 'any method',
39 FETCH_BIN: 'binary download',
40 FETCH_BUILD: 'build from source'
41 }
42
43# Status of tool fetching
44FETCHED, FAIL, PRESENT, STATUS_COUNT = range(4)
45
46DOWNLOAD_DESTDIR = os.path.join(os.getenv('HOME'), 'bin')
47
48class Bintool:
49 """Tool which operates on binaries to help produce entry contents
50
51 This is the base class for all bintools
52 """
53 # List of bintools to regard as missing
54 missing_list = []
55
Quentin Schulze4408432022-09-01 17:51:40 +020056 def __init__(self, name, desc, version_regex=None, version_args='-V'):
Simon Glass252de6b2022-01-09 20:13:49 -070057 self.name = name
58 self.desc = desc
Quentin Schulz723a63e2022-09-01 17:51:37 +020059 self.version_regex = version_regex
Quentin Schulze4408432022-09-01 17:51:40 +020060 self.version_args = version_args
Simon Glass252de6b2022-01-09 20:13:49 -070061
62 @staticmethod
63 def find_bintool_class(btype):
64 """Look up the bintool class for bintool
65
66 Args:
67 byte: Bintool to use, e.g. 'mkimage'
68
69 Returns:
70 The bintool class object if found, else a tuple:
71 module name that could not be found
72 exception received
73 """
74 # Convert something like 'u-boot' to 'u_boot' since we are only
75 # interested in the type.
76 module_name = btype.replace('-', '_')
77 module = modules.get(module_name)
Stefan Herbrechtsmeier0f369d72022-08-19 16:25:35 +020078 class_name = f'Bintool{module_name}'
Simon Glass252de6b2022-01-09 20:13:49 -070079
80 # Import the module if we have not already done so
81 if not module:
82 try:
83 module = importlib.import_module('binman.btool.' + module_name)
84 except ImportError as exc:
Stefan Herbrechtsmeier0f369d72022-08-19 16:25:35 +020085 try:
86 # Deal with classes which must be renamed due to conflicts
87 # with Python libraries
Stefan Herbrechtsmeier0f369d72022-08-19 16:25:35 +020088 module = importlib.import_module('binman.btool.btool_' +
89 module_name)
90 except ImportError:
91 return module_name, exc
Simon Glass252de6b2022-01-09 20:13:49 -070092 modules[module_name] = module
93
94 # Look up the expected class name
Stefan Herbrechtsmeier0f369d72022-08-19 16:25:35 +020095 return getattr(module, class_name)
Simon Glass252de6b2022-01-09 20:13:49 -070096
97 @staticmethod
98 def create(name):
99 """Create a new bintool object
100
101 Args:
102 name (str): Bintool to create, e.g. 'mkimage'
103
104 Returns:
105 A new object of the correct type (a subclass of Binutil)
106 """
107 cls = Bintool.find_bintool_class(name)
108 if isinstance(cls, tuple):
109 raise ValueError("Cannot import bintool module '%s': %s" % cls)
110
111 # Call its constructor to get the object we want.
112 obj = cls(name)
113 return obj
114
115 def show(self):
116 """Show a line of information about a bintool"""
117 if self.is_present():
118 version = self.version()
119 else:
120 version = '-'
121 print(FORMAT % (self.name, version, self.desc,
122 self.get_path() or '(not found)'))
123
124 @classmethod
125 def set_missing_list(cls, missing_list):
126 cls.missing_list = missing_list or []
127
128 @staticmethod
129 def get_tool_list(include_testing=False):
130 """Get a list of the known tools
131
132 Returns:
133 list of str: names of all tools known to binman
134 """
135 files = glob.glob(os.path.join(BINMAN_DIR, 'btool/*'))
136 names = [os.path.splitext(os.path.basename(fname))[0]
137 for fname in files]
138 names = [name for name in names if name[0] != '_']
Quentin Schulz478332a2022-11-07 13:54:54 +0100139 names = [name[6:] if name.startswith('btool_') else name
140 for name in names]
Simon Glass252de6b2022-01-09 20:13:49 -0700141 if include_testing:
142 names.append('_testing')
143 return sorted(names)
144
145 @staticmethod
146 def list_all():
147 """List all the bintools known to binman"""
148 names = Bintool.get_tool_list()
149 print(FORMAT % ('Name', 'Version', 'Description', 'Path'))
150 print(FORMAT % ('-' * 15,'-' * 11, '-' * 25, '-' * 30))
151 for name in names:
152 btool = Bintool.create(name)
153 btool.show()
154
155 def is_present(self):
156 """Check if a bintool is available on the system
157
158 Returns:
159 bool: True if available, False if not
160 """
161 if self.name in self.missing_list:
162 return False
163 return bool(self.get_path())
164
165 def get_path(self):
166 """Get the path of a bintool
167
168 Returns:
169 str: Path to the tool, if available, else None
170 """
171 return tools.tool_find(self.name)
172
173 def fetch_tool(self, method, col, skip_present):
174 """Fetch a single tool
175
176 Args:
177 method (FETCH_...): Method to use
178 col (terminal.Color): Color terminal object
179 skip_present (boo;): Skip fetching if it is already present
180
181 Returns:
182 int: Result of fetch either FETCHED, FAIL, PRESENT
183 """
184 def try_fetch(meth):
185 res = None
186 try:
187 res = self.fetch(meth)
188 except urllib.error.URLError as uerr:
189 message = uerr.reason
Simon Glass252ac582022-01-29 14:14:17 -0700190 print(col.build(col.RED, f'- {message}'))
Simon Glass252de6b2022-01-09 20:13:49 -0700191
192 except ValueError as exc:
193 print(f'Exception: {exc}')
194 return res
195
196 if skip_present and self.is_present():
197 return PRESENT
Simon Glass252ac582022-01-29 14:14:17 -0700198 print(col.build(col.YELLOW, 'Fetch: %s' % self.name))
Simon Glass252de6b2022-01-09 20:13:49 -0700199 if method == FETCH_ANY:
200 for try_method in range(1, FETCH_COUNT):
201 print(f'- trying method: {FETCH_NAMES[try_method]}')
202 result = try_fetch(try_method)
203 if result:
204 break
205 else:
206 result = try_fetch(method)
207 if not result:
208 return FAIL
209 if result is not True:
210 fname, tmpdir = result
211 dest = os.path.join(DOWNLOAD_DESTDIR, self.name)
212 print(f"- writing to '{dest}'")
213 shutil.move(fname, dest)
214 if tmpdir:
215 shutil.rmtree(tmpdir)
216 return FETCHED
217
218 @staticmethod
219 def fetch_tools(method, names_to_fetch):
220 """Fetch bintools from a suitable place
221
222 This fetches or builds the requested bintools so that they can be used
223 by binman
224
225 Args:
226 names_to_fetch (list of str): names of bintools to fetch
227
228 Returns:
229 True on success, False on failure
230 """
231 def show_status(color, prompt, names):
Simon Glass252ac582022-01-29 14:14:17 -0700232 print(col.build(
Simon Glass252de6b2022-01-09 20:13:49 -0700233 color, f'{prompt}:%s{len(names):2}: %s' %
234 (' ' * (16 - len(prompt)), ' '.join(names))))
235
236 col = terminal.Color()
237 skip_present = False
238 name_list = names_to_fetch
239 if len(names_to_fetch) == 1 and names_to_fetch[0] in ['all', 'missing']:
240 name_list = Bintool.get_tool_list()
241 if names_to_fetch[0] == 'missing':
242 skip_present = True
Simon Glass252ac582022-01-29 14:14:17 -0700243 print(col.build(col.YELLOW,
Simon Glass252de6b2022-01-09 20:13:49 -0700244 'Fetching tools: %s' % ' '.join(name_list)))
245 status = collections.defaultdict(list)
246 for name in name_list:
247 btool = Bintool.create(name)
248 result = btool.fetch_tool(method, col, skip_present)
249 status[result].append(name)
250 if result == FAIL:
251 if method == FETCH_ANY:
252 print('- failed to fetch with all methods')
253 else:
254 print(f"- method '{FETCH_NAMES[method]}' is not supported")
255
256 if len(name_list) > 1:
257 if skip_present:
258 show_status(col.GREEN, 'Already present', status[PRESENT])
259 show_status(col.GREEN, 'Tools fetched', status[FETCHED])
260 if status[FAIL]:
261 show_status(col.RED, 'Failures', status[FAIL])
262 return not status[FAIL]
263
264 def run_cmd_result(self, *args, binary=False, raise_on_error=True):
265 """Run the bintool using command-line arguments
266
267 Args:
268 args (list of str): Arguments to provide, in addition to the bintool
269 name
270 binary (bool): True to return output as bytes instead of str
271 raise_on_error (bool): True to raise a ValueError exception if the
272 tool returns a non-zero return code
273
274 Returns:
275 CommandResult: Resulting output from the bintool, or None if the
276 tool is not present
277 """
278 if self.name in self.missing_list:
279 return None
280 name = os.path.expanduser(self.name) # Expand paths containing ~
281 all_args = (name,) + args
282 env = tools.get_env_with_path()
Simon Glassf3385a52022-01-29 14:14:15 -0700283 tout.detail(f"bintool: {' '.join(all_args)}")
Simon Glassd9800692022-01-29 14:14:05 -0700284 result = command.run_pipe(
Simon Glass252de6b2022-01-09 20:13:49 -0700285 [all_args], capture=True, capture_stderr=True, env=env,
286 raise_on_error=False, binary=binary)
287
288 if result.return_code:
289 # Return None if the tool was not found. In this case there is no
290 # output from the tool and it does not appear on the path. We still
291 # try to run it (as above) since RunPipe() allows faking the tool's
292 # output
293 if not any([result.stdout, result.stderr, tools.tool_find(name)]):
Simon Glassf3385a52022-01-29 14:14:15 -0700294 tout.info(f"bintool '{name}' not found")
Simon Glass252de6b2022-01-09 20:13:49 -0700295 return None
296 if raise_on_error:
Simon Glassf3385a52022-01-29 14:14:15 -0700297 tout.info(f"bintool '{name}' failed")
Simon Glass252de6b2022-01-09 20:13:49 -0700298 raise ValueError("Error %d running '%s': %s" %
299 (result.return_code, ' '.join(all_args),
300 result.stderr or result.stdout))
301 if result.stdout:
Simon Glassf3385a52022-01-29 14:14:15 -0700302 tout.debug(result.stdout)
Simon Glass252de6b2022-01-09 20:13:49 -0700303 if result.stderr:
Simon Glassf3385a52022-01-29 14:14:15 -0700304 tout.debug(result.stderr)
Simon Glass252de6b2022-01-09 20:13:49 -0700305 return result
306
307 def run_cmd(self, *args, binary=False):
308 """Run the bintool using command-line arguments
309
310 Args:
311 args (list of str): Arguments to provide, in addition to the bintool
312 name
313 binary (bool): True to return output as bytes instead of str
314
315 Returns:
316 str or bytes: Resulting stdout from the bintool
317 """
318 result = self.run_cmd_result(*args, binary=binary)
319 if result:
320 return result.stdout
321
322 @classmethod
Simon Glassd64af082022-09-17 09:01:19 -0600323 def build_from_git(cls, git_repo, make_target, bintool_path, flags=None):
Simon Glass252de6b2022-01-09 20:13:49 -0700324 """Build a bintool from a git repo
325
326 This clones the repo in a temporary directory, builds it with 'make',
327 then returns the filename of the resulting executable bintool
328
329 Args:
330 git_repo (str): URL of git repo
331 make_target (str): Target to pass to 'make' to build the tool
332 bintool_path (str): Relative path of the tool in the repo, after
333 build is complete
Simon Glassd64af082022-09-17 09:01:19 -0600334 flags (list of str): Flags or variables to pass to make, or None
Simon Glass252de6b2022-01-09 20:13:49 -0700335
336 Returns:
337 tuple:
338 str: Filename of fetched file to copy to a suitable directory
339 str: Name of temp directory to remove, or None
340 or None on error
341 """
342 tmpdir = tempfile.mkdtemp(prefix='binmanf.')
343 print(f"- clone git repo '{git_repo}' to '{tmpdir}'")
Simon Glassc1aa66e2022-01-29 14:14:04 -0700344 tools.run('git', 'clone', '--depth', '1', git_repo, tmpdir)
Simon Glass252de6b2022-01-09 20:13:49 -0700345 print(f"- build target '{make_target}'")
Simon Glassd64af082022-09-17 09:01:19 -0600346 cmd = ['make', '-C', tmpdir, '-j', f'{multiprocessing.cpu_count()}',
347 make_target]
348 if flags:
349 cmd += flags
350 tools.run(*cmd)
Simon Glass252de6b2022-01-09 20:13:49 -0700351 fname = os.path.join(tmpdir, bintool_path)
352 if not os.path.exists(fname):
353 print(f"- File '{fname}' was not produced")
354 return None
355 return fname, tmpdir
356
357 @classmethod
358 def fetch_from_url(cls, url):
359 """Fetch a bintool from a URL
360
361 Args:
362 url (str): URL to fetch from
363
364 Returns:
365 tuple:
366 str: Filename of fetched file to copy to a suitable directory
367 str: Name of temp directory to remove, or None
368 """
Simon Glassc1aa66e2022-01-29 14:14:04 -0700369 fname, tmpdir = tools.download(url)
370 tools.run('chmod', 'a+x', fname)
Simon Glass252de6b2022-01-09 20:13:49 -0700371 return fname, tmpdir
372
373 @classmethod
374 def fetch_from_drive(cls, drive_id):
375 """Fetch a bintool from Google drive
376
377 Args:
378 drive_id (str): ID of file to fetch. For a URL of the form
379 'https://drive.google.com/file/d/xxx/view?usp=sharing' the value
380 passed here should be 'xxx'
381
382 Returns:
383 tuple:
384 str: Filename of fetched file to copy to a suitable directory
385 str: Name of temp directory to remove, or None
386 """
387 url = f'https://drive.google.com/uc?export=download&id={drive_id}'
388 return cls.fetch_from_url(url)
389
390 @classmethod
391 def apt_install(cls, package):
392 """Install a bintool using the 'aot' tool
393
394 This requires use of servo so may request a password
395
396 Args:
397 package (str): Name of package to install
398
399 Returns:
400 True, assuming it completes without error
401 """
402 args = ['sudo', 'apt', 'install', '-y', package]
403 print('- %s' % ' '.join(args))
Simon Glassc1aa66e2022-01-29 14:14:04 -0700404 tools.run(*args)
Simon Glass252de6b2022-01-09 20:13:49 -0700405 return True
406
Simon Glassbc570642022-01-09 20:14:11 -0700407 @staticmethod
408 def WriteDocs(modules, test_missing=None):
409 """Write out documentation about the various bintools to stdout
410
411 Args:
412 modules: List of modules to include
413 test_missing: Used for testing. This is a module to report
414 as missing
415 """
416 print('''.. SPDX-License-Identifier: GPL-2.0+
417
418Binman bintool Documentation
419============================
420
421This file describes the bintools (binary tools) supported by binman. Bintools
422are binman's name for external executables that it runs to generate or process
423binaries. It is fairly easy to create new bintools. Just add a new file to the
424'btool' directory. You can use existing bintools as examples.
425
426
427''')
428 modules = sorted(modules)
429 missing = []
430 for name in modules:
431 module = Bintool.find_bintool_class(name)
432 docs = getattr(module, '__doc__')
433 if test_missing == name:
434 docs = None
435 if docs:
436 lines = docs.splitlines()
437 first_line = lines[0]
438 rest = [line[4:] for line in lines[1:]]
439 hdr = 'Bintool: %s: %s' % (name, first_line)
440 print(hdr)
441 print('-' * len(hdr))
442 print('\n'.join(rest))
443 print()
444 print()
445 else:
446 missing.append(name)
447
448 if missing:
449 raise ValueError('Documentation is missing for modules: %s' %
450 ', '.join(missing))
451
Simon Glass252de6b2022-01-09 20:13:49 -0700452 # pylint: disable=W0613
453 def fetch(self, method):
454 """Fetch handler for a bintool
455
456 This should be implemented by the base class
457
458 Args:
459 method (FETCH_...): Method to use
460
461 Returns:
462 tuple:
463 str: Filename of fetched file to copy to a suitable directory
464 str: Name of temp directory to remove, or None
465 or True if the file was fetched and already installed
466 or None if no fetch() implementation is available
467
468 Raises:
469 Valuerror: Fetching could not be completed
470 """
471 print(f"No method to fetch bintool '{self.name}'")
472 return False
473
Simon Glass252de6b2022-01-09 20:13:49 -0700474 def version(self):
475 """Version handler for a bintool
476
Simon Glass252de6b2022-01-09 20:13:49 -0700477 Returns:
478 str: Version string for this bintool
479 """
Quentin Schulz723a63e2022-09-01 17:51:37 +0200480 if self.version_regex is None:
481 return 'unknown'
482
483 import re
484
Quentin Schulze4408432022-09-01 17:51:40 +0200485 result = self.run_cmd_result(self.version_args)
Quentin Schulz723a63e2022-09-01 17:51:37 +0200486 out = result.stdout.strip()
487 if not out:
488 out = result.stderr.strip()
489 if not out:
490 return 'unknown'
491
492 m_version = re.search(self.version_regex, out)
493 return m_version.group(1) if m_version else out
494
Stefan Herbrechtsmeier867eed12022-08-19 16:25:33 +0200495
496class BintoolPacker(Bintool):
497 """Tool which compression / decompression entry contents
498
499 This is a bintools base class for compression / decompression packer
500
501 Properties:
502 name: Name of packer tool
503 compression: Compression type (COMPRESS_...), value of 'name' property
504 if none
505 compress_args: List of positional args provided to tool for compress,
506 ['--compress'] if none
507 decompress_args: List of positional args provided to tool for
508 decompress, ['--decompress'] if none
509 fetch_package: Name of the tool installed using the apt, value of 'name'
510 property if none
511 version_regex: Regular expressions to extract the version from tool
512 version output, '(v[0-9.]+)' if none
513 """
514 def __init__(self, name, compression=None, compress_args=None,
515 decompress_args=None, fetch_package=None,
Quentin Schulze4408432022-09-01 17:51:40 +0200516 version_regex=r'(v[0-9.]+)', version_args='-V'):
Stefan Herbrechtsmeier867eed12022-08-19 16:25:33 +0200517 desc = '%s compression' % (compression if compression else name)
Quentin Schulze4408432022-09-01 17:51:40 +0200518 super().__init__(name, desc, version_regex, version_args)
Stefan Herbrechtsmeier867eed12022-08-19 16:25:33 +0200519 if compress_args is None:
520 compress_args = ['--compress']
521 self.compress_args = compress_args
522 if decompress_args is None:
523 decompress_args = ['--decompress']
524 self.decompress_args = decompress_args
525 if fetch_package is None:
526 fetch_package = name
527 self.fetch_package = fetch_package
Stefan Herbrechtsmeier867eed12022-08-19 16:25:33 +0200528
529 def compress(self, indata):
530 """Compress data
531
532 Args:
533 indata (bytes): Data to compress
534
535 Returns:
536 bytes: Compressed data
537 """
538 with tempfile.NamedTemporaryFile(prefix='comp.tmp',
539 dir=tools.get_output_dir()) as tmp:
540 tools.write_file(tmp.name, indata)
541 args = self.compress_args + ['--stdout', tmp.name]
542 return self.run_cmd(*args, binary=True)
543
544 def decompress(self, indata):
545 """Decompress data
546
547 Args:
548 indata (bytes): Data to decompress
549
550 Returns:
551 bytes: Decompressed data
552 """
553 with tempfile.NamedTemporaryFile(prefix='decomp.tmp',
554 dir=tools.get_output_dir()) as inf:
555 tools.write_file(inf.name, indata)
556 args = self.decompress_args + ['--stdout', inf.name]
557 return self.run_cmd(*args, binary=True)
558
559 def fetch(self, method):
560 """Fetch handler
561
562 This installs the gzip package using the apt utility.
563
564 Args:
565 method (FETCH_...): Method to use
566
567 Returns:
568 True if the file was fetched and now installed, None if a method
569 other than FETCH_BIN was requested
570
571 Raises:
572 Valuerror: Fetching could not be completed
573 """
574 if method != FETCH_BIN:
575 return None
576 return self.apt_install(self.fetch_package)