| # SPDX-License-Identifier: GPL-2.0+ |
| # Copyright (c) 2012 The Chromium OS Authors. |
| # |
| |
| import re |
| import glob |
| from html.parser import HTMLParser |
| import os |
| import sys |
| import tempfile |
| import urllib.request, urllib.error, urllib.parse |
| |
| from buildman import bsettings |
| from u_boot_pylib import command |
| from u_boot_pylib import terminal |
| from u_boot_pylib import tools |
| |
| (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH, |
| PRIORITY_CALC) = list(range(4)) |
| |
| (VAR_CROSS_COMPILE, VAR_PATH, VAR_ARCH, VAR_MAKE_ARGS) = range(4) |
| |
| # Simple class to collect links from a page |
| class MyHTMLParser(HTMLParser): |
| def __init__(self, arch): |
| """Create a new parser |
| |
| After the parser runs, self.links will be set to a list of the links |
| to .xz archives found in the page, and self.arch_link will be set to |
| the one for the given architecture (or None if not found). |
| |
| Args: |
| arch: Architecture to search for |
| """ |
| HTMLParser.__init__(self) |
| self.arch_link = None |
| self.links = [] |
| self.re_arch = re.compile('[-_]%s-' % arch) |
| |
| def handle_starttag(self, tag, attrs): |
| if tag == 'a': |
| for tag, value in attrs: |
| if tag == 'href': |
| if value and value.endswith('.xz'): |
| self.links.append(value) |
| if self.re_arch.search(value): |
| self.arch_link = value |
| |
| |
| class Toolchain: |
| """A single toolchain |
| |
| Public members: |
| gcc: Full path to C compiler |
| path: Directory path containing C compiler |
| cross: Cross compile string, e.g. 'arm-linux-' |
| arch: Architecture of toolchain as determined from the first |
| component of the filename. E.g. arm-linux-gcc becomes arm |
| priority: Toolchain priority (0=highest, 20=lowest) |
| override_toolchain: Toolchain to use for sandbox, overriding the normal |
| one |
| """ |
| def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC, |
| arch=None, override_toolchain=None): |
| """Create a new toolchain object. |
| |
| Args: |
| fname: Filename of the gcc component |
| test: True to run the toolchain to test it |
| verbose: True to print out the information |
| priority: Priority to use for this toolchain, or PRIORITY_CALC to |
| calculate it |
| """ |
| self.gcc = fname |
| self.path = os.path.dirname(fname) |
| self.override_toolchain = override_toolchain |
| |
| # Find the CROSS_COMPILE prefix to use for U-Boot. For example, |
| # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'. |
| basename = os.path.basename(fname) |
| pos = basename.rfind('-') |
| self.cross = basename[:pos + 1] if pos != -1 else '' |
| |
| # The architecture is the first part of the name |
| pos = self.cross.find('-') |
| if arch: |
| self.arch = arch |
| else: |
| self.arch = self.cross[:pos] if pos != -1 else 'sandbox' |
| if self.arch == 'sandbox' and override_toolchain: |
| self.gcc = override_toolchain |
| |
| env = self.MakeEnvironment(False) |
| |
| # As a basic sanity check, run the C compiler with --version |
| cmd = [fname, '--version'] |
| if priority == PRIORITY_CALC: |
| self.priority = self.GetPriority(fname) |
| else: |
| self.priority = priority |
| if test: |
| result = command.run_pipe([cmd], capture=True, env=env, |
| raise_on_error=False) |
| self.ok = result.return_code == 0 |
| if verbose: |
| print('Tool chain test: ', end=' ') |
| if self.ok: |
| print("OK, arch='%s', priority %d" % (self.arch, |
| self.priority)) |
| else: |
| print('BAD') |
| print('Command: ', cmd) |
| print(result.stdout) |
| print(result.stderr) |
| else: |
| self.ok = True |
| |
| def GetPriority(self, fname): |
| """Return the priority of the toolchain. |
| |
| Toolchains are ranked according to their suitability by their |
| filename prefix. |
| |
| Args: |
| fname: Filename of toolchain |
| Returns: |
| Priority of toolchain, PRIORITY_CALC=highest, 20=lowest. |
| """ |
| priority_list = ['-elf', '-unknown-linux-gnu', '-linux', |
| '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux', |
| '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi', |
| '-linux-gnueabihf', '-le-linux', '-uclinux'] |
| for prio in range(len(priority_list)): |
| if priority_list[prio] in fname: |
| return PRIORITY_CALC + prio |
| return PRIORITY_CALC + prio |
| |
| def GetWrapper(self, show_warning=True): |
| """Get toolchain wrapper from the setting file. |
| """ |
| value = '' |
| for name, value in bsettings.get_items('toolchain-wrapper'): |
| if not value: |
| print("Warning: Wrapper not found") |
| if value: |
| value = value + ' ' |
| |
| return value |
| |
| def GetEnvArgs(self, which): |
| """Get an environment variable/args value based on the the toolchain |
| |
| Args: |
| which: VAR_... value to get |
| |
| Returns: |
| Value of that environment variable or arguments |
| """ |
| if which == VAR_CROSS_COMPILE: |
| wrapper = self.GetWrapper() |
| base = '' if self.arch == 'sandbox' else self.path |
| return wrapper + os.path.join(base, self.cross) |
| elif which == VAR_PATH: |
| return self.path |
| elif which == VAR_ARCH: |
| return self.arch |
| elif which == VAR_MAKE_ARGS: |
| args = self.MakeArgs() |
| if args: |
| return ' '.join(args) |
| return '' |
| else: |
| raise ValueError('Unknown arg to GetEnvArgs (%d)' % which) |
| |
| def MakeEnvironment(self, full_path): |
| """Returns an environment for using the toolchain. |
| |
| Thie takes the current environment and adds CROSS_COMPILE so that |
| the tool chain will operate correctly. This also disables localized |
| output and possibly unicode encoded output of all build tools by |
| adding LC_ALL=C. |
| |
| Note that os.environb is used to obtain the environment, since in some |
| cases the environment many contain non-ASCII characters and we see |
| errors like: |
| |
| UnicodeEncodeError: 'utf-8' codec can't encode characters in position |
| 569-570: surrogates not allowed |
| |
| Args: |
| full_path: Return the full path in CROSS_COMPILE and don't set |
| PATH |
| Returns: |
| Dict containing the (bytes) environment to use. This is based on the |
| current environment, with changes as needed to CROSS_COMPILE, PATH |
| and LC_ALL. |
| """ |
| env = dict(os.environb) |
| wrapper = self.GetWrapper() |
| |
| if self.override_toolchain: |
| # We'll use MakeArgs() to provide this |
| pass |
| elif full_path: |
| env[b'CROSS_COMPILE'] = tools.to_bytes( |
| wrapper + os.path.join(self.path, self.cross)) |
| else: |
| env[b'CROSS_COMPILE'] = tools.to_bytes(wrapper + self.cross) |
| env[b'PATH'] = tools.to_bytes(self.path) + b':' + env[b'PATH'] |
| |
| env[b'LC_ALL'] = b'C' |
| |
| return env |
| |
| def MakeArgs(self): |
| """Create the 'make' arguments for a toolchain |
| |
| This is only used when the toolchain is being overridden. Since the |
| U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the |
| environment (and MakeEnvironment()) to override these values. This |
| function returns the arguments to accomplish this. |
| |
| Returns: |
| List of arguments to pass to 'make' |
| """ |
| if self.override_toolchain: |
| return ['HOSTCC=%s' % self.override_toolchain, |
| 'CC=%s' % self.override_toolchain] |
| return [] |
| |
| |
| class Toolchains: |
| """Manage a list of toolchains for building U-Boot |
| |
| We select one toolchain for each architecture type |
| |
| Public members: |
| toolchains: Dict of Toolchain objects, keyed by architecture name |
| prefixes: Dict of prefixes to check, keyed by architecture. This can |
| be a full path and toolchain prefix, for example |
| {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of |
| something on the search path, for example |
| {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported. |
| paths: List of paths to check for toolchains (may contain wildcards) |
| """ |
| |
| def __init__(self, override_toolchain=None): |
| self.toolchains = {} |
| self.prefixes = {} |
| self.paths = [] |
| self.override_toolchain = override_toolchain |
| self._make_flags = dict(bsettings.get_items('make-flags')) |
| |
| def GetPathList(self, show_warning=True): |
| """Get a list of available toolchain paths |
| |
| Args: |
| show_warning: True to show a warning if there are no tool chains. |
| |
| Returns: |
| List of strings, each a path to a toolchain mentioned in the |
| [toolchain] section of the settings file. |
| """ |
| toolchains = bsettings.get_items('toolchain') |
| if show_warning and not toolchains: |
| print(("Warning: No tool chains. Please run 'buildman " |
| "--fetch-arch all' to download all available toolchains, or " |
| "add a [toolchain] section to your buildman config file " |
| "%s. See buildman.rst for details" % |
| bsettings.config_fname)) |
| |
| paths = [] |
| for name, value in toolchains: |
| if '*' in value: |
| paths += glob.glob(value) |
| else: |
| paths.append(value) |
| return paths |
| |
| def GetSettings(self, show_warning=True): |
| """Get toolchain settings from the settings file. |
| |
| Args: |
| show_warning: True to show a warning if there are no tool chains. |
| """ |
| self.prefixes = bsettings.get_items('toolchain-prefix') |
| self.paths += self.GetPathList(show_warning) |
| |
| def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC, |
| arch=None): |
| """Add a toolchain to our list |
| |
| We select the given toolchain as our preferred one for its |
| architecture if it is a higher priority than the others. |
| |
| Args: |
| fname: Filename of toolchain's gcc driver |
| test: True to run the toolchain to test it |
| priority: Priority to use for this toolchain |
| arch: Toolchain architecture, or None if not known |
| """ |
| toolchain = Toolchain(fname, test, verbose, priority, arch, |
| self.override_toolchain) |
| add_it = toolchain.ok |
| if toolchain.arch in self.toolchains: |
| add_it = (toolchain.priority < |
| self.toolchains[toolchain.arch].priority) |
| if add_it: |
| self.toolchains[toolchain.arch] = toolchain |
| elif verbose: |
| print(("Toolchain '%s' at priority %d will be ignored because " |
| "another toolchain for arch '%s' has priority %d" % |
| (toolchain.gcc, toolchain.priority, toolchain.arch, |
| self.toolchains[toolchain.arch].priority))) |
| |
| def ScanPath(self, path, verbose): |
| """Scan a path for a valid toolchain |
| |
| Args: |
| path: Path to scan |
| verbose: True to print out progress information |
| Returns: |
| Filename of C compiler if found, else None |
| """ |
| fnames = [] |
| for subdir in ['.', 'bin', 'usr/bin']: |
| dirname = os.path.join(path, subdir) |
| if verbose: print(" - looking in '%s'" % dirname) |
| for fname in glob.glob(dirname + '/*gcc'): |
| if verbose: print(" - found '%s'" % fname) |
| fnames.append(fname) |
| return fnames |
| |
| def ScanPathEnv(self, fname): |
| """Scan the PATH environment variable for a given filename. |
| |
| Args: |
| fname: Filename to scan for |
| Returns: |
| List of matching pathanames, or [] if none |
| """ |
| pathname_list = [] |
| for path in os.environ["PATH"].split(os.pathsep): |
| path = path.strip('"') |
| pathname = os.path.join(path, fname) |
| if os.path.exists(pathname): |
| pathname_list.append(pathname) |
| return pathname_list |
| |
| def Scan(self, verbose): |
| """Scan for available toolchains and select the best for each arch. |
| |
| We look for all the toolchains we can file, figure out the |
| architecture for each, and whether it works. Then we select the |
| highest priority toolchain for each arch. |
| |
| Args: |
| verbose: True to print out progress information |
| """ |
| if verbose: print('Scanning for tool chains') |
| for name, value in self.prefixes: |
| if verbose: print(" - scanning prefix '%s'" % value) |
| if os.path.exists(value): |
| self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name) |
| continue |
| fname = value + 'gcc' |
| if os.path.exists(fname): |
| self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name) |
| continue |
| fname_list = self.ScanPathEnv(fname) |
| for f in fname_list: |
| self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name) |
| if not fname_list: |
| raise ValueError("No tool chain found for prefix '%s'" % |
| value) |
| for path in self.paths: |
| if verbose: print(" - scanning path '%s'" % path) |
| fnames = self.ScanPath(path, verbose) |
| for fname in fnames: |
| self.Add(fname, True, verbose) |
| |
| def List(self): |
| """List out the selected toolchains for each architecture""" |
| col = terminal.Color() |
| print(col.build(col.BLUE, 'List of available toolchains (%d):' % |
| len(self.toolchains))) |
| if len(self.toolchains): |
| for key, value in sorted(self.toolchains.items()): |
| print('%-10s: %s' % (key, value.gcc)) |
| else: |
| print('None') |
| |
| def Select(self, arch): |
| """Returns the toolchain for a given architecture |
| |
| Args: |
| args: Name of architecture (e.g. 'arm', 'ppc_8xx') |
| |
| returns: |
| toolchain object, or None if none found |
| """ |
| for tag, value in bsettings.get_items('toolchain-alias'): |
| if arch == tag: |
| for alias in value.split(): |
| if alias in self.toolchains: |
| return self.toolchains[alias] |
| |
| if not arch in self.toolchains: |
| raise ValueError("No tool chain found for arch '%s'" % arch) |
| return self.toolchains[arch] |
| |
| def ResolveReferences(self, var_dict, args): |
| """Resolve variable references in a string |
| |
| This converts ${blah} within the string to the value of blah. |
| This function works recursively. |
| |
| Args: |
| var_dict: Dictionary containing variables and their values |
| args: String containing make arguments |
| Returns: |
| Resolved string |
| |
| >>> bsettings.setup(None) |
| >>> tcs = Toolchains() |
| >>> tcs.Add('fred', False) |
| >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \ |
| 'second' : '2nd'} |
| >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set') |
| 'this=OBLIQUE_set' |
| >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd') |
| 'this=OBLIQUE_setfi2ndrstnd' |
| """ |
| re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})') |
| |
| while True: |
| m = re_var.search(args) |
| if not m: |
| break |
| lookup = m.group(0)[2:-1] |
| value = var_dict.get(lookup, '') |
| args = args[:m.start(0)] + value + args[m.end(0):] |
| return args |
| |
| def GetMakeArguments(self, brd): |
| """Returns 'make' arguments for a given board |
| |
| The flags are in a section called 'make-flags'. Flags are named |
| after the target they represent, for example snapper9260=TESTING=1 |
| will pass TESTING=1 to make when building the snapper9260 board. |
| |
| References to other boards can be added in the string also. For |
| example: |
| |
| [make-flags] |
| at91-boards=ENABLE_AT91_TEST=1 |
| snapper9260=${at91-boards} BUILD_TAG=442 |
| snapper9g45=${at91-boards} BUILD_TAG=443 |
| |
| This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260 |
| and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45. |
| |
| A special 'target' variable is set to the board target. |
| |
| Args: |
| brd: Board object for the board to check. |
| Returns: |
| 'make' flags for that board, or '' if none |
| """ |
| self._make_flags['target'] = brd.target |
| arg_str = self.ResolveReferences(self._make_flags, |
| self._make_flags.get(brd.target, '')) |
| args = re.findall("(?:\".*?\"|\S)+", arg_str) |
| i = 0 |
| while i < len(args): |
| args[i] = args[i].replace('"', '') |
| if not args[i]: |
| del args[i] |
| else: |
| i += 1 |
| return args |
| |
| def LocateArchUrl(self, fetch_arch): |
| """Find a toolchain available online |
| |
| Look in standard places for available toolchains. At present the |
| only standard place is at kernel.org. |
| |
| Args: |
| arch: Architecture to look for, or 'list' for all |
| Returns: |
| If fetch_arch is 'list', a tuple: |
| Machine architecture (e.g. x86_64) |
| List of toolchains |
| else |
| URL containing this toolchain, if avaialble, else None |
| """ |
| arch = command.output_one_line('uname', '-m') |
| if arch == 'aarch64': |
| arch = 'arm64' |
| base = 'https://www.kernel.org/pub/tools/crosstool/files/bin' |
| versions = ['13.1.0', '12.2.0'] |
| links = [] |
| for version in versions: |
| url = '%s/%s/%s/' % (base, arch, version) |
| print('Checking: %s' % url) |
| response = urllib.request.urlopen(url) |
| html = tools.to_string(response.read()) |
| parser = MyHTMLParser(fetch_arch) |
| parser.feed(html) |
| if fetch_arch == 'list': |
| links += parser.links |
| elif parser.arch_link: |
| return url + parser.arch_link |
| if fetch_arch == 'list': |
| return arch, links |
| return None |
| |
| def Unpack(self, fname, dest): |
| """Unpack a tar file |
| |
| Args: |
| fname: Filename to unpack |
| dest: Destination directory |
| Returns: |
| Directory name of the first entry in the archive, without the |
| trailing / |
| """ |
| stdout = command.output('tar', 'xvfJ', fname, '-C', dest) |
| dirs = stdout.splitlines()[1].split('/')[:2] |
| return '/'.join(dirs) |
| |
| def TestSettingsHasPath(self, path): |
| """Check if buildman will find this toolchain |
| |
| Returns: |
| True if the path is in settings, False if not |
| """ |
| paths = self.GetPathList(False) |
| return path in paths |
| |
| def ListArchs(self): |
| """List architectures with available toolchains to download""" |
| host_arch, archives = self.LocateArchUrl('list') |
| re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*') |
| arch_set = set() |
| for archive in archives: |
| # Remove the host architecture from the start |
| arch = re_arch.match(archive[len(host_arch):]) |
| if arch: |
| if arch.group(1) != '2.0' and arch.group(1) != '64': |
| arch_set.add(arch.group(1)) |
| return sorted(arch_set) |
| |
| def FetchAndInstall(self, arch): |
| """Fetch and install a new toolchain |
| |
| arch: |
| Architecture to fetch, or 'list' to list |
| """ |
| # Fist get the URL for this architecture |
| col = terminal.Color() |
| print(col.build(col.BLUE, "Downloading toolchain for arch '%s'" % arch)) |
| url = self.LocateArchUrl(arch) |
| if not url: |
| print(("Cannot find toolchain for arch '%s' - use 'list' to list" % |
| arch)) |
| return 2 |
| home = os.environ['HOME'] |
| dest = os.path.join(home, '.buildman-toolchains') |
| if not os.path.exists(dest): |
| os.mkdir(dest) |
| |
| # Download the tar file for this toolchain and unpack it |
| tarfile, tmpdir = tools.download(url, '.buildman') |
| if not tarfile: |
| return 1 |
| print(col.build(col.GREEN, 'Unpacking to: %s' % dest), end=' ') |
| sys.stdout.flush() |
| path = self.Unpack(tarfile, dest) |
| os.remove(tarfile) |
| os.rmdir(tmpdir) |
| print() |
| |
| # Check that the toolchain works |
| print(col.build(col.GREEN, 'Testing')) |
| dirpath = os.path.join(dest, path) |
| compiler_fname_list = self.ScanPath(dirpath, True) |
| if not compiler_fname_list: |
| print('Could not locate C compiler - fetch failed.') |
| return 1 |
| if len(compiler_fname_list) != 1: |
| print(col.build(col.RED, 'Warning, ambiguous toolchains: %s' % |
| ', '.join(compiler_fname_list))) |
| toolchain = Toolchain(compiler_fname_list[0], True, True) |
| |
| # Make sure that it will be found by buildman |
| if not self.TestSettingsHasPath(dirpath): |
| print(("Adding 'download' to config file '%s'" % |
| bsettings.config_fname)) |
| bsettings.set_item('toolchain', 'download', '%s/*/*' % dest) |
| return 0 |