blob: acb5a29c8fb83af7e4e63b14737adb8d7ff95f81 [file] [log] [blame]
Tom Rini83d290c2018-05-06 17:58:06 -04001# SPDX-License-Identifier: GPL-2.0+
Simon Glassfc3fe1c2013-04-03 11:07:16 +00002# Copyright (c) 2012 The Chromium OS Authors.
3#
Simon Glassfc3fe1c2013-04-03 11:07:16 +00004
Simon Glass4281ad82013-09-23 17:35:17 -06005import re
Simon Glassfc3fe1c2013-04-03 11:07:16 +00006import glob
Simon Glassc05aa032019-10-31 07:42:53 -06007from html.parser import HTMLParser
Simon Glassfc3fe1c2013-04-03 11:07:16 +00008import os
Simon Glass827e37b2014-12-01 17:34:06 -07009import sys
10import tempfile
Simon Glassc05aa032019-10-31 07:42:53 -060011import urllib.request, urllib.error, urllib.parse
Simon Glassfc3fe1c2013-04-03 11:07:16 +000012
Simon Glass0ede00f2020-04-17 18:09:02 -060013from buildman import bsettings
Simon Glassbf776672020-04-17 18:09:04 -060014from patman import command
15from patman import terminal
16from patman import tools
Simon Glassfc3fe1c2013-04-03 11:07:16 +000017
Simon Glass17bce662016-03-12 18:50:32 -070018(PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
Simon Glassc05aa032019-10-31 07:42:53 -060019 PRIORITY_CALC) = list(range(4))
Simon Glassff690df2016-03-06 19:45:37 -070020
Simon Glass57cb9d52019-12-05 15:59:14 -070021(VAR_CROSS_COMPILE, VAR_PATH, VAR_ARCH, VAR_MAKE_ARGS) = range(4)
22
Simon Glass827e37b2014-12-01 17:34:06 -070023# Simple class to collect links from a page
24class MyHTMLParser(HTMLParser):
25 def __init__(self, arch):
26 """Create a new parser
27
28 After the parser runs, self.links will be set to a list of the links
29 to .xz archives found in the page, and self.arch_link will be set to
30 the one for the given architecture (or None if not found).
31
32 Args:
33 arch: Architecture to search for
34 """
35 HTMLParser.__init__(self)
36 self.arch_link = None
37 self.links = []
Daniel Schwierzeck4c58d272018-05-10 07:15:53 -040038 self.re_arch = re.compile('[-_]%s-' % arch)
Simon Glass827e37b2014-12-01 17:34:06 -070039
40 def handle_starttag(self, tag, attrs):
41 if tag == 'a':
42 for tag, value in attrs:
43 if tag == 'href':
44 if value and value.endswith('.xz'):
45 self.links.append(value)
Daniel Schwierzeck4c58d272018-05-10 07:15:53 -040046 if self.re_arch.search(value):
Simon Glass827e37b2014-12-01 17:34:06 -070047 self.arch_link = value
48
49
Simon Glassfc3fe1c2013-04-03 11:07:16 +000050class Toolchain:
51 """A single toolchain
52
53 Public members:
54 gcc: Full path to C compiler
55 path: Directory path containing C compiler
56 cross: Cross compile string, e.g. 'arm-linux-'
57 arch: Architecture of toolchain as determined from the first
58 component of the filename. E.g. arm-linux-gcc becomes arm
Simon Glassff690df2016-03-06 19:45:37 -070059 priority: Toolchain priority (0=highest, 20=lowest)
Simon Glass00beb242019-01-07 16:44:20 -070060 override_toolchain: Toolchain to use for sandbox, overriding the normal
61 one
Simon Glassfc3fe1c2013-04-03 11:07:16 +000062 """
Simon Glass608e3992016-03-06 19:45:38 -070063 def __init__(self, fname, test, verbose=False, priority=PRIORITY_CALC,
Simon Glass00beb242019-01-07 16:44:20 -070064 arch=None, override_toolchain=None):
Simon Glassfc3fe1c2013-04-03 11:07:16 +000065 """Create a new toolchain object.
66
67 Args:
68 fname: Filename of the gcc component
69 test: True to run the toolchain to test it
Simon Glassad24eba2016-03-06 19:45:35 -070070 verbose: True to print out the information
Simon Glassff690df2016-03-06 19:45:37 -070071 priority: Priority to use for this toolchain, or PRIORITY_CALC to
72 calculate it
Simon Glassfc3fe1c2013-04-03 11:07:16 +000073 """
74 self.gcc = fname
75 self.path = os.path.dirname(fname)
Simon Glass00beb242019-01-07 16:44:20 -070076 self.override_toolchain = override_toolchain
Simon Glassb5324122014-12-01 17:33:58 -070077
78 # Find the CROSS_COMPILE prefix to use for U-Boot. For example,
79 # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'.
80 basename = os.path.basename(fname)
81 pos = basename.rfind('-')
82 self.cross = basename[:pos + 1] if pos != -1 else ''
83
84 # The architecture is the first part of the name
Simon Glassfc3fe1c2013-04-03 11:07:16 +000085 pos = self.cross.find('-')
Simon Glass608e3992016-03-06 19:45:38 -070086 if arch:
87 self.arch = arch
88 else:
89 self.arch = self.cross[:pos] if pos != -1 else 'sandbox'
Simon Glass00beb242019-01-07 16:44:20 -070090 if self.arch == 'sandbox' and override_toolchain:
91 self.gcc = override_toolchain
Simon Glassfc3fe1c2013-04-03 11:07:16 +000092
Simon Glassbb1501f2014-12-01 17:34:00 -070093 env = self.MakeEnvironment(False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +000094
95 # As a basic sanity check, run the C compiler with --version
96 cmd = [fname, '--version']
Simon Glassff690df2016-03-06 19:45:37 -070097 if priority == PRIORITY_CALC:
98 self.priority = self.GetPriority(fname)
99 else:
100 self.priority = priority
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000101 if test:
Stephen Warren8bb2bdd2013-10-09 14:28:09 -0600102 result = command.RunPipe([cmd], capture=True, env=env,
103 raise_on_error=False)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000104 self.ok = result.return_code == 0
105 if verbose:
Simon Glassc05aa032019-10-31 07:42:53 -0600106 print('Tool chain test: ', end=' ')
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000107 if self.ok:
Simon Glassc05aa032019-10-31 07:42:53 -0600108 print("OK, arch='%s', priority %d" % (self.arch,
109 self.priority))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000110 else:
Simon Glassc05aa032019-10-31 07:42:53 -0600111 print('BAD')
112 print('Command: ', cmd)
113 print(result.stdout)
114 print(result.stderr)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000115 else:
116 self.ok = True
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000117
118 def GetPriority(self, fname):
119 """Return the priority of the toolchain.
120
121 Toolchains are ranked according to their suitability by their
122 filename prefix.
123
124 Args:
125 fname: Filename of toolchain
126 Returns:
Simon Glassff690df2016-03-06 19:45:37 -0700127 Priority of toolchain, PRIORITY_CALC=highest, 20=lowest.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000128 """
Masahiro Yamada87082672014-07-07 09:47:45 +0900129 priority_list = ['-elf', '-unknown-linux-gnu', '-linux',
Tom Rini546a6f32017-04-14 19:47:50 -0400130 '-none-linux-gnueabi', '-none-linux-gnueabihf', '-uclinux',
131 '-none-eabi', '-gentoo-linux-gnu', '-linux-gnueabi',
132 '-linux-gnueabihf', '-le-linux', '-uclinux']
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000133 for prio in range(len(priority_list)):
134 if priority_list[prio] in fname:
Simon Glassff690df2016-03-06 19:45:37 -0700135 return PRIORITY_CALC + prio
136 return PRIORITY_CALC + prio
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000137
York Sund5fe0132016-10-04 14:33:51 -0700138 def GetWrapper(self, show_warning=True):
139 """Get toolchain wrapper from the setting file.
140 """
Simon Glassccd29792019-01-07 16:44:24 -0700141 value = ''
142 for name, value in bsettings.GetItems('toolchain-wrapper'):
York Sund5fe0132016-10-04 14:33:51 -0700143 if not value:
Simon Glassc05aa032019-10-31 07:42:53 -0600144 print("Warning: Wrapper not found")
York Sund5fe0132016-10-04 14:33:51 -0700145 if value:
146 value = value + ' '
147
148 return value
149
Simon Glass57cb9d52019-12-05 15:59:14 -0700150 def GetEnvArgs(self, which):
151 """Get an environment variable/args value based on the the toolchain
152
153 Args:
154 which: VAR_... value to get
155
156 Returns:
157 Value of that environment variable or arguments
158 """
159 wrapper = self.GetWrapper()
160 if which == VAR_CROSS_COMPILE:
161 return wrapper + os.path.join(self.path, self.cross)
162 elif which == VAR_PATH:
163 return self.path
164 elif which == VAR_ARCH:
165 return self.arch
166 elif which == VAR_MAKE_ARGS:
167 args = self.MakeArgs()
168 if args:
169 return ' '.join(args)
170 return ''
171 else:
172 raise ValueError('Unknown arg to GetEnvArgs (%d)' % which)
173
Simon Glassbb1501f2014-12-01 17:34:00 -0700174 def MakeEnvironment(self, full_path):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000175 """Returns an environment for using the toolchain.
176
Simon Glassbb1501f2014-12-01 17:34:00 -0700177 Thie takes the current environment and adds CROSS_COMPILE so that
Daniel Schwierzeckb0e994c2017-06-08 03:07:08 +0200178 the tool chain will operate correctly. This also disables localized
179 output and possibly unicode encoded output of all build tools by
180 adding LC_ALL=C.
Simon Glassbb1501f2014-12-01 17:34:00 -0700181
182 Args:
183 full_path: Return the full path in CROSS_COMPILE and don't set
184 PATH
Simon Glass00beb242019-01-07 16:44:20 -0700185 Returns:
186 Dict containing the environemnt to use. This is based on the current
187 environment, with changes as needed to CROSS_COMPILE, PATH and
188 LC_ALL.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000189 """
190 env = dict(os.environ)
York Sund5fe0132016-10-04 14:33:51 -0700191 wrapper = self.GetWrapper()
192
Simon Glass00beb242019-01-07 16:44:20 -0700193 if self.override_toolchain:
194 # We'll use MakeArgs() to provide this
195 pass
196 elif full_path:
York Sund5fe0132016-10-04 14:33:51 -0700197 env['CROSS_COMPILE'] = wrapper + os.path.join(self.path, self.cross)
Simon Glassbb1501f2014-12-01 17:34:00 -0700198 else:
York Sund5fe0132016-10-04 14:33:51 -0700199 env['CROSS_COMPILE'] = wrapper + self.cross
Simon Glassbb1501f2014-12-01 17:34:00 -0700200 env['PATH'] = self.path + ':' + env['PATH']
201
Daniel Schwierzeckb0e994c2017-06-08 03:07:08 +0200202 env['LC_ALL'] = 'C'
203
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000204 return env
205
Simon Glass00beb242019-01-07 16:44:20 -0700206 def MakeArgs(self):
207 """Create the 'make' arguments for a toolchain
208
209 This is only used when the toolchain is being overridden. Since the
210 U-Boot Makefile sets CC and HOSTCC explicitly we cannot rely on the
211 environment (and MakeEnvironment()) to override these values. This
212 function returns the arguments to accomplish this.
213
214 Returns:
215 List of arguments to pass to 'make'
216 """
217 if self.override_toolchain:
218 return ['HOSTCC=%s' % self.override_toolchain,
219 'CC=%s' % self.override_toolchain]
220 return []
221
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000222
223class Toolchains:
224 """Manage a list of toolchains for building U-Boot
225
226 We select one toolchain for each architecture type
227
228 Public members:
229 toolchains: Dict of Toolchain objects, keyed by architecture name
Simon Glass17bce662016-03-12 18:50:32 -0700230 prefixes: Dict of prefixes to check, keyed by architecture. This can
231 be a full path and toolchain prefix, for example
232 {'x86', 'opt/i386-linux/bin/i386-linux-'}, or the name of
233 something on the search path, for example
234 {'arm', 'arm-linux-gnueabihf-'}. Wildcards are not supported.
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000235 paths: List of paths to check for toolchains (may contain wildcards)
236 """
237
Simon Glass00beb242019-01-07 16:44:20 -0700238 def __init__(self, override_toolchain=None):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000239 self.toolchains = {}
Simon Glass17bce662016-03-12 18:50:32 -0700240 self.prefixes = {}
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000241 self.paths = []
Simon Glass00beb242019-01-07 16:44:20 -0700242 self.override_toolchain = override_toolchain
Simon Glassd4144e42014-09-05 19:00:13 -0600243 self._make_flags = dict(bsettings.GetItems('make-flags'))
244
Simon Glass80e6a482016-07-27 20:33:01 -0600245 def GetPathList(self, show_warning=True):
Simon Glass827e37b2014-12-01 17:34:06 -0700246 """Get a list of available toolchain paths
247
Simon Glass80e6a482016-07-27 20:33:01 -0600248 Args:
249 show_warning: True to show a warning if there are no tool chains.
250
Simon Glass827e37b2014-12-01 17:34:06 -0700251 Returns:
252 List of strings, each a path to a toolchain mentioned in the
253 [toolchain] section of the settings file.
254 """
Simon Glass4281ad82013-09-23 17:35:17 -0600255 toolchains = bsettings.GetItems('toolchain')
Simon Glass80e6a482016-07-27 20:33:01 -0600256 if show_warning and not toolchains:
Simon Glassc05aa032019-10-31 07:42:53 -0600257 print(("Warning: No tool chains. Please run 'buildman "
Simon Glass713bea32016-07-27 20:33:02 -0600258 "--fetch-arch all' to download all available toolchains, or "
259 "add a [toolchain] section to your buildman config file "
260 "%s. See README for details" %
Simon Glassc05aa032019-10-31 07:42:53 -0600261 bsettings.config_fname))
Simon Glass4281ad82013-09-23 17:35:17 -0600262
Simon Glass827e37b2014-12-01 17:34:06 -0700263 paths = []
Simon Glass4281ad82013-09-23 17:35:17 -0600264 for name, value in toolchains:
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000265 if '*' in value:
Simon Glass827e37b2014-12-01 17:34:06 -0700266 paths += glob.glob(value)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000267 else:
Simon Glass827e37b2014-12-01 17:34:06 -0700268 paths.append(value)
269 return paths
270
Simon Glass80e6a482016-07-27 20:33:01 -0600271 def GetSettings(self, show_warning=True):
272 """Get toolchain settings from the settings file.
273
274 Args:
275 show_warning: True to show a warning if there are no tool chains.
276 """
277 self.prefixes = bsettings.GetItems('toolchain-prefix')
278 self.paths += self.GetPathList(show_warning)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000279
Simon Glass608e3992016-03-06 19:45:38 -0700280 def Add(self, fname, test=True, verbose=False, priority=PRIORITY_CALC,
281 arch=None):
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000282 """Add a toolchain to our list
283
284 We select the given toolchain as our preferred one for its
285 architecture if it is a higher priority than the others.
286
287 Args:
288 fname: Filename of toolchain's gcc driver
289 test: True to run the toolchain to test it
Simon Glassff690df2016-03-06 19:45:37 -0700290 priority: Priority to use for this toolchain
Simon Glass608e3992016-03-06 19:45:38 -0700291 arch: Toolchain architecture, or None if not known
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000292 """
Simon Glass00beb242019-01-07 16:44:20 -0700293 toolchain = Toolchain(fname, test, verbose, priority, arch,
294 self.override_toolchain)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000295 add_it = toolchain.ok
296 if toolchain.arch in self.toolchains:
297 add_it = (toolchain.priority <
298 self.toolchains[toolchain.arch].priority)
299 if add_it:
300 self.toolchains[toolchain.arch] = toolchain
Simon Glassff690df2016-03-06 19:45:37 -0700301 elif verbose:
Simon Glassc05aa032019-10-31 07:42:53 -0600302 print(("Toolchain '%s' at priority %d will be ignored because "
Simon Glassff690df2016-03-06 19:45:37 -0700303 "another toolchain for arch '%s' has priority %d" %
304 (toolchain.gcc, toolchain.priority, toolchain.arch,
Simon Glassc05aa032019-10-31 07:42:53 -0600305 self.toolchains[toolchain.arch].priority)))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000306
Simon Glass827e37b2014-12-01 17:34:06 -0700307 def ScanPath(self, path, verbose):
308 """Scan a path for a valid toolchain
309
310 Args:
311 path: Path to scan
312 verbose: True to print out progress information
313 Returns:
314 Filename of C compiler if found, else None
315 """
Albert ARIBAUDd9088982015-02-01 00:12:44 +0100316 fnames = []
Simon Glass827e37b2014-12-01 17:34:06 -0700317 for subdir in ['.', 'bin', 'usr/bin']:
318 dirname = os.path.join(path, subdir)
Simon Glassc05aa032019-10-31 07:42:53 -0600319 if verbose: print(" - looking in '%s'" % dirname)
Simon Glass827e37b2014-12-01 17:34:06 -0700320 for fname in glob.glob(dirname + '/*gcc'):
Simon Glassc05aa032019-10-31 07:42:53 -0600321 if verbose: print(" - found '%s'" % fname)
Albert ARIBAUDd9088982015-02-01 00:12:44 +0100322 fnames.append(fname)
323 return fnames
Simon Glass827e37b2014-12-01 17:34:06 -0700324
Simon Glass17bce662016-03-12 18:50:32 -0700325 def ScanPathEnv(self, fname):
326 """Scan the PATH environment variable for a given filename.
327
328 Args:
329 fname: Filename to scan for
330 Returns:
331 List of matching pathanames, or [] if none
332 """
333 pathname_list = []
334 for path in os.environ["PATH"].split(os.pathsep):
335 path = path.strip('"')
336 pathname = os.path.join(path, fname)
337 if os.path.exists(pathname):
338 pathname_list.append(pathname)
339 return pathname_list
Simon Glass827e37b2014-12-01 17:34:06 -0700340
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000341 def Scan(self, verbose):
342 """Scan for available toolchains and select the best for each arch.
343
344 We look for all the toolchains we can file, figure out the
345 architecture for each, and whether it works. Then we select the
346 highest priority toolchain for each arch.
347
348 Args:
349 verbose: True to print out progress information
350 """
Simon Glassc05aa032019-10-31 07:42:53 -0600351 if verbose: print('Scanning for tool chains')
Simon Glass17bce662016-03-12 18:50:32 -0700352 for name, value in self.prefixes:
Simon Glassc05aa032019-10-31 07:42:53 -0600353 if verbose: print(" - scanning prefix '%s'" % value)
Simon Glass17bce662016-03-12 18:50:32 -0700354 if os.path.exists(value):
355 self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
356 continue
357 fname = value + 'gcc'
358 if os.path.exists(fname):
359 self.Add(fname, True, verbose, PRIORITY_PREFIX_GCC, name)
360 continue
361 fname_list = self.ScanPathEnv(fname)
362 for f in fname_list:
363 self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
364 if not fname_list:
Simon Glassc05aa032019-10-31 07:42:53 -0600365 raise ValueError("No tool chain found for prefix '%s'" %
Simon Glass17bce662016-03-12 18:50:32 -0700366 value)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000367 for path in self.paths:
Simon Glassc05aa032019-10-31 07:42:53 -0600368 if verbose: print(" - scanning path '%s'" % path)
Albert ARIBAUDd9088982015-02-01 00:12:44 +0100369 fnames = self.ScanPath(path, verbose)
370 for fname in fnames:
Simon Glass827e37b2014-12-01 17:34:06 -0700371 self.Add(fname, True, verbose)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000372
373 def List(self):
374 """List out the selected toolchains for each architecture"""
Simon Glass713bea32016-07-27 20:33:02 -0600375 col = terminal.Color()
Simon Glassc05aa032019-10-31 07:42:53 -0600376 print(col.Color(col.BLUE, 'List of available toolchains (%d):' %
377 len(self.toolchains)))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000378 if len(self.toolchains):
Simon Glassc05aa032019-10-31 07:42:53 -0600379 for key, value in sorted(self.toolchains.items()):
380 print('%-10s: %s' % (key, value.gcc))
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000381 else:
Simon Glassc05aa032019-10-31 07:42:53 -0600382 print('None')
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000383
384 def Select(self, arch):
385 """Returns the toolchain for a given architecture
386
387 Args:
388 args: Name of architecture (e.g. 'arm', 'ppc_8xx')
389
390 returns:
391 toolchain object, or None if none found
392 """
Simon Glass9b83bfd2014-12-01 17:34:05 -0700393 for tag, value in bsettings.GetItems('toolchain-alias'):
394 if arch == tag:
395 for alias in value.split():
396 if alias in self.toolchains:
397 return self.toolchains[alias]
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000398
399 if not arch in self.toolchains:
Simon Glassc05aa032019-10-31 07:42:53 -0600400 raise ValueError("No tool chain found for arch '%s'" % arch)
Simon Glassfc3fe1c2013-04-03 11:07:16 +0000401 return self.toolchains[arch]
Simon Glass4281ad82013-09-23 17:35:17 -0600402
403 def ResolveReferences(self, var_dict, args):
404 """Resolve variable references in a string
405
406 This converts ${blah} within the string to the value of blah.
407 This function works recursively.
408
409 Args:
410 var_dict: Dictionary containing variables and their values
411 args: String containing make arguments
412 Returns:
413 Resolved string
414
415 >>> bsettings.Setup()
416 >>> tcs = Toolchains()
417 >>> tcs.Add('fred', False)
418 >>> var_dict = {'oblique' : 'OBLIQUE', 'first' : 'fi${second}rst', \
419 'second' : '2nd'}
420 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set')
421 'this=OBLIQUE_set'
422 >>> tcs.ResolveReferences(var_dict, 'this=${oblique}_set${first}nd')
423 'this=OBLIQUE_setfi2ndrstnd'
424 """
Simon Glassf60c9d42014-08-28 09:43:40 -0600425 re_var = re.compile('(\$\{[-_a-z0-9A-Z]{1,}\})')
Simon Glass4281ad82013-09-23 17:35:17 -0600426
427 while True:
428 m = re_var.search(args)
429 if not m:
430 break
431 lookup = m.group(0)[2:-1]
432 value = var_dict.get(lookup, '')
433 args = args[:m.start(0)] + value + args[m.end(0):]
434 return args
435
436 def GetMakeArguments(self, board):
437 """Returns 'make' arguments for a given board
438
439 The flags are in a section called 'make-flags'. Flags are named
440 after the target they represent, for example snapper9260=TESTING=1
441 will pass TESTING=1 to make when building the snapper9260 board.
442
443 References to other boards can be added in the string also. For
444 example:
445
446 [make-flags]
447 at91-boards=ENABLE_AT91_TEST=1
448 snapper9260=${at91-boards} BUILD_TAG=442
449 snapper9g45=${at91-boards} BUILD_TAG=443
450
451 This will return 'ENABLE_AT91_TEST=1 BUILD_TAG=442' for snapper9260
452 and 'ENABLE_AT91_TEST=1 BUILD_TAG=443' for snapper9g45.
453
454 A special 'target' variable is set to the board target.
455
456 Args:
457 board: Board object for the board to check.
458 Returns:
459 'make' flags for that board, or '' if none
460 """
461 self._make_flags['target'] = board.target
462 arg_str = self.ResolveReferences(self._make_flags,
463 self._make_flags.get(board.target, ''))
Cristian Ciocaltea4251fbc2019-11-24 22:30:26 +0200464 args = re.findall("(?:\".*?\"|\S)+", arg_str)
Simon Glass4281ad82013-09-23 17:35:17 -0600465 i = 0
466 while i < len(args):
Cristian Ciocaltea4251fbc2019-11-24 22:30:26 +0200467 args[i] = args[i].replace('"', '')
Simon Glass4281ad82013-09-23 17:35:17 -0600468 if not args[i]:
469 del args[i]
470 else:
471 i += 1
472 return args
Simon Glass827e37b2014-12-01 17:34:06 -0700473
474 def LocateArchUrl(self, fetch_arch):
475 """Find a toolchain available online
476
477 Look in standard places for available toolchains. At present the
478 only standard place is at kernel.org.
479
480 Args:
481 arch: Architecture to look for, or 'list' for all
482 Returns:
483 If fetch_arch is 'list', a tuple:
484 Machine architecture (e.g. x86_64)
485 List of toolchains
486 else
487 URL containing this toolchain, if avaialble, else None
488 """
489 arch = command.OutputOneLine('uname', '-m')
Matthias Brugger2f7c53c2020-01-17 10:53:37 +0100490 if arch == 'aarch64':
491 arch = 'arm64'
Simon Glass827e37b2014-12-01 17:34:06 -0700492 base = 'https://www.kernel.org/pub/tools/crosstool/files/bin'
Bin Meng205254e2020-04-06 06:06:59 -0700493 versions = ['9.2.0', '7.3.0', '6.4.0', '4.9.4']
Simon Glass827e37b2014-12-01 17:34:06 -0700494 links = []
495 for version in versions:
496 url = '%s/%s/%s/' % (base, arch, version)
Simon Glassc05aa032019-10-31 07:42:53 -0600497 print('Checking: %s' % url)
498 response = urllib.request.urlopen(url)
499 html = tools.ToString(response.read())
Simon Glass827e37b2014-12-01 17:34:06 -0700500 parser = MyHTMLParser(fetch_arch)
501 parser.feed(html)
502 if fetch_arch == 'list':
503 links += parser.links
504 elif parser.arch_link:
505 return url + parser.arch_link
506 if fetch_arch == 'list':
507 return arch, links
508 return None
509
510 def Download(self, url):
511 """Download a file to a temporary directory
512
513 Args:
514 url: URL to download
515 Returns:
516 Tuple:
517 Temporary directory name
518 Full path to the downloaded archive file in that directory,
519 or None if there was an error while downloading
520 """
Simon Glassc05aa032019-10-31 07:42:53 -0600521 print('Downloading: %s' % url)
Simon Glass827e37b2014-12-01 17:34:06 -0700522 leaf = url.split('/')[-1]
523 tmpdir = tempfile.mkdtemp('.buildman')
Simon Glassc05aa032019-10-31 07:42:53 -0600524 response = urllib.request.urlopen(url)
Simon Glass827e37b2014-12-01 17:34:06 -0700525 fname = os.path.join(tmpdir, leaf)
526 fd = open(fname, 'wb')
527 meta = response.info()
Simon Glassc05aa032019-10-31 07:42:53 -0600528 size = int(meta.get('Content-Length'))
Simon Glass827e37b2014-12-01 17:34:06 -0700529 done = 0
530 block_size = 1 << 16
531 status = ''
532
533 # Read the file in chunks and show progress as we go
534 while True:
535 buffer = response.read(block_size)
536 if not buffer:
Simon Glassc05aa032019-10-31 07:42:53 -0600537 print(chr(8) * (len(status) + 1), '\r', end=' ')
Simon Glass827e37b2014-12-01 17:34:06 -0700538 break
539
540 done += len(buffer)
541 fd.write(buffer)
Simon Glassc05aa032019-10-31 07:42:53 -0600542 status = r'%10d MiB [%3d%%]' % (done // 1024 // 1024,
543 done * 100 // size)
Simon Glass827e37b2014-12-01 17:34:06 -0700544 status = status + chr(8) * (len(status) + 1)
Simon Glassc05aa032019-10-31 07:42:53 -0600545 print(status, end=' ')
Simon Glass827e37b2014-12-01 17:34:06 -0700546 sys.stdout.flush()
547 fd.close()
548 if done != size:
Simon Glassc05aa032019-10-31 07:42:53 -0600549 print('Error, failed to download')
Simon Glass827e37b2014-12-01 17:34:06 -0700550 os.remove(fname)
551 fname = None
552 return tmpdir, fname
553
554 def Unpack(self, fname, dest):
555 """Unpack a tar file
556
557 Args:
558 fname: Filename to unpack
559 dest: Destination directory
560 Returns:
561 Directory name of the first entry in the archive, without the
562 trailing /
563 """
564 stdout = command.Output('tar', 'xvfJ', fname, '-C', dest)
Trevor Woernerd82f5392018-11-21 03:31:12 -0500565 dirs = stdout.splitlines()[1].split('/')[:2]
566 return '/'.join(dirs)
Simon Glass827e37b2014-12-01 17:34:06 -0700567
568 def TestSettingsHasPath(self, path):
Simon Glass2289b272016-07-27 20:33:03 -0600569 """Check if buildman will find this toolchain
Simon Glass827e37b2014-12-01 17:34:06 -0700570
571 Returns:
572 True if the path is in settings, False if not
573 """
Simon Glass80e6a482016-07-27 20:33:01 -0600574 paths = self.GetPathList(False)
Simon Glass827e37b2014-12-01 17:34:06 -0700575 return path in paths
576
577 def ListArchs(self):
578 """List architectures with available toolchains to download"""
579 host_arch, archives = self.LocateArchUrl('list')
Trevor Woernerb11f1262018-11-21 03:31:13 -0500580 re_arch = re.compile('[-a-z0-9.]*[-_]([^-]*)-.*')
Simon Glass827e37b2014-12-01 17:34:06 -0700581 arch_set = set()
582 for archive in archives:
583 # Remove the host architecture from the start
584 arch = re_arch.match(archive[len(host_arch):])
585 if arch:
Trevor Woernerb11f1262018-11-21 03:31:13 -0500586 if arch.group(1) != '2.0' and arch.group(1) != '64':
587 arch_set.add(arch.group(1))
Simon Glass827e37b2014-12-01 17:34:06 -0700588 return sorted(arch_set)
589
590 def FetchAndInstall(self, arch):
591 """Fetch and install a new toolchain
592
593 arch:
594 Architecture to fetch, or 'list' to list
595 """
596 # Fist get the URL for this architecture
Simon Glass713bea32016-07-27 20:33:02 -0600597 col = terminal.Color()
Simon Glassc05aa032019-10-31 07:42:53 -0600598 print(col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch))
Simon Glass827e37b2014-12-01 17:34:06 -0700599 url = self.LocateArchUrl(arch)
600 if not url:
Simon Glassc05aa032019-10-31 07:42:53 -0600601 print(("Cannot find toolchain for arch '%s' - use 'list' to list" %
602 arch))
Simon Glass827e37b2014-12-01 17:34:06 -0700603 return 2
604 home = os.environ['HOME']
605 dest = os.path.join(home, '.buildman-toolchains')
606 if not os.path.exists(dest):
607 os.mkdir(dest)
608
609 # Download the tar file for this toolchain and unpack it
610 tmpdir, tarfile = self.Download(url)
611 if not tarfile:
612 return 1
Simon Glassc05aa032019-10-31 07:42:53 -0600613 print(col.Color(col.GREEN, 'Unpacking to: %s' % dest), end=' ')
Simon Glass827e37b2014-12-01 17:34:06 -0700614 sys.stdout.flush()
615 path = self.Unpack(tarfile, dest)
616 os.remove(tarfile)
617 os.rmdir(tmpdir)
Simon Glassc05aa032019-10-31 07:42:53 -0600618 print()
Simon Glass827e37b2014-12-01 17:34:06 -0700619
620 # Check that the toolchain works
Simon Glassc05aa032019-10-31 07:42:53 -0600621 print(col.Color(col.GREEN, 'Testing'))
Simon Glass827e37b2014-12-01 17:34:06 -0700622 dirpath = os.path.join(dest, path)
Simon Glass2a76a642015-03-02 17:05:15 -0700623 compiler_fname_list = self.ScanPath(dirpath, True)
624 if not compiler_fname_list:
Simon Glassc05aa032019-10-31 07:42:53 -0600625 print('Could not locate C compiler - fetch failed.')
Simon Glass827e37b2014-12-01 17:34:06 -0700626 return 1
Simon Glass2a76a642015-03-02 17:05:15 -0700627 if len(compiler_fname_list) != 1:
Simon Glassc05aa032019-10-31 07:42:53 -0600628 print(col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
629 ', '.join(compiler_fname_list)))
Simon Glass2a76a642015-03-02 17:05:15 -0700630 toolchain = Toolchain(compiler_fname_list[0], True, True)
Simon Glass827e37b2014-12-01 17:34:06 -0700631
632 # Make sure that it will be found by buildman
633 if not self.TestSettingsHasPath(dirpath):
Simon Glassc05aa032019-10-31 07:42:53 -0600634 print(("Adding 'download' to config file '%s'" %
635 bsettings.config_fname))
Simon Glassc8785c52016-07-27 20:33:05 -0600636 bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)
Simon Glass827e37b2014-12-01 17:34:06 -0700637 return 0