blob: 24df13e5008d820d1b3de19dc71457e8cd3644df [file] [log] [blame]
Tom Rini3bc14092019-09-20 17:42:07 -04001#!/usr/bin/env python3
Tom Rini83d290c2018-05-06 17:58:06 -04002# SPDX-License-Identifier: GPL-2.0+
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +09003#
4# Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
5#
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +09006
7"""
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +09008Converter from Kconfig and MAINTAINERS to a board database.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +09009
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090010Run 'tools/genboardscfg.py' to create a board database.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090011
12Run 'tools/genboardscfg.py -h' for available options.
Masahiro Yamada21343422014-08-27 14:05:51 +090013
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090014Python 2.6 or later, but not Python 3.x is necessary to run this script.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090015"""
16
17import errno
18import fnmatch
19import glob
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090020import multiprocessing
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090021import optparse
22import os
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090023import sys
24import tempfile
25import time
26
Peng Fan60801702018-07-24 14:27:18 +080027sys.path.insert(1, os.path.join(os.path.dirname(__file__), 'buildman'))
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090028import kconfiglib
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090029
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090030### constant variables ###
31OUTPUT_FILE = 'boards.cfg'
32CONFIG_DIR = 'configs'
33SLEEP_TIME = 0.03
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090034COMMENT_BLOCK = '''#
35# List of boards
36# Automatically generated by %s: don't edit
37#
Masahiro Yamadaca418dd2014-08-06 13:42:34 +090038# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090039
40''' % __file__
41
42### helper functions ###
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090043def try_remove(f):
44 """Remove a file ignoring 'No such file or directory' error."""
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090045 try:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090046 os.remove(f)
47 except OSError as exception:
48 # Ignore 'No such file or directory' error
49 if exception.errno != errno.ENOENT:
50 raise
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090051
52def check_top_directory():
53 """Exit if we are not at the top of source directory."""
54 for f in ('README', 'Licenses'):
55 if not os.path.exists(f):
Masahiro Yamada31e21412014-08-16 00:59:26 +090056 sys.exit('Please run at the top of source directory.')
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090057
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090058def output_is_new(output):
59 """Check if the output file is up to date.
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090060
61 Returns:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090062 True if the given output file exists and is newer than any of
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090063 *_defconfig, MAINTAINERS and Kconfig*. False otherwise.
64 """
65 try:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090066 ctime = os.path.getctime(output)
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090067 except OSError as exception:
68 if exception.errno == errno.ENOENT:
69 # return False on 'No such file or directory' error
70 return False
71 else:
72 raise
73
74 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
75 for filename in fnmatch.filter(filenames, '*_defconfig'):
76 if fnmatch.fnmatch(filename, '.*'):
77 continue
78 filepath = os.path.join(dirpath, filename)
79 if ctime < os.path.getctime(filepath):
80 return False
81
82 for (dirpath, dirnames, filenames) in os.walk('.'):
83 for filename in filenames:
84 if (fnmatch.fnmatch(filename, '*~') or
85 not fnmatch.fnmatch(filename, 'Kconfig*') and
86 not filename == 'MAINTAINERS'):
87 continue
88 filepath = os.path.join(dirpath, filename)
89 if ctime < os.path.getctime(filepath):
90 return False
91
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090092 # Detect a board that has been removed since the current board database
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090093 # was generated
Tom Rini3bc14092019-09-20 17:42:07 -040094 with open(output, encoding="utf-8") as f:
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090095 for line in f:
96 if line[0] == '#' or line == '\n':
97 continue
98 defconfig = line.split()[6] + '_defconfig'
99 if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
100 return False
101
102 return True
103
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900104### classes ###
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900105class KconfigScanner:
106
107 """Kconfig scanner."""
108
109 ### constant variable only used in this class ###
110 _SYMBOL_TABLE = {
111 'arch' : 'SYS_ARCH',
112 'cpu' : 'SYS_CPU',
113 'soc' : 'SYS_SOC',
114 'vendor' : 'SYS_VENDOR',
115 'board' : 'SYS_BOARD',
116 'config' : 'SYS_CONFIG_NAME',
117 'options' : 'SYS_EXTRA_OPTIONS'
118 }
119
120 def __init__(self):
Tom Rini65e05dd2019-09-20 17:42:09 -0400121 """Scan all the Kconfig files and create a Kconfig object."""
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900122 # Define environment variables referenced from Kconfig
123 os.environ['srctree'] = os.getcwd()
124 os.environ['UBOOTVERSION'] = 'dummy'
125 os.environ['KCONFIG_OBJDIR'] = ''
Tom Rini65e05dd2019-09-20 17:42:09 -0400126 self._conf = kconfiglib.Kconfig(warn=False)
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900127
128 def __del__(self):
129 """Delete a leftover temporary file before exit.
130
131 The scan() method of this class creates a temporay file and deletes
132 it on success. If scan() method throws an exception on the way,
133 the temporary file might be left over. In that case, it should be
134 deleted in this destructor.
135 """
136 if hasattr(self, '_tmpfile') and self._tmpfile:
137 try_remove(self._tmpfile)
138
139 def scan(self, defconfig):
140 """Load a defconfig file to obtain board parameters.
141
142 Arguments:
143 defconfig: path to the defconfig file to be processed
144
145 Returns:
146 A dictionary of board parameters. It has a form of:
147 {
148 'arch': <arch_name>,
149 'cpu': <cpu_name>,
150 'soc': <soc_name>,
151 'vendor': <vendor_name>,
152 'board': <board_name>,
153 'target': <target_name>,
154 'config': <config_header_name>,
155 'options': <extra_options>
156 }
157 """
158 # strip special prefixes and save it in a temporary file
159 fd, self._tmpfile = tempfile.mkstemp()
160 with os.fdopen(fd, 'w') as f:
161 for line in open(defconfig):
162 colon = line.find(':CONFIG_')
163 if colon == -1:
164 f.write(line)
165 else:
166 f.write(line[colon + 1:])
167
Tom Rini5e7c8a32019-09-20 17:42:08 -0400168 self._conf.load_config(self._tmpfile)
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900169 try_remove(self._tmpfile)
170 self._tmpfile = None
171
172 params = {}
173
174 # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
175 # Set '-' if the value is empty.
Tom Rini3bc14092019-09-20 17:42:07 -0400176 for key, symbol in list(self._SYMBOL_TABLE.items()):
Tom Rini65e05dd2019-09-20 17:42:09 -0400177 value = self._conf.syms.get(symbol).str_value
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900178 if value:
179 params[key] = value
180 else:
181 params[key] = '-'
182
183 defconfig = os.path.basename(defconfig)
184 params['target'], match, rear = defconfig.partition('_defconfig')
185 assert match and not rear, '%s : invalid defconfig' % defconfig
186
187 # fix-up for aarch64
188 if params['arch'] == 'arm' and params['cpu'] == 'armv8':
189 params['arch'] = 'aarch64'
190
191 # fix-up options field. It should have the form:
192 # <config name>[:comma separated config options]
193 if params['options'] != '-':
194 params['options'] = params['config'] + ':' + \
195 params['options'].replace(r'\"', '"')
196 elif params['config'] != params['target']:
197 params['options'] = params['config']
198
199 return params
200
201def scan_defconfigs_for_multiprocess(queue, defconfigs):
202 """Scan defconfig files and queue their board parameters
203
204 This function is intended to be passed to
205 multiprocessing.Process() constructor.
206
207 Arguments:
208 queue: An instance of multiprocessing.Queue().
209 The resulting board parameters are written into it.
210 defconfigs: A sequence of defconfig files to be scanned.
211 """
212 kconf_scanner = KconfigScanner()
213 for defconfig in defconfigs:
214 queue.put(kconf_scanner.scan(defconfig))
215
216def read_queues(queues, params_list):
217 """Read the queues and append the data to the paramers list"""
218 for q in queues:
219 while not q.empty():
220 params_list.append(q.get())
221
222def scan_defconfigs(jobs=1):
223 """Collect board parameters for all defconfig files.
224
225 This function invokes multiple processes for faster processing.
226
227 Arguments:
228 jobs: The number of jobs to run simultaneously
229 """
230 all_defconfigs = []
231 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
232 for filename in fnmatch.filter(filenames, '*_defconfig'):
233 if fnmatch.fnmatch(filename, '.*'):
234 continue
235 all_defconfigs.append(os.path.join(dirpath, filename))
236
237 total_boards = len(all_defconfigs)
238 processes = []
239 queues = []
240 for i in range(jobs):
Tom Rini3bc14092019-09-20 17:42:07 -0400241 defconfigs = all_defconfigs[total_boards * i // jobs :
242 total_boards * (i + 1) // jobs]
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900243 q = multiprocessing.Queue(maxsize=-1)
244 p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
245 args=(q, defconfigs))
246 p.start()
247 processes.append(p)
248 queues.append(q)
249
250 # The resulting data should be accumulated to this list
251 params_list = []
252
253 # Data in the queues should be retrieved preriodically.
254 # Otherwise, the queues would become full and subprocesses would get stuck.
255 while any([p.is_alive() for p in processes]):
256 read_queues(queues, params_list)
257 # sleep for a while until the queues are filled
258 time.sleep(SLEEP_TIME)
259
260 # Joining subprocesses just in case
261 # (All subprocesses should already have been finished)
262 for p in processes:
263 p.join()
264
265 # retrieve leftover data
266 read_queues(queues, params_list)
267
268 return params_list
269
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900270class MaintainersDatabase:
271
272 """The database of board status and maintainers."""
273
274 def __init__(self):
275 """Create an empty database."""
276 self.database = {}
277
278 def get_status(self, target):
279 """Return the status of the given board.
280
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900281 The board status is generally either 'Active' or 'Orphan'.
282 Display a warning message and return '-' if status information
283 is not found.
284
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900285 Returns:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900286 'Active', 'Orphan' or '-'.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900287 """
Masahiro Yamadab8828e82014-08-25 12:39:43 +0900288 if not target in self.database:
Tom Rini3bc14092019-09-20 17:42:07 -0400289 print("WARNING: no status info for '%s'" % target, file=sys.stderr)
Masahiro Yamadab8828e82014-08-25 12:39:43 +0900290 return '-'
291
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900292 tmp = self.database[target][0]
293 if tmp.startswith('Maintained'):
294 return 'Active'
Lokesh Vutladee7c682017-05-10 16:19:52 +0530295 elif tmp.startswith('Supported'):
296 return 'Active'
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900297 elif tmp.startswith('Orphan'):
298 return 'Orphan'
299 else:
Tom Rini3bc14092019-09-20 17:42:07 -0400300 print(("WARNING: %s: unknown status for '%s'" %
301 (tmp, target)), file=sys.stderr)
Masahiro Yamadab8828e82014-08-25 12:39:43 +0900302 return '-'
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900303
304 def get_maintainers(self, target):
305 """Return the maintainers of the given board.
306
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900307 Returns:
308 Maintainers of the board. If the board has two or more maintainers,
309 they are separated with colons.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900310 """
Masahiro Yamadab8828e82014-08-25 12:39:43 +0900311 if not target in self.database:
Tom Rini3bc14092019-09-20 17:42:07 -0400312 print("WARNING: no maintainers for '%s'" % target, file=sys.stderr)
Masahiro Yamadab8828e82014-08-25 12:39:43 +0900313 return ''
314
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900315 return ':'.join(self.database[target][1])
316
317 def parse_file(self, file):
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900318 """Parse a MAINTAINERS file.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900319
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900320 Parse a MAINTAINERS file and accumulates board status and
321 maintainers information.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900322
323 Arguments:
324 file: MAINTAINERS file to be parsed
325 """
326 targets = []
327 maintainers = []
328 status = '-'
Tom Rini3bc14092019-09-20 17:42:07 -0400329 for line in open(file, encoding="utf-8"):
Masahiro Yamada5dff8442014-09-16 14:11:49 +0900330 # Check also commented maintainers
331 if line[:3] == '#M:':
332 line = line[1:]
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900333 tag, rest = line[:2], line[2:].strip()
334 if tag == 'M:':
335 maintainers.append(rest)
336 elif tag == 'F:':
337 # expand wildcard and filter by 'configs/*_defconfig'
338 for f in glob.glob(rest):
339 front, match, rear = f.partition('configs/')
340 if not front and match:
341 front, match, rear = rear.rpartition('_defconfig')
342 if match and not rear:
343 targets.append(front)
344 elif tag == 'S:':
345 status = rest
Masahiro Yamada9c2d60c2014-08-22 14:10:43 +0900346 elif line == '\n':
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900347 for target in targets:
348 self.database[target] = (status, maintainers)
349 targets = []
350 maintainers = []
351 status = '-'
352 if targets:
353 for target in targets:
354 self.database[target] = (status, maintainers)
355
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900356def insert_maintainers_info(params_list):
357 """Add Status and Maintainers information to the board parameters list.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900358
359 Arguments:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900360 params_list: A list of the board parameters
361 """
362 database = MaintainersDatabase()
363 for (dirpath, dirnames, filenames) in os.walk('.'):
364 if 'MAINTAINERS' in filenames:
365 database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
366
367 for i, params in enumerate(params_list):
368 target = params['target']
369 params['status'] = database.get_status(target)
370 params['maintainers'] = database.get_maintainers(target)
371 params_list[i] = params
372
373def format_and_output(params_list, output):
374 """Write board parameters into a file.
375
376 Columnate the board parameters, sort lines alphabetically,
377 and then write them to a file.
378
379 Arguments:
380 params_list: The list of board parameters
381 output: The path to the output file
382 """
383 FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
384 'options', 'maintainers')
385
386 # First, decide the width of each column
387 max_length = dict([ (f, 0) for f in FIELDS])
388 for params in params_list:
389 for f in FIELDS:
390 max_length[f] = max(max_length[f], len(params[f]))
391
392 output_lines = []
393 for params in params_list:
394 line = ''
395 for f in FIELDS:
396 # insert two spaces between fields like column -t would
397 line += ' ' + params[f].ljust(max_length[f])
398 output_lines.append(line.strip())
399
400 # ignore case when sorting
401 output_lines.sort(key=str.lower)
402
Tom Rini3bc14092019-09-20 17:42:07 -0400403 with open(output, 'w', encoding="utf-8") as f:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900404 f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
405
Simon Glass69bbdd12019-12-05 15:59:11 -0700406def gen_boards_cfg(output, jobs=1, force=False, quiet=False):
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900407 """Generate a board database file.
408
409 Arguments:
410 output: The name of the output file
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900411 jobs: The number of jobs to run simultaneously
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900412 force: Force to generate the output even if it is new
Simon Glass69bbdd12019-12-05 15:59:11 -0700413 quiet: True to avoid printing a message if nothing needs doing
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900414 """
Masahiro Yamada79d45d32014-08-25 12:39:46 +0900415 check_top_directory()
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900416
417 if not force and output_is_new(output):
Simon Glass69bbdd12019-12-05 15:59:11 -0700418 if not quiet:
419 print("%s is up to date. Nothing to do." % output)
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +0900420 sys.exit(0)
421
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900422 params_list = scan_defconfigs(jobs)
423 insert_maintainers_info(params_list)
424 format_and_output(params_list, output)
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900425
426def main():
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900427 try:
428 cpu_count = multiprocessing.cpu_count()
429 except NotImplementedError:
430 cpu_count = 1
431
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900432 parser = optparse.OptionParser()
433 # Add options here
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +0900434 parser.add_option('-f', '--force', action="store_true", default=False,
435 help='regenerate the output even if it is new')
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900436 parser.add_option('-j', '--jobs', type='int', default=cpu_count,
437 help='the number of jobs to run simultaneously')
438 parser.add_option('-o', '--output', default=OUTPUT_FILE,
439 help='output file [default=%s]' % OUTPUT_FILE)
Simon Glass69bbdd12019-12-05 15:59:11 -0700440 parser.add_option('-q', '--quiet', action="store_true", help='run silently')
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900441 (options, args) = parser.parse_args()
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +0900442
Simon Glass69bbdd12019-12-05 15:59:11 -0700443 gen_boards_cfg(options.output, jobs=options.jobs, force=options.force,
444 quiet=options.quiet)
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900445
446if __name__ == '__main__':
447 main()