blob: 07bf681d1d955518ec630c9b79bdd48b2a80c928 [file] [log] [blame]
Tom Rini3bc14092019-09-20 17:42:07 -04001#!/usr/bin/env python3
Tom Rini83d290c2018-05-06 17:58:06 -04002# SPDX-License-Identifier: GPL-2.0+
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +09003#
4# Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
5#
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +09006
7"""
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +09008Converter from Kconfig and MAINTAINERS to a board database.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +09009
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090010Run 'tools/genboardscfg.py' to create a board database.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090011
12Run 'tools/genboardscfg.py -h' for available options.
13"""
14
15import errno
16import fnmatch
17import glob
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090018import multiprocessing
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090019import optparse
20import os
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090021import sys
22import tempfile
23import time
24
Simon Glassb4fa9492020-04-17 18:09:05 -060025from buildman import kconfiglib
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090026
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090027### constant variables ###
28OUTPUT_FILE = 'boards.cfg'
29CONFIG_DIR = 'configs'
30SLEEP_TIME = 0.03
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090031COMMENT_BLOCK = '''#
32# List of boards
33# Automatically generated by %s: don't edit
34#
Masahiro Yamadaca418dd2014-08-06 13:42:34 +090035# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090036
37''' % __file__
38
39### helper functions ###
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090040def try_remove(f):
41 """Remove a file ignoring 'No such file or directory' error."""
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090042 try:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090043 os.remove(f)
44 except OSError as exception:
45 # Ignore 'No such file or directory' error
46 if exception.errno != errno.ENOENT:
47 raise
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090048
49def check_top_directory():
50 """Exit if we are not at the top of source directory."""
51 for f in ('README', 'Licenses'):
52 if not os.path.exists(f):
Masahiro Yamada31e21412014-08-16 00:59:26 +090053 sys.exit('Please run at the top of source directory.')
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090054
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090055def output_is_new(output):
56 """Check if the output file is up to date.
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090057
58 Returns:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090059 True if the given output file exists and is newer than any of
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090060 *_defconfig, MAINTAINERS and Kconfig*. False otherwise.
61 """
62 try:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090063 ctime = os.path.getctime(output)
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090064 except OSError as exception:
65 if exception.errno == errno.ENOENT:
66 # return False on 'No such file or directory' error
67 return False
68 else:
69 raise
70
71 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
72 for filename in fnmatch.filter(filenames, '*_defconfig'):
73 if fnmatch.fnmatch(filename, '.*'):
74 continue
75 filepath = os.path.join(dirpath, filename)
76 if ctime < os.path.getctime(filepath):
77 return False
78
79 for (dirpath, dirnames, filenames) in os.walk('.'):
80 for filename in filenames:
81 if (fnmatch.fnmatch(filename, '*~') or
82 not fnmatch.fnmatch(filename, 'Kconfig*') and
83 not filename == 'MAINTAINERS'):
84 continue
85 filepath = os.path.join(dirpath, filename)
86 if ctime < os.path.getctime(filepath):
87 return False
88
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090089 # Detect a board that has been removed since the current board database
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090090 # was generated
Tom Rini3bc14092019-09-20 17:42:07 -040091 with open(output, encoding="utf-8") as f:
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090092 for line in f:
93 if line[0] == '#' or line == '\n':
94 continue
95 defconfig = line.split()[6] + '_defconfig'
96 if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
97 return False
98
99 return True
100
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900101### classes ###
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900102class KconfigScanner:
103
104 """Kconfig scanner."""
105
106 ### constant variable only used in this class ###
107 _SYMBOL_TABLE = {
108 'arch' : 'SYS_ARCH',
109 'cpu' : 'SYS_CPU',
110 'soc' : 'SYS_SOC',
111 'vendor' : 'SYS_VENDOR',
112 'board' : 'SYS_BOARD',
113 'config' : 'SYS_CONFIG_NAME',
114 'options' : 'SYS_EXTRA_OPTIONS'
115 }
116
117 def __init__(self):
Tom Rini65e05dd2019-09-20 17:42:09 -0400118 """Scan all the Kconfig files and create a Kconfig object."""
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900119 # Define environment variables referenced from Kconfig
120 os.environ['srctree'] = os.getcwd()
121 os.environ['UBOOTVERSION'] = 'dummy'
122 os.environ['KCONFIG_OBJDIR'] = ''
Tom Rini65e05dd2019-09-20 17:42:09 -0400123 self._conf = kconfiglib.Kconfig(warn=False)
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900124
125 def __del__(self):
126 """Delete a leftover temporary file before exit.
127
128 The scan() method of this class creates a temporay file and deletes
129 it on success. If scan() method throws an exception on the way,
130 the temporary file might be left over. In that case, it should be
131 deleted in this destructor.
132 """
133 if hasattr(self, '_tmpfile') and self._tmpfile:
134 try_remove(self._tmpfile)
135
136 def scan(self, defconfig):
137 """Load a defconfig file to obtain board parameters.
138
139 Arguments:
140 defconfig: path to the defconfig file to be processed
141
142 Returns:
143 A dictionary of board parameters. It has a form of:
144 {
145 'arch': <arch_name>,
146 'cpu': <cpu_name>,
147 'soc': <soc_name>,
148 'vendor': <vendor_name>,
149 'board': <board_name>,
150 'target': <target_name>,
151 'config': <config_header_name>,
152 'options': <extra_options>
153 }
154 """
155 # strip special prefixes and save it in a temporary file
156 fd, self._tmpfile = tempfile.mkstemp()
157 with os.fdopen(fd, 'w') as f:
158 for line in open(defconfig):
159 colon = line.find(':CONFIG_')
160 if colon == -1:
161 f.write(line)
162 else:
163 f.write(line[colon + 1:])
164
Tom Rini5e7c8a32019-09-20 17:42:08 -0400165 self._conf.load_config(self._tmpfile)
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900166 try_remove(self._tmpfile)
167 self._tmpfile = None
168
169 params = {}
170
171 # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
172 # Set '-' if the value is empty.
Tom Rini3bc14092019-09-20 17:42:07 -0400173 for key, symbol in list(self._SYMBOL_TABLE.items()):
Tom Rini65e05dd2019-09-20 17:42:09 -0400174 value = self._conf.syms.get(symbol).str_value
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900175 if value:
176 params[key] = value
177 else:
178 params[key] = '-'
179
180 defconfig = os.path.basename(defconfig)
181 params['target'], match, rear = defconfig.partition('_defconfig')
182 assert match and not rear, '%s : invalid defconfig' % defconfig
183
184 # fix-up for aarch64
185 if params['arch'] == 'arm' and params['cpu'] == 'armv8':
186 params['arch'] = 'aarch64'
187
188 # fix-up options field. It should have the form:
189 # <config name>[:comma separated config options]
190 if params['options'] != '-':
191 params['options'] = params['config'] + ':' + \
192 params['options'].replace(r'\"', '"')
193 elif params['config'] != params['target']:
194 params['options'] = params['config']
195
196 return params
197
198def scan_defconfigs_for_multiprocess(queue, defconfigs):
199 """Scan defconfig files and queue their board parameters
200
201 This function is intended to be passed to
202 multiprocessing.Process() constructor.
203
204 Arguments:
205 queue: An instance of multiprocessing.Queue().
206 The resulting board parameters are written into it.
207 defconfigs: A sequence of defconfig files to be scanned.
208 """
209 kconf_scanner = KconfigScanner()
210 for defconfig in defconfigs:
211 queue.put(kconf_scanner.scan(defconfig))
212
213def read_queues(queues, params_list):
214 """Read the queues and append the data to the paramers list"""
215 for q in queues:
216 while not q.empty():
217 params_list.append(q.get())
218
219def scan_defconfigs(jobs=1):
220 """Collect board parameters for all defconfig files.
221
222 This function invokes multiple processes for faster processing.
223
224 Arguments:
225 jobs: The number of jobs to run simultaneously
226 """
227 all_defconfigs = []
228 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
229 for filename in fnmatch.filter(filenames, '*_defconfig'):
230 if fnmatch.fnmatch(filename, '.*'):
231 continue
232 all_defconfigs.append(os.path.join(dirpath, filename))
233
234 total_boards = len(all_defconfigs)
235 processes = []
236 queues = []
237 for i in range(jobs):
Tom Rini3bc14092019-09-20 17:42:07 -0400238 defconfigs = all_defconfigs[total_boards * i // jobs :
239 total_boards * (i + 1) // jobs]
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900240 q = multiprocessing.Queue(maxsize=-1)
241 p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
242 args=(q, defconfigs))
243 p.start()
244 processes.append(p)
245 queues.append(q)
246
247 # The resulting data should be accumulated to this list
248 params_list = []
249
250 # Data in the queues should be retrieved preriodically.
251 # Otherwise, the queues would become full and subprocesses would get stuck.
252 while any([p.is_alive() for p in processes]):
253 read_queues(queues, params_list)
254 # sleep for a while until the queues are filled
255 time.sleep(SLEEP_TIME)
256
257 # Joining subprocesses just in case
258 # (All subprocesses should already have been finished)
259 for p in processes:
260 p.join()
261
262 # retrieve leftover data
263 read_queues(queues, params_list)
264
265 return params_list
266
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900267class MaintainersDatabase:
268
269 """The database of board status and maintainers."""
270
271 def __init__(self):
272 """Create an empty database."""
273 self.database = {}
274
275 def get_status(self, target):
276 """Return the status of the given board.
277
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900278 The board status is generally either 'Active' or 'Orphan'.
279 Display a warning message and return '-' if status information
280 is not found.
281
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900282 Returns:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900283 'Active', 'Orphan' or '-'.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900284 """
Masahiro Yamadab8828e82014-08-25 12:39:43 +0900285 if not target in self.database:
Tom Rini3bc14092019-09-20 17:42:07 -0400286 print("WARNING: no status info for '%s'" % target, file=sys.stderr)
Masahiro Yamadab8828e82014-08-25 12:39:43 +0900287 return '-'
288
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900289 tmp = self.database[target][0]
290 if tmp.startswith('Maintained'):
291 return 'Active'
Lokesh Vutladee7c682017-05-10 16:19:52 +0530292 elif tmp.startswith('Supported'):
293 return 'Active'
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900294 elif tmp.startswith('Orphan'):
295 return 'Orphan'
296 else:
Tom Rini3bc14092019-09-20 17:42:07 -0400297 print(("WARNING: %s: unknown status for '%s'" %
298 (tmp, target)), file=sys.stderr)
Masahiro Yamadab8828e82014-08-25 12:39:43 +0900299 return '-'
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900300
301 def get_maintainers(self, target):
302 """Return the maintainers of the given board.
303
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900304 Returns:
305 Maintainers of the board. If the board has two or more maintainers,
306 they are separated with colons.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900307 """
Masahiro Yamadab8828e82014-08-25 12:39:43 +0900308 if not target in self.database:
Tom Rini3bc14092019-09-20 17:42:07 -0400309 print("WARNING: no maintainers for '%s'" % target, file=sys.stderr)
Masahiro Yamadab8828e82014-08-25 12:39:43 +0900310 return ''
311
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900312 return ':'.join(self.database[target][1])
313
314 def parse_file(self, file):
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900315 """Parse a MAINTAINERS file.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900316
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900317 Parse a MAINTAINERS file and accumulates board status and
318 maintainers information.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900319
320 Arguments:
321 file: MAINTAINERS file to be parsed
322 """
323 targets = []
324 maintainers = []
325 status = '-'
Tom Rini3bc14092019-09-20 17:42:07 -0400326 for line in open(file, encoding="utf-8"):
Masahiro Yamada5dff8442014-09-16 14:11:49 +0900327 # Check also commented maintainers
328 if line[:3] == '#M:':
329 line = line[1:]
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900330 tag, rest = line[:2], line[2:].strip()
331 if tag == 'M:':
332 maintainers.append(rest)
333 elif tag == 'F:':
334 # expand wildcard and filter by 'configs/*_defconfig'
335 for f in glob.glob(rest):
336 front, match, rear = f.partition('configs/')
337 if not front and match:
338 front, match, rear = rear.rpartition('_defconfig')
339 if match and not rear:
340 targets.append(front)
341 elif tag == 'S:':
342 status = rest
Masahiro Yamada9c2d60c2014-08-22 14:10:43 +0900343 elif line == '\n':
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900344 for target in targets:
345 self.database[target] = (status, maintainers)
346 targets = []
347 maintainers = []
348 status = '-'
349 if targets:
350 for target in targets:
351 self.database[target] = (status, maintainers)
352
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900353def insert_maintainers_info(params_list):
354 """Add Status and Maintainers information to the board parameters list.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900355
356 Arguments:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900357 params_list: A list of the board parameters
358 """
359 database = MaintainersDatabase()
360 for (dirpath, dirnames, filenames) in os.walk('.'):
361 if 'MAINTAINERS' in filenames:
362 database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
363
364 for i, params in enumerate(params_list):
365 target = params['target']
366 params['status'] = database.get_status(target)
367 params['maintainers'] = database.get_maintainers(target)
368 params_list[i] = params
369
370def format_and_output(params_list, output):
371 """Write board parameters into a file.
372
373 Columnate the board parameters, sort lines alphabetically,
374 and then write them to a file.
375
376 Arguments:
377 params_list: The list of board parameters
378 output: The path to the output file
379 """
380 FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
381 'options', 'maintainers')
382
383 # First, decide the width of each column
384 max_length = dict([ (f, 0) for f in FIELDS])
385 for params in params_list:
386 for f in FIELDS:
387 max_length[f] = max(max_length[f], len(params[f]))
388
389 output_lines = []
390 for params in params_list:
391 line = ''
392 for f in FIELDS:
393 # insert two spaces between fields like column -t would
394 line += ' ' + params[f].ljust(max_length[f])
395 output_lines.append(line.strip())
396
397 # ignore case when sorting
398 output_lines.sort(key=str.lower)
399
Tom Rini3bc14092019-09-20 17:42:07 -0400400 with open(output, 'w', encoding="utf-8") as f:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900401 f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
402
Simon Glass69bbdd12019-12-05 15:59:11 -0700403def gen_boards_cfg(output, jobs=1, force=False, quiet=False):
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900404 """Generate a board database file.
405
406 Arguments:
407 output: The name of the output file
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900408 jobs: The number of jobs to run simultaneously
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900409 force: Force to generate the output even if it is new
Simon Glass69bbdd12019-12-05 15:59:11 -0700410 quiet: True to avoid printing a message if nothing needs doing
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900411 """
Masahiro Yamada79d45d32014-08-25 12:39:46 +0900412 check_top_directory()
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900413
414 if not force and output_is_new(output):
Simon Glass69bbdd12019-12-05 15:59:11 -0700415 if not quiet:
416 print("%s is up to date. Nothing to do." % output)
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +0900417 sys.exit(0)
418
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900419 params_list = scan_defconfigs(jobs)
420 insert_maintainers_info(params_list)
421 format_and_output(params_list, output)
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900422
423def main():
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900424 try:
425 cpu_count = multiprocessing.cpu_count()
426 except NotImplementedError:
427 cpu_count = 1
428
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900429 parser = optparse.OptionParser()
430 # Add options here
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +0900431 parser.add_option('-f', '--force', action="store_true", default=False,
432 help='regenerate the output even if it is new')
Andre Przywara5ecdd522022-01-11 15:34:50 +0000433 parser.add_option('-j', '--jobs', type='int', default=min(cpu_count, 240),
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900434 help='the number of jobs to run simultaneously')
435 parser.add_option('-o', '--output', default=OUTPUT_FILE,
436 help='output file [default=%s]' % OUTPUT_FILE)
Simon Glass69bbdd12019-12-05 15:59:11 -0700437 parser.add_option('-q', '--quiet', action="store_true", help='run silently')
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900438 (options, args) = parser.parse_args()
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +0900439
Simon Glass69bbdd12019-12-05 15:59:11 -0700440 gen_boards_cfg(options.output, jobs=options.jobs, force=options.force,
441 quiet=options.quiet)
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900442
443if __name__ == '__main__':
444 main()