blob: e9bbd15e15a937686d19f4ba6eb385e37fb5369b [file] [log] [blame]
Masahiro Yamada21343422014-08-27 14:05:51 +09001#!/usr/bin/env python2
Tom Rini83d290c2018-05-06 17:58:06 -04002# SPDX-License-Identifier: GPL-2.0+
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +09003#
4# Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
5#
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +09006
7"""
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +09008Converter from Kconfig and MAINTAINERS to a board database.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +09009
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090010Run 'tools/genboardscfg.py' to create a board database.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090011
12Run 'tools/genboardscfg.py -h' for available options.
Masahiro Yamada21343422014-08-27 14:05:51 +090013
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090014Python 2.6 or later, but not Python 3.x is necessary to run this script.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090015"""
16
17import errno
18import fnmatch
19import glob
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090020import multiprocessing
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090021import optparse
22import os
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090023import sys
24import tempfile
25import time
26
Peng Fan60801702018-07-24 14:27:18 +080027sys.path.insert(1, os.path.join(os.path.dirname(__file__), 'buildman'))
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090028import kconfiglib
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090029
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090030### constant variables ###
31OUTPUT_FILE = 'boards.cfg'
32CONFIG_DIR = 'configs'
33SLEEP_TIME = 0.03
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090034COMMENT_BLOCK = '''#
35# List of boards
36# Automatically generated by %s: don't edit
37#
Masahiro Yamadaca418dd2014-08-06 13:42:34 +090038# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090039
40''' % __file__
41
42### helper functions ###
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090043def try_remove(f):
44 """Remove a file ignoring 'No such file or directory' error."""
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090045 try:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090046 os.remove(f)
47 except OSError as exception:
48 # Ignore 'No such file or directory' error
49 if exception.errno != errno.ENOENT:
50 raise
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090051
52def check_top_directory():
53 """Exit if we are not at the top of source directory."""
54 for f in ('README', 'Licenses'):
55 if not os.path.exists(f):
Masahiro Yamada31e21412014-08-16 00:59:26 +090056 sys.exit('Please run at the top of source directory.')
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090057
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090058def output_is_new(output):
59 """Check if the output file is up to date.
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090060
61 Returns:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090062 True if the given output file exists and is newer than any of
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090063 *_defconfig, MAINTAINERS and Kconfig*. False otherwise.
64 """
65 try:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090066 ctime = os.path.getctime(output)
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090067 except OSError as exception:
68 if exception.errno == errno.ENOENT:
69 # return False on 'No such file or directory' error
70 return False
71 else:
72 raise
73
74 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
75 for filename in fnmatch.filter(filenames, '*_defconfig'):
76 if fnmatch.fnmatch(filename, '.*'):
77 continue
78 filepath = os.path.join(dirpath, filename)
79 if ctime < os.path.getctime(filepath):
80 return False
81
82 for (dirpath, dirnames, filenames) in os.walk('.'):
83 for filename in filenames:
84 if (fnmatch.fnmatch(filename, '*~') or
85 not fnmatch.fnmatch(filename, 'Kconfig*') and
86 not filename == 'MAINTAINERS'):
87 continue
88 filepath = os.path.join(dirpath, filename)
89 if ctime < os.path.getctime(filepath):
90 return False
91
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090092 # Detect a board that has been removed since the current board database
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090093 # was generated
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090094 with open(output) as f:
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090095 for line in f:
96 if line[0] == '#' or line == '\n':
97 continue
98 defconfig = line.split()[6] + '_defconfig'
99 if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
100 return False
101
102 return True
103
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900104### classes ###
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900105class KconfigScanner:
106
107 """Kconfig scanner."""
108
109 ### constant variable only used in this class ###
110 _SYMBOL_TABLE = {
111 'arch' : 'SYS_ARCH',
112 'cpu' : 'SYS_CPU',
113 'soc' : 'SYS_SOC',
114 'vendor' : 'SYS_VENDOR',
115 'board' : 'SYS_BOARD',
116 'config' : 'SYS_CONFIG_NAME',
117 'options' : 'SYS_EXTRA_OPTIONS'
118 }
119
120 def __init__(self):
121 """Scan all the Kconfig files and create a Config object."""
122 # Define environment variables referenced from Kconfig
123 os.environ['srctree'] = os.getcwd()
124 os.environ['UBOOTVERSION'] = 'dummy'
125 os.environ['KCONFIG_OBJDIR'] = ''
Simon Glass8639f692017-08-04 03:30:30 -0600126 self._conf = kconfiglib.Config(print_warnings=False)
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900127
128 def __del__(self):
129 """Delete a leftover temporary file before exit.
130
131 The scan() method of this class creates a temporay file and deletes
132 it on success. If scan() method throws an exception on the way,
133 the temporary file might be left over. In that case, it should be
134 deleted in this destructor.
135 """
136 if hasattr(self, '_tmpfile') and self._tmpfile:
137 try_remove(self._tmpfile)
138
139 def scan(self, defconfig):
140 """Load a defconfig file to obtain board parameters.
141
142 Arguments:
143 defconfig: path to the defconfig file to be processed
144
145 Returns:
146 A dictionary of board parameters. It has a form of:
147 {
148 'arch': <arch_name>,
149 'cpu': <cpu_name>,
150 'soc': <soc_name>,
151 'vendor': <vendor_name>,
152 'board': <board_name>,
153 'target': <target_name>,
154 'config': <config_header_name>,
155 'options': <extra_options>
156 }
157 """
158 # strip special prefixes and save it in a temporary file
159 fd, self._tmpfile = tempfile.mkstemp()
160 with os.fdopen(fd, 'w') as f:
161 for line in open(defconfig):
162 colon = line.find(':CONFIG_')
163 if colon == -1:
164 f.write(line)
165 else:
166 f.write(line[colon + 1:])
167
Simon Glass8639f692017-08-04 03:30:30 -0600168 warnings = self._conf.load_config(self._tmpfile)
169 if warnings:
170 for warning in warnings:
171 print '%s: %s' % (defconfig, warning)
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900172
173 try_remove(self._tmpfile)
174 self._tmpfile = None
175
176 params = {}
177
178 # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
179 # Set '-' if the value is empty.
180 for key, symbol in self._SYMBOL_TABLE.items():
181 value = self._conf.get_symbol(symbol).get_value()
182 if value:
183 params[key] = value
184 else:
185 params[key] = '-'
186
187 defconfig = os.path.basename(defconfig)
188 params['target'], match, rear = defconfig.partition('_defconfig')
189 assert match and not rear, '%s : invalid defconfig' % defconfig
190
191 # fix-up for aarch64
192 if params['arch'] == 'arm' and params['cpu'] == 'armv8':
193 params['arch'] = 'aarch64'
194
195 # fix-up options field. It should have the form:
196 # <config name>[:comma separated config options]
197 if params['options'] != '-':
198 params['options'] = params['config'] + ':' + \
199 params['options'].replace(r'\"', '"')
200 elif params['config'] != params['target']:
201 params['options'] = params['config']
202
203 return params
204
205def scan_defconfigs_for_multiprocess(queue, defconfigs):
206 """Scan defconfig files and queue their board parameters
207
208 This function is intended to be passed to
209 multiprocessing.Process() constructor.
210
211 Arguments:
212 queue: An instance of multiprocessing.Queue().
213 The resulting board parameters are written into it.
214 defconfigs: A sequence of defconfig files to be scanned.
215 """
216 kconf_scanner = KconfigScanner()
217 for defconfig in defconfigs:
218 queue.put(kconf_scanner.scan(defconfig))
219
220def read_queues(queues, params_list):
221 """Read the queues and append the data to the paramers list"""
222 for q in queues:
223 while not q.empty():
224 params_list.append(q.get())
225
226def scan_defconfigs(jobs=1):
227 """Collect board parameters for all defconfig files.
228
229 This function invokes multiple processes for faster processing.
230
231 Arguments:
232 jobs: The number of jobs to run simultaneously
233 """
234 all_defconfigs = []
235 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
236 for filename in fnmatch.filter(filenames, '*_defconfig'):
237 if fnmatch.fnmatch(filename, '.*'):
238 continue
239 all_defconfigs.append(os.path.join(dirpath, filename))
240
241 total_boards = len(all_defconfigs)
242 processes = []
243 queues = []
244 for i in range(jobs):
245 defconfigs = all_defconfigs[total_boards * i / jobs :
246 total_boards * (i + 1) / jobs]
247 q = multiprocessing.Queue(maxsize=-1)
248 p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
249 args=(q, defconfigs))
250 p.start()
251 processes.append(p)
252 queues.append(q)
253
254 # The resulting data should be accumulated to this list
255 params_list = []
256
257 # Data in the queues should be retrieved preriodically.
258 # Otherwise, the queues would become full and subprocesses would get stuck.
259 while any([p.is_alive() for p in processes]):
260 read_queues(queues, params_list)
261 # sleep for a while until the queues are filled
262 time.sleep(SLEEP_TIME)
263
264 # Joining subprocesses just in case
265 # (All subprocesses should already have been finished)
266 for p in processes:
267 p.join()
268
269 # retrieve leftover data
270 read_queues(queues, params_list)
271
272 return params_list
273
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900274class MaintainersDatabase:
275
276 """The database of board status and maintainers."""
277
278 def __init__(self):
279 """Create an empty database."""
280 self.database = {}
281
282 def get_status(self, target):
283 """Return the status of the given board.
284
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900285 The board status is generally either 'Active' or 'Orphan'.
286 Display a warning message and return '-' if status information
287 is not found.
288
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900289 Returns:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900290 'Active', 'Orphan' or '-'.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900291 """
Masahiro Yamadab8828e82014-08-25 12:39:43 +0900292 if not target in self.database:
293 print >> sys.stderr, "WARNING: no status info for '%s'" % target
294 return '-'
295
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900296 tmp = self.database[target][0]
297 if tmp.startswith('Maintained'):
298 return 'Active'
Lokesh Vutladee7c682017-05-10 16:19:52 +0530299 elif tmp.startswith('Supported'):
300 return 'Active'
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900301 elif tmp.startswith('Orphan'):
302 return 'Orphan'
303 else:
Masahiro Yamadab8828e82014-08-25 12:39:43 +0900304 print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" %
305 (tmp, target))
306 return '-'
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900307
308 def get_maintainers(self, target):
309 """Return the maintainers of the given board.
310
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900311 Returns:
312 Maintainers of the board. If the board has two or more maintainers,
313 they are separated with colons.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900314 """
Masahiro Yamadab8828e82014-08-25 12:39:43 +0900315 if not target in self.database:
316 print >> sys.stderr, "WARNING: no maintainers for '%s'" % target
317 return ''
318
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900319 return ':'.join(self.database[target][1])
320
321 def parse_file(self, file):
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900322 """Parse a MAINTAINERS file.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900323
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900324 Parse a MAINTAINERS file and accumulates board status and
325 maintainers information.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900326
327 Arguments:
328 file: MAINTAINERS file to be parsed
329 """
330 targets = []
331 maintainers = []
332 status = '-'
333 for line in open(file):
Masahiro Yamada5dff8442014-09-16 14:11:49 +0900334 # Check also commented maintainers
335 if line[:3] == '#M:':
336 line = line[1:]
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900337 tag, rest = line[:2], line[2:].strip()
338 if tag == 'M:':
339 maintainers.append(rest)
340 elif tag == 'F:':
341 # expand wildcard and filter by 'configs/*_defconfig'
342 for f in glob.glob(rest):
343 front, match, rear = f.partition('configs/')
344 if not front and match:
345 front, match, rear = rear.rpartition('_defconfig')
346 if match and not rear:
347 targets.append(front)
348 elif tag == 'S:':
349 status = rest
Masahiro Yamada9c2d60c2014-08-22 14:10:43 +0900350 elif line == '\n':
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900351 for target in targets:
352 self.database[target] = (status, maintainers)
353 targets = []
354 maintainers = []
355 status = '-'
356 if targets:
357 for target in targets:
358 self.database[target] = (status, maintainers)
359
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900360def insert_maintainers_info(params_list):
361 """Add Status and Maintainers information to the board parameters list.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900362
363 Arguments:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900364 params_list: A list of the board parameters
365 """
366 database = MaintainersDatabase()
367 for (dirpath, dirnames, filenames) in os.walk('.'):
368 if 'MAINTAINERS' in filenames:
369 database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
370
371 for i, params in enumerate(params_list):
372 target = params['target']
373 params['status'] = database.get_status(target)
374 params['maintainers'] = database.get_maintainers(target)
375 params_list[i] = params
376
377def format_and_output(params_list, output):
378 """Write board parameters into a file.
379
380 Columnate the board parameters, sort lines alphabetically,
381 and then write them to a file.
382
383 Arguments:
384 params_list: The list of board parameters
385 output: The path to the output file
386 """
387 FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
388 'options', 'maintainers')
389
390 # First, decide the width of each column
391 max_length = dict([ (f, 0) for f in FIELDS])
392 for params in params_list:
393 for f in FIELDS:
394 max_length[f] = max(max_length[f], len(params[f]))
395
396 output_lines = []
397 for params in params_list:
398 line = ''
399 for f in FIELDS:
400 # insert two spaces between fields like column -t would
401 line += ' ' + params[f].ljust(max_length[f])
402 output_lines.append(line.strip())
403
404 # ignore case when sorting
405 output_lines.sort(key=str.lower)
406
407 with open(output, 'w') as f:
408 f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
409
410def gen_boards_cfg(output, jobs=1, force=False):
411 """Generate a board database file.
412
413 Arguments:
414 output: The name of the output file
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900415 jobs: The number of jobs to run simultaneously
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900416 force: Force to generate the output even if it is new
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900417 """
Masahiro Yamada79d45d32014-08-25 12:39:46 +0900418 check_top_directory()
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900419
420 if not force and output_is_new(output):
421 print "%s is up to date. Nothing to do." % output
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +0900422 sys.exit(0)
423
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900424 params_list = scan_defconfigs(jobs)
425 insert_maintainers_info(params_list)
426 format_and_output(params_list, output)
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900427
428def main():
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900429 try:
430 cpu_count = multiprocessing.cpu_count()
431 except NotImplementedError:
432 cpu_count = 1
433
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900434 parser = optparse.OptionParser()
435 # Add options here
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +0900436 parser.add_option('-f', '--force', action="store_true", default=False,
437 help='regenerate the output even if it is new')
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900438 parser.add_option('-j', '--jobs', type='int', default=cpu_count,
439 help='the number of jobs to run simultaneously')
440 parser.add_option('-o', '--output', default=OUTPUT_FILE,
441 help='output file [default=%s]' % OUTPUT_FILE)
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900442 (options, args) = parser.parse_args()
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +0900443
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900444 gen_boards_cfg(options.output, jobs=options.jobs, force=options.force)
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900445
446if __name__ == '__main__':
447 main()