blob: 2e871feaf4fa76948aaf80ba7f55bc40a2e10da6 [file] [log] [blame]
Masahiro Yamada21343422014-08-27 14:05:51 +09001#!/usr/bin/env python2
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +09002#
3# Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
4#
5# SPDX-License-Identifier: GPL-2.0+
6#
7
8"""
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +09009Converter from Kconfig and MAINTAINERS to a board database.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090010
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090011Run 'tools/genboardscfg.py' to create a board database.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090012
13Run 'tools/genboardscfg.py -h' for available options.
Masahiro Yamada21343422014-08-27 14:05:51 +090014
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090015Python 2.6 or later, but not Python 3.x is necessary to run this script.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090016"""
17
18import errno
19import fnmatch
20import glob
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090021import multiprocessing
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090022import optparse
23import os
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090024import sys
25import tempfile
26import time
27
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090028sys.path.append(os.path.join(os.path.dirname(__file__), 'buildman'))
29import kconfiglib
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090030
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090031### constant variables ###
32OUTPUT_FILE = 'boards.cfg'
33CONFIG_DIR = 'configs'
34SLEEP_TIME = 0.03
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090035COMMENT_BLOCK = '''#
36# List of boards
37# Automatically generated by %s: don't edit
38#
Masahiro Yamadaca418dd2014-08-06 13:42:34 +090039# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090040
41''' % __file__
42
43### helper functions ###
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090044def try_remove(f):
45 """Remove a file ignoring 'No such file or directory' error."""
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090046 try:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090047 os.remove(f)
48 except OSError as exception:
49 # Ignore 'No such file or directory' error
50 if exception.errno != errno.ENOENT:
51 raise
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090052
53def check_top_directory():
54 """Exit if we are not at the top of source directory."""
55 for f in ('README', 'Licenses'):
56 if not os.path.exists(f):
Masahiro Yamada31e21412014-08-16 00:59:26 +090057 sys.exit('Please run at the top of source directory.')
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +090058
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090059def output_is_new(output):
60 """Check if the output file is up to date.
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090061
62 Returns:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090063 True if the given output file exists and is newer than any of
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090064 *_defconfig, MAINTAINERS and Kconfig*. False otherwise.
65 """
66 try:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090067 ctime = os.path.getctime(output)
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090068 except OSError as exception:
69 if exception.errno == errno.ENOENT:
70 # return False on 'No such file or directory' error
71 return False
72 else:
73 raise
74
75 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
76 for filename in fnmatch.filter(filenames, '*_defconfig'):
77 if fnmatch.fnmatch(filename, '.*'):
78 continue
79 filepath = os.path.join(dirpath, filename)
80 if ctime < os.path.getctime(filepath):
81 return False
82
83 for (dirpath, dirnames, filenames) in os.walk('.'):
84 for filename in filenames:
85 if (fnmatch.fnmatch(filename, '*~') or
86 not fnmatch.fnmatch(filename, 'Kconfig*') and
87 not filename == 'MAINTAINERS'):
88 continue
89 filepath = os.path.join(dirpath, filename)
90 if ctime < os.path.getctime(filepath):
91 return False
92
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090093 # Detect a board that has been removed since the current board database
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090094 # was generated
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +090095 with open(output) as f:
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +090096 for line in f:
97 if line[0] == '#' or line == '\n':
98 continue
99 defconfig = line.split()[6] + '_defconfig'
100 if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
101 return False
102
103 return True
104
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900105### classes ###
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900106class KconfigScanner:
107
108 """Kconfig scanner."""
109
110 ### constant variable only used in this class ###
111 _SYMBOL_TABLE = {
112 'arch' : 'SYS_ARCH',
113 'cpu' : 'SYS_CPU',
114 'soc' : 'SYS_SOC',
115 'vendor' : 'SYS_VENDOR',
116 'board' : 'SYS_BOARD',
117 'config' : 'SYS_CONFIG_NAME',
118 'options' : 'SYS_EXTRA_OPTIONS'
119 }
120
121 def __init__(self):
122 """Scan all the Kconfig files and create a Config object."""
123 # Define environment variables referenced from Kconfig
124 os.environ['srctree'] = os.getcwd()
125 os.environ['UBOOTVERSION'] = 'dummy'
126 os.environ['KCONFIG_OBJDIR'] = ''
127 self._conf = kconfiglib.Config()
128
129 def __del__(self):
130 """Delete a leftover temporary file before exit.
131
132 The scan() method of this class creates a temporay file and deletes
133 it on success. If scan() method throws an exception on the way,
134 the temporary file might be left over. In that case, it should be
135 deleted in this destructor.
136 """
137 if hasattr(self, '_tmpfile') and self._tmpfile:
138 try_remove(self._tmpfile)
139
140 def scan(self, defconfig):
141 """Load a defconfig file to obtain board parameters.
142
143 Arguments:
144 defconfig: path to the defconfig file to be processed
145
146 Returns:
147 A dictionary of board parameters. It has a form of:
148 {
149 'arch': <arch_name>,
150 'cpu': <cpu_name>,
151 'soc': <soc_name>,
152 'vendor': <vendor_name>,
153 'board': <board_name>,
154 'target': <target_name>,
155 'config': <config_header_name>,
156 'options': <extra_options>
157 }
158 """
159 # strip special prefixes and save it in a temporary file
160 fd, self._tmpfile = tempfile.mkstemp()
161 with os.fdopen(fd, 'w') as f:
162 for line in open(defconfig):
163 colon = line.find(':CONFIG_')
164 if colon == -1:
165 f.write(line)
166 else:
167 f.write(line[colon + 1:])
168
169 self._conf.load_config(self._tmpfile)
170
171 try_remove(self._tmpfile)
172 self._tmpfile = None
173
174 params = {}
175
176 # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
177 # Set '-' if the value is empty.
178 for key, symbol in self._SYMBOL_TABLE.items():
179 value = self._conf.get_symbol(symbol).get_value()
180 if value:
181 params[key] = value
182 else:
183 params[key] = '-'
184
185 defconfig = os.path.basename(defconfig)
186 params['target'], match, rear = defconfig.partition('_defconfig')
187 assert match and not rear, '%s : invalid defconfig' % defconfig
188
189 # fix-up for aarch64
190 if params['arch'] == 'arm' and params['cpu'] == 'armv8':
191 params['arch'] = 'aarch64'
192
193 # fix-up options field. It should have the form:
194 # <config name>[:comma separated config options]
195 if params['options'] != '-':
196 params['options'] = params['config'] + ':' + \
197 params['options'].replace(r'\"', '"')
198 elif params['config'] != params['target']:
199 params['options'] = params['config']
200
201 return params
202
203def scan_defconfigs_for_multiprocess(queue, defconfigs):
204 """Scan defconfig files and queue their board parameters
205
206 This function is intended to be passed to
207 multiprocessing.Process() constructor.
208
209 Arguments:
210 queue: An instance of multiprocessing.Queue().
211 The resulting board parameters are written into it.
212 defconfigs: A sequence of defconfig files to be scanned.
213 """
214 kconf_scanner = KconfigScanner()
215 for defconfig in defconfigs:
216 queue.put(kconf_scanner.scan(defconfig))
217
218def read_queues(queues, params_list):
219 """Read the queues and append the data to the paramers list"""
220 for q in queues:
221 while not q.empty():
222 params_list.append(q.get())
223
224def scan_defconfigs(jobs=1):
225 """Collect board parameters for all defconfig files.
226
227 This function invokes multiple processes for faster processing.
228
229 Arguments:
230 jobs: The number of jobs to run simultaneously
231 """
232 all_defconfigs = []
233 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
234 for filename in fnmatch.filter(filenames, '*_defconfig'):
235 if fnmatch.fnmatch(filename, '.*'):
236 continue
237 all_defconfigs.append(os.path.join(dirpath, filename))
238
239 total_boards = len(all_defconfigs)
240 processes = []
241 queues = []
242 for i in range(jobs):
243 defconfigs = all_defconfigs[total_boards * i / jobs :
244 total_boards * (i + 1) / jobs]
245 q = multiprocessing.Queue(maxsize=-1)
246 p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
247 args=(q, defconfigs))
248 p.start()
249 processes.append(p)
250 queues.append(q)
251
252 # The resulting data should be accumulated to this list
253 params_list = []
254
255 # Data in the queues should be retrieved preriodically.
256 # Otherwise, the queues would become full and subprocesses would get stuck.
257 while any([p.is_alive() for p in processes]):
258 read_queues(queues, params_list)
259 # sleep for a while until the queues are filled
260 time.sleep(SLEEP_TIME)
261
262 # Joining subprocesses just in case
263 # (All subprocesses should already have been finished)
264 for p in processes:
265 p.join()
266
267 # retrieve leftover data
268 read_queues(queues, params_list)
269
270 return params_list
271
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900272class MaintainersDatabase:
273
274 """The database of board status and maintainers."""
275
276 def __init__(self):
277 """Create an empty database."""
278 self.database = {}
279
280 def get_status(self, target):
281 """Return the status of the given board.
282
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900283 The board status is generally either 'Active' or 'Orphan'.
284 Display a warning message and return '-' if status information
285 is not found.
286
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900287 Returns:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900288 'Active', 'Orphan' or '-'.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900289 """
Masahiro Yamadab8828e82014-08-25 12:39:43 +0900290 if not target in self.database:
291 print >> sys.stderr, "WARNING: no status info for '%s'" % target
292 return '-'
293
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900294 tmp = self.database[target][0]
295 if tmp.startswith('Maintained'):
296 return 'Active'
Lokesh Vutladee7c682017-05-10 16:19:52 +0530297 elif tmp.startswith('Supported'):
298 return 'Active'
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900299 elif tmp.startswith('Orphan'):
300 return 'Orphan'
301 else:
Masahiro Yamadab8828e82014-08-25 12:39:43 +0900302 print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" %
303 (tmp, target))
304 return '-'
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900305
306 def get_maintainers(self, target):
307 """Return the maintainers of the given board.
308
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900309 Returns:
310 Maintainers of the board. If the board has two or more maintainers,
311 they are separated with colons.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900312 """
Masahiro Yamadab8828e82014-08-25 12:39:43 +0900313 if not target in self.database:
314 print >> sys.stderr, "WARNING: no maintainers for '%s'" % target
315 return ''
316
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900317 return ':'.join(self.database[target][1])
318
319 def parse_file(self, file):
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900320 """Parse a MAINTAINERS file.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900321
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900322 Parse a MAINTAINERS file and accumulates board status and
323 maintainers information.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900324
325 Arguments:
326 file: MAINTAINERS file to be parsed
327 """
328 targets = []
329 maintainers = []
330 status = '-'
331 for line in open(file):
Masahiro Yamada5dff8442014-09-16 14:11:49 +0900332 # Check also commented maintainers
333 if line[:3] == '#M:':
334 line = line[1:]
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900335 tag, rest = line[:2], line[2:].strip()
336 if tag == 'M:':
337 maintainers.append(rest)
338 elif tag == 'F:':
339 # expand wildcard and filter by 'configs/*_defconfig'
340 for f in glob.glob(rest):
341 front, match, rear = f.partition('configs/')
342 if not front and match:
343 front, match, rear = rear.rpartition('_defconfig')
344 if match and not rear:
345 targets.append(front)
346 elif tag == 'S:':
347 status = rest
Masahiro Yamada9c2d60c2014-08-22 14:10:43 +0900348 elif line == '\n':
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900349 for target in targets:
350 self.database[target] = (status, maintainers)
351 targets = []
352 maintainers = []
353 status = '-'
354 if targets:
355 for target in targets:
356 self.database[target] = (status, maintainers)
357
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900358def insert_maintainers_info(params_list):
359 """Add Status and Maintainers information to the board parameters list.
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900360
361 Arguments:
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900362 params_list: A list of the board parameters
363 """
364 database = MaintainersDatabase()
365 for (dirpath, dirnames, filenames) in os.walk('.'):
366 if 'MAINTAINERS' in filenames:
367 database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
368
369 for i, params in enumerate(params_list):
370 target = params['target']
371 params['status'] = database.get_status(target)
372 params['maintainers'] = database.get_maintainers(target)
373 params_list[i] = params
374
375def format_and_output(params_list, output):
376 """Write board parameters into a file.
377
378 Columnate the board parameters, sort lines alphabetically,
379 and then write them to a file.
380
381 Arguments:
382 params_list: The list of board parameters
383 output: The path to the output file
384 """
385 FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
386 'options', 'maintainers')
387
388 # First, decide the width of each column
389 max_length = dict([ (f, 0) for f in FIELDS])
390 for params in params_list:
391 for f in FIELDS:
392 max_length[f] = max(max_length[f], len(params[f]))
393
394 output_lines = []
395 for params in params_list:
396 line = ''
397 for f in FIELDS:
398 # insert two spaces between fields like column -t would
399 line += ' ' + params[f].ljust(max_length[f])
400 output_lines.append(line.strip())
401
402 # ignore case when sorting
403 output_lines.sort(key=str.lower)
404
405 with open(output, 'w') as f:
406 f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
407
408def gen_boards_cfg(output, jobs=1, force=False):
409 """Generate a board database file.
410
411 Arguments:
412 output: The name of the output file
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900413 jobs: The number of jobs to run simultaneously
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900414 force: Force to generate the output even if it is new
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900415 """
Masahiro Yamada79d45d32014-08-25 12:39:46 +0900416 check_top_directory()
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900417
418 if not force and output_is_new(output):
419 print "%s is up to date. Nothing to do." % output
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +0900420 sys.exit(0)
421
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900422 params_list = scan_defconfigs(jobs)
423 insert_maintainers_info(params_list)
424 format_and_output(params_list, output)
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900425
426def main():
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900427 try:
428 cpu_count = multiprocessing.cpu_count()
429 except NotImplementedError:
430 cpu_count = 1
431
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900432 parser = optparse.OptionParser()
433 # Add options here
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +0900434 parser.add_option('-f', '--force', action="store_true", default=False,
435 help='regenerate the output even if it is new')
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900436 parser.add_option('-j', '--jobs', type='int', default=cpu_count,
437 help='the number of jobs to run simultaneously')
438 parser.add_option('-o', '--output', default=OUTPUT_FILE,
439 help='output file [default=%s]' % OUTPUT_FILE)
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900440 (options, args) = parser.parse_args()
Masahiro Yamadad1bf4af2014-08-25 12:39:47 +0900441
Masahiro Yamadaf6c8f382014-09-01 19:57:38 +0900442 gen_boards_cfg(options.output, jobs=options.jobs, force=options.force)
Masahiro Yamada3c08e8b2014-07-30 14:08:19 +0900443
444if __name__ == '__main__':
445 main()