blob: 880a5594be0504c895e48da37e5264aba675ce08 [file] [log] [blame]
Simon Glass793dca32019-10-31 07:42:57 -06001#!/usr/bin/env python3
Tom Rini83d290c2018-05-06 17:58:06 -04002# SPDX-License-Identifier: GPL-2.0+
Masahiro Yamada5a27c732015-05-20 11:36:07 +09003#
4# Author: Masahiro Yamada <yamada.masahiro@socionext.com>
5#
Masahiro Yamada5a27c732015-05-20 11:36:07 +09006
7"""
8Move config options from headers to defconfig files.
9
Simon Glass5c72c0e2021-07-21 21:35:51 -060010See doc/develop/moveconfig.rst for documentation.
Masahiro Yamada5a27c732015-05-20 11:36:07 +090011"""
12
Simon Glassb2e83c62021-12-18 14:54:31 -070013from argparse import ArgumentParser
Markus Klotzbuecherb3192f42020-02-12 20:46:44 +010014import asteval
Simon Glass99b66602017-06-01 19:39:03 -060015import collections
Simon Glass91197aa2021-12-18 14:54:35 -070016from contextlib import ExitStack
Masahiro Yamada8ba1f5d2016-07-25 19:15:24 +090017import copy
Masahiro Yamadaf2f69812016-07-25 19:15:25 +090018import difflib
Simon Glass84067a52021-12-18 08:09:45 -070019import doctest
Masahiro Yamadac8e1b102016-05-19 15:52:07 +090020import filecmp
Masahiro Yamada5a27c732015-05-20 11:36:07 +090021import fnmatch
Masahiro Yamada0dbc9b52016-10-19 14:39:54 +090022import glob
Masahiro Yamada5a27c732015-05-20 11:36:07 +090023import multiprocessing
Masahiro Yamada5a27c732015-05-20 11:36:07 +090024import os
Simon Glass793dca32019-10-31 07:42:57 -060025import queue
Masahiro Yamada5a27c732015-05-20 11:36:07 +090026import re
27import shutil
28import subprocess
29import sys
30import tempfile
Simon Glassd73fcb12017-06-01 19:39:02 -060031import threading
Masahiro Yamada5a27c732015-05-20 11:36:07 +090032import time
Simon Glass84067a52021-12-18 08:09:45 -070033import unittest
Masahiro Yamada5a27c732015-05-20 11:36:07 +090034
Simon Glass0ede00f2020-04-17 18:09:02 -060035from buildman import bsettings
36from buildman import kconfiglib
37from buildman import toolchain
Simon Glasscb008832017-06-15 21:39:33 -060038
Masahiro Yamada5a27c732015-05-20 11:36:07 +090039SHOW_GNU_MAKE = 'scripts/show-gnu-make'
40SLEEP_TIME=0.03
41
Masahiro Yamada5a27c732015-05-20 11:36:07 +090042STATE_IDLE = 0
43STATE_DEFCONFIG = 1
44STATE_AUTOCONF = 2
Joe Hershberger96464ba2015-05-19 13:21:17 -050045STATE_SAVEDEFCONFIG = 3
Masahiro Yamada5a27c732015-05-20 11:36:07 +090046
47ACTION_MOVE = 0
Masahiro Yamadacc008292016-05-19 15:51:56 +090048ACTION_NO_ENTRY = 1
Masahiro Yamada916224c2016-08-22 22:18:21 +090049ACTION_NO_ENTRY_WARN = 2
50ACTION_NO_CHANGE = 3
Masahiro Yamada5a27c732015-05-20 11:36:07 +090051
52COLOR_BLACK = '0;30'
53COLOR_RED = '0;31'
54COLOR_GREEN = '0;32'
55COLOR_BROWN = '0;33'
56COLOR_BLUE = '0;34'
57COLOR_PURPLE = '0;35'
58COLOR_CYAN = '0;36'
59COLOR_LIGHT_GRAY = '0;37'
60COLOR_DARK_GRAY = '1;30'
61COLOR_LIGHT_RED = '1;31'
62COLOR_LIGHT_GREEN = '1;32'
63COLOR_YELLOW = '1;33'
64COLOR_LIGHT_BLUE = '1;34'
65COLOR_LIGHT_PURPLE = '1;35'
66COLOR_LIGHT_CYAN = '1;36'
67COLOR_WHITE = '1;37'
68
Simon Glassf3b8e642017-06-01 19:39:01 -060069AUTO_CONF_PATH = 'include/config/auto.conf'
Simon Glassd73fcb12017-06-01 19:39:02 -060070CONFIG_DATABASE = 'moveconfig.db'
Simon Glassf3b8e642017-06-01 19:39:01 -060071
Simon Glasscb008832017-06-15 21:39:33 -060072CONFIG_LEN = len('CONFIG_')
Simon Glassf3b8e642017-06-01 19:39:01 -060073
Markus Klotzbuecherb237d352019-05-15 15:15:52 +020074SIZES = {
Simon Glassdaa694d2021-12-18 14:54:30 -070075 'SZ_1': 0x00000001, 'SZ_2': 0x00000002,
76 'SZ_4': 0x00000004, 'SZ_8': 0x00000008,
77 'SZ_16': 0x00000010, 'SZ_32': 0x00000020,
78 'SZ_64': 0x00000040, 'SZ_128': 0x00000080,
79 'SZ_256': 0x00000100, 'SZ_512': 0x00000200,
80 'SZ_1K': 0x00000400, 'SZ_2K': 0x00000800,
81 'SZ_4K': 0x00001000, 'SZ_8K': 0x00002000,
82 'SZ_16K': 0x00004000, 'SZ_32K': 0x00008000,
83 'SZ_64K': 0x00010000, 'SZ_128K': 0x00020000,
84 'SZ_256K': 0x00040000, 'SZ_512K': 0x00080000,
85 'SZ_1M': 0x00100000, 'SZ_2M': 0x00200000,
86 'SZ_4M': 0x00400000, 'SZ_8M': 0x00800000,
87 'SZ_16M': 0x01000000, 'SZ_32M': 0x02000000,
88 'SZ_64M': 0x04000000, 'SZ_128M': 0x08000000,
89 'SZ_256M': 0x10000000, 'SZ_512M': 0x20000000,
90 'SZ_1G': 0x40000000, 'SZ_2G': 0x80000000,
91 'SZ_4G': 0x100000000
Markus Klotzbuecherb237d352019-05-15 15:15:52 +020092}
93
Simon Glassb8d11da2022-02-08 11:49:45 -070094RE_REMOVE_DEFCONFIG = re.compile(r'(.*)_defconfig')
95
Simon Glass65e62032023-02-01 13:19:12 -070096# CONFIG symbols present in the build system (from Linux) but not actually used
97# in U-Boot; KCONFIG symbols
98IGNORE_SYMS = ['DEBUG_SECTION_MISMATCH', 'FTRACE_MCOUNT_RECORD', 'GCOV_KERNEL',
99 'GCOV_PROFILE_ALL', 'KALLSYMS', 'KASAN', 'MODVERSIONS', 'SHELL',
100 'TPL_BUILD', 'VPL_BUILD', 'IS_ENABLED', 'FOO', 'IF_ENABLED_INT',
101 'IS_ENABLED_', 'IS_ENABLED_1', 'IS_ENABLED_2', 'IS_ENABLED_3',
102 'SPL_', 'TPL_', 'SPL_FOO', 'TPL_FOO', 'TOOLS_FOO',
103 'ACME', 'SPL_ACME', 'TPL_ACME', 'TRACE_BRANCH_PROFILING',
104 'VAL', '_UNDEFINED', 'SPL_BUILD', ]
105
106SPL_PREFIXES = ['SPL_', 'TPL_', 'VPL_', 'TOOLS_']
107
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900108### helper functions ###
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900109def check_top_directory():
110 """Exit if we are not at the top of source directory."""
Simon Glass91197aa2021-12-18 14:54:35 -0700111 for fname in 'README', 'Licenses':
112 if not os.path.exists(fname):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900113 sys.exit('Please run at the top of source directory.')
114
Masahiro Yamadabd63e5b2016-05-19 15:51:54 +0900115def check_clean_directory():
116 """Exit if the source tree is not clean."""
Simon Glass91197aa2021-12-18 14:54:35 -0700117 for fname in '.config', 'include/config':
118 if os.path.exists(fname):
Masahiro Yamadabd63e5b2016-05-19 15:51:54 +0900119 sys.exit("source tree is not clean, please run 'make mrproper'")
120
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900121def get_make_cmd():
122 """Get the command name of GNU Make.
123
124 U-Boot needs GNU Make for building, but the command name is not
125 necessarily "make". (for example, "gmake" on FreeBSD).
126 Returns the most appropriate command name on your system.
127 """
Simon Glass91197aa2021-12-18 14:54:35 -0700128 with subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) as proc:
129 ret = proc.communicate()
130 if proc.returncode:
131 sys.exit('GNU Make not found')
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900132 return ret[0].rstrip()
133
Simon Glass25f978c2017-06-01 19:38:58 -0600134def get_matched_defconfig(line):
135 """Get the defconfig files that match a pattern
136
137 Args:
Simon Glass91197aa2021-12-18 14:54:35 -0700138 line (str): Path or filename to match, e.g. 'configs/snow_defconfig' or
Simon Glass25f978c2017-06-01 19:38:58 -0600139 'k2*_defconfig'. If no directory is provided, 'configs/' is
140 prepended
141
142 Returns:
Simon Glass91197aa2021-12-18 14:54:35 -0700143 list of str: a list of matching defconfig files
Simon Glass25f978c2017-06-01 19:38:58 -0600144 """
145 dirname = os.path.dirname(line)
146 if dirname:
147 pattern = line
148 else:
149 pattern = os.path.join('configs', line)
150 return glob.glob(pattern) + glob.glob(pattern + '_defconfig')
151
Masahiro Yamada0dbc9b52016-10-19 14:39:54 +0900152def get_matched_defconfigs(defconfigs_file):
Simon Glassee4e61b2017-06-01 19:38:59 -0600153 """Get all the defconfig files that match the patterns in a file.
154
155 Args:
Simon Glass91197aa2021-12-18 14:54:35 -0700156 defconfigs_file (str): File containing a list of defconfigs to process,
157 or '-' to read the list from stdin
Simon Glassee4e61b2017-06-01 19:38:59 -0600158
159 Returns:
Simon Glass91197aa2021-12-18 14:54:35 -0700160 list of str: A list of paths to defconfig files, with no duplicates
Simon Glassee4e61b2017-06-01 19:38:59 -0600161 """
Masahiro Yamada0dbc9b52016-10-19 14:39:54 +0900162 defconfigs = []
Simon Glass91197aa2021-12-18 14:54:35 -0700163 with ExitStack() as stack:
164 if defconfigs_file == '-':
165 inf = sys.stdin
166 defconfigs_file = 'stdin'
167 else:
168 inf = stack.enter_context(open(defconfigs_file, encoding='utf-8'))
169 for i, line in enumerate(inf):
170 line = line.strip()
171 if not line:
172 continue # skip blank lines silently
173 if ' ' in line:
174 line = line.split(' ')[0] # handle 'git log' input
175 matched = get_matched_defconfig(line)
176 if not matched:
177 print(f"warning: {defconfigs_file}:{i + 1}: no defconfig matched '{line}'",
178 file=sys.stderr)
Masahiro Yamada0dbc9b52016-10-19 14:39:54 +0900179
Simon Glass91197aa2021-12-18 14:54:35 -0700180 defconfigs += matched
Masahiro Yamada0dbc9b52016-10-19 14:39:54 +0900181
182 # use set() to drop multiple matching
Simon Glass91197aa2021-12-18 14:54:35 -0700183 return [defconfig[len('configs') + 1:] for defconfig in set(defconfigs)]
Masahiro Yamada0dbc9b52016-10-19 14:39:54 +0900184
Masahiro Yamada684c3062016-07-25 19:15:28 +0900185def get_all_defconfigs():
Simon Glass91197aa2021-12-18 14:54:35 -0700186 """Get all the defconfig files under the configs/ directory.
187
188 Returns:
189 list of str: List of paths to defconfig files
190 """
Masahiro Yamada684c3062016-07-25 19:15:28 +0900191 defconfigs = []
Simon Glass91197aa2021-12-18 14:54:35 -0700192 for (dirpath, _, filenames) in os.walk('configs'):
Masahiro Yamada684c3062016-07-25 19:15:28 +0900193 dirpath = dirpath[len('configs') + 1:]
194 for filename in fnmatch.filter(filenames, '*_defconfig'):
195 defconfigs.append(os.path.join(dirpath, filename))
196
197 return defconfigs
198
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900199def color_text(color_enabled, color, string):
200 """Return colored string."""
201 if color_enabled:
Masahiro Yamada1d085562016-05-19 15:52:02 +0900202 # LF should not be surrounded by the escape sequence.
203 # Otherwise, additional whitespace or line-feed might be printed.
204 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else ''
205 for s in string.split('\n') ])
Simon Glass91197aa2021-12-18 14:54:35 -0700206 return string
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900207
Simon Glass91197aa2021-12-18 14:54:35 -0700208def show_diff(alines, blines, file_path, color_enabled):
Masahiro Yamadaf2f69812016-07-25 19:15:25 +0900209 """Show unidified diff.
210
Simon Glass91197aa2021-12-18 14:54:35 -0700211 Args:
212 alines (list of str): A list of lines (before)
213 blines (list of str): A list of lines (after)
214 file_path (str): Path to the file
215 color_enabled (bool): Display the diff in color
Masahiro Yamadaf2f69812016-07-25 19:15:25 +0900216 """
Simon Glass91197aa2021-12-18 14:54:35 -0700217 diff = difflib.unified_diff(alines, blines,
Masahiro Yamadaf2f69812016-07-25 19:15:25 +0900218 fromfile=os.path.join('a', file_path),
219 tofile=os.path.join('b', file_path))
220
221 for line in diff:
Alper Nebi Yasak6c928c62022-01-29 18:22:08 +0300222 if line.startswith('-') and not line.startswith('--'):
223 print(color_text(color_enabled, COLOR_RED, line))
224 elif line.startswith('+') and not line.startswith('++'):
225 print(color_text(color_enabled, COLOR_GREEN, line))
Masahiro Yamadae9ea1222016-07-25 19:15:26 +0900226 else:
Alper Nebi Yasak6c928c62022-01-29 18:22:08 +0300227 print(line)
Masahiro Yamadaf2f69812016-07-25 19:15:25 +0900228
Simon Glass91197aa2021-12-18 14:54:35 -0700229def extend_matched_lines(lines, matched, pre_patterns, post_patterns,
230 extend_pre, extend_post):
Masahiro Yamada8ba1f5d2016-07-25 19:15:24 +0900231 """Extend matched lines if desired patterns are found before/after already
232 matched lines.
233
Simon Glass91197aa2021-12-18 14:54:35 -0700234 Args:
235 lines (list of str): list of lines handled.
236 matched (list of int): list of line numbers that have been already
237 matched (will be updated by this function)
238 pre_patterns (list of re.Pattern): list of regular expression that should
239 be matched as preamble
240 post_patterns (list of re.Pattern): list of regular expression that should
241 be matched as postamble
242 extend_pre (bool): Add the line number of matched preamble to the matched
243 list
244 extend_post (bool): Add the line number of matched postamble to the
245 matched list
Masahiro Yamada8ba1f5d2016-07-25 19:15:24 +0900246 """
247 extended_matched = []
248
249 j = matched[0]
250
251 for i in matched:
252 if i == 0 or i < j:
253 continue
254 j = i
255 while j in matched:
256 j += 1
257 if j >= len(lines):
258 break
259
Simon Glass91197aa2021-12-18 14:54:35 -0700260 for pat in pre_patterns:
261 if pat.search(lines[i - 1]):
Masahiro Yamada8ba1f5d2016-07-25 19:15:24 +0900262 break
263 else:
264 # not matched
265 continue
266
Simon Glass91197aa2021-12-18 14:54:35 -0700267 for pat in post_patterns:
268 if pat.search(lines[j]):
Masahiro Yamada8ba1f5d2016-07-25 19:15:24 +0900269 break
270 else:
271 # not matched
272 continue
273
274 if extend_pre:
275 extended_matched.append(i - 1)
276 if extend_post:
277 extended_matched.append(j)
278
279 matched += extended_matched
280 matched.sort()
281
Simon Glassb2e83c62021-12-18 14:54:31 -0700282def confirm(args, prompt):
Simon Glass91197aa2021-12-18 14:54:35 -0700283 """Ask the user to confirm something
284
285 Args:
286 args (Namespace ): program arguments
287
288 Returns:
289 bool: True to confirm, False to cancel/stop
290 """
Simon Glassb2e83c62021-12-18 14:54:31 -0700291 if not args.yes:
Chris Packham85edfc12017-05-02 21:30:46 +1200292 while True:
Simon Glass91197aa2021-12-18 14:54:35 -0700293 choice = input(f'{prompt} [y/n]: ')
Chris Packham85edfc12017-05-02 21:30:46 +1200294 choice = choice.lower()
Simon Glass793dca32019-10-31 07:42:57 -0600295 print(choice)
Simon Glass91197aa2021-12-18 14:54:35 -0700296 if choice in ('y', 'n'):
Chris Packham85edfc12017-05-02 21:30:46 +1200297 break
298
299 if choice == 'n':
300 return False
301
302 return True
303
Simon Glass2fd85bd2021-12-18 14:54:33 -0700304def write_file(fname, data):
305 """Write data to a file
306
307 Args:
308 fname (str): Filename to write to
309 data (list of str): Lines to write (with or without trailing newline);
310 or str to write
311 """
312 with open(fname, 'w', encoding='utf-8') as out:
313 if isinstance(data, list):
314 for line in data:
315 print(line.rstrip('\n'), file=out)
316 else:
317 out.write(data)
318
Simon Glass37f815c2021-12-18 14:54:34 -0700319def read_file(fname, as_lines=True, skip_unicode=False):
320 """Read a file and return the contents
321
322 Args:
323 fname (str): Filename to read from
324 as_lines: Return file contents as a list of lines
325 skip_unicode (bool): True to report unicode errors and continue
326
327 Returns:
328 iter of str: List of ;ines from the file with newline removed; str if
329 as_lines is False with newlines intact; or None if a unicode error
330 occurred
331
332 Raises:
333 UnicodeDecodeError: Unicode error occurred when reading
334 """
335 with open(fname, encoding='utf-8') as inf:
336 try:
337 if as_lines:
338 return [line.rstrip('\n') for line in inf.readlines()]
339 else:
340 return inf.read()
341 except UnicodeDecodeError as e:
342 if not skip_unicode:
Simon Glass68a0b712022-02-11 13:23:22 -0700343 raise
Simon Glass37f815c2021-12-18 14:54:34 -0700344 print("Failed on file %s': %s" % (fname, e))
345 return None
346
Simon Glassb2e83c62021-12-18 14:54:31 -0700347def cleanup_empty_blocks(header_path, args):
Chris Packham4d9dbb12019-01-30 20:23:16 +1300348 """Clean up empty conditional blocks
349
Simon Glass91197aa2021-12-18 14:54:35 -0700350 Args:
351 header_path (str): path to the cleaned file.
352 args (Namespace): program arguments
Chris Packham4d9dbb12019-01-30 20:23:16 +1300353 """
354 pattern = re.compile(r'^\s*#\s*if.*$\n^\s*#\s*endif.*$\n*', flags=re.M)
Simon Glass37f815c2021-12-18 14:54:34 -0700355 data = read_file(header_path, as_lines=False, skip_unicode=True)
356 if data is None:
357 return
Chris Packham4d9dbb12019-01-30 20:23:16 +1300358
359 new_data = pattern.sub('\n', data)
360
361 show_diff(data.splitlines(True), new_data.splitlines(True), header_path,
Simon Glassb2e83c62021-12-18 14:54:31 -0700362 args.color)
Chris Packham4d9dbb12019-01-30 20:23:16 +1300363
Simon Glassb2e83c62021-12-18 14:54:31 -0700364 if args.dry_run:
Chris Packham4d9dbb12019-01-30 20:23:16 +1300365 return
366
Simon Glass37f815c2021-12-18 14:54:34 -0700367 if new_data != data:
368 write_file(header_path, new_data)
Chris Packham4d9dbb12019-01-30 20:23:16 +1300369
Simon Glassb2e83c62021-12-18 14:54:31 -0700370def cleanup_one_header(header_path, patterns, args):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900371 """Clean regex-matched lines away from a file.
372
Simon Glass91197aa2021-12-18 14:54:35 -0700373 Args:
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900374 header_path: path to the cleaned file.
375 patterns: list of regex patterns. Any lines matching to these
376 patterns are deleted.
Simon Glass91197aa2021-12-18 14:54:35 -0700377 args (Namespace): program arguments
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900378 """
Simon Glass37f815c2021-12-18 14:54:34 -0700379 lines = read_file(header_path, skip_unicode=True)
380 if lines is None:
381 return
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900382
383 matched = []
384 for i, line in enumerate(lines):
Alper Nebi Yasak6c928c62022-01-29 18:22:08 +0300385 if i - 1 in matched and lines[i - 1].endswith('\\'):
Masahiro Yamadaa3a779f2016-07-25 19:15:27 +0900386 matched.append(i)
387 continue
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900388 for pattern in patterns:
Masahiro Yamada8ba1f5d2016-07-25 19:15:24 +0900389 if pattern.search(line):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900390 matched.append(i)
391 break
392
Masahiro Yamada8ba1f5d2016-07-25 19:15:24 +0900393 if not matched:
394 return
395
396 # remove empty #ifdef ... #endif, successive blank lines
Alper Nebi Yasak6c928c62022-01-29 18:22:08 +0300397 pattern_if = re.compile(r'#\s*if(def|ndef)?\b') # #if, #ifdef, #ifndef
398 pattern_elif = re.compile(r'#\s*el(if|se)\b') # #elif, #else
399 pattern_endif = re.compile(r'#\s*endif\b') # #endif
Masahiro Yamada8ba1f5d2016-07-25 19:15:24 +0900400 pattern_blank = re.compile(r'^\s*$') # empty line
401
402 while True:
403 old_matched = copy.copy(matched)
404 extend_matched_lines(lines, matched, [pattern_if],
405 [pattern_endif], True, True)
406 extend_matched_lines(lines, matched, [pattern_elif],
407 [pattern_elif, pattern_endif], True, False)
408 extend_matched_lines(lines, matched, [pattern_if, pattern_elif],
409 [pattern_blank], False, True)
410 extend_matched_lines(lines, matched, [pattern_blank],
411 [pattern_elif, pattern_endif], True, False)
412 extend_matched_lines(lines, matched, [pattern_blank],
413 [pattern_blank], True, False)
414 if matched == old_matched:
415 break
416
Masahiro Yamadaf2f69812016-07-25 19:15:25 +0900417 tolines = copy.copy(lines)
418
419 for i in reversed(matched):
420 tolines.pop(i)
421
Simon Glassb2e83c62021-12-18 14:54:31 -0700422 show_diff(lines, tolines, header_path, args.color)
Masahiro Yamada8ba1f5d2016-07-25 19:15:24 +0900423
Simon Glassb2e83c62021-12-18 14:54:31 -0700424 if args.dry_run:
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900425 return
426
Simon Glass2fd85bd2021-12-18 14:54:33 -0700427 write_file(header_path, tolines)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900428
Simon Glassb2e83c62021-12-18 14:54:31 -0700429def cleanup_headers(configs, args):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900430 """Delete config defines from board headers.
431
Simon Glass91197aa2021-12-18 14:54:35 -0700432 Args:
Masahiro Yamadab134bc12016-05-19 15:51:57 +0900433 configs: A list of CONFIGs to remove.
Simon Glass91197aa2021-12-18 14:54:35 -0700434 args (Namespace): program arguments
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900435 """
Simon Glassb2e83c62021-12-18 14:54:31 -0700436 if not confirm(args, 'Clean up headers?'):
Chris Packham85edfc12017-05-02 21:30:46 +1200437 return
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900438
439 patterns = []
Masahiro Yamadab134bc12016-05-19 15:51:57 +0900440 for config in configs:
Alper Nebi Yasak6c928c62022-01-29 18:22:08 +0300441 patterns.append(re.compile(r'#\s*define\s+%s\b' % config))
442 patterns.append(re.compile(r'#\s*undef\s+%s\b' % config))
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900443
Joe Hershberger60727f52015-05-19 13:21:21 -0500444 for dir in 'include', 'arch', 'board':
445 for (dirpath, dirnames, filenames) in os.walk(dir):
Masahiro Yamadadc6de502016-07-25 19:15:22 +0900446 if dirpath == os.path.join('include', 'generated'):
447 continue
Joe Hershberger60727f52015-05-19 13:21:21 -0500448 for filename in filenames:
Simon Glassa38cc172020-08-11 11:23:34 -0600449 if not filename.endswith(('~', '.dts', '.dtsi', '.bin',
Trevor Woernerdc514d72021-03-15 12:01:33 -0400450 '.elf','.aml','.dat')):
Chris Packham4d9dbb12019-01-30 20:23:16 +1300451 header_path = os.path.join(dirpath, filename)
Tom Rini02b56702019-11-10 21:19:37 -0500452 # This file contains UTF-16 data and no CONFIG symbols
453 if header_path == 'include/video_font_data.h':
454 continue
Simon Glassb2e83c62021-12-18 14:54:31 -0700455 cleanup_one_header(header_path, patterns, args)
456 cleanup_empty_blocks(header_path, args)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900457
Chris Packhamf90df592017-05-02 21:30:48 +1200458def find_matching(patterns, line):
459 for pat in patterns:
460 if pat.search(line):
461 return True
462 return False
463
Simon Glassb2e83c62021-12-18 14:54:31 -0700464def cleanup_readme(configs, args):
Chris Packhamf90df592017-05-02 21:30:48 +1200465 """Delete config description in README
466
Simon Glass91197aa2021-12-18 14:54:35 -0700467 Args:
Chris Packhamf90df592017-05-02 21:30:48 +1200468 configs: A list of CONFIGs to remove.
Simon Glass91197aa2021-12-18 14:54:35 -0700469 args (Namespace): program arguments
Chris Packhamf90df592017-05-02 21:30:48 +1200470 """
Simon Glassb2e83c62021-12-18 14:54:31 -0700471 if not confirm(args, 'Clean up README?'):
Chris Packhamf90df592017-05-02 21:30:48 +1200472 return
473
474 patterns = []
475 for config in configs:
476 patterns.append(re.compile(r'^\s+%s' % config))
477
Simon Glass37f815c2021-12-18 14:54:34 -0700478 lines = read_file('README')
Chris Packhamf90df592017-05-02 21:30:48 +1200479
480 found = False
481 newlines = []
482 for line in lines:
483 if not found:
484 found = find_matching(patterns, line)
485 if found:
486 continue
487
488 if found and re.search(r'^\s+CONFIG', line):
489 found = False
490
491 if not found:
492 newlines.append(line)
493
Simon Glass2fd85bd2021-12-18 14:54:33 -0700494 write_file('README', newlines)
Chris Packhamf90df592017-05-02 21:30:48 +1200495
Markus Klotzbuecherb237d352019-05-15 15:15:52 +0200496def try_expand(line):
497 """If value looks like an expression, try expanding it
498 Otherwise just return the existing value
499 """
500 if line.find('=') == -1:
501 return line
502
503 try:
Markus Klotzbuecherb3192f42020-02-12 20:46:44 +0100504 aeval = asteval.Interpreter( usersyms=SIZES, minimal=True )
Markus Klotzbuecherb237d352019-05-15 15:15:52 +0200505 cfg, val = re.split("=", line)
506 val= val.strip('\"')
Simon Glassdaa694d2021-12-18 14:54:30 -0700507 if re.search(r'[*+-/]|<<|SZ_+|\(([^\)]+)\)', val):
Markus Klotzbuecherb3192f42020-02-12 20:46:44 +0100508 newval = hex(aeval(val))
Simon Glassdaa694d2021-12-18 14:54:30 -0700509 print('\tExpanded expression %s to %s' % (val, newval))
Markus Klotzbuecherb237d352019-05-15 15:15:52 +0200510 return cfg+'='+newval
511 except:
Simon Glassdaa694d2021-12-18 14:54:30 -0700512 print('\tFailed to expand expression in %s' % line)
Markus Klotzbuecherb237d352019-05-15 15:15:52 +0200513
514 return line
515
Chris Packhamca438342017-05-02 21:30:47 +1200516
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900517### classes ###
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900518class Progress:
519
520 """Progress Indicator"""
521
522 def __init__(self, total):
523 """Create a new progress indicator.
524
Simon Glass91197aa2021-12-18 14:54:35 -0700525 Args:
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900526 total: A number of defconfig files to process.
527 """
528 self.current = 0
529 self.total = total
530
531 def inc(self):
532 """Increment the number of processed defconfig files."""
533
534 self.current += 1
535
536 def show(self):
537 """Display the progress."""
Simon Glass793dca32019-10-31 07:42:57 -0600538 print(' %d defconfigs out of %d\r' % (self.current, self.total), end=' ')
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900539 sys.stdout.flush()
540
Simon Glasscb008832017-06-15 21:39:33 -0600541
542class KconfigScanner:
543 """Kconfig scanner."""
544
545 def __init__(self):
546 """Scan all the Kconfig files and create a Config object."""
547 # Define environment variables referenced from Kconfig
548 os.environ['srctree'] = os.getcwd()
549 os.environ['UBOOTVERSION'] = 'dummy'
550 os.environ['KCONFIG_OBJDIR'] = ''
Simon Glass65e62032023-02-01 13:19:12 -0700551 os.environ['CC'] = 'gcc'
Tom Rini65e05dd2019-09-20 17:42:09 -0400552 self.conf = kconfiglib.Kconfig()
Simon Glasscb008832017-06-15 21:39:33 -0600553
554
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900555class KconfigParser:
556
557 """A parser of .config and include/autoconf.mk."""
558
559 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
560 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
561
Simon Glassb2e83c62021-12-18 14:54:31 -0700562 def __init__(self, configs, args, build_dir):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900563 """Create a new parser.
564
Simon Glass91197aa2021-12-18 14:54:35 -0700565 Args:
Masahiro Yamadab134bc12016-05-19 15:51:57 +0900566 configs: A list of CONFIGs to move.
Simon Glass91197aa2021-12-18 14:54:35 -0700567 args (Namespace): program arguments
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900568 build_dir: Build directory.
569 """
Masahiro Yamadab134bc12016-05-19 15:51:57 +0900570 self.configs = configs
Simon Glassb2e83c62021-12-18 14:54:31 -0700571 self.args = args
Masahiro Yamada1f169922016-05-19 15:52:00 +0900572 self.dotconfig = os.path.join(build_dir, '.config')
573 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk')
Masahiro Yamada07913d12016-08-22 22:18:22 +0900574 self.spl_autoconf = os.path.join(build_dir, 'spl', 'include',
575 'autoconf.mk')
Simon Glassf3b8e642017-06-01 19:39:01 -0600576 self.config_autoconf = os.path.join(build_dir, AUTO_CONF_PATH)
Masahiro Yamada5da4f852016-05-19 15:52:06 +0900577 self.defconfig = os.path.join(build_dir, 'defconfig')
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900578
Simon Glass6821a742017-07-10 14:47:47 -0600579 def get_arch(self):
580 """Parse .config file and return the architecture.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900581
582 Returns:
Simon Glass6821a742017-07-10 14:47:47 -0600583 Architecture name (e.g. 'arm').
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900584 """
585 arch = ''
586 cpu = ''
Simon Glass37f815c2021-12-18 14:54:34 -0700587 for line in read_file(self.dotconfig):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900588 m = self.re_arch.match(line)
589 if m:
590 arch = m.group(1)
591 continue
592 m = self.re_cpu.match(line)
593 if m:
594 cpu = m.group(1)
595
Masahiro Yamada90ed6cb2016-05-19 15:51:53 +0900596 if not arch:
597 return None
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900598
599 # fix-up for aarch64
600 if arch == 'arm' and cpu == 'armv8':
601 arch = 'aarch64'
602
Simon Glass6821a742017-07-10 14:47:47 -0600603 return arch
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900604
Masahiro Yamadab134bc12016-05-19 15:51:57 +0900605 def parse_one_config(self, config, dotconfig_lines, autoconf_lines):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900606 """Parse .config, defconfig, include/autoconf.mk for one config.
607
608 This function looks for the config options in the lines from
609 defconfig, .config, and include/autoconf.mk in order to decide
610 which action should be taken for this defconfig.
611
Simon Glass91197aa2021-12-18 14:54:35 -0700612 Args:
Masahiro Yamadab134bc12016-05-19 15:51:57 +0900613 config: CONFIG name to parse.
Masahiro Yamadacc008292016-05-19 15:51:56 +0900614 dotconfig_lines: lines from the .config file.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900615 autoconf_lines: lines from the include/autoconf.mk file.
616
617 Returns:
618 A tupple of the action for this defconfig and the line
619 matched for the config.
620 """
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900621 not_set = '# %s is not set' % config
622
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900623 for line in autoconf_lines:
624 line = line.rstrip()
625 if line.startswith(config + '='):
Masahiro Yamadacc008292016-05-19 15:51:56 +0900626 new_val = line
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900627 break
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900628 else:
Masahiro Yamadacc008292016-05-19 15:51:56 +0900629 new_val = not_set
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900630
Markus Klotzbuecherb237d352019-05-15 15:15:52 +0200631 new_val = try_expand(new_val)
632
Masahiro Yamada916224c2016-08-22 22:18:21 +0900633 for line in dotconfig_lines:
634 line = line.rstrip()
635 if line.startswith(config + '=') or line == not_set:
636 old_val = line
637 break
638 else:
639 if new_val == not_set:
640 return (ACTION_NO_ENTRY, config)
641 else:
642 return (ACTION_NO_ENTRY_WARN, config)
643
Masahiro Yamadacc008292016-05-19 15:51:56 +0900644 # If this CONFIG is neither bool nor trisate
645 if old_val[-2:] != '=y' and old_val[-2:] != '=m' and old_val != not_set:
646 # tools/scripts/define2mk.sed changes '1' to 'y'.
647 # This is a problem if the CONFIG is int type.
648 # Check the type in Kconfig and handle it correctly.
649 if new_val[-2:] == '=y':
650 new_val = new_val[:-1] + '1'
651
Masahiro Yamada50301592016-06-15 14:33:50 +0900652 return (ACTION_NO_CHANGE if old_val == new_val else ACTION_MOVE,
653 new_val)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900654
Masahiro Yamada1d085562016-05-19 15:52:02 +0900655 def update_dotconfig(self):
Masahiro Yamada6ff36d22016-05-19 15:51:50 +0900656 """Parse files for the config options and update the .config.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900657
Masahiro Yamadacc008292016-05-19 15:51:56 +0900658 This function parses the generated .config and include/autoconf.mk
659 searching the target options.
Masahiro Yamada6ff36d22016-05-19 15:51:50 +0900660 Move the config option(s) to the .config as needed.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900661
Simon Glass91197aa2021-12-18 14:54:35 -0700662 Args:
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900663 defconfig: defconfig name.
Masahiro Yamada522e8dc2016-05-19 15:52:01 +0900664
665 Returns:
Masahiro Yamada7fb0bac2016-05-19 15:52:04 +0900666 Return a tuple of (updated flag, log string).
667 The "updated flag" is True if the .config was updated, False
668 otherwise. The "log string" shows what happend to the .config.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900669 """
670
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900671 results = []
Masahiro Yamada7fb0bac2016-05-19 15:52:04 +0900672 updated = False
Masahiro Yamada916224c2016-08-22 22:18:21 +0900673 suspicious = False
Masahiro Yamada07913d12016-08-22 22:18:22 +0900674 rm_files = [self.config_autoconf, self.autoconf]
675
Simon Glassb2e83c62021-12-18 14:54:31 -0700676 if self.args.spl:
Masahiro Yamada07913d12016-08-22 22:18:22 +0900677 if os.path.exists(self.spl_autoconf):
678 autoconf_path = self.spl_autoconf
679 rm_files.append(self.spl_autoconf)
680 else:
681 for f in rm_files:
682 os.remove(f)
683 return (updated, suspicious,
Simon Glassb2e83c62021-12-18 14:54:31 -0700684 color_text(self.args.color, COLOR_BROWN,
Masahiro Yamada07913d12016-08-22 22:18:22 +0900685 "SPL is not enabled. Skipped.") + '\n')
686 else:
687 autoconf_path = self.autoconf
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900688
Simon Glass37f815c2021-12-18 14:54:34 -0700689 dotconfig_lines = read_file(self.dotconfig)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900690
Simon Glass37f815c2021-12-18 14:54:34 -0700691 autoconf_lines = read_file(autoconf_path)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900692
Masahiro Yamadab134bc12016-05-19 15:51:57 +0900693 for config in self.configs:
694 result = self.parse_one_config(config, dotconfig_lines,
Joe Hershberger96464ba2015-05-19 13:21:17 -0500695 autoconf_lines)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900696 results.append(result)
697
698 log = ''
699
700 for (action, value) in results:
701 if action == ACTION_MOVE:
702 actlog = "Move '%s'" % value
703 log_color = COLOR_LIGHT_GREEN
Masahiro Yamadacc008292016-05-19 15:51:56 +0900704 elif action == ACTION_NO_ENTRY:
Simon Glassdaa694d2021-12-18 14:54:30 -0700705 actlog = '%s is not defined in Kconfig. Do nothing.' % value
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900706 log_color = COLOR_LIGHT_BLUE
Masahiro Yamada916224c2016-08-22 22:18:21 +0900707 elif action == ACTION_NO_ENTRY_WARN:
Simon Glassdaa694d2021-12-18 14:54:30 -0700708 actlog = '%s is not defined in Kconfig (suspicious). Do nothing.' % value
Masahiro Yamada916224c2016-08-22 22:18:21 +0900709 log_color = COLOR_YELLOW
710 suspicious = True
Masahiro Yamadacc008292016-05-19 15:51:56 +0900711 elif action == ACTION_NO_CHANGE:
712 actlog = "'%s' is the same as the define in Kconfig. Do nothing." \
713 % value
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900714 log_color = COLOR_LIGHT_PURPLE
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900715 else:
Simon Glassdaa694d2021-12-18 14:54:30 -0700716 sys.exit('Internal Error. This should not happen.')
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900717
Simon Glassb2e83c62021-12-18 14:54:31 -0700718 log += color_text(self.args.color, log_color, actlog) + '\n'
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900719
Simon Glass91197aa2021-12-18 14:54:35 -0700720 with open(self.dotconfig, 'a', encoding='utf-8') as out:
Masahiro Yamadae423d172016-05-19 15:51:49 +0900721 for (action, value) in results:
722 if action == ACTION_MOVE:
Simon Glass91197aa2021-12-18 14:54:35 -0700723 out.write(value + '\n')
Masahiro Yamada7fb0bac2016-05-19 15:52:04 +0900724 updated = True
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900725
Masahiro Yamada5da4f852016-05-19 15:52:06 +0900726 self.results = results
Masahiro Yamada07913d12016-08-22 22:18:22 +0900727 for f in rm_files:
728 os.remove(f)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900729
Masahiro Yamada916224c2016-08-22 22:18:21 +0900730 return (updated, suspicious, log)
Masahiro Yamada522e8dc2016-05-19 15:52:01 +0900731
Masahiro Yamada5da4f852016-05-19 15:52:06 +0900732 def check_defconfig(self):
733 """Check the defconfig after savedefconfig
734
735 Returns:
736 Return additional log if moved CONFIGs were removed again by
737 'make savedefconfig'.
738 """
739
740 log = ''
741
Simon Glass37f815c2021-12-18 14:54:34 -0700742 defconfig_lines = read_file(self.defconfig)
Masahiro Yamada5da4f852016-05-19 15:52:06 +0900743
744 for (action, value) in self.results:
745 if action != ACTION_MOVE:
746 continue
Alper Nebi Yasak6c928c62022-01-29 18:22:08 +0300747 if not value in defconfig_lines:
Simon Glassb2e83c62021-12-18 14:54:31 -0700748 log += color_text(self.args.color, COLOR_YELLOW,
Masahiro Yamada5da4f852016-05-19 15:52:06 +0900749 "'%s' was removed by savedefconfig.\n" %
750 value)
751
752 return log
753
Simon Glassd73fcb12017-06-01 19:39:02 -0600754
755class DatabaseThread(threading.Thread):
756 """This thread processes results from Slot threads.
757
758 It collects the data in the master config directary. There is only one
759 result thread, and this helps to serialise the build output.
760 """
761 def __init__(self, config_db, db_queue):
762 """Set up a new result thread
763
764 Args:
765 builder: Builder which will be sent each result
766 """
767 threading.Thread.__init__(self)
768 self.config_db = config_db
769 self.db_queue= db_queue
770
771 def run(self):
772 """Called to start up the result thread.
773
774 We collect the next result job and pass it on to the build.
775 """
776 while True:
777 defconfig, configs = self.db_queue.get()
778 self.config_db[defconfig] = configs
779 self.db_queue.task_done()
780
781
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900782class Slot:
783
784 """A slot to store a subprocess.
785
786 Each instance of this class handles one subprocess.
787 This class is useful to control multiple threads
788 for faster processing.
789 """
790
Simon Glassb2e83c62021-12-18 14:54:31 -0700791 def __init__(self, toolchains, configs, args, progress, devnull,
Simon Glass6821a742017-07-10 14:47:47 -0600792 make_cmd, reference_src_dir, db_queue):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900793 """Create a new process slot.
794
Simon Glass91197aa2021-12-18 14:54:35 -0700795 Args:
Simon Glass6821a742017-07-10 14:47:47 -0600796 toolchains: Toolchains object containing toolchains.
Masahiro Yamadab134bc12016-05-19 15:51:57 +0900797 configs: A list of CONFIGs to move.
Simon Glassb2e83c62021-12-18 14:54:31 -0700798 args: Program arguments
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900799 progress: A progress indicator.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900800 devnull: A file object of '/dev/null'.
801 make_cmd: command name of GNU Make.
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500802 reference_src_dir: Determine the true starting config state from this
803 source tree.
Simon Glassd73fcb12017-06-01 19:39:02 -0600804 db_queue: output queue to write config info for the database
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900805 """
Simon Glass6821a742017-07-10 14:47:47 -0600806 self.toolchains = toolchains
Simon Glassb2e83c62021-12-18 14:54:31 -0700807 self.args = args
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900808 self.progress = progress
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900809 self.build_dir = tempfile.mkdtemp()
810 self.devnull = devnull
811 self.make_cmd = (make_cmd, 'O=' + self.build_dir)
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500812 self.reference_src_dir = reference_src_dir
Simon Glassd73fcb12017-06-01 19:39:02 -0600813 self.db_queue = db_queue
Simon Glassb2e83c62021-12-18 14:54:31 -0700814 self.parser = KconfigParser(configs, args, self.build_dir)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900815 self.state = STATE_IDLE
Masahiro Yamada09c6c062016-08-22 22:18:20 +0900816 self.failed_boards = set()
817 self.suspicious_boards = set()
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900818
819 def __del__(self):
820 """Delete the working directory
821
822 This function makes sure the temporary directory is cleaned away
823 even if Python suddenly dies due to error. It should be done in here
Joe Hershbergerf2dae752016-06-10 14:53:29 -0500824 because it is guaranteed the destructor is always invoked when the
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900825 instance of the class gets unreferenced.
826
827 If the subprocess is still running, wait until it finishes.
828 """
829 if self.state != STATE_IDLE:
830 while self.ps.poll() == None:
831 pass
832 shutil.rmtree(self.build_dir)
833
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900834 def add(self, defconfig):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900835 """Assign a new subprocess for defconfig and add it to the slot.
836
837 If the slot is vacant, create a new subprocess for processing the
838 given defconfig and add it to the slot. Just returns False if
839 the slot is occupied (i.e. the current subprocess is still running).
840
Simon Glass91197aa2021-12-18 14:54:35 -0700841 Args:
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900842 defconfig: defconfig name.
843
844 Returns:
845 Return True on success or False on failure
846 """
847 if self.state != STATE_IDLE:
848 return False
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900849
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900850 self.defconfig = defconfig
Masahiro Yamada1d085562016-05-19 15:52:02 +0900851 self.log = ''
Masahiro Yamadaf432c332016-06-15 14:33:52 +0900852 self.current_src_dir = self.reference_src_dir
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900853 self.do_defconfig()
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900854 return True
855
856 def poll(self):
857 """Check the status of the subprocess and handle it as needed.
858
859 Returns True if the slot is vacant (i.e. in idle state).
860 If the configuration is successfully finished, assign a new
861 subprocess to build include/autoconf.mk.
862 If include/autoconf.mk is generated, invoke the parser to
Masahiro Yamada7fb0bac2016-05-19 15:52:04 +0900863 parse the .config and the include/autoconf.mk, moving
864 config options to the .config as needed.
865 If the .config was updated, run "make savedefconfig" to sync
866 it, update the original defconfig, and then set the slot back
867 to the idle state.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900868
869 Returns:
870 Return True if the subprocess is terminated, False otherwise
871 """
872 if self.state == STATE_IDLE:
873 return True
874
875 if self.ps.poll() == None:
876 return False
877
878 if self.ps.poll() != 0:
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900879 self.handle_error()
880 elif self.state == STATE_DEFCONFIG:
Masahiro Yamadaf432c332016-06-15 14:33:52 +0900881 if self.reference_src_dir and not self.current_src_dir:
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500882 self.do_savedefconfig()
883 else:
884 self.do_autoconf()
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900885 elif self.state == STATE_AUTOCONF:
Masahiro Yamadaf432c332016-06-15 14:33:52 +0900886 if self.current_src_dir:
887 self.current_src_dir = None
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500888 self.do_defconfig()
Simon Glassb2e83c62021-12-18 14:54:31 -0700889 elif self.args.build_db:
Simon Glassd73fcb12017-06-01 19:39:02 -0600890 self.do_build_db()
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500891 else:
892 self.do_savedefconfig()
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900893 elif self.state == STATE_SAVEDEFCONFIG:
894 self.update_defconfig()
895 else:
Simon Glassdaa694d2021-12-18 14:54:30 -0700896 sys.exit('Internal Error. This should not happen.')
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900897
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900898 return True if self.state == STATE_IDLE else False
Joe Hershberger96464ba2015-05-19 13:21:17 -0500899
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900900 def handle_error(self):
901 """Handle error cases."""
Masahiro Yamada8513dc02016-05-19 15:52:08 +0900902
Simon Glassb2e83c62021-12-18 14:54:31 -0700903 self.log += color_text(self.args.color, COLOR_LIGHT_RED,
Simon Glassdaa694d2021-12-18 14:54:30 -0700904 'Failed to process.\n')
Simon Glassb2e83c62021-12-18 14:54:31 -0700905 if self.args.verbose:
906 self.log += color_text(self.args.color, COLOR_LIGHT_CYAN,
Markus Klotzbuecher4f5c5e92020-02-12 20:46:45 +0100907 self.ps.stderr.read().decode())
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900908 self.finish(False)
Joe Hershberger96464ba2015-05-19 13:21:17 -0500909
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900910 def do_defconfig(self):
911 """Run 'make <board>_defconfig' to create the .config file."""
Masahiro Yamadac8e1b102016-05-19 15:52:07 +0900912
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900913 cmd = list(self.make_cmd)
914 cmd.append(self.defconfig)
915 self.ps = subprocess.Popen(cmd, stdout=self.devnull,
Masahiro Yamadaf432c332016-06-15 14:33:52 +0900916 stderr=subprocess.PIPE,
917 cwd=self.current_src_dir)
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900918 self.state = STATE_DEFCONFIG
Masahiro Yamadac8e1b102016-05-19 15:52:07 +0900919
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900920 def do_autoconf(self):
Simon Glassf3b8e642017-06-01 19:39:01 -0600921 """Run 'make AUTO_CONF_PATH'."""
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900922
Simon Glass6821a742017-07-10 14:47:47 -0600923 arch = self.parser.get_arch()
924 try:
925 toolchain = self.toolchains.Select(arch)
926 except ValueError:
Simon Glassb2e83c62021-12-18 14:54:31 -0700927 self.log += color_text(self.args.color, COLOR_YELLOW,
Chris Packhamce3ba452017-08-27 20:00:51 +1200928 "Tool chain for '%s' is missing. Do nothing.\n" % arch)
Masahiro Yamada4efef992016-05-19 15:52:03 +0900929 self.finish(False)
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900930 return
Simon Glass793dca32019-10-31 07:42:57 -0600931 env = toolchain.MakeEnvironment(False)
Masahiro Yamada90ed6cb2016-05-19 15:51:53 +0900932
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900933 cmd = list(self.make_cmd)
Joe Hershberger7740f652015-05-19 13:21:18 -0500934 cmd.append('KCONFIG_IGNORE_DUPLICATES=1')
Simon Glassf3b8e642017-06-01 19:39:01 -0600935 cmd.append(AUTO_CONF_PATH)
Simon Glass6821a742017-07-10 14:47:47 -0600936 self.ps = subprocess.Popen(cmd, stdout=self.devnull, env=env,
Masahiro Yamadaf432c332016-06-15 14:33:52 +0900937 stderr=subprocess.PIPE,
938 cwd=self.current_src_dir)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900939 self.state = STATE_AUTOCONF
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900940
Simon Glassd73fcb12017-06-01 19:39:02 -0600941 def do_build_db(self):
942 """Add the board to the database"""
943 configs = {}
Simon Glass37f815c2021-12-18 14:54:34 -0700944 for line in read_file(os.path.join(self.build_dir, AUTO_CONF_PATH)):
945 if line.startswith('CONFIG'):
946 config, value = line.split('=', 1)
947 configs[config] = value.rstrip()
Simon Glassd73fcb12017-06-01 19:39:02 -0600948 self.db_queue.put([self.defconfig, configs])
949 self.finish(True)
950
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900951 def do_savedefconfig(self):
952 """Update the .config and run 'make savedefconfig'."""
953
Masahiro Yamada916224c2016-08-22 22:18:21 +0900954 (updated, suspicious, log) = self.parser.update_dotconfig()
955 if suspicious:
956 self.suspicious_boards.add(self.defconfig)
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900957 self.log += log
958
Simon Glassb2e83c62021-12-18 14:54:31 -0700959 if not self.args.force_sync and not updated:
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900960 self.finish(True)
961 return
962 if updated:
Simon Glassb2e83c62021-12-18 14:54:31 -0700963 self.log += color_text(self.args.color, COLOR_LIGHT_GREEN,
Simon Glassdaa694d2021-12-18 14:54:30 -0700964 'Syncing by savedefconfig...\n')
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900965 else:
Simon Glassdaa694d2021-12-18 14:54:30 -0700966 self.log += 'Syncing by savedefconfig (forced by option)...\n'
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900967
968 cmd = list(self.make_cmd)
969 cmd.append('savedefconfig')
970 self.ps = subprocess.Popen(cmd, stdout=self.devnull,
971 stderr=subprocess.PIPE)
972 self.state = STATE_SAVEDEFCONFIG
973
974 def update_defconfig(self):
975 """Update the input defconfig and go back to the idle state."""
976
Masahiro Yamadafc2661e2016-06-15 14:33:54 +0900977 log = self.parser.check_defconfig()
978 if log:
Masahiro Yamada09c6c062016-08-22 22:18:20 +0900979 self.suspicious_boards.add(self.defconfig)
Masahiro Yamadafc2661e2016-06-15 14:33:54 +0900980 self.log += log
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900981 orig_defconfig = os.path.join('configs', self.defconfig)
982 new_defconfig = os.path.join(self.build_dir, 'defconfig')
983 updated = not filecmp.cmp(orig_defconfig, new_defconfig)
984
985 if updated:
Simon Glassb2e83c62021-12-18 14:54:31 -0700986 self.log += color_text(self.args.color, COLOR_LIGHT_BLUE,
Simon Glassdaa694d2021-12-18 14:54:30 -0700987 'defconfig was updated.\n')
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900988
Simon Glassb2e83c62021-12-18 14:54:31 -0700989 if not self.args.dry_run and updated:
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900990 shutil.move(new_defconfig, orig_defconfig)
991 self.finish(True)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900992
Masahiro Yamada4efef992016-05-19 15:52:03 +0900993 def finish(self, success):
994 """Display log along with progress and go to the idle state.
Masahiro Yamada1d085562016-05-19 15:52:02 +0900995
Simon Glass91197aa2021-12-18 14:54:35 -0700996 Args:
Masahiro Yamada4efef992016-05-19 15:52:03 +0900997 success: Should be True when the defconfig was processed
998 successfully, or False when it fails.
Masahiro Yamada1d085562016-05-19 15:52:02 +0900999 """
1000 # output at least 30 characters to hide the "* defconfigs out of *".
1001 log = self.defconfig.ljust(30) + '\n'
1002
1003 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ])
1004 # Some threads are running in parallel.
1005 # Print log atomically to not mix up logs from different threads.
Simon Glass793dca32019-10-31 07:42:57 -06001006 print(log, file=(sys.stdout if success else sys.stderr))
Masahiro Yamada4efef992016-05-19 15:52:03 +09001007
1008 if not success:
Simon Glassb2e83c62021-12-18 14:54:31 -07001009 if self.args.exit_on_error:
Simon Glassdaa694d2021-12-18 14:54:30 -07001010 sys.exit('Exit on error.')
Masahiro Yamada4efef992016-05-19 15:52:03 +09001011 # If --exit-on-error flag is not set, skip this board and continue.
1012 # Record the failed board.
Masahiro Yamada09c6c062016-08-22 22:18:20 +09001013 self.failed_boards.add(self.defconfig)
Masahiro Yamada4efef992016-05-19 15:52:03 +09001014
Masahiro Yamada1d085562016-05-19 15:52:02 +09001015 self.progress.inc()
1016 self.progress.show()
Masahiro Yamada4efef992016-05-19 15:52:03 +09001017 self.state = STATE_IDLE
Masahiro Yamada1d085562016-05-19 15:52:02 +09001018
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001019 def get_failed_boards(self):
Masahiro Yamada09c6c062016-08-22 22:18:20 +09001020 """Returns a set of failed boards (defconfigs) in this slot.
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001021 """
1022 return self.failed_boards
1023
Masahiro Yamadafc2661e2016-06-15 14:33:54 +09001024 def get_suspicious_boards(self):
Masahiro Yamada09c6c062016-08-22 22:18:20 +09001025 """Returns a set of boards (defconfigs) with possible misconversion.
Masahiro Yamadafc2661e2016-06-15 14:33:54 +09001026 """
Masahiro Yamada916224c2016-08-22 22:18:21 +09001027 return self.suspicious_boards - self.failed_boards
Masahiro Yamadafc2661e2016-06-15 14:33:54 +09001028
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001029class Slots:
1030
1031 """Controller of the array of subprocess slots."""
1032
Simon Glassb2e83c62021-12-18 14:54:31 -07001033 def __init__(self, toolchains, configs, args, progress,
Simon Glass6821a742017-07-10 14:47:47 -06001034 reference_src_dir, db_queue):
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001035 """Create a new slots controller.
1036
Simon Glass91197aa2021-12-18 14:54:35 -07001037 Args:
Simon Glass6821a742017-07-10 14:47:47 -06001038 toolchains: Toolchains object containing toolchains.
Masahiro Yamadab134bc12016-05-19 15:51:57 +09001039 configs: A list of CONFIGs to move.
Simon Glassb2e83c62021-12-18 14:54:31 -07001040 args: Program arguments
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +09001041 progress: A progress indicator.
Joe Hershberger6b96c1a2016-06-10 14:53:32 -05001042 reference_src_dir: Determine the true starting config state from this
1043 source tree.
Simon Glassd73fcb12017-06-01 19:39:02 -06001044 db_queue: output queue to write config info for the database
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001045 """
Simon Glassb2e83c62021-12-18 14:54:31 -07001046 self.args = args
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001047 self.slots = []
Simon Glass478920d2021-12-18 14:54:32 -07001048 devnull = subprocess.DEVNULL
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001049 make_cmd = get_make_cmd()
Simon Glassb2e83c62021-12-18 14:54:31 -07001050 for i in range(args.jobs):
1051 self.slots.append(Slot(toolchains, configs, args, progress,
Simon Glass6821a742017-07-10 14:47:47 -06001052 devnull, make_cmd, reference_src_dir,
1053 db_queue))
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001054
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +09001055 def add(self, defconfig):
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001056 """Add a new subprocess if a vacant slot is found.
1057
Simon Glass91197aa2021-12-18 14:54:35 -07001058 Args:
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001059 defconfig: defconfig name to be put into.
1060
1061 Returns:
1062 Return True on success or False on failure
1063 """
1064 for slot in self.slots:
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +09001065 if slot.add(defconfig):
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001066 return True
1067 return False
1068
1069 def available(self):
1070 """Check if there is a vacant slot.
1071
1072 Returns:
1073 Return True if at lease one vacant slot is found, False otherwise.
1074 """
1075 for slot in self.slots:
1076 if slot.poll():
1077 return True
1078 return False
1079
1080 def empty(self):
1081 """Check if all slots are vacant.
1082
1083 Returns:
1084 Return True if all the slots are vacant, False otherwise.
1085 """
1086 ret = True
1087 for slot in self.slots:
1088 if not slot.poll():
1089 ret = False
1090 return ret
1091
1092 def show_failed_boards(self):
1093 """Display all of the failed boards (defconfigs)."""
Masahiro Yamada09c6c062016-08-22 22:18:20 +09001094 boards = set()
Masahiro Yamada96dccd92016-06-15 14:33:53 +09001095 output_file = 'moveconfig.failed'
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001096
1097 for slot in self.slots:
Masahiro Yamada09c6c062016-08-22 22:18:20 +09001098 boards |= slot.get_failed_boards()
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001099
Masahiro Yamada96dccd92016-06-15 14:33:53 +09001100 if boards:
1101 boards = '\n'.join(boards) + '\n'
Simon Glassdaa694d2021-12-18 14:54:30 -07001102 msg = 'The following boards were not processed due to error:\n'
Masahiro Yamada96dccd92016-06-15 14:33:53 +09001103 msg += boards
Simon Glassdaa694d2021-12-18 14:54:30 -07001104 msg += '(the list has been saved in %s)\n' % output_file
Simon Glassb2e83c62021-12-18 14:54:31 -07001105 print(color_text(self.args.color, COLOR_LIGHT_RED,
Simon Glass793dca32019-10-31 07:42:57 -06001106 msg), file=sys.stderr)
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001107
Simon Glass2fd85bd2021-12-18 14:54:33 -07001108 write_file(output_file, boards)
Joe Hershberger2559cd82015-05-19 13:21:22 -05001109
Masahiro Yamadafc2661e2016-06-15 14:33:54 +09001110 def show_suspicious_boards(self):
1111 """Display all boards (defconfigs) with possible misconversion."""
Masahiro Yamada09c6c062016-08-22 22:18:20 +09001112 boards = set()
Masahiro Yamadafc2661e2016-06-15 14:33:54 +09001113 output_file = 'moveconfig.suspicious'
1114
1115 for slot in self.slots:
Masahiro Yamada09c6c062016-08-22 22:18:20 +09001116 boards |= slot.get_suspicious_boards()
Masahiro Yamadafc2661e2016-06-15 14:33:54 +09001117
1118 if boards:
1119 boards = '\n'.join(boards) + '\n'
Simon Glassdaa694d2021-12-18 14:54:30 -07001120 msg = 'The following boards might have been converted incorrectly.\n'
1121 msg += 'It is highly recommended to check them manually:\n'
Masahiro Yamadafc2661e2016-06-15 14:33:54 +09001122 msg += boards
Simon Glassdaa694d2021-12-18 14:54:30 -07001123 msg += '(the list has been saved in %s)\n' % output_file
Simon Glassb2e83c62021-12-18 14:54:31 -07001124 print(color_text(self.args.color, COLOR_YELLOW,
Simon Glass793dca32019-10-31 07:42:57 -06001125 msg), file=sys.stderr)
Masahiro Yamadafc2661e2016-06-15 14:33:54 +09001126
Simon Glass2fd85bd2021-12-18 14:54:33 -07001127 write_file(output_file, boards)
Masahiro Yamadafc2661e2016-06-15 14:33:54 +09001128
Masahiro Yamada5cc42a52016-06-15 14:33:51 +09001129class ReferenceSource:
1130
1131 """Reference source against which original configs should be parsed."""
1132
1133 def __init__(self, commit):
1134 """Create a reference source directory based on a specified commit.
1135
Simon Glass91197aa2021-12-18 14:54:35 -07001136 Args:
Masahiro Yamada5cc42a52016-06-15 14:33:51 +09001137 commit: commit to git-clone
1138 """
1139 self.src_dir = tempfile.mkdtemp()
Simon Glassdaa694d2021-12-18 14:54:30 -07001140 print('Cloning git repo to a separate work directory...')
Masahiro Yamada5cc42a52016-06-15 14:33:51 +09001141 subprocess.check_output(['git', 'clone', os.getcwd(), '.'],
1142 cwd=self.src_dir)
Simon Glass793dca32019-10-31 07:42:57 -06001143 print("Checkout '%s' to build the original autoconf.mk." % \
1144 subprocess.check_output(['git', 'rev-parse', '--short', commit]).strip())
Masahiro Yamada5cc42a52016-06-15 14:33:51 +09001145 subprocess.check_output(['git', 'checkout', commit],
1146 stderr=subprocess.STDOUT, cwd=self.src_dir)
Joe Hershberger6b96c1a2016-06-10 14:53:32 -05001147
1148 def __del__(self):
Masahiro Yamada5cc42a52016-06-15 14:33:51 +09001149 """Delete the reference source directory
Joe Hershberger6b96c1a2016-06-10 14:53:32 -05001150
1151 This function makes sure the temporary directory is cleaned away
1152 even if Python suddenly dies due to error. It should be done in here
1153 because it is guaranteed the destructor is always invoked when the
1154 instance of the class gets unreferenced.
1155 """
Masahiro Yamada5cc42a52016-06-15 14:33:51 +09001156 shutil.rmtree(self.src_dir)
Joe Hershberger6b96c1a2016-06-10 14:53:32 -05001157
Masahiro Yamada5cc42a52016-06-15 14:33:51 +09001158 def get_dir(self):
1159 """Return the absolute path to the reference source directory."""
1160
1161 return self.src_dir
Joe Hershberger6b96c1a2016-06-10 14:53:32 -05001162
Simon Glassb2e83c62021-12-18 14:54:31 -07001163def move_config(toolchains, configs, args, db_queue):
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001164 """Move config options to defconfig files.
1165
Simon Glass91197aa2021-12-18 14:54:35 -07001166 Args:
Masahiro Yamadab134bc12016-05-19 15:51:57 +09001167 configs: A list of CONFIGs to move.
Simon Glassb2e83c62021-12-18 14:54:31 -07001168 args: Program arguments
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001169 """
Masahiro Yamadab134bc12016-05-19 15:51:57 +09001170 if len(configs) == 0:
Simon Glassb2e83c62021-12-18 14:54:31 -07001171 if args.force_sync:
Simon Glass793dca32019-10-31 07:42:57 -06001172 print('No CONFIG is specified. You are probably syncing defconfigs.', end=' ')
Simon Glassb2e83c62021-12-18 14:54:31 -07001173 elif args.build_db:
Simon Glass793dca32019-10-31 07:42:57 -06001174 print('Building %s database' % CONFIG_DATABASE)
Masahiro Yamada6a9f79f2016-05-19 15:52:09 +09001175 else:
Simon Glass793dca32019-10-31 07:42:57 -06001176 print('Neither CONFIG nor --force-sync is specified. Nothing will happen.', end=' ')
Masahiro Yamada6a9f79f2016-05-19 15:52:09 +09001177 else:
Simon Glass793dca32019-10-31 07:42:57 -06001178 print('Move ' + ', '.join(configs), end=' ')
Simon Glassb2e83c62021-12-18 14:54:31 -07001179 print('(jobs: %d)\n' % args.jobs)
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001180
Simon Glassb2e83c62021-12-18 14:54:31 -07001181 if args.git_ref:
1182 reference_src = ReferenceSource(args.git_ref)
Masahiro Yamada5cc42a52016-06-15 14:33:51 +09001183 reference_src_dir = reference_src.get_dir()
1184 else:
Masahiro Yamadaf432c332016-06-15 14:33:52 +09001185 reference_src_dir = None
Joe Hershberger6b96c1a2016-06-10 14:53:32 -05001186
Simon Glassb2e83c62021-12-18 14:54:31 -07001187 if args.defconfigs:
1188 defconfigs = get_matched_defconfigs(args.defconfigs)
Joe Hershberger91040e82015-05-19 13:21:19 -05001189 else:
Masahiro Yamada684c3062016-07-25 19:15:28 +09001190 defconfigs = get_all_defconfigs()
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001191
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +09001192 progress = Progress(len(defconfigs))
Simon Glassb2e83c62021-12-18 14:54:31 -07001193 slots = Slots(toolchains, configs, args, progress, reference_src_dir,
Simon Glass91197aa2021-12-18 14:54:35 -07001194 db_queue)
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001195
1196 # Main loop to process defconfig files:
1197 # Add a new subprocess into a vacant slot.
1198 # Sleep if there is no available slot.
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +09001199 for defconfig in defconfigs:
1200 while not slots.add(defconfig):
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001201 while not slots.available():
1202 # No available slot: sleep for a while
1203 time.sleep(SLEEP_TIME)
1204
1205 # wait until all the subprocesses finish
1206 while not slots.empty():
1207 time.sleep(SLEEP_TIME)
1208
Simon Glass793dca32019-10-31 07:42:57 -06001209 print('')
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001210 slots.show_failed_boards()
Masahiro Yamadafc2661e2016-06-15 14:33:54 +09001211 slots.show_suspicious_boards()
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001212
Simon Glasscb008832017-06-15 21:39:33 -06001213def find_kconfig_rules(kconf, config, imply_config):
1214 """Check whether a config has a 'select' or 'imply' keyword
1215
1216 Args:
Tom Rini65e05dd2019-09-20 17:42:09 -04001217 kconf: Kconfiglib.Kconfig object
Simon Glasscb008832017-06-15 21:39:33 -06001218 config: Name of config to check (without CONFIG_ prefix)
1219 imply_config: Implying config (without CONFIG_ prefix) which may or
1220 may not have an 'imply' for 'config')
1221
1222 Returns:
1223 Symbol object for 'config' if found, else None
1224 """
Tom Rini65e05dd2019-09-20 17:42:09 -04001225 sym = kconf.syms.get(imply_config)
Simon Glasscb008832017-06-15 21:39:33 -06001226 if sym:
Simon Glassea40b202021-07-21 21:35:53 -06001227 for sel, cond in (sym.selects + sym.implies):
Simon Glassa3627082021-12-18 08:09:42 -07001228 if sel.name == config:
Simon Glasscb008832017-06-15 21:39:33 -06001229 return sym
1230 return None
1231
1232def check_imply_rule(kconf, config, imply_config):
1233 """Check if we can add an 'imply' option
1234
1235 This finds imply_config in the Kconfig and looks to see if it is possible
1236 to add an 'imply' for 'config' to that part of the Kconfig.
1237
1238 Args:
Tom Rini65e05dd2019-09-20 17:42:09 -04001239 kconf: Kconfiglib.Kconfig object
Simon Glasscb008832017-06-15 21:39:33 -06001240 config: Name of config to check (without CONFIG_ prefix)
1241 imply_config: Implying config (without CONFIG_ prefix) which may or
1242 may not have an 'imply' for 'config')
1243
1244 Returns:
1245 tuple:
1246 filename of Kconfig file containing imply_config, or None if none
1247 line number within the Kconfig file, or 0 if none
1248 message indicating the result
1249 """
Tom Rini65e05dd2019-09-20 17:42:09 -04001250 sym = kconf.syms.get(imply_config)
Simon Glasscb008832017-06-15 21:39:33 -06001251 if not sym:
1252 return 'cannot find sym'
Simon Glassea40b202021-07-21 21:35:53 -06001253 nodes = sym.nodes
1254 if len(nodes) != 1:
1255 return '%d locations' % len(nodes)
Simon Glassa3627082021-12-18 08:09:42 -07001256 node = nodes[0]
1257 fname, linenum = node.filename, node.linenr
Simon Glasscb008832017-06-15 21:39:33 -06001258 cwd = os.getcwd()
1259 if cwd and fname.startswith(cwd):
1260 fname = fname[len(cwd) + 1:]
1261 file_line = ' at %s:%d' % (fname, linenum)
Simon Glass37f815c2021-12-18 14:54:34 -07001262 data = read_file(fname)
Simon Glasscb008832017-06-15 21:39:33 -06001263 if data[linenum - 1] != 'config %s' % imply_config:
1264 return None, 0, 'bad sym format %s%s' % (data[linenum], file_line)
1265 return fname, linenum, 'adding%s' % file_line
1266
1267def add_imply_rule(config, fname, linenum):
1268 """Add a new 'imply' option to a Kconfig
1269
1270 Args:
1271 config: config option to add an imply for (without CONFIG_ prefix)
1272 fname: Kconfig filename to update
1273 linenum: Line number to place the 'imply' before
1274
1275 Returns:
1276 Message indicating the result
1277 """
1278 file_line = ' at %s:%d' % (fname, linenum)
Simon Glass37f815c2021-12-18 14:54:34 -07001279 data = read_file(fname)
Simon Glasscb008832017-06-15 21:39:33 -06001280 linenum -= 1
1281
1282 for offset, line in enumerate(data[linenum:]):
1283 if line.strip().startswith('help') or not line:
1284 data.insert(linenum + offset, '\timply %s' % config)
Simon Glass2fd85bd2021-12-18 14:54:33 -07001285 write_file(fname, data)
Simon Glasscb008832017-06-15 21:39:33 -06001286 return 'added%s' % file_line
1287
1288 return 'could not insert%s'
1289
1290(IMPLY_MIN_2, IMPLY_TARGET, IMPLY_CMD, IMPLY_NON_ARCH_BOARD) = (
1291 1, 2, 4, 8)
Simon Glass9b2a2e82017-06-15 21:39:32 -06001292
1293IMPLY_FLAGS = {
1294 'min2': [IMPLY_MIN_2, 'Show options which imply >2 boards (normally >5)'],
1295 'target': [IMPLY_TARGET, 'Allow CONFIG_TARGET_... options to imply'],
1296 'cmd': [IMPLY_CMD, 'Allow CONFIG_CMD_... to imply'],
Simon Glasscb008832017-06-15 21:39:33 -06001297 'non-arch-board': [
1298 IMPLY_NON_ARCH_BOARD,
1299 'Allow Kconfig options outside arch/ and /board/ to imply'],
Simon Glass91197aa2021-12-18 14:54:35 -07001300}
Simon Glass9b2a2e82017-06-15 21:39:32 -06001301
Simon Glass9d603392021-12-18 08:09:43 -07001302
1303def read_database():
1304 """Read in the config database
1305
1306 Returns:
1307 tuple:
1308 set of all config options seen (each a str)
1309 set of all defconfigs seen (each a str)
1310 dict of configs for each defconfig:
1311 key: defconfig name, e.g. "MPC8548CDS_legacy_defconfig"
1312 value: dict:
1313 key: CONFIG option
1314 value: Value of option
1315 dict of defconfigs for each config:
1316 key: CONFIG option
1317 value: set of boards using that option
1318
1319 """
1320 configs = {}
1321
1322 # key is defconfig name, value is dict of (CONFIG_xxx, value)
1323 config_db = {}
1324
1325 # Set of all config options we have seen
1326 all_configs = set()
1327
1328 # Set of all defconfigs we have seen
1329 all_defconfigs = set()
1330
1331 defconfig_db = collections.defaultdict(set)
Simon Glass37f815c2021-12-18 14:54:34 -07001332 for line in read_file(CONFIG_DATABASE):
1333 line = line.rstrip()
1334 if not line: # Separator between defconfigs
1335 config_db[defconfig] = configs
1336 all_defconfigs.add(defconfig)
1337 configs = {}
1338 elif line[0] == ' ': # CONFIG line
1339 config, value = line.strip().split('=', 1)
1340 configs[config] = value
1341 defconfig_db[config].add(defconfig)
1342 all_configs.add(config)
1343 else: # New defconfig
1344 defconfig = line
Simon Glass9d603392021-12-18 08:09:43 -07001345
1346 return all_configs, all_defconfigs, config_db, defconfig_db
1347
1348
Simon Glasscb008832017-06-15 21:39:33 -06001349def do_imply_config(config_list, add_imply, imply_flags, skip_added,
1350 check_kconfig=True, find_superset=False):
Simon Glass99b66602017-06-01 19:39:03 -06001351 """Find CONFIG options which imply those in the list
1352
1353 Some CONFIG options can be implied by others and this can help to reduce
1354 the size of the defconfig files. For example, CONFIG_X86 implies
1355 CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and
1356 all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to
1357 each of the x86 defconfig files.
1358
1359 This function uses the moveconfig database to find such options. It
1360 displays a list of things that could possibly imply those in the list.
1361 The algorithm ignores any that start with CONFIG_TARGET since these
1362 typically refer to only a few defconfigs (often one). It also does not
1363 display a config with less than 5 defconfigs.
1364
1365 The algorithm works using sets. For each target config in config_list:
1366 - Get the set 'defconfigs' which use that target config
1367 - For each config (from a list of all configs):
1368 - Get the set 'imply_defconfig' of defconfigs which use that config
1369 -
1370 - If imply_defconfigs contains anything not in defconfigs then
1371 this config does not imply the target config
1372
1373 Params:
1374 config_list: List of CONFIG options to check (each a string)
Simon Glasscb008832017-06-15 21:39:33 -06001375 add_imply: Automatically add an 'imply' for each config.
Simon Glass9b2a2e82017-06-15 21:39:32 -06001376 imply_flags: Flags which control which implying configs are allowed
1377 (IMPLY_...)
Simon Glasscb008832017-06-15 21:39:33 -06001378 skip_added: Don't show options which already have an imply added.
1379 check_kconfig: Check if implied symbols already have an 'imply' or
1380 'select' for the target config, and show this information if so.
Simon Glass99b66602017-06-01 19:39:03 -06001381 find_superset: True to look for configs which are a superset of those
1382 already found. So for example if CONFIG_EXYNOS5 implies an option,
1383 but CONFIG_EXYNOS covers a larger set of defconfigs and also
1384 implies that option, this will drop the former in favour of the
1385 latter. In practice this option has not proved very used.
1386
1387 Note the terminoloy:
1388 config - a CONFIG_XXX options (a string, e.g. 'CONFIG_CMD_EEPROM')
1389 defconfig - a defconfig file (a string, e.g. 'configs/snow_defconfig')
1390 """
Simon Glasscb008832017-06-15 21:39:33 -06001391 kconf = KconfigScanner().conf if check_kconfig else None
1392 if add_imply and add_imply != 'all':
Simon Glassa3627082021-12-18 08:09:42 -07001393 add_imply = add_imply.split(',')
Simon Glasscb008832017-06-15 21:39:33 -06001394
Simon Glass9d603392021-12-18 08:09:43 -07001395 all_configs, all_defconfigs, config_db, defconfig_db = read_database()
Simon Glass99b66602017-06-01 19:39:03 -06001396
Simon Glassa3627082021-12-18 08:09:42 -07001397 # Work through each target config option in turn, independently
Simon Glass99b66602017-06-01 19:39:03 -06001398 for config in config_list:
1399 defconfigs = defconfig_db.get(config)
1400 if not defconfigs:
Simon Glass793dca32019-10-31 07:42:57 -06001401 print('%s not found in any defconfig' % config)
Simon Glass99b66602017-06-01 19:39:03 -06001402 continue
1403
1404 # Get the set of defconfigs without this one (since a config cannot
1405 # imply itself)
1406 non_defconfigs = all_defconfigs - defconfigs
1407 num_defconfigs = len(defconfigs)
Simon Glass793dca32019-10-31 07:42:57 -06001408 print('%s found in %d/%d defconfigs' % (config, num_defconfigs,
1409 len(all_configs)))
Simon Glass99b66602017-06-01 19:39:03 -06001410
1411 # This will hold the results: key=config, value=defconfigs containing it
1412 imply_configs = {}
1413 rest_configs = all_configs - set([config])
1414
1415 # Look at every possible config, except the target one
1416 for imply_config in rest_configs:
Simon Glass9b2a2e82017-06-15 21:39:32 -06001417 if 'ERRATUM' in imply_config:
Simon Glass99b66602017-06-01 19:39:03 -06001418 continue
Simon Glass91197aa2021-12-18 14:54:35 -07001419 if not imply_flags & IMPLY_CMD:
Simon Glass9b2a2e82017-06-15 21:39:32 -06001420 if 'CONFIG_CMD' in imply_config:
1421 continue
Simon Glass91197aa2021-12-18 14:54:35 -07001422 if not imply_flags & IMPLY_TARGET:
Simon Glass9b2a2e82017-06-15 21:39:32 -06001423 if 'CONFIG_TARGET' in imply_config:
1424 continue
Simon Glass99b66602017-06-01 19:39:03 -06001425
1426 # Find set of defconfigs that have this config
1427 imply_defconfig = defconfig_db[imply_config]
1428
1429 # Get the intersection of this with defconfigs containing the
1430 # target config
1431 common_defconfigs = imply_defconfig & defconfigs
1432
1433 # Get the set of defconfigs containing this config which DO NOT
1434 # also contain the taret config. If this set is non-empty it means
1435 # that this config affects other defconfigs as well as (possibly)
1436 # the ones affected by the target config. This means it implies
1437 # things we don't want to imply.
1438 not_common_defconfigs = imply_defconfig & non_defconfigs
1439 if not_common_defconfigs:
1440 continue
1441
1442 # If there are common defconfigs, imply_config may be useful
1443 if common_defconfigs:
1444 skip = False
1445 if find_superset:
Simon Glass793dca32019-10-31 07:42:57 -06001446 for prev in list(imply_configs.keys()):
Simon Glass99b66602017-06-01 19:39:03 -06001447 prev_count = len(imply_configs[prev])
1448 count = len(common_defconfigs)
1449 if (prev_count > count and
1450 (imply_configs[prev] & common_defconfigs ==
1451 common_defconfigs)):
1452 # skip imply_config because prev is a superset
1453 skip = True
1454 break
1455 elif count > prev_count:
1456 # delete prev because imply_config is a superset
1457 del imply_configs[prev]
1458 if not skip:
1459 imply_configs[imply_config] = common_defconfigs
1460
1461 # Now we have a dict imply_configs of configs which imply each config
1462 # The value of each dict item is the set of defconfigs containing that
1463 # config. Rank them so that we print the configs that imply the largest
1464 # number of defconfigs first.
Simon Glasscb008832017-06-15 21:39:33 -06001465 ranked_iconfigs = sorted(imply_configs,
Simon Glass99b66602017-06-01 19:39:03 -06001466 key=lambda k: len(imply_configs[k]), reverse=True)
Simon Glasscb008832017-06-15 21:39:33 -06001467 kconfig_info = ''
1468 cwd = os.getcwd()
1469 add_list = collections.defaultdict(list)
1470 for iconfig in ranked_iconfigs:
1471 num_common = len(imply_configs[iconfig])
Simon Glass99b66602017-06-01 19:39:03 -06001472
1473 # Don't bother if there are less than 5 defconfigs affected.
Simon Glass9b2a2e82017-06-15 21:39:32 -06001474 if num_common < (2 if imply_flags & IMPLY_MIN_2 else 5):
Simon Glass99b66602017-06-01 19:39:03 -06001475 continue
Simon Glasscb008832017-06-15 21:39:33 -06001476 missing = defconfigs - imply_configs[iconfig]
Simon Glass99b66602017-06-01 19:39:03 -06001477 missing_str = ', '.join(missing) if missing else 'all'
1478 missing_str = ''
Simon Glasscb008832017-06-15 21:39:33 -06001479 show = True
1480 if kconf:
1481 sym = find_kconfig_rules(kconf, config[CONFIG_LEN:],
1482 iconfig[CONFIG_LEN:])
1483 kconfig_info = ''
1484 if sym:
Simon Glassea40b202021-07-21 21:35:53 -06001485 nodes = sym.nodes
1486 if len(nodes) == 1:
1487 fname, linenum = nodes[0].filename, nodes[0].linenr
Simon Glasscb008832017-06-15 21:39:33 -06001488 if cwd and fname.startswith(cwd):
1489 fname = fname[len(cwd) + 1:]
1490 kconfig_info = '%s:%d' % (fname, linenum)
1491 if skip_added:
1492 show = False
1493 else:
Tom Rini65e05dd2019-09-20 17:42:09 -04001494 sym = kconf.syms.get(iconfig[CONFIG_LEN:])
Simon Glasscb008832017-06-15 21:39:33 -06001495 fname = ''
1496 if sym:
Simon Glassea40b202021-07-21 21:35:53 -06001497 nodes = sym.nodes
1498 if len(nodes) == 1:
1499 fname, linenum = nodes[0].filename, nodes[0].linenr
Simon Glasscb008832017-06-15 21:39:33 -06001500 if cwd and fname.startswith(cwd):
1501 fname = fname[len(cwd) + 1:]
1502 in_arch_board = not sym or (fname.startswith('arch') or
1503 fname.startswith('board'))
1504 if (not in_arch_board and
Simon Glass91197aa2021-12-18 14:54:35 -07001505 not imply_flags & IMPLY_NON_ARCH_BOARD):
Simon Glasscb008832017-06-15 21:39:33 -06001506 continue
1507
1508 if add_imply and (add_imply == 'all' or
1509 iconfig in add_imply):
1510 fname, linenum, kconfig_info = (check_imply_rule(kconf,
1511 config[CONFIG_LEN:], iconfig[CONFIG_LEN:]))
1512 if fname:
1513 add_list[fname].append(linenum)
1514
1515 if show and kconfig_info != 'skip':
Simon Glass793dca32019-10-31 07:42:57 -06001516 print('%5d : %-30s%-25s %s' % (num_common, iconfig.ljust(30),
1517 kconfig_info, missing_str))
Simon Glasscb008832017-06-15 21:39:33 -06001518
1519 # Having collected a list of things to add, now we add them. We process
1520 # each file from the largest line number to the smallest so that
1521 # earlier additions do not affect our line numbers. E.g. if we added an
1522 # imply at line 20 it would change the position of each line after
1523 # that.
Simon Glass793dca32019-10-31 07:42:57 -06001524 for fname, linenums in add_list.items():
Simon Glasscb008832017-06-15 21:39:33 -06001525 for linenum in sorted(linenums, reverse=True):
1526 add_imply_rule(config[CONFIG_LEN:], fname, linenum)
Simon Glass99b66602017-06-01 19:39:03 -06001527
Simon Glass941671a2022-02-08 11:49:46 -07001528def defconfig_matches(configs, re_match):
1529 """Check if any CONFIG option matches a regex
1530
1531 The match must be complete, i.e. from the start to end of the CONFIG option.
1532
1533 Args:
1534 configs (dict): Dict of CONFIG options:
1535 key: CONFIG option
1536 value: Value of option
1537 re_match (re.Pattern): Match to check
1538
1539 Returns:
1540 bool: True if any CONFIG matches the regex
1541 """
1542 for cfg in configs:
Simon Glassd9c958f2022-03-05 20:18:54 -07001543 if re_match.fullmatch(cfg):
Simon Glass941671a2022-02-08 11:49:46 -07001544 return True
1545 return False
Simon Glass99b66602017-06-01 19:39:03 -06001546
Simon Glass65d7fce2021-12-18 08:09:46 -07001547def do_find_config(config_list):
1548 """Find boards with a given combination of CONFIGs
1549
1550 Params:
Simon Glass941671a2022-02-08 11:49:46 -07001551 config_list: List of CONFIG options to check (each a regex consisting
Simon Glass65d7fce2021-12-18 08:09:46 -07001552 of a config option, with or without a CONFIG_ prefix. If an option
1553 is preceded by a tilde (~) then it must be false, otherwise it must
1554 be true)
1555 """
1556 all_configs, all_defconfigs, config_db, defconfig_db = read_database()
1557
Simon Glass65d7fce2021-12-18 08:09:46 -07001558 # Start with all defconfigs
1559 out = all_defconfigs
1560
1561 # Work through each config in turn
Simon Glass65d7fce2021-12-18 08:09:46 -07001562 for item in config_list:
1563 # Get the real config name and whether we want this config or not
1564 cfg = item
1565 want = True
1566 if cfg[0] == '~':
1567 want = False
1568 cfg = cfg[1:]
1569
Simon Glass65d7fce2021-12-18 08:09:46 -07001570 # Search everything that is still in the running. If it has a config
1571 # that we want, or doesn't have one that we don't, add it into the
1572 # running for the next stage
1573 in_list = out
1574 out = set()
Simon Glass941671a2022-02-08 11:49:46 -07001575 re_match = re.compile(cfg)
Simon Glass65d7fce2021-12-18 08:09:46 -07001576 for defc in in_list:
Simon Glass941671a2022-02-08 11:49:46 -07001577 has_cfg = defconfig_matches(config_db[defc], re_match)
Simon Glass65d7fce2021-12-18 08:09:46 -07001578 if has_cfg == want:
1579 out.add(defc)
Tom Rini9ef3ba82022-12-04 10:14:16 -05001580 print(f'{len(out)} matches')
1581 print(' '.join(item.split('_defconfig')[0] for item in out))
Simon Glass65d7fce2021-12-18 08:09:46 -07001582
1583
1584def prefix_config(cfg):
1585 """Prefix a config with CONFIG_ if needed
1586
1587 This handles ~ operator, which indicates that the CONFIG should be disabled
1588
1589 >>> prefix_config('FRED')
1590 'CONFIG_FRED'
1591 >>> prefix_config('CONFIG_FRED')
1592 'CONFIG_FRED'
1593 >>> prefix_config('~FRED')
1594 '~CONFIG_FRED'
1595 >>> prefix_config('~CONFIG_FRED')
1596 '~CONFIG_FRED'
1597 >>> prefix_config('A123')
1598 'CONFIG_A123'
1599 """
1600 op = ''
1601 if cfg[0] == '~':
1602 op = cfg[0]
1603 cfg = cfg[1:]
1604 if not cfg.startswith('CONFIG_'):
1605 cfg = 'CONFIG_' + cfg
1606 return op + cfg
1607
1608
Simon Glass65e62032023-02-01 13:19:12 -07001609RE_MK_CONFIGS = re.compile('CONFIG_(\$\(SPL_(?:TPL_)?\))?([A-Za-z0-9_]*)')
1610RE_IFDEF = re.compile('(ifdef|ifndef)')
1611RE_C_CONFIGS = re.compile('CONFIG_([A-Za-z0-9_]*)')
1612RE_CONFIG_IS = re.compile('CONFIG_IS_ENABLED\(([A-Za-z0-9_]*)\)')
1613
1614class ConfigUse:
1615 def __init__(self, cfg, is_spl, fname, rest):
1616 self.cfg = cfg
1617 self.is_spl = is_spl
1618 self.fname = fname
1619 self.rest = rest
1620
1621 def __hash__(self):
1622 return hash((self.cfg, self.is_spl))
1623
1624def scan_makefiles(fnames):
1625 """Scan Makefiles looking for Kconfig options
1626
1627 Looks for uses of CONFIG options in Makefiles
1628
1629 Args:
1630 fnames (list of tuple):
1631 str: Makefile filename where the option was found
1632 str: Line of the Makefile
1633
1634 Returns:
1635 tuple:
1636 dict: all_uses
1637 key (ConfigUse): object
1638 value (list of str): matching lines
1639 dict: Uses by filename
1640 key (str): filename
1641 value (set of ConfigUse): uses in that filename
1642
1643 >>> RE_MK_CONFIGS.search('CONFIG_FRED').groups()
1644 (None, 'FRED')
1645 >>> RE_MK_CONFIGS.search('CONFIG_$(SPL_)MARY').groups()
1646 ('$(SPL_)', 'MARY')
1647 >>> RE_MK_CONFIGS.search('CONFIG_$(SPL_TPL_)MARY').groups()
1648 ('$(SPL_TPL_)', 'MARY')
1649 """
1650 all_uses = collections.defaultdict(list)
1651 fname_uses = {}
1652 for fname, rest in fnames:
1653 m_iter = RE_MK_CONFIGS.finditer(rest)
1654 found = False
1655 for m in m_iter:
1656 found = True
1657 real_opt = m.group(2)
1658 if real_opt == '':
1659 continue
1660 is_spl = False
1661 if m.group(1):
1662 is_spl = True
1663 use = ConfigUse(real_opt, is_spl, fname, rest)
1664 if fname not in fname_uses:
1665 fname_uses[fname] = set()
1666 fname_uses[fname].add(use)
1667 all_uses[use].append(rest)
1668 return all_uses, fname_uses
1669
1670
1671def scan_src_files(fnames):
1672 """Scan source files (other than Makefiles) looking for Kconfig options
1673
1674 Looks for uses of CONFIG options
1675
1676 Args:
1677 fnames (list of tuple):
1678 str: Makefile filename where the option was found
1679 str: Line of the Makefile
1680
1681 Returns:
1682 tuple:
1683 dict: all_uses
1684 key (ConfigUse): object
1685 value (list of str): matching lines
1686 dict: Uses by filename
1687 key (str): filename
1688 value (set of ConfigUse): uses in that filename
1689
1690 >>> RE_C_CONFIGS.search('CONFIG_FRED').groups()
1691 ('FRED',)
1692 >>> RE_CONFIG_IS.search('CONFIG_IS_ENABLED(MARY)').groups()
1693 ('MARY',)
1694 >>> RE_CONFIG_IS.search('#if CONFIG_IS_ENABLED(OF_PLATDATA)').groups()
1695 ('OF_PLATDATA',)
1696 """
1697 def add_uses(m_iter, is_spl):
1698 for m in m_iter:
1699 found = True
1700 real_opt = m.group(1)
1701 if real_opt == '':
1702 continue
1703 use = ConfigUse(real_opt, is_spl, fname, rest)
1704 if fname not in fname_uses:
1705 fname_uses[fname] = set()
1706 fname_uses[fname].add(use)
1707 all_uses[use].append(rest)
1708
1709 all_uses = collections.defaultdict(list)
1710 fname_uses = {}
1711 for fname, rest in fnames:
1712 m_iter = RE_C_CONFIGS.finditer(rest)
1713 add_uses(m_iter, False)
1714
1715 m_iter2 = RE_CONFIG_IS.finditer(rest)
1716 add_uses(m_iter2, True)
1717
1718 return all_uses, fname_uses
1719
1720
1721MODE_NORMAL, MODE_SPL, MODE_PROPER = range(3)
1722
1723def do_scan_source(path, do_update):
1724 """Scan the source tree for Kconfig inconsistencies
1725
1726 Args:
1727 path (str): Path to source tree
1728 do_update (bool) : True to write to scripts/kconf_... files
1729 """
1730 def is_not_proper(name):
1731 for prefix in SPL_PREFIXES:
1732 if name.startswith(prefix):
1733 return name[len(prefix):]
1734 return False
1735
1736 def check_not_found(all_uses, spl_mode):
1737 """Check for Kconfig options mentioned in the source but not in Kconfig
1738
1739 Args:
1740 all_uses (dict):
1741 key (ConfigUse): object
1742 value (list of str): matching lines
1743 spl_mode (int): If MODE_SPL, look at source code which implies
1744 an SPL_ option, but for which there is none;
1745 for MOD_PROPER, look at source code which implies a Proper
1746 option (i.e. use of CONFIG_IS_ENABLED() or $(SPL_) or
1747 $(SPL_TPL_) but for which there none;
1748 if MODE_NORMAL, ignore SPL
1749
1750 Returns:
1751 dict:
1752 key (str): CONFIG name (without 'CONFIG_' prefix
1753 value (list of ConfigUse): List of uses of this CONFIG
1754 """
1755 # Make sure we know about all the options
1756 not_found = collections.defaultdict(list)
1757 for use, rest in all_uses.items():
1758 name = use.cfg
1759 if name in IGNORE_SYMS:
1760 continue
1761 check = True
1762
1763 if spl_mode == MODE_SPL:
1764 check = use.is_spl
1765
1766 # If it is an SPL symbol, try prepending all SPL_ prefixes to
1767 # find at least one SPL symbol
1768 if use.is_spl:
1769 add_to_dict = False
1770 for prefix in SPL_PREFIXES:
1771 try_name = prefix + name
1772 sym = kconf.syms.get(try_name)
1773 if sym:
1774 break
1775 if not sym:
1776 not_found[f'SPL_{name}'].append(use)
1777 continue
1778 elif spl_mode == MODE_PROPER:
1779 # Try to find the Proper version of this symbol, i.e. without
1780 # the SPL_ prefix
1781 proper_name = is_not_proper(name)
1782 if proper_name:
1783 name = proper_name
1784 elif not use.is_spl:
1785 check = False
1786 else: # MODE_NORMAL
1787 debug = False
1788 sym = kconf.syms.get(name)
1789 if not sym:
1790 proper_name = is_not_proper(name)
1791 if proper_name:
1792 name = proper_name
1793 sym = kconf.syms.get(name)
1794 if not sym:
1795 for prefix in SPL_PREFIXES:
1796 try_name = prefix + name
1797 sym = kconf.syms.get(try_name)
1798 if sym:
1799 break
1800 if not sym:
1801 not_found[name].append(use)
1802 continue
1803
1804 sym = kconf.syms.get(name)
1805 if not sym and check:
1806 not_found[name].append(use)
1807 return not_found
1808
1809 def show_uses(uses):
1810 """Show a list of uses along with their filename and code snippet
1811
1812 Args:
1813 uses (dict):
1814 key (str): CONFIG name (without 'CONFIG_' prefix
1815 value (list of ConfigUse): List of uses of this CONFIG
1816 """
1817 for name in sorted(uses):
1818 print(f'{name}: ', end='')
1819 for i, use in enumerate(uses[name]):
1820 print(f'{" " if i else ""}{use.fname}: {use.rest.strip()}')
1821
1822
1823 print('Scanning Kconfig')
1824 kconf = KconfigScanner().conf
1825 print(f'Scanning source in {path}')
1826 args = ['git', 'grep', '-E', r'IS_ENABLED|\bCONFIG']
1827 with subprocess.Popen(args, stdout=subprocess.PIPE) as proc:
1828 out, err = proc.communicate()
1829 lines = out.splitlines()
1830 re_fname = re.compile('^([^:]*):(.*)')
1831 src_list = []
1832 mk_list = []
1833 for line in lines:
1834 linestr = line.decode('utf-8')
1835 m_fname = re_fname.search(linestr)
1836 if not m_fname:
1837 continue
1838 fname, rest = m_fname.groups()
1839 dirname, leaf = os.path.split(fname)
1840 root, ext = os.path.splitext(leaf)
1841 if ext == '.autoconf':
1842 pass
1843 elif ext in ['.c', '.h', '.S', '.lds', '.dts', '.dtsi', '.asl', '.cfg',
1844 '.env', '.tmpl']:
1845 src_list.append([fname, rest])
1846 elif 'Makefile' in root or ext == '.mk':
1847 mk_list.append([fname, rest])
1848 elif ext in ['.yml', '.sh', '.py', '.awk', '.pl', '.rst', '', '.sed']:
1849 pass
1850 elif 'Kconfig' in root or 'Kbuild' in root:
1851 pass
1852 elif 'README' in root:
1853 pass
1854 elif dirname in ['configs']:
1855 pass
1856 elif dirname.startswith('doc') or dirname.startswith('scripts/kconfig'):
1857 pass
1858 else:
1859 print(f'Not sure how to handle file {fname}')
1860
1861 # Scan the Makefiles
1862 all_uses, fname_uses = scan_makefiles(mk_list)
1863
1864 spl_not_found = set()
1865 proper_not_found = set()
1866
1867 # Make sure we know about all the options
1868 print('\nCONFIG options present in Makefiles but not Kconfig:')
1869 not_found = check_not_found(all_uses, MODE_NORMAL)
1870 show_uses(not_found)
1871
1872 print('\nCONFIG options present in Makefiles but not Kconfig (SPL):')
1873 not_found = check_not_found(all_uses, MODE_SPL)
1874 show_uses(not_found)
1875 spl_not_found |= set([is_not_proper(key) or key for key in not_found.keys()])
1876
1877 print('\nCONFIG options used as Proper in Makefiles but without a non-SPL_ variant:')
1878 not_found = check_not_found(all_uses, MODE_PROPER)
1879 show_uses(not_found)
1880 proper_not_found |= set([key for key in not_found.keys()])
1881
1882 # Scan the source code
1883 all_uses, fname_uses = scan_src_files(src_list)
1884
1885 # Make sure we know about all the options
1886 print('\nCONFIG options present in source but not Kconfig:')
1887 not_found = check_not_found(all_uses, MODE_NORMAL)
1888 show_uses(not_found)
1889
1890 print('\nCONFIG options present in source but not Kconfig (SPL):')
1891 not_found = check_not_found(all_uses, MODE_SPL)
1892 show_uses(not_found)
1893 spl_not_found |= set([is_not_proper(key) or key for key in not_found.keys()])
1894
1895 print('\nCONFIG options used as Proper in source but without a non-SPL_ variant:')
1896 not_found = check_not_found(all_uses, MODE_PROPER)
1897 show_uses(not_found)
1898 proper_not_found |= set([key for key in not_found.keys()])
1899
1900 print('\nCONFIG options used as SPL but without an SPL_ variant:')
1901 for item in sorted(spl_not_found):
1902 print(f' {item}')
1903
1904 print('\nCONFIG options used as Proper but without a non-SPL_ variant:')
1905 for item in sorted(proper_not_found):
1906 print(f' {item}')
1907
1908 # Write out the updated information
1909 if do_update:
1910 with open(os.path.join(path, 'scripts', 'conf_nospl'), 'w') as out:
1911 print('# These options should not be enabled in SPL builds\n',
1912 file=out)
1913 for item in sorted(spl_not_found):
1914 print(item, file=out)
1915 with open(os.path.join(path, 'scripts', 'conf_noproper'), 'w') as out:
1916 print('# These options should not be enabled in Proper builds\n',
1917 file=out)
1918 for item in sorted(proper_not_found):
1919 print(item, file=out)
1920
1921
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001922def main():
1923 try:
1924 cpu_count = multiprocessing.cpu_count()
1925 except NotImplementedError:
1926 cpu_count = 1
1927
Simon Glassb2e83c62021-12-18 14:54:31 -07001928 epilog = '''Move config options from headers to defconfig files. See
1929doc/develop/moveconfig.rst for documentation.'''
1930
1931 parser = ArgumentParser(epilog=epilog)
1932 # Add arguments here
1933 parser.add_argument('-a', '--add-imply', type=str, default='',
Simon Glasscb008832017-06-15 21:39:33 -06001934 help='comma-separated list of CONFIG options to add '
1935 "an 'imply' statement to for the CONFIG in -i")
Simon Glassb2e83c62021-12-18 14:54:31 -07001936 parser.add_argument('-A', '--skip-added', action='store_true', default=False,
Simon Glasscb008832017-06-15 21:39:33 -0600