blob: 1dff891593123d58b03046a2990503564983f4b0 [file] [log] [blame]
Simon Glass793dca32019-10-31 07:42:57 -06001#!/usr/bin/env python3
Tom Rini83d290c2018-05-06 17:58:06 -04002# SPDX-License-Identifier: GPL-2.0+
Masahiro Yamada5a27c732015-05-20 11:36:07 +09003#
4# Author: Masahiro Yamada <yamada.masahiro@socionext.com>
5#
Masahiro Yamada5a27c732015-05-20 11:36:07 +09006
7"""
8Move config options from headers to defconfig files.
9
Simon Glass5c72c0e2021-07-21 21:35:51 -060010See doc/develop/moveconfig.rst for documentation.
Masahiro Yamada5a27c732015-05-20 11:36:07 +090011"""
12
Simon Glassb2e83c62021-12-18 14:54:31 -070013from argparse import ArgumentParser
Simon Glass99b66602017-06-01 19:39:03 -060014import collections
Simon Glass91197aa2021-12-18 14:54:35 -070015from contextlib import ExitStack
Simon Glass84067a52021-12-18 08:09:45 -070016import doctest
Masahiro Yamadac8e1b102016-05-19 15:52:07 +090017import filecmp
Masahiro Yamada5a27c732015-05-20 11:36:07 +090018import fnmatch
Masahiro Yamada0dbc9b52016-10-19 14:39:54 +090019import glob
Masahiro Yamada5a27c732015-05-20 11:36:07 +090020import multiprocessing
Masahiro Yamada5a27c732015-05-20 11:36:07 +090021import os
Simon Glass793dca32019-10-31 07:42:57 -060022import queue
Masahiro Yamada5a27c732015-05-20 11:36:07 +090023import re
24import shutil
25import subprocess
26import sys
27import tempfile
Simon Glassd73fcb12017-06-01 19:39:02 -060028import threading
Masahiro Yamada5a27c732015-05-20 11:36:07 +090029import time
Simon Glass84067a52021-12-18 08:09:45 -070030import unittest
Masahiro Yamada5a27c732015-05-20 11:36:07 +090031
Simon Glassb5aa5a32023-09-23 13:43:52 -060032import asteval
Simon Glass0ede00f2020-04-17 18:09:02 -060033from buildman import bsettings
34from buildman import kconfiglib
35from buildman import toolchain
Simon Glasscb008832017-06-15 21:39:33 -060036
Masahiro Yamada5a27c732015-05-20 11:36:07 +090037SHOW_GNU_MAKE = 'scripts/show-gnu-make'
38SLEEP_TIME=0.03
39
Masahiro Yamada5a27c732015-05-20 11:36:07 +090040STATE_IDLE = 0
41STATE_DEFCONFIG = 1
42STATE_AUTOCONF = 2
Joe Hershberger96464ba2015-05-19 13:21:17 -050043STATE_SAVEDEFCONFIG = 3
Masahiro Yamada5a27c732015-05-20 11:36:07 +090044
Masahiro Yamada5a27c732015-05-20 11:36:07 +090045COLOR_BLACK = '0;30'
46COLOR_RED = '0;31'
47COLOR_GREEN = '0;32'
48COLOR_BROWN = '0;33'
49COLOR_BLUE = '0;34'
50COLOR_PURPLE = '0;35'
51COLOR_CYAN = '0;36'
52COLOR_LIGHT_GRAY = '0;37'
53COLOR_DARK_GRAY = '1;30'
54COLOR_LIGHT_RED = '1;31'
55COLOR_LIGHT_GREEN = '1;32'
56COLOR_YELLOW = '1;33'
57COLOR_LIGHT_BLUE = '1;34'
58COLOR_LIGHT_PURPLE = '1;35'
59COLOR_LIGHT_CYAN = '1;36'
60COLOR_WHITE = '1;37'
61
Simon Glassf3b8e642017-06-01 19:39:01 -060062AUTO_CONF_PATH = 'include/config/auto.conf'
Simon Glassd73fcb12017-06-01 19:39:02 -060063CONFIG_DATABASE = 'moveconfig.db'
Simon Glassf3b8e642017-06-01 19:39:01 -060064
Simon Glasscb008832017-06-15 21:39:33 -060065CONFIG_LEN = len('CONFIG_')
Simon Glassf3b8e642017-06-01 19:39:01 -060066
Markus Klotzbuecherb237d352019-05-15 15:15:52 +020067SIZES = {
Simon Glassdaa694d2021-12-18 14:54:30 -070068 'SZ_1': 0x00000001, 'SZ_2': 0x00000002,
69 'SZ_4': 0x00000004, 'SZ_8': 0x00000008,
70 'SZ_16': 0x00000010, 'SZ_32': 0x00000020,
71 'SZ_64': 0x00000040, 'SZ_128': 0x00000080,
72 'SZ_256': 0x00000100, 'SZ_512': 0x00000200,
73 'SZ_1K': 0x00000400, 'SZ_2K': 0x00000800,
74 'SZ_4K': 0x00001000, 'SZ_8K': 0x00002000,
75 'SZ_16K': 0x00004000, 'SZ_32K': 0x00008000,
76 'SZ_64K': 0x00010000, 'SZ_128K': 0x00020000,
77 'SZ_256K': 0x00040000, 'SZ_512K': 0x00080000,
78 'SZ_1M': 0x00100000, 'SZ_2M': 0x00200000,
79 'SZ_4M': 0x00400000, 'SZ_8M': 0x00800000,
80 'SZ_16M': 0x01000000, 'SZ_32M': 0x02000000,
81 'SZ_64M': 0x04000000, 'SZ_128M': 0x08000000,
82 'SZ_256M': 0x10000000, 'SZ_512M': 0x20000000,
83 'SZ_1G': 0x40000000, 'SZ_2G': 0x80000000,
84 'SZ_4G': 0x100000000
Markus Klotzbuecherb237d352019-05-15 15:15:52 +020085}
86
Simon Glassb8d11da2022-02-08 11:49:45 -070087RE_REMOVE_DEFCONFIG = re.compile(r'(.*)_defconfig')
88
Simon Glass65e62032023-02-01 13:19:12 -070089# CONFIG symbols present in the build system (from Linux) but not actually used
90# in U-Boot; KCONFIG symbols
91IGNORE_SYMS = ['DEBUG_SECTION_MISMATCH', 'FTRACE_MCOUNT_RECORD', 'GCOV_KERNEL',
92 'GCOV_PROFILE_ALL', 'KALLSYMS', 'KASAN', 'MODVERSIONS', 'SHELL',
93 'TPL_BUILD', 'VPL_BUILD', 'IS_ENABLED', 'FOO', 'IF_ENABLED_INT',
94 'IS_ENABLED_', 'IS_ENABLED_1', 'IS_ENABLED_2', 'IS_ENABLED_3',
95 'SPL_', 'TPL_', 'SPL_FOO', 'TPL_FOO', 'TOOLS_FOO',
96 'ACME', 'SPL_ACME', 'TPL_ACME', 'TRACE_BRANCH_PROFILING',
97 'VAL', '_UNDEFINED', 'SPL_BUILD', ]
98
99SPL_PREFIXES = ['SPL_', 'TPL_', 'VPL_', 'TOOLS_']
100
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900101### helper functions ###
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900102def check_top_directory():
103 """Exit if we are not at the top of source directory."""
Simon Glass91197aa2021-12-18 14:54:35 -0700104 for fname in 'README', 'Licenses':
105 if not os.path.exists(fname):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900106 sys.exit('Please run at the top of source directory.')
107
Masahiro Yamadabd63e5b2016-05-19 15:51:54 +0900108def check_clean_directory():
109 """Exit if the source tree is not clean."""
Simon Glass91197aa2021-12-18 14:54:35 -0700110 for fname in '.config', 'include/config':
111 if os.path.exists(fname):
Masahiro Yamadabd63e5b2016-05-19 15:51:54 +0900112 sys.exit("source tree is not clean, please run 'make mrproper'")
113
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900114def get_make_cmd():
115 """Get the command name of GNU Make.
116
117 U-Boot needs GNU Make for building, but the command name is not
118 necessarily "make". (for example, "gmake" on FreeBSD).
119 Returns the most appropriate command name on your system.
120 """
Simon Glass91197aa2021-12-18 14:54:35 -0700121 with subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) as proc:
122 ret = proc.communicate()
123 if proc.returncode:
124 sys.exit('GNU Make not found')
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900125 return ret[0].rstrip()
126
Simon Glass25f978c2017-06-01 19:38:58 -0600127def get_matched_defconfig(line):
128 """Get the defconfig files that match a pattern
129
130 Args:
Simon Glass91197aa2021-12-18 14:54:35 -0700131 line (str): Path or filename to match, e.g. 'configs/snow_defconfig' or
Simon Glass25f978c2017-06-01 19:38:58 -0600132 'k2*_defconfig'. If no directory is provided, 'configs/' is
133 prepended
134
135 Returns:
Simon Glass91197aa2021-12-18 14:54:35 -0700136 list of str: a list of matching defconfig files
Simon Glass25f978c2017-06-01 19:38:58 -0600137 """
138 dirname = os.path.dirname(line)
139 if dirname:
140 pattern = line
141 else:
142 pattern = os.path.join('configs', line)
143 return glob.glob(pattern) + glob.glob(pattern + '_defconfig')
144
Masahiro Yamada0dbc9b52016-10-19 14:39:54 +0900145def get_matched_defconfigs(defconfigs_file):
Simon Glassee4e61b2017-06-01 19:38:59 -0600146 """Get all the defconfig files that match the patterns in a file.
147
148 Args:
Simon Glass91197aa2021-12-18 14:54:35 -0700149 defconfigs_file (str): File containing a list of defconfigs to process,
150 or '-' to read the list from stdin
Simon Glassee4e61b2017-06-01 19:38:59 -0600151
152 Returns:
Simon Glass91197aa2021-12-18 14:54:35 -0700153 list of str: A list of paths to defconfig files, with no duplicates
Simon Glassee4e61b2017-06-01 19:38:59 -0600154 """
Masahiro Yamada0dbc9b52016-10-19 14:39:54 +0900155 defconfigs = []
Simon Glass91197aa2021-12-18 14:54:35 -0700156 with ExitStack() as stack:
157 if defconfigs_file == '-':
158 inf = sys.stdin
159 defconfigs_file = 'stdin'
160 else:
161 inf = stack.enter_context(open(defconfigs_file, encoding='utf-8'))
162 for i, line in enumerate(inf):
163 line = line.strip()
164 if not line:
165 continue # skip blank lines silently
166 if ' ' in line:
167 line = line.split(' ')[0] # handle 'git log' input
168 matched = get_matched_defconfig(line)
169 if not matched:
170 print(f"warning: {defconfigs_file}:{i + 1}: no defconfig matched '{line}'",
171 file=sys.stderr)
Masahiro Yamada0dbc9b52016-10-19 14:39:54 +0900172
Simon Glass91197aa2021-12-18 14:54:35 -0700173 defconfigs += matched
Masahiro Yamada0dbc9b52016-10-19 14:39:54 +0900174
175 # use set() to drop multiple matching
Simon Glass91197aa2021-12-18 14:54:35 -0700176 return [defconfig[len('configs') + 1:] for defconfig in set(defconfigs)]
Masahiro Yamada0dbc9b52016-10-19 14:39:54 +0900177
Masahiro Yamada684c3062016-07-25 19:15:28 +0900178def get_all_defconfigs():
Simon Glass91197aa2021-12-18 14:54:35 -0700179 """Get all the defconfig files under the configs/ directory.
180
181 Returns:
182 list of str: List of paths to defconfig files
183 """
Masahiro Yamada684c3062016-07-25 19:15:28 +0900184 defconfigs = []
Simon Glass91197aa2021-12-18 14:54:35 -0700185 for (dirpath, _, filenames) in os.walk('configs'):
Masahiro Yamada684c3062016-07-25 19:15:28 +0900186 dirpath = dirpath[len('configs') + 1:]
187 for filename in fnmatch.filter(filenames, '*_defconfig'):
188 defconfigs.append(os.path.join(dirpath, filename))
189
190 return defconfigs
191
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900192def color_text(color_enabled, color, string):
193 """Return colored string."""
194 if color_enabled:
Masahiro Yamada1d085562016-05-19 15:52:02 +0900195 # LF should not be surrounded by the escape sequence.
196 # Otherwise, additional whitespace or line-feed might be printed.
197 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else ''
198 for s in string.split('\n') ])
Simon Glass91197aa2021-12-18 14:54:35 -0700199 return string
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900200
Simon Glass2fd85bd2021-12-18 14:54:33 -0700201def write_file(fname, data):
202 """Write data to a file
203
204 Args:
205 fname (str): Filename to write to
206 data (list of str): Lines to write (with or without trailing newline);
207 or str to write
208 """
209 with open(fname, 'w', encoding='utf-8') as out:
210 if isinstance(data, list):
211 for line in data:
212 print(line.rstrip('\n'), file=out)
213 else:
214 out.write(data)
215
Simon Glass37f815c2021-12-18 14:54:34 -0700216def read_file(fname, as_lines=True, skip_unicode=False):
217 """Read a file and return the contents
218
219 Args:
220 fname (str): Filename to read from
Simon Glass549d4222023-09-23 13:43:58 -0600221 as_lines (bool): Return file contents as a list of lines
Simon Glass37f815c2021-12-18 14:54:34 -0700222 skip_unicode (bool): True to report unicode errors and continue
223
224 Returns:
225 iter of str: List of ;ines from the file with newline removed; str if
226 as_lines is False with newlines intact; or None if a unicode error
227 occurred
228
229 Raises:
230 UnicodeDecodeError: Unicode error occurred when reading
231 """
232 with open(fname, encoding='utf-8') as inf:
233 try:
234 if as_lines:
235 return [line.rstrip('\n') for line in inf.readlines()]
236 else:
237 return inf.read()
Simon Glassa4c9d172023-09-23 13:44:01 -0600238 except UnicodeDecodeError as exc:
Simon Glass37f815c2021-12-18 14:54:34 -0700239 if not skip_unicode:
Simon Glass68a0b712022-02-11 13:23:22 -0700240 raise
Simon Glassa4c9d172023-09-23 13:44:01 -0600241 print(f"Failed on file '{fname}: {exc}")
Simon Glass37f815c2021-12-18 14:54:34 -0700242 return None
243
Markus Klotzbuecherb237d352019-05-15 15:15:52 +0200244def try_expand(line):
245 """If value looks like an expression, try expanding it
246 Otherwise just return the existing value
247 """
248 if line.find('=') == -1:
249 return line
250
251 try:
Markus Klotzbuecherb3192f42020-02-12 20:46:44 +0100252 aeval = asteval.Interpreter( usersyms=SIZES, minimal=True )
Markus Klotzbuecherb237d352019-05-15 15:15:52 +0200253 cfg, val = re.split("=", line)
254 val= val.strip('\"')
Simon Glassdaa694d2021-12-18 14:54:30 -0700255 if re.search(r'[*+-/]|<<|SZ_+|\(([^\)]+)\)', val):
Markus Klotzbuecherb3192f42020-02-12 20:46:44 +0100256 newval = hex(aeval(val))
Simon Glass1bd43062023-09-23 13:43:59 -0600257 print(f'\tExpanded expression {val} to {newval}')
Markus Klotzbuecherb237d352019-05-15 15:15:52 +0200258 return cfg+'='+newval
259 except:
Simon Glass1bd43062023-09-23 13:43:59 -0600260 print(f'\tFailed to expand expression in {line}')
Markus Klotzbuecherb237d352019-05-15 15:15:52 +0200261
262 return line
263
Chris Packhamca438342017-05-02 21:30:47 +1200264
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900265### classes ###
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900266class Progress:
267
268 """Progress Indicator"""
269
270 def __init__(self, total):
271 """Create a new progress indicator.
272
Simon Glass91197aa2021-12-18 14:54:35 -0700273 Args:
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900274 total: A number of defconfig files to process.
275 """
276 self.current = 0
277 self.total = total
278
279 def inc(self):
280 """Increment the number of processed defconfig files."""
281
282 self.current += 1
283
284 def show(self):
285 """Display the progress."""
Simon Glass1bd43062023-09-23 13:43:59 -0600286 print(f' {self.current} defconfigs out of {self.total}\r', end=' ')
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900287 sys.stdout.flush()
288
Simon Glasscb008832017-06-15 21:39:33 -0600289
290class KconfigScanner:
291 """Kconfig scanner."""
292
293 def __init__(self):
294 """Scan all the Kconfig files and create a Config object."""
295 # Define environment variables referenced from Kconfig
296 os.environ['srctree'] = os.getcwd()
297 os.environ['UBOOTVERSION'] = 'dummy'
298 os.environ['KCONFIG_OBJDIR'] = ''
Simon Glass65e62032023-02-01 13:19:12 -0700299 os.environ['CC'] = 'gcc'
Tom Rini65e05dd2019-09-20 17:42:09 -0400300 self.conf = kconfiglib.Kconfig()
Simon Glasscb008832017-06-15 21:39:33 -0600301
302
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900303class KconfigParser:
304
305 """A parser of .config and include/autoconf.mk."""
306
307 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
308 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
309
Simon Glass882c8e42023-09-23 13:43:54 -0600310 def __init__(self, args, build_dir):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900311 """Create a new parser.
312
Simon Glass91197aa2021-12-18 14:54:35 -0700313 Args:
Simon Glass91197aa2021-12-18 14:54:35 -0700314 args (Namespace): program arguments
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900315 build_dir: Build directory.
316 """
Simon Glassb2e83c62021-12-18 14:54:31 -0700317 self.args = args
Masahiro Yamada1f169922016-05-19 15:52:00 +0900318 self.dotconfig = os.path.join(build_dir, '.config')
319 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk')
Masahiro Yamada07913d12016-08-22 22:18:22 +0900320 self.spl_autoconf = os.path.join(build_dir, 'spl', 'include',
321 'autoconf.mk')
Simon Glassf3b8e642017-06-01 19:39:01 -0600322 self.config_autoconf = os.path.join(build_dir, AUTO_CONF_PATH)
Masahiro Yamada5da4f852016-05-19 15:52:06 +0900323 self.defconfig = os.path.join(build_dir, 'defconfig')
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900324
Simon Glass6821a742017-07-10 14:47:47 -0600325 def get_arch(self):
326 """Parse .config file and return the architecture.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900327
328 Returns:
Simon Glass6821a742017-07-10 14:47:47 -0600329 Architecture name (e.g. 'arm').
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900330 """
331 arch = ''
332 cpu = ''
Simon Glass37f815c2021-12-18 14:54:34 -0700333 for line in read_file(self.dotconfig):
Simon Glassa4c9d172023-09-23 13:44:01 -0600334 m_arch = self.re_arch.match(line)
335 if m_arch:
336 arch = m_arch.group(1)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900337 continue
Simon Glassa4c9d172023-09-23 13:44:01 -0600338 m_cpu = self.re_cpu.match(line)
339 if m_cpu:
340 cpu = m_cpu.group(1)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900341
Masahiro Yamada90ed6cb2016-05-19 15:51:53 +0900342 if not arch:
343 return None
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900344
345 # fix-up for aarch64
346 if arch == 'arm' and cpu == 'armv8':
347 arch = 'aarch64'
348
Simon Glass6821a742017-07-10 14:47:47 -0600349 return arch
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900350
Simon Glassd73fcb12017-06-01 19:39:02 -0600351
352class DatabaseThread(threading.Thread):
353 """This thread processes results from Slot threads.
354
355 It collects the data in the master config directary. There is only one
356 result thread, and this helps to serialise the build output.
357 """
358 def __init__(self, config_db, db_queue):
359 """Set up a new result thread
360
361 Args:
362 builder: Builder which will be sent each result
363 """
364 threading.Thread.__init__(self)
365 self.config_db = config_db
366 self.db_queue= db_queue
367
368 def run(self):
369 """Called to start up the result thread.
370
371 We collect the next result job and pass it on to the build.
372 """
373 while True:
374 defconfig, configs = self.db_queue.get()
375 self.config_db[defconfig] = configs
376 self.db_queue.task_done()
377
378
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900379class Slot:
380
381 """A slot to store a subprocess.
382
383 Each instance of this class handles one subprocess.
384 This class is useful to control multiple threads
385 for faster processing.
386 """
387
Simon Glass882c8e42023-09-23 13:43:54 -0600388 def __init__(self, toolchains, args, progress, devnull,
Simon Glass6821a742017-07-10 14:47:47 -0600389 make_cmd, reference_src_dir, db_queue):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900390 """Create a new process slot.
391
Simon Glass91197aa2021-12-18 14:54:35 -0700392 Args:
Simon Glass6821a742017-07-10 14:47:47 -0600393 toolchains: Toolchains object containing toolchains.
Simon Glassb2e83c62021-12-18 14:54:31 -0700394 args: Program arguments
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900395 progress: A progress indicator.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900396 devnull: A file object of '/dev/null'.
397 make_cmd: command name of GNU Make.
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500398 reference_src_dir: Determine the true starting config state from this
399 source tree.
Simon Glassd73fcb12017-06-01 19:39:02 -0600400 db_queue: output queue to write config info for the database
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900401 """
Simon Glass6821a742017-07-10 14:47:47 -0600402 self.toolchains = toolchains
Simon Glassb2e83c62021-12-18 14:54:31 -0700403 self.args = args
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900404 self.progress = progress
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900405 self.build_dir = tempfile.mkdtemp()
406 self.devnull = devnull
407 self.make_cmd = (make_cmd, 'O=' + self.build_dir)
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500408 self.reference_src_dir = reference_src_dir
Simon Glassd73fcb12017-06-01 19:39:02 -0600409 self.db_queue = db_queue
Simon Glass882c8e42023-09-23 13:43:54 -0600410 self.parser = KconfigParser(args, self.build_dir)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900411 self.state = STATE_IDLE
Masahiro Yamada09c6c062016-08-22 22:18:20 +0900412 self.failed_boards = set()
Simon Glassa6ab4db2023-09-23 13:44:02 -0600413 self.defconfig = None
414 self.log = ''
415 self.current_src_dir = None
416 self.proc = None
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900417
418 def __del__(self):
419 """Delete the working directory
420
421 This function makes sure the temporary directory is cleaned away
422 even if Python suddenly dies due to error. It should be done in here
Joe Hershbergerf2dae752016-06-10 14:53:29 -0500423 because it is guaranteed the destructor is always invoked when the
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900424 instance of the class gets unreferenced.
425
426 If the subprocess is still running, wait until it finishes.
427 """
428 if self.state != STATE_IDLE:
Simon Glassa4c9d172023-09-23 13:44:01 -0600429 while self.proc.poll() == None:
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900430 pass
431 shutil.rmtree(self.build_dir)
432
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900433 def add(self, defconfig):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900434 """Assign a new subprocess for defconfig and add it to the slot.
435
436 If the slot is vacant, create a new subprocess for processing the
437 given defconfig and add it to the slot. Just returns False if
438 the slot is occupied (i.e. the current subprocess is still running).
439
Simon Glass91197aa2021-12-18 14:54:35 -0700440 Args:
Simon Glass549d4222023-09-23 13:43:58 -0600441 defconfig (str): defconfig name.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900442
443 Returns:
444 Return True on success or False on failure
445 """
446 if self.state != STATE_IDLE:
447 return False
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900448
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900449 self.defconfig = defconfig
Masahiro Yamada1d085562016-05-19 15:52:02 +0900450 self.log = ''
Masahiro Yamadaf432c332016-06-15 14:33:52 +0900451 self.current_src_dir = self.reference_src_dir
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900452 self.do_defconfig()
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900453 return True
454
455 def poll(self):
456 """Check the status of the subprocess and handle it as needed.
457
458 Returns True if the slot is vacant (i.e. in idle state).
459 If the configuration is successfully finished, assign a new
460 subprocess to build include/autoconf.mk.
461 If include/autoconf.mk is generated, invoke the parser to
Masahiro Yamada7fb0bac2016-05-19 15:52:04 +0900462 parse the .config and the include/autoconf.mk, moving
463 config options to the .config as needed.
464 If the .config was updated, run "make savedefconfig" to sync
465 it, update the original defconfig, and then set the slot back
466 to the idle state.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900467
468 Returns:
469 Return True if the subprocess is terminated, False otherwise
470 """
471 if self.state == STATE_IDLE:
472 return True
473
Simon Glassa4c9d172023-09-23 13:44:01 -0600474 if self.proc.poll() == None:
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900475 return False
476
Simon Glassa4c9d172023-09-23 13:44:01 -0600477 if self.proc.poll() != 0:
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900478 self.handle_error()
479 elif self.state == STATE_DEFCONFIG:
Masahiro Yamadaf432c332016-06-15 14:33:52 +0900480 if self.reference_src_dir and not self.current_src_dir:
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500481 self.do_savedefconfig()
482 else:
483 self.do_autoconf()
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900484 elif self.state == STATE_AUTOCONF:
Masahiro Yamadaf432c332016-06-15 14:33:52 +0900485 if self.current_src_dir:
486 self.current_src_dir = None
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500487 self.do_defconfig()
Simon Glassb2e83c62021-12-18 14:54:31 -0700488 elif self.args.build_db:
Simon Glassd73fcb12017-06-01 19:39:02 -0600489 self.do_build_db()
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500490 else:
491 self.do_savedefconfig()
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900492 elif self.state == STATE_SAVEDEFCONFIG:
493 self.update_defconfig()
494 else:
Simon Glassdaa694d2021-12-18 14:54:30 -0700495 sys.exit('Internal Error. This should not happen.')
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900496
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900497 return True if self.state == STATE_IDLE else False
Joe Hershberger96464ba2015-05-19 13:21:17 -0500498
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900499 def handle_error(self):
500 """Handle error cases."""
Masahiro Yamada8513dc02016-05-19 15:52:08 +0900501
Simon Glassb2e83c62021-12-18 14:54:31 -0700502 self.log += color_text(self.args.color, COLOR_LIGHT_RED,
Simon Glassdaa694d2021-12-18 14:54:30 -0700503 'Failed to process.\n')
Simon Glassb2e83c62021-12-18 14:54:31 -0700504 if self.args.verbose:
505 self.log += color_text(self.args.color, COLOR_LIGHT_CYAN,
Simon Glassa4c9d172023-09-23 13:44:01 -0600506 self.proc.stderr.read().decode())
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900507 self.finish(False)
Joe Hershberger96464ba2015-05-19 13:21:17 -0500508
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900509 def do_defconfig(self):
510 """Run 'make <board>_defconfig' to create the .config file."""
Masahiro Yamadac8e1b102016-05-19 15:52:07 +0900511
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900512 cmd = list(self.make_cmd)
513 cmd.append(self.defconfig)
Simon Glassa4c9d172023-09-23 13:44:01 -0600514 self.proc = subprocess.Popen(cmd, stdout=self.devnull,
515 stderr=subprocess.PIPE,
516 cwd=self.current_src_dir)
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900517 self.state = STATE_DEFCONFIG
Masahiro Yamadac8e1b102016-05-19 15:52:07 +0900518
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900519 def do_autoconf(self):
Simon Glassf3b8e642017-06-01 19:39:01 -0600520 """Run 'make AUTO_CONF_PATH'."""
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900521
Simon Glass6821a742017-07-10 14:47:47 -0600522 arch = self.parser.get_arch()
523 try:
524 toolchain = self.toolchains.Select(arch)
525 except ValueError:
Simon Glassb2e83c62021-12-18 14:54:31 -0700526 self.log += color_text(self.args.color, COLOR_YELLOW,
Simon Glass1bd43062023-09-23 13:43:59 -0600527 f"Tool chain for '{arch}' is missing. Do nothing.\n")
Masahiro Yamada4efef992016-05-19 15:52:03 +0900528 self.finish(False)
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900529 return
Simon Glass793dca32019-10-31 07:42:57 -0600530 env = toolchain.MakeEnvironment(False)
Masahiro Yamada90ed6cb2016-05-19 15:51:53 +0900531
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900532 cmd = list(self.make_cmd)
Joe Hershberger7740f652015-05-19 13:21:18 -0500533 cmd.append('KCONFIG_IGNORE_DUPLICATES=1')
Simon Glassf3b8e642017-06-01 19:39:01 -0600534 cmd.append(AUTO_CONF_PATH)
Simon Glassa4c9d172023-09-23 13:44:01 -0600535 self.proc = subprocess.Popen(cmd, stdout=self.devnull, env=env,
536 stderr=subprocess.PIPE,
537 cwd=self.current_src_dir)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900538 self.state = STATE_AUTOCONF
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900539
Simon Glassd73fcb12017-06-01 19:39:02 -0600540 def do_build_db(self):
541 """Add the board to the database"""
542 configs = {}
Simon Glass37f815c2021-12-18 14:54:34 -0700543 for line in read_file(os.path.join(self.build_dir, AUTO_CONF_PATH)):
544 if line.startswith('CONFIG'):
545 config, value = line.split('=', 1)
546 configs[config] = value.rstrip()
Simon Glassd73fcb12017-06-01 19:39:02 -0600547 self.db_queue.put([self.defconfig, configs])
548 self.finish(True)
549
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900550 def do_savedefconfig(self):
551 """Update the .config and run 'make savedefconfig'."""
Simon Glassc7345612023-09-23 13:43:55 -0600552 if not self.args.force_sync:
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900553 self.finish(True)
554 return
Simon Glassc7345612023-09-23 13:43:55 -0600555 self.log += 'Syncing by savedefconfig (forced by option)...\n'
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900556
557 cmd = list(self.make_cmd)
558 cmd.append('savedefconfig')
Simon Glassa4c9d172023-09-23 13:44:01 -0600559 self.proc = subprocess.Popen(cmd, stdout=self.devnull,
560 stderr=subprocess.PIPE)
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900561 self.state = STATE_SAVEDEFCONFIG
562
563 def update_defconfig(self):
564 """Update the input defconfig and go back to the idle state."""
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900565 orig_defconfig = os.path.join('configs', self.defconfig)
566 new_defconfig = os.path.join(self.build_dir, 'defconfig')
567 updated = not filecmp.cmp(orig_defconfig, new_defconfig)
568
569 if updated:
Simon Glassb2e83c62021-12-18 14:54:31 -0700570 self.log += color_text(self.args.color, COLOR_LIGHT_BLUE,
Simon Glassdaa694d2021-12-18 14:54:30 -0700571 'defconfig was updated.\n')
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900572
Simon Glassb2e83c62021-12-18 14:54:31 -0700573 if not self.args.dry_run and updated:
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900574 shutil.move(new_defconfig, orig_defconfig)
575 self.finish(True)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900576
Masahiro Yamada4efef992016-05-19 15:52:03 +0900577 def finish(self, success):
578 """Display log along with progress and go to the idle state.
Masahiro Yamada1d085562016-05-19 15:52:02 +0900579
Simon Glass91197aa2021-12-18 14:54:35 -0700580 Args:
Simon Glass549d4222023-09-23 13:43:58 -0600581 success (bool): Should be True when the defconfig was processed
Masahiro Yamada4efef992016-05-19 15:52:03 +0900582 successfully, or False when it fails.
Masahiro Yamada1d085562016-05-19 15:52:02 +0900583 """
584 # output at least 30 characters to hide the "* defconfigs out of *".
585 log = self.defconfig.ljust(30) + '\n'
586
587 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ])
588 # Some threads are running in parallel.
589 # Print log atomically to not mix up logs from different threads.
Simon Glass793dca32019-10-31 07:42:57 -0600590 print(log, file=(sys.stdout if success else sys.stderr))
Masahiro Yamada4efef992016-05-19 15:52:03 +0900591
592 if not success:
Simon Glassb2e83c62021-12-18 14:54:31 -0700593 if self.args.exit_on_error:
Simon Glassdaa694d2021-12-18 14:54:30 -0700594 sys.exit('Exit on error.')
Masahiro Yamada4efef992016-05-19 15:52:03 +0900595 # If --exit-on-error flag is not set, skip this board and continue.
596 # Record the failed board.
Masahiro Yamada09c6c062016-08-22 22:18:20 +0900597 self.failed_boards.add(self.defconfig)
Masahiro Yamada4efef992016-05-19 15:52:03 +0900598
Masahiro Yamada1d085562016-05-19 15:52:02 +0900599 self.progress.inc()
600 self.progress.show()
Masahiro Yamada4efef992016-05-19 15:52:03 +0900601 self.state = STATE_IDLE
Masahiro Yamada1d085562016-05-19 15:52:02 +0900602
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900603 def get_failed_boards(self):
Masahiro Yamada09c6c062016-08-22 22:18:20 +0900604 """Returns a set of failed boards (defconfigs) in this slot.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900605 """
606 return self.failed_boards
607
608class Slots:
609
610 """Controller of the array of subprocess slots."""
611
Simon Glass882c8e42023-09-23 13:43:54 -0600612 def __init__(self, toolchains, args, progress, reference_src_dir, db_queue):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900613 """Create a new slots controller.
614
Simon Glass91197aa2021-12-18 14:54:35 -0700615 Args:
Simon Glass6821a742017-07-10 14:47:47 -0600616 toolchains: Toolchains object containing toolchains.
Simon Glassb2e83c62021-12-18 14:54:31 -0700617 args: Program arguments
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900618 progress: A progress indicator.
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500619 reference_src_dir: Determine the true starting config state from this
620 source tree.
Simon Glassd73fcb12017-06-01 19:39:02 -0600621 db_queue: output queue to write config info for the database
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900622 """
Simon Glassb2e83c62021-12-18 14:54:31 -0700623 self.args = args
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900624 self.slots = []
Simon Glass478920d2021-12-18 14:54:32 -0700625 devnull = subprocess.DEVNULL
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900626 make_cmd = get_make_cmd()
Simon Glass62fae4b2023-09-23 13:44:00 -0600627 for _ in range(args.jobs):
Simon Glass882c8e42023-09-23 13:43:54 -0600628 self.slots.append(Slot(toolchains, args, progress, devnull,
629 make_cmd, reference_src_dir, db_queue))
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900630
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900631 def add(self, defconfig):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900632 """Add a new subprocess if a vacant slot is found.
633
Simon Glass91197aa2021-12-18 14:54:35 -0700634 Args:
Simon Glass549d4222023-09-23 13:43:58 -0600635 defconfig (str): defconfig name to be put into.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900636
637 Returns:
638 Return True on success or False on failure
639 """
640 for slot in self.slots:
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900641 if slot.add(defconfig):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900642 return True
643 return False
644
645 def available(self):
646 """Check if there is a vacant slot.
647
648 Returns:
649 Return True if at lease one vacant slot is found, False otherwise.
650 """
651 for slot in self.slots:
652 if slot.poll():
653 return True
654 return False
655
656 def empty(self):
657 """Check if all slots are vacant.
658
659 Returns:
660 Return True if all the slots are vacant, False otherwise.
661 """
662 ret = True
663 for slot in self.slots:
664 if not slot.poll():
665 ret = False
666 return ret
667
668 def show_failed_boards(self):
669 """Display all of the failed boards (defconfigs)."""
Masahiro Yamada09c6c062016-08-22 22:18:20 +0900670 boards = set()
Masahiro Yamada96dccd92016-06-15 14:33:53 +0900671 output_file = 'moveconfig.failed'
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900672
673 for slot in self.slots:
Masahiro Yamada09c6c062016-08-22 22:18:20 +0900674 boards |= slot.get_failed_boards()
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900675
Masahiro Yamada96dccd92016-06-15 14:33:53 +0900676 if boards:
677 boards = '\n'.join(boards) + '\n'
Simon Glassdaa694d2021-12-18 14:54:30 -0700678 msg = 'The following boards were not processed due to error:\n'
Masahiro Yamada96dccd92016-06-15 14:33:53 +0900679 msg += boards
Simon Glass1bd43062023-09-23 13:43:59 -0600680 msg += f'(the list has been saved in {output_file})\n'
Simon Glassb2e83c62021-12-18 14:54:31 -0700681 print(color_text(self.args.color, COLOR_LIGHT_RED,
Simon Glass793dca32019-10-31 07:42:57 -0600682 msg), file=sys.stderr)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900683
Simon Glass2fd85bd2021-12-18 14:54:33 -0700684 write_file(output_file, boards)
Joe Hershberger2559cd82015-05-19 13:21:22 -0500685
Masahiro Yamada5cc42a52016-06-15 14:33:51 +0900686class ReferenceSource:
687
688 """Reference source against which original configs should be parsed."""
689
690 def __init__(self, commit):
691 """Create a reference source directory based on a specified commit.
692
Simon Glass91197aa2021-12-18 14:54:35 -0700693 Args:
Masahiro Yamada5cc42a52016-06-15 14:33:51 +0900694 commit: commit to git-clone
695 """
696 self.src_dir = tempfile.mkdtemp()
Simon Glassdaa694d2021-12-18 14:54:30 -0700697 print('Cloning git repo to a separate work directory...')
Masahiro Yamada5cc42a52016-06-15 14:33:51 +0900698 subprocess.check_output(['git', 'clone', os.getcwd(), '.'],
699 cwd=self.src_dir)
Simon Glass1bd43062023-09-23 13:43:59 -0600700 rev = subprocess.check_output(['git', 'rev-parse', '--short',
701 commit]).strip()
702 print(f"Checkout '{rev}' to build the original autoconf.mk.")
Masahiro Yamada5cc42a52016-06-15 14:33:51 +0900703 subprocess.check_output(['git', 'checkout', commit],
704 stderr=subprocess.STDOUT, cwd=self.src_dir)
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500705
706 def __del__(self):
Masahiro Yamada5cc42a52016-06-15 14:33:51 +0900707 """Delete the reference source directory
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500708
709 This function makes sure the temporary directory is cleaned away
710 even if Python suddenly dies due to error. It should be done in here
711 because it is guaranteed the destructor is always invoked when the
712 instance of the class gets unreferenced.
713 """
Masahiro Yamada5cc42a52016-06-15 14:33:51 +0900714 shutil.rmtree(self.src_dir)
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500715
Masahiro Yamada5cc42a52016-06-15 14:33:51 +0900716 def get_dir(self):
717 """Return the absolute path to the reference source directory."""
718
719 return self.src_dir
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500720
Simon Glass882c8e42023-09-23 13:43:54 -0600721def move_config(toolchains, args, db_queue):
722 """Build database or sync config options to defconfig files.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900723
Simon Glass91197aa2021-12-18 14:54:35 -0700724 Args:
Simon Glass549d4222023-09-23 13:43:58 -0600725 toolchains (Toolchains): Toolchains to use
726 args (Namespace): Program arguments
727 db_queue (Queue): Queue for database updates
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900728 """
Simon Glass882c8e42023-09-23 13:43:54 -0600729 if args.force_sync:
730 print('Syncing defconfigs', end=' ')
731 elif args.build_db:
Simon Glass1bd43062023-09-23 13:43:59 -0600732 print(f'Building {CONFIG_DATABASE} database')
733 print(f'(jobs: {args.jobs})\n')
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900734
Simon Glassb2e83c62021-12-18 14:54:31 -0700735 if args.git_ref:
736 reference_src = ReferenceSource(args.git_ref)
Masahiro Yamada5cc42a52016-06-15 14:33:51 +0900737 reference_src_dir = reference_src.get_dir()
738 else:
Masahiro Yamadaf432c332016-06-15 14:33:52 +0900739 reference_src_dir = None
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500740
Simon Glassb2e83c62021-12-18 14:54:31 -0700741 if args.defconfigs:
742 defconfigs = get_matched_defconfigs(args.defconfigs)
Joe Hershberger91040e82015-05-19 13:21:19 -0500743 else:
Masahiro Yamada684c3062016-07-25 19:15:28 +0900744 defconfigs = get_all_defconfigs()
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900745
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900746 progress = Progress(len(defconfigs))
Simon Glass882c8e42023-09-23 13:43:54 -0600747 slots = Slots(toolchains, args, progress, reference_src_dir, db_queue)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900748
749 # Main loop to process defconfig files:
750 # Add a new subprocess into a vacant slot.
751 # Sleep if there is no available slot.
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900752 for defconfig in defconfigs:
753 while not slots.add(defconfig):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900754 while not slots.available():
755 # No available slot: sleep for a while
756 time.sleep(SLEEP_TIME)
757
758 # wait until all the subprocesses finish
759 while not slots.empty():
760 time.sleep(SLEEP_TIME)
761
Simon Glass793dca32019-10-31 07:42:57 -0600762 print('')
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900763 slots.show_failed_boards()
764
Simon Glasscb008832017-06-15 21:39:33 -0600765def find_kconfig_rules(kconf, config, imply_config):
766 """Check whether a config has a 'select' or 'imply' keyword
767
768 Args:
Simon Glass549d4222023-09-23 13:43:58 -0600769 kconf (Kconfiglib.Kconfig): Kconfig object
770 config (str): Name of config to check (without CONFIG_ prefix)
771 imply_config (str): Implying config (without CONFIG_ prefix) which may
772 or may not have an 'imply' for 'config')
Simon Glasscb008832017-06-15 21:39:33 -0600773
774 Returns:
775 Symbol object for 'config' if found, else None
776 """
Tom Rini65e05dd2019-09-20 17:42:09 -0400777 sym = kconf.syms.get(imply_config)
Simon Glasscb008832017-06-15 21:39:33 -0600778 if sym:
Simon Glass62fae4b2023-09-23 13:44:00 -0600779 for sel, _ in (sym.selects + sym.implies):
Simon Glassa3627082021-12-18 08:09:42 -0700780 if sel.name == config:
Simon Glasscb008832017-06-15 21:39:33 -0600781 return sym
782 return None
783
784def check_imply_rule(kconf, config, imply_config):
785 """Check if we can add an 'imply' option
786
787 This finds imply_config in the Kconfig and looks to see if it is possible
788 to add an 'imply' for 'config' to that part of the Kconfig.
789
790 Args:
Simon Glass549d4222023-09-23 13:43:58 -0600791 kconf (Kconfiglib.Kconfig): Kconfig object
792 config (str): Name of config to check (without CONFIG_ prefix)
793 imply_config (str): Implying config (without CONFIG_ prefix) which may
794 or may not have an 'imply' for 'config')
Simon Glasscb008832017-06-15 21:39:33 -0600795
796 Returns:
797 tuple:
Simon Glass549d4222023-09-23 13:43:58 -0600798 str: filename of Kconfig file containing imply_config, or None if
799 none
800 int: line number within the Kconfig file, or 0 if none
801 str: message indicating the result
Simon Glasscb008832017-06-15 21:39:33 -0600802 """
Tom Rini65e05dd2019-09-20 17:42:09 -0400803 sym = kconf.syms.get(imply_config)
Simon Glasscb008832017-06-15 21:39:33 -0600804 if not sym:
805 return 'cannot find sym'
Simon Glassea40b202021-07-21 21:35:53 -0600806 nodes = sym.nodes
807 if len(nodes) != 1:
Simon Glass1bd43062023-09-23 13:43:59 -0600808 return f'{len(nodes)} locations'
Simon Glassa3627082021-12-18 08:09:42 -0700809 node = nodes[0]
810 fname, linenum = node.filename, node.linenr
Simon Glasscb008832017-06-15 21:39:33 -0600811 cwd = os.getcwd()
812 if cwd and fname.startswith(cwd):
813 fname = fname[len(cwd) + 1:]
Simon Glass1bd43062023-09-23 13:43:59 -0600814 file_line = f' at {fname}:{linenum}'
Simon Glass37f815c2021-12-18 14:54:34 -0700815 data = read_file(fname)
Simon Glass1bd43062023-09-23 13:43:59 -0600816 if data[linenum - 1] != f'config {imply_config}':
817 return None, 0, f'bad sym format {data[linenum]}{file_line})'
818 return fname, linenum, f'adding{file_line}'
Simon Glasscb008832017-06-15 21:39:33 -0600819
820def add_imply_rule(config, fname, linenum):
821 """Add a new 'imply' option to a Kconfig
822
823 Args:
Simon Glass549d4222023-09-23 13:43:58 -0600824 config (str): config option to add an imply for (without CONFIG_ prefix)
825 fname (str): Kconfig filename to update
826 linenum (int): Line number to place the 'imply' before
Simon Glasscb008832017-06-15 21:39:33 -0600827
828 Returns:
829 Message indicating the result
830 """
Simon Glass1bd43062023-09-23 13:43:59 -0600831 file_line = f' at {fname}:{linenum}'
Simon Glass37f815c2021-12-18 14:54:34 -0700832 data = read_file(fname)
Simon Glasscb008832017-06-15 21:39:33 -0600833 linenum -= 1
834
835 for offset, line in enumerate(data[linenum:]):
836 if line.strip().startswith('help') or not line:
Simon Glass1bd43062023-09-23 13:43:59 -0600837 data.insert(linenum + offset, f'\timply {config}')
Simon Glass2fd85bd2021-12-18 14:54:33 -0700838 write_file(fname, data)
Simon Glass1bd43062023-09-23 13:43:59 -0600839 return f'added{file_line}'
Simon Glasscb008832017-06-15 21:39:33 -0600840
841 return 'could not insert%s'
842
843(IMPLY_MIN_2, IMPLY_TARGET, IMPLY_CMD, IMPLY_NON_ARCH_BOARD) = (
844 1, 2, 4, 8)
Simon Glass9b2a2e82017-06-15 21:39:32 -0600845
846IMPLY_FLAGS = {
847 'min2': [IMPLY_MIN_2, 'Show options which imply >2 boards (normally >5)'],
848 'target': [IMPLY_TARGET, 'Allow CONFIG_TARGET_... options to imply'],
849 'cmd': [IMPLY_CMD, 'Allow CONFIG_CMD_... to imply'],
Simon Glasscb008832017-06-15 21:39:33 -0600850 'non-arch-board': [
851 IMPLY_NON_ARCH_BOARD,
852 'Allow Kconfig options outside arch/ and /board/ to imply'],
Simon Glass91197aa2021-12-18 14:54:35 -0700853}
Simon Glass9b2a2e82017-06-15 21:39:32 -0600854
Simon Glass9d603392021-12-18 08:09:43 -0700855
856def read_database():
857 """Read in the config database
858
859 Returns:
860 tuple:
861 set of all config options seen (each a str)
862 set of all defconfigs seen (each a str)
863 dict of configs for each defconfig:
864 key: defconfig name, e.g. "MPC8548CDS_legacy_defconfig"
865 value: dict:
866 key: CONFIG option
867 value: Value of option
868 dict of defconfigs for each config:
869 key: CONFIG option
870 value: set of boards using that option
871
872 """
873 configs = {}
874
875 # key is defconfig name, value is dict of (CONFIG_xxx, value)
876 config_db = {}
877
878 # Set of all config options we have seen
879 all_configs = set()
880
881 # Set of all defconfigs we have seen
882 all_defconfigs = set()
883
884 defconfig_db = collections.defaultdict(set)
Simon Glass37f815c2021-12-18 14:54:34 -0700885 for line in read_file(CONFIG_DATABASE):
886 line = line.rstrip()
887 if not line: # Separator between defconfigs
888 config_db[defconfig] = configs
889 all_defconfigs.add(defconfig)
890 configs = {}
891 elif line[0] == ' ': # CONFIG line
892 config, value = line.strip().split('=', 1)
893 configs[config] = value
894 defconfig_db[config].add(defconfig)
895 all_configs.add(config)
896 else: # New defconfig
897 defconfig = line
Simon Glass9d603392021-12-18 08:09:43 -0700898
899 return all_configs, all_defconfigs, config_db, defconfig_db
900
901
Simon Glasscb008832017-06-15 21:39:33 -0600902def do_imply_config(config_list, add_imply, imply_flags, skip_added,
903 check_kconfig=True, find_superset=False):
Simon Glass99b66602017-06-01 19:39:03 -0600904 """Find CONFIG options which imply those in the list
905
906 Some CONFIG options can be implied by others and this can help to reduce
907 the size of the defconfig files. For example, CONFIG_X86 implies
908 CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and
909 all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to
910 each of the x86 defconfig files.
911
912 This function uses the moveconfig database to find such options. It
913 displays a list of things that could possibly imply those in the list.
914 The algorithm ignores any that start with CONFIG_TARGET since these
915 typically refer to only a few defconfigs (often one). It also does not
916 display a config with less than 5 defconfigs.
917
918 The algorithm works using sets. For each target config in config_list:
919 - Get the set 'defconfigs' which use that target config
920 - For each config (from a list of all configs):
921 - Get the set 'imply_defconfig' of defconfigs which use that config
922 -
923 - If imply_defconfigs contains anything not in defconfigs then
924 this config does not imply the target config
925
926 Params:
927 config_list: List of CONFIG options to check (each a string)
Simon Glasscb008832017-06-15 21:39:33 -0600928 add_imply: Automatically add an 'imply' for each config.
Simon Glass9b2a2e82017-06-15 21:39:32 -0600929 imply_flags: Flags which control which implying configs are allowed
930 (IMPLY_...)
Simon Glasscb008832017-06-15 21:39:33 -0600931 skip_added: Don't show options which already have an imply added.
932 check_kconfig: Check if implied symbols already have an 'imply' or
933 'select' for the target config, and show this information if so.
Simon Glass99b66602017-06-01 19:39:03 -0600934 find_superset: True to look for configs which are a superset of those
935 already found. So for example if CONFIG_EXYNOS5 implies an option,
936 but CONFIG_EXYNOS covers a larger set of defconfigs and also
937 implies that option, this will drop the former in favour of the
938 latter. In practice this option has not proved very used.
939
940 Note the terminoloy:
941 config - a CONFIG_XXX options (a string, e.g. 'CONFIG_CMD_EEPROM')
942 defconfig - a defconfig file (a string, e.g. 'configs/snow_defconfig')
943 """
Simon Glasscb008832017-06-15 21:39:33 -0600944 kconf = KconfigScanner().conf if check_kconfig else None
945 if add_imply and add_imply != 'all':
Simon Glassa3627082021-12-18 08:09:42 -0700946 add_imply = add_imply.split(',')
Simon Glasscb008832017-06-15 21:39:33 -0600947
Simon Glass62fae4b2023-09-23 13:44:00 -0600948 all_configs, all_defconfigs, _, defconfig_db = read_database()
Simon Glass99b66602017-06-01 19:39:03 -0600949
Simon Glassa3627082021-12-18 08:09:42 -0700950 # Work through each target config option in turn, independently
Simon Glass99b66602017-06-01 19:39:03 -0600951 for config in config_list:
952 defconfigs = defconfig_db.get(config)
953 if not defconfigs:
Simon Glass1bd43062023-09-23 13:43:59 -0600954 print(f'{config} not found in any defconfig')
Simon Glass99b66602017-06-01 19:39:03 -0600955 continue
956
957 # Get the set of defconfigs without this one (since a config cannot
958 # imply itself)
959 non_defconfigs = all_defconfigs - defconfigs
960 num_defconfigs = len(defconfigs)
Simon Glass1bd43062023-09-23 13:43:59 -0600961 print(f'{config} found in {num_defconfigs}/{len(all_configs)} defconfigs')
Simon Glass99b66602017-06-01 19:39:03 -0600962
963 # This will hold the results: key=config, value=defconfigs containing it
964 imply_configs = {}
965 rest_configs = all_configs - set([config])
966
967 # Look at every possible config, except the target one
968 for imply_config in rest_configs:
Simon Glass9b2a2e82017-06-15 21:39:32 -0600969 if 'ERRATUM' in imply_config:
Simon Glass99b66602017-06-01 19:39:03 -0600970 continue
Simon Glass91197aa2021-12-18 14:54:35 -0700971 if not imply_flags & IMPLY_CMD:
Simon Glass9b2a2e82017-06-15 21:39:32 -0600972 if 'CONFIG_CMD' in imply_config:
973 continue
Simon Glass91197aa2021-12-18 14:54:35 -0700974 if not imply_flags & IMPLY_TARGET:
Simon Glass9b2a2e82017-06-15 21:39:32 -0600975 if 'CONFIG_TARGET' in imply_config:
976 continue
Simon Glass99b66602017-06-01 19:39:03 -0600977
978 # Find set of defconfigs that have this config
979 imply_defconfig = defconfig_db[imply_config]
980
981 # Get the intersection of this with defconfigs containing the
982 # target config
983 common_defconfigs = imply_defconfig & defconfigs
984
985 # Get the set of defconfigs containing this config which DO NOT
986 # also contain the taret config. If this set is non-empty it means
987 # that this config affects other defconfigs as well as (possibly)
988 # the ones affected by the target config. This means it implies
989 # things we don't want to imply.
990 not_common_defconfigs = imply_defconfig & non_defconfigs
991 if not_common_defconfigs:
992 continue
993
994 # If there are common defconfigs, imply_config may be useful
995 if common_defconfigs:
996 skip = False
997 if find_superset:
Simon Glass793dca32019-10-31 07:42:57 -0600998 for prev in list(imply_configs.keys()):
Simon Glass99b66602017-06-01 19:39:03 -0600999 prev_count = len(imply_configs[prev])
1000 count = len(common_defconfigs)
1001 if (prev_count > count and
1002 (imply_configs[prev] & common_defconfigs ==
1003 common_defconfigs)):
1004 # skip imply_config because prev is a superset
1005 skip = True
1006 break
1007 elif count > prev_count:
1008 # delete prev because imply_config is a superset
1009 del imply_configs[prev]
1010 if not skip:
1011 imply_configs[imply_config] = common_defconfigs
1012
1013 # Now we have a dict imply_configs of configs which imply each config
1014 # The value of each dict item is the set of defconfigs containing that
1015 # config. Rank them so that we print the configs that imply the largest
1016 # number of defconfigs first.
Simon Glasscb008832017-06-15 21:39:33 -06001017 ranked_iconfigs = sorted(imply_configs,
Simon Glass99b66602017-06-01 19:39:03 -06001018 key=lambda k: len(imply_configs[k]), reverse=True)
Simon Glasscb008832017-06-15 21:39:33 -06001019 kconfig_info = ''
1020 cwd = os.getcwd()
1021 add_list = collections.defaultdict(list)
1022 for iconfig in ranked_iconfigs:
1023 num_common = len(imply_configs[iconfig])
Simon Glass99b66602017-06-01 19:39:03 -06001024
1025 # Don't bother if there are less than 5 defconfigs affected.
Simon Glass9b2a2e82017-06-15 21:39:32 -06001026 if num_common < (2 if imply_flags & IMPLY_MIN_2 else 5):
Simon Glass99b66602017-06-01 19:39:03 -06001027 continue
Simon Glasscb008832017-06-15 21:39:33 -06001028 missing = defconfigs - imply_configs[iconfig]
Simon Glass99b66602017-06-01 19:39:03 -06001029 missing_str = ', '.join(missing) if missing else 'all'
1030 missing_str = ''
Simon Glasscb008832017-06-15 21:39:33 -06001031 show = True
1032 if kconf:
1033 sym = find_kconfig_rules(kconf, config[CONFIG_LEN:],
1034 iconfig[CONFIG_LEN:])
1035 kconfig_info = ''
1036 if sym:
Simon Glassea40b202021-07-21 21:35:53 -06001037 nodes = sym.nodes
1038 if len(nodes) == 1:
1039 fname, linenum = nodes[0].filename, nodes[0].linenr
Simon Glasscb008832017-06-15 21:39:33 -06001040 if cwd and fname.startswith(cwd):
1041 fname = fname[len(cwd) + 1:]
Simon Glass1bd43062023-09-23 13:43:59 -06001042 kconfig_info = f'{fname}:{linenum}'
Simon Glasscb008832017-06-15 21:39:33 -06001043 if skip_added:
1044 show = False
1045 else:
Tom Rini65e05dd2019-09-20 17:42:09 -04001046 sym = kconf.syms.get(iconfig[CONFIG_LEN:])
Simon Glasscb008832017-06-15 21:39:33 -06001047 fname = ''
1048 if sym:
Simon Glassea40b202021-07-21 21:35:53 -06001049 nodes = sym.nodes
1050 if len(nodes) == 1:
1051 fname, linenum = nodes[0].filename, nodes[0].linenr
Simon Glasscb008832017-06-15 21:39:33 -06001052 if cwd and fname.startswith(cwd):
1053 fname = fname[len(cwd) + 1:]
1054 in_arch_board = not sym or (fname.startswith('arch') or
1055 fname.startswith('board'))
1056 if (not in_arch_board and
Simon Glass91197aa2021-12-18 14:54:35 -07001057 not imply_flags & IMPLY_NON_ARCH_BOARD):
Simon Glasscb008832017-06-15 21:39:33 -06001058 continue
1059
1060 if add_imply and (add_imply == 'all' or
1061 iconfig in add_imply):
1062 fname, linenum, kconfig_info = (check_imply_rule(kconf,
1063 config[CONFIG_LEN:], iconfig[CONFIG_LEN:]))
1064 if fname:
1065 add_list[fname].append(linenum)
1066
1067 if show and kconfig_info != 'skip':
Simon Glass1bd43062023-09-23 13:43:59 -06001068 print(f'{num_common:5d} : '
1069 f'{iconfig.ljust(30):-30s}{kconfig_info:-25s} {missing_str}')
Simon Glasscb008832017-06-15 21:39:33 -06001070
1071 # Having collected a list of things to add, now we add them. We process
1072 # each file from the largest line number to the smallest so that
1073 # earlier additions do not affect our line numbers. E.g. if we added an
1074 # imply at line 20 it would change the position of each line after
1075 # that.
Simon Glass793dca32019-10-31 07:42:57 -06001076 for fname, linenums in add_list.items():
Simon Glasscb008832017-06-15 21:39:33 -06001077 for linenum in sorted(linenums, reverse=True):
1078 add_imply_rule(config[CONFIG_LEN:], fname, linenum)
Simon Glass99b66602017-06-01 19:39:03 -06001079
Simon Glass941671a2022-02-08 11:49:46 -07001080def defconfig_matches(configs, re_match):
1081 """Check if any CONFIG option matches a regex
1082
1083 The match must be complete, i.e. from the start to end of the CONFIG option.
1084
1085 Args:
1086 configs (dict): Dict of CONFIG options:
1087 key: CONFIG option
1088 value: Value of option
1089 re_match (re.Pattern): Match to check
1090
1091 Returns:
1092 bool: True if any CONFIG matches the regex
1093 """
1094 for cfg in configs:
Simon Glassd9c958f2022-03-05 20:18:54 -07001095 if re_match.fullmatch(cfg):
Simon Glass941671a2022-02-08 11:49:46 -07001096 return True
1097 return False
Simon Glass99b66602017-06-01 19:39:03 -06001098
Simon Glass65d7fce2021-12-18 08:09:46 -07001099def do_find_config(config_list):
1100 """Find boards with a given combination of CONFIGs
1101
1102 Params:
Simon Glass941671a2022-02-08 11:49:46 -07001103 config_list: List of CONFIG options to check (each a regex consisting
Simon Glass65d7fce2021-12-18 08:09:46 -07001104 of a config option, with or without a CONFIG_ prefix. If an option
1105 is preceded by a tilde (~) then it must be false, otherwise it must
1106 be true)
1107 """
Simon Glass62fae4b2023-09-23 13:44:00 -06001108 _, all_defconfigs, config_db, _ = read_database()
Simon Glass65d7fce2021-12-18 08:09:46 -07001109
Simon Glass65d7fce2021-12-18 08:09:46 -07001110 # Start with all defconfigs
1111 out = all_defconfigs
1112
1113 # Work through each config in turn
Simon Glass65d7fce2021-12-18 08:09:46 -07001114 for item in config_list:
1115 # Get the real config name and whether we want this config or not
1116 cfg = item
1117 want = True
1118 if cfg[0] == '~':
1119 want = False
1120 cfg = cfg[1:]
1121
Simon Glass65d7fce2021-12-18 08:09:46 -07001122 # Search everything that is still in the running. If it has a config
1123 # that we want, or doesn't have one that we don't, add it into the
1124 # running for the next stage
1125 in_list = out
1126 out = set()
Simon Glass941671a2022-02-08 11:49:46 -07001127 re_match = re.compile(cfg)
Simon Glass65d7fce2021-12-18 08:09:46 -07001128 for defc in in_list:
Simon Glass941671a2022-02-08 11:49:46 -07001129 has_cfg = defconfig_matches(config_db[defc], re_match)
Simon Glass65d7fce2021-12-18 08:09:46 -07001130 if has_cfg == want:
1131 out.add(defc)
Tom Rini9ef3ba82022-12-04 10:14:16 -05001132 print(f'{len(out)} matches')
1133 print(' '.join(item.split('_defconfig')[0] for item in out))
Simon Glass65d7fce2021-12-18 08:09:46 -07001134
1135
1136def prefix_config(cfg):
1137 """Prefix a config with CONFIG_ if needed
1138
1139 This handles ~ operator, which indicates that the CONFIG should be disabled
1140
1141 >>> prefix_config('FRED')
1142 'CONFIG_FRED'
1143 >>> prefix_config('CONFIG_FRED')
1144 'CONFIG_FRED'
1145 >>> prefix_config('~FRED')
1146 '~CONFIG_FRED'
1147 >>> prefix_config('~CONFIG_FRED')
1148 '~CONFIG_FRED'
1149 >>> prefix_config('A123')
1150 'CONFIG_A123'
1151 """
Simon Glassa4c9d172023-09-23 13:44:01 -06001152 oper = ''
Simon Glass65d7fce2021-12-18 08:09:46 -07001153 if cfg[0] == '~':
Simon Glassa4c9d172023-09-23 13:44:01 -06001154 oper = cfg[0]
Simon Glass65d7fce2021-12-18 08:09:46 -07001155 cfg = cfg[1:]
1156 if not cfg.startswith('CONFIG_'):
1157 cfg = 'CONFIG_' + cfg
Simon Glassa4c9d172023-09-23 13:44:01 -06001158 return oper + cfg
Simon Glass65d7fce2021-12-18 08:09:46 -07001159
1160
Simon Glass98275712023-09-23 13:43:57 -06001161RE_MK_CONFIGS = re.compile(r'CONFIG_(\$\(SPL_(?:TPL_)?\))?([A-Za-z0-9_]*)')
1162RE_IFDEF = re.compile(r'(ifdef|ifndef)')
1163RE_C_CONFIGS = re.compile(r'CONFIG_([A-Za-z0-9_]*)')
1164RE_CONFIG_IS = re.compile(r'CONFIG_IS_ENABLED\(([A-Za-z0-9_]*)\)')
Simon Glass65e62032023-02-01 13:19:12 -07001165
1166class ConfigUse:
1167 def __init__(self, cfg, is_spl, fname, rest):
1168 self.cfg = cfg
1169 self.is_spl = is_spl
1170 self.fname = fname
1171 self.rest = rest
1172
1173 def __hash__(self):
1174 return hash((self.cfg, self.is_spl))
1175
1176def scan_makefiles(fnames):
1177 """Scan Makefiles looking for Kconfig options
1178
1179 Looks for uses of CONFIG options in Makefiles
1180
1181 Args:
1182 fnames (list of tuple):
1183 str: Makefile filename where the option was found
1184 str: Line of the Makefile
1185
1186 Returns:
1187 tuple:
1188 dict: all_uses
1189 key (ConfigUse): object
1190 value (list of str): matching lines
1191 dict: Uses by filename
1192 key (str): filename
1193 value (set of ConfigUse): uses in that filename
1194
1195 >>> RE_MK_CONFIGS.search('CONFIG_FRED').groups()
1196 (None, 'FRED')
1197 >>> RE_MK_CONFIGS.search('CONFIG_$(SPL_)MARY').groups()
1198 ('$(SPL_)', 'MARY')
1199 >>> RE_MK_CONFIGS.search('CONFIG_$(SPL_TPL_)MARY').groups()
1200 ('$(SPL_TPL_)', 'MARY')
1201 """
1202 all_uses = collections.defaultdict(list)
1203 fname_uses = {}
1204 for fname, rest in fnames:
1205 m_iter = RE_MK_CONFIGS.finditer(rest)
Simon Glassa4c9d172023-09-23 13:44:01 -06001206 for mat in m_iter:
1207 real_opt = mat.group(2)
Simon Glass65e62032023-02-01 13:19:12 -07001208 if real_opt == '':
1209 continue
1210 is_spl = False
Simon Glassa4c9d172023-09-23 13:44:01 -06001211 if mat.group(1):
Simon Glass65e62032023-02-01 13:19:12 -07001212 is_spl = True
1213 use = ConfigUse(real_opt, is_spl, fname, rest)
1214 if fname not in fname_uses:
1215 fname_uses[fname] = set()
1216 fname_uses[fname].add(use)
1217 all_uses[use].append(rest)
1218 return all_uses, fname_uses
1219
1220
1221def scan_src_files(fnames):
1222 """Scan source files (other than Makefiles) looking for Kconfig options
1223
1224 Looks for uses of CONFIG options
1225
1226 Args:
1227 fnames (list of tuple):
1228 str: Makefile filename where the option was found
1229 str: Line of the Makefile
1230
1231 Returns:
1232 tuple:
1233 dict: all_uses
1234 key (ConfigUse): object
1235 value (list of str): matching lines
1236 dict: Uses by filename
1237 key (str): filename
1238 value (set of ConfigUse): uses in that filename
1239
1240 >>> RE_C_CONFIGS.search('CONFIG_FRED').groups()
1241 ('FRED',)
1242 >>> RE_CONFIG_IS.search('CONFIG_IS_ENABLED(MARY)').groups()
1243 ('MARY',)
1244 >>> RE_CONFIG_IS.search('#if CONFIG_IS_ENABLED(OF_PLATDATA)').groups()
1245 ('OF_PLATDATA',)
1246 """
Simon Glass62fae4b2023-09-23 13:44:00 -06001247 fname = None
1248 rest = None
1249
Simon Glass65e62032023-02-01 13:19:12 -07001250 def add_uses(m_iter, is_spl):
Simon Glassa4c9d172023-09-23 13:44:01 -06001251 for mat in m_iter:
1252 real_opt = mat.group(1)
Simon Glass65e62032023-02-01 13:19:12 -07001253 if real_opt == '':
1254 continue
1255 use = ConfigUse(real_opt, is_spl, fname, rest)
1256 if fname not in fname_uses:
1257 fname_uses[fname] = set()
1258 fname_uses[fname].add(use)
1259 all_uses[use].append(rest)
1260
1261 all_uses = collections.defaultdict(list)
1262 fname_uses = {}
1263 for fname, rest in fnames:
1264 m_iter = RE_C_CONFIGS.finditer(rest)
1265 add_uses(m_iter, False)
1266
1267 m_iter2 = RE_CONFIG_IS.finditer(rest)
1268 add_uses(m_iter2, True)
1269
1270 return all_uses, fname_uses
1271
1272
1273MODE_NORMAL, MODE_SPL, MODE_PROPER = range(3)
1274
1275def do_scan_source(path, do_update):
1276 """Scan the source tree for Kconfig inconsistencies
1277
1278 Args:
1279 path (str): Path to source tree
1280 do_update (bool) : True to write to scripts/kconf_... files
1281 """
1282 def is_not_proper(name):
1283 for prefix in SPL_PREFIXES:
1284 if name.startswith(prefix):
1285 return name[len(prefix):]
1286 return False
1287
1288 def check_not_found(all_uses, spl_mode):
1289 """Check for Kconfig options mentioned in the source but not in Kconfig
1290
1291 Args:
1292 all_uses (dict):
1293 key (ConfigUse): object
1294 value (list of str): matching lines
1295 spl_mode (int): If MODE_SPL, look at source code which implies
1296 an SPL_ option, but for which there is none;
1297 for MOD_PROPER, look at source code which implies a Proper
1298 option (i.e. use of CONFIG_IS_ENABLED() or $(SPL_) or
1299 $(SPL_TPL_) but for which there none;
1300 if MODE_NORMAL, ignore SPL
1301
1302 Returns:
1303 dict:
1304 key (str): CONFIG name (without 'CONFIG_' prefix
1305 value (list of ConfigUse): List of uses of this CONFIG
1306 """
1307 # Make sure we know about all the options
1308 not_found = collections.defaultdict(list)
Simon Glass62fae4b2023-09-23 13:44:00 -06001309 for use, _ in all_uses.items():
Simon Glass65e62032023-02-01 13:19:12 -07001310 name = use.cfg
1311 if name in IGNORE_SYMS:
1312 continue
1313 check = True
1314
1315 if spl_mode == MODE_SPL:
1316 check = use.is_spl
1317
1318 # If it is an SPL symbol, try prepending all SPL_ prefixes to
1319 # find at least one SPL symbol
1320 if use.is_spl:
Simon Glass65e62032023-02-01 13:19:12 -07001321 for prefix in SPL_PREFIXES:
1322 try_name = prefix + name
1323 sym = kconf.syms.get(try_name)
1324 if sym:
1325 break
1326 if not sym:
1327 not_found[f'SPL_{name}'].append(use)
1328 continue
1329 elif spl_mode == MODE_PROPER:
1330 # Try to find the Proper version of this symbol, i.e. without
1331 # the SPL_ prefix
1332 proper_name = is_not_proper(name)
1333 if proper_name:
1334 name = proper_name
1335 elif not use.is_spl:
1336 check = False
1337 else: # MODE_NORMAL
Simon Glass65e62032023-02-01 13:19:12 -07001338 sym = kconf.syms.get(name)
1339 if not sym:
1340 proper_name = is_not_proper(name)
1341 if proper_name:
1342 name = proper_name
1343 sym = kconf.syms.get(name)
1344 if not sym:
1345 for prefix in SPL_PREFIXES:
1346 try_name = prefix + name
1347 sym = kconf.syms.get(try_name)
1348 if sym:
1349 break
1350 if not sym:
1351 not_found[name].append(use)
1352 continue
1353
1354 sym = kconf.syms.get(name)
1355 if not sym and check:
1356 not_found[name].append(use)
1357 return not_found
1358
1359 def show_uses(uses):
1360 """Show a list of uses along with their filename and code snippet
1361
1362 Args:
1363 uses (dict):
1364 key (str): CONFIG name (without 'CONFIG_' prefix
1365 value (list of ConfigUse): List of uses of this CONFIG
1366 """
1367 for name in sorted(uses):
1368 print(f'{name}: ', end='')
1369 for i, use in enumerate(uses[name]):
1370 print(f'{" " if i else ""}{use.fname}: {use.rest.strip()}')
1371
1372
1373 print('Scanning Kconfig')
1374 kconf = KconfigScanner().conf
1375 print(f'Scanning source in {path}')
1376 args = ['git', 'grep', '-E', r'IS_ENABLED|\bCONFIG']
1377 with subprocess.Popen(args, stdout=subprocess.PIPE) as proc:
Simon Glass62fae4b2023-09-23 13:44:00 -06001378 out, _ = proc.communicate()
Simon Glass65e62032023-02-01 13:19:12 -07001379 lines = out.splitlines()
1380 re_fname = re.compile('^([^:]*):(.*)')
1381 src_list = []
1382 mk_list = []
1383 for line in lines:
1384 linestr = line.decode('utf-8')
1385 m_fname = re_fname.search(linestr)
1386 if not m_fname:
1387 continue
1388 fname, rest = m_fname.groups()
1389 dirname, leaf = os.path.split(fname)
1390 root, ext = os.path.splitext(leaf)
1391 if ext == '.autoconf':
1392 pass
1393 elif ext in ['.c', '.h', '.S', '.lds', '.dts', '.dtsi', '.asl', '.cfg',
1394 '.env', '.tmpl']:
1395 src_list.append([fname, rest])
1396 elif 'Makefile' in root or ext == '.mk':
1397 mk_list.append([fname, rest])
1398 elif ext in ['.yml', '.sh', '.py', '.awk', '.pl', '.rst', '', '.sed']:
1399 pass
1400 elif 'Kconfig' in root or 'Kbuild' in root:
1401 pass
1402 elif 'README' in root:
1403 pass
1404 elif dirname in ['configs']:
1405 pass
1406 elif dirname.startswith('doc') or dirname.startswith('scripts/kconfig'):
1407 pass
1408 else:
1409 print(f'Not sure how to handle file {fname}')
1410
1411 # Scan the Makefiles
Simon Glass62fae4b2023-09-23 13:44:00 -06001412 all_uses, _ = scan_makefiles(mk_list)
Simon Glass65e62032023-02-01 13:19:12 -07001413
1414 spl_not_found = set()
1415 proper_not_found = set()
1416
1417 # Make sure we know about all the options
1418 print('\nCONFIG options present in Makefiles but not Kconfig:')
1419 not_found = check_not_found(all_uses, MODE_NORMAL)
1420 show_uses(not_found)
1421
1422 print('\nCONFIG options present in Makefiles but not Kconfig (SPL):')
1423 not_found = check_not_found(all_uses, MODE_SPL)
1424 show_uses(not_found)
1425 spl_not_found |= set([is_not_proper(key) or key for key in not_found.keys()])
1426
1427 print('\nCONFIG options used as Proper in Makefiles but without a non-SPL_ variant:')
1428 not_found = check_not_found(all_uses, MODE_PROPER)
1429 show_uses(not_found)
1430 proper_not_found |= set([key for key in not_found.keys()])
1431
1432 # Scan the source code
Simon Glass62fae4b2023-09-23 13:44:00 -06001433 all_uses, _ = scan_src_files(src_list)
Simon Glass65e62032023-02-01 13:19:12 -07001434
1435 # Make sure we know about all the options
1436 print('\nCONFIG options present in source but not Kconfig:')
1437 not_found = check_not_found(all_uses, MODE_NORMAL)
1438 show_uses(not_found)
1439
1440 print('\nCONFIG options present in source but not Kconfig (SPL):')
1441 not_found = check_not_found(all_uses, MODE_SPL)
1442 show_uses(not_found)
1443 spl_not_found |= set([is_not_proper(key) or key for key in not_found.keys()])
1444
1445 print('\nCONFIG options used as Proper in source but without a non-SPL_ variant:')
1446 not_found = check_not_found(all_uses, MODE_PROPER)
1447 show_uses(not_found)
1448 proper_not_found |= set([key for key in not_found.keys()])
1449
1450 print('\nCONFIG options used as SPL but without an SPL_ variant:')
1451 for item in sorted(spl_not_found):
1452 print(f' {item}')
1453
1454 print('\nCONFIG options used as Proper but without a non-SPL_ variant:')
1455 for item in sorted(proper_not_found):
1456 print(f' {item}')
1457
1458 # Write out the updated information
1459 if do_update:
1460 with open(os.path.join(path, 'scripts', 'conf_nospl'), 'w') as out:
1461 print('# These options should not be enabled in SPL builds\n',
1462 file=out)
1463 for item in sorted(spl_not_found):
1464 print(item, file=out)
1465 with open(os.path.join(path, 'scripts', 'conf_noproper'), 'w') as out:
1466 print('# These options should not be enabled in Proper builds\n',
1467 file=out)
1468 for item in sorted(proper_not_found):
1469 print(item, file=out)
1470
1471
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001472def main():
1473 try:
1474 cpu_count = multiprocessing.cpu_count()
1475 except NotImplementedError:
1476 cpu_count = 1
1477
Simon Glassb2e83c62021-12-18 14:54:31 -07001478 epilog = '''Move config options from headers to defconfig files. See
1479doc/develop/moveconfig.rst for documentation.'''
1480
1481 parser = ArgumentParser(epilog=epilog)
1482 # Add arguments here
1483 parser.add_argument('-a', '--add-imply', type=str, default='',
Simon Glasscb008832017-06-15 21:39:33 -06001484 help='comma-separated list of CONFIG options to add '
1485 "an 'imply' statement to for the CONFIG in -i")
Simon Glassb2e83c62021-12-18 14:54:31 -07001486 parser.add_argument('-A', '--skip-added', action='store_true', default=False,
Simon Glasscb008832017-06-15 21:39:33 -06001487 help="don't show options which are already marked as "
1488 'implying others')
Simon Glassb2e83c62021-12-18 14:54:31 -07001489 parser.add_argument('-b', '--build-db', action='store_true', default=False,
Simon Glassd73fcb12017-06-01 19:39:02 -06001490 help='build a CONFIG database')
Simon Glassb2e83c62021-12-18 14:54:31 -07001491 parser.add_argument('-c', '--color', action='store_true', default=False,
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001492 help='display the log in color')
Simon Glassb2e83c62021-12-18 14:54:31 -07001493 parser.add_argument('-C', '--commit', action='store_true', default=False,
Simon Glass9ede2122016-09-12 23:18:21 -06001494 help='Create a git commit for the operation')
Simon Glassb2e83c62021-12-18 14:54:31 -07001495 parser.add_argument('-d', '--defconfigs', type=str,
Simon Glassee4e61b2017-06-01 19:38:59 -06001496 help='a file containing a list of defconfigs to move, '
1497 "one per line (for example 'snow_defconfig') "
1498 "or '-' to read from stdin")
Simon Glassb2e83c62021-12-18 14:54:31 -07001499 parser.add_argument('-e', '--exit-on-error', action='store_true',
Simon Glasse1ae5632021-12-18 08:09:44 -07001500 default=False,
1501 help='exit immediately on any error')
Simon Glassb2e83c62021-12-18 14:54:31 -07001502 parser.add_argument('-f', '--find', action='store_true', default=False,
Simon Glass65d7fce2021-12-18 08:09:46 -07001503 help='Find boards with a given config combination')
Simon Glassb2e83c62021-12-18 14:54:31 -07001504 parser.add_argument('-i', '--imply', action='store_true', default=False,
Simon Glass99b66602017-06-01 19:39:03 -06001505 help='find options which imply others')
Simon Glassb2e83c62021-12-18 14:54:31 -07001506 parser.add_argument('-I', '--imply-flags', type=str, default='',
Simon Glass9b2a2e82017-06-15 21:39:32 -06001507 help="control the -i option ('help' for help")
Simon Glassb2e83c62021-12-18 14:54:31 -07001508 parser.add_argument('-j', '--jobs', type=int, default=cpu_count,
Simon Glasse1ae5632021-12-18 08:09:44 -07001509 help='the number of jobs to run simultaneously')
Simon Glassb2e83c62021-12-18 14:54:31 -07001510 parser.add_argument('-n', '--dry-run', action='store_true', default=False,
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001511 help='perform a trial run (show log with no changes)')
Simon Glassb2e83c62021-12-18 14:54:31 -07001512 parser.add_argument('-r', '--git-ref', type=str,
Simon Glasse1ae5632021-12-18 08:09:44 -07001513 help='the git ref to clone for building the autoconf.mk')
Simon Glassb2e83c62021-12-18 14:54:31 -07001514 parser.add_argument('-s', '--force-sync', action='store_true', default=False,
Masahiro Yamada8513dc02016-05-19 15:52:08 +09001515 help='force sync by savedefconfig')
Simon Glassb2e83c62021-12-18 14:54:31 -07001516 parser.add_argument('-S', '--spl', action='store_true', default=False,
Masahiro Yamada07913d12016-08-22 22:18:22 +09001517 help='parse config options defined for SPL build')
Simon Glass65e62032023-02-01 13:19:12 -07001518 parser.add_argument('--scan-source', action='store_true', default=False,
1519 help='scan source for uses of CONFIG options')
Simon Glassb2e83c62021-12-18 14:54:31 -07001520 parser.add_argument('-t', '--test', action='store_true', default=False,
Simon Glasse1ae5632021-12-18 08:09:44 -07001521 help='run unit tests')
Simon Glassb2e83c62021-12-18 14:54:31 -07001522 parser.add_argument('-y', '--yes', action='store_true', default=False,
Simon Glass6b403df2016-09-12 23:18:20 -06001523 help="respond 'yes' to any prompts")
Simon Glass65e62032023-02-01 13:19:12 -07001524 parser.add_argument('-u', '--update', action='store_true', default=False,
1525 help="update scripts/ files (use with --scan-source)")
Simon Glassb2e83c62021-12-18 14:54:31 -07001526 parser.add_argument('-v', '--verbose', action='store_true', default=False,
Joe Hershberger95bf9c72015-05-19 13:21:24 -05001527 help='show any build errors as boards are built')
Simon Glassb2e83c62021-12-18 14:54:31 -07001528 parser.add_argument('configs', nargs='*')
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001529
Simon Glassb2e83c62021-12-18 14:54:31 -07001530 args = parser.parse_args()
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001531
Simon Glassb2e83c62021-12-18 14:54:31 -07001532 if args.test:
Simon Glass84067a52021-12-18 08:09:45 -07001533 sys.argv = [sys.argv[0]]
Simon Glass62fae4b2023-09-23 13:44:00 -06001534 fail, _ = doctest.testmod()
Simon Glass84067a52021-12-18 08:09:45 -07001535 if fail:
1536 return 1
1537 unittest.main()
1538
Simon Glass65e62032023-02-01 13:19:12 -07001539 if args.scan_source:
1540 do_scan_source(os.getcwd(), args.update)
1541 return
1542
Simon Glass882c8e42023-09-23 13:43:54 -06001543 if not any((args.force_sync, args.build_db, args.imply, args.find)):
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001544 parser.print_usage()
1545 sys.exit(1)
1546
Masahiro Yamadab6ef3932016-05-19 15:51:58 +09001547 # prefix the option name with CONFIG_ if missing
Simon Glass882c8e42023-09-23 13:43:54 -06001548 configs = [prefix_config(cfg) for cfg in args.configs]
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001549
Joe Hershberger2144f882015-05-19 13:21:20 -05001550 check_top_directory()
1551
Simon Glassb2e83c62021-12-18 14:54:31 -07001552 if args.imply:
Simon Glass9b2a2e82017-06-15 21:39:32 -06001553 imply_flags = 0
Simon Glassb2e83c62021-12-18 14:54:31 -07001554 if args.imply_flags == 'all':
Simon Glassdee36c72017-07-10 14:47:46 -06001555 imply_flags = -1
1556
Simon Glassb2e83c62021-12-18 14:54:31 -07001557 elif args.imply_flags:
1558 for flag in args.imply_flags.split(','):
Simon Glassdee36c72017-07-10 14:47:46 -06001559 bad = flag not in IMPLY_FLAGS
1560 if bad:
Simon Glass1bd43062023-09-23 13:43:59 -06001561 print(f"Invalid flag '{flag}'")
Simon Glassdee36c72017-07-10 14:47:46 -06001562 if flag == 'help' or bad:
Simon Glass793dca32019-10-31 07:42:57 -06001563 print("Imply flags: (separate with ',')")
1564 for name, info in IMPLY_FLAGS.items():
Simon Glass1bd43062023-09-23 13:43:59 -06001565 print(f' {name:-15s}: {info[1]}')
Simon Glassdee36c72017-07-10 14:47:46 -06001566 parser.print_usage()
1567 sys.exit(1)
1568 imply_flags |= IMPLY_FLAGS[flag][0]
Simon Glass9b2a2e82017-06-15 21:39:32 -06001569
Simon Glassb2e83c62021-12-18 14:54:31 -07001570 do_imply_config(configs, args.add_imply, imply_flags, args.skip_added)
Simon Glass99b66602017-06-01 19:39:03 -06001571 return
1572
Simon Glassb2e83c62021-12-18 14:54:31 -07001573 if args.find:
Simon Glass65d7fce2021-12-18 08:09:46 -07001574 do_find_config(configs)
1575 return
1576
Simon Glass3481e892023-09-23 13:43:53 -06001577 # We are either building the database or forcing a sync of defconfigs
Simon Glassd73fcb12017-06-01 19:39:02 -06001578 config_db = {}
Simon Glass793dca32019-10-31 07:42:57 -06001579 db_queue = queue.Queue()
Simon Glassa4c9d172023-09-23 13:44:01 -06001580 dbt = DatabaseThread(config_db, db_queue)
1581 dbt.daemon = True
1582 dbt.start()
Simon Glassd73fcb12017-06-01 19:39:02 -06001583
Simon Glass63df2022023-09-23 13:43:50 -06001584 check_clean_directory()
1585 bsettings.setup('')
1586 toolchains = toolchain.Toolchains()
1587 toolchains.GetSettings()
1588 toolchains.Scan(verbose=False)
Simon Glass882c8e42023-09-23 13:43:54 -06001589 move_config(toolchains, args, db_queue)
Simon Glass63df2022023-09-23 13:43:50 -06001590 db_queue.join()
Joe Hershberger2144f882015-05-19 13:21:20 -05001591
Simon Glassb2e83c62021-12-18 14:54:31 -07001592 if args.commit:
Simon Glass9ede2122016-09-12 23:18:21 -06001593 subprocess.call(['git', 'add', '-u'])
1594 if configs:
1595 msg = 'Convert %s %sto Kconfig' % (configs[0],
1596 'et al ' if len(configs) > 1 else '')
1597 msg += ('\n\nThis converts the following to Kconfig:\n %s\n' %
1598 '\n '.join(configs))
1599 else:
1600 msg = 'configs: Resync with savedefconfig'
1601 msg += '\n\nRsync all defconfig files using moveconfig.py'
1602 subprocess.call(['git', 'commit', '-s', '-m', msg])
1603
Simon Glassb2e83c62021-12-18 14:54:31 -07001604 if args.build_db:
Simon Glassa4c9d172023-09-23 13:44:01 -06001605 with open(CONFIG_DATABASE, 'w', encoding='utf-8') as outf:
Simon Glass793dca32019-10-31 07:42:57 -06001606 for defconfig, configs in config_db.items():
Simon Glassa4c9d172023-09-23 13:44:01 -06001607 outf.write(f'{defconfig}\n')
Simon Glassd73fcb12017-06-01 19:39:02 -06001608 for config in sorted(configs.keys()):
Simon Glassa4c9d172023-09-23 13:44:01 -06001609 outf.write(f' {config}={configs[config]}\n')
1610 outf.write('\n')
Simon Glassd73fcb12017-06-01 19:39:02 -06001611
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001612if __name__ == '__main__':
Simon Glass65d7fce2021-12-18 08:09:46 -07001613 sys.exit(main())