blob: 568386f7b9d0a92866437eafffd38c3b01da7238 [file] [log] [blame]
Simon Glass793dca32019-10-31 07:42:57 -06001#!/usr/bin/env python3
Tom Rini83d290c2018-05-06 17:58:06 -04002# SPDX-License-Identifier: GPL-2.0+
Masahiro Yamada5a27c732015-05-20 11:36:07 +09003#
4# Author: Masahiro Yamada <yamada.masahiro@socionext.com>
5#
Masahiro Yamada5a27c732015-05-20 11:36:07 +09006
7"""
8Move config options from headers to defconfig files.
9
Simon Glass5c72c0e2021-07-21 21:35:51 -060010See doc/develop/moveconfig.rst for documentation.
Masahiro Yamada5a27c732015-05-20 11:36:07 +090011"""
12
Simon Glassb2e83c62021-12-18 14:54:31 -070013from argparse import ArgumentParser
Simon Glass99b66602017-06-01 19:39:03 -060014import collections
Simon Glass91197aa2021-12-18 14:54:35 -070015from contextlib import ExitStack
Simon Glass84067a52021-12-18 08:09:45 -070016import doctest
Masahiro Yamadac8e1b102016-05-19 15:52:07 +090017import filecmp
Masahiro Yamada5a27c732015-05-20 11:36:07 +090018import fnmatch
Masahiro Yamada0dbc9b52016-10-19 14:39:54 +090019import glob
Masahiro Yamada5a27c732015-05-20 11:36:07 +090020import multiprocessing
Masahiro Yamada5a27c732015-05-20 11:36:07 +090021import os
Simon Glass793dca32019-10-31 07:42:57 -060022import queue
Masahiro Yamada5a27c732015-05-20 11:36:07 +090023import re
24import shutil
25import subprocess
26import sys
27import tempfile
Simon Glassd73fcb12017-06-01 19:39:02 -060028import threading
Masahiro Yamada5a27c732015-05-20 11:36:07 +090029import time
Simon Glass84067a52021-12-18 08:09:45 -070030import unittest
Masahiro Yamada5a27c732015-05-20 11:36:07 +090031
Simon Glassb5aa5a32023-09-23 13:43:52 -060032import asteval
Simon Glass0ede00f2020-04-17 18:09:02 -060033from buildman import bsettings
34from buildman import kconfiglib
35from buildman import toolchain
Simon Glasscb008832017-06-15 21:39:33 -060036
Masahiro Yamada5a27c732015-05-20 11:36:07 +090037SHOW_GNU_MAKE = 'scripts/show-gnu-make'
38SLEEP_TIME=0.03
39
Masahiro Yamada5a27c732015-05-20 11:36:07 +090040STATE_IDLE = 0
41STATE_DEFCONFIG = 1
42STATE_AUTOCONF = 2
Joe Hershberger96464ba2015-05-19 13:21:17 -050043STATE_SAVEDEFCONFIG = 3
Masahiro Yamada5a27c732015-05-20 11:36:07 +090044
Masahiro Yamada5a27c732015-05-20 11:36:07 +090045COLOR_BLACK = '0;30'
46COLOR_RED = '0;31'
47COLOR_GREEN = '0;32'
48COLOR_BROWN = '0;33'
49COLOR_BLUE = '0;34'
50COLOR_PURPLE = '0;35'
51COLOR_CYAN = '0;36'
52COLOR_LIGHT_GRAY = '0;37'
53COLOR_DARK_GRAY = '1;30'
54COLOR_LIGHT_RED = '1;31'
55COLOR_LIGHT_GREEN = '1;32'
56COLOR_YELLOW = '1;33'
57COLOR_LIGHT_BLUE = '1;34'
58COLOR_LIGHT_PURPLE = '1;35'
59COLOR_LIGHT_CYAN = '1;36'
60COLOR_WHITE = '1;37'
61
Simon Glassf3b8e642017-06-01 19:39:01 -060062AUTO_CONF_PATH = 'include/config/auto.conf'
Simon Glassd73fcb12017-06-01 19:39:02 -060063CONFIG_DATABASE = 'moveconfig.db'
Simon Glassf3b8e642017-06-01 19:39:01 -060064
Simon Glasscb008832017-06-15 21:39:33 -060065CONFIG_LEN = len('CONFIG_')
Simon Glassf3b8e642017-06-01 19:39:01 -060066
Markus Klotzbuecherb237d352019-05-15 15:15:52 +020067SIZES = {
Simon Glassdaa694d2021-12-18 14:54:30 -070068 'SZ_1': 0x00000001, 'SZ_2': 0x00000002,
69 'SZ_4': 0x00000004, 'SZ_8': 0x00000008,
70 'SZ_16': 0x00000010, 'SZ_32': 0x00000020,
71 'SZ_64': 0x00000040, 'SZ_128': 0x00000080,
72 'SZ_256': 0x00000100, 'SZ_512': 0x00000200,
73 'SZ_1K': 0x00000400, 'SZ_2K': 0x00000800,
74 'SZ_4K': 0x00001000, 'SZ_8K': 0x00002000,
75 'SZ_16K': 0x00004000, 'SZ_32K': 0x00008000,
76 'SZ_64K': 0x00010000, 'SZ_128K': 0x00020000,
77 'SZ_256K': 0x00040000, 'SZ_512K': 0x00080000,
78 'SZ_1M': 0x00100000, 'SZ_2M': 0x00200000,
79 'SZ_4M': 0x00400000, 'SZ_8M': 0x00800000,
80 'SZ_16M': 0x01000000, 'SZ_32M': 0x02000000,
81 'SZ_64M': 0x04000000, 'SZ_128M': 0x08000000,
82 'SZ_256M': 0x10000000, 'SZ_512M': 0x20000000,
83 'SZ_1G': 0x40000000, 'SZ_2G': 0x80000000,
84 'SZ_4G': 0x100000000
Markus Klotzbuecherb237d352019-05-15 15:15:52 +020085}
86
Simon Glassb8d11da2022-02-08 11:49:45 -070087RE_REMOVE_DEFCONFIG = re.compile(r'(.*)_defconfig')
88
Simon Glass65e62032023-02-01 13:19:12 -070089# CONFIG symbols present in the build system (from Linux) but not actually used
90# in U-Boot; KCONFIG symbols
91IGNORE_SYMS = ['DEBUG_SECTION_MISMATCH', 'FTRACE_MCOUNT_RECORD', 'GCOV_KERNEL',
92 'GCOV_PROFILE_ALL', 'KALLSYMS', 'KASAN', 'MODVERSIONS', 'SHELL',
93 'TPL_BUILD', 'VPL_BUILD', 'IS_ENABLED', 'FOO', 'IF_ENABLED_INT',
94 'IS_ENABLED_', 'IS_ENABLED_1', 'IS_ENABLED_2', 'IS_ENABLED_3',
95 'SPL_', 'TPL_', 'SPL_FOO', 'TPL_FOO', 'TOOLS_FOO',
96 'ACME', 'SPL_ACME', 'TPL_ACME', 'TRACE_BRANCH_PROFILING',
97 'VAL', '_UNDEFINED', 'SPL_BUILD', ]
98
99SPL_PREFIXES = ['SPL_', 'TPL_', 'VPL_', 'TOOLS_']
100
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900101### helper functions ###
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900102def check_top_directory():
103 """Exit if we are not at the top of source directory."""
Simon Glass91197aa2021-12-18 14:54:35 -0700104 for fname in 'README', 'Licenses':
105 if not os.path.exists(fname):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900106 sys.exit('Please run at the top of source directory.')
107
Masahiro Yamadabd63e5b2016-05-19 15:51:54 +0900108def check_clean_directory():
109 """Exit if the source tree is not clean."""
Simon Glass91197aa2021-12-18 14:54:35 -0700110 for fname in '.config', 'include/config':
111 if os.path.exists(fname):
Masahiro Yamadabd63e5b2016-05-19 15:51:54 +0900112 sys.exit("source tree is not clean, please run 'make mrproper'")
113
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900114def get_make_cmd():
115 """Get the command name of GNU Make.
116
117 U-Boot needs GNU Make for building, but the command name is not
118 necessarily "make". (for example, "gmake" on FreeBSD).
119 Returns the most appropriate command name on your system.
120 """
Simon Glass91197aa2021-12-18 14:54:35 -0700121 with subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) as proc:
122 ret = proc.communicate()
123 if proc.returncode:
124 sys.exit('GNU Make not found')
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900125 return ret[0].rstrip()
126
Simon Glass25f978c2017-06-01 19:38:58 -0600127def get_matched_defconfig(line):
128 """Get the defconfig files that match a pattern
129
130 Args:
Simon Glass91197aa2021-12-18 14:54:35 -0700131 line (str): Path or filename to match, e.g. 'configs/snow_defconfig' or
Simon Glass25f978c2017-06-01 19:38:58 -0600132 'k2*_defconfig'. If no directory is provided, 'configs/' is
133 prepended
134
135 Returns:
Simon Glass91197aa2021-12-18 14:54:35 -0700136 list of str: a list of matching defconfig files
Simon Glass25f978c2017-06-01 19:38:58 -0600137 """
138 dirname = os.path.dirname(line)
139 if dirname:
140 pattern = line
141 else:
142 pattern = os.path.join('configs', line)
143 return glob.glob(pattern) + glob.glob(pattern + '_defconfig')
144
Masahiro Yamada0dbc9b52016-10-19 14:39:54 +0900145def get_matched_defconfigs(defconfigs_file):
Simon Glassee4e61b2017-06-01 19:38:59 -0600146 """Get all the defconfig files that match the patterns in a file.
147
148 Args:
Simon Glass91197aa2021-12-18 14:54:35 -0700149 defconfigs_file (str): File containing a list of defconfigs to process,
150 or '-' to read the list from stdin
Simon Glassee4e61b2017-06-01 19:38:59 -0600151
152 Returns:
Simon Glass91197aa2021-12-18 14:54:35 -0700153 list of str: A list of paths to defconfig files, with no duplicates
Simon Glassee4e61b2017-06-01 19:38:59 -0600154 """
Masahiro Yamada0dbc9b52016-10-19 14:39:54 +0900155 defconfigs = []
Simon Glass91197aa2021-12-18 14:54:35 -0700156 with ExitStack() as stack:
157 if defconfigs_file == '-':
158 inf = sys.stdin
159 defconfigs_file = 'stdin'
160 else:
161 inf = stack.enter_context(open(defconfigs_file, encoding='utf-8'))
162 for i, line in enumerate(inf):
163 line = line.strip()
164 if not line:
165 continue # skip blank lines silently
166 if ' ' in line:
167 line = line.split(' ')[0] # handle 'git log' input
168 matched = get_matched_defconfig(line)
169 if not matched:
170 print(f"warning: {defconfigs_file}:{i + 1}: no defconfig matched '{line}'",
171 file=sys.stderr)
Masahiro Yamada0dbc9b52016-10-19 14:39:54 +0900172
Simon Glass91197aa2021-12-18 14:54:35 -0700173 defconfigs += matched
Masahiro Yamada0dbc9b52016-10-19 14:39:54 +0900174
175 # use set() to drop multiple matching
Simon Glass91197aa2021-12-18 14:54:35 -0700176 return [defconfig[len('configs') + 1:] for defconfig in set(defconfigs)]
Masahiro Yamada0dbc9b52016-10-19 14:39:54 +0900177
Masahiro Yamada684c3062016-07-25 19:15:28 +0900178def get_all_defconfigs():
Simon Glass91197aa2021-12-18 14:54:35 -0700179 """Get all the defconfig files under the configs/ directory.
180
181 Returns:
182 list of str: List of paths to defconfig files
183 """
Masahiro Yamada684c3062016-07-25 19:15:28 +0900184 defconfigs = []
Simon Glass91197aa2021-12-18 14:54:35 -0700185 for (dirpath, _, filenames) in os.walk('configs'):
Masahiro Yamada684c3062016-07-25 19:15:28 +0900186 dirpath = dirpath[len('configs') + 1:]
187 for filename in fnmatch.filter(filenames, '*_defconfig'):
188 defconfigs.append(os.path.join(dirpath, filename))
189
190 return defconfigs
191
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900192def color_text(color_enabled, color, string):
193 """Return colored string."""
194 if color_enabled:
Masahiro Yamada1d085562016-05-19 15:52:02 +0900195 # LF should not be surrounded by the escape sequence.
196 # Otherwise, additional whitespace or line-feed might be printed.
197 return '\n'.join([ '\033[' + color + 'm' + s + '\033[0m' if s else ''
198 for s in string.split('\n') ])
Simon Glass91197aa2021-12-18 14:54:35 -0700199 return string
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900200
Simon Glass2fd85bd2021-12-18 14:54:33 -0700201def write_file(fname, data):
202 """Write data to a file
203
204 Args:
205 fname (str): Filename to write to
206 data (list of str): Lines to write (with or without trailing newline);
207 or str to write
208 """
209 with open(fname, 'w', encoding='utf-8') as out:
210 if isinstance(data, list):
211 for line in data:
212 print(line.rstrip('\n'), file=out)
213 else:
214 out.write(data)
215
Simon Glass37f815c2021-12-18 14:54:34 -0700216def read_file(fname, as_lines=True, skip_unicode=False):
217 """Read a file and return the contents
218
219 Args:
220 fname (str): Filename to read from
Simon Glass549d4222023-09-23 13:43:58 -0600221 as_lines (bool): Return file contents as a list of lines
Simon Glass37f815c2021-12-18 14:54:34 -0700222 skip_unicode (bool): True to report unicode errors and continue
223
224 Returns:
225 iter of str: List of ;ines from the file with newline removed; str if
226 as_lines is False with newlines intact; or None if a unicode error
227 occurred
228
229 Raises:
230 UnicodeDecodeError: Unicode error occurred when reading
231 """
232 with open(fname, encoding='utf-8') as inf:
233 try:
234 if as_lines:
235 return [line.rstrip('\n') for line in inf.readlines()]
Simon Glassf297ba32023-09-23 13:44:05 -0600236 return inf.read()
Simon Glassa4c9d172023-09-23 13:44:01 -0600237 except UnicodeDecodeError as exc:
Simon Glass37f815c2021-12-18 14:54:34 -0700238 if not skip_unicode:
Simon Glass68a0b712022-02-11 13:23:22 -0700239 raise
Simon Glassa4c9d172023-09-23 13:44:01 -0600240 print(f"Failed on file '{fname}: {exc}")
Simon Glass37f815c2021-12-18 14:54:34 -0700241 return None
242
Markus Klotzbuecherb237d352019-05-15 15:15:52 +0200243def try_expand(line):
244 """If value looks like an expression, try expanding it
245 Otherwise just return the existing value
246 """
247 if line.find('=') == -1:
248 return line
249
250 try:
Markus Klotzbuecherb3192f42020-02-12 20:46:44 +0100251 aeval = asteval.Interpreter( usersyms=SIZES, minimal=True )
Markus Klotzbuecherb237d352019-05-15 15:15:52 +0200252 cfg, val = re.split("=", line)
253 val= val.strip('\"')
Simon Glassdaa694d2021-12-18 14:54:30 -0700254 if re.search(r'[*+-/]|<<|SZ_+|\(([^\)]+)\)', val):
Markus Klotzbuecherb3192f42020-02-12 20:46:44 +0100255 newval = hex(aeval(val))
Simon Glass1bd43062023-09-23 13:43:59 -0600256 print(f'\tExpanded expression {val} to {newval}')
Markus Klotzbuecherb237d352019-05-15 15:15:52 +0200257 return cfg+'='+newval
258 except:
Simon Glass1bd43062023-09-23 13:43:59 -0600259 print(f'\tFailed to expand expression in {line}')
Markus Klotzbuecherb237d352019-05-15 15:15:52 +0200260
261 return line
262
Chris Packhamca438342017-05-02 21:30:47 +1200263
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900264### classes ###
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900265class Progress:
266
267 """Progress Indicator"""
268
269 def __init__(self, total):
270 """Create a new progress indicator.
271
Simon Glass91197aa2021-12-18 14:54:35 -0700272 Args:
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900273 total: A number of defconfig files to process.
274 """
275 self.current = 0
276 self.total = total
277
278 def inc(self):
279 """Increment the number of processed defconfig files."""
280
281 self.current += 1
282
283 def show(self):
284 """Display the progress."""
Simon Glass1bd43062023-09-23 13:43:59 -0600285 print(f' {self.current} defconfigs out of {self.total}\r', end=' ')
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900286 sys.stdout.flush()
287
Simon Glasscb008832017-06-15 21:39:33 -0600288
289class KconfigScanner:
290 """Kconfig scanner."""
291
292 def __init__(self):
293 """Scan all the Kconfig files and create a Config object."""
294 # Define environment variables referenced from Kconfig
295 os.environ['srctree'] = os.getcwd()
296 os.environ['UBOOTVERSION'] = 'dummy'
297 os.environ['KCONFIG_OBJDIR'] = ''
Simon Glass65e62032023-02-01 13:19:12 -0700298 os.environ['CC'] = 'gcc'
Tom Rini65e05dd2019-09-20 17:42:09 -0400299 self.conf = kconfiglib.Kconfig()
Simon Glasscb008832017-06-15 21:39:33 -0600300
301
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900302class KconfigParser:
303
304 """A parser of .config and include/autoconf.mk."""
305
306 re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"')
307 re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"')
308
Simon Glass882c8e42023-09-23 13:43:54 -0600309 def __init__(self, args, build_dir):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900310 """Create a new parser.
311
Simon Glass91197aa2021-12-18 14:54:35 -0700312 Args:
Simon Glass91197aa2021-12-18 14:54:35 -0700313 args (Namespace): program arguments
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900314 build_dir: Build directory.
315 """
Simon Glassb2e83c62021-12-18 14:54:31 -0700316 self.args = args
Masahiro Yamada1f169922016-05-19 15:52:00 +0900317 self.dotconfig = os.path.join(build_dir, '.config')
318 self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk')
Masahiro Yamada07913d12016-08-22 22:18:22 +0900319 self.spl_autoconf = os.path.join(build_dir, 'spl', 'include',
320 'autoconf.mk')
Simon Glassf3b8e642017-06-01 19:39:01 -0600321 self.config_autoconf = os.path.join(build_dir, AUTO_CONF_PATH)
Masahiro Yamada5da4f852016-05-19 15:52:06 +0900322 self.defconfig = os.path.join(build_dir, 'defconfig')
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900323
Simon Glass6821a742017-07-10 14:47:47 -0600324 def get_arch(self):
325 """Parse .config file and return the architecture.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900326
327 Returns:
Simon Glass6821a742017-07-10 14:47:47 -0600328 Architecture name (e.g. 'arm').
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900329 """
330 arch = ''
331 cpu = ''
Simon Glass37f815c2021-12-18 14:54:34 -0700332 for line in read_file(self.dotconfig):
Simon Glassa4c9d172023-09-23 13:44:01 -0600333 m_arch = self.re_arch.match(line)
334 if m_arch:
335 arch = m_arch.group(1)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900336 continue
Simon Glassa4c9d172023-09-23 13:44:01 -0600337 m_cpu = self.re_cpu.match(line)
338 if m_cpu:
339 cpu = m_cpu.group(1)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900340
Masahiro Yamada90ed6cb2016-05-19 15:51:53 +0900341 if not arch:
342 return None
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900343
344 # fix-up for aarch64
345 if arch == 'arm' and cpu == 'armv8':
346 arch = 'aarch64'
347
Simon Glass6821a742017-07-10 14:47:47 -0600348 return arch
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900349
Simon Glassd73fcb12017-06-01 19:39:02 -0600350
351class DatabaseThread(threading.Thread):
352 """This thread processes results from Slot threads.
353
354 It collects the data in the master config directary. There is only one
355 result thread, and this helps to serialise the build output.
356 """
357 def __init__(self, config_db, db_queue):
358 """Set up a new result thread
359
360 Args:
361 builder: Builder which will be sent each result
362 """
363 threading.Thread.__init__(self)
364 self.config_db = config_db
365 self.db_queue= db_queue
366
367 def run(self):
368 """Called to start up the result thread.
369
370 We collect the next result job and pass it on to the build.
371 """
372 while True:
373 defconfig, configs = self.db_queue.get()
374 self.config_db[defconfig] = configs
375 self.db_queue.task_done()
376
377
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900378class Slot:
379
380 """A slot to store a subprocess.
381
382 Each instance of this class handles one subprocess.
383 This class is useful to control multiple threads
384 for faster processing.
385 """
386
Simon Glass882c8e42023-09-23 13:43:54 -0600387 def __init__(self, toolchains, args, progress, devnull,
Simon Glass6821a742017-07-10 14:47:47 -0600388 make_cmd, reference_src_dir, db_queue):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900389 """Create a new process slot.
390
Simon Glass91197aa2021-12-18 14:54:35 -0700391 Args:
Simon Glass6821a742017-07-10 14:47:47 -0600392 toolchains: Toolchains object containing toolchains.
Simon Glassb2e83c62021-12-18 14:54:31 -0700393 args: Program arguments
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900394 progress: A progress indicator.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900395 devnull: A file object of '/dev/null'.
396 make_cmd: command name of GNU Make.
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500397 reference_src_dir: Determine the true starting config state from this
398 source tree.
Simon Glassd73fcb12017-06-01 19:39:02 -0600399 db_queue: output queue to write config info for the database
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900400 """
Simon Glass6821a742017-07-10 14:47:47 -0600401 self.toolchains = toolchains
Simon Glassb2e83c62021-12-18 14:54:31 -0700402 self.args = args
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900403 self.progress = progress
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900404 self.build_dir = tempfile.mkdtemp()
405 self.devnull = devnull
406 self.make_cmd = (make_cmd, 'O=' + self.build_dir)
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500407 self.reference_src_dir = reference_src_dir
Simon Glassd73fcb12017-06-01 19:39:02 -0600408 self.db_queue = db_queue
Simon Glass882c8e42023-09-23 13:43:54 -0600409 self.parser = KconfigParser(args, self.build_dir)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900410 self.state = STATE_IDLE
Masahiro Yamada09c6c062016-08-22 22:18:20 +0900411 self.failed_boards = set()
Simon Glassa6ab4db2023-09-23 13:44:02 -0600412 self.defconfig = None
413 self.log = ''
414 self.current_src_dir = None
415 self.proc = None
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900416
417 def __del__(self):
418 """Delete the working directory
419
420 This function makes sure the temporary directory is cleaned away
421 even if Python suddenly dies due to error. It should be done in here
Joe Hershbergerf2dae752016-06-10 14:53:29 -0500422 because it is guaranteed the destructor is always invoked when the
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900423 instance of the class gets unreferenced.
424
425 If the subprocess is still running, wait until it finishes.
426 """
427 if self.state != STATE_IDLE:
Simon Glassf297ba32023-09-23 13:44:05 -0600428 while self.proc.poll() is None:
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900429 pass
430 shutil.rmtree(self.build_dir)
431
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900432 def add(self, defconfig):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900433 """Assign a new subprocess for defconfig and add it to the slot.
434
435 If the slot is vacant, create a new subprocess for processing the
436 given defconfig and add it to the slot. Just returns False if
437 the slot is occupied (i.e. the current subprocess is still running).
438
Simon Glass91197aa2021-12-18 14:54:35 -0700439 Args:
Simon Glass549d4222023-09-23 13:43:58 -0600440 defconfig (str): defconfig name.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900441
442 Returns:
443 Return True on success or False on failure
444 """
445 if self.state != STATE_IDLE:
446 return False
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900447
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900448 self.defconfig = defconfig
Masahiro Yamada1d085562016-05-19 15:52:02 +0900449 self.log = ''
Masahiro Yamadaf432c332016-06-15 14:33:52 +0900450 self.current_src_dir = self.reference_src_dir
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900451 self.do_defconfig()
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900452 return True
453
454 def poll(self):
455 """Check the status of the subprocess and handle it as needed.
456
457 Returns True if the slot is vacant (i.e. in idle state).
458 If the configuration is successfully finished, assign a new
459 subprocess to build include/autoconf.mk.
460 If include/autoconf.mk is generated, invoke the parser to
Masahiro Yamada7fb0bac2016-05-19 15:52:04 +0900461 parse the .config and the include/autoconf.mk, moving
462 config options to the .config as needed.
463 If the .config was updated, run "make savedefconfig" to sync
464 it, update the original defconfig, and then set the slot back
465 to the idle state.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900466
467 Returns:
468 Return True if the subprocess is terminated, False otherwise
469 """
470 if self.state == STATE_IDLE:
471 return True
472
Simon Glassf297ba32023-09-23 13:44:05 -0600473 if self.proc.poll() is None:
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900474 return False
475
Simon Glassa4c9d172023-09-23 13:44:01 -0600476 if self.proc.poll() != 0:
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900477 self.handle_error()
478 elif self.state == STATE_DEFCONFIG:
Masahiro Yamadaf432c332016-06-15 14:33:52 +0900479 if self.reference_src_dir and not self.current_src_dir:
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500480 self.do_savedefconfig()
481 else:
482 self.do_autoconf()
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900483 elif self.state == STATE_AUTOCONF:
Masahiro Yamadaf432c332016-06-15 14:33:52 +0900484 if self.current_src_dir:
485 self.current_src_dir = None
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500486 self.do_defconfig()
Simon Glassb2e83c62021-12-18 14:54:31 -0700487 elif self.args.build_db:
Simon Glassd73fcb12017-06-01 19:39:02 -0600488 self.do_build_db()
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500489 else:
490 self.do_savedefconfig()
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900491 elif self.state == STATE_SAVEDEFCONFIG:
492 self.update_defconfig()
493 else:
Simon Glassdaa694d2021-12-18 14:54:30 -0700494 sys.exit('Internal Error. This should not happen.')
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900495
Simon Glassf297ba32023-09-23 13:44:05 -0600496 return self.state == STATE_IDLE
Joe Hershberger96464ba2015-05-19 13:21:17 -0500497
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900498 def handle_error(self):
499 """Handle error cases."""
Masahiro Yamada8513dc02016-05-19 15:52:08 +0900500
Simon Glassb2e83c62021-12-18 14:54:31 -0700501 self.log += color_text(self.args.color, COLOR_LIGHT_RED,
Simon Glassdaa694d2021-12-18 14:54:30 -0700502 'Failed to process.\n')
Simon Glassb2e83c62021-12-18 14:54:31 -0700503 if self.args.verbose:
504 self.log += color_text(self.args.color, COLOR_LIGHT_CYAN,
Simon Glassa4c9d172023-09-23 13:44:01 -0600505 self.proc.stderr.read().decode())
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900506 self.finish(False)
Joe Hershberger96464ba2015-05-19 13:21:17 -0500507
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900508 def do_defconfig(self):
509 """Run 'make <board>_defconfig' to create the .config file."""
Masahiro Yamadac8e1b102016-05-19 15:52:07 +0900510
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900511 cmd = list(self.make_cmd)
512 cmd.append(self.defconfig)
Simon Glassa4c9d172023-09-23 13:44:01 -0600513 self.proc = subprocess.Popen(cmd, stdout=self.devnull,
514 stderr=subprocess.PIPE,
515 cwd=self.current_src_dir)
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900516 self.state = STATE_DEFCONFIG
Masahiro Yamadac8e1b102016-05-19 15:52:07 +0900517
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900518 def do_autoconf(self):
Simon Glassf3b8e642017-06-01 19:39:01 -0600519 """Run 'make AUTO_CONF_PATH'."""
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900520
Simon Glass6821a742017-07-10 14:47:47 -0600521 arch = self.parser.get_arch()
522 try:
Simon Glassf297ba32023-09-23 13:44:05 -0600523 tchain = self.toolchains.Select(arch)
Simon Glass6821a742017-07-10 14:47:47 -0600524 except ValueError:
Simon Glassb2e83c62021-12-18 14:54:31 -0700525 self.log += color_text(self.args.color, COLOR_YELLOW,
Simon Glass1bd43062023-09-23 13:43:59 -0600526 f"Tool chain for '{arch}' is missing. Do nothing.\n")
Masahiro Yamada4efef992016-05-19 15:52:03 +0900527 self.finish(False)
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900528 return
Simon Glassf297ba32023-09-23 13:44:05 -0600529 env = tchain.MakeEnvironment(False)
Masahiro Yamada90ed6cb2016-05-19 15:51:53 +0900530
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900531 cmd = list(self.make_cmd)
Joe Hershberger7740f652015-05-19 13:21:18 -0500532 cmd.append('KCONFIG_IGNORE_DUPLICATES=1')
Simon Glassf3b8e642017-06-01 19:39:01 -0600533 cmd.append(AUTO_CONF_PATH)
Simon Glassa4c9d172023-09-23 13:44:01 -0600534 self.proc = subprocess.Popen(cmd, stdout=self.devnull, env=env,
535 stderr=subprocess.PIPE,
536 cwd=self.current_src_dir)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900537 self.state = STATE_AUTOCONF
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900538
Simon Glassd73fcb12017-06-01 19:39:02 -0600539 def do_build_db(self):
540 """Add the board to the database"""
541 configs = {}
Simon Glass37f815c2021-12-18 14:54:34 -0700542 for line in read_file(os.path.join(self.build_dir, AUTO_CONF_PATH)):
543 if line.startswith('CONFIG'):
544 config, value = line.split('=', 1)
545 configs[config] = value.rstrip()
Simon Glassd73fcb12017-06-01 19:39:02 -0600546 self.db_queue.put([self.defconfig, configs])
547 self.finish(True)
548
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900549 def do_savedefconfig(self):
550 """Update the .config and run 'make savedefconfig'."""
Simon Glassc7345612023-09-23 13:43:55 -0600551 if not self.args.force_sync:
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900552 self.finish(True)
553 return
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900554
555 cmd = list(self.make_cmd)
556 cmd.append('savedefconfig')
Simon Glassa4c9d172023-09-23 13:44:01 -0600557 self.proc = subprocess.Popen(cmd, stdout=self.devnull,
558 stderr=subprocess.PIPE)
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900559 self.state = STATE_SAVEDEFCONFIG
560
561 def update_defconfig(self):
562 """Update the input defconfig and go back to the idle state."""
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900563 orig_defconfig = os.path.join('configs', self.defconfig)
564 new_defconfig = os.path.join(self.build_dir, 'defconfig')
565 updated = not filecmp.cmp(orig_defconfig, new_defconfig)
566
567 if updated:
Simon Glassb2e83c62021-12-18 14:54:31 -0700568 self.log += color_text(self.args.color, COLOR_LIGHT_BLUE,
Simon Glassdaa694d2021-12-18 14:54:30 -0700569 'defconfig was updated.\n')
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900570
Simon Glassb2e83c62021-12-18 14:54:31 -0700571 if not self.args.dry_run and updated:
Masahiro Yamadae307fa92016-06-08 11:47:37 +0900572 shutil.move(new_defconfig, orig_defconfig)
573 self.finish(True)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900574
Masahiro Yamada4efef992016-05-19 15:52:03 +0900575 def finish(self, success):
576 """Display log along with progress and go to the idle state.
Masahiro Yamada1d085562016-05-19 15:52:02 +0900577
Simon Glass91197aa2021-12-18 14:54:35 -0700578 Args:
Simon Glass549d4222023-09-23 13:43:58 -0600579 success (bool): Should be True when the defconfig was processed
Masahiro Yamada4efef992016-05-19 15:52:03 +0900580 successfully, or False when it fails.
Masahiro Yamada1d085562016-05-19 15:52:02 +0900581 """
582 # output at least 30 characters to hide the "* defconfigs out of *".
Simon Glass5aba58c2023-09-23 13:44:06 -0600583 if self.log:
584 log = self.defconfig.ljust(30) + '\n'
Masahiro Yamada1d085562016-05-19 15:52:02 +0900585
Simon Glass5aba58c2023-09-23 13:44:06 -0600586 log += '\n'.join([ ' ' + s for s in self.log.split('\n') ])
587 # Some threads are running in parallel.
588 # Print log atomically to not mix up logs from different threads.
589 print(log, file=(sys.stdout if success else sys.stderr))
Masahiro Yamada4efef992016-05-19 15:52:03 +0900590
591 if not success:
Simon Glassb2e83c62021-12-18 14:54:31 -0700592 if self.args.exit_on_error:
Simon Glassdaa694d2021-12-18 14:54:30 -0700593 sys.exit('Exit on error.')
Masahiro Yamada4efef992016-05-19 15:52:03 +0900594 # If --exit-on-error flag is not set, skip this board and continue.
595 # Record the failed board.
Masahiro Yamada09c6c062016-08-22 22:18:20 +0900596 self.failed_boards.add(self.defconfig)
Masahiro Yamada4efef992016-05-19 15:52:03 +0900597
Masahiro Yamada1d085562016-05-19 15:52:02 +0900598 self.progress.inc()
599 self.progress.show()
Masahiro Yamada4efef992016-05-19 15:52:03 +0900600 self.state = STATE_IDLE
Masahiro Yamada1d085562016-05-19 15:52:02 +0900601
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900602 def get_failed_boards(self):
Masahiro Yamada09c6c062016-08-22 22:18:20 +0900603 """Returns a set of failed boards (defconfigs) in this slot.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900604 """
605 return self.failed_boards
606
607class Slots:
608
609 """Controller of the array of subprocess slots."""
610
Simon Glass882c8e42023-09-23 13:43:54 -0600611 def __init__(self, toolchains, args, progress, reference_src_dir, db_queue):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900612 """Create a new slots controller.
613
Simon Glass91197aa2021-12-18 14:54:35 -0700614 Args:
Simon Glass6821a742017-07-10 14:47:47 -0600615 toolchains: Toolchains object containing toolchains.
Simon Glassb2e83c62021-12-18 14:54:31 -0700616 args: Program arguments
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900617 progress: A progress indicator.
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500618 reference_src_dir: Determine the true starting config state from this
619 source tree.
Simon Glassd73fcb12017-06-01 19:39:02 -0600620 db_queue: output queue to write config info for the database
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900621 """
Simon Glassb2e83c62021-12-18 14:54:31 -0700622 self.args = args
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900623 self.slots = []
Simon Glass478920d2021-12-18 14:54:32 -0700624 devnull = subprocess.DEVNULL
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900625 make_cmd = get_make_cmd()
Simon Glass62fae4b2023-09-23 13:44:00 -0600626 for _ in range(args.jobs):
Simon Glass882c8e42023-09-23 13:43:54 -0600627 self.slots.append(Slot(toolchains, args, progress, devnull,
628 make_cmd, reference_src_dir, db_queue))
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900629
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900630 def add(self, defconfig):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900631 """Add a new subprocess if a vacant slot is found.
632
Simon Glass91197aa2021-12-18 14:54:35 -0700633 Args:
Simon Glass549d4222023-09-23 13:43:58 -0600634 defconfig (str): defconfig name to be put into.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900635
636 Returns:
637 Return True on success or False on failure
638 """
639 for slot in self.slots:
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900640 if slot.add(defconfig):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900641 return True
642 return False
643
644 def available(self):
645 """Check if there is a vacant slot.
646
647 Returns:
648 Return True if at lease one vacant slot is found, False otherwise.
649 """
650 for slot in self.slots:
651 if slot.poll():
652 return True
653 return False
654
655 def empty(self):
656 """Check if all slots are vacant.
657
658 Returns:
659 Return True if all the slots are vacant, False otherwise.
660 """
661 ret = True
662 for slot in self.slots:
663 if not slot.poll():
664 ret = False
665 return ret
666
667 def show_failed_boards(self):
668 """Display all of the failed boards (defconfigs)."""
Masahiro Yamada09c6c062016-08-22 22:18:20 +0900669 boards = set()
Masahiro Yamada96dccd92016-06-15 14:33:53 +0900670 output_file = 'moveconfig.failed'
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900671
672 for slot in self.slots:
Masahiro Yamada09c6c062016-08-22 22:18:20 +0900673 boards |= slot.get_failed_boards()
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900674
Masahiro Yamada96dccd92016-06-15 14:33:53 +0900675 if boards:
676 boards = '\n'.join(boards) + '\n'
Simon Glassdaa694d2021-12-18 14:54:30 -0700677 msg = 'The following boards were not processed due to error:\n'
Masahiro Yamada96dccd92016-06-15 14:33:53 +0900678 msg += boards
Simon Glass1bd43062023-09-23 13:43:59 -0600679 msg += f'(the list has been saved in {output_file})\n'
Simon Glassb2e83c62021-12-18 14:54:31 -0700680 print(color_text(self.args.color, COLOR_LIGHT_RED,
Simon Glass793dca32019-10-31 07:42:57 -0600681 msg), file=sys.stderr)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900682
Simon Glass2fd85bd2021-12-18 14:54:33 -0700683 write_file(output_file, boards)
Joe Hershberger2559cd82015-05-19 13:21:22 -0500684
Masahiro Yamada5cc42a52016-06-15 14:33:51 +0900685class ReferenceSource:
686
687 """Reference source against which original configs should be parsed."""
688
689 def __init__(self, commit):
690 """Create a reference source directory based on a specified commit.
691
Simon Glass91197aa2021-12-18 14:54:35 -0700692 Args:
Masahiro Yamada5cc42a52016-06-15 14:33:51 +0900693 commit: commit to git-clone
694 """
695 self.src_dir = tempfile.mkdtemp()
Simon Glassdaa694d2021-12-18 14:54:30 -0700696 print('Cloning git repo to a separate work directory...')
Masahiro Yamada5cc42a52016-06-15 14:33:51 +0900697 subprocess.check_output(['git', 'clone', os.getcwd(), '.'],
698 cwd=self.src_dir)
Simon Glass1bd43062023-09-23 13:43:59 -0600699 rev = subprocess.check_output(['git', 'rev-parse', '--short',
700 commit]).strip()
701 print(f"Checkout '{rev}' to build the original autoconf.mk.")
Masahiro Yamada5cc42a52016-06-15 14:33:51 +0900702 subprocess.check_output(['git', 'checkout', commit],
703 stderr=subprocess.STDOUT, cwd=self.src_dir)
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500704
705 def __del__(self):
Masahiro Yamada5cc42a52016-06-15 14:33:51 +0900706 """Delete the reference source directory
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500707
708 This function makes sure the temporary directory is cleaned away
709 even if Python suddenly dies due to error. It should be done in here
710 because it is guaranteed the destructor is always invoked when the
711 instance of the class gets unreferenced.
712 """
Masahiro Yamada5cc42a52016-06-15 14:33:51 +0900713 shutil.rmtree(self.src_dir)
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500714
Masahiro Yamada5cc42a52016-06-15 14:33:51 +0900715 def get_dir(self):
716 """Return the absolute path to the reference source directory."""
717
718 return self.src_dir
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500719
Simon Glass882c8e42023-09-23 13:43:54 -0600720def move_config(toolchains, args, db_queue):
721 """Build database or sync config options to defconfig files.
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900722
Simon Glass91197aa2021-12-18 14:54:35 -0700723 Args:
Simon Glass549d4222023-09-23 13:43:58 -0600724 toolchains (Toolchains): Toolchains to use
725 args (Namespace): Program arguments
726 db_queue (Queue): Queue for database updates
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900727 """
Simon Glass882c8e42023-09-23 13:43:54 -0600728 if args.force_sync:
729 print('Syncing defconfigs', end=' ')
730 elif args.build_db:
Simon Glass1bd43062023-09-23 13:43:59 -0600731 print(f'Building {CONFIG_DATABASE} database')
732 print(f'(jobs: {args.jobs})\n')
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900733
Simon Glassb2e83c62021-12-18 14:54:31 -0700734 if args.git_ref:
735 reference_src = ReferenceSource(args.git_ref)
Masahiro Yamada5cc42a52016-06-15 14:33:51 +0900736 reference_src_dir = reference_src.get_dir()
737 else:
Masahiro Yamadaf432c332016-06-15 14:33:52 +0900738 reference_src_dir = None
Joe Hershberger6b96c1a2016-06-10 14:53:32 -0500739
Simon Glassb2e83c62021-12-18 14:54:31 -0700740 if args.defconfigs:
741 defconfigs = get_matched_defconfigs(args.defconfigs)
Joe Hershberger91040e82015-05-19 13:21:19 -0500742 else:
Masahiro Yamada684c3062016-07-25 19:15:28 +0900743 defconfigs = get_all_defconfigs()
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900744
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900745 progress = Progress(len(defconfigs))
Simon Glass882c8e42023-09-23 13:43:54 -0600746 slots = Slots(toolchains, args, progress, reference_src_dir, db_queue)
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900747
748 # Main loop to process defconfig files:
749 # Add a new subprocess into a vacant slot.
750 # Sleep if there is no available slot.
Masahiro Yamadac5e60fd2016-05-19 15:51:55 +0900751 for defconfig in defconfigs:
752 while not slots.add(defconfig):
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900753 while not slots.available():
754 # No available slot: sleep for a while
755 time.sleep(SLEEP_TIME)
756
757 # wait until all the subprocesses finish
758 while not slots.empty():
759 time.sleep(SLEEP_TIME)
760
Simon Glass793dca32019-10-31 07:42:57 -0600761 print('')
Masahiro Yamada5a27c732015-05-20 11:36:07 +0900762 slots.show_failed_boards()
763
Simon Glasscb008832017-06-15 21:39:33 -0600764def find_kconfig_rules(kconf, config, imply_config):
765 """Check whether a config has a 'select' or 'imply' keyword
766
767 Args:
Simon Glass549d4222023-09-23 13:43:58 -0600768 kconf (Kconfiglib.Kconfig): Kconfig object
769 config (str): Name of config to check (without CONFIG_ prefix)
770 imply_config (str): Implying config (without CONFIG_ prefix) which may
771 or may not have an 'imply' for 'config')
Simon Glasscb008832017-06-15 21:39:33 -0600772
773 Returns:
774 Symbol object for 'config' if found, else None
775 """
Tom Rini65e05dd2019-09-20 17:42:09 -0400776 sym = kconf.syms.get(imply_config)
Simon Glasscb008832017-06-15 21:39:33 -0600777 if sym:
Simon Glass62fae4b2023-09-23 13:44:00 -0600778 for sel, _ in (sym.selects + sym.implies):
Simon Glassa3627082021-12-18 08:09:42 -0700779 if sel.name == config:
Simon Glasscb008832017-06-15 21:39:33 -0600780 return sym
781 return None
782
783def check_imply_rule(kconf, config, imply_config):
784 """Check if we can add an 'imply' option
785
786 This finds imply_config in the Kconfig and looks to see if it is possible
787 to add an 'imply' for 'config' to that part of the Kconfig.
788
789 Args:
Simon Glass549d4222023-09-23 13:43:58 -0600790 kconf (Kconfiglib.Kconfig): Kconfig object
791 config (str): Name of config to check (without CONFIG_ prefix)
792 imply_config (str): Implying config (without CONFIG_ prefix) which may
793 or may not have an 'imply' for 'config')
Simon Glasscb008832017-06-15 21:39:33 -0600794
795 Returns:
796 tuple:
Simon Glass549d4222023-09-23 13:43:58 -0600797 str: filename of Kconfig file containing imply_config, or None if
798 none
799 int: line number within the Kconfig file, or 0 if none
800 str: message indicating the result
Simon Glasscb008832017-06-15 21:39:33 -0600801 """
Tom Rini65e05dd2019-09-20 17:42:09 -0400802 sym = kconf.syms.get(imply_config)
Simon Glasscb008832017-06-15 21:39:33 -0600803 if not sym:
804 return 'cannot find sym'
Simon Glassea40b202021-07-21 21:35:53 -0600805 nodes = sym.nodes
806 if len(nodes) != 1:
Simon Glass1bd43062023-09-23 13:43:59 -0600807 return f'{len(nodes)} locations'
Simon Glassa3627082021-12-18 08:09:42 -0700808 node = nodes[0]
809 fname, linenum = node.filename, node.linenr
Simon Glasscb008832017-06-15 21:39:33 -0600810 cwd = os.getcwd()
811 if cwd and fname.startswith(cwd):
812 fname = fname[len(cwd) + 1:]
Simon Glass1bd43062023-09-23 13:43:59 -0600813 file_line = f' at {fname}:{linenum}'
Simon Glass37f815c2021-12-18 14:54:34 -0700814 data = read_file(fname)
Simon Glass1bd43062023-09-23 13:43:59 -0600815 if data[linenum - 1] != f'config {imply_config}':
816 return None, 0, f'bad sym format {data[linenum]}{file_line})'
817 return fname, linenum, f'adding{file_line}'
Simon Glasscb008832017-06-15 21:39:33 -0600818
819def add_imply_rule(config, fname, linenum):
820 """Add a new 'imply' option to a Kconfig
821
822 Args:
Simon Glass549d4222023-09-23 13:43:58 -0600823 config (str): config option to add an imply for (without CONFIG_ prefix)
824 fname (str): Kconfig filename to update
825 linenum (int): Line number to place the 'imply' before
Simon Glasscb008832017-06-15 21:39:33 -0600826
827 Returns:
828 Message indicating the result
829 """
Simon Glass1bd43062023-09-23 13:43:59 -0600830 file_line = f' at {fname}:{linenum}'
Simon Glass37f815c2021-12-18 14:54:34 -0700831 data = read_file(fname)
Simon Glasscb008832017-06-15 21:39:33 -0600832 linenum -= 1
833
834 for offset, line in enumerate(data[linenum:]):
835 if line.strip().startswith('help') or not line:
Simon Glass1bd43062023-09-23 13:43:59 -0600836 data.insert(linenum + offset, f'\timply {config}')
Simon Glass2fd85bd2021-12-18 14:54:33 -0700837 write_file(fname, data)
Simon Glass1bd43062023-09-23 13:43:59 -0600838 return f'added{file_line}'
Simon Glasscb008832017-06-15 21:39:33 -0600839
840 return 'could not insert%s'
841
842(IMPLY_MIN_2, IMPLY_TARGET, IMPLY_CMD, IMPLY_NON_ARCH_BOARD) = (
843 1, 2, 4, 8)
Simon Glass9b2a2e82017-06-15 21:39:32 -0600844
845IMPLY_FLAGS = {
846 'min2': [IMPLY_MIN_2, 'Show options which imply >2 boards (normally >5)'],
847 'target': [IMPLY_TARGET, 'Allow CONFIG_TARGET_... options to imply'],
848 'cmd': [IMPLY_CMD, 'Allow CONFIG_CMD_... to imply'],
Simon Glasscb008832017-06-15 21:39:33 -0600849 'non-arch-board': [
850 IMPLY_NON_ARCH_BOARD,
851 'Allow Kconfig options outside arch/ and /board/ to imply'],
Simon Glass91197aa2021-12-18 14:54:35 -0700852}
Simon Glass9b2a2e82017-06-15 21:39:32 -0600853
Simon Glass9d603392021-12-18 08:09:43 -0700854
855def read_database():
856 """Read in the config database
857
858 Returns:
859 tuple:
860 set of all config options seen (each a str)
861 set of all defconfigs seen (each a str)
862 dict of configs for each defconfig:
863 key: defconfig name, e.g. "MPC8548CDS_legacy_defconfig"
864 value: dict:
865 key: CONFIG option
866 value: Value of option
867 dict of defconfigs for each config:
868 key: CONFIG option
869 value: set of boards using that option
870
871 """
872 configs = {}
873
874 # key is defconfig name, value is dict of (CONFIG_xxx, value)
875 config_db = {}
876
877 # Set of all config options we have seen
878 all_configs = set()
879
880 # Set of all defconfigs we have seen
881 all_defconfigs = set()
882
883 defconfig_db = collections.defaultdict(set)
Simon Glass37f815c2021-12-18 14:54:34 -0700884 for line in read_file(CONFIG_DATABASE):
885 line = line.rstrip()
886 if not line: # Separator between defconfigs
887 config_db[defconfig] = configs
888 all_defconfigs.add(defconfig)
889 configs = {}
890 elif line[0] == ' ': # CONFIG line
891 config, value = line.strip().split('=', 1)
892 configs[config] = value
893 defconfig_db[config].add(defconfig)
894 all_configs.add(config)
895 else: # New defconfig
896 defconfig = line
Simon Glass9d603392021-12-18 08:09:43 -0700897
898 return all_configs, all_defconfigs, config_db, defconfig_db
899
900
Simon Glasscb008832017-06-15 21:39:33 -0600901def do_imply_config(config_list, add_imply, imply_flags, skip_added,
902 check_kconfig=True, find_superset=False):
Simon Glass99b66602017-06-01 19:39:03 -0600903 """Find CONFIG options which imply those in the list
904
905 Some CONFIG options can be implied by others and this can help to reduce
906 the size of the defconfig files. For example, CONFIG_X86 implies
907 CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and
908 all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to
909 each of the x86 defconfig files.
910
911 This function uses the moveconfig database to find such options. It
912 displays a list of things that could possibly imply those in the list.
913 The algorithm ignores any that start with CONFIG_TARGET since these
914 typically refer to only a few defconfigs (often one). It also does not
915 display a config with less than 5 defconfigs.
916
917 The algorithm works using sets. For each target config in config_list:
918 - Get the set 'defconfigs' which use that target config
919 - For each config (from a list of all configs):
920 - Get the set 'imply_defconfig' of defconfigs which use that config
921 -
922 - If imply_defconfigs contains anything not in defconfigs then
923 this config does not imply the target config
924
925 Params:
926 config_list: List of CONFIG options to check (each a string)
Simon Glasscb008832017-06-15 21:39:33 -0600927 add_imply: Automatically add an 'imply' for each config.
Simon Glass9b2a2e82017-06-15 21:39:32 -0600928 imply_flags: Flags which control which implying configs are allowed
929 (IMPLY_...)
Simon Glasscb008832017-06-15 21:39:33 -0600930 skip_added: Don't show options which already have an imply added.
931 check_kconfig: Check if implied symbols already have an 'imply' or
932 'select' for the target config, and show this information if so.
Simon Glass99b66602017-06-01 19:39:03 -0600933 find_superset: True to look for configs which are a superset of those
934 already found. So for example if CONFIG_EXYNOS5 implies an option,
935 but CONFIG_EXYNOS covers a larger set of defconfigs and also
936 implies that option, this will drop the former in favour of the
937 latter. In practice this option has not proved very used.
938
939 Note the terminoloy:
940 config - a CONFIG_XXX options (a string, e.g. 'CONFIG_CMD_EEPROM')
941 defconfig - a defconfig file (a string, e.g. 'configs/snow_defconfig')
942 """
Simon Glasscb008832017-06-15 21:39:33 -0600943 kconf = KconfigScanner().conf if check_kconfig else None
944 if add_imply and add_imply != 'all':
Simon Glassa3627082021-12-18 08:09:42 -0700945 add_imply = add_imply.split(',')
Simon Glasscb008832017-06-15 21:39:33 -0600946
Simon Glass62fae4b2023-09-23 13:44:00 -0600947 all_configs, all_defconfigs, _, defconfig_db = read_database()
Simon Glass99b66602017-06-01 19:39:03 -0600948
Simon Glassa3627082021-12-18 08:09:42 -0700949 # Work through each target config option in turn, independently
Simon Glass99b66602017-06-01 19:39:03 -0600950 for config in config_list:
951 defconfigs = defconfig_db.get(config)
952 if not defconfigs:
Simon Glass1bd43062023-09-23 13:43:59 -0600953 print(f'{config} not found in any defconfig')
Simon Glass99b66602017-06-01 19:39:03 -0600954 continue
955
956 # Get the set of defconfigs without this one (since a config cannot
957 # imply itself)
958 non_defconfigs = all_defconfigs - defconfigs
959 num_defconfigs = len(defconfigs)
Simon Glass1bd43062023-09-23 13:43:59 -0600960 print(f'{config} found in {num_defconfigs}/{len(all_configs)} defconfigs')
Simon Glass99b66602017-06-01 19:39:03 -0600961
962 # This will hold the results: key=config, value=defconfigs containing it
963 imply_configs = {}
964 rest_configs = all_configs - set([config])
965
966 # Look at every possible config, except the target one
967 for imply_config in rest_configs:
Simon Glass9b2a2e82017-06-15 21:39:32 -0600968 if 'ERRATUM' in imply_config:
Simon Glass99b66602017-06-01 19:39:03 -0600969 continue
Simon Glass91197aa2021-12-18 14:54:35 -0700970 if not imply_flags & IMPLY_CMD:
Simon Glass9b2a2e82017-06-15 21:39:32 -0600971 if 'CONFIG_CMD' in imply_config:
972 continue
Simon Glass91197aa2021-12-18 14:54:35 -0700973 if not imply_flags & IMPLY_TARGET:
Simon Glass9b2a2e82017-06-15 21:39:32 -0600974 if 'CONFIG_TARGET' in imply_config:
975 continue
Simon Glass99b66602017-06-01 19:39:03 -0600976
977 # Find set of defconfigs that have this config
978 imply_defconfig = defconfig_db[imply_config]
979
980 # Get the intersection of this with defconfigs containing the
981 # target config
982 common_defconfigs = imply_defconfig & defconfigs
983
984 # Get the set of defconfigs containing this config which DO NOT
985 # also contain the taret config. If this set is non-empty it means
986 # that this config affects other defconfigs as well as (possibly)
987 # the ones affected by the target config. This means it implies
988 # things we don't want to imply.
989 not_common_defconfigs = imply_defconfig & non_defconfigs
990 if not_common_defconfigs:
991 continue
992
993 # If there are common defconfigs, imply_config may be useful
994 if common_defconfigs:
995 skip = False
996 if find_superset:
Simon Glass793dca32019-10-31 07:42:57 -0600997 for prev in list(imply_configs.keys()):
Simon Glass99b66602017-06-01 19:39:03 -0600998 prev_count = len(imply_configs[prev])
999 count = len(common_defconfigs)
1000 if (prev_count > count and
1001 (imply_configs[prev] & common_defconfigs ==
1002 common_defconfigs)):
1003 # skip imply_config because prev is a superset
1004 skip = True
1005 break
Simon Glassf297ba32023-09-23 13:44:05 -06001006 if count > prev_count:
Simon Glass99b66602017-06-01 19:39:03 -06001007 # delete prev because imply_config is a superset
1008 del imply_configs[prev]
1009 if not skip:
1010 imply_configs[imply_config] = common_defconfigs
1011
1012 # Now we have a dict imply_configs of configs which imply each config
1013 # The value of each dict item is the set of defconfigs containing that
1014 # config. Rank them so that we print the configs that imply the largest
1015 # number of defconfigs first.
Simon Glasscb008832017-06-15 21:39:33 -06001016 ranked_iconfigs = sorted(imply_configs,
Simon Glass99b66602017-06-01 19:39:03 -06001017 key=lambda k: len(imply_configs[k]), reverse=True)
Simon Glasscb008832017-06-15 21:39:33 -06001018 kconfig_info = ''
1019 cwd = os.getcwd()
1020 add_list = collections.defaultdict(list)
1021 for iconfig in ranked_iconfigs:
1022 num_common = len(imply_configs[iconfig])
Simon Glass99b66602017-06-01 19:39:03 -06001023
1024 # Don't bother if there are less than 5 defconfigs affected.
Simon Glass9b2a2e82017-06-15 21:39:32 -06001025 if num_common < (2 if imply_flags & IMPLY_MIN_2 else 5):
Simon Glass99b66602017-06-01 19:39:03 -06001026 continue
Simon Glasscb008832017-06-15 21:39:33 -06001027 missing = defconfigs - imply_configs[iconfig]
Simon Glass99b66602017-06-01 19:39:03 -06001028 missing_str = ', '.join(missing) if missing else 'all'
1029 missing_str = ''
Simon Glasscb008832017-06-15 21:39:33 -06001030 show = True
1031 if kconf:
1032 sym = find_kconfig_rules(kconf, config[CONFIG_LEN:],
1033 iconfig[CONFIG_LEN:])
1034 kconfig_info = ''
1035 if sym:
Simon Glassea40b202021-07-21 21:35:53 -06001036 nodes = sym.nodes
1037 if len(nodes) == 1:
1038 fname, linenum = nodes[0].filename, nodes[0].linenr
Simon Glasscb008832017-06-15 21:39:33 -06001039 if cwd and fname.startswith(cwd):
1040 fname = fname[len(cwd) + 1:]
Simon Glass1bd43062023-09-23 13:43:59 -06001041 kconfig_info = f'{fname}:{linenum}'
Simon Glasscb008832017-06-15 21:39:33 -06001042 if skip_added:
1043 show = False
1044 else:
Tom Rini65e05dd2019-09-20 17:42:09 -04001045 sym = kconf.syms.get(iconfig[CONFIG_LEN:])
Simon Glasscb008832017-06-15 21:39:33 -06001046 fname = ''
1047 if sym:
Simon Glassea40b202021-07-21 21:35:53 -06001048 nodes = sym.nodes
1049 if len(nodes) == 1:
1050 fname, linenum = nodes[0].filename, nodes[0].linenr
Simon Glasscb008832017-06-15 21:39:33 -06001051 if cwd and fname.startswith(cwd):
1052 fname = fname[len(cwd) + 1:]
1053 in_arch_board = not sym or (fname.startswith('arch') or
1054 fname.startswith('board'))
1055 if (not in_arch_board and
Simon Glass91197aa2021-12-18 14:54:35 -07001056 not imply_flags & IMPLY_NON_ARCH_BOARD):
Simon Glasscb008832017-06-15 21:39:33 -06001057 continue
1058
1059 if add_imply and (add_imply == 'all' or
1060 iconfig in add_imply):
1061 fname, linenum, kconfig_info = (check_imply_rule(kconf,
1062 config[CONFIG_LEN:], iconfig[CONFIG_LEN:]))
1063 if fname:
1064 add_list[fname].append(linenum)
1065
1066 if show and kconfig_info != 'skip':
Simon Glass1bd43062023-09-23 13:43:59 -06001067 print(f'{num_common:5d} : '
1068 f'{iconfig.ljust(30):-30s}{kconfig_info:-25s} {missing_str}')
Simon Glasscb008832017-06-15 21:39:33 -06001069
1070 # Having collected a list of things to add, now we add them. We process
1071 # each file from the largest line number to the smallest so that
1072 # earlier additions do not affect our line numbers. E.g. if we added an
1073 # imply at line 20 it would change the position of each line after
1074 # that.
Simon Glass793dca32019-10-31 07:42:57 -06001075 for fname, linenums in add_list.items():
Simon Glasscb008832017-06-15 21:39:33 -06001076 for linenum in sorted(linenums, reverse=True):
1077 add_imply_rule(config[CONFIG_LEN:], fname, linenum)
Simon Glass99b66602017-06-01 19:39:03 -06001078
Simon Glass941671a2022-02-08 11:49:46 -07001079def defconfig_matches(configs, re_match):
1080 """Check if any CONFIG option matches a regex
1081
1082 The match must be complete, i.e. from the start to end of the CONFIG option.
1083
1084 Args:
1085 configs (dict): Dict of CONFIG options:
1086 key: CONFIG option
1087 value: Value of option
1088 re_match (re.Pattern): Match to check
1089
1090 Returns:
1091 bool: True if any CONFIG matches the regex
1092 """
1093 for cfg in configs:
Simon Glassd9c958f2022-03-05 20:18:54 -07001094 if re_match.fullmatch(cfg):
Simon Glass941671a2022-02-08 11:49:46 -07001095 return True
1096 return False
Simon Glass99b66602017-06-01 19:39:03 -06001097
Simon Glass65d7fce2021-12-18 08:09:46 -07001098def do_find_config(config_list):
1099 """Find boards with a given combination of CONFIGs
1100
1101 Params:
Simon Glass941671a2022-02-08 11:49:46 -07001102 config_list: List of CONFIG options to check (each a regex consisting
Simon Glass65d7fce2021-12-18 08:09:46 -07001103 of a config option, with or without a CONFIG_ prefix. If an option
1104 is preceded by a tilde (~) then it must be false, otherwise it must
1105 be true)
1106 """
Simon Glass62fae4b2023-09-23 13:44:00 -06001107 _, all_defconfigs, config_db, _ = read_database()
Simon Glass65d7fce2021-12-18 08:09:46 -07001108
Simon Glass65d7fce2021-12-18 08:09:46 -07001109 # Start with all defconfigs
1110 out = all_defconfigs
1111
1112 # Work through each config in turn
Simon Glass65d7fce2021-12-18 08:09:46 -07001113 for item in config_list:
1114 # Get the real config name and whether we want this config or not
1115 cfg = item
1116 want = True
1117 if cfg[0] == '~':
1118 want = False
1119 cfg = cfg[1:]
1120
Simon Glass65d7fce2021-12-18 08:09:46 -07001121 # Search everything that is still in the running. If it has a config
1122 # that we want, or doesn't have one that we don't, add it into the
1123 # running for the next stage
1124 in_list = out
1125 out = set()
Simon Glass941671a2022-02-08 11:49:46 -07001126 re_match = re.compile(cfg)
Simon Glass65d7fce2021-12-18 08:09:46 -07001127 for defc in in_list:
Simon Glass941671a2022-02-08 11:49:46 -07001128 has_cfg = defconfig_matches(config_db[defc], re_match)
Simon Glass65d7fce2021-12-18 08:09:46 -07001129 if has_cfg == want:
1130 out.add(defc)
Tom Rini9ef3ba82022-12-04 10:14:16 -05001131 print(f'{len(out)} matches')
1132 print(' '.join(item.split('_defconfig')[0] for item in out))
Simon Glass65d7fce2021-12-18 08:09:46 -07001133
1134
1135def prefix_config(cfg):
1136 """Prefix a config with CONFIG_ if needed
1137
1138 This handles ~ operator, which indicates that the CONFIG should be disabled
1139
1140 >>> prefix_config('FRED')
1141 'CONFIG_FRED'
1142 >>> prefix_config('CONFIG_FRED')
1143 'CONFIG_FRED'
1144 >>> prefix_config('~FRED')
1145 '~CONFIG_FRED'
1146 >>> prefix_config('~CONFIG_FRED')
1147 '~CONFIG_FRED'
1148 >>> prefix_config('A123')
1149 'CONFIG_A123'
1150 """
Simon Glassa4c9d172023-09-23 13:44:01 -06001151 oper = ''
Simon Glass65d7fce2021-12-18 08:09:46 -07001152 if cfg[0] == '~':
Simon Glassa4c9d172023-09-23 13:44:01 -06001153 oper = cfg[0]
Simon Glass65d7fce2021-12-18 08:09:46 -07001154 cfg = cfg[1:]
1155 if not cfg.startswith('CONFIG_'):
1156 cfg = 'CONFIG_' + cfg
Simon Glassa4c9d172023-09-23 13:44:01 -06001157 return oper + cfg
Simon Glass65d7fce2021-12-18 08:09:46 -07001158
1159
Simon Glass98275712023-09-23 13:43:57 -06001160RE_MK_CONFIGS = re.compile(r'CONFIG_(\$\(SPL_(?:TPL_)?\))?([A-Za-z0-9_]*)')
1161RE_IFDEF = re.compile(r'(ifdef|ifndef)')
1162RE_C_CONFIGS = re.compile(r'CONFIG_([A-Za-z0-9_]*)')
1163RE_CONFIG_IS = re.compile(r'CONFIG_IS_ENABLED\(([A-Za-z0-9_]*)\)')
Simon Glass65e62032023-02-01 13:19:12 -07001164
1165class ConfigUse:
1166 def __init__(self, cfg, is_spl, fname, rest):
1167 self.cfg = cfg
1168 self.is_spl = is_spl
1169 self.fname = fname
1170 self.rest = rest
1171
1172 def __hash__(self):
1173 return hash((self.cfg, self.is_spl))
1174
1175def scan_makefiles(fnames):
1176 """Scan Makefiles looking for Kconfig options
1177
1178 Looks for uses of CONFIG options in Makefiles
1179
1180 Args:
1181 fnames (list of tuple):
1182 str: Makefile filename where the option was found
1183 str: Line of the Makefile
1184
1185 Returns:
1186 tuple:
1187 dict: all_uses
1188 key (ConfigUse): object
1189 value (list of str): matching lines
1190 dict: Uses by filename
1191 key (str): filename
1192 value (set of ConfigUse): uses in that filename
1193
1194 >>> RE_MK_CONFIGS.search('CONFIG_FRED').groups()
1195 (None, 'FRED')
1196 >>> RE_MK_CONFIGS.search('CONFIG_$(SPL_)MARY').groups()
1197 ('$(SPL_)', 'MARY')
1198 >>> RE_MK_CONFIGS.search('CONFIG_$(SPL_TPL_)MARY').groups()
1199 ('$(SPL_TPL_)', 'MARY')
1200 """
1201 all_uses = collections.defaultdict(list)
1202 fname_uses = {}
1203 for fname, rest in fnames:
1204 m_iter = RE_MK_CONFIGS.finditer(rest)
Simon Glassa4c9d172023-09-23 13:44:01 -06001205 for mat in m_iter:
1206 real_opt = mat.group(2)
Simon Glass65e62032023-02-01 13:19:12 -07001207 if real_opt == '':
1208 continue
1209 is_spl = False
Simon Glassa4c9d172023-09-23 13:44:01 -06001210 if mat.group(1):
Simon Glass65e62032023-02-01 13:19:12 -07001211 is_spl = True
1212 use = ConfigUse(real_opt, is_spl, fname, rest)
1213 if fname not in fname_uses:
1214 fname_uses[fname] = set()
1215 fname_uses[fname].add(use)
1216 all_uses[use].append(rest)
1217 return all_uses, fname_uses
1218
1219
1220def scan_src_files(fnames):
1221 """Scan source files (other than Makefiles) looking for Kconfig options
1222
1223 Looks for uses of CONFIG options
1224
1225 Args:
1226 fnames (list of tuple):
1227 str: Makefile filename where the option was found
1228 str: Line of the Makefile
1229
1230 Returns:
1231 tuple:
1232 dict: all_uses
1233 key (ConfigUse): object
1234 value (list of str): matching lines
1235 dict: Uses by filename
1236 key (str): filename
1237 value (set of ConfigUse): uses in that filename
1238
1239 >>> RE_C_CONFIGS.search('CONFIG_FRED').groups()
1240 ('FRED',)
1241 >>> RE_CONFIG_IS.search('CONFIG_IS_ENABLED(MARY)').groups()
1242 ('MARY',)
1243 >>> RE_CONFIG_IS.search('#if CONFIG_IS_ENABLED(OF_PLATDATA)').groups()
1244 ('OF_PLATDATA',)
1245 """
Simon Glass62fae4b2023-09-23 13:44:00 -06001246 fname = None
1247 rest = None
1248
Simon Glass65e62032023-02-01 13:19:12 -07001249 def add_uses(m_iter, is_spl):
Simon Glassa4c9d172023-09-23 13:44:01 -06001250 for mat in m_iter:
1251 real_opt = mat.group(1)
Simon Glass65e62032023-02-01 13:19:12 -07001252 if real_opt == '':
1253 continue
1254 use = ConfigUse(real_opt, is_spl, fname, rest)
1255 if fname not in fname_uses:
1256 fname_uses[fname] = set()
1257 fname_uses[fname].add(use)
1258 all_uses[use].append(rest)
1259
1260 all_uses = collections.defaultdict(list)
1261 fname_uses = {}
1262 for fname, rest in fnames:
1263 m_iter = RE_C_CONFIGS.finditer(rest)
1264 add_uses(m_iter, False)
1265
1266 m_iter2 = RE_CONFIG_IS.finditer(rest)
1267 add_uses(m_iter2, True)
1268
1269 return all_uses, fname_uses
1270
1271
1272MODE_NORMAL, MODE_SPL, MODE_PROPER = range(3)
1273
1274def do_scan_source(path, do_update):
1275 """Scan the source tree for Kconfig inconsistencies
1276
1277 Args:
1278 path (str): Path to source tree
1279 do_update (bool) : True to write to scripts/kconf_... files
1280 """
1281 def is_not_proper(name):
1282 for prefix in SPL_PREFIXES:
1283 if name.startswith(prefix):
1284 return name[len(prefix):]
1285 return False
1286
1287 def check_not_found(all_uses, spl_mode):
1288 """Check for Kconfig options mentioned in the source but not in Kconfig
1289
1290 Args:
1291 all_uses (dict):
1292 key (ConfigUse): object
1293 value (list of str): matching lines
1294 spl_mode (int): If MODE_SPL, look at source code which implies
1295 an SPL_ option, but for which there is none;
1296 for MOD_PROPER, look at source code which implies a Proper
1297 option (i.e. use of CONFIG_IS_ENABLED() or $(SPL_) or
1298 $(SPL_TPL_) but for which there none;
1299 if MODE_NORMAL, ignore SPL
1300
1301 Returns:
1302 dict:
1303 key (str): CONFIG name (without 'CONFIG_' prefix
1304 value (list of ConfigUse): List of uses of this CONFIG
1305 """
1306 # Make sure we know about all the options
1307 not_found = collections.defaultdict(list)
Simon Glass62fae4b2023-09-23 13:44:00 -06001308 for use, _ in all_uses.items():
Simon Glass65e62032023-02-01 13:19:12 -07001309 name = use.cfg
1310 if name in IGNORE_SYMS:
1311 continue
1312 check = True
1313
1314 if spl_mode == MODE_SPL:
1315 check = use.is_spl
1316
1317 # If it is an SPL symbol, try prepending all SPL_ prefixes to
1318 # find at least one SPL symbol
1319 if use.is_spl:
Simon Glass65e62032023-02-01 13:19:12 -07001320 for prefix in SPL_PREFIXES:
1321 try_name = prefix + name
1322 sym = kconf.syms.get(try_name)
1323 if sym:
1324 break
1325 if not sym:
1326 not_found[f'SPL_{name}'].append(use)
1327 continue
1328 elif spl_mode == MODE_PROPER:
1329 # Try to find the Proper version of this symbol, i.e. without
1330 # the SPL_ prefix
1331 proper_name = is_not_proper(name)
1332 if proper_name:
1333 name = proper_name
1334 elif not use.is_spl:
1335 check = False
1336 else: # MODE_NORMAL
Simon Glass65e62032023-02-01 13:19:12 -07001337 sym = kconf.syms.get(name)
1338 if not sym:
1339 proper_name = is_not_proper(name)
1340 if proper_name:
1341 name = proper_name
1342 sym = kconf.syms.get(name)
1343 if not sym:
1344 for prefix in SPL_PREFIXES:
1345 try_name = prefix + name
1346 sym = kconf.syms.get(try_name)
1347 if sym:
1348 break
1349 if not sym:
1350 not_found[name].append(use)
1351 continue
1352
1353 sym = kconf.syms.get(name)
1354 if not sym and check:
1355 not_found[name].append(use)
1356 return not_found
1357
1358 def show_uses(uses):
1359 """Show a list of uses along with their filename and code snippet
1360
1361 Args:
1362 uses (dict):
1363 key (str): CONFIG name (without 'CONFIG_' prefix
1364 value (list of ConfigUse): List of uses of this CONFIG
1365 """
1366 for name in sorted(uses):
1367 print(f'{name}: ', end='')
1368 for i, use in enumerate(uses[name]):
1369 print(f'{" " if i else ""}{use.fname}: {use.rest.strip()}')
1370
1371
1372 print('Scanning Kconfig')
1373 kconf = KconfigScanner().conf
1374 print(f'Scanning source in {path}')
1375 args = ['git', 'grep', '-E', r'IS_ENABLED|\bCONFIG']
1376 with subprocess.Popen(args, stdout=subprocess.PIPE) as proc:
Simon Glass62fae4b2023-09-23 13:44:00 -06001377 out, _ = proc.communicate()
Simon Glass65e62032023-02-01 13:19:12 -07001378 lines = out.splitlines()
1379 re_fname = re.compile('^([^:]*):(.*)')
1380 src_list = []
1381 mk_list = []
1382 for line in lines:
1383 linestr = line.decode('utf-8')
1384 m_fname = re_fname.search(linestr)
1385 if not m_fname:
1386 continue
1387 fname, rest = m_fname.groups()
1388 dirname, leaf = os.path.split(fname)
1389 root, ext = os.path.splitext(leaf)
1390 if ext == '.autoconf':
1391 pass
1392 elif ext in ['.c', '.h', '.S', '.lds', '.dts', '.dtsi', '.asl', '.cfg',
1393 '.env', '.tmpl']:
1394 src_list.append([fname, rest])
1395 elif 'Makefile' in root or ext == '.mk':
1396 mk_list.append([fname, rest])
1397 elif ext in ['.yml', '.sh', '.py', '.awk', '.pl', '.rst', '', '.sed']:
1398 pass
1399 elif 'Kconfig' in root or 'Kbuild' in root:
1400 pass
1401 elif 'README' in root:
1402 pass
1403 elif dirname in ['configs']:
1404 pass
1405 elif dirname.startswith('doc') or dirname.startswith('scripts/kconfig'):
1406 pass
1407 else:
1408 print(f'Not sure how to handle file {fname}')
1409
1410 # Scan the Makefiles
Simon Glass62fae4b2023-09-23 13:44:00 -06001411 all_uses, _ = scan_makefiles(mk_list)
Simon Glass65e62032023-02-01 13:19:12 -07001412
1413 spl_not_found = set()
1414 proper_not_found = set()
1415
1416 # Make sure we know about all the options
1417 print('\nCONFIG options present in Makefiles but not Kconfig:')
1418 not_found = check_not_found(all_uses, MODE_NORMAL)
1419 show_uses(not_found)
1420
1421 print('\nCONFIG options present in Makefiles but not Kconfig (SPL):')
1422 not_found = check_not_found(all_uses, MODE_SPL)
1423 show_uses(not_found)
Simon Glassb774ba52023-09-23 13:44:03 -06001424 spl_not_found |= {is_not_proper(key) or key for key in not_found.keys()}
Simon Glass65e62032023-02-01 13:19:12 -07001425
1426 print('\nCONFIG options used as Proper in Makefiles but without a non-SPL_ variant:')
1427 not_found = check_not_found(all_uses, MODE_PROPER)
1428 show_uses(not_found)
Simon Glassb774ba52023-09-23 13:44:03 -06001429 proper_not_found |= {not_found.keys()}
Simon Glass65e62032023-02-01 13:19:12 -07001430
1431 # Scan the source code
Simon Glass62fae4b2023-09-23 13:44:00 -06001432 all_uses, _ = scan_src_files(src_list)
Simon Glass65e62032023-02-01 13:19:12 -07001433
1434 # Make sure we know about all the options
1435 print('\nCONFIG options present in source but not Kconfig:')
1436 not_found = check_not_found(all_uses, MODE_NORMAL)
1437 show_uses(not_found)
1438
1439 print('\nCONFIG options present in source but not Kconfig (SPL):')
1440 not_found = check_not_found(all_uses, MODE_SPL)
1441 show_uses(not_found)
Simon Glassb774ba52023-09-23 13:44:03 -06001442 spl_not_found |= {is_not_proper(key) or key for key in not_found.keys()}
Simon Glass65e62032023-02-01 13:19:12 -07001443
1444 print('\nCONFIG options used as Proper in source but without a non-SPL_ variant:')
1445 not_found = check_not_found(all_uses, MODE_PROPER)
1446 show_uses(not_found)
Simon Glassb774ba52023-09-23 13:44:03 -06001447 proper_not_found |= {not_found.keys()}
Simon Glass65e62032023-02-01 13:19:12 -07001448
1449 print('\nCONFIG options used as SPL but without an SPL_ variant:')
1450 for item in sorted(spl_not_found):
1451 print(f' {item}')
1452
1453 print('\nCONFIG options used as Proper but without a non-SPL_ variant:')
1454 for item in sorted(proper_not_found):
1455 print(f' {item}')
1456
1457 # Write out the updated information
1458 if do_update:
Simon Glasse6c686f2023-09-23 13:44:04 -06001459 with open(os.path.join(path, 'scripts', 'conf_nospl'), 'w',
1460 encoding='utf-8') as out:
Simon Glass65e62032023-02-01 13:19:12 -07001461 print('# These options should not be enabled in SPL builds\n',
1462 file=out)
1463 for item in sorted(spl_not_found):
1464 print(item, file=out)
Simon Glasse6c686f2023-09-23 13:44:04 -06001465 with open(os.path.join(path, 'scripts', 'conf_noproper'), 'w',
1466 encoding='utf-8') as out:
Simon Glass65e62032023-02-01 13:19:12 -07001467 print('# These options should not be enabled in Proper builds\n',
1468 file=out)
1469 for item in sorted(proper_not_found):
1470 print(item, file=out)
1471
1472
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001473def main():
1474 try:
1475 cpu_count = multiprocessing.cpu_count()
1476 except NotImplementedError:
1477 cpu_count = 1
1478
Simon Glassb2e83c62021-12-18 14:54:31 -07001479 epilog = '''Move config options from headers to defconfig files. See
1480doc/develop/moveconfig.rst for documentation.'''
1481
1482 parser = ArgumentParser(epilog=epilog)
1483 # Add arguments here
1484 parser.add_argument('-a', '--add-imply', type=str, default='',
Simon Glasscb008832017-06-15 21:39:33 -06001485 help='comma-separated list of CONFIG options to add '
1486 "an 'imply' statement to for the CONFIG in -i")
Simon Glassb2e83c62021-12-18 14:54:31 -07001487 parser.add_argument('-A', '--skip-added', action='store_true', default=False,
Simon Glasscb008832017-06-15 21:39:33 -06001488 help="don't show options which are already marked as "
1489 'implying others')
Simon Glassb2e83c62021-12-18 14:54:31 -07001490 parser.add_argument('-b', '--build-db', action='store_true', default=False,
Simon Glassd73fcb12017-06-01 19:39:02 -06001491 help='build a CONFIG database')
Simon Glassb2e83c62021-12-18 14:54:31 -07001492 parser.add_argument('-c', '--color', action='store_true', default=False,
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001493 help='display the log in color')
Simon Glassb2e83c62021-12-18 14:54:31 -07001494 parser.add_argument('-C', '--commit', action='store_true', default=False,
Simon Glass9ede2122016-09-12 23:18:21 -06001495 help='Create a git commit for the operation')
Simon Glassb2e83c62021-12-18 14:54:31 -07001496 parser.add_argument('-d', '--defconfigs', type=str,
Simon Glassee4e61b2017-06-01 19:38:59 -06001497 help='a file containing a list of defconfigs to move, '
1498 "one per line (for example 'snow_defconfig') "
1499 "or '-' to read from stdin")
Simon Glassb2e83c62021-12-18 14:54:31 -07001500 parser.add_argument('-e', '--exit-on-error', action='store_true',
Simon Glasse1ae5632021-12-18 08:09:44 -07001501 default=False,
1502 help='exit immediately on any error')
Simon Glassb2e83c62021-12-18 14:54:31 -07001503 parser.add_argument('-f', '--find', action='store_true', default=False,
Simon Glass65d7fce2021-12-18 08:09:46 -07001504 help='Find boards with a given config combination')
Simon Glassb2e83c62021-12-18 14:54:31 -07001505 parser.add_argument('-i', '--imply', action='store_true', default=False,
Simon Glass99b66602017-06-01 19:39:03 -06001506 help='find options which imply others')
Simon Glassb2e83c62021-12-18 14:54:31 -07001507 parser.add_argument('-I', '--imply-flags', type=str, default='',
Simon Glass9b2a2e82017-06-15 21:39:32 -06001508 help="control the -i option ('help' for help")
Simon Glassb2e83c62021-12-18 14:54:31 -07001509 parser.add_argument('-j', '--jobs', type=int, default=cpu_count,
Simon Glasse1ae5632021-12-18 08:09:44 -07001510 help='the number of jobs to run simultaneously')
Simon Glassb2e83c62021-12-18 14:54:31 -07001511 parser.add_argument('-n', '--dry-run', action='store_true', default=False,
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001512 help='perform a trial run (show log with no changes)')
Simon Glassb2e83c62021-12-18 14:54:31 -07001513 parser.add_argument('-r', '--git-ref', type=str,
Simon Glasse1ae5632021-12-18 08:09:44 -07001514 help='the git ref to clone for building the autoconf.mk')
Simon Glassb2e83c62021-12-18 14:54:31 -07001515 parser.add_argument('-s', '--force-sync', action='store_true', default=False,
Masahiro Yamada8513dc02016-05-19 15:52:08 +09001516 help='force sync by savedefconfig')
Simon Glassb2e83c62021-12-18 14:54:31 -07001517 parser.add_argument('-S', '--spl', action='store_true', default=False,
Masahiro Yamada07913d12016-08-22 22:18:22 +09001518 help='parse config options defined for SPL build')
Simon Glass65e62032023-02-01 13:19:12 -07001519 parser.add_argument('--scan-source', action='store_true', default=False,
1520 help='scan source for uses of CONFIG options')
Simon Glassb2e83c62021-12-18 14:54:31 -07001521 parser.add_argument('-t', '--test', action='store_true', default=False,
Simon Glasse1ae5632021-12-18 08:09:44 -07001522 help='run unit tests')
Simon Glassb2e83c62021-12-18 14:54:31 -07001523 parser.add_argument('-y', '--yes', action='store_true', default=False,
Simon Glass6b403df2016-09-12 23:18:20 -06001524 help="respond 'yes' to any prompts")
Simon Glass65e62032023-02-01 13:19:12 -07001525 parser.add_argument('-u', '--update', action='store_true', default=False,
1526 help="update scripts/ files (use with --scan-source)")
Simon Glassb2e83c62021-12-18 14:54:31 -07001527 parser.add_argument('-v', '--verbose', action='store_true', default=False,
Joe Hershberger95bf9c72015-05-19 13:21:24 -05001528 help='show any build errors as boards are built')
Simon Glassb2e83c62021-12-18 14:54:31 -07001529 parser.add_argument('configs', nargs='*')
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001530
Simon Glassb2e83c62021-12-18 14:54:31 -07001531 args = parser.parse_args()
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001532
Simon Glassb2e83c62021-12-18 14:54:31 -07001533 if args.test:
Simon Glass84067a52021-12-18 08:09:45 -07001534 sys.argv = [sys.argv[0]]
Simon Glass62fae4b2023-09-23 13:44:00 -06001535 fail, _ = doctest.testmod()
Simon Glass84067a52021-12-18 08:09:45 -07001536 if fail:
1537 return 1
1538 unittest.main()
1539
Simon Glass65e62032023-02-01 13:19:12 -07001540 if args.scan_source:
1541 do_scan_source(os.getcwd(), args.update)
Simon Glassf297ba32023-09-23 13:44:05 -06001542 return 0
Simon Glass65e62032023-02-01 13:19:12 -07001543
Simon Glass882c8e42023-09-23 13:43:54 -06001544 if not any((args.force_sync, args.build_db, args.imply, args.find)):
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001545 parser.print_usage()
1546 sys.exit(1)
1547
Masahiro Yamadab6ef3932016-05-19 15:51:58 +09001548 # prefix the option name with CONFIG_ if missing
Simon Glass882c8e42023-09-23 13:43:54 -06001549 configs = [prefix_config(cfg) for cfg in args.configs]
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001550
Joe Hershberger2144f882015-05-19 13:21:20 -05001551 check_top_directory()
1552
Simon Glassb2e83c62021-12-18 14:54:31 -07001553 if args.imply:
Simon Glass9b2a2e82017-06-15 21:39:32 -06001554 imply_flags = 0
Simon Glassb2e83c62021-12-18 14:54:31 -07001555 if args.imply_flags == 'all':
Simon Glassdee36c72017-07-10 14:47:46 -06001556 imply_flags = -1
1557
Simon Glassb2e83c62021-12-18 14:54:31 -07001558 elif args.imply_flags:
1559 for flag in args.imply_flags.split(','):
Simon Glassdee36c72017-07-10 14:47:46 -06001560 bad = flag not in IMPLY_FLAGS
1561 if bad:
Simon Glass1bd43062023-09-23 13:43:59 -06001562 print(f"Invalid flag '{flag}'")
Simon Glassdee36c72017-07-10 14:47:46 -06001563 if flag == 'help' or bad:
Simon Glass793dca32019-10-31 07:42:57 -06001564 print("Imply flags: (separate with ',')")
1565 for name, info in IMPLY_FLAGS.items():
Simon Glass1bd43062023-09-23 13:43:59 -06001566 print(f' {name:-15s}: {info[1]}')
Simon Glassdee36c72017-07-10 14:47:46 -06001567 parser.print_usage()
1568 sys.exit(1)
1569 imply_flags |= IMPLY_FLAGS[flag][0]
Simon Glass9b2a2e82017-06-15 21:39:32 -06001570
Simon Glassb2e83c62021-12-18 14:54:31 -07001571 do_imply_config(configs, args.add_imply, imply_flags, args.skip_added)
Simon Glassf297ba32023-09-23 13:44:05 -06001572 return 0
Simon Glass99b66602017-06-01 19:39:03 -06001573
Simon Glassb2e83c62021-12-18 14:54:31 -07001574 if args.find:
Simon Glass65d7fce2021-12-18 08:09:46 -07001575 do_find_config(configs)
Simon Glassf297ba32023-09-23 13:44:05 -06001576 return 0
Simon Glass65d7fce2021-12-18 08:09:46 -07001577
Simon Glass3481e892023-09-23 13:43:53 -06001578 # We are either building the database or forcing a sync of defconfigs
Simon Glassd73fcb12017-06-01 19:39:02 -06001579 config_db = {}
Simon Glass793dca32019-10-31 07:42:57 -06001580 db_queue = queue.Queue()
Simon Glassa4c9d172023-09-23 13:44:01 -06001581 dbt = DatabaseThread(config_db, db_queue)
1582 dbt.daemon = True
1583 dbt.start()
Simon Glassd73fcb12017-06-01 19:39:02 -06001584
Simon Glass63df2022023-09-23 13:43:50 -06001585 check_clean_directory()
1586 bsettings.setup('')
1587 toolchains = toolchain.Toolchains()
1588 toolchains.GetSettings()
1589 toolchains.Scan(verbose=False)
Simon Glass882c8e42023-09-23 13:43:54 -06001590 move_config(toolchains, args, db_queue)
Simon Glass63df2022023-09-23 13:43:50 -06001591 db_queue.join()
Joe Hershberger2144f882015-05-19 13:21:20 -05001592
Simon Glassb2e83c62021-12-18 14:54:31 -07001593 if args.commit:
Simon Glass9ede2122016-09-12 23:18:21 -06001594 subprocess.call(['git', 'add', '-u'])
1595 if configs:
1596 msg = 'Convert %s %sto Kconfig' % (configs[0],
1597 'et al ' if len(configs) > 1 else '')
1598 msg += ('\n\nThis converts the following to Kconfig:\n %s\n' %
1599 '\n '.join(configs))
1600 else:
1601 msg = 'configs: Resync with savedefconfig'
1602 msg += '\n\nRsync all defconfig files using moveconfig.py'
1603 subprocess.call(['git', 'commit', '-s', '-m', msg])
1604
Simon Glassb2e83c62021-12-18 14:54:31 -07001605 if args.build_db:
Simon Glassa4c9d172023-09-23 13:44:01 -06001606 with open(CONFIG_DATABASE, 'w', encoding='utf-8') as outf:
Simon Glass793dca32019-10-31 07:42:57 -06001607 for defconfig, configs in config_db.items():
Simon Glassa4c9d172023-09-23 13:44:01 -06001608 outf.write(f'{defconfig}\n')
Simon Glassd73fcb12017-06-01 19:39:02 -06001609 for config in sorted(configs.keys()):
Simon Glassa4c9d172023-09-23 13:44:01 -06001610 outf.write(f' {config}={configs[config]}\n')
1611 outf.write('\n')
Simon Glassf297ba32023-09-23 13:44:05 -06001612 return 0
1613
Simon Glassd73fcb12017-06-01 19:39:02 -06001614
Masahiro Yamada5a27c732015-05-20 11:36:07 +09001615if __name__ == '__main__':
Simon Glass65d7fce2021-12-18 08:09:46 -07001616 sys.exit(main())