blob: f18d385d99515547aee153c71ab174fa2b4d01cb [file] [log] [blame]
Simon Glassff1fd6c2018-07-06 10:27:23 -06001# SPDX-License-Identifier: GPL-2.0+
2#
3# Copyright (c) 2016 Google, Inc
4#
5
Simon Glassc3f94542018-07-06 10:27:34 -06006from contextlib import contextmanager
Simon Glass1d0f30e2022-01-22 05:07:28 -07007import doctest
Simon Glassff1fd6c2018-07-06 10:27:23 -06008import glob
Simon Glassce0dc2e2020-04-17 18:09:01 -06009import multiprocessing
Simon Glassff1fd6c2018-07-06 10:27:23 -060010import os
11import sys
Simon Glassce0dc2e2020-04-17 18:09:01 -060012import unittest
Simon Glassff1fd6c2018-07-06 10:27:23 -060013
Simon Glass4583c002023-02-23 18:18:04 -070014from u_boot_pylib import command
Simon Glassff1fd6c2018-07-06 10:27:23 -060015
Simon Glassc3a13cc2020-04-17 18:08:55 -060016from io import StringIO
Simon Glassc3f94542018-07-06 10:27:34 -060017
Simon Glassce0dc2e2020-04-17 18:09:01 -060018use_concurrent = True
19try:
Simon Glass00290d62023-02-23 18:18:03 -070020 from concurrencytest import ConcurrentTestSuite
21 from concurrencytest import fork_for_tests
Simon Glassce0dc2e2020-04-17 18:09:01 -060022except:
23 use_concurrent = False
24
Simon Glassc3f94542018-07-06 10:27:34 -060025
Simon Glass5e2ab402022-01-29 14:14:14 -070026def run_test_coverage(prog, filter_fname, exclude_list, build_dir, required=None,
Simon Glass407a1412023-07-19 17:49:31 -060027 extra_args=None, single_thread='-P1'):
Simon Glassff1fd6c2018-07-06 10:27:23 -060028 """Run tests and check that we get 100% coverage
29
30 Args:
31 prog: Program to run (with be passed a '-t' argument to run tests
32 filter_fname: Normally all *.py files in the program's directory will
33 be included. If this is not None, then it is used to filter the
34 list so that only filenames that don't contain filter_fname are
35 included.
36 exclude_list: List of file patterns to exclude from the coverage
37 calculation
38 build_dir: Build directory, used to locate libfdt.py
39 required: List of modules which must be in the coverage report
Simon Glass32eb66d2020-07-09 18:39:29 -060040 extra_args (str): Extra arguments to pass to the tool before the -t/test
41 arg
Simon Glass407a1412023-07-19 17:49:31 -060042 single_thread (str): Argument string to make the tests run
43 single-threaded. This is necessary to get proper coverage results.
44 The default is '-P0'
Simon Glassff1fd6c2018-07-06 10:27:23 -060045
46 Raises:
47 ValueError if the code coverage is not 100%
48 """
49 # This uses the build output from sandbox_spl to get _libfdt.so
50 path = os.path.dirname(prog)
51 if filter_fname:
52 glob_list = glob.glob(os.path.join(path, '*.py'))
53 glob_list = [fname for fname in glob_list if filter_fname in fname]
54 else:
55 glob_list = []
56 glob_list += exclude_list
Simon Glass9550f9a2019-05-17 22:00:54 -060057 glob_list += ['*libfdt.py', '*site-packages*', '*dist-packages*']
Simon Glass347e0f02020-07-09 18:39:34 -060058 glob_list += ['*concurrencytest*']
Simon Glass6bb74de2020-07-05 21:41:55 -060059 test_cmd = 'test' if 'binman' in prog or 'patman' in prog else '-t'
Simon Glass428e7732020-04-17 18:09:00 -060060 prefix = ''
61 if build_dir:
62 prefix = 'PYTHONPATH=$PYTHONPATH:%s/sandbox_spl/tools ' % build_dir
63 cmd = ('%spython3-coverage run '
Simon Glass407a1412023-07-19 17:49:31 -060064 '--omit "%s" %s %s %s %s' % (prefix, ','.join(glob_list),
65 prog, extra_args or '', test_cmd,
66 single_thread or '-P1'))
Simon Glassff1fd6c2018-07-06 10:27:23 -060067 os.system(cmd)
Simon Glassd9800692022-01-29 14:14:05 -070068 stdout = command.output('python3-coverage', 'report')
Simon Glassff1fd6c2018-07-06 10:27:23 -060069 lines = stdout.splitlines()
70 if required:
71 # Convert '/path/to/name.py' just the module name 'name'
72 test_set = set([os.path.splitext(os.path.basename(line.split()[0]))[0]
73 for line in lines if '/etype/' in line])
74 missing_list = required
Simon Glasse4304402019-07-08 14:25:32 -060075 missing_list.discard('__init__')
Simon Glassff1fd6c2018-07-06 10:27:23 -060076 missing_list.difference_update(test_set)
77 if missing_list:
Simon Glass5a1af1d2019-05-14 15:53:36 -060078 print('Missing tests for %s' % (', '.join(missing_list)))
79 print(stdout)
Simon Glassff1fd6c2018-07-06 10:27:23 -060080 ok = False
81
82 coverage = lines[-1].split(' ')[-1]
83 ok = True
Simon Glass5a1af1d2019-05-14 15:53:36 -060084 print(coverage)
Simon Glassff1fd6c2018-07-06 10:27:23 -060085 if coverage != '100%':
Simon Glass5a1af1d2019-05-14 15:53:36 -060086 print(stdout)
Simon Glass8816eda2022-08-13 11:40:41 -060087 print("To get a report in 'htmlcov/index.html', type: python3-coverage html")
Simon Glass5a1af1d2019-05-14 15:53:36 -060088 print('Coverage error: %s, but should be 100%%' % coverage)
Simon Glassff1fd6c2018-07-06 10:27:23 -060089 ok = False
90 if not ok:
91 raise ValueError('Test coverage failure')
Simon Glassc3f94542018-07-06 10:27:34 -060092
93
94# Use this to suppress stdout/stderr output:
95# with capture_sys_output() as (stdout, stderr)
96# ...do something...
97@contextmanager
98def capture_sys_output():
99 capture_out, capture_err = StringIO(), StringIO()
100 old_out, old_err = sys.stdout, sys.stderr
101 try:
102 sys.stdout, sys.stderr = capture_out, capture_err
103 yield capture_out, capture_err
104 finally:
105 sys.stdout, sys.stderr = old_out, old_err
Simon Glassce0dc2e2020-04-17 18:09:01 -0600106
107
Alper Nebi Yasakdd6b92b2022-04-02 20:06:07 +0300108class FullTextTestResult(unittest.TextTestResult):
109 """A test result class that can print extended text results to a stream
110
111 This is meant to be used by a TestRunner as a result class. Like
112 TextTestResult, this prints out the names of tests as they are run,
113 errors as they occur, and a summary of the results at the end of the
114 test run. Beyond those, this prints information about skipped tests,
115 expected failures and unexpected successes.
116
117 Args:
118 stream: A file-like object to write results to
119 descriptions (bool): True to print descriptions with test names
120 verbosity (int): Detail of printed output per test as they run
121 Test stdout and stderr always get printed when buffering
122 them is disabled by the test runner. In addition to that,
123 0: Print nothing
124 1: Print a dot per test
125 2: Print test names
126 """
127 def __init__(self, stream, descriptions, verbosity):
128 self.verbosity = verbosity
129 super().__init__(stream, descriptions, verbosity)
130
131 def printErrors(self):
132 "Called by TestRunner after test run to summarize the tests"
133 # The parent class doesn't keep unexpected successes in the same
134 # format as the rest. Adapt it to what printErrorList expects.
135 unexpected_successes = [
136 (test, 'Test was expected to fail, but succeeded.\n')
137 for test in self.unexpectedSuccesses
138 ]
139
140 super().printErrors() # FAIL and ERROR
141 self.printErrorList('SKIP', self.skipped)
142 self.printErrorList('XFAIL', self.expectedFailures)
143 self.printErrorList('XPASS', unexpected_successes)
144
145 def addSkip(self, test, reason):
146 """Called when a test is skipped."""
147 # Add empty line to keep spacing consistent with other results
148 if not reason.endswith('\n'):
149 reason += '\n'
150 super().addSkip(test, reason)
151
152
Alper Nebi Yasakd8318fe2022-04-02 20:06:06 +0300153def run_test_suites(toolname, debug, verbosity, test_preserve_dirs, processes,
Simon Glass5e2ab402022-01-29 14:14:14 -0700154 test_name, toolpath, class_and_module_list):
Simon Glassce0dc2e2020-04-17 18:09:01 -0600155 """Run a series of test suites and collect the results
156
157 Args:
Alper Nebi Yasakd8318fe2022-04-02 20:06:06 +0300158 toolname: Name of the tool that ran the tests
Simon Glassce0dc2e2020-04-17 18:09:01 -0600159 debug: True to enable debugging, which shows a full stack trace on error
160 verbosity: Verbosity level to use (0-4)
161 test_preserve_dirs: True to preserve the input directory used by tests
162 so that it can be examined afterwards (only useful for debugging
163 tests). If a single test is selected (in args[0]) it also preserves
164 the output directory for this test. Both directories are displayed
165 on the command line.
166 processes: Number of processes to use to run tests (None=same as #CPUs)
167 test_name: Name of test to run, or None for all
168 toolpath: List of paths to use for tools
Simon Glass1d0f30e2022-01-22 05:07:28 -0700169 class_and_module_list: List of test classes (type class) and module
170 names (type str) to run
Simon Glassce0dc2e2020-04-17 18:09:01 -0600171 """
Simon Glassce0dc2e2020-04-17 18:09:01 -0600172 sys.argv = [sys.argv[0]]
173 if debug:
174 sys.argv.append('-D')
175 if verbosity:
176 sys.argv.append('-v%d' % verbosity)
177 if toolpath:
178 for path in toolpath:
179 sys.argv += ['--toolpath', path]
180
181 suite = unittest.TestSuite()
182 loader = unittest.TestLoader()
Alper Nebi Yasakd8318fe2022-04-02 20:06:06 +0300183 runner = unittest.TextTestRunner(
184 stream=sys.stdout,
185 verbosity=(1 if verbosity is None else verbosity),
Alper Nebi Yasakdd6b92b2022-04-02 20:06:07 +0300186 resultclass=FullTextTestResult,
Alper Nebi Yasakd8318fe2022-04-02 20:06:06 +0300187 )
188
189 if use_concurrent and processes != 1:
190 suite = ConcurrentTestSuite(suite,
Simon Glass6811fea2023-02-23 18:18:02 -0700191 fork_for_tests(processes or multiprocessing.cpu_count()))
Alper Nebi Yasakd8318fe2022-04-02 20:06:06 +0300192
193 for module in class_and_module_list:
194 if isinstance(module, str) and (not test_name or test_name == module):
195 suite.addTests(doctest.DocTestSuite(module))
196
Simon Glass1d0f30e2022-01-22 05:07:28 -0700197 for module in class_and_module_list:
198 if isinstance(module, str):
199 continue
Simon Glassce0dc2e2020-04-17 18:09:01 -0600200 # Test the test module about our arguments, if it is interested
201 if hasattr(module, 'setup_test_args'):
202 setup_test_args = getattr(module, 'setup_test_args')
203 setup_test_args(preserve_indir=test_preserve_dirs,
204 preserve_outdirs=test_preserve_dirs and test_name is not None,
205 toolpath=toolpath, verbosity=verbosity)
206 if test_name:
Alper Nebi Yasakce12c472022-04-02 20:06:05 +0300207 # Since Python v3.5 If an ImportError or AttributeError occurs
208 # while traversing a name then a synthetic test that raises that
209 # error when run will be returned. Check that the requested test
210 # exists, otherwise these errors are included in the results.
211 if test_name in loader.getTestCaseNames(module):
Simon Glassce0dc2e2020-04-17 18:09:01 -0600212 suite.addTests(loader.loadTestsFromName(test_name, module))
Simon Glassce0dc2e2020-04-17 18:09:01 -0600213 else:
214 suite.addTests(loader.loadTestsFromTestCase(module))
Alper Nebi Yasakd8318fe2022-04-02 20:06:06 +0300215
216 print(f" Running {toolname} tests ".center(70, "="))
217 result = runner.run(suite)
218 print()
219
220 return result