Simon Glass | ff1fd6c | 2018-07-06 10:27:23 -0600 | [diff] [blame] | 1 | # SPDX-License-Identifier: GPL-2.0+ |
| 2 | # |
| 3 | # Copyright (c) 2016 Google, Inc |
| 4 | # |
| 5 | |
Simon Glass | c3f9454 | 2018-07-06 10:27:34 -0600 | [diff] [blame] | 6 | from contextlib import contextmanager |
Simon Glass | 1d0f30e | 2022-01-22 05:07:28 -0700 | [diff] [blame] | 7 | import doctest |
Simon Glass | ff1fd6c | 2018-07-06 10:27:23 -0600 | [diff] [blame] | 8 | import glob |
Simon Glass | ce0dc2e | 2020-04-17 18:09:01 -0600 | [diff] [blame] | 9 | import multiprocessing |
Simon Glass | ff1fd6c | 2018-07-06 10:27:23 -0600 | [diff] [blame] | 10 | import os |
| 11 | import sys |
Simon Glass | ce0dc2e | 2020-04-17 18:09:01 -0600 | [diff] [blame] | 12 | import unittest |
Simon Glass | ff1fd6c | 2018-07-06 10:27:23 -0600 | [diff] [blame] | 13 | |
Simon Glass | 4583c00 | 2023-02-23 18:18:04 -0700 | [diff] [blame] | 14 | from u_boot_pylib import command |
Simon Glass | ff1fd6c | 2018-07-06 10:27:23 -0600 | [diff] [blame] | 15 | |
Simon Glass | c3a13cc | 2020-04-17 18:08:55 -0600 | [diff] [blame] | 16 | from io import StringIO |
Simon Glass | c3f9454 | 2018-07-06 10:27:34 -0600 | [diff] [blame] | 17 | |
Simon Glass | ce0dc2e | 2020-04-17 18:09:01 -0600 | [diff] [blame] | 18 | use_concurrent = True |
| 19 | try: |
Simon Glass | 00290d6 | 2023-02-23 18:18:03 -0700 | [diff] [blame] | 20 | from concurrencytest import ConcurrentTestSuite |
| 21 | from concurrencytest import fork_for_tests |
Simon Glass | ce0dc2e | 2020-04-17 18:09:01 -0600 | [diff] [blame] | 22 | except: |
| 23 | use_concurrent = False |
| 24 | |
Simon Glass | c3f9454 | 2018-07-06 10:27:34 -0600 | [diff] [blame] | 25 | |
Simon Glass | 5e2ab40 | 2022-01-29 14:14:14 -0700 | [diff] [blame] | 26 | def run_test_coverage(prog, filter_fname, exclude_list, build_dir, required=None, |
Simon Glass | 407a141 | 2023-07-19 17:49:31 -0600 | [diff] [blame] | 27 | extra_args=None, single_thread='-P1'): |
Simon Glass | ff1fd6c | 2018-07-06 10:27:23 -0600 | [diff] [blame] | 28 | """Run tests and check that we get 100% coverage |
| 29 | |
| 30 | Args: |
| 31 | prog: Program to run (with be passed a '-t' argument to run tests |
| 32 | filter_fname: Normally all *.py files in the program's directory will |
| 33 | be included. If this is not None, then it is used to filter the |
| 34 | list so that only filenames that don't contain filter_fname are |
| 35 | included. |
| 36 | exclude_list: List of file patterns to exclude from the coverage |
| 37 | calculation |
| 38 | build_dir: Build directory, used to locate libfdt.py |
| 39 | required: List of modules which must be in the coverage report |
Simon Glass | 32eb66d | 2020-07-09 18:39:29 -0600 | [diff] [blame] | 40 | extra_args (str): Extra arguments to pass to the tool before the -t/test |
| 41 | arg |
Simon Glass | 407a141 | 2023-07-19 17:49:31 -0600 | [diff] [blame] | 42 | single_thread (str): Argument string to make the tests run |
| 43 | single-threaded. This is necessary to get proper coverage results. |
| 44 | The default is '-P0' |
Simon Glass | ff1fd6c | 2018-07-06 10:27:23 -0600 | [diff] [blame] | 45 | |
| 46 | Raises: |
| 47 | ValueError if the code coverage is not 100% |
| 48 | """ |
| 49 | # This uses the build output from sandbox_spl to get _libfdt.so |
| 50 | path = os.path.dirname(prog) |
| 51 | if filter_fname: |
| 52 | glob_list = glob.glob(os.path.join(path, '*.py')) |
| 53 | glob_list = [fname for fname in glob_list if filter_fname in fname] |
| 54 | else: |
| 55 | glob_list = [] |
| 56 | glob_list += exclude_list |
Simon Glass | 9550f9a | 2019-05-17 22:00:54 -0600 | [diff] [blame] | 57 | glob_list += ['*libfdt.py', '*site-packages*', '*dist-packages*'] |
Simon Glass | 347e0f0 | 2020-07-09 18:39:34 -0600 | [diff] [blame] | 58 | glob_list += ['*concurrencytest*'] |
Simon Glass | 6bb74de | 2020-07-05 21:41:55 -0600 | [diff] [blame] | 59 | test_cmd = 'test' if 'binman' in prog or 'patman' in prog else '-t' |
Simon Glass | 428e773 | 2020-04-17 18:09:00 -0600 | [diff] [blame] | 60 | prefix = '' |
| 61 | if build_dir: |
| 62 | prefix = 'PYTHONPATH=$PYTHONPATH:%s/sandbox_spl/tools ' % build_dir |
| 63 | cmd = ('%spython3-coverage run ' |
Simon Glass | 407a141 | 2023-07-19 17:49:31 -0600 | [diff] [blame] | 64 | '--omit "%s" %s %s %s %s' % (prefix, ','.join(glob_list), |
| 65 | prog, extra_args or '', test_cmd, |
| 66 | single_thread or '-P1')) |
Simon Glass | ff1fd6c | 2018-07-06 10:27:23 -0600 | [diff] [blame] | 67 | os.system(cmd) |
Simon Glass | d980069 | 2022-01-29 14:14:05 -0700 | [diff] [blame] | 68 | stdout = command.output('python3-coverage', 'report') |
Simon Glass | ff1fd6c | 2018-07-06 10:27:23 -0600 | [diff] [blame] | 69 | lines = stdout.splitlines() |
| 70 | if required: |
| 71 | # Convert '/path/to/name.py' just the module name 'name' |
| 72 | test_set = set([os.path.splitext(os.path.basename(line.split()[0]))[0] |
| 73 | for line in lines if '/etype/' in line]) |
| 74 | missing_list = required |
Simon Glass | e430440 | 2019-07-08 14:25:32 -0600 | [diff] [blame] | 75 | missing_list.discard('__init__') |
Simon Glass | ff1fd6c | 2018-07-06 10:27:23 -0600 | [diff] [blame] | 76 | missing_list.difference_update(test_set) |
| 77 | if missing_list: |
Simon Glass | 5a1af1d | 2019-05-14 15:53:36 -0600 | [diff] [blame] | 78 | print('Missing tests for %s' % (', '.join(missing_list))) |
| 79 | print(stdout) |
Simon Glass | ff1fd6c | 2018-07-06 10:27:23 -0600 | [diff] [blame] | 80 | ok = False |
| 81 | |
| 82 | coverage = lines[-1].split(' ')[-1] |
| 83 | ok = True |
Simon Glass | 5a1af1d | 2019-05-14 15:53:36 -0600 | [diff] [blame] | 84 | print(coverage) |
Simon Glass | ff1fd6c | 2018-07-06 10:27:23 -0600 | [diff] [blame] | 85 | if coverage != '100%': |
Simon Glass | 5a1af1d | 2019-05-14 15:53:36 -0600 | [diff] [blame] | 86 | print(stdout) |
Simon Glass | 8816eda | 2022-08-13 11:40:41 -0600 | [diff] [blame] | 87 | print("To get a report in 'htmlcov/index.html', type: python3-coverage html") |
Simon Glass | 5a1af1d | 2019-05-14 15:53:36 -0600 | [diff] [blame] | 88 | print('Coverage error: %s, but should be 100%%' % coverage) |
Simon Glass | ff1fd6c | 2018-07-06 10:27:23 -0600 | [diff] [blame] | 89 | ok = False |
| 90 | if not ok: |
| 91 | raise ValueError('Test coverage failure') |
Simon Glass | c3f9454 | 2018-07-06 10:27:34 -0600 | [diff] [blame] | 92 | |
| 93 | |
| 94 | # Use this to suppress stdout/stderr output: |
| 95 | # with capture_sys_output() as (stdout, stderr) |
| 96 | # ...do something... |
| 97 | @contextmanager |
| 98 | def capture_sys_output(): |
| 99 | capture_out, capture_err = StringIO(), StringIO() |
| 100 | old_out, old_err = sys.stdout, sys.stderr |
| 101 | try: |
| 102 | sys.stdout, sys.stderr = capture_out, capture_err |
| 103 | yield capture_out, capture_err |
| 104 | finally: |
| 105 | sys.stdout, sys.stderr = old_out, old_err |
Simon Glass | ce0dc2e | 2020-04-17 18:09:01 -0600 | [diff] [blame] | 106 | |
| 107 | |
Alper Nebi Yasak | dd6b92b | 2022-04-02 20:06:07 +0300 | [diff] [blame] | 108 | class FullTextTestResult(unittest.TextTestResult): |
| 109 | """A test result class that can print extended text results to a stream |
| 110 | |
| 111 | This is meant to be used by a TestRunner as a result class. Like |
| 112 | TextTestResult, this prints out the names of tests as they are run, |
| 113 | errors as they occur, and a summary of the results at the end of the |
| 114 | test run. Beyond those, this prints information about skipped tests, |
| 115 | expected failures and unexpected successes. |
| 116 | |
| 117 | Args: |
| 118 | stream: A file-like object to write results to |
| 119 | descriptions (bool): True to print descriptions with test names |
| 120 | verbosity (int): Detail of printed output per test as they run |
| 121 | Test stdout and stderr always get printed when buffering |
| 122 | them is disabled by the test runner. In addition to that, |
| 123 | 0: Print nothing |
| 124 | 1: Print a dot per test |
| 125 | 2: Print test names |
| 126 | """ |
| 127 | def __init__(self, stream, descriptions, verbosity): |
| 128 | self.verbosity = verbosity |
| 129 | super().__init__(stream, descriptions, verbosity) |
| 130 | |
| 131 | def printErrors(self): |
| 132 | "Called by TestRunner after test run to summarize the tests" |
| 133 | # The parent class doesn't keep unexpected successes in the same |
| 134 | # format as the rest. Adapt it to what printErrorList expects. |
| 135 | unexpected_successes = [ |
| 136 | (test, 'Test was expected to fail, but succeeded.\n') |
| 137 | for test in self.unexpectedSuccesses |
| 138 | ] |
| 139 | |
| 140 | super().printErrors() # FAIL and ERROR |
| 141 | self.printErrorList('SKIP', self.skipped) |
| 142 | self.printErrorList('XFAIL', self.expectedFailures) |
| 143 | self.printErrorList('XPASS', unexpected_successes) |
| 144 | |
| 145 | def addSkip(self, test, reason): |
| 146 | """Called when a test is skipped.""" |
| 147 | # Add empty line to keep spacing consistent with other results |
| 148 | if not reason.endswith('\n'): |
| 149 | reason += '\n' |
| 150 | super().addSkip(test, reason) |
| 151 | |
| 152 | |
Alper Nebi Yasak | d8318fe | 2022-04-02 20:06:06 +0300 | [diff] [blame] | 153 | def run_test_suites(toolname, debug, verbosity, test_preserve_dirs, processes, |
Simon Glass | 5e2ab40 | 2022-01-29 14:14:14 -0700 | [diff] [blame] | 154 | test_name, toolpath, class_and_module_list): |
Simon Glass | ce0dc2e | 2020-04-17 18:09:01 -0600 | [diff] [blame] | 155 | """Run a series of test suites and collect the results |
| 156 | |
| 157 | Args: |
Alper Nebi Yasak | d8318fe | 2022-04-02 20:06:06 +0300 | [diff] [blame] | 158 | toolname: Name of the tool that ran the tests |
Simon Glass | ce0dc2e | 2020-04-17 18:09:01 -0600 | [diff] [blame] | 159 | debug: True to enable debugging, which shows a full stack trace on error |
| 160 | verbosity: Verbosity level to use (0-4) |
| 161 | test_preserve_dirs: True to preserve the input directory used by tests |
| 162 | so that it can be examined afterwards (only useful for debugging |
| 163 | tests). If a single test is selected (in args[0]) it also preserves |
| 164 | the output directory for this test. Both directories are displayed |
| 165 | on the command line. |
| 166 | processes: Number of processes to use to run tests (None=same as #CPUs) |
| 167 | test_name: Name of test to run, or None for all |
| 168 | toolpath: List of paths to use for tools |
Simon Glass | 1d0f30e | 2022-01-22 05:07:28 -0700 | [diff] [blame] | 169 | class_and_module_list: List of test classes (type class) and module |
| 170 | names (type str) to run |
Simon Glass | ce0dc2e | 2020-04-17 18:09:01 -0600 | [diff] [blame] | 171 | """ |
Simon Glass | ce0dc2e | 2020-04-17 18:09:01 -0600 | [diff] [blame] | 172 | sys.argv = [sys.argv[0]] |
| 173 | if debug: |
| 174 | sys.argv.append('-D') |
| 175 | if verbosity: |
| 176 | sys.argv.append('-v%d' % verbosity) |
| 177 | if toolpath: |
| 178 | for path in toolpath: |
| 179 | sys.argv += ['--toolpath', path] |
| 180 | |
| 181 | suite = unittest.TestSuite() |
| 182 | loader = unittest.TestLoader() |
Alper Nebi Yasak | d8318fe | 2022-04-02 20:06:06 +0300 | [diff] [blame] | 183 | runner = unittest.TextTestRunner( |
| 184 | stream=sys.stdout, |
| 185 | verbosity=(1 if verbosity is None else verbosity), |
Alper Nebi Yasak | dd6b92b | 2022-04-02 20:06:07 +0300 | [diff] [blame] | 186 | resultclass=FullTextTestResult, |
Alper Nebi Yasak | d8318fe | 2022-04-02 20:06:06 +0300 | [diff] [blame] | 187 | ) |
| 188 | |
| 189 | if use_concurrent and processes != 1: |
| 190 | suite = ConcurrentTestSuite(suite, |
Simon Glass | 6811fea | 2023-02-23 18:18:02 -0700 | [diff] [blame] | 191 | fork_for_tests(processes or multiprocessing.cpu_count())) |
Alper Nebi Yasak | d8318fe | 2022-04-02 20:06:06 +0300 | [diff] [blame] | 192 | |
| 193 | for module in class_and_module_list: |
| 194 | if isinstance(module, str) and (not test_name or test_name == module): |
| 195 | suite.addTests(doctest.DocTestSuite(module)) |
| 196 | |
Simon Glass | 1d0f30e | 2022-01-22 05:07:28 -0700 | [diff] [blame] | 197 | for module in class_and_module_list: |
| 198 | if isinstance(module, str): |
| 199 | continue |
Simon Glass | ce0dc2e | 2020-04-17 18:09:01 -0600 | [diff] [blame] | 200 | # Test the test module about our arguments, if it is interested |
| 201 | if hasattr(module, 'setup_test_args'): |
| 202 | setup_test_args = getattr(module, 'setup_test_args') |
| 203 | setup_test_args(preserve_indir=test_preserve_dirs, |
| 204 | preserve_outdirs=test_preserve_dirs and test_name is not None, |
| 205 | toolpath=toolpath, verbosity=verbosity) |
| 206 | if test_name: |
Alper Nebi Yasak | ce12c47 | 2022-04-02 20:06:05 +0300 | [diff] [blame] | 207 | # Since Python v3.5 If an ImportError or AttributeError occurs |
| 208 | # while traversing a name then a synthetic test that raises that |
| 209 | # error when run will be returned. Check that the requested test |
| 210 | # exists, otherwise these errors are included in the results. |
| 211 | if test_name in loader.getTestCaseNames(module): |
Simon Glass | ce0dc2e | 2020-04-17 18:09:01 -0600 | [diff] [blame] | 212 | suite.addTests(loader.loadTestsFromName(test_name, module)) |
Simon Glass | ce0dc2e | 2020-04-17 18:09:01 -0600 | [diff] [blame] | 213 | else: |
| 214 | suite.addTests(loader.loadTestsFromTestCase(module)) |
Alper Nebi Yasak | d8318fe | 2022-04-02 20:06:06 +0300 | [diff] [blame] | 215 | |
| 216 | print(f" Running {toolname} tests ".center(70, "=")) |
| 217 | result = runner.run(suite) |
| 218 | print() |
| 219 | |
| 220 | return result |