blob: 65e1d75626cf82105b11130458ac9b264e0ffaa1 [file] [log] [blame]
Stephen Warrend2015062016-01-15 11:15:24 -07001# Copyright (c) 2015 Stephen Warren
2# Copyright (c) 2015-2016, NVIDIA CORPORATION. All rights reserved.
3#
4# SPDX-License-Identifier: GPL-2.0
5
6# Implementation of pytest run-time hook functions. These are invoked by
7# pytest at certain points during operation, e.g. startup, for each executed
8# test, at shutdown etc. These hooks perform functions such as:
9# - Parsing custom command-line options.
10# - Pullilng in user-specified board configuration.
11# - Creating the U-Boot console test fixture.
12# - Creating the HTML log file.
13# - Monitoring each test's results.
14# - Implementing custom pytest markers.
15
16import atexit
17import errno
18import os
19import os.path
Stephen Warrend2015062016-01-15 11:15:24 -070020import pytest
21from _pytest.runner import runtestprotocol
22import ConfigParser
Stephen Warren1cd85f52016-02-08 14:44:16 -070023import re
Stephen Warrend2015062016-01-15 11:15:24 -070024import StringIO
25import sys
26
27# Globals: The HTML log file, and the connection to the U-Boot console.
28log = None
29console = None
30
31def mkdir_p(path):
Stephen Warrene8debf32016-01-26 13:41:30 -070032 """Create a directory path.
Stephen Warrend2015062016-01-15 11:15:24 -070033
34 This includes creating any intermediate/parent directories. Any errors
35 caused due to already extant directories are ignored.
36
37 Args:
38 path: The directory path to create.
39
40 Returns:
41 Nothing.
Stephen Warrene8debf32016-01-26 13:41:30 -070042 """
Stephen Warrend2015062016-01-15 11:15:24 -070043
44 try:
45 os.makedirs(path)
46 except OSError as exc:
47 if exc.errno == errno.EEXIST and os.path.isdir(path):
48 pass
49 else:
50 raise
51
52def pytest_addoption(parser):
Stephen Warrene8debf32016-01-26 13:41:30 -070053 """pytest hook: Add custom command-line options to the cmdline parser.
Stephen Warrend2015062016-01-15 11:15:24 -070054
55 Args:
56 parser: The pytest command-line parser.
57
58 Returns:
59 Nothing.
Stephen Warrene8debf32016-01-26 13:41:30 -070060 """
Stephen Warrend2015062016-01-15 11:15:24 -070061
62 parser.addoption('--build-dir', default=None,
63 help='U-Boot build directory (O=)')
64 parser.addoption('--result-dir', default=None,
65 help='U-Boot test result/tmp directory')
66 parser.addoption('--persistent-data-dir', default=None,
67 help='U-Boot test persistent generated data directory')
68 parser.addoption('--board-type', '--bd', '-B', default='sandbox',
69 help='U-Boot board type')
70 parser.addoption('--board-identity', '--id', default='na',
71 help='U-Boot board identity/instance')
72 parser.addoption('--build', default=False, action='store_true',
73 help='Compile U-Boot before running tests')
Stephen Warren89ab8412016-02-04 16:11:50 -070074 parser.addoption('--gdbserver', default=None,
75 help='Run sandbox under gdbserver. The argument is the channel '+
76 'over which gdbserver should communicate, e.g. localhost:1234')
Stephen Warrend2015062016-01-15 11:15:24 -070077
78def pytest_configure(config):
Stephen Warrene8debf32016-01-26 13:41:30 -070079 """pytest hook: Perform custom initialization at startup time.
Stephen Warrend2015062016-01-15 11:15:24 -070080
81 Args:
82 config: The pytest configuration.
83
84 Returns:
85 Nothing.
Stephen Warrene8debf32016-01-26 13:41:30 -070086 """
Stephen Warrend2015062016-01-15 11:15:24 -070087
88 global log
89 global console
90 global ubconfig
91
92 test_py_dir = os.path.dirname(os.path.abspath(__file__))
93 source_dir = os.path.dirname(os.path.dirname(test_py_dir))
94
95 board_type = config.getoption('board_type')
96 board_type_filename = board_type.replace('-', '_')
97
98 board_identity = config.getoption('board_identity')
99 board_identity_filename = board_identity.replace('-', '_')
100
101 build_dir = config.getoption('build_dir')
102 if not build_dir:
103 build_dir = source_dir + '/build-' + board_type
104 mkdir_p(build_dir)
105
106 result_dir = config.getoption('result_dir')
107 if not result_dir:
108 result_dir = build_dir
109 mkdir_p(result_dir)
110
111 persistent_data_dir = config.getoption('persistent_data_dir')
112 if not persistent_data_dir:
113 persistent_data_dir = build_dir + '/persistent-data'
114 mkdir_p(persistent_data_dir)
115
Stephen Warren89ab8412016-02-04 16:11:50 -0700116 gdbserver = config.getoption('gdbserver')
117 if gdbserver and board_type != 'sandbox':
118 raise Exception('--gdbserver only supported with sandbox')
119
Stephen Warrend2015062016-01-15 11:15:24 -0700120 import multiplexed_log
121 log = multiplexed_log.Logfile(result_dir + '/test-log.html')
122
123 if config.getoption('build'):
124 if build_dir != source_dir:
125 o_opt = 'O=%s' % build_dir
126 else:
127 o_opt = ''
128 cmds = (
129 ['make', o_opt, '-s', board_type + '_defconfig'],
130 ['make', o_opt, '-s', '-j8'],
131 )
Stephen Warren83357fd2016-02-03 16:46:34 -0700132 with log.section('make'):
133 runner = log.get_runner('make', sys.stdout)
134 for cmd in cmds:
135 runner.run(cmd, cwd=source_dir)
136 runner.close()
137 log.status_pass('OK')
Stephen Warrend2015062016-01-15 11:15:24 -0700138
139 class ArbitraryAttributeContainer(object):
140 pass
141
142 ubconfig = ArbitraryAttributeContainer()
143 ubconfig.brd = dict()
144 ubconfig.env = dict()
145
146 modules = [
147 (ubconfig.brd, 'u_boot_board_' + board_type_filename),
148 (ubconfig.env, 'u_boot_boardenv_' + board_type_filename),
149 (ubconfig.env, 'u_boot_boardenv_' + board_type_filename + '_' +
150 board_identity_filename),
151 ]
152 for (dict_to_fill, module_name) in modules:
153 try:
154 module = __import__(module_name)
155 except ImportError:
156 continue
157 dict_to_fill.update(module.__dict__)
158
159 ubconfig.buildconfig = dict()
160
161 for conf_file in ('.config', 'include/autoconf.mk'):
162 dot_config = build_dir + '/' + conf_file
163 if not os.path.exists(dot_config):
164 raise Exception(conf_file + ' does not exist; ' +
165 'try passing --build option?')
166
167 with open(dot_config, 'rt') as f:
168 ini_str = '[root]\n' + f.read()
169 ini_sio = StringIO.StringIO(ini_str)
170 parser = ConfigParser.RawConfigParser()
171 parser.readfp(ini_sio)
172 ubconfig.buildconfig.update(parser.items('root'))
173
174 ubconfig.test_py_dir = test_py_dir
175 ubconfig.source_dir = source_dir
176 ubconfig.build_dir = build_dir
177 ubconfig.result_dir = result_dir
178 ubconfig.persistent_data_dir = persistent_data_dir
179 ubconfig.board_type = board_type
180 ubconfig.board_identity = board_identity
Stephen Warren89ab8412016-02-04 16:11:50 -0700181 ubconfig.gdbserver = gdbserver
Simon Glass06719602016-07-03 09:40:36 -0600182 ubconfig.dtb = build_dir + '/arch/sandbox/dts/test.dtb'
Stephen Warrend2015062016-01-15 11:15:24 -0700183
184 env_vars = (
185 'board_type',
186 'board_identity',
187 'source_dir',
188 'test_py_dir',
189 'build_dir',
190 'result_dir',
191 'persistent_data_dir',
192 )
193 for v in env_vars:
194 os.environ['U_BOOT_' + v.upper()] = getattr(ubconfig, v)
195
Simon Glass2fedbaa2016-07-04 11:58:37 -0600196 if board_type.startswith('sandbox'):
Stephen Warrend2015062016-01-15 11:15:24 -0700197 import u_boot_console_sandbox
198 console = u_boot_console_sandbox.ConsoleSandbox(log, ubconfig)
199 else:
200 import u_boot_console_exec_attach
201 console = u_boot_console_exec_attach.ConsoleExecAttach(log, ubconfig)
202
Stephen Warren1cd85f52016-02-08 14:44:16 -0700203re_ut_test_list = re.compile(r'_u_boot_list_2_(dm|env)_test_2_\1_test_(.*)\s*$')
204def generate_ut_subtest(metafunc, fixture_name):
205 """Provide parametrization for a ut_subtest fixture.
206
207 Determines the set of unit tests built into a U-Boot binary by parsing the
208 list of symbols generated by the build process. Provides this information
209 to test functions by parameterizing their ut_subtest fixture parameter.
210
211 Args:
212 metafunc: The pytest test function.
213 fixture_name: The fixture name to test.
214
215 Returns:
216 Nothing.
217 """
218
219 fn = console.config.build_dir + '/u-boot.sym'
220 try:
221 with open(fn, 'rt') as f:
222 lines = f.readlines()
223 except:
224 lines = []
225 lines.sort()
226
227 vals = []
228 for l in lines:
229 m = re_ut_test_list.search(l)
230 if not m:
231 continue
232 vals.append(m.group(1) + ' ' + m.group(2))
233
234 ids = ['ut_' + s.replace(' ', '_') for s in vals]
235 metafunc.parametrize(fixture_name, vals, ids=ids)
236
237def generate_config(metafunc, fixture_name):
238 """Provide parametrization for {env,brd}__ fixtures.
Stephen Warrend2015062016-01-15 11:15:24 -0700239
240 If a test function takes parameter(s) (fixture names) of the form brd__xxx
241 or env__xxx, the brd and env configuration dictionaries are consulted to
242 find the list of values to use for those parameters, and the test is
243 parametrized so that it runs once for each combination of values.
244
245 Args:
246 metafunc: The pytest test function.
Stephen Warren1cd85f52016-02-08 14:44:16 -0700247 fixture_name: The fixture name to test.
Stephen Warrend2015062016-01-15 11:15:24 -0700248
249 Returns:
250 Nothing.
Stephen Warrene8debf32016-01-26 13:41:30 -0700251 """
Stephen Warrend2015062016-01-15 11:15:24 -0700252
253 subconfigs = {
254 'brd': console.config.brd,
255 'env': console.config.env,
256 }
Stephen Warren1cd85f52016-02-08 14:44:16 -0700257 parts = fixture_name.split('__')
258 if len(parts) < 2:
259 return
260 if parts[0] not in subconfigs:
261 return
262 subconfig = subconfigs[parts[0]]
263 vals = []
264 val = subconfig.get(fixture_name, [])
265 # If that exact name is a key in the data source:
266 if val:
267 # ... use the dict value as a single parameter value.
268 vals = (val, )
269 else:
270 # ... otherwise, see if there's a key that contains a list of
271 # values to use instead.
272 vals = subconfig.get(fixture_name+ 's', [])
273 def fixture_id(index, val):
274 try:
275 return val['fixture_id']
276 except:
277 return fixture_name + str(index)
278 ids = [fixture_id(index, val) for (index, val) in enumerate(vals)]
279 metafunc.parametrize(fixture_name, vals, ids=ids)
280
281def pytest_generate_tests(metafunc):
282 """pytest hook: parameterize test functions based on custom rules.
283
284 Check each test function parameter (fixture name) to see if it is one of
285 our custom names, and if so, provide the correct parametrization for that
286 parameter.
287
288 Args:
289 metafunc: The pytest test function.
290
291 Returns:
292 Nothing.
293 """
294
Stephen Warrend2015062016-01-15 11:15:24 -0700295 for fn in metafunc.fixturenames:
Stephen Warren1cd85f52016-02-08 14:44:16 -0700296 if fn == 'ut_subtest':
297 generate_ut_subtest(metafunc, fn)
Stephen Warrend2015062016-01-15 11:15:24 -0700298 continue
Stephen Warren1cd85f52016-02-08 14:44:16 -0700299 generate_config(metafunc, fn)
Stephen Warrend2015062016-01-15 11:15:24 -0700300
Stefan Brünsd8c1e032016-11-05 17:45:32 +0100301@pytest.fixture(scope='session')
302def u_boot_log(request):
303 """Generate the value of a test's log fixture.
304
305 Args:
306 request: The pytest request.
307
308 Returns:
309 The fixture value.
310 """
311
312 return console.log
313
314@pytest.fixture(scope='session')
315def u_boot_config(request):
316 """Generate the value of a test's u_boot_config fixture.
317
318 Args:
319 request: The pytest request.
320
321 Returns:
322 The fixture value.
323 """
324
325 return console.config
326
Stephen Warren636f38d2016-01-22 12:30:08 -0700327@pytest.fixture(scope='function')
Stephen Warrend2015062016-01-15 11:15:24 -0700328def u_boot_console(request):
Stephen Warrene8debf32016-01-26 13:41:30 -0700329 """Generate the value of a test's u_boot_console fixture.
Stephen Warrend2015062016-01-15 11:15:24 -0700330
331 Args:
332 request: The pytest request.
333
334 Returns:
335 The fixture value.
Stephen Warrene8debf32016-01-26 13:41:30 -0700336 """
Stephen Warrend2015062016-01-15 11:15:24 -0700337
Stephen Warren636f38d2016-01-22 12:30:08 -0700338 console.ensure_spawned()
Stephen Warrend2015062016-01-15 11:15:24 -0700339 return console
340
Stephen Warren83357fd2016-02-03 16:46:34 -0700341anchors = {}
Stephen Warren13260222016-02-10 13:47:37 -0700342tests_not_run = []
343tests_failed = []
344tests_xpassed = []
345tests_xfailed = []
346tests_skipped = []
347tests_passed = []
Stephen Warrend2015062016-01-15 11:15:24 -0700348
349def pytest_itemcollected(item):
Stephen Warrene8debf32016-01-26 13:41:30 -0700350 """pytest hook: Called once for each test found during collection.
Stephen Warrend2015062016-01-15 11:15:24 -0700351
352 This enables our custom result analysis code to see the list of all tests
353 that should eventually be run.
354
355 Args:
356 item: The item that was collected.
357
358 Returns:
359 Nothing.
Stephen Warrene8debf32016-01-26 13:41:30 -0700360 """
Stephen Warrend2015062016-01-15 11:15:24 -0700361
Stephen Warren13260222016-02-10 13:47:37 -0700362 tests_not_run.append(item.name)
Stephen Warrend2015062016-01-15 11:15:24 -0700363
364def cleanup():
Stephen Warrene8debf32016-01-26 13:41:30 -0700365 """Clean up all global state.
Stephen Warrend2015062016-01-15 11:15:24 -0700366
367 Executed (via atexit) once the entire test process is complete. This
368 includes logging the status of all tests, and the identity of any failed
369 or skipped tests.
370
371 Args:
372 None.
373
374 Returns:
375 Nothing.
Stephen Warrene8debf32016-01-26 13:41:30 -0700376 """
Stephen Warrend2015062016-01-15 11:15:24 -0700377
378 if console:
379 console.close()
380 if log:
Stephen Warren83357fd2016-02-03 16:46:34 -0700381 with log.section('Status Report', 'status_report'):
382 log.status_pass('%d passed' % len(tests_passed))
383 if tests_skipped:
384 log.status_skipped('%d skipped' % len(tests_skipped))
385 for test in tests_skipped:
386 anchor = anchors.get(test, None)
387 log.status_skipped('... ' + test, anchor)
388 if tests_xpassed:
389 log.status_xpass('%d xpass' % len(tests_xpassed))
390 for test in tests_xpassed:
391 anchor = anchors.get(test, None)
392 log.status_xpass('... ' + test, anchor)
393 if tests_xfailed:
394 log.status_xfail('%d xfail' % len(tests_xfailed))
395 for test in tests_xfailed:
396 anchor = anchors.get(test, None)
397 log.status_xfail('... ' + test, anchor)
398 if tests_failed:
399 log.status_fail('%d failed' % len(tests_failed))
400 for test in tests_failed:
401 anchor = anchors.get(test, None)
402 log.status_fail('... ' + test, anchor)
403 if tests_not_run:
404 log.status_fail('%d not run' % len(tests_not_run))
405 for test in tests_not_run:
406 anchor = anchors.get(test, None)
407 log.status_fail('... ' + test, anchor)
Stephen Warrend2015062016-01-15 11:15:24 -0700408 log.close()
409atexit.register(cleanup)
410
411def setup_boardspec(item):
Stephen Warrene8debf32016-01-26 13:41:30 -0700412 """Process any 'boardspec' marker for a test.
Stephen Warrend2015062016-01-15 11:15:24 -0700413
414 Such a marker lists the set of board types that a test does/doesn't
415 support. If tests are being executed on an unsupported board, the test is
416 marked to be skipped.
417
418 Args:
419 item: The pytest test item.
420
421 Returns:
422 Nothing.
Stephen Warrene8debf32016-01-26 13:41:30 -0700423 """
Stephen Warrend2015062016-01-15 11:15:24 -0700424
425 mark = item.get_marker('boardspec')
426 if not mark:
427 return
428 required_boards = []
429 for board in mark.args:
430 if board.startswith('!'):
431 if ubconfig.board_type == board[1:]:
432 pytest.skip('board not supported')
433 return
434 else:
435 required_boards.append(board)
436 if required_boards and ubconfig.board_type not in required_boards:
437 pytest.skip('board not supported')
438
439def setup_buildconfigspec(item):
Stephen Warrene8debf32016-01-26 13:41:30 -0700440 """Process any 'buildconfigspec' marker for a test.
Stephen Warrend2015062016-01-15 11:15:24 -0700441
442 Such a marker lists some U-Boot configuration feature that the test
443 requires. If tests are being executed on an U-Boot build that doesn't
444 have the required feature, the test is marked to be skipped.
445
446 Args:
447 item: The pytest test item.
448
449 Returns:
450 Nothing.
Stephen Warrene8debf32016-01-26 13:41:30 -0700451 """
Stephen Warrend2015062016-01-15 11:15:24 -0700452
453 mark = item.get_marker('buildconfigspec')
454 if not mark:
455 return
456 for option in mark.args:
457 if not ubconfig.buildconfig.get('config_' + option.lower(), None):
458 pytest.skip('.config feature not enabled')
459
Stephen Warrenb0a928a2016-10-17 17:25:52 -0600460def start_test_section(item):
461 anchors[item.name] = log.start_section(item.name)
462
Stephen Warrend2015062016-01-15 11:15:24 -0700463def pytest_runtest_setup(item):
Stephen Warrene8debf32016-01-26 13:41:30 -0700464 """pytest hook: Configure (set up) a test item.
Stephen Warrend2015062016-01-15 11:15:24 -0700465
466 Called once for each test to perform any custom configuration. This hook
467 is used to skip the test if certain conditions apply.
468
469 Args:
470 item: The pytest test item.
471
472 Returns:
473 Nothing.
Stephen Warrene8debf32016-01-26 13:41:30 -0700474 """
Stephen Warrend2015062016-01-15 11:15:24 -0700475
Stephen Warrenb0a928a2016-10-17 17:25:52 -0600476 start_test_section(item)
Stephen Warrend2015062016-01-15 11:15:24 -0700477 setup_boardspec(item)
478 setup_buildconfigspec(item)
479
480def pytest_runtest_protocol(item, nextitem):
Stephen Warrene8debf32016-01-26 13:41:30 -0700481 """pytest hook: Called to execute a test.
Stephen Warrend2015062016-01-15 11:15:24 -0700482
483 This hook wraps the standard pytest runtestprotocol() function in order
484 to acquire visibility into, and record, each test function's result.
485
486 Args:
487 item: The pytest test item to execute.
488 nextitem: The pytest test item that will be executed after this one.
489
490 Returns:
491 A list of pytest reports (test result data).
Stephen Warrene8debf32016-01-26 13:41:30 -0700492 """
Stephen Warrend2015062016-01-15 11:15:24 -0700493
494 reports = runtestprotocol(item, nextitem=nextitem)
Stephen Warren78b39cc2016-01-27 23:57:51 -0700495
Stephen Warrenb0a928a2016-10-17 17:25:52 -0600496 # In pytest 3, runtestprotocol() may not call pytest_runtest_setup() if
497 # the test is skipped. That call is required to create the test's section
498 # in the log file. The call to log.end_section() requires that the log
499 # contain a section for this test. Create a section for the test if it
500 # doesn't already exist.
501 if not item.name in anchors:
502 start_test_section(item)
503
Stephen Warren78b39cc2016-01-27 23:57:51 -0700504 failure_cleanup = False
505 test_list = tests_passed
506 msg = 'OK'
507 msg_log = log.status_pass
Stephen Warrend2015062016-01-15 11:15:24 -0700508 for report in reports:
509 if report.outcome == 'failed':
Stephen Warren78b39cc2016-01-27 23:57:51 -0700510 if hasattr(report, 'wasxfail'):
511 test_list = tests_xpassed
512 msg = 'XPASSED'
513 msg_log = log.status_xpass
514 else:
515 failure_cleanup = True
516 test_list = tests_failed
517 msg = 'FAILED:\n' + str(report.longrepr)
518 msg_log = log.status_fail
Stephen Warrend2015062016-01-15 11:15:24 -0700519 break
520 if report.outcome == 'skipped':
Stephen Warren78b39cc2016-01-27 23:57:51 -0700521 if hasattr(report, 'wasxfail'):
522 failure_cleanup = True
523 test_list = tests_xfailed
524 msg = 'XFAILED:\n' + str(report.longrepr)
525 msg_log = log.status_xfail
526 break
527 test_list = tests_skipped
528 msg = 'SKIPPED:\n' + str(report.longrepr)
529 msg_log = log.status_skipped
Stephen Warrend2015062016-01-15 11:15:24 -0700530
Stephen Warren78b39cc2016-01-27 23:57:51 -0700531 if failure_cleanup:
Stephen Warrenc10eb9d2016-01-22 12:30:09 -0700532 console.drain_console()
Stephen Warren78b39cc2016-01-27 23:57:51 -0700533
Stephen Warren13260222016-02-10 13:47:37 -0700534 test_list.append(item.name)
Stephen Warrend2015062016-01-15 11:15:24 -0700535 tests_not_run.remove(item.name)
536
537 try:
Stephen Warren78b39cc2016-01-27 23:57:51 -0700538 msg_log(msg)
Stephen Warrend2015062016-01-15 11:15:24 -0700539 except:
540 # If something went wrong with logging, it's better to let the test
541 # process continue, which may report other exceptions that triggered
542 # the logging issue (e.g. console.log wasn't created). Hence, just
543 # squash the exception. If the test setup failed due to e.g. syntax
544 # error somewhere else, this won't be seen. However, once that issue
545 # is fixed, if this exception still exists, it will then be logged as
546 # part of the test's stdout.
547 import traceback
548 print 'Exception occurred while logging runtest status:'
549 traceback.print_exc()
550 # FIXME: Can we force a test failure here?
551
552 log.end_section(item.name)
553
Stephen Warren78b39cc2016-01-27 23:57:51 -0700554 if failure_cleanup:
Stephen Warrend2015062016-01-15 11:15:24 -0700555 console.cleanup_spawn()
556
557 return reports