blob: 5b3a92316bbfae2969f76fc11a61e713b649f30b [file] [log] [blame]
Stephen Warrend2015062016-01-15 11:15:24 -07001# Copyright (c) 2015 Stephen Warren
2# Copyright (c) 2015-2016, NVIDIA CORPORATION. All rights reserved.
3#
4# SPDX-License-Identifier: GPL-2.0
5
6# Implementation of pytest run-time hook functions. These are invoked by
7# pytest at certain points during operation, e.g. startup, for each executed
8# test, at shutdown etc. These hooks perform functions such as:
9# - Parsing custom command-line options.
10# - Pullilng in user-specified board configuration.
11# - Creating the U-Boot console test fixture.
12# - Creating the HTML log file.
13# - Monitoring each test's results.
14# - Implementing custom pytest markers.
15
16import atexit
17import errno
18import os
19import os.path
Stephen Warrend2015062016-01-15 11:15:24 -070020import pytest
21from _pytest.runner import runtestprotocol
22import ConfigParser
Stephen Warren1cd85f52016-02-08 14:44:16 -070023import re
Stephen Warrend2015062016-01-15 11:15:24 -070024import StringIO
25import sys
26
27# Globals: The HTML log file, and the connection to the U-Boot console.
28log = None
29console = None
30
31def mkdir_p(path):
Stephen Warrene8debf32016-01-26 13:41:30 -070032 """Create a directory path.
Stephen Warrend2015062016-01-15 11:15:24 -070033
34 This includes creating any intermediate/parent directories. Any errors
35 caused due to already extant directories are ignored.
36
37 Args:
38 path: The directory path to create.
39
40 Returns:
41 Nothing.
Stephen Warrene8debf32016-01-26 13:41:30 -070042 """
Stephen Warrend2015062016-01-15 11:15:24 -070043
44 try:
45 os.makedirs(path)
46 except OSError as exc:
47 if exc.errno == errno.EEXIST and os.path.isdir(path):
48 pass
49 else:
50 raise
51
52def pytest_addoption(parser):
Stephen Warrene8debf32016-01-26 13:41:30 -070053 """pytest hook: Add custom command-line options to the cmdline parser.
Stephen Warrend2015062016-01-15 11:15:24 -070054
55 Args:
56 parser: The pytest command-line parser.
57
58 Returns:
59 Nothing.
Stephen Warrene8debf32016-01-26 13:41:30 -070060 """
Stephen Warrend2015062016-01-15 11:15:24 -070061
62 parser.addoption('--build-dir', default=None,
63 help='U-Boot build directory (O=)')
64 parser.addoption('--result-dir', default=None,
65 help='U-Boot test result/tmp directory')
66 parser.addoption('--persistent-data-dir', default=None,
67 help='U-Boot test persistent generated data directory')
68 parser.addoption('--board-type', '--bd', '-B', default='sandbox',
69 help='U-Boot board type')
70 parser.addoption('--board-identity', '--id', default='na',
71 help='U-Boot board identity/instance')
72 parser.addoption('--build', default=False, action='store_true',
73 help='Compile U-Boot before running tests')
Stephen Warren89ab8412016-02-04 16:11:50 -070074 parser.addoption('--gdbserver', default=None,
75 help='Run sandbox under gdbserver. The argument is the channel '+
76 'over which gdbserver should communicate, e.g. localhost:1234')
Stephen Warrend2015062016-01-15 11:15:24 -070077
78def pytest_configure(config):
Stephen Warrene8debf32016-01-26 13:41:30 -070079 """pytest hook: Perform custom initialization at startup time.
Stephen Warrend2015062016-01-15 11:15:24 -070080
81 Args:
82 config: The pytest configuration.
83
84 Returns:
85 Nothing.
Stephen Warrene8debf32016-01-26 13:41:30 -070086 """
Stephen Warrend2015062016-01-15 11:15:24 -070087
88 global log
89 global console
90 global ubconfig
91
92 test_py_dir = os.path.dirname(os.path.abspath(__file__))
93 source_dir = os.path.dirname(os.path.dirname(test_py_dir))
94
95 board_type = config.getoption('board_type')
96 board_type_filename = board_type.replace('-', '_')
97
98 board_identity = config.getoption('board_identity')
99 board_identity_filename = board_identity.replace('-', '_')
100
101 build_dir = config.getoption('build_dir')
102 if not build_dir:
103 build_dir = source_dir + '/build-' + board_type
104 mkdir_p(build_dir)
105
106 result_dir = config.getoption('result_dir')
107 if not result_dir:
108 result_dir = build_dir
109 mkdir_p(result_dir)
110
111 persistent_data_dir = config.getoption('persistent_data_dir')
112 if not persistent_data_dir:
113 persistent_data_dir = build_dir + '/persistent-data'
114 mkdir_p(persistent_data_dir)
115
Stephen Warren89ab8412016-02-04 16:11:50 -0700116 gdbserver = config.getoption('gdbserver')
117 if gdbserver and board_type != 'sandbox':
118 raise Exception('--gdbserver only supported with sandbox')
119
Stephen Warrend2015062016-01-15 11:15:24 -0700120 import multiplexed_log
121 log = multiplexed_log.Logfile(result_dir + '/test-log.html')
122
123 if config.getoption('build'):
124 if build_dir != source_dir:
125 o_opt = 'O=%s' % build_dir
126 else:
127 o_opt = ''
128 cmds = (
129 ['make', o_opt, '-s', board_type + '_defconfig'],
130 ['make', o_opt, '-s', '-j8'],
131 )
Stephen Warren83357fd2016-02-03 16:46:34 -0700132 with log.section('make'):
133 runner = log.get_runner('make', sys.stdout)
134 for cmd in cmds:
135 runner.run(cmd, cwd=source_dir)
136 runner.close()
137 log.status_pass('OK')
Stephen Warrend2015062016-01-15 11:15:24 -0700138
139 class ArbitraryAttributeContainer(object):
140 pass
141
142 ubconfig = ArbitraryAttributeContainer()
143 ubconfig.brd = dict()
144 ubconfig.env = dict()
145
146 modules = [
147 (ubconfig.brd, 'u_boot_board_' + board_type_filename),
148 (ubconfig.env, 'u_boot_boardenv_' + board_type_filename),
149 (ubconfig.env, 'u_boot_boardenv_' + board_type_filename + '_' +
150 board_identity_filename),
151 ]
152 for (dict_to_fill, module_name) in modules:
153 try:
154 module = __import__(module_name)
155 except ImportError:
156 continue
157 dict_to_fill.update(module.__dict__)
158
159 ubconfig.buildconfig = dict()
160
161 for conf_file in ('.config', 'include/autoconf.mk'):
162 dot_config = build_dir + '/' + conf_file
163 if not os.path.exists(dot_config):
164 raise Exception(conf_file + ' does not exist; ' +
165 'try passing --build option?')
166
167 with open(dot_config, 'rt') as f:
168 ini_str = '[root]\n' + f.read()
169 ini_sio = StringIO.StringIO(ini_str)
170 parser = ConfigParser.RawConfigParser()
171 parser.readfp(ini_sio)
172 ubconfig.buildconfig.update(parser.items('root'))
173
174 ubconfig.test_py_dir = test_py_dir
175 ubconfig.source_dir = source_dir
176 ubconfig.build_dir = build_dir
177 ubconfig.result_dir = result_dir
178 ubconfig.persistent_data_dir = persistent_data_dir
179 ubconfig.board_type = board_type
180 ubconfig.board_identity = board_identity
Stephen Warren89ab8412016-02-04 16:11:50 -0700181 ubconfig.gdbserver = gdbserver
Simon Glass06719602016-07-03 09:40:36 -0600182 ubconfig.dtb = build_dir + '/arch/sandbox/dts/test.dtb'
Stephen Warrend2015062016-01-15 11:15:24 -0700183
184 env_vars = (
185 'board_type',
186 'board_identity',
187 'source_dir',
188 'test_py_dir',
189 'build_dir',
190 'result_dir',
191 'persistent_data_dir',
192 )
193 for v in env_vars:
194 os.environ['U_BOOT_' + v.upper()] = getattr(ubconfig, v)
195
Simon Glass2fedbaa2016-07-04 11:58:37 -0600196 if board_type.startswith('sandbox'):
Stephen Warrend2015062016-01-15 11:15:24 -0700197 import u_boot_console_sandbox
198 console = u_boot_console_sandbox.ConsoleSandbox(log, ubconfig)
199 else:
200 import u_boot_console_exec_attach
201 console = u_boot_console_exec_attach.ConsoleExecAttach(log, ubconfig)
202
Stephen Warren1cd85f52016-02-08 14:44:16 -0700203re_ut_test_list = re.compile(r'_u_boot_list_2_(dm|env)_test_2_\1_test_(.*)\s*$')
204def generate_ut_subtest(metafunc, fixture_name):
205 """Provide parametrization for a ut_subtest fixture.
206
207 Determines the set of unit tests built into a U-Boot binary by parsing the
208 list of symbols generated by the build process. Provides this information
209 to test functions by parameterizing their ut_subtest fixture parameter.
210
211 Args:
212 metafunc: The pytest test function.
213 fixture_name: The fixture name to test.
214
215 Returns:
216 Nothing.
217 """
218
219 fn = console.config.build_dir + '/u-boot.sym'
220 try:
221 with open(fn, 'rt') as f:
222 lines = f.readlines()
223 except:
224 lines = []
225 lines.sort()
226
227 vals = []
228 for l in lines:
229 m = re_ut_test_list.search(l)
230 if not m:
231 continue
232 vals.append(m.group(1) + ' ' + m.group(2))
233
234 ids = ['ut_' + s.replace(' ', '_') for s in vals]
235 metafunc.parametrize(fixture_name, vals, ids=ids)
236
237def generate_config(metafunc, fixture_name):
238 """Provide parametrization for {env,brd}__ fixtures.
Stephen Warrend2015062016-01-15 11:15:24 -0700239
240 If a test function takes parameter(s) (fixture names) of the form brd__xxx
241 or env__xxx, the brd and env configuration dictionaries are consulted to
242 find the list of values to use for those parameters, and the test is
243 parametrized so that it runs once for each combination of values.
244
245 Args:
246 metafunc: The pytest test function.
Stephen Warren1cd85f52016-02-08 14:44:16 -0700247 fixture_name: The fixture name to test.
Stephen Warrend2015062016-01-15 11:15:24 -0700248
249 Returns:
250 Nothing.
Stephen Warrene8debf32016-01-26 13:41:30 -0700251 """
Stephen Warrend2015062016-01-15 11:15:24 -0700252
253 subconfigs = {
254 'brd': console.config.brd,
255 'env': console.config.env,
256 }
Stephen Warren1cd85f52016-02-08 14:44:16 -0700257 parts = fixture_name.split('__')
258 if len(parts) < 2:
259 return
260 if parts[0] not in subconfigs:
261 return
262 subconfig = subconfigs[parts[0]]
263 vals = []
264 val = subconfig.get(fixture_name, [])
265 # If that exact name is a key in the data source:
266 if val:
267 # ... use the dict value as a single parameter value.
268 vals = (val, )
269 else:
270 # ... otherwise, see if there's a key that contains a list of
271 # values to use instead.
272 vals = subconfig.get(fixture_name+ 's', [])
273 def fixture_id(index, val):
274 try:
275 return val['fixture_id']
276 except:
277 return fixture_name + str(index)
278 ids = [fixture_id(index, val) for (index, val) in enumerate(vals)]
279 metafunc.parametrize(fixture_name, vals, ids=ids)
280
281def pytest_generate_tests(metafunc):
282 """pytest hook: parameterize test functions based on custom rules.
283
284 Check each test function parameter (fixture name) to see if it is one of
285 our custom names, and if so, provide the correct parametrization for that
286 parameter.
287
288 Args:
289 metafunc: The pytest test function.
290
291 Returns:
292 Nothing.
293 """
294
Stephen Warrend2015062016-01-15 11:15:24 -0700295 for fn in metafunc.fixturenames:
Stephen Warren1cd85f52016-02-08 14:44:16 -0700296 if fn == 'ut_subtest':
297 generate_ut_subtest(metafunc, fn)
Stephen Warrend2015062016-01-15 11:15:24 -0700298 continue
Stephen Warren1cd85f52016-02-08 14:44:16 -0700299 generate_config(metafunc, fn)
Stephen Warrend2015062016-01-15 11:15:24 -0700300
Stephen Warren636f38d2016-01-22 12:30:08 -0700301@pytest.fixture(scope='function')
Stephen Warrend2015062016-01-15 11:15:24 -0700302def u_boot_console(request):
Stephen Warrene8debf32016-01-26 13:41:30 -0700303 """Generate the value of a test's u_boot_console fixture.
Stephen Warrend2015062016-01-15 11:15:24 -0700304
305 Args:
306 request: The pytest request.
307
308 Returns:
309 The fixture value.
Stephen Warrene8debf32016-01-26 13:41:30 -0700310 """
Stephen Warrend2015062016-01-15 11:15:24 -0700311
Stephen Warren636f38d2016-01-22 12:30:08 -0700312 console.ensure_spawned()
Stephen Warrend2015062016-01-15 11:15:24 -0700313 return console
314
Stephen Warren83357fd2016-02-03 16:46:34 -0700315anchors = {}
Stephen Warren13260222016-02-10 13:47:37 -0700316tests_not_run = []
317tests_failed = []
318tests_xpassed = []
319tests_xfailed = []
320tests_skipped = []
321tests_passed = []
Stephen Warrend2015062016-01-15 11:15:24 -0700322
323def pytest_itemcollected(item):
Stephen Warrene8debf32016-01-26 13:41:30 -0700324 """pytest hook: Called once for each test found during collection.
Stephen Warrend2015062016-01-15 11:15:24 -0700325
326 This enables our custom result analysis code to see the list of all tests
327 that should eventually be run.
328
329 Args:
330 item: The item that was collected.
331
332 Returns:
333 Nothing.
Stephen Warrene8debf32016-01-26 13:41:30 -0700334 """
Stephen Warrend2015062016-01-15 11:15:24 -0700335
Stephen Warren13260222016-02-10 13:47:37 -0700336 tests_not_run.append(item.name)
Stephen Warrend2015062016-01-15 11:15:24 -0700337
338def cleanup():
Stephen Warrene8debf32016-01-26 13:41:30 -0700339 """Clean up all global state.
Stephen Warrend2015062016-01-15 11:15:24 -0700340
341 Executed (via atexit) once the entire test process is complete. This
342 includes logging the status of all tests, and the identity of any failed
343 or skipped tests.
344
345 Args:
346 None.
347
348 Returns:
349 Nothing.
Stephen Warrene8debf32016-01-26 13:41:30 -0700350 """
Stephen Warrend2015062016-01-15 11:15:24 -0700351
352 if console:
353 console.close()
354 if log:
Stephen Warren83357fd2016-02-03 16:46:34 -0700355 with log.section('Status Report', 'status_report'):
356 log.status_pass('%d passed' % len(tests_passed))
357 if tests_skipped:
358 log.status_skipped('%d skipped' % len(tests_skipped))
359 for test in tests_skipped:
360 anchor = anchors.get(test, None)
361 log.status_skipped('... ' + test, anchor)
362 if tests_xpassed:
363 log.status_xpass('%d xpass' % len(tests_xpassed))
364 for test in tests_xpassed:
365 anchor = anchors.get(test, None)
366 log.status_xpass('... ' + test, anchor)
367 if tests_xfailed:
368 log.status_xfail('%d xfail' % len(tests_xfailed))
369 for test in tests_xfailed:
370 anchor = anchors.get(test, None)
371 log.status_xfail('... ' + test, anchor)
372 if tests_failed:
373 log.status_fail('%d failed' % len(tests_failed))
374 for test in tests_failed:
375 anchor = anchors.get(test, None)
376 log.status_fail('... ' + test, anchor)
377 if tests_not_run:
378 log.status_fail('%d not run' % len(tests_not_run))
379 for test in tests_not_run:
380 anchor = anchors.get(test, None)
381 log.status_fail('... ' + test, anchor)
Stephen Warrend2015062016-01-15 11:15:24 -0700382 log.close()
383atexit.register(cleanup)
384
385def setup_boardspec(item):
Stephen Warrene8debf32016-01-26 13:41:30 -0700386 """Process any 'boardspec' marker for a test.
Stephen Warrend2015062016-01-15 11:15:24 -0700387
388 Such a marker lists the set of board types that a test does/doesn't
389 support. If tests are being executed on an unsupported board, the test is
390 marked to be skipped.
391
392 Args:
393 item: The pytest test item.
394
395 Returns:
396 Nothing.
Stephen Warrene8debf32016-01-26 13:41:30 -0700397 """
Stephen Warrend2015062016-01-15 11:15:24 -0700398
399 mark = item.get_marker('boardspec')
400 if not mark:
401 return
402 required_boards = []
403 for board in mark.args:
404 if board.startswith('!'):
405 if ubconfig.board_type == board[1:]:
406 pytest.skip('board not supported')
407 return
408 else:
409 required_boards.append(board)
410 if required_boards and ubconfig.board_type not in required_boards:
411 pytest.skip('board not supported')
412
413def setup_buildconfigspec(item):
Stephen Warrene8debf32016-01-26 13:41:30 -0700414 """Process any 'buildconfigspec' marker for a test.
Stephen Warrend2015062016-01-15 11:15:24 -0700415
416 Such a marker lists some U-Boot configuration feature that the test
417 requires. If tests are being executed on an U-Boot build that doesn't
418 have the required feature, the test is marked to be skipped.
419
420 Args:
421 item: The pytest test item.
422
423 Returns:
424 Nothing.
Stephen Warrene8debf32016-01-26 13:41:30 -0700425 """
Stephen Warrend2015062016-01-15 11:15:24 -0700426
427 mark = item.get_marker('buildconfigspec')
428 if not mark:
429 return
430 for option in mark.args:
431 if not ubconfig.buildconfig.get('config_' + option.lower(), None):
432 pytest.skip('.config feature not enabled')
433
434def pytest_runtest_setup(item):
Stephen Warrene8debf32016-01-26 13:41:30 -0700435 """pytest hook: Configure (set up) a test item.
Stephen Warrend2015062016-01-15 11:15:24 -0700436
437 Called once for each test to perform any custom configuration. This hook
438 is used to skip the test if certain conditions apply.
439
440 Args:
441 item: The pytest test item.
442
443 Returns:
444 Nothing.
Stephen Warrene8debf32016-01-26 13:41:30 -0700445 """
Stephen Warrend2015062016-01-15 11:15:24 -0700446
Stephen Warren83357fd2016-02-03 16:46:34 -0700447 anchors[item.name] = log.start_section(item.name)
Stephen Warrend2015062016-01-15 11:15:24 -0700448 setup_boardspec(item)
449 setup_buildconfigspec(item)
450
451def pytest_runtest_protocol(item, nextitem):
Stephen Warrene8debf32016-01-26 13:41:30 -0700452 """pytest hook: Called to execute a test.
Stephen Warrend2015062016-01-15 11:15:24 -0700453
454 This hook wraps the standard pytest runtestprotocol() function in order
455 to acquire visibility into, and record, each test function's result.
456
457 Args:
458 item: The pytest test item to execute.
459 nextitem: The pytest test item that will be executed after this one.
460
461 Returns:
462 A list of pytest reports (test result data).
Stephen Warrene8debf32016-01-26 13:41:30 -0700463 """
Stephen Warrend2015062016-01-15 11:15:24 -0700464
465 reports = runtestprotocol(item, nextitem=nextitem)
Stephen Warren78b39cc2016-01-27 23:57:51 -0700466
467 failure_cleanup = False
468 test_list = tests_passed
469 msg = 'OK'
470 msg_log = log.status_pass
Stephen Warrend2015062016-01-15 11:15:24 -0700471 for report in reports:
472 if report.outcome == 'failed':
Stephen Warren78b39cc2016-01-27 23:57:51 -0700473 if hasattr(report, 'wasxfail'):
474 test_list = tests_xpassed
475 msg = 'XPASSED'
476 msg_log = log.status_xpass
477 else:
478 failure_cleanup = True
479 test_list = tests_failed
480 msg = 'FAILED:\n' + str(report.longrepr)
481 msg_log = log.status_fail
Stephen Warrend2015062016-01-15 11:15:24 -0700482 break
483 if report.outcome == 'skipped':
Stephen Warren78b39cc2016-01-27 23:57:51 -0700484 if hasattr(report, 'wasxfail'):
485 failure_cleanup = True
486 test_list = tests_xfailed
487 msg = 'XFAILED:\n' + str(report.longrepr)
488 msg_log = log.status_xfail
489 break
490 test_list = tests_skipped
491 msg = 'SKIPPED:\n' + str(report.longrepr)
492 msg_log = log.status_skipped
Stephen Warrend2015062016-01-15 11:15:24 -0700493
Stephen Warren78b39cc2016-01-27 23:57:51 -0700494 if failure_cleanup:
Stephen Warrenc10eb9d2016-01-22 12:30:09 -0700495 console.drain_console()
Stephen Warren78b39cc2016-01-27 23:57:51 -0700496
Stephen Warren13260222016-02-10 13:47:37 -0700497 test_list.append(item.name)
Stephen Warrend2015062016-01-15 11:15:24 -0700498 tests_not_run.remove(item.name)
499
500 try:
Stephen Warren78b39cc2016-01-27 23:57:51 -0700501 msg_log(msg)
Stephen Warrend2015062016-01-15 11:15:24 -0700502 except:
503 # If something went wrong with logging, it's better to let the test
504 # process continue, which may report other exceptions that triggered
505 # the logging issue (e.g. console.log wasn't created). Hence, just
506 # squash the exception. If the test setup failed due to e.g. syntax
507 # error somewhere else, this won't be seen. However, once that issue
508 # is fixed, if this exception still exists, it will then be logged as
509 # part of the test's stdout.
510 import traceback
511 print 'Exception occurred while logging runtest status:'
512 traceback.print_exc()
513 # FIXME: Can we force a test failure here?
514
515 log.end_section(item.name)
516
Stephen Warren78b39cc2016-01-27 23:57:51 -0700517 if failure_cleanup:
Stephen Warrend2015062016-01-15 11:15:24 -0700518 console.cleanup_spawn()
519
520 return reports