18ec9d54bSJed Brown#!/usr/bin/env python3 28ec9d54bSJed Brown 38ec9d54bSJed Brownimport os 48ec9d54bSJed Brownimport sys 58ec9d54bSJed Brownsys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'junit-xml'))) 68ec9d54bSJed Brownfrom junit_xml import TestCase, TestSuite 78ec9d54bSJed Brown 88ec9d54bSJed Browndef parse_testargs(file): 98ec9d54bSJed Brown if os.path.splitext(file)[1] in ['.c', '.cpp']: 108ec9d54bSJed Brown return sum([line.split()[1:] for line in open(file).readlines() 118ec9d54bSJed Brown if line.startswith('//TESTARGS')], []) 128ec9d54bSJed Brown elif os.path.splitext(file)[1] == '.usr': 138ec9d54bSJed Brown return sum([line.split()[2:] for line in open(file).readlines() 148ec9d54bSJed Brown if line.startswith('C TESTARGS')], []) 159bcbe8bdSJed Brown raise RuntimeError('Unrecognized extension for file: {}'.format(file)) 168ec9d54bSJed Brown 178ec9d54bSJed Browndef get_source(test): 188ec9d54bSJed Brown if test.startswith('petsc-'): 198ec9d54bSJed Brown return os.path.join('examples', 'petsc', test[6:] + '.c') 208ec9d54bSJed Brown elif test.startswith('mfem-'): 218ec9d54bSJed Brown return os.path.join('examples', 'mfem', test[5:] + '.cpp') 228ec9d54bSJed Brown elif test.startswith('nek-'): 238ec9d54bSJed Brown return os.path.join('examples', 'nek5000', test[4:] + '.usr') 248ec9d54bSJed Brown elif test.startswith('ex'): 258ec9d54bSJed Brown return os.path.join('examples', 'ceed', test + '.c') 268ec9d54bSJed Brown 278ec9d54bSJed Browndef get_testargs(test): 288ec9d54bSJed Brown source = get_source(test) 298ec9d54bSJed Brown if source is None: 308ec9d54bSJed Brown return ['{ceed_resource}'] 318ec9d54bSJed Brown return parse_testargs(source) 328ec9d54bSJed Brown 338ec9d54bSJed Browndef run(test, backends): 348ec9d54bSJed Brown import subprocess 358ec9d54bSJed Brown import time 368ec9d54bSJed Brown import difflib 378ec9d54bSJed Brown args = get_testargs(test) 388ec9d54bSJed Brown testcases = [] 398ec9d54bSJed Brown for ceed_resource in backends: 408ec9d54bSJed Brown rargs = [os.path.join('build', test)] + args.copy() 418ec9d54bSJed Brown rargs[rargs.index('{ceed_resource}')] = ceed_resource 428ec9d54bSJed Brown start = time.time() 438ec9d54bSJed Brown proc = subprocess.run(rargs, 448ec9d54bSJed Brown stdout=subprocess.PIPE, 45*73132ccbSJed Brown stderr=subprocess.PIPE) 46*73132ccbSJed Brown proc.stdout = proc.stdout.decode('utf-8') 47*73132ccbSJed Brown proc.stderr = proc.stderr.decode('utf-8') 488ec9d54bSJed Brown 499bcbe8bdSJed Brown case = TestCase('{} {}'.format(test, ceed_resource), 508ec9d54bSJed Brown elapsed_sec=time.time()-start, 518ec9d54bSJed Brown timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(start)), 528ec9d54bSJed Brown stdout=proc.stdout, 538ec9d54bSJed Brown stderr=proc.stderr) 548ec9d54bSJed Brown ref_stdout = os.path.join('output', test + '.out') 558ec9d54bSJed Brown if proc.stderr: 568ec9d54bSJed Brown if 'OCCA backend failed to use' in proc.stderr: 579bcbe8bdSJed Brown case.add_skipped_info('occa mode not supported {} {}'.format(test, ceed_resource)) 588ec9d54bSJed Brown elif 'Backend does not implement' in proc.stderr: 599bcbe8bdSJed Brown case.add_skipped_info('not implemented {} {}'.format(test, ceed_resource)) 608ec9d54bSJed Brown elif 'access' in proc.stderr and test[:4] in 't103 t104 t105 t106 t107'.split(): 619bcbe8bdSJed Brown case.add_skipped_info('expected failure') 628ec9d54bSJed Brown elif 'vectors incompatible' in proc.stderr and test[:4] in ['t308']: 639bcbe8bdSJed Brown case.add_skipped_info('expected failure') 648ec9d54bSJed Brown else: 658ec9d54bSJed Brown case.add_failure_info('stderr', proc.stderr) 668ec9d54bSJed Brown if not case.is_skipped(): 678ec9d54bSJed Brown if proc.returncode != 0: 689bcbe8bdSJed Brown case.add_error_info('returncode = {}'.format(proc.returncode)) 698ec9d54bSJed Brown elif os.path.isfile(ref_stdout): 708ec9d54bSJed Brown with open(ref_stdout) as ref: 718ec9d54bSJed Brown diff = list(difflib.unified_diff(ref.readlines(), 728ec9d54bSJed Brown proc.stdout.splitlines(keepends=True), 738ec9d54bSJed Brown fromfile=ref_stdout, 748ec9d54bSJed Brown tofile='New')) 758ec9d54bSJed Brown if diff: 768ec9d54bSJed Brown case.add_failure_info('stdout', output=''.join(diff)) 778ec9d54bSJed Brown elif proc.stdout: 788ec9d54bSJed Brown case.add_failure_info('stdout', output=proc.stdout) 798ec9d54bSJed Brown testcases.append(case) 808ec9d54bSJed Brown return TestSuite(test, testcases) 818ec9d54bSJed Brown 828ec9d54bSJed Brownif __name__ == '__main__': 838ec9d54bSJed Brown import argparse 848ec9d54bSJed Brown parser = argparse.ArgumentParser('Test runner with JUnit output') 858ec9d54bSJed Brown parser.add_argument('--output', help='Output file to write test', default=None) 868ec9d54bSJed Brown parser.add_argument('--gather', help='Gather all *.junit files into XML', action='store_true') 878ec9d54bSJed Brown parser.add_argument('test', help='Test executable', nargs='?') 888ec9d54bSJed Brown args = parser.parse_args() 898ec9d54bSJed Brown 908ec9d54bSJed Brown if args.gather: 918ec9d54bSJed Brown gather() 928ec9d54bSJed Brown else: 938ec9d54bSJed Brown backends = os.environ['BACKENDS'].split() 948ec9d54bSJed Brown 958ec9d54bSJed Brown result = run(args.test, backends) 968ec9d54bSJed Brown output = (os.path.join('build', args.test + '.junit') 978ec9d54bSJed Brown if args.output is None 988ec9d54bSJed Brown else args.output) 998ec9d54bSJed Brown with open(output, 'w') as fd: 1008ec9d54bSJed Brown TestSuite.to_file(fd, [result]) 1018ec9d54bSJed Brown 102