1*8ec9d54bSJed Brown#!/usr/bin/env python3 2*8ec9d54bSJed Brown 3*8ec9d54bSJed Brownimport os 4*8ec9d54bSJed Brownimport sys 5*8ec9d54bSJed Brownsys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'junit-xml'))) 6*8ec9d54bSJed Brownfrom junit_xml import TestCase, TestSuite 7*8ec9d54bSJed Brown 8*8ec9d54bSJed Browndef parse_testargs(file): 9*8ec9d54bSJed Brown if os.path.splitext(file)[1] in ['.c', '.cpp']: 10*8ec9d54bSJed Brown return sum([line.split()[1:] for line in open(file).readlines() 11*8ec9d54bSJed Brown if line.startswith('//TESTARGS')], []) 12*8ec9d54bSJed Brown elif os.path.splitext(file)[1] == '.usr': 13*8ec9d54bSJed Brown return sum([line.split()[2:] for line in open(file).readlines() 14*8ec9d54bSJed Brown if line.startswith('C TESTARGS')], []) 15*8ec9d54bSJed Brown raise RuntimeError(f'Unrecognized extension for file: {file}') 16*8ec9d54bSJed Brown 17*8ec9d54bSJed Browndef get_source(test): 18*8ec9d54bSJed Brown if test.startswith('petsc-'): 19*8ec9d54bSJed Brown return os.path.join('examples', 'petsc', test[6:] + '.c') 20*8ec9d54bSJed Brown elif test.startswith('mfem-'): 21*8ec9d54bSJed Brown return os.path.join('examples', 'mfem', test[5:] + '.cpp') 22*8ec9d54bSJed Brown elif test.startswith('nek-'): 23*8ec9d54bSJed Brown return os.path.join('examples', 'nek5000', test[4:] + '.usr') 24*8ec9d54bSJed Brown elif test.startswith('ex'): 25*8ec9d54bSJed Brown return os.path.join('examples', 'ceed', test + '.c') 26*8ec9d54bSJed Brown 27*8ec9d54bSJed Browndef get_testargs(test): 28*8ec9d54bSJed Brown source = get_source(test) 29*8ec9d54bSJed Brown if source is None: 30*8ec9d54bSJed Brown return ['{ceed_resource}'] 31*8ec9d54bSJed Brown return parse_testargs(source) 32*8ec9d54bSJed Brown 33*8ec9d54bSJed Browndef run(test, backends): 34*8ec9d54bSJed Brown import subprocess 35*8ec9d54bSJed Brown import time 36*8ec9d54bSJed Brown import difflib 37*8ec9d54bSJed Brown args = get_testargs(test) 38*8ec9d54bSJed Brown testcases = [] 39*8ec9d54bSJed Brown for ceed_resource in backends: 40*8ec9d54bSJed Brown rargs = [os.path.join('build', test)] + args.copy() 41*8ec9d54bSJed Brown rargs[rargs.index('{ceed_resource}')] = ceed_resource 42*8ec9d54bSJed Brown start = time.time() 43*8ec9d54bSJed Brown proc = subprocess.run(rargs, 44*8ec9d54bSJed Brown stdout=subprocess.PIPE, 45*8ec9d54bSJed Brown stderr=subprocess.PIPE, 46*8ec9d54bSJed Brown encoding='utf-8') 47*8ec9d54bSJed Brown 48*8ec9d54bSJed Brown case = TestCase(f'{test} {ceed_resource}', 49*8ec9d54bSJed Brown elapsed_sec=time.time()-start, 50*8ec9d54bSJed Brown timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(start)), 51*8ec9d54bSJed Brown stdout=proc.stdout, 52*8ec9d54bSJed Brown stderr=proc.stderr) 53*8ec9d54bSJed Brown ref_stdout = os.path.join('output', test + '.out') 54*8ec9d54bSJed Brown if proc.stderr: 55*8ec9d54bSJed Brown if 'OCCA backend failed to use' in proc.stderr: 56*8ec9d54bSJed Brown case.add_skipped_info(f'occa mode not supported {test} {ceed_resource}') 57*8ec9d54bSJed Brown elif 'Backend does not implement' in proc.stderr: 58*8ec9d54bSJed Brown case.add_skipped_info(f'not implemented {test} {ceed_resource}') 59*8ec9d54bSJed Brown elif 'access' in proc.stderr and test[:4] in 't103 t104 t105 t106 t107'.split(): 60*8ec9d54bSJed Brown case.add_skipped_info(f'expected failure') 61*8ec9d54bSJed Brown elif 'vectors incompatible' in proc.stderr and test[:4] in ['t308']: 62*8ec9d54bSJed Brown case.add_skipped_info(f'expected failure') 63*8ec9d54bSJed Brown else: 64*8ec9d54bSJed Brown case.add_failure_info('stderr', proc.stderr) 65*8ec9d54bSJed Brown if not case.is_skipped(): 66*8ec9d54bSJed Brown if proc.returncode != 0: 67*8ec9d54bSJed Brown case.add_error_info(f'returncode = {proc.returncode}') 68*8ec9d54bSJed Brown elif os.path.isfile(ref_stdout): 69*8ec9d54bSJed Brown with open(ref_stdout) as ref: 70*8ec9d54bSJed Brown diff = list(difflib.unified_diff(ref.readlines(), 71*8ec9d54bSJed Brown proc.stdout.splitlines(keepends=True), 72*8ec9d54bSJed Brown fromfile=ref_stdout, 73*8ec9d54bSJed Brown tofile='New')) 74*8ec9d54bSJed Brown if diff: 75*8ec9d54bSJed Brown case.add_failure_info('stdout', output=''.join(diff)) 76*8ec9d54bSJed Brown elif proc.stdout: 77*8ec9d54bSJed Brown case.add_failure_info('stdout', output=proc.stdout) 78*8ec9d54bSJed Brown testcases.append(case) 79*8ec9d54bSJed Brown return TestSuite(test, testcases) 80*8ec9d54bSJed Brown 81*8ec9d54bSJed Brownif __name__ == '__main__': 82*8ec9d54bSJed Brown import argparse 83*8ec9d54bSJed Brown parser = argparse.ArgumentParser('Test runner with JUnit output') 84*8ec9d54bSJed Brown parser.add_argument('--output', help='Output file to write test', default=None) 85*8ec9d54bSJed Brown parser.add_argument('--gather', help='Gather all *.junit files into XML', action='store_true') 86*8ec9d54bSJed Brown parser.add_argument('test', help='Test executable', nargs='?') 87*8ec9d54bSJed Brown args = parser.parse_args() 88*8ec9d54bSJed Brown 89*8ec9d54bSJed Brown if args.gather: 90*8ec9d54bSJed Brown gather() 91*8ec9d54bSJed Brown else: 92*8ec9d54bSJed Brown backends = os.environ['BACKENDS'].split() 93*8ec9d54bSJed Brown 94*8ec9d54bSJed Brown result = run(args.test, backends) 95*8ec9d54bSJed Brown output = (os.path.join('build', args.test + '.junit') 96*8ec9d54bSJed Brown if args.output is None 97*8ec9d54bSJed Brown else args.output) 98*8ec9d54bSJed Brown with open(output, 'w') as fd: 99*8ec9d54bSJed Brown TestSuite.to_file(fd, [result]) 100*8ec9d54bSJed Brown 101