Lines Matching +full:- +full:- +full:output +full:- +full:file
19 sys.path.insert(0, str(Path(__file__).parent / "junit-xml"))
65 csv_rtol: float = -1
66 csv_ztol: float = -1
67 cgns_tol: float = -1
88 def get_source_path(self, test: str) -> Path:
89 """Compute path to test source file
95 Path: Path to source file
100 def get_run_path(self, test: str) -> Path:
101 """Compute path to built test executable file
112 def get_output_path(self, test: str, output_file: str) -> Path:
113 """Compute path to expected output file
117 output_file (str): File name of output file
120 Path: Path to expected output file
125 def test_failure_artifacts_path(self) -> Path:
132 return getattr(self, '_cgns_tol', 1.0e-12)
141 return getattr(self, '_csv_ztol', 3e-10)
150 return getattr(self, '_csv_rtol', 1e-6)
157 def csv_comment_diff_fn(self): # -> Any | Callable[..., None]:
172 def post_test_hook(self, test: str, spec: TestSpec, backend: str) -> None:
181 def check_pre_skip(self, test: str, spec: TestSpec, resource: str, nproc: int) -> Optional[str]:
195 … def check_post_skip(self, test: str, spec: TestSpec, resource: str, stderr: str) -> Optional[str]:
196 """Check if a test case should be allowed to fail, based on its stderr output
202 stderr (str): Standard error output from test case execution
209 …def check_required_failure(self, test: str, spec: TestSpec, resource: str, stderr: str) -> Tuple[s…
216 stderr (str): Standard error output from test case execution
223 def check_allowed_stdout(self, test: str) -> bool:
224 """Check whether a test is allowed to print console output
230 bool: True if the test is allowed to print console output
235 def has_cgnsdiff() -> bool:
247 return 'not found' not in proc.stderr.decode('utf-8')
250 def contains_any(base: str, substrings: List[str]) -> bool:
263 def startswith_any(base: str, prefixes: List[str]) -> bool:
276 def find_matching(line: str, open: str = '(', close: str = ')') -> Tuple[int, int]:
297 return -1, -1
303 count -= 1
306 return start, -1
309 def parse_test_line(line: str, fallback_name: str = '') -> TestSpec:
331 …([A-Za-z]+[\w\-]+) # match key starting with alpha, containing alphanumeric, _, or -; captured as…
375 def get_test_args(source_file: Path) -> List[TestSpec]:
376 """Parse all test cases from a given source file
379 source_file (Path): Path to source file
382 RuntimeError: Errors if source file extension is unsupported
397 raise RuntimeError(f'Unrecognized extension for file: {source_file}')
405 …comment_str: str = '#', comment_func: Optional[Callable[[str, str], Optional[str]]] = None) -> str:
406 """Compare CSV results against an expected CSV file with tolerances
409 test_csv (Path): Path to output CSV results
412 rel_tol (float): Relative tolerance for comparing non-zero values.
417 str: Diff output between result and expected CSVs
423 return f'No lines found in test output {test_csv}'
434 output = comment_func(test_line, true_line)
435 if output:
436 return output
451 return f'No CSV columns found in test output {test_csv}'
481 def diff_cgns(test_cgns: Path, true_cgns: Path, cgns_tol: float) -> str:
482 """Compare CGNS results against an expected CGSN file with tolerance
485 test_cgns (Path): Path to output CGNS file
486 true_cgns (Path): Path to expected CGNS file
487 cgns_tol (float): Tolerance for comparing floating-point values
490 str: Diff output between result and expected CGNS files
494 run_args: List[str] = ['cgnsdiff', '-d', '-t', f'{cgns_tol}', str(test_cgns), str(true_cgns)]
501 return proc.stderr.decode('utf-8') + proc.stdout.decode('utf-8')
504 def diff_ascii(test_file: Path, true_file: Path, backend: str) -> str:
505 """Compare ASCII results against an expected ASCII file
508 test_file (Path): Path to output ASCII file
509 true_file (Path): Path to expected ASCII file
512 str: Diff output between result and expected ASCII files
514 tmp_backend: str = backend.replace('/', '-')
524 backend: str, test: str, index: int, verbose: bool) -> str:
527 # print incremental output if TAP mode
529 …output_str += f' ok {index} - {spec.name}, {backend} # SKIP {test_case.skipped[0]["message"]}\n'
531 … output_str += f' not ok {index} - {spec.name}, {backend} ({test_case.elapsed_sec} s)\n'
533 output_str += f' ok {index} - {spec.name}, {backend} ({test_case.elapsed_sec} s)\n'
535 output_str += f' ---\n'
552 output_str += f' -\n'
554 if failure["output"]:
555 out = failure["output"].strip().replace('\n', '\n ')
556 output_str += f' output: |\n {out}\n'
565 … output_str += 'Output: \n{}\n'.format((test_case.errors[0]['output'] or 'NO MESSAGE').strip())
569 … output_str += 'Output: \n{}\n'.format((failure['output'] or 'NO MESSAGE').strip())
573 def save_failure_artifact(suite_spec: SuiteSpec, file: Path) -> Path:
574 """Attach a file to a test case
577 test_case (TestCase): Test case to attach the file to
578 file (Path): Path to the file to attach
580 save_path: Path = suite_spec.test_failure_artifacts_path / file.name
581 shutil.copyfile(file, save_path)
586 mode: RunMode, nproc: int, suite_spec: SuiteSpec, verbose: bool = False) -> TestCase:
594 mode (RunMode): Output mode
597 …verbose (bool, optional): Print detailed output for all runs, not just failures. Defaults to False.
609 run_args[i] = arg.replace('{ceed_resource}', backend.replace('/', '-'))
613 run_args = ['mpiexec', '-n', f'{nproc}', *run_args]
620 … timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime()),
635 elapsed_sec=time.time() - start,
636 timestamp=time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(start)),
637 stdout=proc.stdout.decode('utf-8'),
638 stderr=proc.stderr.decode('utf-8'),
645 ref_csvs = [suite_spec.get_output_path(test, file)
646 for file in output_files if file.endswith('.csv')]
647 ref_ascii = [suite_spec.get_output_path(test, file)
648 for file in output_files if not file.endswith('.csv')]
652 ref_cgns = [suite_spec.get_output_path(test, file) for file in output_files]
683 test_case.add_failure_info('stdout', output=''.join(diff))
685 test_case.add_failure_info('stdout', output=test_case.stdout)
686 # expected CSV output
694 test_case.add_failure_info('csv', output=f'{ref_csv} not found')
696 test_case.add_failure_info('csv', output=f'{out_file} not found')
710 test_case.add_failure_info(f'csv: {save_path}', output=diff)
713 # expected CGNS output
721 test_case.add_failure_info('cgns', output=f'{ref_cgn} not found')
723 test_case.add_failure_info('cgns', output=f'{out_file} not found')
730 test_case.add_failure_info(f'cgns: {save_path}', output=diff)
733 # expected ASCII output
741 test_case.add_failure_info('ascii', output=f'{ref_file} not found')
743 test_case.add_failure_info('ascii', output=f'{out_file} not found')
749 test_case.add_failure_info(f'ascii: {save_path}', output=diff)
769 …suite_spec: SuiteSpec, pool_size: int = 1, search: str = ".*", verbose: bool = False) -> TestSuite:
775 mode (RunMode): Output mode, either `RunMode.TAP` or `RunMode.JUNIT`
780 …verbose (bool, optional): Print detailed output for all runs, not just failures. Defaults to False.
815 print(f'{"" if subtest_ok else "not "}ok {i} - {test_case.category}')
820 def write_junit_xml(test_suite: TestSuite, batch: str = '') -> None:
821 """Write a JUnit XML file containing the results of a `TestSuite`
831 def has_failures(test_suite: TestSuite) -> bool: