diff --git a/reframe_tests/tests/base_tests.py b/reframe_tests/tests/base_tests.py index a1b3e4c4..2e4b2763 100644 --- a/reframe_tests/tests/base_tests.py +++ b/reframe_tests/tests/base_tests.py @@ -22,12 +22,13 @@ from reframe.core.builtins import ( sanity_function, variable, ) -from reframe.utility import udeps +from reframe.utility import osext, udeps from gprMax.receivers import Rx from reframe_tests.utilities.deferrable import path_join -GPRMAX_ROOT_DIR = Path(__file__).parent.parent.resolve() +TESTS_ROOT_DIR = Path(__file__).parent +GPRMAX_ROOT_DIR = Path(__file__).parent.parent.parent.resolve() PATH_TO_PYENV = os.path.join(".venv", "bin", "activate") @@ -126,6 +127,12 @@ class GprMaxRegressionTest(rfm.RunOnlyRegressionTest): path_to_pyenv = os.path.join(CreatePyenvTest(part="login").stagedir, PATH_TO_PYENV) self.prerun_cmds.append(f"source {path_to_pyenv}") + @run_after("setup") + def setup_reference_file(self): + """Build reference file path""" + self.reference_file = Path("regression_checks", self.short_name).with_suffix(".h5") + self.reference_file = os.path.abspath(self.reference_file) + @run_after("setup", always_last=True) def configure_test_run(self, input_file_ext: str = ".in"): """Configure gprMax commandline arguments and plot outputs @@ -138,7 +145,7 @@ class GprMaxRegressionTest(rfm.RunOnlyRegressionTest): self.executable_opts = [self.input_file, "-o", self.output_file] self.executable_opts += self.extra_executable_opts self.postrun_cmds = [ - f"python -m toolboxes.Plotting.plot_Ascan -save {self.output_file} --outputs {' '.join(self.rx_outputs)}" + f"python -m reframe_tests.utilities.plotting {self.output_file} {self.reference_file} -m {self.model}" ] self.keep_files = [self.input_file, self.output_file, f"{self.model}.pdf"] @@ -178,16 +185,17 @@ class GprMaxRegressionTest(rfm.RunOnlyRegressionTest): @run_before("run") def check_input_file_exists(self): """Skip test if input file does not exist""" - self.skip_if( - not os.path.exists(os.path.join(self.sourcesdir, self.input_file)), - f"Input file '{self.input_file}' not present in src directory '{self.sourcesdir}'", - ) - - @run_before("run") - def setup_reference_file(self): - """Build reference file path""" - self.reference_file = Path("regression_checks", self.short_name).with_suffix(".h5") - self.reference_file = os.path.abspath(self.reference_file) + # Current working directory will be where the reframe job was launched + # However reframe assumes the source directory is relative to the test file + with osext.change_dir(TESTS_ROOT_DIR): + self.skip_if( + not os.path.exists(self.sourcesdir), + f"Source directory '{self.sourcesdir}' does not exist. Current working directory: '{os.getcwd()}'", + ) + self.skip_if( + not os.path.exists(os.path.join(self.sourcesdir, self.input_file)), + f"Input file '{self.input_file}' not present in source directory '{self.sourcesdir}'", + ) @run_before("run", always_last=True) def setup_regression_check(self): @@ -317,6 +325,7 @@ class GprMaxBScanRegressionTest(GprMaxRegressionTest): class GprMaxTaskfarmRegressionTest(GprMaxBScanRegressionTest): serial_dependency: type[GprMaxRegressionTest] extra_executable_opts = ["-taskfarm"] + sourcesdir = "src" # Necessary so test is not skipped (set later) num_tasks = required @@ -338,20 +347,21 @@ class GprMaxTaskfarmRegressionTest(GprMaxBScanRegressionTest): """Set the source directory to the same as the serial test""" self.sourcesdir = str(self.serial_dependency.sourcesdir) - @run_before("run") + @run_after("setup") def setup_reference_file(self): """ Set the reference file regression check to the output of the serial test """ target = self.getdep(self._get_variant()) - self.reference_file = os.path.join(target.stagedir, str(self.output_file)) + self.reference_file = os.path.join(target.stagedir, target.output_file) class GprMaxMPIRegressionTest(GprMaxRegressionTest): # TODO: Make this a variable serial_dependency: type[GprMaxRegressionTest] mpi_layout = parameter() + sourcesdir = "src" # Necessary so test is not skipped (set later) @run_after("setup", always_last=True) def configure_test_run(self): @@ -376,11 +386,11 @@ class GprMaxMPIRegressionTest(GprMaxRegressionTest): """Set the source directory to the same as the serial test""" self.sourcesdir = str(self.serial_dependency.sourcesdir) - @run_before("run") + @run_after("setup") def setup_reference_file(self): """ Set the reference file regression check to the output of the serial test """ target = self.getdep(self._get_variant()) - self.reference_file = os.path.join(target.stagedir, str(self.output_file)) + self.reference_file = os.path.join(target.stagedir, target.output_file) diff --git a/reframe_tests/utilities/data.py b/reframe_tests/utilities/data.py index 142d7957..abab8fe1 100644 --- a/reframe_tests/utilities/data.py +++ b/reframe_tests/utilities/data.py @@ -1,7 +1,9 @@ import logging +from typing import Tuple import h5py import numpy as np +import numpy.typing as npt from gprMax.utilities.logging import logging_config @@ -12,7 +14,7 @@ logging_config(name=__name__) FIELD_COMPONENTS_BASE_PATH = "/rxs/rx1/" -def get_data_from_h5_file(h5_filepath): +def get_data_from_h5_file(h5_filepath: str) -> Tuple[npt.NDArray, npt.NDArray]: with h5py.File(h5_filepath, "r") as h5_file: # Get available field output component names and datatype field_components = list(h5_file[FIELD_COMPONENTS_BASE_PATH].keys()) @@ -23,7 +25,9 @@ def get_data_from_h5_file(h5_filepath): if len(shape) == 1: data = np.zeros((h5_file.attrs["Iterations"], len(field_components)), dtype=dtype) else: # Merged B-scan data - data = np.zeros((h5_file.attrs["Iterations"], len(field_components), shape[1]), dtype=dtype) + data = np.zeros( + (h5_file.attrs["Iterations"], len(field_components), shape[1]), dtype=dtype + ) for index, field_component in enumerate(field_components): data[:, index] = h5_file[FIELD_COMPONENTS_BASE_PATH + str(field_component)] if np.any(np.isnan(data[:, index])): @@ -36,18 +40,21 @@ def get_data_from_h5_file(h5_filepath): return time, data -def calculate_diffs(test_data, ref_data): +def calculate_diffs(test_data: npt.NDArray, ref_data: npt.NDArray) -> npt.NDArray: diffs = np.zeros(test_data.shape, dtype=np.float64) for i in range(test_data.shape[1]): maxi = np.amax(np.abs(ref_data[:, i])) diffs[:, i] = np.divide( - np.abs(ref_data[:, i] - test_data[:, i]), maxi, out=np.zeros_like(ref_data[:, i]), where=maxi != 0 + np.abs(ref_data[:, i] - test_data[:, i]), + maxi, + out=np.zeros_like(ref_data[:, i]), + where=maxi != 0, ) # Replace any division by zero with zero # Calculate power (ignore warning from taking a log of any zero values) with np.errstate(divide="ignore"): diffs[:, i] = 20 * np.log10(diffs[:, i]) # Replace any NaNs or Infs from zero division - diffs[:, i][np.invert(np.isfinite(diffs[:, i]))] = 0 + # diffs[:, i][np.invert(np.isfinite(diffs[:, i]))] = 0 return diffs diff --git a/reframe_tests/utilities/plotting.py b/reframe_tests/utilities/plotting.py index 2f501702..da4b8709 100644 --- a/reframe_tests/utilities/plotting.py +++ b/reframe_tests/utilities/plotting.py @@ -1,6 +1,10 @@ +import argparse + import numpy as np from matplotlib import pyplot as plt +from reframe_tests.utilities.data import calculate_diffs, get_data_from_h5_file + def _plot_data(subplots, time, data, label=None, colour="r", line_style="-"): for i in range(data.shape[1]): @@ -76,6 +80,10 @@ def plot_diffs(time, diffs, plot_min=-160): x_max = np.max(time) y_max = np.max(diffs) + + if not np.isfinite(y_max): + y_max = 0 + for i, ax in enumerate(fig.axes): ax.set_ylabel(ylabels[i]) ax.set_xlim(0, x_max) @@ -83,3 +91,24 @@ def plot_diffs(time, diffs, plot_min=-160): ax.grid() return fig + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("input_file", help="Path to input file") + parser.add_argument("reference_file", help="Path to reference file") + parser.add_argument("-model-name", "-name", "-n", help="Name of the model", default="model") + + args = parser.parse_args() + + input_time, input_data = get_data_from_h5_file(args.input_file) + ref_time, ref_data = get_data_from_h5_file(args.reference_file) + + figure = plot_dataset_comparison(input_time, input_data, ref_time, ref_data, args.model_name) + figure.tight_layout(h_pad=3, w_pad=4, pad=2) + figure.savefig(f"{args.model_name}.pdf", dpi=300) + + diffs = calculate_diffs(input_data, ref_data) + figure = plot_diffs(input_time, diffs) + figure.tight_layout(h_pad=3, w_pad=4, pad=2) + figure.savefig(f"{args.model_name}_diffs.pdf", dpi=300)