Add reference data to output plot and plot diffs

reframe_tests/tests/base_tests.py

@@ -22,12 +22,13 @@ from reframe.core.builtins import (
     sanity_function,
     variable,
 )
-from reframe.utility import udeps
+from reframe.utility import osext, udeps
 
 from gprMax.receivers import Rx
 from reframe_tests.utilities.deferrable import path_join
 
-GPRMAX_ROOT_DIR = Path(__file__).parent.parent.resolve()
+TESTS_ROOT_DIR = Path(__file__).parent
+GPRMAX_ROOT_DIR = Path(__file__).parent.parent.parent.resolve()
 PATH_TO_PYENV = os.path.join(".venv", "bin", "activate")
 
 
@@ -126,6 +127,12 @@ class GprMaxRegressionTest(rfm.RunOnlyRegressionTest):
         path_to_pyenv = os.path.join(CreatePyenvTest(part="login").stagedir, PATH_TO_PYENV)
         self.prerun_cmds.append(f"source {path_to_pyenv}")
 
+    @run_after("setup")
+    def setup_reference_file(self):
+        """Build reference file path"""
+        self.reference_file = Path("regression_checks", self.short_name).with_suffix(".h5")
+        self.reference_file = os.path.abspath(self.reference_file)
+
     @run_after("setup", always_last=True)
     def configure_test_run(self, input_file_ext: str = ".in"):
         """Configure gprMax commandline arguments and plot outputs
@@ -138,7 +145,7 @@ class GprMaxRegressionTest(rfm.RunOnlyRegressionTest):
         self.executable_opts = [self.input_file, "-o", self.output_file]
         self.executable_opts += self.extra_executable_opts
         self.postrun_cmds = [
-            f"python -m toolboxes.Plotting.plot_Ascan -save {self.output_file} --outputs {' '.join(self.rx_outputs)}"
+            f"python -m reframe_tests.utilities.plotting {self.output_file} {self.reference_file} -m {self.model}"
         ]
         self.keep_files = [self.input_file, self.output_file, f"{self.model}.pdf"]
 
@@ -178,16 +185,17 @@ class GprMaxRegressionTest(rfm.RunOnlyRegressionTest):
     @run_before("run")
     def check_input_file_exists(self):
         """Skip test if input file does not exist"""
-        self.skip_if(
+        # Current working directory will be where the reframe job was launched
-            not os.path.exists(os.path.join(self.sourcesdir, self.input_file)),
+        # However reframe assumes the source directory is relative to the test file
-            f"Input file '{self.input_file}' not present in src directory '{self.sourcesdir}'",
+        with osext.change_dir(TESTS_ROOT_DIR):
-        )
+            self.skip_if(
-
+                not os.path.exists(self.sourcesdir),
-    @run_before("run")
+                f"Source directory '{self.sourcesdir}' does not exist. Current working directory: '{os.getcwd()}'",
-    def setup_reference_file(self):
+            )
-        """Build reference file path"""
+            self.skip_if(
-        self.reference_file = Path("regression_checks", self.short_name).with_suffix(".h5")
+                not os.path.exists(os.path.join(self.sourcesdir, self.input_file)),
-        self.reference_file = os.path.abspath(self.reference_file)
+                f"Input file '{self.input_file}' not present in source directory '{self.sourcesdir}'",
+            )
 
     @run_before("run", always_last=True)
     def setup_regression_check(self):
@@ -317,6 +325,7 @@ class GprMaxBScanRegressionTest(GprMaxRegressionTest):
 class GprMaxTaskfarmRegressionTest(GprMaxBScanRegressionTest):
     serial_dependency: type[GprMaxRegressionTest]
     extra_executable_opts = ["-taskfarm"]
+    sourcesdir = "src"  # Necessary so test is not skipped (set later)
 
     num_tasks = required
 
@@ -338,20 +347,21 @@ class GprMaxTaskfarmRegressionTest(GprMaxBScanRegressionTest):
         """Set the source directory to the same as the serial test"""
         self.sourcesdir = str(self.serial_dependency.sourcesdir)
 
-    @run_before("run")
+    @run_after("setup")
     def setup_reference_file(self):
         """
         Set the reference file regression check to the output of the
         serial test
         """
         target = self.getdep(self._get_variant())
-        self.reference_file = os.path.join(target.stagedir, str(self.output_file))
+        self.reference_file = os.path.join(target.stagedir, target.output_file)
 
 
 class GprMaxMPIRegressionTest(GprMaxRegressionTest):
     # TODO: Make this a variable
     serial_dependency: type[GprMaxRegressionTest]
     mpi_layout = parameter()
+    sourcesdir = "src"  # Necessary so test is not skipped (set later)
 
     @run_after("setup", always_last=True)
     def configure_test_run(self):
@@ -376,11 +386,11 @@ class GprMaxMPIRegressionTest(GprMaxRegressionTest):
         """Set the source directory to the same as the serial test"""
         self.sourcesdir = str(self.serial_dependency.sourcesdir)
 
-    @run_before("run")
+    @run_after("setup")
     def setup_reference_file(self):
         """
         Set the reference file regression check to the output of the
         serial test
         """
         target = self.getdep(self._get_variant())
-        self.reference_file = os.path.join(target.stagedir, str(self.output_file))
+        self.reference_file = os.path.join(target.stagedir, target.output_file)

reframe_tests/utilities/data.py

@@ -1,7 +1,9 @@
 import logging
+from typing import Tuple
 
 import h5py
 import numpy as np
+import numpy.typing as npt
 
 from gprMax.utilities.logging import logging_config
 
@@ -12,7 +14,7 @@ logging_config(name=__name__)
 FIELD_COMPONENTS_BASE_PATH = "/rxs/rx1/"
 
 
-def get_data_from_h5_file(h5_filepath):
+def get_data_from_h5_file(h5_filepath: str) -> Tuple[npt.NDArray, npt.NDArray]:
     with h5py.File(h5_filepath, "r") as h5_file:
         # Get available field output component names and datatype
         field_components = list(h5_file[FIELD_COMPONENTS_BASE_PATH].keys())
@@ -23,7 +25,9 @@ def get_data_from_h5_file(h5_filepath):
         if len(shape) == 1:
             data = np.zeros((h5_file.attrs["Iterations"], len(field_components)), dtype=dtype)
         else:  # Merged B-scan data
-            data = np.zeros((h5_file.attrs["Iterations"], len(field_components), shape[1]), dtype=dtype)
+            data = np.zeros(
+                (h5_file.attrs["Iterations"], len(field_components), shape[1]), dtype=dtype
+            )
         for index, field_component in enumerate(field_components):
             data[:, index] = h5_file[FIELD_COMPONENTS_BASE_PATH + str(field_component)]
             if np.any(np.isnan(data[:, index])):
@@ -36,18 +40,21 @@ def get_data_from_h5_file(h5_filepath):
     return time, data
 
 
-def calculate_diffs(test_data, ref_data):
+def calculate_diffs(test_data: npt.NDArray, ref_data: npt.NDArray) -> npt.NDArray:
     diffs = np.zeros(test_data.shape, dtype=np.float64)
     for i in range(test_data.shape[1]):
         maxi = np.amax(np.abs(ref_data[:, i]))
         diffs[:, i] = np.divide(
-            np.abs(ref_data[:, i] - test_data[:, i]), maxi, out=np.zeros_like(ref_data[:, i]), where=maxi != 0
+            np.abs(ref_data[:, i] - test_data[:, i]),
+            maxi,
+            out=np.zeros_like(ref_data[:, i]),
+            where=maxi != 0,
         )  # Replace any division by zero with zero
 
         # Calculate power (ignore warning from taking a log of any zero values)
         with np.errstate(divide="ignore"):
             diffs[:, i] = 20 * np.log10(diffs[:, i])
         # Replace any NaNs or Infs from zero division
-        diffs[:, i][np.invert(np.isfinite(diffs[:, i]))] = 0
+        # diffs[:, i][np.invert(np.isfinite(diffs[:, i]))] = 0
 
     return diffs

reframe_tests/utilities/plotting.py

@@ -1,6 +1,10 @@
+import argparse
+
 import numpy as np
 from matplotlib import pyplot as plt
 
+from reframe_tests.utilities.data import calculate_diffs, get_data_from_h5_file
+
 
 def _plot_data(subplots, time, data, label=None, colour="r", line_style="-"):
     for i in range(data.shape[1]):
@@ -76,6 +80,10 @@ def plot_diffs(time, diffs, plot_min=-160):
 
     x_max = np.max(time)
     y_max = np.max(diffs)
+
+    if not np.isfinite(y_max):
+        y_max = 0
+
     for i, ax in enumerate(fig.axes):
         ax.set_ylabel(ylabels[i])
         ax.set_xlim(0, x_max)
@@ -83,3 +91,24 @@ def plot_diffs(time, diffs, plot_min=-160):
         ax.grid()
 
     return fig
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument("input_file", help="Path to input file")
+    parser.add_argument("reference_file", help="Path to reference file")
+    parser.add_argument("-model-name", "-name", "-n", help="Name of the model", default="model")
+
+    args = parser.parse_args()
+
+    input_time, input_data = get_data_from_h5_file(args.input_file)
+    ref_time, ref_data = get_data_from_h5_file(args.reference_file)
+
+    figure = plot_dataset_comparison(input_time, input_data, ref_time, ref_data, args.model_name)
+    figure.tight_layout(h_pad=3, w_pad=4, pad=2)
+    figure.savefig(f"{args.model_name}.pdf", dpi=300)
+
+    diffs = calculate_diffs(input_data, ref_data)
+    figure = plot_diffs(input_time, diffs)
+    figure.tight_layout(h_pad=3, w_pad=4, pad=2)
+    figure.savefig(f"{args.model_name}_diffs.pdf", dpi=300)