Add reframe tests for Snapshots

reframe_tests/tests/base_tests.py

@@ -7,6 +7,7 @@
 import os
 from pathlib import Path
 from shutil import copyfile
+from typing import Literal
 
 import reframe as rfm
 import reframe.utility.sanity as sn
@@ -60,6 +61,16 @@ class CreatePyenvTest(rfm.RunOnlyRegressionTest):
             """
             self.prerun_cmds.insert(3, "CC=cc CXX=CC FC=ftn python -m pip install cloudpickle")
 
+            """
+            A default pip install of h5py does not have MPI support so
+            it needs to built as described here:
+            https://docs.h5py.org/en/stable/mpi.html#building-against-parallel-hdf5
+            """
+            self.modules.append("cray-hdf5-parallel")
+            self.prerun_cmds.insert(
+                4, "CC=mpicc HDF5_MPI='ON' python -m pip install --no-binary=h5py h5py"
+            )
+
     @sanity_function
     def check_requirements_installed(self):
         """
@@ -95,6 +106,8 @@ class GprMaxRegressionTest(rfm.RunOnlyRegressionTest):
 
     model = parameter()
     is_antenna_model = variable(bool, value=False)
+    has_receiver_output = variable(bool, value=True)
+    snapshots = variable(typ.List[str], value=[])
     sourcesdir = required
     extra_executable_opts = variable(typ.List[str], value=[])
     executable = "time -p python -m gprMax --log-level 10 --hide-progress-bars"
@@ -126,11 +139,21 @@ class GprMaxRegressionTest(rfm.RunOnlyRegressionTest):
         path_to_pyenv = os.path.join(CreatePyenvTest(part="login").stagedir, PATH_TO_PYENV)
         self.prerun_cmds.append(f"source {path_to_pyenv}")
 
+    def build_reference_filepath(self, suffix: str = "") -> str:
+        filename = f"{self.short_name}_{suffix}" if len(suffix) > 0 else self.short_name
+        reference_file = Path("regression_checks", filename).with_suffix(".h5")
+        return os.path.abspath(reference_file)
+
+    def build_snapshot_filepath(self, snapshot: str) -> str:
+        return os.path.join(f"{self.model}_snaps", snapshot)
+
     @run_after("setup")
-    def setup_reference_file(self):
-        """Build reference file path"""
-        self.reference_file = Path("regression_checks", self.short_name).with_suffix(".h5")
-        self.reference_file = os.path.abspath(self.reference_file)
+    def setup_reference_files(self):
+        """Build reference file paths"""
+        self.reference_file = self.build_reference_filepath()
+        self.snapshot_reference_files = []
+        for snapshot in self.snapshots:
+            self.snapshot_reference_files.append(self.build_reference_filepath(snapshot))
 
     @run_after("setup", always_last=True)
     def configure_test_run(self, input_file_ext: str = ".in"):
@@ -143,10 +166,13 @@ class GprMaxRegressionTest(rfm.RunOnlyRegressionTest):
         self.output_file = f"{self.model}.h5"
         self.executable_opts = [self.input_file, "-o", self.output_file]
         self.executable_opts += self.extra_executable_opts
-        self.postrun_cmds = [
-            f"python -m reframe_tests.utilities.plotting {self.output_file} {self.reference_file} -m {self.model}"
-        ]
-        self.keep_files = [self.input_file, self.output_file, f"{self.model}.pdf"]
+        self.keep_files = [self.input_file, *self.snapshots]
+
+        if self.has_receiver_output:
+            self.postrun_cmds = [
+                f"python -m reframe_tests.utilities.plotting {self.output_file} {self.reference_file} -m {self.model}"
+            ]
+            self.keep_files += [self.output_file, f"{self.model}.pdf"]
 
         if self.is_antenna_model:
             self.postrun_cmds = [
@@ -196,7 +222,7 @@ class GprMaxRegressionTest(rfm.RunOnlyRegressionTest):
                 f"Input file '{self.input_file}' not present in source directory '{self.sourcesdir}'",
             )
 
-    def test_simulation_complete(self):
+    def test_simulation_complete(self) -> Literal[True]:
         """Check simulation completed successfully"""
         return sn.assert_not_found(
             r"(?i)error",
@@ -206,34 +232,99 @@ class GprMaxRegressionTest(rfm.RunOnlyRegressionTest):
             r"=== Simulation completed in ", self.stdout, "Simulation did not complete"
         )
 
-    def test_output_regression(self):
-        cmd = [f"h5diff {self.output_file} {self.reference_file}"]
-        if self.current_system.name == "archer2":
-            cmd.append("module load cray-hdf5")
+    def run_regression_check(
+        self, output_file: str, reference_file: str, error_msg: str
+    ) -> Literal[True]:
+        """Compare two provided .h5 files using h5diff
+
+        Args:
+            output_file: Filepath of .h5 file output by the test.
+            reference_file: Filepath of reference .h5 file containing
+                the expected output.
+        """
+        if self.current_system.name == "archer2":
+            h5diff = "/opt/cray/pe/hdf5/default/bin/h5diff"
+        else:
+            h5diff = "h5diff"
+
+        h5diff_output = osext.run_command([h5diff, os.path.abspath(output_file), reference_file])
 
-        h5diff_output = osext.run_command(cmd, check=True, shell=True)
         return sn.assert_false(
             h5diff_output.stdout,
             (
-                f"Found h5diff output (see '{path_join(self.stagedir, self.stdout)}')\n"
-                f"For more details run: 'h5diff {os.path.abspath(self.output_file)} {self.reference_file}'\n"
-                f"To re-create regression file, delete '{self.reference_file}' and rerun the test."
+                f"{error_msg}\n"
+                f"For more details run: 'h5diff {os.path.abspath(output_file)} {reference_file}'\n"
+                f"To re-create regression file, delete '{reference_file}' and rerun the test."
             ),
         )
 
-    @sanity_function
-    def regression_check(self):
+    def test_output_regression(self) -> Literal[True]:
+        """Compare the test output with the reference file.
+
+        If the test contains any receivers, a regression check is run,
+        otherwise it checks the test did not generate an output file.
         """
-        Perform regression check by checking for the h5diff output.
-        Create reference file from the test output if it does not exist.
-        """
-        if sn.path_exists(self.reference_file):
-            return self.test_simulation_complete() and self.test_output_regression()
-        else:
-            copyfile(self.output_file, self.reference_file)
-            return sn.assert_true(
-                False, f"No reference file exists. Creating... '{self.reference_file}'"
+        if self.has_receiver_output:
+            return self.run_regression_check(
+                self.output_file, self.reference_file, "Failed output regresssion check"
             )
+        else:
+            return sn.assert_false(
+                sn.path_exists(self.output_file),
+                f"Unexpected output file found: '{self.output_file}'",
+            )
+
+    def test_snapshot_regression(self) -> Literal[True]:
+        """Compare the snapshot outputs with reference files.
+
+        Generates a regression check for each snapshot. Each regression
+        check is a deffered expression, so they all need to be returned
+        so that they are each evaluated.
+        """
+        regression_checks = []
+        for index, snapshot in enumerate(self.snapshots):
+            snapshot_path = self.build_snapshot_filepath(snapshot)
+            reference_file = self.snapshot_reference_files[index]
+            regression_checks.append(
+                self.run_regression_check(
+                    snapshot_path, reference_file, f"Failed snapshot regresssion check '{snapshot}'"
+                )
+            )
+
+        # sn.assert_true is not strictly necessary
+        return sn.assert_true(sn.all(regression_checks))
+
+    @sanity_function
+    def regression_check(self) -> Literal[True]:
+        """Perform regression check for the test output and snapshots
+
+        If not all the reference files exist, then create all the
+        missing reference files from the test output and fail the test.
+        """
+        if (not self.has_receiver_output or sn.path_exists(self.reference_file)) and sn.all(
+            [sn.path_exists(path) for path in self.snapshot_reference_files]
+        ):
+            return (
+                self.test_simulation_complete()
+                and self.test_output_regression()
+                and self.test_snapshot_regression()
+            )
+        else:
+            error_messages = []
+            if self.has_receiver_output and not sn.path_exists(self.reference_file):
+                copyfile(self.output_file, self.reference_file)
+                error_messages.append(
+                    f"Output reference file does not exist. Creating... '{self.reference_file}'"
+                )
+            for index, snapshot in enumerate(self.snapshots):
+                reference_file = self.snapshot_reference_files[index]
+                if not sn.path_exists(reference_file):
+                    copyfile(self.build_snapshot_filepath(snapshot), reference_file)
+                    error_messages.append(
+                        f"Snapshot '{snapshot}' reference file does not exist. Creating... '{reference_file}'"
+                    )
+
+            return sn.assert_true(False, "\n".join(error_messages))
 
     @performance_function("s", perf_key="run_time")
     def extract_run_time(self):
@@ -333,18 +424,21 @@ class GprMaxTaskfarmRegressionTest(GprMaxBScanRegressionTest):
         super().inject_dependencies()
 
     @run_after("init")
-    def setup_source_directory(self):
-        """Set the source directory to the same as the serial test"""
+    def set_variables_from_serial_dependency(self):
+        """Set test dependencies to the same as the serial test"""
         self.sourcesdir = str(self.serial_dependency.sourcesdir)
+        self.has_receiver_output = bool(self.serial_dependency.has_receiver_output)
+        self.snapshots = list(self.serial_dependency.snapshots)
 
     @run_after("setup")
-    def setup_reference_file(self):
+    def setup_reference_files(self):
         """
-        Set the reference file regression check to the output of the
+        Set the reference file regression checks to the output of the
         serial test
         """
         target = self.getdep(self._get_variant())
         self.reference_file = os.path.join(target.stagedir, target.output_file)
+        self.snapshot_reference_files = target.snapshot_reference_files
 
 
 class GprMaxMPIRegressionTest(GprMaxRegressionTest):
@@ -372,15 +466,18 @@ class GprMaxMPIRegressionTest(GprMaxRegressionTest):
         super().inject_dependencies()
 
     @run_after("init")
-    def setup_source_directory(self):
-        """Set the source directory to the same as the serial test"""
+    def set_variables_from_serial_dependency(self):
+        """Set test dependencies to the same as the serial test"""
         self.sourcesdir = str(self.serial_dependency.sourcesdir)
+        self.has_receiver_output = bool(self.serial_dependency.has_receiver_output)
+        self.snapshots = list(self.serial_dependency.snapshots)
 
     @run_after("setup")
-    def setup_reference_file(self):
+    def setup_reference_files(self):
         """
-        Set the reference file regression check to the output of the
+        Set the reference file regression checks to the output of the
         serial test
         """
         target = self.getdep(self._get_variant())
         self.reference_file = os.path.join(target.stagedir, target.output_file)
+        self.snapshot_reference_files = target.snapshot_reference_files

reframe_tests/tests/src/snapshot_tests/whole_domain.in

@@ -6,4 +6,4 @@
 #waveform: gaussiandot 1 1e9 myWave
 #hertzian_dipole: z 0.050 0.050 0.050 myWave
 
-#snapshot: 0 0 0 0.100 0.100 0.100 0.01 0.01 0.01 2e-9 snapshot_3.h5
+#snapshot: 0 0 0 0.100 0.100 0.100 0.01 0.01 0.01 2e-9 snapshot.h5

reframe_tests/tests/src/snapshot_tests/whole_domain_2d.in

@@ -6,4 +6,4 @@
 #waveform: gaussiandot 1 1e9 myWave
 #hertzian_dipole: z 0.050 0.050 0 myWave
 
-#snapshot: 0 0 0 0.100 0.100 0.001 0.01 0.01 0.01 2e-9 snapshot_3.h5
+#snapshot: 0 0 0 0.100 0.100 0.001 0.01 0.01 0.01 2e-9 snapshot.h5

reframe_tests/tests/test_snapshots.py

@@ -0,0 +1,42 @@
+import reframe as rfm
+from reframe.core.builtins import parameter
+
+from reframe_tests.tests.base_tests import GprMaxMPIRegressionTest, GprMaxRegressionTest
+
+
+@rfm.simple_test
+class Test2DSnapshot(GprMaxRegressionTest):
+    tags = {"test", "serial", "2d", "waveform", "hertzian_dipole", "snapshot"}
+    sourcesdir = "src/snapshot_tests"
+    model = parameter(["whole_domain_2d"])
+    has_receiver_output = False
+    snapshots = ["snapshot.h5"]
+
+
+@rfm.simple_test
+class TestSnapshot(GprMaxRegressionTest):
+    tags = {"test", "serial", "2d", "waveform", "hertzian_dipole", "snapshot"}
+    sourcesdir = "src/snapshot_tests"
+    model = parameter(["whole_domain"])
+    has_receiver_output = False
+    snapshots = ["snapshot.h5"]
+
+
+"""Test MPI Functionality
+"""
+
+
+@rfm.simple_test
+class Test2DSnapshotMpi(GprMaxMPIRegressionTest):
+    tags = {"test", "mpi", "2d", "waveform", "hertzian_dipole", "snapshot"}
+    mpi_layout = parameter([[2, 2, 1], [3, 3, 1], [4, 4, 1]])
+    serial_dependency = Test2DSnapshot
+    model = serial_dependency.model
+
+
+@rfm.simple_test
+class TestSnapshotMpi(GprMaxMPIRegressionTest):
+    tags = {"test", "mpi", "2d", "waveform", "hertzian_dipole", "snapshot"}
+    mpi_layout = parameter([[2, 2, 2], [3, 3, 3], [4, 4, 4]])
+    serial_dependency = TestSnapshot
+    model = serial_dependency.model