Refactor base reframe tests

reframe_tests/base_tests.py

@@ -5,9 +5,14 @@ from shutil import copyfile
 
 import reframe as rfm
 import reframe.utility.sanity as sn
+import reframe.utility.typecheck as typ
+from matplotlib.tri import TriContourSet
+from numpy import product
 from reframe.core.builtins import (
+    parameter,
     performance_function,
     require_deps,
+    required,
     run_after,
     run_before,
     sanity_function,
@@ -73,13 +78,21 @@ class CreatePyenvTest(rfm.RunOnlyRegressionTest):
         )
 
 
-class GprMaxBaseTest(rfm.RunOnlyRegressionTest):
+class GprMaxRegressionTest(rfm.RunOnlyRegressionTest):
     valid_systems = ["archer2:compute"]
     valid_prog_environs = ["PrgEnv-gnu"]
     modules = ["cray-python"]
-    executable = "time -p python -m gprMax --log-level 25"
+
+    num_cpus_per_task = 16
     exclusive_access = True
 
+    model = variable(str)
+    # sourcesdir = required
+    extra_executable_opts = variable(typ.List[str], value=[])
+    executable = "time -p python -m gprMax --log-level 25"
+
+    h5diff_header = f"{'=' * 10} h5diff output {'=' * 10}"
+
     @run_after("init")
     def setup_env_vars(self):
         """Set OMP_NUM_THREADS environment variable from num_cpus_per_task"""
@@ -105,10 +118,37 @@ class GprMaxBaseTest(rfm.RunOnlyRegressionTest):
         path_to_pyenv = os.path.join(CreatePyenvTest(part="login").stagedir, PATH_TO_PYENV)
         self.prerun_cmds.append(f"source {path_to_pyenv}")
 
-    @sanity_function
+    @run_after("init", always_last=True)
+    def configure_test_run(self, input_file_ext: str = ".in"):
+        self.input_file = f"{self.model}{input_file_ext}"
+        self.skip_if(
+            not os.path.exists(self.input_file),
+            f"Input file '{self.input_file}' not present in src directory '{self.sourcesdir}'",
+        )
+        self.output_file = f"{self.model}.h5"
+        self.executable_opts = [self.input_file, "-o", self.output_file]
+        self.executable_opts += self.extra_executable_opts
+        self.postrun_cmds = [f"python -m toolboxes.Plotting.plot_Ascan -save {self.output_file}"]
+        self.keep_files = [self.input_file, self.output_file, f"{self.model}.pdf"]
+
+    @run_before("run")
+    def setup_reference_file(self):
+        """Build reference file path"""
+        self.reference_file = Path("regression_checks", self.short_name).with_suffix(".h5")
+        self.reference_file = os.path.abspath(self.reference_file)
+
+    @run_before("run", always_last=True)
+    def setup_regression_check(self):
+        """Add h5diff command to run after the test"""
+        if self.current_system.name == "archer2":
+            self.modules.append("cray-hdf5")
+
+        if os.path.exists(self.reference_file):
+            self.postrun_cmds.append(f"echo {self.h5diff_header}")
+            self.postrun_cmds.append(f"h5diff {self.output_file} {self.reference_file}")
+
     def test_simulation_complete(self):
         """Check simulation completed successfully"""
-        # TODO: Check for correctness/regression rather than just completing
         return sn.assert_not_found(
             r"(?i)error",
             self.stderr,
@@ -117,6 +157,34 @@ class GprMaxBaseTest(rfm.RunOnlyRegressionTest):
             r"=== Simulation completed in ", self.stdout, "Simulation did not complete"
         )
 
+    @sanity_function
+    def regression_check(self):
+        """
+        Perform regression check by checking for the h5diff output.
+        Create reference file from the test output if it does not exist.
+        """
+        if sn.path_exists(self.reference_file):
+            h5diff_output = sn.extractsingle(
+                f"{self.h5diff_header}\n(?P<h5diff>[\S\s]*)", self.stdout, "h5diff"
+            )
+            return (
+                self.test_simulation_complete()
+                and sn.assert_found(self.h5diff_header, self.stdout, "Failed to find h5diff header")
+                and sn.assert_false(
+                    h5diff_output,
+                    (
+                        f"Found h5diff output (see '{path_join(self.stagedir, self.stdout)}')\n"
+                        f"For more details run: 'h5diff {os.path.abspath(self.output_file)} {self.reference_file}'\n"
+                        f"To re-create regression file, delete '{self.reference_file}' and rerun the test."
+                    ),
+                )
+            )
+        else:
+            copyfile(self.output_file, self.reference_file)
+            return sn.assert_true(
+                False, f"No reference file exists. Creating... '{self.reference_file}'"
+            )
+
     @performance_function("s", perf_key="run_time")
     def extract_run_time(self):
         """Extract total runtime from the last task to complete"""
@@ -167,54 +235,74 @@ class GprMaxBaseTest(rfm.RunOnlyRegressionTest):
         return hours * 3600 + minutes * 60 + seconds
 
 
-class GprMaxRegressionTest(GprMaxBaseTest):
-    input_file = variable(str)
-    output_file = variable(str)
+class GprMaxAPIRegressionTest(GprMaxRegressionTest):
+    executable = "time -p python"
 
-    h5diff_header = f"{'=' * 10} h5diff output {'=' * 10}"
+    @run_after("init", always_last=True)
+    def configure_test_run(self):
+        super().configure_test_run(input_file_ext=".py")
+
+
+class GprMaxBScanRegressionTest(GprMaxRegressionTest):
+    num_models = variable(int)
+
+    @run_after("init", always_last=True)
+    def configure_test_run(self):
+        self.input_file = f"{self.model}.in"
+        self.output_file = f"{self.model}_merged.h5"
+        self.executable_opts = [self.input_file, "-n", str(self.num_models)]
+        self.postrun_cmds = [
+            f"python -m toolboxes.Utilities.outputfiles_merge {self.model}",
+            f"python -m toolboxes.Plotting.plot_Bscan -save {self.output_file} Ez",
+        ]
+        self.keep_files = [self.input_file, self.output_file, "{self.model}_merged.pdf"]
+
+
+class GprMaxTaskfarmRegressionTest(GprMaxBScanRegressionTest):
+    serial_dependecy = variable(type[GprMaxRegressionTest])
+    extra_executable_opts = ["-taskfarm"]
+
+    num_tasks = required
+
+    @run_after("init")
+    def inject_dependencies(self):
+        """Test depends on the Python virtual environment building correctly"""
+        variant = self.serial_dependecy.get_variant_nums(
+            model=lambda m: m == self.model, num_models=lambda n: n == self.num_models
+        )
+        self.depends_on(self.serial_dependecy.variant_name(variant[0]), udeps.by_env)
+        super().inject_dependencies()
 
     @run_before("run")
     def setup_reference_file(self):
-        """Build reference file path"""
-        self.reference_file = Path("regression_checks", self.short_name).with_suffix(".h5")
-        self.reference_file = os.path.abspath(self.reference_file)
-
-    @run_before("run", always_last=True)
-    def setup_regression_check(self):
-        """Add h5diff command to run after the test"""
-        self.modules.append("cray-hdf5")
-        if os.path.exists(self.reference_file):
-            self.postrun_cmds.append(f"echo {self.h5diff_header}")
-            self.postrun_cmds.append(f"h5diff {self.output_file} {self.reference_file}")
-
-    @sanity_function
-    def regression_check(self):
-        """
-        Perform regression check by checking for the h5diff output.
-        Create reference file from the test output if it does not exist.
-        """
-        if sn.path_exists(self.reference_file):
-            h5diff_output = sn.extractsingle(
-                f"{self.h5diff_header}\n(?P<h5diff>[\S\s]*)", self.stdout, "h5diff"
-            )
-            return (
-                self.test_simulation_complete()
-                and sn.assert_found(self.h5diff_header, self.stdout, "Failed to find h5diff header")
-                and sn.assert_false(
-                    h5diff_output,
-                    (
-                        f"Found h5diff output (see '{path_join(self.stagedir, self.stdout)}')\n"
-                        f"For more details run: 'h5diff {os.path.abspath(self.output_file)} {self.reference_file}'\n"
-                        f"To re-create regression file, delete '{self.reference_file}' and rerun the test."
-                    ),
-                )
-            )
-        else:
-            copyfile(self.output_file, self.reference_file)
-            return sn.assert_true(
-                False, f"No reference file exists. Creating... '{self.reference_file}'"
-            )
+        """Add prerun command to load the built Python environment"""
+        variant = self.serial_dependecy.get_variant_nums(
+            model=lambda m: m == self.model, num_models=lambda n: n == self.num_models
+        )
+        target = self.getdep(self.serial_dependecy.variant_name(variant[0]))
+        self.reference_file = os.path.join(target.stagedir, str(self.output_file))
 
 
-class GprMaxAPIRegressionTest(GprMaxRegressionTest):
-    executable = "time -p python"
+class GprMaxMPIRegressionTest(GprMaxRegressionTest):
+    mpi_layout = variable(typ.List[int])
+    serial_dependecy = variable(type[GprMaxRegressionTest])
+
+    @run_after("init", always_last=True)
+    def configure_test_run(self):
+        self.num_tasks = product(self.mpi_layout, dtype=int)
+        self.extra_executable_opts = ["-mpi", " ".join(self.mpi_layout)]
+        super().configure_test_run()
+
+    @run_after("init")
+    def inject_dependencies(self):
+        """Test depends on the Python virtual environment building correctly"""
+        variant = self.serial_dependecy.get_variant_nums(model=lambda m: m == self.model)
+        self.depends_on(self.serial_dependecy.variant_name(variant[0]), udeps.by_env)
+        super().inject_dependencies()
+
+    @run_before("run")
+    def setup_reference_file(self):
+        """Add prerun command to load the built Python environment"""
+        variant = self.serial_dependecy.get_variant_nums(model=lambda m: m == self.model)
+        target = self.getdep(self.serial_dependecy.variant_name(variant[0]))
+        self.reference_file = os.path.join(target.stagedir, str(self.output_file))

reframe_tests/reframe_tests.py

@@ -1,9 +1,12 @@
-import os
-
 import reframe as rfm
-from base_tests import GprMaxAPIRegressionTest, GprMaxRegressionTest
-from reframe.core.builtins import parameter, require_deps, run_after, run_before
-from reframe.utility import udeps
+from base_tests import (
+    GprMaxAPIRegressionTest,
+    GprMaxBScanRegressionTest,
+    GprMaxMPIRegressionTest,
+    GprMaxRegressionTest,
+    GprMaxTaskfarmRegressionTest,
+)
+from reframe.core.builtins import parameter, run_after
 
 """ReFrame tests for basic functionality
 
@@ -14,79 +17,70 @@ from reframe.utility import udeps
 
 
 @rfm.simple_test
-class TaskfarmTest(GprMaxRegressionTest):
-    tags = {"test", "mpi", "taskfarm"}
+class TestBscan(GprMaxBScanRegressionTest):
+    tags = {"test", "bscan"}
 
-    model = parameter(["cylinder_Bscan_2D"])
-
-    num_mpi_tasks = parameter([8, 16])
-    num_cpus_per_task = 16
-
-    @run_after("init")
-    def setup_env_vars(self):
-        self.num_tasks = self.num_mpi_tasks
-        super().setup_env_vars()
-
-    @run_after("init")
-    def set_filenames(self):
-        self.input_file = f"{self.model}.in"
-        self.output_file = f"{self.model}_merged.h5"
-        self.executable_opts = [self.input_file, "-n", "64", "-taskfarm"]
-        self.postrun_cmds = [
-            f"python -m toolboxes.Utilities.outputfiles_merge {self.model}",
-            f"python -m toolboxes.Plotting.plot_Bscan -save {self.output_file} Ez",
-        ]
-        self.keep_files = [self.input_file, self.output_file, "{self.model}_merged.pdf"]
+    model = "cylinder_Bscan_2D"
+    num_models = 64
 
 
 @rfm.simple_test
-class BScanTest(GprMaxRegressionTest):
-    tags = {"test", "bscan"}
+class TestSingleNodeTaskfarm(GprMaxTaskfarmRegressionTest):
+    tags = {"test", "mpi", "taskfarm"}
 
-    model = parameter(["cylinder_Bscan_2D"])
+    model = "cylinder_Bscan_2D"
+    num_tasks = 8
+    num_tasks_per_node = 8
+    num_models = 64
+    serial_dependecy = TestBscan
 
-    num_cpus_per_task = 16
 
-    @run_after("init")
-    def set_filenames(self):
-        self.input_file = f"{self.model}.in"
-        self.output_file = f"{self.model}_merged.h5"
-        self.executable_opts = [self.input_file, "-n", "64"]
-        self.postrun_cmds = [
-            f"python -m toolboxes.Utilities.outputfiles_merge {self.model}",
-            f"python -m toolboxes.Plotting.plot_Bscan -save {self.output_file} Ez",
-        ]
-        self.keep_files = [self.input_file, self.output_file, "{self.model}_merged.pdf"]
+@rfm.simple_test
+class TestMultiNodeTaskfarm(GprMaxTaskfarmRegressionTest):
+    tags = {"test", "mpi", "taskfarm"}
+
+    model = "cylinder_Bscan_2D"
+    num_tasks = 32
+    num_tasks_per_node = 8
+    num_models = 64
+    serial_dependecy = TestBscan
+
+
+@rfm.simple_test
+class Test2DModelXY(GprMaxRegressionTest):
+    tags = {"test", "serial", "2d"}
+
+    model = "2D_EzHxHy"
+
+
+@rfm.simple_test
+class Test2DModelYZ(GprMaxRegressionTest):
+    tags = {"test", "serial", "2d"}
+
+    model = "2D_EzHxHy"
 
 
 @rfm.simple_test
 class BasicModelsTest(GprMaxRegressionTest):
     tags = {"test", "serial", "regression"}
 
-    # List of available basic test models
-    model = parameter(
-        [
-            "2D_ExHyHz",
-            "2D_EyHxHz",
-            "2D_EzHxHy",
-            "2D_ExHyHz_hs",
-            "cylinder_Ascan_2D",
-            "hertzian_dipole_fs",
-            "hertzian_dipole_hs",
-            "hertzian_dipole_dispersive",
-            "magnetic_dipole_fs",
-            "magnetic_dipole_hs",
-        ]
-    )
-    num_cpus_per_task = 16
-
-    @run_after("init")
-    def set_filenames(self):
-        self.input_file = f"{self.model}.in"
-        self.output_file = f"{self.model}.h5"
-        self.executable_opts = [self.input_file, "-o", self.output_file]
-        self.postrun_cmds = [f"python -m toolboxes.Plotting.plot_Ascan -save {self.output_file}"]
-        self.keep_files = [self.input_file, self.output_file, f"{self.model}.pdf"]
+    def __init__(self):
+        super().__init__()
+        # List of available basic test models
+        self.model = parameter(
+            [
+                "2D_ExHyHz",
+                "2D_EyHxHz",
+                "2D_EzHxHy",
+                "2D_ExHyHz_hs",
+                "cylinder_Ascan_2D",
+                "hertzian_dipole_fs",
+                "hertzian_dipole_hs",
+                "hertzian_dipole_dispersive",
+                "magnetic_dipole_fs",
+                "magnetic_dipole_hs",
+            ]
+        )
 
 
 @rfm.simple_test
@@ -94,112 +88,115 @@ class AntennaModelsTest(GprMaxRegressionTest):
     tags = {"test", "serial", "regression", "antenna"}
 
     # List of available antenna test models
-    model = parameter(
-        [
-            "antenna_wire_dipole_fs",
-        ]
-    )
-    num_cpus_per_task = 16
-
-    @run_after("init")
-    def set_filenames(self):
-        self.input_file = f"{self.model}.in"
-        self.output_file = f"{self.model}.h5"
-        self.executable_opts = [self.input_file, "-o", self.output_file]
-        self.postrun_cmds = [f"python -m toolboxes.Plotting.plot_Ascan -save {self.output_file}"]
-        self.postrun_cmds = [
-            f"python -m toolboxes.Plotting.plot_antenna_params -save {self.output_file}"
-        ]
-
-        antenna_t1_params = f"{self.model}_t1_params.pdf"
-        antenna_ant_params = f"{self.model}_ant_params.pdf"
-        plot_ascan_output = f"{self.model}.pdf"
-        geometry_view = f"{self.model}.vtu"
-        self.keep_files = [
-            self.input_file,
-            self.output_file,
-            antenna_t1_params,
-            antenna_ant_params,
-            plot_ascan_output,
-            geometry_view,
-        ]
+    model = "antenna_wire_dipole_fs"
 
 
 @rfm.simple_test
 class SubgridTest(GprMaxAPIRegressionTest):
     tags = {"test", "api", "serial", "regression", "subgrid"}
 
-    # List of available subgrid test models
-    model = parameter(
-        [
-            "cylinder_fs",
-            # "gssi_400_over_fractal_subsurface",  # Takes ~1hr 30m on ARCHER2
-        ]
-    )
-    num_cpus_per_task = 16
-
-    @run_after("init")
-    def set_filenames(self):
-        self.input_file = f"{self.model}.py"
-        self.output_file = f"{self.model}.h5"
-        self.executable_opts = [self.input_file, "-o", self.output_file]
-        self.postrun_cmds = [f"python -m toolboxes.Plotting.plot_Ascan -save {self.output_file}"]
-
-        geometry_view = f"{self.model}.vti"
-        subgrid_geometry_view = f"{self.model}_sg.vti"
-        plot_ascan_output = f"{self.model}.pdf"
-        self.keep_files = [
-            self.input_file,
-            self.output_file,
-            geometry_view,
-            subgrid_geometry_view,
-            plot_ascan_output,
-        ]
+    def __init__(self):
+        super().__init__()
+        # List of available subgrid test models
+        self.model = parameter(
+            [
+                "cylinder_fs",
+                # "gssi_400_over_fractal_subsurface",  # Takes ~1hr 30m on ARCHER2
+            ]
+        )
 
 
 @rfm.simple_test
-class MPIBasicModelsTest(GprMaxRegressionTest):
+class MPIBasicModelsTest(GprMaxMPIRegressionTest):
     tags = {"test", "mpi", "regression"}
 
-    # List of available basic test models
-    model = parameter(
-        [
-            "2D_ExHyHz",
-            "2D_EyHxHz",
-            "2D_EzHxHy",
-            "2D_ExHyHz_hs",
-            "cylinder_Ascan_2D",
-            "hertzian_dipole_fs",
-            "hertzian_dipole_hs",
-            "hertzian_dipole_dispersive",
-            "magnetic_dipole_fs",
-            "magnetic_dipole_hs",
-        ]
-    )
-    num_cpus_per_task = 16
-    num_tasks = 8
     num_tasks_per_node = 4
+    mpi_layout = [2, 2, 2]
 
-    mpi_layout = "2 2 2"
+    serial_dependency = BasicModelsTest
 
-    @run_after("init")
-    def inject_dependencies(self):
-        """Test depends on the Python virtual environment building correctly"""
-        variant = BasicModelsTest.get_variant_nums(model=lambda m: m == self.model)
-        self.depends_on(BasicModelsTest.variant_name(variant[0]), udeps.by_env)
-        super().inject_dependencies()
+    def __init__(self):
+        super().__init__()
+        # List of available basic test models
+        self.model = parameter(
+            [
+                "2D_ExHyHz",
+                "2D_EyHxHz",
+                "2D_EzHxHy",
+                "2D_ExHyHz_hs",
+                "cylinder_Ascan_2D",
+                "hertzian_dipole_fs",
+                "hertzian_dipole_hs",
+                "hertzian_dipole_dispersive",
+                "magnetic_dipole_fs",
+                "magnetic_dipole_hs",
+            ]
+        )
 
-    @run_after("init")
-    def set_filenames(self):
-        self.input_file = f"{self.model}.in"
-        self.output_file = f"{self.model}.h5"
-        self.executable_opts = ["-mpi", self.mpi_layout, self.input_file, "-o", self.output_file]
-        self.postrun_cmds = [f"python -m toolboxes.Plotting.plot_Ascan -save {self.output_file}"]
-        self.keep_files = [self.input_file, self.output_file, f"{self.model}.pdf"]
 
-    @run_before("run")
-    def setup_reference_file(self):
-        """Add prerun command to load the built Python environment"""
-        variant = BasicModelsTest.get_variant_nums(model=lambda m: m == self.model)
-        target = self.getdep(BasicModelsTest.variant_name(variant[0]))
-        self.reference_file = os.path.join(target.stagedir, str(self.output_file))
+@rfm.simple_test
+class TestBoxGeometryNoPml(GprMaxRegressionTest):
+    sourcesdir = "src/box_geometry_tests"
+
+    def __init__(self):
+        super().__init__()
+        self.model = parameter(
+            [
+                "box_full_model",
+                "box_half_model",
+            ]
+        )
+
+    @run_after("init", always_last=True)
+    def add_gprmax_commands(self):
+        self.prerun_cmds.append(f"echo '#pml_cells: 0' >> {self.input_file}")
+
+
+@rfm.simple_test
+class TestBoxGeometryDefaultPml(GprMaxRegressionTest):
+    sourcesdir = "src/box_geometry_tests"
+
+    def __init__(self):
+        super().__init__()
+        self.model = parameter(
+            [
+                "box_full_model",
+                "box_half_model",
+            ]
+        )
+
+
+@rfm.simple_test
+class TestBoxGeometryNoPmlMpi(GprMaxMPIRegressionTest):
+    mpi_layout = [2, 2, 2]
+
+    serial_dependency = TestBoxGeometryNoPml
+
+    def __init__(self):
+        super().__init__()
+        self.model = parameter(
+            [
+                "box_full_model",
+                "box_half_model",
+            ]
+        )
+
+    @run_after("init", always_last=True)
+    def add_gprmax_commands(self):
+        self.prerun_cmds.append(f"echo '#pml_cells: 0' >> {self.input_file}")
+
+
+@rfm.simple_test
+class TestBoxGeometryDefaultPmlMpi(GprMaxMPIRegressionTest):
+    mpi_layout = [2, 2, 2]
+
+    serial_dependency = TestBoxGeometryDefaultPml
+
+    def __init__(self):
+        super().__init__()
+        self.model = parameter(
+            [
+                "box_full_model",
+                "box_half_model",
+            ]
+        )