Move reframe test files into seperate directory

reframe_tests/tests/base_tests.py

@@ -0,0 +1,361 @@
+"""ReFrame base classes for GprMax tests"""
+import os
+from pathlib import Path
+from shutil import copyfile
+
+import reframe as rfm
+import reframe.utility.sanity as sn
+import reframe.utility.typecheck as typ
+from numpy import product
+from reframe.core.builtins import (
+    parameter,
+    performance_function,
+    require_deps,
+    required,
+    run_after,
+    run_before,
+    sanity_function,
+    variable,
+)
+from reframe.utility import udeps
+
+from gprMax.receivers import Rx
+from reframe_tests.utilities.deferrable import path_join
+
+GPRMAX_ROOT_DIR = Path(__file__).parent.parent.resolve()
+PATH_TO_PYENV = os.path.join(".venv", "bin", "activate")
+
+
+@rfm.simple_test
+class CreatePyenvTest(rfm.RunOnlyRegressionTest):
+    valid_systems = ["generic", "archer2:login"]
+    valid_prog_environs = ["builtin", "PrgEnv-gnu"]
+    modules = ["cray-python"]
+
+    prerun_cmds = [
+        "python -m venv --system-site-packages --prompt gprMax .venv",
+        f"source {PATH_TO_PYENV}",
+        "CC=cc CXX=CC FC=ftn python -m pip install --upgrade pip",
+        f"CC=cc CXX=CC FC=ftn python -m pip install -r {os.path.join(GPRMAX_ROOT_DIR, 'requirements.txt')}",
+    ]
+    executable = f"CC=cc CXX=CC FC=ftn python -m pip install -e {GPRMAX_ROOT_DIR}"
+
+    @run_after("init")
+    def install_system_specific_dependencies(self):
+        """Install additional dependencies for specific systems"""
+        if self.current_system.name == "archer2":
+            """
+            Needed to prevent a pip install error.
+            dask 2022.2.1 (installed) requires cloudpickle>=1.1.1, which
+            is not installed and is missed by the pip dependency checks.
+
+            Not necessary for gprMax, but any error message is picked up
+            by the sanity checks.
+            """
+            self.prerun_cmds.insert(3, "CC=cc CXX=CC FC=ftn python -m pip install cloudpickle")
+
+    @sanity_function
+    def check_requirements_installed(self):
+        """
+        Check packages successfully installed from requirements.txt
+        Check gprMax installed successfully and no other errors thrown
+        """
+        return (
+            sn.assert_found(
+                r"(Successfully installed pip)|(Requirement already satisfied: pip.*\n(?!Collecting pip))",
+                self.stdout,
+                "Failed to update pip",
+            )
+            and sn.assert_found(
+                r"Successfully installed (?!(gprMax)|(pip))",
+                self.stdout,
+                "Failed to install requirements",
+            )
+            and sn.assert_found(
+                r"Successfully installed gprMax", self.stdout, "Failed to install gprMax"
+            )
+            and sn.assert_not_found(r"finished with status 'error'", self.stdout)
+            and sn.assert_not_found(r"(ERROR|error):", self.stderr)
+        )
+
+
+class GprMaxRegressionTest(rfm.RunOnlyRegressionTest):
+    valid_systems = ["archer2:compute"]
+    valid_prog_environs = ["PrgEnv-gnu"]
+    modules = ["cray-python"]
+
+    num_cpus_per_task = 16
+    exclusive_access = True
+
+    model = parameter()
+    is_antenna_model = variable(bool, value=False)
+    sourcesdir = required
+    extra_executable_opts = variable(typ.List[str], value=[])
+    executable = "time -p python -m gprMax --log-level 10 --hide-progress-bars"
+
+    rx_outputs = variable(typ.List[str], value=Rx.defaultoutputs)
+    h5diff_header = f"{'=' * 10} h5diff output {'=' * 10}"
+
+    @run_after("init")
+    def setup_env_vars(self):
+        """Set OMP_NUM_THREADS environment variable from num_cpus_per_task"""
+        self.env_vars["OMP_NUM_THREADS"] = self.num_cpus_per_task
+
+        if self.current_system.name == "archer2":
+            # Avoid inheriting slurm memory environment variables from any previous slurm job (i.e. the reframe job)
+            self.prerun_cmds.append("unset SLURM_MEM_PER_NODE")
+            self.prerun_cmds.append("unset SLURM_MEM_PER_CPU")
+
+            # Set the matplotlib cache to the work filesystem
+            self.env_vars["MPLCONFIGDIR"] = "${HOME/home/work}/.config/matplotlib"
+
+    # TODO: Change CreatePyenvTest to a fixture instead of a test dependency
+    @run_after("init")
+    def inject_dependencies(self):
+        """Test depends on the Python virtual environment building correctly"""
+        self.depends_on("CreatePyenvTest", udeps.by_env)
+
+    @require_deps
+    def get_pyenv_path(self, CreatePyenvTest):
+        """Add prerun command to load the built Python environment"""
+        path_to_pyenv = os.path.join(CreatePyenvTest(part="login").stagedir, PATH_TO_PYENV)
+        self.prerun_cmds.append(f"source {path_to_pyenv}")
+
+    @run_after("setup", always_last=True)
+    def configure_test_run(self, input_file_ext: str = ".in"):
+        self.input_file = f"{self.model}{input_file_ext}"
+        self.output_file = f"{self.model}.h5"
+        self.executable_opts = [self.input_file, "-o", self.output_file]
+        self.executable_opts += self.extra_executable_opts
+        self.postrun_cmds = [
+            f"python -m toolboxes.Plotting.plot_Ascan -save {self.output_file} --outputs {' '.join(self.rx_outputs)}"
+        ]
+        self.keep_files = [self.input_file, self.output_file, f"{self.model}.pdf"]
+
+        if self.is_antenna_model:
+            self.postrun_cmds = [
+                f"python -m toolboxes.Plotting.plot_antenna_params -save {self.output_file}"
+            ]
+
+            antenna_t1_params = f"{self.model}_t1_params.pdf"
+            antenna_ant_params = f"{self.model}_ant_params.pdf"
+            self.keep_files += [
+                antenna_t1_params,
+                antenna_ant_params,
+            ]
+
+    @run_before("run")
+    def combine_task_outputs(self):
+        if self.num_tasks > 1:
+            stdout = self.stdout.evaluate().split(".")[0]
+            stderr = self.stderr.evaluate().split(".")[0]
+            self.prerun_cmds.append(f"mkdir out")
+            self.prerun_cmds.append(f"mkdir err")
+            self.job.launcher.options = [
+                f"--output=out/{stdout}_%t.out",
+                f"--error=err/{stderr}_%t.err",
+            ]
+            self.executable_opts += ["--log-all-ranks"]
+            self.postrun_cmds.append(f"cat out/{stdout}_*.out >> {self.stdout}")
+            self.postrun_cmds.append(f"cat err/{stderr}_*.err >> {self.stderr}")
+
+    @run_before("run")
+    def check_input_file_exists(self):
+        self.skip_if(
+            not os.path.exists(os.path.join(self.sourcesdir, self.input_file)),
+            f"Input file '{self.input_file}' not present in src directory '{self.sourcesdir}'",
+        )
+
+    @run_before("run")
+    def setup_reference_file(self):
+        """Build reference file path"""
+        self.reference_file = Path("regression_checks", self.short_name).with_suffix(".h5")
+        self.reference_file = os.path.abspath(self.reference_file)
+
+    @run_before("run", always_last=True)
+    def setup_regression_check(self):
+        """Add h5diff command to run after the test"""
+        if self.current_system.name == "archer2":
+            self.modules.append("cray-hdf5")
+
+        if os.path.exists(self.reference_file):
+            self.postrun_cmds.append(f"echo {self.h5diff_header}")
+            self.postrun_cmds.append(f"h5diff {self.output_file} {self.reference_file}")
+
+    def test_simulation_complete(self):
+        """Check simulation completed successfully"""
+        return sn.assert_not_found(
+            r"(?i)error",
+            self.stderr,
+            f"An error occured. See '{path_join(self.stagedir, self.stderr)}' for details.",
+        ) and sn.assert_found(
+            r"=== Simulation completed in ", self.stdout, "Simulation did not complete"
+        )
+
+    @sanity_function
+    def regression_check(self):
+        """
+        Perform regression check by checking for the h5diff output.
+        Create reference file from the test output if it does not exist.
+        """
+        if sn.path_exists(self.reference_file):
+            h5diff_output = sn.extractsingle(
+                f"{self.h5diff_header}\n(?P<h5diff>[\S\s]*)", self.stdout, "h5diff"
+            )
+            return (
+                self.test_simulation_complete()
+                and sn.assert_found(self.h5diff_header, self.stdout, "Failed to find h5diff header")
+                and sn.assert_false(
+                    h5diff_output,
+                    (
+                        f"Found h5diff output (see '{path_join(self.stagedir, self.stdout)}')\n"
+                        f"For more details run: 'h5diff {os.path.abspath(self.output_file)} {self.reference_file}'\n"
+                        f"To re-create regression file, delete '{self.reference_file}' and rerun the test."
+                    ),
+                )
+            )
+        else:
+            copyfile(self.output_file, self.reference_file)
+            return sn.assert_true(
+                False, f"No reference file exists. Creating... '{self.reference_file}'"
+            )
+
+    @performance_function("s", perf_key="run_time")
+    def extract_run_time(self):
+        """Extract total runtime from the last task to complete"""
+        return sn.extractsingle(
+            r"real\s+(?P<run_time>\S+)", self.stderr, "run_time", float, self.num_tasks - 1
+        )
+
+    @performance_function("s", perf_key="simulation_time")
+    def extract_simulation_time(self):
+        """Extract simulation time reported by gprMax"""
+
+        # sn.extractall throws an error if a group has value None.
+        # Therefore have to handle the < 1 min, >= 1 min and >= 1 hour cases separately.
+        timeframe = sn.extractsingle(
+            r"=== Simulation completed in \S+ (?P<timeframe>hour|minute|second)",
+            self.stdout,
+            "timeframe",
+        )
+        if timeframe == "hour":
+            simulation_time = sn.extractall(
+                r"=== Simulation completed in (?P<hours>\S+) hours?, (?P<minutes>\S+) minutes? and (?P<seconds>\S+) seconds? =*",
+                self.stdout,
+                ["hours", "minutes", "seconds"],
+                float,
+            )
+            hours = simulation_time[0][0]
+            minutes = simulation_time[0][1]
+            seconds = simulation_time[0][2]
+        elif timeframe == "minute":
+            hours = 0
+            simulation_time = sn.extractall(
+                r"=== Simulation completed in (?P<minutes>\S+) minutes? and (?P<seconds>\S+) seconds? =*",
+                self.stdout,
+                ["minutes", "seconds"],
+                float,
+            )
+            minutes = simulation_time[0][0]
+            seconds = simulation_time[0][1]
+        else:
+            hours = 0
+            minutes = 0
+            seconds = sn.extractsingle(
+                r"=== Simulation completed in (?P<seconds>\S+) seconds? =*",
+                self.stdout,
+                "seconds",
+                float,
+            )
+        return hours * 3600 + minutes * 60 + seconds
+
+
+class GprMaxAPIRegressionTest(GprMaxRegressionTest):
+    executable = "time -p python"
+
+    @run_after("setup", always_last=True)
+    def configure_test_run(self):
+        super().configure_test_run(input_file_ext=".py")
+
+
+class GprMaxBScanRegressionTest(GprMaxRegressionTest):
+    num_models = parameter()
+
+    @run_after("setup", always_last=True)
+    def configure_test_run(self):
+        self.extra_executable_opts += ["-n", str(self.num_models)]
+        super().configure_test_run()
+
+        self.postrun_cmds = [
+            f"python -m toolboxes.Utilities.outputfiles_merge {self.model}",
+            f"mv {self.model}_merged.h5 {self.output_file}",
+            f"python -m toolboxes.Plotting.plot_Bscan -save {self.output_file} Ez",
+        ]
+
+
+class GprMaxTaskfarmRegressionTest(GprMaxBScanRegressionTest):
+    serial_dependency: type[GprMaxRegressionTest]
+    extra_executable_opts = ["-taskfarm"]
+    sourcesdir = "src"
+
+    num_tasks = required
+
+    def _get_variant(self) -> str:
+        variant = self.serial_dependency.get_variant_nums(
+            model=lambda m: m == self.model, num_models=lambda n: n == self.num_models
+        )
+        return self.serial_dependency.variant_name(variant[0])
+
+    @run_after("init")
+    def inject_dependencies(self):
+        """Test depends on the Python virtual environment building correctly"""
+        self.depends_on(self._get_variant(), udeps.by_env)
+        super().inject_dependencies()
+
+    @run_after("init")
+    def setup_source_directory(self):
+        """Set the source directory to the same as the serial test"""
+        self.sourcesdir = str(self.serial_dependency.sourcesdir)
+
+    @run_before("run")
+    def setup_reference_file(self):
+        """Add prerun command to load the built Python environment"""
+        target = self.getdep(self._get_variant())
+        self.reference_file = os.path.join(target.stagedir, str(self.output_file))
+
+
+class GprMaxMPIRegressionTest(GprMaxRegressionTest):
+    # TODO: Make this a variable
+    serial_dependency: type[GprMaxRegressionTest]
+    mpi_layout = parameter()
+    sourcesdir = "src"
+
+    @run_after("setup", always_last=True)
+    def configure_test_run(self):
+        self.num_tasks = int(product(self.mpi_layout))
+        self.extra_executable_opts = ["-mpi", *map(str, self.mpi_layout)]
+        super().configure_test_run()
+
+    def _get_variant(self) -> str:
+        variant = self.serial_dependency.get_variant_nums(model=lambda m: m == self.model)
+        return self.serial_dependency.variant_name(variant[0])
+
+    @run_after("init")
+    def inject_dependencies(self):
+        """Test depends on the specified serial test"""
+        self.depends_on(self._get_variant(), udeps.by_env)
+        super().inject_dependencies()
+
+    @run_after("init")
+    def setup_source_directory(self):
+        """Set the source directory to the same as the serial test"""
+        self.sourcesdir = str(self.serial_dependency.sourcesdir)
+
+    @run_before("run")
+    def setup_reference_file(self):
+        """
+        Set the reference file regression check to the output of the
+        serial test
+        """
+        target = self.getdep(self._get_variant())
+        self.reference_file = os.path.join(target.stagedir, str(self.output_file))

reframe_tests/tests/reframe_tests.py

@@ -0,0 +1,315 @@
+import reframe as rfm
+from base_tests import (
+    GprMaxAPIRegressionTest,
+    GprMaxBScanRegressionTest,
+    GprMaxMPIRegressionTest,
+    GprMaxRegressionTest,
+    GprMaxTaskfarmRegressionTest,
+)
+from reframe.core.builtins import parameter, run_after, run_before
+
+"""ReFrame tests for basic functionality
+
+    Usage:
+        cd gprMax/reframe_tests
+        reframe -C configuraiton/{CONFIG_FILE} -c reframe_tests.py -c base_tests.py -r
+"""
+
+
+@rfm.simple_test
+class TestAscan(GprMaxRegressionTest):
+    tags = {
+        "test",
+        "serial",
+        "ascan",
+        "2d",
+        "hertzian_dipole",
+        "waveform",
+        "material",
+        "box",
+        "cylinder",
+    }
+    sourcesdir = "src/example_models"
+    model = parameter(["cylinder_Ascan_2D"])
+
+
+@rfm.simple_test
+class TestBscan(GprMaxBScanRegressionTest):
+    tags = {
+        "test",
+        "serial",
+        "bscan",
+        "steps",
+        "waveform",
+        "hertzian_dipole",
+        "material",
+        "box",
+        "cylinder",
+    }
+    sourcesdir = "src/bscan_tests"
+    model = parameter(["cylinder_Bscan_2D"])
+    num_models = parameter([64])
+
+
+@rfm.simple_test
+class TestSingleNodeTaskfarm(GprMaxTaskfarmRegressionTest):
+    tags = {
+        "test",
+        "mpi",
+        "taskfarm",
+        "steps",
+        "waveform",
+        "hertzian_dipole",
+        "material",
+        "box",
+        "cylinder",
+    }
+    num_tasks = 8
+    num_tasks_per_node = 8
+    serial_dependency = TestBscan
+    model = serial_dependency.model
+    num_models = serial_dependency.num_models
+
+
+@rfm.simple_test
+class TestMultiNodeTaskfarm(GprMaxTaskfarmRegressionTest):
+    tags = {
+        "test",
+        "mpi",
+        "taskfarm",
+        "steps",
+        "waveform",
+        "hertzian_dipole",
+        "material",
+        "box",
+        "cylinder",
+    }
+    num_tasks = 32
+    num_tasks_per_node = 8
+    serial_dependency = TestBscan
+    model = serial_dependency.model
+    num_models = serial_dependency.num_models
+
+
+@rfm.simple_test
+class Test2DModelXY(GprMaxRegressionTest):
+    tags = {"test", "serial", "2d", "waveform", "hertzian_dipole"}
+    sourcesdir = "src/2d_tests"
+    model = parameter(["2D_EzHxHy"])
+
+
+@rfm.simple_test
+class Test2DModelXZ(GprMaxRegressionTest):
+    tags = {"test", "serial", "2d", "waveform", "hertzian_dipole"}
+    sourcesdir = "src/2d_tests"
+    model = parameter(["2D_EyHxHz"])
+
+
+@rfm.simple_test
+class Test2DModelYZ(GprMaxRegressionTest):
+    tags = {"test", "serial", "2d", "waveform", "hertzian_dipole"}
+    sourcesdir = "src/2d_tests"
+    model = parameter(["2D_ExHyHz"])
+
+
+@rfm.simple_test
+class TestHertzianDipoleSource(GprMaxRegressionTest):
+    tags = {"test", "serial", "hertzian_dipole", "waveform"}
+    sourcesdir = "src/source_tests"
+    model = parameter(["hertzian_dipole_fs"])
+
+
+@rfm.simple_test
+class TestMagneticDipoleSource(GprMaxRegressionTest):
+    tags = {"test", "serial", "magnetic_dipole", "waveform"}
+    sourcesdir = "src/source_tests"
+    model = parameter(["magnetic_dipole_fs"])
+
+
+@rfm.simple_test
+class TestDispersiveMaterials(GprMaxRegressionTest):
+    tags = {"test", "serial", "hertzian_dipole", "waveform", "material", "dispersive", "box"}
+    sourcesdir = "src/material_tests"
+    model = parameter(["hertzian_dipole_dispersive"])
+
+
+@rfm.simple_test
+class TestTransmissionLineSource(GprMaxRegressionTest):
+    tags = {"test", "serial", "transmission_line", "waveform"}
+    sourcesdir = "src/source_tests"
+    model = parameter(["transmission_line_fs"])
+
+
+@rfm.simple_test
+class TestEdgeGeometry(GprMaxRegressionTest):
+    tags = {"test", "serial", "geometry", "edge", "transmission_line", "waveform", "antenna"}
+    sourcesdir = "src/geomtry_tests/edge_geometry"
+    model = parameter(["antenna_wire_dipole_fs"])
+    is_antenna_model = True
+
+
+@rfm.simple_test
+class TestSubgrids(GprMaxAPIRegressionTest):
+    tags = {
+        "test",
+        "api",
+        "serial",
+        "subgrid",
+        "hertzian_dipole",
+        "waveform",
+        "material",
+        "dispersive",
+        "cylinder",
+    }
+    sourcesdir = "src/subgrid_tests"
+    model = parameter(["cylinder_fs"])
+
+
+@rfm.simple_test
+class TestSubgridsWithAntennaModel(GprMaxAPIRegressionTest):
+    tags = {
+        "test",
+        "api",
+        "serial",
+        "subgrid",
+        "antenna",
+        "material",
+        "box",
+        "fractal_box",
+        "add_surface_roughness",
+    }
+    sourcesdir = "src/subgrid_tests"
+    model = parameter(["gssi_400_over_fractal_subsurface"])
+    is_antenna_model = True
+
+    @run_after("init")
+    def skip_test(self):
+        self.skip_if(self.current_system.name == "archer2", "Takes ~1hr 30m on ARCHER2")
+
+
+@rfm.simple_test
+class Test2DModelXYMpi(GprMaxMPIRegressionTest):
+    tags = {"test", "mpi", "2d", "waveform", "hertzian_dipole"}
+    mpi_layout = parameter([[4, 4, 1]])
+    serial_dependency = Test2DModelXY
+    model = serial_dependency.model
+
+
+@rfm.simple_test
+class Test2DModelXZMpi(GprMaxMPIRegressionTest):
+    tags = {"test", "mpi", "2d", "waveform", "hertzian_dipole"}
+    mpi_layout = parameter([[4, 1, 4]])
+    serial_dependency = Test2DModelXZ
+    model = serial_dependency.model
+
+
+@rfm.simple_test
+class Test2DModelYZMpi(GprMaxMPIRegressionTest):
+    tags = {"test", "mpi", "2d", "waveform", "hertzian_dipole"}
+    mpi_layout = parameter([[1, 4, 4]])
+    serial_dependency = Test2DModelYZ
+    model = serial_dependency.model
+
+
+@rfm.simple_test
+class TestHertzianDipoleSourceMpi(GprMaxMPIRegressionTest):
+    tags = {"test", "mpi", "hertzian_dipole", "waveform"}
+    mpi_layout = parameter([[3, 3, 3]])
+    serial_dependency = TestHertzianDipoleSource
+    model = serial_dependency.model
+
+
+@rfm.simple_test
+class TestMagneticDipoleSourceMpi(GprMaxMPIRegressionTest):
+    tags = {"test", "mpi", "magnetic_dipole", "waveform"}
+    mpi_layout = parameter([[3, 3, 3]])
+    serial_dependency = TestMagneticDipoleSource
+    model = serial_dependency.model
+
+
+@rfm.simple_test
+class TestDispersiveMaterialsMpi(GprMaxMPIRegressionTest):
+    tags = {"test", "mpi", "hertzian_dipole", "waveform", "material", "dispersive", "box"}
+    mpi_layout = parameter([[3, 3, 3]])
+    serial_dependency = TestDispersiveMaterials
+    model = serial_dependency.model
+
+
+@rfm.simple_test
+class TestTransmissionLineSourceMpi(GprMaxMPIRegressionTest):
+    tags = {"test", "mpi", "transmission_line", "waveform"}
+    mpi_layout = parameter([[3, 3, 3]])
+    serial_dependency = TestTransmissionLineSource
+    model = serial_dependency.model
+
+
+@rfm.simple_test
+class TestEdgeGeometryMpi(GprMaxMPIRegressionTest):
+    tags = {"test", "mpi", "geometry", "edge", "transmission_line", "waveform", "antenna"}
+    mpi_layout = parameter([[3, 3, 3]])
+    serial_dependency = TestEdgeGeometry
+    model = serial_dependency.model
+    is_antenna_model = True
+
+
+@rfm.simple_test
+class TestBoxGeometryNoPml(GprMaxRegressionTest):
+    tags = {"test", "serial", "geometery", "box"}
+    sourcesdir = "src/geometry_tests/box_geometry"
+    model = parameter(["box_full_model", "box_half_model", "box_single_rank"])
+
+    @run_before("run")
+    def add_gprmax_commands(self):
+        self.prerun_cmds.append(f"echo '#pml_cells: 0' >> {self.input_file}")
+
+
+@rfm.simple_test
+class TestBoxGeometryDefaultPml(GprMaxRegressionTest):
+    tags = {"test", "serial", "geometery", "box"}
+    sourcesdir = "src/geometry_tests/box_geometry"
+    model = parameter(
+        [
+            "box_full_model",
+            "box_half_model",
+            "box_single_rank",
+            "box_outside_pml",
+            "box_single_rank_outside_pml",
+        ]
+    )
+
+
+@rfm.simple_test
+class TestBoxGeometryNoPmlMpi(GprMaxMPIRegressionTest):
+    tags = {"test", "mpi", "geometery", "box"}
+    mpi_layout = parameter([[2, 2, 2], [3, 3, 3], [4, 4, 4]])
+    serial_dependency = TestBoxGeometryNoPml
+    model = serial_dependency.model
+
+    @run_before("run")
+    def add_gprmax_commands(self):
+        self.prerun_cmds.append(f"echo '#pml_cells: 0' >> {self.input_file}")
+
+
+@rfm.simple_test
+class TestBoxGeometryDefaultPmlMpi(GprMaxMPIRegressionTest):
+    tags = {"test", "mpi", "geometery", "box"}
+    mpi_layout = parameter([[2, 2, 2], [3, 3, 3], [4, 4, 4]])
+    serial_dependency = TestBoxGeometryDefaultPml
+    model = serial_dependency.model
+
+
+@rfm.simple_test
+class TestSingleCellPml(GprMaxRegressionTest):
+    tags = {"test", "serial", "geometery", "box", "pml"}
+    sourcesdir = "src/pml_tests"
+    model = parameter(["single_cell_pml_2d"])
+    rx_outputs = ["Hx"]
+
+
+@rfm.simple_test
+class TestSingleCellPmlMpi(GprMaxMPIRegressionTest):
+    tags = {"test", "mpi", "geometery", "box", "pml"}
+    mpi_layout = parameter([[2, 2, 1], [3, 3, 1]])
+    serial_dependency = TestSingleCellPml
+    model = serial_dependency.model
+    rx_outputs = ["Hx"]

reframe_tests/tests/src/2d_tests/2D_ExHyHz.in

@@ -0,0 +1,8 @@
+#title: 2D test Ex, Hy, Hz components
+#domain: 0.001 0.100 0.100
+#dx_dy_dz: 0.001 0.001 0.001
+#time_window: 3e-9
+
+#waveform: gaussiandot 1 1e9 myWave
+#hertzian_dipole: x 0 0.050 0.050 myWave
+#rx: 0 0.070 0.070

reframe_tests/tests/src/2d_tests/2D_ExHyHz_hs.in

@@ -0,0 +1,13 @@
+#title: 2D test Ex, Hy, Hz components
+#domain: 0.001 0.010 0.010
+#dx_dy_dz: 0.001 0.001 0.001
+#time_window: 3e-9
+
+#pml_cells: 0
+
+#waveform: gaussiandot 1 1e9 myWave
+#hertzian_dipole: x 0 0.005 0.005 myWave
+#rx: 0 0.002 0.002
+
+#material: 8 0 1 0 half_space
+#box: 0 0 0 0.001 0.004 0.004 half_space

reframe_tests/tests/src/2d_tests/2D_EyHxHz.in

@@ -0,0 +1,8 @@
+#title: 2D test Ey, Hx, Hz components
+#domain: 0.100 0.001 0.100
+#dx_dy_dz: 0.001 0.001 0.001
+#time_window: 3e-9
+
+#waveform: gaussiandot 1 1e9 myWave
+#hertzian_dipole: y 0.050 0 0.050 myWave
+#rx: 0.070 0 0.070

reframe_tests/tests/src/2d_tests/2D_EzHxHy.in

@@ -0,0 +1,8 @@
+#title: 2D test Ez, Hx, Hy components
+#domain: 0.100 0.100 0.001
+#dx_dy_dz: 0.001 0.001 0.001
+#time_window: 3e-9
+
+#waveform: gaussiandot 1 1e9 myWave
+#hertzian_dipole: z 0.050 0.050 0 myWave
+#rx: 0.070 0.070 0

reframe_tests/tests/src/bscan_tests/cylinder_Bscan_2D.in

@@ -0,0 +1,15 @@
+#title: B-scan from a metal cylinder buried in a dielectric half-space
+#domain: 0.240 0.210 0.002
+#dx_dy_dz: 0.002 0.002 0.002
+#time_window: 3e-9
+
+#material: 6 0 1 0 half_space
+
+#waveform: ricker 1 1.5e9 my_ricker
+#hertzian_dipole: z 0.040 0.170 0 my_ricker
+#rx: 0.080 0.170 0
+#src_steps: 0.002 0 0
+#rx_steps: 0.002 0 0
+
+#box: 0 0 0 0.240 0.170 0.002 half_space
+#cylinder: 0.120 0.080 0 0.120 0.080 0.002 0.010 pec

reframe_tests/tests/src/example_models/cylinder_Ascan_2D.in

@@ -0,0 +1,13 @@
+#title: A-scan from a metal cylinder buried in a dielectric half-space
+#domain: 0.240 0.210 0.002
+#dx_dy_dz: 0.002 0.002 0.002
+#time_window: 3e-9
+
+#material: 6 0 1 0 half_space
+
+#waveform: ricker 1 1.5e9 my_ricker
+#hertzian_dipole: z 0.100 0.170 0 my_ricker
+#rx: 0.140 0.170 0
+
+#box: 0 0 0 0.240 0.170 0.002 half_space
+#cylinder: 0.120 0.080 0 0.120 0.080 0.002 0.010 pec

reframe_tests/tests/src/geometry_tests/box_geometry/box_full_model.in

@@ -0,0 +1,11 @@
+#title: Hertzian dipole over a half-space
+#domain: 0.100 0.100 0.100
+#dx_dy_dz: 0.001 0.001 0.001
+#time_window: 3e-9
+
+#waveform: gaussiandot 1 1e9 myWave
+#hertzian_dipole: z 0.050 0.050 0.050 myWave
+#rx: 0.070 0.070 0.070
+
+#material: 8 0 1 0 half_space
+#box: 0 0 0 0.100 0.100 0.100 half_space

reframe_tests/tests/src/geometry_tests/box_geometry/box_half_model.in

@@ -0,0 +1,11 @@
+#title: Hertzian dipole over a half-space
+#domain: 0.100 0.100 0.100
+#dx_dy_dz: 0.001 0.001 0.001
+#time_window: 3e-9
+
+#waveform: gaussiandot 1 1e9 myWave
+#hertzian_dipole: z 0.050 0.050 0.050 myWave
+#rx: 0.070 0.070 0.070
+
+#material: 8 0 1 0 half_space
+#box: 0 0 0 0.100 0.100 0.050 half_space

reframe_tests/tests/src/geometry_tests/box_geometry/box_outside_pml.in

@@ -0,0 +1,11 @@
+#title: Hertzian dipole over a half-space
+#domain: 0.100 0.100 0.100
+#dx_dy_dz: 0.001 0.001 0.001
+#time_window: 3e-9
+
+#waveform: gaussiandot 1 1e9 myWave
+#hertzian_dipole: z 0.050 0.050 0.050 myWave
+#rx: 0.070 0.070 0.070
+
+#material: 8 0 1 0 half_space
+#box: 0.020 0.020 0.020 0.080 0.080 0.050 half_space

reframe_tests/tests/src/geometry_tests/box_geometry/box_single_rank.in

@@ -0,0 +1,11 @@
+#title: Hertzian dipole over a half-space
+#domain: 0.100 0.100 0.100
+#dx_dy_dz: 0.001 0.001 0.001
+#time_window: 3e-9
+
+#waveform: gaussiandot 1 1e9 myWave
+#hertzian_dipole: z 0.050 0.050 0.050 myWave
+#rx: 0.070 0.070 0.070
+
+#material: 8 0 1 0 half_space
+#box: 0 0 0 0.020 0.020 0.020 half_space

reframe_tests/tests/src/geometry_tests/box_geometry/box_single_rank_outside_pml.in

@@ -0,0 +1,11 @@
+#title: Hertzian dipole over a half-space
+#domain: 0.100 0.100 0.100
+#dx_dy_dz: 0.001 0.001 0.001
+#time_window: 3e-9
+
+#waveform: gaussiandot 1 1e9 myWave
+#hertzian_dipole: z 0.050 0.050 0.050 myWave
+#rx: 0.070 0.070 0.070
+
+#material: 8 0 1 0 half_space
+#box: 0.030 0.030 0.030 0.045 0.045 0.045 half_space

reframe_tests/tests/src/geometry_tests/edge_geometry/antenna_wire_dipole_fs.in

@@ -0,0 +1,15 @@
+#title: Wire antenna - half-wavelength dipole in free-space
+#domain: 0.050 0.050 0.200
+#dx_dy_dz: 0.001 0.001 0.001
+#time_window: 60e-9
+
+#waveform: gaussian 1 1e9 mypulse
+#transmission_line: z 0.025 0.025 0.100 73 mypulse
+
+## 150mm length
+#edge: 0.025 0.025 0.025 0.025 0.025 0.175 pec
+
+## 1mm gap at centre of dipole
+#edge: 0.025 0.025 0.100 0.025 0.025 0.101 free_space
+
+#geometry_view: 0.020 0.020 0.020 0.030 0.030 0.180 0.001 0.001 0.001 antenna_wire_dipole_fs f

reframe_tests/tests/src/material_tests/hertzian_dipole_dispersive.in

@@ -0,0 +1,12 @@
+#title: Hertzian dipole in water
+#domain: 0.100 0.100 0.100
+#dx_dy_dz: 0.001 0.001 0.001
+#time_window: 3e-9
+
+#waveform: gaussiandot 1 1e9 myWave
+#hertzian_dipole: z 0.050 0.050 0.050 myWave
+#rx: 0.070 0.070 0.070
+
+#material: 4.9 0 1 0 myWater
+#add_dispersion_debye: 1 75.2 9.231e-12 myWater
+#box: 0 0 0 0.100 0.100 0.100 myWater

reframe_tests/tests/src/material_tests/magnetic_dipole_hs.in

@@ -0,0 +1,11 @@
+#title: Magnetic dipole over a half-space
+#domain: 0.100 0.100 0.100
+#dx_dy_dz: 0.001 0.001 0.001
+#time_window: 3e-9
+
+#waveform: gaussiandot 1 1e9 myWave
+#magnetic_dipole: z 0.050 0.050 0.050 myWave
+#rx: 0.070 0.070 0.070
+
+#material: 8 0 1 0 half_space
+#box: 0 0 0 0.100 0.100 0.050 half_space

reframe_tests/tests/src/pml_tests/single_cell_pml_2d.in

@@ -0,0 +1,31 @@
+#title: Single depth PML
+#domain: 0.6 0.6 0.1
+#dx_dy_dz: 0.1 0.1 0.1
+#time_window: 10
+
+#waveform: gaussiandot 1 1e7 myWave
+#hertzian_dipole: z 0.3 0.3 0 myWave
+#rx: 0.1 0.1 0 r01 Hx
+#rx: 0.1 0.2 0 r02 Hx
+#rx: 0.1 0.3 0 r03 Hx
+#rx: 0.1 0.4 0 r04 Hx
+
+#rx: 0.2 0.1 0 r05 Hx
+#rx: 0.2 0.2 0 r06 Hx
+#rx: 0.2 0.3 0 r07 Hx
+#rx: 0.2 0.4 0 r08 Hx
+
+#rx: 0.3 0.1 0 r09 Hx
+#rx: 0.3 0.2 0 r10 Hx
+#rx: 0.3 0.3 0 r11 Hx
+#rx: 0.3 0.4 0 r12 Hx
+
+#rx: 0.4 0.1 0 r13 Hx
+#rx: 0.4 0.2 0 r14 Hx
+#rx: 0.4 0.3 0 r15 Hx
+#rx: 0.4 0.4 0 r16 Hx
+
+#material: 8 0 1 0 half_space
+#box: 0 0 0 0.6 0.3 0.1 half_space
+
+#pml_cells: 1 1 0 1 1 0

reframe_tests/tests/src/snapshot_tests/whole_domain.in

@@ -0,0 +1,9 @@
+#title: Hertzian dipole in free-space
+#domain: 0.100 0.100 0.100
+#dx_dy_dz: 0.001 0.001 0.001
+#time_window: 3e-9
+
+#waveform: gaussiandot 1 1e9 myWave
+#hertzian_dipole: z 0.050 0.050 0.050 myWave
+
+#snapshot: 0 0 0 0.100 0.100 0.100 0.01 0.01 0.01 2e-9 snapshot_3.h5

reframe_tests/tests/src/snapshot_tests/whole_domain_2d.in

@@ -0,0 +1,9 @@
+#title: Hertzian dipole in free-space
+#domain: 0.100 0.100 0.001
+#dx_dy_dz: 0.001 0.001 0.001
+#time_window: 3e-9
+
+#waveform: gaussiandot 1 1e9 myWave
+#hertzian_dipole: z 0.050 0.050 0 myWave
+
+#snapshot: 0 0 0 0.100 0.100 0.001 0.01 0.01 0.01 2e-9 snapshot_3.h5

reframe_tests/tests/src/source_tests/hertzian_dipole_fs.in

@@ -0,0 +1,8 @@
+#title: Hertzian dipole in free-space
+#domain: 0.100 0.100 0.100
+#dx_dy_dz: 0.001 0.001 0.001
+#time_window: 3e-9
+
+#waveform: gaussiandot 1 1e9 myWave
+#hertzian_dipole: z 0.050 0.050 0.050 myWave
+#rx: 0.070 0.070 0.070

reframe_tests/tests/src/source_tests/magnetic_dipole_fs.in

@@ -0,0 +1,8 @@
+#title: Magnetic dipole in free-space
+#domain: 0.100 0.100 0.100
+#dx_dy_dz: 0.001 0.001 0.001
+#time_window: 3e-9
+
+#waveform: gaussiandot 1 1e9 myWave
+#magnetic_dipole: z 0.050 0.050 0.050 myWave
+#rx: 0.070 0.070 0.070

reframe_tests/tests/src/source_tests/transmission_line_fs.in

@@ -0,0 +1,8 @@
+#title: Transmission line in free-space
+#domain: 0.100 0.100 0.100
+#dx_dy_dz: 0.001 0.001 0.001
+#time_window: 3e-9
+
+#waveform: gaussian 1 1e9 mypulse
+#transmission_line: x 0.050 0.050 0.050 35 mypulse
+#rx: 0.070 0.070 0.070

reframe_tests/tests/src/subgrid_tests/cylinder_fs.py

@@ -0,0 +1,115 @@
+"""Cylinder in freespace
+
+This example model demonstrates how to use subgrids at a basic level.
+
+The geometry is 3D (required for any use of subgrids) and is of a water-filled
+cylindrical object in freespace. The subgrid encloses the cylinderical object
+using a fine spatial discretisation (1mm), and a courser spatial discretisation
+(5mm) is used in the rest of the model (main grid). A simple Hertzian dipole
+source is used with a waveform shaped as the first derivative of a gaussian.
+"""
+
+from pathlib import Path
+
+import gprMax
+from gprMax.materials import calculate_water_properties
+
+# File path - used later to specify name of output files
+fn = Path(__file__)
+parts = fn.parts
+
+# Subgrid spatial discretisation in x, y, z directions
+dl_sg = 1e-3
+
+# Subgrid ratio - must always be an odd integer multiple
+ratio = 5
+dl = dl_sg * ratio
+
+# Domain extent
+x = 0.500
+y = 0.500
+z = 0.500
+
+# Time window
+tw = 6e-9
+
+scene = gprMax.Scene()
+
+title = gprMax.Title(name=fn.name)
+dxdydz = gprMax.Discretisation(p1=(dl, dl, dl))
+domain = gprMax.Domain(p1=(x, y, z))
+time_window = gprMax.TimeWindow(time=tw)
+
+wf = gprMax.Waveform(wave_type="gaussiandot", amp=1, freq=1.5e9, id="mypulse")
+hd = gprMax.HertzianDipole(polarisation="z", p1=(0.205, 0.400, 0.250), waveform_id="mypulse")
+rx = gprMax.Rx(p1=(0.245, 0.400, 0.250))
+
+scene.add(title)
+scene.add(dxdydz)
+scene.add(domain)
+scene.add(time_window)
+scene.add(wf)
+scene.add(hd)
+scene.add(rx)
+
+# Cylinder parameters
+c1 = (0.225, 0.250, 0.100)
+c2 = (0.225, 0.250, 0.400)
+r = 0.010
+sg1 = (c1[0] - r, c1[1] - r, c1[2])
+sg2 = (c2[0] + r, c2[1] + r, c2[2])
+
+# Create subgrid
+subgrid = gprMax.SubGridHSG(p1=sg1, p2=sg2, ratio=ratio, id="sg")
+scene.add(subgrid)
+
+# Create water material
+eri, er, tau, sig = calculate_water_properties()
+water = gprMax.Material(er=eri, se=sig, mr=1, sm=0, id="water")
+subgrid.add(water)
+water = gprMax.AddDebyeDispersion(poles=1, er_delta=[er - eri], tau=[tau], material_ids=["water"])
+subgrid.add(water)
+
+# Add cylinder to subgrid
+cylinder = gprMax.Cylinder(p1=c1, p2=c2, r=r, material_id="water")
+subgrid.add(cylinder)
+
+# Create some geometry views for both subgrid and main grid
+gvsg = gprMax.GeometryView(
+    p1=sg1,
+    p2=sg2,
+    dl=(dl_sg, dl_sg, dl_sg),
+    filename=fn.with_suffix("").parts[-1] + "_sg",
+    output_type="n",
+)
+subgrid.add(gvsg)
+
+gv1 = gprMax.GeometryView(
+    p1=(0, 0, 0),
+    p2=(x, y, z),
+    dl=(dl, dl, dl),
+    filename=fn.with_suffix("").parts[-1],
+    output_type="n",
+)
+scene.add(gv1)
+
+# Create some snapshots of entire domain
+for i in range(5):
+    s = gprMax.Snapshot(
+        p1=(0, 0, 0),
+        p2=(x, y, z),
+        dl=(dl, dl, dl),
+        time=(i + 0.5) * 1e-9,
+        filename=fn.with_suffix("").parts[-1] + "_" + str(i + 1),
+    )
+    scene.add(s)
+
+gprMax.run(
+    scenes=[scene],
+    n=1,
+    geometry_only=False,
+    outputfile=fn,
+    subgrid=True,
+    autotranslate=True,
+    log_level=25,
+)

reframe_tests/tests/src/subgrid_tests/gssi_400_over_fractal_subsurface.py

@@ -0,0 +1,166 @@
+"""GPR antenna model (like a GSSI 400MHz antenna) over layered media with a
+    rough subsurface interface.
+
+This example model demonstrates how to use subgrids at a more advanced level -
+    combining use of an imported antenna model and rough subsurface interface.
+
+The geometry is 3D (required for any use of subgrids) and is of a 2 layered
+subsurface. The top layer in a sandy soil and the bottom layer a soil with
+higher permittivity (both have some simple conductive loss). There is a rough
+interface between the soil layers. A GPR antenna model (like a GSSI 400MHz
+antenna) is imported and placed on the surface of the layered media. The antenna
+is meshed using a subgrid with a fine spatial discretisation (1mm), and a
+courser spatial discretisation (9mm) is used in the rest of the model (main
+grid).
+"""
+
+from pathlib import Path
+
+import numpy as np
+
+import gprMax
+from toolboxes.GPRAntennaModels.GSSI import antenna_like_GSSI_400
+
+# File path - used later to specify name of output files
+fn = Path(__file__)
+parts = fn.parts
+
+# Subgrid spatial discretisation in x, y, z directions
+dl_sg = 1e-3
+
+# Subgrid ratio - must always be an odd integer multiple
+ratio = 9
+dl = dl_sg * ratio
+
+# Domain extent
+x = 3
+y = 1
+z = 2
+
+# Time window
+# Estimated two way travel time over 1 metre in material with highest
+# permittivity, slowest velocity.
+tw = 2 / 3e8 * (np.sqrt(3.2) + np.sqrt(9))
+
+scene = gprMax.Scene()
+
+title = gprMax.Title(name=fn.name)
+dxdydz = gprMax.Discretisation(p1=(dl, dl, dl))
+domain = gprMax.Domain(p1=(x, y, z))
+time_window = gprMax.TimeWindow(time=tw)
+
+scene.add(title)
+scene.add(dxdydz)
+scene.add(domain)
+scene.add(time_window)
+
+# Dimensions of antenna case
+antenna_case = (0.3, 0.3, 0.178)
+
+# Position of antenna
+antenna_p = (x / 2, y / 2, 170 * dl)
+
+# Extra distance surrounding antenna for subgrid
+bounding_box = 2 * dl
+
+# Subgrid extent
+sg_x0 = antenna_p[0] - antenna_case[0] / 2 - bounding_box
+sg_y0 = antenna_p[1] - antenna_case[1] / 2 - bounding_box
+sg_z0 = antenna_p[2] - bounding_box
+sg_x1 = antenna_p[0] + antenna_case[0] / 2 + bounding_box
+sg_y1 = antenna_p[1] + antenna_case[1] / 2 + bounding_box
+sg_z1 = antenna_p[2] + antenna_case[2] + bounding_box
+
+# Create subgrid
+sg = gprMax.SubGridHSG(p1=[sg_x0, sg_y0, sg_z0], p2=[sg_x1, sg_y1, sg_z1], ratio=ratio, id="sg")
+scene.add(sg)
+
+# Create and add a box of homogeneous material to main grid - sandy_soil
+sandy_soil = gprMax.Material(er=3.2, se=0.397e-3, mr=1, sm=0, id="sandy_soil")
+scene.add(sandy_soil)
+b1 = gprMax.Box(p1=(0, 0, 0), p2=(x, y, antenna_p[2]), material_id="sandy_soil")
+scene.add(b1)
+
+# Position box of sandy_soil in the subgrid.
+#   It has to be positioned manually because it traverses the main grid/subgrid
+#   interface. Grid traversal is when objects extend beyond the outer surface.
+#   Setting autotranslate to false allows you to place objects beyond the outer
+#   surface.
+
+# PML separation from the outer surface
+ps = ratio // 2 + 2
+# Number of PML cells in the subgrid
+pc = 6
+# Inner surface/outer surface separation
+isos = 3 * ratio
+
+# Calculate maximum z-coordinate (height) for box of sandy_soil in subgrid
+h = antenna_p[2] - sg_z0 + (ps + pc + isos) * dl_sg
+
+# Create and add a box of homogeneous material to subgrid - sandy_soil
+sg.add(sandy_soil)
+b2 = gprMax.Box(p1=(0, 0, 0), p2=(411 * dl_sg, 411 * dl_sg, h), material_id="sandy_soil")
+# Set autotranslate for the box object to false
+b2.autotranslate = False
+sg.add(b2)
+
+# Import antenna model and add components to subgrid
+gssi_objects = antenna_like_GSSI_400(*antenna_p, resolution=dl_sg)
+for obj in gssi_objects:
+    sg.add(obj)
+
+# Create and add a homogeneous material with a rough surface
+soil = gprMax.Material(er=9, se=0.397e-3, mr=1, sm=0, id="soil")
+scene.add(soil)
+
+fb = gprMax.FractalBox(
+    p1=(0, 0, 0),
+    p2=(3, 1, 1),
+    frac_dim=1.5,
+    weighting=(1, 1, 1),
+    n_materials=1,
+    mixing_model_id="soil",
+    id="fbox",
+    seed=1,
+)
+scene.add(fb)
+
+rough_surf = gprMax.AddSurfaceRoughness(
+    p1=(0, 0, 1),
+    p2=(3, 1, 1),
+    frac_dim=1.5,
+    weighting=(1, 1),
+    limits=(0.4, 1.2),
+    fractal_box_id="fbox",
+    seed=1,
+)
+scene.add(rough_surf)
+
+# Create some snapshots and geometry views
+for i in range(1, 51):
+    snap = gprMax.Snapshot(
+        p1=(0, y / 2, 0),
+        p2=(x, y / 2 + dl, z),
+        dl=(dl, dl, dl),
+        filename=Path(*parts[:-1], f"{parts[-1]}_{str(i)}").name,
+        time=i * tw / 50,
+    )
+    scene.add(snap)
+
+gvsg = gprMax.GeometryView(
+    p1=(sg_x0, sg_y0, sg_z0),
+    p2=(sg_x1, sg_y1, sg_z1),
+    dl=(dl_sg, dl_sg, dl_sg),
+    filename=fn.with_suffix("").parts[-1] + "_sg",
+    output_type="n",
+)
+sg.add(gvsg)
+
+gv1 = gprMax.GeometryView(
+    p1=(0, 0, 0), p2=domain.props.p1, dl=dl, filename=fn.with_suffix("").parts[-1], output_type="n"
+)
+scene.add(gv1)
+
+gprMax.run(
+    scenes=[scene], n=1, geometry_only=False, outputfile=fn, subgrid=True, autotranslate=True
+)