From 0e6d13898c7b09d47fd11a3b3594c93b76f3c69e Mon Sep 17 00:00:00 2001 From: nmannall Date: Tue, 18 Jun 2024 11:55:51 +0100 Subject: [PATCH] Refactor base reframe tests --- reframe_tests/base_tests.py | 186 ++++++++++++++------ reframe_tests/reframe_tests.py | 313 ++++++++++++++++----------------- 2 files changed, 292 insertions(+), 207 deletions(-) diff --git a/reframe_tests/base_tests.py b/reframe_tests/base_tests.py index 6f8be160..a3762314 100644 --- a/reframe_tests/base_tests.py +++ b/reframe_tests/base_tests.py @@ -5,9 +5,14 @@ from shutil import copyfile import reframe as rfm import reframe.utility.sanity as sn +import reframe.utility.typecheck as typ +from matplotlib.tri import TriContourSet +from numpy import product from reframe.core.builtins import ( + parameter, performance_function, require_deps, + required, run_after, run_before, sanity_function, @@ -73,13 +78,21 @@ class CreatePyenvTest(rfm.RunOnlyRegressionTest): ) -class GprMaxBaseTest(rfm.RunOnlyRegressionTest): +class GprMaxRegressionTest(rfm.RunOnlyRegressionTest): valid_systems = ["archer2:compute"] valid_prog_environs = ["PrgEnv-gnu"] modules = ["cray-python"] - executable = "time -p python -m gprMax --log-level 25" + + num_cpus_per_task = 16 exclusive_access = True + model = variable(str) + # sourcesdir = required + extra_executable_opts = variable(typ.List[str], value=[]) + executable = "time -p python -m gprMax --log-level 25" + + h5diff_header = f"{'=' * 10} h5diff output {'=' * 10}" + @run_after("init") def setup_env_vars(self): """Set OMP_NUM_THREADS environment variable from num_cpus_per_task""" @@ -105,10 +118,37 @@ class GprMaxBaseTest(rfm.RunOnlyRegressionTest): path_to_pyenv = os.path.join(CreatePyenvTest(part="login").stagedir, PATH_TO_PYENV) self.prerun_cmds.append(f"source {path_to_pyenv}") - @sanity_function + @run_after("init", always_last=True) + def configure_test_run(self, input_file_ext: str = ".in"): + self.input_file = f"{self.model}{input_file_ext}" + self.skip_if( + not os.path.exists(self.input_file), + f"Input file '{self.input_file}' not present in src directory '{self.sourcesdir}'", + ) + self.output_file = f"{self.model}.h5" + self.executable_opts = [self.input_file, "-o", self.output_file] + self.executable_opts += self.extra_executable_opts + self.postrun_cmds = [f"python -m toolboxes.Plotting.plot_Ascan -save {self.output_file}"] + self.keep_files = [self.input_file, self.output_file, f"{self.model}.pdf"] + + @run_before("run") + def setup_reference_file(self): + """Build reference file path""" + self.reference_file = Path("regression_checks", self.short_name).with_suffix(".h5") + self.reference_file = os.path.abspath(self.reference_file) + + @run_before("run", always_last=True) + def setup_regression_check(self): + """Add h5diff command to run after the test""" + if self.current_system.name == "archer2": + self.modules.append("cray-hdf5") + + if os.path.exists(self.reference_file): + self.postrun_cmds.append(f"echo {self.h5diff_header}") + self.postrun_cmds.append(f"h5diff {self.output_file} {self.reference_file}") + def test_simulation_complete(self): """Check simulation completed successfully""" - # TODO: Check for correctness/regression rather than just completing return sn.assert_not_found( r"(?i)error", self.stderr, @@ -117,6 +157,34 @@ class GprMaxBaseTest(rfm.RunOnlyRegressionTest): r"=== Simulation completed in ", self.stdout, "Simulation did not complete" ) + @sanity_function + def regression_check(self): + """ + Perform regression check by checking for the h5diff output. + Create reference file from the test output if it does not exist. + """ + if sn.path_exists(self.reference_file): + h5diff_output = sn.extractsingle( + f"{self.h5diff_header}\n(?P[\S\s]*)", self.stdout, "h5diff" + ) + return ( + self.test_simulation_complete() + and sn.assert_found(self.h5diff_header, self.stdout, "Failed to find h5diff header") + and sn.assert_false( + h5diff_output, + ( + f"Found h5diff output (see '{path_join(self.stagedir, self.stdout)}')\n" + f"For more details run: 'h5diff {os.path.abspath(self.output_file)} {self.reference_file}'\n" + f"To re-create regression file, delete '{self.reference_file}' and rerun the test." + ), + ) + ) + else: + copyfile(self.output_file, self.reference_file) + return sn.assert_true( + False, f"No reference file exists. Creating... '{self.reference_file}'" + ) + @performance_function("s", perf_key="run_time") def extract_run_time(self): """Extract total runtime from the last task to complete""" @@ -167,54 +235,74 @@ class GprMaxBaseTest(rfm.RunOnlyRegressionTest): return hours * 3600 + minutes * 60 + seconds -class GprMaxRegressionTest(GprMaxBaseTest): - input_file = variable(str) - output_file = variable(str) +class GprMaxAPIRegressionTest(GprMaxRegressionTest): + executable = "time -p python" - h5diff_header = f"{'=' * 10} h5diff output {'=' * 10}" + @run_after("init", always_last=True) + def configure_test_run(self): + super().configure_test_run(input_file_ext=".py") + + +class GprMaxBScanRegressionTest(GprMaxRegressionTest): + num_models = variable(int) + + @run_after("init", always_last=True) + def configure_test_run(self): + self.input_file = f"{self.model}.in" + self.output_file = f"{self.model}_merged.h5" + self.executable_opts = [self.input_file, "-n", str(self.num_models)] + self.postrun_cmds = [ + f"python -m toolboxes.Utilities.outputfiles_merge {self.model}", + f"python -m toolboxes.Plotting.plot_Bscan -save {self.output_file} Ez", + ] + self.keep_files = [self.input_file, self.output_file, "{self.model}_merged.pdf"] + + +class GprMaxTaskfarmRegressionTest(GprMaxBScanRegressionTest): + serial_dependecy = variable(type[GprMaxRegressionTest]) + extra_executable_opts = ["-taskfarm"] + + num_tasks = required + + @run_after("init") + def inject_dependencies(self): + """Test depends on the Python virtual environment building correctly""" + variant = self.serial_dependecy.get_variant_nums( + model=lambda m: m == self.model, num_models=lambda n: n == self.num_models + ) + self.depends_on(self.serial_dependecy.variant_name(variant[0]), udeps.by_env) + super().inject_dependencies() @run_before("run") def setup_reference_file(self): - """Build reference file path""" - self.reference_file = Path("regression_checks", self.short_name).with_suffix(".h5") - self.reference_file = os.path.abspath(self.reference_file) - - @run_before("run", always_last=True) - def setup_regression_check(self): - """Add h5diff command to run after the test""" - self.modules.append("cray-hdf5") - if os.path.exists(self.reference_file): - self.postrun_cmds.append(f"echo {self.h5diff_header}") - self.postrun_cmds.append(f"h5diff {self.output_file} {self.reference_file}") - - @sanity_function - def regression_check(self): - """ - Perform regression check by checking for the h5diff output. - Create reference file from the test output if it does not exist. - """ - if sn.path_exists(self.reference_file): - h5diff_output = sn.extractsingle( - f"{self.h5diff_header}\n(?P[\S\s]*)", self.stdout, "h5diff" - ) - return ( - self.test_simulation_complete() - and sn.assert_found(self.h5diff_header, self.stdout, "Failed to find h5diff header") - and sn.assert_false( - h5diff_output, - ( - f"Found h5diff output (see '{path_join(self.stagedir, self.stdout)}')\n" - f"For more details run: 'h5diff {os.path.abspath(self.output_file)} {self.reference_file}'\n" - f"To re-create regression file, delete '{self.reference_file}' and rerun the test." - ), - ) - ) - else: - copyfile(self.output_file, self.reference_file) - return sn.assert_true( - False, f"No reference file exists. Creating... '{self.reference_file}'" - ) + """Add prerun command to load the built Python environment""" + variant = self.serial_dependecy.get_variant_nums( + model=lambda m: m == self.model, num_models=lambda n: n == self.num_models + ) + target = self.getdep(self.serial_dependecy.variant_name(variant[0])) + self.reference_file = os.path.join(target.stagedir, str(self.output_file)) -class GprMaxAPIRegressionTest(GprMaxRegressionTest): - executable = "time -p python" +class GprMaxMPIRegressionTest(GprMaxRegressionTest): + mpi_layout = variable(typ.List[int]) + serial_dependecy = variable(type[GprMaxRegressionTest]) + + @run_after("init", always_last=True) + def configure_test_run(self): + self.num_tasks = product(self.mpi_layout, dtype=int) + self.extra_executable_opts = ["-mpi", " ".join(self.mpi_layout)] + super().configure_test_run() + + @run_after("init") + def inject_dependencies(self): + """Test depends on the Python virtual environment building correctly""" + variant = self.serial_dependecy.get_variant_nums(model=lambda m: m == self.model) + self.depends_on(self.serial_dependecy.variant_name(variant[0]), udeps.by_env) + super().inject_dependencies() + + @run_before("run") + def setup_reference_file(self): + """Add prerun command to load the built Python environment""" + variant = self.serial_dependecy.get_variant_nums(model=lambda m: m == self.model) + target = self.getdep(self.serial_dependecy.variant_name(variant[0])) + self.reference_file = os.path.join(target.stagedir, str(self.output_file)) diff --git a/reframe_tests/reframe_tests.py b/reframe_tests/reframe_tests.py index e24c27ec..f65469fe 100644 --- a/reframe_tests/reframe_tests.py +++ b/reframe_tests/reframe_tests.py @@ -1,9 +1,12 @@ -import os - import reframe as rfm -from base_tests import GprMaxAPIRegressionTest, GprMaxRegressionTest -from reframe.core.builtins import parameter, require_deps, run_after, run_before -from reframe.utility import udeps +from base_tests import ( + GprMaxAPIRegressionTest, + GprMaxBScanRegressionTest, + GprMaxMPIRegressionTest, + GprMaxRegressionTest, + GprMaxTaskfarmRegressionTest, +) +from reframe.core.builtins import parameter, run_after """ReFrame tests for basic functionality @@ -14,79 +17,70 @@ from reframe.utility import udeps @rfm.simple_test -class TaskfarmTest(GprMaxRegressionTest): - tags = {"test", "mpi", "taskfarm"} +class TestBscan(GprMaxBScanRegressionTest): + tags = {"test", "bscan"} - model = parameter(["cylinder_Bscan_2D"]) - - num_mpi_tasks = parameter([8, 16]) - num_cpus_per_task = 16 - - @run_after("init") - def setup_env_vars(self): - self.num_tasks = self.num_mpi_tasks - super().setup_env_vars() - - @run_after("init") - def set_filenames(self): - self.input_file = f"{self.model}.in" - self.output_file = f"{self.model}_merged.h5" - self.executable_opts = [self.input_file, "-n", "64", "-taskfarm"] - self.postrun_cmds = [ - f"python -m toolboxes.Utilities.outputfiles_merge {self.model}", - f"python -m toolboxes.Plotting.plot_Bscan -save {self.output_file} Ez", - ] - self.keep_files = [self.input_file, self.output_file, "{self.model}_merged.pdf"] + model = "cylinder_Bscan_2D" + num_models = 64 @rfm.simple_test -class BScanTest(GprMaxRegressionTest): - tags = {"test", "bscan"} +class TestSingleNodeTaskfarm(GprMaxTaskfarmRegressionTest): + tags = {"test", "mpi", "taskfarm"} - model = parameter(["cylinder_Bscan_2D"]) + model = "cylinder_Bscan_2D" + num_tasks = 8 + num_tasks_per_node = 8 + num_models = 64 + serial_dependecy = TestBscan - num_cpus_per_task = 16 - @run_after("init") - def set_filenames(self): - self.input_file = f"{self.model}.in" - self.output_file = f"{self.model}_merged.h5" - self.executable_opts = [self.input_file, "-n", "64"] - self.postrun_cmds = [ - f"python -m toolboxes.Utilities.outputfiles_merge {self.model}", - f"python -m toolboxes.Plotting.plot_Bscan -save {self.output_file} Ez", - ] - self.keep_files = [self.input_file, self.output_file, "{self.model}_merged.pdf"] +@rfm.simple_test +class TestMultiNodeTaskfarm(GprMaxTaskfarmRegressionTest): + tags = {"test", "mpi", "taskfarm"} + + model = "cylinder_Bscan_2D" + num_tasks = 32 + num_tasks_per_node = 8 + num_models = 64 + serial_dependecy = TestBscan + + +@rfm.simple_test +class Test2DModelXY(GprMaxRegressionTest): + tags = {"test", "serial", "2d"} + + model = "2D_EzHxHy" + + +@rfm.simple_test +class Test2DModelYZ(GprMaxRegressionTest): + tags = {"test", "serial", "2d"} + + model = "2D_EzHxHy" @rfm.simple_test class BasicModelsTest(GprMaxRegressionTest): tags = {"test", "serial", "regression"} - # List of available basic test models - model = parameter( - [ - "2D_ExHyHz", - "2D_EyHxHz", - "2D_EzHxHy", - "2D_ExHyHz_hs", - "cylinder_Ascan_2D", - "hertzian_dipole_fs", - "hertzian_dipole_hs", - "hertzian_dipole_dispersive", - "magnetic_dipole_fs", - "magnetic_dipole_hs", - ] - ) - num_cpus_per_task = 16 - - @run_after("init") - def set_filenames(self): - self.input_file = f"{self.model}.in" - self.output_file = f"{self.model}.h5" - self.executable_opts = [self.input_file, "-o", self.output_file] - self.postrun_cmds = [f"python -m toolboxes.Plotting.plot_Ascan -save {self.output_file}"] - self.keep_files = [self.input_file, self.output_file, f"{self.model}.pdf"] + def __init__(self): + super().__init__() + # List of available basic test models + self.model = parameter( + [ + "2D_ExHyHz", + "2D_EyHxHz", + "2D_EzHxHy", + "2D_ExHyHz_hs", + "cylinder_Ascan_2D", + "hertzian_dipole_fs", + "hertzian_dipole_hs", + "hertzian_dipole_dispersive", + "magnetic_dipole_fs", + "magnetic_dipole_hs", + ] + ) @rfm.simple_test @@ -94,112 +88,115 @@ class AntennaModelsTest(GprMaxRegressionTest): tags = {"test", "serial", "regression", "antenna"} # List of available antenna test models - model = parameter( - [ - "antenna_wire_dipole_fs", - ] - ) - num_cpus_per_task = 16 - - @run_after("init") - def set_filenames(self): - self.input_file = f"{self.model}.in" - self.output_file = f"{self.model}.h5" - self.executable_opts = [self.input_file, "-o", self.output_file] - self.postrun_cmds = [f"python -m toolboxes.Plotting.plot_Ascan -save {self.output_file}"] - self.postrun_cmds = [ - f"python -m toolboxes.Plotting.plot_antenna_params -save {self.output_file}" - ] - - antenna_t1_params = f"{self.model}_t1_params.pdf" - antenna_ant_params = f"{self.model}_ant_params.pdf" - plot_ascan_output = f"{self.model}.pdf" - geometry_view = f"{self.model}.vtu" - self.keep_files = [ - self.input_file, - self.output_file, - antenna_t1_params, - antenna_ant_params, - plot_ascan_output, - geometry_view, - ] + model = "antenna_wire_dipole_fs" @rfm.simple_test class SubgridTest(GprMaxAPIRegressionTest): tags = {"test", "api", "serial", "regression", "subgrid"} - # List of available subgrid test models - model = parameter( - [ - "cylinder_fs", - # "gssi_400_over_fractal_subsurface", # Takes ~1hr 30m on ARCHER2 - ] - ) - num_cpus_per_task = 16 - - @run_after("init") - def set_filenames(self): - self.input_file = f"{self.model}.py" - self.output_file = f"{self.model}.h5" - self.executable_opts = [self.input_file, "-o", self.output_file] - self.postrun_cmds = [f"python -m toolboxes.Plotting.plot_Ascan -save {self.output_file}"] - - geometry_view = f"{self.model}.vti" - subgrid_geometry_view = f"{self.model}_sg.vti" - plot_ascan_output = f"{self.model}.pdf" - self.keep_files = [ - self.input_file, - self.output_file, - geometry_view, - subgrid_geometry_view, - plot_ascan_output, - ] + def __init__(self): + super().__init__() + # List of available subgrid test models + self.model = parameter( + [ + "cylinder_fs", + # "gssi_400_over_fractal_subsurface", # Takes ~1hr 30m on ARCHER2 + ] + ) @rfm.simple_test -class MPIBasicModelsTest(GprMaxRegressionTest): +class MPIBasicModelsTest(GprMaxMPIRegressionTest): tags = {"test", "mpi", "regression"} - # List of available basic test models - model = parameter( - [ - "2D_ExHyHz", - "2D_EyHxHz", - "2D_EzHxHy", - "2D_ExHyHz_hs", - "cylinder_Ascan_2D", - "hertzian_dipole_fs", - "hertzian_dipole_hs", - "hertzian_dipole_dispersive", - "magnetic_dipole_fs", - "magnetic_dipole_hs", - ] - ) - num_cpus_per_task = 16 - num_tasks = 8 num_tasks_per_node = 4 + mpi_layout = [2, 2, 2] - mpi_layout = "2 2 2" + serial_dependency = BasicModelsTest - @run_after("init") - def inject_dependencies(self): - """Test depends on the Python virtual environment building correctly""" - variant = BasicModelsTest.get_variant_nums(model=lambda m: m == self.model) - self.depends_on(BasicModelsTest.variant_name(variant[0]), udeps.by_env) - super().inject_dependencies() + def __init__(self): + super().__init__() + # List of available basic test models + self.model = parameter( + [ + "2D_ExHyHz", + "2D_EyHxHz", + "2D_EzHxHy", + "2D_ExHyHz_hs", + "cylinder_Ascan_2D", + "hertzian_dipole_fs", + "hertzian_dipole_hs", + "hertzian_dipole_dispersive", + "magnetic_dipole_fs", + "magnetic_dipole_hs", + ] + ) - @run_after("init") - def set_filenames(self): - self.input_file = f"{self.model}.in" - self.output_file = f"{self.model}.h5" - self.executable_opts = ["-mpi", self.mpi_layout, self.input_file, "-o", self.output_file] - self.postrun_cmds = [f"python -m toolboxes.Plotting.plot_Ascan -save {self.output_file}"] - self.keep_files = [self.input_file, self.output_file, f"{self.model}.pdf"] - @run_before("run") - def setup_reference_file(self): - """Add prerun command to load the built Python environment""" - variant = BasicModelsTest.get_variant_nums(model=lambda m: m == self.model) - target = self.getdep(BasicModelsTest.variant_name(variant[0])) - self.reference_file = os.path.join(target.stagedir, str(self.output_file)) +@rfm.simple_test +class TestBoxGeometryNoPml(GprMaxRegressionTest): + sourcesdir = "src/box_geometry_tests" + + def __init__(self): + super().__init__() + self.model = parameter( + [ + "box_full_model", + "box_half_model", + ] + ) + + @run_after("init", always_last=True) + def add_gprmax_commands(self): + self.prerun_cmds.append(f"echo '#pml_cells: 0' >> {self.input_file}") + + +@rfm.simple_test +class TestBoxGeometryDefaultPml(GprMaxRegressionTest): + sourcesdir = "src/box_geometry_tests" + + def __init__(self): + super().__init__() + self.model = parameter( + [ + "box_full_model", + "box_half_model", + ] + ) + + +@rfm.simple_test +class TestBoxGeometryNoPmlMpi(GprMaxMPIRegressionTest): + mpi_layout = [2, 2, 2] + + serial_dependency = TestBoxGeometryNoPml + + def __init__(self): + super().__init__() + self.model = parameter( + [ + "box_full_model", + "box_half_model", + ] + ) + + @run_after("init", always_last=True) + def add_gprmax_commands(self): + self.prerun_cmds.append(f"echo '#pml_cells: 0' >> {self.input_file}") + + +@rfm.simple_test +class TestBoxGeometryDefaultPmlMpi(GprMaxMPIRegressionTest): + mpi_layout = [2, 2, 2] + + serial_dependency = TestBoxGeometryDefaultPml + + def __init__(self): + super().__init__() + self.model = parameter( + [ + "box_full_model", + "box_half_model", + ] + )