diff --git a/gprMax/grid/fdtd_grid.py b/gprMax/grid/fdtd_grid.py index 82297f94..edd6ecb1 100644 --- a/gprMax/grid/fdtd_grid.py +++ b/gprMax/grid/fdtd_grid.py @@ -166,6 +166,9 @@ class FDTDGrid: if thickness > 0: pml = self._construct_pml(pml_id, thickness) averageer, averagemr = self._calculate_average_pml_material_properties(pml) + logger.debug( + f"PML {pml.ID}: Average permittivity = {averageer}, Average permeability = {averagemr}" + ) pml.calculate_update_coeffs(averageer, averagemr) self.pmls["slabs"].append(pml) pbar.update() @@ -282,6 +285,8 @@ class FDTDGrid: n1 = self.nx n2 = self.ny solid = self.solid[:, :, pml.zs] + else: + raise ValueError(f"Unknown PML ID '{pml.ID}'") return pml_average_er_mr(n1, n2, config.get_model_config().ompthreads, solid, ers, mrs) diff --git a/gprMax/grid/mpi_grid.py b/gprMax/grid/mpi_grid.py index 2b4dda20..de15c610 100644 --- a/gprMax/grid/mpi_grid.py +++ b/gprMax/grid/mpi_grid.py @@ -313,8 +313,14 @@ class MPIGrid(FDTDGrid): sumer, summr = pml_sum_er_mr(n1, n2, config.get_model_config().ompthreads, solid, ers, mrs) n = pml.comm.allreduce(n1 * n2, MPI.SUM) - averageer = pml.comm.allreduce(sumer, MPI.SUM) / n - averagemr = pml.comm.allreduce(summr, MPI.SUM) / n + sumer = pml.comm.allreduce(sumer, MPI.SUM) + summr = pml.comm.allreduce(summr, MPI.SUM) + averageer = sumer / n + averagemr = summr / n + + logger.debug( + f"PML {pml.ID} has size {n}. Permittivity sum = {sumer}, Permeability sum = {summr}" + ) return averageer, averagemr @@ -405,3 +411,7 @@ class MPIGrid(FDTDGrid): self.size += self.negative_halo_offset self.lower_extent -= self.negative_halo_offset self.upper_extent = self.lower_extent + self.size + + logger.debug( + f"[Rank {self.rank}] Grid size: {self.size}, Lower extent: {self.lower_extent}, Upper extent: {self.upper_extent}" + ) diff --git a/gprMax/pml.py b/gprMax/pml.py index 4e8b52f8..da370558 100644 --- a/gprMax/pml.py +++ b/gprMax/pml.py @@ -18,6 +18,7 @@ import logging from importlib import import_module +from typing import List import numpy as np from mpi4py import MPI @@ -238,7 +239,7 @@ class PML: self.d = self.G.dz self.thickness = self.nz - self.CFS = self.G.pmls["cfs"] + self.CFS: List[CFS] = self.G.pmls["cfs"] self.check_kappamin() self.initialise_field_arrays() @@ -346,6 +347,9 @@ class PML: for x, cfs in enumerate(self.CFS): if not cfs.sigma.max: cfs.calculate_sigmamax(self.d, er, mr) + logger.debug( + f"PML {self.ID}: sigma.max set to {cfs.sigma.max} for {'first' if x == 0 else 'second'} order CFS parameter" + ) Ealpha, Halpha = cfs.calculate_values(self.thickness, cfs.alpha) Ekappa, Hkappa = cfs.calculate_values(self.thickness, cfs.kappa) Esigma, Hsigma = cfs.calculate_values(self.thickness, cfs.sigma) @@ -737,206 +741,3 @@ def print_pml_info(G): f"\nPML boundaries [{G.name}]: {{formulation: {G.pmls['formulation']}, " f"order: {len(G.pmls['cfs'])}, thickness (cells): {pmlinfo}}}" ) - - -def build_pml(G, pml_ID, thickness): - """Builds instances of the PML and calculates the initial parameters and - coefficients including setting profile (based on underlying material - er and mr from solid array). - - Args: - G: FDTDGrid class describing a grid in a model. - pml_ID: string identifier of PML slab. - thickness: int with thickness of PML slab in cells. - """ - - # Arrays to hold values of permittivity and permeability (avoids accessing - # Material class in Cython.) - ers = np.zeros(len(G.materials)) - mrs = np.zeros(len(G.materials)) - - for i, m in enumerate(G.materials): - ers[i] = m.er - mrs[i] = m.mr - - if config.sim_config.general["solver"] == "cpu": - pml_type = PML - elif config.sim_config.general["solver"] == "cuda": - pml_type = CUDAPML - elif config.sim_config.general["solver"] == "opencl": - pml_type = OpenCLPML - - if pml_ID == "x0": - pml = pml_type( - G, ID=pml_ID, direction="xminus", xs=0, xf=thickness, ys=0, yf=G.ny, zs=0, zf=G.nz - ) - elif pml_ID == "xmax": - pml = pml_type( - G, - ID=pml_ID, - direction="xplus", - xs=G.nx - thickness, - xf=G.nx, - ys=0, - yf=G.ny, - zs=0, - zf=G.nz, - ) - elif pml_ID == "y0": - pml = pml_type( - G, ID=pml_ID, direction="yminus", xs=0, xf=G.nx, ys=0, yf=thickness, zs=0, zf=G.nz - ) - elif pml_ID == "ymax": - pml = pml_type( - G, - ID=pml_ID, - direction="yplus", - xs=0, - xf=G.nx, - ys=G.ny - thickness, - yf=G.ny, - zs=0, - zf=G.nz, - ) - elif pml_ID == "z0": - pml = pml_type( - G, ID=pml_ID, direction="zminus", xs=0, xf=G.nx, ys=0, yf=G.ny, zs=0, zf=thickness - ) - elif pml_ID == "zmax": - pml = pml_type( - G, - ID=pml_ID, - direction="zplus", - xs=0, - xf=G.nx, - ys=0, - yf=G.ny, - zs=G.nz - thickness, - zf=G.nz, - ) - - if pml_ID[0] == "x": - averageer, averagemr = pml_average_er_mr( - G.ny, G.nz, config.get_model_config().ompthreads, G.solid[pml.xs, :, :], ers, mrs - ) - elif pml_ID[0] == "y": - averageer, averagemr = pml_average_er_mr( - G.nx, G.nz, config.get_model_config().ompthreads, G.solid[:, pml.ys, :], ers, mrs - ) - elif pml_ID[0] == "z": - averageer, averagemr = pml_average_er_mr( - G.nx, G.ny, config.get_model_config().ompthreads, G.solid[:, :, pml.zs], ers, mrs - ) - - pml.calculate_update_coeffs(averageer, averagemr) - G.pmls["slabs"].append(pml) - - -def build_pml_mpi(G, pml_ID, thickness): - """Builds instances of the PML and calculates the initial parameters and - coefficients including setting profile (based on underlying material - er and mr from solid array). - - Args: - G: FDTDGrid class describing a grid in a model. - pml_ID: string identifier of PML slab. - thickness: int with thickness of PML slab in cells. - """ - - # Arrays to hold values of permittivity and permeability (avoids accessing - # Material class in Cython.) - ers = np.zeros(len(G.materials)) - mrs = np.zeros(len(G.materials)) - - for i, m in enumerate(G.materials): - ers[i] = m.er - mrs[i] = m.mr - - if config.sim_config.general["solver"] == "cpu": - pml_type = PML - elif config.sim_config.general["solver"] == "cuda": - pml_type = CUDAPML - elif config.sim_config.general["solver"] == "opencl": - pml_type = OpenCLPML - - if pml_ID == "x0": - pml = pml_type( - G, ID=pml_ID, direction="xminus", xs=0, xf=thickness, ys=0, yf=G.ny, zs=0, zf=G.nz - ) - elif pml_ID == "xmax": - pml = pml_type( - G, - ID=pml_ID, - direction="xplus", - xs=G.nx - thickness, - xf=G.nx, - ys=0, - yf=G.ny, - zs=0, - zf=G.nz, - ) - elif pml_ID == "y0": - pml = pml_type( - G, ID=pml_ID, direction="yminus", xs=0, xf=G.nx, ys=0, yf=thickness, zs=0, zf=G.nz - ) - elif pml_ID == "ymax": - pml = pml_type( - G, - ID=pml_ID, - direction="yplus", - xs=0, - xf=G.nx, - ys=G.ny - thickness, - yf=G.ny, - zs=0, - zf=G.nz, - ) - elif pml_ID == "z0": - pml = pml_type( - G, ID=pml_ID, direction="zminus", xs=0, xf=G.nx, ys=0, yf=G.ny, zs=0, zf=thickness - ) - elif pml_ID == "zmax": - pml = pml_type( - G, - ID=pml_ID, - direction="zplus", - xs=0, - xf=G.nx, - ys=0, - yf=G.ny, - zs=G.nz - thickness, - zf=G.nz, - ) - - # Need to account for the negative halo (remove it) to avoid double - # counting. The solid array does not have a positive halo so we - # don't need to consider that. - if pml_ID[0] == "x": - o1 = G.negative_halo_offset[1] - o2 = G.negative_halo_offset[2] - n1 = G.ny - o1 - n2 = G.nz - o2 - solid = G.solid[pml.xs, o1:, o2:] - comm = G.x_comm - elif pml_ID[0] == "y": - o1 = G.negative_halo_offset[0] - o2 = G.negative_halo_offset[2] - n1 = G.nx - o1 - n2 = G.nz - o2 - solid = G.solid[o1:, pml.ys, o2:] - comm = G.y_comm - elif pml_ID[0] == "z": - o1 = G.negative_halo_offset[0] - o2 = G.negative_halo_offset[1] - n1 = G.nx - o1 - n2 = G.ny - o2 - solid = G.solid[o1:, o2:, pml.zs] - comm = G.z_comm - - sumer, summr = pml_sum_er_mr(n1, n2, config.get_model_config().ompthreads, solid, ers, mrs) - n = comm.allreduce(n1 * n2, MPI.SUM) - averageer = comm.allreduce(sumer, MPI.SUM) / n - averagemr = comm.allreduce(summr, MPI.SUM) / n - - pml.calculate_update_coeffs(averageer, averagemr) - G.pmls["slabs"].append(pml) diff --git a/reframe_tests/base_tests.py b/reframe_tests/base_tests.py index 9d042c9a..ec2b5441 100644 --- a/reframe_tests/base_tests.py +++ b/reframe_tests/base_tests.py @@ -144,6 +144,13 @@ class GprMaxRegressionTest(rfm.RunOnlyRegressionTest): antenna_ant_params, ] + if self.num_tasks > 1: + stdout = self.stdout.evaluate().split(".")[0] + stderr = self.stderr.evaluate().split(".")[0] + self.job.launcher.options = [f"--output={stdout}_%t.out", f"--error={stderr}_%t.err"] + self.postrun_cmds.append(f"cat {stdout}_*.out >> {self.stdout}") + self.postrun_cmds.append(f"cat {stderr}_*.err >> {self.stderr}") + @run_before("run") def check_input_file_exists(self): self.skip_if( @@ -258,7 +265,7 @@ class GprMaxRegressionTest(rfm.RunOnlyRegressionTest): class GprMaxAPIRegressionTest(GprMaxRegressionTest): executable = "time -p python" - @run_after("init", always_last=True) + @run_after("setup", always_last=True) def configure_test_run(self): super().configure_test_run(input_file_ext=".py") @@ -307,8 +314,6 @@ class GprMaxMPIRegressionTest(GprMaxRegressionTest): # TODO: Make this a variable serial_dependency: type[GprMaxRegressionTest] mpi_layout = parameter() - _stdout = "rfm_job-%t.out" - _stderr = "rfm_job-%t.err" @run_after("setup", always_last=True) def configure_test_run(self): diff --git a/reframe_tests/reframe_tests.py b/reframe_tests/reframe_tests.py index ab3f693a..ac656799 100644 --- a/reframe_tests/reframe_tests.py +++ b/reframe_tests/reframe_tests.py @@ -244,7 +244,7 @@ class TestEdgeGeometryMpi(GprMaxMPIRegressionTest): class TestBoxGeometryNoPml(GprMaxRegressionTest): tags = {"test", "serial", "geometery", "box"} sourcesdir = "src/box_geometry_tests" - model = parameter(["box_full_model", "box_half_model"]) + model = parameter(["box_full_model", "box_half_model", "box_single_rank"]) @run_before("run") def add_gprmax_commands(self): diff --git a/reframe_tests/regression_checks/TestBoxGeometryNoPml_7804157f.h5 b/reframe_tests/regression_checks/TestBoxGeometryNoPml_7804157f.h5 new file mode 100644 index 00000000..343ebae5 Binary files /dev/null and b/reframe_tests/regression_checks/TestBoxGeometryNoPml_7804157f.h5 differ