Merge branch 'devel' into mpi

这个提交包含在:
nmannall
2023-12-20 15:32:52 +00:00
当前提交 5661fa8445
共有 15 个文件被更改,包括 123 次插入52 次删除

查看文件

@@ -9,12 +9,12 @@ repos:
- id: check-added-large-files
- id: check-toml
- repo: https://github.com/psf/black-pre-commit-mirror
rev: 23.11.0
rev: 23.12.0
hooks:
- id: black
args: ["--line-length", "120"] # Adjust the max line length value as needed.
- repo: https://github.com/pycqa/isort
rev: 5.12.0
rev: 5.13.2
hooks:
- id: isort
args: ["--line-length", "120", "--profile", "black"]

查看文件

@@ -210,4 +210,4 @@ Periodically you should update conda and the required Python packages. With the
Thanks To Our Contributors ✨🔗
==========================
.. image:: https://contrib.rocks/image?repo=gprMax/gprMax
:target: https://github.com/gprMax/gprMax/graphs/contributors
:target: https://github.com/gprMax/gprMax/graphs/contributors

查看文件

@@ -767,11 +767,11 @@ For example, to specify the normalised first derivative of a Gaussian waveform w
Allows you to specify an ASCII file that contains amplitude values that specify custom waveform(s) that can be used with sources in the model.
The first row of each column must begin with a identifier string that will be used as the name of each waveform. Subsequent rows should contain amplitude values for the custom waveform you want to use. You can import multiple different waveforms (as columns of amplitude data) in a single file.
The first row of each column must begin with a identifier string that will be used as the name of each waveform. Subsequent rows should contain amplitude values for the custom waveform you want to use. You can import multiple different waveforms (as columns of amplitude data) in a single file.
Ideally, there should be the same number of amplitude values as number of iterations in your model. If there are less amplitude values than the number of iterations in the model, the end of the sequence of amplitude values will be padded with zero values up to the number of iterations. If extra amplitude values are specified than needed then they are ignored.
Optionally, in the first column of the file you may specify your own time vector of values (which must use the identifier ``time``) to use with the amplitude values of the waveform.
Optionally, in the first column of the file you may specify your own time vector of values (which must use the identifier ``time``) to use with the amplitude values of the waveform.
The amplitude values will be interpolated using either the aforementioned user specified time vector, or if none was supplied, a vector of time values corresponding to the simulation time step and number of iterations will be used. Key parameters used for the interpolation can be specified in the command.

查看文件

@@ -21,11 +21,11 @@ import logging
import numpy as np
import gprMax.config as config
from gprMax.cmds_geometry.cmds_geometry import UserObjectGeometry, rotate_2point_object
from gprMax.fractals import FractalVolume
from gprMax.materials import ListMaterial
from ..cython.geometry_primitives import build_voxels_from_array, build_voxels_from_array_mask
from gprMax.cmds_geometry.cmds_geometry import UserObjectGeometry, rotate_2point_object
logger = logging.getLogger(__name__)
@@ -172,7 +172,7 @@ class FractalBox(UserObjectGeometry):
f"is {dielectricsmoothing}."
)
grid.fractalvolumes.append(self.volume)
def build(self, grid, uip):
if self.do_pre_build:
self.pre_build(grid, uip)
@@ -218,7 +218,8 @@ class FractalBox(UserObjectGeometry):
# otherwise a mixing model
if self.volume.nbins == 1:
self.volume.fractalvolume = np.ones(
(self.volume.nx, self.volume.ny, self.volume.nz), dtype=config.sim_config.dtypes["float_or_double"]
(self.volume.nx, self.volume.ny, self.volume.nz),
dtype=config.sim_config.dtypes["float_or_double"],
)
materialnumID = next(x.numID for x in grid.materials if x.ID == self.volume.operatingonID)
self.volume.fractalvolume *= materialnumID
@@ -253,11 +254,17 @@ class FractalBox(UserObjectGeometry):
for j in range(surface.ys, surface.yf):
for k in range(surface.zs, surface.zf):
if i < surface.fractalsurface[j - surface.ys, k - surface.zs]:
self.volume.mask[i - self.volume.xs, j - self.volume.ys, k - self.volume.zs] = 1
self.volume.mask[
i - self.volume.xs, j - self.volume.ys, k - self.volume.zs
] = 1
elif surface.filldepth > 0 and i < surface.filldepth:
self.volume.mask[i - self.volume.xs, j - self.volume.ys, k - self.volume.zs] = 2
self.volume.mask[
i - self.volume.xs, j - self.volume.ys, k - self.volume.zs
] = 2
else:
self.volume.mask[i - self.volume.xs, j - self.volume.ys, k - self.volume.zs] = 0
self.volume.mask[
i - self.volume.xs, j - self.volume.ys, k - self.volume.zs
] = 0
elif surface.ID == "grass":
g = surface.grass[0]
# Build the blades of the grass
@@ -269,7 +276,10 @@ class FractalBox(UserObjectGeometry):
for i in range(self.volume.xs, surface.fractalrange[1]):
if (
i < surface.fractalsurface[j - surface.ys, k - surface.zs]
and self.volume.mask[i - self.volume.xs, j - self.volume.ys, k - self.volume.zs] != 1
and self.volume.mask[
i - self.volume.xs, j - self.volume.ys, k - self.volume.zs
]
!= 1
):
y, z = g.calculate_blade_geometry(blade, height)
# Add y, z coordinates to existing location
@@ -304,7 +314,10 @@ class FractalBox(UserObjectGeometry):
surface.fractalsurface[j - surface.ys, k - surface.zs]
- self.volume.originalxf
)
and self.volume.mask[i - self.volume.xs, j - self.volume.ys, k - self.volume.zs] == 1
and self.volume.mask[
i - self.volume.xs, j - self.volume.ys, k - self.volume.zs
]
== 1
):
y, z = g.calculate_root_geometry(root, depth)
# Add y, z coordinates to existing location
@@ -342,11 +355,17 @@ class FractalBox(UserObjectGeometry):
for j in range(surface.fractalrange[0], surface.fractalrange[1]):
for k in range(surface.zs, surface.zf):
if j < surface.fractalsurface[i - surface.xs, k - surface.zs]:
self.volume.mask[i - self.volume.xs, j - self.volume.ys, k - self.volume.zs] = 1
self.volume.mask[
i - self.volume.xs, j - self.volume.ys, k - self.volume.zs
] = 1
elif surface.filldepth > 0 and j < surface.filldepth:
self.volume.mask[i - self.volume.xs, j - self.volume.ys, k - self.volume.zs] = 2
self.volume.mask[
i - self.volume.xs, j - self.volume.ys, k - self.volume.zs
] = 2
else:
self.volume.mask[i - self.volume.xs, j - self.volume.ys, k - self.volume.zs] = 0
self.volume.mask[
i - self.volume.xs, j - self.volume.ys, k - self.volume.zs
] = 0
elif surface.ID == "grass":
g = surface.grass[0]
# Build the blades of the grass
@@ -358,7 +377,10 @@ class FractalBox(UserObjectGeometry):
for j in range(self.volume.ys, surface.fractalrange[1]):
if (
j < surface.fractalsurface[i - surface.xs, k - surface.zs]
and self.volume.mask[i - self.volume.xs, j - self.volume.ys, k - self.volume.zs] != 1
and self.volume.mask[
i - self.volume.xs, j - self.volume.ys, k - self.volume.zs
]
!= 1
):
x, z = g.calculate_blade_geometry(blade, height)
# Add x, z coordinates to existing location
@@ -393,7 +415,10 @@ class FractalBox(UserObjectGeometry):
surface.fractalsurface[i - surface.xs, k - surface.zs]
- self.volume.originalyf
)
and self.volume.mask[i - self.volume.xs, j - self.volume.ys, k - self.volume.zs] == 1
and self.volume.mask[
i - self.volume.xs, j - self.volume.ys, k - self.volume.zs
]
== 1
):
x, z = g.calculate_root_geometry(root, depth)
# Add x, z coordinates to existing location
@@ -431,11 +456,17 @@ class FractalBox(UserObjectGeometry):
for j in range(surface.ys, surface.yf):
for k in range(surface.fractalrange[0], surface.fractalrange[1]):
if k < surface.fractalsurface[i - surface.xs, j - surface.ys]:
self.volume.mask[i - self.volume.xs, j - self.volume.ys, k - self.volume.zs] = 1
self.volume.mask[
i - self.volume.xs, j - self.volume.ys, k - self.volume.zs
] = 1
elif surface.filldepth > 0 and k < surface.filldepth:
self.volume.mask[i - self.volume.xs, j - self.volume.ys, k - self.volume.zs] = 2
self.volume.mask[
i - self.volume.xs, j - self.volume.ys, k - self.volume.zs
] = 2
else:
self.volume.mask[i - self.volume.xs, j - self.volume.ys, k - self.volume.zs] = 0
self.volume.mask[
i - self.volume.xs, j - self.volume.ys, k - self.volume.zs
] = 0
elif surface.ID == "grass":
g = surface.grass[0]
# Build the blades of the grass
@@ -447,7 +478,10 @@ class FractalBox(UserObjectGeometry):
for k in range(self.volume.zs, surface.fractalrange[1]):
if (
k < surface.fractalsurface[i - surface.xs, j - surface.ys]
and self.volume.mask[i - self.volume.xs, j - self.volume.ys, k - self.volume.zs] != 1
and self.volume.mask[
i - self.volume.xs, j - self.volume.ys, k - self.volume.zs
]
!= 1
):
x, y = g.calculate_blade_geometry(blade, height)
# Add x, y coordinates to existing location
@@ -482,7 +516,10 @@ class FractalBox(UserObjectGeometry):
surface.fractalsurface[i - surface.xs, j - surface.ys]
- self.volume.originalzf
)
and self.volume.mask[i - self.volume.xs, j - self.volume.ys, k - self.volume.zs] == 1
and self.volume.mask[
i - self.volume.xs, j - self.volume.ys, k - self.volume.zs
]
== 1
):
x, y = g.calculate_root_geometry(root, depth)
# Add x, y coordinates to existing location
@@ -553,4 +590,4 @@ class FractalBox(UserObjectGeometry):
grid.rigidE,
grid.rigidH,
grid.ID,
)
)

查看文件

@@ -82,7 +82,7 @@ class GeometryObjectsRead(UserObjectGeometry):
scene.add(material_obj)
# Creates the internal simulation objects
scene.process_cmds(material_objs, grid, sort=False)
scene.process_cmds(material_objs, grid)
# Update material type
for material in grid.materials:

查看文件

@@ -90,7 +90,7 @@ class ModelConfig:
except:
deviceID = 0
self.device = {"dev": sim_config.set_model_device(deviceID), "snapsgpu2cpu": False}
self.device = {"dev": sim_config.set_model_device(deviceID), "deviceID": deviceID, "snapsgpu2cpu": False}
# Total memory usage for all grids in the model. Starts with 50MB overhead.
self.mem_overhead = 65e6
@@ -121,14 +121,14 @@ class ModelConfig:
# dispersivedtype: Data type for dispersive materials.
# dispersiveCdtype: Data type for dispersive materials in Cython.
# drudelorentz: True/False model contains Drude or Lorentz materials.
# cudarealfunc: String to substitute into CUDA kernels for fields
# crealfunc: String to substitute into CUDA/OpenCL kernels for fields
# dependent on dispersive material type.
self.materials = {
"maxpoles": 0,
"dispersivedtype": None,
"dispersiveCdtype": None,
"drudelorentz": None,
"cudarealfunc": "",
"crealfunc": None,
}
def get_scene(self):

查看文件

@@ -17,10 +17,14 @@
# along with gprMax. If not, see <http://www.gnu.org/licenses/>.
import datetime
import gc
import logging
import sys
import humanize
from colorama import Fore, Style, init
init()
import gprMax.config as config
@@ -81,6 +85,14 @@ class Context:
if not config.sim_config.args.geometry_only:
solver = create_solver(G)
model.solve(solver)
del solver, model
if not config.sim_config.args.geometry_fixed:
# Manual garbage collection required to stop memory leak on GPUs
# when using pycuda
del G
gc.collect()
self.tsimend = timer()
self.print_sim_time_taken()
@@ -130,7 +142,11 @@ class MPIContext(Context):
model_config = config.ModelConfig()
# Set GPU deviceID according to worker rank
if config.sim_config.general["solver"] == "cuda":
model_config.device = {"dev": config.sim_config.devices["devs"][self.rank - 1], "snapsgpu2cpu": False}
model_config.device = {
"dev": config.sim_config.devices["devs"][self.rank - 1],
"deviceID": self.rank - 1,
"snapsgpu2cpu": False,
}
config.model_configs = model_config
G = create_G()
@@ -140,6 +156,12 @@ class MPIContext(Context):
if not config.sim_config.args.geometry_only:
solver = create_solver(G)
model.solve(solver)
del solver, model
# Manual garbage collection required to stop memory leak on GPUs when
# using pycuda
del G
gc.collect()
def run(self):
"""Specialise how the models are run.
@@ -156,6 +178,10 @@ class MPIContext(Context):
print_cuda_info(config.sim_config.devices["devs"])
elif config.sim_config.general["solver"] == "opencl":
print_opencl_info(config.sim_config.devices["devs"])
s = f"\n--- Input file: {config.sim_config.input_file_path}"
logger.basic(Fore.GREEN + f"{s} {'-' * (get_terminal_width() - 1 - len(s))}\n" + Style.RESET_ALL)
sys.stdout.flush()
# Contruct MPIExecutor

查看文件

@@ -9,7 +9,7 @@
#define IDX2D_SRCWAVES(m, n) (m)*({{NY_SRCWAVES}})+(n)
#define IDX3D_FIELDS(i, j, k) (i)*({{NY_FIELDS}})*({{NZ_FIELDS}})+(j)*({{NZ_FIELDS}})+(k)
#define IDX3D_RXS(i,j,k) (i)*({{NY_RXS}})*({{NZ_RXS}})+(j)*({{NZ_RXS}})+(k)
#define IDX3D_RXS(i, j, k) (i)*({{NY_RXS}})*({{NZ_RXS}})+(j)*({{NZ_RXS}})+(k)
#define IDX4D_ID(p, i, j, k) (p)*({{NX_ID}})*({{NY_ID}})*({{NZ_ID}})+(i)*({{NY_ID}})*({{NZ_ID}})+(j)*({{NZ_ID}})+(k)
#define IDX4D_SNAPS(p, i, j, k) (p)*({{NX_SNAPS}})*({{NY_SNAPS}})*({{NZ_SNAPS}})+(i)*({{NY_SNAPS}})*({{NZ_SNAPS}})+(j)*({{NZ_SNAPS}})+(k)

查看文件

@@ -260,6 +260,7 @@ class Grass:
self.numblades = numblades
self.geometryparams = np.zeros((self.numblades, 6), dtype=config.sim_config.dtypes["float_or_double"])
self.seed = seed
self.set_geometry_parameters()
def set_geometry_parameters(self):
"""Sets randomly defined parameters that will be used to calculate

查看文件

@@ -96,9 +96,7 @@ class GeometryView:
def set_filename(self):
"""Constructs filename from user-supplied name and model run number."""
parts = config.get_model_config().output_file_path.parts
self.filename = Path(*parts[:-1],
self.filenamebase +
config.get_model_config().appendmodelnumber)
self.filename = Path(*parts[:-1], self.filenamebase + config.get_model_config().appendmodelnumber)
class GeometryViewLines(GeometryView):

查看文件

@@ -325,7 +325,6 @@ class PeplinskiSoil:
m.se = sig
m.deltaer.append(er - eri)
m.tau.append(DispersiveMaterial.watertau)
matID = m.er * m.se
m.ID = f"|{float(m.er):.4f}+{float(m.se):.4f}+{float(m.mr):.4f}+{float(m.sm):.4f}|"
G.materials.append(m)
self.matID.append(m.numID)

查看文件

@@ -58,8 +58,8 @@ class ModelBuildRun:
# Set number of OpenMP threads to physical threads at this point to be
# used with threaded model building methods, e.g. fractals. Can be
# changed by #num_threads command in input file or via API later for
# use with CPU solver.
# changed by the user via #num_threads command in input file or via API
# later for use with CPU solver.
config.get_model_config().ompthreads = set_omp_threads(config.get_model_config().ompthreads)
def build(self):
@@ -309,14 +309,21 @@ class ModelBuildRun:
if config.sim_config.general["solver"] == "opencl":
solvername = "OpenCL"
platformname = " on " + " ".join(config.get_model_config().device["dev"].platform.name.split())
devicename = " ".join(config.get_model_config().device["dev"].name.split())
devicename = (
f'Device {config.get_model_config().device["deviceID"]}: '
f'{" ".join(config.get_model_config().device["dev"].name.split())}'
)
else:
solvername = "CUDA"
platformname = ""
devicename = " ".join(config.get_model_config().device["dev"].name().split())
devicename = (
f'Device {config.get_model_config().device["deviceID"]}: '
f'{" ".join(config.get_model_config().device["dev"].name().split())}'
)
logger.basic(
f"\nModel {config.model_num + 1}/{config.sim_config.model_end} "
f"on {config.sim_config.hostinfo['hostname']} "
f"solving on {config.sim_config.hostinfo['hostname']} "
f"with {solvername} backend using {devicename}{platformname}"
)

查看文件

@@ -18,11 +18,11 @@
import logging
from gprMax.cmds_geometry.cmds_geometry import UserObjectGeometry
from gprMax.cmds_geometry.fractal_box import FractalBox
from gprMax.cmds_geometry.add_grass import AddGrass
from gprMax.cmds_geometry.add_surface_roughness import AddSurfaceRoughness
from gprMax.cmds_geometry.add_surface_water import AddSurfaceWater
from gprMax.cmds_geometry.cmds_geometry import UserObjectGeometry
from gprMax.cmds_geometry.fractal_box import FractalBox
from gprMax.cmds_multiuse import UserObjectMulti
from gprMax.cmds_singleuse import Discretisation, Domain, TimeWindow, UserObjectSingle
from gprMax.materials import create_built_in_materials
@@ -57,10 +57,10 @@ class Scene:
else:
logger.exception("This object is unknown to gprMax")
raise ValueError
def build_obj(self, obj, grid):
"""Builds objects.
Args:
obj: user object
grid: FDTDGrid class describing a grid in a model.
@@ -71,9 +71,10 @@ class Scene:
except ValueError:
logger.exception("Error creating user input object")
raise
def process_subgrid_cmds(self):
"""Process all commands in any sub-grids."""
def func(obj):
if isinstance(obj, SubGridUserBase):
return True
@@ -99,13 +100,12 @@ class Scene:
self.build_obj(obj, grid)
return self
def process_geocmds(self, commands, grid):
# Check for fractal boxes and modifications and pre-process them first
proc_cmds = []
for obj in commands:
if isinstance(obj, (FractalBox, AddGrass,
AddSurfaceRoughness, AddSurfaceWater)):
if isinstance(obj, (FractalBox, AddGrass, AddSurfaceRoughness, AddSurfaceWater)):
self.build_obj(obj, grid)
if isinstance(obj, (FractalBox)):
proc_cmds.append(obj)

查看文件

@@ -19,6 +19,7 @@
import logging
from importlib import import_module
import humanize
import numpy as np
from jinja2 import Environment, PackageLoader
@@ -359,7 +360,7 @@ class CUDAUpdates:
self.subs_func.update(
{
"REAL": config.sim_config.dtypes["C_float_or_double"],
"REALFUNC": config.get_model_config().materials["cudarealfunc"],
"REALFUNC": config.get_model_config().materials["crealfunc"],
"NX_T": self.grid.Tx.shape[1],
"NY_T": self.grid.Tx.shape[2],
"NZ_T": self.grid.Tx.shape[3],
@@ -369,10 +370,12 @@ class CUDAUpdates:
bld = self._build_knl(knl_fields_updates.update_electric_dispersive_A, self.subs_name_args, self.subs_func)
knl = self.source_module(bld, options=config.sim_config.devices["nvcc_opts"])
self.dispersive_update_a = knl.get_function("update_electric_dispersive_A")
self._copy_mat_coeffs(knl, knl)
bld = self._build_knl(knl_fields_updates.update_electric_dispersive_B, self.subs_name_args, self.subs_func)
knl = self.source_module(bld, options=config.sim_config.devices["nvcc_opts"])
self.dispersive_update_b = knl.get_function("update_electric_dispersive_B")
self._copy_mat_coeffs(knl, knl)
# Set blocks per grid and initialise field arrays on GPU
self.grid.set_blocks_per_grid()
@@ -784,9 +787,9 @@ class CUDAUpdates:
def cleanup(self):
"""Cleanup GPU context."""
# Remove context from top of stack and delete
# Remove context from top of stack and clear
self.ctx.pop()
del self.ctx
self.ctx = None
class OpenCLUpdates:

查看文件

@@ -183,4 +183,4 @@ def fft_power(waveform, dt):
def timer():
"""Time in fractional seconds."""
return timer_fn()
return timer_fn()