From 379d03fe6b895b3006bf3cc0d06fb47fc9d4d4fe Mon Sep 17 00:00:00 2001 From: Craig Warren Date: Wed, 16 Oct 2019 14:06:06 +0100 Subject: [PATCH] Multiple models and geometry-fixed partially working. --- gprMax/cmds_single_use.py | 26 ++++---- gprMax/config.py | 26 ++++---- gprMax/config_parser.py | 6 +- gprMax/contexts.py | 3 +- gprMax/grid.py | 26 ++++++-- gprMax/model_build_run.py | 122 +++++++++++++++++++------------------- gprMax/utilities.py | 28 +++++++++ 7 files changed, 144 insertions(+), 93 deletions(-) diff --git a/gprMax/cmds_single_use.py b/gprMax/cmds_single_use.py index 9b265c48..85d169e9 100644 --- a/gprMax/cmds_single_use.py +++ b/gprMax/cmds_single_use.py @@ -16,7 +16,6 @@ # You should have received a copy of the GNU General Public License # along with gprMax. If not, see . -import decimal as d import inspect import logging import os @@ -206,7 +205,8 @@ class TimeWindow(UserObjectSingle): class Messages(UserObjectSingle): - """Allows you to control the amount of information displayed on screen when gprMax is run + """Allows you to control the amount of information displayed on screen + when gprMax is run :param yn: Whether information should be displayed. :type yn: bool, optional @@ -260,7 +260,7 @@ class Title(UserObjectSingle): class NumThreads(UserObjectSingle): """Allows you to control how many OpenMP threads (usually the number of - physical CPU cores available) are used when running the model. + physical CPU cores available) are used when running the model. :param n: Number of threads. :type n: int, optional @@ -369,7 +369,8 @@ class PMLCells(UserObjectSingle): class SrcSteps(UserObjectSingle): - """Provides a simple method to allow you to move the location of all simple sources + """Provides a simple method to allow you to move the location of all simple + sources. :param p1: increments (x,y,z) to move all simple sources :type p1: list, non-optional @@ -391,7 +392,8 @@ class SrcSteps(UserObjectSingle): class RxSteps(UserObjectSingle): - """Provides a simple method to allow you to move the location of all simple receivers + """Provides a simple method to allow you to move the location of all simple + receivers. :param p1: increments (x,y,z) to move all simple receivers :type p1: list, non-optional @@ -414,7 +416,8 @@ class RxSteps(UserObjectSingle): class ExcitationFile(UserObjectSingle): """Allows you to specify an ASCII file that contains columns of amplitude - values that specify custom waveform shapes that can be used with sources in the model. + values that specify custom waveform shapes that can be used with sources + in the model. :param filepath: Excitation file path. :type filepath: str, non-optional @@ -477,14 +480,15 @@ class ExcitationFile(UserObjectSingle): singlewaveformvalues = singlewaveformvalues[:len(waveformtime)] # Zero-pad end of waveform array if it is shorter than time array elif len(singlewaveformvalues) < len(waveformtime): - singlewaveformvalues = np.lib.pad(singlewaveformvalues, (0, len(singlewaveformvalues) - len(waveformvalues)), 'constant', constant_values=0) + singlewaveformvalues = np.lib.pad(singlewaveformvalues, + (0, len(singlewaveformvalues) - + len(waveformvalues)), + 'constant', constant_values=0) # Interpolate waveform values w.userfunc = interpolate.interp1d(waveformtime, singlewaveformvalues, **kwargs) - log.info(f"User waveform {w.ID} created using {timestr} and, if \ - required, interpolation parameters (kind: {kwargs['kind']}, \ - fill value: {kwargs['fill_value']}).") + log.info(f"User waveform {w.ID} created using {timestr} and, if required, interpolation parameters (kind: {kwargs['kind']}, fill value: {kwargs['fill_value']}).") G.waveforms.append(w) @@ -505,7 +509,7 @@ class OutputDir(UserObjectSingle): class NumberOfModelRuns(UserObjectSingle): """Number of times to run the simulation. This required when using multiple - class:Scene instances. + class:Scene instances. :param n: File path to directory. :type n: str, non-optional diff --git a/gprMax/config.py b/gprMax/config.py index eefadb99..4945227c 100644 --- a/gprMax/config.py +++ b/gprMax/config.py @@ -53,6 +53,7 @@ class ModelConfig: """ self.i = model_num # Indexed from 0 + self.grids = [] self.ompthreads = None # Number of OpenMP threads # Store information for CUDA solver type @@ -61,7 +62,10 @@ class ModelConfig: # N.B. This will happen if the requested snapshots are too large to fit # on the memory of the GPU. If True this will slow performance significantly self.cuda = {'gpu': None, 'snapsgpu2cpu': False} - self.memoryusage = 0 # Total memory usage for all grids in the model + + # Total memory usage for all grids in the model. Starts with 50MB overhead. + self.mem_use = 50e6 + self.reuse_geometry = False if not sim_config.single_model: @@ -79,7 +83,7 @@ class ModelConfig: # String to print at start of each model run inputfilestr = f'\n--- Model {self.i + 1}/{sim_config.model_end}, input file: {sim_config.input_file_path}' - self.next_model = Fore.GREEN + f"{inputfilestr} {'-' * (get_terminal_width() - 1 - len(inputfilestr))}\n" + Style.RESET_ALL + self.set_inputfilestr(inputfilestr) # Numerical dispersion analysis parameters # highestfreqthres: threshold (dB) down from maximum power (0dB) of main frequency used @@ -98,13 +102,6 @@ class ModelConfig: 'dispersivedtype': None, 'dispersiveCdtype': None} - def memory_check(self): - """Check if the required amount of memory (RAM) is available to build - and/or run model on the host. - """ - if self.memoryusage > config.sim_config.hostinfo['ram']: - raise GeneralError(f"Memory (RAM) required ~{human_size(self.memoryusage)} exceeds {human_size(config.hostinfo['ram'], a_kilobyte_is_1024_bytes=True)} detected!\n") - def get_scene(self): if sim_config.scenes: return sim_config.scenes[self.i] @@ -119,6 +116,14 @@ class ModelConfig: 'current_model_run': self.i + 1, 'inputfile': sim_config.input_file_path.resolve()} + def set_inputfilestr(self, inputfilestr): + """Set string describing model. + + Args: + inputfilestr (str): Description of model. + """ + self.inputfilestr = Fore.GREEN + f"{inputfilestr} {'-' * (get_terminal_width() - 1 - len(inputfilestr))}\n" + Style.RESET_ALL + class SimulationConfig: """Configuration parameters for a standard simulation. @@ -166,9 +171,6 @@ class SimulationConfig: # Information about any GPUs as a list of GPU objects self.cuda_gpus = [] - # Data type (precision) for electromagnetic field output - self.dtypes = None - # Subgrid parameter may not exist if user enters via CLI try: self.subgrid = args.subgrid diff --git a/gprMax/config_parser.py b/gprMax/config_parser.py index a14ceaec..23f90ea7 100755 --- a/gprMax/config_parser.py +++ b/gprMax/config_parser.py @@ -34,14 +34,14 @@ def write_simulation_config(args): config.sim_config = config.SimulationConfig(args) -def write_model_config(i): +def write_model_config(model_num): """Write model level configuration parameters to config module. As there can only be one instance of the config module objects are always found via 'import gprMax.config' Args: - i (int): Model number. + model_num (int): Model number. """ - model_config = config.ModelConfig(i) + model_config = config.ModelConfig(model_num) config.model_configs.append(model_config) diff --git a/gprMax/contexts.py b/gprMax/contexts.py index f4320349..f136f3c9 100644 --- a/gprMax/contexts.py +++ b/gprMax/contexts.py @@ -82,12 +82,13 @@ class NoMPIContext(Context): for i in self.model_range: write_model_config(i) - # Always create a solver for the first model. # The next model to run only gets a new solver if the # geometry is not re-used. if i != 0 and config.sim_config.args.geometry_fixed: config.model_configs[i].reuse_geometry = True + # Ensure re-used G is associated correctly with model + G.model_num = i else: G = create_G(i) diff --git a/gprMax/grid.py b/gprMax/grid.py index 501b69d9..2b36edec 100644 --- a/gprMax/grid.py +++ b/gprMax/grid.py @@ -171,12 +171,14 @@ class FDTDGrid: self.Tx = np.zeros((config.materials['maxpoles'], self.nx + 1, self.ny + 1, self.nz + 1), dtype=dtype) self.Ty = np.zeros((config.materials['maxpoles'], self.nx + 1, self.ny + 1, self.nz + 1), dtype=dtype) self.Tz = np.zeros((config.materials['maxpoles'], self.nx + 1, self.ny + 1, self.nz + 1), dtype=dtype) - self.updatecoeffsdispersive = np.zeros((len(self.materials), 3 * config.materials['maxpoles']), dtype=dtype) + self.updatecoeffsdispersive = np.zeros((len(self.materials), 3 * config.model_configs[self.model_num].materials['maxpoles']), dtype=dtype) - def memory_estimate_basic(self): - """Estimate the amount of memory (RAM) required to run a model.""" + def mem_est_basic(self): + """Estimate the amount of memory (RAM) required for grid arrays. - stdoverhead = 50e6 + Returns: + mem_use (int): Memory (bytes). + """ solidarray = self.nx * self.ny * self.nz * np.dtype(np.uint32).itemsize @@ -206,7 +208,21 @@ class FDTDGrid: pmlarrays += ((self.nx + 1) * self.ny * v) pmlarrays += (self.nx * (self.ny + 1) * v) - self.memoryusage = int(stdoverhead + fieldarrays + solidarray + rigidarrays + pmlarrays) + mem_use = int(fieldarrays + solidarray + rigidarrays + pmlarrays) + + return mem_use + + def mem_est_dispersive(self): + """Estimate the amount of memory (RAM) required for dispersive grid arrays. + + Returns: + mem_use (int): Memory (bytes). + """ + + mem_use = int(3 * config.model_configs[self.model_num].materials['maxpoles'] * + (G.nx + 1) * (G.ny + 1) * (G.nz + 1) * + np.dtype(config.model_configs[self.model_num].materials['dispersivedtype']).itemsize) + return mem_use def tmx(self): """Add PEC boundaries to invariant direction in 2D TMx mode. diff --git a/gprMax/model_build_run.py b/gprMax/model_build_run.py index 7f659aa2..3760671b 100644 --- a/gprMax/model_build_run.py +++ b/gprMax/model_build_run.py @@ -69,6 +69,7 @@ from .solvers import create_solver from .sources import gpu_initialise_src_arrays from .utilities import get_terminal_width from .utilities import human_size +from .utilities import mem_check from .utilities import open_path_file from .utilities import round32 from .utilities import set_omp_threads @@ -100,34 +101,33 @@ class ModelBuildRun: if G.srcsteps[0] != 0 or G.srcsteps[1] != 0 or G.srcsteps[2] != 0: for source in itertools.chain(G.hertziandipoles, G.magneticdipoles): if config.model_configs[G.model_num] == 0: - if (source.xcoord + G.srcsteps[0] * self.sim_config.model_end < 0 or - source.xcoord + G.srcsteps[0] * self.sim_config.model_end > G.nx or - source.ycoord + G.srcsteps[1] * self.sim_config.model_end < 0 or - source.ycoord + G.srcsteps[1] * self.sim_config.model_end > G.ny or - source.zcoord + G.srcsteps[2] * self.sim_config.model_end < 0 or - source.zcoord + G.srcsteps[2] * self.sim_config.model_end > G.nz): + if (source.xcoord + G.srcsteps[0] * config.sim_config.model_end < 0 or + source.xcoord + G.srcsteps[0] * config.sim_config.model_end > G.nx or + source.ycoord + G.srcsteps[1] * config.sim_config.model_end < 0 or + source.ycoord + G.srcsteps[1] * config.sim_config.model_end > G.ny or + source.zcoord + G.srcsteps[2] * config.sim_config.model_end < 0 or + source.zcoord + G.srcsteps[2] * config.sim_config.model_end > G.nz): raise GeneralError('Source(s) will be stepped to a position outside the domain.') - source.xcoord = source.xcoordorigin + config.model_configs[G.model_num] * G.srcsteps[0] - source.ycoord = source.ycoordorigin + config.model_configs[G.model_num] * G.srcsteps[1] - source.zcoord = source.zcoordorigin + config.model_configs[G.model_num] * G.srcsteps[2] + source.xcoord = source.xcoordorigin + G.model_num * G.srcsteps[0] + source.ycoord = source.ycoordorigin + G.model_num * G.srcsteps[1] + source.zcoord = source.zcoordorigin + G.model_num * G.srcsteps[2] if G.rxsteps[0] != 0 or G.rxsteps[1] != 0 or G.rxsteps[2] != 0: for receiver in G.rxs: if config.model_configs[G.model_num] == 0: - if (receiver.xcoord + G.rxsteps[0] * self.sim_config.model_end < 0 or - receiver.xcoord + G.rxsteps[0] * self.sim_config.model_end > G.nx or - receiver.ycoord + G.rxsteps[1] * self.sim_config.model_end < 0 or - receiver.ycoord + G.rxsteps[1] * self.sim_config.model_end > G.ny or - receiver.zcoord + G.rxsteps[2] * self.sim_config.model_end < 0 or - receiver.zcoord + G.rxsteps[2] * self.sim_config.model_end > G.nz): + if (receiver.xcoord + G.rxsteps[0] * config.sim_config.model_end < 0 or + receiver.xcoord + G.rxsteps[0] * config.sim_config.model_end > G.nx or + receiver.ycoord + G.rxsteps[1] * config.sim_config.model_end < 0 or + receiver.ycoord + G.rxsteps[1] * config.sim_config.model_end > G.ny or + receiver.zcoord + G.rxsteps[2] * config.sim_config.model_end < 0 or + receiver.zcoord + G.rxsteps[2] * config.sim_config.model_end > G.nz): raise GeneralError('Receiver(s) will be stepped to a position outside the domain.') - receiver.xcoord = receiver.xcoordorigin + config.model_configs[G.model_num] * G.rxsteps[0] - receiver.ycoord = receiver.ycoordorigin + config.model_configs[G.model_num] * G.rxsteps[1] - receiver.zcoord = receiver.zcoordorigin + config.model_configs[G.model_num] * G.rxsteps[2] + receiver.xcoord = receiver.xcoordorigin + G.model_num * G.rxsteps[0] + receiver.ycoord = receiver.ycoordorigin + G.model_num * G.rxsteps[1] + receiver.zcoord = receiver.zcoordorigin + G.model_num * G.rxsteps[2] # Write files for any geometry views and geometry object outputs - if not (G.geometryviews or G.geometryobjectswrite) and self.sim_config.geometry_only and config.is_messages(): - log.warning(Fore.RED + f'\nNo geometry views or geometry objects to output found.' + Style.RESET_ALL) - if config.sim_config.is_messages(): log.info('') + if not (G.geometryviews or G.geometryobjectswrite) and config.sim_config.args.geometry_only: + log.warning(Fore.RED + f'\nNo geometry views or geometry objects found.' + Style.RESET_ALL) for i, geometryview in enumerate(G.geometryviews): geometryview.set_filename(config.model_configs[G.model_num].appendmodelnumber) pbar = tqdm(total=geometryview.datawritesize, unit='byte', unit_scale=True, @@ -143,17 +143,21 @@ class ModelBuildRun: disable=not config.sim_config.general['progressbars']) geometryobject.write_hdf5(G, pbar) pbar.close() - if config.sim_config.is_messages(): log.info('') def build_geometry(self): G = self.G - log.info(config.model_configs[G.model_num].next_model) + log.info(config.model_configs[G.model_num].inputfilestr) scene = self.build_scene() - # Combine available grids + # Combine available grids and check memory requirements grids = [G] + G.subgrids + for grid in grids: + config.model_configs[G.model_num].mem_use += grid.mem_est_basic() + mem_check(config.model_configs[G.model_num].mem_use) + log.info(f'\nMemory (RAM) required: ~{human_size(config.model_configs[G.model_num].mem_use)}') + gridbuilders = [GridBuilder(grid) for grid in grids] for gb in gridbuilders: @@ -168,33 +172,32 @@ class ModelBuildRun: if config.model_configs[G.model_num].materials['maxpoles'] != 0: drudelorentz = any([m for m in G.materials if 'drude' in m.type or 'lorentz' in m.type]) if drudelorentz: - config.materials['dispersivedtype'] = config.dtypes['complex'] - config.materials['dispersiveCdtype'] = config.dtypes['C_complex'] + config.model_configs[G.model_num].materials['dispersivedtype'] = config.sim_config.dtypes['complex'] + config.model_configs[G.model_num].materials['dispersiveCdtype'] = config.sim_config.dtypes['C_complex'] else: - config.materials['dispersivedtype'] = config.dtypes['float_or_double'] - config.materials['dispersiveCdtype'] = config.dtypes['C_float_or_double'] + config.model_configs[G.model_num].materials['dispersivedtype'] = config.sim_config.dtypes['float_or_double'] + config.model_configs[G.model_num].materials['dispersiveCdtype'] = config.sim_config.dtypes['C_float_or_double'] # Update estimated memory (RAM) usage - config.model_configs[G.model_num].memoryusage += int(3 * - config.materials['maxpoles'] * - (G.nx + 1) * (G.ny + 1) * (G.nz + 1) * - np.dtype(config.materials['dispersivedtype']).itemsize) - G.memory_check() - log.info(f'\nMemory (RAM) required - updated (dispersive): ~{human_size(config.model_configs[G.model_num].memoryusage)}\n') + config.model_configs[G.model_num].mem_use += G.mem_est_dispersive() + mem_check(config.model_configs[G.model_num].mem_use) + log.info(f'Memory (RAM) required - updated (dispersive): ~{human_size(config.model_configs[G.model_num].mem_use)}') for gb in gridbuilders: - gb.grid.initialise_dispersive_arrays(config.materials['dispersivedtype']) + gb.grid.initialise_dispersive_arrays(config.model_configs[G.model_num].materials['dispersivedtype']) # Check there is sufficient memory to store any snapshots if G.snapshots: - snapsmemsize = 0 + snaps_mem = 0 for snap in G.snapshots: # 2 x required to account for electric and magnetic fields - snapsmemsize += (2 * snap.datasizefield) - G.memoryusage += int(snapsmemsize) - G.memory_check(snapsmemsize=int(snapsmemsize)) - - log.info(f'\nMemory (RAM) required - updated (snapshots): ~{human_size(G.memoryusage)}\n') + snaps_mem += int(2 * snap.datasizefield) + config.model_configs[G.model_num].mem_use += snaps_mem + # Check if there is sufficient memory on host + mem_check(config.model_configs[G.model_num].mem_use) + if config.sim_config.general['cuda']: + mem_check_gpu_snaps(G.model_num, snaps_mem) + log.info(f'Memory (RAM) required - updated (snapshots): ~{human_size(config.model_configs[G.model_num].mem_use)}') # Build materials for gb in gridbuilders: @@ -215,8 +218,8 @@ class ModelBuildRun: def reuse_geometry(self): # Reset iteration number self.G.iteration = 0 - config.model_configs[self.G.model_num].inputfilestr = f'\n--- Model {config.model_configs[self.G.model_num].appendmodelnumber}/{self.sim_config.model_end}, input file (not re-processed, i.e. geometry fixed): {self.sim_config.input_file_path}' - log.info(Fore.GREEN + f"{config.model_configs[self.G.model_num].inputfilestr} {'-' * (get_terminal_width() - 1 - len(config.model_configs[self.G.model_num].inputfilestr))}" + Style.RESET_ALL) + config.model_configs[self.G.model_num].set_inputfilestr(f'\n--- Model {config.model_configs[self.G.model_num].appendmodelnumber}/{config.sim_config.model_end}, input file (not re-processed, i.e. geometry fixed): {config.sim_config.input_file_path}') + log.info(config.model_configs[self.G.model_num].inputfilestr) for grid in [self.G] + self.G.subgrids: grid.reset_fields() @@ -224,13 +227,13 @@ class ModelBuildRun: # API for multiple scenes / model runs scene = config.model_configs[self.G.model_num].get_scene() - # If there is no scene - process the hash commands instead + # If there is no scene, process the hash commands if not scene: scene = Scene() # Parse the input file into user objects and add them to the scene scene = parse_hash_commands(config.model_configs[self.G.model_num], self.G, scene) - # Creates the internal simulation objects. + # Creates the internal simulation objects scene.create_internal_objects(self.G) return scene @@ -281,11 +284,11 @@ class ModelBuildRun: tsolve (float): time taken to execute solving (seconds). """ - memGPU = '' + mem_GPU = '' if config.sim_config.general['cuda']: - memGPU = f' host + ~{human_size(self.solver.get_memsolve())} GPU' + mem_GPU = f' host + ~{human_size(self.solver.get_memsolve())} GPU' - log.info(f'\nMemory (RAM) used: ~{human_size(self.p.memory_full_info().uss)}{memGPU}') + log.info(f'\nMemory (RAM) used: ~{human_size(self.p.memory_full_info().uss)}{mem_GPU}') log.info(f'Solving time [HH:MM:SS]: {datetime.timedelta(seconds=tsolve)}') def solve(self, solver): @@ -299,24 +302,23 @@ class ModelBuildRun: """ self.create_output_directory() - log.info(f'Output file: {config.model_configs[self.G.model_num].output_file_path_ext}') + log.info(f'\nOutput file: {config.model_configs[self.G.model_num].output_file_path_ext}') # Set and check number of OpenMP threads if config.sim_config.general['cpu']: config.model_configs[self.G.model_num].ompthreads = set_omp_threads(config.model_configs[self.G.model_num].ompthreads) - log.info(f'CPU (OpenMP) threads for solving: {config.model_configs[self.G.model_num].ompthreads}') + log.info(f'CPU (OpenMP) threads for solving: {config.model_configs[self.G.model_num].ompthreads}\n') if config.model_configs[self.G.model_num].ompthreads > config.sim_config.hostinfo['physicalcores']: log.warning(Fore.RED + f"You have specified more threads ({config.model_configs[self.G.model_num].ompthreads}) than available physical CPU cores ({config.sim_config.hostinfo['physicalcores']}). This may lead to degraded performance." + Style.RESET_ALL) - # Print information about any GPU in use - if config.sim_config.general['cuda']: - log.info(f"GPU for solving: {config.model_configs[self.G.model_num].cuda['gpu'].deviceID} - {config.model_configs[self.G.model_num].cuda['gpu'].name}") + elif config.sim_config.general['cuda']: + log.info(f"GPU for solving: {config.model_configs[self.G.model_num].cuda['gpu'].deviceID} - {config.model_configs[self.G.model_num].cuda['gpu'].name}\n") # Prepare iterator if config.sim_config.is_messages(): - iterator = tqdm(range(self.G.iterations), desc='\nRunning simulation, model ' + str(self.G.model_num + 1) + '/' + str(config.sim_config.model_end), ncols=get_terminal_width() - 1, file=sys.stdout, disable=not config.sim_config.general['progressbars']) + iterator = tqdm(range(self.G.iterations), desc='Running simulation, model ' + str(self.G.model_num + 1) + '/' + str(config.sim_config.model_end), ncols=get_terminal_width() - 1, file=sys.stdout, disable=not config.sim_config.general['progressbars']) else: - iterator = range(0, self.G.iterations) + iterator = range(self.G.iterations) # Run solver tsolve = solver.solve(iterator) @@ -335,16 +337,14 @@ class GridBuilder: self.grid = grid def build_pmls(self): - grid = self.grid - - # Build the PMLS - pbar = tqdm(total=sum(1 for value in grid.pmlthickness.values() if value > 0), + log.info('') + pbar = tqdm(total=sum(1 for value in self.grid.pmlthickness.values() if value > 0), desc=f'Building {self.grid.name} Grid PML boundaries', ncols=get_terminal_width() - 1, file=sys.stdout, disable=not config.sim_config.general['progressbars']) - for pml_id, thickness in grid.pmlthickness.items(): + for pml_id, thickness in self.grid.pmlthickness.items(): if thickness > 0: - build_pml(grid, pml_id, thickness) + build_pml(self.grid, pml_id, thickness) pbar.update() pbar.close() diff --git a/gprMax/utilities.py b/gprMax/utilities.py index c49c5f3c..47a85b92 100644 --- a/gprMax/utilities.py +++ b/gprMax/utilities.py @@ -379,6 +379,34 @@ def set_omp_threads(nthreads=None): return nthreads +def mem_check(mem): + """Check if the required amount of memory (RAM) is available on host. + + Args: + mem (int): Memory required (bytes). + """ + if mem > config.sim_config.hostinfo['ram']: + raise GeneralError(f"Memory (RAM) required ~{human_size(mem)} exceeds {human_size(config.sim_config.hostinfo['ram'], a_kilobyte_is_1024_bytes=True)} detected!\n") + + +def mem_check_gpu_snaps(model_num, snaps_mem): + """Check if the required amount of memory (RAM) for all snapshots can fit + on specified GPU. + + Args: + model_num (int): Model number. + snaps_mem (int): Memory required for all snapshots (bytes). + """ + if config.model_configs[G.model_num].mem_use - snaps_mem > gpu.totalmem: + raise GeneralError(f'Memory (RAM) required ~{human_size(config.model_configs[G.model_num].mem_use)} exceeds {human_size(config.model_configs[G.model_num].gpu.totalmem, a_kilobyte_is_1024_bytes=True)} detected on specified {config.model_configs[G.model_num].gpu.deviceID} - {config.model_configs[G.model_num].gpu.name} GPU!\n') + + # If the required memory without the snapshots will fit on the GPU then + # transfer and store snaphots on host + if (snaps_mem != 0 and config.model_configs[G.model_num].mem_use - + snaps_mem < config.model_configs[G.model_num].gpu.totalmem): + config.model_configs[G.model_num].cuda['snapsgpu2cpu'] = True + + class GPU: """GPU information."""