Clean up experiments backend code

Author: Lucas Rebscher

netpyne_ui/experiments.py

@@ -20,13 +20,20 @@
 NET_PARAMS_FILE = "netParams.json"
 EXPERIMENT_FILE = "experiment.json"
 
+# Id for the base model trial created for an Experiment without parameters.
 BASE_TRIAL_ID = "model_output"
 
+
 class ExperimentsError(Exception):
+    """ Base Exception for any specific Experiment errors. """
     pass
 
 
 def get_experiments() -> List[dict]:
+    """Returns the current list of experiments.
+
+    Scans the experiments folder to read experiments from disk and includes experiment in design.
+    """
     # Only update Experiments stored on filesystem
     stored_experiments = _scan_experiments_directory()
     model.experiments = [
@@ -77,6 +84,11 @@ def edit_experiment(name: str, experiment: dict):
 
 
 def replace_current_with(name: str):
+    """Replaces the experiment in design with a new experiment based on experiment with `name`.
+
+    :param name: name of experiment to be cloned.
+    :raises ExperimentsError: thrown if Experiment with `name` doesn't exist.
+    """
     exp = _get_by_name(name)
     if not exp:
         raise ExperimentsError(f"Experiment with name {name} does not exist")
@@ -116,6 +128,7 @@ def any_in_state(states: List[model.ExperimentState]) -> model.Experiment:
 
 
 def set_to_error(experiment: model.Experiment):
+    """Sets the state of `experiment` to ERROR. """
     path = os.path.join(
         constants.NETPYNE_WORKDIR_PATH, constants.EXPERIMENTS_FOLDER, experiment.name
     )
@@ -216,7 +229,7 @@ def _parse_experiment(directory: str) -> model.Experiment:
         * netParams.json
         * simConfig.json
         * json file for each trial in case of batch
-        * output files for each trial (if available)
+        * data files for each trial (if available)
 
     :raises ExperimentsError
     """
@@ -230,13 +243,6 @@ def _parse_experiment(directory: str) -> model.Experiment:
     except IOError:
         raise ExperimentsError(f"Could not find {EXPERIMENT_FILE}")
 
-    with open(os.path.join(path, NET_PARAMS_FILE), "r") as f:
-        net_params = json.load(f)
-
-    with open(os.path.join(path, SIM_CONFIG_FILE), "r") as f:
-        sim_config = json.load(f)
-
-    run_cfg = experiment_config["runCfg"]
     del experiment_config["runCfg"]
 
     # Convert timestamp to datetime
@@ -251,7 +257,7 @@ def _parse_experiment(directory: str) -> model.Experiment:
 
 
 def _delete_experiment_folder(experiment: model.Experiment):
-    """Recursively deletes the associated experiment folder."""
+    """Recursively deletes the associated `experiment` folder."""
 
     def onerror(func, path, exc_info):
         # TODO: error handling
@@ -271,6 +277,11 @@ def _create_base_model_trial() -> model.Trial:
 
 
 def _create_trials(experiment: model.Experiment) -> List[model.Trial]:
+    """Gerates based on `experiment.params` all possible trial combinations.
+
+    :param experiment: given experiment.
+    :return: list of generated trials.
+    """
     params = copy.deepcopy(experiment.params)
     params = process_params(params)
 
@@ -314,6 +325,12 @@ def _create_trials(experiment: model.Experiment) -> List[model.Trial]:
 
 
 def process_params(params: List[model.ExplorationParameter]) -> List[model.ExplorationParameter]:
+    """Cleaning, filtering and converting of experiment `params`.
+
+    :param params: given parameters.
+    :raises ExperimentsError: thrown for invalid parameter configuration.
+    :return: processed list of params.
+    """
     params = [p for p in params if p.mapsTo != '']
 
     for param in params:

netpyne_ui/model.py

@@ -5,87 +5,6 @@
 
 
 def register(metadata):
-    metadata['batch_config'] = {
-        "label": "Batch configuration",
-        "help": "",
-        "suggestions": "",
-        "hintText": "",
-        'children': {
-            'enabled': {
-                "label": "Run as batch",
-                "help": "Activates batch",
-                "suggestions": "",
-                "hintText": "",
-                "type": "bool"
-            },
-            'name': {
-                "label": "Name of the batch",
-                "help": "",
-                "suggestions": "",
-                "hintText": "",
-                "type": "str"
-            },
-            'saveFolder': {
-                "label": "Save folder",
-                "help": "",
-                "suggestions": "",
-                "hintText": "",
-                "type": "str"
-            },
-            'seed': {
-                "label": "Seed",
-                "help": "",
-                "suggestions": "",
-                "hintText": "",
-                "type": "int"
-            },
-            'method': {
-                "label": "Exploration method",
-                "help": "",
-                "suggestions": "",
-                "hintText": "",
-                "type": "str"
-            }
-        }
-    }
-
-    metadata["run_config"] = {
-        "label": "Run configuration",
-        "help": "",
-        "suggestions": "",
-        "hintText": "",
-        'children': {
-            'parallel': {
-                "label": "Parallel",
-                "help": "",
-                "suggestions": "",
-                "hintText": "",
-                "type": "bool"
-            },
-            'asynchronous': {
-                "label": "Run in background",
-                "help": "",
-                "suggestions": "",
-                "hintText": "",
-                "type": "bool"
-            },
-            'type': {
-                "label": "Execution type",
-                "help": "",
-                "suggestions": "",
-                "hintText": "",
-                "type": "str"
-            },
-            'cores': {
-                "label": "Number of cores",
-                "help": "",
-                "suggestions": "",
-                "hintText": "",
-                "type": "int"
-            }
-        }
-    }
-
     # Modification of metadata to enable validation of experiment parameters.
     metadata['netParams']['children']['popParams']['container'] = True
     metadata['netParams']['children']['cellParams']['container'] = True
@@ -96,6 +15,7 @@ def register(metadata):
     metadata['netParams']['children']['stimSourceParams']['container'] = True
     metadata['netParams']['children']['stimTargetParams']['container'] = True
 
+
 class ExperimentState:
     DESIGN = "DESIGN"
     PENDING = "PENDING"
@@ -136,7 +56,7 @@ class Trial:
     # [{ "paramX": 0.2, "paramY": "0.4"}]
     params: list = field(default_factory=list)
     indices: list = field(default_factory=list)
-    
+
 
 @dataclass
 class Experiment:

netpyne_ui/netpyne_geppetto.py

@@ -302,6 +302,7 @@ def simulateNetPyNEModelInGeppetto(self, args):
             return utils.getJSONError(message, sys.exc_info())
 
     def _prepare_simulation_files(self, experiment: model.Experiment = None, use_prev_inst: bool = False) -> str:
+        """Prepares template files and netpyne model files for a single simulation """
         exp = copy.deepcopy(experiment)
         # Remove parameter & trials for single run
         exp.params = []
@@ -358,6 +359,13 @@ def _prepare_simulation_files(self, experiment: model.Experiment = None, use_pre
         return save_folder_path
 
     def _prepare_batch_files(self, experiment: model.Experiment) -> str:
+        """Creates template files and netpyne model files in the experiment folder.
+        
+        Only for an experiment consisting of many trials.
+
+        :param experiment: given experiment
+        :return: working directory path
+        """
         exp = copy.deepcopy(experiment)
         exp.params = self.experiments.process_params(exp.params)
 

netpyne_ui/simulations.py

@@ -3,11 +3,11 @@
 import os
 import multiprocessing
 import platform
-from dataclasses import dataclass
 
 from netpyne import sim
 from netpyne_ui import constants
 
+# Available netpyne.batch.Batch methods.
 MPI_DIRECT = "mpi_direct"
 MPI_BULLETIN = "mpi_bulletin"
 
@@ -27,15 +27,6 @@ class InvalidConfigError(Exception):
     """ Thrown if invalid number of CPUs were specified. """
 
 
-@dataclass
-class Simulation:
-    cores: int
-    name: str
-    asynchronous: bool
-    batch: bool
-    subprocess = None
-
-
 class LocalSimulationPool:
     """ Pool that manages simulation running on the same machine as Netpyne-UI. """
 
@@ -47,18 +38,25 @@ def __init__(self):
 
     def run(self, parallel, cores, method="", batch=False, asynchronous=False, working_directory=None):
         if int(cores) > self.cpus:
-            raise InvalidConfigError(f"Specified {cores} cores, but only {self.cpus} are available")
+            raise InvalidConfigError(
+                f"Specified {cores} cores, but only {self.cpus} are available")
 
         logging.info(f"Scheduling simulation on {cores} cores ...")
 
         if batch or asynchronous or parallel:
             if method == MPI_DIRECT:
-                self._run_in_subprocess(_python_command(working_directory), asynchronous=asynchronous, working_directory=working_directory)
+                self._run_in_subprocess(
+                    _python_command(working_directory), asynchronous=asynchronous, working_directory=working_directory
+                )
             elif method == MPI_BULLETIN:
                 if parallel:
-                    self._run_in_subprocess(_bulletin_board_cmd(cores, working_directory), asynchronous=asynchronous, working_directory=working_directory)
+                    self._run_in_subprocess(
+                        _bulletin_board_cmd(cores, working_directory), asynchronous=asynchronous, working_directory=working_directory
+                    )
                 else:
-                    self._run_in_subprocess(_python_command(working_directory), asynchronous=asynchronous, working_directory=working_directory)
+                    self._run_in_subprocess(
+                        _python_command(working_directory), asynchronous=asynchronous, working_directory=working_directory
+                    )
             else:
                 raise InvalidConfigError(f"Unsupported method {method}")
         else:
@@ -89,7 +87,7 @@ def _run_in_same_process(self):
         sim.simulate()
         sim.saveData()
 
-    def _run_in_subprocess(self, cmds, asynchronous=False, working_directory: str=None):
+    def _run_in_subprocess(self, cmds, asynchronous=False, working_directory: str = None):
         if self.is_running():
             logging.info("Another simulation is still running")
             return False
@@ -110,6 +108,11 @@ def _run_in_subprocess(self, cmds, asynchronous=False, working_directory: str=No
 
 
 def _bulletin_board_cmd(cores, working_directory=None):
+    """ Creates command to run batch or single simulations in parallel.
+
+    Uses mpiexec that implements the OpenMPI protocol to schedule simulations.
+    Uses the master/worker pattern.
+    """
     return ["mpiexec", "-n", str(cores), "--use-hwthread-cpus", "nrniv", "-python", "-mpi", _script_path(working_directory)]