Merge branch 'development' of https://github.com/metacell/netpyne-ui into feature/depandabot_20231030

Author: ddelpiano

Dockerfile

@@ -1,67 +1,83 @@
-FROM node:14.21.3-bullseye as jsbuild
+FROM node:14 as jsbuild
 
-WORKDIR /app
-
-COPY --chown=1000:1000 webapp/package.json .
-COPY --chown=1000:1000 webapp/yarn.lock .
+ENV FOLDER=netpyne
 
+WORKDIR $FOLDER/webapp
+COPY webapp/package.json .
+COPY webapp/yarn.lock .
 
 RUN yarn install  --network-timeout 1000000000
 
-COPY webapp/ .
+COPY webapp .
 RUN yarn build-dev
 
-
-RUN rm -Rf node_modules/*
-
 ###
 FROM jupyter/base-notebook:hub-1.5.0
 ENV NB_UID=jovyan
 ENV FOLDER=netpyne
-ARG BUILD_ARGS=""
-ARG WORKSPACE_VERSION=master
-# ARG GEPPETTO_VERSION=development
-# ARG NETPYNE_VERSION=master
-
-ENV FOLDER=/home/jovyan/work/NetPyNE-UI
+ENV NP_LFPYKIT_HEAD_FILE=/home/jovyan/nyhead.mat
 
 USER root
 
 RUN rm -rf /var/lib/apt/lists
 RUN apt-get update -qq &&\
-    apt-get install python3-tk vim nano unzip git make libtool g++ -qq pkg-config libfreetype6-dev libpng-dev libopenmpi-dev  openjdk-11-jre-headless -y -y
+    apt-get install python3-tk vim nano unzip git make libtool g++ -qq pkg-config libfreetype6-dev libpng-dev libopenmpi-dev -y
+RUN apt-get install openjdk-11-jre-headless -y
 RUN conda install python=3.7 -y
 
+
 WORKDIR $FOLDER
-COPY --chown=1000:1000 requirements.txt requirements.txt
-RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip && pip install -r requirements.txt --prefer-binary
+COPY  --chown=1000:1000 requirements.txt requirements.txt
+RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\
+    pip install -r requirements.txt --prefer-binary
 
-COPY --chown=jovyan:1000 . .
-COPY --from=jsbuild --chown=jovyan:1000 /app webapp
+COPY . .
+COPY --from=jsbuild --chown=1000:1000 $FOLDER/webapp/build webapp/build
 
+# ToDo: fixme, for now remove the jupyter hub config json file because it overrides the default
+# and thus removes the frame ancestor cors settings
+RUN rm -f ~/.jupyter/*.json
+RUN chown $NB_UID .
+RUN chown $NB_UID /opt
+RUN rm -Rf workspace
+
+USER $NB_UID
+
+# sym link workspace pvc to $FOLDER
+RUN mkdir -p /opt/workspace
+RUN mkdir -p /opt/user
+
+
+ENV NEURON_HOME=/opt/conda
+
+
+USER root
 
 RUN jupyter nbextension install --py --symlink --sys-prefix jupyter_geppetto
 RUN jupyter nbextension enable --py --sys-prefix jupyter_geppetto
 RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension
 RUN jupyter serverextension enable --py --sys-prefix jupyter_geppetto
 
-RUN python utilities/install.py ${BUILD_ARGS} --workspace $WORKSPACE_VERSION
+ARG BUILD_ARGS=""
+ARG WORKSPACE_VERSION=master
+RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\
+  python utilities/install.py ${BUILD_ARGS} --workspace $WORKSPACE_VERSION
 
-RUN jupyter labextension disable @jupyterlab/hub-extension
 
-RUN chown  $NB_UID .
-RUN chown -R $NB_UID workspace
+RUN mv workspace /opt/workspace/tutorials
+RUN chown -R $NB_UID /opt/workspace/tutorials
+RUN ln -s /opt/workspace workspace
 
-# Temp fixes for eeg plots
-ENV NEURON_HOME=/opt/conda
-# For lfpykit 0.4
-# RUN wget -P $(pip show LFPykit | grep "Location:" | awk '{print $2"/lfpykit"}') https://www.parralab.org/nyhead/sa_nyhead.mat
-# For lpfykit 0.5
-ENV NP_LFPYKIT_HEAD_FILE=/home/jovyan/nyhead.mat
+RUN jupyter labextension disable @jupyterlab/hub-extension
 RUN wget --no-check-certificate -O $NP_LFPYKIT_HEAD_FILE https://www.parralab.org/nyhead/sa_nyhead.mat
 
 USER $NB_UID
 
+
 EXPOSE 8888
 
+ENTRYPOINT ["tini", "-g", "--"]
+
+
 
+CMD ./NetPyNE-UI

README.md

@@ -30,26 +30,59 @@ your system and you have had troubles installing it you can opt for the Virtual
 
 ### Python Dependencies
 
-We recommend the use of a new python 3 virtual environment:
+We recommend the use of a new python 3.7 virtual environment.
+Currently, NetPyNE-UI only supports Python 3.7, but that can change in the future.
+
+For NetPyNE-UI, the preferred way of creating a virtual env is to pass by conda/miniconda or mamba/micromamba.
+The pyenv tool can be also used, but it requires to be compiled with some special options to have NEURON running properly the simulation using `nrniv`.
+The reason behind this is that NEURON is distributed as a Python wheel with a specific option which force NEURON to look for the CPython dynamic lib.
+However, pyenv by default installs the static version of the CPython lib, resulting in `nrniv -python` not being able to run.
+
+The way of creating the virtualenv using (mini)conda is the following
+
+```bash
+conda create -n netpyne python=3.7
+conda activate netpyne
+```
+
+Here is how to create the virtualenv using (micro)mamba
+
+```bash
+mamba create -n netpyne python=3.7 -c conda-forge
+mamba activate netpyne
+```
+
+You can also directly create a virtualenv using your `python3` executable, obviously if it's the 3.7 version.
 
 ```bash
 python3 -m venv npenv
 source npenv/bin/activate
 ```
 
-Or, with conda
+If you want to use anyway pyenv, here is how to properly create the virtualenv and activate it.
 
 ```bash
-conda create -n netpyne python=3.7
-conda activate netpyne
+env PYTHON_CONFIGURE_OPTS="--enable-shared"  pyenv install --verbose 3.7.17
+pyenv virtualenv 3.7.17 netpyne
+pyenv shell netpyne
 ```
 
 ### Run install script
 
+When you are in your virtualenv, here is how you can install the "basic" version of NetPyNE-UI:
+
 ```bash
 python utilities/install.py
 ```
 
+If you want to have a different version of NetPyNE or geppetto meta, you can pass the version you want to the installer:
+
+```bash
+python utilities/install.py --dev --netpyne development --geppetto development --no-test
+```
+
+This command will install the `development` version of netpyne and geppetto and disable the tests during the installation.
+
 ### Start application
 
 ```bash
@@ -62,6 +95,15 @@ For debugging you can use `run.py` instead
 python run.py
 ```
 
+To run the UI in dev mode, you need to run `python run.py` in one terminal, and use `yarn` from the `webapp` folder, using node v14 (use `nvm` if you need to have a different version of node than the one installed on your system):
+
+```bash
+cd webapp
+yarn start
+```
+
+You can then navigate to `http://127.0.0.1:8081/` to access the dev version of the UI.
+
 ## Run NetPyNE User Interface in Docker
 
 Ensure that you have Docker installed on your system.
@@ -75,7 +117,7 @@ docker build -t netpyne-ui .
 Run the image
 
 ```bash
-docker run -p 8888:8888 netpyne-ui 
+docker run -p 8888:8888 netpyne-ui
 ```
 
 ## End-to-end tests

jupyter_hub/Dockerfile_spawner

@@ -16,15 +16,15 @@ RUN /bin/bash -c "INCUBATOR_VER=${INCUBATOR_VER} source activate snakes && pip i
 # Clone NetPyNE-UI and install the development version
 RUN wget https://github.com/MetaCell/NetPyNE-UI/archive/$netpyneuiBranch.zip -q
 RUN unzip $netpyneuiBranch.zip
-WORKDIR /home/jovyan/work/NetPyNE-UI-$netpyneuiBranch/utilities
+WORKDIR /opt/workspace-$netpyneuiBranch/utilities
 RUN /bin/bash -c "source activate snakes && python --version"
 RUN /bin/bash -c "source activate snakes && exec python install.py branch $netpyneuiBranch"
 WORKDIR /home/jovyan
 RUN git clone https://github.com/Neurosim-lab/netpyne_workspace
 WORKDIR /home/jovyan/workspace
-RUN ln -sfn /home/jovyan/work/NetPyNE-UI-$netpyneuiBranch/netpyne_ui/tests tests
+RUN ln -sfn /opt/workspace-$netpyneuiBranch/netpyne_ui/tests tests
 
 # Copy jupyterhub_config
-# RUN cp /home/jovyan/work/NetPyNE-UI-$netpyneuiBranch/jupyterhub_config.py .
+# RUN cp /opt/workspace-$netpyneuiBranch/jupyterhub_config.py .
 
 CMD /bin/bash -c "source activate snakes && exec jupyterhub-singleuser --debug --NotebookApp.default_url=/geppetto --NotebookApp.token='' --library=netpyne_ui"

k8s/cf_values.yaml

@@ -11,10 +11,10 @@ hub:
   shutdownOnLogout: true
   extraConfig:
     timing: |
-      c.Spawner.notebook_dir = '/home/jovyan/work/NetPyNE-UI'
+      c.Spawner.notebook_dir = '/opt/workspace'
       c.Spawner.default_url = '/geppetto'
     spawner: >-
-      c.Spawner.args = ["--library=netpyne_ui"]
+      c.Spawner.args = ["--library=netpyne_ui", "--NotebookApp.default_url=/geppetto"]
 singleuser:
   storage:
     type: none

k8s/minikube_values.yaml

@@ -20,7 +20,7 @@ hub:
       c.Spawner.port = 8000
       c.Spawner.http_timeout = 300
       c.Spawner.start_timeout = 300
-      c.Spawner.notebook_dir = '/home/jovyan/work/NetPyNE-UI'
+      c.Spawner.notebook_dir = '/opt/workspace'
       c.Spawner.default_url = '/geppetto'
       c.Spawner.debug = True
       c.Spawner.disable_check_xsrf=True

netpyne_ui/netpyne_geppetto.py

@@ -47,6 +47,40 @@
 os.chdir(constants.NETPYNE_WORKDIR_PATH)
 
 
+class NetpyneValidationError(Exception):
+    ...
+
+
+def deepcopy_wout_empty(d, memo=None):
+    def is_empty(x):
+        return x == '' or x == [] or x == () or x == set() or x == {} or x is None
+
+    # if is_empty(d):
+    #     return None
+    memo = {} if memo is None else memo
+    if id(d) in memo:
+        return memo[id(d)]
+    if isinstance(d, dict):
+        cpy = {}
+        for k, v in d.items():
+            v_cpy = deepcopy_wout_empty(v, memo=memo)
+            if is_empty(v_cpy):
+                continue
+            cpy[k] = memo.setdefault(id(v), v_cpy)
+        return cpy
+    elif isinstance(d, (list, set, tuple)):
+        return d.__class__(memo.setdefault(id(v), deepcopy_wout_empty(v, memo=memo)) for v in d if not is_empty(v))
+    elif hasattr(d, '__dict__'):
+        cpy = d.__new__(d.__class__)  # We skip the initializer, in case it needs some arguments
+        for k, v in d.__dict__.items():
+            v_cpy = deepcopy_wout_empty(v, memo=memo)
+            # if is_empty(v_cpy):  #
+            #     continue
+            cpy.__dict__[k] = memo.setdefault(id(v), v_cpy)
+        return cpy
+    return d
+
+
 class NetPyNEGeppetto:
 
     def __init__(self):
@@ -185,6 +219,11 @@ def instantiateNetPyNEModelInGeppetto(self, args):
                     self.geppetto_model = self.model_interpreter.getGeppettoModel(netpyne_model)
 
                 return json.loads(GeppettoModelSerializer.serialize(self.geppetto_model))
+        except NetpyneValidationError as e:
+            message = ("Error while validating the NetPyNE model before instantiation.\n"
+                       "One or more components in your model have issues, see details below:")
+            logging.exception(message)
+            return utils.getJSONError(message, '\n'.join(e.args))
         except Exception as e:
             message = "Error while instantiating the NetPyNE model"
             logging.exception(message)
@@ -256,6 +295,26 @@ def simulate_single_model(self, experiment: model.Experiment = None, use_prev_in
                 response = json.loads(GeppettoModelSerializer.serialize(self.geppetto_model))
                 return response
 
+
+    def validate_netParams(self):
+        cpy = deepcopy_wout_empty(self.netParams)
+        _, failed = sim.validator.validateNetParams(cpy)
+        if failed:
+            message = ""
+            components_error = {}
+            for entry in failed:
+                components_error.setdefault(entry.component, []).append((entry.keyPath, entry.summary))
+            for component, details in components_error.items():
+                message = message + f"* Error validating {component}\n"
+                for (keyPath, summary) in details:
+                    path = ' -> '.join(f"{key}" for key in keyPath)
+                    message = message + f"    Error in {path}\n"
+                    for line in summary:
+                        message = message + f"      {line}\n"
+                message = message + "\n"
+            raise NetpyneValidationError(message)
+
+
     def simulateNetPyNEModelInGeppetto(self, args):
         """ Starts simulation of the currently loaded NetPyNe model.
 
@@ -270,8 +329,9 @@ def simulateNetPyNEModelInGeppetto(self, args):
         allTrials = args.get('allTrials', True)
         use_prev_inst = args.get('usePrevInst', False)
         sim_id = args.get('simId', 0)
-
         try:
+            self.validate_netParams()
+
             experiment = experiments.get_current()
             if experiment:
                 if self.experiments.any_in_state([model.ExperimentState.PENDING, model.ExperimentState.SIMULATING]):
@@ -300,6 +360,11 @@ def simulateNetPyNEModelInGeppetto(self, args):
 
             else:
                 return self.simulate_single_model(use_prev_inst=use_prev_inst)
+        except NetpyneValidationError as e:
+            message = (f"Error while validating the NetPyNE model before simulation {sim_id}.\n"
+                       "One or more components in your model have issues, see details below:")
+            logging.exception(message)
+            return utils.getJSONError(message, '\n'.join(e.args))
 
         except Exception as e :
             message = f"Error while simulating the NetPyNE model: {e}. SimulationId {sim_id}"
@@ -740,6 +805,7 @@ def deleteModel(self, modelParams):
         return utils.getJSONReply()
 
     def instantiateNetPyNEModel(self):
+        self.validate_netParams()
         with redirect_stdout(sys.__stdout__):
             saveData = sim.allSimData if hasattr(sim, 'allSimData') and 'spkt' in sim.allSimData.keys() and len(
                 sim.allSimData['spkt']) > 0 else False
@@ -774,7 +840,8 @@ def doIhaveInstOrSimData(self):
         return {'haveInstance': out[0], 'haveSimData': out[1]}
 
     def rename(self, path, oldValue, newValue):
-        command = 'sim.rename(self.' + path + ',"' + oldValue + '","' + newValue + '")'
+        # command = 'sim.rename(self.' + path + ',"' + oldValue + '","' + newValue + '")'
+        command = f'sim.rename(self.{path}, {oldValue!r}, {newValue!r})'
         logging.debug('renaming ' + command)
 
         eval(command)
@@ -912,11 +979,15 @@ def getAvailableCellModels(self):
                     cell_models.add(cm)
         return list(cell_models)
 
-    def getAvailableCellTypes(self):
-        cell_types = set([])
-        for p in self.netParams.cellParams:
-            cell_types.add(p)
-        return list(cell_types)
+    def getAvailableCellModels(self):
+        return ["", "VecStim", "NetStim", "IntFire1"]
+
+    def getAvailableStimulationDistribution(self):
+        return ["normal", "uniform"]
+
+    def getAvailableStimulationPattern(self):
+        # self.netParams.popParams[name]['spikePattern'] = {}
+        return ["", "rhythmic", "evoked", "poisson", "gauss"]
 
     def getAvailableSections(self):
         sections = {}

tests/frontend/e2e/jest-puppeteer.config.js

@@ -1,6 +1,6 @@
 module.exports = {
   launch: {
-    headless: true,
+    headless: 'new',
     defaultViewport: {
       width: 1300,
       height: 1024