Merge pull request #825 from MetaCell/release/1.1.0

sync master

Author: ddelpiano

.github/workflows/tests.yml

@@ -1,67 +1,101 @@
-FROM node:14.21.3-bullseye as jsbuild
+FROM node:18 as jsbuild
 
-WORKDIR /app
-
-COPY --chown=1000:1000 webapp/package.json .
-COPY --chown=1000:1000 webapp/yarn.lock .
+ENV FOLDER=netpyne
 
+WORKDIR $FOLDER/webapp
+COPY webapp/package.json .
+COPY webapp/yarn.lock .
 
 RUN yarn install  --network-timeout 1000000000
 
-COPY webapp/ .
+COPY webapp .
 RUN yarn build-dev
 
-
-RUN rm -Rf node_modules/*
+### Download on a separate stage to run in parallel with buildkit
+FROM jupyter/base-notebook:hub-1.5.0 as downloads
+USER root
+RUN wget --no-check-certificate -O /nyhead.mat https://www.parralab.org/nyhead/sa_nyhead.mat
 
 ###
 FROM jupyter/base-notebook:hub-1.5.0
 ENV NB_UID=jovyan
 ENV FOLDER=netpyne
-ARG BUILD_ARGS=""
-ARG WORKSPACE_VERSION=master
-# ARG GEPPETTO_VERSION=development
-# ARG NETPYNE_VERSION=master
-
-ENV FOLDER=/home/jovyan/work/NetPyNE-UI
+ENV NP_LFPYKIT_HEAD_FILE=/home/jovyan/nyhead.mat
 
 USER root
 
 RUN rm -rf /var/lib/apt/lists
 RUN apt-get update -qq &&\
-    apt-get install python3-tk vim nano unzip git make libtool g++ -qq pkg-config libfreetype6-dev libpng-dev libopenmpi-dev  openjdk-11-jre-headless -y -y
+    apt-get install python3-tk vim nano unzip git make libtool g++ -qq pkg-config libfreetype6-dev libpng-dev libopenmpi-dev -y
+RUN apt-get install openjdk-11-jre-headless -y
 RUN conda install python=3.7 -y
 
+
 WORKDIR $FOLDER
-COPY --chown=1000:1000 requirements.txt requirements.txt
-RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip && pip install -r requirements.txt --prefer-binary
+COPY  --chown=1000:1000 requirements.txt requirements.txt
+RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\
+    pip install -r requirements.txt --prefer-binary
+
+
+# ToDo: fixme, for now remove the jupyter hub config json file because it overrides the default
+# and thus removes the frame ancestor cors settings
+RUN rm -f ~/.jupyter/*.json
+RUN chown $NB_UID .
+RUN chown $NB_UID /opt
+RUN rm -Rf workspace
+# sym link workspace pvc to $FOLDER
+RUN mkdir -p /opt/workspace
+RUN mkdir -p /opt/user
+
+COPY netpyne_ui netpyne_ui
+COPY utilities utilities
+COPY setup.py .
+COPY tests tests
+COPY NetPyNE-UI .
+COPY README.rst .
+COPY requirements-test.txt .
+
+
+USER $NB_UID
 
-COPY --chown=jovyan:1000 . .
-COPY --from=jsbuild --chown=jovyan:1000 /app webapp
 
 
+
+ENV NEURON_HOME=/opt/conda
+
+
+USER root
+
 RUN jupyter nbextension install --py --symlink --sys-prefix jupyter_geppetto
 RUN jupyter nbextension enable --py --sys-prefix jupyter_geppetto
 RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension
 RUN jupyter serverextension enable --py --sys-prefix jupyter_geppetto
 
-RUN python utilities/install.py ${BUILD_ARGS} --workspace $WORKSPACE_VERSION
+ARG BUILD_ARGS=""
+ARG WORKSPACE_VERSION=master
 
-RUN jupyter labextension disable @jupyterlab/hub-extension
+RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\
+  python utilities/install.py ${BUILD_ARGS} --workspace $WORKSPACE_VERSION --npm-skip
 
-RUN chown  $NB_UID .
-RUN chown -R $NB_UID workspace
 
-# Temp fixes for eeg plots
-ENV NEURON_HOME=/opt/conda
-# For lfpykit 0.4
-# RUN wget -P $(pip show LFPykit | grep "Location:" | awk '{print $2"/lfpykit"}') https://www.parralab.org/nyhead/sa_nyhead.mat
-# For lpfykit 0.5
-ENV NP_LFPYKIT_HEAD_FILE=/home/jovyan/nyhead.mat
-RUN wget --no-check-certificate -O $NP_LFPYKIT_HEAD_FILE https://www.parralab.org/nyhead/sa_nyhead.mat
+RUN mv workspace /opt/workspace/tutorials
+RUN chown -R $NB_UID /opt/workspace
+RUN ln -s /opt/workspace workspace
+
+RUN jupyter labextension disable @jupyterlab/hub-extension
+
+COPY --from=downloads --chown=1000:1000 /nyhead.mat $NP_LFPYKIT_HEAD_FILE
+COPY --from=jsbuild --chown=1000:1000 $FOLDER/webapp/build webapp/build
 
+RUN chown -R $NB_UID /home/jovyan/.jupyter
+RUN touch app.log && chown $NB_UID app.log
 USER $NB_UID
 
+
 EXPOSE 8888
 
+ENTRYPOINT ["tini", "-g", "--"]
+
+
 
+CMD ./NetPyNE-UI

Dockerfile

@@ -30,26 +30,59 @@ your system and you have had troubles installing it you can opt for the Virtual
 
 ### Python Dependencies
 
-We recommend the use of a new python 3 virtual environment:
+We recommend the use of a new python 3.7 virtual environment.
+Currently, NetPyNE-UI only supports Python 3.7, but that can change in the future.
+
+For NetPyNE-UI, the preferred way of creating a virtual env is to pass by conda/miniconda or mamba/micromamba.
+The pyenv tool can be also used, but it requires to be compiled with some special options to have NEURON running properly the simulation using `nrniv`.
+The reason behind this is that NEURON is distributed as a Python wheel with a specific option which force NEURON to look for the CPython dynamic lib.
+However, pyenv by default installs the static version of the CPython lib, resulting in `nrniv -python` not being able to run.
+
+The way of creating the virtualenv using (mini)conda is the following
+
+```bash
+conda create -n netpyne python=3.7
+conda activate netpyne
+```
+
+Here is how to create the virtualenv using (micro)mamba
+
+```bash
+mamba create -n netpyne python=3.7 -c conda-forge
+mamba activate netpyne
+```
+
+You can also directly create a virtualenv using your `python3` executable, obviously if it's the 3.7 version.
 
 ```bash
 python3 -m venv npenv
 source npenv/bin/activate
 ```
 
-Or, with conda
+If you want to use anyway pyenv, here is how to properly create the virtualenv and activate it.
 
 ```bash
-conda create -n netpyne python=3.7
-conda activate netpyne
+env PYTHON_CONFIGURE_OPTS="--enable-shared"  pyenv install --verbose 3.7.17
+pyenv virtualenv 3.7.17 netpyne
+pyenv shell netpyne
 ```
 
 ### Run install script
 
+When you are in your virtualenv, here is how you can install the "basic" version of NetPyNE-UI:
+
 ```bash
 python utilities/install.py
 ```
 
+If you want to have a different version of NetPyNE or geppetto meta, you can pass the version you want to the installer:
+
+```bash
+python utilities/install.py --dev --netpyne development --geppetto development --no-test
+```
+
+This command will install the `development` version of netpyne and geppetto and disable the tests during the installation.
+
 ### Start application
 
 ```bash
@@ -62,6 +95,15 @@ For debugging you can use `run.py` instead
 python run.py
 ```
 
+To run the UI in dev mode, you need to run `python run.py` in one terminal, and use `yarn` from the `webapp` folder, using node v14 (use `nvm` if you need to have a different version of node than the one installed on your system):
+
+```bash
+cd webapp
+yarn start
+```
+
+You can then navigate to `http://127.0.0.1:8081/` to access the dev version of the UI.
+
 ## Run NetPyNE User Interface in Docker
 
 Ensure that you have Docker installed on your system.
@@ -75,7 +117,7 @@ docker build -t netpyne-ui .
 Run the image
 
 ```bash
-docker run -p 8888:8888 netpyne-ui 
+docker run -p 8888:8888 netpyne-ui
 ```
 
 ## End-to-end tests

README.md

@@ -16,15 +16,15 @@ RUN /bin/bash -c "INCUBATOR_VER=${INCUBATOR_VER} source activate snakes && pip i
 # Clone NetPyNE-UI and install the development version
 RUN wget https://github.com/MetaCell/NetPyNE-UI/archive/$netpyneuiBranch.zip -q
 RUN unzip $netpyneuiBranch.zip
-WORKDIR /home/jovyan/work/NetPyNE-UI-$netpyneuiBranch/utilities
+WORKDIR /opt/workspace-$netpyneuiBranch/utilities
 RUN /bin/bash -c "source activate snakes && python --version"
 RUN /bin/bash -c "source activate snakes && exec python install.py branch $netpyneuiBranch"
 WORKDIR /home/jovyan
 RUN git clone https://github.com/Neurosim-lab/netpyne_workspace
 WORKDIR /home/jovyan/workspace
-RUN ln -sfn /home/jovyan/work/NetPyNE-UI-$netpyneuiBranch/netpyne_ui/tests tests
+RUN ln -sfn /opt/workspace-$netpyneuiBranch/netpyne_ui/tests tests
 
 # Copy jupyterhub_config
-# RUN cp /home/jovyan/work/NetPyNE-UI-$netpyneuiBranch/jupyterhub_config.py .
+# RUN cp /opt/workspace-$netpyneuiBranch/jupyterhub_config.py .
 
 CMD /bin/bash -c "source activate snakes && exec jupyterhub-singleuser --debug --NotebookApp.default_url=/geppetto --NotebookApp.token='' --library=netpyne_ui"

jupyter_hub/Dockerfile_spawner

@@ -11,10 +11,10 @@ hub:
   shutdownOnLogout: true
   extraConfig:
     timing: |
-      c.Spawner.notebook_dir = '/home/jovyan/work/NetPyNE-UI'
+      c.Spawner.notebook_dir = '/opt/workspace'
       c.Spawner.default_url = '/geppetto'
     spawner: >-
-      c.Spawner.args = ["--library=netpyne_ui"]
+      c.Spawner.args = ["--library=netpyne_ui", "--NotebookApp.default_url=/geppetto"]
 singleuser:
   storage:
     type: none

k8s/cf_values.yaml

@@ -20,7 +20,7 @@ hub:
       c.Spawner.port = 8000
       c.Spawner.http_timeout = 300
       c.Spawner.start_timeout = 300
-      c.Spawner.notebook_dir = '/home/jovyan/work/NetPyNE-UI'
+      c.Spawner.notebook_dir = '/opt/workspace'
       c.Spawner.default_url = '/geppetto'
       c.Spawner.debug = True
       c.Spawner.disable_check_xsrf=True

k8s/minikube_values.yaml

@@ -2,7 +2,9 @@
 import sys
 import logging
 from netpyne.specs import simConfig
+from packaging import version
 
+import pyneuroml
 from pyneuroml import pynml
 from pyneuroml.lems import generate_lems_file_for_neuroml
 from pyneuroml.pynml import read_neuroml2_file
@@ -13,57 +15,67 @@
 def convertLEMSSimulation(lemsFileName, compileMod=True):
     """Converts a LEMS Simulation file
 
-    Converts a LEMS Simulation file (https://docs.neuroml.org/Userdocs/LEMSSimulation.html)
-    pointing to a NeuroML 2 file into the equivalent in NetPyNE
+    Converts a LEMS Simulation file
+    (https://docs.neuroml.org/Userdocs/LEMSSimulation.html) pointing to a
+    NeuroML 2 file into the equivalent in NetPyNE
+
     Returns:
         simConfig, netParams for the model in NetPyNE
     """
     current_path = os.getcwd()
     try:
-        
+
         fullLemsFileName = os.path.abspath(lemsFileName)
-        tmp_path = os.path.dirname(fullLemsFileName) 
+        tmp_path = os.path.dirname(fullLemsFileName)
         if tmp_path:
             os.chdir(tmp_path)
         logging.info(
             "Importing LEMSSimulation with NeuroML 2 network from: %s"
             % fullLemsFileName
         )
 
-        result = pynml.run_lems_with_jneuroml_netpyne(
-            lemsFileName, only_generate_json=True, exit_on_fail=False)
-        
-        if result == False:
-            raise Exception("Error loading lems file")
+        # feature to return output added in 1.0.9
+        if version.parse(pyneuroml.__version__) >= version.parse("1.0.9"):
+            result, output_msg = pynml.run_lems_with_jneuroml_netpyne(
+                lemsFileName, only_generate_json=True, exit_on_fail=False,
+                return_string=True, max_memory="1G")
+
+            if result is False:
+                raise Exception(f"Error loading lems file: {output_msg}")
+        else:
+            result = pynml.run_lems_with_jneuroml_netpyne(
+                lemsFileName, only_generate_json=True, exit_on_fail=False,
+                max_memory="1G")
+
+            if result is False:
+                raise Exception("Error loading lems file")
+
         lems = pynml.read_lems_file(lemsFileName)
 
         np_json_fname = fullLemsFileName.replace('.xml','_netpyne_data.json')
-    
+
         return np_json_fname
     finally:
         os.chdir(current_path)
 
 
-
-
 def convertNeuroML2(nml2FileName,  compileMod=True):
     """Loads a NeuroML 2 file into NetPyNE
     Loads a NeuroML 2 file into NetPyNE by creating a new LEMS Simulation
-    file (https://docs.neuroml.org/Userdocs/LEMSSimulation.html) and using jNeuroML
-    to convert it.
+    file (https://docs.neuroml.org/Userdocs/LEMSSimulation.html) and using
+    jNeuroML to convert it.
 
     Returns:
         simConfig, netParams for the model in NetPyNE
     """
     current_path = os.getcwd()
     try:
         fullNmlFileName = os.path.abspath(nml2FileName)
-        work_path = os.path.dirname(fullNmlFileName) 
+        work_path = os.path.dirname(fullNmlFileName)
         if not os.path.exists(work_path):
             os.makedirs(work_path)
         os.chdir(work_path)
         sys.path.append(work_path)
-        
 
         logging.info(
                 "Importing NeuroML 2 network from: %s"