Merge pull request #738 from MetaCell/release/1.0.0

Release/1.0.0

Author: enicolasgomez

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:13.14 as jsbuild
+FROM node:14.21.3-bullseye as jsbuild
 
 WORKDIR /app
 
@@ -18,10 +18,10 @@ RUN rm -Rf node_modules/*
 FROM jupyter/base-notebook:hub-1.5.0
 ENV NB_UID=jovyan
 ENV FOLDER=netpyne
-ARG GEPPETTO_VERSION=development
 ARG BUILD_ARGS=""
-ARG NETPYNE_VERSION=master
 ARG WORKSPACE_VERSION=master
+# ARG GEPPETTO_VERSION=development
+# ARG NETPYNE_VERSION=master
 
 ENV FOLDER=/home/jovyan/work/NetPyNE-UI
 
@@ -32,10 +32,9 @@ RUN apt-get update -qq &&\
     apt-get install python3-tk vim nano unzip git make libtool g++ -qq pkg-config libfreetype6-dev libpng-dev libopenmpi-dev  openjdk-11-jre-headless -y -y
 RUN conda install python=3.7 -y
 
-
 WORKDIR $FOLDER
 COPY --chown=1000:1000 requirements.txt requirements.txt
-RUN pip install -r requirements.txt --no-cache-dir --prefer-binary
+RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip && pip install -r requirements.txt --prefer-binary
 
 COPY --chown=$NB_UID:1000 . .
 COPY --from=jsbuild --chown=$NB_UID:1000 /app webapp
@@ -46,7 +45,7 @@ RUN jupyter nbextension enable --py --sys-prefix jupyter_geppetto
 RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension
 RUN jupyter serverextension enable --py --sys-prefix jupyter_geppetto
 
-RUN python utilities/install.py ${BUILD_ARGS} --geppetto ${GEPPETTO_VERSION} --netpyne $NETPYNE_VERSION --workspace WORKSPACE_VERSION --npm-skip
+RUN python utilities/install.py ${BUILD_ARGS} --workspace $WORKSPACE_VERSION
 
 RUN jupyter labextension disable @jupyterlab/hub-extension
 
@@ -58,7 +57,8 @@ ENV NEURON_HOME=/opt/conda
 # For lfpykit 0.4
 # RUN wget -P $(pip show LFPykit | grep "Location:" | awk '{print $2"/lfpykit"}') https://www.parralab.org/nyhead/sa_nyhead.mat
 # For lpfykit 0.5
-RUN wget --no-check-certificate -P ${FOLDER}/workspace https://www.parralab.org/nyhead/sa_nyhead.mat
+ENV NP_LFPYKIT_HEAD_FILE=/home/jovyan/nyhead.mat
+RUN wget --no-check-certificate -O $NP_LFPYKIT_HEAD_FILE https://www.parralab.org/nyhead/sa_nyhead.mat
 
 USER $NB_UID
 

netpyne_ui/__init__.py

@@ -11,21 +11,23 @@
     "https://d8bf7e40eec34cb9891f6dd8207b5e83@sentry.metacell.us/6"
 )
 
+HEAD_MODEL_FILE = os.getenv("NP_LFPYKIT_HEAD_FILE", "sa_nyhead.mat")
+
 def init_eeg():
     import sys
     from netpyne_ui.constants import HERE
     # FIXES library asking for input to download
     sys.stdin = open(os.path.join(HERE, "resources/stdin.txt"),'r') 
     from lfpykit.eegmegcalc import NYHeadModel
     try:
-        NYHeadModel() # Downloads the model if does not exist
+        NYHeadModel(HEAD_MODEL_FILE) # Downloads the model if does not exist
     except:
         logging.error("Error initializing the EEG head model", exc_info=True)
 
-from multiprocessing import Process
-thread = Process(target = init_eeg)
-thread.start()
-
+if not os.path.exists(HEAD_MODEL_FILE):
+    from multiprocessing import Process
+    thread = Process(target = init_eeg)
+    thread.start()
 
 RouteManager.add_controller(api.NetPyNEController)
 

netpyne_ui/mod_utils.py

@@ -1,3 +1,4 @@
+import logging
 import shutil
 import subprocess
 import os
@@ -26,19 +27,30 @@ def is_loaded_mechanisms():
         return True
 
 
-def loadModMechFiles(compileMod, modFolder):
+def loadModMechFiles(compileMod, modFolder, forceRecompile=False):
     # Create Symbolic link
-    if compileMod:
-        modPath = os.path.join(str(modFolder), "x86_64")
 
-        if os.path.exists(modPath):
-            shutil.rmtree(modPath)
-
-        os.chdir(modFolder)
-        subprocess.call(["nrnivmodl"])
-        os.chdir('..')
-
-    try:
-        neuron.load_mechanisms(str(modFolder))
-    except:
-        raise
+        try:
+
+            owd = os.getcwd()
+            if compileMod:
+                
+                compiledModPath = os.path.join(str(modFolder), "x86_64")
+
+                if os.path.exists(compiledModPath) and forceRecompile:
+                        logging.info("Forcing mod files to recompile in %s" % modFolder)
+                        shutil.rmtree(compiledModPath)
+
+                if not os.path.exists(compiledModPath):
+                    logging.info("Compiling mod files in %s" % modFolder)
+                    os.chdir(modFolder)
+                    subprocess.call(["nrnivmodl"])
+                    os.chdir('..')
+
+            try:
+                neuron.load_mechanisms(str(modFolder))
+            except:
+                logging.exception("Error loading mechanisms")
+                raise
+        finally:
+            os.chdir(owd)

netpyne_ui/netpyne_geppetto.py

@@ -13,6 +13,7 @@
 import json
 import logging
 import os
+from pathlib import Path
 import pprint
 import re
 import sys
@@ -45,7 +46,6 @@
 
 os.chdir(constants.NETPYNE_WORKDIR_PATH)
 
-neuron.nrn_dll_loaded.append(os.path.join(NETPYNE_WORKDIR_PATH, 'mod'))  # Avoids to load workspace modfiles twice
 
 class NetPyNEGeppetto:
 
@@ -79,17 +79,6 @@ def __init__(self):
         if not simulations.local.is_running():
             [experiments.set_to_error(e) for e in running_exps]
 
-        # sys.stdin = open(os.path.join(constants.HERE, "stdin.txt"),'r') # FIXES library asking for input to download -- eg lfpykit models
-
-        # from ipykernel import kernelbase
-
-        # def raw_input(self, prompt=''):
-        #     return "y"
-        # kernelbase.Kernel.raw_input = raw_input
-
-        # from lfpykit.eegmegcalc import NYHeadModel
-        # NYHeadModel()
-
     def getData(self):
         return {
             "metadata": metadata,
@@ -511,7 +500,32 @@ def _create3D_shapes(self, json_path: str):
                 # Load again because gatherData removed simData
                 sim.loadSimData(json_path)
 
+    def loadFromIndexFile(self, json_path: str):
+        cfg, netParams = sim.loadModel(json_path, loadMechs=True, ignoreMechAlreadyExistsError=True)
+        self.simConfig = cfg
+        self.netParams = netParams
+
+        if isinstance(self.netParams, dict):
+          self.netParams = specs.NetParams(self.netParams)
+
+        if isinstance(self.simConfig, dict):
+          self.simConfig = specs.SimConfig(self.simConfig)
 
+        for key, value in self.netParams.cellParams.items():
+            if hasattr(value, 'todict'):
+                self.netParams.cellParams[key] = value.todict()
+
+        # TODO: when should sim.initialize be called?
+        #   Only on import or better before every simulation or network instantiation?
+        sim.initialize()
+
+    def saveToIndexFile(self, srcPath, dstPath, exportNetParamsAsPython, exportSimConfigAsPython):
+        sim.saveModel(netParams=self.netParams,
+                      simConfig=self.simConfig,
+                      srcPath=srcPath,
+                      dstPath=dstPath,
+                      exportNetParamsAsPython=exportNetParamsAsPython,
+                      exportSimConfigAsPython=exportSimConfigAsPython)
 
     def importModel(self, modelParameters):
         """ Imports a model stored in form of Python files.
@@ -524,7 +538,7 @@ def importModel(self, modelParameters):
             #   Shouldn't be specific to Import
             sim.clearAll()
         try:
-            loadModMechFiles(modelParameters['compileMod'], modelParameters['modFolder'])
+            loadModMechFiles(modelParameters['compileMod'], modelParameters['modFolder'], modelParameters.get("forceRecompile", True))
         except Exception:
             message = "Error while importing/compiling mods"
             logging.exception(message)
@@ -566,7 +580,7 @@ def importModel(self, modelParameters):
                 #   Only on import or better before every simulation or network instantiation?
                 sim.initialize()
             return utils.getJSONReply()
-        except Exception:
+        except:
             message = "Error while importing the NetPyNE model"
             logging.exception(message)
             return utils.getJSONError(message, sys.exc_info())
@@ -575,7 +589,7 @@ def importModel(self, modelParameters):
 
     def importNeuroML(self, modelParameters):
         from netpyne_ui.helpers import neuroml
-       
+
 
         try:
             # Get Current dir
@@ -584,9 +598,9 @@ def importNeuroML(self, modelParameters):
             with redirect_stdout(sys.__stdout__):
                 # NetParams
                 filename = str(modelParameters["fileName"])
-               
+
                 json_fname = neuroml.convertNeuroML2(filename, compileMod=modelParameters["compileMod"])
-                   
+
             return self.loadModel(args=dict(
                 compileMod=True,
                 modFolder=os.path.dirname(json_fname),
@@ -605,7 +619,7 @@ def importNeuroML(self, modelParameters):
 
     def importLEMS(self, modelParameters):
         from netpyne_ui.helpers import neuroml
-       
+
 
         try:
             # Get Current dir
@@ -614,7 +628,7 @@ def importLEMS(self, modelParameters):
             with redirect_stdout(sys.__stdout__):
                 # NetParams
                 filename = str(modelParameters["fileName"])
-               
+
                 json_fname =  neuroml.convertLEMSSimulation(filename)
 
             return self.loadModel(args=dict(
@@ -789,10 +803,21 @@ def getPlotSettings(self, plot_name):
             return self.simConfig.analysis[plot_name]
         return {}
 
-    def getDirList(self, dir=None, onlyDirs=False, filterFiles=False):
-        # Get Current dir
+    def checkFileExists(self, path):
+        path = Path(path or '')
+        return path.exists()
+
+    def getFullPath(self, dir, subDir):
         if dir is None or dir == '':
-            dir = os.path.join(os.getcwd(), constants.NETPYNE_WORKDIR_PATH)
+            base = constants.NETPYNE_WORKDIR_PATH
+            if subDir:
+                base = os.path.join(base, subDir)
+            dir = os.path.join(os.getcwd(), base)
+        return dir
+
+    def getDirList(self, dir=None, onlyDirs=False, filterFiles=False, subDir=None):
+        # Get Current dir
+        dir = self.getFullPath(dir, subDir)
         dir_list = []
         file_list = []
         for f in sorted(os.listdir(str(dir)), key=str.lower):
@@ -810,6 +835,13 @@ def checkAvailablePlots(self):
     def getPlot(self, plotName, LFPflavour, theme='gui'):
         try:
             with redirect_stdout(sys.__stdout__):
+                availablePlots = self.checkAvailablePlots()
+                checkCondition = availablePlots.get(plotName.replace('iplot', 'plot'), False)
+
+                if checkCondition is False:
+                    logging.info("Plot " + plotName + " not available")
+                    return -1
+
                 args = self.getPlotSettings(plotName)
                 if LFPflavour:
                     args['plots'] = [LFPflavour]
@@ -899,10 +931,10 @@ def getAvailableCellTypes(self):
             cell_types.add(p)
         return sorted(cell_types)
 
-    def getAvailableRxDSections(self, selectedRegion):
+    def getAvailableRxDSections(self, selectedRegion = None):
         sections = set([])
         sections.add('all')
-        if selectedRegion in self.netParams.rxdParams.regions and self.netParams.rxdParams.regions[selectedRegion].get('cells'):
+        if selectedRegion and selectedRegion in self.netParams.rxdParams.regions and self.netParams.rxdParams.regions[selectedRegion].get('cells'):
             if 'all' in self.netParams.rxdParams.regions[selectedRegion]['cells']:
                 for cellRule in self.netParams.cellParams:
                     for cellSect in self.netParams.cellParams[cellRule]['secs']:

requirements.txt

@@ -34,7 +34,7 @@ jupyterthemes==0.20.0
 kiwisolver==1.2.0
 lesscpy==0.14.0
 libNeuroML==0.4.0
-lfpykit==0.5
+lfpykit==0.5.1
 lxml==4.5.1
 Mako==1.1.0
 MarkupSafe==1.1.1
@@ -44,7 +44,7 @@ mistune==0.8.4
 multimethod==1.3
 nbconvert==5.6.1
 nbformat==5.0.6
-netpyne==1.0.3.1
+netpyne==1.0.4.1
 NEURON==8.2.2
 numpy==1.18.5
 oauthlib==3.0.1

setup.py

@@ -18,7 +18,7 @@
 
 setuptools.setup(
     name="netpyne_ui",
-    version="0.9.1",
+    version="1.0.0",
     url="https://github.com/MetaCell/NetPyNE-UI",
     author="MetaCell",
     author_email="info@metacell.us",
@@ -41,8 +41,8 @@
     ],
     install_requires=[
         'jupyter-geppetto>=1.0.0',
-        'NEURON>=8.0.2',
-        'netpyne>=1.0.3.1',
+        'NEURON>=8.2.2',
+        'netpyne>=1.0.4.1',
         'neuromllite==0.5.1',
         'pyNeuroML>=0.7.1',
         'sentry_sdk>=1.5.2',

tests/frontend/e2e/package.json

@@ -4,7 +4,7 @@
   "description": "NetPyNe UI tests",
   "license": "unlicensed",
   "scripts": {
-    "test": "jest --verbose ",
+    "test": "jest --verbose",
     "EEG_Dipole_test": "jest --verbose EEG_and_Dipole_Tut#1 ",
     "Experiment_Manager_test": "jest --verbose ExperimentManager_Tut#1 ",
     "Tutorial_1_test":"jest --verbose Tut#1_smoke ",

tests/frontend/e2e/tests/ControlPanel.test.js

@@ -9,7 +9,7 @@ import * as selectors from './selectors'
 
 
 //PAGE INFO:
-const baseURL = process.env.url || 'https://stage.netpyne.metacell.us/'
+const baseURL = process.env.url || 'https://test.netpyne.metacell.us/'
 const PAGE_WAIT = 3000;
 const TIMEOUT = 60000;
 

tests/frontend/e2e/tests/EEG_and_Dipole_Tut#1.test.js

@@ -9,7 +9,7 @@ import * as selectors from './selectors'
 
 
 //PAGE INFO:
-const baseURL = process.env.url || 'https://stage.netpyne.metacell.us/'
+const baseURL = process.env.url || 'https://test.netpyne.metacell.us/'
 const PAGE_WAIT = 3000;
 const TIMEOUT = 60000;
 
@@ -60,7 +60,7 @@ beforeAll(async () => {
 
 describe('EEG and Dipole Plot Test using Tutorial#1', () => {
 
-    it('Open new page', async () => {
+    it.skip('Open new page', async () => {
 
         console.log('Opening a new NetPyNE page')
 
@@ -120,7 +120,7 @@ describe('EEG and Dipole Plot Test using Tutorial#1', () => {
         await page.waitForSelector(selectors.TRACES_TO_RECORD_SELECTOR)
         await page.waitForTimeout(PAGE_WAIT)
         await page.waitForSelector(selectors.DIPOLE_LFPYKIT_SELECTOR)
-        await expect(page).toClick(selectors.DIPOLE_LFPYKIT_SELECTOR)
+        // await expect(page).toClick(selectors.DIPOLE_LFPYKIT_SELECTOR)
         await page.waitForTimeout(PAGE_WAIT)
         await page.click(selectors.DIPOLE_LFPYKIT_SELECTOR)
         await page.waitForTimeout(PAGE_WAIT)

tests/frontend/e2e/tests/ExperimentManager_Tut#1.test.js

@@ -9,7 +9,7 @@ import * as selectors from './selectors'
 
 
 //PAGE INFO:
-const baseURL = process.env.url || 'https://stage.netpyne.metacell.us/'
+const baseURL = process.env.url || 'https://test.netpyne.metacell.us/'
 const PAGE_WAIT = 3000;
 const TIMEOUT = 60000;