Merge pull request #618 from MetaCell/osb2-dev

Osb2 dev

Author: ddelpiano

.gitignore

@@ -27,4 +27,7 @@ utilities/x86_64
 x86_64
 .jupyter-config
 venv
+webapp/build
+webapp/.yalc
 node_modules
+sa_nyhead.mat

Dockerfile

@@ -20,14 +20,16 @@ ENV NB_UID=jovyan
 ENV FOLDER=netpyne
 ARG GEPPETTO_VERSION=development
 ARG BUILD_ARGS=""
+ARG NETPYNE_VERSION=master
+ARG WORKSPACE_VERSION=master
 
 ENV FOLDER=/home/jovyan/work/NetPyNE-UI
 
 USER root
 
 RUN rm -rf /var/lib/apt/lists
 RUN apt-get update -qq &&\
-    apt-get install python3-tk vim nano unzip git make libtool g++ -qq pkg-config libfreetype6-dev libpng-dev libopenmpi-dev -y
+    apt-get install python3-tk vim nano unzip git make libtool g++ -qq pkg-config libfreetype6-dev libpng-dev libopenmpi-dev  openjdk-11-jre-headless -y -y
 RUN conda install python=3.7 -y
 
 
@@ -44,25 +46,22 @@ RUN jupyter nbextension enable --py --sys-prefix jupyter_geppetto
 RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension
 RUN jupyter serverextension enable --py --sys-prefix jupyter_geppetto
 
-RUN python utilities/install.py ${BUILD_ARGS} --geppetto ${GEPPETTO_VERSION} --npm-skip
+RUN python utilities/install.py ${BUILD_ARGS} --geppetto ${GEPPETTO_VERSION} --netpyne $NETPYNE_VERSION --workspace WORKSPACE_VERSION --npm-skip
 
 RUN jupyter labextension disable @jupyterlab/hub-extension
 
 RUN chown  $NB_UID .
 RUN chown -R $NB_UID workspace
 
 # Temp fixes for eeg plots
+ENV NEURON_HOME=/opt/conda
 # For lfpykit 0.4
 # RUN wget -P $(pip show LFPykit | grep "Location:" | awk '{print $2"/lfpykit"}') https://www.parralab.org/nyhead/sa_nyhead.mat
 # For lpfykit 0.5
 RUN wget --no-check-certificate -P ${FOLDER}/workspace https://www.parralab.org/nyhead/sa_nyhead.mat
 
 USER $NB_UID
 
-
 EXPOSE 8888
 
 
-
-
-

NetPyNE-UI

@@ -1,4 +1,4 @@
 #!/bin/sh
 CONFIG=$(pwd)/.jupyter-config
 export JUPYTER_CONFIG_DIR=$CONFIG
-exec jupyter notebook --NotebookApp.default_url=/geppetto --NotebookApp.token='' --library=netpyne_ui --NotebookApp.disable_check_xsrf=True
+exec jupyter notebook --NotebookApp.default_url=/geppetto --NotebookApp.token='' --NotebookApp.notebook_dir=workspace --library=netpyne_ui --NotebookApp.disable_check_xsrf=True

netpyne_ui/api.py

@@ -19,7 +19,8 @@ def allowed_file(filename, allowed_extensions=ALLOWED_EXTENSIONS):
 def send_files(handler, file_path, filename):
     with open(file_path, "rb") as f:
         handler.set_header('Content-Type', 'application/force-download')
-        handler.set_header('Content-Disposition', f"attachment; filename={filename}")
+        handler.set_header('Content-Disposition',
+                           f"attachment; filename={filename}")
 
         try:
             while True:
@@ -36,7 +37,8 @@ def get_file_paths(handler):
     file_paths = False
     if 'uri' in handler.request.arguments:
         file_paths = []
-        tmp_file_paths = [path.decode('utf-8') for path in handler.request.arguments['uri']]
+        tmp_file_paths = [path.decode('utf-8')
+                          for path in handler.request.arguments['uri']]
         for path in tmp_file_paths:
             if os.path.exists(path):
                 file_paths.append(path)
@@ -69,12 +71,14 @@ def uploads(handler: IPythonHandler):
         files_saved = 0
 
         if len(files) == 0 or 'file' not in files:
-            handler.set_status(400, f"Can't find 'file' or filename is empty. Files received {len(files)}")
+            handler.set_status(
+                400, f"Can't find 'file' or filename is empty. Files received {len(files)}")
         else:
 
             for f in files['file']:
                 if not allowed_file(f.filename):
-                    logging.warn(f"Can't store file {f.filename}. Extension not allowed")
+                    logging.warn(
+                        f"Can't store file {f.filename}. Extension not allowed")
                     continue
 
                 # Save to file
@@ -120,8 +124,42 @@ def downloads(handler: IPythonHandler):
                     tar_gz_file_path = os.path.join(dir_path, tar_gz_file_name)
                     with tarfile.open(tar_gz_file_path, mode='w:gz') as tar:
                         for file_path in file_paths:
-                            tar.add(file_path, os.path.join('download', file_path.split('/')[-1]))
+                            tar.add(file_path, os.path.join(
+                                'download', file_path.split('/')[-1]))
 
                     send_files(handler, tar_gz_file_path, tar_gz_file_name)
 
         handler.finish()
+
+
+def create_notebook(filename):
+    import nbformat as nbf
+    from nbformat.v4.nbbase import new_notebook
+    from nbformat import sign
+    import codecs
+
+    directory = os.path.dirname(filename)
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+    nb0 = new_notebook(cells=[nbf.v4.new_markdown_cell("""# Welcome to the NetPyNE-ui!
+    
+    """),
+                              nbf.v4.new_code_cell(
+                                  'netpyne_geppetto.netParams'),
+                              nbf.v4.new_code_cell(
+                                  'netpyne_geppetto.simConfig')
+                              ], metadata={"kernelspec": {
+                                  "display_name": "Python 3",
+                                  "language": "python",
+                                  "name": "python3"
+                              }})
+
+    f = codecs.open(filename, encoding='utf-8', mode='w')
+
+    nbf.write(nb0, filename)
+    f.close()
+
+
+# TODO move to jupyter geppetto, using notebook dir path
+if os.path.exists('workspace') and not os.path.exists('workspace/notebook.ipynb'):
+    create_notebook('workspace/notebook.ipynb')

netpyne_ui/constants.py

@@ -11,7 +11,7 @@
 TEMPLATE_FILENAME_BATCH_RUN = "batch_run_single.py"
 TEMPLATE_FILENAME_BATCH = "batch.py"
 
-ALLOWED_EXTENSIONS = ["py", "zip", "gz", ".tar.gz", "pdf", "txt", "xls", "png", "jpeg", "hoc", "json"]
+ALLOWED_EXTENSIONS = ["py", "zip", "gz", ".tar.gz", "pdf", "txt", "xls", "png", "jpeg", "hoc", "json", "xml", "nml"]
 HERE = os.path.dirname(os.path.abspath(__file__))
 ROOT = os.path.dirname(HERE)
 UPLOAD_FOLDER_PATH = os.path.join(ROOT, NETPYNE_WORKDIR, UPLOAD_FOLDER_NAME)

netpyne_ui/helpers/neuroml.py

@@ -0,0 +1,117 @@
+import os
+import sys
+import logging
+from netpyne.specs import simConfig
+
+from pyneuroml import pynml
+from pyneuroml.lems import generate_lems_file_for_neuroml
+from pyneuroml.pynml import read_neuroml2_file
+
+from netpyne_ui.mod_utils import loadModMechFiles
+
+
+def convertLEMSSimulation(lemsFileName, compileMod=True):
+    """Converts a LEMS Simulation file
+
+    Converts a LEMS Simulation file (https://docs.neuroml.org/Userdocs/LEMSSimulation.html)
+    pointing to a NeuroML 2 file into the equivalent in NetPyNE
+    Returns:
+        simConfig, netParams for the model in NetPyNE
+    """
+    current_path = os.getcwd()
+    try:
+        
+        fullLemsFileName = os.path.abspath(lemsFileName)
+        tmp_path = os.path.dirname(fullLemsFileName) 
+        if tmp_path:
+            os.chdir(tmp_path)
+        logging.info(
+            "Importing LEMSSimulation with NeuroML 2 network from: %s"
+            % fullLemsFileName
+        )
+
+        result = pynml.run_lems_with_jneuroml_netpyne(
+            lemsFileName, only_generate_json=True, exit_on_fail=False)
+        
+        if result == False:
+            raise Exception("Error loading lems file")
+        lems = pynml.read_lems_file(lemsFileName)
+
+        np_json_fname = fullLemsFileName.replace('.xml','_netpyne_data.json')
+    
+        return np_json_fname
+    finally:
+        os.chdir(current_path)
+
+
+
+
+def convertNeuroML2(nml2FileName,  compileMod=True):
+    """Loads a NeuroML 2 file into NetPyNE
+    Loads a NeuroML 2 file into NetPyNE by creating a new LEMS Simulation
+    file (https://docs.neuroml.org/Userdocs/LEMSSimulation.html) and using jNeuroML
+    to convert it.
+
+    Returns:
+        simConfig, netParams for the model in NetPyNE
+    """
+    current_path = os.getcwd()
+    try:
+        fullNmlFileName = os.path.abspath(nml2FileName)
+        work_path = os.path.dirname(fullNmlFileName) 
+        if not os.path.exists(work_path):
+            os.makedirs(work_path)
+        os.chdir(work_path)
+        sys.path.append(work_path)
+        
+
+        logging.info(
+                "Importing NeuroML 2 network from: %s"
+                % fullNmlFileName
+            )
+        nml_model = read_neuroml2_file(fullNmlFileName)
+
+        target = nml_model.networks[0].id
+        sim_id = "Sim_%s" % target
+        duration = 1000
+        dt = 0.025
+        lems_file_name = os.path.join(os.path.dirname(fullNmlFileName), "LEMS_%s.xml" % sim_id)
+        lems_file_name = "LEMS_%s.xml" % sim_id
+        target_dir = os.path.dirname(fullNmlFileName)
+        target_dir = "."
+
+        generate_lems_file_for_neuroml(
+            sim_id,
+            fullNmlFileName,
+            target,
+            duration,
+            dt,
+            lems_file_name,
+            target_dir,
+            include_extra_files=["PyNN.xml"],
+            gen_plots_for_all_v=True,
+            plot_all_segments=False,
+            gen_plots_for_quantities={},  # Dict with displays vs lists of quantity paths
+            gen_plots_for_only_populations=[],  # List of populations, all pops if = []
+            gen_saves_for_all_v=True,
+            save_all_segments=False,
+            gen_saves_for_only_populations=[],  # List of populations, all pops if = []
+            gen_saves_for_quantities={},  # Dict with file names vs lists of quantity paths
+            gen_spike_saves_for_all_somas=True,
+            report_file_name="report.txt",
+            copy_neuroml=True,
+            verbose=True,
+        )
+        os.chdir(work_path)
+        res = convertLEMSSimulation(lems_file_name, compileMod=compileMod)
+    finally:
+        os.chdir(current_path)
+    return res
+
+
+if __name__ == "__main__":
+
+    if '-nml' in sys.argv:
+        convertNeuroML2("../../NeuroML2/Spikers.net.nml")
+    else:
+        convertLEMSSimulation("LEMS_HHSimple.xml")

netpyne_ui/mod_utils.py

@@ -26,7 +26,7 @@ def is_loaded_mechanisms():
         return True
 
 
-def compileModMechFiles(compileMod, modFolder):
+def loadModMechFiles(compileMod, modFolder):
     # Create Symbolic link
     if compileMod:
         modPath = os.path.join(str(modFolder), "x86_64")
@@ -38,7 +38,7 @@ def compileModMechFiles(compileMod, modFolder):
         subprocess.call(["nrnivmodl"])
         os.chdir('..')
 
-        try:
-            neuron.load_mechanisms(str(modFolder))
-        except:
-            raise
+    try:
+        neuron.load_mechanisms(str(modFolder))
+    except:
+        raise