Merge branch 'development' into feature/127

Author: Facu

.dockerignore

@@ -0,0 +1,13 @@
+org.geppetto.frontend.jupyter
+pygeppetto
+.vscode
+.ipynb_checkpoints
+netpyne_ui/geppetto
+*.egg-info
+notebook.ipynb
+.git
+netpyne
+Dockerfile
+Dockerfile_base
+Dockerfile_dev
+netpyne_workspace

.gitignore

@@ -11,4 +11,4 @@ org.geppetto.frontend.jupyter/
 init.py
 npm*
 .vscode
-app.log
+app.log

Dockerfile

@@ -23,4 +23,4 @@ RUN unzip $netpyneuiBranch.zip
 WORKDIR /home/jovyan/netpyne_workspace
 RUN ln -sfn /home/jovyan/work/NetPyNE-UI-$netpyneuiBranch/netpyne_ui/tests tests
 
-CMD /bin/bash -c "source activate snakes && exec jupyter notebook --debug --NotebookApp.default_url=/geppetto --NotebookApp.token='' --library=netpyne_ui"
+CMD /bin/bash -c "source activate snakes && exec jupyter notebook --debug --NotebookApp.default_url=/geppetto --NotebookApp.token='' --library=netpyne_ui --NotebookApp.disable_check_xsrf=True"

Dockerfile_dev

@@ -1,18 +1,23 @@
-FROM metacell/jupyter-neuron:latest
+# FROM metacell/jupyter-neuron:latest
+FROM frodriguez4600/jupyter-neuron:latest
 USER $NB_USER
 
 ARG netpyneuiBranch=development 
 ENV netpyneuiBranch=${netpyneuiBranch}  
 RUN echo "$netpyneuiBranch";
 
-# Clone NetPyNE-UI and install the development version
-RUN wget https://github.com/MetaCell/NetPyNE-UI/archive/$netpyneuiBranch.zip -q
-RUN unzip $netpyneuiBranch.zip
-WORKDIR /home/jovyan/work/NetPyNE-UI-$netpyneuiBranch/utilities
-RUN /bin/bash -c "source activate snakes && python --version"
-RUN /bin/bash -c "source activate snakes && exec python install.py branch $netpyneuiBranch"
-WORKDIR /home/jovyan
-RUN git clone https://github.com/Neurosim-lab/netpyne_workspace
-WORKDIR /home/jovyan/netpyne_workspace
-RUN ln -sfn /home/jovyan/work/NetPyNE-UI-$netpyneuiBranch/netpyne_ui/tests tests
-CMD /bin/bash -c "source activate snakes && exec jupyter notebook --debug --NotebookApp.default_url=/geppetto --NotebookApp.token='' --library=netpyne_ui"
+ENV FOLDER=NetPyNE-UI
+WORKDIR /home/jovyan/work
+
+COPY --chown=1000:1000 . NetPyNE-UI
+WORKDIR ${FOLDER}/utilities
+
+RUN python install.py branch $netpyneuiBranch
+
+WORKDIR /home/jovyan/work/NetPyNE-UI
+
+RUN git clone https://github.com/Neurosim-lab/netpyne_workspace && rm -rf netpyne_workspace/.git
+WORKDIR /home/jovyan/work/NetPyNE-UI/netpyne_workspace
+
+RUN ln -sfn /home/jovyan/work/NetPyNE-UI/netpyne_ui/tests ../tests
+CMD /bin/bash -c "cd .. && NetPyNE-UI"

NetPyNE-UI

@@ -1,2 +1,2 @@
 #!/bin/sh
-exec jupyter notebook --NotebookApp.default_url=/geppetto --NotebookApp.token='' --library=netpyne_ui
+exec jupyter notebook --NotebookApp.default_url=/geppetto --NotebookApp.token='' --library=netpyne_ui --NotebookApp.disable_check_xsrf=True

netpyne_ui/__init__.py

@@ -0,0 +1,7 @@
+import logging
+from jupyter_geppetto.webapi import RouteManager
+
+from netpyne_ui import api
+
+RouteManager.add_controller(api.NetPyNEController)
+logging.info("Adding NWBModelInterpreter")

netpyne_ui/api.py

@@ -0,0 +1,111 @@
+import os
+import logging
+import uuid
+import gzip
+import tarfile
+import shutil
+from zipfile import ZipFile
+from tempfile import TemporaryDirectory
+from jupyter_geppetto.webapi import get, post
+from notebook.base.handlers import IPythonHandler
+from netpyne_ui.constants import NETPYNE_WORKDIR, UPLOAD_FOLDER_NAME, ALLOWED_EXTENSIONS, UPLOAD_FOLDER_PATH
+
+def allowed_file(filename, allowed_extensions=ALLOWED_EXTENSIONS):
+    return '.' in filename and \
+           filename.rsplit('.', 1)[1].lower() in allowed_extensions
+
+
+def send_files(handler, file_path, filename):
+    with open(file_path, "rb") as f:
+        handler.set_header('Content-Type', 'application/force-download')
+        handler.set_header('Content-Disposition', f"attachment; filename={filename}")
+
+        try:
+            while True:
+                _buffer = f.read(4096)
+                if _buffer:
+                    handler.write(_buffer)
+                else:
+                    return 
+        except:
+            handler.set_status(500, f"Error sending files")
+
+
+def get_file_paths(handler):
+    file_paths = False
+    if 'uri' in handler.request.arguments:
+        file_paths = []
+        tmp_file_paths = [path.decode('utf-8') for path in handler.request.arguments['uri']]
+        for path in tmp_file_paths:
+            if os.path.exists(path):
+                file_paths.append(path)
+    
+    return file_paths
+
+class NetPyNEController:  # pytest: no cover
+
+    @post('/uploads')
+    def uploads(handler: IPythonHandler):
+        files = handler.request.files
+        files_saved = 0
+        
+        if len(files) == 0 or 'file' not in files:
+            handler.set_status(400, f"Can't find 'file' or filename is empty. Files received {len(files)}")
+        else:
+
+            for f in files['file']:
+                if not allowed_file(f.filename):
+                    logging.warn(f"Can't store file {f.filename}. Extension not allowed")
+                    continue
+
+                ## Save to file
+                filename = f.filename
+                file_path = os.path.join(UPLOAD_FOLDER_PATH, filename)
+                
+                with open(file_path, 'wb') as zf:
+                    zf.write(f['body'])
+                
+                files_saved += 1
+
+                if filename.endswith('.zip'):
+                    with ZipFile(file_path) as zipObj:
+                        zipObj.extractall(UPLOAD_FOLDER_PATH)
+
+                elif filename.endswith('.tar.gz'):
+                    with tarfile.open(file_path, mode='r:gz') as tar:
+                        tar.extractall(UPLOAD_FOLDER_PATH)
+
+                elif filename.endswith('.gz'):
+                    with gzip.open(file_path, "rb") as gz, open(file_path.replace('.gz', ''), 'wb') as ff:
+                        shutil.copyfileobj(gz, ff)
+            
+            handler.set_status(200, f"Number of files saved: {files_saved}. Number of files sent: {len(files['file'])}")
+
+        handler.finish()
+    
+    @get('/downloads')
+    def downloads(handler: IPythonHandler):
+
+        file_paths = get_file_paths(handler)
+        
+        if file_paths:
+            
+            if len(file_paths) == 0:
+                handler.set_status(400, f"Files not found.")
+                handler.finish()
+                return
+
+            if len(file_paths) == 1:
+                send_files(handler, file_paths[0], file_paths[0].split('/')[-1])
+                
+            else :
+                with TemporaryDirectory() as dir_path:
+                    tar_gz_file_name = f'{str(uuid.uuid4())}.tar.gz'
+                    tar_gz_file_path = os.path.join(dir_path, tar_gz_file_name)
+                    with tarfile.open(tar_gz_file_path, mode='w:gz') as tar:
+                        for file_path in file_paths:
+                            tar.add(file_path, os.path.join('download', file_path.split('/')[-1]))
+
+                    send_files(handler, tar_gz_file_path, tar_gz_file_name)
+        
+        handler.finish()

netpyne_ui/constants.py

@@ -0,0 +1,15 @@
+import os
+
+UPLOAD_FOLDER_NAME = 'uploads'
+NETPYNE_WORKDIR = 'netpyne_workspace'
+
+ALLOWED_EXTENSIONS = ["py", "zip", "gz", ".tar.gz", "pdf", "txt", "xls", "png", "jpeg"]
+
+UPLOAD_FOLDER_PATH = os.path.join(os.getcwd(), NETPYNE_WORKDIR, UPLOAD_FOLDER_NAME)
+NETPYNE_WORKDIR_PATH = os.path.join(os.getcwd(), NETPYNE_WORKDIR)
+
+if not os.path.exists(UPLOAD_FOLDER_PATH):
+    os.makedirs(UPLOAD_FOLDER_PATH)
+
+if not os.path.exists(NETPYNE_WORKDIR_PATH):
+    os.makedirs(NETPYNE_WORKDIR_PATH)

netpyne_ui/netpyne_geppetto.py

@@ -30,8 +30,9 @@
 from jupyter_geppetto import jupyter_geppetto, synchronization, utils
 import imp
 from contextlib import redirect_stdout, redirect_stderr
+from netpyne_ui.constants import NETPYNE_WORKDIR_PATH
 
-
+os.chdir(NETPYNE_WORKDIR_PATH)
 class NetPyNEGeppetto():
 
     def __init__(self):
@@ -263,6 +264,11 @@ def exportModel(self, args):
                 sim.cfg.saveJson = True
                 sim.saveData(include)
                 sim.cfg.saveJson = False
+
+                with open(f"{sim.cfg.filename}.json") as json_file:
+                    data = json.load(json_file)
+                    return data
+
             return utils.getJSONReply()
         except:
             return utils.getJSONError("Error while exporting the NetPyNE model", sys.exc_info())
@@ -287,18 +293,25 @@ def importNeuroML(self, modelParams):
             return utils.getJSONError("Error while exporting the NetPyNE model", sys.exc_info())
 
     def deleteModel(self, modelParams):
+        
         try:
             with redirect_stdout(sys.__stdout__):       
                 self.netParams = specs.NetParams()
                 self.simConfig = specs.SimConfig()
-                self.netParams.todict()
-                self.netParams.todict()
-                if self.doIhaveInstOrSimData()['haveInstance']: sim.clearAll()
+                sim.initialize(specs.NetParams(), specs.SimConfig())
                 self.geppetto_model = None
-            return utils.getJSONReply()
-
         except:
             return utils.getJSONError("Error while exporting the NetPyNE model", sys.exc_info())
+
+        try:
+            # This function fails is some keys don't exists
+            # sim.clearAll()
+            self.clearSim()
+            
+        except:
+            pass
+
+        return utils.getJSONReply()
 
     def instantiateNetPyNEModel(self):
         with redirect_stdout(sys.__stdout__):
@@ -360,16 +373,17 @@ def getPlotSettings(self, plot):
     def getDirList(self, dir=None, onlyDirs = False, filterFiles=False):
         # Get Current dir
         if dir == None or dir == '':
-            dir = os.getcwd()
+            dir = os.path.join(os.getcwd(), NETPYNE_WORKDIR_PATH)
         dir_list = []
+        file_list = []
         for f in sorted(os.listdir(str(dir)), key=str.lower):
             ff=os.path.join(dir,f)
             if os.path.isdir(ff):
-                dir_list.insert(0, {'title': f, 'path': ff, 'load': False, 'children': [{'title': 'Loading...'}]})
+                dir_list.append({'title': f, 'path': ff, 'load': False, 'children': [{'title': 'Loading...'}]})
             elif not onlyDirs:
                 if not filterFiles or os.path.isfile(ff) and ff.endswith(filterFiles):
-                    dir_list.append({'title': f, 'path': ff})
-        return dir_list
+                    file_list.append({'title': f, 'path': ff})
+        return dir_list + file_list
 
     def getPlot(self, plotName, LFPflavour):
         try:
@@ -545,7 +559,8 @@ def header(title, spacer='-'):
 
                 script.write(header('end script', spacer='='))
 
-            return utils.getJSONReply()
+            with open(fname) as f: 
+                return f.read()
 
         except:
             return utils.getJSONError("Error while importing the NetPyNE model", sys.exc_info())
@@ -652,6 +667,52 @@ def propagate_syn_mech_rename(self, new, old):
                                     self.netParams.stimTargetParams[label]['synMech'] = new
 
 
+    def clearSim(self):
+        # clean up
+        sim.pc.barrier()
+        sim.pc.gid_clear()                    # clear previous gid settings
+
+        # clean cells and simData in all nodes
+        sim.clearObj([cell.__dict__ if hasattr(cell, '__dict__') else cell for cell in sim.net.cells])
+        if 'stims' in list(sim.simData.keys()):
+            sim.clearObj([stim for stim in sim.simData['stims']])
+
+        for key in list(sim.simData.keys()): del sim.simData[key]
+
+        if hasattr(sim, 'net'):
+            for c in sim.net.cells: del c
+            for p in sim.net.pops: del p
+            if hasattr(sim.net, 'params'):
+                del sim.net.params
+
+
+        # clean cells and simData gathered in master node
+        if sim.rank == 0:
+            if hasattr(sim.net, 'allCells'):
+                sim.clearObj([cell.__dict__ if hasattr(cell, '__dict__') else cell for cell in sim.net.allCells])
+            if hasattr(sim, 'allSimData'):
+                if 'stims' in list(sim.allSimData.keys()):
+                    sim.clearObj([stim for stim in sim.allSimData['stims']])
+                for key in list(sim.allSimData.keys()): del sim.allSimData[key]
+                del sim.allSimData
+            
+            
+            import matplotlib
+            matplotlib.pyplot.clf()
+            matplotlib.pyplot.close('all')
+
+        if hasattr(sim, 'net'):
+            if hasattr(sim.net, 'allCells'):
+                for c in sim.net.allCells: del c
+                del sim.net.allCells
+            if hasattr(sim.net, 'allPops'):
+                for p in sim.net.allPops: del p
+
+            del sim.net
+
+        import gc; gc.collect()
+
+
 logging.info("Initialising NetPyNE UI")
 netpyne_geppetto = NetPyNEGeppetto()
 logging.info("NetPyNE UI initialised")

run.py

@@ -13,6 +13,7 @@
     sys.argv.append('--NotebookApp.default_url=/geppetto')
     sys.argv.append("--NotebookApp.token=''")
     sys.argv.append('--library=netpyne_ui')
+    sys.argv.append('--NotebookApp.disable_check_xsrf=True')
 
     app = NotebookApp.instance()
     app.initialize(sys.argv)