Skip to content

Commit 8482174

Browse files
filippomcfilippomc
authored
Merge pull request #128 from MetaCell/feature/116
Feature/116
2 parents e3ef6b7 + aecbcbc commit 8482174

11 files changed

Lines changed: 292 additions & 39 deletions

File tree

‎.dockerignore‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
org.geppetto.frontend.jupyter
2+
pygeppetto
3+
.vscode
4+
.ipynb_checkpoints
5+
netpyne_ui/geppetto
6+
*.egg-info
7+
notebook.ipynb
8+
.git
9+
netpyne
10+
Dockerfile
11+
Dockerfile_base
12+
Dockerfile_dev
13+
netpyne_workspace

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,5 @@ pygeppetto/
88
netpyne/
99
org.geppetto.frontend.jupyter/
1010
*.ipynb
11-
init.py
11+
init.py
12+
.vscode

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,4 +23,4 @@ RUN unzip $netpyneuiBranch.zip
2323
WORKDIR /home/jovyan/netpyne_workspace
2424
RUN ln -sfn /home/jovyan/work/NetPyNE-UI-$netpyneuiBranch/netpyne_ui/tests tests
2525

26-
CMD /bin/bash -c "source activate snakes && exec jupyter notebook --debug --NotebookApp.default_url=/geppetto --NotebookApp.token='' --library=netpyne_ui"
26+
CMD /bin/bash -c "source activate snakes && exec jupyter notebook --debug --NotebookApp.default_url=/geppetto --NotebookApp.token='' --library=netpyne_ui --NotebookApp.disable_check_xsrf=True"

‎Dockerfile_dev‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,23 @@
1-
FROM metacell/jupyter-neuron:latest
1+
# FROM metacell/jupyter-neuron:latest
2+
FROM frodriguez4600/jupyter-neuron:latest
23
USER $NB_USER
34

45
ARG netpyneuiBranch=development
56
ENV netpyneuiBranch=${netpyneuiBranch}
67
RUN echo "$netpyneuiBranch";
78

8-
# Clone NetPyNE-UI and install the development version
9-
RUN wget https://github.com/MetaCell/NetPyNE-UI/archive/$netpyneuiBranch.zip -q
10-
RUN unzip $netpyneuiBranch.zip
11-
WORKDIR /home/jovyan/work/NetPyNE-UI-$netpyneuiBranch/utilities
12-
RUN /bin/bash -c "source activate snakes && python --version"
13-
RUN /bin/bash -c "source activate snakes && exec python install.py branch $netpyneuiBranch"
14-
WORKDIR /home/jovyan
15-
RUN git clone https://github.com/Neurosim-lab/netpyne_workspace
16-
WORKDIR /home/jovyan/netpyne_workspace
17-
RUN ln -sfn /home/jovyan/work/NetPyNE-UI-$netpyneuiBranch/netpyne_ui/tests tests
18-
CMD /bin/bash -c "source activate snakes && exec jupyter notebook --debug --NotebookApp.default_url=/geppetto --NotebookApp.token='' --library=netpyne_ui"
9+
ENV FOLDER=NetPyNE-UI
10+
WORKDIR /home/jovyan/work
11+
12+
COPY --chown=1000:1000 . NetPyNE-UI
13+
WORKDIR ${FOLDER}/utilities
14+
15+
RUN python install.py branch $netpyneuiBranch
16+
17+
WORKDIR /home/jovyan/work/NetPyNE-UI
18+
19+
RUN git clone https://github.com/Neurosim-lab/netpyne_workspace && rm -rf netpyne_workspace/.git
20+
WORKDIR /home/jovyan/work/NetPyNE-UI/netpyne_workspace
21+
22+
RUN ln -sfn /home/jovyan/work/NetPyNE-UI/netpyne_ui/tests ../tests
23+
CMD /bin/bash -c "cd .. && NetPyNE-UI"

‎NetPyNE-UI‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
#!/bin/sh
2-
exec jupyter notebook --NotebookApp.default_url=/geppetto --NotebookApp.token='' --library=netpyne_ui
2+
exec jupyter notebook --NotebookApp.default_url=/geppetto --NotebookApp.token='' --library=netpyne_ui --NotebookApp.disable_check_xsrf=True

‎netpyne_ui/__init__.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
import logging
2+
from jupyter_geppetto.webapi import RouteManager
3+
4+
from netpyne_ui import api
5+
6+
RouteManager.add_controller(api.NetPyNEController)
7+
logging.info("Adding NWBModelInterpreter")

‎netpyne_ui/api.py‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
import os
2+
import logging
3+
import uuid
4+
import gzip
5+
import tarfile
6+
import shutil
7+
from zipfile import ZipFile
8+
from tempfile import TemporaryDirectory
9+
from jupyter_geppetto.webapi import get, post
10+
from notebook.base.handlers import IPythonHandler
11+
from netpyne_ui.constants import NETPYNE_WORKDIR, UPLOAD_FOLDER_NAME, ALLOWED_EXTENSIONS, UPLOAD_FOLDER_PATH
12+
13+
def allowed_file(filename, allowed_extensions=ALLOWED_EXTENSIONS):
14+
return '.' in filename and \
15+
filename.rsplit('.', 1)[1].lower() in allowed_extensions
16+
17+
18+
def send_files(handler, file_path, filename):
19+
with open(file_path, "rb") as f:
20+
handler.set_header('Content-Type', 'application/force-download')
21+
handler.set_header('Content-Disposition', f"attachment; filename={filename}")
22+
23+
try:
24+
while True:
25+
_buffer = f.read(4096)
26+
if _buffer:
27+
handler.write(_buffer)
28+
else:
29+
return
30+
except:
31+
handler.set_status(500, f"Error sending files")
32+
33+
34+
def get_file_paths(handler):
35+
file_paths = False
36+
if 'uri' in handler.request.arguments:
37+
file_paths = []
38+
tmp_file_paths = [path.decode('utf-8') for path in handler.request.arguments['uri']]
39+
for path in tmp_file_paths:
40+
if os.path.exists(path):
41+
file_paths.append(path)
42+
43+
return file_paths
44+
45+
class NetPyNEController: # pytest: no cover
46+
47+
@post('/uploads')
48+
def uploads(handler: IPythonHandler):
49+
files = handler.request.files
50+
files_saved = 0
51+
52+
if len(files) == 0 or 'file' not in files:
53+
handler.set_status(400, f"Can't find 'file' or filename is empty. Files received {len(files)}")
54+
else:
55+
56+
for f in files['file']:
57+
if not allowed_file(f.filename):
58+
logging.warn(f"Can't store file {f.filename}. Extension not allowed")
59+
continue
60+
61+
## Save to file
62+
filename = f.filename
63+
file_path = os.path.join(UPLOAD_FOLDER_PATH, filename)
64+
65+
with open(file_path, 'wb') as zf:
66+
zf.write(f['body'])
67+
68+
files_saved += 1
69+
70+
if filename.endswith('.zip'):
71+
with ZipFile(file_path) as zipObj:
72+
zipObj.extractall(UPLOAD_FOLDER_PATH)
73+
74+
elif filename.endswith('.tar.gz'):
75+
with tarfile.open(file_path, mode='r:gz') as tar:
76+
tar.extractall(UPLOAD_FOLDER_PATH)
77+
78+
elif filename.endswith('.gz'):
79+
with gzip.open(file_path, "rb") as gz, open(file_path.replace('.gz', ''), 'wb') as ff:
80+
shutil.copyfileobj(gz, ff)
81+
82+
handler.set_status(200, f"Number of files saved: {files_saved}. Number of files sent: {len(files['file'])}")
83+
84+
handler.finish()
85+
86+
@get('/downloads')
87+
def downloads(handler: IPythonHandler):
88+
89+
file_paths = get_file_paths(handler)
90+
91+
if file_paths:
92+
93+
if len(file_paths) == 0:
94+
handler.set_status(400, f"Files not found.")
95+
handler.finish()
96+
return
97+
98+
if len(file_paths) == 1:
99+
send_files(handler, file_paths[0], file_paths[0].split('/')[-1])
100+
101+
else :
102+
with TemporaryDirectory() as dir_path:
103+
tar_gz_file_name = f'{str(uuid.uuid4())}.tar.gz'
104+
tar_gz_file_path = os.path.join(dir_path, tar_gz_file_name)
105+
with tarfile.open(tar_gz_file_path, mode='w:gz') as tar:
106+
for file_path in file_paths:
107+
tar.add(file_path, os.path.join('download', file_path.split('/')[-1]))
108+
109+
send_files(handler, tar_gz_file_path, tar_gz_file_name)
110+
111+
handler.finish()

‎netpyne_ui/constants.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import os
2+
3+
UPLOAD_FOLDER_NAME = 'uploads'
4+
NETPYNE_WORKDIR = 'netpyne_workspace'
5+
6+
ALLOWED_EXTENSIONS = ["py", "zip", "gz", ".tar.gz", "pdf", "txt", "xls", "png", "jpeg"]
7+
8+
UPLOAD_FOLDER_PATH = os.path.join(os.getcwd(), NETPYNE_WORKDIR, UPLOAD_FOLDER_NAME)
9+
NETPYNE_WORKDIR_PATH = os.path.join(os.getcwd(), NETPYNE_WORKDIR)
10+
11+
if not os.path.exists(UPLOAD_FOLDER_PATH):
12+
os.makedirs(UPLOAD_FOLDER_PATH)
13+
14+
if not os.path.exists(NETPYNE_WORKDIR_PATH):
15+
os.makedirs(NETPYNE_WORKDIR_PATH)

‎netpyne_ui/netpyne_geppetto.py‎

Lines changed: 75 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,9 @@
3030
from jupyter_geppetto import jupyter_geppetto, synchronization, utils
3131
import imp
3232
from contextlib import redirect_stdout, redirect_stderr
33+
from netpyne_ui.constants import NETPYNE_WORKDIR_PATH
3334

34-
35+
os.chdir(NETPYNE_WORKDIR_PATH)
3536
class NetPyNEGeppetto():
3637

3738
def __init__(self):
@@ -59,7 +60,7 @@ def instantiateNetPyNEModelInGeppetto(self, args):
5960
netpyne_model = self.instantiateNetPyNEModel()
6061
self.geppetto_model = self.model_interpreter.getGeppettoModel(netpyne_model)
6162

62-
return json.loads(GeppettoModelSerializer().serialize(self.geppetto_model))
63+
return json.loads(GeppettoModelSerializer.serialize(self.geppetto_model))
6364
except:
6465
return utils.getJSONError("Error while instantiating the NetPyNE model", sys.exc_info())
6566

@@ -185,7 +186,7 @@ def remove(dictionary):
185186
sim.loadSimData(args['jsonModelFolder'])
186187
self.geppetto_model = self.model_interpreter.getGeppettoModel(sim)
187188

188-
return json.loads(GeppettoModelSerializer().serialize(self.geppetto_model))
189+
return json.loads(GeppettoModelSerializer.serialize(self.geppetto_model))
189190
else:
190191
return utils.getJSONReply()
191192
except:
@@ -263,6 +264,11 @@ def exportModel(self, args):
263264
sim.cfg.saveJson = True
264265
sim.saveData(include)
265266
sim.cfg.saveJson = False
267+
268+
with open(f"{sim.cfg.filename}.json") as json_file:
269+
data = json.load(json_file)
270+
return data
271+
266272
return utils.getJSONReply()
267273
except:
268274
return utils.getJSONError("Error while exporting the NetPyNE model", sys.exc_info())
@@ -281,24 +287,31 @@ def importNeuroML(self, modelParams):
281287
sim.initialize()
282288
sim.importNeuroML2(modelParams['neuroMLFolder'], simConfig=specs.SimConfig(), simulate=False, analyze=False)
283289
self.geppetto_model = self.model_interpreter.getGeppettoModel(sim)
284-
return json.loads(GeppettoModelSerializer().serialize(self.geppetto_model))
290+
return json.loads(GeppettoModelSerializer.serialize(self.geppetto_model))
285291

286292
except:
287293
return utils.getJSONError("Error while exporting the NetPyNE model", sys.exc_info())
288294

289295
def deleteModel(self, modelParams):
296+
290297
try:
291298
with redirect_stdout(sys.__stdout__):
292299
self.netParams = specs.NetParams()
293300
self.simConfig = specs.SimConfig()
294-
self.netParams.todict()
295-
self.netParams.todict()
296-
if self.doIhaveInstOrSimData()['haveInstance']: sim.clearAll()
301+
sim.initialize(specs.NetParams(), specs.SimConfig())
297302
self.geppetto_model = None
298-
return utils.getJSONReply()
299-
300303
except:
301304
return utils.getJSONError("Error while exporting the NetPyNE model", sys.exc_info())
305+
306+
try:
307+
# This function fails is some keys don't exists
308+
# sim.clearAll()
309+
self.clearSim()
310+
311+
except:
312+
pass
313+
314+
return utils.getJSONReply()
302315

303316
def instantiateNetPyNEModel(self):
304317
with redirect_stdout(sys.__stdout__):
@@ -360,16 +373,17 @@ def getPlotSettings(self, plot):
360373
def getDirList(self, dir=None, onlyDirs = False, filterFiles=False):
361374
# Get Current dir
362375
if dir == None or dir == '':
363-
dir = os.getcwd()
376+
dir = os.path.join(os.getcwd(), NETPYNE_WORKDIR_PATH)
364377
dir_list = []
378+
file_list = []
365379
for f in sorted(os.listdir(str(dir)), key=str.lower):
366380
ff=os.path.join(dir,f)
367381
if os.path.isdir(ff):
368-
dir_list.insert(0, {'title': f, 'path': ff, 'load': False, 'children': [{'title': 'Loading...'}]})
382+
dir_list.append({'title': f, 'path': ff, 'load': False, 'children': [{'title': 'Loading...'}]})
369383
elif not onlyDirs:
370384
if not filterFiles or os.path.isfile(ff) and ff.endswith(filterFiles):
371-
dir_list.append({'title': f, 'path': ff})
372-
return dir_list
385+
file_list.append({'title': f, 'path': ff})
386+
return dir_list + file_list
373387

374388
def getPlot(self, plotName, LFPflavour):
375389
try:
@@ -541,7 +555,8 @@ def header(title, spacer='-'):
541555

542556
script.write(header('end script', spacer='='))
543557

544-
return utils.getJSONReply()
558+
with open(fname) as f:
559+
return f.read()
545560

546561
except:
547562
return utils.getJSONError("Error while importing the NetPyNE model", sys.exc_info())
@@ -648,6 +663,52 @@ def propagate_syn_mech_rename(self, new, old):
648663
self.netParams.stimTargetParams[label]['synMech'] = new
649664

650665

666+
def clearSim(self):
667+
# clean up
668+
sim.pc.barrier()
669+
sim.pc.gid_clear() # clear previous gid settings
670+
671+
# clean cells and simData in all nodes
672+
sim.clearObj([cell.__dict__ if hasattr(cell, '__dict__') else cell for cell in sim.net.cells])
673+
if 'stims' in list(sim.simData.keys()):
674+
sim.clearObj([stim for stim in sim.simData['stims']])
675+
676+
for key in list(sim.simData.keys()): del sim.simData[key]
677+
678+
if hasattr(sim, 'net'):
679+
for c in sim.net.cells: del c
680+
for p in sim.net.pops: del p
681+
if hasattr(sim.net, 'params'):
682+
del sim.net.params
683+
684+
685+
# clean cells and simData gathered in master node
686+
if sim.rank == 0:
687+
if hasattr(sim.net, 'allCells'):
688+
sim.clearObj([cell.__dict__ if hasattr(cell, '__dict__') else cell for cell in sim.net.allCells])
689+
if hasattr(sim, 'allSimData'):
690+
if 'stims' in list(sim.allSimData.keys()):
691+
sim.clearObj([stim for stim in sim.allSimData['stims']])
692+
for key in list(sim.allSimData.keys()): del sim.allSimData[key]
693+
del sim.allSimData
694+
695+
696+
import matplotlib
697+
matplotlib.pyplot.clf()
698+
matplotlib.pyplot.close('all')
699+
700+
if hasattr(sim, 'net'):
701+
if hasattr(sim.net, 'allCells'):
702+
for c in sim.net.allCells: del c
703+
del sim.net.allCells
704+
if hasattr(sim.net, 'allPops'):
705+
for p in sim.net.allPops: del p
706+
707+
del sim.net
708+
709+
import gc; gc.collect()
710+
711+
651712
logging.info("Initialising NetPyNE UI")
652713
netpyne_geppetto = NetPyNEGeppetto()
653714
logging.info("NetPyNE UI initialised")

‎run.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
sys.argv.append('--NotebookApp.default_url=/geppetto')
1414
sys.argv.append("--NotebookApp.token=''")
1515
sys.argv.append('--library=netpyne_ui')
16+
sys.argv.append('--NotebookApp.disable_check_xsrf=True')
1617

1718
app = NotebookApp.instance()
1819
app.initialize(sys.argv)

0 commit comments

Comments
 (0)