Merge pull request #74 from MetaCell/py3

Migrate to latest jupyter libs

Author: adrianq

.travis.yml

@@ -67,6 +67,7 @@ script:
   - "curl -s -I $1 http://localhost:8888/geppetto | grep HTTP/1.1 | awk {'print $2'}"
   - firefox --version
   - travis_retry xvfb-run -a --server-args="-screen 0 1024x768x24"  casperjs test netpyne-tests.js --host=http://localhost:8888/ --engine=slimerjs
-  - sudo docker logs -t netpyne-ui_container
+  - python -c "import fcntl; fcntl.fcntl(1, fcntl.F_SETFL, 0)"
+  - docker logs -t netpyne-ui_container
   - docker stop $(docker ps -a -q)
   - docker rm $(docker ps -a -q)

Dockerfile

@@ -9,7 +9,6 @@ RUN apt-get install -y \
         g++ \
         build-essential \
         libncurses-dev \
-        python2.7 \
         libpython-dev \
         cython \
         libx11-dev \
@@ -20,36 +19,31 @@ RUN apt-get install -y \
         libtool \ 
         libxext-dev \
         libncurses-dev \
-        python2.7-dev \
         xfonts-100dpi \ 
         libopenmpi-dev \
-        python2.7-scipy \
         make \
         zlib1g-dev \
         unzip \
         vim \
         libpng-dev
 
-# Install latest NEURON
-RUN git clone --branch 7.6.1crxd https://github.com/adamjhn/nrn.git
+# Switch to non sudo, create a Python 3 virtual environment 
+USER $NB_USER
+RUN conda create --name snakes python=3.7
+
+# Install latest iv and NEURON
+RUN git clone --branch 7.6.2 https://github.com/neuronsimulator/nrn
 WORKDIR nrn
 RUN ./build.sh
-RUN ./configure --without-x --with-nrnpython=python2 --without-paranrn --prefix="/home/jovyan/work/nrn/" --without-iv
+# Activate conda to configure nrn with the right python version
+RUN /bin/bash -c "source activate snakes && ./configure --without-x --with-nrnpython=python3 --without-paranrn --prefix='/home/jovyan/work/nrn/' --without-iv"
 RUN make --silent -j4
 RUN make --silent install -j4
 
-# Switch to non sudo, create a Python 2 virtual environment 
-USER $NB_USER
-# Commenting out the conda update things broke!
-# RUN conda update conda
-RUN conda create --name snakes python=2
-
 # Install NEURON python
 WORKDIR src/nrnpython
 ENV PATH="/home/jovyan/work/nrn/x86_64/bin:${PATH}"
 RUN /bin/bash -c "source activate snakes && python setup.py install"
-# Install Bokeh
-RUN /bin/bash -c "source activate snakes && conda install bokeh=0.12.7"
 
 ARG INCUBATOR_VER=unknown
 RUN /bin/bash -c "INCUBATOR_VER=${INCUBATOR_VER} source activate snakes && pip install netpyne_ui"

Dockerfile_dev

@@ -13,7 +13,6 @@ RUN apt-get install -y \
         g++ \
         build-essential \
         libncurses-dev \
-        python2.7 \
         libpython-dev \
         cython \
         libx11-dev \
@@ -24,36 +23,32 @@ RUN apt-get install -y \
         libtool \ 
         libxext-dev \
         libncurses-dev \
-        python2.7-dev \
         xfonts-100dpi \ 
         libopenmpi-dev \
-        python2.7-scipy \
         make \
         zlib1g-dev \
         unzip \
         vim \
         libpng-dev
 
+# Switch to non sudo, create a Python 3 virtual environment 
+USER $NB_USER
+RUN conda create --name snakes python=3.7
+
 # Install latest iv and NEURON
-RUN git clone --branch 7.6.1crxd https://github.com/adamjhn/nrn.git
+RUN git clone --branch 7.6.2 https://github.com/neuronsimulator/nrn
 WORKDIR nrn
 RUN ./build.sh
-RUN ./configure --without-x --with-nrnpython=python2 --without-paranrn --prefix="/home/jovyan/work/nrn/" --without-iv
+# Activate conda to configure nrn with the right python version
+RUN /bin/bash -c "source activate snakes && ./configure --without-x --with-nrnpython=python3 --without-paranrn --prefix='/home/jovyan/work/nrn/' --without-iv"
 RUN make --silent -j4
 RUN make --silent install -j4
 
-# Switch to non sudo, create a Python 2 virtual environment 
-USER $NB_USER
-# Commenting out the conda update things broke!
-# RUN conda update conda
-RUN conda create --name snakes python=2
 
 # Install NEURON python
 WORKDIR src/nrnpython
 ENV PATH="/home/jovyan/work/nrn/x86_64/bin:${PATH}"
 RUN /bin/bash -c "source activate snakes && python setup.py install"
-# Install Bokeh
-RUN /bin/bash -c "source activate snakes && conda install bokeh=0.12.7"
 
 # Clone NetPyNE-UI and install the development version
 WORKDIR ../../../

netpyne_ui/netpyne_geppetto.py

@@ -2,7 +2,7 @@
 netpyne_geppetto.py
 Initialise NetPyNE Geppetto, this class contains methods to connect NetPyNE with the Geppetto based UI
 """
-import StringIO
+import io
 import json
 import os
 import importlib
@@ -14,9 +14,11 @@
 import traceback
 
 
-from netpyne import specs, sim, analysis, utils
-from netpyne.metadata import metadata, api
-from netpyne_model_interpreter import NetPyNEModelInterpreter
+from netpyne import specs, sim, analysis
+from netpyne.specs.utils import validateFunction
+from netpyne.conversion.neuronPyHoc import mechVarList
+from netpyne.metadata import metadata
+from netpyne_ui.netpyne_model_interpreter import NetPyNEModelInterpreter
 from pygeppetto.model.model_serializer import GeppettoModelSerializer
 import matplotlib.pyplot as plt
 from pygeppetto import ui
@@ -27,6 +29,7 @@
 from shutil import copyfile
 from jupyter_geppetto.geppetto_comm import GeppettoJupyterModelSync, GeppettoJupyterGUISync
 from jupyter_geppetto.geppetto_comm import GeppettoCoreAPI as G
+import imp
 
 
 
@@ -108,7 +111,7 @@ def importModel(self, modelParameters):
 
             self.compileModMechFiles(modelParameters['compileMod'], modelParameters['modFolder'])
 
-            import netpyne_geppetto
+            from . import netpyne_geppetto
 
             if modelParameters['importFormat']=='py':
                 # NetParams
@@ -120,7 +123,7 @@ def importModel(self, modelParameters):
                 # Import Model attributes
                 netpyne_geppetto.netParams = getattr(netParamsModuleName, str(modelParameters["netParamsVariable"]))
 
-                for key, value in netpyne_geppetto.netParams.cellParams.iteritems():
+                for key, value in netpyne_geppetto.netParams.cellParams.items():
                     if hasattr(value, 'todict'):
                         netpyne_geppetto.netParams.cellParams[key] = value.todict()
 
@@ -159,7 +162,7 @@ def importCellTemplate(self, modelParameters, modFolder, compileMod):
             # Get Current dir
             owd = os.getcwd()
 
-            from netpyne_geppetto import netParams
+            from .netpyne_geppetto import netParams
 
             self.compileModMechFiles(compileMod, modFolder)
 
@@ -185,7 +188,7 @@ def exportModel(self, modelParameters):
             return self.getJSONError("Error while exporting the NetPyNE model",traceback.format_exc())
 
     def instantiateNetPyNEModel(self):
-        import sys; reload(sys)
+        import sys; imp.reload(sys)
         sim.initialize(netParams, simConfig)  # create network object and set cfg and net params
         sim.net.createPops()                  # instantiate network populations
         sim.net.createCells()                 # instantiate network cells based on defined populations
@@ -198,7 +201,7 @@ def instantiateNetPyNEModel(self):
         return sim
 
     def simulateNetPyNEModel(self):
-        import sys; reload(sys)
+        import sys; imp.reload(sys)
         sim.setupRecording() 
         sim.simulate()
         sim.saveData()
@@ -244,15 +247,15 @@ def getPlot(self, plotName, LFPflavour):
             return [ui.getSVG(fig[0])].__str__()
         elif isinstance(fig, dict):
             svgs = []
-            for key, value in fig.iteritems():
+            for key, value in fig.items():
                 logging.debug("Found plot for "+ key)
                 svgs.append(ui.getSVG(value))
             return svgs.__str__()
         else:
             return [ui.getSVG(fig)].__str__()
 
     def getAvailablePops(self):
-        return netParams.popParams.keys()
+        return list(netParams.popParams.keys())
 
     def getAvailableCellModels(self):
         cellModels = set([])
@@ -275,29 +278,29 @@ def getAvailableCellTypes(self):
     def getAvailableSections(self):
         sections = {}
         for cellRule in netParams.cellParams:
-            sections[cellRule] = netParams.cellParams[cellRule]['secs'].keys()
+            sections[cellRule] = list(netParams.cellParams[cellRule]['secs'].keys())
         return sections
 
     def getAvailableStimSources(self):
-        return netParams.stimSourceParams.keys()
+        return list(netParams.stimSourceParams.keys())
 
     def getAvailableSynMech(self):
-        return netParams.synMechParams.keys()
+        return list(netParams.synMechParams.keys())
 
     def getAvailableMechs(self):
-        mechs = utils.mechVarList()['mechs']
-        for key in mechs.keys():
+        mechs = mechVarList()['mechs']
+        for key in list(mechs.keys()):
             if 'ion' in key: del mechs[key]
         for key in ["morphology", "capacitance", "extracellular"]: del mechs[key]
-        return mechs.keys()
+        return list(mechs.keys())
 
     def getMechParams(self, mechanism):
-        params = utils.mechVarList()['mechs'][mechanism]
+        params = mechVarList()['mechs'][mechanism]
         return [value[:-(len(mechanism) + 1)] for value in params]
 
     def getAvailablePlots(self):
         plots  = ["plotRaster", "plotSpikeHist", "plotSpikeStats","plotRatePSD", "plotTraces", "plotLFP", "plotShape", "plot2Dnet", "plotConn", "granger"]
-        return [plot for plot in plots if plot not in simConfig.analysis.keys()]
+        return [plot for plot in plots if plot not in list(simConfig.analysis.keys())]
 
     def deleteParam(self, paramToDel):
         logging.debug("Checking if netParams."+paramToDel+" is not null")
@@ -308,7 +311,7 @@ def deleteParam(self, paramToDel):
             logging.debug('Parameter '+paramToDel+' is null, not deleted')
 
     def validateFunction(self, functionString):
-        return utils.ValidateFunction(functionString, netParams.__dict__)
+        return ValidateFunction(functionString, netParams.__dict__)
 
     def generateScript(self, metadata):
         def convert2bool(string):
@@ -332,20 +335,20 @@ def header(title, spacer='-'):
                 script.write('netParams = specs.NetParams()\n')
                 script.write('simConfig = specs.SimConfig()\n')
                 script.write(header('single value attributes'))
-                for attr, value in netParams.__dict__.items():
+                for attr, value in list(netParams.__dict__.items()):
                     if attr not in params:
                         if value!=getattr(specs.NetParams(), attr):
                             script.write('netParams.' + attr + ' = ')
                             script.write(convert2bool(json.dumps(value, indent=4))+'\n')
 
                 script.write(header('network attributes'))
                 for param in params:
-                    for key, value in getattr(netParams, param).items():
+                    for key, value in list(getattr(netParams, param).items()):
                         script.write("netParams." + param + "['" + key + "'] = ")
                         script.write(convert2bool(json.dumps(value, indent=4))+'\n')
 
                 script.write(header('network configuration'))
-                for attr, value in simConfig.__dict__.items():
+                for attr, value in list(simConfig.__dict__.items()):
                     if value!=getattr(specs.SimConfig(), attr):
                         script.write('netParams.' + attr + ' = ')
                         script.write(convert2bool(json.dumps(value, indent=4))+'\n')
@@ -427,8 +430,11 @@ def globalMessageHandler(identifier, command, parameters):
             response = eval(command)
         else:
             response = eval(command + '(*parameters)')
+        import sys
+        imp.reload(sys)
+        print(type(response))
         GeppettoJupyterModelSync.events_controller.triggerEvent(
-            "receive_python_message", {'id': identifier, 'response': response})
+            "receive_python_message", {'id': identifier, 'response': response.decode("utf-8") if isinstance(response, bytes) else response})
     except:
         response = netpyne_geppetto.getJSONError("Error while executing command "+command,traceback.format_exc())
         GeppettoJupyterModelSync.events_controller.triggerEvent(
@@ -481,7 +487,7 @@ def GeppettoInit():
 
 GeppettoJupyterModelSync.current_model.original_model = json.dumps({'netParams': netParams.__dict__,
                                                                     'simConfig': simConfig.__dict__,
-                                                                    'metadata': metadata.metadata,
+                                                                    'metadata': metadata,
                                                                     'requirement': 'from netpyne_ui.netpyne_geppetto import *',
                                                                     'isDocker': os.path.isfile('/.dockerenv'),
                                                                     'currentFolder': os.getcwd()})

netpyne_ui/netpyne_model_interpreter.py

@@ -68,7 +68,7 @@ def extractPopulations(self, netpyne_model, netpyne_geppetto_library, geppetto_m
 
                 # Iterate sections creating spheres and cylinders
                 if hasattr(secs, 'items'):
-                    for sec_name, sec in secs.items():
+                    for sec_name, sec in list(secs.items()):
                         if 'pt3d' in sec['geom']:
                             points = sec['geom']['pt3d']
                             for i in range(len(points) - 1):

netpyne_ui/tests/M1detailed/M1detailed.json

@@ -116,7 +116,7 @@ def geom_nseg(self):
       sec.nseg = int((sec.L/(d_lambda*self.lambda_f(sec))+0.9)/2)*2 + 1
     for sec in self.all:
       after += sec.nseg 
-    print "geom_nseg: changed from ", before, " to ", after, " total segments"
+    print(("geom_nseg: changed from ", before, " to ", after, " total segments"))
 
   def lambda_f(self, section): 
     # these are reasonable values for most models
@@ -147,7 +147,7 @@ def optimize_nseg (self):
     for sec in self.all:
       sec.Ra = rall
       sec.cm = cap
-    for i in xrange(2,len(self.apic),1): 
+    for i in range(2,len(self.apic),1): 
       self.apic[i].cm = spinecapfactor * self.apic[i].cm  	 # spinecapfactor * cm
       self.apic[i].g_pas = spinecapfactor / rm  # spinecapfactor * (1.0/rm)
     for sec in self.dend:  
@@ -174,7 +174,7 @@ def init_once (self):
     self.apicchanprop()
     self.addbasalchan()
     self.basalchanprop()
-    for i in xrange(2,len(self.apic),1): 
+    for i in range(2,len(self.apic),1): 
       self.apic[i].cm = spinecapfactor * self.apic[i].cm  	 # spinecapfactor * cm
       self.apic[i].g_pas = spinecapfactor / rm  # spinecapfactor * (1.0/rm)
     for sec in self.dend:  

netpyne_ui/tests/M1detailed/cells/ITcell.py

@@ -66,8 +66,8 @@
 # L5B PV = 6180, SOM = 2600 (~70-30%)
 # L6  PV = 2640, SOM = 1820 (~60-40%)
 
-print density
-print relDensityI
+print(density)
+print(relDensityI)
 
 with open('popColors.pkl', 'r') as fileObj: popColors = pickle.load(fileObj)['popColors']  # load popColors
 
@@ -81,15 +81,15 @@
 	layers['5B'] = {'IT': 1155, 'PT': 1155, 'SOM': 202, 'PV':411}
 	layers['6'] = {'IT': 1465, 'CT': 1465, 'SOM': 107, 'PV':218}
 
-	for layer,pops in layers.iteritems():
+	for layer,pops in list(layers.items()):
 		# make a square figure and axes
 		figure(1, figsize=(6,6))
 		ax = axes([0.1, 0.1, 0.8, 0.8])
 
 		# The slices will be ordered and plotted counter-clockwise.
-		labels = pops.keys()
+		labels = list(pops.keys())
 		tot = float(sum(pops.values()))
-		fracs = [float(pop)/tot for pop in pops.values()]
+		fracs = [float(pop)/tot for pop in list(pops.values())]
 		#explode=(0, 0.05, 0, 0)
 		# if layer=='6':
 		# 	colors = [ 'gold', 'purple', 'red', 'green']
@@ -117,7 +117,7 @@
 popColors.update({'SOM2': [187, 143, 206], 'SOM5A': [165, 105, 189], 'SOM5B': [142, 68, 173], 'SOM6': [142, 68, 173]}) # SOM purple tones
 popColors.update({'PV2': [247, 220, 111], 'PV5A': [244, 208, 63], 'PV5B': [241, 196, 15], 'PV6': [212, 172, 13]}) # PV yellow tones
 popColors.update({'SOM2': [248, 196, 113], 'SOM5A': [245, 176, 65], 'SOM5B': [243, 156, 18], 'SOM6': [214, 137, 16]}) # SOM orange tones
-for pop,col in popColors.iteritems(): popColors[pop]=[c/256.0 for c in col] #normalize
+for pop,col in list(popColors.items()): popColors[pop]=[c/256.0 for c in col] #normalize
 with open('popColors.pkl', 'wb') as fileObj:        
         pickle.dump({'popColors': popColors}, fileObj)
 

netpyne_ui/tests/M1detailed/cells/PT5B_full_cellParams.pkl

@@ -64,7 +64,8 @@
 #------------------------------------------------------------------------------
 # Analysis and plotting 
 #------------------------------------------------------------------------------
-with open('cells/popColors.pkl', 'r') as fileObj: popColors = pickle.load(fileObj)['popColors']
+with open('cells/popColors.pkl', 'rb') as fileObj:
+	popColors = pickle.load(fileObj)['popColors']
 cfg.analysis['plotTraces'] = {'include': [('IT5A',0), ('PT5B',00)], 'timeRange': [0,500], 'oneFigPer': 'cell', 'figSize': (10,4), 'saveFig': True, 'showFig': False} 
 
 cfg.analysis['plotRaster'] = {'include': allpops, 'saveFig': True, 'showFig': False, 'labels': 'overlay', 'popRates': True, 'orderInverse': True,