Skip to content

Commit 87d0140

Browse files
aranegaaranega
committed
netpyne-44 Add handling of kernel crash loop
1 parent f945d4a commit 87d0140

6 files changed

Lines changed: 104 additions & 17 deletions

File tree

‎webapp/components/NetPyNE.js‎

Lines changed: 47 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,8 @@ class NetPyNE extends React.Component {
5050
this.loaded = false;
5151
this.kernelRestartState = {
5252
state: "idle",
53-
kernelID: undefined
53+
kernelID: undefined,
54+
crashLoop: false
5455
}
5556
}
5657

@@ -99,32 +100,38 @@ class NetPyNE extends React.Component {
99100
console.log("Replaying all commands since the beginning of the session")
100101
replayAll(this.kernelRestartState.kernelID)
101102
this.kernelRestartState = {
103+
...this.kernelRestartState,
102104
state: "idle",
103-
kernelID: undefined
105+
kernelID: undefined,
104106
}
105107
}
106108
case "idle":
107109
if (type === "kernel_connected") {
108110
console.log("Kernel is connecting/starting, being init")
109111
this.kernelRestartState = {
112+
...this.kernelRestartState,
110113
state: "init",
111114
kernelID: kernel.id
112115
}
113116
}
114117
else if (type === "kernel_autorestarting") {
115118
console.log("Kernel restart event caught, trying to re-init the current model")
116119
this.kernelRestartState = {
120+
...this.kernelRestartState,
117121
state: "restarting",
118122
kernelID: kernel.id
119123
}
120-
this.props.dispatchAction(openDialog({
121-
title: "Kernel restart",
122-
message: "An action occured that made the kernel restart. We are reloading your model and all the actions you applied on it."
123-
}))
124+
if (!this.kernelRestartState.crashLoop) {
125+
this.props.dispatchAction(openDialog({
126+
title: "Kernel restart",
127+
message: "An action occured that made the kernel restart. We are reloading your model and all the actions you applied on it."
128+
}))
129+
}
124130
}
125131
else if (type === "kernel_restarting") {
126132
console.log("Kernel restart, perhaps it's a special restart?")
127133
this.kernelRestartState = {
134+
...this.kernelRestartState,
128135
state: "special_restart",
129136
kernelID: kernel.id
130137
}
@@ -133,27 +140,58 @@ class NetPyNE extends React.Component {
133140
if (type === "kernel_ready") {
134141
console.log("Kernel properly initialized")
135142
this.kernelRestartState = {
143+
...this.kernelRestartState,
136144
state: "idle",
137-
kernelID: undefined
145+
kernelID: undefined,
138146
}
139147
}
140148
case "special_restart":
141149
if (type == "kernel_autorestarting") {
142150
console.log("Kernel autorestart after a start, we might not have the ready event, we force it then")
143151
replayAll(this.kernelRestartState.kernelID)
144152
this.kernelRestartState = {
153+
...this.kernelRestartState,
145154
state: 'restarting',
146-
kernelID: kernel.id
155+
kernelID: kernel.id,
147156
}
148157
} else {
158+
console.log("Regular restart detected")
149159
this.kernelRestartState = {
160+
...this.kernelRestartState,
150161
state: 'idle',
151-
kernelID: undefined
162+
kernelID: undefined,
152163
}
153164
}
154165
}
155166
}
156167
window.addEventListener('kernelstatus', handleKernelRestart)
168+
169+
// Dedicated code to handle crash loops
170+
const kernelRestartLoopHandler = ( ) => {
171+
if (!this.kernelRestartState.crashLoop) {
172+
this.props.dispatchAction(openDialog({
173+
title: "Kernel restart loop stabilization",
174+
message: "One of your actions triggered a kernel restart loop. We are trying to identify the faulty command and to restore your model until this point. Close this window and wait for the kernel stabilization notification."
175+
}))
176+
} else {
177+
clearTimeout(this.kernelRestartState.crashLoop)
178+
}
179+
const taskID = setTimeout((_this) => {
180+
this.props.dispatchAction(openDialog({
181+
title: "Kernel restart loop stabilized",
182+
message: "The kernel is now stabilized."
183+
}));
184+
this.kernelRestartState = {
185+
...this.kernelRestartState,
186+
crashLoop: false
187+
}
188+
}, 8000, this)
189+
this.kernelRestartState = {
190+
...this.kernelRestartState,
191+
crashLoop: taskID
192+
}
193+
}
194+
window.addEventListener('kernelRestartLoop', kernelRestartLoopHandler)
157195
}
158196

159197
componentWillUnmount () {

‎webapp/components/general/CommandRecorder.js‎

Lines changed: 31 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1+
import { KERNEL_HANDLING } from '../../constants';
12
import { store } from '../../redux/actiondomainStore'
2-
import { recordCommand, dropLastCommand } from '../../redux/actions/actiondomain';
3-
import { execPythonMessageWithoutRecording } from './GeppettoJupyterUtils';
3+
import { recordCommand, dropLastCommand, dropFromIndex } from '../../redux/actions/actiondomain';
4+
import { execPythonMessage, execPythonMessageWithoutRecording } from './GeppettoJupyterUtils';
45

56

67
const registerKernelListeners = () => {
@@ -58,18 +59,42 @@ const record = (kernelID, command) => {
5859
store.dispatch(recordCommand(kernelID, command))
5960
}
6061

61-
const replayAll = (kernelID) => {
62-
const commands = [
62+
const TIMEFRAME = 10 * 1000; // 10s
63+
let lastReplayTime = 0
64+
65+
const getCommands = (kernelID) => {
66+
return [
6367
"from jupyter_geppetto import jupyter_geppetto",
6468
"from jupyter_geppetto import utils",
6569
"from netpyne_ui.netpyne_geppetto import netpyne_geppetto",
6670
"netpyne_geppetto.deleteModel({})",
67-
"netpyne_geppetto.loadFromIndexFile('/tmp/tmpmodel')",
68-
...store.getState()[kernelID]];
71+
`netpyne_geppetto.loadFromIndexFile("${KERNEL_HANDLING.tmpModelPath}")`,
72+
...store.getState()[kernelID]
73+
]
74+
}
75+
76+
const replayAll = (kernelID, fromRec = false) => {
77+
const currentTimestamp = Date.now();
78+
const commands = getCommands(kernelID);
79+
80+
if (!fromRec && currentTimestamp - lastReplayTime < TIMEFRAME) {
81+
const restartLoop = new CustomEvent("kernelRestartLoop", {
82+
detail: {
83+
"kernel": kernelID,
84+
"state": "looping"
85+
}
86+
});
87+
window.dispatchEvent(restartLoop);
88+
store.dispatch(dropLastCommand(kernelID))
89+
replayAll(kernelID, true)
90+
return
91+
}
92+
6993
const lastCommand = commands.pop() // we drop the last command which is probably the faulty one
7094
const script = commands.join('\n')
7195
console.log("Playing", script)
7296
console.log("Skipping last command", lastCommand)
97+
lastReplayTime = currentTimestamp
7398
execPythonMessageWithoutRecording(script).then(() => {
7499
store.dispatch(dropLastCommand(kernelID))
75100
})

‎webapp/constants.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -746,3 +746,7 @@ export const geometryStrings = [
746746
'inside',
747747
'membrane'
748748
]
749+
750+
export const KERNEL_HANDLING = {
751+
tmpModelPath: '/tmp/tmpmodel'
752+
}

‎webapp/redux/actiondomainStore.js‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
// import action types
2-
import { DROP_LAST_COMMAND, FLUSH_COMMANDS, RECORD_COMMAND } from "./actions/actiondomain";
2+
import { DROP_FROM_INDEX, DROP_LAST_COMMAND, FLUSH_COMMANDS, RECORD_COMMAND } from "./actions/actiondomain";
33
// import redux from 'redux'
44

55
// Default state for general
@@ -30,6 +30,15 @@ const flushCommands = (state, { kernel }) => {
3030
return { ...state, [kernel]: [] };
3131
}
3232

33+
const dropFromIndex = (state, { kernel, index }) => {
34+
const newState = { ...state };
35+
if (!newState[kernel]) {
36+
newState[kernel] = []
37+
}
38+
newState[kernel] = newState[kernel].splice(0, index)
39+
return newState;
40+
}
41+
3342
// reducer
3443
const reducer = (state = ACTION_DOMAIN_DEFAULT_STATE, action) => {
3544
switch (action.type) {
@@ -39,6 +48,8 @@ const reducer = (state = ACTION_DOMAIN_DEFAULT_STATE, action) => {
3948
return dropLastCommand(state, action.payload)
4049
case FLUSH_COMMANDS:
4150
return flushCommands(state, action.payload)
51+
case DROP_FROM_INDEX:
52+
return dropFromIndex(state, action.payload)
4253
default:
4354
return state
4455
}

‎webapp/redux/actions/actiondomain.js‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Action Types
22
export const RECORD_COMMAND = 'RECORD_COMMAND';
33
export const DROP_LAST_COMMAND = 'DROP_LAST_COMMAND';
4+
export const DROP_FROM_INDEX = 'DROP_FROM_INDEX';
45
export const FLUSH_COMMANDS = 'FLUSH_COMMANDS';
56

67
// Actions
@@ -19,6 +20,14 @@ export const dropLastCommand = (kernelID) => ({
1920
}
2021
})
2122

23+
export const dropFromIndex = (kernelID, index) => ({
24+
type: DROP_FROM_INDEX,
25+
payload: {
26+
kernel: kernelID,
27+
index: index
28+
}
29+
})
30+
2231
export const flushCommands = (kernelID) => ({
2332
type: FLUSH_COMMANDS,
2433
payload: {

‎webapp/redux/middleware/middleware.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -290,7 +290,7 @@ export default (store) => (next) => (action) => {
290290
next(GeppettoActions.modelLoaded())
291291
getExperiments()
292292
});
293-
Utils.execPythonMessage('netpyne_geppetto.saveToIndexFile(None, "/tmp/tmpmodel", True, True)', ()=>{}, false);
293+
Utils.execPythonMessage(`netpyne_geppetto.saveToIndexFile(None, "${Constants.KERNEL_HANDLING.tmpModelPath}", True, True)`, ()=>{}, false);
294294

295295
setTimeout(() => {
296296
if (!responded) {

0 commit comments

Comments
 (0)