Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

fix(execd): end a Jupyter execution when its kernel dies or the connection drops

Maohao2b2cb6c11e07
diff --git a/components/execd/pkg/jupyter/execute/execute.go b/components/execd/pkg/jupyter/execute/execute.go
index 883dd8f64..8c65f551c 100644
--- a/components/execd/pkg/jupyter/execute/execute.go
+++ b/components/execd/pkg/jupyter/execute/execute.go
@@ -41,6 +41,10 @@ type Client struct {
handlers map[MessageType]func(*Message)
+ // onConnectionLost is called when the kernel connection fails while it is
+ // still in use, i.e. not after Disconnect.
+ onConnectionLost func(error)
+
session string
msgCounter int
@@ -220,6 +224,14 @@ func (c *Client) registerExecuteCodeStreamHandlers(state *streamExecutionState,
c.registerHandler(MsgStatus, func(msg *Message) {
c.handleExecutionStatus(msg, state, resultChan)
})
+ c.mu.Lock()
+ c.onConnectionLost = func(err error) {
+ c.failExecution(state, resultChan, ErrorOutput{
+ EName: "KernelConnectionLost",
+ EValue: err.Error(),
+ })
+ }
+ c.mu.Unlock()
}
func (c *Client) handleExecuteReply(msg *Message, state *streamExecutionState) {
@@ -301,7 +313,16 @@ func (c *Client) handleExecutionStatus(msg *Message, state *streamExecutionState
if err := json.Unmarshal(msg.Content, &status); err != nil {
return
}
- if status.ExecutionState != StateIdle {
+ switch status.ExecutionState {
+ case StateIdle:
+ case StateRestarting, StateDead:
+ // The kernel died, so this execution will never get a reply.
+ c.failExecution(state, resultChan, ErrorOutput{
+ EName: "KernelDied",
+ EValue: "the kernel died while executing the code (kernel state: " + string(status.ExecutionState) + ")",
+ })
+ return
+ default:
return
}
@@ -314,6 +335,26 @@ func (c *Client) handleExecutionStatus(msg *Message, state *streamExecutionState
safego.Go(func() { c.finalizeExecution(state, resultChan) })
}
+// failExecution ends an execution that will never complete: it reports
+// errOutput and closes resultChan, or lets a running finalizeExecution close it,
+// since that stops waiting once an error is recorded.
+func (c *Client) failExecution(state *streamExecutionState, resultChan chan *ExecutionResult, errOutput ErrorOutput) {
+ state.trySend(resultChan, &ExecutionResult{Error: &errOutput, Status: "error"})
+
+ state.resultMutex.Lock()
+ state.result.Status = "error"
+ state.result.Error = &errOutput
+ state.resultMutex.Unlock()
+
+ state.executeMutex.Lock()
+ finalizing := state.executeDone
+ state.executeDone = true
+ state.executeMutex.Unlock()
+ if !finalizing {
+ state.closeResultChan(resultChan)
+ }
+}
+
func (c *Client) finalizeExecution(state *streamExecutionState, resultChan chan *ExecutionResult) {
// See handleExecuteResult: resultChan send must stay outside resultMutex.
state.resultMutex.Lock()
@@ -461,6 +502,7 @@ func (c *Client) clearTemporaryHandlers() {
c.mu.Lock()
defer c.mu.Unlock()
c.handlers = make(map[MessageType]func(*Message))
+ c.onConnectionLost = nil
c.registerDefaultHandlers()
}
@@ -477,7 +519,17 @@ func (c *Client) receiveMessages() {
var msg Message
err := conn.ReadJSON(&msg)
if err != nil {
- // connection may already be closed
+ // After Disconnect, or a new Connect, the connection was closed on
+ // purpose; otherwise tell the execution waiting on it.
+ c.mu.Lock()
+ var onConnectionLost func(error)
+ if c.conn == conn {
+ onConnectionLost = c.onConnectionLost
+ }
+ c.mu.Unlock()
+ if onConnectionLost != nil {
+ onConnectionLost(err)
+ }
break
}
diff --git a/components/execd/pkg/jupyter/execute/execute_test.go b/components/execd/pkg/jupyter/execute/execute_test.go
index 96b0601d0..c98a027e1 100644
--- a/components/execd/pkg/jupyter/execute/execute_test.go
+++ b/components/execd/pkg/jupyter/execute/execute_test.go
@@ -282,3 +282,141 @@ func TestExecuteCodeStreamFallsBackWhenPollIntervalIsNonPositive(t *testing.T) {
require.GreaterOrEqual(t, elapsed, 90*time.Millisecond, "expected non-positive poll interval to fall back to runtime default (100ms)")
require.Less(t, elapsed, 300*time.Millisecond, "expected fallback poll interval to still close stream promptly")
}
+
+// collectUntilClosed drains resultChan and fails the test if it is not closed
+// within the timeout.
+func collectUntilClosed(t *testing.T, resultChan chan *ExecutionResult, timeout time.Duration) []*ExecutionResult {
+ t.Helper()
+ var results []*ExecutionResult
+ deadline := time.After(timeout)
+ for {
+ select {
+ case result, ok := <-resultChan:
+ if !ok {
+ return results
+ }
+ results = append(results, result)
+ case <-deadline:
+ t.Fatalf("resultChan was not closed within %s; got %d results", timeout, len(results))
+ }
+ }
+}
+
+func sendStatus(conn *websocket.Conn, state ExecutionState, parent Header) {
+ content, _ := json.Marshal(StatusUpdate{ExecutionState: state})
+ _ = conn.WriteJSON(Message{
+ Header: Header{MessageID: "status-" + string(state), MessageType: string(MsgStatus)},
+ ParentHeader: parent,
+ Content: json.RawMessage(content),
+ })
+}
+
+func TestExecuteCodeStreamFailsWhenConnectionDrops(t *testing.T) {
+ server := createTestServer(t, func(conn *websocket.Conn) {
+ var executeRequest Message
+ if err := conn.ReadJSON(&executeRequest); err != nil {
+ return
+ }
+ streamContent, _ := json.Marshal(StreamOutput{Name: StreamStdout, Text: "before drop\n"})
+ _ = conn.WriteJSON(Message{
+ Header: Header{MessageID: "stream-msg-id", MessageType: string(MsgStream)},
+ ParentHeader: executeRequest.Header,
+ Content: json.RawMessage(streamContent),
+ })
+ // Returning closes the connection without an execute_reply.
+ })
+ defer server.Close()
+
+ client := NewClient("", nil)
+ if err := client.Connect("ws" + strings.TrimPrefix(server.URL, "http") + "/api/kernels/k/channels"); err != nil {
+ t.Fatalf("failed to connect: %v", err)
+ }
+ defer client.Disconnect()
+
+ resultChan := make(chan *ExecutionResult, 10)
+ if err := client.ExecuteCodeStream("print('before drop')", resultChan); err != nil {
+ t.Fatalf("ExecuteCodeStream: %v", err)
+ }
+
+ results := collectUntilClosed(t, resultChan, 3*time.Second)
+ last := results[len(results)-1]
+ if last.Error == nil {
+ t.Fatalf("last result = %+v, want an error", last)
+ }
+}
+
+func TestExecuteCodeStreamFailsWhenKernelRestarts(t *testing.T) {
+ serverDone := make(chan struct{})
+ defer close(serverDone)
+ server := createTestServer(t, func(conn *websocket.Conn) {
+ var executeRequest Message
+ if err := conn.ReadJSON(&executeRequest); err != nil {
+ return
+ }
+ sendStatus(conn, StateBusy, executeRequest.Header)
+ // The kernel died; the server restarts it, and the new kernel reports
+ // idle with no parent. The execute_request never gets a reply.
+ sendStatus(conn, StateRestarting, Header{})
+ sendStatus(conn, StateStarting, Header{})
+ sendStatus(conn, StateIdle, Header{})
+ <-serverDone
+ })
+ defer server.Close()
+
+ client := NewClient("", nil)
+ if err := client.Connect("ws" + strings.TrimPrefix(server.URL, "http") + "/api/kernels/k/channels"); err != nil {
+ t.Fatalf("failed to connect: %v", err)
+ }
+ defer client.Disconnect()
+
+ resultChan := make(chan *ExecutionResult, 10)
+ if err := client.ExecuteCodeStream("import os; os._exit(1)", resultChan); err != nil {
+ t.Fatalf("ExecuteCodeStream: %v", err)
+ }
+
+ results := collectUntilClosed(t, resultChan, 3*time.Second)
+ if len(results) == 0 || results[len(results)-1].Error == nil {
+ t.Fatalf("results = %+v, want the last one to be an error", results)
+ }
+ for _, result := range results {
+ if result.ExecutionTime > 0 {
+ t.Fatalf("got a completion result %+v for an execution that never finished", result)
+ }
+ }
+}
+
+func TestExecuteCodeStreamDisconnectIsNotReportedAsLostConnection(t *testing.T) {
+ serverDone := make(chan struct{})
+ defer close(serverDone)
+ requestRead := make(chan struct{})
+ server := createTestServer(t, func(conn *websocket.Conn) {
+ var executeRequest Message
+ if err := conn.ReadJSON(&executeRequest); err != nil {
+ return
+ }
+ close(requestRead)
+ <-serverDone
+ })
+ defer server.Close()
+
+ client := NewClient("", nil)
+ if err := client.Connect("ws" + strings.TrimPrefix(server.URL, "http") + "/api/kernels/k/channels"); err != nil {
+ t.Fatalf("failed to connect: %v", err)
+ }
+
+ resultChan := make(chan *ExecutionResult, 10)
+ if err := client.ExecuteCodeStream("import time; time.sleep(60)", resultChan); err != nil {
+ t.Fatalf("ExecuteCodeStream: %v", err)
+ }
+ <-requestRead
+
+ // The caller stopped waiting (e.g. its request was cancelled) and closes
+ // the connection itself.
+ client.Disconnect()
+ time.Sleep(200 * time.Millisecond)
+ select {
+ case result := <-resultChan:
+ t.Fatalf("got %+v after Disconnect, want nothing", result)
+ default:
+ }
+}
diff --git a/components/execd/pkg/jupyter/execute/types.go b/components/execd/pkg/jupyter/execute/types.go
index 3550fce82..c02091d97 100644
--- a/components/execd/pkg/jupyter/execute/types.go
+++ b/components/execd/pkg/jupyter/execute/types.go
@@ -72,6 +72,12 @@ const (
StateBusy ExecutionState = "busy"
StateStarting ExecutionState = "starting"
+
+ // StateRestarting and StateDead are sent by the Jupyter server, not the
+ // kernel, when the kernel died and is being restarted or could not be.
+ StateRestarting ExecutionState = "restarting"
+
+ StateDead ExecutionState = "dead"
)
type Header struct {