Secure, Fast, and Extensible Sandbox runtime for AI agents.
fix(execd): end a Jupyter execution when its kernel dies or the connection drops
Maohao2b2cb6c11e07
diff --git a/components/execd/pkg/jupyter/execute/execute.go b/components/execd/pkg/jupyter/execute/execute.goindex 883dd8f64..8c65f551c 100644--- a/components/execd/pkg/jupyter/execute/execute.go+++ b/components/execd/pkg/jupyter/execute/execute.go@@ -41,6 +41,10 @@ type Client struct {handlers map[MessageType]func(*Message)+ // onConnectionLost is called when the kernel connection fails while it is+ // still in use, i.e. not after Disconnect.+ onConnectionLost func(error)+session stringmsgCounter int@@ -220,6 +224,14 @@ func (c *Client) registerExecuteCodeStreamHandlers(state *streamExecutionState,c.registerHandler(MsgStatus, func(msg *Message) {c.handleExecutionStatus(msg, state, resultChan)})+ c.mu.Lock()+ c.onConnectionLost = func(err error) {+ c.failExecution(state, resultChan, ErrorOutput{+ EName: "KernelConnectionLost",+ EValue: err.Error(),+ })+ }+ c.mu.Unlock()}func (c *Client) handleExecuteReply(msg *Message, state *streamExecutionState) {@@ -301,7 +313,16 @@ func (c *Client) handleExecutionStatus(msg *Message, state *streamExecutionStateif err := json.Unmarshal(msg.Content, &status); err != nil {return}- if status.ExecutionState != StateIdle {+ switch status.ExecutionState {+ case StateIdle:+ case StateRestarting, StateDead:+ // The kernel died, so this execution will never get a reply.+ c.failExecution(state, resultChan, ErrorOutput{+ EName: "KernelDied",+ EValue: "the kernel died while executing the code (kernel state: " + string(status.ExecutionState) + ")",+ })+ return+ default:return}@@ -314,6 +335,26 @@ func (c *Client) handleExecutionStatus(msg *Message, state *streamExecutionStatesafego.Go(func() { c.finalizeExecution(state, resultChan) })}+// failExecution ends an execution that will never complete: it reports+// errOutput and closes resultChan, or lets a running finalizeExecution close it,+// since that stops waiting once an error is recorded.+func (c *Client) failExecution(state *streamExecutionState, resultChan chan *ExecutionResult, errOutput ErrorOutput) {+ state.trySend(resultChan, &ExecutionResult{Error: &errOutput, Status: "error"})++ state.resultMutex.Lock()+ state.result.Status = "error"+ state.result.Error = &errOutput+ state.resultMutex.Unlock()++ state.executeMutex.Lock()+ finalizing := state.executeDone+ state.executeDone = true+ state.executeMutex.Unlock()+ if !finalizing {+ state.closeResultChan(resultChan)+ }+}+func (c *Client) finalizeExecution(state *streamExecutionState, resultChan chan *ExecutionResult) {// See handleExecuteResult: resultChan send must stay outside resultMutex.state.resultMutex.Lock()@@ -461,6 +502,7 @@ func (c *Client) clearTemporaryHandlers() {c.mu.Lock()defer c.mu.Unlock()c.handlers = make(map[MessageType]func(*Message))+ c.onConnectionLost = nilc.registerDefaultHandlers()}@@ -477,7 +519,17 @@ func (c *Client) receiveMessages() {var msg Messageerr := conn.ReadJSON(&msg)if err != nil {- // connection may already be closed+ // After Disconnect, or a new Connect, the connection was closed on+ // purpose; otherwise tell the execution waiting on it.+ c.mu.Lock()+ var onConnectionLost func(error)+ if c.conn == conn {+ onConnectionLost = c.onConnectionLost+ }+ c.mu.Unlock()+ if onConnectionLost != nil {+ onConnectionLost(err)+ }break}diff --git a/components/execd/pkg/jupyter/execute/execute_test.go b/components/execd/pkg/jupyter/execute/execute_test.goindex 96b0601d0..c98a027e1 100644--- a/components/execd/pkg/jupyter/execute/execute_test.go+++ b/components/execd/pkg/jupyter/execute/execute_test.go@@ -282,3 +282,141 @@ func TestExecuteCodeStreamFallsBackWhenPollIntervalIsNonPositive(t *testing.T) {require.GreaterOrEqual(t, elapsed, 90*time.Millisecond, "expected non-positive poll interval to fall back to runtime default (100ms)")require.Less(t, elapsed, 300*time.Millisecond, "expected fallback poll interval to still close stream promptly")}++// collectUntilClosed drains resultChan and fails the test if it is not closed+// within the timeout.+func collectUntilClosed(t *testing.T, resultChan chan *ExecutionResult, timeout time.Duration) []*ExecutionResult {+ t.Helper()+ var results []*ExecutionResult+ deadline := time.After(timeout)+ for {+ select {+ case result, ok := <-resultChan:+ if !ok {+ return results+ }+ results = append(results, result)+ case <-deadline:+ t.Fatalf("resultChan was not closed within %s; got %d results", timeout, len(results))+ }+ }+}++func sendStatus(conn *websocket.Conn, state ExecutionState, parent Header) {+ content, _ := json.Marshal(StatusUpdate{ExecutionState: state})+ _ = conn.WriteJSON(Message{+ Header: Header{MessageID: "status-" + string(state), MessageType: string(MsgStatus)},+ ParentHeader: parent,+ Content: json.RawMessage(content),+ })+}++func TestExecuteCodeStreamFailsWhenConnectionDrops(t *testing.T) {+ server := createTestServer(t, func(conn *websocket.Conn) {+ var executeRequest Message+ if err := conn.ReadJSON(&executeRequest); err != nil {+ return+ }+ streamContent, _ := json.Marshal(StreamOutput{Name: StreamStdout, Text: "before drop\n"})+ _ = conn.WriteJSON(Message{+ Header: Header{MessageID: "stream-msg-id", MessageType: string(MsgStream)},+ ParentHeader: executeRequest.Header,+ Content: json.RawMessage(streamContent),+ })+ // Returning closes the connection without an execute_reply.+ })+ defer server.Close()++ client := NewClient("", nil)+ if err := client.Connect("ws" + strings.TrimPrefix(server.URL, "http") + "/api/kernels/k/channels"); err != nil {+ t.Fatalf("failed to connect: %v", err)+ }+ defer client.Disconnect()++ resultChan := make(chan *ExecutionResult, 10)+ if err := client.ExecuteCodeStream("print('before drop')", resultChan); err != nil {+ t.Fatalf("ExecuteCodeStream: %v", err)+ }++ results := collectUntilClosed(t, resultChan, 3*time.Second)+ last := results[len(results)-1]+ if last.Error == nil {+ t.Fatalf("last result = %+v, want an error", last)+ }+}++func TestExecuteCodeStreamFailsWhenKernelRestarts(t *testing.T) {+ serverDone := make(chan struct{})+ defer close(serverDone)+ server := createTestServer(t, func(conn *websocket.Conn) {+ var executeRequest Message+ if err := conn.ReadJSON(&executeRequest); err != nil {+ return+ }+ sendStatus(conn, StateBusy, executeRequest.Header)+ // The kernel died; the server restarts it, and the new kernel reports+ // idle with no parent. The execute_request never gets a reply.+ sendStatus(conn, StateRestarting, Header{})+ sendStatus(conn, StateStarting, Header{})+ sendStatus(conn, StateIdle, Header{})+ <-serverDone+ })+ defer server.Close()++ client := NewClient("", nil)+ if err := client.Connect("ws" + strings.TrimPrefix(server.URL, "http") + "/api/kernels/k/channels"); err != nil {+ t.Fatalf("failed to connect: %v", err)+ }+ defer client.Disconnect()++ resultChan := make(chan *ExecutionResult, 10)+ if err := client.ExecuteCodeStream("import os; os._exit(1)", resultChan); err != nil {+ t.Fatalf("ExecuteCodeStream: %v", err)+ }++ results := collectUntilClosed(t, resultChan, 3*time.Second)+ if len(results) == 0 || results[len(results)-1].Error == nil {+ t.Fatalf("results = %+v, want the last one to be an error", results)+ }+ for _, result := range results {+ if result.ExecutionTime > 0 {+ t.Fatalf("got a completion result %+v for an execution that never finished", result)+ }+ }+}++func TestExecuteCodeStreamDisconnectIsNotReportedAsLostConnection(t *testing.T) {+ serverDone := make(chan struct{})+ defer close(serverDone)+ requestRead := make(chan struct{})+ server := createTestServer(t, func(conn *websocket.Conn) {+ var executeRequest Message+ if err := conn.ReadJSON(&executeRequest); err != nil {+ return+ }+ close(requestRead)+ <-serverDone+ })+ defer server.Close()++ client := NewClient("", nil)+ if err := client.Connect("ws" + strings.TrimPrefix(server.URL, "http") + "/api/kernels/k/channels"); err != nil {+ t.Fatalf("failed to connect: %v", err)+ }++ resultChan := make(chan *ExecutionResult, 10)+ if err := client.ExecuteCodeStream("import time; time.sleep(60)", resultChan); err != nil {+ t.Fatalf("ExecuteCodeStream: %v", err)+ }+ <-requestRead++ // The caller stopped waiting (e.g. its request was cancelled) and closes+ // the connection itself.+ client.Disconnect()+ time.Sleep(200 * time.Millisecond)+ select {+ case result := <-resultChan:+ t.Fatalf("got %+v after Disconnect, want nothing", result)+ default:+ }+}diff --git a/components/execd/pkg/jupyter/execute/types.go b/components/execd/pkg/jupyter/execute/types.goindex 3550fce82..c02091d97 100644--- a/components/execd/pkg/jupyter/execute/types.go+++ b/components/execd/pkg/jupyter/execute/types.go@@ -72,6 +72,12 @@ const (StateBusy ExecutionState = "busy"StateStarting ExecutionState = "starting"++ // StateRestarting and StateDead are sent by the Jupyter server, not the+ // kernel, when the kernel died and is being restarted or could not be.+ StateRestarting ExecutionState = "restarting"++ StateDead ExecutionState = "dead")type Header struct {