Secure, Fast, and Extensible Sandbox runtime for AI agents.
fix(execd): keep isolated session alive when a foreground run is cancelled or disconnected
高然08a9fc33935f
diff --git a/components/execd/pkg/runtime/isolated_session_ctrl.go b/components/execd/pkg/runtime/isolated_session_ctrl.goindex 6ce53e2a8..488cde259 100644--- a/components/execd/pkg/runtime/isolated_session_ctrl.go+++ b/components/execd/pkg/runtime/isolated_session_ctrl.go@@ -41,6 +41,12 @@ import (const isolatedRunEndMarkerPrefix = "__ISOLATED_RUN_END__"+// isolatedInterruptEscalationTimeout bounds how long a cancelled run waits+// after SIGINT before escalating to SIGKILL. It gives a well-behaved command+// time to wind down while guaranteeing a SIGINT-immune command cannot hold+// the session's run mutex forever.+var isolatedInterruptEscalationTimeout = 10 * time.Second+// IsolatedRunner is the concrete isolated session runner.type IsolatedRunner struct {ctrl *Controller@@ -645,24 +651,29 @@ func (r *IsolatedRunner) RunInIsolatedSession(ctx context.Context, id string, co// Prepend env exports before user code.runMarker := fmt.Sprintf("%s_%s", isolatedRunEndMarkerPrefix, uuid.New().String())- var script string+ // trap : INT keeps the persistent shell alive when the session process+ // group is signalled: the shell catches SIGINT (the trap runs after the+ // foreground child exits) while the running command dies with the default+ // disposition. When envs are supplied the code already runs in a subshell,+ // so an interrupt ends the whole run; without envs the code must stay in+ // the main shell to keep user exports visible to later runs, and an+ // interrupt then stops the currently executing line only.+ var script strings.Builder+ script.WriteString("trap : INT\n")if len(envs) > 0 {- script += "(\n"+ script.WriteString("(\n")for k, v := range envs {- script += fmt.Sprintf("export %s=%s\n", shellescape(k), shellescape(v))- }- script += code- if !strings.HasSuffix(script, "\n") {- script += "\n"- }- script += ")\n"- } else {- script += code- if !strings.HasSuffix(script, "\n") {- script += "\n"+ script.WriteString(fmt.Sprintf("export %s=%s\n", shellescape(k), shellescape(v)))}}- script += fmt.Sprintf("echo %s $?\n", runMarker)+ script.WriteString(code)+ if !strings.HasSuffix(code, "\n") {+ script.WriteString("\n")+ }+ if len(envs) > 0 {+ script.WriteString(")\n")+ }+ script.WriteString(fmt.Sprintf("echo %s $?\n", runMarker))// On timeout/cancel, send SIGINT to interrupt the running command// without killing the persistent shell session. Closing stdin would@@ -681,11 +692,24 @@ func (r *IsolatedRunner) RunInIsolatedSession(ctx context.Context, id string, coselect {case <-ctx.Done():_ = s.signalProcessGroupIfRunning(syscall.SIGINT)+ // If the command ignores SIGINT the marker never arrives and+ // this run would hold the session's run mutex forever. Escalate+ // to SIGKILL so the run (and with it the wedged session) ends+ // deterministically instead of hanging.+ select {+ case <-done:+ case <-time.After(isolatedInterruptEscalationTimeout):+ log.Warn(+ "isolated session: %s command ignored SIGINT; escalating to SIGKILL",+ id,+ )+ _ = s.signalProcessGroupIfRunning(syscall.SIGKILL)+ }case <-done:}}()- if _, err := io.WriteString(stdin, script); err != nil {+ if _, err := io.WriteString(stdin, script.String()); err != nil {return fmt.Errorf("write stdin: %w", err)}diff --git a/components/execd/pkg/runtime/isolated_session_interrupt_test.go b/components/execd/pkg/runtime/isolated_session_interrupt_test.gonew file mode 100644index 000000000..b6fcc8e6c--- /dev/null+++ b/components/execd/pkg/runtime/isolated_session_interrupt_test.go@@ -0,0 +1,153 @@+// Copyright 2026 The OpenSandbox Authors++//go:build !windows++// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package runtime++import (+ "context"+ "errors"+ "testing"+ "time"+)++func newInterruptTestSession(t *testing.T) (*IsolatedRunner, string) {+ t.Helper()+ runner := newTestRunner(t)+ id, err := runner.CreateIsolatedSession(&IsolatedSessionOptions{+ WorkspacePath: t.TempDir(),+ WorkspaceMode: "rw",+ })+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = runner.DeleteIsolatedSession(id) })+ return runner, id+}++// TestRunInIsolatedSession_InterruptKeepsSessionAlive verifies the public+// contract that a cancelled run (client disconnect or timeout_seconds expiry)+// interrupts the running command without killing or deregistering the+// persistent session, so the next run on the same session succeeds.+func TestRunInIsolatedSession_InterruptKeepsSessionAlive(t *testing.T) {+ runner, id := newInterruptTestSession(t)++ ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)+ defer cancel()+ err := runner.RunInIsolatedSession(ctx, id, "sleep 30", nil, nil)+ if !errors.Is(err, context.DeadlineExceeded) {+ t.Fatalf("interrupted run error = %v, want context.DeadlineExceeded", err)+ }++ s := runner.lookup(id)+ if s == nil {+ t.Fatal("session was removed after run cancellation")+ }+ if s.dead() {+ t.Fatal("session is dead after run cancellation")+ }++ // The same session must accept a follow-up run.+ var lines []string+ runCtx, runCancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer runCancel()+ err = runner.RunInIsolatedSession(runCtx, id, "echo next", nil, func(line string) {+ lines = append(lines, line)+ })+ if err != nil {+ t.Fatalf("follow-up run: %v", err)+ }+ if len(lines) != 1 || lines[0] != "next" {+ t.Fatalf("follow-up output = %v, want [next]", lines)+ }+}++// TestRunInIsolatedSession_InterruptedRunDoesNotLeakIntoNext verifies that an+// interrupt ends the whole run when the code is wrapped in a subshell (envs+// supplied): later lines of the interrupted code must never execute and must+// not pollute the next run's output.+func TestRunInIsolatedSession_InterruptedRunDoesNotLeakIntoNext(t *testing.T) {+ runner, id := newInterruptTestSession(t)++ ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)+ defer cancel()+ err := runner.RunInIsolatedSession(+ ctx,+ id,+ "sleep 30\necho SHOULD_NOT_APPEAR",+ map[string]string{"RUN_ENVELOPE": "1"},+ nil,+ )+ if !errors.Is(err, context.DeadlineExceeded) {+ t.Fatalf("interrupted run error = %v, want context.DeadlineExceeded", err)+ }++ var lines []string+ runCtx, runCancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer runCancel()+ err = runner.RunInIsolatedSession(runCtx, id, "echo next", nil, func(line string) {+ lines = append(lines, line)+ })+ if err != nil {+ t.Fatalf("follow-up run: %v", err)+ }+ if len(lines) != 1 || lines[0] != "next" {+ t.Fatalf("follow-up output = %v, want exactly [next]", lines)+ }+}++// TestRunInIsolatedSession_EscalatesWhenInterruptIgnored verifies that a+// command which ignores SIGINT cannot wedge the session's run mutex forever:+// the watcher escalates to SIGKILL, the run fails, and the wedged session is+// torn down so the failure is observable instead of a silent hang.+func TestRunInIsolatedSession_EscalatesWhenInterruptIgnored(t *testing.T) {+ runner, id := newInterruptTestSession(t)++ originalEscalation := isolatedInterruptEscalationTimeout+ isolatedInterruptEscalationTimeout = 300 * time.Millisecond+ t.Cleanup(func() { isolatedInterruptEscalationTimeout = originalEscalation })++ ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)+ defer cancel()+ start := time.Now()+ err := runner.RunInIsolatedSession(ctx, id, "trap '' INT; sleep 30", nil, nil)+ if !errors.Is(err, context.DeadlineExceeded) {+ t.Fatalf("escalated run error = %v, want context.DeadlineExceeded", err)+ }+ if elapsed := time.Since(start); elapsed > 5*time.Second {+ t.Fatalf("escalated run took %v; SIGINT-immune command was not killed", elapsed)+ }++ // The SIGKILL reaps the session process, so the session must turn dead+ // and reject further work. (The test isolator's lifecycle only drains on+ // explicit delete, so registry removal is not asserted here.)+ deadline := time.Now().Add(5 * time.Second)+ for {+ s := runner.lookup(id)+ if s == nil {+ t.Fatal("session record should still exist until deletion")+ }+ if s.dead() {+ break+ }+ if time.Now().After(deadline) {+ t.Fatal("session should be dead after SIGKILL escalation")+ }+ time.Sleep(50 * time.Millisecond)+ }+ if err := runner.RunInIsolatedSession(context.Background(), id, "echo hi", nil, nil); !errors.Is(err, ErrSessionNotActive) {+ t.Fatalf("run on escalated session error = %v, want ErrSessionNotActive", err)+ }+}diff --git a/components/execd/pkg/web/controller/isolated_session.go b/components/execd/pkg/web/controller/isolated_session.goindex 704a8a258..dbfde4660 100644--- a/components/execd/pkg/web/controller/isolated_session.go+++ b/components/execd/pkg/web/controller/isolated_session.go@@ -20,6 +20,7 @@ import ("net/http""strconv""strings"+ "sync""time""github.com/gin-gonic/gin"@@ -265,6 +266,18 @@ func (c *IsolatedSessionController) Run() {}defer cancel()+ // Fail fast on an unknown session while the response is still JSON:+ // once SSE headers are committed below, errors can only be delivered as+ // stream events.+ if _, err := isolatedRunner.GetIsolatedSession(sessionID); err != nil {+ if errors.Is(err, runtime.ErrContextNotFound) {+ c.RespondError(http.StatusNotFound, model.ErrorCodeSessionNotFound, "session not found")+ return+ }+ c.RespondError(http.StatusInternalServerError, model.ErrorCodeRuntimeError, err.Error())+ return+ }+// Background mode: submit detached, return a run handle, let the client// poll status/logs. Deliberately independent of this request's context so// a client disconnect cannot cancel detached work. 202 Accepted keeps the@@ -284,6 +297,20 @@ func (c *IsolatedSessionController) Run() {return}+ // Foreground mode: stream via SSE. Commit the response headers up front+ // and keep the stream alive with periodic pings so gateway idle timeouts+ // cannot reap the connection while a silent command (e.g. pip install)+ // produces no stdout for long stretches.+ c.setupSSEResponse()++ pingCtx, pingCancel := context.WithCancel(ctx)+ var pingWg sync.WaitGroup+ pingWg.Add(1)+ go func() {+ defer pingWg.Done()+ c.ping(pingCtx)+ }()+onStdout := func(line string) {if line == "" {return@@ -300,6 +327,12 @@ func (c *IsolatedSessionController) Run() {err := isolatedRunner.RunInIsolatedSession(ctx, sessionID, req.Code, req.Envs, onStdout)durationMs := float64(time.Since(startTime)) / float64(time.Millisecond)+ // Stop the ping goroutine before the final event and before the handler+ // returns, so nothing writes to the response writer after Go's net/http+ // closes it.+ pingCancel()+ pingWg.Wait()+if err != nil {if errors.Is(err, runtime.ErrContextNotFound) {c.RespondError(http.StatusNotFound, model.ErrorCodeSessionNotFound, "session not found")diff --git a/components/execd/pkg/web/controller/isolated_session_controller_test.go b/components/execd/pkg/web/controller/isolated_session_controller_test.goindex cbb445fd9..940a1a68a 100644--- a/components/execd/pkg/web/controller/isolated_session_controller_test.go+++ b/components/execd/pkg/web/controller/isolated_session_controller_test.go@@ -539,3 +539,104 @@ func TestGetRunLogs_RejectsMalformedCursor(t *testing.T) {}}}++// TestIsolatedRun_ForegroundStreamSendsPings verifies the foreground run+// streams SSE with periodic ping keepalives: a command that produces no+// stdout for a while must still emit bytes so gateway idle timeouts cannot+// reap the connection, and the stream must end with the complete event.+func TestIsolatedRun_ForegroundStreamSendsPings(t *testing.T) {+ previousRunner := isolatedRunner+ previousInterval := ssePingInterval+ runner, err := runtime.NewIsolatedRunner(+ runtime.NewController("", ""),+ lifecycleTestIsolator{},+ isolation.Config{+ UpperRoot: t.TempDir(),+ UpperMaxBytes: 8 << 30,+ },+ )+ if err != nil {+ t.Fatal(err)+ }+ isolatedRunner = runner+ ssePingInterval = 30 * time.Millisecond+ t.Cleanup(func() {+ runner.StopGC()+ isolatedRunner = previousRunner+ ssePingInterval = previousInterval+ })++ sessionID, err := runner.CreateIsolatedSession(&runtime.IsolatedSessionOptions{+ WorkspacePath: filepath.Join(t.TempDir(), "ws"),+ WorkspaceMode: "rw",+ })+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = runner.DeleteIsolatedSession(sessionID) })++ runCtx, runRecorder := newTestContext(+ http.MethodPost,+ "/v1/isolated/session/"+sessionID+"/run",+ []byte(`{"code":"sleep 0.4","timeout_seconds":10}`),+ )+ runCtx.Params = gin.Params{{Key: "sessionId", Value: sessionID}}+ NewIsolatedSessionController(runCtx).Run()++ if runRecorder.Code != http.StatusOK {+ t.Fatalf("run status = %d, want 200; body: %s", runRecorder.Code, runRecorder.Body.String())+ }+ if got := runRecorder.Header().Get("Content-Type"); got != "text/event-stream" {+ t.Fatalf("content type = %q, want text/event-stream", got)+ }++ body := runRecorder.Body.String()+ pings := strings.Count(body, `"type":"ping"`)+ if pings == 0 {+ t.Fatalf("expected at least one ping event during silent command, body:\n%s", body)+ }+ if !strings.Contains(body, `"type":"execution_complete"`) {+ t.Fatalf("stream must end with execution_complete, body:\n%s", body)+ }+}++// TestIsolatedRun_UnknownSessionReturnsJSON verifies that a run on a missing+// session still returns the structured 404 JSON error, not an SSE stream+// (headers are not committed before the session check).+func TestIsolatedRun_UnknownSessionReturnsJSON(t *testing.T) {+ previousRunner := isolatedRunner+ runner, err := runtime.NewIsolatedRunner(+ runtime.NewController("", ""),+ lifecycleTestIsolator{},+ isolation.Config{+ UpperRoot: t.TempDir(),+ UpperMaxBytes: 8 << 30,+ },+ )+ if err != nil {+ t.Fatal(err)+ }+ isolatedRunner = runner+ t.Cleanup(func() {+ runner.StopGC()+ isolatedRunner = previousRunner+ })++ runCtx, runRecorder := newTestContext(+ http.MethodPost,+ "/v1/isolated/session/nope/run",+ []byte(`{"code":"echo hi"}`),+ )+ runCtx.Params = gin.Params{{Key: "sessionId", Value: "nope"}}+ NewIsolatedSessionController(runCtx).Run()++ if runRecorder.Code != http.StatusNotFound {+ t.Fatalf("status = %d, want 404; body: %s", runRecorder.Code, runRecorder.Body.String())+ }+ if got := runRecorder.Header().Get("Content-Type"); got == "text/event-stream" {+ t.Fatal("unknown-session run must return JSON, not an SSE stream")+ }+ if !strings.Contains(runRecorder.Body.String(), "session not found") {+ t.Fatalf("body = %s, want session-not-found error", runRecorder.Body.String())+ }+}diff --git a/components/execd/pkg/web/controller/sse.go b/components/execd/pkg/web/controller/sse.goindex 1162f185b..2d8d251a1 100644--- a/components/execd/pkg/web/controller/sse.go+++ b/components/execd/pkg/web/controller/sse.go@@ -37,6 +37,11 @@ var sseHeaders = map[string]string{"X-Accel-Buffering": "no",}+// ssePingInterval is the keepalive period for SSE streams. It must stay well+// below downstream gateway/proxy idle timeouts so a silent command (e.g. a+// long pip install producing no stdout) cannot get the stream reaped.+var ssePingInterval = 3 * time.Second+// setupSSEResponse is idempotent: once headers are committed, subsequent calls// no-op. Callers that need the headers up front (e.g. long-running streaming// endpoints with no early-error path) can call it explicitly. Endpoints that@@ -228,7 +233,7 @@ func (c *basicController) writeSingleEvent(handler string, data []byte, verbose}// ping periodically keeps the SSE connection alive.-func (c *CodeInterpretingController) ping(ctx context.Context) {+func (c *basicController) ping(ctx context.Context) {wait.Until(func() {if c.ctx.Writer == nil {return@@ -240,5 +245,5 @@ func (c *CodeInterpretingController) ping(ctx context.Context) {}payload := event.ToJSON()c.writeSingleEvent("Ping", payload, false, event.Summary())- }, 3*time.Second, ctx.Done())+ }, ssePingInterval, ctx.Done())}diff --git a/docs/guides/isolation-sessions.md b/docs/guides/isolation-sessions.mdindex 340118088..10213fb88 100644--- a/docs/guides/isolation-sessions.md+++ b/docs/guides/isolation-sessions.md@@ -84,10 +84,14 @@ Non-happy paths:| Event | What happens ||---|---|-| Run `timeout_seconds` elapsed | execd cancels the run context, which sends `SIGINT` to the bwrap process group and emits an `IsolatedError` SSE event; the session itself stays alive. |+| Run `timeout_seconds` elapsed | execd cancels the run context and sends `SIGINT` to the session process group, which interrupts the running command while the persistent shell stays alive; the run ends with an `IsolatedError` SSE event and the same session accepts the next `run`. || `bwrap` process exited | The next `run` returns an `IsolatedError` SSE event with `session process has exited`; only `ErrContextNotFound` (session ID unknown) becomes an HTTP-level error. `DELETE` + recreate. || Idle timeout reached | GC runs the same teardown as `DELETE`. |-| Client disconnects mid-SSE | The Gin request context is cancelled and execd sends `SIGINT` to the running command; the run does **not** continue in the background. |+| Client disconnects mid-SSE | The Gin request context is cancelled and execd sends `SIGINT` to the running command; the run does **not** continue in the background, and the session stays alive for the next `run`. |++::: tip SSE keepalive+Foreground `run` streams commit their SSE response headers immediately and send a `ping` event every few seconds, so proxies and gateways with idle-timeout policies do not reap the connection while a long command (e.g. `pip install`) produces no stdout. Treat `ping` as a no-op event type.+:::---diff --git a/sdks/sandbox/javascript/src/api/execd.ts b/sdks/sandbox/javascript/src/api/execd.tsindex 2fe3e8b6e..060159930 100644--- a/sdks/sandbox/javascript/src/api/execd.ts+++ b/sdks/sandbox/javascript/src/api/execd.ts@@ -969,7 +969,7 @@ export interface paths {put?: never;/*** Run code in an isolated session- * @description Runs code inside an existing isolated session. Foreground mode (default) streams output via SSE (200). Background mode (`background: true`) starts the command detached and returns a JSON run handle with 202 Accepted for polling status and logs via `/v1/isolated/session/{sessionId}/runs/{runId}` and `.../runs/{runId}/logs`. `timeout_seconds` applies to foreground runs only; background runs are not time-limited (idle GC is suspended while one is active) and are bounded by the session lifetime.+ * @description Runs code inside an existing isolated session. Foreground mode (default) streams output via SSE (200). Background mode (`background: true`) starts the command detached and returns a JSON run handle with 202 Accepted for polling status and logs via `/v1/isolated/session/{sessionId}/runs/{runId}` and `.../runs/{runId}/logs`. `timeout_seconds` applies to foreground runs only; background runs are not time-limited (idle GC is suspended while one is active) and are bounded by the session lifetime. Foreground streams emit periodic `ping` keepalive events so idle gateway timeouts do not reap the connection while the command produces no output; clients must ignore unrecognized event types.*/post: operations["runInIsolatedSession"];delete?: never;diff --git a/sdks/sandbox/python/src/opensandbox/api/execd/api/isolated_execution/run_in_isolated_session.py b/sdks/sandbox/python/src/opensandbox/api/execd/api/isolated_execution/run_in_isolated_session.pyindex ed947d6d3..82803237e 100644--- a/sdks/sandbox/python/src/opensandbox/api/execd/api/isolated_execution/run_in_isolated_session.py+++ b/sdks/sandbox/python/src/opensandbox/api/execd/api/isolated_execution/run_in_isolated_session.py@@ -110,7 +110,9 @@ def sync_detailed(handle with 202 Accepted for polling status and logs via`/v1/isolated/session/{sessionId}/runs/{runId}` and `.../runs/{runId}/logs`. `timeout_seconds`applies to foreground runs only; background runs are not time-limited (idle GC is suspended while- one is active) and are bounded by the session lifetime.+ one is active) and are bounded by the session lifetime. Foreground streams emit periodic `ping`+ keepalive events so idle gateway timeouts do not reap the connection while the command produces no+ output; clients must ignore unrecognized event types.Args:session_id (UUID):@@ -149,7 +151,9 @@ def sync(handle with 202 Accepted for polling status and logs via`/v1/isolated/session/{sessionId}/runs/{runId}` and `.../runs/{runId}/logs`. `timeout_seconds`applies to foreground runs only; background runs are not time-limited (idle GC is suspended while- one is active) and are bounded by the session lifetime.+ one is active) and are bounded by the session lifetime. Foreground streams emit periodic `ping`+ keepalive events so idle gateway timeouts do not reap the connection while the command produces no+ output; clients must ignore unrecognized event types.Args:session_id (UUID):@@ -183,7 +187,9 @@ async def asyncio_detailed(handle with 202 Accepted for polling status and logs via`/v1/isolated/session/{sessionId}/runs/{runId}` and `.../runs/{runId}/logs`. `timeout_seconds`applies to foreground runs only; background runs are not time-limited (idle GC is suspended while- one is active) and are bounded by the session lifetime.+ one is active) and are bounded by the session lifetime. Foreground streams emit periodic `ping`+ keepalive events so idle gateway timeouts do not reap the connection while the command produces no+ output; clients must ignore unrecognized event types.Args:session_id (UUID):@@ -220,7 +226,9 @@ async def asyncio(handle with 202 Accepted for polling status and logs via`/v1/isolated/session/{sessionId}/runs/{runId}` and `.../runs/{runId}/logs`. `timeout_seconds`applies to foreground runs only; background runs are not time-limited (idle GC is suspended while- one is active) and are bounded by the session lifetime.+ one is active) and are bounded by the session lifetime. Foreground streams emit periodic `ping`+ keepalive events so idle gateway timeouts do not reap the connection while the command produces no+ output; clients must ignore unrecognized event types.Args:session_id (UUID):diff --git a/specs/execd-api.yaml b/specs/execd-api.yamlindex 092e86085..8ddf26a8f 100644--- a/specs/execd-api.yaml+++ b/specs/execd-api.yaml@@ -1925,6 +1925,9 @@ paths:`.../runs/{runId}/logs`. `timeout_seconds` applies to foregroundruns only; background runs are not time-limited (idle GC is suspendedwhile one is active) and are bounded by the session lifetime.+ Foreground streams emit periodic `ping` keepalive events so idle+ gateway timeouts do not reap the connection while the command+ produces no output; clients must ignore unrecognized event types.operationId: runInIsolatedSessiontags:- IsolatedExecution