Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

fix(execd): send run errors after the stream started as SSE error events

Maohaodfdc3e692f6c
diff --git a/components/execd/pkg/runtime/command.go b/components/execd/pkg/runtime/command.go
index e1a787308..152157848 100644
--- a/components/execd/pkg/runtime/command.go
+++ b/components/execd/pkg/runtime/command.go
@@ -394,12 +394,14 @@ func (c *Controller) runBackgroundCommand(ctx context.Context, cancel context.Ca
cmd, err := prepareCommand(ctx, request)
if err != nil {
cancel()
+ _ = pipe.Close()
return fmt.Errorf("resolve cwd: %w", err)
}
cred, err := buildCredential(request.Uid, request.Gid)
if err != nil {
cancel()
+ _ = pipe.Close()
return fmt.Errorf("build credential: %w", err)
}
cmd.SysProcAttr = &syscall.SysProcAttr{
@@ -429,6 +431,7 @@ func (c *Controller) runBackgroundCommand(ctx context.Context, cancel context.Ca
}
if err != nil {
cancel()
+ _ = pipe.Close()
startErr := credentialStartHint(err, cred)
log.Error("command: start failed: %v", startErr)
kernel.running = false
diff --git a/components/execd/pkg/runtime/command_cleanup_test.go b/components/execd/pkg/runtime/command_cleanup_test.go
index 242b8feba..9fd10abf7 100644
--- a/components/execd/pkg/runtime/command_cleanup_test.go
+++ b/components/execd/pkg/runtime/command_cleanup_test.go
@@ -50,6 +50,32 @@ func TestRunCommandRemovesForegroundOutputFiles(t *testing.T) {
require.False(t, status.Running)
}
+func TestRunBackgroundCommandClosesOutputFileWhenStartFails(t *testing.T) {
+ const fdDir = "/proc/self/fd"
+ if _, err := os.Stat(fdDir); err != nil {
+ t.Skip("needs /proc/self/fd")
+ }
+ openFDs := func() int {
+ entries, err := os.ReadDir(fdDir)
+ require.NoError(t, err)
+ return len(entries)
+ }
+
+ t.Setenv("TMPDIR", t.TempDir())
+ c := NewController("", "")
+ missingCwd := filepath.Join(t.TempDir(), "missing")
+
+ before := openFDs()
+ for range 20 {
+ ctx, cancel := context.WithCancel(context.Background())
+ req := &ExecuteCodeRequest{Code: "true", Cwd: missingCwd}
+ req.SetDefaultHooks()
+ require.Error(t, c.runBackgroundCommand(ctx, cancel, req))
+ cancel()
+ }
+ require.LessOrEqual(t, openFDs(), before)
+}
+
func TestCommandOutputDirRejectsSymlink(t *testing.T) {
if goruntime.GOOS == "windows" {
t.Skip("symlink creation requires additional privileges on Windows")
diff --git a/components/execd/pkg/runtime/command_windows.go b/components/execd/pkg/runtime/command_windows.go
index 775f19940..4671cb22e 100644
--- a/components/execd/pkg/runtime/command_windows.go
+++ b/components/execd/pkg/runtime/command_windows.go
@@ -145,6 +145,7 @@ func (c *Controller) runBackgroundCommand(ctx context.Context, cancel context.Ca
log.Info("command: received %v", log.SanitizeCommand(request.commandContent()))
cmd, err := prepareCommand(ctx, request)
if err != nil {
+ pipe.Close() // best-effort
return fmt.Errorf("resolve cwd: %w", err)
}
diff --git a/components/execd/pkg/web/controller/codeinterpreting.go b/components/execd/pkg/web/controller/codeinterpreting.go
index 809a3c599..dfa85ce68 100644
--- a/components/execd/pkg/web/controller/codeinterpreting.go
+++ b/components/execd/pkg/web/controller/codeinterpreting.go
@@ -173,11 +173,7 @@ func (c *CodeInterpretingController) RunCode() {
err = codeRunner.Execute(runCodeRequest)
if err != nil {
recordExecution("failure")
- c.RespondError(
- http.StatusInternalServerError,
- model.ErrorCodeRuntimeError,
- fmt.Sprintf("error running codes %v", err),
- )
+ c.respondRunError(runCodeRequest.Hooks, completeCh, fmt.Sprintf("error running codes %v", err))
return
}
@@ -415,11 +411,7 @@ func (c *CodeInterpretingController) RunInSession() {
err := codeRunner.RunInBashSession(ctx, runReq)
if err != nil {
recordExecution("failure")
- c.RespondError(
- http.StatusInternalServerError,
- model.ErrorCodeRuntimeError,
- fmt.Sprintf("error running in session. %v", err),
- )
+ c.respondRunError(runReq.Hooks, completeCh, fmt.Sprintf("error running in session. %v", err))
return
}
diff --git a/components/execd/pkg/web/controller/codeinterpreting_test.go b/components/execd/pkg/web/controller/codeinterpreting_test.go
index 245d64bd7..19c8d2a40 100644
--- a/components/execd/pkg/web/controller/codeinterpreting_test.go
+++ b/components/execd/pkg/web/controller/codeinterpreting_test.go
@@ -19,6 +19,7 @@ import (
"encoding/json"
"errors"
"net/http"
+ "strings"
"testing"
"time"
@@ -430,6 +431,118 @@ func TestRunCodeSuccessStillEmitsSSE(t *testing.T) {
require.NotEmpty(t, w.Body.Bytes(), "successful run should write SSE events")
}
+// An error returned after the init event cannot change the response status
+// any more, so it must reach the client as an SSE error event instead of a
+// JSON error body appended to the stream.
+func TestRunErrorAfterInitEmitsErrorEvent(t *testing.T) {
+ failAfterInit := func(request *runtime.ExecuteCodeRequest) error {
+ request.Hooks.OnExecuteInit("session-1")
+ return errors.New("failure after init")
+ }
+ previousRunner := codeRunner
+ previousTimeout := flag.ApiGracefulShutdownTimeout
+ codeRunner = &fakeCodeRunner{
+ execute: failAfterInit,
+ runInBashSession: func(_ context.Context, request *runtime.ExecuteCodeRequest) error {
+ return failAfterInit(request)
+ },
+ }
+ flag.ApiGracefulShutdownTimeout = 50 * time.Millisecond
+ t.Cleanup(func() {
+ codeRunner = previousRunner
+ flag.ApiGracefulShutdownTimeout = previousTimeout
+ })
+
+ cases := []struct {
+ name string
+ path string
+ body string
+ params gin.Params
+ run func(*CodeInterpretingController)
+ }{
+ {
+ name: "RunCode",
+ path: "/code/run",
+ body: `{"code":"print(1)","context":{"id":"ctx-1","language":"python"}}`,
+ run: (*CodeInterpretingController).RunCode,
+ },
+ {
+ name: "RunCommand",
+ path: "/command",
+ body: `{"command":"echo hi","background":true}`,
+ run: (*CodeInterpretingController).RunCommand,
+ },
+ {
+ name: "RunInSession",
+ path: "/sessions/session-1/run",
+ body: `{"command":"echo hi","timeout":0}`,
+ params: gin.Params{{Key: "sessionId", Value: "session-1"}},
+ run: (*CodeInterpretingController).RunInSession,
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ ctx, w := newTestContext(http.MethodPost, tc.path, []byte(tc.body))
+ ctx.Params = append(ctx.Params, tc.params...)
+
+ tc.run(NewCodeInterpretingController(ctx))
+
+ require.Equal(t, http.StatusOK, w.Code)
+ require.Contains(t, w.Header().Get("Content-Type"), "text/event-stream")
+ body := w.Body.String()
+ require.True(t, strings.HasSuffix(body, "\n\n"), "every event must be terminated, got %q", body)
+ var errorEvent *model.ServerStreamEvent
+ for _, frame := range strings.Split(strings.TrimSuffix(body, "\n\n"), "\n\n") {
+ var event model.ServerStreamEvent
+ require.NoError(t, json.Unmarshal([]byte(frame), &event))
+ require.NotEmpty(t, event.Type, "frame without an event type: %s", frame)
+ if event.Type == model.StreamEventTypeError {
+ errorEvent = &event
+ }
+ }
+ require.NotNil(t, errorEvent, "no error event in %q", body)
+ require.NotNil(t, errorEvent.Error)
+ require.Contains(t, errorEvent.Error.EValue, "failure after init")
+ })
+ }
+}
+
+// Some runtime paths report a specific error event and then also return an
+// error. The run has already ended, so no second, generic error may follow:
+// clients keep the last error event they see.
+func TestRunErrorAfterReportedErrorIsNotRepeated(t *testing.T) {
+ previousRunner := codeRunner
+ previousTimeout := flag.ApiGracefulShutdownTimeout
+ codeRunner = &fakeCodeRunner{
+ execute: func(request *runtime.ExecuteCodeRequest) error {
+ request.Hooks.OnExecuteInit("session-1")
+ request.Hooks.OnExecuteError(&execute.ErrorOutput{EName: "DBInitError", EValue: "no database"})
+ return errors.New("no database")
+ },
+ }
+ flag.ApiGracefulShutdownTimeout = 50 * time.Millisecond
+ t.Cleanup(func() {
+ codeRunner = previousRunner
+ flag.ApiGracefulShutdownTimeout = previousTimeout
+ })
+
+ body := []byte(`{"code":"select 1","context":{"id":"ctx-1","language":"sql"}}`)
+ ctx, w := newTestContext(http.MethodPost, "/code/run", body)
+
+ NewCodeInterpretingController(ctx).RunCode()
+
+ var errorEvents []model.ServerStreamEvent
+ for _, frame := range strings.Split(strings.TrimSuffix(w.Body.String(), "\n\n"), "\n\n") {
+ var event model.ServerStreamEvent
+ require.NoError(t, json.Unmarshal([]byte(frame), &event))
+ if event.Type == model.StreamEventTypeError {
+ errorEvents = append(errorEvents, event)
+ }
+ }
+ require.Len(t, errorEvents, 1)
+ require.Equal(t, "DBInitError", errorEvents[0].Error.EName)
+}
+
func TestRunInSession_InvalidCwdReturns400(t *testing.T) {
previousRunner := codeRunner
codeRunner = &fakeCodeRunner{
diff --git a/components/execd/pkg/web/controller/command.go b/components/execd/pkg/web/controller/command.go
index 30c2d5a31..8037a4728 100644
--- a/components/execd/pkg/web/controller/command.go
+++ b/components/execd/pkg/web/controller/command.go
@@ -108,11 +108,7 @@ func (c *CodeInterpretingController) RunCommand() {
err = codeRunner.Execute(runCodeRequest)
if err != nil {
recordExecution("failure")
- c.RespondError(
- http.StatusInternalServerError,
- model.ErrorCodeRuntimeError,
- fmt.Sprintf("error running commands %v", err),
- )
+ c.respondRunError(runCodeRequest.Hooks, completeCh, fmt.Sprintf("error running commands %v", err))
return
}
diff --git a/components/execd/pkg/web/controller/sse.go b/components/execd/pkg/web/controller/sse.go
index 1162f185b..d54829b18 100644
--- a/components/execd/pkg/web/controller/sse.go
+++ b/components/execd/pkg/web/controller/sse.go
@@ -54,6 +54,14 @@ func (c *basicController) setupSSEResponse() {
})
}
+// streamStarted reports whether the response has been committed. It takes the
+// event write lock because the ping goroutine may be writing concurrently.
+func (c *basicController) streamStarted() bool {
+ c.chunkWriter.Lock()
+ defer c.chunkWriter.Unlock()
+ return c.ctx.Writer.Written()
+}
+
// setServerEventsHandler adapts runtime callbacks to SSE events.
//
// It returns the hooks and a cleanup function. The cleanup function blocks
@@ -178,6 +186,28 @@ func (c *CodeInterpretingController) setServerEventsHandler(ctx context.Context)
return hooks, stopSSE
}
+// respondRunError reports an error returned by a run. Before any event has
+// been written it is a JSON error response. After that the status is already
+// committed, so it is sent as an SSE error event through hooks instead, unless
+// the run has already ended (ended is closed) with its own complete or error
+// event.
+func (c *CodeInterpretingController) respondRunError(hooks runtime.ExecuteResultHook, ended <-chan struct{}, message string) {
+ if !c.streamStarted() {
+ c.RespondError(http.StatusInternalServerError, model.ErrorCodeRuntimeError, message)
+ return
+ }
+ select {
+ case <-ended:
+ return
+ default:
+ }
+ hooks.OnExecuteError(&execute.ErrorOutput{
+ EName: "RuntimeError",
+ EValue: message,
+ Traceback: []string{message},
+ })
+}
+
func (c *CodeInterpretingController) writeExecutionComplete(executionTime time.Duration, exitCode *int) {
event := model.ServerStreamEvent{
Type: model.StreamEventTypeComplete,