Secure, Fast, and Extensible Sandbox runtime for AI agents.
fix(execd): send run errors after the stream started as SSE error events
Maohaodfdc3e692f6c
diff --git a/components/execd/pkg/runtime/command.go b/components/execd/pkg/runtime/command.goindex e1a787308..152157848 100644--- a/components/execd/pkg/runtime/command.go+++ b/components/execd/pkg/runtime/command.go@@ -394,12 +394,14 @@ func (c *Controller) runBackgroundCommand(ctx context.Context, cancel context.Cacmd, err := prepareCommand(ctx, request)if err != nil {cancel()+ _ = pipe.Close()return fmt.Errorf("resolve cwd: %w", err)}cred, err := buildCredential(request.Uid, request.Gid)if err != nil {cancel()+ _ = pipe.Close()return fmt.Errorf("build credential: %w", err)}cmd.SysProcAttr = &syscall.SysProcAttr{@@ -429,6 +431,7 @@ func (c *Controller) runBackgroundCommand(ctx context.Context, cancel context.Ca}if err != nil {cancel()+ _ = pipe.Close()startErr := credentialStartHint(err, cred)log.Error("command: start failed: %v", startErr)kernel.running = falsediff --git a/components/execd/pkg/runtime/command_cleanup_test.go b/components/execd/pkg/runtime/command_cleanup_test.goindex 242b8feba..9fd10abf7 100644--- a/components/execd/pkg/runtime/command_cleanup_test.go+++ b/components/execd/pkg/runtime/command_cleanup_test.go@@ -50,6 +50,32 @@ func TestRunCommandRemovesForegroundOutputFiles(t *testing.T) {require.False(t, status.Running)}+func TestRunBackgroundCommandClosesOutputFileWhenStartFails(t *testing.T) {+ const fdDir = "/proc/self/fd"+ if _, err := os.Stat(fdDir); err != nil {+ t.Skip("needs /proc/self/fd")+ }+ openFDs := func() int {+ entries, err := os.ReadDir(fdDir)+ require.NoError(t, err)+ return len(entries)+ }++ t.Setenv("TMPDIR", t.TempDir())+ c := NewController("", "")+ missingCwd := filepath.Join(t.TempDir(), "missing")++ before := openFDs()+ for range 20 {+ ctx, cancel := context.WithCancel(context.Background())+ req := &ExecuteCodeRequest{Code: "true", Cwd: missingCwd}+ req.SetDefaultHooks()+ require.Error(t, c.runBackgroundCommand(ctx, cancel, req))+ cancel()+ }+ require.LessOrEqual(t, openFDs(), before)+}+func TestCommandOutputDirRejectsSymlink(t *testing.T) {if goruntime.GOOS == "windows" {t.Skip("symlink creation requires additional privileges on Windows")diff --git a/components/execd/pkg/runtime/command_windows.go b/components/execd/pkg/runtime/command_windows.goindex 775f19940..4671cb22e 100644--- a/components/execd/pkg/runtime/command_windows.go+++ b/components/execd/pkg/runtime/command_windows.go@@ -145,6 +145,7 @@ func (c *Controller) runBackgroundCommand(ctx context.Context, cancel context.Calog.Info("command: received %v", log.SanitizeCommand(request.commandContent()))cmd, err := prepareCommand(ctx, request)if err != nil {+ pipe.Close() // best-effortreturn fmt.Errorf("resolve cwd: %w", err)}diff --git a/components/execd/pkg/web/controller/codeinterpreting.go b/components/execd/pkg/web/controller/codeinterpreting.goindex 809a3c599..dfa85ce68 100644--- a/components/execd/pkg/web/controller/codeinterpreting.go+++ b/components/execd/pkg/web/controller/codeinterpreting.go@@ -173,11 +173,7 @@ func (c *CodeInterpretingController) RunCode() {err = codeRunner.Execute(runCodeRequest)if err != nil {recordExecution("failure")- c.RespondError(- http.StatusInternalServerError,- model.ErrorCodeRuntimeError,- fmt.Sprintf("error running codes %v", err),- )+ c.respondRunError(runCodeRequest.Hooks, completeCh, fmt.Sprintf("error running codes %v", err))return}@@ -415,11 +411,7 @@ func (c *CodeInterpretingController) RunInSession() {err := codeRunner.RunInBashSession(ctx, runReq)if err != nil {recordExecution("failure")- c.RespondError(- http.StatusInternalServerError,- model.ErrorCodeRuntimeError,- fmt.Sprintf("error running in session. %v", err),- )+ c.respondRunError(runReq.Hooks, completeCh, fmt.Sprintf("error running in session. %v", err))return}diff --git a/components/execd/pkg/web/controller/codeinterpreting_test.go b/components/execd/pkg/web/controller/codeinterpreting_test.goindex 245d64bd7..19c8d2a40 100644--- a/components/execd/pkg/web/controller/codeinterpreting_test.go+++ b/components/execd/pkg/web/controller/codeinterpreting_test.go@@ -19,6 +19,7 @@ import ("encoding/json""errors""net/http"+ "strings""testing""time"@@ -430,6 +431,118 @@ func TestRunCodeSuccessStillEmitsSSE(t *testing.T) {require.NotEmpty(t, w.Body.Bytes(), "successful run should write SSE events")}+// An error returned after the init event cannot change the response status+// any more, so it must reach the client as an SSE error event instead of a+// JSON error body appended to the stream.+func TestRunErrorAfterInitEmitsErrorEvent(t *testing.T) {+ failAfterInit := func(request *runtime.ExecuteCodeRequest) error {+ request.Hooks.OnExecuteInit("session-1")+ return errors.New("failure after init")+ }+ previousRunner := codeRunner+ previousTimeout := flag.ApiGracefulShutdownTimeout+ codeRunner = &fakeCodeRunner{+ execute: failAfterInit,+ runInBashSession: func(_ context.Context, request *runtime.ExecuteCodeRequest) error {+ return failAfterInit(request)+ },+ }+ flag.ApiGracefulShutdownTimeout = 50 * time.Millisecond+ t.Cleanup(func() {+ codeRunner = previousRunner+ flag.ApiGracefulShutdownTimeout = previousTimeout+ })++ cases := []struct {+ name string+ path string+ body string+ params gin.Params+ run func(*CodeInterpretingController)+ }{+ {+ name: "RunCode",+ path: "/code/run",+ body: `{"code":"print(1)","context":{"id":"ctx-1","language":"python"}}`,+ run: (*CodeInterpretingController).RunCode,+ },+ {+ name: "RunCommand",+ path: "/command",+ body: `{"command":"echo hi","background":true}`,+ run: (*CodeInterpretingController).RunCommand,+ },+ {+ name: "RunInSession",+ path: "/sessions/session-1/run",+ body: `{"command":"echo hi","timeout":0}`,+ params: gin.Params{{Key: "sessionId", Value: "session-1"}},+ run: (*CodeInterpretingController).RunInSession,+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ ctx, w := newTestContext(http.MethodPost, tc.path, []byte(tc.body))+ ctx.Params = append(ctx.Params, tc.params...)++ tc.run(NewCodeInterpretingController(ctx))++ require.Equal(t, http.StatusOK, w.Code)+ require.Contains(t, w.Header().Get("Content-Type"), "text/event-stream")+ body := w.Body.String()+ require.True(t, strings.HasSuffix(body, "\n\n"), "every event must be terminated, got %q", body)+ var errorEvent *model.ServerStreamEvent+ for _, frame := range strings.Split(strings.TrimSuffix(body, "\n\n"), "\n\n") {+ var event model.ServerStreamEvent+ require.NoError(t, json.Unmarshal([]byte(frame), &event))+ require.NotEmpty(t, event.Type, "frame without an event type: %s", frame)+ if event.Type == model.StreamEventTypeError {+ errorEvent = &event+ }+ }+ require.NotNil(t, errorEvent, "no error event in %q", body)+ require.NotNil(t, errorEvent.Error)+ require.Contains(t, errorEvent.Error.EValue, "failure after init")+ })+ }+}++// Some runtime paths report a specific error event and then also return an+// error. The run has already ended, so no second, generic error may follow:+// clients keep the last error event they see.+func TestRunErrorAfterReportedErrorIsNotRepeated(t *testing.T) {+ previousRunner := codeRunner+ previousTimeout := flag.ApiGracefulShutdownTimeout+ codeRunner = &fakeCodeRunner{+ execute: func(request *runtime.ExecuteCodeRequest) error {+ request.Hooks.OnExecuteInit("session-1")+ request.Hooks.OnExecuteError(&execute.ErrorOutput{EName: "DBInitError", EValue: "no database"})+ return errors.New("no database")+ },+ }+ flag.ApiGracefulShutdownTimeout = 50 * time.Millisecond+ t.Cleanup(func() {+ codeRunner = previousRunner+ flag.ApiGracefulShutdownTimeout = previousTimeout+ })++ body := []byte(`{"code":"select 1","context":{"id":"ctx-1","language":"sql"}}`)+ ctx, w := newTestContext(http.MethodPost, "/code/run", body)++ NewCodeInterpretingController(ctx).RunCode()++ var errorEvents []model.ServerStreamEvent+ for _, frame := range strings.Split(strings.TrimSuffix(w.Body.String(), "\n\n"), "\n\n") {+ var event model.ServerStreamEvent+ require.NoError(t, json.Unmarshal([]byte(frame), &event))+ if event.Type == model.StreamEventTypeError {+ errorEvents = append(errorEvents, event)+ }+ }+ require.Len(t, errorEvents, 1)+ require.Equal(t, "DBInitError", errorEvents[0].Error.EName)+}+func TestRunInSession_InvalidCwdReturns400(t *testing.T) {previousRunner := codeRunnercodeRunner = &fakeCodeRunner{diff --git a/components/execd/pkg/web/controller/command.go b/components/execd/pkg/web/controller/command.goindex 30c2d5a31..8037a4728 100644--- a/components/execd/pkg/web/controller/command.go+++ b/components/execd/pkg/web/controller/command.go@@ -108,11 +108,7 @@ func (c *CodeInterpretingController) RunCommand() {err = codeRunner.Execute(runCodeRequest)if err != nil {recordExecution("failure")- c.RespondError(- http.StatusInternalServerError,- model.ErrorCodeRuntimeError,- fmt.Sprintf("error running commands %v", err),- )+ c.respondRunError(runCodeRequest.Hooks, completeCh, fmt.Sprintf("error running commands %v", err))return}diff --git a/components/execd/pkg/web/controller/sse.go b/components/execd/pkg/web/controller/sse.goindex 1162f185b..d54829b18 100644--- a/components/execd/pkg/web/controller/sse.go+++ b/components/execd/pkg/web/controller/sse.go@@ -54,6 +54,14 @@ func (c *basicController) setupSSEResponse() {})}+// streamStarted reports whether the response has been committed. It takes the+// event write lock because the ping goroutine may be writing concurrently.+func (c *basicController) streamStarted() bool {+ c.chunkWriter.Lock()+ defer c.chunkWriter.Unlock()+ return c.ctx.Writer.Written()+}+// setServerEventsHandler adapts runtime callbacks to SSE events.//// It returns the hooks and a cleanup function. The cleanup function blocks@@ -178,6 +186,28 @@ func (c *CodeInterpretingController) setServerEventsHandler(ctx context.Context)return hooks, stopSSE}+// respondRunError reports an error returned by a run. Before any event has+// been written it is a JSON error response. After that the status is already+// committed, so it is sent as an SSE error event through hooks instead, unless+// the run has already ended (ended is closed) with its own complete or error+// event.+func (c *CodeInterpretingController) respondRunError(hooks runtime.ExecuteResultHook, ended <-chan struct{}, message string) {+ if !c.streamStarted() {+ c.RespondError(http.StatusInternalServerError, model.ErrorCodeRuntimeError, message)+ return+ }+ select {+ case <-ended:+ return+ default:+ }+ hooks.OnExecuteError(&execute.ErrorOutput{+ EName: "RuntimeError",+ EValue: message,+ Traceback: []string{message},+ })+}+func (c *CodeInterpretingController) writeExecutionComplete(executionTime time.Duration, exitCode *int) {event := model.ServerStreamEvent{Type: model.StreamEventTypeComplete,