Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

fix(execd): address review: drain cancelled script tails and stream late 404s

高然13eec485db54
diff --git a/components/execd/pkg/runtime/isolated_session_ctrl.go b/components/execd/pkg/runtime/isolated_session_ctrl.go
index 488cde259..947d1dfd3 100644
--- a/components/execd/pkg/runtime/isolated_session_ctrl.go
+++ b/components/execd/pkg/runtime/isolated_session_ctrl.go
@@ -47,6 +47,11 @@ const isolatedRunEndMarkerPrefix = "__ISOLATED_RUN_END__"
// the session's run mutex forever.
var isolatedInterruptEscalationTimeout = 10 * time.Second
+// isolatedInterruptRepeatInterval is the SIGINT re-send period during the
+// escalation window. Repeated SIGINTs drain a no-envs script tail statement
+// by statement without harming the trapped shell.
+var isolatedInterruptRepeatInterval = 500 * time.Millisecond
+
// IsolatedRunner is the concrete isolated session runner.
type IsolatedRunner struct {
ctrl *Controller
@@ -692,18 +697,32 @@ func (r *IsolatedRunner) RunInIsolatedSession(ctx context.Context, id string, co
select {
case <-ctx.Done():
_ = s.signalProcessGroupIfRunning(syscall.SIGINT)
- // If the command ignores SIGINT the marker never arrives and
- // this run would hold the session's run mutex forever. Escalate
- // to SIGKILL so the run (and with it the wedged session) ends
- // deterministically instead of hanging.
- select {
- case <-done:
- case <-time.After(isolatedInterruptEscalationTimeout):
- log.Warn(
- "isolated session: %s command ignored SIGINT; escalating to SIGKILL",
- id,
- )
- _ = s.signalProcessGroupIfRunning(syscall.SIGKILL)
+ // Without envs the script is not subshell-wrapped: after the
+ // first SIGINT the persistent shell advances to the remaining
+ // buffered statements of the cancelled run. Repeating SIGINT
+ // kills each newly started statement (the trap keeps the shell
+ // alive) so the tail drains to the end marker quickly instead
+ // of running to completion. Only if the marker still has not
+ // arrived after the escalation window is the command assumed
+ // to ignore SIGINT; SIGKILL then ends the wedged run (and the
+ // session) deterministically instead of hanging forever.
+ ticker := time.NewTicker(isolatedInterruptRepeatInterval)
+ defer ticker.Stop()
+ deadline := time.After(isolatedInterruptEscalationTimeout)
+ for {
+ select {
+ case <-done:
+ return
+ case <-ticker.C:
+ _ = s.signalProcessGroupIfRunning(syscall.SIGINT)
+ case <-deadline:
+ log.Warn(
+ "isolated session: %s command ignored SIGINT; escalating to SIGKILL",
+ id,
+ )
+ _ = s.signalProcessGroupIfRunning(syscall.SIGKILL)
+ return
+ }
}
case <-done:
}
diff --git a/components/execd/pkg/runtime/isolated_session_interrupt_test.go b/components/execd/pkg/runtime/isolated_session_interrupt_test.go
index b6fcc8e6c..4a890614b 100644
--- a/components/execd/pkg/runtime/isolated_session_interrupt_test.go
+++ b/components/execd/pkg/runtime/isolated_session_interrupt_test.go
@@ -108,6 +108,45 @@ func TestRunInIsolatedSession_InterruptedRunDoesNotLeakIntoNext(t *testing.T) {
}
}
+// TestRunInIsolatedSession_CancelledScriptTailDoesNotKillSession verifies
+// that a cancelled no-envs run whose remaining statements execute after the
+// interrupt cannot trip the SIGKILL escalation: repeated SIGINTs drain the
+// buffered tail statement by statement while the trapped shell survives, so
+// the session stays alive and accepts follow-up runs.
+func TestRunInIsolatedSession_CancelledScriptTailDoesNotKillSession(t *testing.T) {
+ runner, id := newInterruptTestSession(t)
+
+ originalEscalation := isolatedInterruptEscalationTimeout
+ isolatedInterruptEscalationTimeout = 1500 * time.Millisecond
+ t.Cleanup(func() { isolatedInterruptEscalationTimeout = originalEscalation })
+
+ ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
+ defer cancel()
+ err := runner.RunInIsolatedSession(ctx, id, "sleep 30\nsleep 30", nil, nil)
+ if !errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("interrupted run error = %v, want context.DeadlineExceeded", err)
+ }
+
+ // Stay past the escalation deadline (300ms cancel + 1500ms grace). With
+ // only a single SIGINT the second statement would still be running and
+ // the watcher would have SIGKILLed the whole session by now.
+ time.Sleep(2 * time.Second)
+
+ s := runner.lookup(id)
+ if s == nil {
+ t.Fatal("session was removed after cancelled-run tail drained")
+ }
+ if s.dead() {
+ t.Fatal("session was killed after cancelled-run tail drained")
+ }
+
+ runCtx, runCancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer runCancel()
+ if err := runner.RunInIsolatedSession(runCtx, id, "echo next", nil, nil); err != nil {
+ t.Fatalf("follow-up run: %v", err)
+ }
+}
+
// TestRunInIsolatedSession_EscalatesWhenInterruptIgnored verifies that a
// command which ignores SIGINT cannot wedge the session's run mutex forever:
// the watcher escalates to SIGKILL, the run fails, and the wedged session is
diff --git a/components/execd/pkg/web/controller/isolated_session.go b/components/execd/pkg/web/controller/isolated_session.go
index dbfde4660..a3ae64586 100644
--- a/components/execd/pkg/web/controller/isolated_session.go
+++ b/components/execd/pkg/web/controller/isolated_session.go
@@ -30,6 +30,7 @@ import (
"github.com/alibaba/opensandbox/execd/pkg/runtime"
"github.com/alibaba/opensandbox/execd/pkg/telemetry"
"github.com/alibaba/opensandbox/execd/pkg/web/model"
+ "github.com/alibaba/opensandbox/internal/safego"
)
var isolatedRunner *runtime.IsolatedRunner
@@ -306,10 +307,10 @@ func (c *IsolatedSessionController) Run() {
pingCtx, pingCancel := context.WithCancel(ctx)
var pingWg sync.WaitGroup
pingWg.Add(1)
- go func() {
+ safego.Go(func() {
defer pingWg.Done()
c.ping(pingCtx)
- }()
+ })
onStdout := func(line string) {
if line == "" {
@@ -335,7 +336,14 @@ func (c *IsolatedSessionController) Run() {
if err != nil {
if errors.Is(err, runtime.ErrContextNotFound) {
- c.RespondError(http.StatusNotFound, model.ErrorCodeSessionNotFound, "session not found")
+ // The session can vanish between the pre-flight probe and the
+ // run's internal lookup (concurrent DELETE, exited-session
+ // cleanup). SSE headers are already committed at this point, so
+ // the status code can no longer change — surface the failure as
+ // a terminal stream event instead of appending a JSON body
+ // mid-stream.
+ telemetry.RecordIsolatedRun(ctx, "error", durationMs)
+ c.writeIsolatedRunError("SessionNotFound", "session not found")
return
}
telemetry.RecordIsolatedRun(ctx, "error", durationMs)
@@ -345,16 +353,7 @@ func (c *IsolatedSessionController) Run() {
ename = "ExitError"
evalue = strings.TrimPrefix(evalue, "command exited with code ")
}
- event := model.ServerStreamEvent{
- Type: model.StreamEventTypeError,
- Text: err.Error(),
- Timestamp: time.Now().UnixMilli(),
- Error: &execute.ErrorOutput{
- EName: ename,
- EValue: evalue,
- },
- }
- c.writeSingleEvent("IsolatedError", event.ToJSON(), true, event.Summary())
+ c.writeIsolatedRunError(ename, evalue)
return
}
telemetry.RecordIsolatedRun(ctx, "success", durationMs)
@@ -365,6 +364,19 @@ func (c *IsolatedSessionController) Run() {
c.writeSingleEvent("IsolatedComplete", event.ToJSON(), true, event.Summary())
}
+func (c *IsolatedSessionController) writeIsolatedRunError(ename, evalue string) {
+ event := model.ServerStreamEvent{
+ Type: model.StreamEventTypeError,
+ Text: evalue,
+ Timestamp: time.Now().UnixMilli(),
+ Error: &execute.ErrorOutput{
+ EName: ename,
+ EValue: evalue,
+ },
+ }
+ c.writeSingleEvent("IsolatedError", event.ToJSON(), true, event.Summary())
+}
+
func (c *IsolatedSessionController) respondRunError(err error) {
if errors.Is(err, runtime.ErrContextNotFound) {
c.RespondError(http.StatusNotFound, model.ErrorCodeSessionNotFound, "session not found")