Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(sandbox): recover timed-out filesystem operations and staged attachments (#3921)

c56425b151d8f
diff --git a/internal/application/service/session_attachment_staging.go b/internal/application/service/session_attachment_staging.go
index fc4d88694..06670d091 100644
--- a/internal/application/service/session_attachment_staging.go
+++ b/internal/application/service/session_attachment_staging.go
@@ -8,6 +8,7 @@ import (
"path"
"sort"
"strings"
+ "time"
"github.com/Tencent/WeKnora/internal/application/repository"
"github.com/Tencent/WeKnora/internal/sandbox"
@@ -151,6 +152,10 @@ func (s *agentService) stageSessionAttachments(
desired := make(map[string]struct{}, len(attachments))
var assets []string
staged := make([]stagedSessionAttachment, 0, len(attachments))
+ inputs := make([]stagedInput, 0, len(attachments))
+ // A timeout retry may move the session to a new container. The initial
+ // directory snapshot cannot then establish that later inputs still exist.
+ reuseExisting := true
maxBytes := int64(secutils.GetMaxFileSizeMB()) * 1024 * 1024
for _, attachment := range attachments {
remotePath, pathErr := sandboxAttachmentPath(attachment, layout.InputDir)
@@ -161,37 +166,60 @@ func (s *agentService) stageSessionAttachments(
assets = append(assets, remotePath+attachmentAssetsSuffix)
entry, present := existing[remotePath]
- if !present || attachment.FileSize <= 0 || entry.Size != attachment.FileSize {
- reader, getErr := s.fileService.GetFile(ctx, attachment.URL)
- if getErr != nil {
- return nil, fmt.Errorf("open attachment %q: %w", attachment.FileName, getErr)
- }
- content, readErr := io.ReadAll(io.LimitReader(reader, maxBytes+1))
- closeErr := reader.Close()
+ if !reuseExisting || !present || attachment.FileSize <= 0 || entry.Size != attachment.FileSize {
+ content, readErr := s.readAttachmentContent(ctx, attachment, maxBytes)
if readErr != nil {
- return nil, fmt.Errorf("read attachment %q: %w", attachment.FileName, readErr)
- }
- if closeErr != nil {
- return nil, fmt.Errorf("close attachment %q: %w", attachment.FileName, closeErr)
- }
- if int64(len(content)) > maxBytes {
- return nil, fmt.Errorf("attachment %q exceeds sandbox staging limit of %d bytes", attachment.FileName, maxBytes)
+ return nil, readErr
}
- if writeErr := store.WriteSessionInputFile(ctx, sessionID, remotePath, content); writeErr != nil {
+ rebound, writeErr := writeSessionInputWithRetry(ctx, store, sessionID, remotePath, content)
+ if writeErr != nil {
return nil, fmt.Errorf("stage attachment %q: %w", attachment.FileName, writeErr)
}
attachment.FileSize = int64(len(content))
+ inputs = append(inputs, stagedInput{
+ remotePath: remotePath,
+ name: attachment.FileName,
+ url: attachment.URL,
+ size: attachment.FileSize,
+ })
+ if rebound {
+ reuseExisting = false
+ // The retry re-resolved the session, which may have
+ // rebound it onto a fresh container: everything staged
+ // before this attachment lived in the old one.
+ if restoreErr := s.reestablishStagedInputs(
+ ctx, store, sessionID, layout.InputDir, inputs, maxBytes,
+ ); restoreErr != nil {
+ return nil, restoreErr
+ }
+ }
+ } else {
+ // Reused from the previous pass: no content kept in memory,
+ // but the rebinding recovery must still be able to
+ // re-create it from durable storage.
+ inputs = append(inputs, stagedInput{
+ remotePath: remotePath,
+ name: attachment.FileName,
+ url: attachment.URL,
+ size: attachment.FileSize,
+ })
}
parsedPath := ""
if text, ok := parsed[strings.TrimSpace(attachment.URL)]; ok {
parsedPath = remotePath + parsedSidecarSuffix
desired[parsedPath] = struct{}{}
- if entry, present := existing[parsedPath]; !present || entry.Size != int64(len(text)) {
+ if entry, present := existing[parsedPath]; !reuseExisting || !present || entry.Size != int64(len(text)) {
if writeErr := store.WriteSessionInputFile(ctx, sessionID, parsedPath, []byte(text)); writeErr != nil {
return nil, fmt.Errorf("stage parsed text of %q: %w", attachment.FileName, writeErr)
}
}
+ inputs = append(inputs, stagedInput{
+ remotePath: parsedPath,
+ name: attachment.FileName + parsedSidecarSuffix,
+ content: []byte(text),
+ size: int64(len(text)),
+ })
}
staged = append(staged, stagedSessionAttachment{
@@ -234,6 +262,149 @@ func (s *agentService) stageSessionAttachments(
return staged, nil
}
+// sandboxStagingRetryDelay separates the two attempts a transient sandbox
+// filesystem stall gets before staging fails the turn.
+const sandboxStagingRetryDelay = 2 * time.Second
+
+// sandboxStagingRebindPasses bounds how many times the rebinding recovery
+// below re-verifies the input set after a retry landed on yet another fresh
+// container. A session whose inputs keep getting killed this many times is
+// not going to heal; the error says so instead of looping forever.
+const sandboxStagingRebindPasses = 3
+
+// stagedInput tracks one attachment staged during this pass so the rebinding
+// recovery can re-establish what an earlier container already held. Original
+// files are re-read from durable storage on recovery rather than retaining
+// every attachment in memory. Only derived text sidecars carry content.
+type stagedInput struct {
+ remotePath string
+ name string
+ url string
+ content []byte
+ size int64
+}
+
+// readAttachmentContent reads one attachment from durable storage, bounded by
+// the sandbox staging limit.
+func (s *agentService) readAttachmentContent(
+ ctx context.Context,
+ attachment types.MessageAttachment,
+ maxBytes int64,
+) ([]byte, error) {
+ reader, getErr := s.fileService.GetFile(ctx, attachment.URL)
+ if getErr != nil {
+ return nil, fmt.Errorf("open attachment %q: %w", attachment.FileName, getErr)
+ }
+ content, readErr := io.ReadAll(io.LimitReader(reader, maxBytes+1))
+ closeErr := reader.Close()
+ if readErr != nil {
+ return nil, fmt.Errorf("read attachment %q: %w", attachment.FileName, readErr)
+ }
+ if closeErr != nil {
+ return nil, fmt.Errorf("close attachment %q: %w", attachment.FileName, closeErr)
+ }
+ if int64(len(content)) > maxBytes {
+ return nil, fmt.Errorf("attachment %q exceeds sandbox staging limit of %d bytes", attachment.FileName, maxBytes)
+ }
+ return content, nil
+}
+
+// reestablishStagedInputs re-writes staged inputs that went missing after a
+// timeout retry rebound the session onto a fresh container: the retry's
+// WriteSessionInputFile re-resolves the session, and the new container's
+// input directory does not carry anything the old one held. Without this
+// pass, staging would report success while the sandbox only has the one
+// attachment whose retry happened to land.
+//
+// Each pass re-lists the input directory and rewrites whatever is missing or
+// has the wrong size. A rewrite that itself only succeeds after a timeout
+// retry may have rebound onto yet another container, so the pass repeats
+// until a full pass lands without a rebind (or the pass budget runs out).
+func (s *agentService) reestablishStagedInputs(
+ ctx context.Context,
+ store sandbox.SessionFileStore,
+ sessionID, inputDir string,
+ inputs []stagedInput,
+ maxBytes int64,
+) error {
+ for pass := 0; pass < sandboxStagingRebindPasses; pass++ {
+ entries, listErr := store.ListSessionFiles(ctx, sessionID, inputDir)
+ if listErr != nil {
+ return fmt.Errorf("relist staged session inputs after filesystem-op timeout: %w", listErr)
+ }
+ present := make(map[string]int64, len(entries))
+ for _, entry := range entries {
+ present[path.Clean(entry.Path)] = entry.Size
+ }
+ reboundAgain := false
+ for _, input := range inputs {
+ if size, ok := present[input.remotePath]; ok && size == input.size {
+ continue
+ }
+ content := input.content
+ if content == nil {
+ var readErr error
+ content, readErr = s.readAttachmentContent(ctx, types.MessageAttachment{
+ URL: input.url,
+ FileName: input.name,
+ }, maxBytes)
+ if readErr != nil {
+ return fmt.Errorf("re-stage attachment %q: %w", input.name, readErr)
+ }
+ if int64(len(content)) != input.size {
+ return fmt.Errorf(
+ "re-stage attachment %q: durable content is %d bytes, expected %d",
+ input.name, len(content), input.size)
+ }
+ }
+ rebound, writeErr := writeSessionInputWithRetry(ctx, store, sessionID, input.remotePath, content)
+ if writeErr != nil {
+ return fmt.Errorf("re-stage attachment %q after filesystem-op timeout: %w", input.name, writeErr)
+ }
+ reboundAgain = reboundAgain || rebound
+ }
+ if !reboundAgain {
+ return nil
+ }
+ }
+ return fmt.Errorf(
+ "session inputs kept getting killed past the filesystem-op timeout across %d rebinding passes",
+ sandboxStagingRebindPasses)
+}
+
+// writeSessionInputWithRetry retries once when the sandbox killed the write
+// past its filesystem-op timeout (issue #3910): a killed op means the
+// container filesystem stalled — the request itself was valid, and such
+// stalls are usually transient. Any other failure surfaces immediately. When
+// the retry also times out, both failures stay attached to the returned error
+// so the log shows it was not a one-off.
+//
+// The returned rebound flag reports that the write only succeeded on the
+// retry: WriteSessionInputFile re-resolves the session, so the second attempt
+// may have landed in a fresh container. Callers staging more than one input
+// must treat a rebound as "everything staged before this write may be gone"
+// and re-establish it.
+func writeSessionInputWithRetry(
+ ctx context.Context,
+ store sandbox.SessionFileStore,
+ sessionID, remotePath string,
+ content []byte,
+) (bool, error) {
+ err := store.WriteSessionInputFile(ctx, sessionID, remotePath, content)
+ if !sandbox.IsRemoteTimeout(err) {
+ return false, err
+ }
+ select {
+ case <-ctx.Done():
+ return false, err
+ case <-time.After(sandboxStagingRetryDelay):
+ }
+ if retryErr := store.WriteSessionInputFile(ctx, sessionID, remotePath, content); retryErr != nil {
+ return false, fmt.Errorf("%w (retry after filesystem-op timeout failed: %v)", err, retryErr)
+ }
+ return true, nil
+}
+
// resolveSessionAttachmentURLs fills in the storage handle for attachments
// whose URL was not persisted on the message row. MessageAttachment.URL is
// deliberately excluded from DB serialization (json:"-") so a cross-session
diff --git a/internal/application/service/session_attachment_staging_test.go b/internal/application/service/session_attachment_staging_test.go
index c2ff61db4..2f3298050 100644
--- a/internal/application/service/session_attachment_staging_test.go
+++ b/internal/application/service/session_attachment_staging_test.go
@@ -613,3 +613,268 @@ func TestStageSessionAttachmentsKeepsRemoteInputRoot(t *testing.T) {
require.Equal(t, sandbox.SessionInputRoot, store.listedDir)
require.True(t, strings.HasPrefix(staged[0].Path, sandbox.SessionInputRoot))
}
+
+// retryTestStore fails WriteSessionInputFile per the hook so the one-retry
+// policy on killed filesystem ops (issue #3910) can be driven without a
+// sandbox. Only WriteSessionInputFile is reachable in these tests.
+type retryTestStore struct {
+ sandbox.SessionFileStore
+ onWrite func() error
+}
+
+func (s *retryTestStore) WriteSessionInputFile(context.Context, string, string, []byte) error {
+ return s.onWrite()
+}
+
+func TestWriteSessionInputWithRetryRetriesKilledOpOnce(t *testing.T) {
+ calls := 0
+ store := &retryTestStore{onWrite: func() error {
+ calls++
+ if calls == 1 {
+ return sandbox.NewRemoteError(
+ sandbox.SandboxTypeDocker, "MakeDir", sandbox.RemoteErrorKindTimeout,
+ "killed after 30s (filesystem-op exec budget 30s), exit=137, no output", nil)
+ }
+ return nil
+ }}
+ rebound, err := writeSessionInputWithRetry(
+ context.Background(), store, "session-1", "/workspace/input/abc/faq.txt", []byte("hi"))
+ require.NoError(t, err)
+ require.True(t, rebound, "a write that only landed on the retry must be reported as a possible rebind")
+ require.Equal(t, 2, calls, "a killed op must be retried exactly once")
+}
+
+func TestWriteSessionInputWithRetrySurfacesOtherFailuresImmediately(t *testing.T) {
+ calls := 0
+ store := &retryTestStore{onWrite: func() error {
+ calls++
+ return sandbox.NewRemoteError(
+ sandbox.SandboxTypeDocker, "MakeDir", sandbox.RemoteErrorKindInvalidRequest,
+ "MakeDir /workspace/input: Permission denied", nil)
+ }}
+ rebound, err := writeSessionInputWithRetry(
+ context.Background(), store, "session-1", "/workspace/input/abc/faq.txt", []byte("hi"))
+ require.Error(t, err)
+ require.False(t, rebound)
+ require.Equal(t, 1, calls, "a non-timeout failure must not be retried")
+}
+
+func TestWriteSessionInputWithRetryRetryFailureKeepsBothErrors(t *testing.T) {
+ calls := 0
+ store := &retryTestStore{onWrite: func() error {
+ calls++
+ return sandbox.NewRemoteError(
+ sandbox.SandboxTypeDocker, "MakeDir", sandbox.RemoteErrorKindTimeout,
+ "killed after 30s, exit=137, no output", nil)
+ }}
+ rebound, err := writeSessionInputWithRetry(
+ context.Background(), store, "session-1", "/workspace/input/abc/faq.txt", []byte("hi"))
+ require.Error(t, err)
+ require.False(t, rebound)
+ require.Equal(t, 2, calls)
+ require.Contains(t, err.Error(), "retry after filesystem-op timeout failed",
+ "both the original kill and the retry failure must stay in the log")
+}
+
+// rebindingInputStore simulates the sequence the one-file retry cannot heal
+// on its own: the path in slowPath gets its first WriteSessionInputFile
+// killed past the filesystem-op timeout, and the retry — which re-resolves
+// the session — lands in a fresh container whose input directory does not
+// hold anything staged before the kill.
+type rebindingInputStore struct {
+ stagingSandboxManager
+ slowPath string
+ killedOnce bool
+ reboundOnce bool
+}
+
+// SessionFileStore must return the outer type: the embedded copy would hand
+// staging the base WriteSessionInputFile and this test's rebind override
+// would never run.
+func (s *rebindingInputStore) SessionFileStore() sandbox.SessionFileStore {
+ return s
+}
+
+func (s *rebindingInputStore) WriteSessionInputFile(_ context.Context, _, filePath string, content []byte) error {
+ if filePath == s.slowPath && !s.killedOnce {
+ s.killedOnce = true
+ return sandbox.NewRemoteError(
+ sandbox.SandboxTypeDocker, "MakeDir", sandbox.RemoteErrorKindTimeout,
+ "MakeDir /workspace/input: killed after 30s, exit=137, no output", nil)
+ }
+ if filePath == s.slowPath && s.killedOnce && !s.reboundOnce {
+ // The retry re-resolved the session onto a fresh container: the
+ // inputs the old container held went away with it.
+ s.reboundOnce = true
+ s.files = make(map[string][]byte)
+ }
+ if s.files == nil {
+ s.files = make(map[string][]byte)
+ }
+ s.files[filePath] = append([]byte(nil), content...)
+ s.writes = append(s.writes, filePath)
+ return nil
+}
+
+// The first attachment is staged into container 1; the second attachment's
+// first write is killed, and its retry lands in a fresh container. Staging
+// must re-establish the first attachment there before reporting success —
+// without the recovery pass, the sandbox would end up holding only the
+// attachment whose retry happened to land.
+func TestStageSessionAttachmentsRestoresEarlierInputsAfterRebind(t *testing.T) {
+ first := types.MessageAttachment{
+ URL: "local://tenant/attachment-1",
+ FileName: "report.pdf",
+ FileType: ".pdf",
+ }
+ second := types.MessageAttachment{
+ URL: "local://tenant/attachment-2",
+ FileName: "notes.txt",
+ FileType: ".txt",
+ }
+ layout := sandbox.RemoteWorkspaceLayout()
+ remoteFirst, err := sandboxAttachmentPath(first, layout.InputDir)
+ require.NoError(t, err)
+ remoteSecond, err := sandboxAttachmentPath(second, layout.InputDir)
+ require.NoError(t, err)
+
+ store := &rebindingInputStore{slowPath: remoteSecond}
+ fileService := &stagingFileService{
+ files: map[string][]byte{
+ first.URL: []byte("content-first"),
+ second.URL: []byte("content-second"),
+ },
+ }
+ service := &agentService{
+ sandboxMgr: store,
+ fileService: fileService,
+ sandboxResolver: stubSandboxResolver{mgr: store},
+ }
+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7))
+
+ staged, err := service.stageSessionAttachments(
+ ctx, "session-1", "cfg-remote", 7,
+ types.MessageAttachments{first, second}, layout,
+ )
+
+ require.NoError(t, err)
+ require.Len(t, staged, 2)
+ require.Equal(t, []byte("content-first"), store.files[remoteFirst],
+ "the attachment staged before the killed write must be re-established in the fresh container")
+ require.Equal(t, []byte("content-second"), store.files[remoteSecond])
+ require.Equal(t, []string{remoteFirst, remoteSecond, remoteFirst}, store.writes,
+ "write order: first attachment, killed attachment's retry, then the re-established first attachment")
+ // Recovery re-reads the earlier original instead of retaining every
+ // attachment in memory for the entire staging pass.
+ require.Equal(t, 2, fileService.getCalls[first.URL])
+ require.Equal(t, 1, fileService.getCalls[second.URL])
+}
+
+// Same sequence, but the earlier attachment was reused from a previous pass
+// (present with a matching size, so its content was never read). The
+// recovery must re-read it from durable storage and stage it into the fresh
+// container.
+func TestStageSessionAttachmentsRestoresReusedInputsAfterRebind(t *testing.T) {
+ first := types.MessageAttachment{
+ URL: "local://tenant/attachment-1",
+ FileName: "report.pdf",
+ FileType: ".pdf",
+ FileSize: int64(len("content-first")),
+ }
+ second := types.MessageAttachment{
+ URL: "local://tenant/attachment-2",
+ FileName: "notes.txt",
+ FileType: ".txt",
+ }
+ layout := sandbox.RemoteWorkspaceLayout()
+ remoteFirst, err := sandboxAttachmentPath(first, layout.InputDir)
+ require.NoError(t, err)
+ remoteSecond, err := sandboxAttachmentPath(second, layout.InputDir)
+ require.NoError(t, err)
+
+ store := &rebindingInputStore{
+ slowPath: remoteSecond,
+ stagingSandboxManager: stagingSandboxManager{
+ sandboxType: sandbox.SandboxTypeCube,
+ files: map[string][]byte{remoteFirst: []byte("content-first")},
+ },
+ }
+ fileService := &stagingFileService{
+ files: map[string][]byte{
+ first.URL: []byte("content-first"),
+ second.URL: []byte("content-second"),
+ },
+ }
+ service := &agentService{
+ sandboxMgr: store,
+ fileService: fileService,
+ sandboxResolver: stubSandboxResolver{mgr: store},
+ }
+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7))
+
+ staged, err := service.stageSessionAttachments(
+ ctx, "session-1", "cfg-remote", 7,
+ types.MessageAttachments{first, second}, layout,
+ )
+
+ require.NoError(t, err)
+ require.Len(t, staged, 2)
+ require.Equal(t, []byte("content-first"), store.files[remoteFirst],
+ "the reused attachment must be re-established after the rebind dropped it")
+ require.Equal(t, []byte("content-second"), store.files[remoteSecond])
+ require.Equal(t, 1, fileService.getCalls[first.URL],
+ "the reused attachment's content must be re-read from durable storage during the recovery")
+}
+
+func TestStageSessionAttachmentsRebindRestoresSidecarsAndLaterInputs(t *testing.T) {
+ for _, sidecarPresent := range []bool{false, true} {
+ name := "new-sidecar"
+ if sidecarPresent {
+ name = "reused-sidecar"
+ }
+ t.Run(name, func(t *testing.T) {
+ db := stagingTempDocDB(t)
+ attachments := types.MessageAttachments{
+ {ID: "parsed-doc", URL: "local://first", FileName: "report.pdf", FileType: ".pdf", FileSize: 3},
+ {URL: "local://second", FileName: "notes.txt", FileType: ".txt", FileSize: 3},
+ {URL: "local://third", FileName: "later.txt", FileType: ".txt", FileSize: 3},
+ }
+ require.NoError(t, db.Create(&types.TemporaryDocument{
+ ID: "parsed-doc", TenantID: 7, SessionID: "session-1", ResourceRef: attachments[0].URL,
+ FileName: "report.pdf", FileType: ".pdf", FileSize: 3,
+ Status: types.TemporaryDocumentStatusReady, Content: "extracted report",
+ ExpiresAt: time.Now().Add(time.Hour),
+ }).Error)
+ layout := sandbox.RemoteWorkspaceLayout()
+ paths := make([]string, len(attachments))
+ for i, attachment := range attachments {
+ var err error
+ paths[i], err = sandboxAttachmentPath(attachment, layout.InputDir)
+ require.NoError(t, err)
+ }
+ store := &rebindingInputStore{
+ slowPath: paths[1],
+ stagingSandboxManager: stagingSandboxManager{
+ files: map[string][]byte{paths[0]: []byte("one"), paths[2]: []byte("tri")},
+ },
+ }
+ if sidecarPresent {
+ store.files[paths[0]+parsedSidecarSuffix] = []byte("extracted report")
+ }
+ fileService := &stagingFileService{files: map[string][]byte{
+ attachments[0].URL: []byte("one"), attachments[1].URL: []byte("two"), attachments[2].URL: []byte("tri"),
+ }}
+ svc := &agentService{
+ db: db, sandboxMgr: store, fileService: fileService, sandboxResolver: stubSandboxResolver{mgr: store},
+ }
+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7))
+ staged, err := svc.stageSessionAttachments(ctx, "session-1", "cfg-remote", 7, attachments, layout)
+ require.NoError(t, err)
+ require.Len(t, staged, 3)
+ assert.Equal(t, []byte("extracted report"), store.files[paths[0]+parsedSidecarSuffix])
+ for i, attachment := range attachments {
+ assert.Equal(t, fileService.files[attachment.URL], store.files[paths[i]], attachment.FileName)
+ }
+ })
+ }
+}
diff --git a/internal/sandbox/docker_remote_client.go b/internal/sandbox/docker_remote_client.go
index 62522d245..3c819d925 100644
--- a/internal/sandbox/docker_remote_client.go
+++ b/internal/sandbox/docker_remote_client.go
@@ -859,9 +859,13 @@ func dockerExecUser(user string) string {
return DefaultSandboxExecUser
}
-// dockerExecWasKilled reports whether an exit code means the wrapper killed
-// the process. 137 is SIGKILL (timeout -s KILL), 124 is timeout(1) reporting
-// that it had to intervene.
+// dockerExecWasKilled reports whether an exit code means the process died to
+// a KILL-class signal: 137 is SIGKILL (which the in-container `timeout -s
+// KILL` wrapper sends, but so does an OOM kill), 124 is timeout(1) reporting
+// that it had to intervene. It is a fact about the process, not a verdict on
+// the cause — and it says nothing about the container: a re-inspect that
+// confirms the container is gone classifies ahead of it in
+// dockerFileOpFailure.
func dockerExecWasKilled(exitCode int) bool {
return exitCode == 137 || exitCode == 124
}
@@ -910,7 +914,7 @@ func (c *DockerRemoteClient) WriteFile(
return dockerError("WriteFile", err)
}
if result.ExitCode != 0 {
- return c.dockerFileOpFailed(ctx, id, "WriteFile", clean, result.Stderr, result.ExitCode)
+ return c.dockerFileOpFailure(ctx, id, "WriteFile", clean, result, RemoteErrorKindInvalidRequest)
}
return nil
}
@@ -945,7 +949,7 @@ func (c *DockerRemoteClient) ReadFile(
return nil, dockerError("ReadFile", err)
}
if result.ExitCode != 0 {
- return nil, c.dockerFileOpFailed(ctx, id, "ReadFile", clean, result.Stderr, result.ExitCode)
+ return nil, c.dockerFileOpFailure(ctx, id, "ReadFile", clean, result, RemoteErrorKindInvalidRequest)
}
return []byte(result.Stdout), nil
}
@@ -989,7 +993,7 @@ func (c *DockerRemoteClient) Stat(
return nil, dockerError("Stat", err)
}
if result.ExitCode != 0 {
- return nil, c.dockerFileOpFailed(ctx, id, "Stat", clean, result.Stderr, result.ExitCode)
+ return nil, c.dockerFileOpFailure(ctx, id, "Stat", clean, result, RemoteErrorKindInvalidRequest)
}
entries := parseDockerFindOutput(result.Stdout)
if len(entries) == 0 {
@@ -1008,23 +1012,26 @@ func (c *DockerRemoteClient) Stat(
}, nil
}
-// dockerFileOpError classifies a failed filesystem helper. A missing path is
-// NotFound so callers can treat it as "nothing there"; everything else,
-// permission denials included, is an invalid request carrying the tool's own
-// complaint rather than a synthesised one.
-// dockerFileOpFailed classifies a filesystem op's non-zero exec exit. The
-// message carries the exit code — with an empty stderr it used to be the
-// only missing clue (issue #3942). When the exec raced the idle sweeper's
-// delete (ensureRunning's inspect passes, the sweep removes the container,
-// the exec dies mid-flight), the failure surfaces as exit != 0 with an
-// empty stderr; the re-inspect reclassifies it as NotFound so
-// CanReplaceRemoteBinding lets the rebinding self-heal take over — making
-// the sweeper's "deleting needs no coordination with the binding store"
-// premise hold for this window too (issue #3942, mechanism A).
-func (c *DockerRemoteClient) dockerFileOpFailed(
- ctx context.Context, id, op, clean, stderr string, exitCode int,
+// dockerFileOpFailure classifies a filesystem op whose exec ran but exited
+// non-zero. The full result goes in so every filesystem op shares one fixed
+// classification order:
+//
+// 1. the tool's own missing-path complaint → NotFound;
+// 2. a re-inspect confirming the container is gone → NotFound, so the
+// rebinding self-heal CanReplaceRemoteBinding unlocks — the exec raced
+// the idle sweeper's delete (issue #3942, mechanism A);
+// 3. a killed exit (137 = SIGKILL, 124 = timeout(1) reporting) → Timeout
+// (retryable, binding preserved): a killed op means the sandbox
+// filesystem stalled, not that the caller asked for something invalid
+// (issue #3910);
+// 4. anything else → fallbackKind, carrying the exit code and the tool's
+// own complaint. ListDir passes Internal to keep its historical
+// classification; the other ops pass InvalidRequest.
+func (c *DockerRemoteClient) dockerFileOpFailure(
+ ctx context.Context, id, op, clean string,
+ result *RemoteExecResult, fallbackKind RemoteErrorKind,
) error {
- if strings.Contains(stderr, "No such file or directory") {
+ if strings.Contains(result.Stderr, "No such file or directory") {
return &RemoteError{
Kind: RemoteErrorKindNotFound,
Provider: SandboxTypeDocker,
@@ -1038,14 +1045,56 @@ func (c *DockerRemoteClient) dockerFileOpFailed(
Provider: SandboxTypeDocker,
Op: op,
Message: fmt.Sprintf("%s %s: exit=%d: container vanished mid-op (likely idle sweep)",
- op, clean, exitCode),
+ op, clean, result.ExitCode),
}
}
+ if result.Killed {
+ return dockerKilledOpFailure(op, clean, result)
+ }
+ detail := fmt.Sprintf("%s %s", op, clean)
+ if line := firstNonEmptyLine(result.Stderr); line != "" {
+ detail += fmt.Sprintf(": exit=%d: %s", result.ExitCode, line)
+ } else if line := firstNonEmptyLine(result.Stdout); line != "" {
+ detail += fmt.Sprintf(": exit=%d, stdout: %s", result.ExitCode, line)
+ } else {
+ // A tool that died without writing to stderr used to leave a
+ // dangling colon; say so and keep the exit code attributable.
+ detail += fmt.Sprintf(" (exit=%d, no stderr)", result.ExitCode)
+ }
return &RemoteError{
- Kind: RemoteErrorKindInvalidRequest,
+ Kind: fallbackKind,
Provider: SandboxTypeDocker,
Op: op,
- Message: fmt.Sprintf("%s %s: exit=%d: %s", op, clean, exitCode, firstNonEmptyLine(stderr)),
+ Message: detail,
+ }
+}
+
+// dockerKilledOpFailure builds the Timeout error for a killed filesystem op.
+//
+// The exit code only says the process died to a KILL-class signal — 137 is
+// SIGKILL (which the in-container `timeout -s KILL` wrapper sends, but OOM
+// and other kills do too) and 124 is timeout(1) reporting that it
+// intervened — so the message reports the facts (exit code, elapsed, output)
+// and names the configured budget without asserting it as the cause: an
+// elapsed close to the budget is a clue, not a verdict.
+func dockerKilledOpFailure(op, clean string, result *RemoteExecResult) error {
+ detail := fmt.Sprintf(
+ "%s %s: killed after %s, exit=%d (filesystem-op exec budget %s; "+
+ "a near-budget elapsed points at the timeout wrapper, but the exit code alone cannot prove the cause)",
+ op, clean, result.Duration.Round(time.Millisecond), result.ExitCode, dockerFilesystemOpTimeout,
+ )
+ if line := firstNonEmptyLine(result.Stderr); line != "" {
+ detail += ", stderr: " + line
+ } else if line := firstNonEmptyLine(result.Stdout); line != "" {
+ detail += ", stdout: " + line
+ } else {
+ detail += ", no output"
+ }
+ return &RemoteError{
+ Kind: RemoteErrorKindTimeout,
+ Provider: SandboxTypeDocker,
+ Op: op,
+ Message: detail,
}
}
@@ -1103,7 +1152,7 @@ func (c *DockerRemoteClient) makeDir(ctx context.Context, id, dir, op string) er
return dockerError(op, err)
}
if result.ExitCode != 0 {
- return c.dockerFileOpFailed(ctx, id, op, dir, result.Stderr, result.ExitCode)
+ return c.dockerFileOpFailure(ctx, id, op, dir, result, RemoteErrorKindInvalidRequest)
}
return nil
}
@@ -1135,7 +1184,7 @@ func (c *DockerRemoteClient) Remove(
return dockerError("Remove", err)
}
if result.ExitCode != 0 {
- return c.dockerFileOpFailed(ctx, id, "Remove", clean, result.Stderr, result.ExitCode)
+ return c.dockerFileOpFailure(ctx, id, "Remove", clean, result, RemoteErrorKindInvalidRequest)
}
return nil
}
@@ -1171,29 +1220,10 @@ func (c *DockerRemoteClient) ListDir(
return nil, dockerError("ListDir", err)
}
if result.ExitCode != 0 {
- if strings.Contains(result.Stderr, "No such file or directory") {
- return nil, &RemoteError{
- Kind: RemoteErrorKindNotFound,
- Provider: SandboxTypeDocker,
- Op: "ListDir",
- Message: clean + " does not exist",
- }
- }
- if c.containerVanished(ctx, id, "ListDir") {
- return nil, &RemoteError{
- Kind: RemoteErrorKindNotFound,
- Provider: SandboxTypeDocker,
- Op: "ListDir",
- Message: fmt.Sprintf("ListDir %s: exit=%d: container vanished mid-op (likely idle sweep)",
- clean, result.ExitCode),
- }
- }
- return nil, &RemoteError{
- Kind: RemoteErrorKindInternal,
- Provider: SandboxTypeDocker,
- Op: "ListDir",
- Message: fmt.Sprintf("find %s: %s", clean, firstNonEmptyLine(result.Stderr)),
- }
+ // The shared classifier keeps ListDir's historical Internal
+ // classification for genuine find failures; killed ops still map
+ // to Timeout like every other filesystem op.
+ return nil, c.dockerFileOpFailure(ctx, id, "ListDir", clean, result, RemoteErrorKindInternal)
}
return parseDockerFindOutput(result.Stdout), nil
}
diff --git a/internal/sandbox/docker_remote_client_test.go b/internal/sandbox/docker_remote_client_test.go
index 6c17d088d..d4d9dec2e 100644
--- a/internal/sandbox/docker_remote_client_test.go
+++ b/internal/sandbox/docker_remote_client_test.go
@@ -1329,11 +1329,72 @@ func TestDockerExecObservesBothOutputStreamsWithoutChangingResult(t *testing.T)
require.Equal(t, result.Stderr, observed["stderr"])
}
-// The idle sweeper can delete a container between ensureRunning's inspect
-// gate and the exec's completion (#3942 mechanism A): the exec dies with a
-// non-zero exit and an empty stderr. The filesystem ops must reclassify
-// that collision as NotFound so CanReplaceRemoteBinding lets the rebinding
-// self-heal run instead of surfacing InvalidRequest to the user.
+// A filesystem op killed by the in-container `timeout -s KILL` wrapper (issue
+// #3910) leaves no stderr: it must classify as Timeout with the evidence in
+// the message, not as invalid_request with a dangling colon.
+func TestDockerFileOpKilledByTimeoutClassifiesAsTimeout(t *testing.T) {
+ engine := newFakeDockerEngine()
+ engine.execExit = 137
+ engine.execStderr = ""
+ // The container must survive the re-inspect: a confirmed-missing
+ // container classifies ahead of the killed exit code.
+ engine.inspect["c"] = container.InspectResponse{
+ ID: "c",
+ State: &container.State{Status: "running"},
+ }
+ docker := newTestDockerClient(t, engine)
+
+ err := docker.MakeDir(context.Background(), testHandle("c"), "/workspace/input/94088e061da2")
+ require.Error(t, err)
+ require.True(t, IsRemoteTimeout(err), "killed op must be Timeout: %q", err.Error())
+ require.False(t, IsRemoteInvalidRequest(err), "killed op is not the caller's fault")
+ require.Contains(t, err.Error(), "killed after")
+ require.Contains(t, err.Error(), "exit=137")
+ require.Contains(t, err.Error(), "no output")
+ // The budget is evidence, not a verdict: the message must present it as
+ // a configured budget instead of asserting the wrapper fired.
+ require.Contains(t, err.Error(), "filesystem-op exec budget")
+ require.NotContains(t, err.Error(), "filesystem-op timeout)", "must not assert the cause of the kill")
+ require.NotRegexp(t, `: $`, err.Error(), "no dangling colon: %q", err.Error())
+}
+
+// A genuine tool failure keeps its stderr complaint, and a failure with no
+// stderr at least carries the exit code instead of ending in a bare colon.
+func TestDockerFileOpExitErrorsCarryEvidence(t *testing.T) {
+ denied := newFakeDockerEngine()
+ denied.execExit = 1
+ denied.execStderr = "mkdir: /workspace/input: Permission denied"
+ denied.inspect["c"] = container.InspectResponse{
+ ID: "c",
+ State: &container.State{Status: "running"},
+ }
+ docker := newTestDockerClient(t, denied)
+
+ err := docker.MakeDir(context.Background(), testHandle("c"), "/workspace/input/x")
+ require.Error(t, err)
+ require.True(t, IsRemoteInvalidRequest(err))
+ require.Contains(t, err.Error(), "Permission denied")
+
+ silent := newFakeDockerEngine()
+ silent.execExit = 1
+ silent.execStderr = ""
+ silent.inspect["c"] = container.InspectResponse{
+ ID: "c",
+ State: &container.State{Status: "running"},
+ }
+ docker = newTestDockerClient(t, silent)
+
+ err = docker.MakeDir(context.Background(), testHandle("c"), "/workspace/input/x")
+ require.Error(t, err)
+ require.True(t, IsRemoteInvalidRequest(err))
+ require.Contains(t, err.Error(), "(exit=1, no stderr)",
+ "silent failure must say so instead of a dangling colon: %q", err.Error())
+}
+
+// A filesystem op that raced the idle sweeper's delete (#3942 mechanism A)
+// surfaces as a non-zero exit with an empty stderr; the re-inspect confirms
+// the container is gone and reclassifies the failure as NotFound so the
+// rebinding self-heal CanReplaceRemoteBinding unlocks.
func TestFileOpAgainstVanishedContainerRebinds(t *testing.T) {
engine := newFakeDockerEngine()
engine.execExit = 1
@@ -1408,3 +1469,63 @@ func TestFileOpGenuineFailureKeepsBinding(t *testing.T) {
require.True(t, IsRemoteInvalidRequest(err), "got %+v", err)
require.False(t, CanReplaceRemoteBinding(err))
}
+
+// The classification order is fixed: a killed exit code does not outrank a
+// re-inspect that confirms the container is gone. exit=137 with the container
+// vanished is NotFound (so the rebinding self-heal can take over), not
+// Timeout — for MakeDir and ListDir alike.
+func TestDockerFileOpKilledWithVanishedContainerIsNotFound(t *testing.T) {
+ engine := newFakeDockerEngine()
+ engine.execExit = 137
+ engine.execStderr = ""
+ docker := newTestDockerClient(t, engine)
+
+ err := docker.MakeDir(context.Background(), testHandle("swept-away"), "/workspace/input/x")
+ require.Error(t, err)
+ require.True(t, IsRemoteNotFound(err), "got %+v", err)
+ require.False(t, IsRemoteTimeout(err),
+ "a confirmed-missing container classifies ahead of the killed exit: %q", err.Error())
+ require.True(t, CanReplaceRemoteBinding(err))
+
+ _, err = docker.ListDir(context.Background(), testHandle("swept-away"), "/workspace/input")
+ require.Error(t, err)
+ require.True(t, IsRemoteNotFound(err), "got %+v", err)
+ require.False(t, IsRemoteTimeout(err),
+ "ListDir must follow the same order: %q", err.Error())
+ require.True(t, CanReplaceRemoteBinding(err))
+}
+
+// A killed ListDir with the container still alive maps to Timeout like every
+// other filesystem op, while a genuine find failure keeps ListDir's
+// historical Internal classification.
+func TestDockerListDirKilledIsTimeoutButGenuineFailureStaysInternal(t *testing.T) {
+ killed := newFakeDockerEngine()
+ killed.execExit = 137
+ killed.execStderr = ""
+ killed.inspect["c"] = container.InspectResponse{
+ ID: "c",
+ State: &container.State{Status: "running"},
+ }
+ docker := newTestDockerClient(t, killed)
+
+ _, err := docker.ListDir(context.Background(), testHandle("c"), "/workspace/output")
+ require.Error(t, err)
+ require.True(t, IsRemoteTimeout(err), "got %+v", err)
+ require.False(t, CanReplaceRemoteBinding(err), "the container is alive: the binding must stay")
+
+ genuine := newFakeDockerEngine()
+ genuine.execExit = 2
+ genuine.execStderr = "find: unrecognized option"
+ genuine.inspect["c"] = container.InspectResponse{
+ ID: "c",
+ State: &container.State{Status: "running"},
+ }
+ docker = newTestDockerClient(t, genuine)
+
+ _, err = docker.ListDir(context.Background(), testHandle("c"), "/workspace/output")
+ require.Error(t, err)
+ var remote *RemoteError
+ require.True(t, errors.As(err, &remote), "got %+v", err)
+ require.Equal(t, RemoteErrorKindInternal, remote.Kind,
+ "ListDir's genuine failures keep their historical classification: %q", err.Error())
+}
diff --git a/internal/sandbox/issue3910_repro_integration_test.go b/internal/sandbox/issue3910_repro_integration_test.go
new file mode 100644
index 000000000..b38387ddb
--- /dev/null
+++ b/internal/sandbox/issue3910_repro_integration_test.go
@@ -0,0 +1,106 @@
+//go:build docker_integration
+
+// Reproduction + regression for GitHub issue #3910.
+//
+// Issue: calling the agent-chat API failed with
+//
+// restore session attachments into sandbox: stage attachment "FAQ知识库.txt":
+// sandbox: create input directory: docker MakeDir: invalid_request:
+// mkdir -p /workspace/input/94088e061da2:
+//
+// Mechanism under test: every docker filesystem op is wrapped in an
+// in-container `timeout -s KILL <N>` (dockerExecCommand). A filesystem op
+// that hangs past dockerFilesystemOpTimeout is SIGKILLed — exit != 0 and,
+// by definition of SIGKILL, no stderr at all. Pre-fix makeDir classifies
+// that as invalid_request with the (empty) stderr as the whole explanation.
+//
+// The test makes `mkdir` hang deterministically by shadowing it with a
+// sleeping script earlier in PATH (/usr/local/bin precedes /usr/bin on the
+// sandbox image), so the in-container timeout wrapper kills it exactly the
+// way it killed the reporter's mkdir.
+//
+// Run like the rest of the conformance suite:
+//
+// DOCKER_INTEGRATION_IMAGE=wechatopenai/weknora-sandbox:dev \
+// go test -tags=docker_integration ./internal/sandbox \
+// -run '^TestDocker3910' -count=1 -v -timeout=15m
+package sandbox
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestDocker3910KilledMakeDirIntegration(t *testing.T) {
+ cfg := dockerIntegrationConfig(t)
+ docker, err := NewDockerRemoteClient(cfg)
+ require.NoError(t, err)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+ defer cancel()
+
+ handle, err := docker.Create(ctx, RemoteCreateRequest{
+ TemplateID: cfg.DockerImage,
+ Metadata: map[string]string{"weknora.test": "issue-3910-repro"},
+ })
+ require.NoError(t, err, "create sandbox container")
+ t.Cleanup(func() {
+ cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), time.Minute)
+ defer cleanupCancel()
+ _ = docker.Delete(cleanupCtx, handle.ID())
+ })
+
+ // Sanity: MakeDir works before the sabotage.
+ require.NoError(t, docker.MakeDir(ctx, handle, "/workspace/input/sane"),
+ "MakeDir must succeed on a healthy container")
+
+ // Shadow mkdir with a script that hangs (120s >> the 30s filesystem-op
+ // timeout). /usr/local/bin precedes /usr/bin in PATH, so the in-container
+ // `timeout -s KILL 30 mkdir -p ...` resolves to the sleeper.
+ sabotage, err := docker.Exec(ctx, handle, RemoteExecRequest{
+ Command: "sh",
+ Args: []string{"-c",
+ `printf '#!/bin/sh\nsleep 120\n' > /usr/local/bin/mkdir && chmod +x /usr/local/bin/mkdir`},
+ User: "root",
+ Timeout: 30 * time.Second,
+ })
+ require.NoError(t, err, "sabotage exec failed")
+ require.Equal(t, 0, sabotage.ExitCode, "sabotage script failed: %s", sabotage.Stderr)
+
+ start := time.Now()
+ err = docker.MakeDir(ctx, handle, "/workspace/input/94088e061da2")
+ elapsed := time.Since(start)
+ require.Error(t, err, "MakeDir against a hung mkdir must fail")
+ t.Logf("MakeDir failed after %s with: %q", elapsed.Round(time.Millisecond), err.Error())
+
+ // Fixed behavior: the SIGKILLed filesystem op is a Timeout (retryable,
+ // binding preserved), not an invalid_request, and the message carries the
+ // evidence a killed process leaves behind — exit code and elapsed time —
+ // instead of the dangling colon from #3910.
+ require.False(t, IsRemoteInvalidRequest(err),
+ "a killed filesystem op is not the caller's fault: got %q", err.Error())
+ require.True(t, IsRemoteTimeout(err),
+ "a killed filesystem op must classify as Timeout: got %q", err.Error())
+ require.Contains(t, err.Error(), "killed after",
+ "message must say the op was killed: %q", err.Error())
+ require.Regexp(t, `exit=1(24|37)`, err.Error(),
+ "message must carry the exit code: %q", err.Error())
+ require.NotRegexp(t, `: $`, err.Error(),
+ "message must not end in a dangling colon: %q", err.Error())
+
+ // The failed MakeDir must not poison the sandbox: a subsequent healthy
+ // operation still works (the binding was preserved, not dropped).
+ recovered, err := docker.Exec(ctx, handle, RemoteExecRequest{
+ Command: "sh",
+ Args: []string{"-c", `rm -f /usr/local/bin/mkdir`},
+ User: "root",
+ Timeout: 30 * time.Second,
+ })
+ require.NoError(t, err)
+ require.Equal(t, 0, recovered.ExitCode)
+ require.NoError(t, docker.MakeDir(ctx, handle, "/workspace/input/recovered"),
+ "sandbox must stay usable after a killed filesystem op")
+}
diff --git a/internal/sandbox/remote_errors.go b/internal/sandbox/remote_errors.go
index a3ee10d2f..917e62b5c 100644
--- a/internal/sandbox/remote_errors.go
+++ b/internal/sandbox/remote_errors.go
@@ -190,6 +190,14 @@ func IsRemoteConflict(err error) bool {
return remoteKind(err) == RemoteErrorKindConflict
}
+// IsRemoteTimeout reports whether err classifies as a deadline the op exceeded
+// (in-container kill wrapper, transport deadline). The caller did nothing
+// wrong and the binding is still valid: such failures may be retried, and a
+// transient provider stall usually clears on the next attempt.
+func IsRemoteTimeout(err error) bool {
+ return remoteKind(err) == RemoteErrorKindTimeout
+}
+
// snapshotDeleteKind reclassifies a snapshot/template delete that failed
// because sandboxes still reference it. E2B returns that as HTTP 400
// invalid_request; it is Conflict: the caller did nothing wrong and should