Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(sandbox): recover timed-out filesystem operations and staged attachments (#3921)
c56425b151d8f
diff --git a/internal/application/service/session_attachment_staging.go b/internal/application/service/session_attachment_staging.goindex fc4d88694..06670d091 100644--- a/internal/application/service/session_attachment_staging.go+++ b/internal/application/service/session_attachment_staging.go@@ -8,6 +8,7 @@ import ("path""sort""strings"+ "time""github.com/Tencent/WeKnora/internal/application/repository""github.com/Tencent/WeKnora/internal/sandbox"@@ -151,6 +152,10 @@ func (s *agentService) stageSessionAttachments(desired := make(map[string]struct{}, len(attachments))var assets []stringstaged := make([]stagedSessionAttachment, 0, len(attachments))+ inputs := make([]stagedInput, 0, len(attachments))+ // A timeout retry may move the session to a new container. The initial+ // directory snapshot cannot then establish that later inputs still exist.+ reuseExisting := truemaxBytes := int64(secutils.GetMaxFileSizeMB()) * 1024 * 1024for _, attachment := range attachments {remotePath, pathErr := sandboxAttachmentPath(attachment, layout.InputDir)@@ -161,37 +166,60 @@ func (s *agentService) stageSessionAttachments(assets = append(assets, remotePath+attachmentAssetsSuffix)entry, present := existing[remotePath]- if !present || attachment.FileSize <= 0 || entry.Size != attachment.FileSize {- reader, getErr := s.fileService.GetFile(ctx, attachment.URL)- if getErr != nil {- return nil, fmt.Errorf("open attachment %q: %w", attachment.FileName, getErr)- }- content, readErr := io.ReadAll(io.LimitReader(reader, maxBytes+1))- closeErr := reader.Close()+ if !reuseExisting || !present || attachment.FileSize <= 0 || entry.Size != attachment.FileSize {+ content, readErr := s.readAttachmentContent(ctx, attachment, maxBytes)if readErr != nil {- return nil, fmt.Errorf("read attachment %q: %w", attachment.FileName, readErr)- }- if closeErr != nil {- return nil, fmt.Errorf("close attachment %q: %w", attachment.FileName, closeErr)- }- if int64(len(content)) > maxBytes {- return nil, fmt.Errorf("attachment %q exceeds sandbox staging limit of %d bytes", attachment.FileName, maxBytes)+ return nil, readErr}- if writeErr := store.WriteSessionInputFile(ctx, sessionID, remotePath, content); writeErr != nil {+ rebound, writeErr := writeSessionInputWithRetry(ctx, store, sessionID, remotePath, content)+ if writeErr != nil {return nil, fmt.Errorf("stage attachment %q: %w", attachment.FileName, writeErr)}attachment.FileSize = int64(len(content))+ inputs = append(inputs, stagedInput{+ remotePath: remotePath,+ name: attachment.FileName,+ url: attachment.URL,+ size: attachment.FileSize,+ })+ if rebound {+ reuseExisting = false+ // The retry re-resolved the session, which may have+ // rebound it onto a fresh container: everything staged+ // before this attachment lived in the old one.+ if restoreErr := s.reestablishStagedInputs(+ ctx, store, sessionID, layout.InputDir, inputs, maxBytes,+ ); restoreErr != nil {+ return nil, restoreErr+ }+ }+ } else {+ // Reused from the previous pass: no content kept in memory,+ // but the rebinding recovery must still be able to+ // re-create it from durable storage.+ inputs = append(inputs, stagedInput{+ remotePath: remotePath,+ name: attachment.FileName,+ url: attachment.URL,+ size: attachment.FileSize,+ })}parsedPath := ""if text, ok := parsed[strings.TrimSpace(attachment.URL)]; ok {parsedPath = remotePath + parsedSidecarSuffixdesired[parsedPath] = struct{}{}- if entry, present := existing[parsedPath]; !present || entry.Size != int64(len(text)) {+ if entry, present := existing[parsedPath]; !reuseExisting || !present || entry.Size != int64(len(text)) {if writeErr := store.WriteSessionInputFile(ctx, sessionID, parsedPath, []byte(text)); writeErr != nil {return nil, fmt.Errorf("stage parsed text of %q: %w", attachment.FileName, writeErr)}}+ inputs = append(inputs, stagedInput{+ remotePath: parsedPath,+ name: attachment.FileName + parsedSidecarSuffix,+ content: []byte(text),+ size: int64(len(text)),+ })}staged = append(staged, stagedSessionAttachment{@@ -234,6 +262,149 @@ func (s *agentService) stageSessionAttachments(return staged, nil}+// sandboxStagingRetryDelay separates the two attempts a transient sandbox+// filesystem stall gets before staging fails the turn.+const sandboxStagingRetryDelay = 2 * time.Second++// sandboxStagingRebindPasses bounds how many times the rebinding recovery+// below re-verifies the input set after a retry landed on yet another fresh+// container. A session whose inputs keep getting killed this many times is+// not going to heal; the error says so instead of looping forever.+const sandboxStagingRebindPasses = 3++// stagedInput tracks one attachment staged during this pass so the rebinding+// recovery can re-establish what an earlier container already held. Original+// files are re-read from durable storage on recovery rather than retaining+// every attachment in memory. Only derived text sidecars carry content.+type stagedInput struct {+ remotePath string+ name string+ url string+ content []byte+ size int64+}++// readAttachmentContent reads one attachment from durable storage, bounded by+// the sandbox staging limit.+func (s *agentService) readAttachmentContent(+ ctx context.Context,+ attachment types.MessageAttachment,+ maxBytes int64,+) ([]byte, error) {+ reader, getErr := s.fileService.GetFile(ctx, attachment.URL)+ if getErr != nil {+ return nil, fmt.Errorf("open attachment %q: %w", attachment.FileName, getErr)+ }+ content, readErr := io.ReadAll(io.LimitReader(reader, maxBytes+1))+ closeErr := reader.Close()+ if readErr != nil {+ return nil, fmt.Errorf("read attachment %q: %w", attachment.FileName, readErr)+ }+ if closeErr != nil {+ return nil, fmt.Errorf("close attachment %q: %w", attachment.FileName, closeErr)+ }+ if int64(len(content)) > maxBytes {+ return nil, fmt.Errorf("attachment %q exceeds sandbox staging limit of %d bytes", attachment.FileName, maxBytes)+ }+ return content, nil+}++// reestablishStagedInputs re-writes staged inputs that went missing after a+// timeout retry rebound the session onto a fresh container: the retry's+// WriteSessionInputFile re-resolves the session, and the new container's+// input directory does not carry anything the old one held. Without this+// pass, staging would report success while the sandbox only has the one+// attachment whose retry happened to land.+//+// Each pass re-lists the input directory and rewrites whatever is missing or+// has the wrong size. A rewrite that itself only succeeds after a timeout+// retry may have rebound onto yet another container, so the pass repeats+// until a full pass lands without a rebind (or the pass budget runs out).+func (s *agentService) reestablishStagedInputs(+ ctx context.Context,+ store sandbox.SessionFileStore,+ sessionID, inputDir string,+ inputs []stagedInput,+ maxBytes int64,+) error {+ for pass := 0; pass < sandboxStagingRebindPasses; pass++ {+ entries, listErr := store.ListSessionFiles(ctx, sessionID, inputDir)+ if listErr != nil {+ return fmt.Errorf("relist staged session inputs after filesystem-op timeout: %w", listErr)+ }+ present := make(map[string]int64, len(entries))+ for _, entry := range entries {+ present[path.Clean(entry.Path)] = entry.Size+ }+ reboundAgain := false+ for _, input := range inputs {+ if size, ok := present[input.remotePath]; ok && size == input.size {+ continue+ }+ content := input.content+ if content == nil {+ var readErr error+ content, readErr = s.readAttachmentContent(ctx, types.MessageAttachment{+ URL: input.url,+ FileName: input.name,+ }, maxBytes)+ if readErr != nil {+ return fmt.Errorf("re-stage attachment %q: %w", input.name, readErr)+ }+ if int64(len(content)) != input.size {+ return fmt.Errorf(+ "re-stage attachment %q: durable content is %d bytes, expected %d",+ input.name, len(content), input.size)+ }+ }+ rebound, writeErr := writeSessionInputWithRetry(ctx, store, sessionID, input.remotePath, content)+ if writeErr != nil {+ return fmt.Errorf("re-stage attachment %q after filesystem-op timeout: %w", input.name, writeErr)+ }+ reboundAgain = reboundAgain || rebound+ }+ if !reboundAgain {+ return nil+ }+ }+ return fmt.Errorf(+ "session inputs kept getting killed past the filesystem-op timeout across %d rebinding passes",+ sandboxStagingRebindPasses)+}++// writeSessionInputWithRetry retries once when the sandbox killed the write+// past its filesystem-op timeout (issue #3910): a killed op means the+// container filesystem stalled — the request itself was valid, and such+// stalls are usually transient. Any other failure surfaces immediately. When+// the retry also times out, both failures stay attached to the returned error+// so the log shows it was not a one-off.+//+// The returned rebound flag reports that the write only succeeded on the+// retry: WriteSessionInputFile re-resolves the session, so the second attempt+// may have landed in a fresh container. Callers staging more than one input+// must treat a rebound as "everything staged before this write may be gone"+// and re-establish it.+func writeSessionInputWithRetry(+ ctx context.Context,+ store sandbox.SessionFileStore,+ sessionID, remotePath string,+ content []byte,+) (bool, error) {+ err := store.WriteSessionInputFile(ctx, sessionID, remotePath, content)+ if !sandbox.IsRemoteTimeout(err) {+ return false, err+ }+ select {+ case <-ctx.Done():+ return false, err+ case <-time.After(sandboxStagingRetryDelay):+ }+ if retryErr := store.WriteSessionInputFile(ctx, sessionID, remotePath, content); retryErr != nil {+ return false, fmt.Errorf("%w (retry after filesystem-op timeout failed: %v)", err, retryErr)+ }+ return true, nil+}+// resolveSessionAttachmentURLs fills in the storage handle for attachments// whose URL was not persisted on the message row. MessageAttachment.URL is// deliberately excluded from DB serialization (json:"-") so a cross-sessiondiff --git a/internal/application/service/session_attachment_staging_test.go b/internal/application/service/session_attachment_staging_test.goindex c2ff61db4..2f3298050 100644--- a/internal/application/service/session_attachment_staging_test.go+++ b/internal/application/service/session_attachment_staging_test.go@@ -613,3 +613,268 @@ func TestStageSessionAttachmentsKeepsRemoteInputRoot(t *testing.T) {require.Equal(t, sandbox.SessionInputRoot, store.listedDir)require.True(t, strings.HasPrefix(staged[0].Path, sandbox.SessionInputRoot))}++// retryTestStore fails WriteSessionInputFile per the hook so the one-retry+// policy on killed filesystem ops (issue #3910) can be driven without a+// sandbox. Only WriteSessionInputFile is reachable in these tests.+type retryTestStore struct {+ sandbox.SessionFileStore+ onWrite func() error+}++func (s *retryTestStore) WriteSessionInputFile(context.Context, string, string, []byte) error {+ return s.onWrite()+}++func TestWriteSessionInputWithRetryRetriesKilledOpOnce(t *testing.T) {+ calls := 0+ store := &retryTestStore{onWrite: func() error {+ calls+++ if calls == 1 {+ return sandbox.NewRemoteError(+ sandbox.SandboxTypeDocker, "MakeDir", sandbox.RemoteErrorKindTimeout,+ "killed after 30s (filesystem-op exec budget 30s), exit=137, no output", nil)+ }+ return nil+ }}+ rebound, err := writeSessionInputWithRetry(+ context.Background(), store, "session-1", "/workspace/input/abc/faq.txt", []byte("hi"))+ require.NoError(t, err)+ require.True(t, rebound, "a write that only landed on the retry must be reported as a possible rebind")+ require.Equal(t, 2, calls, "a killed op must be retried exactly once")+}++func TestWriteSessionInputWithRetrySurfacesOtherFailuresImmediately(t *testing.T) {+ calls := 0+ store := &retryTestStore{onWrite: func() error {+ calls+++ return sandbox.NewRemoteError(+ sandbox.SandboxTypeDocker, "MakeDir", sandbox.RemoteErrorKindInvalidRequest,+ "MakeDir /workspace/input: Permission denied", nil)+ }}+ rebound, err := writeSessionInputWithRetry(+ context.Background(), store, "session-1", "/workspace/input/abc/faq.txt", []byte("hi"))+ require.Error(t, err)+ require.False(t, rebound)+ require.Equal(t, 1, calls, "a non-timeout failure must not be retried")+}++func TestWriteSessionInputWithRetryRetryFailureKeepsBothErrors(t *testing.T) {+ calls := 0+ store := &retryTestStore{onWrite: func() error {+ calls+++ return sandbox.NewRemoteError(+ sandbox.SandboxTypeDocker, "MakeDir", sandbox.RemoteErrorKindTimeout,+ "killed after 30s, exit=137, no output", nil)+ }}+ rebound, err := writeSessionInputWithRetry(+ context.Background(), store, "session-1", "/workspace/input/abc/faq.txt", []byte("hi"))+ require.Error(t, err)+ require.False(t, rebound)+ require.Equal(t, 2, calls)+ require.Contains(t, err.Error(), "retry after filesystem-op timeout failed",+ "both the original kill and the retry failure must stay in the log")+}++// rebindingInputStore simulates the sequence the one-file retry cannot heal+// on its own: the path in slowPath gets its first WriteSessionInputFile+// killed past the filesystem-op timeout, and the retry — which re-resolves+// the session — lands in a fresh container whose input directory does not+// hold anything staged before the kill.+type rebindingInputStore struct {+ stagingSandboxManager+ slowPath string+ killedOnce bool+ reboundOnce bool+}++// SessionFileStore must return the outer type: the embedded copy would hand+// staging the base WriteSessionInputFile and this test's rebind override+// would never run.+func (s *rebindingInputStore) SessionFileStore() sandbox.SessionFileStore {+ return s+}++func (s *rebindingInputStore) WriteSessionInputFile(_ context.Context, _, filePath string, content []byte) error {+ if filePath == s.slowPath && !s.killedOnce {+ s.killedOnce = true+ return sandbox.NewRemoteError(+ sandbox.SandboxTypeDocker, "MakeDir", sandbox.RemoteErrorKindTimeout,+ "MakeDir /workspace/input: killed after 30s, exit=137, no output", nil)+ }+ if filePath == s.slowPath && s.killedOnce && !s.reboundOnce {+ // The retry re-resolved the session onto a fresh container: the+ // inputs the old container held went away with it.+ s.reboundOnce = true+ s.files = make(map[string][]byte)+ }+ if s.files == nil {+ s.files = make(map[string][]byte)+ }+ s.files[filePath] = append([]byte(nil), content...)+ s.writes = append(s.writes, filePath)+ return nil+}++// The first attachment is staged into container 1; the second attachment's+// first write is killed, and its retry lands in a fresh container. Staging+// must re-establish the first attachment there before reporting success —+// without the recovery pass, the sandbox would end up holding only the+// attachment whose retry happened to land.+func TestStageSessionAttachmentsRestoresEarlierInputsAfterRebind(t *testing.T) {+ first := types.MessageAttachment{+ URL: "local://tenant/attachment-1",+ FileName: "report.pdf",+ FileType: ".pdf",+ }+ second := types.MessageAttachment{+ URL: "local://tenant/attachment-2",+ FileName: "notes.txt",+ FileType: ".txt",+ }+ layout := sandbox.RemoteWorkspaceLayout()+ remoteFirst, err := sandboxAttachmentPath(first, layout.InputDir)+ require.NoError(t, err)+ remoteSecond, err := sandboxAttachmentPath(second, layout.InputDir)+ require.NoError(t, err)++ store := &rebindingInputStore{slowPath: remoteSecond}+ fileService := &stagingFileService{+ files: map[string][]byte{+ first.URL: []byte("content-first"),+ second.URL: []byte("content-second"),+ },+ }+ service := &agentService{+ sandboxMgr: store,+ fileService: fileService,+ sandboxResolver: stubSandboxResolver{mgr: store},+ }+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7))++ staged, err := service.stageSessionAttachments(+ ctx, "session-1", "cfg-remote", 7,+ types.MessageAttachments{first, second}, layout,+ )++ require.NoError(t, err)+ require.Len(t, staged, 2)+ require.Equal(t, []byte("content-first"), store.files[remoteFirst],+ "the attachment staged before the killed write must be re-established in the fresh container")+ require.Equal(t, []byte("content-second"), store.files[remoteSecond])+ require.Equal(t, []string{remoteFirst, remoteSecond, remoteFirst}, store.writes,+ "write order: first attachment, killed attachment's retry, then the re-established first attachment")+ // Recovery re-reads the earlier original instead of retaining every+ // attachment in memory for the entire staging pass.+ require.Equal(t, 2, fileService.getCalls[first.URL])+ require.Equal(t, 1, fileService.getCalls[second.URL])+}++// Same sequence, but the earlier attachment was reused from a previous pass+// (present with a matching size, so its content was never read). The+// recovery must re-read it from durable storage and stage it into the fresh+// container.+func TestStageSessionAttachmentsRestoresReusedInputsAfterRebind(t *testing.T) {+ first := types.MessageAttachment{+ URL: "local://tenant/attachment-1",+ FileName: "report.pdf",+ FileType: ".pdf",+ FileSize: int64(len("content-first")),+ }+ second := types.MessageAttachment{+ URL: "local://tenant/attachment-2",+ FileName: "notes.txt",+ FileType: ".txt",+ }+ layout := sandbox.RemoteWorkspaceLayout()+ remoteFirst, err := sandboxAttachmentPath(first, layout.InputDir)+ require.NoError(t, err)+ remoteSecond, err := sandboxAttachmentPath(second, layout.InputDir)+ require.NoError(t, err)++ store := &rebindingInputStore{+ slowPath: remoteSecond,+ stagingSandboxManager: stagingSandboxManager{+ sandboxType: sandbox.SandboxTypeCube,+ files: map[string][]byte{remoteFirst: []byte("content-first")},+ },+ }+ fileService := &stagingFileService{+ files: map[string][]byte{+ first.URL: []byte("content-first"),+ second.URL: []byte("content-second"),+ },+ }+ service := &agentService{+ sandboxMgr: store,+ fileService: fileService,+ sandboxResolver: stubSandboxResolver{mgr: store},+ }+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7))++ staged, err := service.stageSessionAttachments(+ ctx, "session-1", "cfg-remote", 7,+ types.MessageAttachments{first, second}, layout,+ )++ require.NoError(t, err)+ require.Len(t, staged, 2)+ require.Equal(t, []byte("content-first"), store.files[remoteFirst],+ "the reused attachment must be re-established after the rebind dropped it")+ require.Equal(t, []byte("content-second"), store.files[remoteSecond])+ require.Equal(t, 1, fileService.getCalls[first.URL],+ "the reused attachment's content must be re-read from durable storage during the recovery")+}++func TestStageSessionAttachmentsRebindRestoresSidecarsAndLaterInputs(t *testing.T) {+ for _, sidecarPresent := range []bool{false, true} {+ name := "new-sidecar"+ if sidecarPresent {+ name = "reused-sidecar"+ }+ t.Run(name, func(t *testing.T) {+ db := stagingTempDocDB(t)+ attachments := types.MessageAttachments{+ {ID: "parsed-doc", URL: "local://first", FileName: "report.pdf", FileType: ".pdf", FileSize: 3},+ {URL: "local://second", FileName: "notes.txt", FileType: ".txt", FileSize: 3},+ {URL: "local://third", FileName: "later.txt", FileType: ".txt", FileSize: 3},+ }+ require.NoError(t, db.Create(&types.TemporaryDocument{+ ID: "parsed-doc", TenantID: 7, SessionID: "session-1", ResourceRef: attachments[0].URL,+ FileName: "report.pdf", FileType: ".pdf", FileSize: 3,+ Status: types.TemporaryDocumentStatusReady, Content: "extracted report",+ ExpiresAt: time.Now().Add(time.Hour),+ }).Error)+ layout := sandbox.RemoteWorkspaceLayout()+ paths := make([]string, len(attachments))+ for i, attachment := range attachments {+ var err error+ paths[i], err = sandboxAttachmentPath(attachment, layout.InputDir)+ require.NoError(t, err)+ }+ store := &rebindingInputStore{+ slowPath: paths[1],+ stagingSandboxManager: stagingSandboxManager{+ files: map[string][]byte{paths[0]: []byte("one"), paths[2]: []byte("tri")},+ },+ }+ if sidecarPresent {+ store.files[paths[0]+parsedSidecarSuffix] = []byte("extracted report")+ }+ fileService := &stagingFileService{files: map[string][]byte{+ attachments[0].URL: []byte("one"), attachments[1].URL: []byte("two"), attachments[2].URL: []byte("tri"),+ }}+ svc := &agentService{+ db: db, sandboxMgr: store, fileService: fileService, sandboxResolver: stubSandboxResolver{mgr: store},+ }+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7))+ staged, err := svc.stageSessionAttachments(ctx, "session-1", "cfg-remote", 7, attachments, layout)+ require.NoError(t, err)+ require.Len(t, staged, 3)+ assert.Equal(t, []byte("extracted report"), store.files[paths[0]+parsedSidecarSuffix])+ for i, attachment := range attachments {+ assert.Equal(t, fileService.files[attachment.URL], store.files[paths[i]], attachment.FileName)+ }+ })+ }+}diff --git a/internal/sandbox/docker_remote_client.go b/internal/sandbox/docker_remote_client.goindex 62522d245..3c819d925 100644--- a/internal/sandbox/docker_remote_client.go+++ b/internal/sandbox/docker_remote_client.go@@ -859,9 +859,13 @@ func dockerExecUser(user string) string {return DefaultSandboxExecUser}-// dockerExecWasKilled reports whether an exit code means the wrapper killed-// the process. 137 is SIGKILL (timeout -s KILL), 124 is timeout(1) reporting-// that it had to intervene.+// dockerExecWasKilled reports whether an exit code means the process died to+// a KILL-class signal: 137 is SIGKILL (which the in-container `timeout -s+// KILL` wrapper sends, but so does an OOM kill), 124 is timeout(1) reporting+// that it had to intervene. It is a fact about the process, not a verdict on+// the cause — and it says nothing about the container: a re-inspect that+// confirms the container is gone classifies ahead of it in+// dockerFileOpFailure.func dockerExecWasKilled(exitCode int) bool {return exitCode == 137 || exitCode == 124}@@ -910,7 +914,7 @@ func (c *DockerRemoteClient) WriteFile(return dockerError("WriteFile", err)}if result.ExitCode != 0 {- return c.dockerFileOpFailed(ctx, id, "WriteFile", clean, result.Stderr, result.ExitCode)+ return c.dockerFileOpFailure(ctx, id, "WriteFile", clean, result, RemoteErrorKindInvalidRequest)}return nil}@@ -945,7 +949,7 @@ func (c *DockerRemoteClient) ReadFile(return nil, dockerError("ReadFile", err)}if result.ExitCode != 0 {- return nil, c.dockerFileOpFailed(ctx, id, "ReadFile", clean, result.Stderr, result.ExitCode)+ return nil, c.dockerFileOpFailure(ctx, id, "ReadFile", clean, result, RemoteErrorKindInvalidRequest)}return []byte(result.Stdout), nil}@@ -989,7 +993,7 @@ func (c *DockerRemoteClient) Stat(return nil, dockerError("Stat", err)}if result.ExitCode != 0 {- return nil, c.dockerFileOpFailed(ctx, id, "Stat", clean, result.Stderr, result.ExitCode)+ return nil, c.dockerFileOpFailure(ctx, id, "Stat", clean, result, RemoteErrorKindInvalidRequest)}entries := parseDockerFindOutput(result.Stdout)if len(entries) == 0 {@@ -1008,23 +1012,26 @@ func (c *DockerRemoteClient) Stat(}, nil}-// dockerFileOpError classifies a failed filesystem helper. A missing path is-// NotFound so callers can treat it as "nothing there"; everything else,-// permission denials included, is an invalid request carrying the tool's own-// complaint rather than a synthesised one.-// dockerFileOpFailed classifies a filesystem op's non-zero exec exit. The-// message carries the exit code — with an empty stderr it used to be the-// only missing clue (issue #3942). When the exec raced the idle sweeper's-// delete (ensureRunning's inspect passes, the sweep removes the container,-// the exec dies mid-flight), the failure surfaces as exit != 0 with an-// empty stderr; the re-inspect reclassifies it as NotFound so-// CanReplaceRemoteBinding lets the rebinding self-heal take over — making-// the sweeper's "deleting needs no coordination with the binding store"-// premise hold for this window too (issue #3942, mechanism A).-func (c *DockerRemoteClient) dockerFileOpFailed(- ctx context.Context, id, op, clean, stderr string, exitCode int,+// dockerFileOpFailure classifies a filesystem op whose exec ran but exited+// non-zero. The full result goes in so every filesystem op shares one fixed+// classification order:+//+// 1. the tool's own missing-path complaint → NotFound;+// 2. a re-inspect confirming the container is gone → NotFound, so the+// rebinding self-heal CanReplaceRemoteBinding unlocks — the exec raced+// the idle sweeper's delete (issue #3942, mechanism A);+// 3. a killed exit (137 = SIGKILL, 124 = timeout(1) reporting) → Timeout+// (retryable, binding preserved): a killed op means the sandbox+// filesystem stalled, not that the caller asked for something invalid+// (issue #3910);+// 4. anything else → fallbackKind, carrying the exit code and the tool's+// own complaint. ListDir passes Internal to keep its historical+// classification; the other ops pass InvalidRequest.+func (c *DockerRemoteClient) dockerFileOpFailure(+ ctx context.Context, id, op, clean string,+ result *RemoteExecResult, fallbackKind RemoteErrorKind,) error {- if strings.Contains(stderr, "No such file or directory") {+ if strings.Contains(result.Stderr, "No such file or directory") {return &RemoteError{Kind: RemoteErrorKindNotFound,Provider: SandboxTypeDocker,@@ -1038,14 +1045,56 @@ func (c *DockerRemoteClient) dockerFileOpFailed(Provider: SandboxTypeDocker,Op: op,Message: fmt.Sprintf("%s %s: exit=%d: container vanished mid-op (likely idle sweep)",- op, clean, exitCode),+ op, clean, result.ExitCode),}}+ if result.Killed {+ return dockerKilledOpFailure(op, clean, result)+ }+ detail := fmt.Sprintf("%s %s", op, clean)+ if line := firstNonEmptyLine(result.Stderr); line != "" {+ detail += fmt.Sprintf(": exit=%d: %s", result.ExitCode, line)+ } else if line := firstNonEmptyLine(result.Stdout); line != "" {+ detail += fmt.Sprintf(": exit=%d, stdout: %s", result.ExitCode, line)+ } else {+ // A tool that died without writing to stderr used to leave a+ // dangling colon; say so and keep the exit code attributable.+ detail += fmt.Sprintf(" (exit=%d, no stderr)", result.ExitCode)+ }return &RemoteError{- Kind: RemoteErrorKindInvalidRequest,+ Kind: fallbackKind,Provider: SandboxTypeDocker,Op: op,- Message: fmt.Sprintf("%s %s: exit=%d: %s", op, clean, exitCode, firstNonEmptyLine(stderr)),+ Message: detail,+ }+}++// dockerKilledOpFailure builds the Timeout error for a killed filesystem op.+//+// The exit code only says the process died to a KILL-class signal — 137 is+// SIGKILL (which the in-container `timeout -s KILL` wrapper sends, but OOM+// and other kills do too) and 124 is timeout(1) reporting that it+// intervened — so the message reports the facts (exit code, elapsed, output)+// and names the configured budget without asserting it as the cause: an+// elapsed close to the budget is a clue, not a verdict.+func dockerKilledOpFailure(op, clean string, result *RemoteExecResult) error {+ detail := fmt.Sprintf(+ "%s %s: killed after %s, exit=%d (filesystem-op exec budget %s; "++ "a near-budget elapsed points at the timeout wrapper, but the exit code alone cannot prove the cause)",+ op, clean, result.Duration.Round(time.Millisecond), result.ExitCode, dockerFilesystemOpTimeout,+ )+ if line := firstNonEmptyLine(result.Stderr); line != "" {+ detail += ", stderr: " + line+ } else if line := firstNonEmptyLine(result.Stdout); line != "" {+ detail += ", stdout: " + line+ } else {+ detail += ", no output"+ }+ return &RemoteError{+ Kind: RemoteErrorKindTimeout,+ Provider: SandboxTypeDocker,+ Op: op,+ Message: detail,}}@@ -1103,7 +1152,7 @@ func (c *DockerRemoteClient) makeDir(ctx context.Context, id, dir, op string) erreturn dockerError(op, err)}if result.ExitCode != 0 {- return c.dockerFileOpFailed(ctx, id, op, dir, result.Stderr, result.ExitCode)+ return c.dockerFileOpFailure(ctx, id, op, dir, result, RemoteErrorKindInvalidRequest)}return nil}@@ -1135,7 +1184,7 @@ func (c *DockerRemoteClient) Remove(return dockerError("Remove", err)}if result.ExitCode != 0 {- return c.dockerFileOpFailed(ctx, id, "Remove", clean, result.Stderr, result.ExitCode)+ return c.dockerFileOpFailure(ctx, id, "Remove", clean, result, RemoteErrorKindInvalidRequest)}return nil}@@ -1171,29 +1220,10 @@ func (c *DockerRemoteClient) ListDir(return nil, dockerError("ListDir", err)}if result.ExitCode != 0 {- if strings.Contains(result.Stderr, "No such file or directory") {- return nil, &RemoteError{- Kind: RemoteErrorKindNotFound,- Provider: SandboxTypeDocker,- Op: "ListDir",- Message: clean + " does not exist",- }- }- if c.containerVanished(ctx, id, "ListDir") {- return nil, &RemoteError{- Kind: RemoteErrorKindNotFound,- Provider: SandboxTypeDocker,- Op: "ListDir",- Message: fmt.Sprintf("ListDir %s: exit=%d: container vanished mid-op (likely idle sweep)",- clean, result.ExitCode),- }- }- return nil, &RemoteError{- Kind: RemoteErrorKindInternal,- Provider: SandboxTypeDocker,- Op: "ListDir",- Message: fmt.Sprintf("find %s: %s", clean, firstNonEmptyLine(result.Stderr)),- }+ // The shared classifier keeps ListDir's historical Internal+ // classification for genuine find failures; killed ops still map+ // to Timeout like every other filesystem op.+ return nil, c.dockerFileOpFailure(ctx, id, "ListDir", clean, result, RemoteErrorKindInternal)}return parseDockerFindOutput(result.Stdout), nil}diff --git a/internal/sandbox/docker_remote_client_test.go b/internal/sandbox/docker_remote_client_test.goindex 6c17d088d..d4d9dec2e 100644--- a/internal/sandbox/docker_remote_client_test.go+++ b/internal/sandbox/docker_remote_client_test.go@@ -1329,11 +1329,72 @@ func TestDockerExecObservesBothOutputStreamsWithoutChangingResult(t *testing.T)require.Equal(t, result.Stderr, observed["stderr"])}-// The idle sweeper can delete a container between ensureRunning's inspect-// gate and the exec's completion (#3942 mechanism A): the exec dies with a-// non-zero exit and an empty stderr. The filesystem ops must reclassify-// that collision as NotFound so CanReplaceRemoteBinding lets the rebinding-// self-heal run instead of surfacing InvalidRequest to the user.+// A filesystem op killed by the in-container `timeout -s KILL` wrapper (issue+// #3910) leaves no stderr: it must classify as Timeout with the evidence in+// the message, not as invalid_request with a dangling colon.+func TestDockerFileOpKilledByTimeoutClassifiesAsTimeout(t *testing.T) {+ engine := newFakeDockerEngine()+ engine.execExit = 137+ engine.execStderr = ""+ // The container must survive the re-inspect: a confirmed-missing+ // container classifies ahead of the killed exit code.+ engine.inspect["c"] = container.InspectResponse{+ ID: "c",+ State: &container.State{Status: "running"},+ }+ docker := newTestDockerClient(t, engine)++ err := docker.MakeDir(context.Background(), testHandle("c"), "/workspace/input/94088e061da2")+ require.Error(t, err)+ require.True(t, IsRemoteTimeout(err), "killed op must be Timeout: %q", err.Error())+ require.False(t, IsRemoteInvalidRequest(err), "killed op is not the caller's fault")+ require.Contains(t, err.Error(), "killed after")+ require.Contains(t, err.Error(), "exit=137")+ require.Contains(t, err.Error(), "no output")+ // The budget is evidence, not a verdict: the message must present it as+ // a configured budget instead of asserting the wrapper fired.+ require.Contains(t, err.Error(), "filesystem-op exec budget")+ require.NotContains(t, err.Error(), "filesystem-op timeout)", "must not assert the cause of the kill")+ require.NotRegexp(t, `: $`, err.Error(), "no dangling colon: %q", err.Error())+}++// A genuine tool failure keeps its stderr complaint, and a failure with no+// stderr at least carries the exit code instead of ending in a bare colon.+func TestDockerFileOpExitErrorsCarryEvidence(t *testing.T) {+ denied := newFakeDockerEngine()+ denied.execExit = 1+ denied.execStderr = "mkdir: /workspace/input: Permission denied"+ denied.inspect["c"] = container.InspectResponse{+ ID: "c",+ State: &container.State{Status: "running"},+ }+ docker := newTestDockerClient(t, denied)++ err := docker.MakeDir(context.Background(), testHandle("c"), "/workspace/input/x")+ require.Error(t, err)+ require.True(t, IsRemoteInvalidRequest(err))+ require.Contains(t, err.Error(), "Permission denied")++ silent := newFakeDockerEngine()+ silent.execExit = 1+ silent.execStderr = ""+ silent.inspect["c"] = container.InspectResponse{+ ID: "c",+ State: &container.State{Status: "running"},+ }+ docker = newTestDockerClient(t, silent)++ err = docker.MakeDir(context.Background(), testHandle("c"), "/workspace/input/x")+ require.Error(t, err)+ require.True(t, IsRemoteInvalidRequest(err))+ require.Contains(t, err.Error(), "(exit=1, no stderr)",+ "silent failure must say so instead of a dangling colon: %q", err.Error())+}++// A filesystem op that raced the idle sweeper's delete (#3942 mechanism A)+// surfaces as a non-zero exit with an empty stderr; the re-inspect confirms+// the container is gone and reclassifies the failure as NotFound so the+// rebinding self-heal CanReplaceRemoteBinding unlocks.func TestFileOpAgainstVanishedContainerRebinds(t *testing.T) {engine := newFakeDockerEngine()engine.execExit = 1@@ -1408,3 +1469,63 @@ func TestFileOpGenuineFailureKeepsBinding(t *testing.T) {require.True(t, IsRemoteInvalidRequest(err), "got %+v", err)require.False(t, CanReplaceRemoteBinding(err))}++// The classification order is fixed: a killed exit code does not outrank a+// re-inspect that confirms the container is gone. exit=137 with the container+// vanished is NotFound (so the rebinding self-heal can take over), not+// Timeout — for MakeDir and ListDir alike.+func TestDockerFileOpKilledWithVanishedContainerIsNotFound(t *testing.T) {+ engine := newFakeDockerEngine()+ engine.execExit = 137+ engine.execStderr = ""+ docker := newTestDockerClient(t, engine)++ err := docker.MakeDir(context.Background(), testHandle("swept-away"), "/workspace/input/x")+ require.Error(t, err)+ require.True(t, IsRemoteNotFound(err), "got %+v", err)+ require.False(t, IsRemoteTimeout(err),+ "a confirmed-missing container classifies ahead of the killed exit: %q", err.Error())+ require.True(t, CanReplaceRemoteBinding(err))++ _, err = docker.ListDir(context.Background(), testHandle("swept-away"), "/workspace/input")+ require.Error(t, err)+ require.True(t, IsRemoteNotFound(err), "got %+v", err)+ require.False(t, IsRemoteTimeout(err),+ "ListDir must follow the same order: %q", err.Error())+ require.True(t, CanReplaceRemoteBinding(err))+}++// A killed ListDir with the container still alive maps to Timeout like every+// other filesystem op, while a genuine find failure keeps ListDir's+// historical Internal classification.+func TestDockerListDirKilledIsTimeoutButGenuineFailureStaysInternal(t *testing.T) {+ killed := newFakeDockerEngine()+ killed.execExit = 137+ killed.execStderr = ""+ killed.inspect["c"] = container.InspectResponse{+ ID: "c",+ State: &container.State{Status: "running"},+ }+ docker := newTestDockerClient(t, killed)++ _, err := docker.ListDir(context.Background(), testHandle("c"), "/workspace/output")+ require.Error(t, err)+ require.True(t, IsRemoteTimeout(err), "got %+v", err)+ require.False(t, CanReplaceRemoteBinding(err), "the container is alive: the binding must stay")++ genuine := newFakeDockerEngine()+ genuine.execExit = 2+ genuine.execStderr = "find: unrecognized option"+ genuine.inspect["c"] = container.InspectResponse{+ ID: "c",+ State: &container.State{Status: "running"},+ }+ docker = newTestDockerClient(t, genuine)++ _, err = docker.ListDir(context.Background(), testHandle("c"), "/workspace/output")+ require.Error(t, err)+ var remote *RemoteError+ require.True(t, errors.As(err, &remote), "got %+v", err)+ require.Equal(t, RemoteErrorKindInternal, remote.Kind,+ "ListDir's genuine failures keep their historical classification: %q", err.Error())+}diff --git a/internal/sandbox/issue3910_repro_integration_test.go b/internal/sandbox/issue3910_repro_integration_test.gonew file mode 100644index 000000000..b38387ddb--- /dev/null+++ b/internal/sandbox/issue3910_repro_integration_test.go@@ -0,0 +1,106 @@+//go:build docker_integration++// Reproduction + regression for GitHub issue #3910.+//+// Issue: calling the agent-chat API failed with+//+// restore session attachments into sandbox: stage attachment "FAQ知识库.txt":+// sandbox: create input directory: docker MakeDir: invalid_request:+// mkdir -p /workspace/input/94088e061da2:+//+// Mechanism under test: every docker filesystem op is wrapped in an+// in-container `timeout -s KILL <N>` (dockerExecCommand). A filesystem op+// that hangs past dockerFilesystemOpTimeout is SIGKILLed — exit != 0 and,+// by definition of SIGKILL, no stderr at all. Pre-fix makeDir classifies+// that as invalid_request with the (empty) stderr as the whole explanation.+//+// The test makes `mkdir` hang deterministically by shadowing it with a+// sleeping script earlier in PATH (/usr/local/bin precedes /usr/bin on the+// sandbox image), so the in-container timeout wrapper kills it exactly the+// way it killed the reporter's mkdir.+//+// Run like the rest of the conformance suite:+//+// DOCKER_INTEGRATION_IMAGE=wechatopenai/weknora-sandbox:dev \+// go test -tags=docker_integration ./internal/sandbox \+// -run '^TestDocker3910' -count=1 -v -timeout=15m+package sandbox++import (+ "context"+ "testing"+ "time"++ "github.com/stretchr/testify/require"+)++func TestDocker3910KilledMakeDirIntegration(t *testing.T) {+ cfg := dockerIntegrationConfig(t)+ docker, err := NewDockerRemoteClient(cfg)+ require.NoError(t, err)++ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)+ defer cancel()++ handle, err := docker.Create(ctx, RemoteCreateRequest{+ TemplateID: cfg.DockerImage,+ Metadata: map[string]string{"weknora.test": "issue-3910-repro"},+ })+ require.NoError(t, err, "create sandbox container")+ t.Cleanup(func() {+ cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), time.Minute)+ defer cleanupCancel()+ _ = docker.Delete(cleanupCtx, handle.ID())+ })++ // Sanity: MakeDir works before the sabotage.+ require.NoError(t, docker.MakeDir(ctx, handle, "/workspace/input/sane"),+ "MakeDir must succeed on a healthy container")++ // Shadow mkdir with a script that hangs (120s >> the 30s filesystem-op+ // timeout). /usr/local/bin precedes /usr/bin in PATH, so the in-container+ // `timeout -s KILL 30 mkdir -p ...` resolves to the sleeper.+ sabotage, err := docker.Exec(ctx, handle, RemoteExecRequest{+ Command: "sh",+ Args: []string{"-c",+ `printf '#!/bin/sh\nsleep 120\n' > /usr/local/bin/mkdir && chmod +x /usr/local/bin/mkdir`},+ User: "root",+ Timeout: 30 * time.Second,+ })+ require.NoError(t, err, "sabotage exec failed")+ require.Equal(t, 0, sabotage.ExitCode, "sabotage script failed: %s", sabotage.Stderr)++ start := time.Now()+ err = docker.MakeDir(ctx, handle, "/workspace/input/94088e061da2")+ elapsed := time.Since(start)+ require.Error(t, err, "MakeDir against a hung mkdir must fail")+ t.Logf("MakeDir failed after %s with: %q", elapsed.Round(time.Millisecond), err.Error())++ // Fixed behavior: the SIGKILLed filesystem op is a Timeout (retryable,+ // binding preserved), not an invalid_request, and the message carries the+ // evidence a killed process leaves behind — exit code and elapsed time —+ // instead of the dangling colon from #3910.+ require.False(t, IsRemoteInvalidRequest(err),+ "a killed filesystem op is not the caller's fault: got %q", err.Error())+ require.True(t, IsRemoteTimeout(err),+ "a killed filesystem op must classify as Timeout: got %q", err.Error())+ require.Contains(t, err.Error(), "killed after",+ "message must say the op was killed: %q", err.Error())+ require.Regexp(t, `exit=1(24|37)`, err.Error(),+ "message must carry the exit code: %q", err.Error())+ require.NotRegexp(t, `: $`, err.Error(),+ "message must not end in a dangling colon: %q", err.Error())++ // The failed MakeDir must not poison the sandbox: a subsequent healthy+ // operation still works (the binding was preserved, not dropped).+ recovered, err := docker.Exec(ctx, handle, RemoteExecRequest{+ Command: "sh",+ Args: []string{"-c", `rm -f /usr/local/bin/mkdir`},+ User: "root",+ Timeout: 30 * time.Second,+ })+ require.NoError(t, err)+ require.Equal(t, 0, recovered.ExitCode)+ require.NoError(t, docker.MakeDir(ctx, handle, "/workspace/input/recovered"),+ "sandbox must stay usable after a killed filesystem op")+}diff --git a/internal/sandbox/remote_errors.go b/internal/sandbox/remote_errors.goindex a3ee10d2f..917e62b5c 100644--- a/internal/sandbox/remote_errors.go+++ b/internal/sandbox/remote_errors.go@@ -190,6 +190,14 @@ func IsRemoteConflict(err error) bool {return remoteKind(err) == RemoteErrorKindConflict}+// IsRemoteTimeout reports whether err classifies as a deadline the op exceeded+// (in-container kill wrapper, transport deadline). The caller did nothing+// wrong and the binding is still valid: such failures may be retried, and a+// transient provider stall usually clears on the next attempt.+func IsRemoteTimeout(err error) bool {+ return remoteKind(err) == RemoteErrorKindTimeout+}+// snapshotDeleteKind reclassifies a snapshot/template delete that failed// because sandboxes still reference it. E2B returns that as HTTP 400// invalid_request; it is Conflict: the caller did nothing wrong and should