Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(sandbox): treat in-use snapshot deletes as retryable conflicts
wizardchenaf0aeb7cc284
diff --git a/internal/application/service/tenant_skill_reaper.go b/internal/application/service/tenant_skill_reaper.goindex e6f70f69a..fada45dbc 100644--- a/internal/application/service/tenant_skill_reaper.go+++ b/internal/application/service/tenant_skill_reaper.go@@ -422,10 +422,10 @@ func configuredSandboxTTL(cfg *types.TenantSandboxConfig) time.Duration {// image is never touched, nor is anything the ledger does not name: extras// belong to other environments on a shared provider account.//-// A live sandbox does not need the template it was created from in order to-// keep running, so the only reason to wait is in-flight creates that resolved-// the previous pointer. Twenty-four hours is far past every backend's default-// TTL; a config that sets a longer one extends the wait.+// Retention is a lower bound, not a guarantee the provider will accept the+// delete. Session sandboxes pause instead of dying when idle, and a paused+// sandbox still pins its template; a Conflict is left on the ledger for the+// next sweep.func (s *TenantSkillService) PruneSupersededSnapshots(ctx context.Context) (int, error) {if s == nil || s.skills == nil {return 0, nil@@ -497,6 +497,12 @@ func (s *TenantSkillService) pruneConfigSnapshots(continue}if err := deleter.DeleteSnapshot(ctx, row.SnapshotID); err != nil && !sandbox.IsRemoteNotFound(err) {+ if sandbox.IsRemoteConflict(err) {+ logger.Infof(ctx,+ "[skill] snapshot %s still in use; leaving it until sandboxes release it: %s",+ row.SnapshotID, sandbox.RemoteErrorDiagnostics(err))+ continue+ }logger.Warnf(ctx, "[skill] delete superseded snapshot %s failed: %v", row.SnapshotID, err)continue}@@ -568,6 +574,12 @@ func (s *TenantSkillService) reapAbandonedBuilds(continue}if err := deleter.DeleteSnapshot(ctx, found); err != nil && !sandbox.IsRemoteNotFound(err) {+ if sandbox.IsRemoteConflict(err) {+ logger.Infof(ctx,+ "[skill] abandoned build %s of config %s still in use; leaving it until sandboxes release it: %s",+ found, cfg.ID, sandbox.RemoteErrorDiagnostics(err))+ continue+ }logger.Warnf(ctx, "[skill] delete abandoned build %s of config %s failed: %v",found, cfg.ID, err)continuediff --git a/internal/application/service/tenant_skill_reaper_test.go b/internal/application/service/tenant_skill_reaper_test.goindex f69657517..c56f34eea 100644--- a/internal/application/service/tenant_skill_reaper_test.go+++ b/internal/application/service/tenant_skill_reaper_test.go@@ -390,6 +390,28 @@ func TestPruneSupersededSnapshotsLeavesTheRowWhenDeleteFails(t *testing.T) {"a failed provider delete must not be recorded as deleted")}+func TestPruneSupersededSnapshotsRetriesWhenSnapshotStillInUse(t *testing.T) {+ fx := newReaperFixture(t)+ fx.svc.snapshotRetention = time.Hour+ old := fx.now.Add(-2 * time.Hour)+ fx.live("snap-live")+ fx.superseded("sk-1", "snap-old", "", old)+ fx.provider.deleteErr = sandbox.NewRemoteError(+ sandbox.SandboxTypeE2B, "DeleteSnapshot", sandbox.RemoteErrorKindConflict,+ "cannot delete template because there are paused sandboxes using it", nil,+ )++ n, err := fx.svc.PruneSupersededSnapshots(context.Background())++ require.NoError(t, err)+ require.Zero(t, n)+ require.Empty(t, fx.provider.deleted)+ rows, err := fx.skills.ListSnapshotsByConfig(context.Background(), 7, "cfg-1")+ require.NoError(t, err)+ require.Equal(t, types.SkillSnapshotStateSuperseded, rows[0].State,+ "an in-use template must stay on the ledger so the next sweep can retry")+}+func TestPruneSupersededSnapshotsTreatsMissingProviderSnapshotAsDeleted(t *testing.T) {fx := newReaperFixture(t)fx.svc.snapshotRetention = time.Hourdiff --git a/internal/application/service/tenant_skill_service.go b/internal/application/service/tenant_skill_service.goindex 247775cc3..6dabe2e6c 100644--- a/internal/application/service/tenant_skill_service.go+++ b/internal/application/service/tenant_skill_service.go@@ -43,11 +43,10 @@ const (skillInstallInFlightSkip = 3 * time.Minute// skillSnapshotRetention is how long a superseded snapshot stays on the- // provider after the pointer has moved. Live sandboxes may still have- // been created from it (especially SkillRolloutNewSession); once they- // expire, the template is only a billed leftover. Twenty-four hours is- // well past every backend's default sandbox TTL. A config that sets a- // longer sandbox TTL extends this via snapshotRetentionFor.+ // provider after the pointer has moved before prune even tries. Paused+ // session sandboxes can pin the template past this window; prune retries+ // on Conflict. A config that sets a longer sandbox TTL extends this via+ // snapshotRetentionFor.skillSnapshotRetention = 24 * time.Hour// skillSnapshotTTLMargin is added on top of a config's own sandbox TTLdiff --git a/internal/sandbox/cube_remote_client.go b/internal/sandbox/cube_remote_client.goindex c10de2fb4..562ae6c22 100644--- a/internal/sandbox/cube_remote_client.go+++ b/internal/sandbox/cube_remote_client.go@@ -1123,6 +1123,7 @@ func normalizeCubeError(op string, err error) error {kind = RemoteErrorKindUnavailable}}+ kind = snapshotDeleteKind(op, kind, err.Error())remoteErr := NewRemoteError(SandboxTypeCube, op, kind, err.Error(), err)remoteErr.StatusCode = statusreturn remoteErrdiff --git a/internal/sandbox/cube_remote_client_test.go b/internal/sandbox/cube_remote_client_test.goindex c5eca9104..0e487bcdd 100644--- a/internal/sandbox/cube_remote_client_test.go+++ b/internal/sandbox/cube_remote_client_test.go@@ -470,6 +470,14 @@ func TestNormalizeCubeError(t *testing.T) {{"bad gateway", "List", &cubesandbox.APIError{StatusCode: http.StatusBadGateway}, RemoteErrorKindUnavailable},{"deadline", "Exec", context.DeadlineExceeded, RemoteErrorKindTimeout},{"unknown", "List", errors.New("unknown"), RemoteErrorKindInternal},+ {"delete snapshot in use", "DeleteSnapshot", &cubesandbox.APIError{+ StatusCode: http.StatusBadRequest,+ Message: "cannot delete template x because there are paused sandboxes using it",+ }, RemoteErrorKindConflict},+ {"delete snapshot bad id", "DeleteSnapshot", &cubesandbox.APIError{+ StatusCode: http.StatusBadRequest,+ Message: "invalid snapshot id",+ }, RemoteErrorKindInvalidRequest},}for _, tt := range tests {diff --git a/internal/sandbox/docker_engine.go b/internal/sandbox/docker_engine.goindex c5e07d13b..0e08558bd 100644--- a/internal/sandbox/docker_engine.go+++ b/internal/sandbox/docker_engine.go@@ -373,7 +373,7 @@ func dockerError(op string, err error) error {return err}return &RemoteError{- Kind: dockerErrorKind(op, err),+ Kind: snapshotDeleteKind(op, dockerErrorKind(op, err), err.Error()),Provider: SandboxTypeDocker,Op: op,Message: err.Error(),diff --git a/internal/sandbox/e2b_remote_client.go b/internal/sandbox/e2b_remote_client.goindex f835044f9..1d93906cb 100644--- a/internal/sandbox/e2b_remote_client.go+++ b/internal/sandbox/e2b_remote_client.go@@ -1253,6 +1253,7 @@ func normalizeE2BError(op string, err error) error {}}}+ kind = snapshotDeleteKind(op, kind, err.Error())remoteErr := NewRemoteError(SandboxTypeE2B, op, kind, err.Error(), err)remoteErr.StatusCode = statusreturn remoteErrdiff --git a/internal/sandbox/e2b_remote_client_test.go b/internal/sandbox/e2b_remote_client_test.goindex 09737b58a..e2b43b88e 100644--- a/internal/sandbox/e2b_remote_client_test.go+++ b/internal/sandbox/e2b_remote_client_test.go@@ -1391,6 +1391,14 @@ func TestNormalizeE2BError(t *testing.T) {{"connect data loss", "List", connect.NewError(connect.CodeDataLoss, errors.New("data loss")), RemoteErrorKindInternal},{"connect unknown", "List", connect.NewError(connect.CodeUnknown, errors.New("unknown")), RemoteErrorKindInternal},{"unknown", "List", errors.New("mystery"), RemoteErrorKindInternal},+ {"delete snapshot in use", "DeleteSnapshot", &e2b.Error{+ StatusCode: http.StatusBadRequest,+ Message: "cannot delete template 'tpl-1' because there are paused sandboxes using it",+ }, RemoteErrorKindConflict},+ {"delete snapshot bad id", "DeleteSnapshot", &e2b.Error{+ StatusCode: http.StatusBadRequest,+ Message: "invalid snapshot id",+ }, RemoteErrorKindInvalidRequest},}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {diff --git a/internal/sandbox/langfuse.go b/internal/sandbox/langfuse.goindex 8cab1e67e..363b653b6 100644--- a/internal/sandbox/langfuse.go+++ b/internal/sandbox/langfuse.go@@ -215,10 +215,24 @@ func (c *langfuseSnapshotClient) DeleteSnapshot(ctx context.Context, snapshotID"snapshot_id": snapshotID,}, nil)err := inner.DeleteSnapshot(ctx, snapshotID)- span.Finish(nil, nil, err)+ out, spanErr := snapshotDeleteSpanResult(err)+ span.Finish(out, nil, spanErr)return err}+// snapshotDeleteSpanResult keeps an in-use Conflict off the ERROR status.+// Session sandboxes pause against the template they booted from, so the skill+// reaper hitting this every few minutes is expected retry, not a fault.+func snapshotDeleteSpanResult(err error) (map[string]interface{}, error) {+ if !IsRemoteConflict(err) {+ return nil, err+ }+ return map[string]interface{}{+ "deferred": true,+ "reason": "in_use",+ }, nil+}+func (c *langfuseSnapshotClient) ListSnapshots(ctx context.Context, sandboxID string,) ([]RemoteSnapshotRef, error) {diff --git a/internal/sandbox/langfuse_test.go b/internal/sandbox/langfuse_test.goindex 84c11d272..8687bbf14 100644--- a/internal/sandbox/langfuse_test.go+++ b/internal/sandbox/langfuse_test.go@@ -61,3 +61,18 @@ func TestTruncateSandboxPreview(t *testing.T) {require.True(t, strings.HasSuffix(got, "…"))require.Equal(t, sandboxSpanPreviewRunes+1, utf8.RuneCountInString(got))}++func TestSnapshotDeleteSpanResultDefersConflict(t *testing.T) {+ out, spanErr := snapshotDeleteSpanResult(NewRemoteError(+ SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindConflict,+ "paused sandboxes using it", nil,+ ))+ require.NoError(t, spanErr, "in-use must not mark the Langfuse span ERROR")+ require.Equal(t, true, out["deferred"])+ require.Equal(t, "in_use", out["reason"])++ boom := NewRemoteError(SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindInternal, "boom", nil)+ out, spanErr = snapshotDeleteSpanResult(boom)+ require.Equal(t, boom, spanErr)+ require.Nil(t, out)+}diff --git a/internal/sandbox/remote_errors.go b/internal/sandbox/remote_errors.goindex b50412401..b89eec0d8 100644--- a/internal/sandbox/remote_errors.go+++ b/internal/sandbox/remote_errors.go@@ -184,6 +184,37 @@ func IsRemoteInvalidRequest(err error) bool {return remoteKind(err) == RemoteErrorKindInvalidRequest}+// IsRemoteConflict reports whether the provider refused the call because the+// resource is busy (concurrent mutation, snapshot still referenced, …).+func IsRemoteConflict(err error) bool {+ return remoteKind(err) == RemoteErrorKindConflict+}++// snapshotDeleteKind reclassifies a snapshot/template delete that failed+// because sandboxes still reference it. E2B returns that as HTTP 400+// invalid_request; it is Conflict: the caller did nothing wrong and should+// retry after those sandboxes (typically paused) go away.+func snapshotDeleteKind(op string, kind RemoteErrorKind, message string) RemoteErrorKind {+ if op != "DeleteSnapshot" && op != "DeleteTemplate" {+ return kind+ }+ if kind == RemoteErrorKindConflict || snapshotInUseBySandboxes(message) {+ return RemoteErrorKindConflict+ }+ return kind+}++func snapshotInUseBySandboxes(message string) bool {+ msg := strings.ToLower(message)+ if strings.Contains(msg, "paused sandbox") {+ return true+ }+ if strings.Contains(msg, "sandboxes using") {+ return true+ }+ return strings.Contains(msg, "cannot delete template") && strings.Contains(msg, "using it")+}+// IsRemoteDirAlreadyExists reports whether MakeDir failed because the// directory is already present. Cube's envd MakeDir is not mkdir -p: a// directory created by a previous call (or by a shell `mkdir -p` indiff --git a/internal/sandbox/remote_errors_test.go b/internal/sandbox/remote_errors_test.goindex cc7df8316..0ad3dee98 100644--- a/internal/sandbox/remote_errors_test.go+++ b/internal/sandbox/remote_errors_test.go@@ -177,3 +177,88 @@ func TestIsRemoteDirAlreadyExists(t *testing.T) {})}}++func TestIsRemoteConflict(t *testing.T) {+ t.Parallel()++ if IsRemoteConflict(nil) {+ t.Fatal("nil is not a conflict")+ }+ conflict := NewRemoteError(SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindConflict, "busy", nil)+ if !IsRemoteConflict(conflict) {+ t.Fatal("Conflict kind must match")+ }+ invalid := NewRemoteError(+ SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindInvalidRequest, "bad id", nil,+ )+ if IsRemoteConflict(invalid) {+ t.Fatal("InvalidRequest must not match")+ }+}++func TestSnapshotDeleteKindPromotesInUseToConflict(t *testing.T) {+ t.Parallel()++ inUse := "cannot delete template 'upfvuzpq0q6foo1mkpbkl' because there are paused sandboxes using it"+ jsonBody := `e2b: status 400: {"code":400,"message":"` + inUse + `"}`++ tests := []struct {+ name string+ op string+ kind RemoteErrorKind+ msg string+ want RemoteErrorKind+ }{+ {+ name: "e2b paused sandboxes 400",+ op: "DeleteSnapshot",+ kind: RemoteErrorKindInvalidRequest,+ msg: inUse,+ want: RemoteErrorKindConflict,+ },+ {+ name: "json-wrapped e2b 400",+ op: "DeleteSnapshot",+ kind: RemoteErrorKindInvalidRequest,+ msg: jsonBody,+ want: RemoteErrorKindConflict,+ },+ {+ name: "cube delete template",+ op: "DeleteTemplate",+ kind: RemoteErrorKindInvalidRequest,+ msg: "cannot delete template x because there are sandboxes using it",+ want: RemoteErrorKindConflict,+ },+ {+ name: "already conflict stays conflict",+ op: "DeleteSnapshot",+ kind: RemoteErrorKindConflict,+ msg: "image is in use",+ want: RemoteErrorKindConflict,+ },+ {+ name: "generic 400 stays invalid",+ op: "DeleteSnapshot",+ kind: RemoteErrorKindInvalidRequest,+ msg: "invalid snapshot id",+ want: RemoteErrorKindInvalidRequest,+ },+ {+ name: "other ops are unchanged",+ op: "Create",+ kind: RemoteErrorKindInvalidRequest,+ msg: inUse,+ want: RemoteErrorKindInvalidRequest,+ },+ }+ for _, tt := range tests {+ tt := tt+ t.Run(tt.name, func(t *testing.T) {+ t.Parallel()+ if got := snapshotDeleteKind(tt.op, tt.kind, tt.msg); got != tt.want {+ t.Fatalf("snapshotDeleteKind() = %q, want %q", got, tt.want)+ }+ })+ }+}