Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(sandbox): treat in-use snapshot deletes as retryable conflicts

wizardchenaf0aeb7cc284
diff --git a/internal/application/service/tenant_skill_reaper.go b/internal/application/service/tenant_skill_reaper.go
index e6f70f69a..fada45dbc 100644
--- a/internal/application/service/tenant_skill_reaper.go
+++ b/internal/application/service/tenant_skill_reaper.go
@@ -422,10 +422,10 @@ func configuredSandboxTTL(cfg *types.TenantSandboxConfig) time.Duration {
// image is never touched, nor is anything the ledger does not name: extras
// belong to other environments on a shared provider account.
//
-// A live sandbox does not need the template it was created from in order to
-// keep running, so the only reason to wait is in-flight creates that resolved
-// the previous pointer. Twenty-four hours is far past every backend's default
-// TTL; a config that sets a longer one extends the wait.
+// Retention is a lower bound, not a guarantee the provider will accept the
+// delete. Session sandboxes pause instead of dying when idle, and a paused
+// sandbox still pins its template; a Conflict is left on the ledger for the
+// next sweep.
func (s *TenantSkillService) PruneSupersededSnapshots(ctx context.Context) (int, error) {
if s == nil || s.skills == nil {
return 0, nil
@@ -497,6 +497,12 @@ func (s *TenantSkillService) pruneConfigSnapshots(
continue
}
if err := deleter.DeleteSnapshot(ctx, row.SnapshotID); err != nil && !sandbox.IsRemoteNotFound(err) {
+ if sandbox.IsRemoteConflict(err) {
+ logger.Infof(ctx,
+ "[skill] snapshot %s still in use; leaving it until sandboxes release it: %s",
+ row.SnapshotID, sandbox.RemoteErrorDiagnostics(err))
+ continue
+ }
logger.Warnf(ctx, "[skill] delete superseded snapshot %s failed: %v", row.SnapshotID, err)
continue
}
@@ -568,6 +574,12 @@ func (s *TenantSkillService) reapAbandonedBuilds(
continue
}
if err := deleter.DeleteSnapshot(ctx, found); err != nil && !sandbox.IsRemoteNotFound(err) {
+ if sandbox.IsRemoteConflict(err) {
+ logger.Infof(ctx,
+ "[skill] abandoned build %s of config %s still in use; leaving it until sandboxes release it: %s",
+ found, cfg.ID, sandbox.RemoteErrorDiagnostics(err))
+ continue
+ }
logger.Warnf(ctx, "[skill] delete abandoned build %s of config %s failed: %v",
found, cfg.ID, err)
continue
diff --git a/internal/application/service/tenant_skill_reaper_test.go b/internal/application/service/tenant_skill_reaper_test.go
index f69657517..c56f34eea 100644
--- a/internal/application/service/tenant_skill_reaper_test.go
+++ b/internal/application/service/tenant_skill_reaper_test.go
@@ -390,6 +390,28 @@ func TestPruneSupersededSnapshotsLeavesTheRowWhenDeleteFails(t *testing.T) {
"a failed provider delete must not be recorded as deleted")
}
+func TestPruneSupersededSnapshotsRetriesWhenSnapshotStillInUse(t *testing.T) {
+ fx := newReaperFixture(t)
+ fx.svc.snapshotRetention = time.Hour
+ old := fx.now.Add(-2 * time.Hour)
+ fx.live("snap-live")
+ fx.superseded("sk-1", "snap-old", "", old)
+ fx.provider.deleteErr = sandbox.NewRemoteError(
+ sandbox.SandboxTypeE2B, "DeleteSnapshot", sandbox.RemoteErrorKindConflict,
+ "cannot delete template because there are paused sandboxes using it", nil,
+ )
+
+ n, err := fx.svc.PruneSupersededSnapshots(context.Background())
+
+ require.NoError(t, err)
+ require.Zero(t, n)
+ require.Empty(t, fx.provider.deleted)
+ rows, err := fx.skills.ListSnapshotsByConfig(context.Background(), 7, "cfg-1")
+ require.NoError(t, err)
+ require.Equal(t, types.SkillSnapshotStateSuperseded, rows[0].State,
+ "an in-use template must stay on the ledger so the next sweep can retry")
+}
+
func TestPruneSupersededSnapshotsTreatsMissingProviderSnapshotAsDeleted(t *testing.T) {
fx := newReaperFixture(t)
fx.svc.snapshotRetention = time.Hour
diff --git a/internal/application/service/tenant_skill_service.go b/internal/application/service/tenant_skill_service.go
index 247775cc3..6dabe2e6c 100644
--- a/internal/application/service/tenant_skill_service.go
+++ b/internal/application/service/tenant_skill_service.go
@@ -43,11 +43,10 @@ const (
skillInstallInFlightSkip = 3 * time.Minute
// skillSnapshotRetention is how long a superseded snapshot stays on the
- // provider after the pointer has moved. Live sandboxes may still have
- // been created from it (especially SkillRolloutNewSession); once they
- // expire, the template is only a billed leftover. Twenty-four hours is
- // well past every backend's default sandbox TTL. A config that sets a
- // longer sandbox TTL extends this via snapshotRetentionFor.
+ // provider after the pointer has moved before prune even tries. Paused
+ // session sandboxes can pin the template past this window; prune retries
+ // on Conflict. A config that sets a longer sandbox TTL extends this via
+ // snapshotRetentionFor.
skillSnapshotRetention = 24 * time.Hour
// skillSnapshotTTLMargin is added on top of a config's own sandbox TTL
diff --git a/internal/sandbox/cube_remote_client.go b/internal/sandbox/cube_remote_client.go
index c10de2fb4..562ae6c22 100644
--- a/internal/sandbox/cube_remote_client.go
+++ b/internal/sandbox/cube_remote_client.go
@@ -1123,6 +1123,7 @@ func normalizeCubeError(op string, err error) error {
kind = RemoteErrorKindUnavailable
}
}
+ kind = snapshotDeleteKind(op, kind, err.Error())
remoteErr := NewRemoteError(SandboxTypeCube, op, kind, err.Error(), err)
remoteErr.StatusCode = status
return remoteErr
diff --git a/internal/sandbox/cube_remote_client_test.go b/internal/sandbox/cube_remote_client_test.go
index c5eca9104..0e487bcdd 100644
--- a/internal/sandbox/cube_remote_client_test.go
+++ b/internal/sandbox/cube_remote_client_test.go
@@ -470,6 +470,14 @@ func TestNormalizeCubeError(t *testing.T) {
{"bad gateway", "List", &cubesandbox.APIError{StatusCode: http.StatusBadGateway}, RemoteErrorKindUnavailable},
{"deadline", "Exec", context.DeadlineExceeded, RemoteErrorKindTimeout},
{"unknown", "List", errors.New("unknown"), RemoteErrorKindInternal},
+ {"delete snapshot in use", "DeleteSnapshot", &cubesandbox.APIError{
+ StatusCode: http.StatusBadRequest,
+ Message: "cannot delete template x because there are paused sandboxes using it",
+ }, RemoteErrorKindConflict},
+ {"delete snapshot bad id", "DeleteSnapshot", &cubesandbox.APIError{
+ StatusCode: http.StatusBadRequest,
+ Message: "invalid snapshot id",
+ }, RemoteErrorKindInvalidRequest},
}
for _, tt := range tests {
diff --git a/internal/sandbox/docker_engine.go b/internal/sandbox/docker_engine.go
index c5e07d13b..0e08558bd 100644
--- a/internal/sandbox/docker_engine.go
+++ b/internal/sandbox/docker_engine.go
@@ -373,7 +373,7 @@ func dockerError(op string, err error) error {
return err
}
return &RemoteError{
- Kind: dockerErrorKind(op, err),
+ Kind: snapshotDeleteKind(op, dockerErrorKind(op, err), err.Error()),
Provider: SandboxTypeDocker,
Op: op,
Message: err.Error(),
diff --git a/internal/sandbox/e2b_remote_client.go b/internal/sandbox/e2b_remote_client.go
index f835044f9..1d93906cb 100644
--- a/internal/sandbox/e2b_remote_client.go
+++ b/internal/sandbox/e2b_remote_client.go
@@ -1253,6 +1253,7 @@ func normalizeE2BError(op string, err error) error {
}
}
}
+ kind = snapshotDeleteKind(op, kind, err.Error())
remoteErr := NewRemoteError(SandboxTypeE2B, op, kind, err.Error(), err)
remoteErr.StatusCode = status
return remoteErr
diff --git a/internal/sandbox/e2b_remote_client_test.go b/internal/sandbox/e2b_remote_client_test.go
index 09737b58a..e2b43b88e 100644
--- a/internal/sandbox/e2b_remote_client_test.go
+++ b/internal/sandbox/e2b_remote_client_test.go
@@ -1391,6 +1391,14 @@ func TestNormalizeE2BError(t *testing.T) {
{"connect data loss", "List", connect.NewError(connect.CodeDataLoss, errors.New("data loss")), RemoteErrorKindInternal},
{"connect unknown", "List", connect.NewError(connect.CodeUnknown, errors.New("unknown")), RemoteErrorKindInternal},
{"unknown", "List", errors.New("mystery"), RemoteErrorKindInternal},
+ {"delete snapshot in use", "DeleteSnapshot", &e2b.Error{
+ StatusCode: http.StatusBadRequest,
+ Message: "cannot delete template 'tpl-1' because there are paused sandboxes using it",
+ }, RemoteErrorKindConflict},
+ {"delete snapshot bad id", "DeleteSnapshot", &e2b.Error{
+ StatusCode: http.StatusBadRequest,
+ Message: "invalid snapshot id",
+ }, RemoteErrorKindInvalidRequest},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
diff --git a/internal/sandbox/langfuse.go b/internal/sandbox/langfuse.go
index 8cab1e67e..363b653b6 100644
--- a/internal/sandbox/langfuse.go
+++ b/internal/sandbox/langfuse.go
@@ -215,10 +215,24 @@ func (c *langfuseSnapshotClient) DeleteSnapshot(ctx context.Context, snapshotID
"snapshot_id": snapshotID,
}, nil)
err := inner.DeleteSnapshot(ctx, snapshotID)
- span.Finish(nil, nil, err)
+ out, spanErr := snapshotDeleteSpanResult(err)
+ span.Finish(out, nil, spanErr)
return err
}
+// snapshotDeleteSpanResult keeps an in-use Conflict off the ERROR status.
+// Session sandboxes pause against the template they booted from, so the skill
+// reaper hitting this every few minutes is expected retry, not a fault.
+func snapshotDeleteSpanResult(err error) (map[string]interface{}, error) {
+ if !IsRemoteConflict(err) {
+ return nil, err
+ }
+ return map[string]interface{}{
+ "deferred": true,
+ "reason": "in_use",
+ }, nil
+}
+
func (c *langfuseSnapshotClient) ListSnapshots(
ctx context.Context, sandboxID string,
) ([]RemoteSnapshotRef, error) {
diff --git a/internal/sandbox/langfuse_test.go b/internal/sandbox/langfuse_test.go
index 84c11d272..8687bbf14 100644
--- a/internal/sandbox/langfuse_test.go
+++ b/internal/sandbox/langfuse_test.go
@@ -61,3 +61,18 @@ func TestTruncateSandboxPreview(t *testing.T) {
require.True(t, strings.HasSuffix(got, "…"))
require.Equal(t, sandboxSpanPreviewRunes+1, utf8.RuneCountInString(got))
}
+
+func TestSnapshotDeleteSpanResultDefersConflict(t *testing.T) {
+ out, spanErr := snapshotDeleteSpanResult(NewRemoteError(
+ SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindConflict,
+ "paused sandboxes using it", nil,
+ ))
+ require.NoError(t, spanErr, "in-use must not mark the Langfuse span ERROR")
+ require.Equal(t, true, out["deferred"])
+ require.Equal(t, "in_use", out["reason"])
+
+ boom := NewRemoteError(SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindInternal, "boom", nil)
+ out, spanErr = snapshotDeleteSpanResult(boom)
+ require.Equal(t, boom, spanErr)
+ require.Nil(t, out)
+}
diff --git a/internal/sandbox/remote_errors.go b/internal/sandbox/remote_errors.go
index b50412401..b89eec0d8 100644
--- a/internal/sandbox/remote_errors.go
+++ b/internal/sandbox/remote_errors.go
@@ -184,6 +184,37 @@ func IsRemoteInvalidRequest(err error) bool {
return remoteKind(err) == RemoteErrorKindInvalidRequest
}
+// IsRemoteConflict reports whether the provider refused the call because the
+// resource is busy (concurrent mutation, snapshot still referenced, …).
+func IsRemoteConflict(err error) bool {
+ return remoteKind(err) == RemoteErrorKindConflict
+}
+
+// snapshotDeleteKind reclassifies a snapshot/template delete that failed
+// because sandboxes still reference it. E2B returns that as HTTP 400
+// invalid_request; it is Conflict: the caller did nothing wrong and should
+// retry after those sandboxes (typically paused) go away.
+func snapshotDeleteKind(op string, kind RemoteErrorKind, message string) RemoteErrorKind {
+ if op != "DeleteSnapshot" && op != "DeleteTemplate" {
+ return kind
+ }
+ if kind == RemoteErrorKindConflict || snapshotInUseBySandboxes(message) {
+ return RemoteErrorKindConflict
+ }
+ return kind
+}
+
+func snapshotInUseBySandboxes(message string) bool {
+ msg := strings.ToLower(message)
+ if strings.Contains(msg, "paused sandbox") {
+ return true
+ }
+ if strings.Contains(msg, "sandboxes using") {
+ return true
+ }
+ return strings.Contains(msg, "cannot delete template") && strings.Contains(msg, "using it")
+}
+
// IsRemoteDirAlreadyExists reports whether MakeDir failed because the
// directory is already present. Cube's envd MakeDir is not mkdir -p: a
// directory created by a previous call (or by a shell `mkdir -p` in
diff --git a/internal/sandbox/remote_errors_test.go b/internal/sandbox/remote_errors_test.go
index cc7df8316..0ad3dee98 100644
--- a/internal/sandbox/remote_errors_test.go
+++ b/internal/sandbox/remote_errors_test.go
@@ -177,3 +177,88 @@ func TestIsRemoteDirAlreadyExists(t *testing.T) {
})
}
}
+
+func TestIsRemoteConflict(t *testing.T) {
+ t.Parallel()
+
+ if IsRemoteConflict(nil) {
+ t.Fatal("nil is not a conflict")
+ }
+ conflict := NewRemoteError(SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindConflict, "busy", nil)
+ if !IsRemoteConflict(conflict) {
+ t.Fatal("Conflict kind must match")
+ }
+ invalid := NewRemoteError(
+ SandboxTypeE2B, "DeleteSnapshot", RemoteErrorKindInvalidRequest, "bad id", nil,
+ )
+ if IsRemoteConflict(invalid) {
+ t.Fatal("InvalidRequest must not match")
+ }
+}
+
+func TestSnapshotDeleteKindPromotesInUseToConflict(t *testing.T) {
+ t.Parallel()
+
+ inUse := "cannot delete template 'upfvuzpq0q6foo1mkpbkl' because there are paused sandboxes using it"
+ jsonBody := `e2b: status 400: {"code":400,"message":"` + inUse + `"}`
+
+ tests := []struct {
+ name string
+ op string
+ kind RemoteErrorKind
+ msg string
+ want RemoteErrorKind
+ }{
+ {
+ name: "e2b paused sandboxes 400",
+ op: "DeleteSnapshot",
+ kind: RemoteErrorKindInvalidRequest,
+ msg: inUse,
+ want: RemoteErrorKindConflict,
+ },
+ {
+ name: "json-wrapped e2b 400",
+ op: "DeleteSnapshot",
+ kind: RemoteErrorKindInvalidRequest,
+ msg: jsonBody,
+ want: RemoteErrorKindConflict,
+ },
+ {
+ name: "cube delete template",
+ op: "DeleteTemplate",
+ kind: RemoteErrorKindInvalidRequest,
+ msg: "cannot delete template x because there are sandboxes using it",
+ want: RemoteErrorKindConflict,
+ },
+ {
+ name: "already conflict stays conflict",
+ op: "DeleteSnapshot",
+ kind: RemoteErrorKindConflict,
+ msg: "image is in use",
+ want: RemoteErrorKindConflict,
+ },
+ {
+ name: "generic 400 stays invalid",
+ op: "DeleteSnapshot",
+ kind: RemoteErrorKindInvalidRequest,
+ msg: "invalid snapshot id",
+ want: RemoteErrorKindInvalidRequest,
+ },
+ {
+ name: "other ops are unchanged",
+ op: "Create",
+ kind: RemoteErrorKindInvalidRequest,
+ msg: inUse,
+ want: RemoteErrorKindInvalidRequest,
+ },
+ }
+ for _, tt := range tests {
+ tt := tt
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+ if got := snapshotDeleteKind(tt.op, tt.kind, tt.msg); got != tt.want {
+ t.Fatalf("snapshotDeleteKind() = %q, want %q", got, tt.want)
+ }
+ })
+ }
+}