Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(sandbox): skip identical skill installs and prune old snapshots (#2791)
lyingbugb4128552b41d
diff --git a/internal/application/service/tenant_skill_bundle.go b/internal/application/service/tenant_skill_bundle.goindex fd23bf3a2..5f5bc18b6 100644--- a/internal/application/service/tenant_skill_bundle.go+++ b/internal/application/service/tenant_skill_bundle.go@@ -36,7 +36,8 @@ type SkillBundle struct {Description stringInstructions string// SHA256 is over the uploaded bytes, so re-uploading the same archive is- // recognisable in the UI and in the ledger.+ // recognisable in the UI and in the ledger, and a ready skill with this+ // digest can skip a billed snapshot rebuild.SHA256 string// Files maps skill-root-relative paths to contents, SKILL.md included.Files map[string][]bytediff --git a/internal/application/service/tenant_skill_install.go b/internal/application/service/tenant_skill_install.goindex 0644016dd..83b24929f 100644--- a/internal/application/service/tenant_skill_install.go+++ b/internal/application/service/tenant_skill_install.go@@ -8,6 +8,7 @@ import ("path""sort""strings"+ "sync""time""github.com/google/uuid"@@ -63,6 +64,12 @@ func (s *TenantSkillService) InstallSkill(if err != nil {return "", err}+ if s.canSkipInstall(ctx, existing, bundle) {+ if err := s.refreshSkippedBundle(ctx, existing, archive); err != nil {+ return "", fmt.Errorf("store bundle for skill %s: %w", existing.ID, err)+ }+ return existing.ID, nil+ }skillID := uuid.NewString()now := s.now()@@ -201,6 +208,14 @@ func (s *TenantSkillService) runInstall(return nil}+ // From here on this run is the row's owner, and everything below can take+ // minutes. The heartbeat is what tells a second upload of the same archive+ // (and the reaper) that those minutes are work rather than a dead process.+ // It is deferred before it is stopped explicitly below, so a failure path+ // still stops it ahead of the deferred failSkill.+ stopHeartbeat := s.startInstallHeartbeat(ctx, tenantID, configID, skillID)+ defer stopHeartbeat()+// The name comes from SKILL.md and is already validated on parse, so a// rejection here means the bundle was accepted by a looser rule than the// one the image path enforces. Failing before any sandbox work keeps that@@ -349,6 +364,10 @@ func (s *TenantSkillService) runInstall(return err}pointerSwitched = true+ // The heartbeat writes the whole row, so it must be gone before the+ // terminal "ready" write below: a beat landing after it would put the row+ // back to installing and have the reaper fail a skill that is serving.+ stopHeartbeat()s.markPreviousSnapshotsSuperseded(ctx, tenantID, configID, installRowID)// The terminal write is the one that must not be best-effort: the pointer@@ -873,9 +892,10 @@ func (s *TenantSkillService) failSkill(// installStillOwnsTheRow is the lock-side counterpart of InstallSkill's// optimistic row write. A remove that ran first deleted the row; a newer// upload of the same name replaced BundleSHA256; a queued remove flipped the-// status. Any of those means this run must not snapshot — failSkill would-// stamp the newer owner's row, and a snapshot with no matching row is an-// orphan the ledger cannot name.+// status; a sibling retry of the same archive found the first run had already+// landed in the live image. Any of those means this run must not snapshot —+// failSkill would stamp the newer owner's row, and a snapshot with no matching+// row is an orphan the ledger cannot name.func (s *TenantSkillService) installStillOwnsTheRow(ctx context.Context, tenantID uint64, configID, skillID string, bundle *SkillBundle,) (bool, error) {@@ -892,9 +912,147 @@ func (s *TenantSkillService) installStillOwnsTheRow(if bundle != nil && current.BundleSHA256 != "" && current.BundleSHA256 != bundle.SHA256 {return false, nil}+ if current.Status == types.SkillStatusReady {+ _, inImage, ok := s.skillFilesInLiveImage(ctx, current)+ if ok && inImage {+ return false, nil+ }+ }return true, nil}+// canSkipInstall reports whether this upload is a no-op. Re-uploading the+// exact archive of a skill that is already ready (and still in the live image)+// must not boot a billed sandbox or grow a new snapshot. An install of the+// same bytes that is still beating is the same situation: the first run owns+// the work.+//+// Only a ready row is answered from the image. An installing row is answered+// from the heartbeat alone, deliberately: the ledger records which skill an+// install snapshotted, not which archive, so a row that is installing bundle+// B while the image still carries the earlier bundle A would look "already+// installed" and this upload would report a success that never happened.+//+// A failed skill with the same digest is a retry: the previous attempt never+// made it into the image. A removal in flight is not a skip either — taking+// the row back to installing is how an upload cancels it.+func (s *TenantSkillService) canSkipInstall(+ ctx context.Context, existing *types.TenantSkillEntity, bundle *SkillBundle,+) bool {+ if existing == nil || bundle == nil {+ return false+ }+ if existing.BundleSHA256 == "" || existing.BundleSHA256 != bundle.SHA256 {+ return false+ }+ switch existing.Status {+ case types.SkillStatusInstalling:+ return s.installIsInFlight(existing)+ case types.SkillStatusReady:+ _, inImage, ok := s.skillFilesInLiveImage(ctx, existing)+ return ok && inImage+ default:+ return false+ }+}++// installIsInFlight reports whether an installing row still belongs to a live+// process. The answer is the heartbeat: a running install restamps+// InstallingSince every skillInstallHeartbeatInterval, so silence past+// skillInstallInFlightSkip means the process is gone and the next upload must+// be allowed to start a new run rather than wait for the stuck-run reaper.+//+// Reading the submission time instead would force a choice between calling a+// slow install dead — a single agent command may take installCommandTimeout,+// and an install runs several — and leaving a dead one unrecoverable.+func (s *TenantSkillService) installIsInFlight(existing *types.TenantSkillEntity) bool {+ if existing == nil || existing.InstallingSince == nil {+ return false+ }+ return !existing.InstallingSince.Before(s.clock()().Add(-skillInstallInFlightSkip))+}++// startInstallHeartbeat keeps this run's liveness visible while it works, and+// returns the stop function that must be called before any terminal write.+//+// The heartbeat writes the whole row, so it would otherwise race the "ready"+// write past the pointer switch and revive an installing status. Both callers+// stop it before that point: runInstall stops it the moment the pointer moves,+// and the deferred stop runs before the deferred failSkill.+func (s *TenantSkillService) startInstallHeartbeat(+ ctx context.Context, tenantID uint64, configID, skillID string,+) func() {+ interval := s.installHeartbeat+ if interval <= 0 {+ interval = skillInstallHeartbeatInterval+ }+ beatCtx, stop := context.WithCancel(ctx)+ done := make(chan struct{})+ go func() {+ defer close(done)+ ticker := time.NewTicker(interval)+ defer ticker.Stop()+ for {+ select {+ case <-beatCtx.Done():+ return+ case <-ticker.C:+ s.beatInstallHeartbeat(beatCtx, tenantID, configID, skillID)+ }+ }+ }()+ var once sync.Once+ return func() {+ once.Do(func() {+ stop()+ <-done+ })+ }+}++// beatInstallHeartbeat stamps InstallingSince for a row this run still owns.+// A row that has left the installing status belongs to a newer upload, a+// queued removal, or a finished run, and reviving its timestamp would hide+// one of those from the reaper.+func (s *TenantSkillService) beatInstallHeartbeat(+ ctx context.Context, tenantID uint64, configID, skillID string,+) {+ current, err := s.skills.GetSkill(ctx, tenantID, configID, skillID)+ if err != nil {+ logger.Warnf(ctx, "[skill] load %s for install heartbeat failed: %v", skillID, err)+ return+ }+ if current == nil || current.Status != types.SkillStatusInstalling {+ return+ }+ at := s.clock()()+ current.InstallingSince = &at+ if err := s.skills.UpdateSkill(ctx, current); err != nil {+ logger.Warnf(ctx, "[skill] install heartbeat for %s failed: %v", skillID, err)+ }+}++// refreshSkippedBundle stores the uploaded archive even when the image work+// is skipped. read_skill serves file contents from it, so a re-upload of a+// ready skill is how a missing object-store blob gets repaired without+// growing a new snapshot. A failure here is returned to the caller rather+// than turning the ready row into a failed install.+func (s *TenantSkillService) refreshSkippedBundle(+ ctx context.Context, existing *types.TenantSkillEntity, archive []byte,+) error {+ if existing == nil {+ return nil+ }+ ref, err := s.saveBundle(ctx, existing.TenantID, existing.ID, archive)+ if err != nil {+ return err+ }+ return s.updateSkillFields(ctx, existing.TenantID, existing.SandboxConfigID, existing.ID,+ func(e *types.TenantSkillEntity) {+ e.BundleRef = ref+ })+}+// startMaintenanceSession opens the session one image operation runs in. The// operation name is carried into the session because the transcript is kept// deliberately, for troubleshooting: filing a removal's under "Skill install"diff --git a/internal/application/service/tenant_skill_install_test.go b/internal/application/service/tenant_skill_install_test.goindex 9b9f73cc3..e8c03a41d 100644--- a/internal/application/service/tenant_skill_install_test.go+++ b/internal/application/service/tenant_skill_install_test.go@@ -310,6 +310,321 @@ func TestInstallSkillRefusesWhenBundleCannotBeStored(t *testing.T) {require.NotContains(t, fx.events, "create-session")}+func TestInstallSkillSkipsWhenReadyWithTheSameArchive(t *testing.T) {+ fx := newInstallFixture(t)+ archive := zipBundle(t, map[string]string{+ "SKILL.md": validSkillMD,+ "scripts/extract.py": "print('hi')\n",+ })+ bundle, err := ParseSkillBundle(archive)+ require.NoError(t, err)+ fx.seedReadySkillWithSHA(bundle.SHA256, "snap-live")++ id, err := fx.svc.InstallSkill(context.Background(), 7, "cfg-1", archive)++ require.NoError(t, err)+ require.Equal(t, "sk-1", id)+ skill, getErr := fx.skillRepo.GetSkill(context.Background(), 7, "cfg-1", "sk-1")+ require.NoError(t, getErr)+ require.Equal(t, types.SkillStatusReady, skill.Status,+ "a ready skill whose archive did not change must not be flipped to installing")+ require.Empty(t, fx.sessionCalls, "the same bytes must not boot a billed sandbox")+ require.NotContains(t, fx.events, "create-snapshot")+ require.Nil(t, fx.configRepo.saved, "the image pointer must stay where it is")+ require.Equal(t, 1, fx.savedBundles,+ "a no-op re-upload must still refresh the stored archive for read_skill")+ require.Equal(t, "file://bundle.zip", skill.BundleRef)+}++func TestInstallSkillRetriesAFailedSkillWithTheSameArchive(t *testing.T) {+ fx := newInstallFixture(t)+ archive := zipBundle(t, map[string]string{+ "SKILL.md": validSkillMD,+ "scripts/extract.py": "print('hi')\n",+ })+ bundle, err := ParseSkillBundle(archive)+ require.NoError(t, err)+ require.NoError(t, fx.skillRepo.UpdateSkill(context.Background(), &types.TenantSkillEntity{+ ID: "sk-1", TenantID: 7, SandboxConfigID: "cfg-1",+ Name: bundle.Name, BundleSHA256: bundle.SHA256,+ Status: types.SkillStatusFailed, Error: "previous run died",+ }))++ id, err := fx.svc.InstallSkill(context.Background(), 7, "cfg-1", archive)++ require.NoError(t, err)+ require.Equal(t, "sk-1", id)+ skill, getErr := fx.skillRepo.GetSkill(context.Background(), 7, "cfg-1", "sk-1")+ require.NoError(t, getErr)+ require.Equal(t, types.SkillStatusInstalling, skill.Status,+ "a failed skill is a retry even when the archive digest is unchanged")+}++func TestInstallSkillReinstallsWhenTheLiveImageNoLongerCarriesTheSkill(t *testing.T) {+ fx := newInstallFixture(t)+ archive := zipBundle(t, map[string]string{+ "SKILL.md": validSkillMD,+ "scripts/extract.py": "print('hi')\n",+ })+ bundle, err := ParseSkillBundle(archive)+ require.NoError(t, err)+ require.NoError(t, fx.skillRepo.UpdateSkill(context.Background(), &types.TenantSkillEntity{+ ID: "sk-1", TenantID: 7, SandboxConfigID: "cfg-1",+ Name: bundle.Name, BundleSHA256: bundle.SHA256,+ Status: types.SkillStatusReady,+ }))+ // The pointer was cleared (last-skill removal, or a rebuild from base).+ // The row still says ready, but the files are gone from every new session.++ id, err := fx.svc.InstallSkill(context.Background(), 7, "cfg-1", archive)++ require.NoError(t, err)+ require.Equal(t, "sk-1", id)+ skill, getErr := fx.skillRepo.GetSkill(context.Background(), 7, "cfg-1", "sk-1")+ require.NoError(t, getErr)+ require.Equal(t, types.SkillStatusInstalling, skill.Status,+ "a ready row whose files left the image is a repair, not a skip")+}++func TestInstallSkillSkipsAnInFlightInstallOfTheSameArchive(t *testing.T) {+ fx := newInstallFixture(t)+ archive := zipBundle(t, map[string]string{+ "SKILL.md": validSkillMD,+ "scripts/extract.py": "print('hi')\n",+ })+ bundle, err := ParseSkillBundle(archive)+ require.NoError(t, err)+ // One heartbeat ago: the first run is slow, not gone.+ beat := fx.now().Add(-skillInstallHeartbeatInterval)+ require.NoError(t, fx.skillRepo.UpdateSkill(context.Background(), &types.TenantSkillEntity{+ ID: "sk-1", TenantID: 7, SandboxConfigID: "cfg-1",+ Name: bundle.Name, BundleSHA256: bundle.SHA256,+ Status: types.SkillStatusInstalling, InstallingSince: &beat,+ }))++ id, err := fx.svc.InstallSkill(context.Background(), 7, "cfg-1", archive)++ require.NoError(t, err)+ require.Equal(t, "sk-1", id)+ require.Empty(t, fx.sessionCalls,+ "a second upload of the same bytes must not start another billed run")+}++// A run that keeps beating is left alone however long it takes: a single agent+// command may take installCommandTimeout, and an install runs several.+func TestInstallSkillSkipsAnInstallThatIsSlowButStillBeating(t *testing.T) {+ fx := newInstallFixture(t)+ archive := zipBundle(t, map[string]string{+ "SKILL.md": validSkillMD,+ "scripts/extract.py": "print('hi')\n",+ })+ bundle, err := ParseSkillBundle(archive)+ require.NoError(t, err)+ submitted := fx.now().Add(-3 * installCommandTimeout)+ beat := fx.now().Add(-skillInstallHeartbeatInterval)+ require.NoError(t, fx.skillRepo.UpdateSkill(context.Background(), &types.TenantSkillEntity{+ ID: "sk-1", TenantID: 7, SandboxConfigID: "cfg-1",+ Name: bundle.Name, BundleSHA256: bundle.SHA256,+ Status: types.SkillStatusInstalling, InstallingSince: &beat,+ CreatedAt: submitted,+ }))++ id, err := fx.svc.InstallSkill(context.Background(), 7, "cfg-1", archive)++ require.NoError(t, err)+ require.Equal(t, "sk-1", id)+ require.Empty(t, fx.sessionCalls,+ "an install that started long ago but is still beating must not be restarted")+}++func TestInstallSkillRetriesAStaleInFlightInstallOfTheSameArchive(t *testing.T) {+ fx := newInstallFixture(t)+ archive := zipBundle(t, map[string]string{+ "SKILL.md": validSkillMD,+ "scripts/extract.py": "print('hi')\n",+ })+ bundle, err := ParseSkillBundle(archive)+ require.NoError(t, err)+ // The heartbeat stopped: the process that owned this row is gone.+ stale := fx.now().Add(-skillInstallInFlightSkip - time.Minute)+ require.NoError(t, fx.skillRepo.UpdateSkill(context.Background(), &types.TenantSkillEntity{+ ID: "sk-1", TenantID: 7, SandboxConfigID: "cfg-1",+ Name: bundle.Name, BundleSHA256: bundle.SHA256,+ Status: types.SkillStatusInstalling, InstallingSince: &stale,+ Error: "the previous process is gone",+ }))++ id, err := fx.svc.InstallSkill(context.Background(), 7, "cfg-1", archive)++ require.NoError(t, err)+ require.Equal(t, "sk-1", id)+ skill, getErr := fx.skillRepo.GetSkill(context.Background(), 7, "cfg-1", "sk-1")+ require.NoError(t, getErr)+ require.Equal(t, types.SkillStatusInstalling, skill.Status)+ require.NotNil(t, skill.InstallingSince)+ require.Equal(t, fx.now(), *skill.InstallingSince,+ "a dead in-flight row must be allowed to start a new run, not wait for the reaper")+ require.Empty(t, skill.Error)+}++// The ledger records which skill an install snapshotted, not which archive, so+// an installing row must never be answered from the image: the files there may+// belong to the previous bundle of the same skill, and skipping would report a+// success that never happened.+func TestCanSkipInstallNeverAnswersAnInstallingRowFromTheImage(t *testing.T) {+ fx := newInstallFixture(t)+ ctx := context.Background()+ fx.seedReadySkillWithSHA(fx.bundle.SHA256, "snap-live")+ existing, err := fx.skillRepo.GetSkill(ctx, 7, "cfg-1", "sk-1")+ require.NoError(t, err)+ stale := fx.now().Add(-skillInstallInFlightSkip - time.Minute)+ existing.Status = types.SkillStatusInstalling+ existing.InstallingSince = &stale++ require.False(t, fx.svc.canSkipInstall(ctx, existing, fx.bundle),+ "a dead install must be retried, not declared done from another bundle's snapshot")+}++// A ready skill is only skipped when the ledger can actually say the files are+// still in the live image. An unreadable ledger must reinstall rather than+// report a success nobody verified.+func TestCanSkipInstallRequiresAReadableLedger(t *testing.T) {+ fx := newInstallFixture(t)+ ctx := context.Background()+ fx.seedReadySkillWithSHA(fx.bundle.SHA256, "snap-live")+ existing, err := fx.skillRepo.GetSkill(ctx, 7, "cfg-1", "sk-1")+ require.NoError(t, err)+ require.True(t, fx.svc.canSkipInstall(ctx, existing, fx.bundle),+ "the same archive of a ready skill still in the image is a no-op")++ fx.skillRepo.listSnapshotsErr = errors.New("ledger unavailable")++ require.False(t, fx.svc.canSkipInstall(ctx, existing, fx.bundle),+ "a skip must be earned by a readable ledger, not assumed")+}++func TestBeatInstallHeartbeatRestampsOnlyAnInstallingRow(t *testing.T) {+ fx := newInstallFixture(t)+ ctx := context.Background()+ stale := fx.now().Add(-time.Hour)+ require.NoError(t, fx.svc.updateSkillFields(ctx, 7, "cfg-1", "sk-1",+ func(e *types.TenantSkillEntity) { e.InstallingSince = &stale }))++ fx.svc.beatInstallHeartbeat(ctx, 7, "cfg-1", "sk-1")++ skill, err := fx.skillRepo.GetSkill(ctx, 7, "cfg-1", "sk-1")+ require.NoError(t, err)+ require.Equal(t, fx.now(), *skill.InstallingSince,+ "a live install must keep its liveness timestamp current")++ // A finished run's row is no longer this install's to touch: reviving the+ // timestamp would hide a ready skill from nothing and a newer upload from+ // the reaper.+ require.NoError(t, fx.svc.updateSkillFields(ctx, 7, "cfg-1", "sk-1",+ func(e *types.TenantSkillEntity) {+ e.Status = types.SkillStatusReady+ e.InstallingSince = nil+ }))++ fx.svc.beatInstallHeartbeat(ctx, 7, "cfg-1", "sk-1")++ skill, err = fx.skillRepo.GetSkill(ctx, 7, "cfg-1", "sk-1")+ require.NoError(t, err)+ require.Equal(t, types.SkillStatusReady, skill.Status)+ require.Nil(t, skill.InstallingSince,+ "a row that left the installing status must not be stamped alive again")+}++func TestStartInstallHeartbeatBeatsUntilStopped(t *testing.T) {+ fx := newInstallFixture(t)+ ctx := context.Background()+ fx.svc.installHeartbeat = time.Millisecond+ stale := fx.now().Add(-time.Hour)+ require.NoError(t, fx.svc.updateSkillFields(ctx, 7, "cfg-1", "sk-1",+ func(e *types.TenantSkillEntity) { e.InstallingSince = &stale }))++ stop := fx.svc.startInstallHeartbeat(ctx, 7, "cfg-1", "sk-1")+ require.Eventually(t, func() bool {+ skill, err := fx.skillRepo.GetSkill(ctx, 7, "cfg-1", "sk-1")+ return err == nil && skill.InstallingSince != nil && skill.InstallingSince.Equal(fx.now())+ }, 2*time.Second, time.Millisecond, "the heartbeat must restamp the row while the run works")+ stop()++ // Stopping is what lets the terminal write stand: a beat landing after it+ // would put a serving skill back to installing.+ require.NoError(t, fx.svc.updateSkillFields(ctx, 7, "cfg-1", "sk-1",+ func(e *types.TenantSkillEntity) {+ e.Status = types.SkillStatusReady+ e.InstallingSince = nil+ }))+ time.Sleep(20 * time.Millisecond)+ skill, err := fx.skillRepo.GetSkill(ctx, 7, "cfg-1", "sk-1")+ require.NoError(t, err)+ require.Equal(t, types.SkillStatusReady, skill.Status)+ require.Nil(t, skill.InstallingSince)+ stop()+}++func TestInstallSkillDoesNotSkipARemovalOfTheSameArchive(t *testing.T) {+ fx := newInstallFixture(t)+ archive := zipBundle(t, map[string]string{+ "SKILL.md": validSkillMD,+ "scripts/extract.py": "print('hi')\n",+ })+ bundle, err := ParseSkillBundle(archive)+ require.NoError(t, err)+ require.NoError(t, fx.skillRepo.UpdateSkill(context.Background(), &types.TenantSkillEntity{+ ID: "sk-1", TenantID: 7, SandboxConfigID: "cfg-1",+ Name: bundle.Name, BundleSHA256: bundle.SHA256,+ Status: types.SkillStatusRemoving,+ }))++ id, err := fx.svc.InstallSkill(context.Background(), 7, "cfg-1", archive)++ require.NoError(t, err)+ require.Equal(t, "sk-1", id)+ skill, getErr := fx.skillRepo.GetSkill(context.Background(), 7, "cfg-1", "sk-1")+ require.NoError(t, getErr)+ require.Equal(t, types.SkillStatusInstalling, skill.Status,+ "re-uploading during a removal is how the upload cancels it")+}++func TestInstallSkillSkipRefusesToPretendSuccessWhenBundleCannotBeStored(t *testing.T) {+ fx := newInstallFixture(t)+ archive := zipBundle(t, map[string]string{+ "SKILL.md": validSkillMD,+ "scripts/extract.py": "print('hi')\n",+ })+ bundle, err := ParseSkillBundle(archive)+ require.NoError(t, err)+ fx.seedReadySkillWithSHA(bundle.SHA256, "snap-live")+ fx.saveErr = errors.New("object store down")++ _, err = fx.svc.InstallSkill(context.Background(), 7, "cfg-1", archive)++ require.Error(t, err)+ require.ErrorContains(t, err, "store bundle")+ skill, getErr := fx.skillRepo.GetSkill(context.Background(), 7, "cfg-1", "sk-1")+ require.NoError(t, getErr)+ require.Equal(t, types.SkillStatusReady, skill.Status,+ "a storage failure on a no-op re-upload must not flip a serving skill to failed")+ require.Empty(t, fx.sessionCalls)+}++func TestRunInstallAbortsWhenTheSameArchiveIsAlreadyServing(t *testing.T) {+ fx := newInstallFixture(t)+ fx.seedReadySkillWithSHA(fx.bundle.SHA256, "snap-live")++ require.NoError(t, fx.svc.runInstall(context.Background(), 7, "cfg-1", "sk-1", fx.bundle))++ require.NotContains(t, fx.events, "create-snapshot",+ "a sibling retry that lost the race to the first run must not grow another snapshot")+ require.Nil(t, fx.configRepo.saved)+ skill, err := fx.skillRepo.GetSkill(context.Background(), 7, "cfg-1", "sk-1")+ require.NoError(t, err)+ require.Equal(t, types.SkillStatusReady, skill.Status)+}+func TestTenantForStoragePrefersMatchingContextTenant(t *testing.T) {svc := &TenantSkillService{}backendID := "backend-1"@@ -833,7 +1148,8 @@ type installFixture struct {engineModel chat.Chat// saveErr fails bundle storage so InstallSkill cannot accept a skill// whose archive will later be unreadable.- saveErr error+ saveErr error+ savedBundles int}func newInstallFixture(t *testing.T) *installFixture {@@ -905,6 +1221,10 @@ func (f *installFixture) record(event string) {f.events = append(f.events, event)}+// now is the fixture's clock, so a test can express "one heartbeat ago"+// against the same instant the service reads.+func (f *installFixture) now() time.Time { return f.svc.now() }+// seedInstalledSkill puts the fixture in the state a removal starts from: the// skill is ready inside the config's current image, the ledger holds the active// row that produced that image, and the image manifest lists the skill.@@ -945,6 +1265,33 @@ func (f *installFixture) seedInstalledSkill(skillID, snapshotID string, generatif.sandboxMgr.manifest = payload}+// seedReadySkillWithSHA puts the fixture in the state a no-op re-upload starts+// from: the skill is ready, the digest matches the archive about to be posted,+// and the ledger says those files are still on the live image.+func (f *installFixture) seedReadySkillWithSHA(sha256, snapshotID string) {+ f.t.Helper()+ ctx := context.Background()+ skill, err := f.skillRepo.GetSkill(ctx, 7, "cfg-1", "sk-1")+ require.NoError(f.t, err)+ require.NotNil(f.t, skill)+ skill.Status = types.SkillStatusReady+ skill.BundleSHA256 = sha256+ skill.InstalledSnapshotID = snapshotID+ skill.Error = ""+ skill.InstallingSince = nil+ require.NoError(f.t, f.skillRepo.UpdateSkill(ctx, skill))++ require.NoError(f.t, f.skillRepo.CreateSnapshotRow(ctx, &types.TenantSkillSnapshotEntity{+ ID: "row-live", TenantID: 7, SandboxConfigID: "cfg-1", SkillID: "sk-1",+ SnapshotID: snapshotID, Generation: 1,+ Trigger: types.SkillSnapshotTriggerInstall, State: types.SkillSnapshotStateActive,+ }))+ f.configRepo.entity.Config.SkillImage = &types.SkillImageConfig{+ SnapshotID: snapshotID, Generation: 1,+ BaseTemplateID: "base-template", OwnerFingerprint: f.fingerprint,+ }+}+type installConfigRepo struct {fx *installFixtureentity *types.TenantSandboxConfigEntity@@ -1046,6 +1393,9 @@ type installSkillRepo struct {// snapshot failing for one state, leaving the snapshot ID nowhere but a// local variable.markStateFails func(state string) bool+ // listSnapshotsErr models an unreadable ledger, which is what stands+ // between "the image still carries this skill" and a guess.+ listSnapshotsErr error// deleteSkillErr models the row delete failing past the point of no// return.deleteSkillErr error@@ -1215,6 +1565,9 @@ func (r *installSkillRepo) ListSnapshotsByConfig() ([]*types.TenantSkillSnapshotEntity, error) {r.mu.Lock()defer r.mu.Unlock()+ if r.listSnapshotsErr != nil {+ return nil, r.listSnapshotsErr+ }var out []*types.TenantSkillSnapshotEntityfor _, e := range r.snapshots {if e.TenantID == tenantID && e.SandboxConfigID == configID {@@ -1751,8 +2104,11 @@ func (installFileService) SaveFile(context.Context, *multipart.FileHeader, uint6}func (s installFileService) SaveBytes(context.Context, []byte, uint64, string, bool) (string, error) {- if s.fx != nil && s.fx.saveErr != nil {- return "", s.fx.saveErr+ if s.fx != nil {+ s.fx.savedBundles+++ if s.fx.saveErr != nil {+ return "", s.fx.saveErr+ }}return "file://bundle.zip", nil}diff --git a/internal/application/service/tenant_skill_reaper.go b/internal/application/service/tenant_skill_reaper.goindex ed16f4e97..882351bb6 100644--- a/internal/application/service/tenant_skill_reaper.go+++ b/internal/application/service/tenant_skill_reaper.go@@ -47,6 +47,13 @@ type skillSnapshotLister interface {ListSnapshots(ctx context.Context, sandboxID string) ([]sandbox.RemoteSnapshotRef, error)}+// skillSnapshotDeleter is the provider delete PruneSupersededSnapshots is+// allowed to call. It is a separate surface from the lister so reconcile+// cannot grow a delete by accident: extras not in the ledger stay untouched.+type skillSnapshotDeleter interface {+ DeleteSnapshot(ctx context.Context, snapshotID string) error+}+// sandboxConfigEnumerator walks every sandbox config for the orphan-snapshot// sweep. ListAll is housekeeping-only.type sandboxConfigEnumerator interface {@@ -59,6 +66,7 @@ var (_ skillReaperConfigReader = (repository.TenantSandboxConfigRepository)(nil)_ sandboxConfigEnumerator = (repository.TenantSandboxConfigRepository)(nil)_ skillSnapshotLister = (sandbox.RemoteSnapshotManager)(nil)+ _ skillSnapshotDeleter = (*sandbox.SessionBoundManager)(nil))// ReapStuckRuns recovers skill rows whose install or remove process died.@@ -67,11 +75,12 @@ var (// still carry this skill's files — and skillFilesInLiveImage answers it from// the snapshot ledger rather than from the row.//-// An installing row older than skillInstallStuckTTL is healed to ready when-// the files are there: a re-install that died before the pointer moved, or a-// terminal ready write that never landed. Leaving it at installing would hide-// a skill the image still carries. Otherwise it becomes failed so the UI stops-// spinning.+// An installing row whose heartbeat has been silent for skillInstallStuckTTL+// is healed to ready when the files are there: a re-install that died before+// the pointer moved, or a terminal ready write that never landed. Leaving it+// at installing would hide a skill the image still carries. Otherwise it+// becomes failed so the UI stops spinning. A live install keeps stamping+// InstallingSince, so a run that is merely slow is never swept.//// A removing row is restored to ready while the files are still there, so the// operator can retry. Once they are gone the leftover row is deleted, so the@@ -80,11 +89,7 @@ func (s *TenantSkillService) ReapStuckRuns(ctx context.Context) (int, error) {if s == nil || s.skills == nil {return 0, nil}- now := s.now- if now == nil {- now = time.Now- }- cutoff := now().Add(-skillInstallStuckTTL)+ cutoff := s.clock()().Add(-skillInstallStuckTTL)stale, err := s.skills.ListStaleInstalling(ctx, cutoff)if err != nil {return 0, err@@ -344,6 +349,204 @@ func snapshotListerFrom(return lister}+func snapshotDeleterFrom(+ ctx context.Context, resolver sandbox.TenantSandboxResolver, tenantID uint64, configID string,+) skillSnapshotDeleter {+ if resolver == nil {+ return nil+ }+ mgr, err := resolver.Resolve(ctx, tenantID, configID)+ if err != nil {+ logger.Warnf(ctx, "[skill] resolve sandbox for snapshot prune of %s failed: %v", configID, err)+ return nil+ }+ if mgr == nil {+ return nil+ }+ deleter, ok := mgr.(skillSnapshotDeleter)+ if !ok {+ return nil+ }+ return deleter+}++func (s *TenantSkillService) snapshotRetentionWindow() time.Duration {+ if s != nil && s.snapshotRetention > 0 {+ return s.snapshotRetention+ }+ return skillSnapshotRetention+}++// snapshotRetentionFor is how long this config's retired snapshots stay on+// the provider. The floor is snapshotRetentionWindow; a config that asked+// for a sandbox TTL longer than that keeps the previous template at least+// that long plus a margin, so a session created from it can still exist.+func (s *TenantSkillService) snapshotRetentionFor(cfg *types.TenantSandboxConfigEntity) time.Duration {+ window := s.snapshotRetentionWindow()+ ttl := time.Duration(0)+ if cfg != nil && cfg.Config != nil {+ ttl = configuredSandboxTTL(cfg.Config)+ }+ if needed := ttl + skillSnapshotTTLMargin; needed > window {+ return needed+ }+ return window+}++func configuredSandboxTTL(cfg *types.TenantSandboxConfig) time.Duration {+ if cfg == nil {+ return 0+ }+ seconds := 0+ if cfg.Cube != nil && cfg.Cube.CubeSandboxTTLSeconds > seconds {+ seconds = cfg.Cube.CubeSandboxTTLSeconds+ }+ if cfg.E2B != nil && cfg.E2B.E2BSandboxTTLSeconds > seconds {+ seconds = cfg.E2B.E2BSandboxTTLSeconds+ }+ if seconds <= 0 {+ return 0+ }+ return time.Duration(seconds) * time.Second+}++// PruneSupersededSnapshots deletes provider snapshots the ledger has already+// retired, once they are older than this config's retention. The current+// image is never touched, nor is anything the ledger does not name: extras+// belong to other environments on a shared provider account.+//+// A live sandbox does not need the template it was created from in order to+// keep running, so the only reason to wait is in-flight creates that resolved+// the previous pointer. Twenty-four hours is far past every backend's default+// TTL; a config that sets a longer one extends the wait.+func (s *TenantSkillService) PruneSupersededSnapshots(ctx context.Context) (int, error) {+ if s == nil || s.skills == nil {+ return 0, nil+ }+ enum, ok := s.configs.(sandboxConfigEnumerator)+ if !ok {+ return 0, nil+ }+ configs, err := enum.ListAll(ctx)+ if err != nil {+ return 0, err+ }+ now := s.clock()+ pruned := 0+ for _, cfg := range configs {+ if cfg == nil || types.IsSandboxWorkspacePolicyRow(cfg) {+ continue+ }+ cutoff := now().Add(-s.snapshotRetentionFor(cfg))+ n, err := s.pruneConfigSnapshots(ctx, cfg, cutoff)+ if err != nil {+ logger.Warnf(ctx, "[skill] prune superseded snapshots of config %s failed: %v",+ cfg.ID, err)+ continue+ }+ pruned += n+ }+ return pruned, nil+}++func (s *TenantSkillService) pruneConfigSnapshots(+ ctx context.Context, cfg *types.TenantSandboxConfigEntity, cutoff time.Time,+) (int, error) {+ if cfg == nil {+ return 0, nil+ }+ // A rotated credential points at a different provider account, where the+ // ledger's snapshot IDs do not exist. The delete would come back+ // not-found, which this sweep reads as "already gone", so the account that+ // really holds those snapshots would keep being billed for them while the+ // ledger recorded them as deleted. ensureUsableImage stops installs for+ // the same reason; this stops the irreversible half.+ if err := ensureUsableImage(cfg); err != nil {+ logger.Warnf(ctx, "[skill] skip snapshot prune of config %s: %v", cfg.ID, err)+ return 0, nil+ }+ rows, err := s.skills.ListSnapshotsByConfig(ctx, cfg.TenantID, cfg.ID)+ if err != nil {+ return 0, err+ }+ live := currentSnapshotID(cfg)+ // Resolving builds a provider client, so it waits until a row is actually+ // eligible: most configs have nothing to prune on most sweeps.+ var deleter skillSnapshotDeleter+ resolved := false+ pruned := 0+ for _, row := range rows {+ if !snapshotEligibleForPrune(row, live, cutoff) {+ continue+ }+ if !resolved {+ deleter = snapshotDeleterFrom(ctx, s.sandboxes, cfg.TenantID, cfg.ID)+ resolved = true+ }+ if deleter == nil {+ logger.Warnf(ctx,+ "[skill] cannot prune snapshot %s of config %s: provider does not support delete",+ row.SnapshotID, cfg.ID)+ continue+ }+ if err := deleter.DeleteSnapshot(ctx, row.SnapshotID); err != nil && !sandbox.IsRemoteNotFound(err) {+ logger.Warnf(ctx, "[skill] delete superseded snapshot %s failed: %v", row.SnapshotID, err)+ continue+ }+ if err := s.skills.MarkSnapshotState(+ ctx, cfg.TenantID, row.ID, types.SkillSnapshotStateDeleted, row.SnapshotID,+ ); err != nil {+ logger.Warnf(ctx, "[skill] mark snapshot %s deleted after prune failed: %v", row.ID, err)+ continue+ }+ pruned+++ }+ return pruned, nil+}++// snapshotEligibleForPrune is the ledger-side gate. The provider delete is+// what costs money; this is what keeps it from touching the live image or a+// snapshot another environment created on the same account.+//+// Superseded rows are the normal case. Active rows that are not the live+// pointer are the crash window between switchImagePointer and+// markPreviousSnapshotsSuperseded: they are billed leftovers too, aged from+// UpdatedAt / CreatedAt because they never got a SupersededAt.+func snapshotEligibleForPrune(+ row *types.TenantSkillSnapshotEntity, liveSnapshotID string, cutoff time.Time,+) bool {+ if row == nil {+ return false+ }+ id := strings.TrimSpace(row.SnapshotID)+ if id == "" || id == strings.TrimSpace(liveSnapshotID) {+ return false+ }+ switch row.State {+ case types.SkillSnapshotStateSuperseded, types.SkillSnapshotStateActive:+ default:+ return false+ }+ aged := snapshotPruneAge(row)+ return aged != nil && aged.Before(cutoff)+}++func snapshotPruneAge(row *types.TenantSkillSnapshotEntity) *time.Time {+ if row == nil {+ return nil+ }+ if row.State == types.SkillSnapshotStateSuperseded && row.SupersededAt != nil {+ return row.SupersededAt+ }+ if !row.UpdatedAt.IsZero() {+ return &row.UpdatedAt+ }+ if !row.CreatedAt.IsZero() {+ return &row.CreatedAt+ }+ return nil+}+func (s *TenantSkillService) reconcileAllSnapshots(ctx context.Context) {enum, ok := s.configs.(sandboxConfigEnumerator)if !ok {@@ -368,6 +571,9 @@ func (s *TenantSkillService) runSkillReaper(ctx context.Context) {if _, err := s.ReapStuckRuns(ctx); err != nil {logger.Warnf(ctx, "[skill] reap stuck runs failed: %v", err)}+ if _, err := s.PruneSupersededSnapshots(ctx); err != nil {+ logger.Warnf(ctx, "[skill] prune superseded snapshots failed: %v", err)+ }s.reconcileAllSnapshots(ctx)}diff --git a/internal/application/service/tenant_skill_reaper_test.go b/internal/application/service/tenant_skill_reaper_test.goindex 4b2aa544f..49cb6c426 100644--- a/internal/application/service/tenant_skill_reaper_test.go+++ b/internal/application/service/tenant_skill_reaper_test.go@@ -2,6 +2,7 @@ package serviceimport ("context"+ "errors""strings""testing""time"@@ -234,6 +235,200 @@ func TestReconcileSnapshotsWarnsExtrasWithoutDeleting(t *testing.T) {"extras are only warned; the same provider account may be shared across environments")}+func TestPruneSupersededSnapshotsDeletesOldLedgerSnapshots(t *testing.T) {+ fx := newReaperFixture(t)+ fx.svc.snapshotRetention = time.Hour+ old := fx.now.Add(-2 * time.Hour)+ recent := fx.now.Add(-30 * time.Minute)+ fx.live("snap-live")+ fx.superseded("sk-1", "snap-old", "", old)+ fx.superseded("sk-2", "snap-recent", "snap-old", recent)+ fx.installed("sk-3", "snap-live", "snap-recent")+ fx.provider.listed = []sandbox.RemoteSnapshotRef{+ {ID: "snap-old"}, {ID: "snap-recent"}, {ID: "snap-live"}, {ID: "snap-foreign"},+ }++ n, err := fx.svc.PruneSupersededSnapshots(context.Background())++ require.NoError(t, err)+ require.Equal(t, 1, n)+ require.Equal(t, []string{"snap-old"}, fx.provider.deleted,+ "only a superseded snapshot older than retention is a billed leftover")+ rows, err := fx.skills.ListSnapshotsByConfig(context.Background(), 7, "cfg-1")+ require.NoError(t, err)+ states := map[string]string{}+ for _, row := range rows {+ states[row.SnapshotID] = row.State+ }+ require.Equal(t, types.SkillSnapshotStateDeleted, states["snap-old"])+ require.Equal(t, types.SkillSnapshotStateSuperseded, states["snap-recent"],+ "a snapshot still inside the retention window may have live sandboxes")+ require.Equal(t, types.SkillSnapshotStateActive, states["snap-live"])+}++func TestPruneSupersededSnapshotsNeverDeletesTheLiveImage(t *testing.T) {+ fx := newReaperFixture(t)+ fx.svc.snapshotRetention = time.Hour+ old := fx.now.Add(-2 * time.Hour)+ fx.live("snap-live")+ fx.skills.snapshots = append(fx.skills.snapshots, &types.TenantSkillSnapshotEntity{+ ID: "row-wrong", TenantID: 7, SandboxConfigID: "cfg-1", SkillID: "sk-1",+ SnapshotID: "snap-live", Trigger: types.SkillSnapshotTriggerInstall,+ State: types.SkillSnapshotStateSuperseded, SupersededAt: &old,+ })++ n, err := fx.svc.PruneSupersededSnapshots(context.Background())++ require.NoError(t, err)+ require.Zero(t, n)+ require.Empty(t, fx.provider.deleted,+ "a ledger bug that marks the live snapshot superseded must not delete it")+}++func TestPruneSupersededSnapshotsNeverDeletesUnknownProviderSnapshots(t *testing.T) {+ fx := newReaperFixture(t)+ fx.svc.snapshotRetention = time.Hour+ fx.live("snap-live")+ fx.installed("sk-1", "snap-live", "")+ fx.provider.listed = []sandbox.RemoteSnapshotRef{+ {ID: "snap-live"}, {ID: "snap-foreign"},+ }++ n, err := fx.svc.PruneSupersededSnapshots(context.Background())++ require.NoError(t, err)+ require.Zero(t, n)+ require.Empty(t, fx.provider.deleted,+ "a snapshot the ledger does not name belongs to another environment")+}++func TestPruneSupersededSnapshotsDeletesStaleActiveLeftovers(t *testing.T) {+ fx := newReaperFixture(t)+ fx.svc.snapshotRetention = time.Hour+ old := fx.now.Add(-2 * time.Hour)+ fx.live("snap-live")+ fx.installed("sk-2", "snap-live", "snap-old")+ fx.skills.snapshots = append(fx.skills.snapshots, &types.TenantSkillSnapshotEntity{+ ID: "row-snap-old", TenantID: 7, SandboxConfigID: "cfg-1", SkillID: "sk-1",+ SnapshotID: "snap-old", Trigger: types.SkillSnapshotTriggerInstall,+ State: types.SkillSnapshotStateActive, CreatedAt: old, UpdatedAt: old,+ })++ n, err := fx.svc.PruneSupersededSnapshots(context.Background())++ require.NoError(t, err)+ require.Equal(t, 1, n)+ require.Equal(t, []string{"snap-old"}, fx.provider.deleted,+ "a pointer switch that never marked the previous row superseded still leaves a billed snapshot")+ rows, err := fx.skills.ListSnapshotsByConfig(context.Background(), 7, "cfg-1")+ require.NoError(t, err)+ states := map[string]string{}+ for _, row := range rows {+ states[row.SnapshotID] = row.State+ }+ require.Equal(t, types.SkillSnapshotStateDeleted, states["snap-old"])+ require.Equal(t, types.SkillSnapshotStateActive, states["snap-live"])+}++// A rotated credential points at another provider account, where these IDs do+// not exist. The delete would come back not-found, which the sweep reads as+// "already gone", so the account that really holds them would keep being+// billed while the ledger recorded them as deleted.+func TestPruneSupersededSnapshotsSkipsAConfigBuiltByAnotherAccount(t *testing.T) {+ fx := newReaperFixture(t)+ fx.svc.snapshotRetention = time.Hour+ old := fx.now.Add(-2 * time.Hour)+ fx.live("snap-live")+ fx.superseded("sk-1", "snap-old", "", old)+ fx.configs.entity.Config.E2B.APIKey = "rotated-key"++ n, err := fx.svc.PruneSupersededSnapshots(context.Background())++ require.NoError(t, err)+ require.Zero(t, n)+ require.Empty(t, fx.provider.deleted,+ "snapshots of an account we can no longer address must not be recorded as deleted")+ rows, err := fx.skills.ListSnapshotsByConfig(context.Background(), 7, "cfg-1")+ require.NoError(t, err)+ require.Equal(t, types.SkillSnapshotStateSuperseded, rows[0].State)+}++// Resolving a provider builds a client. Most configs have nothing to prune on+// most sweeps, and the sweep runs every five minutes across every workspace.+func TestPruneSupersededSnapshotsBuildsNoProviderClientWithNothingToPrune(t *testing.T) {+ fx := newReaperFixture(t)+ fx.svc.snapshotRetention = time.Hour+ recent := fx.now.Add(-30 * time.Minute)+ fx.live("snap-live")+ fx.installed("sk-1", "snap-live", "snap-recent")+ fx.superseded("sk-2", "snap-recent", "", recent)++ n, err := fx.svc.PruneSupersededSnapshots(context.Background())++ require.NoError(t, err)+ require.Zero(t, n)+ require.Zero(t, fx.resolver.resolves,+ "a sweep with no eligible row must not pay for a provider client")+}++func TestPruneSupersededSnapshotsLeavesTheRowWhenDeleteFails(t *testing.T) {+ fx := newReaperFixture(t)+ fx.svc.snapshotRetention = time.Hour+ old := fx.now.Add(-2 * time.Hour)+ fx.live("snap-live")+ fx.superseded("sk-1", "snap-old", "", old)+ fx.provider.deleteErr = errors.New("provider down")++ n, err := fx.svc.PruneSupersededSnapshots(context.Background())++ require.NoError(t, err)+ require.Zero(t, n)+ require.Empty(t, fx.provider.deleted)+ rows, err := fx.skills.ListSnapshotsByConfig(context.Background(), 7, "cfg-1")+ require.NoError(t, err)+ require.Equal(t, types.SkillSnapshotStateSuperseded, rows[0].State,+ "a failed provider delete must not be recorded as deleted")+}++func TestPruneSupersededSnapshotsTreatsMissingProviderSnapshotAsDeleted(t *testing.T) {+ fx := newReaperFixture(t)+ fx.svc.snapshotRetention = time.Hour+ old := fx.now.Add(-2 * time.Hour)+ fx.live("snap-live")+ fx.superseded("sk-1", "snap-old", "", old)+ fx.provider.deleteErr = &sandbox.RemoteError{+ Kind: sandbox.RemoteErrorKindNotFound, Op: "DeleteSnapshot",+ }++ n, err := fx.svc.PruneSupersededSnapshots(context.Background())++ require.NoError(t, err)+ require.Equal(t, 1, n)+ rows, err := fx.skills.ListSnapshotsByConfig(context.Background(), 7, "cfg-1")+ require.NoError(t, err)+ require.Equal(t, types.SkillSnapshotStateDeleted, rows[0].State,+ "a snapshot the provider already dropped is gone; the ledger must catch up")+}++func TestPruneSupersededSnapshotsHonoursALongerConfiguredSandboxTTL(t *testing.T) {+ fx := newReaperFixture(t)+ fx.svc.snapshotRetention = time.Hour+ fx.configs.entity.Config.E2B.E2BSandboxTTLSeconds = int((48 * time.Hour).Seconds())+ young := fx.now.Add(-25 * time.Hour)+ old := fx.now.Add(-50 * time.Hour)+ fx.live("snap-live")+ fx.superseded("sk-1", "snap-young", "", young)+ fx.superseded("sk-2", "snap-old", "snap-young", old)+ fx.installed("sk-3", "snap-live", "snap-old")++ n, err := fx.svc.PruneSupersededSnapshots(context.Background())++ require.NoError(t, err)+ require.Equal(t, 1, n)+ require.Equal(t, []string{"snap-old"}, fx.provider.deleted,+ "a config whose sandbox TTL exceeds the floor must keep templates that young")+}+func TestTenantSkillServiceStartIsIdempotent(t *testing.T) {svc := NewTenantSkillService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)@@ -249,28 +444,45 @@ type reaperFixture struct {skills *reaperSkillStoreconfigs *reaperConfigStoreprovider *reaperSnapshotProvider- now time.Time+ resolver *reaperSandboxResolver+ // fingerprint is the credential identity the stored image was built with.+ // A prune that could not tell it from another account's would delete+ // snapshots it cannot even see, so the fixture carries a real one.+ fingerprint string+ now time.Time}func newReaperFixture(t *testing.T) *reaperFixture {t.Helper()now := time.Date(2026, 8, 20, 12, 0, 0, 0, time.UTC)+ fingerprint := sandbox.SkillImageFingerprint("e2b", "key-1", "https://e2b.example")skills := &reaperSkillStore{rows: map[string]*types.TenantSkillEntity{}}configs := &reaperConfigStore{entity: &types.TenantSandboxConfigEntity{ID: "cfg-1", TenantID: 7,+ SandboxType: string(sandbox.SandboxTypeE2B),Config: &types.TenantSandboxConfig{- SkillImage: &types.SkillImageConfig{SnapshotID: "snap-other"},+ SandboxType: string(sandbox.SandboxTypeE2B),+ E2B: &types.E2BSandboxConfig{+ APIURL: "https://e2b.example", APIKey: "key-1", TemplateID: "base-template",+ },+ SkillImage: &types.SkillImageConfig{+ SnapshotID: "snap-other", OwnerFingerprint: fingerprint,+ },},},}provider := &reaperSnapshotProvider{}+ resolver := &reaperSandboxResolver{provider: provider}svc := NewTenantSkillService(- skills, configs, nil, &reaperSandboxResolver{provider: provider},+ skills, configs, nil, resolver,nil, nil, nil, nil, nil, nil, nil, nil,)svc.now = func() time.Time { return now }- return &reaperFixture{svc: svc, skills: skills, configs: configs, provider: provider, now: now}+ return &reaperFixture{+ svc: svc, skills: skills, configs: configs, provider: provider,+ resolver: resolver, fingerprint: fingerprint, now: now,+ }}// installed and removed write the ledger row an install or a removal leaves@@ -291,10 +503,21 @@ func (f *reaperFixture) snapshotRow(skillID, snapshotID, parentSnapshotID, trigg})}+func (f *reaperFixture) superseded(skillID, snapshotID, parentSnapshotID string, at time.Time) {+ f.skills.snapshots = append(f.skills.snapshots, &types.TenantSkillSnapshotEntity{+ ID: "row-" + snapshotID, TenantID: 7, SandboxConfigID: "cfg-1", SkillID: skillID,+ SnapshotID: snapshotID, ParentSnapshotID: parentSnapshotID,+ Trigger: types.SkillSnapshotTriggerInstall, State: types.SkillSnapshotStateSuperseded,+ SupersededAt: &at,+ })+}+// live points the config at a snapshot, the way an install's pointer switch// does.func (f *reaperFixture) live(snapshotID string) {- f.configs.entity.Config.SkillImage = &types.SkillImageConfig{SnapshotID: snapshotID}+ f.configs.entity.Config.SkillImage = &types.SkillImageConfig{+ SnapshotID: snapshotID, OwnerFingerprint: f.fingerprint,+ }}var (@@ -304,6 +527,7 @@ var (_ sandboxConfigEnumerator = (*reaperConfigStore)(nil)_ sandbox.TenantSandboxResolver = (*reaperSandboxResolver)(nil)_ skillSnapshotLister = (*reaperSnapshotProvider)(nil)+ _ skillSnapshotDeleter = (*reaperSnapshotProvider)(nil))type reaperSkillStore struct {@@ -395,9 +619,23 @@ func (r *reaperSkillStore) CreateSnapshotRow(_ context.Context, e *types.TenantS}func (r *reaperSkillStore) MarkSnapshotState(- context.Context, uint64, string, string, string,+ _ context.Context, tenantID uint64, id, state, snapshotID string,) error {- panic("MarkSnapshotState is outside the reaper surface")+ for _, e := range r.snapshots {+ if e == nil || e.ID != id || e.TenantID != tenantID {+ continue+ }+ e.State = state+ if snapshotID != "" {+ e.SnapshotID = snapshotID+ }+ if state == types.SkillSnapshotStateSuperseded {+ now := time.Now()+ e.SupersededAt = &now+ }+ return nil+ }+ return nil}func (r *reaperSkillStore) DeleteSnapshotRowsByConfig(context.Context, uint64, string) error {@@ -452,9 +690,13 @@ func (r *reaperConfigStore) ClearCordon(context.Context, uint64, string) error {type reaperSandboxResolver struct {provider *reaperSnapshotProvider+ // resolves counts provider constructions. Resolving builds a client, so a+ // sweep with nothing to do must not pay for one.+ resolves int}func (r *reaperSandboxResolver) Resolve(context.Context, uint64, string) (sandbox.Manager, error) {+ r.resolves++return r.provider, nil}@@ -464,6 +706,7 @@ type reaperSnapshotProvider struct {listed []sandbox.RemoteSnapshotReflistCalls []stringdeleted []string+ deleteErr error}func (p *reaperSnapshotProvider) ListSnapshots(@@ -474,6 +717,9 @@ func (p *reaperSnapshotProvider) ListSnapshots(}func (p *reaperSnapshotProvider) DeleteSnapshot(_ context.Context, snapshotID string) error {+ if p.deleteErr != nil {+ return p.deleteErr+ }p.deleted = append(p.deleted, snapshotID)return nil}diff --git a/internal/application/service/tenant_skill_service.go b/internal/application/service/tenant_skill_service.goindex ed4c0d3fa..8da4e70d5 100644--- a/internal/application/service/tenant_skill_service.go+++ b/internal/application/service/tenant_skill_service.go@@ -19,9 +19,39 @@ import (// lock without renewing. Installs run for minutes, so the lease is renewed by// redislock rather than being set long.const (- skillImageLockLease = 30 * time.Second- skillImageLockRenew = 10 * time.Second+ skillImageLockLease = 30 * time.Second+ skillImageLockRenew = 10 * time.Second++ // skillInstallStuckTTL is how long a run may go without a heartbeat+ // before the reaper treats it as abandoned. It is a silence budget, not+ // a duration budget: a legitimate install that spends two hours in the+ // agent keeps beating and is left alone.skillInstallStuckTTL = 60 * time.Minute++ // skillInstallHeartbeatInterval is how often a running install stamps+ // InstallingSince to say its process is still alive. Everything that has+ // to tell "still working" from "died" reads that timestamp.+ skillInstallHeartbeatInterval = 30 * time.Second++ // skillInstallInFlightSkip is how much heartbeat silence makes a second+ // upload of the same archive stop deferring to the run that owns the row.+ // It is a multiple of the heartbeat so a slow install is never mistaken+ // for a dead one, and short enough that a re-upload recovers a dead+ // process in minutes instead of waiting for skillInstallStuckTTL.+ skillInstallInFlightSkip = 3 * time.Minute++ // skillSnapshotRetention is how long a superseded snapshot stays on the+ // provider after the pointer has moved. Live sandboxes may still have+ // been created from it (especially SkillRolloutNewSession); once they+ // expire, the template is only a billed leftover. Twenty-four hours is+ // well past every backend's default sandbox TTL. A config that sets a+ // longer sandbox TTL extends this via snapshotRetentionFor.+ skillSnapshotRetention = 24 * time.Hour++ // skillSnapshotTTLMargin is added on top of a config's own sandbox TTL+ // so an in-flight create that resolved the previous pointer still has+ // a template to boot from.+ skillSnapshotTTLMargin = time.Hour)// TenantSkillService owns the skill image lifecycle for sandbox configs.@@ -52,6 +82,14 @@ type TenantSkillService struct {// test can let an install outlast it, which every real install does.cleanupTimeout time.Duration+ // snapshotRetention is how long a superseded snapshot is kept on the+ // provider. Injectable so a prune test can age a row without waiting a day.+ snapshotRetention time.Duration++ // installHeartbeat is how often a running install restamps its liveness.+ // Injectable so a test can observe a beat without waiting half a minute.+ installHeartbeat time.Duration+// localLocks serialises installs when Redis is absent. It only guards this// process; multi-replica deployments require Redis for cross-process safety.localLocks *keyedMutex@@ -79,21 +117,23 @@ func NewTenantSkillService(messages interfaces.MessageRepository,) *TenantSkillService {return &TenantSkillService{- skills: skillsRepo,- configs: configsRepo,- resolver: resolver,- sandboxes: sandboxes,- sandboxPolicy: sandboxPolicy,- agents: agents,- installerAgents: customAgents,- sessions: sessions,- models: models,- redis: redisClient,- streams: streams,- messages: messages,- now: time.Now,- cleanupTimeout: installCleanupTimeout,- localLocks: newKeyedMutex(),+ skills: skillsRepo,+ configs: configsRepo,+ resolver: resolver,+ sandboxes: sandboxes,+ sandboxPolicy: sandboxPolicy,+ agents: agents,+ installerAgents: customAgents,+ sessions: sessions,+ models: models,+ redis: redisClient,+ streams: streams,+ messages: messages,+ now: time.Now,+ cleanupTimeout: installCleanupTimeout,+ snapshotRetention: skillSnapshotRetention,+ installHeartbeat: skillInstallHeartbeatInterval,+ localLocks: newKeyedMutex(),cron: cron.New(cron.WithSeconds(), cron.WithChain(cron.Recover(cron.DefaultLogger),)),@@ -127,6 +167,15 @@ func skillImageLockKey(tenantID uint64, configID string) string {return fmt.Sprintf("weknora-skill-image-lock:%d:%s", tenantID, configID)}+// clock is this service's time source. Tests inject one; a service built+// without NewTenantSkillService still gets a working default.+func (s *TenantSkillService) clock() func() time.Time {+ if s != nil && s.now != nil {+ return s.now+ }+ return time.Now+}+// keyedMutex is the no-Redis fallback for withConfigLock.type keyedMutex struct {mu sync.Mutexdiff --git a/internal/sandbox/session_manager.go b/internal/sandbox/session_manager.goindex 0be39fd9a..3b503fae9 100644--- a/internal/sandbox/session_manager.go+++ b/internal/sandbox/session_manager.go@@ -426,8 +426,9 @@ func (m *SessionBoundManager) CreateSnapshot(return snapshots.CreateSnapshot(ctx, handle.ID(), name)}-// DeleteSnapshot forwards provider snapshot deletion. It is used only to clean-// up a just-created orphan when the DB pointer switch fails.+// DeleteSnapshot forwards provider snapshot deletion. The skill install path+// uses it to abandon an orphan when the pointer switch fails; the reaper uses+// it to prune superseded snapshots that have aged past retention.func (m *SessionBoundManager) DeleteSnapshot(ctx context.Context, snapshotID string) error {if err := m.requireRemoteBackend(); err != nil {return errdiff --git a/internal/types/tenant_skill.go b/internal/types/tenant_skill.goindex 8841b42aa..fad685f96 100644--- a/internal/types/tenant_skill.go+++ b/internal/types/tenant_skill.go@@ -16,7 +16,8 @@ const ()// Snapshot ledger states. deleted is written only after a real provider-side-// delete, which today happens only when the whole sandbox config is removed.+// delete: either the whole sandbox config is removed, or the reaper has+// pruned a retired snapshot past its retention window.const (SkillSnapshotStateBuilding = "building"SkillSnapshotStateActive = "active"