Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: preserve active cleanup owners when deleting completed agent history (#13430)
Adam Eivya14f3615181b
diff --git a/client/src/components/cos/tabs/AgentsTab.jsx b/client/src/components/cos/tabs/AgentsTab.jsxindex c168a46ed6..4a509fe79f 100644--- a/client/src/components/cos/tabs/AgentsTab.jsx+++ b/client/src/components/cos/tabs/AgentsTab.jsx@@ -188,7 +188,14 @@ export default function AgentsTab({ agentsLoaded = true, agentsError = null, onRsetConfirmingClear(false);const result = await api.clearCompletedCosAgents({ silent: true }).catch(err => { toast.error(err.message); return null; });if (!result) return;- toast.success('Cleared completed agents');+ if (result.retained > 0) {+ const retainedWord = result.retained === 1 ? '1 active cleanup owner retained' : `${result.retained} active cleanup owners retained`;+ toast.success(result.cleared > 0+ ? `Cleared ${result.cleared} completed agent${result.cleared === 1 ? '' : 's'} (${retainedWord})`+ : `No completed agents cleared (${retainedWord})`);+ } else {+ toast.success('Cleared completed agents');+ }refresh();};diff --git a/client/src/components/cos/tabs/AgentsTab.test.jsx b/client/src/components/cos/tabs/AgentsTab.test.jsxindex 049ad614f9..ca6a2b331e 100644--- a/client/src/components/cos/tabs/AgentsTab.test.jsx+++ b/client/src/components/cos/tabs/AgentsTab.test.jsx@@ -93,7 +93,14 @@ vi.mock('./RelaunchAgentModal', () => ({</button>),}));-vi.mock('../../ui/InlineConfirmRow', () => ({ default: ({ question }) => <p>{question}</p> }));+vi.mock('../../ui/InlineConfirmRow', () => ({+ default: ({ question, confirmText, onConfirm }) => (+ <div>+ <p>{question}</p>+ {onConfirm && <button type="button" onClick={onConfirm}>{confirmText || 'Confirm'}</button>}+ </div>+ ),+}));import * as api from '../../../services/api';import toast from '../../ui/Toast';@@ -489,6 +496,19 @@ it('makes the clear scope explicit when the feedback filter hides completed recoexpect(screen.queryByText(/removes 1 agent record/)).not.toBeInTheDocument();});+it('explains retained active cleanup owners when clearing completed agents', async () => {+ const user = userEvent.setup();+ api.getCosCompletedAgents.mockResolvedValue({ items: [completedAgent('done', 'Completed work')], total: 2, nextCursor: null });+ api.clearCompletedCosAgents.mockResolvedValue({ cleared: 1, retained: 1 });+ const onRefresh = vi.fn();+ renderTab([], onRefresh);+ await screen.findByText('Completed work');+ await user.click(screen.getByRole('button', { name: 'Clear all completed agents' }));+ await user.click(screen.getByRole('button', { name: 'Clear all' }));+ expect(toast.success).toHaveBeenCalledWith('Cleared 1 completed agent (1 active cleanup owner retained)');+ expect(onRefresh).toHaveBeenCalled();+});+const deferred = () => {let resolve;diff --git a/server/lib/README.md b/server/lib/README.mdindex dd344b8831..222008e920 100644--- a/server/lib/README.md+++ b/server/lib/README.md@@ -582,7 +582,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub| `taskPauseHold.js` | The user-PAUSE hold — pure metadata helpers shared by the writer (`pauseAgent`), the reader (`resumeAgent`), and the clearer (`updateTask`). `AGENT_PAUSED_CATEGORY` is the `blockedCategory` a pause stamps; `PAUSE_METADATA_KEYS` names every key it writes; `pauseMetadata({agentId, pausedAt, workspacePath, runId})` arms it and `clearedPauseMetadata()` releases it (undefined-valued, so `updateTask` deletes rather than serializing `"null"`); `isAgentPausedTask(task)` / `isResumablePausedTask(task, agentId)` answer whether a pause is still live and whether it is THIS agent's. `updateTask` clears the whole set whenever the task leaves `blocked`, so a revive by any path (dedupe, cooldown expiry, manual unblock) can't leave a task running under a stale pause. Also carries the cycle-safe RELEASE adapter (#3730) `agentManagement.js` registers at load — `resolvePausedTaskResume(task)` hands `updateTask` the paused run's resume pointer before the write (so any un-block resumes on the preserved worktree, not just the Resume dialog) and `retirePausedAgent(agentId, taskId, branchName)` retires the paused record after it. || `taskBlockCategories.js` | The `blockedCategory` vocabulary — which blocks are the system's to clear and which are a person's, read by the pause logic, the failure reaper, and the investigation auto-retry so they can't drift into three literal sets. `PAUSED_BLOCKED_CATEGORIES` (paused until something outside the task changes — keeps the resume pointer), `USER_DECISION_BLOCKED_CATEGORIES` (user intent / open decision — the reaper's exemption), `NON_AUTO_RETRY_BLOCK_CATEGORIES` (their union — a completed investigation must never revive these), `TIMED_COOLDOWN_BLOCKED_CATEGORIES` (a timer clears these — the cooldown sweeper revives them, so block reporters stay quiet), `PROVIDER_CONFIG_BLOCKED_CATEGORY` (the shared literal the agent-spawn stamp site uses, so the stamp cannot drift from the two sets it belongs to). Pure. || `taskRequeue.js` | The REQUEUE stamp (#3376) — pure metadata helpers for the one BACKWARD lifecycle step (`in_progress → pending`, performed by the orphan sweep and the retry-hold release). `REQUEUED_AT_KEY` / `LAST_SPAWNED_AT_KEY` name the two stamps; `isPostSpawnRequeue(pendingTask, inProgressTask)` answers whether the requeue happened strictly AFTER that spawn, which is how the federated merge tells a real requeue apart from an ordinary edit landing on a peer's stale `pending` copy. Returns false when either stamp is missing, so callers fall back to the lifecycle rank. |-| `taskRetryHold.js` | The failed-task RETRY HOLD state (#3373) — pure metadata helpers shared by the failure verdict, the post-cleanup release, the spawn guard, and the orphan sweep. `retryHoldMetadata(agentId, now)` arms it (task stays `in_progress`, so no dequeue tier can claim the retry before its resume pointer is resolved); `clearedRetryHoldMetadata()` releases it in the same write that flips the task to `pending`; `isRetryHeld(metadata)` / `isRetryHoldOwner(metadata, agentId)` gate the spawn and the owner-scoped release; `isStaleRetryHold(metadata, now, graceMs)` + `RETRY_HOLD_GRACE_MS` let the orphan sweep finish a transition whose process died. |+| `taskRetryHold.js` | The failed-task RETRY HOLD and delivery-cleanup ownership state (#3373, #13430) — pure metadata helpers shared by the failure verdict, the post-cleanup release, the spawn guard, the orphan sweep, and completed-agent deletion guards. `retryHoldMetadata(agentId, now)` arms it (task stays `in_progress`, so no dequeue tier can claim the retry before its resume pointer is resolved); `clearedRetryHoldMetadata()` releases it in the same write that flips the task to `pending`; `isRetryHeld(metadata)` / `isRetryHoldOwner(metadata, agentId)` gate the spawn and the owner-scoped release; `isStaleRetryHold(metadata, now, graceMs)` + `RETRY_HOLD_GRACE_MS` let the orphan sweep finish a transition whose process died; `isDeliveryCleanupHeld(metadata)` / `collectProtectedCleanupOwners(tasks)` identify active delivery-cleanup and retry-cleanup owners across `in_progress` tasks so single deletion and bulk completed-agent clearing retain live recovery evidence. || `taskStatusTransition.js` | The CoS task STATUS TRANSITION vocabulary (#6620) — `resolveTaskStatusTransition(previousStatus, nextStatus)` classifies one update's status pair ONCE and returns `{ action, isRevive, isRequeue, entersInProgress, leavesInProgress, leavesBlocked, isTerminal }`, so `writeTaskUpdate`'s blocked/pause clear, resume-pointer drop, retry-hold retire, claim release, requeue stamp and emitted `tasks:changed` action all read one answer instead of re-deriving seven. A falsy `nextStatus` means "this patch carries no status" (a lease-renewal heartbeat) and releases nothing. `isTerminalTaskStatus(status)` is the shared `completed`/`blocked` predicate `cosTaskStore` and `cosTaskMerge` both read. Pure — metadata predicates (is this block a pause?) stay at the application site. || `taskTargetBranch.js` | Pure task-branch contract: `resolveTaskTargetBranch(metadata)` reads a retry's legacy `existingBranch` or a review-loop follow-up's canonical `reviewLoopPRBranch`; `shouldStripTaskTargetBranch(metadata)` identifies only retry-owned pointers for terminal cleanup. || `claimContinuation.js` | Claim-flow relaunch pointer. `claimContinuationBranch(claimTarget)` is the `claim/issue-<n>` or `claim/<key>` branch a pinned target checks out. `claimContinuationPointer(...)` names the `claim-*` worktree already holding that branch when a relaunch should finish it in place (locked, prunable, unmanaged, or occupied by another running/paused agent → null). `claimContinuationWorkspace(...)` turns a still-on-disk pointer into the cwd the next agent uses without moving the directory. `reconcileAssignmentForTask`, `reconcileAssignedTarget`, and `reconcileMergeTarget` preserve scheduler assignments and match their registered/live branch, checkout, and completion policy for reconciliation admission; `claimOwnershipBinding(task)` is the branch (or "picks its own", for a swarm orchestrator; a pinned run is also "still choosing" until it binds) a claim run registers on its agent record; `bindClaimBranch` / `releaseClaimBranch` patch the concrete branches a run binds before cutting them (an epic's child, a picked issue), `ownsClaimBranch(agent, branch)` is explicit registered/bound ownership only (never a picker's "may pick"), `claimCheckoutOwnerReason(...)` names a live/possible claim owner for branch-reconcile (`claimCheckoutOwner(...)` adds the holder run ids a blocked cleanup records), and `claimBranchAdmission(...)` gates coordinator adoption and detach with the same branch/repository owners; `claimContinuationAdmission(...)` is the launch-time ownership check that re-reads holder, branch and live owners before a cached pointer may be entered (anything unreadable or changed → refuse, the caller defers; a possible swarm owner holds the launch until that run ends). |diff --git a/server/lib/taskRetryHold.js b/server/lib/taskRetryHold.jsindex a9a2747ecc..e6f77053cc 100644--- a/server/lib/taskRetryHold.js+++ b/server/lib/taskRetryHold.js@@ -113,3 +113,31 @@ export function isStaleRetryHold(metadata, now = Date.now(), graceMs = RETRY_HOL/** Repository verification owns these tasks across completion and restarts. */export const isDeliveryCleanupHeld = metadata => metadata?.deliveryRecovery === 'cleanup'&& typeof metadata.deliveryRecoveryAgentId === 'string' && metadata.deliveryRecoveryAgentId.length > 0;++/**+ * Extract the protected cleanup/retry owner ids from a list of tasks.+ * Only active (`in_progress`) task metadata counts; historical flags on+ * completed agent results do not establish current ownership.+ * `hasUnattributedRetryHold` is true when any `in_progress` task carries a+ * legacy boolean retry hold (`retryPendingCleanup: true`), which could belong+ * to any completed run until the hold is resolved.+ */+export function collectProtectedCleanupOwners(tasks) {+ const ownerIds = new Set();+ let hasUnattributedRetryHold = false;+ for (const task of tasks || []) {+ if (task?.status !== 'in_progress') continue;+ const meta = task.metadata;+ if (isDeliveryCleanupHeld(meta)) ownerIds.add(meta.deliveryRecoveryAgentId);+ if (isRetryHeld(meta)) {+ const holder = meta[RETRY_HOLD_KEY];+ if (isTruthyMeta(holder)) hasUnattributedRetryHold = true;+ else if (typeof holder === 'string' && holder) ownerIds.add(holder);+ }+ }+ return {+ ownerIds,+ hasUnattributedRetryHold,+ isProtected: agentId => Boolean(agentId) && (hasUnattributedRetryHold || ownerIds.has(agentId)),+ };+}diff --git a/server/lib/taskRetryHold.test.js b/server/lib/taskRetryHold.test.jsindex 2b852da46d..9764812ae1 100644--- a/server/lib/taskRetryHold.test.js+++ b/server/lib/taskRetryHold.test.js@@ -6,6 +6,8 @@ import {isRetryHeld,isRetryHoldOwner,isStaleRetryHold,+ isDeliveryCleanupHeld,+ collectProtectedCleanupOwners,} from './taskRetryHold.js';const NOW = Date.parse('2026-08-02T12:00:00.000Z');@@ -100,3 +102,41 @@ describe('isStaleRetryHold', () => {expect(isStaleRetryHold({}, NOW)).toBe(false);});});++describe('collectProtectedCleanupOwners (#13430)', () => {+ it('identifies active delivery-cleanup and retry-cleanup owners only while the task is in_progress', () => {+ const deliveryHeld = {+ id: 'sys-rl-example',+ status: 'in_progress',+ metadata: { deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: 'agent-example' },+ };+ const retryHeld = {+ id: 'task-retry',+ status: 'in_progress',+ metadata: { retryPendingCleanup: 'agent-retry' },+ };+ const completedTask = {+ id: 'task-done',+ status: 'completed',+ metadata: { deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: 'agent-stale' },+ };++ expect(isDeliveryCleanupHeld(deliveryHeld.metadata)).toBe(true);++ const collected = collectProtectedCleanupOwners([deliveryHeld, retryHeld, completedTask]);+ expect([...collected.ownerIds]).toEqual(['agent-example', 'agent-retry']);+ expect(collected.hasUnattributedRetryHold).toBe(false);+ expect(collected.isProtected('agent-example')).toBe(true);+ expect(collected.isProtected('agent-other')).toBe(false);+ expect(collected.isProtected('agent-retry')).toBe(true);+ expect(collected.isProtected('agent-stale')).toBe(false);+ });++ it('protects all candidate agents when an in_progress task carries an unattributed legacy retry hold', () => {+ const collected = collectProtectedCleanupOwners([+ { id: 'task-legacy', status: 'in_progress', metadata: { retryPendingCleanup: true } },+ ]);+ expect(collected.hasUnattributedRetryHold).toBe(true);+ expect(collected.isProtected('agent-any')).toBe(true);+ });+});diff --git a/server/routes/cos.test.js b/server/routes/cos.test.jsindex 09e063a747..28e69b9ad3 100644--- a/server/routes/cos.test.js+++ b/server/routes/cos.test.js@@ -828,6 +828,18 @@ describe('CoS Routes', () => {expect(response.status).toBe(404);});++ it('should return 409 when agent is still owned by an active task cleanup hold', async () => {+ deleteAgent.mockRejectedValue(new ServerError(+ 'Agent agent-001 is still owned by an active task delivery or retry cleanup hold and cannot be deleted until cleanup finishes',+ { status: 409, code: 'AGENT_CLEANUP_OWNED' },+ ));++ const response = await request(app).delete('/api/cos/agents/agent-001');++ expect(response.status).toBe(409);+ expect(response.body.code).toBe('AGENT_CLEANUP_OWNED');+ });});describe('GET /api/cos/health', () => {diff --git a/server/services/agentPromptBuilder.test.js b/server/services/agentPromptBuilder.test.jsindex e2042fd6d0..82a1923ec9 100644--- a/server/services/agentPromptBuilder.test.js+++ b/server/services/agentPromptBuilder.test.js@@ -1790,11 +1790,13 @@ describe('buildLightContextPrompt', () => {expect(prompt).toMatch(/\/data\/slashdo-resolved\/local-agent-review-loop\.md/);expect(prompt).not.toMatch(/RECIPE HEADER/);// This ceiling detects accidentally inlining the staged 40KB recipe.- // Exclude the exact shared safety contracts, whose independently tested- // required prose is not recipe content; retain the original ceiling.+ // Exclude the exact shared safety contracts and the checkout root prefix+ // embedded in doc pointers, whose independently tested required prose and+ // worktree path length are not recipe content; retain the original ceiling.expect(prompt).toContain(MANAGED_WORKTREE_CLEANUP_CONTRACT);const withoutFixedGuidance = prompt.replace(MANDATORY_DISPATCH_HINT_GUIDANCE, '')- .replace(MANAGED_WORKTREE_CLEANUP_CONTRACT, '');+ .replace(MANAGED_WORKTREE_CLEANUP_CONTRACT, '')+ .replaceAll(PATHS.root, '');expect(withoutFixedGuidance.length).toBeLessThan(33_000);});diff --git a/server/services/cosAgentArchive.js b/server/services/cosAgentArchive.jsindex 90131c285d..0b40a71b67 100644--- a/server/services/cosAgentArchive.js+++ b/server/services/cosAgentArchive.js@@ -16,9 +16,9 @@ import { join } from 'path';import { cosEvents } from './cosEvents.js';import { loadState, saveState, withStateLock, AGENTS_DIR } from './cosState.js';import { atomicWrite, ensureDir } from '../lib/fileUtils.js';-import { ServerError } from '../lib/errorHandler.js';-import { isDeliveryCleanupHeld } from '../lib/taskRetryHold.js';import { loadAgentIndex, saveAgentIndex, recordArchivedAgentOrder, moveAgentDir } from './cosAgentIndex.js';+import { readProtectedCleanupOwners } from './cosAgentLifecycle.js';+import { releaseAgentReservation } from './agentMaintenanceReservations.js';// Archive stale completed agents from state.json.// Completed agents are already persisted to per-agent metadata files on disk@@ -88,51 +88,72 @@ export async function archiveStaleAgents() {// Clear completed agents from state, cache, and diskexport async function clearCompletedAgents() {return withBackupAssetPublication(() => withStateLock(async () => {- // Completion is persisted before its task hold; both records protect restart evidence.- const { getTaskView } = await import('./cosTaskStore.js');- const { readBranchCompletionPolicies } = await import('./branchCompletionPolicy.js');- const [tasks, agents] = await Promise.all([- getTaskView({ kind: 'queue' }),- readBranchCompletionPolicies({ repairIndex: false }),- ]);- if ([...(tasks.user?.tasks || []), ...(tasks.cos?.tasks || [])]- .some(task => task.status === 'in_progress' && isDeliveryCleanupHeld(task.metadata))- || agents.some(agent => agent.status === 'completed' && agent.metadata?.repoStateVerification?.status === 'pending')) {- throw new ServerError('Wait for delivery verification and cleanup before clearing completed agents.', {- status: 409, code: 'DELIVERY_CLEANUP_PENDING',- });- }const state = await loadState();const idx = await loadAgentIndex();+ const protectedOwners = await readProtectedCleanupOwners({ state, index: idx });- // Remove completed agents from state+ // Remove unprotected completed agents from state; retain current cleanup/retry ownersconst stateCompleted = Object.keys(state.agents).filter(id => state.agents[id].status === 'completed');+ const removableStateIds = [];+ const retainedIds = new Set();for (const id of stateCompleted) {+ if (protectedOwners.isProtected(id)) {+ retainedIds.add(id);+ continue;+ }+ removableStateIds.push(id);delete state.agents[id];}- await saveState(state);+ if (removableStateIds.length > 0) {+ await saveState(state, { changedAgentIds: removableStateIds });+ }++ const removedIndexEntries = [];+ const retainedDates = new Set();+ const allDates = new Set();+ for (const [id, date] of idx) {+ allDates.add(date);+ if (protectedOwners.isProtected(id)) {+ retainedIds.add(id);+ retainedDates.add(date);+ } else {+ removedIndexEntries.push([id, date]);+ }+ }- // Collect all unique dates from index, then remove date bucket dirs- const dates = new Set(idx.values());- const totalCleared = idx.size + stateCompleted.filter(id => !idx.has(id)).length;+ const totalCleared = removedIndexEntries.length + removableStateIds.filter(id => !idx.has(id)).length;// Remove durable references before bytes; a failed index write must leave// the old archive trees readable and its in-memory lookup retryable.- const previousEntries = [...idx];- idx.clear();- try { await saveAgentIndex(); }- catch (err) { for (const [id, date] of previousEntries) idx.set(id, date); throw err; }+ if (removedIndexEntries.length > 0) {+ for (const [id] of removedIndexEntries) idx.delete(id);+ try { await saveAgentIndex(); }+ catch (err) { for (const [id, date] of removedIndexEntries) idx.set(id, date); throw err; }+ }- const removals = [...dates].map(date => {+ // Whole buckets with no retained entries can be removed in one pass; mixed+ // buckets remove only the unprotected agent directories inside them.+ const removals = [];+ for (const date of allDates) {+ if (retainedDates.has(date)) continue;const dateDir = join(AGENTS_DIR, date);- return existsSync(dateDir)- ? rm(dateDir, { recursive: true }).catch(() => {})- : Promise.resolve();- });+ if (existsSync(dateDir)) {+ removals.push(rm(dateDir, { recursive: true }).catch(() => {}));+ }+ }+ for (const [id, date] of removedIndexEntries) {+ if (!retainedDates.has(date)) continue;+ const agentDir = join(AGENTS_DIR, date, id);+ if (existsSync(agentDir)) {+ removals.push(rm(agentDir, { recursive: true }).catch(() => {}));+ }+ }await Promise.all(removals);- return { cleared: totalCleared };+ for (const id of removableStateIds) await releaseAgentReservation(id);+ cosEvents.emit('agents:changed', { action: 'cleared-completed', cleared: totalCleared, retained: retainedIds.size });+ return { cleared: totalCleared, retained: retainedIds.size };}));}diff --git a/server/services/cosAgentArchives.backupAdmission.test.js b/server/services/cosAgentArchives.backupAdmission.test.jsindex 1cd1043c8c..14e49d97aa 100644--- a/server/services/cosAgentArchives.backupAdmission.test.js+++ b/server/services/cosAgentArchives.backupAdmission.test.js@@ -7,15 +7,15 @@ const fixture = await vi.hoisted(async () => {const { mkdtemp } = await import('node:fs/promises');const { tmpdir } = await import('node:os');const { join } = await import('node:path');+ const { createFileWriteQueue } = await import('../lib/fileWriteQueue.js');const base = await mkdtemp(join(tmpdir(), 'cos-archive-backup-'));- return { base, root: join(base, 'agents'), state: null, tasks: null, beforeIndex: null, bypass: false };+ return { base, root: join(base, 'agents'), state: null, beforeIndex: null, bypass: false, withStateLock: createFileWriteQueue() };});-vi.mock('./cosTaskStore.js', () => ({ getTaskView: async () => fixture.tasks }));vi.mock('./cosState.js', () => ({AGENTS_DIR: fixture.root,loadState: async () => structuredClone(fixture.state),saveState: async state => { fixture.state = structuredClone(state); await writeFile(join(fixture.root, 'state.json'), JSON.stringify(state)); },- withStateLock: createFileWriteQueue(),+ withStateLock: fn => fixture.withStateLock(fn),readAgentsStateForSafetyCheck: async () => ({ trusted: true, agents: fixture.state.agents }),}));vi.mock('../lib/fileUtils.js', async importOriginal => {@@ -42,6 +42,8 @@ let index;let acquireCut;const date = '2001-01-01';const id = 'agent-example';+let taskStore;+let taskRepo;const readIndex = async () => JSON.parse(await readFile(join(fixture.root, 'index.json'), 'utf8'));const assertIndexedTrees = async () => {for (const [agentId, bucket] of Object.entries(await readIndex())) {@@ -49,22 +51,34 @@ const assertIndexedTrees = async () => {expect(JSON.parse(await readFile(join(fixture.root, bucket, agentId, 'metadata.json'), 'utf8')).id).toBe(agentId);}};-async function seed({ completed = false, indexed = false } = {}) {- fixture.state.agents[id] = { id, status: completed ? 'completed' : 'running', startedAt: `${date}T01:00:00Z`,- ...(completed ? { completedAt: `${date}T02:00:00Z`, result: { success: true } } : {}), metadata: {} };- const dir = indexed ? join(fixture.root, date, id) : join(fixture.root, id);+async function seedAgent(agentId, { completed = false, indexed = false, resident = true, bucketDate = date, result = { success: true } } = {}) {+ const record = {+ id: agentId,+ status: completed ? 'completed' : 'running',+ startedAt: `${bucketDate}T01:00:00Z`,+ ...(completed ? { completedAt: `${bucketDate}T02:00:00Z`, result } : {}),+ metadata: {},+ };+ if (resident) fixture.state.agents[agentId] = record;+ const dir = indexed ? join(fixture.root, bucketDate, agentId) : join(fixture.root, agentId);await mkdir(dir, { recursive: true });await writeFile(join(dir, 'output.txt'), 'Example preserved output');- await writeFile(join(dir, 'metadata.json'), JSON.stringify(fixture.state.agents[id]));- if (indexed) { (await index.loadAgentIndex()).set(id, date); await index.saveAgentIndex(); }+ await writeFile(join(dir, 'metadata.json'), JSON.stringify(record));+ if (indexed) { (await index.loadAgentIndex()).set(agentId, bucketDate); await index.saveAgentIndex(); }+}+async function seed({ completed = false, indexed = false } = {}) {+ await seedAgent(id, { completed, indexed });}beforeEach(async () => {vi.resetModules(); fixture.beforeIndex = null; fixture.bypass = false;fixture.state = { agents: {}, config: { completedAgentRetentionMs: 1 }, stats: { tasksCompleted: 0, errors: 0 } };- fixture.tasks = { user: { tasks: [] }, cos: { tasks: [] } };await rm(fixture.root, { recursive: true, force: true }); await mkdir(fixture.root);await writeFile(join(fixture.root, 'index.json'), '{}');({ acquireBackupSnapshotCut: acquireCut } = await import('../lib/backupSnapshotBoundary.js'));+ taskRepo = await import('./cosTaskRepository.js');+ taskRepo.__resetCosTaskRepository();+ taskStore = await import('./cosTaskStore.js');+ taskStore.__resetTaskCache();index = await import('./cosAgentIndex.js');api = { ...await import('./cosAgentLifecycle.js'), ...await import('./cosAgentArchive.js') };});@@ -78,35 +92,40 @@ it('preserves pending verification before the task hold is written and after arcconst metadataPath = join(fixture.root, date, id, 'metadata.json');await writeFile(metadataPath, JSON.stringify(agent));- await expect(api.clearCompletedAgents()).rejects.toMatchObject({ code: 'DELIVERY_CLEANUP_PENDING' });+ expect(await api.clearCompletedAgents()).toEqual({ cleared: 0, retained: 1 });expect(fixture.state.agents[id]).toEqual(agent);delete fixture.state.agents[id];- await expect(api.clearCompletedAgents()).rejects.toMatchObject({ code: 'DELIVERY_CLEANUP_PENDING' });+ expect(await api.clearCompletedAgents()).toEqual({ cleared: 0, retained: 1 });await assertIndexedTrees();agent.metadata.repoStateVerification.status = 'completed';await writeFile(metadataPath, JSON.stringify(agent));- expect(await api.clearCompletedAgents()).toEqual({ cleared: 1 });+ expect(await api.clearCompletedAgents()).toEqual({ cleared: 1, retained: 0 });});it.each(['user', 'cos'])('preserves cleanup-owner evidence in the %s queue until delivery settles', async source => {await seed({ completed: true, indexed: true });// An archived owner is absent from resident state but still required after restart.delete fixture.state.agents[id];- const task = { id: 'task-example', status: 'in_progress', metadata: {- deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: id,- } };- fixture.tasks[source].tasks.push(task);+ const taskType = source === 'cos' ? 'internal' : 'user';+ await taskStore.addTask({+ id: 'task-example',+ status: 'in_progress',+ priority: 'MEDIUM',+ priorityValue: 2,+ description: 'Cleanup task',+ metadata: { deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: id },+ }, taskType, { raw: true });const beforeState = structuredClone(fixture.state);const beforeIndex = await readIndex();- await expect(api.clearCompletedAgents()).rejects.toMatchObject({ status: 409, code: 'DELIVERY_CLEANUP_PENDING' });+ expect(await api.clearCompletedAgents()).toEqual({ cleared: 0, retained: 1 });expect(fixture.state).toEqual(beforeState);expect(await readIndex()).toEqual(beforeIndex);await assertIndexedTrees();- task.status = 'completed';- expect(await api.clearCompletedAgents()).toEqual({ cleared: 1 });+ await taskStore.updateTask('task-example', { status: 'completed' }, taskType);+ expect(await api.clearCompletedAgents()).toEqual({ cleared: 1, retained: 0 });expect(await readIndex()).toEqual({});expect(await readdir(fixture.root)).not.toContain(date);});@@ -212,3 +231,157 @@ it('does not let a cold reader queue its shared initialization ahead of an admittry { await assertIndexedTrees(); } finally { release(); }await reader;});++describe('active cleanup and retry owner preservation (#13430)', () => {+ async function seedCleanupTask(taskId, metadata, taskType = 'internal') {+ return taskStore.addTask({+ id: taskId,+ status: 'in_progress',+ priority: 'MEDIUM',+ priorityValue: 2,+ description: `Cleanup task ${taskId}`,+ metadata,+ }, taskType, { raw: true });+ }++ it.each([+ { label: 'resident record', indexed: false, resident: true },+ { label: 'indexed archive', indexed: true, resident: false },+ ])('refuses single delete and retains $label while task holds deliveryRecovery cleanup', async ({ indexed, resident }) => {+ await seedAgent(id, { completed: true, indexed, resident });+ await seedCleanupTask('sys-rl-example', {+ deliveryRecovery: 'cleanup',+ deliveryRecoveryAgentId: id,+ });++ await expect(api.deleteAgent(id)).rejects.toMatchObject({+ status: 409,+ code: 'AGENT_CLEANUP_OWNED',+ });++ const record = await api.getAgent(id);+ expect(record?.id).toBe(id);+ if (resident) expect(fixture.state.agents[id]?.id).toBe(id);+ if (indexed) await assertIndexedTrees();++ await taskStore.updateTask('sys-rl-example', {+ status: 'completed',+ metadata: { deliveryRecovery: undefined, deliveryRecoveryAgentId: undefined },+ }, 'internal');++ await expect(api.deleteAgent(id)).resolves.toEqual({ success: true, agentId: id });+ expect(await api.getAgent(id)).toBeNull();+ });++ it('retains protected resident and archived runs across mixed buckets during clearCompletedAgents', async () => {+ const protectedResident = 'agent-protected-resident';+ const protectedArchived = 'agent-protected-archived';+ const unprotectedSameBucket = 'agent-unprotected-same-bucket';+ const unprotectedOtherBucket = 'agent-unprotected-other-bucket';+ const otherDate = '2001-01-02';++ await seedAgent(protectedResident, { completed: true, indexed: false, resident: true });+ await seedAgent(protectedArchived, { completed: true, indexed: true, resident: false, bucketDate: date });+ await seedAgent(unprotectedSameBucket, { completed: true, indexed: true, resident: true, bucketDate: date });+ await seedAgent(unprotectedOtherBucket, { completed: true, indexed: true, resident: false, bucketDate: otherDate });++ await seedCleanupTask('sys-rl-example', {+ deliveryRecovery: 'cleanup',+ deliveryRecoveryAgentId: protectedResident,+ }, 'internal');+ await seedCleanupTask('task-retry-example', {+ retryPendingCleanup: protectedArchived,+ retryPendingSince: `${date}T02:00:00.000Z`,+ }, 'user');++ const cleared = await api.clearCompletedAgents();+ expect(cleared).toEqual({ cleared: 2, retained: 2 });++ expect(fixture.state.agents[protectedResident]?.id).toBe(protectedResident);+ expect(fixture.state.agents[unprotectedSameBucket]).toBeUndefined();+ expect(await readIndex()).toEqual({ [protectedArchived]: date });+ await assertIndexedTrees();+ await expect(readFile(join(fixture.root, date, unprotectedSameBucket, 'metadata.json'), 'utf8')).rejects.toThrow();+ await expect(readdir(join(fixture.root, otherDate))).rejects.toThrow();++ await taskStore.updateTask('sys-rl-example', { status: 'completed' }, 'internal');+ await taskStore.updateTask('task-retry-example', { status: 'pending' }, 'user');++ const secondClear = await api.clearCompletedAgents();+ expect(secondClear).toEqual({ cleared: 2, retained: 0 });+ expect(fixture.state.agents).toEqual({});+ expect(await readIndex()).toEqual({});+ });++ it('does not retain completed history solely because a stale agent result still carries cleanupPending', async () => {+ await seedAgent(id, {+ completed: true,+ indexed: true,+ resident: true,+ result: { success: true, cleanupPending: true },+ });+ await taskStore.addTask({+ id: 'sys-rl-settled',+ status: 'completed',+ priority: 'MEDIUM',+ priorityValue: 2,+ description: 'Already settled task',+ metadata: {},+ }, 'internal', { raw: true });++ await expect(api.clearCompletedAgents()).resolves.toEqual({ cleared: 1, retained: 0 });+ expect(await api.getAgent(id)).toBeNull();+ });++ it('revalidates current task ownership under the state lock so a concurrent reassignment is not deleted', async () => {+ const oldOwner = 'agent-old-owner';+ const newOwner = 'agent-new-owner';+ await seedAgent(oldOwner, { completed: true, indexed: true, resident: true });+ await seedAgent(newOwner, { completed: true, indexed: true, resident: true });+ await seedCleanupTask('sys-rl-race', {+ deliveryRecovery: 'cleanup',+ deliveryRecoveryAgentId: oldOwner,+ }, 'internal');++ // Start clearCompletedAgents while an earlier task reassignment is holding the shared+ // state lock; clearCompletedAgents must read task ownership AFTER acquiring the lock,+ // not from a stale pre-lock snapshot.+ const entered = deferred();+ const release = deferred();+ const blocker = fixture.withStateLock(async () => {+ entered.resolve();+ await release.promise;+ });+ await entered.promise;++ const reassign = taskStore.updateTask('sys-rl-race', {+ status: 'in_progress',+ metadata: { deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: newOwner },+ }, 'internal');+ await settle();+ const clear = api.clearCompletedAgents();+ await settle();++ release.resolve();+ await blocker;+ const [, clearResult] = await Promise.all([reassign, clear]);+ expect(clearResult).toEqual({ cleared: 1, retained: 1 });+ expect((await api.getAgent(newOwner))?.id).toBe(newOwner);+ expect(await api.getAgent(oldOwner)).toBeNull();+ });++ it('preserves evidence when task cleanup ownership cannot be verified', async () => {+ await seedAgent(id, { completed: true, indexed: true, resident: true });+ const spy = vi.spyOn(taskStore, 'getTaskView').mockRejectedValueOnce(new Error('task store unavailable'));+ try {+ await expect(api.deleteAgent(id)).rejects.toMatchObject({+ status: 503,+ code: 'CLEANUP_OWNERSHIP_UNAVAILABLE',+ });+ expect((await api.getAgent(id))?.id).toBe(id);+ await assertIndexedTrees();+ } finally {+ spy.mockRestore();+ }+ });+});diff --git a/server/services/cosAgentLifecycle.js b/server/services/cosAgentLifecycle.jsindex 621ac983fe..174e73c712 100644--- a/server/services/cosAgentLifecycle.js+++ b/server/services/cosAgentLifecycle.js@@ -24,8 +24,56 @@ import { repairCodexTaskSummary } from './codexSummaryRepair.js';import { loadAgentIndex, saveAgentIndex, getAgentDir, recordArchivedAgentOrder, moveAgentDir } from './cosAgentIndex.js';import { hasClaimCheckoutBindings } from './completedCheckoutEvidence.js';import { isAgentHandoff } from '../lib/agentOutcome.js';+import { collectProtectedCleanupOwners } from '../lib/taskRetryHold.js';import { reconcileStaleAgentReservations, releaseAgentReservation } from './agentMaintenanceReservations.js';+function hasPendingRepoStateVerification(agent) {+ return agent?.status === 'completed' && agent?.metadata?.repoStateVerification?.status === 'pending';+}++export async function readProtectedCleanupOwners({ state = null, index = null, targetAgentId = null } = {}) {+ const store = await import('./cosTaskStore.js').catch(err => {+ throw new ServerError(`Cannot verify task cleanup ownership: ${err.message}`, {+ status: 503, code: 'CLEANUP_OWNERSHIP_UNAVAILABLE',+ });+ });+ const getTaskView = 'getTaskView' in store ? store.getTaskView : null;+ if (typeof getTaskView !== 'function') {+ throw new ServerError('Cannot verify task cleanup ownership before deleting agent history', {+ status: 503, code: 'CLEANUP_OWNERSHIP_UNAVAILABLE',+ });+ }+ const view = await getTaskView({ kind: 'queue' }).catch(err => {+ throw new ServerError(`Cannot verify task cleanup ownership: ${err.message}`, {+ status: 503, code: 'CLEANUP_OWNERSHIP_UNAVAILABLE',+ });+ });+ if (!Array.isArray(view?.user?.tasks) || !Array.isArray(view?.cos?.tasks)) {+ throw new ServerError('Cannot verify task cleanup ownership before deleting agent history', {+ status: 503, code: 'CLEANUP_OWNERSHIP_UNAVAILABLE',+ });+ }+ const protectedOwners = collectProtectedCleanupOwners([...view.user.tasks, ...view.cos.tasks]);+ // Completion is persisted before its task hold; pending repo-state verification+ // on a resident or archived completion record also protects restart evidence.+ for (const [id, agent] of Object.entries(state?.agents || {})) {+ if (hasPendingRepoStateVerification(agent)) protectedOwners.ownerIds.add(id);+ }+ if (index instanceof Map) {+ const candidates = targetAgentId+ ? (index.has(targetAgentId) ? [[targetAgentId, index.get(targetAgentId)]] : [])+ : [...index.entries()];+ for (const [id, date] of candidates) {+ if (protectedOwners.ownerIds.has(id) || state?.agents?.[id] || !date) continue;+ const content = await tryReadFile(join(AGENTS_DIR, date, id, 'metadata.json'));+ if (!content) continue;+ const parsed = safeJSONParse(content, null, { logError: false });+ if (hasPendingRepoStateVerification(parsed)) protectedOwners.ownerIds.add(id);+ }+ }+ return protectedOwners;+}+export async function registerAgent(agentId, taskId, metadata = {}) {return withStateLock(async () => {const state = await loadState();@@ -1045,6 +1093,14 @@ export async function deleteAgent(agentId) {throw new ServerError('Agent not found', { status: 404, code: 'NOT_FOUND' });}+ const protectedOwners = await readProtectedCleanupOwners({ state, index: idx, targetAgentId: agentId });+ if (protectedOwners.isProtected(agentId)) {+ throw new ServerError(+ `Agent ${agentId} is still owned by an active task delivery or retry cleanup hold and cannot be deleted until cleanup finishes`,+ { status: 409, code: 'AGENT_CLEANUP_OWNED' },+ );+ }+const wasRunning = state.agents[agentId]?.status === 'running';delete state.agents[agentId];await saveState(state, { changedAgentIds: [agentId] });diff --git a/server/services/cosAgentLifecycle.test.js b/server/services/cosAgentLifecycle.test.jsindex 69ba54d3f0..a90f4d8309 100644--- a/server/services/cosAgentLifecycle.test.js+++ b/server/services/cosAgentLifecycle.test.js@@ -28,7 +28,10 @@ vi.mock('./cosState.js', () => ({vi.mock('./branchCompletionPolicy.js', () => ({ readBranchCompletionPolicies: vi.fn(async () => Object.values(mockCosState.state?.agents || {})) }));const recoveryTasks = vi.hoisted(() => new Map());-vi.mock('./cosTaskStore.js', () => ({ getTaskById: vi.fn(async id => recoveryTasks.get(id) || null) }));+vi.mock('./cosTaskStore.js', () => ({+ getTaskById: vi.fn(async id => recoveryTasks.get(id) || null),+ getTaskView: vi.fn(async () => ({ user: { tasks: [...recoveryTasks.values()] }, cos: { tasks: [] } })),+}));vi.mock('./domainUsage.js', () => ({recordDomainUsage: vi.fn(async () => {})