Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: preserve active cleanup owners when deleting completed agent history (#13430)

Adam Eivya14f3615181b
diff --git a/client/src/components/cos/tabs/AgentsTab.jsx b/client/src/components/cos/tabs/AgentsTab.jsx
index c168a46ed6..4a509fe79f 100644
--- a/client/src/components/cos/tabs/AgentsTab.jsx
+++ b/client/src/components/cos/tabs/AgentsTab.jsx
@@ -188,7 +188,14 @@ export default function AgentsTab({ agentsLoaded = true, agentsError = null, onR
setConfirmingClear(false);
const result = await api.clearCompletedCosAgents({ silent: true }).catch(err => { toast.error(err.message); return null; });
if (!result) return;
- toast.success('Cleared completed agents');
+ if (result.retained > 0) {
+ const retainedWord = result.retained === 1 ? '1 active cleanup owner retained' : `${result.retained} active cleanup owners retained`;
+ toast.success(result.cleared > 0
+ ? `Cleared ${result.cleared} completed agent${result.cleared === 1 ? '' : 's'} (${retainedWord})`
+ : `No completed agents cleared (${retainedWord})`);
+ } else {
+ toast.success('Cleared completed agents');
+ }
refresh();
};
diff --git a/client/src/components/cos/tabs/AgentsTab.test.jsx b/client/src/components/cos/tabs/AgentsTab.test.jsx
index 049ad614f9..ca6a2b331e 100644
--- a/client/src/components/cos/tabs/AgentsTab.test.jsx
+++ b/client/src/components/cos/tabs/AgentsTab.test.jsx
@@ -93,7 +93,14 @@ vi.mock('./RelaunchAgentModal', () => ({
</button>
),
}));
-vi.mock('../../ui/InlineConfirmRow', () => ({ default: ({ question }) => <p>{question}</p> }));
+vi.mock('../../ui/InlineConfirmRow', () => ({
+ default: ({ question, confirmText, onConfirm }) => (
+ <div>
+ <p>{question}</p>
+ {onConfirm && <button type="button" onClick={onConfirm}>{confirmText || 'Confirm'}</button>}
+ </div>
+ ),
+}));
import * as api from '../../../services/api';
import toast from '../../ui/Toast';
@@ -489,6 +496,19 @@ it('makes the clear scope explicit when the feedback filter hides completed reco
expect(screen.queryByText(/removes 1 agent record/)).not.toBeInTheDocument();
});
+it('explains retained active cleanup owners when clearing completed agents', async () => {
+ const user = userEvent.setup();
+ api.getCosCompletedAgents.mockResolvedValue({ items: [completedAgent('done', 'Completed work')], total: 2, nextCursor: null });
+ api.clearCompletedCosAgents.mockResolvedValue({ cleared: 1, retained: 1 });
+ const onRefresh = vi.fn();
+ renderTab([], onRefresh);
+ await screen.findByText('Completed work');
+ await user.click(screen.getByRole('button', { name: 'Clear all completed agents' }));
+ await user.click(screen.getByRole('button', { name: 'Clear all' }));
+ expect(toast.success).toHaveBeenCalledWith('Cleared 1 completed agent (1 active cleanup owner retained)');
+ expect(onRefresh).toHaveBeenCalled();
+});
+
const deferred = () => {
let resolve;
diff --git a/server/lib/README.md b/server/lib/README.md
index dd344b8831..222008e920 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -582,7 +582,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `taskPauseHold.js` | The user-PAUSE hold — pure metadata helpers shared by the writer (`pauseAgent`), the reader (`resumeAgent`), and the clearer (`updateTask`). `AGENT_PAUSED_CATEGORY` is the `blockedCategory` a pause stamps; `PAUSE_METADATA_KEYS` names every key it writes; `pauseMetadata({agentId, pausedAt, workspacePath, runId})` arms it and `clearedPauseMetadata()` releases it (undefined-valued, so `updateTask` deletes rather than serializing `"null"`); `isAgentPausedTask(task)` / `isResumablePausedTask(task, agentId)` answer whether a pause is still live and whether it is THIS agent's. `updateTask` clears the whole set whenever the task leaves `blocked`, so a revive by any path (dedupe, cooldown expiry, manual unblock) can't leave a task running under a stale pause. Also carries the cycle-safe RELEASE adapter (#3730) `agentManagement.js` registers at load — `resolvePausedTaskResume(task)` hands `updateTask` the paused run's resume pointer before the write (so any un-block resumes on the preserved worktree, not just the Resume dialog) and `retirePausedAgent(agentId, taskId, branchName)` retires the paused record after it. |
| `taskBlockCategories.js` | The `blockedCategory` vocabulary — which blocks are the system's to clear and which are a person's, read by the pause logic, the failure reaper, and the investigation auto-retry so they can't drift into three literal sets. `PAUSED_BLOCKED_CATEGORIES` (paused until something outside the task changes — keeps the resume pointer), `USER_DECISION_BLOCKED_CATEGORIES` (user intent / open decision — the reaper's exemption), `NON_AUTO_RETRY_BLOCK_CATEGORIES` (their union — a completed investigation must never revive these), `TIMED_COOLDOWN_BLOCKED_CATEGORIES` (a timer clears these — the cooldown sweeper revives them, so block reporters stay quiet), `PROVIDER_CONFIG_BLOCKED_CATEGORY` (the shared literal the agent-spawn stamp site uses, so the stamp cannot drift from the two sets it belongs to). Pure. |
| `taskRequeue.js` | The REQUEUE stamp (#3376) — pure metadata helpers for the one BACKWARD lifecycle step (`in_progress → pending`, performed by the orphan sweep and the retry-hold release). `REQUEUED_AT_KEY` / `LAST_SPAWNED_AT_KEY` name the two stamps; `isPostSpawnRequeue(pendingTask, inProgressTask)` answers whether the requeue happened strictly AFTER that spawn, which is how the federated merge tells a real requeue apart from an ordinary edit landing on a peer's stale `pending` copy. Returns false when either stamp is missing, so callers fall back to the lifecycle rank. |
-| `taskRetryHold.js` | The failed-task RETRY HOLD state (#3373) — pure metadata helpers shared by the failure verdict, the post-cleanup release, the spawn guard, and the orphan sweep. `retryHoldMetadata(agentId, now)` arms it (task stays `in_progress`, so no dequeue tier can claim the retry before its resume pointer is resolved); `clearedRetryHoldMetadata()` releases it in the same write that flips the task to `pending`; `isRetryHeld(metadata)` / `isRetryHoldOwner(metadata, agentId)` gate the spawn and the owner-scoped release; `isStaleRetryHold(metadata, now, graceMs)` + `RETRY_HOLD_GRACE_MS` let the orphan sweep finish a transition whose process died. |
+| `taskRetryHold.js` | The failed-task RETRY HOLD and delivery-cleanup ownership state (#3373, #13430) — pure metadata helpers shared by the failure verdict, the post-cleanup release, the spawn guard, the orphan sweep, and completed-agent deletion guards. `retryHoldMetadata(agentId, now)` arms it (task stays `in_progress`, so no dequeue tier can claim the retry before its resume pointer is resolved); `clearedRetryHoldMetadata()` releases it in the same write that flips the task to `pending`; `isRetryHeld(metadata)` / `isRetryHoldOwner(metadata, agentId)` gate the spawn and the owner-scoped release; `isStaleRetryHold(metadata, now, graceMs)` + `RETRY_HOLD_GRACE_MS` let the orphan sweep finish a transition whose process died; `isDeliveryCleanupHeld(metadata)` / `collectProtectedCleanupOwners(tasks)` identify active delivery-cleanup and retry-cleanup owners across `in_progress` tasks so single deletion and bulk completed-agent clearing retain live recovery evidence. |
| `taskStatusTransition.js` | The CoS task STATUS TRANSITION vocabulary (#6620) — `resolveTaskStatusTransition(previousStatus, nextStatus)` classifies one update's status pair ONCE and returns `{ action, isRevive, isRequeue, entersInProgress, leavesInProgress, leavesBlocked, isTerminal }`, so `writeTaskUpdate`'s blocked/pause clear, resume-pointer drop, retry-hold retire, claim release, requeue stamp and emitted `tasks:changed` action all read one answer instead of re-deriving seven. A falsy `nextStatus` means "this patch carries no status" (a lease-renewal heartbeat) and releases nothing. `isTerminalTaskStatus(status)` is the shared `completed`/`blocked` predicate `cosTaskStore` and `cosTaskMerge` both read. Pure — metadata predicates (is this block a pause?) stay at the application site. |
| `taskTargetBranch.js` | Pure task-branch contract: `resolveTaskTargetBranch(metadata)` reads a retry's legacy `existingBranch` or a review-loop follow-up's canonical `reviewLoopPRBranch`; `shouldStripTaskTargetBranch(metadata)` identifies only retry-owned pointers for terminal cleanup. |
| `claimContinuation.js` | Claim-flow relaunch pointer. `claimContinuationBranch(claimTarget)` is the `claim/issue-<n>` or `claim/<key>` branch a pinned target checks out. `claimContinuationPointer(...)` names the `claim-*` worktree already holding that branch when a relaunch should finish it in place (locked, prunable, unmanaged, or occupied by another running/paused agent → null). `claimContinuationWorkspace(...)` turns a still-on-disk pointer into the cwd the next agent uses without moving the directory. `reconcileAssignmentForTask`, `reconcileAssignedTarget`, and `reconcileMergeTarget` preserve scheduler assignments and match their registered/live branch, checkout, and completion policy for reconciliation admission; `claimOwnershipBinding(task)` is the branch (or "picks its own", for a swarm orchestrator; a pinned run is also "still choosing" until it binds) a claim run registers on its agent record; `bindClaimBranch` / `releaseClaimBranch` patch the concrete branches a run binds before cutting them (an epic's child, a picked issue), `ownsClaimBranch(agent, branch)` is explicit registered/bound ownership only (never a picker's "may pick"), `claimCheckoutOwnerReason(...)` names a live/possible claim owner for branch-reconcile (`claimCheckoutOwner(...)` adds the holder run ids a blocked cleanup records), and `claimBranchAdmission(...)` gates coordinator adoption and detach with the same branch/repository owners; `claimContinuationAdmission(...)` is the launch-time ownership check that re-reads holder, branch and live owners before a cached pointer may be entered (anything unreadable or changed → refuse, the caller defers; a possible swarm owner holds the launch until that run ends). |
diff --git a/server/lib/taskRetryHold.js b/server/lib/taskRetryHold.js
index a9a2747ecc..e6f77053cc 100644
--- a/server/lib/taskRetryHold.js
+++ b/server/lib/taskRetryHold.js
@@ -113,3 +113,31 @@ export function isStaleRetryHold(metadata, now = Date.now(), graceMs = RETRY_HOL
/** Repository verification owns these tasks across completion and restarts. */
export const isDeliveryCleanupHeld = metadata => metadata?.deliveryRecovery === 'cleanup'
&& typeof metadata.deliveryRecoveryAgentId === 'string' && metadata.deliveryRecoveryAgentId.length > 0;
+
+/**
+ * Extract the protected cleanup/retry owner ids from a list of tasks.
+ * Only active (`in_progress`) task metadata counts; historical flags on
+ * completed agent results do not establish current ownership.
+ * `hasUnattributedRetryHold` is true when any `in_progress` task carries a
+ * legacy boolean retry hold (`retryPendingCleanup: true`), which could belong
+ * to any completed run until the hold is resolved.
+ */
+export function collectProtectedCleanupOwners(tasks) {
+ const ownerIds = new Set();
+ let hasUnattributedRetryHold = false;
+ for (const task of tasks || []) {
+ if (task?.status !== 'in_progress') continue;
+ const meta = task.metadata;
+ if (isDeliveryCleanupHeld(meta)) ownerIds.add(meta.deliveryRecoveryAgentId);
+ if (isRetryHeld(meta)) {
+ const holder = meta[RETRY_HOLD_KEY];
+ if (isTruthyMeta(holder)) hasUnattributedRetryHold = true;
+ else if (typeof holder === 'string' && holder) ownerIds.add(holder);
+ }
+ }
+ return {
+ ownerIds,
+ hasUnattributedRetryHold,
+ isProtected: agentId => Boolean(agentId) && (hasUnattributedRetryHold || ownerIds.has(agentId)),
+ };
+}
diff --git a/server/lib/taskRetryHold.test.js b/server/lib/taskRetryHold.test.js
index 2b852da46d..9764812ae1 100644
--- a/server/lib/taskRetryHold.test.js
+++ b/server/lib/taskRetryHold.test.js
@@ -6,6 +6,8 @@ import {
isRetryHeld,
isRetryHoldOwner,
isStaleRetryHold,
+ isDeliveryCleanupHeld,
+ collectProtectedCleanupOwners,
} from './taskRetryHold.js';
const NOW = Date.parse('2026-08-02T12:00:00.000Z');
@@ -100,3 +102,41 @@ describe('isStaleRetryHold', () => {
expect(isStaleRetryHold({}, NOW)).toBe(false);
});
});
+
+describe('collectProtectedCleanupOwners (#13430)', () => {
+ it('identifies active delivery-cleanup and retry-cleanup owners only while the task is in_progress', () => {
+ const deliveryHeld = {
+ id: 'sys-rl-example',
+ status: 'in_progress',
+ metadata: { deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: 'agent-example' },
+ };
+ const retryHeld = {
+ id: 'task-retry',
+ status: 'in_progress',
+ metadata: { retryPendingCleanup: 'agent-retry' },
+ };
+ const completedTask = {
+ id: 'task-done',
+ status: 'completed',
+ metadata: { deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: 'agent-stale' },
+ };
+
+ expect(isDeliveryCleanupHeld(deliveryHeld.metadata)).toBe(true);
+
+ const collected = collectProtectedCleanupOwners([deliveryHeld, retryHeld, completedTask]);
+ expect([...collected.ownerIds]).toEqual(['agent-example', 'agent-retry']);
+ expect(collected.hasUnattributedRetryHold).toBe(false);
+ expect(collected.isProtected('agent-example')).toBe(true);
+ expect(collected.isProtected('agent-other')).toBe(false);
+ expect(collected.isProtected('agent-retry')).toBe(true);
+ expect(collected.isProtected('agent-stale')).toBe(false);
+ });
+
+ it('protects all candidate agents when an in_progress task carries an unattributed legacy retry hold', () => {
+ const collected = collectProtectedCleanupOwners([
+ { id: 'task-legacy', status: 'in_progress', metadata: { retryPendingCleanup: true } },
+ ]);
+ expect(collected.hasUnattributedRetryHold).toBe(true);
+ expect(collected.isProtected('agent-any')).toBe(true);
+ });
+});
diff --git a/server/routes/cos.test.js b/server/routes/cos.test.js
index 09e063a747..28e69b9ad3 100644
--- a/server/routes/cos.test.js
+++ b/server/routes/cos.test.js
@@ -828,6 +828,18 @@ describe('CoS Routes', () => {
expect(response.status).toBe(404);
});
+
+ it('should return 409 when agent is still owned by an active task cleanup hold', async () => {
+ deleteAgent.mockRejectedValue(new ServerError(
+ 'Agent agent-001 is still owned by an active task delivery or retry cleanup hold and cannot be deleted until cleanup finishes',
+ { status: 409, code: 'AGENT_CLEANUP_OWNED' },
+ ));
+
+ const response = await request(app).delete('/api/cos/agents/agent-001');
+
+ expect(response.status).toBe(409);
+ expect(response.body.code).toBe('AGENT_CLEANUP_OWNED');
+ });
});
describe('GET /api/cos/health', () => {
diff --git a/server/services/agentPromptBuilder.test.js b/server/services/agentPromptBuilder.test.js
index e2042fd6d0..82a1923ec9 100644
--- a/server/services/agentPromptBuilder.test.js
+++ b/server/services/agentPromptBuilder.test.js
@@ -1790,11 +1790,13 @@ describe('buildLightContextPrompt', () => {
expect(prompt).toMatch(/\/data\/slashdo-resolved\/local-agent-review-loop\.md/);
expect(prompt).not.toMatch(/RECIPE HEADER/);
// This ceiling detects accidentally inlining the staged 40KB recipe.
- // Exclude the exact shared safety contracts, whose independently tested
- // required prose is not recipe content; retain the original ceiling.
+ // Exclude the exact shared safety contracts and the checkout root prefix
+ // embedded in doc pointers, whose independently tested required prose and
+ // worktree path length are not recipe content; retain the original ceiling.
expect(prompt).toContain(MANAGED_WORKTREE_CLEANUP_CONTRACT);
const withoutFixedGuidance = prompt.replace(MANDATORY_DISPATCH_HINT_GUIDANCE, '')
- .replace(MANAGED_WORKTREE_CLEANUP_CONTRACT, '');
+ .replace(MANAGED_WORKTREE_CLEANUP_CONTRACT, '')
+ .replaceAll(PATHS.root, '');
expect(withoutFixedGuidance.length).toBeLessThan(33_000);
});
diff --git a/server/services/cosAgentArchive.js b/server/services/cosAgentArchive.js
index 90131c285d..0b40a71b67 100644
--- a/server/services/cosAgentArchive.js
+++ b/server/services/cosAgentArchive.js
@@ -16,9 +16,9 @@ import { join } from 'path';
import { cosEvents } from './cosEvents.js';
import { loadState, saveState, withStateLock, AGENTS_DIR } from './cosState.js';
import { atomicWrite, ensureDir } from '../lib/fileUtils.js';
-import { ServerError } from '../lib/errorHandler.js';
-import { isDeliveryCleanupHeld } from '../lib/taskRetryHold.js';
import { loadAgentIndex, saveAgentIndex, recordArchivedAgentOrder, moveAgentDir } from './cosAgentIndex.js';
+import { readProtectedCleanupOwners } from './cosAgentLifecycle.js';
+import { releaseAgentReservation } from './agentMaintenanceReservations.js';
// Archive stale completed agents from state.json.
// Completed agents are already persisted to per-agent metadata files on disk
@@ -88,51 +88,72 @@ export async function archiveStaleAgents() {
// Clear completed agents from state, cache, and disk
export async function clearCompletedAgents() {
return withBackupAssetPublication(() => withStateLock(async () => {
- // Completion is persisted before its task hold; both records protect restart evidence.
- const { getTaskView } = await import('./cosTaskStore.js');
- const { readBranchCompletionPolicies } = await import('./branchCompletionPolicy.js');
- const [tasks, agents] = await Promise.all([
- getTaskView({ kind: 'queue' }),
- readBranchCompletionPolicies({ repairIndex: false }),
- ]);
- if ([...(tasks.user?.tasks || []), ...(tasks.cos?.tasks || [])]
- .some(task => task.status === 'in_progress' && isDeliveryCleanupHeld(task.metadata))
- || agents.some(agent => agent.status === 'completed' && agent.metadata?.repoStateVerification?.status === 'pending')) {
- throw new ServerError('Wait for delivery verification and cleanup before clearing completed agents.', {
- status: 409, code: 'DELIVERY_CLEANUP_PENDING',
- });
- }
const state = await loadState();
const idx = await loadAgentIndex();
+ const protectedOwners = await readProtectedCleanupOwners({ state, index: idx });
- // Remove completed agents from state
+ // Remove unprotected completed agents from state; retain current cleanup/retry owners
const stateCompleted = Object.keys(state.agents).filter(
id => state.agents[id].status === 'completed'
);
+ const removableStateIds = [];
+ const retainedIds = new Set();
for (const id of stateCompleted) {
+ if (protectedOwners.isProtected(id)) {
+ retainedIds.add(id);
+ continue;
+ }
+ removableStateIds.push(id);
delete state.agents[id];
}
- await saveState(state);
+ if (removableStateIds.length > 0) {
+ await saveState(state, { changedAgentIds: removableStateIds });
+ }
+
+ const removedIndexEntries = [];
+ const retainedDates = new Set();
+ const allDates = new Set();
+ for (const [id, date] of idx) {
+ allDates.add(date);
+ if (protectedOwners.isProtected(id)) {
+ retainedIds.add(id);
+ retainedDates.add(date);
+ } else {
+ removedIndexEntries.push([id, date]);
+ }
+ }
- // Collect all unique dates from index, then remove date bucket dirs
- const dates = new Set(idx.values());
- const totalCleared = idx.size + stateCompleted.filter(id => !idx.has(id)).length;
+ const totalCleared = removedIndexEntries.length + removableStateIds.filter(id => !idx.has(id)).length;
// Remove durable references before bytes; a failed index write must leave
// the old archive trees readable and its in-memory lookup retryable.
- const previousEntries = [...idx];
- idx.clear();
- try { await saveAgentIndex(); }
- catch (err) { for (const [id, date] of previousEntries) idx.set(id, date); throw err; }
+ if (removedIndexEntries.length > 0) {
+ for (const [id] of removedIndexEntries) idx.delete(id);
+ try { await saveAgentIndex(); }
+ catch (err) { for (const [id, date] of removedIndexEntries) idx.set(id, date); throw err; }
+ }
- const removals = [...dates].map(date => {
+ // Whole buckets with no retained entries can be removed in one pass; mixed
+ // buckets remove only the unprotected agent directories inside them.
+ const removals = [];
+ for (const date of allDates) {
+ if (retainedDates.has(date)) continue;
const dateDir = join(AGENTS_DIR, date);
- return existsSync(dateDir)
- ? rm(dateDir, { recursive: true }).catch(() => {})
- : Promise.resolve();
- });
+ if (existsSync(dateDir)) {
+ removals.push(rm(dateDir, { recursive: true }).catch(() => {}));
+ }
+ }
+ for (const [id, date] of removedIndexEntries) {
+ if (!retainedDates.has(date)) continue;
+ const agentDir = join(AGENTS_DIR, date, id);
+ if (existsSync(agentDir)) {
+ removals.push(rm(agentDir, { recursive: true }).catch(() => {}));
+ }
+ }
await Promise.all(removals);
- return { cleared: totalCleared };
+ for (const id of removableStateIds) await releaseAgentReservation(id);
+ cosEvents.emit('agents:changed', { action: 'cleared-completed', cleared: totalCleared, retained: retainedIds.size });
+ return { cleared: totalCleared, retained: retainedIds.size };
}));
}
diff --git a/server/services/cosAgentArchives.backupAdmission.test.js b/server/services/cosAgentArchives.backupAdmission.test.js
index 1cd1043c8c..14e49d97aa 100644
--- a/server/services/cosAgentArchives.backupAdmission.test.js
+++ b/server/services/cosAgentArchives.backupAdmission.test.js
@@ -7,15 +7,15 @@ const fixture = await vi.hoisted(async () => {
const { mkdtemp } = await import('node:fs/promises');
const { tmpdir } = await import('node:os');
const { join } = await import('node:path');
+ const { createFileWriteQueue } = await import('../lib/fileWriteQueue.js');
const base = await mkdtemp(join(tmpdir(), 'cos-archive-backup-'));
- return { base, root: join(base, 'agents'), state: null, tasks: null, beforeIndex: null, bypass: false };
+ return { base, root: join(base, 'agents'), state: null, beforeIndex: null, bypass: false, withStateLock: createFileWriteQueue() };
});
-vi.mock('./cosTaskStore.js', () => ({ getTaskView: async () => fixture.tasks }));
vi.mock('./cosState.js', () => ({
AGENTS_DIR: fixture.root,
loadState: async () => structuredClone(fixture.state),
saveState: async state => { fixture.state = structuredClone(state); await writeFile(join(fixture.root, 'state.json'), JSON.stringify(state)); },
- withStateLock: createFileWriteQueue(),
+ withStateLock: fn => fixture.withStateLock(fn),
readAgentsStateForSafetyCheck: async () => ({ trusted: true, agents: fixture.state.agents }),
}));
vi.mock('../lib/fileUtils.js', async importOriginal => {
@@ -42,6 +42,8 @@ let index;
let acquireCut;
const date = '2001-01-01';
const id = 'agent-example';
+let taskStore;
+let taskRepo;
const readIndex = async () => JSON.parse(await readFile(join(fixture.root, 'index.json'), 'utf8'));
const assertIndexedTrees = async () => {
for (const [agentId, bucket] of Object.entries(await readIndex())) {
@@ -49,22 +51,34 @@ const assertIndexedTrees = async () => {
expect(JSON.parse(await readFile(join(fixture.root, bucket, agentId, 'metadata.json'), 'utf8')).id).toBe(agentId);
}
};
-async function seed({ completed = false, indexed = false } = {}) {
- fixture.state.agents[id] = { id, status: completed ? 'completed' : 'running', startedAt: `${date}T01:00:00Z`,
- ...(completed ? { completedAt: `${date}T02:00:00Z`, result: { success: true } } : {}), metadata: {} };
- const dir = indexed ? join(fixture.root, date, id) : join(fixture.root, id);
+async function seedAgent(agentId, { completed = false, indexed = false, resident = true, bucketDate = date, result = { success: true } } = {}) {
+ const record = {
+ id: agentId,
+ status: completed ? 'completed' : 'running',
+ startedAt: `${bucketDate}T01:00:00Z`,
+ ...(completed ? { completedAt: `${bucketDate}T02:00:00Z`, result } : {}),
+ metadata: {},
+ };
+ if (resident) fixture.state.agents[agentId] = record;
+ const dir = indexed ? join(fixture.root, bucketDate, agentId) : join(fixture.root, agentId);
await mkdir(dir, { recursive: true });
await writeFile(join(dir, 'output.txt'), 'Example preserved output');
- await writeFile(join(dir, 'metadata.json'), JSON.stringify(fixture.state.agents[id]));
- if (indexed) { (await index.loadAgentIndex()).set(id, date); await index.saveAgentIndex(); }
+ await writeFile(join(dir, 'metadata.json'), JSON.stringify(record));
+ if (indexed) { (await index.loadAgentIndex()).set(agentId, bucketDate); await index.saveAgentIndex(); }
+}
+async function seed({ completed = false, indexed = false } = {}) {
+ await seedAgent(id, { completed, indexed });
}
beforeEach(async () => {
vi.resetModules(); fixture.beforeIndex = null; fixture.bypass = false;
fixture.state = { agents: {}, config: { completedAgentRetentionMs: 1 }, stats: { tasksCompleted: 0, errors: 0 } };
- fixture.tasks = { user: { tasks: [] }, cos: { tasks: [] } };
await rm(fixture.root, { recursive: true, force: true }); await mkdir(fixture.root);
await writeFile(join(fixture.root, 'index.json'), '{}');
({ acquireBackupSnapshotCut: acquireCut } = await import('../lib/backupSnapshotBoundary.js'));
+ taskRepo = await import('./cosTaskRepository.js');
+ taskRepo.__resetCosTaskRepository();
+ taskStore = await import('./cosTaskStore.js');
+ taskStore.__resetTaskCache();
index = await import('./cosAgentIndex.js');
api = { ...await import('./cosAgentLifecycle.js'), ...await import('./cosAgentArchive.js') };
});
@@ -78,35 +92,40 @@ it('preserves pending verification before the task hold is written and after arc
const metadataPath = join(fixture.root, date, id, 'metadata.json');
await writeFile(metadataPath, JSON.stringify(agent));
- await expect(api.clearCompletedAgents()).rejects.toMatchObject({ code: 'DELIVERY_CLEANUP_PENDING' });
+ expect(await api.clearCompletedAgents()).toEqual({ cleared: 0, retained: 1 });
expect(fixture.state.agents[id]).toEqual(agent);
delete fixture.state.agents[id];
- await expect(api.clearCompletedAgents()).rejects.toMatchObject({ code: 'DELIVERY_CLEANUP_PENDING' });
+ expect(await api.clearCompletedAgents()).toEqual({ cleared: 0, retained: 1 });
await assertIndexedTrees();
agent.metadata.repoStateVerification.status = 'completed';
await writeFile(metadataPath, JSON.stringify(agent));
- expect(await api.clearCompletedAgents()).toEqual({ cleared: 1 });
+ expect(await api.clearCompletedAgents()).toEqual({ cleared: 1, retained: 0 });
});
it.each(['user', 'cos'])('preserves cleanup-owner evidence in the %s queue until delivery settles', async source => {
await seed({ completed: true, indexed: true });
// An archived owner is absent from resident state but still required after restart.
delete fixture.state.agents[id];
- const task = { id: 'task-example', status: 'in_progress', metadata: {
- deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: id,
- } };
- fixture.tasks[source].tasks.push(task);
+ const taskType = source === 'cos' ? 'internal' : 'user';
+ await taskStore.addTask({
+ id: 'task-example',
+ status: 'in_progress',
+ priority: 'MEDIUM',
+ priorityValue: 2,
+ description: 'Cleanup task',
+ metadata: { deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: id },
+ }, taskType, { raw: true });
const beforeState = structuredClone(fixture.state);
const beforeIndex = await readIndex();
- await expect(api.clearCompletedAgents()).rejects.toMatchObject({ status: 409, code: 'DELIVERY_CLEANUP_PENDING' });
+ expect(await api.clearCompletedAgents()).toEqual({ cleared: 0, retained: 1 });
expect(fixture.state).toEqual(beforeState);
expect(await readIndex()).toEqual(beforeIndex);
await assertIndexedTrees();
- task.status = 'completed';
- expect(await api.clearCompletedAgents()).toEqual({ cleared: 1 });
+ await taskStore.updateTask('task-example', { status: 'completed' }, taskType);
+ expect(await api.clearCompletedAgents()).toEqual({ cleared: 1, retained: 0 });
expect(await readIndex()).toEqual({});
expect(await readdir(fixture.root)).not.toContain(date);
});
@@ -212,3 +231,157 @@ it('does not let a cold reader queue its shared initialization ahead of an admit
try { await assertIndexedTrees(); } finally { release(); }
await reader;
});
+
+describe('active cleanup and retry owner preservation (#13430)', () => {
+ async function seedCleanupTask(taskId, metadata, taskType = 'internal') {
+ return taskStore.addTask({
+ id: taskId,
+ status: 'in_progress',
+ priority: 'MEDIUM',
+ priorityValue: 2,
+ description: `Cleanup task ${taskId}`,
+ metadata,
+ }, taskType, { raw: true });
+ }
+
+ it.each([
+ { label: 'resident record', indexed: false, resident: true },
+ { label: 'indexed archive', indexed: true, resident: false },
+ ])('refuses single delete and retains $label while task holds deliveryRecovery cleanup', async ({ indexed, resident }) => {
+ await seedAgent(id, { completed: true, indexed, resident });
+ await seedCleanupTask('sys-rl-example', {
+ deliveryRecovery: 'cleanup',
+ deliveryRecoveryAgentId: id,
+ });
+
+ await expect(api.deleteAgent(id)).rejects.toMatchObject({
+ status: 409,
+ code: 'AGENT_CLEANUP_OWNED',
+ });
+
+ const record = await api.getAgent(id);
+ expect(record?.id).toBe(id);
+ if (resident) expect(fixture.state.agents[id]?.id).toBe(id);
+ if (indexed) await assertIndexedTrees();
+
+ await taskStore.updateTask('sys-rl-example', {
+ status: 'completed',
+ metadata: { deliveryRecovery: undefined, deliveryRecoveryAgentId: undefined },
+ }, 'internal');
+
+ await expect(api.deleteAgent(id)).resolves.toEqual({ success: true, agentId: id });
+ expect(await api.getAgent(id)).toBeNull();
+ });
+
+ it('retains protected resident and archived runs across mixed buckets during clearCompletedAgents', async () => {
+ const protectedResident = 'agent-protected-resident';
+ const protectedArchived = 'agent-protected-archived';
+ const unprotectedSameBucket = 'agent-unprotected-same-bucket';
+ const unprotectedOtherBucket = 'agent-unprotected-other-bucket';
+ const otherDate = '2001-01-02';
+
+ await seedAgent(protectedResident, { completed: true, indexed: false, resident: true });
+ await seedAgent(protectedArchived, { completed: true, indexed: true, resident: false, bucketDate: date });
+ await seedAgent(unprotectedSameBucket, { completed: true, indexed: true, resident: true, bucketDate: date });
+ await seedAgent(unprotectedOtherBucket, { completed: true, indexed: true, resident: false, bucketDate: otherDate });
+
+ await seedCleanupTask('sys-rl-example', {
+ deliveryRecovery: 'cleanup',
+ deliveryRecoveryAgentId: protectedResident,
+ }, 'internal');
+ await seedCleanupTask('task-retry-example', {
+ retryPendingCleanup: protectedArchived,
+ retryPendingSince: `${date}T02:00:00.000Z`,
+ }, 'user');
+
+ const cleared = await api.clearCompletedAgents();
+ expect(cleared).toEqual({ cleared: 2, retained: 2 });
+
+ expect(fixture.state.agents[protectedResident]?.id).toBe(protectedResident);
+ expect(fixture.state.agents[unprotectedSameBucket]).toBeUndefined();
+ expect(await readIndex()).toEqual({ [protectedArchived]: date });
+ await assertIndexedTrees();
+ await expect(readFile(join(fixture.root, date, unprotectedSameBucket, 'metadata.json'), 'utf8')).rejects.toThrow();
+ await expect(readdir(join(fixture.root, otherDate))).rejects.toThrow();
+
+ await taskStore.updateTask('sys-rl-example', { status: 'completed' }, 'internal');
+ await taskStore.updateTask('task-retry-example', { status: 'pending' }, 'user');
+
+ const secondClear = await api.clearCompletedAgents();
+ expect(secondClear).toEqual({ cleared: 2, retained: 0 });
+ expect(fixture.state.agents).toEqual({});
+ expect(await readIndex()).toEqual({});
+ });
+
+ it('does not retain completed history solely because a stale agent result still carries cleanupPending', async () => {
+ await seedAgent(id, {
+ completed: true,
+ indexed: true,
+ resident: true,
+ result: { success: true, cleanupPending: true },
+ });
+ await taskStore.addTask({
+ id: 'sys-rl-settled',
+ status: 'completed',
+ priority: 'MEDIUM',
+ priorityValue: 2,
+ description: 'Already settled task',
+ metadata: {},
+ }, 'internal', { raw: true });
+
+ await expect(api.clearCompletedAgents()).resolves.toEqual({ cleared: 1, retained: 0 });
+ expect(await api.getAgent(id)).toBeNull();
+ });
+
+ it('revalidates current task ownership under the state lock so a concurrent reassignment is not deleted', async () => {
+ const oldOwner = 'agent-old-owner';
+ const newOwner = 'agent-new-owner';
+ await seedAgent(oldOwner, { completed: true, indexed: true, resident: true });
+ await seedAgent(newOwner, { completed: true, indexed: true, resident: true });
+ await seedCleanupTask('sys-rl-race', {
+ deliveryRecovery: 'cleanup',
+ deliveryRecoveryAgentId: oldOwner,
+ }, 'internal');
+
+ // Start clearCompletedAgents while an earlier task reassignment is holding the shared
+ // state lock; clearCompletedAgents must read task ownership AFTER acquiring the lock,
+ // not from a stale pre-lock snapshot.
+ const entered = deferred();
+ const release = deferred();
+ const blocker = fixture.withStateLock(async () => {
+ entered.resolve();
+ await release.promise;
+ });
+ await entered.promise;
+
+ const reassign = taskStore.updateTask('sys-rl-race', {
+ status: 'in_progress',
+ metadata: { deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: newOwner },
+ }, 'internal');
+ await settle();
+ const clear = api.clearCompletedAgents();
+ await settle();
+
+ release.resolve();
+ await blocker;
+ const [, clearResult] = await Promise.all([reassign, clear]);
+ expect(clearResult).toEqual({ cleared: 1, retained: 1 });
+ expect((await api.getAgent(newOwner))?.id).toBe(newOwner);
+ expect(await api.getAgent(oldOwner)).toBeNull();
+ });
+
+ it('preserves evidence when task cleanup ownership cannot be verified', async () => {
+ await seedAgent(id, { completed: true, indexed: true, resident: true });
+ const spy = vi.spyOn(taskStore, 'getTaskView').mockRejectedValueOnce(new Error('task store unavailable'));
+ try {
+ await expect(api.deleteAgent(id)).rejects.toMatchObject({
+ status: 503,
+ code: 'CLEANUP_OWNERSHIP_UNAVAILABLE',
+ });
+ expect((await api.getAgent(id))?.id).toBe(id);
+ await assertIndexedTrees();
+ } finally {
+ spy.mockRestore();
+ }
+ });
+});
diff --git a/server/services/cosAgentLifecycle.js b/server/services/cosAgentLifecycle.js
index 621ac983fe..174e73c712 100644
--- a/server/services/cosAgentLifecycle.js
+++ b/server/services/cosAgentLifecycle.js
@@ -24,8 +24,56 @@ import { repairCodexTaskSummary } from './codexSummaryRepair.js';
import { loadAgentIndex, saveAgentIndex, getAgentDir, recordArchivedAgentOrder, moveAgentDir } from './cosAgentIndex.js';
import { hasClaimCheckoutBindings } from './completedCheckoutEvidence.js';
import { isAgentHandoff } from '../lib/agentOutcome.js';
+import { collectProtectedCleanupOwners } from '../lib/taskRetryHold.js';
import { reconcileStaleAgentReservations, releaseAgentReservation } from './agentMaintenanceReservations.js';
+function hasPendingRepoStateVerification(agent) {
+ return agent?.status === 'completed' && agent?.metadata?.repoStateVerification?.status === 'pending';
+}
+
+export async function readProtectedCleanupOwners({ state = null, index = null, targetAgentId = null } = {}) {
+ const store = await import('./cosTaskStore.js').catch(err => {
+ throw new ServerError(`Cannot verify task cleanup ownership: ${err.message}`, {
+ status: 503, code: 'CLEANUP_OWNERSHIP_UNAVAILABLE',
+ });
+ });
+ const getTaskView = 'getTaskView' in store ? store.getTaskView : null;
+ if (typeof getTaskView !== 'function') {
+ throw new ServerError('Cannot verify task cleanup ownership before deleting agent history', {
+ status: 503, code: 'CLEANUP_OWNERSHIP_UNAVAILABLE',
+ });
+ }
+ const view = await getTaskView({ kind: 'queue' }).catch(err => {
+ throw new ServerError(`Cannot verify task cleanup ownership: ${err.message}`, {
+ status: 503, code: 'CLEANUP_OWNERSHIP_UNAVAILABLE',
+ });
+ });
+ if (!Array.isArray(view?.user?.tasks) || !Array.isArray(view?.cos?.tasks)) {
+ throw new ServerError('Cannot verify task cleanup ownership before deleting agent history', {
+ status: 503, code: 'CLEANUP_OWNERSHIP_UNAVAILABLE',
+ });
+ }
+ const protectedOwners = collectProtectedCleanupOwners([...view.user.tasks, ...view.cos.tasks]);
+ // Completion is persisted before its task hold; pending repo-state verification
+ // on a resident or archived completion record also protects restart evidence.
+ for (const [id, agent] of Object.entries(state?.agents || {})) {
+ if (hasPendingRepoStateVerification(agent)) protectedOwners.ownerIds.add(id);
+ }
+ if (index instanceof Map) {
+ const candidates = targetAgentId
+ ? (index.has(targetAgentId) ? [[targetAgentId, index.get(targetAgentId)]] : [])
+ : [...index.entries()];
+ for (const [id, date] of candidates) {
+ if (protectedOwners.ownerIds.has(id) || state?.agents?.[id] || !date) continue;
+ const content = await tryReadFile(join(AGENTS_DIR, date, id, 'metadata.json'));
+ if (!content) continue;
+ const parsed = safeJSONParse(content, null, { logError: false });
+ if (hasPendingRepoStateVerification(parsed)) protectedOwners.ownerIds.add(id);
+ }
+ }
+ return protectedOwners;
+}
+
export async function registerAgent(agentId, taskId, metadata = {}) {
return withStateLock(async () => {
const state = await loadState();
@@ -1045,6 +1093,14 @@ export async function deleteAgent(agentId) {
throw new ServerError('Agent not found', { status: 404, code: 'NOT_FOUND' });
}
+ const protectedOwners = await readProtectedCleanupOwners({ state, index: idx, targetAgentId: agentId });
+ if (protectedOwners.isProtected(agentId)) {
+ throw new ServerError(
+ `Agent ${agentId} is still owned by an active task delivery or retry cleanup hold and cannot be deleted until cleanup finishes`,
+ { status: 409, code: 'AGENT_CLEANUP_OWNED' },
+ );
+ }
+
const wasRunning = state.agents[agentId]?.status === 'running';
delete state.agents[agentId];
await saveState(state, { changedAgentIds: [agentId] });
diff --git a/server/services/cosAgentLifecycle.test.js b/server/services/cosAgentLifecycle.test.js
index 69ba54d3f0..a90f4d8309 100644
--- a/server/services/cosAgentLifecycle.test.js
+++ b/server/services/cosAgentLifecycle.test.js
@@ -28,7 +28,10 @@ vi.mock('./cosState.js', () => ({
vi.mock('./branchCompletionPolicy.js', () => ({ readBranchCompletionPolicies: vi.fn(async () => Object.values(mockCosState.state?.agents || {})) }));
const recoveryTasks = vi.hoisted(() => new Map());
-vi.mock('./cosTaskStore.js', () => ({ getTaskById: vi.fn(async id => recoveryTasks.get(id) || null) }));
+vi.mock('./cosTaskStore.js', () => ({
+ getTaskById: vi.fn(async id => recoveryTasks.get(id) || null),
+ getTaskView: vi.fn(async () => ({ user: { tasks: [...recoveryTasks.values()] }, cos: { tasks: [] } })),
+}));
vi.mock('./domainUsage.js', () => ({
recordDomainUsage: vi.fn(async () => {})