Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: bind delivery recovery to current owners and preterminal checkout evidence

[._.]/ Adam Eivyeb3d512cb3df
diff --git a/server/services/agentFinalization.js b/server/services/agentFinalization.js
index f78ff7b411..db5ee42d05 100644
--- a/server/services/agentFinalization.js
+++ b/server/services/agentFinalization.js
@@ -1301,18 +1301,23 @@ export async function retireDeadAgent({ agent, task, success, exitCode, duration
const cleanupAllowed = savedCleanup || (executionSuccess && validationPassed !== false
&& (agent.metadata?.isWorktree === true || hasClaimCheckoutBindings(agent.metadata)));
const recoveryOutcome = !cleanupAllowed && taskOutcome?.recovery?.mode === 'cleanup' ? { ...taskOutcome, recovery: undefined } : taskOutcome;
- const recoveryUpdate = deliveryHeld && !refused && !isPrivateSecurity && task?.id && task.status !== 'completed'
+ let recoveryUpdate = deliveryHeld && !refused && !isPrivateSecurity && task?.id && task.status !== 'completed'
? resolveUndeliveredTaskUpdate(task, recoveryOutcome, agent.id) : null;
+ let recoveryOwnershipLost = false;
if (recoveryUpdate) {
- const update = await updateTask(task.id, recoveryUpdate, task.taskType || 'user');
+ const update = await updateTask(task.id, recoveryUpdate, task.taskType || 'user', {
+ expectedStatus: task.status || null, expectedTask: { description: task.description, priority: task.priority, metadata: task.metadata },
+ recoveryOwner: { id: agent.id, startedAt: agent.startedAt },
+ });
if (update?.error) throw new Error(`Could not persist recovered task outcome: ${update.error}`);
+ if (update?.statusChanged) { recoveryUpdate = null; recoveryOwnershipLost = true; }
}
await completeAgent(agent.id, {
success: effectiveSuccess,
executionSuccess,
...(recoveryUpdate ? { deliveryRecoveryOwned: true } : {}),
...(recoveryUpdate?.metadata?.deliveryRecoveryAgentId === agent.id ? { cleanupPending: true } : {}),
- ...(savedCleanup ? { cleanupDeliveryVerified: true } : {}),
+ ...(savedCleanup && !recoveryOwnershipLost ? { cleanupDeliveryVerified: true } : {}),
taskOutcome,
validationPassed,
...(claimDisposition ? { claimDisposition } : {}),
diff --git a/server/services/agentFinalization.retireDeadAgent.test.js b/server/services/agentFinalization.retireDeadAgent.test.js
index 353204393a..fea10d8b9c 100644
--- a/server/services/agentFinalization.retireDeadAgent.test.js
+++ b/server/services/agentFinalization.retireDeadAgent.test.js
@@ -128,7 +128,7 @@ describe('retireDeadAgent (#8440)', () => {
task: TASK, success: true, exitCode: 0, duration: 100 });
expect(result).toEqual({ success: false, taskOutcome, deliveryHeld: true });
expect(completeAgent).toHaveBeenCalledWith(AGENT.id, expect.objectContaining({ success: false, executionSuccess: true, taskOutcome }));
- expect(updateTask).toHaveBeenCalledWith(TASK.id, expect.objectContaining({ status: 'in_progress', metadata: expect.objectContaining({ taskOutcome, retryPendingCleanup: AGENT.id, failureCount: 1 }) }), 'user');
+ expect(updateTask).toHaveBeenCalledWith(TASK.id, expect.objectContaining({ status: 'in_progress', metadata: expect.objectContaining({ taskOutcome, retryPendingCleanup: AGENT.id, failureCount: 1 }) }), 'user', expect.objectContaining({ recoveryOwner: { id: AGENT.id, startedAt: AGENT.startedAt } }));
expect(order).toEqual(['task', 'remove', 'release']);
} finally { await rm(workspacePath, { recursive: true, force: true }); }
});
@@ -139,10 +139,26 @@ describe('retireDeadAgent (#8440)', () => {
} };
await retireDeadAgent({ agent: { ...AGENT, result: { taskOutcome } }, task: TASK, success: true, exitCode: 0, duration: 100 });
expect(updateTask).toHaveBeenCalledWith(TASK.id, expect.objectContaining({ status: 'blocked',
- metadata: expect.objectContaining({ deliveryDecisionAgentId: AGENT.id }) }), 'user');
+ metadata: expect.objectContaining({ deliveryDecisionAgentId: AGENT.id }) }), 'user', expect.objectContaining({ expectedTask: { description: TASK.description, priority: TASK.priority, metadata: TASK.metadata } }));
expect(releaseRetryHold).toHaveBeenCalledWith({ agentId: AGENT.id, task: TASK, success: false, agentMetadata: AGENT.metadata });
});
+ it.each([false, true])('retires only the dead run when the guarded recovery write loses task ownership (cleanup=%s)', async cleanup => {
+ const taskOutcome = { status: 'incomplete', reason: 'Original attempt needs validation.' };
+ const task = { ...TASK, status: 'in_progress', metadata: { claimedBy: 'original', lastSpawnedAt: AGENT.startedAt,
+ ...(cleanup && { deliveryRecovery: 'cleanup', deliveryRecoveryAgentId: AGENT.id }) } };
+ updateTask.mockResolvedValueOnce({ statusChanged: true, task: { ...task, metadata: { ...task.metadata, blocker: 'Operator hold' } } });
+ await retireDeadAgent({ agent: { ...AGENT, result: { taskOutcome } }, task, success: true, exitCode: 0, duration: 100 });
+ expect(updateTask).toHaveBeenCalledWith(TASK.id, expect.any(Object), 'user', expect.objectContaining({
+ expectedStatus: 'in_progress', expectedTask: { description: task.description, priority: task.priority, metadata: task.metadata },
+ recoveryOwner: { id: AGENT.id, startedAt: AGENT.startedAt },
+ }));
+ expect(completeAgent.mock.calls.at(-1)[1].deliveryRecoveryOwned).not.toBe(true);
+ expect(completeAgent.mock.calls.at(-1)[1].cleanupPending).not.toBe(true);
+ expect(completeAgent.mock.calls.at(-1)[1].cleanupDeliveryVerified).not.toBe(true);
+ expect(releaseRetryHold).not.toHaveBeenCalled();
+ });
+
it('accepts a recovered no-op only after its output hook verifies delivery', async () => {
const taskOutcome = { status: 'no-action-needed', reason: 'Invented verification found no remaining work.' };
const hook = vi.fn(async () => ({ accepted: true }));
diff --git a/server/services/agentRepoStateVerification.js b/server/services/agentRepoStateVerification.js
index ca69afdc57..a2ea82a2c1 100644
--- a/server/services/agentRepoStateVerification.js
+++ b/server/services/agentRepoStateVerification.js
@@ -617,6 +617,7 @@ async function settleAbsentCheckout({ agentId, agentState, metadata, app, source
const integrated = forge && (await execGit(['merge-base', '--is-ancestor', forge.prHead, forge.defaultHead], sourceWorkspace,
{ ignoreExitCode: true, env: { ...process.env, GIT_OPTIONAL_LOCKS: '0', GIT_NO_LAZY_FETCH: '1' } }).catch(() => null))?.exitCode === 0;
const completed = metadata.completedCheckouts?.find(row => [1, 2].includes(row.version) && row.agentId === agentId
+ && row.startedAt === agentState.startedAt
&& row.repoPath === sourceWorkspace && row.branch === branchName && row.path === worktreePath
&& Number.isFinite(Date.parse(row.startedAt)) && Date.parse(row.capturedAt) >= Date.parse(row.startedAt));
if (completed?.version === 2) {
diff --git a/server/services/agentRepoStateVerification.test.js b/server/services/agentRepoStateVerification.test.js
index c0268fcd07..1cfcb3e4a9 100644
--- a/server/services/agentRepoStateVerification.test.js
+++ b/server/services/agentRepoStateVerification.test.js
@@ -108,6 +108,7 @@ const RELEASE = { branch: BRANCH, path: WORKTREE, head: HEAD, locked: false, loc
const CREATED_AT = '2026-01-01T00:00:00.000Z';
const agentState = (overrides = {}) => ({
+ startedAt: CREATED_AT,
metadata: {
isWorktree: true,
sourceWorkspace: '/repo',
@@ -286,6 +287,16 @@ describe('verifyAgentRepoState — clean runs', () => {
expect(await run({ agentState: state })).toMatchObject({ verified: false, skipReason: 'already-absent-preservation-unverified' });
});
+ it('withholds absent-checkout settlement for a receipt from an earlier execution generation', async () => {
+ const completed = { version: 1, agentId: 'agent-1', repoPath: '/repo', branch: BRANCH, path: WORKTREE, head: HEAD,
+ startedAt: '2025-01-01T00:00:00.000Z', capturedAt: CREATED_AT };
+ const state = agentState({ completedCheckouts: [completed], reviewLoopMergeTarget: { url: 'https://github.com/example/repo/pull/7' } });
+ findMissingCheckoutPreservation.mockResolvedValue({ archiveDir: '/example/archive', evidenceDigest: 'a'.repeat(64), recheck: async () => true });
+ expect(await run({ agentState: state })).toMatchObject({ verified: false });
+ expect(findMissingCheckoutPreservation).not.toHaveBeenCalled();
+ expect(recordCleanupOutcome).not.toHaveBeenCalledWith('agent-1', expect.objectContaining({ status: 'verified' }));
+ });
+
it.each([['broken leaf symlink', true], ['unreadable EACCES leaf', null]])('never treats %s as absent or verified cleanup', async (_name, presence) => {
readCheckoutPathPresence.mockResolvedValue(presence);
const result = await run();
diff --git a/server/services/completedCheckoutEvidence.js b/server/services/completedCheckoutEvidence.js
index f67a49e66b..191123e52d 100644
--- a/server/services/completedCheckoutEvidence.js
+++ b/server/services/completedCheckoutEvidence.js
@@ -113,19 +113,29 @@ export function completedCheckoutReceiptValid(agent, receipt) {
/** Read the final head only inside the pre-existing physical binding. No fetch
* or branch-name discovery is allowed after terminal ingestion.
*/
-export async function captureSeededCompletionCheckouts(agent) {
+export async function captureSeededCompletionCheckouts(agent, { preterminal = false } = {}) {
+ if (preterminal && !['running', 'paused'].includes(agent.status)) return null;
+ const terminalAt = preterminal ? Date.now() : Date.parse(agent.completedAt);
const bound = hasClaimCheckoutBindings(agent.metadata) ? agent.metadata.claimBranches : [agent.metadata?.worktreeBranch];
const seeds = agent.metadata?.checkoutSeeds?.filter(seed => Array.isArray(bound) && bound.includes(seed.branch));
if (!Array.isArray(seeds) || !seeds.length) return null;
const receipts = [];
for (const seed of seeds) {
if (!checkoutSeedSchema.safeParse(seed).success || seed.agentId !== agent.id || seed.startedAt !== agent.startedAt
- || Date.parse(seed.seededAt) < Date.parse(agent.startedAt) || Date.parse(seed.seededAt) > Date.parse(agent.completedAt)
+ || Date.parse(seed.seededAt) < Date.parse(agent.startedAt) || Date.parse(seed.seededAt) > terminalAt
|| !same(seed.repository, await filesystemIdentity(seed.repoPath, 'directory'))
|| !same(seed.checkout, await filesystemIdentity(seed.path, 'directory'))) return null;
const head = (await execGit(['rev-parse', '--verify', 'HEAD^{commit}'], seed.path, {
env: { ...process.env, GIT_NO_LAZY_FETCH: '1', GIT_OPTIONAL_LOCKS: '0', GIT_NO_REPLACE_OBJECTS: '1' }, timeout: 5000,
})).stdout.trim();
+ // A physical seed survives later commits in the same checkout. Only a head
+ // observed while this execution still owned it may authorize late capture.
+ const terminal = agent.metadata?.checkoutCapture?.targets?.find(row => row.agentId === agent.id
+ && row.startedAt === agent.startedAt && row.repoPath === seed.repoPath && row.branch === seed.branch
+ && row.path === seed.path && same(row.incarnation, seed.incarnation)
+ && Number.isFinite(Date.parse(row.capturedAt)) && Date.parse(row.capturedAt) >= Date.parse(agent.startedAt)
+ && Date.parse(row.capturedAt) <= terminalAt);
+ if (!preterminal && (!terminal || terminal.head !== head)) return null;
const target = { repoPath: seed.repoPath, branch: seed.branch, path: seed.path, head };
const current = await readCompletedCheckoutIdentity(target);
if (!current || !same(current.incarnation, seed.incarnation)) return null;
diff --git a/server/services/completedCheckoutEvidence.test.js b/server/services/completedCheckoutEvidence.test.js
index dde38d95d8..32241d996c 100644
--- a/server/services/completedCheckoutEvidence.test.js
+++ b/server/services/completedCheckoutEvidence.test.js
@@ -4,7 +4,7 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { execGit } from '../lib/execGit.js';
import { materializeGitRepo } from '../lib/gitTestRepo.js';
-import { captureCompletedCheckouts, hasCompletedCheckoutEvidence, readCompletedCheckoutIdentity } from './completedCheckoutEvidence.js';
+import { captureCheckoutSeeds, captureCompletedCheckouts, captureSeededCompletionCheckouts, hasCompletedCheckoutEvidence, readCompletedCheckoutIdentity } from './completedCheckoutEvidence.js';
describe('completed claim checkout evidence', () => {
let scratch, repoPath, path, head, agent;
@@ -99,4 +99,13 @@ describe('completed claim checkout evidence', () => {
expect(await hasCompletedCheckoutEvidence([record, { ...newer, startedAt: null }], target())).toBe(false);
}
});
+
+ it('withholds late capture when only a physical seed survived and no terminal head was observed', async () => {
+ const checkoutSeeds = await captureCheckoutSeeds(agent);
+ const record = { ...completed([]), metadata: { ...agent.metadata, checkoutSeeds,
+ checkoutCapture: { status: 'pending', targets: [], unknown: true } } };
+ await execGit(['commit', '--allow-empty', '-m', 'later writer in the same physical checkout'], path);
+ expect(await captureSeededCompletionCheckouts(record)).toBeNull();
+ expect(await hasCompletedCheckoutEvidence([record], { ...target(), head: (await execGit(['rev-parse', 'HEAD'], path)).stdout.trim() })).toBe(false);
+ });
});
diff --git a/server/services/cosAgentLifecycle.js b/server/services/cosAgentLifecycle.js
index 21312623b7..293f093f8e 100644
--- a/server/services/cosAgentLifecycle.js
+++ b/server/services/cosAgentLifecycle.js
@@ -428,7 +428,7 @@ export async function completeAgent(agentId, result = {}) {
completedCheckouts = [];
// Identity capture describes the owned physical checkout, not a passing
// execution verdict. Failed/orphaned runs may already have delivered code.
- const { captureCompletedCheckouts } = await import('./completedCheckoutEvidence.js');
+ const { captureCompletedCheckouts, captureSeededCompletionCheckouts } = await import('./completedCheckoutEvidence.js');
try { completedCheckouts = await captureCompletedCheckouts(agent, { success: true, strict: true }); }
catch (error) {
// Ingestion remains terminal. Retry only an originally captured exact
@@ -443,6 +443,12 @@ export async function completeAgent(agentId, result = {}) {
recovery: { mode: 'cleanup' } } };
}
}
+ if (!completedCheckouts.length && agent.metadata?.checkoutSeeds?.length) {
+ // Capture a terminal head before publishing completion, even when the
+ // ordinary inventory read failed. A later retry must match this head.
+ const targets = await captureSeededCompletionCheckouts(agent, { preterminal: true }).catch(() => null);
+ if (targets?.length) checkoutCapture = { ...(checkoutCapture || { status: 'captured' }), targets, unknown: false };
+ }
}
// The execution can end before repository preservation is complete. Keep
diff --git a/server/services/cosAgentLifecycle.test.js b/server/services/cosAgentLifecycle.test.js
index a44ae75b58..3e1a6c3a1b 100644
--- a/server/services/cosAgentLifecycle.test.js
+++ b/server/services/cosAgentLifecycle.test.js
@@ -646,7 +646,7 @@ describe('completeAgent idempotence (#3384)', () => {
expect(persisted.metadata.completedCheckouts).toEqual([evidence]);
});
- it.each(['unchanged', 'physical-generation', 'new-owner', 'orphan-empty', 'orphan-physical-generation', 'orphan-new-owner'])('recovers only the original completion physical seed after restart, scenario=%s', async scenario => {
+ it.each(['unchanged', 'physical-generation', 'new-owner', 'later-head', 'orphan-empty', 'orphan-physical-generation', 'orphan-new-owner', 'orphan-later-head'])('recovers only the original completion physical seed after restart, scenario=%s', async scenario => {
const orphan = scenario.startsWith('orphan-');
const changed = !['unchanged', 'orphan-empty'].includes(scenario);
const scratch = await mkdtemp(join(tmpdir(), 'portos-completion-capture-seed-'));
@@ -674,9 +674,14 @@ describe('completeAgent idempotence (#3384)', () => {
if (orphan) expect(completed).toMatchObject({ result: { success: false, executionSuccess: false, orphaned: true },
metadata: { completedCheckouts: [], checkoutCapture: { status: 'captured' } } });
else expect(completed).toMatchObject({ status: 'completed', result: { cleanupPending: true, executionSuccess: true },
- metadata: { checkoutCapture: { status: 'pending', unknown: true } } });
+ metadata: { checkoutCapture: { status: 'pending', unknown: false, targets: [expect.objectContaining({ head })] } } });
expect(cosEvents.listenerCount('agent:completed')).toBeGreaterThan(0);
delete mockCosState.state.agents[agentId]; // Restart: only the real archived metadata remains.
+ let retainedHead = head;
+ if (scenario.endsWith('later-head')) {
+ await execGit(['-c', 'user.name=Test', '-c', '[email protected]', 'commit', '--allow-empty', '-m', 'later unrelated writer'], checkout);
+ retainedHead = (await execGit(['rev-parse', 'HEAD'], checkout)).stdout.trim();
+ }
if (scenario.endsWith('new-owner')) {
mockCosState.state.agents['new-owner'] = { id: 'new-owner', status: 'completed', startedAt: new Date().toISOString(),
metadata: { sourceWorkspace: repo, workspacePath: checkout, worktreeBranch: 'feature/seeded' } };
@@ -699,7 +704,7 @@ describe('completeAgent idempotence (#3384)', () => {
}
if (orphan) expect(recovered.result).toEqual(completed.result);
expect(await readFile(join(checkout, 'delivery.txt'), 'utf8')).toBe('wanted original work');
- expect((await execGit(['rev-parse', 'HEAD'], checkout)).stdout.trim()).toBe(head);
+ expect((await execGit(['rev-parse', 'HEAD'], checkout)).stdout.trim()).toBe(retainedHead);
expect(emittedCompletions).toHaveLength(1);
} finally {
failFirstRead?.mockRestore();
diff --git a/server/services/cosTaskStore.js b/server/services/cosTaskStore.js
index 3a7ce92c35..b1e4e69aff 100644
--- a/server/services/cosTaskStore.js
+++ b/server/services/cosTaskStore.js
@@ -1,6 +1,7 @@
import { auditWorkflow, newAuditMetadata, isLegacyDeepAudit } from '../lib/auditWorkflow.js';
import { isForgeMaintenanceTask, hasCurrentForgeMaintenanceEvidence, LEGACY_FORGE_MAINTENANCE_REASON } from '../lib/forgeMaintenanceTasks.js';
import { getSkipReason } from './cosTaskClaim.js';
+import { isDeepStrictEqual } from 'node:util';
/**
* CoS Task Store Module
*
@@ -31,7 +32,7 @@ import { PAUSED_BLOCKED_CATEGORIES, USER_DECISION_BLOCKED_CATEGORIES } from '../
import { splitTaskPromptFields, TASK_PROMPT_KEY, TASK_CONTEXT_KEY } from '../lib/cosTaskPrompt.js';
import { normalizeOrchestrationMode, normalizeOrchestrationProfile } from '../lib/orchestrationProfile.js';
import { loadState, withStateLock } from './cosState.js';
-import { hasActiveTaskOwner } from './agentState.js';
+import { activeAgents, runnerAgents, hasActiveTaskOwner } from './agentState.js';
import { cosEvents } from './cosEvents.js';
import { CLAIM_METADATA_KEYS } from './cosTaskClaim.js';
import { mergeTaskLists } from './cosTaskMerge.js';
@@ -678,10 +679,10 @@ async function addTaskAttempt(taskData, taskType, { raw = false, ignoreTaskId =
* `expectedStatus`, when supplied, is checked under the task-file lock. A mismatch
* returns `{ statusChanged: true }` without writing or retiring pause state.
*/
-export async function updateTask(taskId, updates, taskType = 'user', { now = Date.now(), suppressDequeue = false, expectedStatus = null, expectedMetadata = null } = {}) {
+export async function updateTask(taskId, updates, taskType = 'user', { now = Date.now(), suppressDequeue = false, expectedStatus = null, expectedMetadata = null, expectedTask = null, recoveryOwner = null, preserveRetryBudget = false } = {}) {
validateRequest(cosTaskTypeSchema, taskType);
const release = await preparePauseRelease(taskId, updates);
- const result = await writeTaskUpdate(taskId, release ? { ...updates, metadata: release.metadata } : updates, taskType, { now, suppressDequeue, expectedStatus, expectedMetadata });
+ const result = await writeTaskUpdate(taskId, release ? { ...updates, metadata: release.metadata } : updates, taskType, { now, suppressDequeue, expectedStatus, expectedMetadata, expectedTask, recoveryOwner, preserveRetryBudget });
if (release && !result?.error && !result?.statusChanged) {
await retirePausedAgent(release.agentId, taskId, resolveTaskTargetBranch(result?.metadata));
}
@@ -789,12 +790,12 @@ function mergeUpdateMetadata(existingMetadata, updates) {
return merged;
}
-async function writeTaskUpdate(taskId, updates, taskType, { now, suppressDequeue = false, expectedStatus = null, expectedMetadata = null }) {
- return withStateLock(() => writeTaskUpdateLocked(taskId, updates, taskType, { now, suppressDequeue, expectedStatus, expectedMetadata }));
+async function writeTaskUpdate(taskId, updates, taskType, options) {
+ return withStateLock(() => writeTaskUpdateLocked(taskId, updates, taskType, options));
}
// Caller holds withStateLock, including its status and ownership decision.
-async function writeTaskUpdateLocked(taskId, updates, taskType, { now, suppressDequeue = false, expectedStatus = null, expectedMetadata = null }) {
+async function writeTaskUpdateLocked(taskId, updates, taskType, { now, suppressDequeue = false, expectedStatus = null, expectedMetadata = null, expectedTask = null, recoveryOwner = null, preserveRetryBudget = false }) {
const tasks = await readTaskFile(taskType);
let taskIndex = tasks.findIndex(t => t.id === taskId);
@@ -820,6 +821,18 @@ async function writeTaskUpdateLocked(taskId, updates, taskType, { now, suppressD
}
const current = tasks[taskIndex];
+ if (expectedTask && Object.entries(expectedTask).some(([key, value]) => !isDeepStrictEqual(current[key], value))) {
+ return { statusChanged: true, task: current };
+ }
+ if (recoveryOwner) {
+ const state = await loadState();
+ const registered = state.agents?.[recoveryOwner.id];
+ const competing = Object.values(state.agents || {}).some(agent => agent.id !== recoveryOwner.id
+ && agent.taskId === taskId && ['running', 'paused'].includes(agent.status))
+ || [activeAgents, runnerAgents].some(entries => [...entries].some(([id, agent]) => id !== recoveryOwner.id
+ && (agent.taskId || agent.task?.id) === taskId));
+ if (competing || (registered && registered.startedAt !== recoveryOwner.startedAt)) return { statusChanged: true, task: current };
+ }
if (expectedMetadata && Object.entries(expectedMetadata).some(([key, value]) => current.metadata?.[key] !== value)) {
return { statusChanged: true, task: current };
}
@@ -853,6 +866,14 @@ async function writeTaskUpdateLocked(taskId, updates, taskType, { now, suppressD
if (!transition[field]) continue;
for (const key of keys) delete updatedMetadata[key];
}
+ // Automatic delivery recovery continues the charged attempt, unlike an
+ // explicit human revive. Persist its budget in the same write as its hold.
+ if (preserveRetryBudget && expectedStatus === 'blocked' && expectedTask && transition.leavesBlocked
+ && updates.status === 'in_progress' && typeof updates.metadata?.retryPendingCleanup === 'string') {
+ for (const key of ['failureCount', 'lastErrorCategory', 'lastFailureAt']) {
+ if (Object.hasOwn(updates.metadata, key)) updatedMetadata[key] = updates.metadata[key];
+ }
+ }
// Drop the resume pointer once the task reaches a terminal state. `existingBranch`
// + `resumeWorktreePath` + `resumedFromAgentId` are stamped by
diff --git a/server/services/cosTaskStore.test.js b/server/services/cosTaskStore.test.js
index 9f9ea47ea3..06ea9daedf 100644
--- a/server/services/cosTaskStore.test.js
+++ b/server/services/cosTaskStore.test.js
@@ -1891,6 +1891,47 @@ describe('cosTaskStore.updateTask', () => {
}
});
+ it('keeps the charged recovery budget durable if completion stops immediately after the recovery write', async () => {
+ const { recoverLegacyDeliveryBlocks } = await import('./deliveryRecovery.js');
+ const outcome = { status: 'incomplete', reason: 'Remaining delivery validation.' };
+ await addTask({ description: 'recover charged delivery', id: 'task-delivery-budget' }, 'user');
+ await updateTask('task-delivery-budget', { status: 'blocked', metadata: {
+ blockedCategory: 'task-incomplete', blockedAt: '2026-10-23T00:00:00.000Z', failureCount: 1, totalSpawnCount: 2, taskOutcome: outcome,
+ } }, 'user');
+ const task = await getTaskById('task-delivery-budget');
+ const owner = { id: 'agent-delivery-budget', taskId: task.id, status: 'completed', result: { taskOutcome: outcome }, metadata: {} };
+ mock.afterWrite = () => { throw new Error('interrupted after committed recovery transition'); };
+ const release = vi.fn();
+ await expect(recoverLegacyDeliveryBlocks([owner], { readTasks: async () => [task], update: updateTask, release })).rejects.toThrow('interrupted after committed recovery transition');
+ const saved = (await stored('user')).find(row => row.id === task.id);
+ expect(saved.status).toBe('in_progress');
+ expect(Number(saved.metadata.failureCount)).toBe(2);
+ expect(Number(saved.metadata.totalSpawnCount)).toBe(2);
+ expect(saved.metadata.retryPendingCleanup).toBe(owner.id);
+ expect(saved.metadata.legacyDeliveryRecovery).toBe(owner.id);
+ expect(release).not.toHaveBeenCalled();
+ });
+
+ it.each(['manual-blocker', 'new-owner'])('preserves a %s arriving after delivery recovery reads its task', async race => {
+ const { recoverLegacyDeliveryBlocks } = await import('./deliveryRecovery.js');
+ const outcome = { status: 'incomplete', reason: 'Remaining delivery validation.' };
+ await addTask({ description: 'recover ownership', id: 'task-delivery-owner' }, 'user');
+ await updateTask('task-delivery-owner', { status: 'blocked', metadata: {
+ blockedCategory: 'task-incomplete', blockedAt: '2026-10-23T00:00:00.000Z', failureCount: 1, taskOutcome: outcome,
+ } }, 'user');
+ const task = await getTaskById('task-delivery-owner');
+ const owner = { id: 'agent-delivery-owner', taskId: task.id, status: 'completed', result: { taskOutcome: outcome }, metadata: {} };
+ if (race === 'manual-blocker') await updateTask(task.id, { metadata: { blocker: 'Operator hold' } }, 'user');
+ else mock.state.agents.newOwner = { id: 'newOwner', taskId: task.id, status: 'running' };
+ const release = vi.fn();
+ expect(await recoverLegacyDeliveryBlocks([owner], { readTasks: async () => [task], update: updateTask, release })).toEqual({ recovered: 0 });
+ const saved = await getTaskById(task.id);
+ expect(saved.status).toBe('blocked');
+ if (race === 'manual-blocker') expect(saved.metadata.blocker).toBe('Operator hold');
+ expect(saved.metadata.legacyDeliveryRecovery).toBeUndefined();
+ expect(release).not.toHaveBeenCalled();
+ });
+
// A requeue releases the in-flight-only markers in the SAME write that makes
// the task spawnable again: a retained lease blocks this instance's own retry
// (and its peer's) for a full lease window, and a retained hold makes the
diff --git a/server/services/deliveryRecovery.js b/server/services/deliveryRecovery.js
index 4bb63b7b15..4f5a44ace9 100644
--- a/server/services/deliveryRecovery.js
+++ b/server/services/deliveryRecovery.js
@@ -25,16 +25,11 @@ export async function recoverLegacyDeliveryBlocks(agents, deps = {}) {
|| owner.result?.taskOutcome?.status !== outcome.status || owner.metadata?.executionInterruption || owner.result?.executionInterruption || isPrivateSecurityTask(owner)) continue;
const planned = resolveUndeliveredTaskUpdate(task, outcome, owner.id);
const result = await update(task.id, { ...planned, metadata: { ...planned.metadata, legacyDeliveryRecovery: owner.id } }, task.taskType || 'user', {
- expectedStatus: 'blocked', expectedMetadata: { blockedCategory: task.metadata.blockedCategory, blockedAt: task.metadata.blockedAt },
+ expectedStatus: 'blocked', expectedTask: { description: task.description, priority: task.priority, metadata: task.metadata },
+ recoveryOwner: { id: owner.id, startedAt: owner.startedAt }, preserveRetryBudget: true,
});
if (result?.error || result?.statusChanged || result?.status !== planned.status) continue;
if (planned.metadata.retryPendingCleanup) {
- // Leaving blocked clears legacy failure fields in the task store. Restore
- // the charged budget under the hold before making the retry spawnable.
- const charged = await update(task.id, { metadata: planned.metadata }, task.taskType || 'user', {
- expectedStatus: 'in_progress', expectedMetadata: { retryPendingCleanup: owner.id },
- });
- if (charged?.error || charged?.statusChanged || charged?.status !== 'in_progress') continue;
await release({ agentId: owner.id, task, success: false, agentMetadata: owner.metadata });
}
recovered++;
diff --git a/server/services/deliveryRecovery.test.js b/server/services/deliveryRecovery.test.js
index 6a519dcd67..c8a93cb6c5 100644
--- a/server/services/deliveryRecovery.test.js
+++ b/server/services/deliveryRecovery.test.js
@@ -13,8 +13,9 @@ describe('legacy delivery recovery', () => {
it('charges the existing retry budget under an owner fence before releasing the preserved checkout', async () => {
const deps = setup();
expect(await recoverLegacyDeliveryBlocks([owner], deps)).toEqual({ recovered: 1 });
- expect(deps.update).toHaveBeenCalledTimes(2);
- expect(deps.update.mock.calls[1][1].metadata).toMatchObject({ failureCount: 2, totalSpawnCount: 2, retryPendingCleanup: owner.id });
+ expect(deps.update).toHaveBeenCalledTimes(1);
+ expect(deps.update.mock.calls[0][1].metadata).toMatchObject({ failureCount: 2, totalSpawnCount: 2, retryPendingCleanup: owner.id });
+ expect(deps.update.mock.calls[0][3]).toMatchObject({ expectedTask: { metadata: task.metadata }, preserveRetryBudget: true });
expect(deps.release).toHaveBeenCalledWith({ agentId: owner.id, task, success: false, agentMetadata: owner.metadata });
});