Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: bind runtime holds to live dispatch and accepted task evidence (#13484)

Adam Eivyf481e825b72d
diff --git a/server/services/agentCompletionTaskHandoff.js b/server/services/agentCompletionTaskHandoff.js
index d68445c5d3..351e58a404 100644
--- a/server/services/agentCompletionTaskHandoff.js
+++ b/server/services/agentCompletionTaskHandoff.js
@@ -13,6 +13,13 @@ const matches = (metadata, expected) => expected.every(([key, value]) => key ===
? spawnCount(value) !== null && spawnCount(metadata?.[key]) === spawnCount(value)
: (metadata?.[key] ?? null) === value);
+/** Match the server's dispatch binding, including metadata scalar round-trips. */
+export function matchesCompletionTaskDispatch(agent, task) {
+ const binding = agent?.metadata?.completionTaskDispatch;
+ return !!binding && binding.taskId === task?.id && binding.taskId === agent.taskId
+ && binding.taskType === (task.taskType || 'user') && matches(task.metadata, binding.expected);
+}
+
/** Called by the trusted dispatcher after its task write, before provider launch. */
export async function bindCompletionTaskDispatch(agentId, task) {
if (task?.status !== 'in_progress' || !task.metadata?.lastSpawnedAt || !task.metadata?.claimedBy
diff --git a/server/services/agentFinalization.js b/server/services/agentFinalization.js
index 8c84dba2b9..8b07659aa7 100644
--- a/server/services/agentFinalization.js
+++ b/server/services/agentFinalization.js
@@ -1304,7 +1304,7 @@ export async function retireDeadAgent({ agent, task, success, exitCode, duration
const recoveryUpdate = deliveryHeld && !refused && !isPrivateSecurity && task?.id && task.status !== 'completed'
? await resolveDeliveryRecoveryUpdate(task, recoveryOutcome, agent.id) : null;
if (recoveryUpdate) {
- const update = await updateTask(task.id, recoveryUpdate, task.taskType || 'user', ...runtimeRecoveryFence(task, recoveryUpdate));
+ const update = await updateTask(task.id, recoveryUpdate, task.taskType || 'user', ...runtimeRecoveryFence(recoveryUpdate));
if (update?.error || update?.statusChanged) throw new Error(`Could not persist recovered task outcome: ${update.error || 'task ownership changed'}`);
}
await completeAgent(agent.id, {
@@ -2032,7 +2032,7 @@ export async function finalizeAgent({
// Persist the prerequisite BEFORE retiring execution. A crash between the
// agent and task writes must never expose this dispatch to generic retry.
const runtimeTaskResult = taskUpdate.metadata?.blockedCategory === 'runtime-verification'
- ? await updateTask(task.id, taskUpdate, taskType, ...runtimeRecoveryFence(preHookTask, taskUpdate)) : null;
+ ? await updateTask(task.id, taskUpdate, taskType, ...runtimeRecoveryFence(taskUpdate)) : null;
if (runtimeTaskResult?.error || runtimeTaskResult?.statusChanged) throw new Error('Runtime verification task ownership changed before completion');
const completedAgent = await completeAgent(agentId, { ...completionRecord({
task, verdict: deliveryVerdict, hookResult, fidelity, executionInterruption, exitCode, duration, outputBuffer, validationPassed,
@@ -2071,7 +2071,7 @@ export async function finalizeAgent({
}
const taskResult = runtimeTaskResult || (capturePending
? await recoverCompletionTaskHandoff(await getAgentRecord(agentId)) || { error: 'Completion cleanup task ownership changed' }
- : await updateTask(task.id, taskUpdate, taskType, ...runtimeRecoveryFence(preHookTask, taskUpdate)));
+ : await updateTask(task.id, taskUpdate, taskType, ...runtimeRecoveryFence(taskUpdate)));
if (taskResult?.error && !isExpectedUnpersistedMiss(task, taskResult.error)) {
const label = terminatedByUser ? 'blocked' : verdict.success ? 'completed' : 'failed';
emitLog('warn', `⚠️ Failed to update ${label} task ${task.id}: ${taskResult.error} (taskType=${taskType})`, { taskId: task.id, agentId, error: taskResult.error });
@@ -2442,11 +2442,11 @@ async function resolveDeliveryRecoveryUpdate(task, outcome, agentId) {
return resolveUndeliveredTaskUpdate(task, outcome, agentId);
}
-function runtimeRecoveryFence(task, update) {
- return update.metadata?.blockedCategory === 'runtime-verification' ? [{
- expectedStatus: task.status === 'blocked' ? 'blocked' : 'in_progress', expectedMetadata: {
- ...(task.status === 'blocked' ? { runtimeVerificationId: task.metadata?.runtimeVerificationId } : {}),
- lastSpawnedAt: task.metadata?.lastSpawnedAt,
- totalSpawnCount: task.metadata?.totalSpawnCount, claimedBy: task.metadata?.claimedBy },
- }] : [];
+function runtimeRecoveryFence(update) {
+ if (update.metadata?.blockedCategory !== 'runtime-verification') return [];
+ const p = update.metadata.runtimeVerification;
+ return [{ expectedStatus: p.dispatchStatus, expectedMetadata: {
+ ...(p.dispatchStatus === 'blocked' ? { runtimeVerificationId: update.metadata.runtimeVerificationId } : {}),
+ ...Object.fromEntries(['claimedBy', 'claimedAt', 'lastSpawnedAt', 'totalSpawnCount'].map(key => [key, p.dispatch?.[key]])),
+ } }];
}
diff --git a/server/services/agentFinalization.outputHooks.test.js b/server/services/agentFinalization.outputHooks.test.js
index 949b794544..9639c36ed7 100644
--- a/server/services/agentFinalization.outputHooks.test.js
+++ b/server/services/agentFinalization.outputHooks.test.js
@@ -68,6 +68,7 @@ import {
vi.mock('./cosTaskStore.js', () => ({
updateTask: vi.fn(async () => ({})),
+ getTaskById: vi.fn(async () => null),
addTask: vi.fn(async () => ({ id: 'investigation-test' })),
}));
vi.mock('./investigationTaskProducer.js', () => ({
@@ -89,7 +90,7 @@ vi.mock('./taskSchedule.js', () => ({
recordTaskTypeSuccess: vi.fn(async () => null),
recordRecommendedAudits: vi.fn(async () => null),
}));
-import { updateTask, addTask } from './cosTaskStore.js';
+import { updateTask, addTask, getTaskById } from './cosTaskStore.js';
import { completeAgentRun } from './agentRunTracking.js';
import { processAgentCompletion } from './agentCompletion.js';
import { MAX_TASK_RETRIES, resolveFailedTaskUpdate } from './agentErrorAnalysis.js';
@@ -481,6 +482,9 @@ describe('recovery output-hook dispatch (#3182)', () => {
isInvestigation: true, totalSpawnCount: 3, failureCount: 1, lastSpawnedAt: '2026-09-10T00:00:00Z',
} };
persistedAgent.taskId = task.id;
+ persistedAgent.metadata.completionTaskDispatch = { taskId: task.id, taskType: 'internal',
+ expected: ['claimedBy', 'claimedAt', 'lastSpawnedAt', 'totalSpawnCount'].map(key => [key, task.metadata[key] ?? null]) };
+ getTaskById.mockResolvedValueOnce(task);
mergedTarget.getOriginInfo.mockResolvedValue({ isGithub: true, host: 'github.com', fullName: 'example/PortOS' });
const github = await import('./github.js');
const git = await import('../lib/execGit.js');
@@ -496,10 +500,12 @@ describe('recovery output-hook dispatch (#3182)', () => {
const order = [];
updateTask.mockImplementationOnce(async () => { order.push('prerequisite'); return {}; });
completeAgent.mockImplementationOnce(async () => { order.push('terminal'); return null; });
+ // The dispatcher increments counters after the agent receives this snapshot.
+ const staleTask = { ...task, status: 'pending', metadata: { ...task.metadata, totalSpawnCount: 2, lastSpawnedAt: undefined } };
const result = mode === 'normal'
- ? await finalizeAgent({ agentId: persistedAgent.id, task, success: true, exitCode: 0,
+ ? await finalizeAgent({ agentId: persistedAgent.id, task: staleTask, success: true, exitCode: 0,
finalSummary: JSON.stringify({ summary: taskOutcome.reason, taskOutcome }) })
- : await retireDeadAgent({ agent: persistedAgent, task, success: true, exitCode: 0 });
+ : await retireDeadAgent({ agent: persistedAgent, task: staleTask, success: true, exitCode: 0 });
expect(result).toMatchObject({ success: false, taskOutcome });
expect(order).toEqual(['prerequisite', 'terminal']);
expect(updateTask).toHaveBeenCalledWith(task.id, expect.objectContaining({ status: 'blocked', metadata: expect.objectContaining({
@@ -514,12 +520,16 @@ describe('recovery output-hook dispatch (#3182)', () => {
it('stops recovered runtime finalization when its conditional prerequisite write loses ownership', async () => {
isProgrammaticIoTaskType.mockReturnValue(false);
- persistedAgent.taskId = TASK.id;
+ const task = { id: TASK.id, taskType: 'internal', status: 'in_progress', metadata: { totalSpawnCount: 1 } };
+ persistedAgent.taskId = task.id;
+ persistedAgent.metadata.completionTaskDispatch = { taskId: task.id, taskType: 'internal',
+ expected: ['claimedBy', 'claimedAt', 'lastSpawnedAt', 'totalSpawnCount'].map(key => [key, task.metadata[key] ?? null]) };
+ getTaskById.mockResolvedValueOnce(task);
persistedAgent.metadata.taskOutcome = { status: 'blocked', reason: 'The running fix remains unverified.',
recovery: { mode: 'runtime', prUrl: 'https://github.com/example/PortOS/pull/123', requiredCommit: 'a'.repeat(40) } };
persistedAgent.metadata.taskOutcomeSource = 'terminal-result';
updateTask.mockResolvedValueOnce({ statusChanged: true });
- await expect(retireDeadAgent({ agent: persistedAgent, task: { id: TASK.id, status: 'in_progress', metadata: {} },
+ await expect(retireDeadAgent({ agent: persistedAgent, task,
success: true, exitCode: 0 })).rejects.toThrow('task ownership changed');
expect(completeAgent).not.toHaveBeenCalled();
expect(addTask).not.toHaveBeenCalled();
diff --git a/server/services/agentWorktreeCleanup.js b/server/services/agentWorktreeCleanup.js
index 1328e5b368..4adf295679 100644
--- a/server/services/agentWorktreeCleanup.js
+++ b/server/services/agentWorktreeCleanup.js
@@ -209,7 +209,8 @@ async function worktreeCleanupContext(agentId, options) {
discardWorktree: isTruthyMeta(normalized.originalTask?.metadata?.discardWorktree),
worktreePath: metadata.workspacePath || join(PATHS.worktrees, agentId),
publicationValidation: publicationHolds.get(agentId) || metadata.publicationValidation,
- runtimeVerificationHeld: agentState?.result?.taskOutcome?.recovery?.mode === 'runtime',
+ runtimeVerificationRequested: agentState?.result?.taskOutcome?.recovery?.mode === 'runtime',
+ taskId: agentState?.taskId || normalized.originalTask?.id,
securityScan: metadata.securityScan,
securityScanControl: metadata.securityScanControl,
warnings: [],
@@ -239,7 +240,12 @@ async function probeWorktreePr(context, success) {
async function runCleanupAgentWorktree(agentId, success, options = {}) {
const context = await worktreeCleanupContext(agentId, options);
- if (context.runtimeVerificationHeld) return []; // The same checkout resumes after verified boot ancestry.
+ if (context.runtimeVerificationRequested) {
+ const { getTaskById } = await import('./cosTaskStore.js');
+ const held = context.taskId ? await getTaskById(context.taskId) : null;
+ if (held?.metadata?.runtimeVerificationId === `${held.id}:${agentId}`
+ && held.metadata.runtimeVerification?.agentId === agentId) return [];
+ }
if (['running', 'blocked'].includes(context.publicationValidation?.status)) {
return [`Publication validation ${context.publicationValidation.status} for ${context.worktreeBranch}; worktree preserved at ${context.worktreePath}. Resume the task after resolving the validation outcome.`];
}
diff --git a/server/services/cleanupAgentWorktree.test.js b/server/services/cleanupAgentWorktree.test.js
index 35d03a0512..40d3a1316d 100644
--- a/server/services/cleanupAgentWorktree.test.js
+++ b/server/services/cleanupAgentWorktree.test.js
@@ -283,11 +283,15 @@ function mockWorktreeAgent(overrides = {}) {
}
describe('cleanupAgentWorktree - PR-creation path', () => {
- it('preserves the runtime continuation checkout even with an old discard policy', async () => {
+ it.each([true, false])('preserves only an accepted runtime prerequisite when accepted=%s', async accepted => {
getAgent.mockResolvedValue({ ...mockWorktreeAgent(), result: { success: false,
taskOutcome: { status: 'blocked', recovery: { mode: 'runtime' } } } });
- await cleanupAgentWorktree('agent-runtime', false, { originalTask: { metadata: { discardWorktree: true } } });
- expect(removeWorktree).not.toHaveBeenCalled();
+ getTaskById.mockResolvedValue({ id: 'task-runtime', metadata: accepted ? {
+ runtimeVerificationId: 'task-runtime:agent-runtime', runtimeVerification: { agentId: 'agent-runtime' },
+ } : {} });
+ await cleanupAgentWorktree('agent-runtime', false, { originalTask: { id: 'task-runtime', metadata: { discardWorktree: true } } });
+ if (accepted) expect(removeWorktree).not.toHaveBeenCalled();
+ else expect(removeWorktree).toHaveBeenCalled();
expect(git.push).not.toHaveBeenCalled();
});
diff --git a/server/services/runtimeVerification.js b/server/services/runtimeVerification.js
index cdab2ca025..b16539576f 100644
--- a/server/services/runtimeVerification.js
+++ b/server/services/runtimeVerification.js
@@ -10,7 +10,8 @@ import { execGit } from '../lib/execGit.js';
import { getOriginInfo } from '../lib/gitRemote.js';
import { getBootCommit } from './installState.js';
import { getAgentRecord } from './cosAgentLifecycle.js';
-import { getAllTasks, updateTask } from './cosTaskStore.js';
+import { matchesCompletionTaskDispatch } from './agentCompletionTaskHandoff.js';
+import { getAllTasks, getTaskById, updateTask } from './cosTaskStore.js';
const SHA = /^[a-f0-9]{40}$/;
const CATEGORY = 'runtime-verification';
@@ -40,7 +41,7 @@ async function integrationEvidence(request) {
}
// A squash/rebase need not contain the old PR head. The forge's integration
// commit is the runtime prerequisite; never replace it with disk HEAD.
- await execGit(['fetch', 'origin', pr.baseRefName], PATHS.root);
+ await execGit(['fetch', '--', 'origin', pr.baseRefName], PATHS.root);
const target = await execGit(['rev-parse', '--verify', `refs/remotes/origin/${pr.baseRefName}^{commit}`], PATHS.root);
if (!await contains(pr.mergeCommit.oid, target.stdout.trim())) throw new Error('The merged fix is not integrated in the fetched default branch');
return { prUrl: request.prUrl, requiredCommit: request.requiredCommit, integratedCommit: pr.mergeCommit.oid };
@@ -58,6 +59,13 @@ async function prerequisiteReason(prerequisite) {
export async function resolveRuntimeVerificationUpdate(task, outcome, agentId) {
const now = new Date().toISOString();
+ const owner = await getAgentRecord(agentId);
+ const live = await getTaskById(task.id);
+ const replay = live?.status === 'blocked' && live.metadata?.runtimeVerificationId === `${task.id}:${agentId}`
+ && live.metadata?.runtimeVerification?.agentId === agentId;
+ if (!live || owner?.taskId !== task.id || (!replay && (live.status !== 'in_progress'
+ || !matchesCompletionTaskDispatch(owner, live)))) throw new Error('Runtime verification dispatch ownership is unverified');
+ task = live;
const prior = task.metadata?.runtimeVerification;
// Replaying finalization retains the original dispatch identity and evidence.
const same = prior?.agentId === agentId && task.metadata?.runtimeVerificationId === `${task.id}:${agentId}`;
@@ -66,8 +74,7 @@ export async function resolveRuntimeVerificationUpdate(task, outcome, agentId) {
totalSpawnCount: String(task.metadata?.totalSpawnCount ?? ''), requestedAt: now,
prUrl: outcome.recovery.prUrl, requiredCommit: outcome.recovery.requiredCommit };
let reason;
- const owner = await getAgentRecord(agentId);
- if (owner?.taskId !== task.id || (task.metadata?.app && task.metadata.app !== PORTOS_APP_ID)) {
+ if (task.metadata?.app && task.metadata.app !== PORTOS_APP_ID) {
reason = 'Runtime verification owner or PortOS task scope could not be verified';
} else {
const evidence = await integrationEvidence(prerequisite).catch(() => null);
@@ -77,7 +84,9 @@ export async function resolveRuntimeVerificationUpdate(task, outcome, agentId) {
} else reason = 'Runtime prerequisite retained: merged-fix integration evidence is unreadable or invalid';
}
return { status: 'blocked', metadata: { ...task.metadata,
- runtimeVerificationId: `${task.id}:${agentId}`, runtimeVerification: prerequisite,
+ runtimeVerificationId: `${task.id}:${agentId}`, runtimeVerification: { ...prerequisite, dispatchStatus: task.status,
+ dispatch: Object.fromEntries(['claimedBy', 'claimedAt', 'lastSpawnedAt', 'totalSpawnCount']
+ .filter(key => task.metadata?.[key] !== undefined).map(key => [key, task.metadata[key]])) },
deliveryDecisionAgentId: agentId, blockedCategory: CATEGORY, blockedAt: same ? task.metadata.blockedAt : now,
blockedReason: reason, taskOutcome: outcome,
...clearedRetryHoldMetadata(),
diff --git a/server/services/runtimeVerification.test.js b/server/services/runtimeVerification.test.js
index 033ccca9dd..ea3293d2fc 100644
--- a/server/services/runtimeVerification.test.js
+++ b/server/services/runtimeVerification.test.js
@@ -41,12 +41,18 @@ import { execGh } from './github.js';
const outcome = () => normalizeTaskOutcome({ status: 'blocked', reason: 'The fix must be running before verification.',
recovery: { mode: 'runtime', prUrl: 'https://github.com/example/PortOS/pull/123', requiredCommit: 'a'.repeat(40) } });
-async function park({ ready = true } = {}) {
+async function runningTask() {
const task = await addTask({ id: 'sys-example', description: 'Verify merged fix', priority: 'MEDIUM', status: 'in_progress', metadata: {
isInvestigation: true, lastSpawnedAt: '2026-01-01T00:00:00Z', totalSpawnCount: 4, failureCount: 2,
orphanRetryCount: 1, existingBranch: 'claim/issue-123', resumeWorktreePath: '/example/worktree',
} }, 'internal', { raw: true });
- await updateTask(task.id, await resolveRuntimeVerificationUpdate(task, outcome(), 'agent-example'), 'internal');
+ facts.owner.metadata.completionTaskDispatch = { taskId: task.id, taskType: 'internal',
+ expected: ['claimedBy', 'claimedAt', 'lastSpawnedAt', 'totalSpawnCount'].map(key => [key, task.metadata[key] ?? null]) };
+ return task;
+}
+async function park({ ready = true } = {}) {
+ const task = await runningTask();
+ await updateTask(task.id, await resolveRuntimeVerificationUpdate({ ...task, metadata: { totalSpawnCount: 0 } }, outcome(), 'agent-example'), 'internal');
if (ready) await updateTask(task.id, { metadata: { runtimeVerificationReadyAgentId: 'agent-example' } }, 'internal');
return getTaskById(task.id);
}
@@ -107,11 +113,19 @@ describe('persisted runtime continuation', () => {
if (kind === 'foreign-repository') request.recovery.prUrl = 'https://github.com/another/project/pull/123';
if (kind === 'wrong-commit') request.recovery.requiredCommit = 'f'.repeat(40);
if (kind === 'unmerged') execGh.mockResolvedValueOnce(JSON.stringify({ state: 'OPEN', mergeCommit: null }));
- const update = await resolveRuntimeVerificationUpdate({ id: 'sys-example', metadata: {} }, request, 'agent-example');
+ const task = await runningTask();
+ const update = await resolveRuntimeVerificationUpdate(task, request, 'agent-example');
expect(update).toMatchObject({ status: 'blocked', metadata: { blockedReason: expect.stringContaining('unreadable or invalid') } });
expect(update.metadata.runtimeVerification.integratedCommit).toBeUndefined();
});
+ it('refuses a predecessor dispatch when a successor has already claimed the task', async () => {
+ const task = await runningTask();
+ await updateTask(task.id, { metadata: { totalSpawnCount: 5, lastSpawnedAt: '2026-01-02T00:00:00Z' } }, 'internal');
+ await expect(resolveRuntimeVerificationUpdate(task, outcome(), 'agent-example')).rejects.toThrow('dispatch ownership');
+ expect((await getTaskById(task.id)).status).toBe('in_progress');
+ });
+
it('retains unreadable integration as a prerequisite without spending another run, then verifies it afresh', async () => {
facts.forge = false; await park(); facts.ancestor = true;
expect((await getTaskById('sys-example')).metadata.blockedReason).toContain('unreadable or invalid');