Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: bind runtime holds to live dispatch and accepted task evidence (#13484)
Adam Eivyf481e825b72d
diff --git a/server/services/agentCompletionTaskHandoff.js b/server/services/agentCompletionTaskHandoff.jsindex d68445c5d3..351e58a404 100644--- a/server/services/agentCompletionTaskHandoff.js+++ b/server/services/agentCompletionTaskHandoff.js@@ -13,6 +13,13 @@ const matches = (metadata, expected) => expected.every(([key, value]) => key ===? spawnCount(value) !== null && spawnCount(metadata?.[key]) === spawnCount(value): (metadata?.[key] ?? null) === value);+/** Match the server's dispatch binding, including metadata scalar round-trips. */+export function matchesCompletionTaskDispatch(agent, task) {+ const binding = agent?.metadata?.completionTaskDispatch;+ return !!binding && binding.taskId === task?.id && binding.taskId === agent.taskId+ && binding.taskType === (task.taskType || 'user') && matches(task.metadata, binding.expected);+}+/** Called by the trusted dispatcher after its task write, before provider launch. */export async function bindCompletionTaskDispatch(agentId, task) {if (task?.status !== 'in_progress' || !task.metadata?.lastSpawnedAt || !task.metadata?.claimedBydiff --git a/server/services/agentFinalization.js b/server/services/agentFinalization.jsindex 8c84dba2b9..8b07659aa7 100644--- a/server/services/agentFinalization.js+++ b/server/services/agentFinalization.js@@ -1304,7 +1304,7 @@ export async function retireDeadAgent({ agent, task, success, exitCode, durationconst recoveryUpdate = deliveryHeld && !refused && !isPrivateSecurity && task?.id && task.status !== 'completed'? await resolveDeliveryRecoveryUpdate(task, recoveryOutcome, agent.id) : null;if (recoveryUpdate) {- const update = await updateTask(task.id, recoveryUpdate, task.taskType || 'user', ...runtimeRecoveryFence(task, recoveryUpdate));+ const update = await updateTask(task.id, recoveryUpdate, task.taskType || 'user', ...runtimeRecoveryFence(recoveryUpdate));if (update?.error || update?.statusChanged) throw new Error(`Could not persist recovered task outcome: ${update.error || 'task ownership changed'}`);}await completeAgent(agent.id, {@@ -2032,7 +2032,7 @@ export async function finalizeAgent({// Persist the prerequisite BEFORE retiring execution. A crash between the// agent and task writes must never expose this dispatch to generic retry.const runtimeTaskResult = taskUpdate.metadata?.blockedCategory === 'runtime-verification'- ? await updateTask(task.id, taskUpdate, taskType, ...runtimeRecoveryFence(preHookTask, taskUpdate)) : null;+ ? await updateTask(task.id, taskUpdate, taskType, ...runtimeRecoveryFence(taskUpdate)) : null;if (runtimeTaskResult?.error || runtimeTaskResult?.statusChanged) throw new Error('Runtime verification task ownership changed before completion');const completedAgent = await completeAgent(agentId, { ...completionRecord({task, verdict: deliveryVerdict, hookResult, fidelity, executionInterruption, exitCode, duration, outputBuffer, validationPassed,@@ -2071,7 +2071,7 @@ export async function finalizeAgent({}const taskResult = runtimeTaskResult || (capturePending? await recoverCompletionTaskHandoff(await getAgentRecord(agentId)) || { error: 'Completion cleanup task ownership changed' }- : await updateTask(task.id, taskUpdate, taskType, ...runtimeRecoveryFence(preHookTask, taskUpdate)));+ : await updateTask(task.id, taskUpdate, taskType, ...runtimeRecoveryFence(taskUpdate)));if (taskResult?.error && !isExpectedUnpersistedMiss(task, taskResult.error)) {const label = terminatedByUser ? 'blocked' : verdict.success ? 'completed' : 'failed';emitLog('warn', `⚠️ Failed to update ${label} task ${task.id}: ${taskResult.error} (taskType=${taskType})`, { taskId: task.id, agentId, error: taskResult.error });@@ -2442,11 +2442,11 @@ async function resolveDeliveryRecoveryUpdate(task, outcome, agentId) {return resolveUndeliveredTaskUpdate(task, outcome, agentId);}-function runtimeRecoveryFence(task, update) {- return update.metadata?.blockedCategory === 'runtime-verification' ? [{- expectedStatus: task.status === 'blocked' ? 'blocked' : 'in_progress', expectedMetadata: {- ...(task.status === 'blocked' ? { runtimeVerificationId: task.metadata?.runtimeVerificationId } : {}),- lastSpawnedAt: task.metadata?.lastSpawnedAt,- totalSpawnCount: task.metadata?.totalSpawnCount, claimedBy: task.metadata?.claimedBy },- }] : [];+function runtimeRecoveryFence(update) {+ if (update.metadata?.blockedCategory !== 'runtime-verification') return [];+ const p = update.metadata.runtimeVerification;+ return [{ expectedStatus: p.dispatchStatus, expectedMetadata: {+ ...(p.dispatchStatus === 'blocked' ? { runtimeVerificationId: update.metadata.runtimeVerificationId } : {}),+ ...Object.fromEntries(['claimedBy', 'claimedAt', 'lastSpawnedAt', 'totalSpawnCount'].map(key => [key, p.dispatch?.[key]])),+ } }];}diff --git a/server/services/agentFinalization.outputHooks.test.js b/server/services/agentFinalization.outputHooks.test.jsindex 949b794544..9639c36ed7 100644--- a/server/services/agentFinalization.outputHooks.test.js+++ b/server/services/agentFinalization.outputHooks.test.js@@ -68,6 +68,7 @@ import {vi.mock('./cosTaskStore.js', () => ({updateTask: vi.fn(async () => ({})),+ getTaskById: vi.fn(async () => null),addTask: vi.fn(async () => ({ id: 'investigation-test' })),}));vi.mock('./investigationTaskProducer.js', () => ({@@ -89,7 +90,7 @@ vi.mock('./taskSchedule.js', () => ({recordTaskTypeSuccess: vi.fn(async () => null),recordRecommendedAudits: vi.fn(async () => null),}));-import { updateTask, addTask } from './cosTaskStore.js';+import { updateTask, addTask, getTaskById } from './cosTaskStore.js';import { completeAgentRun } from './agentRunTracking.js';import { processAgentCompletion } from './agentCompletion.js';import { MAX_TASK_RETRIES, resolveFailedTaskUpdate } from './agentErrorAnalysis.js';@@ -481,6 +482,9 @@ describe('recovery output-hook dispatch (#3182)', () => {isInvestigation: true, totalSpawnCount: 3, failureCount: 1, lastSpawnedAt: '2026-09-10T00:00:00Z',} };persistedAgent.taskId = task.id;+ persistedAgent.metadata.completionTaskDispatch = { taskId: task.id, taskType: 'internal',+ expected: ['claimedBy', 'claimedAt', 'lastSpawnedAt', 'totalSpawnCount'].map(key => [key, task.metadata[key] ?? null]) };+ getTaskById.mockResolvedValueOnce(task);mergedTarget.getOriginInfo.mockResolvedValue({ isGithub: true, host: 'github.com', fullName: 'example/PortOS' });const github = await import('./github.js');const git = await import('../lib/execGit.js');@@ -496,10 +500,12 @@ describe('recovery output-hook dispatch (#3182)', () => {const order = [];updateTask.mockImplementationOnce(async () => { order.push('prerequisite'); return {}; });completeAgent.mockImplementationOnce(async () => { order.push('terminal'); return null; });+ // The dispatcher increments counters after the agent receives this snapshot.+ const staleTask = { ...task, status: 'pending', metadata: { ...task.metadata, totalSpawnCount: 2, lastSpawnedAt: undefined } };const result = mode === 'normal'- ? await finalizeAgent({ agentId: persistedAgent.id, task, success: true, exitCode: 0,+ ? await finalizeAgent({ agentId: persistedAgent.id, task: staleTask, success: true, exitCode: 0,finalSummary: JSON.stringify({ summary: taskOutcome.reason, taskOutcome }) })- : await retireDeadAgent({ agent: persistedAgent, task, success: true, exitCode: 0 });+ : await retireDeadAgent({ agent: persistedAgent, task: staleTask, success: true, exitCode: 0 });expect(result).toMatchObject({ success: false, taskOutcome });expect(order).toEqual(['prerequisite', 'terminal']);expect(updateTask).toHaveBeenCalledWith(task.id, expect.objectContaining({ status: 'blocked', metadata: expect.objectContaining({@@ -514,12 +520,16 @@ describe('recovery output-hook dispatch (#3182)', () => {it('stops recovered runtime finalization when its conditional prerequisite write loses ownership', async () => {isProgrammaticIoTaskType.mockReturnValue(false);- persistedAgent.taskId = TASK.id;+ const task = { id: TASK.id, taskType: 'internal', status: 'in_progress', metadata: { totalSpawnCount: 1 } };+ persistedAgent.taskId = task.id;+ persistedAgent.metadata.completionTaskDispatch = { taskId: task.id, taskType: 'internal',+ expected: ['claimedBy', 'claimedAt', 'lastSpawnedAt', 'totalSpawnCount'].map(key => [key, task.metadata[key] ?? null]) };+ getTaskById.mockResolvedValueOnce(task);persistedAgent.metadata.taskOutcome = { status: 'blocked', reason: 'The running fix remains unverified.',recovery: { mode: 'runtime', prUrl: 'https://github.com/example/PortOS/pull/123', requiredCommit: 'a'.repeat(40) } };persistedAgent.metadata.taskOutcomeSource = 'terminal-result';updateTask.mockResolvedValueOnce({ statusChanged: true });- await expect(retireDeadAgent({ agent: persistedAgent, task: { id: TASK.id, status: 'in_progress', metadata: {} },+ await expect(retireDeadAgent({ agent: persistedAgent, task,success: true, exitCode: 0 })).rejects.toThrow('task ownership changed');expect(completeAgent).not.toHaveBeenCalled();expect(addTask).not.toHaveBeenCalled();diff --git a/server/services/agentWorktreeCleanup.js b/server/services/agentWorktreeCleanup.jsindex 1328e5b368..4adf295679 100644--- a/server/services/agentWorktreeCleanup.js+++ b/server/services/agentWorktreeCleanup.js@@ -209,7 +209,8 @@ async function worktreeCleanupContext(agentId, options) {discardWorktree: isTruthyMeta(normalized.originalTask?.metadata?.discardWorktree),worktreePath: metadata.workspacePath || join(PATHS.worktrees, agentId),publicationValidation: publicationHolds.get(agentId) || metadata.publicationValidation,- runtimeVerificationHeld: agentState?.result?.taskOutcome?.recovery?.mode === 'runtime',+ runtimeVerificationRequested: agentState?.result?.taskOutcome?.recovery?.mode === 'runtime',+ taskId: agentState?.taskId || normalized.originalTask?.id,securityScan: metadata.securityScan,securityScanControl: metadata.securityScanControl,warnings: [],@@ -239,7 +240,12 @@ async function probeWorktreePr(context, success) {async function runCleanupAgentWorktree(agentId, success, options = {}) {const context = await worktreeCleanupContext(agentId, options);- if (context.runtimeVerificationHeld) return []; // The same checkout resumes after verified boot ancestry.+ if (context.runtimeVerificationRequested) {+ const { getTaskById } = await import('./cosTaskStore.js');+ const held = context.taskId ? await getTaskById(context.taskId) : null;+ if (held?.metadata?.runtimeVerificationId === `${held.id}:${agentId}`+ && held.metadata.runtimeVerification?.agentId === agentId) return [];+ }if (['running', 'blocked'].includes(context.publicationValidation?.status)) {return [`Publication validation ${context.publicationValidation.status} for ${context.worktreeBranch}; worktree preserved at ${context.worktreePath}. Resume the task after resolving the validation outcome.`];}diff --git a/server/services/cleanupAgentWorktree.test.js b/server/services/cleanupAgentWorktree.test.jsindex 35d03a0512..40d3a1316d 100644--- a/server/services/cleanupAgentWorktree.test.js+++ b/server/services/cleanupAgentWorktree.test.js@@ -283,11 +283,15 @@ function mockWorktreeAgent(overrides = {}) {}describe('cleanupAgentWorktree - PR-creation path', () => {- it('preserves the runtime continuation checkout even with an old discard policy', async () => {+ it.each([true, false])('preserves only an accepted runtime prerequisite when accepted=%s', async accepted => {getAgent.mockResolvedValue({ ...mockWorktreeAgent(), result: { success: false,taskOutcome: { status: 'blocked', recovery: { mode: 'runtime' } } } });- await cleanupAgentWorktree('agent-runtime', false, { originalTask: { metadata: { discardWorktree: true } } });- expect(removeWorktree).not.toHaveBeenCalled();+ getTaskById.mockResolvedValue({ id: 'task-runtime', metadata: accepted ? {+ runtimeVerificationId: 'task-runtime:agent-runtime', runtimeVerification: { agentId: 'agent-runtime' },+ } : {} });+ await cleanupAgentWorktree('agent-runtime', false, { originalTask: { id: 'task-runtime', metadata: { discardWorktree: true } } });+ if (accepted) expect(removeWorktree).not.toHaveBeenCalled();+ else expect(removeWorktree).toHaveBeenCalled();expect(git.push).not.toHaveBeenCalled();});diff --git a/server/services/runtimeVerification.js b/server/services/runtimeVerification.jsindex cdab2ca025..b16539576f 100644--- a/server/services/runtimeVerification.js+++ b/server/services/runtimeVerification.js@@ -10,7 +10,8 @@ import { execGit } from '../lib/execGit.js';import { getOriginInfo } from '../lib/gitRemote.js';import { getBootCommit } from './installState.js';import { getAgentRecord } from './cosAgentLifecycle.js';-import { getAllTasks, updateTask } from './cosTaskStore.js';+import { matchesCompletionTaskDispatch } from './agentCompletionTaskHandoff.js';+import { getAllTasks, getTaskById, updateTask } from './cosTaskStore.js';const SHA = /^[a-f0-9]{40}$/;const CATEGORY = 'runtime-verification';@@ -40,7 +41,7 @@ async function integrationEvidence(request) {}// A squash/rebase need not contain the old PR head. The forge's integration// commit is the runtime prerequisite; never replace it with disk HEAD.- await execGit(['fetch', 'origin', pr.baseRefName], PATHS.root);+ await execGit(['fetch', '--', 'origin', pr.baseRefName], PATHS.root);const target = await execGit(['rev-parse', '--verify', `refs/remotes/origin/${pr.baseRefName}^{commit}`], PATHS.root);if (!await contains(pr.mergeCommit.oid, target.stdout.trim())) throw new Error('The merged fix is not integrated in the fetched default branch');return { prUrl: request.prUrl, requiredCommit: request.requiredCommit, integratedCommit: pr.mergeCommit.oid };@@ -58,6 +59,13 @@ async function prerequisiteReason(prerequisite) {export async function resolveRuntimeVerificationUpdate(task, outcome, agentId) {const now = new Date().toISOString();+ const owner = await getAgentRecord(agentId);+ const live = await getTaskById(task.id);+ const replay = live?.status === 'blocked' && live.metadata?.runtimeVerificationId === `${task.id}:${agentId}`+ && live.metadata?.runtimeVerification?.agentId === agentId;+ if (!live || owner?.taskId !== task.id || (!replay && (live.status !== 'in_progress'+ || !matchesCompletionTaskDispatch(owner, live)))) throw new Error('Runtime verification dispatch ownership is unverified');+ task = live;const prior = task.metadata?.runtimeVerification;// Replaying finalization retains the original dispatch identity and evidence.const same = prior?.agentId === agentId && task.metadata?.runtimeVerificationId === `${task.id}:${agentId}`;@@ -66,8 +74,7 @@ export async function resolveRuntimeVerificationUpdate(task, outcome, agentId) {totalSpawnCount: String(task.metadata?.totalSpawnCount ?? ''), requestedAt: now,prUrl: outcome.recovery.prUrl, requiredCommit: outcome.recovery.requiredCommit };let reason;- const owner = await getAgentRecord(agentId);- if (owner?.taskId !== task.id || (task.metadata?.app && task.metadata.app !== PORTOS_APP_ID)) {+ if (task.metadata?.app && task.metadata.app !== PORTOS_APP_ID) {reason = 'Runtime verification owner or PortOS task scope could not be verified';} else {const evidence = await integrationEvidence(prerequisite).catch(() => null);@@ -77,7 +84,9 @@ export async function resolveRuntimeVerificationUpdate(task, outcome, agentId) {} else reason = 'Runtime prerequisite retained: merged-fix integration evidence is unreadable or invalid';}return { status: 'blocked', metadata: { ...task.metadata,- runtimeVerificationId: `${task.id}:${agentId}`, runtimeVerification: prerequisite,+ runtimeVerificationId: `${task.id}:${agentId}`, runtimeVerification: { ...prerequisite, dispatchStatus: task.status,+ dispatch: Object.fromEntries(['claimedBy', 'claimedAt', 'lastSpawnedAt', 'totalSpawnCount']+ .filter(key => task.metadata?.[key] !== undefined).map(key => [key, task.metadata[key]])) },deliveryDecisionAgentId: agentId, blockedCategory: CATEGORY, blockedAt: same ? task.metadata.blockedAt : now,blockedReason: reason, taskOutcome: outcome,...clearedRetryHoldMetadata(),diff --git a/server/services/runtimeVerification.test.js b/server/services/runtimeVerification.test.jsindex 033ccca9dd..ea3293d2fc 100644--- a/server/services/runtimeVerification.test.js+++ b/server/services/runtimeVerification.test.js@@ -41,12 +41,18 @@ import { execGh } from './github.js';const outcome = () => normalizeTaskOutcome({ status: 'blocked', reason: 'The fix must be running before verification.',recovery: { mode: 'runtime', prUrl: 'https://github.com/example/PortOS/pull/123', requiredCommit: 'a'.repeat(40) } });-async function park({ ready = true } = {}) {+async function runningTask() {const task = await addTask({ id: 'sys-example', description: 'Verify merged fix', priority: 'MEDIUM', status: 'in_progress', metadata: {isInvestigation: true, lastSpawnedAt: '2026-01-01T00:00:00Z', totalSpawnCount: 4, failureCount: 2,orphanRetryCount: 1, existingBranch: 'claim/issue-123', resumeWorktreePath: '/example/worktree',} }, 'internal', { raw: true });- await updateTask(task.id, await resolveRuntimeVerificationUpdate(task, outcome(), 'agent-example'), 'internal');+ facts.owner.metadata.completionTaskDispatch = { taskId: task.id, taskType: 'internal',+ expected: ['claimedBy', 'claimedAt', 'lastSpawnedAt', 'totalSpawnCount'].map(key => [key, task.metadata[key] ?? null]) };+ return task;+}+async function park({ ready = true } = {}) {+ const task = await runningTask();+ await updateTask(task.id, await resolveRuntimeVerificationUpdate({ ...task, metadata: { totalSpawnCount: 0 } }, outcome(), 'agent-example'), 'internal');if (ready) await updateTask(task.id, { metadata: { runtimeVerificationReadyAgentId: 'agent-example' } }, 'internal');return getTaskById(task.id);}@@ -107,11 +113,19 @@ describe('persisted runtime continuation', () => {if (kind === 'foreign-repository') request.recovery.prUrl = 'https://github.com/another/project/pull/123';if (kind === 'wrong-commit') request.recovery.requiredCommit = 'f'.repeat(40);if (kind === 'unmerged') execGh.mockResolvedValueOnce(JSON.stringify({ state: 'OPEN', mergeCommit: null }));- const update = await resolveRuntimeVerificationUpdate({ id: 'sys-example', metadata: {} }, request, 'agent-example');+ const task = await runningTask();+ const update = await resolveRuntimeVerificationUpdate(task, request, 'agent-example');expect(update).toMatchObject({ status: 'blocked', metadata: { blockedReason: expect.stringContaining('unreadable or invalid') } });expect(update.metadata.runtimeVerification.integratedCommit).toBeUndefined();});+ it('refuses a predecessor dispatch when a successor has already claimed the task', async () => {+ const task = await runningTask();+ await updateTask(task.id, { metadata: { totalSpawnCount: 5, lastSpawnedAt: '2026-01-02T00:00:00Z' } }, 'internal');+ await expect(resolveRuntimeVerificationUpdate(task, outcome(), 'agent-example')).rejects.toThrow('dispatch ownership');+ expect((await getTaskById(task.id)).status).toBe('in_progress');+ });+it('retains unreadable integration as a prerequisite without spending another run, then verifies it afresh', async () => {facts.forge = false; await park(); facts.ancestor = true;expect((await getTaskById('sys-example')).metadata.blockedReason).toContain('unreadable or invalid');