Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: wait for running merged fixes before retrying verification (#13484)

Adam Eivy583ea68e4ed9
diff --git a/docs/features/chief-of-staff.md b/docs/features/chief-of-staff.md
index 635075af5b..f24afb5a2e 100644
--- a/docs/features/chief-of-staff.md
+++ b/docs/features/chief-of-staff.md
@@ -46,6 +46,27 @@ say “Execution completed.” Historical user feedback is unchanged. GitHub
notification resolution requires explicit completed or verified no-action-needed
delivery; exit success alone cannot mark a thread done.
+### Runtime verification prerequisites
+
+When verification requires a merged PortOS fix to be running, an agent may report
+`blocked` with `recovery: { "mode": "runtime", "prUrl": "https://github.com/example/PortOS/pull/123", "requiredCommit": "<full PR head or merge SHA>" }`.
+The server validates the configured repository, merged PR, default-branch
+integration and captured boot ancestry. A changed disk HEAD or completion
+sentinel does not satisfy this prerequisite.
+
+The original task retains its dispatch identity, merged checkout, receipt
+obligations and remaining retry/spawn budgets in the existing `cos_tasks` record.
+It stays blocked under `runtime-verification`, exempt from stale-failure expiry
+and generic revival. No new store, seed or migration is needed. The configured
+idle updater remains the only automatic update path: disabling it, cooldowns,
+active work and checkout-readiness gates continue to hold updates. A supported
+manual update can also satisfy the prerequisite.
+
+A CoS sweep resumes that same task once after verified boot ancestry and the
+original run's resume-pointer handoff. Concurrent sweeps use a conditional task
+write, and persisted evidence survives restart. This resumes verification; only
+its successful delivery can revive the investigation's affected tasks.
+
## Features
1. **Dual Task Lists**: User tasks and system tasks, both stored in `cos_tasks`
diff --git a/server/lib/agentOutcome.js b/server/lib/agentOutcome.js
index 17b2795fd6..f242b45059 100644
--- a/server/lib/agentOutcome.js
+++ b/server/lib/agentOutcome.js
@@ -99,6 +99,9 @@ export function normalizeTaskOutcome(value) {
if (typeof value.reason !== 'string' || !value.reason.trim()) return null;
const outcome = { status: value.status, reason: value.reason.trim() };
if (['blocked', 'incomplete'].includes(value.status)) {
+ if (value.recovery?.mode === 'runtime') outcome.recovery = { mode: 'runtime',
+ prUrl: typeof value.recovery.prUrl === 'string' ? value.recovery.prUrl : '',
+ requiredCommit: typeof value.recovery.requiredCommit === 'string' ? value.recovery.requiredCommit : '' };
if (value.recovery?.mode === 'cleanup') outcome.recovery = { mode: 'cleanup' };
if (value.recovery?.mode === 'approval') {
const options = Array.isArray(value.recovery.options)
diff --git a/server/lib/taskBlockCategories.js b/server/lib/taskBlockCategories.js
index 5e5f5bc62a..ee7d43c308 100644
--- a/server/lib/taskBlockCategories.js
+++ b/server/lib/taskBlockCategories.js
@@ -58,6 +58,7 @@ export const TIMED_COOLDOWN_BLOCKED_CATEGORIES = new Set([
*/
export const PAUSED_BLOCKED_CATEGORIES = new Set([
...TIMED_COOLDOWN_BLOCKED_CATEGORIES,
+ 'runtime-verification', // Server-owned boot prerequisite; retain work and never expire or auto-retry.
'task-approval', // A user decision must retain its checkout and never silently expire.
'task-recovery-exhausted', // Continuing beyond the retry/spawn limit needs an explicit decision.
'deep-audit-partial', // Explicit resume only; keep retained remediation and never auto-expire.
@@ -88,6 +89,7 @@ export const USER_DECISION_BLOCKED_CATEGORIES = new Set([
'user-terminated', // user explicitly stopped the agent
AGENT_PAUSED_CATEGORY, // user paused; resumable on demand
'challenge-escalation', // parked awaiting the user's arbitration
+ 'runtime-verification', // Server-owned boot prerequisite; retain work and never expire or auto-retry.
'task-approval', // A user decision must retain its checkout and never silently expire.
'task-recovery-exhausted', // Continuing beyond the retry/spawn limit needs an explicit decision.
'deep-audit-partial', // Explicit resume only; keep retained remediation and never auto-expire.
diff --git a/server/services/agentFinalization.js b/server/services/agentFinalization.js
index 27f1f14db6..90be315c83 100644
--- a/server/services/agentFinalization.js
+++ b/server/services/agentFinalization.js
@@ -1302,9 +1302,9 @@ export async function retireDeadAgent({ agent, task, success, exitCode, duration
&& (agent.metadata?.isWorktree === true || hasClaimCheckoutBindings(agent.metadata)));
const recoveryOutcome = !cleanupAllowed && taskOutcome?.recovery?.mode === 'cleanup' ? { ...taskOutcome, recovery: undefined } : taskOutcome;
const recoveryUpdate = deliveryHeld && !refused && !isPrivateSecurity && task?.id && task.status !== 'completed'
- ? resolveUndeliveredTaskUpdate(task, recoveryOutcome, agent.id) : null;
+ ? await resolveDeliveryRecoveryUpdate(task, recoveryOutcome, agent.id) : null;
if (recoveryUpdate) {
- const update = await updateTask(task.id, recoveryUpdate, task.taskType || 'user');
+ const update = await updateTask(task.id, recoveryUpdate, task.taskType || 'user', ...runtimeRecoveryFence(task, recoveryUpdate));
if (update?.error) throw new Error(`Could not persist recovered task outcome: ${update.error}`);
}
await completeAgent(agent.id, {
@@ -1337,7 +1337,7 @@ export async function retireDeadAgent({ agent, task, success, exitCode, duration
.catch(err => emitLog('warn', `Completion sentinel removal failed for ${agent.id}: ${err.message}`, { agentId: agent.id }));
if (recoveryUpdate?.metadata?.retryPendingCleanup
- || ['task-approval', 'task-recovery-exhausted'].includes(recoveryUpdate?.metadata?.blockedCategory)) {
+ || ['task-approval', 'task-recovery-exhausted', 'runtime-verification'].includes(recoveryUpdate?.metadata?.blockedCategory)) {
const { releaseRetryHold } = await import('./agentWorktreeCleanup.js');
await releaseRetryHold({ agentId: agent.id, task, success: false, agentMetadata: agent.metadata })
.catch(err => emitLog('warn', `Recovered delivery retry deferred for ${agent.id}: ${err.message}`, { agentId: agent.id }));
@@ -1996,7 +1996,7 @@ export async function finalizeAgent({
&& !(verdict.source === 'reported' && verdict.errorAnalysis?.category === `task-${reportedTaskOutcome?.status}`);
const taskUpdate = (deliveryHeld || (verdict.success && outcomeHeld)) && !terminatedByUser
&& !terminalFailure && !executionInterruption && !isPrivateSecurityTask(task) && hookResult?.outcome?.accepted !== false
- ? resolveUndeliveredTaskUpdate(preHookTask, (!executionSuccess || !verdict.success || validationPassed === false || !cleanupEligible) && taskOutcome.recovery?.mode === 'cleanup'
+ ? await resolveDeliveryRecoveryUpdate(preHookTask, (!executionSuccess || !verdict.success || validationPassed === false || !cleanupEligible) && taskOutcome.recovery?.mode === 'cleanup'
? { ...taskOutcome, recovery: undefined } : taskOutcome, agentId)
: await resolveFinalTaskUpdate({ task, preHookTask, hookMetadata, verdict, terminatedByUser, agentId });
taskUpdate.metadata = { ...(taskUpdate.metadata || task.metadata), taskOutcome };
@@ -2029,13 +2029,18 @@ export async function finalizeAgent({
if (cleanupEligible) await prepareCompletionTaskHandoff(await getAgentRecord(agentId),
Object.fromEntries(Object.entries(taskUpdate.metadata || {}).filter(([key, value]) =>
key !== 'privateSecurityScope' && JSON.stringify(value) !== JSON.stringify(preHookTask.metadata?.[key]))));
+ // Persist the prerequisite BEFORE retiring execution. A crash between the
+ // agent and task writes must never expose this dispatch to generic retry.
+ const runtimeTaskResult = taskUpdate.metadata?.blockedCategory === 'runtime-verification'
+ ? await updateTask(task.id, taskUpdate, taskType, ...runtimeRecoveryFence(preHookTask, taskUpdate)) : null;
+ if (runtimeTaskResult?.error || runtimeTaskResult?.statusChanged) throw new Error('Runtime verification task ownership changed before completion');
const completedAgent = await completeAgent(agentId, { ...completionRecord({
task, verdict: deliveryVerdict, hookResult, fidelity, executionInterruption, exitCode, duration, outputBuffer, validationPassed,
}), executionSuccess, taskOutcome,
requireCheckoutSettlement: deliverySuccess && taskUpdate.status === 'completed' && cleanupEligible
&& resolvePrCompletion(task.metadata) === PR_COMPLETIONS.REVIEW_THEN_MERGE,
...(claimDisposition ? { claimDisposition } : {}),
- ...(['task-approval', 'task-recovery-exhausted'].includes(taskUpdate.metadata?.blockedCategory)
+ ...(['task-approval', 'task-recovery-exhausted', 'runtime-verification'].includes(taskUpdate.metadata?.blockedCategory)
|| taskUpdate.metadata?.retryPendingCleanup === agentId || taskUpdate.metadata?.deliveryRecoveryAgentId === agentId
? { deliveryRecoveryOwned: true } : {}),
...(taskUpdate.metadata?.deliveryRecoveryAgentId === agentId ? { cleanupPending: true } : {}) });
@@ -2064,9 +2069,9 @@ export async function finalizeAgent({
delete task.privateSecurityScope;
if (taskUpdate.metadata) delete taskUpdate.metadata.privateSecurityScope;
}
- const taskResult = capturePending
+ const taskResult = runtimeTaskResult || (capturePending
? await recoverCompletionTaskHandoff(await getAgentRecord(agentId)) || { error: 'Completion cleanup task ownership changed' }
- : await updateTask(task.id, taskUpdate, taskType);
+ : await updateTask(task.id, taskUpdate, taskType, ...runtimeRecoveryFence(preHookTask, taskUpdate)));
if (taskResult?.error && !isExpectedUnpersistedMiss(task, taskResult.error)) {
const label = terminatedByUser ? 'blocked' : verdict.success ? 'completed' : 'failed';
emitLog('warn', `⚠️ Failed to update ${label} task ${task.id}: ${taskResult.error} (taskType=${taskType})`, { taskId: task.id, agentId, error: taskResult.error });
@@ -2427,3 +2432,21 @@ async function persistCompletionSummary(agentId, { workspacePath, finalSummary =
} });
return taskOutcome;
}
+
+// Runtime evidence is server-owned and never inferred from completion prose.
+async function resolveDeliveryRecoveryUpdate(task, outcome, agentId) {
+ if (outcome?.recovery?.mode === 'runtime') {
+ const { resolveRuntimeVerificationUpdate } = await import('./runtimeVerification.js');
+ return resolveRuntimeVerificationUpdate(task, outcome, agentId);
+ }
+ return resolveUndeliveredTaskUpdate(task, outcome, agentId);
+}
+
+function runtimeRecoveryFence(task, update) {
+ return update.metadata?.blockedCategory === 'runtime-verification' ? [{
+ expectedStatus: task.status === 'blocked' ? 'blocked' : 'in_progress', expectedMetadata: {
+ ...(task.status === 'blocked' ? { runtimeVerificationId: task.metadata?.runtimeVerificationId } : {}),
+ lastSpawnedAt: task.metadata?.lastSpawnedAt,
+ totalSpawnCount: task.metadata?.totalSpawnCount, claimedBy: task.metadata?.claimedBy },
+ }] : [];
+}
diff --git a/server/services/agentFinalization.outputHooks.test.js b/server/services/agentFinalization.outputHooks.test.js
index dbef138c0f..d221408b70 100644
--- a/server/services/agentFinalization.outputHooks.test.js
+++ b/server/services/agentFinalization.outputHooks.test.js
@@ -475,6 +475,43 @@ describe('recovery output-hook dispatch (#3182)', () => {
expect(addTask).not.toHaveBeenCalled();
});
+ it.each(['normal', 'recovered'])('parks a runtime prerequisite through %s public completion without a provider retry', async mode => {
+ isProgrammaticIoTaskType.mockReturnValue(false);
+ const task = { id: TASK.id, taskType: 'internal', status: 'in_progress', metadata: {
+ isInvestigation: true, totalSpawnCount: 3, failureCount: 1, lastSpawnedAt: '2026-09-10T00:00:00Z',
+ } };
+ persistedAgent.taskId = task.id;
+ mergedTarget.getOriginInfo.mockResolvedValue({ isGithub: true, host: 'github.com', fullName: 'example/PortOS' });
+ const github = await import('./github.js');
+ const git = await import('../lib/execGit.js');
+ const forge = vi.spyOn(github, 'execGh').mockImplementation(async args => JSON.stringify(args[0] === 'pr'
+ ? { state: 'MERGED', url: 'https://github.com/example/PortOS/pull/123', headRefOid: 'a'.repeat(40), mergeCommit: { oid: 'c'.repeat(40) }, baseRefName: 'main' }
+ : { defaultBranchRef: { name: 'main' } }));
+ const gitRead = vi.spyOn(git, 'execGit').mockResolvedValue({ stdout: 'd'.repeat(40), exitCode: 0 });
+ const taskOutcome = { status: 'blocked', reason: 'Update preflight refuses this active investigation; boot is stale.',
+ recovery: { mode: 'runtime', prUrl: 'https://github.com/example/PortOS/pull/123', requiredCommit: 'a'.repeat(40) } };
+ try {
+ persistedAgent.metadata.taskOutcome = taskOutcome;
+ persistedAgent.metadata.taskOutcomeSource = 'terminal-result';
+ const order = [];
+ updateTask.mockImplementationOnce(async () => { order.push('prerequisite'); return {}; });
+ completeAgent.mockImplementationOnce(async () => { order.push('terminal'); return null; });
+ const result = mode === 'normal'
+ ? await finalizeAgent({ agentId: persistedAgent.id, task, success: true, exitCode: 0,
+ finalSummary: JSON.stringify({ summary: taskOutcome.reason, taskOutcome }) })
+ : await retireDeadAgent({ agent: persistedAgent, task, success: true, exitCode: 0 });
+ expect(result).toMatchObject({ success: false, taskOutcome });
+ expect(order).toEqual(['prerequisite', 'terminal']);
+ expect(updateTask).toHaveBeenCalledWith(task.id, expect.objectContaining({ status: 'blocked', metadata: expect.objectContaining({
+ blockedCategory: 'runtime-verification', failureCount: 1, totalSpawnCount: 3,
+ runtimeVerification: expect.objectContaining({ taskId: task.id, agentId: persistedAgent.id, integratedCommit: 'c'.repeat(40) }),
+ }) }), 'internal', expect.objectContaining({ expectedStatus: 'in_progress' }));
+ expect(completeAgent).toHaveBeenCalledWith(persistedAgent.id, expect.objectContaining({ success: false, deliveryRecoveryOwned: true }));
+ expect(resolveFailedTaskUpdate).not.toHaveBeenCalled();
+ expect(addTask).not.toHaveBeenCalled();
+ } finally { forge.mockRestore(); gitRead.mockRestore(); }
+ });
+
it.each(['worktree', 'pinned-claim'])('hands %s cleanup-only work to completion recovery without another agent or success claim', async mode => {
const taskOutcome = { status: 'incomplete', reason: 'Verified delivery; owner still holds checkout.', recovery: { mode: 'cleanup' } };
isProgrammaticIoTaskType.mockReturnValue(false);
diff --git a/server/services/agentWorktreeCleanup.js b/server/services/agentWorktreeCleanup.js
index 91dccc86a4..3e47aa7987 100644
--- a/server/services/agentWorktreeCleanup.js
+++ b/server/services/agentWorktreeCleanup.js
@@ -208,6 +208,7 @@ async function worktreeCleanupContext(agentId, options) {
discardWorktree: isTruthyMeta(normalized.originalTask?.metadata?.discardWorktree),
worktreePath: metadata.workspacePath || join(PATHS.worktrees, agentId),
publicationValidation: publicationHolds.get(agentId) || metadata.publicationValidation,
+ runtimeVerificationHeld: agentState?.result?.taskOutcome?.recovery?.mode === 'runtime',
securityScan: metadata.securityScan,
securityScanControl: metadata.securityScanControl,
warnings: [],
@@ -237,6 +238,7 @@ async function probeWorktreePr(context, success) {
async function runCleanupAgentWorktree(agentId, success, options = {}) {
const context = await worktreeCleanupContext(agentId, options);
+ if (context.runtimeVerificationHeld) return []; // The same checkout resumes after verified boot ancestry.
if (['running', 'blocked'].includes(context.publicationValidation?.status)) {
return [`Publication validation ${context.publicationValidation.status} for ${context.worktreeBranch}; worktree preserved at ${context.worktreePath}. Resume the task after resolving the validation outcome.`];
}
@@ -991,11 +993,14 @@ export async function releaseRetryHold({ agentId, task, success, agentMetadata }
});
if (!persisted) return {};
- if (persisted.status === 'blocked' && ['task-approval', 'task-recovery-exhausted'].includes(persisted.metadata?.blockedCategory)) {
+ if (persisted.status === 'blocked' && ['task-approval', 'task-recovery-exhausted', 'runtime-verification'].includes(persisted.metadata?.blockedCategory)) {
if (persisted.metadata.deliveryDecisionAgentId !== agentId) return {};
const metadata = agentMetadata === undefined ? (await getAgentRecord(agentId))?.metadata : agentMetadata;
- const patch = await resolveTaskResumePatch({ task, agentId, agentMetadata: metadata });
- const result = await updateTask(task.id, { metadata: patch }, task.taskType || persisted.taskType || 'user', {
+ const patch = persisted.metadata.blockedCategory === 'runtime-verification'
+ ? await runtimeResumePatch({ task: persisted, agentId, agentMetadata: metadata })
+ : await resolveTaskResumePatch({ task, agentId, agentMetadata: metadata });
+ const result = await updateTask(task.id, { metadata: { ...patch,
+ ...(persisted.metadata.blockedCategory === 'runtime-verification' ? { runtimeVerificationReadyAgentId: agentId } : {}) } }, task.taskType || persisted.taskType || 'user', {
expectedStatus: 'blocked', expectedMetadata: { blockedCategory: persisted.metadata.blockedCategory, blockedAt: persisted.metadata.blockedAt, deliveryDecisionAgentId: agentId },
});
return result?.error || result?.statusChanged ? {} : patch;
@@ -1485,3 +1490,17 @@ export async function spawnMergeRecoveryTask(cleanupWarnings, agentId, task, app
emitLog('info', 'Cleanup warning retained for deterministic completion recovery', { agentId, appName });
return null;
}
+
+// A verification continuation deliberately retains even an already-merged
+// branch. Ordinary failed-task retry correctly discards those resume pointers.
+async function runtimeResumePatch({ task, agentId, agentMetadata }) {
+ const claim = await resolveClaimContinuationPatch({ task, agentId, agentMetadata });
+ if (claim) return claim;
+ if (!agentMetadata?.isWorktree || agentMetadata.isPersistentWorktree) return {};
+ const branch = agentMetadata.worktreeBranch;
+ const path = agentMetadata.workspacePath || join(PATHS.worktrees, agentId);
+ if (!branch || !existsSync(path) || await git.getBranch(path) !== branch) {
+ throw new Error('Runtime verification checkout identity is unavailable; retain the prerequisite');
+ }
+ return { existingBranch: branch, resumeWorktreePath: path, resumedFromAgentId: agentId };
+}
diff --git a/server/services/cleanupAgentWorktree.test.js b/server/services/cleanupAgentWorktree.test.js
index 9ea7558017..35d03a0512 100644
--- a/server/services/cleanupAgentWorktree.test.js
+++ b/server/services/cleanupAgentWorktree.test.js
@@ -283,6 +283,14 @@ function mockWorktreeAgent(overrides = {}) {
}
describe('cleanupAgentWorktree - PR-creation path', () => {
+ it('preserves the runtime continuation checkout even with an old discard policy', async () => {
+ getAgent.mockResolvedValue({ ...mockWorktreeAgent(), result: { success: false,
+ taskOutcome: { status: 'blocked', recovery: { mode: 'runtime' } } } });
+ await cleanupAgentWorktree('agent-runtime', false, { originalTask: { metadata: { discardWorktree: true } } });
+ expect(removeWorktree).not.toHaveBeenCalled();
+ expect(git.push).not.toHaveBeenCalled();
+ });
+
it('publishes only after PortOS validation succeeds and persists that distinct outcome', async () => {
validatePublication.mockResolvedValue({ status: 'passed', head: 'verified-head' });
git.push.mockImplementation(async () => {
@@ -1744,6 +1752,25 @@ describe('releaseRetryHold', () => {
}) }, 'user', { expectedStatus: 'blocked', expectedMetadata: { blockedCategory, blockedAt: 'now', deliveryDecisionAgentId: 'agent-x' } });
});
+ it('retains an already-merged checkout for runtime verification and never makes it pending', async () => {
+ getTaskById.mockResolvedValue({ id: 'task-1', status: 'blocked', metadata: {
+ blockedCategory: 'runtime-verification', blockedAt: 'now', deliveryDecisionAgentId: 'agent-x',
+ } });
+ git.hasBranchMergeEvidence.mockResolvedValue(true);
+ git.getBranch.mockResolvedValue(DEAD_BRANCH);
+ existsSyncMock.mockReturnValue(true);
+ await releaseRetryHold({ agentId: 'agent-x', task: task(), success: false, agentMetadata });
+ expect(updateTask).toHaveBeenCalledWith('task-1', { metadata: {
+ existingBranch: DEAD_BRANCH, resumedFromAgentId: 'agent-x', resumeWorktreePath: DEAD_TREE,
+ runtimeVerificationReadyAgentId: 'agent-x',
+ } }, 'user', expect.objectContaining({ expectedStatus: 'blocked' }));
+ updateTask.mockClear();
+ git.getBranch.mockResolvedValue('another-owner');
+ await expect(releaseRetryHold({ agentId: 'agent-x', task: task(), success: false, agentMetadata }))
+ .rejects.toThrow('checkout identity');
+ expect(updateTask).not.toHaveBeenCalled();
+ });
+
it('records the pointer for a failed run whose task is still pending', async () => {
await releaseRetryHold({ agentId: 'agent-x', task: task(), success: false });
diff --git a/server/services/cos.js b/server/services/cos.js
index 67d9092dfa..8c20172352 100644
--- a/server/services/cos.js
+++ b/server/services/cos.js
@@ -521,6 +521,15 @@ async function bootDaemon(state) {
// Existing retirement journals must recover even when branch-reconcile is
// disabled or every agent slot is occupied. The service reuses its exact cut;
// it never goes through normal agent/worktree admission.
+ scheduleEvent({
+ id: 'cos-runtime-verification', type: 'interval', intervalMs: 60 * 1000,
+ handler: async () => {
+ if (await pendingMergeSweepBlocked()) return;
+ const { sweepRuntimeVerifications } = await import('./runtimeVerification.js');
+ await sweepRuntimeVerifications().catch(error => console.warn(`⚠️ Runtime verification deferred: ${error.message}`));
+ },
+ metadata: { description: 'Resume runtime verification after a verified boot' },
+ });
const { INITIALIZED_RECOVERY_INTERVAL_MS, sweepInitializedCheckoutRecoveries } = await import('./initializedCheckoutRecoverySweep.js');
scheduleEvent({
id: 'cos-initialized-retirement-recovery',
@@ -630,6 +639,7 @@ async function teardownDaemon() {
cancelEvent('cos-learning-insights');
cancelEvent('cos-rehabilitation-check');
cancelEvent('cos-improvement-check');
+ cancelEvent('cos-runtime-verification');
cancelEvent('cos-pending-merge-sweep');
cancelEvent('cos-initialized-retirement-recovery');
(await import('./prWatcher.js')).cancelHandoffFollowUps();
diff --git a/server/services/cosTaskStore.js b/server/services/cosTaskStore.js
index 3a7ce92c35..22d2a2dd46 100644
--- a/server/services/cosTaskStore.js
+++ b/server/services/cosTaskStore.js
@@ -836,6 +836,13 @@ async function writeTaskUpdateLocked(taskId, updates, taskType, { now, suppressD
if (!releasesCurrentHold) return { error: 'Deep checkpoint cleanup is still pending; resume after cleanup finishes' };
}
+ const runtimeHeld = current.status === 'blocked' && current.metadata?.blockedCategory === 'runtime-verification';
+ const runtimeRelease = runtimeHeld && updates.status === 'pending'
+ && expectedStatus === 'blocked' && expectedMetadata?.runtimeVerificationId === current.metadata.runtimeVerificationId
+ && updates.metadata?.runtimeVerification?.status === 'resumed';
+ if (runtimeHeld && updates.status && updates.status !== 'blocked' && !runtimeRelease) {
+ return { error: 'Runtime verification prerequisite must be released by its server coordinator' };
+ }
const updatedMetadata = mergeUpdateMetadata(tasks[taskIndex].metadata, updates);
// Ask "what status transition is this?" ONCE. Every lifecycle rule below reads a
@@ -851,7 +858,10 @@ async function writeTaskUpdateLocked(taskId, updates, taskType, { now, suppressD
// the `blockedCategory` clear feeding the resume-pointer check below.
for (const [field, keys] of TRANSITION_METADATA_CLEARS) {
if (!transition[field]) continue;
- for (const key of keys) delete updatedMetadata[key];
+ for (const key of keys) {
+ if (runtimeRelease && ['failureCount', 'lastErrorCategory', 'lastFailureAt'].includes(key)) continue;
+ delete updatedMetadata[key];
+ }
}
// Drop the resume pointer once the task reaches a terminal state. `existingBranch`
diff --git a/server/services/promptSections/completion.js b/server/services/promptSections/completion.js
index 1203cc0ff1..14600760a1 100644
--- a/server/services/promptSections/completion.js
+++ b/server/services/promptSections/completion.js
@@ -77,7 +77,7 @@ const SENTINEL_WATCH_TAIL = 'PortOS watches this sentinel and finalizes the run
* per-section splice the next mode had to remember. Composing it here keeps
* "prints a sentinel path" and "says the write is permitted" one fact.
*/
-const TASK_OUTCOME_GUIDANCE = 'Write a JSON envelope with `summary` (your markdown report), `taskOutcome: { "status": "completed" | "no-action-needed" | "blocked" | "incomplete", "reason": "specific evidence or blocker" }`, and the existing `payload` when your task requires one. Use completed only when the requested deliverable is finished; no-action-needed only when checks verified no work remains; blocked for an unmet prerequisite; incomplete for unfinished work. Unfinished work resumes automatically within the existing retry/spawn budget. If ONLY repository cleanup remains after verified delivery, add `recovery: { "mode": "cleanup" }` inside taskOutcome: the server owns cleanup after this run releases its ownership, without another agent. Do not use cleanup mode for unmerged PRs, red validation, missing deliverables, or unresolved review findings. Only when proceeding needs a human safety or preference decision, add `recovery: { "mode": "approval", "options": ["concrete option and consequence", "alternative and consequence"] }`; explain the decision in reason. Never request approval for routine mechanical recovery. Process exit and writing this file do not prove task achievement. Preserve unfinished work and name what remains in the reason.';
+const TASK_OUTCOME_GUIDANCE = 'Write a JSON envelope with `summary` (your markdown report), `taskOutcome: { "status": "completed" | "no-action-needed" | "blocked" | "incomplete", "reason": "specific evidence or blocker" }`, and the existing `payload` when your task requires one. Use completed only when the requested deliverable is finished; no-action-needed only when checks verified no work remains; blocked for an unmet prerequisite; incomplete for unfinished work. Unfinished work resumes automatically within the existing retry/spawn budget. If ONLY repository cleanup remains after verified delivery, add `recovery: { "mode": "cleanup" }` inside taskOutcome: the server owns cleanup after this run releases its ownership, without another agent. Do not use cleanup mode for unmerged PRs, red validation, missing deliverables, or unresolved review findings. For a PortOS investigation whose merged fix must be running before verification, report blocked with `recovery: { "mode": "runtime", "prUrl": "https://github.com/owner/repository/pull/123", "requiredCommit": "<full merged PR head or merge commit>" }`. The server verifies integration and boot ancestry, preserves the same task and remaining budgets, and waits for the configured idle update policy; never restart or force an update yourself. Only when proceeding needs a human safety or preference decision, add `recovery: { "mode": "approval", "options": ["concrete option and consequence", "alternative and consequence"] }`; explain the decision in reason. Never request approval for routine mechanical recovery. Process exit and writing this file do not prove task achievement. Preserve unfinished work and name what remains in the reason.';
const sentinelCompletionSection = (notice, writeInstruction) =>
[notice, '', `${writeInstruction} ${SENTINEL_WATCH_TAIL}`, '', TASK_OUTCOME_GUIDANCE, '', SENTINEL_WRITE_PERMISSION_NOTE].join('\n');
diff --git a/server/services/runtimeVerification.js b/server/services/runtimeVerification.js
new file mode 100644
index 0000000000..bdd5784755
--- /dev/null
+++ b/server/services/runtimeVerification.js
@@ -0,0 +1,133 @@
+/**
+ * A boot prerequisite is unfinished delivery, never a failed provider attempt.
+ * Persist it on the original task and use the task store's compare-and-set to
+ * release it once. Updating remains entirely with the configured idle updater.
+ */
+import { clearedRetryHoldMetadata } from '../lib/taskRetryHold.js';
+import { PATHS } from '../lib/fileUtils.js';
+import { PORTOS_APP_ID } from '../lib/appIdentity.js';
+import { execGit } from '../lib/execGit.js';
+import { getOriginInfo } from '../lib/gitRemote.js';
+import { getBootCommit } from './installState.js';
+import { getAgentRecord } from './cosAgentLifecycle.js';
+import { getAllTasks, updateTask } from './cosTaskStore.js';
+
+const SHA = /^[a-f0-9]{40}$/;
+const CATEGORY = 'runtime-verification';
+
+async function contains(ancestor, descendant) {
+ if (!SHA.test(ancestor || '') || !SHA.test(descendant || '')) return false;
+ const result = await execGit(['merge-base', '--is-ancestor', ancestor, descendant], PATHS.root, { ignoreExitCode: true });
+ return result.exitCode === 0;
+}
+
+/** Structured forge fields only; neither PR prose nor agent assertions are proof. */
+async function integrationEvidence(request) {
+ if (!SHA.test(request.requiredCommit || '')) throw new Error('A full required commit is missing');
+ const origin = await getOriginInfo(PATHS.root);
+ const url = new URL(request.prUrl);
+ if (!origin.isGithub || url.origin !== `https://${origin.host}` || url.search || url.hash
+ || url.pathname.split('/').slice(1, 3).join('/').toLowerCase() !== origin.fullName.toLowerCase()
+ || !/^\/[^/]+\/[^/]+\/pull\/[1-9]\d*$/.test(url.pathname)) throw new Error('The fix must belong to this PortOS repository');
+ const { execGh } = await import('./github.js');
+ const raw = await execGh(['pr', 'view', url.pathname.split('/').at(-1), '--repo', origin.fullName,
+ '--json', 'url,state,headRefOid,mergeCommit,baseRefName'], 15000, { cwd: PATHS.root });
+ const pr = JSON.parse(raw);
+ const repo = JSON.parse(await execGh(['repo', 'view', origin.fullName, '--json', 'defaultBranchRef'], 15000, { cwd: PATHS.root }));
+ if (pr.state !== 'MERGED' || pr.url !== request.prUrl || pr.baseRefName !== repo.defaultBranchRef?.name
+ || !SHA.test(pr.mergeCommit?.oid || '') || ![pr.headRefOid, pr.mergeCommit.oid].includes(request.requiredCommit)) {
+ throw new Error('The required commit is not the verified merged fix on the default branch');
+ }
+ // A squash/rebase need not contain the old PR head. The forge's integration
+ // commit is the runtime prerequisite; never replace it with disk HEAD.
+ await execGit(['fetch', 'origin', pr.baseRefName], PATHS.root);
+ const target = await execGit(['rev-parse', '--verify', `refs/remotes/origin/${pr.baseRefName}^{commit}`], PATHS.root);
+ if (!await contains(pr.mergeCommit.oid, target.stdout.trim())) throw new Error('The merged fix is not integrated in the fetched default branch');
+ return { prUrl: request.prUrl, requiredCommit: request.requiredCommit, integratedCommit: pr.mergeCommit.oid };
+}
+
+async function prerequisiteReason(prerequisite) {
+ const { getSettingsWithStatus } = await import('./settings.js');
+ const policy = await getSettingsWithStatus().catch(() => ({ corrupt: true }));
+ const update = policy.corrupt ? 'Update policy is unreadable; retain the prerequisite.'
+ : policy.settings?.autoUpdate?.enabled === true
+ ? 'Waiting for the configured idle update policy, including cooldown and readiness gates.'
+ : 'Automatic updates are disabled; a supported operator update is required.';
+ return `Runtime verification requires ${prerequisite.integratedCommit || prerequisite.requiredCommit || 'a verified merged fix'}; captured boot is ${getBootCommit() || 'unreadable'}. ${update}`;
+}
+
+export async function resolveRuntimeVerificationUpdate(task, outcome, agentId) {
+ const now = new Date().toISOString();
+ const prior = task.metadata?.runtimeVerification;
+ // Replaying finalization retains the original dispatch identity and evidence.
+ const same = prior?.agentId === agentId && task.metadata?.runtimeVerificationId === `${task.id}:${agentId}`;
+ let prerequisite = same ? prior : { version: 1, status: 'waiting', taskId: task.id, agentId,
+ lastSpawnedAt: task.metadata?.lastSpawnedAt || null,
+ totalSpawnCount: String(task.metadata?.totalSpawnCount ?? ''), requestedAt: now,
+ prUrl: outcome.recovery.prUrl, requiredCommit: outcome.recovery.requiredCommit };
+ let reason;
+ const owner = await getAgentRecord(agentId);
+ if (owner?.taskId !== task.id || (task.metadata?.app && task.metadata.app !== PORTOS_APP_ID)) {
+ reason = 'Runtime verification owner or PortOS task scope could not be verified';
+ } else {
+ const evidence = await integrationEvidence(prerequisite).catch(() => null);
+ if (evidence) {
+ prerequisite = { ...prerequisite, ...evidence };
+ reason = await prerequisiteReason(prerequisite);
+ } else reason = 'Runtime prerequisite retained: merged-fix integration evidence is unreadable or invalid';
+ }
+ return { status: 'blocked', metadata: { ...task.metadata,
+ runtimeVerificationId: `${task.id}:${agentId}`, runtimeVerification: prerequisite,
+ deliveryDecisionAgentId: agentId, blockedCategory: CATEGORY, blockedAt: same ? task.metadata.blockedAt : now,
+ blockedReason: reason, taskOutcome: outcome,
+ ...clearedRetryHoldMetadata(),
+ deliveryRecovery: undefined, deliveryRecoveryAgentId: undefined } };
+}
+
+/** Every sweep rechecks boot/forge/dispatch evidence; an unreadable fact holds. */
+export async function sweepRuntimeVerifications() {
+ const queues = await getAllTasks();
+ const { readAgentsStateForSafetyCheck } = await import('./cosState.js');
+ const owners = await readAgentsStateForSafetyCheck();
+ if (!owners.trusted) return { resumed: [] };
+ const resumed = [];
+ for (const [taskType, queue] of [['user', queues.user], ['internal', queues.cos]]) {
+ for (const task of queue?.tasks || []) {
+ const m = task.metadata || {};
+ const p = m.runtimeVerification;
+ if (task.status !== 'blocked' || m.blockedCategory !== CATEGORY || p?.version !== 1 || p.status !== 'waiting'
+ || p.taskId !== task.id || m.runtimeVerificationId !== `${task.id}:${p.agentId}`
+ || m.deliveryDecisionAgentId !== p.agentId || (m.app && m.app !== PORTOS_APP_ID)
+ || (m.lastSpawnedAt || null) !== p.lastSpawnedAt
+ || String(m.totalSpawnCount ?? '') !== p.totalSpawnCount
+ || Object.values(owners.agents).some(a => a.taskId === task.id && a.status === 'running')) continue;
+ const owner = await getAgentRecord(p.agentId);
+ if (owner?.taskId !== task.id || !['completed', 'failed'].includes(owner.status)) continue;
+ // A restart may interrupt the failed-delivery cleanup before its resume
+ // pointer is saved. Reuse its guarded preservation path, without re-running
+ // finalization, resetting budgets or declaring checkout retirement complete.
+ if (m.runtimeVerificationReadyAgentId !== p.agentId) {
+ const { releaseRetryHold } = await import('./agentWorktreeCleanup.js');
+ await releaseRetryHold({ agentId: p.agentId, task: { ...task, taskType }, success: false, agentMetadata: owner.metadata });
+ continue;
+ }
+ const boot = getBootCommit();
+ if (!SHA.test(boot || '')) continue;
+ // A stale boot cannot change until restart. Do not poll the forge or fetch
+ // every minute during the operator's potentially hours-long cooldown.
+ if (p.integratedCommit && !await contains(p.integratedCommit, boot).catch(() => false)) continue;
+ const evidence = await integrationEvidence(p).catch(() => null);
+ if (!evidence || (p.integratedCommit && p.integratedCommit !== evidence.integratedCommit)) continue;
+ if (!await contains(evidence.integratedCommit, boot).catch(() => false)) continue;
+ const result = await updateTask(task.id, { status: 'pending', metadata: {
+ runtimeVerification: { ...p, ...evidence, status: 'resumed', bootCommit: boot, resumedAt: new Date().toISOString() },
+ runtimeVerificationReadyAgentId: undefined,
+ } }, taskType, { expectedStatus: 'blocked', expectedMetadata: {
+ runtimeVerificationId: m.runtimeVerificationId, deliveryDecisionAgentId: p.agentId,
+ blockedCategory: CATEGORY, blockedAt: m.blockedAt, lastSpawnedAt: m.lastSpawnedAt, totalSpawnCount: m.totalSpawnCount,
+ } });
+ if (!result?.error && !result?.statusChanged) resumed.push(task.id);
+ }
+ }
+ return { resumed };
+}
diff --git a/server/services/runtimeVerification.test.js b/server/services/runtimeVerification.test.js
new file mode 100644
index 0000000000..2263bda924
--- /dev/null
+++ b/server/services/runtimeVerification.test.js
@@ -0,0 +1,123 @@
+import { beforeEach, afterAll, describe, expect, it, vi } from 'vitest';
+import { makePathsProxy, lazyTempDataRoot, cleanupTempDataRoots } from '../lib/mockPathsDataRoot.js';
+vi.mock('../lib/paths.js', async original => makePathsProxy(await original(), { dataRoot: () => lazyTempDataRoot('runtime-verification-') }));
+afterAll(cleanupTempDataRoots);
+const facts = vi.hoisted(() => ({ boot: 'b'.repeat(40), integrated: 'c'.repeat(40),
+ owner: { id: 'agent-example', taskId: 'sys-example', status: 'completed', metadata: {} },
+ trusted: true, running: false, forge: true, ancestor: false }));
+vi.mock('./cosState.js', async () => ({
+ loadState: vi.fn(async () => ({ agents: {} })),
+ withStateLock: (await import('../lib/fileWriteQueue.js')).createFileWriteQueue(),
+ readAgentsStateForSafetyCheck: vi.fn(async () => ({ trusted: facts.trusted,
+ agents: facts.running ? { sibling: { taskId: 'sys-example', status: 'running' } } : {} })),
+}));
+vi.mock('./cosAgentLifecycle.js', () => ({ getAgentRecord: vi.fn(async () => facts.owner) }));
+vi.mock('./cosEvents.js', () => ({ emitLog: vi.fn(), cosEvents: { emit: vi.fn() } }));
+vi.mock('./settings.js', () => ({ getSettingsWithStatus: vi.fn(async () => ({ corrupt: false, settings: { autoUpdate: { enabled: false } } })) }));
+vi.mock('./installState.js', () => ({ getBootCommit: () => facts.boot }));
+vi.mock('../lib/gitRemote.js', () => ({ getOriginInfo: vi.fn(async () => ({ isGithub: true, host: 'github.com', fullName: 'example/PortOS' })) }));
+vi.mock('./github.js', () => ({ execGh: vi.fn(async args => {
+ if (!facts.forge) throw new Error('Unavailable');
+ return JSON.stringify(args[0] === 'pr'
+ ? { state: 'MERGED', url: 'https://github.com/example/PortOS/pull/123', headRefOid: 'a'.repeat(40), mergeCommit: { oid: facts.integrated }, baseRefName: 'main' }
+ : { defaultBranchRef: { name: 'main' } });
+}) }));
+vi.mock('../lib/execGit.js', () => ({ execGit: vi.fn(async args => {
+ if (args[0] === 'rev-parse') return { stdout: 'd'.repeat(40), exitCode: 0 };
+ if (args[0] === 'merge-base') return { exitCode: args[3] === facts.boot ? (facts.ancestor ? 0 : 1) : 0 };
+ return { stdout: '', exitCode: 0 };
+}) }));
+vi.mock('./agentWorktreeCleanup.js', () => ({ releaseRetryHold: vi.fn(async ({ task, agentId }) => {
+ const { updateTask } = await import('./cosTaskStore.js');
+ await updateTask(task.id, { metadata: { runtimeVerificationReadyAgentId: agentId } }, task.taskType);
+}) }));
+
+import { resolveRuntimeVerificationUpdate, sweepRuntimeVerifications } from './runtimeVerification.js';
+import { addTask, getTaskById, updateTask, reviveBlockedTask, __resetTaskCache, sweepResolvedFailureTasks } from './cosTaskStore.js';
+import { __resetCosTaskRepository } from './cosTaskRepository.js';
+import { cosEvents } from './cosEvents.js';
+import { normalizeTaskOutcome } from '../lib/agentOutcome.js';
+import { execGh } from './github.js';
+
+const outcome = () => normalizeTaskOutcome({ status: 'blocked', reason: 'The fix must be running before verification.',
+ recovery: { mode: 'runtime', prUrl: 'https://github.com/example/PortOS/pull/123', requiredCommit: 'a'.repeat(40) } });
+async function park({ ready = true } = {}) {
+ const task = await addTask({ id: 'sys-example', description: 'Verify merged fix', priority: 'MEDIUM', status: 'in_progress', metadata: {
+ isInvestigation: true, lastSpawnedAt: '2026-01-01T00:00:00Z', totalSpawnCount: 4, failureCount: 2,
+ orphanRetryCount: 1, existingBranch: 'claim/issue-123', resumeWorktreePath: '/example/worktree',
+ } }, 'internal', { raw: true });
+ await updateTask(task.id, await resolveRuntimeVerificationUpdate(task, outcome(), 'agent-example'), 'internal');
+ if (ready) await updateTask(task.id, { metadata: { runtimeVerificationReadyAgentId: 'agent-example' } }, 'internal');
+ return getTaskById(task.id);
+}
+
+beforeEach(() => {
+ __resetCosTaskRepository(); __resetTaskCache(); vi.clearAllMocks();
+ Object.assign(facts, { boot: 'b'.repeat(40), integrated: 'c'.repeat(40), ancestor: false, forge: true, trusted: true, running: false,
+ owner: { id: 'agent-example', taskId: 'sys-example', status: 'completed', metadata: {} } });
+});
+
+describe('persisted runtime continuation', () => {
+ it('parks stale code without budget changes, expiry, generic revival or affected-task completion', async () => {
+ const task = await park();
+ expect(task).toMatchObject({ status: 'blocked', metadata: { blockedCategory: 'runtime-verification', failureCount: '2', totalSpawnCount: '4' } });
+ expect(task.metadata.blockedReason).toContain('Automatic updates are disabled');
+ expect(await reviveBlockedTask(task.id)).toMatchObject({ error: expect.stringContaining('Runtime verification') });
+ await sweepResolvedFailureTasks({ now: Date.now() + 60 * 86400000 });
+ expect(await sweepRuntimeVerifications()).toEqual({ resumed: [] });
+ expect(await getTaskById(task.id)).toMatchObject({ status: 'blocked', metadata: { existingBranch: 'claim/issue-123', failureCount: '2' } });
+ });
+
+ it('resumes the same task exactly once through concurrent sweeps and cache reload, retaining its budget and checkout', async () => {
+ await park();
+ facts.ancestor = true;
+ __resetTaskCache(); // the persisted record, not process-local continuation state
+ cosEvents.emit.mockClear();
+ const results = await Promise.all([sweepRuntimeVerifications(), sweepRuntimeVerifications()]);
+ expect(results.flatMap(r => r.resumed)).toEqual(['sys-example']);
+ const task = await getTaskById('sys-example');
+ expect(task).toMatchObject({ status: 'pending', metadata: { failureCount: '2', totalSpawnCount: '4', orphanRetryCount: '1',
+ existingBranch: 'claim/issue-123', resumeWorktreePath: '/example/worktree',
+ runtimeVerification: { status: 'resumed', integratedCommit: 'c'.repeat(40), bootCommit: facts.boot } } });
+ expect(cosEvents.emit.mock.calls.filter(([name, e]) => name === 'tasks:changed' && e.action === 'unblocked')).toHaveLength(1);
+ __resetTaskCache();
+ expect(await sweepRuntimeVerifications()).toEqual({ resumed: [] });
+ });
+
+ it.each(['unknown-boot', 'unreadable-forge', 'changed-integration', 'live-owner', 'unknown-owners', 'changed-dispatch'])('retains the hold for %s evidence', async kind => {
+ await park(); facts.ancestor = true;
+ if (kind === 'unknown-boot') facts.boot = null;
+ if (kind === 'unreadable-forge') facts.forge = false;
+ if (kind === 'changed-integration') facts.integrated = 'e'.repeat(40);
+ if (kind === 'live-owner') facts.running = true;
+ if (kind === 'unknown-owners') facts.trusted = false;
+ if (kind === 'changed-dispatch') await updateTask('sys-example', { metadata: { totalSpawnCount: 5 } }, 'internal');
+ expect(await sweepRuntimeVerifications()).toEqual({ resumed: [] });
+ expect((await getTaskById('sys-example')).status).toBe('blocked');
+ });
+
+ it('recovers the original resume-pointer handoff after restart before releasing a verified boot', async () => {
+ await park({ ready: false }); facts.ancestor = true;
+ expect(await sweepRuntimeVerifications()).toEqual({ resumed: [] });
+ expect(await sweepRuntimeVerifications()).toEqual({ resumed: ['sys-example'] });
+ });
+
+ it.each(['unmerged', 'wrong-commit', 'foreign-repository'])('does not accept %s assertions as merged-fix evidence', async kind => {
+ const request = outcome();
+ if (kind === 'foreign-repository') request.recovery.prUrl = 'https://github.com/another/project/pull/123';
+ if (kind === 'wrong-commit') request.recovery.requiredCommit = 'f'.repeat(40);
+ if (kind === 'unmerged') execGh.mockResolvedValueOnce(JSON.stringify({ state: 'OPEN', mergeCommit: null }));
+ const update = await resolveRuntimeVerificationUpdate({ id: 'sys-example', metadata: {} }, request, 'agent-example');
+ expect(update).toMatchObject({ status: 'blocked', metadata: { blockedReason: expect.stringContaining('unreadable or invalid') } });
+ expect(update.metadata.runtimeVerification.integratedCommit).toBeUndefined();
+ });
+
+ it('retains unreadable integration as a prerequisite without spending another run, then verifies it afresh', async () => {
+ facts.forge = false; await park(); facts.ancestor = true;
+ expect((await getTaskById('sys-example')).metadata.blockedReason).toContain('unreadable or invalid');
+ expect(await sweepRuntimeVerifications()).toEqual({ resumed: [] });
+ facts.forge = true;
+ expect(await sweepRuntimeVerifications()).toEqual({ resumed: ['sys-example'] });
+ expect(execGh).toHaveBeenCalled();
+ });
+});