Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: investigate blocked system delivery without resetting budgets (#13541)
picodot258b33280cfb
diff --git a/client/src/components/cos/BlockedSystemRecovery.jsx b/client/src/components/cos/BlockedSystemRecovery.jsxnew file mode 100644index 0000000000..a8f74429ac--- /dev/null+++ b/client/src/components/cos/BlockedSystemRecovery.jsx@@ -0,0 +1,20 @@+import { Link } from 'react-router';+import { formatDateTime } from '../../utils/formatters';++const messages = {+ creating: 'Preparing an automatic investigation.',+ queued: 'An automatic investigation is queued.',+ investigating: 'An agent is investigating the cause.',+ waiting: 'Automatic investigation is waiting for its retry window.',+ diagnosed: 'The investigation finished. Original work remains stopped under its existing limits.',+ 'investigation-blocked': 'The investigation stopped within its configured limits. Original work is preserved.',+ 'budget-exhausted': 'Automatic investigation reached its configured limit. Original work is preserved.',+};++export default function BlockedSystemRecovery({ recovery }) {+ return <div className="text-sm text-port-text-muted space-y-1" aria-label="Automatic investigation">+ <p>{messages[recovery.phase] || 'Automatic investigation status is unavailable.'}</p>+ {recovery.retryAt && <p>Next check: {formatDateTime(recovery.retryAt)}</p>}+ {recovery.taskId && <Link className="text-port-accent underline" to={`/cos/tasks?task=${encodeURIComponent(recovery.taskId)}`}>View automatic investigation</Link>}+ </div>;+}diff --git a/client/src/components/cos/tabs/TaskItem.jsx b/client/src/components/cos/tabs/TaskItem.jsxindex 317f0e2941..469051e5e7 100644--- a/client/src/components/cos/tabs/TaskItem.jsx+++ b/client/src/components/cos/tabs/TaskItem.jsx@@ -37,6 +37,7 @@ import RelaunchAgentModal from './RelaunchAgentModal';import AgentCard from './AgentCard';import PreflightProgress from './PreflightProgress';import DeliveryStatus from '../DeliveryStatus';+import BlockedSystemRecovery from '../BlockedSystemRecovery';import { isDeliveryCleanupHeld } from '../../../../../server/lib/taskRetryHold.js';const statusIcons = {@@ -685,7 +686,15 @@ export default function TaskItem({ task, agent = null, liveOutput, isSystem, spaback to `blockedReason` — the field every server-side blockwrites (max-spawns, retries, provider-config, terminated, …), sowithout this fallback an auto-blocked task shows no reason at all. */}- {task.status === 'blocked' && (task.metadata?.blocker || task.metadata?.blockedReason) && (+ {task.status === 'blocked' && !task.metadata?.blocker+ && task.metadata?.blockedInvestigation?.sourceCategory === task.metadata?.blockedCategory+ && task.metadata?.blockedInvestigation?.sourceBlockedAt === (task.metadata?.blockedAt || task.metadata?.lastSpawnedAt || 'legacy') ? <div className="mt-2">+ <BlockedSystemRecovery recovery={task.metadata.blockedInvestigation} />+ {task.metadata.blockedReason && <details className="mt-1">+ <summary className="cursor-pointer text-sm text-port-text-muted">Original failure details</summary>+ <CollapsibleText id={`task-blocker-${idScope}-${task.id}`} text={task.metadata.blockedReason} className="text-port-error/90" />+ </details>}+ </div> : task.status === 'blocked' && (task.metadata?.blocker || task.metadata?.blockedReason) && (<div className="flex items-start gap-2 mt-2 px-2 py-1.5 bg-port-error/10 border border-port-error/20 rounded text-sm"><AlertCircle size={14} className="text-port-error shrink-0 mt-0.5" aria-hidden="true" />{/* Clamped: server-side auto-blocks write `blockedReason` fromdiff --git a/client/src/components/cos/tabs/TaskItem.test.jsx b/client/src/components/cos/tabs/TaskItem.test.jsxindex e048e7d69a..0ac4ebb2be 100644--- a/client/src/components/cos/tabs/TaskItem.test.jsx+++ b/client/src/components/cos/tabs/TaskItem.test.jsx@@ -1,4 +1,5 @@import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';+import { MemoryRouter } from 'react-router';import { fireEvent, render, screen, waitFor } from '@testing-library/react';const api = vi.hoisted(() => ({@@ -214,6 +215,20 @@ describe('TaskItem task source', () => {});describe('TaskItem blocked reason', () => {+ it.each([+ ['queued', 'An automatic investigation is queued.'],+ ['investigating', 'An agent is investigating the cause.'],+ ['investigation-blocked', 'The investigation stopped within its configured limits. Original work is preserved.'],+ ])('shows actual %s recovery ownership while retaining the original failure', (phase, message) => {+ const task = { id: 'system-recovery', description: 'Original work', status: 'blocked', metadata: {+ blockedCategory: 'task-recovery-exhausted', blockedAt: '2026-01-01', blockedReason: 'Original incomplete delivery',+ blockedInvestigation: { phase, taskId: 'investigator', sourceCategory: 'task-recovery-exhausted', sourceBlockedAt: '2026-01-01' } } };+ render(<MemoryRouter><TaskItem task={task} isSystem onRefresh={vi.fn()} providers={providers} /></MemoryRouter>);+ expect(screen.getByText(message)).toBeInTheDocument();+ expect(screen.getByRole('link', { name: 'View automatic investigation' })).toHaveAttribute('href', '/cos/tasks?task=investigator');+ expect(screen.getByText('Original incomplete delivery')).toBeInTheDocument();+ });+it('renders blockedReason when a blocked task has no user-set blocker', () => {// Every server-side auto-block (max-spawns, retries, provider-config, …) writes// metadata.blockedReason, never `blocker`, so the display must fall back to it.diff --git a/server/lib/investigationTasks.js b/server/lib/investigationTasks.jsindex ccb9321c05..b5ee874944 100644--- a/server/lib/investigationTasks.js+++ b/server/lib/investigationTasks.js@@ -361,6 +361,7 @@ export function autoRetryCount(task) {export function couldReleaseBlockedTasks(investigation) {return isInvestigationTask(investigation)&& investigation.status === 'completed'+ && !isTruthyMeta(investigation.metadata?.preserveOriginalBudget)&& investigation.metadata?.resolution !== 'auto-expired'&& affectedTaskIds(investigation).length > 0;}diff --git a/server/services/agentErrorAnalysis.js b/server/services/agentErrorAnalysis.jsindex badbe30ef5..acb0641973 100644--- a/server/services/agentErrorAnalysis.js+++ b/server/services/agentErrorAnalysis.js@@ -24,6 +24,7 @@ import {INVESTIGATION_CIRCUIT_MAX_CREATIONS,INVESTIGATION_HEADLINE_PREFIX,INVESTIGATION_TASK_DELIVERY,+ SUPERVISED_INVESTIGATION_DELIVERY,LOOP_REASON_PROSE,MAX_AUTO_RETRIES_PER_TASK,buildInvestigationFingerprint,@@ -1344,6 +1345,16 @@ async function doCreateInvestigationTask(agentId, originalTask, errorAnalysis) {const { approvalRequired, loopReason, approvalReason } = resolveInvestigationApproval({fingerprint, tasks: allTasks, recentCreations: recentInvestigationCreations(now), now});+ // The blocked-system watcher owns durable backoff; a repeat failure must not+ // allocate another blocked task asking the user to supervise machine recovery.+ const systemRecovery = analysis.blockedSystemRecovery === true;+ if (systemRecovery && approvalRequired) return null;+ let targetInstanceId;+ if (systemRecovery) {+ const { getInstanceId, UNKNOWN_INSTANCE_ID } = await import('./instanceIdentity.js');+ targetInstanceId = await getInstanceId();+ if (!targetInstanceId || targetInstanceId === UNKNOWN_INSTANCE_ID) return null;+ }// Every interpolated free-text field is redacted before it lands in the body —// this task is human-facing and may sync across federated peers, so no@@ -1369,11 +1380,15 @@ async function doCreateInvestigationTask(agentId, originalTask, errorAnalysis) {`## What to approve\n${remedy}`,`## Why this is held for you\n${LOOP_REASON_PROSE[loopReason]}`].join('\n\n')- : `## What to do\nNo approval needed — this investigation runs unattended. Diagnose the failure above, apply the fix, and leave the task's findings in your summary. Do NOT un-block the original task by hand; completing this one does it.\n\n${remedy}`;+ : systemRecovery+ ? `## What to do\nInvestigate and repair the machine cause with required review and CI. Preserve original work, approvals and retry/spawn budgets. Do not requeue or mark the original task complete by hand.\n\n${remedy}`+ : `## What to do\nNo approval needed — this investigation runs unattended. Diagnose the failure above, apply the fix, and leave the task's findings in your summary. Do NOT un-block the original task by hand; completing this one does it.\n\n${remedy}`;// Completing this task revives every blocked task in `affectedTasks`// automatically (investigationRetry.js), so nobody has to go find them.- const unblocks = `${approvalRequired ? 'Approving and applying' : 'Applying'} the fix and completing this task automatically retries the original task \`${originalTask.id}\`; it will resume: ${originalDesc}. If the same cause blocks it again after ${MAX_AUTO_RETRIES_PER_TASK} automatic retries, it stays blocked for you.`;+ const unblocks = systemRecovery+ ? `This investigation does not refund or reset task \`${originalTask.id}\` budgets. Its existing delivery verifier may settle proven completion; unfinished work remains stopped under its existing limits. Report the actual remaining work or user-owned decision.`+ : `${approvalRequired ? 'Approving and applying' : 'Applying'} the fix and completing this task automatically retries the original task \`${originalTask.id}\`; it will resume: ${originalDesc}. If the same cause blocks it again after ${MAX_AUTO_RETRIES_PER_TASK} automatic retries, it stays blocked for you.`;// The fingerprint rides in the headline so addTask's first-line dedup —// which sees no `metadata.app` on investigation tasks — tracks fingerprint@@ -1396,7 +1411,8 @@ ${actionBlock}${unblocks}`;const investigationTask = await addTask({- ...INVESTIGATION_TASK_DELIVERY,+ ...(systemRecovery ? SUPERVISED_INVESTIGATION_DELIVERY : INVESTIGATION_TASK_DELIVERY),+ ...(systemRecovery && { targetInstanceId, metadata: { preserveOriginalBudget: true, blockedSystemRecovery: true } }),description,priority: 'HIGH',context: `Auto-generated from agent ${agentId} failure`,diff --git a/server/services/agentErrorAnalysis.test.js b/server/services/agentErrorAnalysis.test.jsindex a9aef44f56..383d98e32f 100644--- a/server/services/agentErrorAnalysis.test.js+++ b/server/services/agentErrorAnalysis.test.js@@ -3,6 +3,7 @@ import { describe, expect, it, vi, beforeEach } from 'vitest';// maybeCreateInvestigationTask → createInvestigationTask → addTask does real// store I/O; stub it so the skip-vs-create gate can be asserted in isolation.// getAllTasks feeds the fingerprint dedup scan (#2615).+vi.mock('./instanceIdentity.js', () => ({ getInstanceId: vi.fn().mockResolvedValue('fixture-local'), UNKNOWN_INSTANCE_ID: 'unknown' }));vi.mock('./cosTaskStore.js', () => ({ addTask: vi.fn(), updateTask: vi.fn(), getAllTasks: vi.fn() }));import {@@ -997,6 +998,22 @@ describe('createInvestigationTask approval wiring (#3714)', () => {__resetInvestigationCircuit();});+ it('creates local supervised diagnosis without refunding exhausted original work', async () => {+ await createInvestigationTask('agent-original', { ...failedTask, taskType: 'internal' },+ { ...analysis, blockedSystemRecovery: true });+ expect(addTask.mock.calls[0][0]).toMatchObject({ targetInstanceId: 'fixture-local',+ metadata: { preserveOriginalBudget: true, blockedSystemRecovery: true } });+ expect(addTask.mock.calls[0][0].description).toContain('existing');+ });++ it('defers repeated system diagnosis instead of creating another blocked approval card', async () => {+ getAllTasks.mockResolvedValue({ user: { tasks: [] }, cos: { tasks: [{ id: 'prior', status: 'completed', metadata: {+ investigationFingerprint: 'startup-failure:internal:none', updatedAt: new Date().toISOString() } }] } });+ expect(await createInvestigationTask('agent-original', { ...failedTask, taskType: 'internal' },+ { ...analysis, blockedSystemRecovery: true })).toBeNull();+ expect(addTask).not.toHaveBeenCalled();+ });+it('runs unattended up to the storm threshold, then holds and records the loop reason', async () => {// Distinct categories so each call mints its own fingerprint and reaches// the circuit rather than deduping into the previous investigation.diff --git a/server/services/blockedSystemInvestigation.js b/server/services/blockedSystemInvestigation.jsnew file mode 100644index 0000000000..5e3741d89a--- /dev/null+++ b/server/services/blockedSystemInvestigation.js@@ -0,0 +1,147 @@+/** Reuse the existing investigator for a locally executed system failure.+ * This diagnoses the cause; it never revives work or refunds its retry budget. */+import { buildInvestigationFingerprint, isInvestigationTask } from '../lib/investigationTasks.js';+import { USER_DECISION_BLOCKED_CATEGORIES, PAUSED_BLOCKED_CATEGORIES } from '../lib/taskBlockCategories.js';+import { explicitPrCompletion, PR_COMPLETIONS } from '../lib/prDisposition.js';+import { isPrivateSecurityTask } from '../lib/privateSecurityPolicy.js';+import { isTruthyMeta } from '../lib/metadataFlags.js';++const BACKOFF_MS = 60000;+const MAX_BACKOFF_MS = 3600000;+const humanBoundaries = new Set(['auth-error', 'forbidden', 'permission-denied', 'permission-error', 'approval-required',+ 'tool-approval', 'access-denied', 'security-refusal', 'sandbox-denied', 'content-refusal', 'content-filtered',+ 'quota-exhausted', 'usage-limit', 'budget-exceeded', 'spending-limit', 'credits-exhausted', 'private-security-assessment-failed']);+const flatten = queues => [...(queues.cos?.tasks || []).map(task => ({ ...task, taskType: 'internal' })),+ ...(queues.user?.tasks || []).map(task => ({ ...task, taskType: 'user' }))];+const eligible = task => {+ const m = task?.metadata || {};+ const category = m.blockedCategory;+ return task?.taskType === 'internal' && task.status === 'blocked' && typeof category === 'string'+ && !isInvestigationTask(task) && !isPrivateSecurityTask(task) && !isTruthyMeta(task.approvalRequired)+ && !(typeof m.blocker === 'string' && m.blocker.trim())+ && m.recoveryOrigin?.subsystem !== 'repository-cleanup'+ && m.taskOutcome?.recovery?.mode !== 'approval'+ && explicitPrCompletion(task) !== PR_COMPLETIONS.LEAVE_OPEN+ && (!USER_DECISION_BLOCKED_CATEGORIES.has(category) || category === 'task-recovery-exhausted')+ && (!PAUSED_BLOCKED_CATEGORIES.has(category) || category === 'task-recovery-exhausted')+ && ![category, m.lastErrorCategory].some(value => typeof value === 'string' && humanBoundaries.has(value.toLowerCase()));+};+const analysisFor = task => ({+ category: `blocked-system-${task.metadata.blockedCategory}-${task.metadata.lastErrorCategory || task.metadata.taskOutcome?.status || 'failure'}`,+ message: 'A system task stopped before its work reached verified completion.',+ snippet: task.metadata.blockedReason || task.metadata.taskOutcome?.reason,+ blockedSystemRecovery: true,+ preserveOriginalBudget: true,+ escalation: 'Re-read the original task, current owners and delivery evidence first. Reuse active fixes. Diagnose and repair the underlying machine/product failure in an isolated checkout with required review, CI and merge admission. Preserve unfinished work and original failure/spawn budgets. Do not requeue, reset, or mark the original task completed by hand; its existing verifier and budget rules own continuation. Do not retry denied operations, change credentials, spend limits or permissions, or bypass a genuine human approval. Report actual human-required boundaries on the original record; machine-recoverable failures must not create personal Brain/todo actions.',+});+const phaseFor = task => task.status === 'in_progress' ? 'investigating'+ : task.status === 'completed' ? 'diagnosed'+ : task.status === 'blocked' || isTruthyMeta(task.approvalRequired) ? 'investigation-blocked' : 'queued';++async function localOwner(task) {+ const agentId = task.metadata?.deliveryDecisionAgentId || task.metadata?.lastAgentId || task.metadata?.failedAgentId;+ if (!agentId) return null;+ const [{ getAgentRecord }, { getInstanceId, UNKNOWN_INSTANCE_ID }] = await Promise.all([+ import('./cosAgentLifecycle.js'), import('./instanceIdentity.js'),+ ]);+ const [agent, instanceId] = await Promise.all([getAgentRecord(agentId), getInstanceId()]);+ return instanceId && instanceId !== UNKNOWN_INSTANCE_ID && agent?.status === 'completed'+ && agent.taskId === task.id && agent.metadata?.instanceId === instanceId+ ? { agentId, instanceId } : null;+}++/** Persistence/producer seams let lifecycle tests exercise crash and replay with+ * a real task store adapter, without running providers or touching host tasks. */+export function _createBlockedSystemInvestigationRecovery(deps = {}) {+ const now = deps.now || Date.now;+ const read = deps.read || (async () => (await import('./cosTaskStore.js')).getAllTasks());+ const ownerFor = deps.ownerFor || localOwner;+ const update = deps.update || (async (task, state) => (await import('./cosTaskStore.js')).updateTask(task.id,+ { metadata: { blockedInvestigation: state } }, 'internal', { expectedStatus: 'blocked',+ expectedTask: { description: task.description, priority: task.priority, metadata: task.metadata } }));+ const permission = deps.permission || (async (fingerprint, tasks) => {+ const producer = await import('./investigationTaskProducer.js');+ if (producer.investigationCircuitOpen(now())) return 'circuit-open';+ return (await producer.resolveAutoInvestigationApproval(fingerprint, { now: now(), tasks })).loopReason;+ });+ const create = deps.create || (async (task, analysis, owner) => (await import('./agentErrorAnalysis.js'))+ .createInvestigationTask(owner.agentId, task, analysis));+ let tail = Promise.resolve();+ const write = async (task, state) => {+ if (JSON.stringify(task.metadata?.blockedInvestigation) === JSON.stringify(state)) return task;+ const saved = await update(task, state);+ return saved?.error || saved?.statusChanged ? null : saved;+ };+ const one = async id => {+ const tasks = flatten(await read());+ let task = tasks.find(row => row.id === id && row.taskType === 'internal');+ if (!eligible(task)) return { handled: false };+ const owner = await ownerFor(task);+ if (!owner) return { handled: false, reason: 'local-owner-unverified' };+ const analysis = analysisFor(task);+ const fingerprint = buildInvestigationFingerprint(task, analysis);+ const epoch = `${fingerprint}:${task.metadata.blockedAt || task.metadata.lastSpawnedAt || 'legacy'}`;+ const old = task.metadata.blockedInvestigation;+ let state = old?.epoch === epoch ? { ...old } : { version: 1, epoch, fingerprint, attempts: 0, failures: 0,+ ownerInstanceId: owner.instanceId, sourceCategory: task.metadata.blockedCategory,+ sourceBlockedAt: task.metadata.blockedAt || task.metadata.lastSpawnedAt || 'legacy' };+ const related = tasks.filter(row => isInvestigationTask(row) && row.metadata?.investigationFingerprint === fingerprint);+ const existing = related.find(row => row.id === state.taskId)+ || related.find(row => ['pending', 'in_progress', 'blocked'].includes(row.status));+ if (existing) {+ state = { ...state, taskId: existing.id, attempts: 1, phase: phaseFor(existing), retryAt: null };+ if (!existing.metadata?.affectedTasks?.includes(task.id)) await create(task, analysis, owner);+ await write(task, state);+ return { handled: true, phase: state.phase, taskId: existing.id };+ }+ if (Number(state.version) !== 1 || !Number.isSafeInteger(Number(state.attempts)) || Number(state.attempts) < 0) {+ return { handled: false, reason: 'recovery-state-unreadable' };+ }+ // One investigator owns this blocked episode, including its own configured+ // retries. Losing/failing that investigator does not mint a fresh budget.+ if (Number(state.attempts) >= 1) {+ await write(task, { ...state, phase: 'budget-exhausted', retryAt: null });+ return { handled: true, phase: 'budget-exhausted' };+ }+ if (Number.isFinite(Date.parse(state.retryAt)) && Date.parse(state.retryAt) > now()) return { handled: true, phase: 'waiting' };+ const defer = async reason => {+ const failures = Math.min(16, Math.max(0, Number(state.failures) || 0) + 1);+ await write(task, { ...state, phase: 'waiting', failures, reason,+ retryAt: new Date(now() + Math.min(MAX_BACKOFF_MS, BACKOFF_MS * 2 ** (failures - 1))).toISOString() });+ return { handled: true, phase: 'waiting', reason };+ };+ const held = await permission(fingerprint, tasks);+ if (held) return defer(held);+ task = await write(task, { ...state, phase: 'creating', retryAt: new Date(now() + BACKOFF_MS).toISOString() });+ if (!task) return { handled: false, reason: 'task-changed' };+ // An explicit user/owner transition during preparation wins over recovery.+ const current = flatten(await read()).find(row => row.id === task.id && row.taskType === 'internal');+ if (!eligible(current) || current.metadata.blockedInvestigation?.epoch !== epoch) return { handled: false, reason: 'task-changed' };+ task = current;+ const created = await create(task, analysis, owner).catch(() => null);+ if (!created?.id) return defer('producer-unavailable');+ state = { ...state, attempts: 1, taskId: created.id, phase: phaseFor(created), retryAt: null };+ await write(task, state);+ return { handled: true, phase: state.phase, taskId: created.id };+ };+ const recover = id => {+ const result = tail.then(() => one(id));+ tail = result.catch(() => {});+ return result;+ };+ let cursor = 0;+ const sweep = async () => {+ const all = flatten(await read()).filter(eligible);+ const start = all.length ? cursor % all.length : 0;+ const candidates = [...all.slice(start), ...all.slice(0, start)].slice(0, 3);+ cursor = all.length ? (start + candidates.length) % all.length : 0;+ const results = [];+ for (const task of candidates) results.push(await recover(task.id));+ return results;+ };+ return { recover, sweep };+}++const recovery = _createBlockedSystemInvestigationRecovery();+export const recoverBlockedSystemTask = taskId => recovery.recover(taskId);+export const recoverBlockedSystemTasks = () => recovery.sweep();diff --git a/server/services/blockedSystemInvestigation.test.js b/server/services/blockedSystemInvestigation.test.jsnew file mode 100644index 0000000000..96e1c507b1--- /dev/null+++ b/server/services/blockedSystemInvestigation.test.js@@ -0,0 +1,171 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';+import { join } from 'node:path';+import { tmpdir } from 'node:os';+import { _createBlockedSystemInvestigationRecovery } from './blockedSystemInvestigation.js';+import { resolveUndeliveredTaskUpdate, MAX_TASK_RETRIES } from './agentErrorAnalysis.js';+import { buildInvestigationFingerprint, couldReleaseBlockedTasks } from '../lib/investigationTasks.js';++const identity = vi.hoisted(() => ({ agent: null }));+vi.mock('./cosAgentLifecycle.js', () => ({ getAgentRecord: vi.fn(async () => identity.agent) }));+vi.mock('./instanceIdentity.js', () => ({ getInstanceId: vi.fn(async () => 'fixture-local'), UNKNOWN_INSTANCE_ID: 'unknown' }));++let root, path, time, created, calls, hold, failAfterCreate, race, failLink;+const read = async () => JSON.parse(await readFile(path, 'utf8'));+const write = queues => writeFile(path, JSON.stringify(queues));+const source = () => ({ id: 'system-original', taskType: 'internal', status: 'in_progress', priority: 'MEDIUM',+ description: 'Finish retained claims, including unfinished source work.', metadata: { app: 'fixture', analysisType: 'claim-issue',+ failureCount: String(MAX_TASK_RETRIES - 1), totalSpawnCount: '1' } });+const blocked = () => {+ const task = source();+ const update = resolveUndeliveredTaskUpdate(task, { status: 'incomplete', reason: 'Merged delivery still has unfinished siblings.' }, 'agent-original', time);+ return { ...task, ...update };+};+const create = async (task, analysis) => {+ calls++;+ expect(analysis.preserveOriginalBudget).toBe(true);+ const queues = await read();+ const fingerprint = buildInvestigationFingerprint(task, analysis);+ let child = queues.cos.tasks.find(row => row.metadata?.investigationFingerprint === fingerprint && row.status !== 'completed');+ if (!child) {+ created++;+ child = { id: `investigator-${created}`, status: 'pending', taskType: 'internal', metadata: {+ isInvestigation: true, preserveOriginalBudget: true, investigationFingerprint: fingerprint, affectedTasks: [] } };+ queues.cos.tasks.push(child);+ }+ child.metadata.affectedTasks = [...new Set([...child.metadata.affectedTasks, task.id])];+ await write(queues);+ if (failAfterCreate) { failAfterCreate = false; throw new Error('interrupted after durable creation'); }+ return child;+};+const update = async (task, state) => {+ const queues = await read();+ const current = queues.cos.tasks.find(row => row.id === task.id);+ if (current.status !== 'blocked' || JSON.stringify(current.metadata) !== JSON.stringify(task.metadata)) return { statusChanged: true };+ if (failLink && state.taskId) { failLink = false; throw new Error('link interrupted'); }+ current.metadata = { ...current.metadata, blockedInvestigation: state };+ if (race && state.phase === 'creating') { race = false; current.status = 'completed'; }+ await write(queues);+ return current;+};+const factory = (realOwner = false) => _createBlockedSystemInvestigationRecovery({ read, update, create, now: () => time,+ ...(!realOwner && { ownerFor: async task => task.metadata?.foreign ? null : { agentId: 'agent-original', instanceId: 'fixture-local' } }),+ permission: async () => hold });++beforeEach(async () => {+ root = await mkdtemp(join(tmpdir(), 'blocked-investigation-'));+ path = join(root, 'queues.json');+ time = Date.parse('2026-01-01T00:00:00.000Z');+ created = calls = 0; hold = null; failAfterCreate = race = failLink = false;+ await write({ user: { tasks: [] }, cos: { tasks: [blocked()] } });+});+afterEach(async () => { await rm(root, { recursive: true, force: true }); });++describe('blocked system recovery persisted workflow', () => {+ it('investigates exhausted incomplete delivery once, groups duplicate observations and preserves original budgets', async () => {+ const recovery = factory();+ await Promise.all([recovery.recover('system-original'), recovery.recover('system-original')]);+ let queues = await read();+ const original = queues.cos.tasks.find(row => row.id === 'system-original');+ expect(original.status).toBe('blocked');+ expect(original.metadata).toMatchObject({ failureCount: MAX_TASK_RETRIES, totalSpawnCount: '1',+ taskOutcome: { status: 'incomplete' }, blockedInvestigation: { phase: 'queued', attempts: 1, taskId: 'investigator-1' } });+ expect(original.description).toBe(source().description);+ queues.cos.tasks.push({ ...blocked(), id: 'same-cause', metadata: { ...blocked().metadata } });+ await write(queues);+ await factory().recover('same-cause');+ queues = await read();+ expect(created).toBe(1);+ expect(queues.cos.tasks.find(row => row.id === 'investigator-1').metadata.affectedTasks).toEqual(['system-original', 'same-cause']);+ await factory().sweep();+ expect(created).toBe(1);+ });++ it.each(['creation', 'link'])('recovers from a crash after durable %s without minting another investigator', async cut => {+ failAfterCreate = cut === 'creation'; failLink = cut === 'link';+ await factory().recover('system-original').catch(() => {});+ await factory().recover('system-original');+ const original = (await read()).cos.tasks.find(row => row.id === 'system-original');+ expect(created).toBe(1);+ expect(original.metadata.blockedInvestigation).toMatchObject({ phase: 'queued', attempts: 1, taskId: 'investigator-1' });+ });++ it('backs off a loop/circuit without creating a blocked approval task, and resumes after the persisted window', async () => {+ hold = 'repeat-fingerprint';+ await factory().recover('system-original');+ let state = (await read()).cos.tasks[0].metadata.blockedInvestigation;+ expect(state.phase).toBe('waiting');+ expect(Date.parse(state.retryAt)).toBeGreaterThan(time);+ hold = null;+ await factory().recover('system-original');+ expect(calls).toBe(0);+ time = Date.parse(state.retryAt) + 1;+ await factory().recover('system-original');+ expect(created).toBe(1);+ });++ it('keeps an investigator failure with its existing owner and never creates a successor or auto-revives originals', async () => {+ await factory().recover('system-original');+ const queues = await read();+ const investigator = queues.cos.tasks.find(row => row.id === 'investigator-1');+ investigator.status = 'blocked';+ investigator.metadata.blockedCategory = 'task-recovery-exhausted';+ await write(queues);+ await factory().sweep();+ expect(created).toBe(1);+ const original = (await read()).cos.tasks.find(row => row.id === 'system-original');+ expect(original.metadata.blockedInvestigation.phase).toBe('investigation-blocked');+ expect(original.metadata.failureCount).toBe(MAX_TASK_RETRIES);+ expect(couldReleaseBlockedTasks({ ...investigator, status: 'completed' })).toBe(false);+ expect(couldReleaseBlockedTasks({ ...investigator, status: 'completed', metadata: { ...investigator.metadata, preserveOriginalBudget: 'true' } })).toBe(false);+ });++ it.each(['task-approval', 'agent-paused', 'provider-config', 'auth-error', 'usage-limit', 'content-refusal', 'worktree-busy', 'foreign-owner', 'cleanup-owner', 'user-blocker', 'user-task'])(+ 'preserves the real %s boundary without filing another task', async boundary => {+ const task = blocked();+ if (boundary === 'foreign-owner') task.metadata.foreign = true;+ else if (boundary === 'cleanup-owner') task.metadata.recoveryOrigin = { subsystem: 'repository-cleanup' };+ else if (boundary === 'user-blocker') task.metadata.blocker = 'Wait for my decision.';+ else task.metadata.blockedCategory = boundary;+ const queues = { user: { tasks: boundary === 'user-task' ? [task] : [] }, cos: { tasks: boundary === 'user-task' ? [] : [task] } };+ await write(queues);+ await factory().sweep();+ expect(await read()).toEqual(queues);+ expect(created).toBe(0);+ });++ it('does not starve later failures behind already-owned investigations', async () => {+ const queues = await read();+ queues.cos.tasks = Array.from({ length: 5 }, (_, i) => ({ ...blocked(), id: `source-${i}` }));+ await write(queues);+ const recovery = factory();+ await recovery.sweep();+ await recovery.sweep();+ const originals = (await read()).cos.tasks.filter(task => task.id.startsWith('source-'));+ expect(originals.every(task => task.metadata.blockedInvestigation?.taskId === 'investigator-1')).toBe(true);+ expect(created).toBe(1);+ });++ it('requires the actual completed local agent record before owning a historical blocked task', async () => {+ const queues = await read();+ queues.cos.tasks[0].metadata.deliveryDecisionAgentId = 'agent-original';+ await write(queues);+ const good = { status: 'completed', taskId: 'system-original', metadata: { instanceId: 'fixture-local' } };+ for (const record of [null, { ...good, status: 'running' }, { ...good, taskId: 'other-task' },+ { ...good, metadata: { instanceId: 'another-instance' } }]) {+ identity.agent = record;+ expect(await factory(true).recover('system-original')).toMatchObject({ handled: false, reason: 'local-owner-unverified' });+ expect(created).toBe(0);+ }+ identity.agent = good;+ expect(await factory(true).recover('system-original')).toMatchObject({ handled: true, taskId: 'investigator-1' });+ expect(created).toBe(1);+ });++ it('lets a user/owner transition win before creation', async () => {+ race = true;+ expect(await factory().recover('system-original')).toMatchObject({ handled: false, reason: 'task-changed' });+ expect(created).toBe(0);+ expect((await read()).cos.tasks[0].status).toBe('completed');+ });+});diff --git a/server/services/cos.js b/server/services/cos.jsindex 8c20172352..d75d098dbb 100644--- a/server/services/cos.js+++ b/server/services/cos.js@@ -408,6 +408,8 @@ async function bootDaemon(state) {emitLog('info', `🧹 Periodic cleanup: ${cleaned} orphaned agent(s)`);}await resetOrphanedTasks();+ const { recoverBlockedSystemTasks } = await import('./blockedSystemInvestigation.js');+ await recoverBlockedSystemTasks().catch(err => console.error(`❌ Blocked-system investigation recovery failed: ${err.message}`));// Evaluation is event-driven, so without this cadence a timed pause// (`worktree-busy`, `orphan-cooldown`) whose `cooldownUntil` lapsed would// sit blocked until the next restart or manual evaluate.@@ -1532,6 +1534,12 @@ export async function init() {await settleBranchReconcileRun(agent);}).catch(err => console.error(`❌ Branch reconciliation completion failed: ${err.message}`))+ .then(async () => {+ if (!isDaemonRunning() || !agent?.taskId) return;+ const { recoverBlockedSystemTask } = await import('./blockedSystemInvestigation.js');+ await recoverBlockedSystemTask(agent.taskId);+ })+ .catch(err => console.error(`❌ Blocked-system investigation recovery failed: ${err.message}`)).then(() => refillPerpetualForCompletedAgent(agent)).catch(err => console.error(`❌ Perpetual refill after ${agent?.id} failed: ${err.message}`))// Same window as the refill above: this runs before the completing@@ -1634,6 +1642,10 @@ export async function init() {cosEvents.on('tasks:changed', (data) => {handBackBlockedPrRemediation(data).catch(err => console.error(`❌ Blocked PR remediation hand-back failed for task ${data?.task?.id}: ${err.message}`));+ if (isDaemonRunning() && data?.type === 'internal' && data.task?.status === 'blocked') {+ import('./blockedSystemInvestigation.js').then(({ recoverBlockedSystemTask }) => recoverBlockedSystemTask(data.task.id))+ .catch(err => console.error(`❌ Blocked-system investigation recovery failed: ${err.message}`));+ }});// A critical health transition reaches the bell even when no CoS view is open.diff --git a/server/services/cos.test.js b/server/services/cos.test.jsindex df5c301c64..e4aa9855d7 100644--- a/server/services/cos.test.js+++ b/server/services/cos.test.js@@ -2554,7 +2554,7 @@ describe('cos.js source — agent:completed triggers perpetual refill', () => {// freed slot with idle-review work before the perpetual task is queued,// breaking the back-to-back drain. The handler must chain refill → dequeue.const onIdx = COS_SRC.indexOf("cosEvents.on('agent:completed'");- const handlerSlice = COS_SRC.slice(onIdx, onIdx + 1400);+ const handlerSlice = COS_SRC.slice(onIdx, COS_SRC.indexOf('// Create notification when a daily briefing completes', onIdx));expect(/refillPerpetualForCompletedAgent\(agent\)[\s\S]*\.then\(\s*\(\)\s*=>\s*dequeueNextTask\(/.test(handlerSlice),'handler must run dequeueNextTask in a .then() AFTER the refill resolves'diff --git a/server/services/investigationRetry.test.js b/server/services/investigationRetry.test.jsindex 31a4397eac..aee3e36184 100644--- a/server/services/investigationRetry.test.js+++ b/server/services/investigationRetry.test.js@@ -25,6 +25,13 @@ describe('retryTasksResolvedByInvestigation', () => {updateTask.mockResolvedValue({ id: 'ok' });});+ it('does not refund original budgets after a diagnostic-only system investigation', async () => {+ expect(await retryTasksResolvedByInvestigation(investigation({ preserveOriginalBudget: true })))+ .toEqual({ retried: [], skipped: [] });+ expect(getAllTasks).not.toHaveBeenCalled();+ expect(reviveBlockedTask).not.toHaveBeenCalled();+ });+it('revives the blocked task the investigation was diagnosing, stamping the auto-retry budget', async () => {getAllTasks.mockResolvedValue(queues({user: [{ id: 'task-1', status: 'blocked', metadata: { blockedCategory: 'test-failure' } }]