Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: investigate blocked system delivery without resetting budgets (#13541)

picodot258b33280cfb
diff --git a/client/src/components/cos/BlockedSystemRecovery.jsx b/client/src/components/cos/BlockedSystemRecovery.jsx
new file mode 100644
index 0000000000..a8f74429ac
--- /dev/null
+++ b/client/src/components/cos/BlockedSystemRecovery.jsx
@@ -0,0 +1,20 @@
+import { Link } from 'react-router';
+import { formatDateTime } from '../../utils/formatters';
+
+const messages = {
+ creating: 'Preparing an automatic investigation.',
+ queued: 'An automatic investigation is queued.',
+ investigating: 'An agent is investigating the cause.',
+ waiting: 'Automatic investigation is waiting for its retry window.',
+ diagnosed: 'The investigation finished. Original work remains stopped under its existing limits.',
+ 'investigation-blocked': 'The investigation stopped within its configured limits. Original work is preserved.',
+ 'budget-exhausted': 'Automatic investigation reached its configured limit. Original work is preserved.',
+};
+
+export default function BlockedSystemRecovery({ recovery }) {
+ return <div className="text-sm text-port-text-muted space-y-1" aria-label="Automatic investigation">
+ <p>{messages[recovery.phase] || 'Automatic investigation status is unavailable.'}</p>
+ {recovery.retryAt && <p>Next check: {formatDateTime(recovery.retryAt)}</p>}
+ {recovery.taskId && <Link className="text-port-accent underline" to={`/cos/tasks?task=${encodeURIComponent(recovery.taskId)}`}>View automatic investigation</Link>}
+ </div>;
+}
diff --git a/client/src/components/cos/tabs/TaskItem.jsx b/client/src/components/cos/tabs/TaskItem.jsx
index 317f0e2941..469051e5e7 100644
--- a/client/src/components/cos/tabs/TaskItem.jsx
+++ b/client/src/components/cos/tabs/TaskItem.jsx
@@ -37,6 +37,7 @@ import RelaunchAgentModal from './RelaunchAgentModal';
import AgentCard from './AgentCard';
import PreflightProgress from './PreflightProgress';
import DeliveryStatus from '../DeliveryStatus';
+import BlockedSystemRecovery from '../BlockedSystemRecovery';
import { isDeliveryCleanupHeld } from '../../../../../server/lib/taskRetryHold.js';
const statusIcons = {
@@ -685,7 +686,15 @@ export default function TaskItem({ task, agent = null, liveOutput, isSystem, spa
back to `blockedReason` — the field every server-side block
writes (max-spawns, retries, provider-config, terminated, …), so
without this fallback an auto-blocked task shows no reason at all. */}
- {task.status === 'blocked' && (task.metadata?.blocker || task.metadata?.blockedReason) && (
+ {task.status === 'blocked' && !task.metadata?.blocker
+ && task.metadata?.blockedInvestigation?.sourceCategory === task.metadata?.blockedCategory
+ && task.metadata?.blockedInvestigation?.sourceBlockedAt === (task.metadata?.blockedAt || task.metadata?.lastSpawnedAt || 'legacy') ? <div className="mt-2">
+ <BlockedSystemRecovery recovery={task.metadata.blockedInvestigation} />
+ {task.metadata.blockedReason && <details className="mt-1">
+ <summary className="cursor-pointer text-sm text-port-text-muted">Original failure details</summary>
+ <CollapsibleText id={`task-blocker-${idScope}-${task.id}`} text={task.metadata.blockedReason} className="text-port-error/90" />
+ </details>}
+ </div> : task.status === 'blocked' && (task.metadata?.blocker || task.metadata?.blockedReason) && (
<div className="flex items-start gap-2 mt-2 px-2 py-1.5 bg-port-error/10 border border-port-error/20 rounded text-sm">
<AlertCircle size={14} className="text-port-error shrink-0 mt-0.5" aria-hidden="true" />
{/* Clamped: server-side auto-blocks write `blockedReason` from
diff --git a/client/src/components/cos/tabs/TaskItem.test.jsx b/client/src/components/cos/tabs/TaskItem.test.jsx
index e048e7d69a..0ac4ebb2be 100644
--- a/client/src/components/cos/tabs/TaskItem.test.jsx
+++ b/client/src/components/cos/tabs/TaskItem.test.jsx
@@ -1,4 +1,5 @@
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { MemoryRouter } from 'react-router';
import { fireEvent, render, screen, waitFor } from '@testing-library/react';
const api = vi.hoisted(() => ({
@@ -214,6 +215,20 @@ describe('TaskItem task source', () => {
});
describe('TaskItem blocked reason', () => {
+ it.each([
+ ['queued', 'An automatic investigation is queued.'],
+ ['investigating', 'An agent is investigating the cause.'],
+ ['investigation-blocked', 'The investigation stopped within its configured limits. Original work is preserved.'],
+ ])('shows actual %s recovery ownership while retaining the original failure', (phase, message) => {
+ const task = { id: 'system-recovery', description: 'Original work', status: 'blocked', metadata: {
+ blockedCategory: 'task-recovery-exhausted', blockedAt: '2026-01-01', blockedReason: 'Original incomplete delivery',
+ blockedInvestigation: { phase, taskId: 'investigator', sourceCategory: 'task-recovery-exhausted', sourceBlockedAt: '2026-01-01' } } };
+ render(<MemoryRouter><TaskItem task={task} isSystem onRefresh={vi.fn()} providers={providers} /></MemoryRouter>);
+ expect(screen.getByText(message)).toBeInTheDocument();
+ expect(screen.getByRole('link', { name: 'View automatic investigation' })).toHaveAttribute('href', '/cos/tasks?task=investigator');
+ expect(screen.getByText('Original incomplete delivery')).toBeInTheDocument();
+ });
+
it('renders blockedReason when a blocked task has no user-set blocker', () => {
// Every server-side auto-block (max-spawns, retries, provider-config, …) writes
// metadata.blockedReason, never `blocker`, so the display must fall back to it.
diff --git a/server/lib/investigationTasks.js b/server/lib/investigationTasks.js
index ccb9321c05..b5ee874944 100644
--- a/server/lib/investigationTasks.js
+++ b/server/lib/investigationTasks.js
@@ -361,6 +361,7 @@ export function autoRetryCount(task) {
export function couldReleaseBlockedTasks(investigation) {
return isInvestigationTask(investigation)
&& investigation.status === 'completed'
+ && !isTruthyMeta(investigation.metadata?.preserveOriginalBudget)
&& investigation.metadata?.resolution !== 'auto-expired'
&& affectedTaskIds(investigation).length > 0;
}
diff --git a/server/services/agentErrorAnalysis.js b/server/services/agentErrorAnalysis.js
index badbe30ef5..acb0641973 100644
--- a/server/services/agentErrorAnalysis.js
+++ b/server/services/agentErrorAnalysis.js
@@ -24,6 +24,7 @@ import {
INVESTIGATION_CIRCUIT_MAX_CREATIONS,
INVESTIGATION_HEADLINE_PREFIX,
INVESTIGATION_TASK_DELIVERY,
+ SUPERVISED_INVESTIGATION_DELIVERY,
LOOP_REASON_PROSE,
MAX_AUTO_RETRIES_PER_TASK,
buildInvestigationFingerprint,
@@ -1344,6 +1345,16 @@ async function doCreateInvestigationTask(agentId, originalTask, errorAnalysis) {
const { approvalRequired, loopReason, approvalReason } = resolveInvestigationApproval({
fingerprint, tasks: allTasks, recentCreations: recentInvestigationCreations(now), now
});
+ // The blocked-system watcher owns durable backoff; a repeat failure must not
+ // allocate another blocked task asking the user to supervise machine recovery.
+ const systemRecovery = analysis.blockedSystemRecovery === true;
+ if (systemRecovery && approvalRequired) return null;
+ let targetInstanceId;
+ if (systemRecovery) {
+ const { getInstanceId, UNKNOWN_INSTANCE_ID } = await import('./instanceIdentity.js');
+ targetInstanceId = await getInstanceId();
+ if (!targetInstanceId || targetInstanceId === UNKNOWN_INSTANCE_ID) return null;
+ }
// Every interpolated free-text field is redacted before it lands in the body —
// this task is human-facing and may sync across federated peers, so no
@@ -1369,11 +1380,15 @@ async function doCreateInvestigationTask(agentId, originalTask, errorAnalysis) {
`## What to approve\n${remedy}`,
`## Why this is held for you\n${LOOP_REASON_PROSE[loopReason]}`
].join('\n\n')
- : `## What to do\nNo approval needed — this investigation runs unattended. Diagnose the failure above, apply the fix, and leave the task's findings in your summary. Do NOT un-block the original task by hand; completing this one does it.\n\n${remedy}`;
+ : systemRecovery
+ ? `## What to do\nInvestigate and repair the machine cause with required review and CI. Preserve original work, approvals and retry/spawn budgets. Do not requeue or mark the original task complete by hand.\n\n${remedy}`
+ : `## What to do\nNo approval needed — this investigation runs unattended. Diagnose the failure above, apply the fix, and leave the task's findings in your summary. Do NOT un-block the original task by hand; completing this one does it.\n\n${remedy}`;
// Completing this task revives every blocked task in `affectedTasks`
// automatically (investigationRetry.js), so nobody has to go find them.
- const unblocks = `${approvalRequired ? 'Approving and applying' : 'Applying'} the fix and completing this task automatically retries the original task \`${originalTask.id}\`; it will resume: ${originalDesc}. If the same cause blocks it again after ${MAX_AUTO_RETRIES_PER_TASK} automatic retries, it stays blocked for you.`;
+ const unblocks = systemRecovery
+ ? `This investigation does not refund or reset task \`${originalTask.id}\` budgets. Its existing delivery verifier may settle proven completion; unfinished work remains stopped under its existing limits. Report the actual remaining work or user-owned decision.`
+ : `${approvalRequired ? 'Approving and applying' : 'Applying'} the fix and completing this task automatically retries the original task \`${originalTask.id}\`; it will resume: ${originalDesc}. If the same cause blocks it again after ${MAX_AUTO_RETRIES_PER_TASK} automatic retries, it stays blocked for you.`;
// The fingerprint rides in the headline so addTask's first-line dedup —
// which sees no `metadata.app` on investigation tasks — tracks fingerprint
@@ -1396,7 +1411,8 @@ ${actionBlock}
${unblocks}`;
const investigationTask = await addTask({
- ...INVESTIGATION_TASK_DELIVERY,
+ ...(systemRecovery ? SUPERVISED_INVESTIGATION_DELIVERY : INVESTIGATION_TASK_DELIVERY),
+ ...(systemRecovery && { targetInstanceId, metadata: { preserveOriginalBudget: true, blockedSystemRecovery: true } }),
description,
priority: 'HIGH',
context: `Auto-generated from agent ${agentId} failure`,
diff --git a/server/services/agentErrorAnalysis.test.js b/server/services/agentErrorAnalysis.test.js
index a9aef44f56..383d98e32f 100644
--- a/server/services/agentErrorAnalysis.test.js
+++ b/server/services/agentErrorAnalysis.test.js
@@ -3,6 +3,7 @@ import { describe, expect, it, vi, beforeEach } from 'vitest';
// maybeCreateInvestigationTask → createInvestigationTask → addTask does real
// store I/O; stub it so the skip-vs-create gate can be asserted in isolation.
// getAllTasks feeds the fingerprint dedup scan (#2615).
+vi.mock('./instanceIdentity.js', () => ({ getInstanceId: vi.fn().mockResolvedValue('fixture-local'), UNKNOWN_INSTANCE_ID: 'unknown' }));
vi.mock('./cosTaskStore.js', () => ({ addTask: vi.fn(), updateTask: vi.fn(), getAllTasks: vi.fn() }));
import {
@@ -997,6 +998,22 @@ describe('createInvestigationTask approval wiring (#3714)', () => {
__resetInvestigationCircuit();
});
+ it('creates local supervised diagnosis without refunding exhausted original work', async () => {
+ await createInvestigationTask('agent-original', { ...failedTask, taskType: 'internal' },
+ { ...analysis, blockedSystemRecovery: true });
+ expect(addTask.mock.calls[0][0]).toMatchObject({ targetInstanceId: 'fixture-local',
+ metadata: { preserveOriginalBudget: true, blockedSystemRecovery: true } });
+ expect(addTask.mock.calls[0][0].description).toContain('existing');
+ });
+
+ it('defers repeated system diagnosis instead of creating another blocked approval card', async () => {
+ getAllTasks.mockResolvedValue({ user: { tasks: [] }, cos: { tasks: [{ id: 'prior', status: 'completed', metadata: {
+ investigationFingerprint: 'startup-failure:internal:none', updatedAt: new Date().toISOString() } }] } });
+ expect(await createInvestigationTask('agent-original', { ...failedTask, taskType: 'internal' },
+ { ...analysis, blockedSystemRecovery: true })).toBeNull();
+ expect(addTask).not.toHaveBeenCalled();
+ });
+
it('runs unattended up to the storm threshold, then holds and records the loop reason', async () => {
// Distinct categories so each call mints its own fingerprint and reaches
// the circuit rather than deduping into the previous investigation.
diff --git a/server/services/blockedSystemInvestigation.js b/server/services/blockedSystemInvestigation.js
new file mode 100644
index 0000000000..5e3741d89a
--- /dev/null
+++ b/server/services/blockedSystemInvestigation.js
@@ -0,0 +1,147 @@
+/** Reuse the existing investigator for a locally executed system failure.
+ * This diagnoses the cause; it never revives work or refunds its retry budget. */
+import { buildInvestigationFingerprint, isInvestigationTask } from '../lib/investigationTasks.js';
+import { USER_DECISION_BLOCKED_CATEGORIES, PAUSED_BLOCKED_CATEGORIES } from '../lib/taskBlockCategories.js';
+import { explicitPrCompletion, PR_COMPLETIONS } from '../lib/prDisposition.js';
+import { isPrivateSecurityTask } from '../lib/privateSecurityPolicy.js';
+import { isTruthyMeta } from '../lib/metadataFlags.js';
+
+const BACKOFF_MS = 60000;
+const MAX_BACKOFF_MS = 3600000;
+const humanBoundaries = new Set(['auth-error', 'forbidden', 'permission-denied', 'permission-error', 'approval-required',
+ 'tool-approval', 'access-denied', 'security-refusal', 'sandbox-denied', 'content-refusal', 'content-filtered',
+ 'quota-exhausted', 'usage-limit', 'budget-exceeded', 'spending-limit', 'credits-exhausted', 'private-security-assessment-failed']);
+const flatten = queues => [...(queues.cos?.tasks || []).map(task => ({ ...task, taskType: 'internal' })),
+ ...(queues.user?.tasks || []).map(task => ({ ...task, taskType: 'user' }))];
+const eligible = task => {
+ const m = task?.metadata || {};
+ const category = m.blockedCategory;
+ return task?.taskType === 'internal' && task.status === 'blocked' && typeof category === 'string'
+ && !isInvestigationTask(task) && !isPrivateSecurityTask(task) && !isTruthyMeta(task.approvalRequired)
+ && !(typeof m.blocker === 'string' && m.blocker.trim())
+ && m.recoveryOrigin?.subsystem !== 'repository-cleanup'
+ && m.taskOutcome?.recovery?.mode !== 'approval'
+ && explicitPrCompletion(task) !== PR_COMPLETIONS.LEAVE_OPEN
+ && (!USER_DECISION_BLOCKED_CATEGORIES.has(category) || category === 'task-recovery-exhausted')
+ && (!PAUSED_BLOCKED_CATEGORIES.has(category) || category === 'task-recovery-exhausted')
+ && ![category, m.lastErrorCategory].some(value => typeof value === 'string' && humanBoundaries.has(value.toLowerCase()));
+};
+const analysisFor = task => ({
+ category: `blocked-system-${task.metadata.blockedCategory}-${task.metadata.lastErrorCategory || task.metadata.taskOutcome?.status || 'failure'}`,
+ message: 'A system task stopped before its work reached verified completion.',
+ snippet: task.metadata.blockedReason || task.metadata.taskOutcome?.reason,
+ blockedSystemRecovery: true,
+ preserveOriginalBudget: true,
+ escalation: 'Re-read the original task, current owners and delivery evidence first. Reuse active fixes. Diagnose and repair the underlying machine/product failure in an isolated checkout with required review, CI and merge admission. Preserve unfinished work and original failure/spawn budgets. Do not requeue, reset, or mark the original task completed by hand; its existing verifier and budget rules own continuation. Do not retry denied operations, change credentials, spend limits or permissions, or bypass a genuine human approval. Report actual human-required boundaries on the original record; machine-recoverable failures must not create personal Brain/todo actions.',
+});
+const phaseFor = task => task.status === 'in_progress' ? 'investigating'
+ : task.status === 'completed' ? 'diagnosed'
+ : task.status === 'blocked' || isTruthyMeta(task.approvalRequired) ? 'investigation-blocked' : 'queued';
+
+async function localOwner(task) {
+ const agentId = task.metadata?.deliveryDecisionAgentId || task.metadata?.lastAgentId || task.metadata?.failedAgentId;
+ if (!agentId) return null;
+ const [{ getAgentRecord }, { getInstanceId, UNKNOWN_INSTANCE_ID }] = await Promise.all([
+ import('./cosAgentLifecycle.js'), import('./instanceIdentity.js'),
+ ]);
+ const [agent, instanceId] = await Promise.all([getAgentRecord(agentId), getInstanceId()]);
+ return instanceId && instanceId !== UNKNOWN_INSTANCE_ID && agent?.status === 'completed'
+ && agent.taskId === task.id && agent.metadata?.instanceId === instanceId
+ ? { agentId, instanceId } : null;
+}
+
+/** Persistence/producer seams let lifecycle tests exercise crash and replay with
+ * a real task store adapter, without running providers or touching host tasks. */
+export function _createBlockedSystemInvestigationRecovery(deps = {}) {
+ const now = deps.now || Date.now;
+ const read = deps.read || (async () => (await import('./cosTaskStore.js')).getAllTasks());
+ const ownerFor = deps.ownerFor || localOwner;
+ const update = deps.update || (async (task, state) => (await import('./cosTaskStore.js')).updateTask(task.id,
+ { metadata: { blockedInvestigation: state } }, 'internal', { expectedStatus: 'blocked',
+ expectedTask: { description: task.description, priority: task.priority, metadata: task.metadata } }));
+ const permission = deps.permission || (async (fingerprint, tasks) => {
+ const producer = await import('./investigationTaskProducer.js');
+ if (producer.investigationCircuitOpen(now())) return 'circuit-open';
+ return (await producer.resolveAutoInvestigationApproval(fingerprint, { now: now(), tasks })).loopReason;
+ });
+ const create = deps.create || (async (task, analysis, owner) => (await import('./agentErrorAnalysis.js'))
+ .createInvestigationTask(owner.agentId, task, analysis));
+ let tail = Promise.resolve();
+ const write = async (task, state) => {
+ if (JSON.stringify(task.metadata?.blockedInvestigation) === JSON.stringify(state)) return task;
+ const saved = await update(task, state);
+ return saved?.error || saved?.statusChanged ? null : saved;
+ };
+ const one = async id => {
+ const tasks = flatten(await read());
+ let task = tasks.find(row => row.id === id && row.taskType === 'internal');
+ if (!eligible(task)) return { handled: false };
+ const owner = await ownerFor(task);
+ if (!owner) return { handled: false, reason: 'local-owner-unverified' };
+ const analysis = analysisFor(task);
+ const fingerprint = buildInvestigationFingerprint(task, analysis);
+ const epoch = `${fingerprint}:${task.metadata.blockedAt || task.metadata.lastSpawnedAt || 'legacy'}`;
+ const old = task.metadata.blockedInvestigation;
+ let state = old?.epoch === epoch ? { ...old } : { version: 1, epoch, fingerprint, attempts: 0, failures: 0,
+ ownerInstanceId: owner.instanceId, sourceCategory: task.metadata.blockedCategory,
+ sourceBlockedAt: task.metadata.blockedAt || task.metadata.lastSpawnedAt || 'legacy' };
+ const related = tasks.filter(row => isInvestigationTask(row) && row.metadata?.investigationFingerprint === fingerprint);
+ const existing = related.find(row => row.id === state.taskId)
+ || related.find(row => ['pending', 'in_progress', 'blocked'].includes(row.status));
+ if (existing) {
+ state = { ...state, taskId: existing.id, attempts: 1, phase: phaseFor(existing), retryAt: null };
+ if (!existing.metadata?.affectedTasks?.includes(task.id)) await create(task, analysis, owner);
+ await write(task, state);
+ return { handled: true, phase: state.phase, taskId: existing.id };
+ }
+ if (Number(state.version) !== 1 || !Number.isSafeInteger(Number(state.attempts)) || Number(state.attempts) < 0) {
+ return { handled: false, reason: 'recovery-state-unreadable' };
+ }
+ // One investigator owns this blocked episode, including its own configured
+ // retries. Losing/failing that investigator does not mint a fresh budget.
+ if (Number(state.attempts) >= 1) {
+ await write(task, { ...state, phase: 'budget-exhausted', retryAt: null });
+ return { handled: true, phase: 'budget-exhausted' };
+ }
+ if (Number.isFinite(Date.parse(state.retryAt)) && Date.parse(state.retryAt) > now()) return { handled: true, phase: 'waiting' };
+ const defer = async reason => {
+ const failures = Math.min(16, Math.max(0, Number(state.failures) || 0) + 1);
+ await write(task, { ...state, phase: 'waiting', failures, reason,
+ retryAt: new Date(now() + Math.min(MAX_BACKOFF_MS, BACKOFF_MS * 2 ** (failures - 1))).toISOString() });
+ return { handled: true, phase: 'waiting', reason };
+ };
+ const held = await permission(fingerprint, tasks);
+ if (held) return defer(held);
+ task = await write(task, { ...state, phase: 'creating', retryAt: new Date(now() + BACKOFF_MS).toISOString() });
+ if (!task) return { handled: false, reason: 'task-changed' };
+ // An explicit user/owner transition during preparation wins over recovery.
+ const current = flatten(await read()).find(row => row.id === task.id && row.taskType === 'internal');
+ if (!eligible(current) || current.metadata.blockedInvestigation?.epoch !== epoch) return { handled: false, reason: 'task-changed' };
+ task = current;
+ const created = await create(task, analysis, owner).catch(() => null);
+ if (!created?.id) return defer('producer-unavailable');
+ state = { ...state, attempts: 1, taskId: created.id, phase: phaseFor(created), retryAt: null };
+ await write(task, state);
+ return { handled: true, phase: state.phase, taskId: created.id };
+ };
+ const recover = id => {
+ const result = tail.then(() => one(id));
+ tail = result.catch(() => {});
+ return result;
+ };
+ let cursor = 0;
+ const sweep = async () => {
+ const all = flatten(await read()).filter(eligible);
+ const start = all.length ? cursor % all.length : 0;
+ const candidates = [...all.slice(start), ...all.slice(0, start)].slice(0, 3);
+ cursor = all.length ? (start + candidates.length) % all.length : 0;
+ const results = [];
+ for (const task of candidates) results.push(await recover(task.id));
+ return results;
+ };
+ return { recover, sweep };
+}
+
+const recovery = _createBlockedSystemInvestigationRecovery();
+export const recoverBlockedSystemTask = taskId => recovery.recover(taskId);
+export const recoverBlockedSystemTasks = () => recovery.sweep();
diff --git a/server/services/blockedSystemInvestigation.test.js b/server/services/blockedSystemInvestigation.test.js
new file mode 100644
index 0000000000..96e1c507b1
--- /dev/null
+++ b/server/services/blockedSystemInvestigation.test.js
@@ -0,0 +1,171 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { _createBlockedSystemInvestigationRecovery } from './blockedSystemInvestigation.js';
+import { resolveUndeliveredTaskUpdate, MAX_TASK_RETRIES } from './agentErrorAnalysis.js';
+import { buildInvestigationFingerprint, couldReleaseBlockedTasks } from '../lib/investigationTasks.js';
+
+const identity = vi.hoisted(() => ({ agent: null }));
+vi.mock('./cosAgentLifecycle.js', () => ({ getAgentRecord: vi.fn(async () => identity.agent) }));
+vi.mock('./instanceIdentity.js', () => ({ getInstanceId: vi.fn(async () => 'fixture-local'), UNKNOWN_INSTANCE_ID: 'unknown' }));
+
+let root, path, time, created, calls, hold, failAfterCreate, race, failLink;
+const read = async () => JSON.parse(await readFile(path, 'utf8'));
+const write = queues => writeFile(path, JSON.stringify(queues));
+const source = () => ({ id: 'system-original', taskType: 'internal', status: 'in_progress', priority: 'MEDIUM',
+ description: 'Finish retained claims, including unfinished source work.', metadata: { app: 'fixture', analysisType: 'claim-issue',
+ failureCount: String(MAX_TASK_RETRIES - 1), totalSpawnCount: '1' } });
+const blocked = () => {
+ const task = source();
+ const update = resolveUndeliveredTaskUpdate(task, { status: 'incomplete', reason: 'Merged delivery still has unfinished siblings.' }, 'agent-original', time);
+ return { ...task, ...update };
+};
+const create = async (task, analysis) => {
+ calls++;
+ expect(analysis.preserveOriginalBudget).toBe(true);
+ const queues = await read();
+ const fingerprint = buildInvestigationFingerprint(task, analysis);
+ let child = queues.cos.tasks.find(row => row.metadata?.investigationFingerprint === fingerprint && row.status !== 'completed');
+ if (!child) {
+ created++;
+ child = { id: `investigator-${created}`, status: 'pending', taskType: 'internal', metadata: {
+ isInvestigation: true, preserveOriginalBudget: true, investigationFingerprint: fingerprint, affectedTasks: [] } };
+ queues.cos.tasks.push(child);
+ }
+ child.metadata.affectedTasks = [...new Set([...child.metadata.affectedTasks, task.id])];
+ await write(queues);
+ if (failAfterCreate) { failAfterCreate = false; throw new Error('interrupted after durable creation'); }
+ return child;
+};
+const update = async (task, state) => {
+ const queues = await read();
+ const current = queues.cos.tasks.find(row => row.id === task.id);
+ if (current.status !== 'blocked' || JSON.stringify(current.metadata) !== JSON.stringify(task.metadata)) return { statusChanged: true };
+ if (failLink && state.taskId) { failLink = false; throw new Error('link interrupted'); }
+ current.metadata = { ...current.metadata, blockedInvestigation: state };
+ if (race && state.phase === 'creating') { race = false; current.status = 'completed'; }
+ await write(queues);
+ return current;
+};
+const factory = (realOwner = false) => _createBlockedSystemInvestigationRecovery({ read, update, create, now: () => time,
+ ...(!realOwner && { ownerFor: async task => task.metadata?.foreign ? null : { agentId: 'agent-original', instanceId: 'fixture-local' } }),
+ permission: async () => hold });
+
+beforeEach(async () => {
+ root = await mkdtemp(join(tmpdir(), 'blocked-investigation-'));
+ path = join(root, 'queues.json');
+ time = Date.parse('2026-01-01T00:00:00.000Z');
+ created = calls = 0; hold = null; failAfterCreate = race = failLink = false;
+ await write({ user: { tasks: [] }, cos: { tasks: [blocked()] } });
+});
+afterEach(async () => { await rm(root, { recursive: true, force: true }); });
+
+describe('blocked system recovery persisted workflow', () => {
+ it('investigates exhausted incomplete delivery once, groups duplicate observations and preserves original budgets', async () => {
+ const recovery = factory();
+ await Promise.all([recovery.recover('system-original'), recovery.recover('system-original')]);
+ let queues = await read();
+ const original = queues.cos.tasks.find(row => row.id === 'system-original');
+ expect(original.status).toBe('blocked');
+ expect(original.metadata).toMatchObject({ failureCount: MAX_TASK_RETRIES, totalSpawnCount: '1',
+ taskOutcome: { status: 'incomplete' }, blockedInvestigation: { phase: 'queued', attempts: 1, taskId: 'investigator-1' } });
+ expect(original.description).toBe(source().description);
+ queues.cos.tasks.push({ ...blocked(), id: 'same-cause', metadata: { ...blocked().metadata } });
+ await write(queues);
+ await factory().recover('same-cause');
+ queues = await read();
+ expect(created).toBe(1);
+ expect(queues.cos.tasks.find(row => row.id === 'investigator-1').metadata.affectedTasks).toEqual(['system-original', 'same-cause']);
+ await factory().sweep();
+ expect(created).toBe(1);
+ });
+
+ it.each(['creation', 'link'])('recovers from a crash after durable %s without minting another investigator', async cut => {
+ failAfterCreate = cut === 'creation'; failLink = cut === 'link';
+ await factory().recover('system-original').catch(() => {});
+ await factory().recover('system-original');
+ const original = (await read()).cos.tasks.find(row => row.id === 'system-original');
+ expect(created).toBe(1);
+ expect(original.metadata.blockedInvestigation).toMatchObject({ phase: 'queued', attempts: 1, taskId: 'investigator-1' });
+ });
+
+ it('backs off a loop/circuit without creating a blocked approval task, and resumes after the persisted window', async () => {
+ hold = 'repeat-fingerprint';
+ await factory().recover('system-original');
+ let state = (await read()).cos.tasks[0].metadata.blockedInvestigation;
+ expect(state.phase).toBe('waiting');
+ expect(Date.parse(state.retryAt)).toBeGreaterThan(time);
+ hold = null;
+ await factory().recover('system-original');
+ expect(calls).toBe(0);
+ time = Date.parse(state.retryAt) + 1;
+ await factory().recover('system-original');
+ expect(created).toBe(1);
+ });
+
+ it('keeps an investigator failure with its existing owner and never creates a successor or auto-revives originals', async () => {
+ await factory().recover('system-original');
+ const queues = await read();
+ const investigator = queues.cos.tasks.find(row => row.id === 'investigator-1');
+ investigator.status = 'blocked';
+ investigator.metadata.blockedCategory = 'task-recovery-exhausted';
+ await write(queues);
+ await factory().sweep();
+ expect(created).toBe(1);
+ const original = (await read()).cos.tasks.find(row => row.id === 'system-original');
+ expect(original.metadata.blockedInvestigation.phase).toBe('investigation-blocked');
+ expect(original.metadata.failureCount).toBe(MAX_TASK_RETRIES);
+ expect(couldReleaseBlockedTasks({ ...investigator, status: 'completed' })).toBe(false);
+ expect(couldReleaseBlockedTasks({ ...investigator, status: 'completed', metadata: { ...investigator.metadata, preserveOriginalBudget: 'true' } })).toBe(false);
+ });
+
+ it.each(['task-approval', 'agent-paused', 'provider-config', 'auth-error', 'usage-limit', 'content-refusal', 'worktree-busy', 'foreign-owner', 'cleanup-owner', 'user-blocker', 'user-task'])(
+ 'preserves the real %s boundary without filing another task', async boundary => {
+ const task = blocked();
+ if (boundary === 'foreign-owner') task.metadata.foreign = true;
+ else if (boundary === 'cleanup-owner') task.metadata.recoveryOrigin = { subsystem: 'repository-cleanup' };
+ else if (boundary === 'user-blocker') task.metadata.blocker = 'Wait for my decision.';
+ else task.metadata.blockedCategory = boundary;
+ const queues = { user: { tasks: boundary === 'user-task' ? [task] : [] }, cos: { tasks: boundary === 'user-task' ? [] : [task] } };
+ await write(queues);
+ await factory().sweep();
+ expect(await read()).toEqual(queues);
+ expect(created).toBe(0);
+ });
+
+ it('does not starve later failures behind already-owned investigations', async () => {
+ const queues = await read();
+ queues.cos.tasks = Array.from({ length: 5 }, (_, i) => ({ ...blocked(), id: `source-${i}` }));
+ await write(queues);
+ const recovery = factory();
+ await recovery.sweep();
+ await recovery.sweep();
+ const originals = (await read()).cos.tasks.filter(task => task.id.startsWith('source-'));
+ expect(originals.every(task => task.metadata.blockedInvestigation?.taskId === 'investigator-1')).toBe(true);
+ expect(created).toBe(1);
+ });
+
+ it('requires the actual completed local agent record before owning a historical blocked task', async () => {
+ const queues = await read();
+ queues.cos.tasks[0].metadata.deliveryDecisionAgentId = 'agent-original';
+ await write(queues);
+ const good = { status: 'completed', taskId: 'system-original', metadata: { instanceId: 'fixture-local' } };
+ for (const record of [null, { ...good, status: 'running' }, { ...good, taskId: 'other-task' },
+ { ...good, metadata: { instanceId: 'another-instance' } }]) {
+ identity.agent = record;
+ expect(await factory(true).recover('system-original')).toMatchObject({ handled: false, reason: 'local-owner-unverified' });
+ expect(created).toBe(0);
+ }
+ identity.agent = good;
+ expect(await factory(true).recover('system-original')).toMatchObject({ handled: true, taskId: 'investigator-1' });
+ expect(created).toBe(1);
+ });
+
+ it('lets a user/owner transition win before creation', async () => {
+ race = true;
+ expect(await factory().recover('system-original')).toMatchObject({ handled: false, reason: 'task-changed' });
+ expect(created).toBe(0);
+ expect((await read()).cos.tasks[0].status).toBe('completed');
+ });
+});
diff --git a/server/services/cos.js b/server/services/cos.js
index 8c20172352..d75d098dbb 100644
--- a/server/services/cos.js
+++ b/server/services/cos.js
@@ -408,6 +408,8 @@ async function bootDaemon(state) {
emitLog('info', `🧹 Periodic cleanup: ${cleaned} orphaned agent(s)`);
}
await resetOrphanedTasks();
+ const { recoverBlockedSystemTasks } = await import('./blockedSystemInvestigation.js');
+ await recoverBlockedSystemTasks().catch(err => console.error(`❌ Blocked-system investigation recovery failed: ${err.message}`));
// Evaluation is event-driven, so without this cadence a timed pause
// (`worktree-busy`, `orphan-cooldown`) whose `cooldownUntil` lapsed would
// sit blocked until the next restart or manual evaluate.
@@ -1532,6 +1534,12 @@ export async function init() {
await settleBranchReconcileRun(agent);
})
.catch(err => console.error(`❌ Branch reconciliation completion failed: ${err.message}`))
+ .then(async () => {
+ if (!isDaemonRunning() || !agent?.taskId) return;
+ const { recoverBlockedSystemTask } = await import('./blockedSystemInvestigation.js');
+ await recoverBlockedSystemTask(agent.taskId);
+ })
+ .catch(err => console.error(`❌ Blocked-system investigation recovery failed: ${err.message}`))
.then(() => refillPerpetualForCompletedAgent(agent))
.catch(err => console.error(`❌ Perpetual refill after ${agent?.id} failed: ${err.message}`))
// Same window as the refill above: this runs before the completing
@@ -1634,6 +1642,10 @@ export async function init() {
cosEvents.on('tasks:changed', (data) => {
handBackBlockedPrRemediation(data)
.catch(err => console.error(`❌ Blocked PR remediation hand-back failed for task ${data?.task?.id}: ${err.message}`));
+ if (isDaemonRunning() && data?.type === 'internal' && data.task?.status === 'blocked') {
+ import('./blockedSystemInvestigation.js').then(({ recoverBlockedSystemTask }) => recoverBlockedSystemTask(data.task.id))
+ .catch(err => console.error(`❌ Blocked-system investigation recovery failed: ${err.message}`));
+ }
});
// A critical health transition reaches the bell even when no CoS view is open.
diff --git a/server/services/cos.test.js b/server/services/cos.test.js
index df5c301c64..e4aa9855d7 100644
--- a/server/services/cos.test.js
+++ b/server/services/cos.test.js
@@ -2554,7 +2554,7 @@ describe('cos.js source — agent:completed triggers perpetual refill', () => {
// freed slot with idle-review work before the perpetual task is queued,
// breaking the back-to-back drain. The handler must chain refill → dequeue.
const onIdx = COS_SRC.indexOf("cosEvents.on('agent:completed'");
- const handlerSlice = COS_SRC.slice(onIdx, onIdx + 1400);
+ const handlerSlice = COS_SRC.slice(onIdx, COS_SRC.indexOf('// Create notification when a daily briefing completes', onIdx));
expect(
/refillPerpetualForCompletedAgent\(agent\)[\s\S]*\.then\(\s*\(\)\s*=>\s*dequeueNextTask\(/.test(handlerSlice),
'handler must run dequeueNextTask in a .then() AFTER the refill resolves'
diff --git a/server/services/investigationRetry.test.js b/server/services/investigationRetry.test.js
index 31a4397eac..aee3e36184 100644
--- a/server/services/investigationRetry.test.js
+++ b/server/services/investigationRetry.test.js
@@ -25,6 +25,13 @@ describe('retryTasksResolvedByInvestigation', () => {
updateTask.mockResolvedValue({ id: 'ok' });
});
+ it('does not refund original budgets after a diagnostic-only system investigation', async () => {
+ expect(await retryTasksResolvedByInvestigation(investigation({ preserveOriginalBudget: true })))
+ .toEqual({ retried: [], skipped: [] });
+ expect(getAllTasks).not.toHaveBeenCalled();
+ expect(reviveBlockedTask).not.toHaveBeenCalled();
+ });
+
it('revives the blocked task the investigation was diagnosing, stamping the auto-retry budget', async () => {
getAllTasks.mockResolvedValue(queues({
user: [{ id: 'task-1', status: 'blocked', metadata: { blockedCategory: 'test-failure' } }]