Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Auto-Fixer Service
*
* Handles automatic agent spawning for critical errors
* Integrates with error handler and CoS task system
*/
import { isRunning } from './cos.js';
import { getBootCommit, getInstallState } from './installState.js';
import { fileInvestigationTask, __resetInvestigationCircuit } from './investigationTaskProducer.js';
import { investigationFingerprint } from '../lib/investigationTasks.js';
import { errorEvents } from '../lib/errorHandler.js';
import { ERROR_CATEGORIES } from '../lib/aiToolkit/errorDetection.js';
import { registerProviderRecoveryHandlers } from './providerRecoveryHooks.js';
// Track recent errors to prevent duplicate auto-fix tasks
const recentErrors = new Map();
const ERROR_DEDUPE_WINDOW = 60000; // 1 minute
// Defer task creation as a backstop for the case where NO fallback is
// attempted (e.g. no fallback configured) — the timer fires and the
// investigation task is created. When a fallback IS attempted, promptRunner.js
// drives the lifecycle explicitly (noteFallbackStarted → noteFallbackHandled /
// noteFallbackFailed), which is authoritative regardless of how long the
// fallback takes. The fixed timer alone was a bug: a slow CLI fallback (Claude
// Code can take 20–30s) outran the window, so a successfully-recovered failure
// still left an investigation task in the user's plan.
const TASK_DEFER_MS = 5000;
// Invariant: every path that removes a timer here MUST also delete the
// matching map entry (the setTimeout callback, noteFallbackHandled, and
// _resetAutoFixerForTests all uphold this) — otherwise the map grows
// unbounded across the lifetime of the process.
const deferredTasks = new Map(); // errorKey -> { timer }
// Error keys whose failure is currently being retried via a fallback. While a
// key is in this set, the backstop timer is suppressed (we wait for the
// fallback's real outcome). Cleared by noteFallbackHandled (success → no task)
// and noteFallbackFailed (failure → the fallback's own task already covers it).
const inFlightFallbacks = new Set(); // errorKey
// Error keys explicitly escalated via escalateProviderFailure, with the ms
// timestamp of the escalation. Deduped on its own window (NOT recentErrors,
// which the primary failure may already have populated) so a concurrent
// no-fallback failure storm collapses to ONE escalated investigation task
// instead of one per call. Pruned + reset alongside the other maps.
const escalatedKeys = new Map(); // errorKey -> number (ms timestamp)
// Circuit breaker: if the SAME resource (errorKey) trips auto-fix more than
// CIRCUIT_MAX_FAILURES times within CIRCUIT_WINDOW_MS, stop creating
// investigation/fix tasks for it. A resource failing this persistently won't be
// healed by yet another identical task — repeating only exhausts the plan queue
// and can drive an unbounded retry loop. The window is rolling: timestamps older
// than CIRCUIT_WINDOW_MS are pruned on every check, so the circuit auto-closes
// once the failure rate ages out (no manual reset needed).
const CIRCUIT_WINDOW_MS = 60 * 60 * 1000; // 1 hour
const CIRCUIT_MAX_FAILURES = 3;
const failureTimestamps = new Map(); // errorKey -> number[] (ms timestamps, newest-last)
// Record a task-worthy failure for `errorKey` and report whether the circuit is
// now OPEN — i.e. this failure is the (CIRCUIT_MAX_FAILURES + 1)th within the
// rolling window and the caller should SUPPRESS task creation. Only genuinely
// new failures (past the dedupe + in-flight-fallback guards) should reach here,
// so the count reflects distinct recovery attempts, not log spam.
function tripCircuit(errorKey) {
const now = Date.now();
// Sweep fully-aged keys so the map can't grow unbounded — `error.message`
// (part of the generic errorKey) can carry dynamic text, so distinct keys
// accumulate over the process lifetime otherwise. Mirrors isDuplicateError's
// global cleanup of recentErrors.
for (const [key, stamps] of failureTimestamps.entries()) {
if (stamps.every((t) => now - t >= CIRCUIT_WINDOW_MS)) {
failureTimestamps.delete(key);
}
}
const recent = (failureTimestamps.get(errorKey) || []).filter((t) => now - t < CIRCUIT_WINDOW_MS);
recent.push(now);
failureTimestamps.set(errorKey, recent);
return recent.length > CIRCUIT_MAX_FAILURES;
}
// Collapse a (possibly multi-line) error string to one capped line for logging
// — the single-line logging convention forbids multi-line blobs; the untruncated
// text always remains in the run record's `error` field.
const oneLine = (s, max = 300) => {
const t = String(s ?? '').replace(/\s+/g, ' ').trim();
return t.length > max ? `${t.slice(0, max - 1)}…` : t;
};
// ── Tiered fallback cascade (issue #2328) ─────────────────────────────────
// A failure is routed to the CHEAPEST tier that can plausibly recover it,
// escalating only when no deterministic fix applies:
// 1 config/env — wrong provider/model/key/path/env; fixable by config
// 2 schema/type — malformed request/response; type/format/parse/build
// 3 constrained-agent-retry — transient/recoverable; a bounded retry may clear it
// 4 escalate — unknown or human-judgement-required; hand to investigation
export const FIX_TIERS = {
CONFIG_ENV: 1,
SCHEMA_TYPE: 2,
CONSTRAINED_RETRY: 3,
ESCALATE: 4,
};
const FIX_TIER_META = {
[FIX_TIERS.CONFIG_ENV]: { strategy: 'config/env', label: 'config/env correction' },
[FIX_TIERS.SCHEMA_TYPE]: { strategy: 'schema/type', label: 'schema/type correction' },
[FIX_TIERS.CONSTRAINED_RETRY]: { strategy: 'constrained-agent-retry', label: 'constrained agent retry' },
[FIX_TIERS.ESCALATE]: { strategy: 'escalate', label: 'escalate for investigation' },
};
// Deterministic error-category → tier map. Categories come from BOTH
// errorDetection.js (AI-provider failures) and agentErrorAnalysis.js (CoS agent
// failures); the two vocabularies overlap and are unioned here. Anything
// unmapped falls through to Tier 4 (escalate) — an unrecognized failure has no
// deterministic fix, so an investigation task carries it to a human/agent.
const CATEGORY_TO_TIER = {
// Tier 1 — config/env (wrong key/model/path/permissions/env)
'auth-error': FIX_TIERS.CONFIG_ENV,
forbidden: FIX_TIERS.CONFIG_ENV,
'model-not-found': FIX_TIERS.CONFIG_ENV,
'model-not-supported': FIX_TIERS.CONFIG_ENV,
'quota-exceeded': FIX_TIERS.CONFIG_ENV,
'billing-error': FIX_TIERS.CONFIG_ENV,
'usage-limit': FIX_TIERS.CONFIG_ENV,
'spawn-error': FIX_TIERS.CONFIG_ENV,
// A provider CLI refusing to load its own config file: deterministic, and the
// fix is literally a config edit — never worth three identical retries plus an
// investigation task.
'cli-config-invalid': FIX_TIERS.CONFIG_ENV,
'permission-denied': FIX_TIERS.CONFIG_ENV,
'file-not-found': FIX_TIERS.CONFIG_ENV,
// Tier 2 — schema/type (malformed request/response, parse/build/format)
'parse-error': FIX_TIERS.SCHEMA_TYPE,
'bad-request': FIX_TIERS.SCHEMA_TYPE,
'context-length': FIX_TIERS.SCHEMA_TYPE,
'ollama-context-window': FIX_TIERS.SCHEMA_TYPE,
'output-length': FIX_TIERS.SCHEMA_TYPE,
'build-error': FIX_TIERS.SCHEMA_TYPE,
'lint-error': FIX_TIERS.SCHEMA_TYPE,
// Tier 3 — constrained-agent-retry (transient/recoverable)
'rate-limit': FIX_TIERS.CONSTRAINED_RETRY,
'network-error': FIX_TIERS.CONSTRAINED_RETRY,
timeout: FIX_TIERS.CONSTRAINED_RETRY,
'server-error': FIX_TIERS.CONSTRAINED_RETRY,
'tool-error': FIX_TIERS.CONSTRAINED_RETRY,
'mcp-error': FIX_TIERS.CONSTRAINED_RETRY,
'test-failure': FIX_TIERS.CONSTRAINED_RETRY,
'npm-error': FIX_TIERS.CONSTRAINED_RETRY,
'memory-error': FIX_TIERS.CONSTRAINED_RETRY,
'turn-limit': FIX_TIERS.CONSTRAINED_RETRY,
'process-killed': FIX_TIERS.CONSTRAINED_RETRY,
'browser-error': FIX_TIERS.CONSTRAINED_RETRY,
'locator-error': FIX_TIERS.CONSTRAINED_RETRY,
'claude-error': FIX_TIERS.CONSTRAINED_RETRY,
'startup-failure': FIX_TIERS.CONSTRAINED_RETRY,
// Tier 4 — escalate (explicit entries; also the fall-through default)
'content-refusal': FIX_TIERS.ESCALATE,
'content-filtered': FIX_TIERS.ESCALATE,
'task-rejected': FIX_TIERS.ESCALATE,
'git-conflict': FIX_TIERS.ESCALATE,
'git-error': FIX_TIERS.ESCALATE,
'no-changes': FIX_TIERS.ESCALATE,
unknown: FIX_TIERS.ESCALATE,
};
/**
* Deterministically map an error category to a fallback tier (issue #2328).
* Pure — no I/O. Unknown/absent categories escalate (Tier 4), so a failure the
* cascade doesn't recognize is never silently swallowed.
* @param {string} [category]
* @returns {{ tier: number, strategy: string, label: string }}
*/
export function classifyFixTier(category) {
const tier = CATEGORY_TO_TIER[category] ?? FIX_TIERS.ESCALATE;
return { tier, ...FIX_TIER_META[tier] };
}
/**
* Look up the human-facing metadata for a fallback tier NUMBER (issue #2328).
* The persisted diagnostics record already carries the tier number, so the
* telemetry aggregator resolves labels from the number rather than re-running
* the category classifier. Unknown tiers degrade to an explicit 'unknown'
* rather than throwing. Pure — no I/O.
* @param {number} tier
* @returns {{ strategy: string, label: string }}
*/
export function fixTierMeta(tier) {
return FIX_TIER_META[tier] || { strategy: 'unknown', label: 'unknown' };
}
/**
* Build a structured, per-attempt auto-fix diagnostics record (issue #2328).
* Beyond the single-line log, this object rides on the task/return shape so
* downstream telemetry can break failures out by tier / category / reason /
* time-to-recovery. Pure — no I/O.
* @returns {{ triggerEvent: string, target: string, errorType: string,
* category: string, tier: number, fixStrategy: string, failureReason: string,
* observedAt: string }}
*/
export function buildFixDiagnostics({ triggerEvent, target, category, failureReason, observedAt } = {}) {
const cat = category || 'unknown';
const { tier, strategy } = classifyFixTier(cat);
return {
triggerEvent: triggerEvent || 'unknown',
target: target || 'unknown',
errorType: cat,
category: cat,
tier,
fixStrategy: strategy,
failureReason: oneLine(failureReason) || 'no error text captured',
// ISO timestamp of when the failure was observed (issue #2328). Rides on the
// persisted diagnostics so the telemetry aggregator can compute
// time-to-recovery (task completion time − this) without a second timestamp
// source. Injectable so callers pass the failure's real timestamp (and tests
// stay deterministic); defaults to now when the caller has none.
observedAt: observedAt || new Date().toISOString(),
};
}
/**
* Derive the diagnostics record for an AI-provider failure `error`. Kept as a
* single helper so the deferred log line and the investigation task record are
* always built from the same fields (no drift).
*/
function providerFixDiagnostics(error) {
const ctx = error.context || {};
return buildFixDiagnostics({
triggerEvent: error.code,
target: `${ctx.provider || 'Unknown'} (${ctx.model || 'N/A'})`,
category: ctx.errorAnalysis?.category,
failureReason: ctx.errorDetails || ctx.errorAnalysis?.message,
// Pin observedAt to the failure's own timestamp so the log-line record and
// the persisted task record (both built from this helper) agree, and so
// time-to-recovery measures from the actual failure, not from whenever the
// deferred handler ran.
observedAt: error.timestamp ? new Date(error.timestamp).toISOString() : undefined,
});
}
function aiProviderErrorKey(providerName, model) {
// NUL separator: provider names ("Claude Code CLI") and model ids
// ("gpt-4o-mini") both commonly contain `-`, so a `-`-joined key would
// collide for pairs like ("gpt-4o", "mini") vs ("gpt", "4o-mini") and
// silently dedupe distinct failures together. NUL never appears in
// legitimate provider/model identifiers, so the key is unambiguous.
return `AI_PROVIDER_EXECUTION_FAILED\x00${providerName}\x00${model}`;
}
/**
* Cancel a deferred investigation task for `provider`/`model` because a
* fallback retry succeeded. Called from `runPromptThroughProvider` after
* a successful fallback — the user got their result, so the queued task
* would be noise. Also clears the dedupe entry so a *future* failure of
* the same provider can still raise a task (otherwise the dedupe window
* would silently suppress real failures for up to 60s).
*
* `provider` matches `ctx.provider` (the provider's display name, not id)
* because that's what the failure event payload uses — see the
* `onRunFailed` hook in server/index.js.
*/
export function noteFallbackHandled({ provider, model }) {
const errorKey = aiProviderErrorKey(provider, model);
inFlightFallbacks.delete(errorKey);
const pending = deferredTasks.get(errorKey);
if (pending) {
clearTimeout(pending.timer);
deferredTasks.delete(errorKey);
console.log(`✅ Suppressed investigation task: fallback handled failure for ${provider} (${model})`);
}
// Always clear the dedupe entry — whether or not there was a timer to cancel
// (noteFallbackStarted may have already cancelled it) — so a *future*
// identical failure can still raise a task.
recentErrors.delete(errorKey);
return !!pending;
}
/**
* Mark that promptRunner is about to retry `provider`/`model` via a fallback.
* Cancels the deferred investigation task immediately and suppresses any that
* would otherwise be scheduled (handleAIProviderError checks the in-flight set),
* so a slow fallback that exceeds TASK_DEFER_MS can't leave a task behind. The
* fallback's eventual outcome (noteFallbackHandled / noteFallbackFailed) clears
* the in-flight marker.
*/
export function noteFallbackStarted({ provider, model }) {
const errorKey = aiProviderErrorKey(provider, model);
inFlightFallbacks.add(errorKey);
const pending = deferredTasks.get(errorKey);
if (pending) {
clearTimeout(pending.timer);
deferredTasks.delete(errorKey);
}
}
/**
* Mark that the fallback retry for `provider`/`model` ALSO failed. Releases the
* in-flight suppression without creating a task for the primary: the fallback
* provider's own failure already queued its investigation task, so one task per
* user action is enough. Also clears the dedupe entry so a later retry isn't
* silently suppressed.
*/
export function noteFallbackFailed({ provider, model }) {
const errorKey = aiProviderErrorKey(provider, model);
inFlightFallbacks.delete(errorKey);
recentErrors.delete(errorKey);
}
/**
* Explicitly escalate a provider failure to a Tier-4 investigation task
* (issue #2342). Called by promptRunner's fallback cascade ONLY when the
* deterministic tiers all declined/failed AND no other queued task will
* survive to represent the failure — specifically when a Tier-1 corrected
* retry was pre-suppressed but threw BEFORE execution (so its onRunFailed
* never queued a task) and no fallback provider exists.
*
* Bypasses the defer window (we already KNOW recovery failed) but still honors
* the per-resource circuit breaker, and first clears any lingering
* timer/in-flight/dedupe state for the key so the escalation isn't itself
* suppressed and a future failure isn't wrongly blocked.
*
* `error` is a synthetic `AI_PROVIDER_EXECUTION_FAILED`-shaped object built by
* the caller from the failure it holds (it lacks the server hook's full run
* metadata, so `runId`/`exitCode` may be absent — the investigation task
* tolerates that). Returns the created task, or null when the circuit is open.
*/
export async function escalateProviderFailure(error) {
const ctx = error?.context || {};
// Before dedupe: a later real fault on this route must still investigate.
if (ctx.errorAnalysis?.category === ERROR_CATEGORIES.RUNTIME_BUDGET_EXHAUSTED) return;
const errorKey = aiProviderErrorKey(ctx.provider, ctx.model);
const now = Date.now();
// Cancel any pending backstop timer + drop the in-flight suppression for this
// key (the cascade already cancelled the timer; make sure it's gone).
inFlightFallbacks.delete(errorKey);
const pending = deferredTasks.get(errorKey);
if (pending) {
clearTimeout(pending.timer);
deferredTasks.delete(errorKey);
}
// Dedupe on the escalation window, checked+set synchronously (single-threaded)
// BEFORE the first await so a concurrent no-fallback failure storm collapses
// to ONE task. We can't use isDuplicateError/recentErrors here — the primary
// failure's handleAIProviderError may already hold a recentErrors entry for
// this key, which would make our FIRST escalation look like a duplicate and
// silently swallow it.
const lastEscalated = escalatedKeys.get(errorKey);
if (lastEscalated && now - lastEscalated < ERROR_DEDUPE_WINDOW) {
console.log(`⏭️ Skipping duplicate escalated AI provider failure: ${ctx.provider} (${ctx.model})`);
return null;
}
escalatedKeys.set(errorKey, now);
for (const [key, ts] of escalatedKeys.entries()) {
if (now - ts >= ERROR_DEDUPE_WINDOW) escalatedKeys.delete(key);
}
// Clear the primary's dedupe entry so a genuinely NEW failure after this
// window can still raise a task through the normal deferred path.
recentErrors.delete(errorKey);
if (tripCircuit(errorKey)) {
console.log(`🔌 Auto-fix circuit OPEN for ${ctx.provider} (${ctx.model}) — suppressing escalated investigation task`);
return null;
}
// Clear the dedupe marker if task creation fails so an identical escalation
// isn't suppressed for the window (mirrors the deferred path's catch arm) —
// otherwise an addTask rejection would silently swallow every retry for 60s.
return createAIProviderInvestigationTask(error).catch((err) => {
escalatedKeys.delete(errorKey);
console.error(`❌ Escalated AI provider task creation failed: ${err.message}`);
return null;
});
}
/**
* Check if an error is a duplicate within the dedupe window
* Also cleans up expired entries
* @returns {boolean} true if this is a duplicate error
*/
function isDuplicateError(errorKey) {
const now = Date.now();
const lastSeen = recentErrors.get(errorKey);
// Clean up expired entries
for (const [key, timestamp] of recentErrors.entries()) {
if (now - timestamp > ERROR_DEDUPE_WINDOW) {
recentErrors.delete(key);
}
}
if (lastSeen && (now - lastSeen) < ERROR_DEDUPE_WINDOW) {
return true;
}
recentErrors.set(errorKey, now);
return false;
}
let autoFixerInitialized = false;
/**
* Initialize auto-fixer event listeners
*/
export function initAutoFixer() {
if (autoFixerInitialized) return;
autoFixerInitialized = true;
// promptRunner.js reaches these through the registry, not an import (#12285).
registerProviderRecoveryHandlers({ noteFallbackStarted, noteFallbackFailed, noteFallbackHandled, escalateProviderFailure });
errorEvents.on('error', (error) => {
(async () => {
// Always handle AI provider errors (even if CoS not running)
if (error.code === 'AI_PROVIDER_EXECUTION_FAILED') {
await handleAIProviderError(error);
return;
}
if (shouldAutoFix(error)) {
await createAutoFixTask(error);
}
})().catch(err => console.error(`❌ autoFixer handler failed: ${err.message}`));
});
console.log('🔧 Auto-fixer initialized');
}
/**
* Test-only: drop all deferred timers + dedupe entries so the next call
* starts from a clean slate. Production code paths never call this.
*/
export function _resetAutoFixerForTests() {
for (const { timer } of deferredTasks.values()) clearTimeout(timer);
deferredTasks.clear();
inFlightFallbacks.clear();
recentErrors.clear();
failureTimestamps.clear();
escalatedKeys.clear();
// This module's task-creation path now goes through the SHARED investigation
// circuit (investigationTaskProducer.js), so a reset that stopped at the local
// maps would leave earlier cases' creations counted against later ones — every
// test past the storm threshold would then assert against a held task.
__resetInvestigationCircuit();
}
/**
* Defer task creation by TASK_DEFER_MS. If `noteFallbackHandled` is called
* for the same provider/model within the window, the timer is cancelled
* and no task is created. Otherwise, the deferred handler runs and
* creates the investigation task.
*/
async function handleAIProviderError(error) {
const ctx = error.context || {};
if (ctx.errorAnalysis?.category === ERROR_CATEGORIES.RUNTIME_BUDGET_EXHAUSTED) return;
// A content/safety refusal is not a provider fault — we know exactly why it
// failed (the model declined the prompt), so there's nothing for a CoS agent
// to investigate. promptRunner.js stops without fallback and the UI
// is told what happened. Bail before deferring/creating a task. (server's
// onRunFailed already emits a distinct code for refusals so this handler
// normally isn't even reached; this guard covers any other emitter.)
if (ctx.errorAnalysis?.category === ERROR_CATEGORIES.CONTENT_REFUSAL) {
console.warn(`🛟 AI model refused prompt on content/safety grounds: ${ctx.provider} (${ctx.model}) — no investigation task or automatic fallback`);
return;
}
const errorKey = aiProviderErrorKey(ctx.provider, ctx.model);
// A fallback retry for this exact failure is already in flight (promptRunner
// called noteFallbackStarted). Don't schedule the backstop timer — the
// fallback's outcome decides whether to investigate. This covers the case
// where the fallback started before this handler ran (microtask ordering).
if (inFlightFallbacks.has(errorKey)) {
return;
}
if (isDuplicateError(errorKey)) {
console.log(`⏭️ Skipping duplicate AI provider error: ${ctx.provider} (${ctx.model})`);
return;
}
if (deferredTasks.has(errorKey)) {
return;
}
// Surface the actual failure reason + category inline so pm2 logs explain
// WHY a run failed without spelunking into data/runs/<id>/metadata.json. The
// reason is collapsed to a single line and capped (logging convention: no
// multi-line blobs) — the full text stays in the run record's `error` field.
const reason = oneLine(ctx.errorDetails || ctx.errorAnalysis?.message) || 'no error text captured';
const category = ctx.errorAnalysis?.category || 'unknown';
// Structured per-attempt diagnostics (issue #2328): classify the failure into
// a fallback tier and surface it inline so pm2 logs (and the task record
// below) break failures out by tier/strategy without spelunking metadata.
const diagnostics = providerFixDiagnostics(error);
console.log(`🤖 AI provider error detected: ${ctx.provider} (${ctx.model}) [${category}] tier=${diagnostics.tier} (${diagnostics.fixStrategy}) exit=${ctx.exitCode ?? '?'} - run ${ctx.runId}: ${reason} (deferring ${TASK_DEFER_MS}ms for possible fallback retry)`);
const timer = setTimeout(() => {
deferredTasks.delete(errorKey);
// Circuit breaker: only count a failure that actually survives the defer
// window (a fallback that recovered the failure has already cancelled this
// timer, so it never reaches here). A provider/model that keeps producing
// real investigation tasks this often won't be fixed by yet another — trip
// the circuit and suppress. Auto-closes once failures age out of the window.
if (tripCircuit(errorKey)) {
console.log(`🔌 Auto-fix circuit OPEN for ${ctx.provider} (${ctx.model}) — >${CIRCUIT_MAX_FAILURES} failures within the last hour; suppressing investigation task`);
return;
}
createAIProviderInvestigationTask(error).catch(err => {
console.error(`❌ Deferred AI provider task creation failed: ${err.message}`);
// Clear the dedupe entry so the next identical failure isn't
// silently suppressed for up to 60s — without this, an addTask
// failure here would block legitimate retries that might succeed.
recentErrors.delete(errorKey);
});
}, TASK_DEFER_MS);
// Keep the timer from preventing process exit (e.g. in tests / shutdown).
timer.unref?.();
deferredTasks.set(errorKey, { timer });
}
async function createAIProviderInvestigationTask(error) {
const ctx = error.context || {};
// Structured diagnostics ride on the task record so downstream telemetry can
// break auto-fix outcomes out by tier / category / failure reason (#2328).
const diagnostics = providerFixDiagnostics(error);
// A server that booted BEFORE the current checkout can keep reproducing a
// failure that is already fixed on disk — exactly what happened when a
// local-LLM playground timeout was filed a second time hours after its fix
// landed (#5771), costing a whole investigation agent to rediscover. Surface
// the boot-vs-HEAD gap in the task body so the agent checks that first.
// Skipped entirely when no boot commit was captured (tarball install, tests,
// any process that never called captureBootCommit): without it the comparison
// is meaningless, and skipping keeps this off the git/fs path in those cases.
const installState = getBootCommit() ? await getInstallState().catch(() => null) : null;
// Build specialized context for AI provider errors
const context = buildAIProviderErrorContext(error, diagnostics, installState);
const taskData = {
// Mirror the `|| 'Unknown'` fallbacks buildAIProviderErrorContext already
// applies to the body. Without them an unattributable failure filed a task
// titled "Investigate AI provider failure: undefined (undefined)".
description: `Investigate AI provider failure: ${ctx.provider || 'Unknown provider'} (${ctx.model || 'unknown model'})`,
priority: 'MEDIUM',
context,
diagnostics,
// No `app`. `metadata.app` is WORKSPACE ROUTING — it must name a record in
// `data/apps.json` — and 'portos' matches neither the seeded id
// ('portos-default') nor its name ('PortOS'), so it resolved to nothing.
// That was harmless while an unresolvable app fell through to the PortOS
// root, which is where this work belongs anyway; since the #3180 guard,
// `prepareAgentWorkspace` refuses to spawn an agent whose app doesn't
// resolve to a repo path, so every provider investigation was filed and
// then rejected. Absent `app` resolves to that same PortOS root, and
// matches the other two task producers in this file.
};
// Filed durably whether or not the CoS daemon is running: `addTask` persists
// without it and CoS picks the task up when it next runs. (An in-memory queue
// here once had no consumer and silently dropped every failure while stopped.)
// Unattended by default, matching every other investigation producer (#3714):
// a provider failure is exactly the diagnosis CoS exists to do for itself, and
// gating it behind an approval only meant the queue filled with approvals while
// the provider stayed broken. The loop guards are unchanged and still upstream
// of here — `isDuplicateError` collapses a repeat inside the dedupe window and
// `tripCircuit` suppresses the task entirely past CIRCUIT_MAX_FAILURES.
const { task } = await fileInvestigationTask({
...taskData,
fingerprint: investigationFingerprint({
category: diagnostics.category,
kind: 'provider-failure',
scope: ctx.provider
})
});
console.log(`✅ AI provider investigation task filed: ${task.id} [tier ${diagnostics.tier}: ${diagnostics.fixStrategy}]`);
return task;
}
/**
* Determine if an error should trigger auto-fix
*/
function shouldAutoFix(error) {
// Only auto-fix if CoS is running
if (!isRunning()) {
return false;
}
// Only auto-fix critical errors or those explicitly marked as auto-fixable
if (error.severity !== 'critical' && !error.canAutoFix) {
return false;
}
const errorKey = `${error.code}-${error.message}`;
if (isDuplicateError(errorKey)) {
console.log(`⏭️ Skipping duplicate error: ${error.code}`);
return false;
}
// Circuit breaker: the same critical error recurring past the threshold won't
// be resolved by spawning another identical fix task — suppress to prevent a
// runaway loop. Auto-closes once the failure rate ages out of the window.
if (tripCircuit(errorKey)) {
console.log(`🔌 Auto-fix circuit OPEN for ${error.code} — >${CIRCUIT_MAX_FAILURES} failures within the last hour; suppressing auto-fix task`);
return false;
}
return true;
}
/**
* Build detailed context for AI provider execution errors
*
* @param {object} [installState] result of `getInstallState()`, when available —
* only `runningStaleCode` / `bootCommit` / `currentCommit` are read, to warn
* that the failing process predates the checkout being investigated.
*/
function buildAIProviderErrorContext(error, diagnostics, installState = null) {
const ctx = error.context || {};
const lines = [
'# AI Provider Execution Failure',
'',
'## Run Details',
`- **Run ID:** ${ctx.runId || 'N/A'}`,
`- **Provider:** ${ctx.provider || 'Unknown'}`,
`- **Provider ID:** ${ctx.providerId || 'N/A'}`,
`- **Model:** ${ctx.model || 'N/A'}`,
`- **Exit Code:** ${ctx.exitCode ?? 'N/A'}`,
`- **Duration:** ${ctx.duration ? `${(ctx.duration / 1000).toFixed(1)}s` : 'N/A'}`,
`- **Workspace:** ${ctx.workspaceName || ctx.workspacePath || 'N/A'}`,
''
];
// The running process is behind the on-disk checkout, so the failure may have
// been fixed since it booted. Named first (right under Run Details) because it
// changes what the whole investigation should do: read the diff before
// debugging, and restart rather than write code if the fix is already there.
if (installState?.runningStaleCode) {
const boot = (installState.bootCommit || '').slice(0, 7) || 'unknown';
const head = (installState.currentCommit || '').slice(0, 7) || 'unknown';
lines.push('## Deployed Build: STALE');
lines.push(`- **Booted at commit:** ${boot}`);
lines.push(`- **Checkout HEAD:** ${head}`);
lines.push(`- The running server started before the current checkout, so it does NOT include every fix on disk. Check \`git log ${boot}..${head}\` for a fix covering this failure before debugging it — if one exists, the remedy is a restart (\`npm run pm2:restart\`), not a code change.`);
lines.push('');
}
// Fallback-tier diagnostics (issue #2328) — tells the investigating agent
// which class of fix to try first before escalating to open-ended debugging.
if (diagnostics) {
lines.push('## Fallback Tier');
lines.push(`- **Tier:** ${diagnostics.tier} (${diagnostics.fixStrategy})`);
lines.push(`- **Error Type:** ${diagnostics.errorType}`);
lines.push(`- **Failure Reason:** ${diagnostics.failureReason}`);
lines.push('');
}
// Add error category if available
if (ctx.errorCategory) {
lines.push(`## Error Category: ${ctx.errorCategory}`);
if (ctx.suggestedFix) {
lines.push(`**Suggested Fix:** ${ctx.suggestedFix}`);
}
lines.push('');
}
// Add error details
lines.push('## Error Details');
if (ctx.errorDetails) {
lines.push('```');
lines.push(ctx.errorDetails);
lines.push('```');
} else {
lines.push(error.message || 'No error details available');
}
lines.push('');
// Add prompt preview if available
if (ctx.promptPreview) {
lines.push('## Prompt Preview');
lines.push('```');
lines.push(ctx.promptPreview);
lines.push('```');
lines.push('');
}
// Add output tail if available (last part of output for debugging)
if (ctx.outputTail) {
lines.push('## Output Tail (last 2KB)');
lines.push('```');
lines.push(ctx.outputTail);
lines.push('```');
lines.push('');
}
lines.push('## Investigation Steps');
const steps = [
'Check if the AI provider is configured correctly in /devtools/providers',
'Verify API keys and endpoints are valid',
'Check server logs for additional context (pm2 logs portos-server)',
'If this is a CLI provider, verify the command is installed and accessible',
'Check for rate limiting or quota issues with the provider',
'Review the output tail for specific error messages',
];
// Numbered here rather than hardcoded so the stale-build step can lead the
// list without renumbering the rest by hand.
if (installState?.runningStaleCode) {
steps.unshift('Rule out the stale deployed build above — a fix already in the checkout only needs a restart');
}
steps.forEach((step, index) => lines.push(`${index + 1}. ${step}`));
return lines.join('\n');
}
/**
* Create a CoS task to fix the error
*/
async function createAutoFixTask(error) {
// Structured diagnostics (issue #2328): a bare crash usually has no recognized
// category, so it classifies to Tier 4 (escalate) — an investigation rather
// than a deterministic correction.
const diagnostics = buildFixDiagnostics({
triggerEvent: error.code,
target: error.code,
category: error.context?.errorAnalysis?.category,
failureReason: error.message,
observedAt: error.timestamp ? new Date(error.timestamp).toISOString() : undefined,
});
console.log(`🤖 Creating auto-fix task for error: ${error.code} [tier ${diagnostics.tier}: ${diagnostics.fixStrategy}]`);
// Build context for the agent
const context = buildErrorContext(error);
// Unattended by default, matching every other error-driven task creator in this
// file. `shouldAutoFix` has already applied the dedupe + circuit breaker before
// we get here, so this only fires for a genuinely new critical error, and the
// shared policy still holds it for a human when the same cause is looping or
// failures are cascading.
const { task } = await fileInvestigationTask({
description: `Fix critical error: ${error.message}`,
priority: 'HIGH',
context,
diagnostics,
fingerprint: investigationFingerprint({
category: diagnostics.category,
kind: 'critical-error',
scope: error.code
})
});
console.log(`✅ Auto-fix task created: ${task.id}`);
return task;
}
/**
* Build detailed context for the auto-fix agent
*/
function buildErrorContext(error) {
const lines = [
'# Error Details',
'',
`**Error Code:** ${error.code}`,
`**Severity:** ${error.severity}`,
`**Timestamp:** ${new Date(error.timestamp).toISOString()}`,
'',
'## Error Message',
error.message,
''
];
// Add stack trace if available
if (error.stack) {
lines.push('## Stack Trace');
lines.push('```');
lines.push(error.stack);
lines.push('```');
lines.push('');
}
// Add context if available
if (error.context && Object.keys(error.context).length > 0) {
lines.push('## Context');
for (const [key, value] of Object.entries(error.context)) {
lines.push(`- **${key}:** ${JSON.stringify(value)}`);
}
lines.push('');
}
lines.push('## Instructions');
lines.push('1. Analyze the error and identify the root cause');
lines.push('2. Check server logs and browser console for additional context');
lines.push('3. Fix the issue in the codebase');
lines.push('4. Verify the fix works by testing the affected functionality');
lines.push('5. If you cannot fix the issue, document your findings in a comment');
return lines.join('\n');
}
/**
* Handle manual error recovery request from UI
*/
export async function handleErrorRecovery(errorCode, context) {
console.log(`🔧 Manual error recovery requested: ${errorCode}`);
// No standing approval gate: the user clicking "investigate this error" IS the
// approval, and asking them to approve their own request a second time is the
// friction this whole path exists to remove. The shared loop policy can still
// hold it — if they ask again about a cause we investigated hours ago, another
// unattended agent would just repeat that run.
const { task } = await fileInvestigationTask({
description: `Investigate and fix error: ${errorCode}`,
priority: 'MEDIUM',
context: context || `User requested investigation of error code: ${errorCode}`,
fingerprint: investigationFingerprint({ kind: 'manual-recovery', scope: errorCode })
});
console.log(`✅ Recovery task created: ${task.id}`);
return task;
}