Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { readFileSync } from 'node:fs';
import { runInNewContext } from 'node:vm';
import { extractDeclaration } from '../lib/mirrorParity.js';
import { ERROR_CATEGORIES } from '../lib/aiToolkit/errorDetection.js';
// cos.js drags in a giant dependency graph (PM2, fs, sockets…) — mock it
// so autoFixer's defer/cancel behavior can be tested in isolation. `addTask`
// records what would be queued, `isRunning` toggles the running/not-running
// branches, and `getAllTasks` feeds the shared investigation-approval policy
// (reached via agentErrorAnalysis) an empty backlog — no prior investigation and
// no storm, i.e. the ordinary unattended case.
vi.mock('./cosTaskStore.js', () => ({
addTask: vi.fn().mockResolvedValue({ id: 'task-1' }),
updateTask: vi.fn().mockResolvedValue(true),
getAllTasks: vi.fn().mockResolvedValue({ user: { tasks: [] }, cos: { tasks: [] } }),
}));
vi.mock('./cos.js', () => ({
isRunning: vi.fn().mockReturnValue(true),
}));
// installState reaches out to git/fs; stub it so the stale-deployed-build
// branch is driven from the test rather than from this checkout's real HEAD.
vi.mock('./installState.js', () => ({
getBootCommit: vi.fn(() => null),
getInstallState: vi.fn(async () => null),
}));
const installState = await import('./installState.js');
const { addTask, getAllTasks } = await import('./cosTaskStore.js');
const { isRunning } = await import('./cos.js');
const {
noteFallbackHandled,
initAutoFixer,
_resetAutoFixerForTests,
classifyFixTier,
buildFixDiagnostics,
escalateProviderFailure,
FIX_TIERS,
} = await import('./autoFixer.js');
const { errorEvents } = await import('../lib/errorHandler.js');
// initAutoFixer is idempotent — it must run once for the error-event
// listener to be attached, regardless of which test file boots first.
initAutoFixer();
// Tasks handed to the durable store (addTask's first argument).
function filedTasks() {
return addTask.mock.calls.map(([task]) => task);
}
function emitProviderFailure({ provider = 'Primary CLI', model = 'm-1', runId = 'run-1' } = {}) {
errorEvents.emit('error', {
code: 'AI_PROVIDER_EXECUTION_FAILED',
message: `AI provider ${provider} execution failed: boom`,
severity: 'error',
canAutoFix: true,
timestamp: Date.now(),
context: {
runId,
provider,
providerId: provider.toLowerCase().replace(/\s+/g, '-'),
model,
exitCode: 1,
},
});
}
describe('autoFixer — defer + noteFallbackHandled', () => {
beforeEach(() => {
vi.useFakeTimers();
addTask.mockClear();
isRunning.mockReturnValue(true);
_resetAutoFixerForTests();
});
afterEach(() => {
vi.useRealTimers();
_resetAutoFixerForTests();
});
it('defers task creation by ~5s instead of creating it immediately', async () => {
emitProviderFailure({ provider: 'Primary CLI', model: 'm-1' });
// Allow the synchronous emit + the IIFE's microtask queue to drain so
// the deferred timer is set before we assert on it.
await vi.advanceTimersByTimeAsync(0);
// Task is NOT created right away — it's deferred.
expect(addTask).not.toHaveBeenCalled();
// After the defer window elapses, the task is created.
await vi.advanceTimersByTimeAsync(5500);
expect(addTask).toHaveBeenCalledTimes(1);
expect(addTask.mock.calls[0][0]).toMatchObject({
description: 'Investigate AI provider failure: Primary CLI (m-1)',
});
});
it('logs the actual failure reason + category + exit code inline (not just the provider)', async () => {
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
try {
errorEvents.emit('error', {
code: 'AI_PROVIDER_EXECUTION_FAILED',
message: 'AI provider Claude Code CLI execution failed: …',
severity: 'error',
canAutoFix: true,
timestamp: Date.now(),
context: {
runId: 'run-bad-model',
provider: 'Claude Code CLI',
providerId: 'claude-code',
model: 'claude-opus-4-8',
exitCode: 1,
errorDetails: 'API Error (claude-opus-4-8): 400 The provided model identifier is invalid.',
errorAnalysis: { category: 'model-not-found', message: 'API Error (claude-opus-4-8): 400 The provided model identifier is invalid.' },
},
});
await vi.advanceTimersByTimeAsync(0);
const line = logSpy.mock.calls.map((c) => c[0]).find((m) => typeof m === 'string' && m.includes('AI provider error detected'));
expect(line).toBeTruthy();
expect(line).toContain('model identifier is invalid'); // the real reason, inline
expect(line).toContain('model-not-found'); // the category
expect(line).toContain('exit=1');
expect(line).toContain('claude-opus-4-8'); // the model
} finally {
logSpy.mockRestore();
}
});
it('publishes caller runtime exhaustion as a bounded warning without an investigation', async () => {
// Execute the real bootstrap hook without booting daemons/DB/schedulers.
const source = readFileSync(new URL('./bootstrap.js', import.meta.url), 'utf8')
.replace('onRunFailed: (metadata, error) => {', 'function failureHook(metadata, error) {');
const declaration = extractDeclaration(source, 'failureHook');
expect(declaration).toBeTruthy();
const hook = runInNewContext(`${declaration}; failureHook`, { errorEvents, ERROR_CATEGORIES, Date });
const observed = vi.fn();
errorEvents.on('error', observed);
try {
hook({ id: 'budget-run', providerName: 'Example API', providerId: 'example-api', model: 'm-1',
timeoutBound: 'absolute', timeoutOrigin: 'caller-budget', runtimeBudgetMs: 120000,
errorAnalysis: { category: 'runtime-budget-exhausted' } }, 'Caller runtime budget exhausted after 120000ms');
expect(observed).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({
code: 'AI_RUNTIME_BUDGET_EXHAUSTED', severity: 'warning', canAutoFix: false,
context: expect.objectContaining({ timeoutOrigin: 'caller-budget', runtimeBudgetMs: 120000 }),
}));
await vi.advanceTimersByTimeAsync(6000);
expect(addTask).not.toHaveBeenCalled();
} finally { errorEvents.off('error', observed); }
});
it('does not investigate caller budgets or poison dedupe for a later provider fault', async () => {
const budget = { code: 'AI_PROVIDER_EXECUTION_FAILED', severity: 'error', canAutoFix: true,
context: { provider: 'Example API', model: 'm-1', runId: 'budget-run',
errorAnalysis: { category: 'runtime-budget-exhausted' } } };
errorEvents.emit('error', budget);
await escalateProviderFailure(budget);
await vi.advanceTimersByTimeAsync(6000);
expect(addTask).not.toHaveBeenCalled();
emitProviderFailure({ provider: 'Example API', model: 'm-1', runId: 'fault-run' });
await vi.advanceTimersByTimeAsync(6000);
expect(addTask).toHaveBeenCalledTimes(1);
});
it('never creates an investigation task for a content/safety refusal', async () => {
// A refusal is self-explanatory (the model declined the prompt) — there's
// nothing for a CoS agent to investigate, and the fallback path handles
// recovery. Even if the failure arrives via AI_PROVIDER_EXECUTION_FAILED
// with refusal analysis, the guard must suppress the task.
errorEvents.emit('error', {
code: 'AI_PROVIDER_EXECUTION_FAILED',
message: 'AI provider Codex CLI execution failed: content refused',
severity: 'error',
canAutoFix: true,
timestamp: Date.now(),
context: {
runId: 'run-refuse', provider: 'Codex CLI', providerId: 'codex', model: 'm-1',
errorAnalysis: { category: 'content-refusal' },
},
});
await vi.advanceTimersByTimeAsync(6000);
expect(addTask).not.toHaveBeenCalled();
});
it('suppresses the task for a SLOW fallback that finishes after the defer window', async () => {
// The bug this fixes: a CLI fallback (Claude Code) can take 20–30s, far
// longer than TASK_DEFER_MS. noteFallbackStarted cancels the backstop timer
// immediately, so even though the success notice arrives late, no task fires.
const { noteFallbackStarted } = await import('./autoFixer.js');
emitProviderFailure({ provider: 'Ollama', model: 'command-r' });
await vi.advanceTimersByTimeAsync(0);
// Fallback starts almost immediately — cancels the deferred task.
noteFallbackStarted({ provider: 'Ollama', model: 'command-r' });
// The backstop window elapses with the fallback still running — no task.
await vi.advanceTimersByTimeAsync(6000);
expect(addTask).not.toHaveBeenCalled();
// Fallback finally succeeds ~25s later — still no task.
noteFallbackHandled({ provider: 'Ollama', model: 'command-r' });
await vi.advanceTimersByTimeAsync(25000);
expect(addTask).not.toHaveBeenCalled();
});
it('suppresses the task even when noteFallbackStarted races ahead of the error event', async () => {
// Microtask-ordering guard: if the fallback is announced before the error
// handler schedules its timer, the in-flight set still suppresses it.
const { noteFallbackStarted } = await import('./autoFixer.js');
noteFallbackStarted({ provider: 'Ollama', model: 'command-r' });
emitProviderFailure({ provider: 'Ollama', model: 'command-r' });
await vi.advanceTimersByTimeAsync(6000);
expect(addTask).not.toHaveBeenCalled();
});
it('noteFallbackFailed releases suppression so a later identical failure can raise a task', async () => {
const { noteFallbackStarted, noteFallbackFailed } = await import('./autoFixer.js');
noteFallbackStarted({ provider: 'Ollama', model: 'command-r' });
noteFallbackFailed({ provider: 'Ollama', model: 'command-r' });
// A fresh failure with no fallback in flight now schedules + fires a task.
emitProviderFailure({ provider: 'Ollama', model: 'command-r' });
await vi.advanceTimersByTimeAsync(6000);
expect(addTask).toHaveBeenCalledTimes(1);
});
it('cancels the deferred task when noteFallbackHandled fires within the window', async () => {
emitProviderFailure({ provider: 'Primary CLI', model: 'm-1' });
await vi.advanceTimersByTimeAsync(0);
// Fallback succeeded — call noteFallbackHandled BEFORE the defer elapses.
const handled = noteFallbackHandled({ provider: 'Primary CLI', model: 'm-1' });
expect(handled).toBe(true);
// Advancing past the defer window must NOT trigger task creation.
await vi.advanceTimersByTimeAsync(10000);
expect(addTask).not.toHaveBeenCalled();
});
it('returns false when noteFallbackHandled has no matching deferred task', async () => {
const handled = noteFallbackHandled({ provider: 'Unknown', model: 'nope' });
expect(handled).toBe(false);
});
it('matches the deferred task by exact provider+model (mismatched key does not cancel)', async () => {
emitProviderFailure({ provider: 'Primary CLI', model: 'm-1' });
await vi.advanceTimersByTimeAsync(0);
// Wrong model — should not cancel.
expect(noteFallbackHandled({ provider: 'Primary CLI', model: 'wrong-model' })).toBe(false);
// Wrong provider name — should not cancel.
expect(noteFallbackHandled({ provider: 'Other CLI', model: 'm-1' })).toBe(false);
await vi.advanceTimersByTimeAsync(5500);
expect(addTask).toHaveBeenCalledTimes(1);
});
it('allows a future failure of the same provider to raise a new task after suppression (no 60s dedupe lockout)', async () => {
emitProviderFailure({ provider: 'Primary CLI', model: 'm-1', runId: 'r-1' });
await vi.advanceTimersByTimeAsync(0);
noteFallbackHandled({ provider: 'Primary CLI', model: 'm-1' });
await vi.advanceTimersByTimeAsync(10000);
expect(addTask).not.toHaveBeenCalled();
// Same provider fails again later — must NOT be suppressed by the
// dedupe map (which would otherwise hold the key for 60s).
emitProviderFailure({ provider: 'Primary CLI', model: 'm-1', runId: 'r-2' });
await vi.advanceTimersByTimeAsync(0);
await vi.advanceTimersByTimeAsync(5500);
expect(addTask).toHaveBeenCalledTimes(1);
});
it('files a durable investigation task when CoS is not running', async () => {
isRunning.mockReturnValue(false);
emitProviderFailure({ provider: 'Primary CLI', model: 'm-1' });
await vi.advanceTimersByTimeAsync(5500);
expect(addTask).toHaveBeenCalledTimes(1);
expect(filedTasks()[0].description).toContain('Investigate AI provider failure: Primary CLI (m-1)');
expect(filedTasks()[0].isInvestigation).toBe(true);
// A repeat inside the dedupe window files nothing new.
emitProviderFailure({ provider: 'Primary CLI', model: 'm-1', runId: 'run-2' });
await vi.advanceTimersByTimeAsync(5500);
expect(addTask).toHaveBeenCalledTimes(1);
});
// The body built by buildAIProviderErrorContext has always fallen back to
// 'Unknown'/'N/A'; the title did not, so a failure that reached the hook with
// no attribution filed a task literally called
// "Investigate AI provider failure: undefined (undefined)".
it('never titles a task with a literal undefined provider or model', async () => {
isRunning.mockReturnValue(false);
// Emitted raw, not through emitProviderFailure — that helper defaults
// provider/model, which is exactly the attribution this case lacks.
errorEvents.emit('error', {
code: 'AI_PROVIDER_EXECUTION_FAILED',
message: 'AI provider undefined execution failed: TUI exited with code 1',
severity: 'error',
canAutoFix: true,
timestamp: Date.now(),
context: { exitCode: 1, duration: 86, errorDetails: 'TUI exited with code 1' },
});
await vi.advanceTimersByTimeAsync(5500);
const pending = filedTasks();
expect(pending).toHaveLength(1);
expect(pending[0].description).not.toMatch(/undefined/);
});
it('clears the dedupe entry when deferred task creation fails so future failures can still raise tasks', async () => {
// Simulate addTask rejecting (e.g. PLAN.md write error). Without the
// dedupe-clear in the catch arm, the next identical failure would be
// suppressed for 60s even though no investigation task ever landed.
addTask.mockRejectedValueOnce(new Error('plan write failed'));
emitProviderFailure({ provider: 'Primary CLI', model: 'm-1', runId: 'r-1' });
await vi.advanceTimersByTimeAsync(0);
await vi.advanceTimersByTimeAsync(5500);
// Drain the rejected addTask's microtask + the .catch's recentErrors.delete.
await vi.advanceTimersByTimeAsync(0);
// First attempt: addTask threw, so no task was created.
expect(addTask).toHaveBeenCalledTimes(1);
// Second identical failure within the 60s dedupe window must NOT be
// suppressed — the stale dedupe entry should have been cleared.
emitProviderFailure({ provider: 'Primary CLI', model: 'm-1', runId: 'r-2' });
await vi.advanceTimersByTimeAsync(5500);
expect(addTask).toHaveBeenCalledTimes(2);
});
it('does NOT collide on hyphenated provider/model pairs (uses an unambiguous separator)', async () => {
// Regression: a `-`-joined dedupe key would treat
// ("gpt-4o", "mini") and ("gpt", "4o-mini") as the same failure,
// silently suppressing one with the other's deferred task.
emitProviderFailure({ provider: 'gpt-4o', model: 'mini', runId: 'r-1' });
emitProviderFailure({ provider: 'gpt', model: '4o-mini', runId: 'r-2' });
await vi.advanceTimersByTimeAsync(5500);
// Both distinct failures must schedule their own investigation task.
expect(addTask).toHaveBeenCalledTimes(2);
});
it('dedupes within the defer window — a second identical failure does not double-schedule', async () => {
emitProviderFailure({ provider: 'Primary CLI', model: 'm-1', runId: 'r-1' });
await vi.advanceTimersByTimeAsync(0);
emitProviderFailure({ provider: 'Primary CLI', model: 'm-1', runId: 'r-2' });
await vi.advanceTimersByTimeAsync(5500);
// Only the first failure schedules a task; the second is dropped by
// either isDuplicateError or the deferredTasks.has guard.
expect(addTask).toHaveBeenCalledTimes(1);
});
});
function emitCriticalError(overrides = {}) {
errorEvents.emit('error', {
code: 'UNCAUGHT_EXCEPTION',
message: 'boom',
severity: 'critical',
canAutoFix: true,
timestamp: Date.now(),
...overrides,
});
}
describe('autoFixer — generic critical-error auto-fix path', () => {
beforeEach(() => {
vi.useFakeTimers();
addTask.mockClear();
isRunning.mockReturnValue(true);
_resetAutoFixerForTests();
});
afterEach(() => {
vi.useRealTimers();
_resetAutoFixerForTests();
});
it('files the fix task unattended — an isolated crash is what CoS diagnoses for itself', async () => {
emitCriticalError({ message: 'Cannot read properties of undefined', stack: 'Error: boom\n at foo (file.js:1:1)' });
await vi.advanceTimersByTimeAsync(0);
expect(addTask).toHaveBeenCalledTimes(1);
expect(addTask.mock.calls[0][0]).toMatchObject({
description: 'Fix critical error: Cannot read properties of undefined',
approvalRequired: false,
approvalReason: null,
});
});
it('triggers on canAutoFix even when severity is not critical', async () => {
emitCriticalError({ code: 'SOME_ERROR', message: 'recoverable-ish failure', severity: 'error' });
await vi.advanceTimersByTimeAsync(0);
expect(addTask).toHaveBeenCalledTimes(1);
expect(addTask.mock.calls[0][0]).toMatchObject({ approvalRequired: false });
});
it('stamps the fingerprint the loop policy reads, so a repeat of the same cause is recognizable', async () => {
emitCriticalError({ code: 'SOME_ERROR', message: 'first time', severity: 'error' });
await vi.advanceTimersByTimeAsync(0);
expect(addTask.mock.calls[0][0]).toMatchObject({
isInvestigation: true,
investigationFingerprint: 'unknown:critical-error:SOME_ERROR',
});
});
it('holds the fix task for a human when the same cause was already investigated today', async () => {
// A prior investigation carrying the fingerprint the production path above
// actually writes, settled an hour ago — the fix did not hold, so another
// unattended agent would just repeat it.
getAllTasks.mockResolvedValueOnce({
user: { tasks: [] },
cos: {
tasks: [{
id: 'sys-prior',
status: 'completed',
metadata: {
isInvestigation: true,
investigationFingerprint: 'unknown:critical-error:SOME_ERROR',
updatedAt: new Date(Date.now() - 60 * 60 * 1000).toISOString(),
}
}]
}
});
emitCriticalError({ code: 'SOME_ERROR', message: 'the same failure, again', severity: 'error' });
await vi.advanceTimersByTimeAsync(0);
expect(addTask).toHaveBeenCalledTimes(1);
expect(addTask.mock.calls[0][0]).toMatchObject({
approvalRequired: true,
approvalReason: 'investigation-loop:repeat-fingerprint',
});
expect(addTask.mock.calls[0][0].description).toContain('Why this is held for you');
});
it('does not fire for a non-critical error that is not marked auto-fixable', async () => {
emitCriticalError({ code: 'VALIDATION_ERROR', message: 'bad input', severity: 'error', canAutoFix: false });
await vi.advanceTimersByTimeAsync(0);
expect(addTask).not.toHaveBeenCalled();
});
it('does not fire when CoS is not running', async () => {
isRunning.mockReturnValue(false);
emitCriticalError();
await vi.advanceTimersByTimeAsync(0);
expect(addTask).not.toHaveBeenCalled();
});
it('dedupes identical critical errors within the window', async () => {
emitCriticalError();
emitCriticalError();
await vi.advanceTimersByTimeAsync(0);
expect(addTask).toHaveBeenCalledTimes(1);
});
});
describe('autoFixer — circuit breaker (guardrail #3)', () => {
beforeEach(() => {
vi.useFakeTimers();
addTask.mockClear();
isRunning.mockReturnValue(true);
_resetAutoFixerForTests();
});
afterEach(() => {
vi.useRealTimers();
_resetAutoFixerForTests();
});
// Distinct same-resource failures must be spaced past the 60s dedupe window,
// otherwise isDuplicateError swallows them before the circuit ever counts.
const DEDUPE_GAP_MS = 61000;
it('AI-provider path: suppresses the task once the same resource fails >3 times within the hour', async () => {
// First 3 distinct failures each raise an investigation task.
for (let i = 0; i < 3; i++) {
emitProviderFailure({ provider: 'Ollama', model: 'command-r', runId: `r-${i}` });
await vi.advanceTimersByTimeAsync(5500); // let the deferred task fire
await vi.advanceTimersByTimeAsync(DEDUPE_GAP_MS - 5500); // clear the dedupe window
}
expect(addTask).toHaveBeenCalledTimes(3);
// 4th failure within the same hour trips the circuit — no task scheduled.
emitProviderFailure({ provider: 'Ollama', model: 'command-r', runId: 'r-4' });
await vi.advanceTimersByTimeAsync(6000);
expect(addTask).toHaveBeenCalledTimes(3);
});
it('AI-provider path: the circuit auto-closes once failures age out of the 1h window', async () => {
for (let i = 0; i < 4; i++) {
emitProviderFailure({ provider: 'Ollama', model: 'command-r', runId: `r-${i}` });
await vi.advanceTimersByTimeAsync(5500);
await vi.advanceTimersByTimeAsync(DEDUPE_GAP_MS - 5500);
}
// 3 tasks (4th suppressed by the open circuit).
expect(addTask).toHaveBeenCalledTimes(3);
// Let the whole failure burst age past the rolling 1h window.
await vi.advanceTimersByTimeAsync(60 * 60 * 1000 + 1000);
// A fresh failure now raises a task again — the circuit auto-closed.
emitProviderFailure({ provider: 'Ollama', model: 'command-r', runId: 'r-late' });
await vi.advanceTimersByTimeAsync(5500);
expect(addTask).toHaveBeenCalledTimes(4);
});
it('AI-provider path: the circuit is per-resource — a different provider is unaffected', async () => {
for (let i = 0; i < 4; i++) {
emitProviderFailure({ provider: 'Ollama', model: 'command-r', runId: `a-${i}` });
await vi.advanceTimersByTimeAsync(5500);
await vi.advanceTimersByTimeAsync(DEDUPE_GAP_MS - 5500);
}
expect(addTask).toHaveBeenCalledTimes(3); // Ollama circuit open
// A distinct provider/model still gets its investigation task.
emitProviderFailure({ provider: 'LM Studio', model: 'qwen', runId: 'b-0' });
await vi.advanceTimersByTimeAsync(5500);
expect(addTask).toHaveBeenCalledTimes(4);
});
it('AI-provider path: fallback-recovered failures do NOT count toward the circuit', async () => {
// A failure that a fallback recovers never produces an investigation task,
// so it must not push the resource toward the circuit threshold. Otherwise
// a provider that always recovers via fallback would eventually suppress a
// GENUINE unrecovered failure's investigation task.
const { noteFallbackStarted, noteFallbackHandled } = await import('./autoFixer.js');
for (let i = 0; i < 4; i++) {
emitProviderFailure({ provider: 'Ollama', model: 'command-r', runId: `f-${i}` });
await vi.advanceTimersByTimeAsync(0);
noteFallbackStarted({ provider: 'Ollama', model: 'command-r' }); // cancels the deferred timer
noteFallbackHandled({ provider: 'Ollama', model: 'command-r' }); // success — clears dedupe/in-flight
await vi.advanceTimersByTimeAsync(DEDUPE_GAP_MS);
}
expect(addTask).not.toHaveBeenCalled();
// A genuine unrecovered failure now STILL raises a task — the circuit never
// opened because recovered failures were never counted.
emitProviderFailure({ provider: 'Ollama', model: 'command-r', runId: 'real' });
await vi.advanceTimersByTimeAsync(5500);
expect(addTask).toHaveBeenCalledTimes(1);
});
it('generic critical-error path: suppresses the fix task once the same error fires >3 times within the hour', async () => {
for (let i = 0; i < 3; i++) {
emitCriticalError({ message: 'recurring boom' });
await vi.advanceTimersByTimeAsync(0);
await vi.advanceTimersByTimeAsync(DEDUPE_GAP_MS);
}
expect(addTask).toHaveBeenCalledTimes(3);
emitCriticalError({ message: 'recurring boom' });
await vi.advanceTimersByTimeAsync(0);
expect(addTask).toHaveBeenCalledTimes(3); // circuit open — suppressed
});
});
describe('autoFixer — tiered fallback classifier (guardrail #1, issue #2328)', () => {
it('maps config/env-fixable categories to Tier 1', () => {
for (const cat of ['auth-error', 'forbidden', 'model-not-found', 'model-not-supported',
'quota-exceeded', 'billing-error', 'usage-limit', 'spawn-error', 'permission-denied', 'file-not-found']) {
expect(classifyFixTier(cat).tier, cat).toBe(FIX_TIERS.CONFIG_ENV);
}
});
it('maps schema/type/format categories to Tier 2', () => {
for (const cat of ['parse-error', 'bad-request', 'context-length', 'output-length', 'build-error', 'lint-error']) {
expect(classifyFixTier(cat).tier, cat).toBe(FIX_TIERS.SCHEMA_TYPE);
}
});
it('maps transient/recoverable categories to Tier 3 (constrained-agent-retry)', () => {
for (const cat of ['rate-limit', 'network-error', 'timeout', 'server-error', 'tool-error',
'mcp-error', 'test-failure', 'npm-error', 'memory-error', 'turn-limit']) {
expect(classifyFixTier(cat).tier, cat).toBe(FIX_TIERS.CONSTRAINED_RETRY);
}
});
it('maps human-judgement categories to Tier 4 (escalate)', () => {
for (const cat of ['content-refusal', 'content-filtered', 'task-rejected', 'git-conflict', 'unknown']) {
expect(classifyFixTier(cat).tier, cat).toBe(FIX_TIERS.ESCALATE);
}
});
it('escalates unknown/absent categories to Tier 4 (no silent swallow)', () => {
expect(classifyFixTier('some-brand-new-category').tier).toBe(FIX_TIERS.ESCALATE);
expect(classifyFixTier(undefined).tier).toBe(FIX_TIERS.ESCALATE);
expect(classifyFixTier('').tier).toBe(FIX_TIERS.ESCALATE);
});
it('returns a stable {tier, strategy, label} shape', () => {
const t1 = classifyFixTier('auth-error');
expect(t1).toMatchObject({ tier: 1, strategy: 'config/env' });
expect(typeof t1.label).toBe('string');
expect(classifyFixTier('rate-limit').strategy).toBe('constrained-agent-retry');
expect(classifyFixTier('unknown').strategy).toBe('escalate');
});
});
describe('autoFixer — structured per-attempt diagnostics (issue #2328)', () => {
it('builds a full diagnostics record from trigger/target/category/reason', () => {
const d = buildFixDiagnostics({
triggerEvent: 'AI_PROVIDER_EXECUTION_FAILED',
target: 'Claude Code CLI (claude-opus-4-8)',
category: 'model-not-found',
failureReason: 'API Error (claude-opus-4-8): 400 The provided model identifier is invalid.',
});
expect(d).toMatchObject({
triggerEvent: 'AI_PROVIDER_EXECUTION_FAILED',
target: 'Claude Code CLI (claude-opus-4-8)',
errorType: 'model-not-found',
category: 'model-not-found',
tier: FIX_TIERS.CONFIG_ENV,
fixStrategy: 'config/env',
});
expect(d.failureReason).toContain('model identifier is invalid');
});
it('collapses a multi-line failure reason to one line and fills sensible defaults', () => {
const d = buildFixDiagnostics({ failureReason: 'line one\n\n line two \nline three' });
expect(d.failureReason).toBe('line one line two line three');
expect(d.triggerEvent).toBe('unknown');
expect(d.target).toBe('unknown');
expect(d.category).toBe('unknown');
expect(d.tier).toBe(FIX_TIERS.ESCALATE);
});
it("reports 'no error text captured' when no reason is supplied", () => {
expect(buildFixDiagnostics({ category: 'auth-error' }).failureReason).toBe('no error text captured');
});
it('stamps observedAt — the injected value when given, else a valid ISO now (#2328)', () => {
const at = '2026-07-09T10:00:00.000Z';
expect(buildFixDiagnostics({ category: 'auth-error', observedAt: at }).observedAt).toBe(at);
// Default: a parseable ISO timestamp so the telemetry aggregator can derive
// time-to-recovery from it.
const defaulted = buildFixDiagnostics({ category: 'auth-error' }).observedAt;
expect(typeof defaulted).toBe('string');
expect(Number.isFinite(Date.parse(defaulted))).toBe(true);
});
});
describe('autoFixer — diagnostics ride on the created task record (issue #2328)', () => {
beforeEach(() => {
vi.useFakeTimers();
addTask.mockClear();
isRunning.mockReturnValue(true);
_resetAutoFixerForTests();
});
afterEach(() => {
vi.useRealTimers();
_resetAutoFixerForTests();
});
it('attaches tier-classified diagnostics to the AI-provider investigation task', async () => {
errorEvents.emit('error', {
code: 'AI_PROVIDER_EXECUTION_FAILED',
message: 'AI provider Claude Code CLI execution failed: …',
severity: 'error',
canAutoFix: true,
timestamp: Date.now(),
context: {
runId: 'run-diag', provider: 'Claude Code CLI', providerId: 'claude-code',
model: 'claude-opus-4-8', exitCode: 1,
errorDetails: 'API Error (claude-opus-4-8): 400 The provided model identifier is invalid.',
errorAnalysis: { category: 'model-not-found', message: 'model identifier is invalid' },
},
});
await vi.advanceTimersByTimeAsync(5500);
expect(addTask).toHaveBeenCalledTimes(1);
const taskArg = addTask.mock.calls[0][0];
expect(taskArg.diagnostics).toMatchObject({
triggerEvent: 'AI_PROVIDER_EXECUTION_FAILED',
target: 'Claude Code CLI (claude-opus-4-8)',
category: 'model-not-found',
tier: FIX_TIERS.CONFIG_ENV,
fixStrategy: 'config/env',
});
expect(taskArg.diagnostics.failureReason).toContain('model identifier is invalid');
// Diagnostics are also embedded in the agent-facing context markdown.
expect(taskArg.context).toContain('## Fallback Tier');
expect(taskArg.context).toContain('Tier:** 1 (config/env)');
});
it('emits the tier inline on the "AI provider error detected" log line', async () => {
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
try {
emitProviderFailure({ provider: 'Ollama', model: 'command-r' });
await vi.advanceTimersByTimeAsync(0);
const line = logSpy.mock.calls.map((c) => c[0]).find((m) => typeof m === 'string' && m.includes('AI provider error detected'));
expect(line).toBeTruthy();
expect(line).toMatch(/tier=\d/);
} finally {
logSpy.mockRestore();
}
});
it('attaches diagnostics to the filed task when CoS is not running', async () => {
isRunning.mockReturnValue(false);
emitProviderFailure({ provider: 'LM Studio', model: 'qwen', runId: 'p-1' });
await vi.advanceTimersByTimeAsync(5500);
const pending = filedTasks();
expect(pending).toHaveLength(1);
expect(pending[0].diagnostics).toMatchObject({
triggerEvent: 'AI_PROVIDER_EXECUTION_FAILED',
target: 'LM Studio (qwen)',
tier: FIX_TIERS.ESCALATE, // no errorAnalysis category → unknown → escalate
});
});
it('attaches diagnostics to the generic critical-error fix task', async () => {
emitCriticalError({ code: 'UNCAUGHT_EXCEPTION', message: 'Cannot read properties of undefined' });
await vi.advanceTimersByTimeAsync(0);
expect(addTask).toHaveBeenCalledTimes(1);
expect(addTask.mock.calls[0][0].diagnostics).toMatchObject({
triggerEvent: 'UNCAUGHT_EXCEPTION',
target: 'UNCAUGHT_EXCEPTION',
tier: FIX_TIERS.ESCALATE,
});
});
});
describe('autoFixer — escalateProviderFailure (explicit Tier-4, issue #2342)', () => {
beforeEach(() => {
addTask.mockClear();
isRunning.mockReturnValue(true);
_resetAutoFixerForTests();
});
afterEach(() => {
_resetAutoFixerForTests();
});
const providerError = (provider = 'Primary API', model = 'primary-model') => ({
code: 'AI_PROVIDER_EXECUTION_FAILED',
message: `AI provider ${provider} execution failed: model gone`,
timestamp: Date.now(),
context: {
provider, providerId: provider.toLowerCase().replace(/\s+/g, '-'), model,
errorDetails: 'model gone', errorAnalysis: { category: 'model-not-found' },
},
});
it('creates an investigation task immediately (no defer window)', async () => {
const task = await escalateProviderFailure(providerError());
expect(addTask).toHaveBeenCalledTimes(1);
expect(addTask.mock.calls[0][0]).toMatchObject({
description: 'Investigate AI provider failure: Primary API (primary-model)',
});
// Diagnostics still ride on the escalated task (tier-classified from category).
expect(addTask.mock.calls[0][0].diagnostics).toMatchObject({
category: 'model-not-found',
tier: FIX_TIERS.CONFIG_ENV,
});
expect(task).toBeTruthy();
});
it('dedupes concurrent identical escalations to a single task (no-fallback failure storm)', async () => {
const results = await Promise.all([
escalateProviderFailure(providerError('Ollama', 'command-r')),
escalateProviderFailure(providerError('Ollama', 'command-r')),
escalateProviderFailure(providerError('Ollama', 'command-r')),
]);
// Only the first escalation creates a task; the rest are deduped within the
// window (returns null).
expect(addTask).toHaveBeenCalledTimes(1);
expect(results.filter(Boolean)).toHaveLength(1);
});
it('honors the per-resource circuit breaker (suppresses after >3 escalations in the window)', async () => {
vi.useFakeTimers();
try {
// Space escalations past the 60s dedupe window so each counts as a
// distinct task-worthy failure toward the circuit threshold.
for (let i = 0; i < 3; i++) {
await escalateProviderFailure(providerError('Ollama', 'command-r'));
await vi.advanceTimersByTimeAsync(61000);
}
expect(addTask).toHaveBeenCalledTimes(3);
// 4th distinct escalation (still within the 1h circuit window) trips it.
const suppressed = await escalateProviderFailure(providerError('Ollama', 'command-r'));
expect(suppressed).toBeNull();
expect(addTask).toHaveBeenCalledTimes(3);
} finally {
vi.useRealTimers();
}
});
});
describe('autoFixer — escalateProviderFailure dedupe clears on task-creation failure (#2342)', () => {
beforeEach(() => {
addTask.mockReset();
isRunning.mockReturnValue(true);
_resetAutoFixerForTests();
});
afterEach(() => {
addTask.mockReset();
addTask.mockResolvedValue({ id: 'task-1' });
_resetAutoFixerForTests();
});
it('does not suppress a retry when the escalated task creation fails', async () => {
const err = {
code: 'AI_PROVIDER_EXECUTION_FAILED',
message: 'AI provider Ollama execution failed: model gone',
timestamp: Date.now(),
context: { provider: 'Ollama', providerId: 'ollama', model: 'command-r', errorDetails: 'model gone', errorAnalysis: { category: 'model-not-found' } },
};
// First escalation: addTask rejects → dedupe marker must be cleared.
addTask.mockRejectedValueOnce(new Error('plan write failed'));
const first = await (await import('./autoFixer.js')).escalateProviderFailure(err);
expect(first).toBeNull();
// Second identical escalation within the window must NOT be deduped — it
// creates the task the failed first attempt never did.
addTask.mockResolvedValueOnce({ id: 'task-ok' });
const second = await (await import('./autoFixer.js')).escalateProviderFailure(err);
expect(second).toBeTruthy();
expect(addTask).toHaveBeenCalledTimes(2);
});
});
// A server still running code from before the current checkout can keep
// reproducing a failure that is already fixed on disk — a local-LLM playground
// timeout was filed a second time hours after #5771 fixed it, and the
// investigation agent had to rediscover that from scratch. The task body now
// says so up front.
describe('autoFixer — stale deployed build in the investigation body', () => {
beforeEach(() => {
vi.useFakeTimers();
addTask.mockClear();
isRunning.mockReturnValue(false);
installState.getBootCommit.mockReturnValue(null);
installState.getInstallState.mockClear();
installState.getInstallState.mockResolvedValue(null);
_resetAutoFixerForTests();
});
afterEach(() => {
vi.useRealTimers();
_resetAutoFixerForTests();
});
it('names the boot-vs-HEAD gap and leads the steps with ruling it out', async () => {
installState.getBootCommit.mockReturnValue('aaaaaaa1111111111111111111111111111111a');
installState.getInstallState.mockResolvedValue({
runningStaleCode: true,
bootCommit: 'aaaaaaa1111111111111111111111111111111a',
currentCommit: 'bbbbbbb2222222222222222222222222222222b',
});
emitProviderFailure({ provider: 'Ollama', model: 'm-1' });
await vi.advanceTimersByTimeAsync(5500);
const { context } = filedTasks()[0];
expect(context).toContain('## Deployed Build: STALE');
expect(context).toContain('git log aaaaaaa..bbbbbbb');
expect(context).toContain('1. Rule out the stale deployed build');
// The pre-existing steps keep their order, just renumbered behind it.
expect(context).toContain('2. Check if the AI provider is configured correctly');
expect(context).toContain('7. Review the output tail for specific error messages');
});
it('omits the section — and never probes git — when the build is current', async () => {
installState.getBootCommit.mockReturnValue('aaaaaaa1111111111111111111111111111111a');
installState.getInstallState.mockResolvedValue({
runningStaleCode: false,
bootCommit: 'aaaaaaa1111111111111111111111111111111a',
currentCommit: 'aaaaaaa1111111111111111111111111111111a',
});
emitProviderFailure({ provider: 'Ollama', model: 'm-1' });
await vi.advanceTimersByTimeAsync(5500);
const { context } = filedTasks()[0];
expect(context).not.toContain('Deployed Build');
expect(context).toContain('1. Check if the AI provider is configured correctly');
});
// No captured boot commit (tarball install, or any process that never called
// captureBootCommit) makes the comparison meaningless — skip the git/fs work
// entirely rather than filing an "unknown..unknown" section.
it('skips the install-state probe when no boot commit was captured', async () => {
installState.getBootCommit.mockReturnValue(null);
emitProviderFailure({ provider: 'Ollama', model: 'm-1' });
await vi.advanceTimersByTimeAsync(5500);
expect(installState.getInstallState).not.toHaveBeenCalled();
expect(filedTasks()[0].context).not.toContain('Deployed Build');
});
});