Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { setFallbackChainCache } from '../lib/providerFallbackChainCache.js';
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
// Usage-limit tests exercise routing, not the install's persisted usage ledger.
vi.mock('./usage.js', () => ({ recordLimitBlock: vi.fn().mockResolvedValue(undefined) }));
vi.mock('./runner.js', () => ({
createRun: vi.fn(),
executeApiRun: vi.fn(),
executeCliRun: vi.fn(),
hasModelFlag: vi.fn(() => false),
extractBakedModel: vi.fn(() => null),
// promptRunner.js imports stopRun for the API-timeout cancel path —
// without this mock, the timer firing would TypeError on `stopRun is
// not a function` and crash any test that triggers the API timeout.
stopRun: vi.fn().mockResolvedValue(undefined),
// Called best-effort by promptRunner.js when createRun proactively
// swapped to a fallback provider — the metadata patch updates the
// run record's providerId/model so /runs attribution matches what
// actually ran. Mocked as a no-op resolve.
patchRunMetadata: vi.fn().mockResolvedValue(undefined),
// Settles a run record a pre-execution refusal throws past (the TUI readiness
// check and the context gate), so the Runs page never shows an attempt that
// never ends.
finalizeRunRecord: vi.fn().mockResolvedValue(undefined),
}));
// TUI runner is in lib (different module from services/runner.js) — mock it
// here so the central handler's tui branch is testable without spawning a
// real PTY. executeTuiRun is responsible for its own response cleanup
// (file-write directive + screen-scrape fallback), so the central handler
// just forwards result.text. Tests drive the cleaned text directly via the
// mock's onComplete payload.
vi.mock('./tuiPromptRunner.js', () => ({
executeTuiRun: vi.fn(),
}));
vi.mock('./providerExecutionReadiness.js', () => ({
ensureProviderReadyForExecution: vi.fn().mockResolvedValue({ success: true }),
}));
// providers.js is a compatibility shim that throws when the toolkit hasn't
// been initialized via setAIToolkit(). Mock it so the resolveProviderAndModel
// tests can drive the active/by-id lookups directly. `getAllProviders` is
// mocked too so the retry-with-fallback path (which enumerates providers to
// look up the configured fallback) can be driven directly from tests.
vi.mock('./providers.js', () => ({
getActiveProvider: vi.fn(),
getProviderById: vi.fn(),
getAllProviders: vi.fn().mockResolvedValue({ activeProvider: null, providers: [] }),
}));
// initAutoFixer() registers these at boot; runPromptThroughProvider reports
// fallback lifecycle through the registry. Spies stand in for the auto-fixer
// so tests can assert it was/wasn't invoked without wiring up the task system.
const autoFixer = {
noteFallbackStarted: vi.fn(),
noteFallbackHandled: vi.fn(),
noteFallbackFailed: vi.fn(),
escalateProviderFailure: vi.fn().mockResolvedValue(undefined),
};
// aiToolkitState lookups gate the retry path on a real providerStatus
// service being present. By default the mock returns null so the retry
// short-circuits and the original error is rethrown — individual tests
// override the return value to enable the retry branch.
vi.mock('../lib/aiToolkitState.js', () => ({
getAIToolkitInstance: vi.fn().mockReturnValue(null),
}));
// The pre-dispatch context gate reads the daemon's live `/v1/models` through
// this module (lazily). Mocked so the suite can drive "the endpoint says 32K"
// without a daemon; the default is the no-observation answer, which is what
// every provider that is not daemon-backed (and every install whose daemon is
// down) gets — so the whole rest of the suite routes untouched.
vi.mock('./observedContextWindows.js', () => ({
withObservedContextWindows: vi.fn(async (provider) => provider),
}));
const runner = await import('./runner.js');
const tuiRunner = await import('./tuiPromptRunner.js');
const executionReadiness = await import('./providerExecutionReadiness.js');
const providers = await import('./providers.js');
(await import('./providerRecoveryHooks.js')).registerProviderRecoveryHandlers(autoFixer);
const toolkitState = await import('../lib/aiToolkitState.js');
const observedWindows = await import('./observedContextWindows.js');
const { ERROR_CATEGORIES } = await import('../lib/aiToolkit/errorDetection.js');
const { apiRunAbsoluteTimeoutMs } = await import('../lib/aiToolkit/internal/runTimeouts.js');
const { CREATIVE_LATITUDE_HEADING, withCreativeLatitude } = await import('../lib/creativeLatitude.js');
const { runPromptThroughProvider, resolveProviderAndModel, resolveEffectiveModel, pickConfigCorrectedModel, normalizeResponseSchema, coerceResponseToSchema, isSchemaTypeCategory, buildRequestCapabilities, assertVisionRunUsedImages, resolveLocalPromptBudget } = await import('./promptRunner.js');
const apiProvider = (extra = {}) => ({
id: 'mock-api', type: 'api', defaultModel: 'm-default', ...extra,
});
const cliProvider = (extra = {}) => ({
id: 'codex', type: 'cli', defaultModel: 'm-default', timeout: 5000, ...extra,
});
const tuiProvider = (extra = {}) => ({
id: 'claude-code-tui', type: 'tui', defaultModel: 'm-default', timeout: 5000, ...extra,
});
beforeEach(() => {
setFallbackChainCache([]);
vi.clearAllMocks();
runner.createRun.mockResolvedValue({ runId: 'run-xyz' });
// vi.clearAllMocks() clears calls but NOT mockReturnValue overrides,
// so the default implementations set in vi.mock() above don't auto-reset
// between tests. Re-apply the defaults that individual tests override
// (hasModelFlag, extractBakedModel) so a leak-over value can't poison
// the next test's resolveEffectiveModel run.
runner.hasModelFlag.mockReturnValue(false);
runner.extractBakedModel.mockReturnValue(null);
// Defaults reset for fallback-path mocks too — same staleness concern.
providers.getAllProviders.mockResolvedValue({ activeProvider: null, providers: [] });
toolkitState.getAIToolkitInstance.mockReturnValue(null);
observedWindows.withObservedContextWindows.mockImplementation(async (provider) => provider);
});
describe('promptRunner — happy paths', () => {
it('routes CLI providers through executeCliRun, accumulates text, resolves { text, runId, model }', async () => {
runner.executeCliRun.mockImplementation(async ({ onData, onComplete }) => {
onData('hello ');
onData('world');
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: cliProvider(),
prompt: 'p',
source: 'test',
});
// `provider` is the effective provider that actually ran (reflects a
// proactive createRun swap); here no swap, so it echoes the input.
expect(out).toEqual({ text: 'hello world', runId: 'run-xyz', model: 'm-default', provider: cliProvider() });
expect(runner.executeCliRun).toHaveBeenCalledTimes(1);
expect(runner.executeApiRun).not.toHaveBeenCalled();
});
it('uses the CLI final response even when diagnostics contain valid prompt JSON', async () => {
runner.executeCliRun.mockImplementation(async ({ onData, onComplete }) => {
onData('OpenAI Codex v1\nuser\n{"message":"example"}\n');
onData('{"message":"actual answer"}');
onComplete({ success: true, text: '{"message":"actual answer"}' });
});
const out = await runPromptThroughProvider({ provider: cliProvider(), prompt: 'p', source: 'test' });
expect(JSON.parse(out.text)).toEqual({ message: 'actual answer' });
});
it('forwards images through the ordinary Codex CLI lifecycle and rejects unsupported CLIs', async () => {
const codex = cliProvider({ command: 'codex' });
runner.executeCliRun.mockImplementation(async ({ screenshots, onComplete }) => {
expect(screenshots).toEqual(['/tmp/example.png']);
onComplete({ success: true });
});
const output = await runPromptThroughProvider({
provider: codex,
prompt: 'Inspect the image.',
source: 'test',
screenshots: ['/tmp/example.png'],
allowFallback: false,
});
expect(assertVisionRunUsedImages(output, codex)).toBe(codex);
await expect(runPromptThroughProvider({
provider: cliProvider({ command: 'opencode' }),
prompt: 'Inspect the image.',
source: 'test',
screenshots: ['/tmp/example.png'],
})).rejects.toMatchObject({ code: 'VISION_PROVIDER_UNSUPPORTED' });
});
it('forwards image attachments through the pinned Codex TUI', async () => {
const provider = { id: 'codex-tui', type: 'tui', command: 'codex', defaultModel: 'gpt-6-astra' };
tuiRunner.executeTuiRun.mockImplementation(async ({ screenshots, onComplete }) => {
expect(screenshots).toEqual(['/tmp/example.png']);
onComplete({ success: true, text: 'Image description' });
});
const output = await runPromptThroughProvider({
provider, prompt: 'Inspect image', screenshots: ['/tmp/example.png'], source: 'test', allowFallback: false,
});
expect(output.text).toBe('Image description');
expect(assertVisionRunUsedImages(output, provider)).toBe(provider);
expect(runner.executeCliRun).not.toHaveBeenCalled();
await expect(runPromptThroughProvider({
provider: { ...provider, command: 'claude' }, prompt: 'Inspect image',
screenshots: ['/tmp/example.png'], source: 'test', allowFallback: false,
})).rejects.toMatchObject({ code: 'VISION_PROVIDER_UNSUPPORTED' });
});
it('routes API providers through executeApiRun, accumulates text, resolves { text, runId, model }', async () => {
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('foo');
onData({ text: 'bar' }); // API streams sometimes ship {text} chunks
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: apiProvider(),
prompt: 'p',
source: 'test',
model: 'gpt-test',
});
expect(out).toEqual({ text: 'foobar', runId: 'run-xyz', model: 'gpt-test', provider: apiProvider() });
expect(runner.executeApiRun).toHaveBeenCalledTimes(1);
expect(runner.executeCliRun).not.toHaveBeenCalled();
});
it('passes explicit API output and absolute spending caps without changing the route', async () => {
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => { onData('bounded'); onComplete({ success: true }); });
await runPromptThroughProvider({ provider: apiProvider(), prompt: 'p', source: 'test',
model: 'gpt-test', allowFallback: false, maxTokens: 8192, absoluteTimeoutMs: 120000 });
expect(runner.executeApiRun).toHaveBeenCalledWith(expect.objectContaining({ maxTokens: 8192, absoluteTimeoutMs: 120000, model: 'gpt-test' }));
});
it('reports the concrete run lifecycle so a parent workflow can stop it', async () => {
const events = [];
runner.executeApiRun.mockImplementation(async ({ runId, onComplete }) => {
events.push(`execute:${runId}`);
onComplete({ success: true });
});
await runPromptThroughProvider({
provider: apiProvider(),
prompt: 'p',
source: 'test',
onRunCreated: (runId) => events.push(`created:${runId}`),
onRunSettled: (runId) => events.push(`settled:${runId}`),
});
expect(events).toEqual(['created:run-xyz', 'execute:run-xyz', 'settled:run-xyz']);
});
it('forwards the attachable-shell readiness callback for TUI providers', async () => {
const onRunReady = vi.fn();
tuiRunner.executeTuiRun.mockImplementation(async ({ runId, provider, onReady, onComplete }) => {
onReady?.({
runId,
providerId: provider.id,
providerName: provider.name || provider.id,
model: provider.defaultModel,
providerType: provider.type,
shellReady: true,
});
onComplete({ success: true });
});
await runPromptThroughProvider({
provider: tuiProvider({ name: 'Codex TUI' }),
prompt: 'p',
source: 'test',
onRunReady,
});
expect(onRunReady).toHaveBeenCalledWith({
runId: 'run-xyz',
providerId: 'claude-code-tui',
providerName: 'Codex TUI',
model: 'm-default',
providerType: 'tui',
shellReady: true,
});
});
it('forwards screenshots to executeApiRun for a vision/multimodal call', async () => {
runner.executeApiRun.mockImplementation(async ({ onComplete }) => onComplete({ success: true }));
await runPromptThroughProvider({
provider: apiProvider(),
prompt: 'describe',
source: 'test',
model: 'gpt-vision',
screenshots: ['a.png', 'b.png'],
});
expect(runner.executeApiRun).toHaveBeenCalledWith(
expect.objectContaining({ screenshots: ['a.png', 'b.png'] }),
);
expect(runner.createRun).toHaveBeenCalledWith(expect.objectContaining({
requestCapabilities: { hasImages: true, requiredContextTokens: 8_002 },
}));
});
it('builds a request budget from prompt input plus the caller output reserve', () => {
expect(buildRequestCapabilities({
prompt: '12345678', screenshots: [], outputReserveTokens: 1_000,
})).toEqual({ hasImages: false, requiredContextTokens: 1_002 });
});
it('refuses an ineligible EXPLICIT provider under a caller mode policy, before any run record', async () => {
// A CLI/API-only caller must not run a TUI route it was handed directly —
// and it must fail before createRun, or a refused call still leaves a run
// record claiming a provider that never executed.
await expect(runPromptThroughProvider({
provider: tuiProvider(),
prompt: 'p',
source: 'test',
callerPolicy: 'cli-harness',
})).rejects.toMatchObject({ code: 'PROVIDER_MODE_NOT_PERMITTED' });
expect(runner.createRun).not.toHaveBeenCalled();
expect(runner.executeCliRun).not.toHaveBeenCalled();
});
it('sends the caller mode policy to fallback selection alongside the request budget', async () => {
runner.executeApiRun.mockImplementation(async ({ onComplete }) => onComplete({ success: true }));
await runPromptThroughProvider({
provider: apiProvider(), prompt: 'p', source: 'test', callerPolicy: 'direct-api',
});
// Carried on requestCapabilities so createRun's PROACTIVE swap is bound by
// the same rule the explicit pin was — not just the pin.
expect(runner.createRun).toHaveBeenCalledWith(expect.objectContaining({
requestCapabilities: expect.objectContaining({ allowedModes: ['api'] }),
}));
// An undeclared caller keeps the unconstrained shape it always had.
expect(buildRequestCapabilities({ prompt: 'p', screenshots: [] }).allowedModes).toBeUndefined();
});
it('defaults screenshots to [] when omitted', async () => {
runner.executeApiRun.mockImplementation(async ({ onComplete }) => onComplete({ success: true }));
await runPromptThroughProvider({ provider: apiProvider(), prompt: 'p', source: 'test' });
expect(runner.executeApiRun).toHaveBeenCalledWith(
expect.objectContaining({ screenshots: [] }),
);
});
it('runs the fallbackModel (not the primary model) when createRun proactively swaps to a fallback', async () => {
// Primary is benched at call time, so the toolkit createRun swaps to an
// API fallback and surfaces the configured fallbackModel. The run must
// execute that model on the fallback — NOT the primary's resolved model
// (the leak that sent `codex-configured-default` to LM Studio). This is
// the common caller path (no pre-created runId), distinct from the
// runtime-retry path covered below.
const fallback = apiProvider({ id: 'fb-api', defaultModel: 'fb-default' });
runner.createRun.mockResolvedValue({
runId: 'run-fb',
provider: fallback,
fallbackModel: 'pinned-fb',
});
let ranModel;
runner.executeApiRun.mockImplementation(async ({ model, onData, onComplete }) => {
ranModel = model;
onData('ok');
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: cliProvider({ defaultModel: 'codex-configured-default' }),
prompt: 'p',
source: 'test',
});
expect(runner.executeApiRun).toHaveBeenCalledTimes(1);
expect(runner.executeCliRun).not.toHaveBeenCalled();
expect(ranModel).toBe('pinned-fb');
expect(out.model).toBe('pinned-fb');
// The resolved result reports the proactively-swapped provider (not the
// requested one), so callers can detect a swap even though usedFallback
// stays unset on this path.
expect(out.provider).toEqual(fallback);
});
it('forwards the provider id + model + source to createRun', async () => {
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('ok');
onComplete({ success: true });
});
await runPromptThroughProvider({
provider: apiProvider({ id: 'openai' }),
prompt: 'p',
// A non-creative source: this test is about forwarding, and a creative
// one would arrive at createRun with the IP-latitude clause prepended
// (covered by the 'creative IP-latitude clause' block below).
source: 'unit-test-source',
model: 'gpt-5',
});
expect(runner.createRun).toHaveBeenCalledWith(expect.objectContaining({
providerId: 'openai',
model: 'gpt-5',
prompt: 'p',
source: 'unit-test-source',
}));
// workspacePath now also goes through (defaults to process.cwd() when
// the caller doesn't pass `cwd`) so /runs reflects the actual spawn dir.
expect(runner.createRun.mock.calls[0][0]).toHaveProperty('workspacePath');
});
it('forwards a per-call cwd through to createRun as workspacePath', async () => {
runner.executeCliRun.mockImplementation(async ({ workspacePath, onData, onComplete }) => {
onData(workspacePath); // echo back the cwd so the assertion below can check it
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: cliProvider(),
prompt: 'p',
source: 't',
cwd: '/some/other/dir',
});
expect(out.text).toBe('/some/other/dir');
expect(runner.createRun).toHaveBeenCalledWith(
expect.objectContaining({ workspacePath: '/some/other/dir' })
);
});
it('reuses a caller-supplied runId (no createRun round-trip)', async () => {
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('ok');
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: apiProvider(),
prompt: 'p',
source: 't',
runId: 'caller-supplied-run',
});
expect(out.runId).toBe('caller-supplied-run');
expect(runner.createRun).not.toHaveBeenCalled();
});
it('passes a CLI provider clone with overridden defaultModel for codex (which honors --model)', async () => {
runner.executeCliRun.mockImplementation(async ({ provider: providerArg, onData, onComplete }) => {
onData(`ran with model=${providerArg.defaultModel}`);
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: cliProvider({ defaultModel: 'old' }), // id: 'codex'
prompt: 'p',
source: 't',
model: 'new',
});
expect(out.text).toBe('ran with model=new');
});
it('clones non-codex CLI providers with the model override when args have no baked-in model flag', async () => {
// Post-#222: runner.js#buildCliArgs honors `provider.defaultModel`
// for codex / claude-code / gemini-cli when the user hasn't already
// baked a model flag into provider.args. So the clone is safe and
// the run record correctly reflects the user's selection.
runner.hasModelFlag.mockReturnValue(false);
runner.executeCliRun.mockImplementation(async ({ provider: providerArg, onData, onComplete }) => {
onData(`ran with defaultModel=${providerArg.defaultModel}`);
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: { id: 'claude-code', type: 'cli', defaultModel: 'old', timeout: 5000, args: [] },
prompt: 'p',
source: 't',
model: 'user-picked-this',
});
expect(out.text).toBe('ran with defaultModel=user-picked-this');
});
// `effort` has no parameter on executeCliRun/executeTuiRun — it rides the
// provider clone, which is what buildCliArgs/buildTuiInvocation read.
it('pins a per-call effort onto the CLI provider clone alongside the model', async () => {
runner.executeCliRun.mockImplementation(async ({ provider: providerArg, onData, onComplete }) => {
onData(`${providerArg.defaultModel}/${providerArg.effort}`);
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: cliProvider({ defaultModel: 'old' }), // id: 'codex'
prompt: 'p',
source: 't',
model: 'new',
effort: 'xhigh',
});
expect(out.text).toBe('new/xhigh');
});
it('pins a per-call effort even when the model needs no override', async () => {
runner.executeCliRun.mockImplementation(async ({ provider: providerArg, onData, onComplete }) => {
onData(`${providerArg.defaultModel}/${providerArg.effort}`);
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: cliProvider({ defaultModel: 'm-default' }),
prompt: 'p',
source: 't',
effort: 'high',
});
expect(out.text).toBe('m-default/high');
});
it('leaves the provider untouched when no effort is given', async () => {
runner.executeCliRun.mockImplementation(async ({ provider: providerArg, onData, onComplete }) => {
onData(`effort=${'effort' in providerArg}`);
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: cliProvider({ defaultModel: 'm-default' }),
prompt: 'p',
source: 't',
});
expect(out.text).toBe('effort=false');
});
it('pins a per-call effort onto the TUI provider clone', async () => {
tuiRunner.executeTuiRun.mockImplementation(async ({ provider: providerArg, onComplete }) => {
onComplete({ success: true, text: `${providerArg.defaultModel}/${providerArg.effort}` });
});
const out = await runPromptThroughProvider({
provider: tuiProvider({ defaultModel: 'm-default' }),
prompt: 'p',
source: 't',
effort: 'max',
});
expect(out.text).toBe('m-default/max');
});
it('does NOT clone CLI providers when args have a baked-in --model flag (args win)', async () => {
// When the user has pinned a model in provider.args, runner.js
// suppresses its own --model injection. Per-call override is
// silently dropped and the args-baked model wins — keep the run
// record honest by not pretending the override applied.
runner.hasModelFlag.mockReturnValue(true);
runner.executeCliRun.mockImplementation(async ({ provider: providerArg, onData, onComplete }) => {
onData(`ran with defaultModel=${providerArg.defaultModel}`);
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: { id: 'claude-code', type: 'cli', defaultModel: 'fallback', timeout: 5000, args: ['--model', 'baked-in'] },
prompt: 'p',
source: 't',
model: 'user-picked-this',
});
expect(out.text).toBe('ran with defaultModel=fallback');
});
it('returns the args-baked model id (not defaultModel) when extractBakedModel succeeds', async () => {
// Regression: previously the non-honoring CLI branch fell through to
// provider.defaultModel, so a baked args model id would never reach
// the run record / return value. Now resolveEffectiveModel extracts
// the args-pinned id directly.
runner.hasModelFlag.mockReturnValue(true);
runner.extractBakedModel.mockReturnValue('baked-in');
runner.executeCliRun.mockImplementation(async ({ provider: providerArg, onData, onComplete }) => {
onData(`ran with defaultModel=${providerArg.defaultModel}`);
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: { id: 'claude-code', type: 'cli', defaultModel: 'fallback', timeout: 5000, args: ['--model', 'baked-in'] },
prompt: 'p',
source: 't',
model: 'user-picked-this',
});
// effectiveModel = baked-in (not defaultModel='fallback'). Clone
// fires because baked-in !== defaultModel; the CLI receives the
// cloned provider whose defaultModel === baked-in.
expect(out.model).toBe('baked-in');
expect(out.text).toBe('ran with defaultModel=baked-in');
// Run-record side of the bugfix: createRun must persist the
// args-baked model id so the recorded run reflects what actually
// executed (not the caller's silently-dropped override or the
// provider.defaultModel fallback).
expect(runner.createRun).toHaveBeenCalledWith(expect.objectContaining({
providerId: 'claude-code',
model: 'baked-in',
prompt: 'p',
source: 't',
}));
});
});
describe('promptRunner — strictest-discriminator rejection', () => {
it('rejects when CLI onComplete reports success: false (even with no error string)', async () => {
runner.executeCliRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false });
});
await expect(runPromptThroughProvider({
provider: cliProvider(),
prompt: 'p',
source: 't',
})).rejects.toThrow(/CLI execution failed/);
});
it('rejects when CLI onComplete reports a non-zero error', async () => {
runner.executeCliRun.mockImplementation(async ({ onComplete }) => {
onComplete({ error: 'codex timeout' });
});
await expect(runPromptThroughProvider({
provider: cliProvider(),
prompt: 'p',
source: 't',
})).rejects.toThrow(/codex timeout/);
});
it('rejects when API onComplete reports success: false (this is the discriminator drift fix)', async () => {
// Before the unification, API sites only checked `error` and would
// resolve with empty text on success: false — silently swallowing
// soft failures. The unified runner rejects on success: false too.
runner.executeApiRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false });
});
await expect(runPromptThroughProvider({
provider: apiProvider(),
prompt: 'p',
source: 't',
})).rejects.toThrow(/API execution failed/);
});
it('rejects when API onComplete reports a non-zero error', async () => {
runner.executeApiRun.mockImplementation(async ({ onComplete }) => {
onComplete({ error: 'upstream 500' });
});
await expect(runPromptThroughProvider({
provider: apiProvider(),
prompt: 'p',
source: 't',
})).rejects.toThrow(/upstream 500/);
});
it('forwards executeCliRun rejection (e.g. ensureDir/spawn failure pre-onComplete)', async () => {
runner.executeCliRun.mockRejectedValue(new Error('ensureDir failed'));
await expect(runPromptThroughProvider({
provider: cliProvider(),
prompt: 'p',
source: 't',
})).rejects.toThrow(/ensureDir failed/);
});
it('forwards executeApiRun rejection (toolkit not initialized, etc.)', async () => {
runner.executeApiRun.mockRejectedValue(new Error('AI Toolkit not initialized'));
await expect(runPromptThroughProvider({
provider: apiProvider(),
prompt: 'p',
source: 't',
})).rejects.toThrow(/AI Toolkit not initialized/);
});
it('rejects unsupported provider types up-front', async () => {
await expect(runPromptThroughProvider({
provider: { id: 'bogus', type: 'rpc' },
prompt: 'p',
source: 't',
})).rejects.toThrow(/Unsupported provider type: rpc/);
});
it('rejects when provider is null or missing entirely', async () => {
await expect(runPromptThroughProvider({
provider: null, prompt: 'p', source: 't',
})).rejects.toThrow(/provider is required/);
await expect(runPromptThroughProvider({
prompt: 'p', source: 't',
})).rejects.toThrow(/provider is required/);
});
it('rejects when provider.id is missing or non-string', async () => {
await expect(runPromptThroughProvider({
provider: { type: 'api' }, prompt: 'p', source: 't',
})).rejects.toThrow(/provider.id must be a non-empty string/);
await expect(runPromptThroughProvider({
provider: { id: '', type: 'api' }, prompt: 'p', source: 't',
})).rejects.toThrow(/provider.id must be a non-empty string/);
});
it('rejects when prompt or source is missing/empty', async () => {
await expect(runPromptThroughProvider({
provider: apiProvider(), prompt: '', source: 't',
})).rejects.toThrow(/prompt must be a non-empty string/);
await expect(runPromptThroughProvider({
provider: apiProvider(), prompt: 'p', source: '',
})).rejects.toThrow(/source must be a non-empty string/);
});
});
describe('promptRunner — multi-chunk text accumulation', () => {
it('accumulates many CLI string chunks in order', async () => {
runner.executeCliRun.mockImplementation(async ({ onData, onComplete }) => {
for (const c of ['a', 'b', 'c', 'd', 'e']) onData(c);
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: cliProvider(),
prompt: 'p',
source: 't',
});
expect(out.text).toBe('abcde');
});
it('accumulates mixed string + {text} API chunks in order', async () => {
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('A');
onData({ text: 'B' });
onData({ text: 'C' });
onData('D');
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: apiProvider(),
prompt: 'p',
source: 't',
});
expect(out.text).toBe('ABCD');
});
it('ignores non-string non-{text} chunks (e.g. heartbeat events)', async () => {
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('hello');
onData({ heartbeat: true }); // chunk with no text — ignored
onData(null); // ignored
onData(' world');
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: apiProvider(),
prompt: 'p',
source: 't',
});
expect(out.text).toBe('hello world');
});
});
// =============================================================================
// TUI routing — the third dispatch branch added alongside cli/api. Mirrors
// the cli/api coverage above so the central handler's tui path can't quietly
// regress (was the entry point for the Pipeline-prose crash bug).
// =============================================================================
describe('promptRunner — TUI provider routing', () => {
it('routes TUI providers through executeTuiRun and resolves with result.text', async () => {
tuiRunner.executeTuiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('chrome chunk ');
onData('more chrome');
onComplete({ success: true, exitCode: 0, text: 'once upon a time' });
});
const out = await runPromptThroughProvider({
provider: tuiProvider(),
prompt: 'tell me a story',
source: 'pipeline-text-stage',
});
expect(out).toEqual({ text: 'once upon a time', runId: 'run-xyz', model: 'm-default', provider: tuiProvider() });
expect(tuiRunner.executeTuiRun).toHaveBeenCalledTimes(1);
expect(runner.executeCliRun).not.toHaveBeenCalled();
expect(runner.executeApiRun).not.toHaveBeenCalled();
});
it('wakes an MTPLX-backed TUI before spawning OpenCode', async () => {
const provider = tuiProvider({
id: 'opencode-mtplx-tui',
mtplxBacked: true,
endpoint: 'http://127.0.0.1:8000/v1',
});
tuiRunner.executeTuiRun.mockImplementation(async ({ onComplete }) => onComplete({ success: true, text: 'ready' }));
await runPromptThroughProvider({ provider, prompt: 'p', source: 'test' });
expect(executionReadiness.ensureProviderReadyForExecution).toHaveBeenCalledWith(provider);
expect(tuiRunner.executeTuiRun).toHaveBeenCalledTimes(1);
});
it('passes cwd + timeout overrides through to executeTuiRun', async () => {
tuiRunner.executeTuiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('ok');
onComplete({ success: true });
});
await runPromptThroughProvider({
provider: tuiProvider(),
prompt: 'p', source: 't',
cwd: '/tmp/some-other-repo',
timeout: 60000,
});
const opts = tuiRunner.executeTuiRun.mock.calls[0][0];
expect(opts.workspacePath).toBe('/tmp/some-other-repo'); // workspacePath option
expect(opts.timeout).toBe(60000); // timeout option
});
it('returns result.text from executeTuiRun (which owns its own response cleanup)', async () => {
tuiRunner.executeTuiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('raw with chrome');
onComplete({ success: true, text: 'cleaned response from file' });
});
const out = await runPromptThroughProvider({
provider: tuiProvider(),
prompt: 'p', source: 't',
});
expect(out.text).toBe('cleaned response from file');
});
it('falls back to empty string when executeTuiRun omits result.text', async () => {
tuiRunner.executeTuiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('streamed chrome');
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: tuiProvider(),
prompt: 'p', source: 't',
});
expect(out.text).toBe('');
});
it('rejects with TUI-labeled error when executeTuiRun fires onComplete with success: false', async () => {
tuiRunner.executeTuiRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, exitCode: 124 });
});
await expect(runPromptThroughProvider({
provider: tuiProvider(),
prompt: 'p', source: 't',
})).rejects.toThrow(/TUI execution failed/);
});
it('rejects when executeTuiRun rejects (spawn failure path)', async () => {
tuiRunner.executeTuiRun.mockRejectedValue(new Error("Failed to spawn TUI 'claude'"));
await expect(runPromptThroughProvider({
provider: tuiProvider(),
prompt: 'p', source: 't',
})).rejects.toThrow(/Failed to spawn TUI/);
});
});
// =============================================================================
// Gateway-inherited credentials — a gateway-backed OpenCode wrapper (OrcaRouter,
// OpenRouter, NVIDIA NIM) stores no key; `withGatewayApiKey` attaches the
// sibling API record's key as a NON-ENUMERABLE property at read time. Anything
// that clones or re-resolves the provider must carry that key along, or the
// run goes out with no Authorization header (NVIDIA NIM 401s "Header of type
// `authorization` was missing") despite a stored key.
// =============================================================================
describe('promptRunner — gateway sibling-key preservation', () => {
// Mirror of `withGatewayApiKey`: the key is readable but invisible to
// spread/JSON, which is exactly what made the clone below drop it.
const gatewayAttached = (record, apiKey) => {
const attached = { ...record };
Object.defineProperty(attached, 'apiKey', { value: apiKey, enumerable: false, configurable: true });
return attached;
};
const nimCli = () => gatewayAttached({
id: 'opencode-nvidia-nim',
type: 'cli',
command: 'opencode',
gatewayBacked: 'nvidia-nim',
models: ['google/gemma-4-31b-it', 'poolside/laguna-xs-2.1'],
defaultModel: 'poolside/laguna-xs-2.1',
timeout: 5000,
}, 'nvapi-test-key');
const nimTui = () => gatewayAttached({
id: 'opencode-nvidia-nim-tui',
type: 'tui',
command: 'opencode',
gatewayBacked: 'nvidia-nim',
models: ['google/gemma-4-31b-it', 'poolside/laguna-xs-2.1'],
defaultModel: 'poolside/laguna-xs-2.1',
timeout: 5000,
}, 'nvapi-test-key');
it('carries the sibling key onto the model-pinned CLI provider clone', async () => {
runner.executeCliRun.mockImplementation(async ({ provider, onComplete }) => {
expect(provider.apiKey).toBe('nvapi-test-key');
expect(provider.defaultModel).toBe('google/gemma-4-31b-it');
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: nimCli(),
model: 'google/gemma-4-31b-it',
prompt: 'p',
source: 'test',
});
expect(runner.executeCliRun).toHaveBeenCalledTimes(1);
expect(out.model).toBe('google/gemma-4-31b-it');
});
it('carries the sibling key onto the model-pinned TUI provider clone', async () => {
tuiRunner.executeTuiRun.mockImplementation(async ({ provider, onComplete }) => {
expect(provider.apiKey).toBe('nvapi-test-key');
expect(provider.defaultModel).toBe('google/gemma-4-31b-it');
onComplete({ success: true, text: 'done' });
});
await runPromptThroughProvider({
provider: nimTui(),
model: 'google/gemma-4-31b-it',
prompt: 'p',
source: 'test',
});
expect(tuiRunner.executeTuiRun).toHaveBeenCalledTimes(1);
});
it('carries the sibling key when only an effort override forces the clone', async () => {
runner.executeCliRun.mockImplementation(async ({ provider, onComplete }) => {
expect(provider.apiKey).toBe('nvapi-test-key');
onComplete({ success: true });
});
await runPromptThroughProvider({
provider: nimCli(),
prompt: 'p',
source: 'test',
effort: 'high',
});
expect(runner.executeCliRun).toHaveBeenCalledTimes(1);
});
it('leaves the provider untouched when no clone is needed', async () => {
const wrapper = nimCli();
runner.executeCliRun.mockImplementation(async ({ provider, onComplete }) => {
expect(provider).toBe(wrapper);
onComplete({ success: true });
});
await runPromptThroughProvider({
provider: wrapper,
prompt: 'p',
source: 'test',
});
expect(runner.executeCliRun).toHaveBeenCalledTimes(1);
});
});
// =============================================================================
// API timeout enforcement — executeApiRun owns BOTH real deadlines: a
// re-armable no-progress bound (`timeout`) and a 30-minute absolute
// total-runtime cap (#7560). promptRunner keeps a SECONDARY backstop timer,
// armed against the ABSOLUTE bound plus API_TIMEOUT_BACKSTOP_GRACE_MS, purely
// for the pathological case where the runner (here a mock) neither throws nor
// ever calls onComplete. Verify that a stuck API run still rejects with the
// timeout error AND invokes stopRun — and, crucially, that the backstop does
// NOT fire at `timeout + grace`, which is where it used to kill healthy
// streaming runs and persist them as user cancels (#7665).
// =============================================================================
describe('promptRunner — API timeout enforcement', () => {
it('backstop rejects with timeout error when executeApiRun never completes and calls stopRun', async () => {
vi.useFakeTimers();
// executeApiRun hangs — never invokes onComplete or onData (as a mock it
// has no internal timer, so only promptRunner's backstop can fire).
runner.executeApiRun.mockImplementation(() => new Promise(() => {}));
// Kick off the call and attach the rejection assertion BEFORE advancing
// timers — otherwise the timer-driven reject lands without a handler
// attached and vitest flags an unhandled rejection.
const promise = runPromptThroughProvider({
provider: apiProvider(),
prompt: 'p', source: 't',
timeout: 5000,
});
const backstopMs = apiRunAbsoluteTimeoutMs(5000);
const assertion = expect(promise).rejects.toThrow(
new RegExp(`API execution timed out after ${backstopMs}ms`)
);
// Backstop fires at the ABSOLUTE bound + grace — advance past it.
await vi.advanceTimersByTimeAsync(backstopMs + 2000 + 1);
await assertion;
expect(runner.stopRun).toHaveBeenCalledWith('run-xyz');
// The runner's effective timeout is threaded through for its own internal
// no-progress timer, unchanged — only the backstop moved.
expect(runner.executeApiRun).toHaveBeenCalledWith(expect.objectContaining({ timeout: 5000 }));
vi.useRealTimers();
});
// The regression itself. #7560 made `timeout` a bound the runner RE-ARMS on
// every byte, so a run that streams past it is healthy, not hung. This
// backstop kept firing at `timeout + 2s` regardless of progress and killed it
// via `stopRun`, which the runner records as a user CANCEL — suppressing
// `onRunFailed`, the provider bench and the fallback cascade. Sixteen NVIDIA
// NIM nemotron runs died at exactly 302s this way (#7665).
it('backstop does not fire at the runner\'s no-progress bound while a run is still streaming', async () => {
vi.useFakeTimers();
let finish;
runner.executeApiRun.mockImplementation(({ onComplete }) => {
finish = () => onComplete({ success: true });
return new Promise(() => {});
});
const promise = runPromptThroughProvider({
provider: apiProvider(),
prompt: 'p', source: 't',
timeout: 5000,
});
// Well past the old backstop deadline (5000 + 2000) and past the default
// 300s provider bound, but nowhere near the absolute cap.
await vi.advanceTimersByTimeAsync(600_000);
expect(runner.stopRun).not.toHaveBeenCalled();
finish();
await promise;
expect(runner.stopRun).not.toHaveBeenCalled();
vi.useRealTimers();
});
it('keeps an explicitly unbounded mind call open past provider and absolute defaults', async () => {
vi.useFakeTimers();
let finish;
runner.executeApiRun.mockImplementation(({ onComplete }) => {
finish = () => onComplete({ success: true });
return new Promise(() => {});
});
const pending = runPromptThroughProvider({ provider: apiProvider(), prompt: 'p',
source: 'cos-persistent-mind', timeout: 0, absoluteTimeoutMs: 0 });
await vi.advanceTimersByTimeAsync(86_400_000);
expect(runner.executeApiRun).toHaveBeenCalledWith(expect.objectContaining({ timeout: 0, absoluteTimeoutMs: 0 }));
expect(runner.stopRun).not.toHaveBeenCalled();
finish();
await pending;
vi.useRealTimers();
});
it('does not call stopRun when API completes within the timeout', async () => {
vi.useFakeTimers();
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('quick');
onComplete({ success: true });
});
const promise = runPromptThroughProvider({
provider: apiProvider(),
prompt: 'p', source: 't',
timeout: 5000,
});
const out = await promise;
expect(out.text).toBe('quick');
// Advance time past what would have been the timeout — should not fire
// because settle-once guards cleared the handle.
await vi.advanceTimersByTimeAsync(10000);
expect(runner.stopRun).not.toHaveBeenCalled();
vi.useRealTimers();
});
});
// =============================================================================
// Retry-with-fallback — when a run fails and a fallback provider is
// available, swap to it transparently instead of letting the failure
// queue an "Investigate AI provider failure" task. Regression guard for
// the issue where LM Studio / Claude Code CLI failures used to create
// noisy plan tasks even with a configured fallback.
// =============================================================================
describe('promptRunner — retry-with-fallback', () => {
// Reuse the top-level factories so the retry tests pick up future
// changes to the canonical provider shapes — only overriding fields
// these tests assert on (id + name, since noteFallbackHandled keys on
// the display name).
const fallbackApi = apiProvider({ id: 'fallback-api', name: 'Fallback API', defaultModel: 'fb-model' });
const primaryCli = cliProvider({ id: 'primary-cli', name: 'Primary CLI', defaultModel: 'primary-model' });
const primaryApi = apiProvider({ id: 'primary-api', name: 'Primary API', defaultModel: 'primary-model' });
function mockToolkitWithFallback(fallback = fallbackApi) {
// isAvailable returns true so promptRunner's "skip if toolkit already
// marked it" gate doesn't short-circuit the mark in these tests.
const isAvailable = vi.fn().mockReturnValue(true);
const markUnavailable = vi.fn().mockResolvedValue(undefined);
const markUsageLimit = vi.fn().mockResolvedValue(undefined);
const getFallbackProvider = vi.fn().mockReturnValue(
fallback ? { provider: fallback, source: 'provider' } : null
);
toolkitState.getAIToolkitInstance.mockReturnValue({
services: { providerStatus: { isAvailable, markUnavailable, markUsageLimit, getFallbackProvider } },
});
providers.getAllProviders.mockResolvedValue({
activeProvider: null,
providers: fallback ? [primaryCli, primaryApi, fallback] : [primaryCli, primaryApi],
});
return { isAvailable, markUnavailable, markUsageLimit, getFallbackProvider };
}
it('resolves composite fallback candidates through the configured provider lookup', async () => {
const primary = apiProvider({ id: 'openai.api@primary', name: 'Primary composite' });
const fallback = apiProvider({ id: 'openai.api@fallback', name: 'Fallback composite' });
const unavailable = 'openai.api@disabled';
const status = mockToolkitWithFallback(fallback);
providers.getAllProviders.mockResolvedValue({ providers: [] });
providers.getProviderById.mockImplementation(async id => ({ [primary.id]: primary, [fallback.id]: fallback }[id] ?? null));
setFallbackChainCache([{ providerId: unavailable }, { providerId: fallback.id }, { providerId: fallback.id }]);
status.getFallbackProvider.mockImplementation((failedId, candidates) => {
expect(failedId).toBe(primary.id);
expect(candidates).toEqual({ [primary.id]: primary, [fallback.id]: fallback });
return { provider: candidates[fallback.id], source: 'system' };
});
runner.executeApiRun
.mockImplementationOnce(async ({ onComplete }) => onComplete({ success: false, error: 'Primary unavailable' }))
.mockImplementationOnce(async ({ onData, onComplete }) => { onData('recovered'); onComplete({ success: true }); });
const result = await runPromptThroughProvider({ provider: primary, prompt: 'p', source: 'test' });
expect(result.text).toBe('recovered');
expect(result.usedFallback).toBe(true);
expect(providers.getProviderById.mock.calls.filter(([id]) => id === unavailable)).toHaveLength(1);
expect(providers.getProviderById.mock.calls.filter(([id]) => id === fallback.id)).toHaveLength(1); // duplicate chain entries reuse the materialized candidate
});
it('refuses a stopped caller after local concurrency wait without provider fallback, benching, or escalation', async () => {
const status = mockToolkitWithFallback();
const local = apiProvider({ id: 'local-guard', endpoint: 'http://localhost:11434/v1' });
let release;
const waiting = new Promise((resolve) => { release = resolve; });
let invoked = 0;
runner.executeApiRun.mockImplementation(async ({ beforeExecute, onComplete }) => {
if (beforeExecute) await beforeExecute();
invoked += 1;
await waiting;
onComplete({ success: true });
});
const first = runPromptThroughProvider({ provider: local, prompt: 'first', source: 'test' });
await vi.waitFor(() => expect(invoked).toBe(1));
let active = true;
const beforeExecute = vi.fn(() => { if (!active) throw Object.assign(new Error('Production stopped'), { code: 'PRODUCTION_NOT_RUNNING' }); });
const second = expect(runPromptThroughProvider({ provider: local, prompt: 'second', source: 'test', beforeExecute }))
.rejects.toMatchObject({ code: 'PRODUCTION_NOT_RUNNING', canceled: true });
active = false;
expect(beforeExecute).not.toHaveBeenCalled();
release(); await first; await second;
expect(invoked).toBe(1);
expect(status.getFallbackProvider).not.toHaveBeenCalled();
expect(status.markUnavailable).not.toHaveBeenCalled();
expect(autoFixer.escalateProviderFailure).not.toHaveBeenCalled();
});
it('keeps an exact provider pin and skips every fallback tier when disabled', async () => {
const status = mockToolkitWithFallback();
runner.executeCliRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, error: 'Pinned provider failed' });
});
await expect(runPromptThroughProvider({
provider: primaryCli,
prompt: 'p',
source: 'test',
allowFallback: false,
})).rejects.toThrow('Pinned provider failed');
expect(runner.createRun).toHaveBeenCalledWith(expect.objectContaining({ allowFallback: false }));
expect(status.getFallbackProvider).not.toHaveBeenCalled();
expect(autoFixer.escalateProviderFailure).not.toHaveBeenCalled();
});
// A pin opts out of ROUTING, not out of health reporting. Leaving the mark to
// the fallback cascade let a provider fail every pinned call for hours while
// the Providers page stayed green and unpinned callers kept choosing it.
it('stops an initial policy refusal without selecting a provider or scheduling recovery', async () => {
const status = mockToolkitWithFallback();
runner.executeApiRun.mockImplementation(async ({ onComplete }) => onComplete({
success: false, error: 'Policy refusal', errorAnalysis: { category: 'content-refusal' },
}));
await expect(runPromptThroughProvider({ provider: primaryApi, prompt: 'p', source: 'test' }))
.rejects.toMatchObject({ errorAnalysis: { category: 'content-refusal' } });
expect(runner.createRun).toHaveBeenCalledTimes(1);
expect(status.getFallbackProvider).not.toHaveBeenCalled();
expect(status.markUnavailable).not.toHaveBeenCalled();
expect(autoFixer.escalateProviderFailure).not.toHaveBeenCalled();
});
it.each([false, true])('rejects caller runtime exhaustion without recovery (allowFallback=%s)', async (allowFallback) => {
const status = mockToolkitWithFallback();
runner.executeApiRun.mockImplementation(async ({ onComplete }) => onComplete({
success: false, error: 'Caller runtime budget exhausted after 120000ms', timeoutBound: 'absolute',
timeoutOrigin: 'caller-budget', runtimeBudgetMs: 120000,
errorAnalysis: { category: 'runtime-budget-exhausted' },
}));
await expect(runPromptThroughProvider({ provider: primaryApi, prompt: 'p', source: 'test', allowFallback }))
.rejects.toMatchObject({ code: 'RUN_RUNTIME_BUDGET_EXHAUSTED', timeoutBound: 'absolute',
timeoutOrigin: 'caller-budget', runtimeBudgetMs: 120000 });
expect(status.markUnavailable).not.toHaveBeenCalled();
expect(status.getFallbackProvider).not.toHaveBeenCalled();
expect(autoFixer.escalateProviderFailure).not.toHaveBeenCalled();
expect(runner.createRun).toHaveBeenCalledTimes(1);
});
it('still benches the provider a pinned run failed on', async () => {
const status = mockToolkitWithFallback();
runner.executeCliRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, error: 'CLI run timed out after 600000ms' });
});
await expect(runPromptThroughProvider({
provider: primaryCli,
prompt: 'p',
source: 'test',
allowFallback: false,
})).rejects.toThrow('CLI run timed out after 600000ms');
expect(status.markUnavailable).toHaveBeenCalledWith(primaryCli.id, expect.objectContaining({
reason: 'timeout',
// The bench covers the budget the run actually burned, so the next pinned
// wake is refused cheaply instead of stalling for another ten minutes.
waitTimeMs: 600000,
}));
});
// A usage limit is the one marker a pinned run leaves alone: the caller that
// cares (persistentMindSupervisor) calls markProviderUsageLimit for this same
// failure, and its observed-block ledger write is deliberately not deduped, so
// marking here too would record one block twice.
it('leaves a pinned usage-limit failure to the caller that owns the marker', async () => {
const status = mockToolkitWithFallback();
runner.executeCliRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, error: "You've hit your usage limit. Try again in 5 hours" });
});
await expect(runPromptThroughProvider({
provider: primaryCli,
prompt: 'p',
source: 'test',
allowFallback: false,
})).rejects.toThrow(/usage limit/);
expect(status.markUsageLimit).not.toHaveBeenCalled();
expect(status.markUnavailable).not.toHaveBeenCalled();
});
// Request-specific categories resolve to "don't bench" in providerCooldown, so
// a pinned caller's bad prompt can't take a healthy provider offline.
it('does not bench a pinned run that failed for a request-specific reason', async () => {
const status = mockToolkitWithFallback();
runner.executeCliRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, error: 'model not found: bogus-model' });
});
await expect(runPromptThroughProvider({
provider: primaryCli,
prompt: 'p',
source: 'test',
allowFallback: false,
})).rejects.toThrow(/model not found/);
expect(status.markUnavailable).not.toHaveBeenCalled();
});
it('does not retry, bench, or escalate a canceled TUI run', async () => {
const status = mockToolkitWithFallback();
const primaryTui = tuiProvider({ id: 'primary-tui', name: 'Primary TUI', defaultModel: 'primary-model' });
tuiRunner.executeTuiRun.mockImplementation(async ({ onComplete }) => {
onComplete({
success: false,
canceled: true,
error: 'TUI interrupted by PortOS shutdown (signal 2)',
});
});
await expect(runPromptThroughProvider({
provider: primaryTui,
prompt: 'p',
source: 'test',
})).rejects.toMatchObject({ code: 'RUN_CANCELED', canceled: true });
expect(runner.executeApiRun).not.toHaveBeenCalled();
expect(status.getFallbackProvider).not.toHaveBeenCalled();
expect(status.markUnavailable).not.toHaveBeenCalled();
expect(autoFixer.noteFallbackStarted).not.toHaveBeenCalled();
expect(autoFixer.escalateProviderFailure).not.toHaveBeenCalled();
});
it('retries with the configured fallback and resolves with usedFallback flag when primary CLI fails', async () => {
const status = mockToolkitWithFallback();
// First call: primary CLI fails. Second call: fallback API succeeds.
runner.executeCliRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, error: 'Process exited with code 1' });
});
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('fallback content');
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: primaryCli,
prompt: 'p',
source: 'test',
});
expect(out.text).toBe('fallback content');
expect(out.usedFallback).toBe(true);
expect(out.fallbackFrom).toEqual({ id: 'primary-cli', name: 'Primary CLI' });
// fallbackProvider exposes the full provider object that actually ran
// so attribution callers (stageRunner persisting runId for history /
// restore) can record providerId without re-picking the fallback.
expect(out.fallbackProvider).toMatchObject({ id: 'fallback-api', name: 'Fallback API' });
// Primary was marked unavailable before retry; fallback was looked up
// and used; the deferred autoFixer task was cancelled.
expect(status.markUnavailable).toHaveBeenCalledWith('primary-cli', expect.objectContaining({
reason: expect.any(String),
}));
expect(status.getFallbackProvider).toHaveBeenCalledWith(
'primary-cli', expect.any(Object), null, null,
expect.objectContaining({ hasImages: false, requiredContextTokens: expect.any(Number) }),
);
// The fallback lifecycle is announced up front (so a slow fallback can't
// outrun the backstop timer) and resolved on success.
expect(autoFixer.noteFallbackStarted).toHaveBeenCalledWith({
provider: 'Primary CLI',
model: 'primary-model',
});
expect(autoFixer.noteFallbackHandled).toHaveBeenCalledWith({
provider: 'Primary CLI',
model: 'primary-model',
});
expect(autoFixer.noteFallbackFailed).not.toHaveBeenCalled();
});
it('resolves a function timeout against each provider that runs, fallback included', async () => {
mockToolkitWithFallback();
runner.executeCliRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, error: 'Process exited with code 1' });
});
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('fallback content');
onComplete({ success: true });
});
await runPromptThroughProvider({
provider: primaryCli, prompt: 'p', source: 'test',
timeout: (provider) => (provider.type === 'api' ? 900000 : 0),
});
expect(runner.executeCliRun).toHaveBeenCalledWith(expect.objectContaining({ timeout: 0 }));
expect(runner.executeApiRun).toHaveBeenCalledWith(expect.objectContaining({ timeout: 900000 }));
});
it('threads screenshot capability into retry selection and the fallback run', async () => {
const status = mockToolkitWithFallback();
runner.executeApiRun
.mockImplementationOnce(async ({ onComplete }) => {
onComplete({ success: false, error: 'vision primary failed' });
})
.mockImplementationOnce(async ({ screenshots, onData, onComplete }) => {
expect(screenshots).toEqual(['frame.png']);
onData('described');
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: primaryApi,
prompt: '12345678',
source: 'test',
screenshots: ['frame.png'],
outputReserveTokens: 1_000,
});
expect(out.text).toBe('described');
expect(status.getFallbackProvider).toHaveBeenCalledWith(
'primary-api', expect.any(Object), null, null,
{ hasImages: true, requiredContextTokens: 1_002 },
);
});
it('does NOT bench the provider on a model-not-found (request-specific bad model id), but still falls back for this call', async () => {
const status = mockToolkitWithFallback();
// Primary API fails with a model-not-found — a bad model id in the request,
// not a provider outage. The provider must stay available for its other
// (valid) models; only this call falls back.
runner.executeApiRun
.mockImplementationOnce(async ({ onComplete }) => {
onComplete({ success: false, error: "model 'bogus' not found", errorAnalysis: { category: ERROR_CATEGORIES.MODEL_NOT_FOUND } });
})
.mockImplementationOnce(async ({ onData, onComplete }) => {
onData('fallback content');
onComplete({ success: true });
});
const out = await runPromptThroughProvider({ provider: primaryApi, prompt: 'p', source: 'test' });
expect(out.text).toBe('fallback content');
expect(out.usedFallback).toBe(true);
// The carve-out: the provider was NOT benched (mirrors CONTENT_REFUSAL).
expect(status.markUnavailable).not.toHaveBeenCalled();
// …but the failing call still recovered via the fallback.
expect(status.getFallbackProvider).toHaveBeenCalledWith(
'primary-api', expect.any(Object), null, null,
expect.objectContaining({ hasImages: false, requiredContextTokens: expect.any(Number) }),
);
});
it('runs the configured fallbackModel on the fallback (never the primary model) when one is pinned', async () => {
const status = mockToolkitWithFallback();
// Provider-level fallback that pins a specific model to run on the fallback.
status.getFallbackProvider.mockReturnValue({
provider: fallbackApi,
source: 'provider',
model: 'pinned-fb-model',
});
runner.executeCliRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, error: 'Process exited with code 1' });
});
let ranModel;
runner.executeApiRun.mockImplementation(async ({ model, onData, onComplete }) => {
ranModel = model;
onData('fallback content');
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: primaryCli,
prompt: 'p',
source: 'test',
});
expect(out.usedFallback).toBe(true);
// The pinned fallbackModel must reach the fallback run — NOT the primary's
// 'primary-model' (the leak this fix closes), and NOT the fallback's own
// 'fb-model' default (the pin must win).
expect(ranModel).toBe('pinned-fb-model');
expect(out.model).toBe('pinned-fb-model');
});
it('retries with fallback when primary API fails', async () => {
mockToolkitWithFallback();
let calls = 0;
runner.executeApiRun.mockImplementation(async ({ provider: providerArg, onData, onComplete }) => {
calls += 1;
if (providerArg.id === 'primary-api') {
onComplete({ success: false, error: 'upstream 500' });
} else {
onData('recovered');
onComplete({ success: true });
}
});
const out = await runPromptThroughProvider({
provider: primaryApi,
prompt: 'p',
source: 'test',
});
expect(calls).toBe(2);
expect(out.text).toBe('recovered');
expect(out.usedFallback).toBe(true);
expect(autoFixer.noteFallbackHandled).toHaveBeenCalledTimes(1);
});
it('uses runner-provided errorAnalysis when marking the failed provider', async () => {
const status = mockToolkitWithFallback();
const runnerErrorAnalysis = {
hasError: true,
category: 'usage-limit',
message: 'Usage limit reset at 5pm',
waitTime: 12345,
};
runner.executeCliRun.mockImplementation(async ({ onComplete }) => {
onComplete({
success: false,
error: 'plain wrapper error',
errorAnalysis: runnerErrorAnalysis,
});
});
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('recovered');
onComplete({ success: true });
});
await runPromptThroughProvider({
provider: primaryCli,
prompt: 'p',
source: 'test',
});
expect(status.markUsageLimit).toHaveBeenCalledWith('primary-cli', {
message: 'Usage limit reset at 5pm',
waitTime: 12345,
});
expect(status.markUnavailable).not.toHaveBeenCalled();
});
it('does NOT suppress the investigation task when the fallback ALSO fails (both errors must surface)', async () => {
mockToolkitWithFallback();
runner.executeCliRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, error: 'primary boom' });
});
runner.executeApiRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, error: 'fallback also boom' });
});
await expect(runPromptThroughProvider({
provider: primaryCli,
prompt: 'p',
source: 'test',
})).rejects.toThrow(/fallback also boom/);
// noteFallbackHandled is reserved for SUCCESSFUL fallback. When both fail,
// the primary's suppression is released via noteFallbackFailed (the
// fallback provider's own failure already queued its investigation task).
expect(autoFixer.noteFallbackStarted).toHaveBeenCalledWith({ provider: 'Primary CLI', model: 'primary-model' });
expect(autoFixer.noteFallbackFailed).toHaveBeenCalledWith({ provider: 'Primary CLI', model: 'primary-model' });
expect(autoFixer.noteFallbackHandled).not.toHaveBeenCalled();
});
it('rethrows the original error when no fallback is available', async () => {
mockToolkitWithFallback(null);
runner.executeCliRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, error: 'no recovery path' });
});
await expect(runPromptThroughProvider({
provider: primaryCli,
prompt: 'p',
source: 'test',
})).rejects.toThrow(/no recovery path/);
expect(runner.executeApiRun).not.toHaveBeenCalled();
expect(autoFixer.noteFallbackHandled).not.toHaveBeenCalled();
// No retry suppressed the original event, so it remains the sole owner.
expect(autoFixer.noteFallbackStarted).not.toHaveBeenCalled();
expect(autoFixer.noteFallbackFailed).not.toHaveBeenCalled();
expect(autoFixer.escalateProviderFailure).not.toHaveBeenCalled();
});
it('rethrows when toolkit/providerStatus is not initialized (no retry path possible)', async () => {
// Default mock returns null toolkit — no retry attempted.
runner.executeCliRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, error: 'init not ready' });
});
await expect(runPromptThroughProvider({
provider: primaryCli,
prompt: 'p',
source: 'test',
})).rejects.toThrow(/init not ready/);
expect(autoFixer.noteFallbackHandled).not.toHaveBeenCalled();
});
it('strips internal effectiveProvider/effectiveModel annotations from rethrown errors', async () => {
// No fallback → original error rethrown. The annotation fields are
// implementation-detail-only and should never leak to callers.
mockToolkitWithFallback(null);
runner.executeCliRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, error: 'plain failure' });
});
try {
await runPromptThroughProvider({
provider: primaryCli,
prompt: 'p',
source: 'test',
});
throw new Error('should have rejected');
} catch (err) {
expect(err.message).toMatch(/plain failure/);
expect(err.effectiveProvider).toBeUndefined();
expect(err.effectiveModel).toBeUndefined();
}
});
it('skips re-marking when the toolkit already marked the provider unavailable (prevents double failureCount)', async () => {
const status = mockToolkitWithFallback();
// Simulate the toolkit's executeApiRun having already called markUsageLimit
// before onComplete fired — providerStatus.isAvailable now returns false.
status.isAvailable.mockReturnValue(false);
runner.executeApiRun.mockImplementation(async ({ provider: providerArg, onData, onComplete }) => {
if (providerArg.id === 'primary-api') {
onComplete({ success: false, error: 'rate limit hit' });
} else {
onData('recovered');
onComplete({ success: true });
}
});
await runPromptThroughProvider({
provider: primaryApi,
prompt: 'p',
source: 'test',
});
// Fallback still ran; investigation task still suppressed; but neither
// markUnavailable nor markUsageLimit was called from promptRunner
// because the toolkit's runner already marked it.
expect(status.markUnavailable).not.toHaveBeenCalled();
expect(status.markUsageLimit).not.toHaveBeenCalled();
expect(autoFixer.noteFallbackHandled).toHaveBeenCalledTimes(1);
});
it('keys noteFallbackHandled to the provider that actually ran (not the caller-intended one) when createRun proactively swapped', async () => {
// Setup: caller asks for primary-cli, but createRun has proactively
// swapped to a different intermediate fallback (mockedFallback in
// runResult.provider) because primary-cli was already marked unavailable.
// That intermediate then fails; promptRunner's catch retries with yet
// another fallback. The cancelled-task key MUST match the intermediate
// (what server's onRunFailed actually published) — not the caller's
// original primary-cli.
const intermediate = cliProvider({ id: 'intermediate-cli', name: 'Intermediate CLI', defaultModel: 'intermediate-model' });
const finalFallback = apiProvider({ id: 'final-api', name: 'Final API', defaultModel: 'final-model' });
const status = mockToolkitWithFallback(finalFallback);
// Simulate createRun's proactive swap: runner.createRun returns the
// intermediate provider rather than the caller-passed primary.
runner.createRun.mockResolvedValueOnce({
runId: 'run-via-intermediate',
provider: intermediate,
});
runner.executeCliRun.mockImplementation(async ({ provider: providerArg, onComplete }) => {
// Intermediate fails on first attempt.
onComplete({ success: false, error: `intermediate boom (${providerArg.id})` });
});
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('recovered via final');
onComplete({ success: true });
});
await runPromptThroughProvider({
provider: primaryCli, // caller-intended primary
prompt: 'p',
source: 'test',
});
// The deferred task in autoFixer is keyed off the provider that
// ACTUALLY failed (intermediate), so noteFallbackHandled must use that
// — not the caller-intended primary-cli.
expect(autoFixer.noteFallbackHandled).toHaveBeenCalledWith({
provider: 'Intermediate CLI',
model: 'intermediate-model',
});
// markUnavailable likewise applies to the failed intermediate, not
// the primary the caller asked for.
expect(status.markUnavailable).toHaveBeenCalledWith('intermediate-cli', expect.any(Object));
});
it('rethrows pre-execution failures (e.g. createRun throwing) without retry — there is no investigation task to suppress', async () => {
const status = mockToolkitWithFallback();
// createRun throws before any execution happens (disk error, disabled
// provider, etc.). No onRunFailed event will fire, so no deferred
// investigation task exists. The retry path must NOT engage.
runner.createRun.mockRejectedValueOnce(new Error('Provider is disabled'));
await expect(runPromptThroughProvider({
provider: primaryCli,
prompt: 'p',
source: 'test',
})).rejects.toThrow(/Provider is disabled/);
expect(runner.executeCliRun).not.toHaveBeenCalled();
expect(runner.executeApiRun).not.toHaveBeenCalled();
expect(status.markUnavailable).not.toHaveBeenCalled();
expect(status.markUsageLimit).not.toHaveBeenCalled();
expect(autoFixer.noteFallbackHandled).not.toHaveBeenCalled();
});
it('keeps the failed primary in the providersMap so provider-level fallbackProvider can be looked up', async () => {
// Regression: an earlier attempt at the self-fallback guard removed
// the primary from the providersMap, which broke provider-level
// fallback selection — getFallbackProvider needs the primary entry
// to read its `fallbackProvider` field. The guard against
// `fallbackProvider === self` now lives in getFallbackProvider; this
// test pins the call-site contract that the primary stays in the map.
const status = mockToolkitWithFallback();
runner.executeCliRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, error: 'primary boom' });
});
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('recovered');
onComplete({ success: true });
});
await runPromptThroughProvider({
provider: primaryCli,
prompt: 'p',
source: 'test',
});
// The map passed to getFallbackProvider must contain the failed
// primary so provider-level fallbackProvider can be read from it.
const mapPassed = status.getFallbackProvider.mock.calls[0][1];
expect(mapPassed).toHaveProperty('primary-cli');
});
it('attaches the sibling key when a Tier-3 fallback pick is a gateway wrapper', async () => {
// The fallback pick comes from the RAW provider map, which carries no
// gateway-inherited key. Without the attach the wrapper executes with
// no Authorization header (NVIDIA NIM 401s "Header of type `authorization`
// was missing") despite a stored key.
const sibling = { id: 'nvidia-nim', type: 'api', apiKey: 'nvapi-test-key' };
const rawWrapper = {
id: 'opencode-nvidia-nim',
name: 'OpenCode NVIDIA NIM',
type: 'cli',
command: 'opencode',
gatewayBacked: 'nvidia-nim',
models: ['poolside/laguna-xs-2.1'],
defaultModel: 'poolside/laguna-xs-2.1',
};
const status = mockToolkitWithFallback(rawWrapper);
providers.getAllProviders.mockResolvedValue({
activeProvider: null,
providers: [primaryCli, primaryApi, sibling, rawWrapper],
});
const executedProviders = [];
runner.executeCliRun
.mockImplementationOnce(async ({ onComplete }) => {
onComplete({ success: false, error: 'primary boom' });
})
.mockImplementationOnce(async ({ provider, onData, onComplete }) => {
executedProviders.push(provider);
onData('recovered');
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: primaryCli,
prompt: 'p',
source: 'test',
});
expect(out.usedFallback).toBe(true);
expect(out.fallbackProvider).toMatchObject({ id: 'opencode-nvidia-nim' });
expect(out.fallbackProvider.apiKey).toBe('nvapi-test-key');
expect(executedProviders).toHaveLength(1);
expect(executedProviders[0].apiKey).toBe('nvapi-test-key');
expect(status.markUnavailable).toHaveBeenCalledWith('primary-cli', expect.any(Object));
});
it('does not turn a successful fallback into a failure when noteFallbackHandled itself throws (best-effort suppression)', async () => {
mockToolkitWithFallback();
autoFixer.noteFallbackHandled.mockImplementation(() => {
throw new Error('autoFixer is offline');
});
runner.executeCliRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, error: 'primary boom' });
});
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('still works');
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: primaryCli,
prompt: 'p',
source: 'test',
});
// Fallback ran and returned its result — the suppression failure was
// logged but did not surface as a rejection.
expect(out.text).toBe('still works');
expect(out.usedFallback).toBe(true);
});
it('routes USAGE_LIMIT failures through markUsageLimit (parses wait time)', async () => {
const status = mockToolkitWithFallback();
runner.executeCliRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, error: "You've hit your usage limit. Try again in 5 hours" });
});
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('recovered');
onComplete({ success: true });
});
await runPromptThroughProvider({
provider: primaryCli,
prompt: 'p',
source: 'test',
});
expect(status.markUsageLimit).toHaveBeenCalledWith('primary-cli', expect.objectContaining({
waitTime: expect.any(String),
}));
expect(status.markUnavailable).not.toHaveBeenCalled();
});
it('coalesces a 10-concurrent-failure storm into one provider-map read + one mark-unavailable', async () => {
// Acceptance for [coalesce-fallback-provider-failure-storm]: when a
// stuck provider has N simultaneous in-flight calls, the per-provider
// mark-and-pick (getAllProviders + markUnavailable) must run ONCE for
// the whole storm — not once per failing call — while every call still
// recovers via its own fallback run.
const status = mockToolkitWithFallback();
// clearAllMocks() resets call history but NOT mockImplementation, so a
// throwing noteFallbackHandled impl set by an earlier test would leak in
// here as best-effort-suppressed stderr. Reset it to a clean no-op.
autoFixer.noteFallbackHandled.mockImplementation(() => {});
// Gate the provider-map read so the first failure's mark-and-pick stays
// in-flight long enough for all 10 failures to pile onto it. Without
// this hold the first could settle before later calls reach their catch,
// defeating the coalescing the test means to assert.
let releaseGetAll;
const getAllGate = new Promise((resolve) => { releaseGetAll = resolve; });
providers.getAllProviders.mockReturnValue(
getAllGate.then(() => ({ activeProvider: null, providers: [primaryCli, primaryApi, fallbackApi] }))
);
runner.executeCliRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, error: 'storm boom' });
});
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('fallback content');
onComplete({ success: true });
});
const calls = Array.from({ length: 10 }, () => runPromptThroughProvider({
provider: primaryCli,
prompt: 'p',
source: 'test',
}));
// Let every call fail its primary run and register on the shared
// in-flight mark-and-pick, THEN release the gated read.
await new Promise((r) => setTimeout(r, 0));
releaseGetAll();
const results = await Promise.all(calls);
// Every call recovered through its own fallback run.
expect(results).toHaveLength(10);
for (const out of results) {
expect(out.text).toBe('fallback content');
expect(out.usedFallback).toBe(true);
}
expect(runner.executeApiRun).toHaveBeenCalledTimes(10);
// ...but the storm read the provider map and benched the provider once.
expect(providers.getAllProviders).toHaveBeenCalledTimes(1);
expect(status.getFallbackProvider).toHaveBeenCalledTimes(1);
expect(status.markUnavailable).toHaveBeenCalledTimes(1);
});
});
// =============================================================================
// pickConfigCorrectedModel — pure Tier-1 (config/env) model corrector.
// =============================================================================
describe('pickConfigCorrectedModel', () => {
it('returns null when the provider carries no enumerable models list', () => {
expect(pickConfigCorrectedModel({ id: 'p', defaultModel: 'x' }, 'x')).toBeNull();
expect(pickConfigCorrectedModel({ id: 'p', models: null }, 'x')).toBeNull();
});
it('picks a different valid generation model, excluding the failed id', () => {
const p = { id: 'p', models: ['broken-model', 'good-model', 'other-model'] };
expect(pickConfigCorrectedModel(p, 'broken-model')).toBe('good-model');
});
it('skips embedding-only models so a generation call is never corrected onto one', () => {
const p = { id: 'p', models: ['broken-model', 'nomic-embed-text:latest', 'qwen3.6:35b'] };
expect(pickConfigCorrectedModel(p, 'broken-model')).toBe('qwen3.6:35b');
});
it('returns null when the only alternatives are the failed id and/or embedding models', () => {
expect(pickConfigCorrectedModel({ id: 'p', models: ['broken-model'] }, 'broken-model')).toBeNull();
expect(pickConfigCorrectedModel({ id: 'p', models: ['broken-model', 'nomic-embed-text'] }, 'broken-model')).toBeNull();
});
});
// =============================================================================
// Tiered fallback cascade (issue #2342) — a model-not-found failure attempts a
// deterministic Tier-1 config/env correction (retry the SAME provider with a
// valid listed model) BEFORE escalating to a Tier-3 fallback provider, and only
// an unrecovered failure escalates to a Tier-4 investigation task.
// =============================================================================
describe('promptRunner — Tier 1 config/env correction (issue #2342)', () => {
const fallbackApi = apiProvider({ id: 'fallback-api', name: 'Fallback API', defaultModel: 'fb-model' });
function mockToolkitWithFallback(fallback = fallbackApi) {
const isAvailable = vi.fn().mockReturnValue(true);
const markUnavailable = vi.fn().mockResolvedValue(undefined);
const markUsageLimit = vi.fn().mockResolvedValue(undefined);
const getFallbackProvider = vi.fn().mockReturnValue(fallback ? { provider: fallback, source: 'provider' } : null);
toolkitState.getAIToolkitInstance.mockReturnValue({
services: { providerStatus: { isAvailable, markUnavailable, markUsageLimit, getFallbackProvider } },
});
providers.getAllProviders.mockResolvedValue({ activeProvider: null, providers: fallback ? [fallback] : [] });
return { isAvailable, markUnavailable, markUsageLimit, getFallbackProvider };
}
it('recovers a model-not-found by retrying the SAME provider with a valid listed model (never reaching Tier 3)', async () => {
const status = mockToolkitWithFallback();
const primary = apiProvider({ id: 'primary-api', name: 'Primary API', defaultModel: 'primary-model', models: ['primary-model', 'good-model'] });
runner.executeApiRun.mockImplementation(async ({ model, onData, onComplete }) => {
if (model === 'primary-model') {
onComplete({ success: false, error: "model 'primary-model' not found", errorAnalysis: { category: ERROR_CATEGORIES.MODEL_NOT_FOUND } });
} else {
onData('corrected content');
onComplete({ success: true });
}
});
const out = await runPromptThroughProvider({ provider: primary, prompt: 'p', source: 'test' });
expect(out.text).toBe('corrected content');
expect(out.usedFallback).toBe(true);
expect(out.fixTier).toBe(1);
expect(out.fixStrategy).toBe('config/env');
// Recovered on the SAME provider with the corrected model — Tier 3 untouched.
expect(out.fallbackProvider).toMatchObject({ id: 'primary-api' });
expect(out.model).toBe('good-model');
expect(status.getFallbackProvider).not.toHaveBeenCalled();
expect(status.markUnavailable).not.toHaveBeenCalled();
// The primary's investigation task was suppressed up front and resolved.
expect(autoFixer.noteFallbackStarted).toHaveBeenCalledWith({ provider: 'Primary API', model: 'primary-model' });
expect(autoFixer.noteFallbackHandled).toHaveBeenCalledWith({ provider: 'Primary API', model: 'primary-model' });
expect(autoFixer.noteFallbackFailed).not.toHaveBeenCalled();
});
it('declines Tier 1 when no alternative model exists and falls through to the Tier-3 fallback provider', async () => {
const status = mockToolkitWithFallback();
// Only the failed model is listed → pickConfigCorrectedModel returns null.
const primary = apiProvider({ id: 'primary-api', name: 'Primary API', defaultModel: 'primary-model', models: ['primary-model'] });
runner.executeApiRun.mockImplementation(async ({ provider: p, onData, onComplete }) => {
if (p.id === 'primary-api') {
onComplete({ success: false, error: 'model missing', errorAnalysis: { category: ERROR_CATEGORIES.MODEL_NOT_FOUND } });
} else {
onData('fallback content');
onComplete({ success: true });
}
});
const out = await runPromptThroughProvider({ provider: primary, prompt: 'p', source: 'test' });
expect(out.text).toBe('fallback content');
expect(out.usedFallback).toBe(true);
expect(out.fixTier).toBe(3);
expect(out.fixStrategy).toBe('constrained-agent-retry');
expect(out.fallbackProvider).toMatchObject({ id: 'fallback-api' });
// Tier 3 engaged: the fallback was looked up (MODEL_NOT_FOUND stays unbenched).
expect(status.getFallbackProvider).toHaveBeenCalledWith(
'primary-api', expect.any(Object), null, null,
expect.objectContaining({ hasImages: false, requiredContextTokens: expect.any(Number) }),
);
expect(status.markUnavailable).not.toHaveBeenCalled();
});
it('escalates to Tier 3 when the Tier-1 corrected retry ALSO fails, and cancels both queued tasks on recovery', async () => {
mockToolkitWithFallback();
const primary = apiProvider({ id: 'primary-api', name: 'Primary API', defaultModel: 'primary-model', models: ['primary-model', 'good-model'] });
runner.executeApiRun.mockImplementation(async ({ provider: p, model, onData, onComplete }) => {
if (p.id === 'primary-api') {
// Both the primary model AND the Tier-1 corrected model fail on this provider.
onComplete({ success: false, error: `still broken (${model})`, errorAnalysis: { category: ERROR_CATEGORIES.MODEL_NOT_FOUND } });
} else {
onData('fallback content');
onComplete({ success: true });
}
});
const out = await runPromptThroughProvider({ provider: primary, prompt: 'p', source: 'test' });
expect(out.text).toBe('fallback content');
expect(out.fixTier).toBe(3);
// Both the primary key and the Tier-1 corrected-model key are cancelled so a
// fully-recovered action leaves ZERO investigation tasks.
expect(autoFixer.noteFallbackHandled).toHaveBeenCalledWith({ provider: 'Primary API', model: 'primary-model' });
expect(autoFixer.noteFallbackHandled).toHaveBeenCalledWith({ provider: 'Primary API', model: 'good-model' });
expect(autoFixer.noteFallbackFailed).not.toHaveBeenCalled();
});
it('escalates once and releases both suppressed keys when the corrected retry fails and no fallback exists', async () => {
mockToolkitWithFallback(null); // no fallback available
const primary = apiProvider({ id: 'primary-api', name: 'Primary API', defaultModel: 'primary-model', models: ['primary-model', 'good-model'] });
runner.executeApiRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, error: 'model gone', errorAnalysis: { category: ERROR_CATEGORIES.MODEL_NOT_FOUND } });
});
await expect(runPromptThroughProvider({ provider: primary, prompt: 'p', source: 'test' }))
.rejects.toThrow(/model gone/);
// Both the primary AND the pre-suppressed corrected-retry key were
// suppressed, so nothing incidental would surface the failure — the cascade
// escalates exactly one investigation task explicitly, then releases both
// keys so the in-flight set doesn't leak.
expect(autoFixer.noteFallbackStarted).toHaveBeenCalledWith({ provider: 'Primary API', model: 'primary-model' });
expect(autoFixer.noteFallbackStarted).toHaveBeenCalledWith({ provider: 'Primary API', model: 'good-model' });
expect(autoFixer.escalateProviderFailure).toHaveBeenCalledWith(expect.objectContaining({
context: expect.objectContaining({ provider: 'Primary API', model: 'primary-model' }),
}));
expect(autoFixer.noteFallbackFailed).toHaveBeenCalledWith({ provider: 'Primary API', model: 'primary-model' });
expect(autoFixer.noteFallbackFailed).toHaveBeenCalledWith({ provider: 'Primary API', model: 'good-model' });
expect(autoFixer.noteFallbackHandled).not.toHaveBeenCalled();
expect(autoFixer.escalateProviderFailure).toHaveBeenCalledTimes(1);
expect(autoFixer.noteFallbackFailed).toHaveBeenCalledTimes(2);
// The replacement must exist before suppression is released; finally must
// not escalate again after this explicit terminal operation.
expect(autoFixer.escalateProviderFailure.mock.invocationCallOrder[0])
.toBeLessThan(autoFixer.noteFallbackFailed.mock.invocationCallOrder[0]);
});
it('skips Tier 1 for a CLI provider whose model flag is baked into args (override ignored)', async () => {
// providerHonorsModelOverride is false when the CLI has a baked --model, so
// the "corrected" model would silently be the same failed one. Tier 1 must
// decline and the cascade fall through to the fallback provider instead.
runner.hasModelFlag.mockReturnValue(true); // args carry a baked model flag
const status = mockToolkitWithFallback();
const primary = { id: 'primary-cli', name: 'Primary CLI', type: 'cli', defaultModel: 'baked-model', timeout: 5000, args: ['--model', 'baked-model'], models: ['baked-model', 'good-model'] };
let primaryCalls = 0;
runner.executeCliRun.mockImplementation(async ({ onComplete }) => {
primaryCalls += 1;
onComplete({ success: false, error: 'model not found', errorAnalysis: { category: ERROR_CATEGORIES.MODEL_NOT_FOUND } });
});
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('fallback content');
onComplete({ success: true });
});
const out = await runPromptThroughProvider({ provider: primary, prompt: 'p', source: 'test' });
// No same-provider corrected retry (primary ran once), straight to Tier 3.
expect(primaryCalls).toBe(1);
expect(out.fixTier).toBe(3);
expect(status.getFallbackProvider).toHaveBeenCalledWith(
'primary-cli', expect.any(Object), null, null,
expect.objectContaining({ hasImages: false, requiredContextTokens: expect.any(Number) }),
);
});
it('does NOT attempt a Tier-1 correction for non-model failure categories', async () => {
const status = mockToolkitWithFallback();
// A rate-limit (Tier 3) failure must skip the config/env correction even
// though the provider lists an alternative model.
const primary = apiProvider({ id: 'primary-api', name: 'Primary API', defaultModel: 'primary-model', models: ['primary-model', 'good-model'] });
let primaryCalls = 0;
runner.executeApiRun.mockImplementation(async ({ provider: p, onData, onComplete }) => {
if (p.id === 'primary-api') {
primaryCalls += 1;
onComplete({ success: false, error: 'rate limited', errorAnalysis: { category: ERROR_CATEGORIES.RATE_LIMIT } });
} else {
onData('fallback content');
onComplete({ success: true });
}
});
const out = await runPromptThroughProvider({ provider: primary, prompt: 'p', source: 'test' });
// The primary provider ran exactly once (no Tier-1 corrected retry) before
// going straight to the Tier-3 fallback.
expect(primaryCalls).toBe(1);
expect(out.fixTier).toBe(3);
expect(status.getFallbackProvider).toHaveBeenCalledWith(
'primary-api', expect.any(Object), null, null,
expect.objectContaining({ hasImages: false, requiredContextTokens: expect.any(Number) }),
);
});
it('escalates explicitly when a Tier-1 pre-execution throw leaves no survivor and no fallback exists', async () => {
// The corrected retry throws BEFORE execution (createRun rejects), so its
// onRunFailed never queues a task. With no fallback either, the primary's
// pre-suppressed backstop task would be swallowed — so the cascade must
// escalate exactly one investigation task explicitly.
mockToolkitWithFallback(null); // no fallback
const primary = apiProvider({ id: 'primary-api', name: 'Primary API', defaultModel: 'primary-model', models: ['primary-model', 'good-model'] });
runner.createRun
.mockResolvedValueOnce({ runId: 'run-primary' }) // primary run
.mockRejectedValueOnce(new Error('provider disabled')); // corrected retry: pre-execution throw
runner.executeApiRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, error: 'model gone', errorAnalysis: { category: ERROR_CATEGORIES.MODEL_NOT_FOUND } });
});
await expect(runPromptThroughProvider({ provider: primary, prompt: 'p', source: 'test' }))
.rejects.toThrow(/model gone/);
expect(autoFixer.escalateProviderFailure).toHaveBeenCalledWith(expect.objectContaining({
code: 'AI_PROVIDER_EXECUTION_FAILED',
context: expect.objectContaining({ provider: 'Primary API', model: 'primary-model' }),
}));
expect(autoFixer.noteFallbackStarted).toHaveBeenCalledWith({ provider: 'Primary API', model: 'primary-model' });
});
it('escalates explicitly when the Tier-3 fallback throws before execution (no fallback task queued)', async () => {
// A fallback that rejects in createRun (pre-execution) fires no onRunFailed,
// so no fallback task is queued. With the primary suppressed, that would be
// zero investigation tasks — the cascade must escalate one explicitly.
mockToolkitWithFallback();
const primary = apiProvider({ id: 'primary-api', name: 'Primary API', defaultModel: 'primary-model' }); // no models → Tier 1 declines
runner.createRun
.mockResolvedValueOnce({ runId: 'run-primary' }) // primary
.mockRejectedValueOnce(new Error('fallback disabled')); // fallback: pre-execution throw
runner.executeApiRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, error: 'model gone', errorAnalysis: { category: ERROR_CATEGORIES.MODEL_NOT_FOUND } });
});
await expect(runPromptThroughProvider({ provider: primary, prompt: 'p', source: 'test' }))
.rejects.toThrow(/fallback disabled/);
expect(autoFixer.escalateProviderFailure).toHaveBeenCalledWith(expect.objectContaining({
context: expect.objectContaining({ provider: 'Primary API', model: 'primary-model' }),
}));
});
it('keys the Tier-1 corrected-retry task on the run that actually ran when createRun proactively swaps', async () => {
// The corrected retry's createRun proactively swaps to a different provider,
// which then fails — its queued task is keyed on the SWAPPED provider/model,
// so the recovery must cancel THAT key (not the pre-run guess).
const swapped = apiProvider({ id: 'swapped-api', name: 'Swapped API', defaultModel: 'swapped-model' });
mockToolkitWithFallback(); // fallback available for the Tier-3 recovery
const primary = apiProvider({ id: 'primary-api', name: 'Primary API', defaultModel: 'primary-model', models: ['primary-model', 'good-model'] });
runner.createRun
.mockResolvedValueOnce({ runId: 'run-primary' }) // primary
.mockResolvedValueOnce({ runId: 'run-corrected', provider: swapped }) // corrected retry swaps
.mockResolvedValue({ runId: 'run-fb' }); // fallback run
runner.executeApiRun.mockImplementation(async ({ provider: p, onData, onComplete }) => {
if (p.id === 'primary-api') {
onComplete({ success: false, error: 'model gone', errorAnalysis: { category: ERROR_CATEGORIES.MODEL_NOT_FOUND } });
} else if (p.id === 'swapped-api') {
onComplete({ success: false, error: 'swapped also failed', errorAnalysis: { category: ERROR_CATEGORIES.MODEL_NOT_FOUND } });
} else {
onData('fallback content');
onComplete({ success: true });
}
});
const out = await runPromptThroughProvider({ provider: primary, prompt: 'p', source: 'test' });
expect(out.text).toBe('fallback content');
expect(out.fixTier).toBe(3);
expect(autoFixer.noteFallbackHandled).toHaveBeenCalledWith({ provider: 'Swapped API', model: 'swapped-model' });
expect(autoFixer.noteFallbackHandled).toHaveBeenCalledWith({ provider: 'Primary API', model: 'primary-model' });
});
it('escalates when the fallback lookup itself throws after Tier 1 suppressed the keys', async () => {
// coalesceFallbackMarkAndPick throwing (e.g. getFallbackProvider blowing up)
// after Tier 1 cancelled the primary's backstop task would otherwise leave
// zero investigation tasks — the finally safety-net must escalate one.
const status = mockToolkitWithFallback();
status.getFallbackProvider.mockImplementation(() => { throw new Error('provider registry corrupt'); });
const primary = apiProvider({ id: 'primary-api', name: 'Primary API', defaultModel: 'primary-model', models: ['primary-model', 'good-model'] });
runner.executeApiRun.mockImplementation(async ({ onComplete }) => {
onComplete({ success: false, error: 'model gone', errorAnalysis: { category: ERROR_CATEGORIES.MODEL_NOT_FOUND } });
});
await expect(runPromptThroughProvider({ provider: primary, prompt: 'p', source: 'test' })).rejects.toThrow('provider registry corrupt');
expect(autoFixer.escalateProviderFailure).toHaveBeenCalledWith(expect.objectContaining({
context: expect.objectContaining({ provider: 'Primary API', model: 'primary-model' }),
}));
expect(autoFixer.escalateProviderFailure).toHaveBeenCalledTimes(1);
expect(autoFixer.noteFallbackFailed.mock.calls).toEqual([
[{ provider: 'Primary API', model: 'primary-model' }],
[{ provider: 'Primary API', model: 'good-model' }],
]);
expect(autoFixer.noteFallbackHandled).not.toHaveBeenCalled();
});
it('does NOT engage Tier 1 for a category-less failure even when the provider lists alternatives', async () => {
// Regression: a failure with no errorAnalysis has category === undefined,
// which must NOT match the model-not-found guard (the `undefined ===
// ERROR_CATEGORIES.MODEL_NOT_SUPPORTED` footgun). The primary must run once,
// then go straight to Tier 3 — never a same-provider model swap.
const status = mockToolkitWithFallback();
const primary = apiProvider({ id: 'primary-api', name: 'Primary API', defaultModel: 'primary-model', models: ['primary-model', 'good-model'] });
let primaryCalls = 0;
runner.executeApiRun.mockImplementation(async ({ provider: p, onData, onComplete }) => {
if (p.id === 'primary-api') {
primaryCalls += 1;
onComplete({ success: false, error: 'exited with code 1' }); // no errorAnalysis
} else {
onData('fallback content');
onComplete({ success: true });
}
});
const out = await runPromptThroughProvider({ provider: primary, prompt: 'p', source: 'test' });
expect(primaryCalls).toBe(1);
expect(out.fixTier).toBe(3);
expect(status.getFallbackProvider).toHaveBeenCalledWith(
'primary-api', expect.any(Object), null, null,
expect.objectContaining({ hasImages: false, requiredContextTokens: expect.any(Number) }),
);
});
});
describe('resolveEffectiveModel — embedding-model skip', () => {
it('skips an embedding-first models list when defaultModel is null (the nomic-embed-text fallback bug)', () => {
// Mirrors the real Ollama provider: defaultModel null, embedding model
// listed first. A generation/fallback run must NOT resolve the embedding.
const ollama = {
id: 'ollama', type: 'api', defaultModel: null,
models: ['nomic-embed-text:latest', 'qwen3.6:35b', 'gpt-oss:20b'],
};
expect(resolveEffectiveModel(ollama, null)).toBe('qwen3.6:35b');
});
it('still honors an explicit defaultModel', () => {
const p = { id: 'ollama', type: 'api', defaultModel: 'llama3.2:latest', models: ['nomic-embed-text:latest', 'llama3.2:latest'] };
expect(resolveEffectiveModel(p, null)).toBe('llama3.2:latest');
});
it('falls back to models[0] only when every model is an embedding model', () => {
const p = { id: 'ollama', type: 'api', defaultModel: null, models: ['nomic-embed-text:latest', 'mxbai-embed-large'] };
expect(resolveEffectiveModel(p, null)).toBe('nomic-embed-text:latest');
});
it('skips a stale embedding-only defaultModel (older config the UI never re-edited)', () => {
// A pre-existing local provider can carry an embedding id in defaultModel;
// a generation/fallback run must not route to it — fall through to the
// first generation model in the list.
const p = { id: 'ollama', type: 'api', defaultModel: 'nomic-embed-text:latest', models: ['nomic-embed-text:latest', 'qwen3.6:35b'] };
expect(resolveEffectiveModel(p, null)).toBe('qwen3.6:35b');
});
it('skips an embedding-only callerModel and falls through to a generation model', () => {
const p = { id: 'ollama', type: 'api', defaultModel: 'qwen3.6:35b', models: ['nomic-embed-text:latest', 'qwen3.6:35b'] };
expect(resolveEffectiveModel(p, 'nomic-embed-text:latest')).toBe('qwen3.6:35b');
});
});
describe('resolveProviderAndModel', () => {
beforeEach(() => {
providers.getActiveProvider.mockReset();
providers.getProviderById.mockReset();
});
it('uses providerId when it resolves; selectedModel falls back to defaultModel', async () => {
providers.getProviderById.mockResolvedValue({ id: 'p-1', type: 'api', defaultModel: 'm-default' });
const out = await resolveProviderAndModel({ providerId: 'p-1' });
expect(providers.getProviderById).toHaveBeenCalledWith('p-1');
expect(providers.getActiveProvider).not.toHaveBeenCalled();
expect(out.provider?.id).toBe('p-1');
expect(out.selectedModel).toBe('m-default');
});
it('caller model wins over provider.defaultModel when the provider honors overrides', async () => {
providers.getProviderById.mockResolvedValue({ id: 'p-1', type: 'api', defaultModel: 'm-default' });
const out = await resolveProviderAndModel({ providerId: 'p-1', model: 'm-override' });
expect(out.selectedModel).toBe('m-override');
});
it('falls back to getActiveProvider when providerId lookup throws', async () => {
providers.getProviderById.mockRejectedValue(new Error('stale id'));
providers.getActiveProvider.mockResolvedValue({ id: 'active-1', type: 'api', defaultModel: 'm-default' });
const out = await resolveProviderAndModel({ providerId: 'p-stale' });
expect(out.provider?.id).toBe('active-1');
expect(out.selectedModel).toBe('m-default');
});
it('falls back to getActiveProvider when providerId resolves to null', async () => {
providers.getProviderById.mockResolvedValue(null);
providers.getActiveProvider.mockResolvedValue({ id: 'active-1', type: 'api', defaultModel: 'm-default' });
const out = await resolveProviderAndModel({ providerId: 'p-missing' });
expect(out.provider?.id).toBe('active-1');
});
it('skips getProviderById entirely when no providerId is given', async () => {
providers.getActiveProvider.mockResolvedValue({ id: 'active-1', type: 'api', defaultModel: 'm-default' });
const out = await resolveProviderAndModel({});
expect(providers.getProviderById).not.toHaveBeenCalled();
expect(out.provider?.id).toBe('active-1');
});
it('returns { provider: null, selectedModel: null } when neither resolves', async () => {
providers.getActiveProvider.mockResolvedValue(null);
const out = await resolveProviderAndModel({});
expect(out).toEqual({ provider: null, selectedModel: null });
});
});
// =============================================================================
// Tier-2 (schema/type) request/response correction (issue #2350). At the runner
// layer a schema/type failure surfaces as a run that SUCCEEDED at the transport
// layer but whose response doesn't satisfy the caller's declared schema. The
// runner validates + coerces in place, then re-requests the SAME provider with a
// schema-corrected prompt, before falling through to a Tier-3 fallback.
// =============================================================================
describe('normalizeResponseSchema', () => {
it('returns null for a falsy schema (feature off)', () => {
expect(normalizeResponseSchema(null)).toBeNull();
expect(normalizeResponseSchema(undefined)).toBeNull();
expect(normalizeResponseSchema(0)).toBeNull();
});
it('wraps a bare predicate, swallowing predicate throws as false', () => {
const pred = normalizeResponseSchema((v) => v.ok === true);
expect(pred({ ok: true })).toBe(true);
expect(pred({ ok: false })).toBe(false);
expect(pred(null)).toBe(false); // predicate throws on null.ok → false, not a crash
});
it('adapts a Zod-style schema via safeParse', () => {
const schema = { safeParse: (v) => ({ success: typeof v === 'string' }) };
const pred = normalizeResponseSchema(schema);
expect(pred('hi')).toBe(true);
expect(pred(42)).toBe(false);
});
it('degrades an unrecognized truthy schema to always-true', () => {
const pred = normalizeResponseSchema({ description: 'a loose hint' });
expect(pred({ anything: 1 })).toBe(true);
});
});
describe('coerceResponseToSchema', () => {
const okShape = (v) => v && v.ok === true;
it('pulls a fenced JSON object matching the schema', () => {
const out = coerceResponseToSchema('```json\n{"ok":true,"n":1}\n```', okShape);
expect(out).toEqual({ ok: true, value: { ok: true, n: 1 }, text: '{"ok":true,"n":1}' });
});
it('strips leading prose and extracts the JSON block', () => {
const out = coerceResponseToSchema('Sure! Here you go:\n{"ok":true}', okShape);
expect(out.ok).toBe(true);
expect(out.value).toEqual({ ok: true });
});
it('coerces a top-level array when the schema expects one', () => {
const out = coerceResponseToSchema('```json\n[1,2,3]\n```', (v) => Array.isArray(v));
expect(out.ok).toBe(true);
expect(out.value).toEqual([1, 2, 3]);
});
it('returns { ok:false } when no block matches the schema', () => {
expect(coerceResponseToSchema('no json here', okShape)).toEqual({ ok: false });
expect(coerceResponseToSchema('{"ok":false}', okShape)).toEqual({ ok: false });
expect(coerceResponseToSchema('', okShape)).toEqual({ ok: false });
});
});
describe('isSchemaTypeCategory', () => {
it('recognizes the schema/type categories and rejects others', () => {
expect(isSchemaTypeCategory('parse-error')).toBe(true);
expect(isSchemaTypeCategory('bad-request')).toBe(true);
expect(isSchemaTypeCategory('output-length')).toBe(true);
expect(isSchemaTypeCategory('rate-limit')).toBe(false);
expect(isSchemaTypeCategory('model-not-found')).toBe(false);
expect(isSchemaTypeCategory(undefined)).toBe(false);
});
});
describe('promptRunner — Tier 2 schema/type correction (issue #2350)', () => {
const okShape = (v) => v && v.ok === true;
const fallbackApi = apiProvider({ id: 'fallback-api', name: 'Fallback API', defaultModel: 'fb-model' });
function mockToolkitWithFallback(fallback = fallbackApi) {
const isAvailable = vi.fn().mockReturnValue(true);
const markUnavailable = vi.fn().mockResolvedValue(undefined);
const markUsageLimit = vi.fn().mockResolvedValue(undefined);
const getFallbackProvider = vi.fn().mockReturnValue(fallback ? { provider: fallback, source: 'provider' } : null);
toolkitState.getAIToolkitInstance.mockReturnValue({
services: { providerStatus: { isAvailable, markUnavailable, markUsageLimit, getFallbackProvider } },
});
providers.getAllProviders.mockResolvedValue({ activeProvider: null, providers: fallback ? [fallback] : [] });
return { isAvailable, markUnavailable, markUsageLimit, getFallbackProvider };
}
it('returns a schema-valid response untouched (no coercion, no fixTier)', async () => {
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('{"ok":true}');
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: apiProvider({ name: 'P' }), prompt: 'p', source: 'test', responseSchema: okShape,
});
expect(out.text).toBe('{"ok":true}');
expect(out.fixTier).toBeUndefined();
expect(out.coercedResponse).toBeUndefined();
});
it('coerces a fenced/prose-wrapped response IN PLACE (fixTier 2, no retry, no fallback lookup)', async () => {
const status = mockToolkitWithFallback();
let calls = 0;
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
calls += 1;
onData('Here is your answer:\n```json\n{"ok":true,"v":9}\n```');
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: apiProvider({ name: 'P' }), prompt: 'p', source: 'test', responseSchema: okShape,
});
expect(out.text).toBe('{"ok":true,"v":9}');
expect(out.fixTier).toBe(2);
expect(out.fixStrategy).toBe('schema/type');
expect(out.coercedResponse).toBe(true);
expect(out.usedFallback).toBeUndefined(); // in-place coercion, same provider succeeded
expect(calls).toBe(1); // no retry needed
expect(status.getFallbackProvider).not.toHaveBeenCalled();
expect(autoFixer.noteFallbackStarted).not.toHaveBeenCalled();
});
it('uses a caller repair({ phase:"response" }) before the built-in coercion', async () => {
const repair = vi.fn(({ phase }) => (phase === 'response' ? { text: '{"ok":true,"fixed":1}' } : null));
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('total garbage, not even close');
onComplete({ success: true });
});
const out = await runPromptThroughProvider({
provider: apiProvider({ name: 'P' }), prompt: 'p', source: 'test', responseSchema: okShape, repair,
});
expect(out.text).toBe('{"ok":true,"fixed":1}');
expect(out.fixTier).toBe(2);
expect(repair).toHaveBeenCalledWith(expect.objectContaining({ phase: 'response' }));
});
it('stops the correction cascade when its retry is refused by policy', async () => {
const status = mockToolkitWithFallback();
let attempts = 0;
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
if (++attempts === 1) { onData('not json'); onComplete({ success: true }); }
else onComplete({ success: false, error: 'Policy refusal', errorAnalysis: { category: 'content-refusal' } });
});
await expect(runPromptThroughProvider({ provider: apiProvider({ name: 'P' }), prompt: 'p', source: 'test',
responseSchema: okShape })).rejects.toMatchObject({ errorAnalysis: { category: 'content-refusal' } });
expect(attempts).toBe(2);
expect(status.getFallbackProvider).not.toHaveBeenCalled();
expect(status.markUnavailable).not.toHaveBeenCalled();
expect(autoFixer.escalateProviderFailure).not.toHaveBeenCalled();
expect(autoFixer.noteFallbackFailed).not.toHaveBeenCalled();
});
it('stops the correction cascade when its retry exhausts a caller runtime budget', async () => {
const status = mockToolkitWithFallback();
let attempts = 0;
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
if (++attempts === 1) { onData('not json'); onComplete({ success: true }); }
else onComplete({ success: false, error: 'Caller runtime budget exhausted',
errorAnalysis: { category: 'runtime-budget-exhausted' } });
});
await expect(runPromptThroughProvider({ provider: apiProvider({ name: 'P' }), prompt: 'p', source: 'test',
responseSchema: okShape, absoluteTimeoutMs: 120000 })).rejects.toMatchObject({ code: 'RUN_RUNTIME_BUDGET_EXHAUSTED' });
expect(attempts).toBe(2);
expect(status.getFallbackProvider).not.toHaveBeenCalled();
expect(status.markUnavailable).not.toHaveBeenCalled();
expect(autoFixer.escalateProviderFailure).not.toHaveBeenCalled();
expect(autoFixer.noteFallbackFailed).not.toHaveBeenCalled();
});
it('re-requests the SAME provider with a schema-strengthened prompt when the response is uncoercible', async () => {
const status = mockToolkitWithFallback();
const primary = apiProvider({ id: 'primary-api', name: 'Primary API', defaultModel: 'm1' });
const prompts = [];
runner.executeApiRun.mockImplementation(async ({ prompt, onData, onComplete }) => {
prompts.push(prompt);
if (prompts.length === 1) { onData('not json'); onComplete({ success: true }); }
else { onData('{"ok":true}'); onComplete({ success: true }); }
});
const out = await runPromptThroughProvider({ provider: primary, prompt: 'p', source: 'test', responseSchema: okShape });
expect(out.text).toBe('{"ok":true}');
expect(out.fixTier).toBe(2);
expect(out.usedFallback).toBe(true);
expect(out.fallbackProvider).toMatchObject({ id: 'primary-api' });
// Re-request went to the SAME provider with a strengthened prompt.
expect(prompts).toHaveLength(2);
expect(prompts[1]).toContain('previous response could not be parsed');
// Tier-3 never engaged; the healthy provider was NOT benched for an off-shape response.
expect(status.getFallbackProvider).not.toHaveBeenCalled();
expect(status.markUnavailable).not.toHaveBeenCalled();
// The primary's incidental task was suppressed up front and resolved.
expect(autoFixer.noteFallbackStarted).toHaveBeenCalledWith({ provider: 'Primary API', model: 'm1' });
expect(autoFixer.noteFallbackHandled).toHaveBeenCalledWith({ provider: 'Primary API', model: 'm1' });
expect(autoFixer.noteFallbackFailed).not.toHaveBeenCalled();
});
it('prefers a caller repair({ phase:"request" }) prompt for the Tier-2 re-request', async () => {
mockToolkitWithFallback();
const repair = vi.fn(({ phase }) => (phase === 'request' ? { prompt: 'CORRECTED REQUEST' } : null));
const prompts = [];
runner.executeApiRun.mockImplementation(async ({ prompt, onData, onComplete }) => {
prompts.push(prompt);
if (prompts.length === 1) { onData('nope'); onComplete({ success: true }); }
else { onData('{"ok":true}'); onComplete({ success: true }); }
});
const out = await runPromptThroughProvider({
provider: apiProvider({ name: 'P' }), prompt: 'p', source: 'test', responseSchema: okShape, repair,
});
expect(out.fixTier).toBe(2);
expect(prompts[1]).toBe('CORRECTED REQUEST');
});
it('falls through to a Tier-3 fallback provider when the Tier-2 re-request is still off-schema', async () => {
const status = mockToolkitWithFallback();
const primary = apiProvider({ id: 'primary-api', name: 'Primary API', defaultModel: 'm1' });
runner.executeApiRun.mockImplementation(async ({ provider: p, onData, onComplete }) => {
if (p.id === 'primary-api') { onData('never valid'); onComplete({ success: true }); }
else { onData('{"ok":true}'); onComplete({ success: true }); }
});
const out = await runPromptThroughProvider({ provider: primary, prompt: 'p', source: 'test', responseSchema: okShape });
expect(out.text).toBe('{"ok":true}');
expect(out.fixTier).toBe(3);
expect(out.fallbackProvider).toMatchObject({ id: 'fallback-api' });
expect(status.getFallbackProvider).toHaveBeenCalledWith(
'primary-api', expect.any(Object), null, null,
expect.objectContaining({ hasImages: false, requiredContextTokens: expect.any(Number) }),
);
// Schema/type failure never benches the primary (it's response-specific, not an outage).
expect(status.markUnavailable).not.toHaveBeenCalled();
expect(autoFixer.noteFallbackHandled).toHaveBeenCalledWith({ provider: 'Primary API', model: 'm1' });
});
it('escalates and throws when even the Tier-3 fallback returns an uncoercible response', async () => {
mockToolkitWithFallback();
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('never ever valid json'); // every provider returns off-schema
onComplete({ success: true });
});
await expect(runPromptThroughProvider({
provider: apiProvider({ id: 'primary-api', name: 'Primary API' }), prompt: 'p', source: 'test', responseSchema: okShape,
})).rejects.toThrow(/did not match the declared schema/);
expect(autoFixer.escalateProviderFailure).toHaveBeenCalledTimes(1);
expect(autoFixer.noteFallbackHandled).not.toHaveBeenCalled();
});
it('leaves a messy response untouched when NO responseSchema is declared (regression)', async () => {
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('```json\n{"ok":true}\n```');
onComplete({ success: true });
});
const out = await runPromptThroughProvider({ provider: apiProvider(), prompt: 'p', source: 'test' });
expect(out.text).toBe('```json\n{"ok":true}\n```'); // no coercion without a schema
expect(out.fixTier).toBeUndefined();
});
});
describe('creative IP-latitude clause', () => {
beforeEach(() => {
runner.executeApiRun.mockImplementation(async ({ onData, onComplete }) => {
onData('ok');
onComplete({ success: true });
});
});
const promptSentToRunner = () => runner.executeApiRun.mock.calls[0][0].prompt;
it('prepends the clause when the run source names a creative request', async () => {
await runPromptThroughProvider({
provider: apiProvider(), prompt: 'Draw the vault door.', source: 'media-prompt-refine',
});
expect(promptSentToRunner()).toContain(CREATIVE_LATITUDE_HEADING);
expect(promptSentToRunner()).toContain('Draw the vault door.');
});
it('leaves an operational run source untouched', async () => {
await runPromptThroughProvider({
provider: apiProvider(), prompt: 'Classify this note.', source: 'brain-classify',
});
expect(promptSentToRunner()).toBe('Classify this note.');
});
it('does not double-stamp a prompt that already carries the clause', async () => {
const prompt = withCreativeLatitude('Write the fight beat.');
await runPromptThroughProvider({ provider: apiProvider(), prompt, source: 'music-lyrics' });
expect(promptSentToRunner().split(CREATIVE_LATITUDE_HEADING)).toHaveLength(2);
});
it('persists the stamped prompt on the run record so /runs shows what ran', async () => {
await runPromptThroughProvider({
provider: apiProvider(), prompt: 'Score the loop.', source: 'chiptune-score',
});
expect(runner.createRun.mock.calls[0][0].prompt).toContain(CREATIVE_LATITUDE_HEADING);
});
});
// ── Pre-dispatch context gate (#7441) ────────────────────────────────────────
// A local daemon accepts an oversized request, emits its banner, then produces
// nothing until the wall-clock timeout fires. Refusing up front is only correct
// if it fires ONLY on a known window, so both the refusal and the two
// unknown-window regressions live here.
describe('promptRunner — context gate on the requested provider', () => {
// ~13 tokens of prompt + the 8000-token default output reserve, against a
// daemon whose listing declared 4096 — a request that provably cannot fit.
const vllmCli = () => cliProvider({
id: 'opencode-vllm',
name: 'OpenCode vLLM',
defaultModel: 'qwen3.8-27b',
models: ['qwen3.8-27b'],
vllmBacked: true,
endpoint: 'http://127.0.0.1:18020/v1',
});
// ~5,000 prompt tokens on its own — over a 4096-token window with no
// reserve arithmetic needed.
const OVERSIZED_PROMPT = 'x'.repeat(20_000);
const servingWindow = (tokens) => async (provider) => ({
...provider,
modelContextWindows: { 'qwen3.8-27b': tokens },
});
// Tier-3 harness: a wider fallback the cascade can route to.
function mockToolkitWithFallbackFor(primary) {
const fallback = apiProvider({ id: 'fallback-api', name: 'Fallback API', defaultModel: 'fb-model' });
const markUnavailable = vi.fn().mockResolvedValue(undefined);
toolkitState.getAIToolkitInstance.mockReturnValue({
services: {
providerStatus: {
isAvailable: vi.fn().mockReturnValue(true),
markUnavailable,
markUsageLimit: vi.fn().mockResolvedValue(undefined),
getFallbackProvider: vi.fn().mockReturnValue({ provider: fallback, source: 'provider' }),
},
},
});
providers.getAllProviders.mockResolvedValue({ activeProvider: null, providers: [primary, fallback] });
return { markUnavailable };
}
it('refuses before dispatch, naming both token counts, for a pinned caller', async () => {
const status = mockToolkitWithFallbackFor(vllmCli());
observedWindows.withObservedContextWindows.mockImplementation(servingWindow(4096));
const rejection = await runPromptThroughProvider({
provider: vllmCli(),
prompt: OVERSIZED_PROMPT,
source: 'test',
allowFallback: false,
}).catch((err) => err);
// A pin opts out of ROUTING, not of health reporting — but this failure is
// the REQUEST's, not the provider's, so the pinned path must not bench a
// daemon that still serves smaller prompts perfectly well.
expect(status.markUnavailable).not.toHaveBeenCalled();
// Both numbers, because the operator's next move depends on the gap:
// a smaller prompt, or a provider with a wider window.
expect(rejection.message).toMatch(/known 4096-token context is below the \d+-token request budget/);
expect(rejection.message).toContain('OpenCode vLLM (qwen3.8-27b)');
// The whole point: no run record, no spawn, no ten-minute stall.
expect(runner.createRun).not.toHaveBeenCalled();
expect(runner.executeCliRun).not.toHaveBeenCalled();
});
// The refusal must act on what is PROVABLE. `requiredContextTokens` adds an
// 8,000-token output reserve that most callers never asked for; refusing on
// it would kill a 26K-token prompt to a 32K endpoint that expects three
// sentences back and succeeds today.
it('does not refuse on an output reserve the caller never asked for', async () => {
observedWindows.withObservedContextWindows.mockImplementation(servingWindow(4096));
runner.executeCliRun.mockImplementation(async ({ onComplete }) => onComplete({ success: true }));
// ~2,000 prompt tokens: under the 4096 window on its own, over it once the
// default 8,000-token reserve is added.
await runPromptThroughProvider({ provider: vllmCli(), prompt: 'x'.repeat(8_000), source: 'test' });
expect(runner.executeCliRun).toHaveBeenCalledTimes(1);
});
it('honors a reserve the caller DID declare', async () => {
observedWindows.withObservedContextWindows.mockImplementation(servingWindow(4096));
// Same prompt, but now the caller says it needs 4,000 tokens of answer —
// that genuinely does not fit, and it said so itself.
const rejection = await runPromptThroughProvider({
provider: vllmCli(),
prompt: 'x'.repeat(8_000),
source: 'test',
outputReserveTokens: 4_000,
allowFallback: false,
}).catch((err) => err);
expect(rejection.message).toContain('known 4096-token context is below');
expect(runner.executeCliRun).not.toHaveBeenCalled();
});
// stageRunner and the loops create the run record themselves. Throwing past
// one leaves the Runs page showing an attempt that never ends, and every
// `success === null` reconciler seeing a phantom active run.
it('settles a caller-supplied run record instead of stranding it open', async () => {
observedWindows.withObservedContextWindows.mockImplementation(servingWindow(4096));
await runPromptThroughProvider({
provider: vllmCli(),
prompt: OVERSIZED_PROMPT,
source: 'test',
runId: 'caller-owned-run',
allowFallback: false,
}).catch(() => {});
expect(runner.finalizeRunRecord).toHaveBeenCalledWith(expect.objectContaining({
runId: 'caller-owned-run',
success: false,
exitCode: 1,
error: expect.stringContaining('4096-token context'),
}));
});
it('routes an oversized request to a fallback instead of benching a healthy daemon', async () => {
const status = mockToolkitWithFallbackFor(vllmCli());
observedWindows.withObservedContextWindows.mockImplementation(async (provider) => (
provider.id === 'opencode-vllm' ? (await servingWindow(4096)(provider)) : provider
));
runner.executeApiRun.mockImplementation(async ({ onComplete }) => onComplete({ success: true }));
const out = await runPromptThroughProvider({
provider: vllmCli(),
prompt: OVERSIZED_PROMPT,
source: 'test',
});
expect(out.usedFallback).toBe(true);
expect(out.provider.id).toBe('fallback-api');
// The endpoint is healthy — a smaller prompt still works there — so this
// must not take it offline for every other caller.
expect(status.markUnavailable).not.toHaveBeenCalled();
});
// #9437 made the persistent mind declare an 8,192-token reserve on every
// wake. On a 16K local daemon that refused a ~13.5K-token prompt that fits
// ("known 16384-token context is below the 21744-token request budget").
// A local provider plans a window-sized reserve instead, and caps generation
// at the same number so the answer cannot outgrow the room the gate planned.
describe('local output reserve', () => {
beforeEach(() => {
vi.stubEnv('OLLAMA_CONTEXT_LENGTH', undefined);
vi.stubEnv('OLLAMA_URL', undefined);
vi.stubEnv('OLLAMA_HOST', undefined);
});
afterEach(() => {
vi.unstubAllEnvs();
});
const localOllama = (extra = {}) => apiProvider({
id: 'ollama',
name: 'Ollama',
defaultModel: 'qwen3.8-27b',
models: ['qwen3.8-27b'],
endpoint: 'http://localhost:11434/v1',
...extra,
});
// ~13,552 prompt tokens: fits 16K with a 2,048 reserve, not with 8,192.
const HELM_SIZED_PROMPT = 'x'.repeat(13_552 * 4);
it('dispatches a prompt that fits a 16K local window and caps max tokens at the local reserve', async () => {
observedWindows.withObservedContextWindows.mockImplementation(servingWindow(16_384));
runner.executeApiRun.mockImplementation(async ({ onComplete }) => onComplete({ success: true }));
await runPromptThroughProvider({
provider: localOllama(),
prompt: HELM_SIZED_PROMPT,
source: 'test',
outputReserveTokens: 8_192,
allowFallback: false,
});
expect(runner.executeApiRun).toHaveBeenCalledTimes(1);
expect(runner.executeApiRun.mock.calls[0][0].maxTokens).toBe(2_048);
});
it('still refuses a local prompt that cannot fit even with the smaller reserve', async () => {
observedWindows.withObservedContextWindows.mockImplementation(servingWindow(16_384));
const rejection = await runPromptThroughProvider({
provider: localOllama(),
prompt: 'x'.repeat(15_000 * 4),
source: 'test',
outputReserveTokens: 8_192,
allowFallback: false,
}).catch((err) => err);
expect(rejection.message).toMatch(/known 16384-token context is below the 17048-token request budget/);
expect(runner.executeApiRun).not.toHaveBeenCalled();
});
it('leaves an explicit caller maxTokens alone', async () => {
observedWindows.withObservedContextWindows.mockImplementation(servingWindow(32_768));
runner.executeApiRun.mockImplementation(async ({ onComplete }) => onComplete({ success: true }));
await runPromptThroughProvider({
provider: localOllama(),
prompt: 'short',
source: 'test',
outputReserveTokens: 8_192,
maxTokens: 6_000,
allowFallback: false,
});
expect(runner.executeApiRun.mock.calls[0][0].maxTokens).toBe(6_000);
});
it('reports the prompt room a local window leaves after its reserve', async () => {
observedWindows.withObservedContextWindows.mockImplementation(servingWindow(20_480));
await expect(resolveLocalPromptBudget({ provider: localOllama(), outputReserveTokens: 8_192 })).resolves.toEqual({
contextWindow: 20_480, outputReserveTokens: 2_560, maxPromptTokens: 17_920,
});
await expect(resolveLocalPromptBudget({
provider: apiProvider({ id: 'cloud-api', defaultModel: 'qwen3.8-27b', endpoint: 'https://api.example.com/v1' }),
outputReserveTokens: 8_192,
})).resolves.toBeNull();
observedWindows.withObservedContextWindows.mockImplementation(async (provider) => provider);
await expect(resolveLocalPromptBudget({ provider: localOllama(), outputReserveTokens: 8_192 })).resolves.toBeNull();
});
it('budgets from the configured Ollama window when no observed window is available', async () => {
vi.stubEnv('OLLAMA_CONTEXT_LENGTH', '32768');
vi.stubEnv('OLLAMA_URL', 'http://localhost:11434');
observedWindows.withObservedContextWindows.mockImplementation(async (provider) => provider);
await expect(resolveLocalPromptBudget({ provider: localOllama(), outputReserveTokens: 8_192 })).resolves.toEqual({
contextWindow: 32_768, outputReserveTokens: 4_096, maxPromptTokens: 28_672,
});
});
it('keeps the declared reserve for a cloud provider', async () => {
observedWindows.withObservedContextWindows.mockImplementation(servingWindow(16_384));
const rejection = await runPromptThroughProvider({
provider: apiProvider({ id: 'cloud-api', name: 'Cloud API', defaultModel: 'qwen3.8-27b', endpoint: 'https://api.example.com/v1' }),
prompt: HELM_SIZED_PROMPT,
source: 'test',
outputReserveTokens: 8_192,
allowFallback: false,
}).catch((err) => err);
expect(rejection.message).toMatch(/known 16384-token context is below the 21744-token request budget/);
expect(runner.executeApiRun).not.toHaveBeenCalled();
});
});
it('dispatches unchanged when nothing declared a window', async () => {
// The default mock is the no-observation answer: a daemon that is down,
// silent about windows, or not daemon-backed at all. Unknown must keep
// meaning "no constraint", never "refuse".
runner.executeCliRun.mockImplementation(async ({ onComplete }) => onComplete({ success: true }));
await runPromptThroughProvider({ provider: vllmCli(), prompt: 'x'.repeat(200_000), source: 'test' });
expect(runner.executeCliRun).toHaveBeenCalledTimes(1);
});
it('dispatches when the known window is wide enough', async () => {
observedWindows.withObservedContextWindows.mockImplementation(servingWindow(131_072));
runner.executeCliRun.mockImplementation(async ({ onComplete }) => onComplete({ success: true }));
await runPromptThroughProvider({ provider: vllmCli(), prompt: 'small', source: 'test' });
expect(runner.executeCliRun).toHaveBeenCalledTimes(1);
});
});