Claude Code. Any Model. The most powerful AI coding agent now speaks every language.
/**
* End-to-end tests around fallback routing and the LiteLLM-demotion refactor.
*
* Black-box tests. The proxy is invoked in-process via the public
* `createProxyServer()` entry point. Each test redirects the active global
* config to an ephemeral temp file so mutations never touch the real user config.
*
* The hermetic group runs in a HOME-sandboxed child over a checked-in catalog.
* Groups B and C are live provider roundtrip smoke tests only; they do not prove
* that the proxy honours defaultProvider. Their tests skipIf on missing credentials.
*
* Group D's D1b asserts the slim-catalog aggregators[] SHAPE (the fields
* claudish indexes) and that the live catalog still overlaps claudish's
* routing map — it does NOT police which providers the catalog carries,
* since that dataset is owned by the models-index repo and legitimately
* includes providers claudish doesn't route. The soft-skip remains only
* for the (now historical) case of a catalog with zero aggregators[].
*
* Run: bun test packages/cli/src/handlers/default-provider-e2e.test.ts
*/
import { afterAll, afterEach, beforeEach, describe, expect, test } from "bun:test";
import { copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { setConfigFileOverride } from "../config-override.js";
import { resolveDefaultProvider } from "../default-provider.js";
import { pickerProviderToFirebaseSlug } from "../model-selector.js";
import { createProxyServer } from "../proxy-server.js";
import { hasCredential } from "../test-helpers/credential-gate.js";
import type { ProxyServer } from "../types.js";
const STAGE4_CATALOG_FIXTURE = join(
import.meta.dir,
"..",
"test-fixtures",
"stage4-default-provider-catalog.json"
);
// ---------------------------------------------------------------------------
// Shared test infrastructure
// ---------------------------------------------------------------------------
const PORT_BASE = 19200;
let portCounter = 0;
function nextPort(): number {
return PORT_BASE + (portCounter++ % 400);
}
let activeProxy: ProxyServer | null = null;
let tempConfigPath: string | null = null;
let tempConfigCounter = 0;
let stderrRestore: (() => void) | null = null;
let stderrBuffer = "";
function captureStderr(): void {
stderrBuffer = "";
// Bun's console.error writes directly to fd 2, bypassing process.stderr.write.
// We must patch BOTH process.stderr.write AND console.error/console.warn
// to reliably observe what the proxy emits.
const originalWrite = process.stderr.write.bind(process.stderr);
const originalError = console.error.bind(console);
const originalWarn = console.warn.bind(console);
const append = (parts: unknown[]) => {
for (const p of parts) {
stderrBuffer += typeof p === "string" ? p : String(p);
stderrBuffer += " ";
}
stderrBuffer += "\n";
};
const writeReplacement = ((chunk: any, encoding?: any, cb?: any) => {
try {
stderrBuffer += typeof chunk === "string" ? chunk : chunk.toString("utf8");
} catch {}
return originalWrite(chunk, encoding, cb);
}) as typeof process.stderr.write;
process.stderr.write = writeReplacement;
console.error = (...args: unknown[]) => {
append(args);
originalError(...args);
};
console.warn = (...args: unknown[]) => {
append(args);
originalWarn(...args);
};
stderrRestore = () => {
process.stderr.write = originalWrite;
console.error = originalError;
console.warn = originalWarn;
};
}
function releaseStderr(): string {
const out = stderrBuffer;
stderrRestore?.();
stderrRestore = null;
stderrBuffer = "";
return out;
}
// setConfigFileOverride isolates fixtures so a crashed run cannot damage the developer's real config.
function sandboxHome(configJson?: Record<string, unknown>): void {
clearHomeSandbox();
tempConfigPath = join(
tmpdir(),
`default-provider-e2e.test.ts-${process.pid}-${tempConfigCounter++}.json`
);
if (configJson) {
writeFileSync(tempConfigPath, JSON.stringify(configJson, null, 2), "utf8");
}
setConfigFileOverride(tempConfigPath);
}
function clearHomeSandbox(): void {
setConfigFileOverride(null);
if (tempConfigPath) {
rmSync(tempConfigPath, { force: true });
tempConfigPath = null;
}
}
async function spinProxy(opts: {
defaultModel?: string;
quiet?: boolean;
}): Promise<number> {
const port = nextPort();
activeProxy = await createProxyServer(
port,
process.env.OPENROUTER_API_KEY,
opts.defaultModel,
false,
process.env.ANTHROPIC_API_KEY,
undefined,
{ quiet: opts.quiet ?? false }
);
return port;
}
async function killProxy(): Promise<void> {
if (activeProxy) {
try {
await activeProxy.shutdown();
} catch {}
activeProxy = null;
}
}
afterEach(async () => {
await killProxy();
if (stderrRestore) releaseStderr();
clearHomeSandbox();
});
afterAll(async () => {
await killProxy();
if (stderrRestore) releaseStderr();
clearHomeSandbox();
});
/**
* POST /v1/messages against the in-process proxy. Returns {ok, status, text}
* where text is the concatenated response content (JSON or SSE).
*
* maxTokens defaults to 64 — lower values (16) cause some providers to emit
* zero output tokens on "say hi" prompts and return an empty SSE stream.
*/
async function askProxy(
port: number,
model: string,
prompt: string,
maxTokens = 64
): Promise<{ ok: boolean; status: number; text: string; raw: any }> {
const res = await fetch(`http://127.0.0.1:${port}/v1/messages`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
model,
max_tokens: maxTokens,
stream: false,
messages: [{ role: "user", content: prompt }],
}),
});
const ct = res.headers.get("content-type") || "";
if (ct.includes("text/event-stream")) {
const raw = await res.text();
const parts: string[] = [];
let sawStop = false;
let sawError = false;
for (const line of raw.split("\n")) {
if (!line.startsWith("data:")) continue;
const data = line.replace(/^data:\s*/, "").trim();
if (!data || data === "[DONE]") continue;
try {
const p = JSON.parse(data);
if (p.type === "content_block_delta" && p.delta?.text) parts.push(p.delta.text);
if (p.type === "message_start" && Array.isArray(p.message?.content)) {
for (const b of p.message.content) if (b.text) parts.push(b.text);
}
if (p.choices?.[0]?.delta?.content) parts.push(p.choices[0].delta.content);
if (p.type === "message_stop") sawStop = true;
if (p.type === "error" || p.error) sawError = true;
} catch {}
}
// HTTP-level success: 2xx AND stream reached completion without error.
// Empty text with message_stop = provider accepted the request but
// produced no tokens (still a valid transport-level success).
const httpOk = res.ok && sawStop && !sawError;
return { ok: httpOk, status: res.status, text: parts.join(""), raw };
}
try {
const body = (await res.json()) as { content?: Array<{ text?: string }> };
let text = "";
if (Array.isArray(body?.content)) {
for (const b of body.content) if (b?.text) text += b.text;
}
return { ok: res.ok, status: res.status, text, raw: body };
} catch {
const raw = await res.text();
return { ok: false, status: res.status, text: "", raw };
}
}
const MARKER = () => `x${Math.random().toString(36).slice(2, 8)}`;
// ---------------------------------------------------------------------------
// Group A — Default provider precedence
// Most Group A scenarios (CLI > env > config > legacy > openrouter > hardcoded)
// are already exhaustively covered by the sibling unit files:
// - packages/cli/src/default-provider.test.ts
// - packages/cli/src/providers/auto-route-default-provider.test.ts
// This file only adds the one scenario those miss: the on-disk legacy-hint
// throttle-marker file lifecycle, observed from the filesystem as a user would.
// ---------------------------------------------------------------------------
describe("Group A — legacy LiteLLM auto-promotion removed (commit 5)", () => {
beforeEach(() => {
sandboxHome();
});
test("A1 — LITELLM env vars no longer auto-promote LiteLLM as default", () => {
// Pre-commit-5, LITELLM_BASE_URL + LITELLM_API_KEY together would resolve
// provider="litellm" with source="legacy-litellm" and legacyAutoPromoted=true
// (and the CLI emitted a one-shot stderr hint, throttled by a marker file).
// After commit 5 of the model-catalog and routing redesign, this auto-
// promotion path is gone — the resolver falls through to OPENROUTER_API_KEY
// or hardcoded "openrouter". Users wanting LiteLLM as default must set
// defaultProvider: "litellm" in the global config or set
// CLAUDISH_DEFAULT_PROVIDER=litellm. The marker file is no longer produced.
const env: NodeJS.ProcessEnv = {
HOME: tmpdir(),
LITELLM_BASE_URL: "http://example.invalid:4000",
LITELLM_API_KEY: "ll-test-key",
};
const result = resolveDefaultProvider({
env,
config: { version: "1.0.0", defaultProfile: "default", profiles: {} },
});
expect(result.provider).toBe("openrouter");
expect(result.source).toBe("hardcoded");
expect(result.legacyAutoPromoted).toBe(false);
});
});
// ---------------------------------------------------------------------------
// Hermetic proxy coverage — defaultProvider reaches bare-name routing
// ---------------------------------------------------------------------------
type HermeticProxyScenario = "config-empty" | "env-empty" | "custom-no-key";
interface HermeticProxyResult {
status: number;
raw: { error?: { message?: string } };
debugLog: string;
}
function runHermeticProxy(scenario: HermeticProxyScenario): HermeticProxyResult {
const home = mkdtempSync(join(tmpdir(), "claudish-default-provider-proxy-"));
const configDir = join(home, ".claudish");
mkdirSync(configDir, { recursive: true });
copyFileSync(STAGE4_CATALOG_FIXTURE, join(configDir, "cloud-models-catalog-v3.json"));
const config: Record<string, unknown> = {
version: "1.0.0",
defaultProfile: "default",
profiles: {},
defaultProvider: scenario === "config-empty" ? "" : "openrouter",
};
if (scenario === "custom-no-key") {
config.defaultProvider = "sandbox-no-key";
config.customEndpoints = {
"sandbox-no-key": {
kind: "simple",
url: "http://127.0.0.1:9/v1",
format: "openai",
apiKey: "${STAGE4_MISSING_KEY}",
},
};
}
writeFileSync(join(configDir, "config.json"), JSON.stringify(config), "utf8");
const proxyModuleUrl = new URL("../proxy-server.ts", import.meta.url).href;
const loggerModuleUrl = new URL("../logger.ts", import.meta.url).href;
const script = `
const { readFileSync, rmSync } = await import("node:fs");
const { createProxyServer } = await import(${JSON.stringify(proxyModuleUrl)});
const { getLogFilePath, initLogger } = await import(${JSON.stringify(loggerModuleUrl)});
let proxy;
let output;
try {
proxy = await createProxyServer(0, undefined, undefined, false, undefined, undefined, {
quiet: true,
});
initLogger(true, "debug", true);
const response = await fetch(proxy.url + "/v1/messages", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
model: "no-such-model-xyz",
max_tokens: 16,
stream: false,
messages: [{ role: "user", content: "hi" }],
}),
});
const raw = await response.json();
await Bun.sleep(150);
const logPath = getLogFilePath();
if (!logPath) throw new Error("debug logger did not expose its path");
const debugLog = readFileSync(logPath, "utf8");
initLogger(false, "info", true);
rmSync(logPath, { force: true });
output = JSON.stringify({
status: response.status,
raw,
debugLog,
});
} finally {
if (proxy) await proxy.shutdown();
}
process.stdout.write(output, () => process.exit(0));
`;
const env: Record<string, string> = {
HOME: home,
PATH: process.env.PATH ?? "",
TMPDIR: process.env.TMPDIR ?? tmpdir(),
OPENROUTER_API_KEY: "",
STAGE4_MISSING_KEY: "",
CLAUDISH_DISABLE_CATALOG_WARM: "1",
CLAUDISH_DISABLE_KEYCHAIN: "1",
CLAUDISH_DISABLE_OP: "1",
CLAUDISH_SKIP_LIVE_E2E: "1",
};
if (scenario === "env-empty") env.CLAUDISH_DEFAULT_PROVIDER = "";
try {
const result = Bun.spawnSync([process.execPath, "-e", script], {
cwd: join(import.meta.dir, "../../../.."),
env,
stdout: "pipe",
stderr: "pipe",
});
const stdout = result.stdout.toString();
const stderr = result.stderr.toString();
expect(result.exitCode, stderr || stdout).toBe(0);
return JSON.parse(stdout) as HermeticProxyResult;
} finally {
rmSync(home, { recursive: true, force: true });
}
}
describe("Hermetic proxy defaultProvider routing", () => {
test('config defaultProvider "" returns the catalog-empty no-route without an upstream call', () => {
const result = runHermeticProxy("config-empty");
expect(result.status).toBe(400);
expect(result.raw.error?.message).toContain(
'[Route] No provider in the catalog serves "no-such-model-xyz".'
);
expect(result.debugLog).not.toContain("[OpenRouter] Response status:");
});
test('CLAUDISH_DEFAULT_PROVIDER="" returns the same no-route without an upstream call', () => {
const result = runHermeticProxy("env-empty");
expect(result.status).toBe(400);
expect(result.raw.error?.message).toContain(
'[Route] No provider in the catalog serves "no-such-model-xyz".'
);
expect(result.debugLog).not.toContain("[OpenRouter] Response status:");
});
test("an uncredentialed custom fallback is named in the tried reason", () => {
const result = runHermeticProxy("custom-no-key");
expect(result.status).toBe(400);
expect(result.raw.error?.message).toContain(
'[Route] No credentialed providers in chain for "no-such-model-xyz" ' +
"(tried: sandbox-no-key)."
);
expect(result.debugLog).not.toContain("[sandbox-no-key] Response status:");
});
});
// ---------------------------------------------------------------------------
// Group B — Live provider roundtrips (not fallback-selection proof)
// ---------------------------------------------------------------------------
// Asked via claudish's OWN credential authority (env → aliases →
// global config apiKeys → 1Password), NOT raw process.env: a key
// stored any supported way makes claudish able to serve these models, so these
// tests must run — an env-only gate would silently skip a suite that could
// actually test something. LiteLLM additionally needs its base URL, which is an
// endpoint (not a credential) and so stays an env check.
//
// The credential check answers "can this run at all"; it does NOT answer "may this
// SPEND". Those are two different questions and this file only ever asked the first,
// so on any machine where credentials resolve (here, through 1Password) every group
// below called a real, metered provider on every `bun run test:safe` — including runs
// that had set `CLAUDISH_SKIP_LIVE_E2E=1` precisely to avoid that. Measured: B1a hit
// OpenRouter and B3 hit xAI with the flag set, and a spent subscription fell through
// the chain to a per-token provider. The suite's own failure count then tracked quota
// balance rather than code, which is what makes it unusable as a regression baseline.
//
// The flag is the same one `mcp-session/e2e-sessions.test.ts` already honours and that
// `.github/workflows/test.yml` sets for CI. It is ANDed with the credential check, not
// a replacement for it: "do not spend" and "cannot run" both have to be false before a
// live call is made.
const SKIP_LIVE_E2E = process.env.CLAUDISH_SKIP_LIVE_E2E === "1";
const HAS_OR = SKIP_LIVE_E2E ? false : await hasCredential("openrouter");
const HAS_LL = SKIP_LIVE_E2E
? false
: !!process.env.LITELLM_BASE_URL && (await hasCredential("litellm"));
const HAS_XAI = SKIP_LIVE_E2E ? false : await hasCredential("x-ai");
describe("Group B — live provider roundtrips", () => {
test.skipIf(!HAS_OR)(
"B1a — catalog-gathered gpt-5.4 bare → served by credentialed OpenRouter",
async () => {
// No user rule: the chain must come from the cloud models catalog. The
// gpt-5.4 row publishes OpenRouter, so defaultProvider is deduplicated
// rather than manufacturing a second connection.
sandboxHome({
version: "1.0.0",
defaultProfile: "default",
profiles: {},
defaultProvider: "openrouter",
});
captureStderr();
const t0 = Date.now();
const port = await spinProxy({ quiet: false });
const marker = MARKER();
const { ok, status, text, raw } = await askProxy(
port,
"gpt-5.4",
`say hi with marker ${marker}`
);
const stderr = releaseStderr();
const elapsed = Date.now() - t0;
if (!ok) {
console.error("[B1a] failed", { status, text, raw, stderr });
}
expect(ok).toBe(true);
expect(text.length).toBeGreaterThan(0);
console.log(
`[B1a] model=gpt-5.4 provider=openrouter elapsed=${elapsed}ms text="${text.slice(0, 60)}"`
);
// Route provenance.
//
// The proxy emits its `[Route] N providers for <model>: A → B` line ONLY
// when the credential-filtered chain has MORE THAN ONE candidate
// (proxy-server.ts: `if (!options.quiet && candidates.length > 1)`). The
// gathered chain includes OpenAI and OpenRouter, so on a machine WITHOUT
// an OpenAI credential it filters down to a single candidate and NO line
// is emitted — which is the common case for someone holding only an
// OpenRouter key. A bare `expect(stderr).toContain("openrouter")` therefore
// failed deterministically in exactly that setup, while the routing under
// test worked perfectly (ok + non-empty text, above, prove OpenRouter
// served it: it is the only credentialed provider left in the chain).
//
// This is NOT the "Bun async logging is flaky" effect the sibling tests
// assume — it is deterministic, and reproducible by setting/unsetting
// OPENAI_API_KEY. So: assert the line's CONTENT when it exists, and always
// guard the regression this test actually exists for.
const lower = stderr.toLowerCase();
const routeLine = lower.split("\n").find((l) => l.includes("[route]"));
if (routeLine) {
expect(routeLine).toContain("openrouter");
}
// Legacy auto-promotion guard (commit 5): litellm must NEVER lead the chain.
expect(routeLine ?? "").not.toMatch(/\[route\][^:]*:\s*litellm/);
},
90_000
);
test.skipIf(!HAS_OR)(
"B1b — defaultProvider=openrouter + gemini-3.1-pro-preview bare → served by OpenRouter",
async () => {
sandboxHome({
version: "1.0.0",
defaultProfile: "default",
profiles: {},
defaultProvider: "openrouter",
});
captureStderr();
const t0 = Date.now();
const port = await spinProxy({ quiet: false });
const marker = MARKER();
// 256, not the 64 default: gemini-3.1-pro-preview is a THINKING model and
// its reasoning is billed against max_tokens. Measured against OpenRouter
// on 2026-08-06 with this exact prompt — at 64 the reasoning consumed the
// whole budget (1/3 runs returned empty content with finish_reason=length,
// 2/3 truncated mid-sentence); at 256, 3/3 complete with finish_reason=stop.
const { ok, status, text, raw } = await askProxy(
port,
"gemini-3.1-pro-preview",
`say hi marker ${marker}`,
256
);
const stderr = releaseStderr();
const elapsed = Date.now() - t0;
if (!ok) {
console.error("[B1b] failed", { status, text, raw, stderr });
}
console.log(
`[B1b] model=gemini-3.1-pro-preview provider=openrouter elapsed=${elapsed}ms text="${text.slice(0, 60)}"`
);
// The load-bearing assertion is that the request succeeded end-to-end.
// Empty text is NOT treated as acceptable here: with an adequate budget
// this prompt returned content 5/5 upstream, so a blank answer means
// something in the pipeline lost it (that is exactly how the Gemini
// reasoning-filter bug hid for seven months).
expect(ok).toBe(true);
expect(text.trim()).not.toBe("");
// Best-effort stderr provenance check — Bun async logging is flaky
const lower = stderr.toLowerCase();
if (!lower.includes("openrouter")) {
console.log("[B1b] stderr capture missed openrouter route marker (Bun async timing)");
}
},
90_000
);
test.skipIf(!HAS_LL)(
"B2 — defaultProvider=litellm + minimax-m2.5 bare → LiteLLM remains the fallback",
async () => {
sandboxHome({
version: "1.0.0",
defaultProfile: "default",
profiles: {},
defaultProvider: "litellm",
});
captureStderr();
const t0 = Date.now();
const port = await spinProxy({ quiet: false });
const { ok, status, text, raw } = await askProxy(port, "minimax-m2.5", `say hi ${MARKER()}`);
const stderr = releaseStderr();
const elapsed = Date.now() - t0;
if (!ok) {
console.error("[B2] failed", { status, text, raw, stderr });
}
// LiteLLM may or may not resolve the bare name — the critical assertion
// is that the request succeeded end-to-end. Stderr observability is
// best-effort due to Bun async timing.
const lower = stderr.toLowerCase();
const llIdx = lower.indexOf("litellm");
const orIdx = lower.indexOf("openrouter");
console.log(
`[B2] model=minimax-m2.5 ok=${ok} elapsed=${elapsed}ms litellm@${llIdx} openrouter@${orIdx} textLen=${text.length}`
);
expect(ok).toBe(true);
// `defaultProvider` is the last-resort fallback position. If both names
// are visible, the catalog-gathered OpenRouter connection must precede it.
if (llIdx >= 0 && orIdx >= 0) expect(orIdx).toBeLessThan(llIdx);
},
90_000
);
test.skipIf(!HAS_XAI)(
"B3 — explicit xai@grok-code-fast-1 bypasses default-provider (no openrouter route)",
async () => {
sandboxHome({
version: "1.0.0",
defaultProfile: "default",
profiles: {},
defaultProvider: "openrouter",
});
captureStderr();
const t0 = Date.now();
const port = await spinProxy({ quiet: false });
const { ok, status, text, raw } = await askProxy(
port,
"xai@grok-code-fast-1",
`say hi ${MARKER()}`
);
const stderr = releaseStderr();
const elapsed = Date.now() - t0;
if (!ok) {
console.error("[B3] failed", { status, text, raw, stderr });
}
console.log(
`[B3] model=xai@grok-code-fast-1 ok=${ok} elapsed=${elapsed}ms text="${text.slice(0, 60)}"`
);
// Explicit provider path must hit XAI. We assert success OR a single-
// provider error (never a fallback chain error).
if (!ok) {
const r = typeof raw === "string" ? raw : JSON.stringify(raw);
expect(r).not.toContain("all_providers_failed");
} else {
expect(text.length).toBeGreaterThan(0);
}
},
90_000
);
test.skipIf(!HAS_LL)(
"B4 — legacy auto-promotion gone (commit 5): no banner ever appears",
async () => {
// Pre-commit-5 this test asserted the one-shot hint fired on the first
// call and was throttled (via marker file) on the second. Commit 5 of
// the model-catalog and routing redesign removed the auto-promotion
// path entirely — the resolver no longer treats `LITELLM_*` env vars
// as "make LiteLLM default". Now both calls must be banner-free.
sandboxHome({ version: "1.0.0", defaultProfile: "default", profiles: {} });
captureStderr();
const port = await spinProxy({ quiet: false });
await askProxy(port, "minimax-m2.5", `hi ${MARKER()}`);
await killProxy();
const firstStderr = releaseStderr();
const firstLower = firstStderr.toLowerCase();
const firstBanner = firstLower.includes("deprecat") && firstLower.includes("litellm");
expect(firstBanner).toBe(false);
captureStderr();
const port2 = await spinProxy({ quiet: false });
await askProxy(port2, "minimax-m2.5", `hi ${MARKER()}`);
const secondStderr = releaseStderr();
const secondLower = secondStderr.toLowerCase();
const secondBanner = secondLower.includes("deprecat") && secondLower.includes("litellm");
expect(secondBanner).toBe(false);
},
120_000
);
});
// ---------------------------------------------------------------------------
// Group C — Live custom-endpoint roundtrips (not fallback-selection proof)
// ---------------------------------------------------------------------------
describe("Group C — live custom-endpoint roundtrips", () => {
// The model is INCIDENTAL to Group C — these tests prove custom-endpoint
// registration + `${VAR}` expansion + roundtrip, and the endpoint routes
// through the OpenAI adapter regardless of the model name. We deliberately
// use a reliable non-reasoning chat model: an earlier choice of
// `minimax-m2.5` (a reasoning model) intermittently returned 0 visible
// tokens on a "say hi" prompt, which made the `text.length > 0` roundtrip
// assertion flaky even though the HTTP call succeeded. gpt-4o-mini returns
// deterministic text, so the strict assertion can stay.
const C_MODEL = "e2e-test-ep@openai/gpt-4o-mini";
test.skipIf(!HAS_OR)(
"C1 — custom endpoint e2e-test-ep with ${OPENROUTER_API_KEY} works",
async () => {
sandboxHome({
version: "1.0.0",
defaultProfile: "default",
profiles: {},
customEndpoints: {
"e2e-test-ep": {
kind: "simple",
url: "https://openrouter.ai/api/v1",
format: "openai",
apiKey: "${OPENROUTER_API_KEY}",
},
},
defaultProvider: "e2e-test-ep",
});
captureStderr();
const t0 = Date.now();
const port = await spinProxy({ quiet: false });
const { ok, status, text, raw } = await askProxy(port, C_MODEL, `say hi ${MARKER()}`);
const stderr = releaseStderr();
const elapsed = Date.now() - t0;
if (!ok) console.error("[C1] failed", { status, text, raw, stderr });
console.log(
`[C1] model=${C_MODEL} ok=${ok} elapsed=${elapsed}ms text="${text.slice(0, 60)}"`
);
// Correctness signal: the request succeeded with non-empty output, which
// proves the custom endpoint was registered + handler created + ${VAR}
// expanded + request roundtripped. Stderr observability is best-effort
// because the proxy logs registration counts (not names) and Bun's
// async logging timing makes capture flaky.
expect(ok).toBe(true);
expect(text.length).toBeGreaterThan(0);
},
90_000
);
test.skipIf(!HAS_OR)(
"C2 — invalid custom endpoint is warned but bare call still succeeds",
async () => {
sandboxHome({
version: "1.0.0",
defaultProfile: "default",
profiles: {},
customEndpoints: {
"e2e-test-ep": {
kind: "simple",
url: "https://openrouter.ai/api/v1",
format: "openai",
apiKey: "${OPENROUTER_API_KEY}",
},
"broken-ep": {
kind: "simple",
// missing url on purpose
format: "openai",
apiKey: "ignored",
},
},
defaultProvider: "openrouter",
});
captureStderr();
const t0 = Date.now();
const port = await spinProxy({ quiet: false });
const { ok, status, text, raw } = await askProxy(port, "gpt-5.4", `hi ${MARKER()}`);
const stderr = releaseStderr();
const elapsed = Date.now() - t0;
if (!ok) console.error("[C2] failed", { status, text, raw, stderr });
console.log(`[C2] ok=${ok} elapsed=${elapsed}ms text="${text.slice(0, 60)}"`);
// Best-effort warning observation — Bun's async console capture is flaky.
// The bun-test stdout stream often shows the warning even when the
// patched JS-level capture misses it. The CRITICAL assertion is that
// the bare call STILL succeeded (the broken endpoint didn't crash startup).
const lower = stderr.toLowerCase();
const mentionsBroken = lower.includes("broken-ep");
const mentionsWarn =
lower.includes("warn") || lower.includes("invalid") || lower.includes("skip");
if (!(mentionsBroken || mentionsWarn)) {
console.log(
"[C2] stderr capture missed the broken-ep warning (Bun async timing) " +
"— continuing because the bare call succeeded which is the load-bearing assertion"
);
}
// Bare call still works
if (ok) {
expect(text.length).toBeGreaterThan(0);
}
},
90_000
);
test.skipIf(!HAS_OR)(
"C3 — ${E2E_TEST_KEY} template is expanded from process env",
async () => {
const savedKey = process.env.E2E_TEST_KEY;
process.env.E2E_TEST_KEY = process.env.OPENROUTER_API_KEY;
try {
sandboxHome({
version: "1.0.0",
defaultProfile: "default",
profiles: {},
customEndpoints: {
"e2e-test-ep": {
kind: "simple",
url: "https://openrouter.ai/api/v1",
format: "openai",
apiKey: "${E2E_TEST_KEY}",
},
},
defaultProvider: "e2e-test-ep",
});
captureStderr();
const t0 = Date.now();
const port = await spinProxy({ quiet: false });
const { ok, status, text, raw } = await askProxy(port, C_MODEL, `hi ${MARKER()}`);
const stderr = releaseStderr();
const elapsed = Date.now() - t0;
if (!ok) console.error("[C3] failed", { status, text, raw, stderr });
console.log(`[C3] ok=${ok} elapsed=${elapsed}ms text="${text.slice(0, 60)}"`);
// If the literal ${E2E_TEST_KEY} string was passed to OpenRouter, we'd
// get HTTP 401. The fact that we got HTTP 200 (ok=true) IS the proof
// that the template was expanded. Empty text content is independent —
// some models occasionally return 0 tokens on "say hi" prompts even
// on a successful HTTP roundtrip. The expansion is what we're testing.
if (ok) {
expect(ok).toBe(true);
} else {
// If we failed, it MUST NOT be because the literal placeholder was forwarded
const r = typeof raw === "string" ? raw : JSON.stringify(raw);
expect(r).not.toContain("${E2E_TEST_KEY}");
}
} finally {
if (savedKey === undefined) delete process.env.E2E_TEST_KEY;
else process.env.E2E_TEST_KEY = savedKey;
}
},
90_000
);
});
// ---------------------------------------------------------------------------
// Group D — Firebase slim catalog aggregators[] contract
// ---------------------------------------------------------------------------
// The Firebase provider slugs claudish can actually route an aggregator entry
// to — derived from the REAL routing map (`pickerProviderToFirebaseSlug` in
// model-selector.ts, consumed by `resolveProviderAggregatorEntry`), NOT a
// hand-kept list. The catalog is owned by a DIFFERENT repo (models-index); it
// legitimately carries providers claudish doesn't route (e.g. `mistralai` — no
// Mistral provider here; `anthropic` — handled natively, not via the picker).
// Those are inert by construction (unmatched → ignored), so their presence is
// NOT a claudish defect and must never fail this suite. A stale allowlist here
// used to hard-fail CI every time models-index shipped a new provider.
//
// SKIPPED under CLAUDISH_SKIP_LIVE_E2E, and the group now needs that gate for a
// second reason. These tests assert the v2 slim-catalog shape. As observed on
// 2026-09-18 the live endpoint serves contract v3: 426 without the v3 Accept
// header, 503 with it. The shape asserted below is therefore no longer served at
// all. Rewrite them against v3 as part of the v3 reader work — do not delete
// them, because the aggregator-shape coverage is still wanted.
const ROUTABLE_FIREBASE_SLUGS = new Set(Object.values(pickerProviderToFirebaseSlug));
describe.skipIf(SKIP_LIVE_E2E)("Group D — Firebase slim catalog", () => {
let cachedBody: any = null;
async function fetchCatalog(): Promise<any> {
if (cachedBody) return cachedBody;
const res = await fetch(
"https://us-central1-claudish-6da10.cloudfunctions.net/queryModels?status=active&catalog=slim&limit=100",
{ headers: { Accept: "application/vnd.models-index.catalog+json;version=3" } }
);
expect(res.status).toBe(200);
const envelope = await res.json();
expect(envelope.contractVersion).toBe(3);
cachedBody = envelope.data;
return cachedBody;
}
test("D1 — catalog returns {models: [...]} with at least one entry", async () => {
const body = await fetchCatalog();
expect(body).toBeDefined();
expect(Array.isArray(body.models)).toBe(true);
expect(body.models.length).toBeGreaterThan(0);
console.log(`[D1] slim catalog models count=${body.models.length}`);
}, 15_000);
test("D1b — aggregators[] shape contract + routing-map overlap (soft-skip if Phase 4 not deployed)", async () => {
const body = await fetchCatalog();
const withAgg = (body.models as any[]).filter(
(m) => Array.isArray(m?.aggregators) && m.aggregators.length > 0
);
if (withAgg.length === 0) {
console.log("[D1b] PENDING DEPLOY — no models have aggregators[] yet");
return;
}
console.log(`[D1b] ${withAgg.length}/${body.models.length} models have aggregators[]`);
// (1) SHAPE is a hard contract: claudish's `resolveProviderAggregatorEntry`
// and `resolveProviderExternalId` index these fields directly, so a
// non-string provider/externalId/confidence would break consumption.
const seenProviders = new Set<string>();
for (const m of withAgg) {
for (const agg of m.aggregators) {
expect(typeof agg.sourceProviderId).toBe("string");
if (agg.routeStatus !== "mapped") continue;
expect(typeof agg.route?.routeId).toBe("string");
expect(typeof agg.route?.routeProfileId).toBe("string");
expect(typeof agg.externalModelId).toBe("string");
expect(typeof agg.confidence).toBe("string");
seenProviders.add(agg.sourceProviderId);
}
}
// (2) ROUTABILITY is reported, not policed. Providers claudish maps to are
// routable; the rest are inert catalog data owned by models-index and are
// simply ignored at runtime — NOT a failure of this repo.
const routable = [...seenProviders].filter((p) => ROUTABLE_FIREBASE_SLUGS.has(p)).sort();
const unroutable = [...seenProviders].filter((p) => !ROUTABLE_FIREBASE_SLUGS.has(p)).sort();
console.log(`[D1b] routable by claudish: ${routable.join(", ") || "(none)"}`);
if (unroutable.length > 0) {
console.log(
`[D1b] catalog providers claudish does not route (inert, expected): ${unroutable.join(", ")}`
);
}
// (3) DRIFT sanity: at least one aggregator provider must be routable. Zero
// overlap would mean the catalog's slug convention diverged from claudish's
// `pickerProviderToFirebaseSlug` entirely — a real, claudish-side routing
// break — without being brittle to any single provider coming or going.
expect(routable.length).toBeGreaterThan(0);
}, 15_000);
test("D2 — entries without aggregators[] parse cleanly (field is optional)", async () => {
const body = await fetchCatalog();
const withoutAgg = (body.models as any[]).filter(
(m) => !Array.isArray(m?.aggregators) || m.aggregators.length === 0
);
console.log(`[D2] models without aggregators[]: ${withoutAgg.length}`);
// Just a shape sanity: each should still have SOMETHING identifiable.
// The slim catalog uses `modelId` (not `id` or `name`).
for (const m of withoutAgg.slice(0, 20)) {
const hasIdentifier =
typeof m.modelId === "string" || typeof m.id === "string" || typeof m.name === "string";
expect(hasIdentifier).toBe(true);
}
}, 15_000);
});
// ---------------------------------------------------------------------------
// Group E — End-to-end config flip happy path
// ---------------------------------------------------------------------------
describe("Group E — config flip happy path", () => {
test.skipIf(!HAS_OR || !HAS_LL)(
"E1 — openrouter → litellm flip with grok-4.20 bare",
async () => {
// Phase 1: defaultProvider=openrouter
sandboxHome({
version: "1.0.0",
defaultProfile: "default",
profiles: {},
defaultProvider: "openrouter",
});
captureStderr();
const t0 = Date.now();
const port = await spinProxy({ quiet: false });
const phase1 = await askProxy(port, "grok-4.20", `say hi ${MARKER()}`);
await killProxy();
const phase1Stderr = releaseStderr();
const elapsed1 = Date.now() - t0;
const lower1 = phase1Stderr.toLowerCase();
console.log(
`[E1-openrouter] ok=${phase1.ok} elapsed=${elapsed1}ms text="${phase1.text.slice(0, 40)}"`
);
// Phase 1 correctness: bare-model invocation succeeded with non-empty
// response. We don't assert on stderr provenance here because Bun's
// async stderr capture is unreliable from inside test handlers — the
// upstream proxy logs land in the bun-test output pipe but skip the
// patched JS-level capture. The non-empty response IS the proof.
expect(phase1.ok).toBe(true);
expect(phase1.text.length).toBeGreaterThan(0);
// No legacy migration banner on explicit defaultProvider (when captured)
const legacyBanner1 = lower1.includes("deprecat") && lower1.includes("litellm");
expect(legacyBanner1).toBe(false);
// Phase 2: flip to litellm
writeFileSync(
tempConfigPath!,
JSON.stringify({
version: "1.0.0",
defaultProfile: "default",
profiles: {},
defaultProvider: "litellm",
}),
"utf8"
);
captureStderr();
const t1 = Date.now();
const port2 = await spinProxy({ quiet: false });
const phase2 = await askProxy(port2, "grok-4.20", `hi ${MARKER()}`);
const phase2Stderr = releaseStderr();
const elapsed2 = Date.now() - t1;
const lower2 = phase2Stderr.toLowerCase();
console.log(
`[E1-litellm] ok=${phase2.ok} elapsed=${elapsed2}ms text="${phase2.text.slice(0, 40)}"`
);
// LiteLLM should appear in the route; legacy banner should NOT (explicit config)
const legacyBanner2 = lower2.includes("deprecat") && lower2.includes("litellm");
expect(legacyBanner2).toBe(false);
// We expect either a litellm route attempt or a successful litellm response
const llMentioned = lower2.includes("litellm");
console.log(`[E1-litellm] litellmMentioned=${llMentioned}`);
},
180_000
);
});