Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

39.0 KB
Raw Blame
/**
 * End-to-end tests around fallback routing and the LiteLLM-demotion refactor.
 *
 * Black-box tests. The proxy is invoked in-process via the public
 * `createProxyServer()` entry point. Each test redirects the active global
 * config to an ephemeral temp file so mutations never touch the real user config.
 *
 * The hermetic group runs in a HOME-sandboxed child over a checked-in catalog.
 * Groups B and C are live provider roundtrip smoke tests only; they do not prove
 * that the proxy honours defaultProvider. Their tests skipIf on missing credentials.
 *
 * Group D's D1b asserts the slim-catalog aggregators[] SHAPE (the fields
 * claudish indexes) and that the live catalog still overlaps claudish's
 * routing map — it does NOT police which providers the catalog carries,
 * since that dataset is owned by the models-index repo and legitimately
 * includes providers claudish doesn't route. The soft-skip remains only
 * for the (now historical) case of a catalog with zero aggregators[].
 *
 * Run: bun test packages/cli/src/handlers/default-provider-e2e.test.ts
 */

import { afterAll, afterEach, beforeEach, describe, expect, test } from "bun:test";
import { copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { setConfigFileOverride } from "../config-override.js";
import { resolveDefaultProvider } from "../default-provider.js";
import { pickerProviderToFirebaseSlug } from "../model-selector.js";
import { createProxyServer } from "../proxy-server.js";
import { hasCredential } from "../test-helpers/credential-gate.js";
import type { ProxyServer } from "../types.js";

const STAGE4_CATALOG_FIXTURE = join(
  import.meta.dir,
  "..",
  "test-fixtures",
  "stage4-default-provider-catalog.json"
);

// ---------------------------------------------------------------------------
// Shared test infrastructure
// ---------------------------------------------------------------------------

const PORT_BASE = 19200;
let portCounter = 0;
function nextPort(): number {
  return PORT_BASE + (portCounter++ % 400);
}

let activeProxy: ProxyServer | null = null;
let tempConfigPath: string | null = null;
let tempConfigCounter = 0;
let stderrRestore: (() => void) | null = null;
let stderrBuffer = "";

function captureStderr(): void {
  stderrBuffer = "";
  // Bun's console.error writes directly to fd 2, bypassing process.stderr.write.
  // We must patch BOTH process.stderr.write AND console.error/console.warn
  // to reliably observe what the proxy emits.
  const originalWrite = process.stderr.write.bind(process.stderr);
  const originalError = console.error.bind(console);
  const originalWarn = console.warn.bind(console);
  const append = (parts: unknown[]) => {
    for (const p of parts) {
      stderrBuffer += typeof p === "string" ? p : String(p);
      stderrBuffer += " ";
    }
    stderrBuffer += "\n";
  };
  const writeReplacement = ((chunk: any, encoding?: any, cb?: any) => {
    try {
      stderrBuffer += typeof chunk === "string" ? chunk : chunk.toString("utf8");
    } catch {}
    return originalWrite(chunk, encoding, cb);
  }) as typeof process.stderr.write;
  process.stderr.write = writeReplacement;
  console.error = (...args: unknown[]) => {
    append(args);
    originalError(...args);
  };
  console.warn = (...args: unknown[]) => {
    append(args);
    originalWarn(...args);
  };
  stderrRestore = () => {
    process.stderr.write = originalWrite;
    console.error = originalError;
    console.warn = originalWarn;
  };
}

function releaseStderr(): string {
  const out = stderrBuffer;
  stderrRestore?.();
  stderrRestore = null;
  stderrBuffer = "";
  return out;
}

// setConfigFileOverride isolates fixtures so a crashed run cannot damage the developer's real config.
function sandboxHome(configJson?: Record<string, unknown>): void {
  clearHomeSandbox();
  tempConfigPath = join(
    tmpdir(),
    `default-provider-e2e.test.ts-${process.pid}-${tempConfigCounter++}.json`
  );
  if (configJson) {
    writeFileSync(tempConfigPath, JSON.stringify(configJson, null, 2), "utf8");
  }
  setConfigFileOverride(tempConfigPath);
}

function clearHomeSandbox(): void {
  setConfigFileOverride(null);
  if (tempConfigPath) {
    rmSync(tempConfigPath, { force: true });
    tempConfigPath = null;
  }
}

async function spinProxy(opts: {
  defaultModel?: string;
  quiet?: boolean;
}): Promise<number> {
  const port = nextPort();
  activeProxy = await createProxyServer(
    port,
    process.env.OPENROUTER_API_KEY,
    opts.defaultModel,
    false,
    process.env.ANTHROPIC_API_KEY,
    undefined,
    { quiet: opts.quiet ?? false }
  );
  return port;
}

async function killProxy(): Promise<void> {
  if (activeProxy) {
    try {
      await activeProxy.shutdown();
    } catch {}
    activeProxy = null;
  }
}

afterEach(async () => {
  await killProxy();
  if (stderrRestore) releaseStderr();
  clearHomeSandbox();
});

afterAll(async () => {
  await killProxy();
  if (stderrRestore) releaseStderr();
  clearHomeSandbox();
});

/**
 * POST /v1/messages against the in-process proxy. Returns {ok, status, text}
 * where text is the concatenated response content (JSON or SSE).
 *
 * maxTokens defaults to 64 — lower values (16) cause some providers to emit
 * zero output tokens on "say hi" prompts and return an empty SSE stream.
 */
async function askProxy(
  port: number,
  model: string,
  prompt: string,
  maxTokens = 64
): Promise<{ ok: boolean; status: number; text: string; raw: any }> {
  const res = await fetch(`http://127.0.0.1:${port}/v1/messages`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model,
      max_tokens: maxTokens,
      stream: false,
      messages: [{ role: "user", content: prompt }],
    }),
  });

  const ct = res.headers.get("content-type") || "";
  if (ct.includes("text/event-stream")) {
    const raw = await res.text();
    const parts: string[] = [];
    let sawStop = false;
    let sawError = false;
    for (const line of raw.split("\n")) {
      if (!line.startsWith("data:")) continue;
      const data = line.replace(/^data:\s*/, "").trim();
      if (!data || data === "[DONE]") continue;
      try {
        const p = JSON.parse(data);
        if (p.type === "content_block_delta" && p.delta?.text) parts.push(p.delta.text);
        if (p.type === "message_start" && Array.isArray(p.message?.content)) {
          for (const b of p.message.content) if (b.text) parts.push(b.text);
        }
        if (p.choices?.[0]?.delta?.content) parts.push(p.choices[0].delta.content);
        if (p.type === "message_stop") sawStop = true;
        if (p.type === "error" || p.error) sawError = true;
      } catch {}
    }
    // HTTP-level success: 2xx AND stream reached completion without error.
    // Empty text with message_stop = provider accepted the request but
    // produced no tokens (still a valid transport-level success).
    const httpOk = res.ok && sawStop && !sawError;
    return { ok: httpOk, status: res.status, text: parts.join(""), raw };
  }

  try {
    const body = (await res.json()) as { content?: Array<{ text?: string }> };
    let text = "";
    if (Array.isArray(body?.content)) {
      for (const b of body.content) if (b?.text) text += b.text;
    }
    return { ok: res.ok, status: res.status, text, raw: body };
  } catch {
    const raw = await res.text();
    return { ok: false, status: res.status, text: "", raw };
  }
}

const MARKER = () => `x${Math.random().toString(36).slice(2, 8)}`;

// ---------------------------------------------------------------------------
// Group A — Default provider precedence
// Most Group A scenarios (CLI > env > config > legacy > openrouter > hardcoded)
// are already exhaustively covered by the sibling unit files:
//   - packages/cli/src/default-provider.test.ts
//   - packages/cli/src/providers/auto-route-default-provider.test.ts
// This file only adds the one scenario those miss: the on-disk legacy-hint
// throttle-marker file lifecycle, observed from the filesystem as a user would.
// ---------------------------------------------------------------------------

describe("Group A — legacy LiteLLM auto-promotion removed (commit 5)", () => {
  beforeEach(() => {
    sandboxHome();
  });

  test("A1 — LITELLM env vars no longer auto-promote LiteLLM as default", () => {
    // Pre-commit-5, LITELLM_BASE_URL + LITELLM_API_KEY together would resolve
    // provider="litellm" with source="legacy-litellm" and legacyAutoPromoted=true
    // (and the CLI emitted a one-shot stderr hint, throttled by a marker file).
    // After commit 5 of the model-catalog and routing redesign, this auto-
    // promotion path is gone — the resolver falls through to OPENROUTER_API_KEY
    // or hardcoded "openrouter". Users wanting LiteLLM as default must set
    // defaultProvider: "litellm" in the global config or set
    // CLAUDISH_DEFAULT_PROVIDER=litellm. The marker file is no longer produced.
    const env: NodeJS.ProcessEnv = {
      HOME: tmpdir(),
      LITELLM_BASE_URL: "http://example.invalid:4000",
      LITELLM_API_KEY: "ll-test-key",
    };

    const result = resolveDefaultProvider({
      env,
      config: { version: "1.0.0", defaultProfile: "default", profiles: {} },
    });
    expect(result.provider).toBe("openrouter");
    expect(result.source).toBe("hardcoded");
    expect(result.legacyAutoPromoted).toBe(false);
  });
});

// ---------------------------------------------------------------------------
// Hermetic proxy coverage — defaultProvider reaches bare-name routing
// ---------------------------------------------------------------------------

type HermeticProxyScenario = "config-empty" | "env-empty" | "custom-no-key";

interface HermeticProxyResult {
  status: number;
  raw: { error?: { message?: string } };
  debugLog: string;
}

function runHermeticProxy(scenario: HermeticProxyScenario): HermeticProxyResult {
  const home = mkdtempSync(join(tmpdir(), "claudish-default-provider-proxy-"));
  const configDir = join(home, ".claudish");
  mkdirSync(configDir, { recursive: true });
  copyFileSync(STAGE4_CATALOG_FIXTURE, join(configDir, "cloud-models-catalog-v3.json"));

  const config: Record<string, unknown> = {
    version: "1.0.0",
    defaultProfile: "default",
    profiles: {},
    defaultProvider: scenario === "config-empty" ? "" : "openrouter",
  };
  if (scenario === "custom-no-key") {
    config.defaultProvider = "sandbox-no-key";
    config.customEndpoints = {
      "sandbox-no-key": {
        kind: "simple",
        url: "http://127.0.0.1:9/v1",
        format: "openai",
        apiKey: "${STAGE4_MISSING_KEY}",
      },
    };
  }
  writeFileSync(join(configDir, "config.json"), JSON.stringify(config), "utf8");

  const proxyModuleUrl = new URL("../proxy-server.ts", import.meta.url).href;
  const loggerModuleUrl = new URL("../logger.ts", import.meta.url).href;
  const script = `
    const { readFileSync, rmSync } = await import("node:fs");
    const { createProxyServer } = await import(${JSON.stringify(proxyModuleUrl)});
    const { getLogFilePath, initLogger } = await import(${JSON.stringify(loggerModuleUrl)});

    let proxy;
    let output;
    try {
      proxy = await createProxyServer(0, undefined, undefined, false, undefined, undefined, {
        quiet: true,
      });
      initLogger(true, "debug", true);
      const response = await fetch(proxy.url + "/v1/messages", {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({
          model: "no-such-model-xyz",
          max_tokens: 16,
          stream: false,
          messages: [{ role: "user", content: "hi" }],
        }),
      });
      const raw = await response.json();
      await Bun.sleep(150);
      const logPath = getLogFilePath();
      if (!logPath) throw new Error("debug logger did not expose its path");
      const debugLog = readFileSync(logPath, "utf8");
      initLogger(false, "info", true);
      rmSync(logPath, { force: true });
      output = JSON.stringify({
        status: response.status,
        raw,
        debugLog,
      });
    } finally {
      if (proxy) await proxy.shutdown();
    }
    process.stdout.write(output, () => process.exit(0));
  `;
  const env: Record<string, string> = {
    HOME: home,
    PATH: process.env.PATH ?? "",
    TMPDIR: process.env.TMPDIR ?? tmpdir(),
    OPENROUTER_API_KEY: "",
    STAGE4_MISSING_KEY: "",
    CLAUDISH_DISABLE_CATALOG_WARM: "1",
    CLAUDISH_DISABLE_KEYCHAIN: "1",
    CLAUDISH_DISABLE_OP: "1",
    CLAUDISH_SKIP_LIVE_E2E: "1",
  };
  if (scenario === "env-empty") env.CLAUDISH_DEFAULT_PROVIDER = "";

  try {
    const result = Bun.spawnSync([process.execPath, "-e", script], {
      cwd: join(import.meta.dir, "../../../.."),
      env,
      stdout: "pipe",
      stderr: "pipe",
    });
    const stdout = result.stdout.toString();
    const stderr = result.stderr.toString();
    expect(result.exitCode, stderr || stdout).toBe(0);
    return JSON.parse(stdout) as HermeticProxyResult;
  } finally {
    rmSync(home, { recursive: true, force: true });
  }
}

describe("Hermetic proxy defaultProvider routing", () => {
  test('config defaultProvider "" returns the catalog-empty no-route without an upstream call', () => {
    const result = runHermeticProxy("config-empty");
    expect(result.status).toBe(400);
    expect(result.raw.error?.message).toContain(
      '[Route] No provider in the catalog serves "no-such-model-xyz".'
    );
    expect(result.debugLog).not.toContain("[OpenRouter] Response status:");
  });

  test('CLAUDISH_DEFAULT_PROVIDER="" returns the same no-route without an upstream call', () => {
    const result = runHermeticProxy("env-empty");
    expect(result.status).toBe(400);
    expect(result.raw.error?.message).toContain(
      '[Route] No provider in the catalog serves "no-such-model-xyz".'
    );
    expect(result.debugLog).not.toContain("[OpenRouter] Response status:");
  });

  test("an uncredentialed custom fallback is named in the tried reason", () => {
    const result = runHermeticProxy("custom-no-key");
    expect(result.status).toBe(400);
    expect(result.raw.error?.message).toContain(
      '[Route] No credentialed providers in chain for "no-such-model-xyz" ' +
        "(tried: sandbox-no-key)."
    );
    expect(result.debugLog).not.toContain("[sandbox-no-key] Response status:");
  });
});

// ---------------------------------------------------------------------------
// Group B — Live provider roundtrips (not fallback-selection proof)
// ---------------------------------------------------------------------------

// Asked via claudish's OWN credential authority (env → aliases →
// global config apiKeys → 1Password), NOT raw process.env: a key
// stored any supported way makes claudish able to serve these models, so these
// tests must run — an env-only gate would silently skip a suite that could
// actually test something. LiteLLM additionally needs its base URL, which is an
// endpoint (not a credential) and so stays an env check.
//
// The credential check answers "can this run at all"; it does NOT answer "may this
// SPEND". Those are two different questions and this file only ever asked the first,
// so on any machine where credentials resolve (here, through 1Password) every group
// below called a real, metered provider on every `bun run test:safe` — including runs
// that had set `CLAUDISH_SKIP_LIVE_E2E=1` precisely to avoid that. Measured: B1a hit
// OpenRouter and B3 hit xAI with the flag set, and a spent subscription fell through
// the chain to a per-token provider. The suite's own failure count then tracked quota
// balance rather than code, which is what makes it unusable as a regression baseline.
//
// The flag is the same one `mcp-session/e2e-sessions.test.ts` already honours and that
// `.github/workflows/test.yml` sets for CI. It is ANDed with the credential check, not
// a replacement for it: "do not spend" and "cannot run" both have to be false before a
// live call is made.
const SKIP_LIVE_E2E = process.env.CLAUDISH_SKIP_LIVE_E2E === "1";
const HAS_OR = SKIP_LIVE_E2E ? false : await hasCredential("openrouter");
const HAS_LL = SKIP_LIVE_E2E
  ? false
  : !!process.env.LITELLM_BASE_URL && (await hasCredential("litellm"));
const HAS_XAI = SKIP_LIVE_E2E ? false : await hasCredential("x-ai");

describe("Group B — live provider roundtrips", () => {
  test.skipIf(!HAS_OR)(
    "B1a — catalog-gathered gpt-5.4 bare → served by credentialed OpenRouter",
    async () => {
      // No user rule: the chain must come from the cloud models catalog. The
      // gpt-5.4 row publishes OpenRouter, so defaultProvider is deduplicated
      // rather than manufacturing a second connection.
      sandboxHome({
        version: "1.0.0",
        defaultProfile: "default",
        profiles: {},
        defaultProvider: "openrouter",
      });
      captureStderr();
      const t0 = Date.now();
      const port = await spinProxy({ quiet: false });
      const marker = MARKER();
      const { ok, status, text, raw } = await askProxy(
        port,
        "gpt-5.4",
        `say hi with marker ${marker}`
      );
      const stderr = releaseStderr();
      const elapsed = Date.now() - t0;

      if (!ok) {
        console.error("[B1a] failed", { status, text, raw, stderr });
      }
      expect(ok).toBe(true);
      expect(text.length).toBeGreaterThan(0);
      console.log(
        `[B1a] model=gpt-5.4 provider=openrouter elapsed=${elapsed}ms text="${text.slice(0, 60)}"`
      );

      // Route provenance.
      //
      // The proxy emits its `[Route] N providers for <model>: A → B` line ONLY
      // when the credential-filtered chain has MORE THAN ONE candidate
      // (proxy-server.ts: `if (!options.quiet && candidates.length > 1)`). The
      // gathered chain includes OpenAI and OpenRouter, so on a machine WITHOUT
      // an OpenAI credential it filters down to a single candidate and NO line
      // is emitted — which is the common case for someone holding only an
      // OpenRouter key. A bare `expect(stderr).toContain("openrouter")` therefore
      // failed deterministically in exactly that setup, while the routing under
      // test worked perfectly (ok + non-empty text, above, prove OpenRouter
      // served it: it is the only credentialed provider left in the chain).
      //
      // This is NOT the "Bun async logging is flaky" effect the sibling tests
      // assume — it is deterministic, and reproducible by setting/unsetting
      // OPENAI_API_KEY. So: assert the line's CONTENT when it exists, and always
      // guard the regression this test actually exists for.
      const lower = stderr.toLowerCase();
      const routeLine = lower.split("\n").find((l) => l.includes("[route]"));
      if (routeLine) {
        expect(routeLine).toContain("openrouter");
      }
      // Legacy auto-promotion guard (commit 5): litellm must NEVER lead the chain.
      expect(routeLine ?? "").not.toMatch(/\[route\][^:]*:\s*litellm/);
    },
    90_000
  );

  test.skipIf(!HAS_OR)(
    "B1b — defaultProvider=openrouter + gemini-3.1-pro-preview bare → served by OpenRouter",
    async () => {
      sandboxHome({
        version: "1.0.0",
        defaultProfile: "default",
        profiles: {},
        defaultProvider: "openrouter",
      });
      captureStderr();
      const t0 = Date.now();
      const port = await spinProxy({ quiet: false });
      const marker = MARKER();
      // 256, not the 64 default: gemini-3.1-pro-preview is a THINKING model and
      // its reasoning is billed against max_tokens. Measured against OpenRouter
      // on 2026-08-06 with this exact prompt — at 64 the reasoning consumed the
      // whole budget (1/3 runs returned empty content with finish_reason=length,
      // 2/3 truncated mid-sentence); at 256, 3/3 complete with finish_reason=stop.
      const { ok, status, text, raw } = await askProxy(
        port,
        "gemini-3.1-pro-preview",
        `say hi marker ${marker}`,
        256
      );
      const stderr = releaseStderr();
      const elapsed = Date.now() - t0;

      if (!ok) {
        console.error("[B1b] failed", { status, text, raw, stderr });
      }
      console.log(
        `[B1b] model=gemini-3.1-pro-preview provider=openrouter elapsed=${elapsed}ms text="${text.slice(0, 60)}"`
      );
      // The load-bearing assertion is that the request succeeded end-to-end.
      // Empty text is NOT treated as acceptable here: with an adequate budget
      // this prompt returned content 5/5 upstream, so a blank answer means
      // something in the pipeline lost it (that is exactly how the Gemini
      // reasoning-filter bug hid for seven months).
      expect(ok).toBe(true);
      expect(text.trim()).not.toBe("");
      // Best-effort stderr provenance check — Bun async logging is flaky
      const lower = stderr.toLowerCase();
      if (!lower.includes("openrouter")) {
        console.log("[B1b] stderr capture missed openrouter route marker (Bun async timing)");
      }
    },
    90_000
  );

  test.skipIf(!HAS_LL)(
    "B2 — defaultProvider=litellm + minimax-m2.5 bare → LiteLLM remains the fallback",
    async () => {
      sandboxHome({
        version: "1.0.0",
        defaultProfile: "default",
        profiles: {},
        defaultProvider: "litellm",
      });
      captureStderr();
      const t0 = Date.now();
      const port = await spinProxy({ quiet: false });
      const { ok, status, text, raw } = await askProxy(port, "minimax-m2.5", `say hi ${MARKER()}`);
      const stderr = releaseStderr();
      const elapsed = Date.now() - t0;

      if (!ok) {
        console.error("[B2] failed", { status, text, raw, stderr });
      }
      // LiteLLM may or may not resolve the bare name — the critical assertion
      // is that the request succeeded end-to-end. Stderr observability is
      // best-effort due to Bun async timing.
      const lower = stderr.toLowerCase();
      const llIdx = lower.indexOf("litellm");
      const orIdx = lower.indexOf("openrouter");
      console.log(
        `[B2] model=minimax-m2.5 ok=${ok} elapsed=${elapsed}ms litellm@${llIdx} openrouter@${orIdx} textLen=${text.length}`
      );
      expect(ok).toBe(true);
      // `defaultProvider` is the last-resort fallback position. If both names
      // are visible, the catalog-gathered OpenRouter connection must precede it.
      if (llIdx >= 0 && orIdx >= 0) expect(orIdx).toBeLessThan(llIdx);
    },
    90_000
  );

  test.skipIf(!HAS_XAI)(
    "B3 — explicit xai@grok-code-fast-1 bypasses default-provider (no openrouter route)",
    async () => {
      sandboxHome({
        version: "1.0.0",
        defaultProfile: "default",
        profiles: {},
        defaultProvider: "openrouter",
      });
      captureStderr();
      const t0 = Date.now();
      const port = await spinProxy({ quiet: false });
      const { ok, status, text, raw } = await askProxy(
        port,
        "xai@grok-code-fast-1",
        `say hi ${MARKER()}`
      );
      const stderr = releaseStderr();
      const elapsed = Date.now() - t0;

      if (!ok) {
        console.error("[B3] failed", { status, text, raw, stderr });
      }
      console.log(
        `[B3] model=xai@grok-code-fast-1 ok=${ok} elapsed=${elapsed}ms text="${text.slice(0, 60)}"`
      );
      // Explicit provider path must hit XAI. We assert success OR a single-
      // provider error (never a fallback chain error).
      if (!ok) {
        const r = typeof raw === "string" ? raw : JSON.stringify(raw);
        expect(r).not.toContain("all_providers_failed");
      } else {
        expect(text.length).toBeGreaterThan(0);
      }
    },
    90_000
  );

  test.skipIf(!HAS_LL)(
    "B4 — legacy auto-promotion gone (commit 5): no banner ever appears",
    async () => {
      // Pre-commit-5 this test asserted the one-shot hint fired on the first
      // call and was throttled (via marker file) on the second. Commit 5 of
      // the model-catalog and routing redesign removed the auto-promotion
      // path entirely — the resolver no longer treats `LITELLM_*` env vars
      // as "make LiteLLM default". Now both calls must be banner-free.
      sandboxHome({ version: "1.0.0", defaultProfile: "default", profiles: {} });

      captureStderr();
      const port = await spinProxy({ quiet: false });
      await askProxy(port, "minimax-m2.5", `hi ${MARKER()}`);
      await killProxy();
      const firstStderr = releaseStderr();
      const firstLower = firstStderr.toLowerCase();
      const firstBanner = firstLower.includes("deprecat") && firstLower.includes("litellm");
      expect(firstBanner).toBe(false);

      captureStderr();
      const port2 = await spinProxy({ quiet: false });
      await askProxy(port2, "minimax-m2.5", `hi ${MARKER()}`);
      const secondStderr = releaseStderr();
      const secondLower = secondStderr.toLowerCase();
      const secondBanner = secondLower.includes("deprecat") && secondLower.includes("litellm");
      expect(secondBanner).toBe(false);
    },
    120_000
  );
});

// ---------------------------------------------------------------------------
// Group C — Live custom-endpoint roundtrips (not fallback-selection proof)
// ---------------------------------------------------------------------------

describe("Group C — live custom-endpoint roundtrips", () => {
  // The model is INCIDENTAL to Group C — these tests prove custom-endpoint
  // registration + `${VAR}` expansion + roundtrip, and the endpoint routes
  // through the OpenAI adapter regardless of the model name. We deliberately
  // use a reliable non-reasoning chat model: an earlier choice of
  // `minimax-m2.5` (a reasoning model) intermittently returned 0 visible
  // tokens on a "say hi" prompt, which made the `text.length > 0` roundtrip
  // assertion flaky even though the HTTP call succeeded. gpt-4o-mini returns
  // deterministic text, so the strict assertion can stay.
  const C_MODEL = "e2e-test-ep@openai/gpt-4o-mini";

  test.skipIf(!HAS_OR)(
    "C1 — custom endpoint e2e-test-ep with ${OPENROUTER_API_KEY} works",
    async () => {
      sandboxHome({
        version: "1.0.0",
        defaultProfile: "default",
        profiles: {},
        customEndpoints: {
          "e2e-test-ep": {
            kind: "simple",
            url: "https://openrouter.ai/api/v1",
            format: "openai",
            apiKey: "${OPENROUTER_API_KEY}",
          },
        },
        defaultProvider: "e2e-test-ep",
      });

      captureStderr();
      const t0 = Date.now();
      const port = await spinProxy({ quiet: false });
      const { ok, status, text, raw } = await askProxy(port, C_MODEL, `say hi ${MARKER()}`);
      const stderr = releaseStderr();
      const elapsed = Date.now() - t0;

      if (!ok) console.error("[C1] failed", { status, text, raw, stderr });
      console.log(
        `[C1] model=${C_MODEL} ok=${ok} elapsed=${elapsed}ms text="${text.slice(0, 60)}"`
      );
      // Correctness signal: the request succeeded with non-empty output, which
      // proves the custom endpoint was registered + handler created + ${VAR}
      // expanded + request roundtripped. Stderr observability is best-effort
      // because the proxy logs registration counts (not names) and Bun's
      // async logging timing makes capture flaky.
      expect(ok).toBe(true);
      expect(text.length).toBeGreaterThan(0);
    },
    90_000
  );

  test.skipIf(!HAS_OR)(
    "C2 — invalid custom endpoint is warned but bare call still succeeds",
    async () => {
      sandboxHome({
        version: "1.0.0",
        defaultProfile: "default",
        profiles: {},
        customEndpoints: {
          "e2e-test-ep": {
            kind: "simple",
            url: "https://openrouter.ai/api/v1",
            format: "openai",
            apiKey: "${OPENROUTER_API_KEY}",
          },
          "broken-ep": {
            kind: "simple",
            // missing url on purpose
            format: "openai",
            apiKey: "ignored",
          },
        },
        defaultProvider: "openrouter",
      });

      captureStderr();
      const t0 = Date.now();
      const port = await spinProxy({ quiet: false });
      const { ok, status, text, raw } = await askProxy(port, "gpt-5.4", `hi ${MARKER()}`);
      const stderr = releaseStderr();
      const elapsed = Date.now() - t0;

      if (!ok) console.error("[C2] failed", { status, text, raw, stderr });
      console.log(`[C2] ok=${ok} elapsed=${elapsed}ms text="${text.slice(0, 60)}"`);
      // Best-effort warning observation — Bun's async console capture is flaky.
      // The bun-test stdout stream often shows the warning even when the
      // patched JS-level capture misses it. The CRITICAL assertion is that
      // the bare call STILL succeeded (the broken endpoint didn't crash startup).
      const lower = stderr.toLowerCase();
      const mentionsBroken = lower.includes("broken-ep");
      const mentionsWarn =
        lower.includes("warn") || lower.includes("invalid") || lower.includes("skip");
      if (!(mentionsBroken || mentionsWarn)) {
        console.log(
          "[C2] stderr capture missed the broken-ep warning (Bun async timing) " +
            "— continuing because the bare call succeeded which is the load-bearing assertion"
        );
      }
      // Bare call still works
      if (ok) {
        expect(text.length).toBeGreaterThan(0);
      }
    },
    90_000
  );

  test.skipIf(!HAS_OR)(
    "C3 — ${E2E_TEST_KEY} template is expanded from process env",
    async () => {
      const savedKey = process.env.E2E_TEST_KEY;
      process.env.E2E_TEST_KEY = process.env.OPENROUTER_API_KEY;
      try {
        sandboxHome({
          version: "1.0.0",
          defaultProfile: "default",
          profiles: {},
          customEndpoints: {
            "e2e-test-ep": {
              kind: "simple",
              url: "https://openrouter.ai/api/v1",
              format: "openai",
              apiKey: "${E2E_TEST_KEY}",
            },
          },
          defaultProvider: "e2e-test-ep",
        });

        captureStderr();
        const t0 = Date.now();
        const port = await spinProxy({ quiet: false });
        const { ok, status, text, raw } = await askProxy(port, C_MODEL, `hi ${MARKER()}`);
        const stderr = releaseStderr();
        const elapsed = Date.now() - t0;

        if (!ok) console.error("[C3] failed", { status, text, raw, stderr });
        console.log(`[C3] ok=${ok} elapsed=${elapsed}ms text="${text.slice(0, 60)}"`);
        // If the literal ${E2E_TEST_KEY} string was passed to OpenRouter, we'd
        // get HTTP 401. The fact that we got HTTP 200 (ok=true) IS the proof
        // that the template was expanded. Empty text content is independent —
        // some models occasionally return 0 tokens on "say hi" prompts even
        // on a successful HTTP roundtrip. The expansion is what we're testing.
        if (ok) {
          expect(ok).toBe(true);
        } else {
          // If we failed, it MUST NOT be because the literal placeholder was forwarded
          const r = typeof raw === "string" ? raw : JSON.stringify(raw);
          expect(r).not.toContain("${E2E_TEST_KEY}");
        }
      } finally {
        if (savedKey === undefined) delete process.env.E2E_TEST_KEY;
        else process.env.E2E_TEST_KEY = savedKey;
      }
    },
    90_000
  );
});

// ---------------------------------------------------------------------------
// Group D — Firebase slim catalog aggregators[] contract
// ---------------------------------------------------------------------------

// The Firebase provider slugs claudish can actually route an aggregator entry
// to — derived from the REAL routing map (`pickerProviderToFirebaseSlug` in
// model-selector.ts, consumed by `resolveProviderAggregatorEntry`), NOT a
// hand-kept list. The catalog is owned by a DIFFERENT repo (models-index); it
// legitimately carries providers claudish doesn't route (e.g. `mistralai` — no
// Mistral provider here; `anthropic` — handled natively, not via the picker).
// Those are inert by construction (unmatched → ignored), so their presence is
// NOT a claudish defect and must never fail this suite. A stale allowlist here
// used to hard-fail CI every time models-index shipped a new provider.
//
// SKIPPED under CLAUDISH_SKIP_LIVE_E2E, and the group now needs that gate for a
// second reason. These tests assert the v2 slim-catalog shape. As observed on
// 2026-09-18 the live endpoint serves contract v3: 426 without the v3 Accept
// header, 503 with it. The shape asserted below is therefore no longer served at
// all. Rewrite them against v3 as part of the v3 reader work — do not delete
// them, because the aggregator-shape coverage is still wanted.
const ROUTABLE_FIREBASE_SLUGS = new Set(Object.values(pickerProviderToFirebaseSlug));

describe.skipIf(SKIP_LIVE_E2E)("Group D — Firebase slim catalog", () => {
  let cachedBody: any = null;

  async function fetchCatalog(): Promise<any> {
    if (cachedBody) return cachedBody;
    const res = await fetch(
      "https://us-central1-claudish-6da10.cloudfunctions.net/queryModels?status=active&catalog=slim&limit=100",
      { headers: { Accept: "application/vnd.models-index.catalog+json;version=3" } }
    );
    expect(res.status).toBe(200);
    const envelope = await res.json();
    expect(envelope.contractVersion).toBe(3);
    cachedBody = envelope.data;
    return cachedBody;
  }

  test("D1 — catalog returns {models: [...]} with at least one entry", async () => {
    const body = await fetchCatalog();
    expect(body).toBeDefined();
    expect(Array.isArray(body.models)).toBe(true);
    expect(body.models.length).toBeGreaterThan(0);
    console.log(`[D1] slim catalog models count=${body.models.length}`);
  }, 15_000);

  test("D1b — aggregators[] shape contract + routing-map overlap (soft-skip if Phase 4 not deployed)", async () => {
    const body = await fetchCatalog();
    const withAgg = (body.models as any[]).filter(
      (m) => Array.isArray(m?.aggregators) && m.aggregators.length > 0
    );
    if (withAgg.length === 0) {
      console.log("[D1b] PENDING DEPLOY — no models have aggregators[] yet");
      return;
    }
    console.log(`[D1b] ${withAgg.length}/${body.models.length} models have aggregators[]`);

    // (1) SHAPE is a hard contract: claudish's `resolveProviderAggregatorEntry`
    // and `resolveProviderExternalId` index these fields directly, so a
    // non-string provider/externalId/confidence would break consumption.
    const seenProviders = new Set<string>();
    for (const m of withAgg) {
      for (const agg of m.aggregators) {
        expect(typeof agg.sourceProviderId).toBe("string");
        if (agg.routeStatus !== "mapped") continue;
        expect(typeof agg.route?.routeId).toBe("string");
        expect(typeof agg.route?.routeProfileId).toBe("string");
        expect(typeof agg.externalModelId).toBe("string");
        expect(typeof agg.confidence).toBe("string");
        seenProviders.add(agg.sourceProviderId);
      }
    }

    // (2) ROUTABILITY is reported, not policed. Providers claudish maps to are
    // routable; the rest are inert catalog data owned by models-index and are
    // simply ignored at runtime — NOT a failure of this repo.
    const routable = [...seenProviders].filter((p) => ROUTABLE_FIREBASE_SLUGS.has(p)).sort();
    const unroutable = [...seenProviders].filter((p) => !ROUTABLE_FIREBASE_SLUGS.has(p)).sort();
    console.log(`[D1b] routable by claudish: ${routable.join(", ") || "(none)"}`);
    if (unroutable.length > 0) {
      console.log(
        `[D1b] catalog providers claudish does not route (inert, expected): ${unroutable.join(", ")}`
      );
    }

    // (3) DRIFT sanity: at least one aggregator provider must be routable. Zero
    // overlap would mean the catalog's slug convention diverged from claudish's
    // `pickerProviderToFirebaseSlug` entirely — a real, claudish-side routing
    // break — without being brittle to any single provider coming or going.
    expect(routable.length).toBeGreaterThan(0);
  }, 15_000);

  test("D2 — entries without aggregators[] parse cleanly (field is optional)", async () => {
    const body = await fetchCatalog();
    const withoutAgg = (body.models as any[]).filter(
      (m) => !Array.isArray(m?.aggregators) || m.aggregators.length === 0
    );
    console.log(`[D2] models without aggregators[]: ${withoutAgg.length}`);
    // Just a shape sanity: each should still have SOMETHING identifiable.
    // The slim catalog uses `modelId` (not `id` or `name`).
    for (const m of withoutAgg.slice(0, 20)) {
      const hasIdentifier =
        typeof m.modelId === "string" || typeof m.id === "string" || typeof m.name === "string";
      expect(hasIdentifier).toBe(true);
    }
  }, 15_000);
});

// ---------------------------------------------------------------------------
// Group E — End-to-end config flip happy path
// ---------------------------------------------------------------------------

describe("Group E — config flip happy path", () => {
  test.skipIf(!HAS_OR || !HAS_LL)(
    "E1 — openrouter → litellm flip with grok-4.20 bare",
    async () => {
      // Phase 1: defaultProvider=openrouter
      sandboxHome({
        version: "1.0.0",
        defaultProfile: "default",
        profiles: {},
        defaultProvider: "openrouter",
      });

      captureStderr();
      const t0 = Date.now();
      const port = await spinProxy({ quiet: false });
      const phase1 = await askProxy(port, "grok-4.20", `say hi ${MARKER()}`);
      await killProxy();
      const phase1Stderr = releaseStderr();
      const elapsed1 = Date.now() - t0;
      const lower1 = phase1Stderr.toLowerCase();

      console.log(
        `[E1-openrouter] ok=${phase1.ok} elapsed=${elapsed1}ms text="${phase1.text.slice(0, 40)}"`
      );
      // Phase 1 correctness: bare-model invocation succeeded with non-empty
      // response. We don't assert on stderr provenance here because Bun's
      // async stderr capture is unreliable from inside test handlers — the
      // upstream proxy logs land in the bun-test output pipe but skip the
      // patched JS-level capture. The non-empty response IS the proof.
      expect(phase1.ok).toBe(true);
      expect(phase1.text.length).toBeGreaterThan(0);
      // No legacy migration banner on explicit defaultProvider (when captured)
      const legacyBanner1 = lower1.includes("deprecat") && lower1.includes("litellm");
      expect(legacyBanner1).toBe(false);

      // Phase 2: flip to litellm
      writeFileSync(
        tempConfigPath!,
        JSON.stringify({
          version: "1.0.0",
          defaultProfile: "default",
          profiles: {},
          defaultProvider: "litellm",
        }),
        "utf8"
      );

      captureStderr();
      const t1 = Date.now();
      const port2 = await spinProxy({ quiet: false });
      const phase2 = await askProxy(port2, "grok-4.20", `hi ${MARKER()}`);
      const phase2Stderr = releaseStderr();
      const elapsed2 = Date.now() - t1;
      const lower2 = phase2Stderr.toLowerCase();

      console.log(
        `[E1-litellm] ok=${phase2.ok} elapsed=${elapsed2}ms text="${phase2.text.slice(0, 40)}"`
      );
      // LiteLLM should appear in the route; legacy banner should NOT (explicit config)
      const legacyBanner2 = lower2.includes("deprecat") && lower2.includes("litellm");
      expect(legacyBanner2).toBe(false);
      // We expect either a litellm route attempt or a successful litellm response
      const llMentioned = lower2.includes("litellm");
      console.log(`[E1-litellm] litellmMentioned=${llMentioned}`);
    },
    180_000
  );
});