Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

10.6 KB
Raw Blame
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { catalogRouteForProvider } from "./catalog-route-bindings.js";

import { credentials } from "../auth/credentials/authority.js";
import type { SlimModelEntry } from "./all-models-cache.js";
import { _resetCatalogClient, _setCatalogEntriesForTest } from "./catalog-client.js";
import { decodeModelConfigs } from "./devin/devin-models.js";
import { providerServesModel } from "./model-availability.js";
import {
  type DiscoveredModel,
  getModelDiscoveryFetcher,
  invalidateModelDiscovery,
  registerModelDiscoveryFetcher,
} from "./model-discovery.js";
import "./model-discovery-builtins.js";
import { devinModelsCatalogEntry } from "./model-resolvers/devin.js";
import type { ProviderDefinition } from "./provider-definitions.js";
import { route } from "./routing-rules.js";
import { clearRuntimeRegistry, registerRuntimeProvider } from "./runtime-providers.js";

const CATALOG_PROVIDER = "openrouter";
const OTHER_CATALOG_PROVIDER = "fireworks";
const DISCOVERY_PROVIDER = "availability-discovery-test";

const realFetch = globalThis.fetch;
const realGetRequestAuth = credentials.getRequestAuth;
const realIsAvailable = credentials.isAvailable;
const realDescribeReadiness = credentials.describeReadiness;
const realDevinDiscovery = getModelDiscoveryFetcher("devin-connect");

if (!realDevinDiscovery) throw new Error("Devin discovery fetcher was not registered");

const DEVIN_BUG_UIDS = new Set(["swe-1-7", "swe-1-7-medium"]);
const devinFixturePath = join(import.meta.dir, "../test-fixtures/devin/GetCliModelConfigs.res.bin");
const devinModelsCatalog: DiscoveredModel[] = decodeModelConfigs(readFileSync(devinFixturePath))
  .filter((model) => DEVIN_BUG_UIDS.has(model.uid))
  .map(devinModelsCatalogEntry)
  .map(({ wireId, ...rest }) => ({ id: wireId, ...rest }));

if (devinModelsCatalog.length !== DEVIN_BUG_UIDS.size) {
  throw new Error("Captured Devin fixture is missing the SWE-1.7 regression uids");
}

function catalogEntry(
  modelId: string,
  providers: Array<{ provider: string; externalId?: string }> = [],
  aliases: string[] = []
): SlimModelEntry {
  return {
    modelId,
    aliases,

    aggregators: providers.map(({ provider, externalId }) => ({
      sourceProviderId: provider,
      sourceCollectorId: "test",
      routeStatus: "mapped",
      route: catalogRouteForProvider(provider),
      externalModelId: externalId ?? modelId,
      confidence: "api_official",
    })),
  };
}

function discoveryProvider(): ProviderDefinition {
  return {
    name: DISCOVERY_PROVIDER,
    displayName: "Availability Discovery Test",
    transport: "openai",
    baseUrl: "https://discovery.invalid",
    apiPath: "/v1/chat/completions",
    apiKeyEnvVar: "AVAILABILITY_DISCOVERY_TEST_API_KEY",
    apiKeyDescription: "Offline test key",
    apiKeyUrl: "https://discovery.invalid/key",
    shortcuts: [],
    legacyPrefixes: [],
    modelDiscovery: { path: "/v1/models", format: "openai-models-list" },
    // Required because handler wiring once lived in PROVIDER_PROFILES, where a missing entry silently routed to OpenRouter; colocating it prevents that half-add.
    createHandler: {
      kind: "none",
      reason: "virtual",
      note: "Test fixture — never builds a handler.",
    },
    isDirectApi: true,
  };
}

function stubModelsCatalog(...ids: string[]): void {
  globalThis.fetch = mock(
    async () =>
      new Response(JSON.stringify({ data: ids.map((id) => ({ id })) }), {
        status: 200,
        headers: { "Content-Type": "application/json" },
      })
  ) as unknown as typeof fetch;
}

function stubDevinModelsCatalog(): void {
  registerModelDiscoveryFetcher("devin-connect", async () => ({
    kind: "models",
    models: devinModelsCatalog,
  }));
}

beforeEach(() => {
  _resetCatalogClient();
  // Keep every test off the real on-disk catalog, including discovery tests.
  _setCatalogEntriesForTest([]);
  invalidateModelDiscovery();
  clearRuntimeRegistry();
  registerRuntimeProvider(discoveryProvider());

  credentials.getRequestAuth = mock(async () => ({
    headers: { Authorization: "Bearer offline-test-token" },
  }));
  globalThis.fetch = mock(async () => {
    throw new Error("Unexpected fetch call");
  }) as unknown as typeof fetch;
});

afterEach(() => {
  _resetCatalogClient();
  invalidateModelDiscovery();
  clearRuntimeRegistry();
  credentials.getRequestAuth = realGetRequestAuth;
  credentials.isAvailable = realIsAvailable;
  credentials.describeReadiness = realDescribeReadiness;
  registerModelDiscoveryFetcher("devin-connect", realDevinDiscovery);
  globalThis.fetch = realFetch;
});

describe("providerServesModel catalog evidence", () => {
  test("still returns serves when the matching row lists the provider", async () => {
    _setCatalogEntriesForTest([
      catalogEntry("model-one", [{ provider: CATALOG_PROVIDER, externalId: "vendor/model-one" }]),
    ]);

    // Partial coverage cannot prove a denial, but a listed provider is still
    // positive catalog evidence and remains safe to confirm.
    expect(await providerServesModel(CATALOG_PROVIDER, "model-one")).toBe("serves");
  });

  test("returns unknown when a catalog-known provider is absent from the matching row", async () => {
    _setCatalogEntriesForTest([
      catalogEntry("model-one", [{ provider: OTHER_CATALOG_PROVIDER }]),
      catalogEntry("model-two", [{ provider: CATALOG_PROVIDER }]),
    ]);

    // Catalog coverage is partial, so absence from one row cannot deny service:
    // openai-codex appears on only 1 of ~760 rows yet genuinely serves gpt-5.
    // Only a complete dynamic models catalog may turn absence into "not-served".
    expect(await providerServesModel(CATALOG_PROVIDER, "model-one")).toBe("unknown");
  });

  test("returns unknown for the openai-codex thin-coverage shape", async () => {
    _setCatalogEntriesForTest([
      catalogEntry("the-one-listed-row", [{ provider: "openai-codex" }]),
      catalogEntry("gpt-5", [{ provider: OTHER_CATALOG_PROVIDER }]),
    ]);

    // This models openai-codex appearing on one row but being absent from the
    // queried row. Its real 1-of-~760 coverage is partial evidence, not a denial.
    expect(await providerServesModel("openai-codex", "gpt-5")).toBe("unknown");
  });

  test("returns unknown when the provider never appears in the catalog vocabulary", async () => {
    _setCatalogEntriesForTest([catalogEntry("model-one", [{ provider: OTHER_CATALOG_PROVIDER }])]);

    // This guard stops subscription providers, which the aggregator catalog
    // usually does not cover, from being dropped from every paid routing chain.
    expect(await providerServesModel("subscription-provider", "model-one")).toBe("unknown");
  });

  test("returns unknown when the catalog is cold", async () => {
    _setCatalogEntriesForTest(null);

    expect(await providerServesModel(CATALOG_PROVIDER, "model-one")).toBe("unknown");
  });

  test("returns unknown when a vocabulary provider has no matching row", async () => {
    _setCatalogEntriesForTest([catalogEntry("known-model", [{ provider: CATALOG_PROVIDER }])]);

    expect(await providerServesModel(CATALOG_PROVIDER, "newer-than-cache")).toBe("unknown");
  });

  test.each([
    [
      "alias",
      "MODEL-ALIAS",
      catalogEntry("canonical-model", [{ provider: CATALOG_PROVIDER }], ["model-alias"]),
    ],
    [
      "aggregator external id",
      "VENDOR/MODEL-ONE",
      catalogEntry("canonical-model", [
        { provider: CATALOG_PROVIDER, externalId: "vendor/model-one" },
      ]),
    ],
  ])("matches a row by %s, not only modelId", async (_kind, wireId, entry) => {
    _setCatalogEntriesForTest([entry]);

    expect(await providerServesModel(CATALOG_PROVIDER, wireId)).toBe("serves");
  });
});

describe("providerServesModel live discovery", () => {
  test("returns serves for exact and case-insensitive matches in the dynamic models catalog", async () => {
    stubModelsCatalog("MiniMax-M3");

    expect(await providerServesModel(DISCOVERY_PROVIDER, "MiniMax-M3")).toBe("serves");
    expect(await providerServesModel(DISCOVERY_PROVIDER, "minimax-m3")).toBe("serves");
  });

  test("returns not-served when a non-empty dynamic models catalog omits the model", async () => {
    stubModelsCatalog("available-model");

    expect(await providerServesModel(DISCOVERY_PROVIDER, "missing-model")).toBe("not-served");
  });

  test("returns unknown when discovery returns an empty dynamic models catalog", async () => {
    stubModelsCatalog();

    // A temporarily unavailable listing endpoint must not silently switch the
    // user away from the provider they selected and may already pay for.
    expect(await providerServesModel(DISCOVERY_PROVIDER, "model-one")).toBe("unknown");
  });

  test("uses discovery before conflicting catalog evidence", async () => {
    _setCatalogEntriesForTest([catalogEntry("model-one", [{ provider: DISCOVERY_PROVIDER }])]);
    stubModelsCatalog("different-model");

    expect(await providerServesModel(DISCOVERY_PROVIDER, "model-one")).toBe("not-served");
  });
});

describe("providerServesModel resolver-backed discovery", () => {
  test("serves a canonical Devin family represented only by knob-encoded uids", async () => {
    stubDevinModelsCatalog();

    expect(devinModelsCatalog.map((model) => model.id)).toEqual(["swe-1-7", "swe-1-7-medium"]);
    expect(devinModelsCatalog.some((model) => model.id === "swe-1.7")).toBe(false);
    expect(await providerServesModel("devin", "swe-1.7")).toBe("serves");
  });

  test("does not turn an unknown Devin family into a served model", async () => {
    stubDevinModelsCatalog();

    expect(await providerServesModel("devin", "not-a-devin-model")).toBe("not-served");
  });

  test("preserves exact-match behavior for a provider with no resolver", async () => {
    stubModelsCatalog("present-model");

    expect(await providerServesModel(DISCOVERY_PROVIDER, "present-model")).toBe("serves");
    expect(await providerServesModel(DISCOVERY_PROVIDER, "absent-model")).toBe("not-served");
  });

  test("keeps Devin primary when routing a bare canonical family", async () => {
    stubDevinModelsCatalog();
    credentials.isAvailable = async (provider: string) =>
      provider === "devin" || provider === "openrouter";
    credentials.describeReadiness = async (provider: string) => ({
      readiness: provider === "devin" || provider === "openrouter" ? "present" : "absent",
    });

    const plan = await route("swe-1.7", {}, "openrouter");

    expect(plan.kind).toBe("ok");
    if (plan.kind !== "ok") return;
    expect(plan.primary.provider).toBe("devin");
    expect(plan.primary.modelSpec).toBe("[email protected]");
  });
});