Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

21.1 KB
Raw Blame
import { describe, expect, test } from "bun:test";
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { EFFORT_LEVELS } from "../../adapters/base-api-format.js";
import { decodeModelConfigs } from "../devin/devin-models.js";
import { DevinModelResolver, devinModelsCatalogEntry } from "./devin.js";
import type { ExpandContext, ModelChoice, ModelsCatalogEntry } from "./types.js";
import { groupKeyOf, nonEmpty, offerIsLive } from "./types.js";

const fixturePath = join(import.meta.dir, "../../test-fixtures/devin/GetCliModelConfigs.res.bin");
const modelsCatalog = decodeModelConfigs(readFileSync(fixturePath))
  .filter((config) => config.contextWindow > 0)
  .map(devinModelsCatalogEntry);
const resolver = new DevinModelResolver();
const choices = resolver.collapse(modelsCatalog);
const modelsCatalogById = new Map(modelsCatalog.map((entry) => [entry.wireId, entry]));

function groupKey(entry: ModelsCatalogEntry): string {
  return JSON.stringify([groupKeyOf(entry), entry.contextWindow ?? 0]);
}

function groupEntries(entries: ModelsCatalogEntry[]): Map<string, ModelsCatalogEntry[]> {
  const groups = new Map<string, ModelsCatalogEntry[]>();
  for (const entry of entries) {
    const key = groupKey(entry);
    const group = groups.get(key);
    if (group) group.push(entry);
    else groups.set(key, [entry]);
  }
  return groups;
}

function entryForChoice(choice: ModelChoice): ModelsCatalogEntry {
  const entry = modelsCatalogById.get(choice.id);
  if (!entry)
    throw new Error(`Collapsed id is not in the fixture dynamic models catalog: ${choice.id}`);
  return entry;
}

function seededPermutation<T>(values: T[], seed: number): T[] {
  const result = [...values];
  let state = seed >>> 0;
  for (let index = result.length - 1; index > 0; index--) {
    state = (Math.imul(state, 1_664_525) + 1_013_904_223) >>> 0;
    const swapIndex = state % (index + 1);
    [result[index], result[swapIndex]] = [result[swapIndex]!, result[index]!];
  }
  return result;
}

function exactNamesOf(entry: ModelsCatalogEntry): string[] {
  const names = new Map<string, string>();
  for (const value of [entry.groupLabel, entry.family]) {
    const name = nonEmpty(value);
    if (!name) continue;
    if (!names.has(name.toLowerCase())) names.set(name.toLowerCase(), name);
  }
  return [...names.values()];
}

function fixtureNameGroups(entries: ModelsCatalogEntry[]): Array<{
  selection: string;
  entries: ModelsCatalogEntry[];
}> {
  const byName = new Map<string, { selection: string; entries: ModelsCatalogEntry[] }>();
  for (const entry of entries) {
    for (const selection of exactNamesOf(entry)) {
      const key = selection.toLowerCase();
      const named = byName.get(key);
      if (named) named.entries.push(entry);
      else byName.set(key, { selection, entries: [entry] });
    }
  }
  return [...byName.values()];
}

type DuplicateEffortCase = {
  choice: ModelChoice;
  effort: (typeof EFFORT_LEVELS)[number];
  candidates: ModelsCatalogEntry[];
};

function fixtureDuplicateEffortCases(): DuplicateEffortCase[] {
  const cases: DuplicateEffortCase[] = [];
  for (const choice of choices) {
    const plain = choice.variants.filter((variant) => variant.modifiers.length === 0);
    const pool = plain.length > 0 ? plain : choice.variants;

    for (const effort of EFFORT_LEVELS) {
      const candidates = pool
        .filter((variant) => variant.effort === effort)
        .map((variant) => modelsCatalogById.get(variant.wireId))
        .filter((entry): entry is ModelsCatalogEntry => entry !== undefined);
      if (candidates.length > 1) cases.push({ choice, effort, candidates });
    }
  }
  return cases;
}

function expectDuplicateEffortTieBreak(
  duplicate: DuplicateEffortCase,
  permutations: ModelsCatalogEntry[][]
): void {
  const { choice, effort, candidates } = duplicate;
  const resolvedIds = new Set(
    [modelsCatalog, ...permutations].map((entries) =>
      resolver.expand(choice.id, entries, { effort })
    )
  );
  expect(resolvedIds.size).toBe(1);

  const [resolvedId] = resolvedIds;
  const vendorDefault = candidates.find((entry) => entry.isFamilyDefault);
  if (vendorDefault) {
    expect(resolvedId).toBe(vendorDefault.wireId);
    return;
  }

  const cheapestCost = Math.min(
    ...candidates.map((entry) => entry.costFactor ?? Number.MAX_SAFE_INTEGER)
  );
  const cheapestIds = candidates
    .filter((entry) => (entry.costFactor ?? Number.MAX_SAFE_INTEGER) === cheapestCost)
    .map((entry) => entry.wireId);
  expect(cheapestIds).toContain(resolvedId);
}

function exactNamePermutationFlaps(
  exactNames: string[],
  contexts: Array<{ label: string; ctx: ExpandContext }>,
  permutations: ModelsCatalogEntry[][]
): string[] {
  const flaps: string[] = [];
  for (const selection of exactNames) {
    for (const { label, ctx } of contexts) {
      const resolvedIds = new Set(
        [modelsCatalog, ...permutations].map((entries) => resolver.expand(selection, entries, ctx))
      );
      if (resolvedIds.size > 1) {
        flaps.push(`${selection} @ ${label}: ${[...resolvedIds].sort().join(" | ")}`);
        continue;
      }

      const [resolvedId] = resolvedIds;
      expect(modelsCatalogById.has(resolvedId!)).toBe(true);
    }
  }
  return flaps;
}

function deriveAmbiguousUidPrefixes(entries: ModelsCatalogEntry[]): Array<{
  prefix: string;
  sourceUid: string;
  matches: ModelsCatalogEntry[];
}> {
  const exactUids = new Set(entries.map((entry) => entry.wireId.toLowerCase()));
  const exactNames = new Set(entries.flatMap(exactNamesOf).map((name) => name.toLowerCase()));
  const usedRoots = new Set<string>();
  const derived: Array<{ prefix: string; sourceUid: string; matches: ModelsCatalogEntry[] }> = [];

  for (const source of entries) {
    const parts = source.wireId.split("-");
    for (let length = parts.length - 1; length > 0; length--) {
      const prefix = parts.slice(0, length).join("-");
      const lower = prefix.toLowerCase();
      if (exactUids.has(lower) || exactNames.has(lower)) continue;

      const matches = entries.filter((entry) => entry.wireId.toLowerCase().startsWith(`${lower}-`));
      if (new Set(matches.map(groupKey)).size <= 1) continue;

      const root = parts[0]!.toLowerCase();
      if (!usedRoots.has(root)) {
        usedRoots.add(root);
        derived.push({ prefix, sourceUid: source.wireId, matches });
      }
      break;
    }
  }
  return derived;
}

function firstGroupForPrefix(entries: ModelsCatalogEntry[], prefix: string): string {
  const first = entries.find((entry) =>
    entry.wireId.toLowerCase().startsWith(`${prefix.toLowerCase()}-`)
  );
  if (!first) throw new Error(`Permutation lost fixture prefix: ${prefix}`);
  return groupKey(first);
}

function expectPermutationInvariant(
  selection: string,
  passesThrough: boolean,
  ctx: ExpandContext,
  permutations: ModelsCatalogEntry[][]
): void {
  const expected = resolver.expand(selection, modelsCatalog, ctx);
  if (passesThrough) expect(expected).toBe(selection);
  else expect(modelsCatalogById.has(expected)).toBe(true);

  for (const entries of permutations) {
    expect(resolver.expand(selection, entries, ctx)).toBe(expected);
  }
}

const groups = groupEntries(modelsCatalog);
const windowsByLabel = new Map<string, Set<number>>();
for (const entry of modelsCatalog) {
  const label = groupKeyOf(entry);
  const windows = windowsByLabel.get(label) ?? new Set<number>();
  windows.add(entry.contextWindow ?? 0);
  windowsByLabel.set(label, windows);
}

describe("DevinModelResolver collapse", () => {
  test("collapses the captured dynamic models catalog to substantially fewer real, unique wire ids", () => {
    expect(modelsCatalog).toHaveLength(170);
    expect(choices.length).toBeLessThan(modelsCatalog.length / 2);
    expect(choices).toHaveLength(groups.size);
    expect(choices.every((choice) => modelsCatalogById.has(choice.id))).toBe(true);
    expect(new Set(choices.map((choice) => choice.id)).size).toBe(choices.length);
  });

  test("creates sibling rows exactly for labels with distinct context windows", () => {
    const rowsByLabel = new Map<string, number>();
    for (const choice of choices) {
      const label = groupKeyOf(entryForChoice(choice));
      rowsByLabel.set(label, (rowsByLabel.get(label) ?? 0) + 1);
    }

    const expectedMultiWindowLabels = [...windowsByLabel]
      .filter(([, windows]) => windows.size > 1)
      .map(([label]) => label)
      .sort();
    const actualMultiWindowLabels = [...rowsByLabel]
      .filter(([, count]) => count > 1)
      .map(([label]) => label)
      .sort();
    expect(actualMultiWindowLabels).toEqual(expectedMultiWindowLabels);

    for (const [label, windows] of windowsByLabel) {
      expect(rowsByLabel.get(label)).toBe(windows.size);
      if (windows.size === 1) expect(rowsByLabel.get(label)).toBe(1);
      else expect(rowsByLabel.get(label)).toBeGreaterThan(1);
    }
  });

  test("makes every row's variants its exact group and partitions the dynamic models catalog", () => {
    const allVariantIds: string[] = [];

    for (const choice of choices) {
      const expectedIds = groups
        .get(groupKey(entryForChoice(choice)))!
        .map((entry) => entry.wireId)
        .sort();
      const actualIds = choice.variants.map((variant) => variant.wireId).sort();
      expect(actualIds).toEqual(expectedIds);
      allVariantIds.push(...actualIds);
    }

    expect(allVariantIds.sort()).toEqual(modelsCatalog.map((entry) => entry.wireId).sort());
  });

  test("uses the provider-declared family default as the row id", () => {
    for (const choice of choices) {
      const defaults = groups
        .get(groupKey(entryForChoice(choice)))!
        .filter((entry) => entry.isFamilyDefault)
        .map((entry) => entry.wireId);
      if (defaults.length > 0) expect(defaults).toContain(choice.id);
    }
  });
});

describe("DevinModelResolver expand", () => {
  test("reaches the 1M GLM effort variants", () => {
    expect(resolver.expand("glm-5-2-1m", modelsCatalog, { effort: "max" })).toBe("glm-5-2-max-1m");
    expect(resolver.expand("glm-5-2-1m", modelsCatalog, { effort: "none" })).toBe(
      "glm-5-2-none-1m"
    );
    expect(resolver.expand("glm-5-2-1m", modelsCatalog, {})).toBe("glm-5-2-1m");
  });

  test("never auto-selects a premium variant when a plain variant exists", () => {
    for (const choice of choices) {
      if (!choice.variants.some((variant) => variant.modifiers.length === 0)) continue;

      for (const effort of EFFORT_LEVELS) {
        const resolved = resolver.expand(choice.id, modelsCatalog, { effort });
        const variant = choice.variants.find((candidate) => candidate.wireId === resolved);
        expect(variant).toBeDefined();
        if (resolved !== choice.id) expect(variant?.modifiers).toEqual([]);
      }
    }
  });

  test("round-trips every collapsed row id when effort is absent", () => {
    for (const choice of choices) {
      expect(resolver.expand(choice.id, modelsCatalog, {})).toBe(choice.id);
    }
  });

  test("honours an explicitly tier-named fixture uid despite conflicting effort", () => {
    const explicit = choices
      .flatMap((choice) => choice.variants.map((variant) => ({ choiceId: choice.id, variant })))
      .find(
        ({ choiceId, variant }) =>
          variant.wireId !== choiceId &&
          variant.effort !== undefined &&
          variant.wireId.toLowerCase().split("-").includes(variant.effort)
      );
    if (!explicit?.variant.effort) {
      throw new Error("Fixture has no non-default uid that explicitly names its effort tier");
    }
    const conflictingEffort = EFFORT_LEVELS.find((effort) => effort !== explicit.variant.effort);
    if (!conflictingEffort) throw new Error("No conflicting effort level is available");

    expect(
      resolver.expand(explicit.variant.wireId, modelsCatalog, { effort: conflictingEffort })
    ).toBe(explicit.variant.wireId);
  });

  test("keeps a bare multi-window label inside one context group for every effort", () => {
    const multiWindowLabels = [...windowsByLabel]
      .filter(([, windows]) => windows.size > 1)
      .map(([label]) => label);
    expect(multiWindowLabels.length).toBeGreaterThan(0);

    for (const label of multiWindowLabels) {
      const baselineId = resolver.expand(label, modelsCatalog, {});
      const baseline = modelsCatalogById.get(baselineId);
      if (!baseline) throw new Error(`Bare fixture label did not resolve: ${label}`);
      expect(groupKeyOf(baseline)).toBe(label);

      for (const effort of EFFORT_LEVELS) {
        const resolvedId = resolver.expand(label, modelsCatalog, { effort });
        const resolved = modelsCatalogById.get(resolvedId);
        if (!resolved) throw new Error(`Bare fixture label did not resolve: ${label}`);
        expect(groupKeyOf(resolved)).toBe(label);
        expect(resolved.contextWindow).toBe(baseline.contextWindow);
      }
    }
  });

  test("passes an unknown selection through unchanged", () => {
    expect(
      resolver.expand("not-in-the-captured-models-catalog", modelsCatalog, { effort: "high" })
    ).toBe("not-in-the-captured-models-catalog");
  });

  test("passes a selection through unchanged when the dynamic models catalog is empty", () => {
    expect(resolver.expand("any-model", [], { effort: "high" })).toBe("any-model");
  });

  test("resolves vendor labels case-insensitively", () => {
    const capitalisedLabel = modelsCatalog
      .map((entry) => nonEmpty(entry.groupLabel))
      .find((label) => label !== undefined && label !== label.toLowerCase());
    if (!capitalisedLabel) throw new Error("Fixture has no capitalised vendor label");

    expect(resolver.expand(capitalisedLabel.toLowerCase(), modelsCatalog, {})).toBe(
      resolver.expand(capitalisedLabel, modelsCatalog, {})
    );
  });

  test("prefers the vendor default, then lower cost, for duplicate effort levels", () => {
    const duplicateEffortCases = fixtureDuplicateEffortCases();

    // The property is fixture-derived: a future dynamic models catalog with no duplicate
    // declared levels has no tie-break case to exercise.
    if (duplicateEffortCases.length === 0) return;

    const permutations = Array.from({ length: 16 }, (_, index) =>
      seededPermutation(modelsCatalog, index + 1)
    );
    for (const duplicate of duplicateEffortCases) {
      expectDuplicateEffortTieBreak(duplicate, permutations);
    }
  });

  test("is invariant for every exact fixture name and effort across dynamic models catalog permutations", () => {
    const permutations = Array.from({ length: 16 }, (_, index) =>
      seededPermutation(modelsCatalog, index + 1)
    );
    expect(
      new Set(permutations.map((entries) => entries.map((entry) => entry.wireId).join("\0"))).size
    ).toBe(permutations.length);

    const exactNames = fixtureNameGroups(modelsCatalog).map(({ selection }) => selection);
    expect(exactNames.length).toBeGreaterThan(0);

    const contexts: Array<{ label: string; ctx: ExpandContext }> = [
      { label: "no effort", ctx: {} },
      ...EFFORT_LEVELS.map((effort) => ({ label: effort, ctx: { effort } })),
    ];
    const flaps = exactNamePermutationFlaps(exactNames, contexts, permutations);

    // Bun prints every string in this list on failure, naming each exact
    // (label/family, effort) pair and every wire id it alternated between.
    expect(flaps).toEqual([]);

    const ambiguousPrefixes = deriveAmbiguousUidPrefixes(modelsCatalog).slice(0, 2);
    if (ambiguousPrefixes.length < 2) {
      throw new Error("Fixture has fewer than two independently derived ambiguous uid prefixes");
    }

    for (const { prefix } of ambiguousPrefixes) {
      const firstMatchGroups = new Set(
        permutations.map((entries) => firstGroupForPrefix(entries, prefix))
      );
      expect(firstMatchGroups.size).toBeGreaterThan(1);
    }

    const passThroughSelections = [
      ...ambiguousPrefixes.map(({ prefix: selection }) => ({ selection, passesThrough: true })),
      { selection: "not-in-the-captured-models-catalog", passesThrough: true },
    ];
    for (const choice of choices) {
      expectPermutationInvariant(choice.id, false, {}, permutations);
    }
    for (const ctx of [{}, { effort: "high" as const }]) {
      for (const { selection, passesThrough } of passThroughSelections) {
        expectPermutationInvariant(selection, passesThrough, ctx, permutations);
      }
    }
  });

  test("passes independently fixture-derived multi-group uid prefixes through unchanged", () => {
    const derived = deriveAmbiguousUidPrefixes(modelsCatalog).slice(0, 2);
    expect(derived).toHaveLength(2);
    expect(new Set(derived.map(({ sourceUid }) => sourceUid)).size).toBe(2);
    expect(new Set(derived.map(({ prefix }) => prefix.split("-")[0]!.toLowerCase())).size).toBe(2);

    const exactUids = new Set(modelsCatalog.map((entry) => entry.wireId.toLowerCase()));
    const exactNames = new Set(
      modelsCatalog.flatMap(exactNamesOf).map((name) => name.toLowerCase())
    );
    for (const { prefix, sourceUid, matches } of derived) {
      expect(sourceUid.toLowerCase().startsWith(`${prefix.toLowerCase()}-`)).toBe(true);
      expect(new Set(matches.map(groupKey)).size).toBeGreaterThan(1);
      expect(exactUids.has(prefix.toLowerCase())).toBe(false);
      expect(exactNames.has(prefix.toLowerCase())).toBe(false);
      expect(resolver.expand(prefix, modelsCatalog, {})).toBe(prefix);
      expect(resolver.expand(prefix, modelsCatalog, { effort: "high" })).toBe(prefix);
    }
  });

  test("resolves fixture-derived exact names spanning context groups via the vendor default", () => {
    const exactUids = new Set(modelsCatalog.map((entry) => entry.wireId.toLowerCase()));
    const multiWindowNames = fixtureNameGroups(modelsCatalog).filter(
      ({ selection, entries }) =>
        !exactUids.has(selection.toLowerCase()) &&
        new Set(entries.map((entry) => entry.contextWindow)).size > 1
    );
    expect(multiWindowNames.length).toBeGreaterThan(0);

    for (const { selection, entries } of multiWindowNames) {
      expect(new Set(entries.map(groupKey)).size).toBeGreaterThan(1);
      const defaults = entries.filter((entry) => entry.isFamilyDefault);
      expect(defaults).toHaveLength(1);

      for (const ctx of [{}, { effort: "high" as const }]) {
        const resolvedId = resolver.expand(selection, modelsCatalog, ctx);
        const resolved = modelsCatalogById.get(resolvedId);
        if (!resolved) throw new Error(`Fixture name did not resolve to a real uid: ${selection}`);
        expect(entries.map((entry) => entry.wireId)).toContain(resolvedId);
        expect(groupKey(resolved)).toBe(groupKey(defaults[0]!));
      }
    }
  });

  test("keeps exact-name effort resolution inside its selected context group", () => {
    const exactUids = new Set(modelsCatalog.map((entry) => entry.wireId.toLowerCase()));
    const multiWindowNames = fixtureNameGroups(modelsCatalog).filter(
      ({ selection, entries }) =>
        !exactUids.has(selection.toLowerCase()) &&
        new Set(entries.map((entry) => entry.contextWindow)).size > 1
    );
    expect(multiWindowNames.length).toBeGreaterThan(0);

    for (const { selection, entries } of multiWindowNames) {
      const baselineId = resolver.expand(selection, modelsCatalog, {});
      const baseline = modelsCatalogById.get(baselineId);
      if (!baseline) throw new Error(`Fixture name did not resolve to a real uid: ${selection}`);

      const selectedGroup = groups.get(groupKey(baseline));
      if (!selectedGroup) throw new Error(`Resolved fixture group is missing: ${selection}`);
      const selectedIds = new Set(selectedGroup.map((entry) => entry.wireId));
      const siblingWindows = new Set(
        entries
          .map((entry) => entry.contextWindow)
          .filter((window) => window !== baseline.contextWindow)
      );
      expect(siblingWindows.size).toBeGreaterThan(0);

      for (const effort of EFFORT_LEVELS) {
        const resolvedId = resolver.expand(selection, modelsCatalog, { effort });
        const resolved = modelsCatalogById.get(resolvedId);
        if (!resolved) throw new Error(`Fixture name did not resolve to a real uid: ${selection}`);
        expect(selectedIds.has(resolvedId)).toBe(true);
        expect(resolved.contextWindow).toBe(baseline.contextWindow);
        expect(siblingWindows.has(resolved.contextWindow)).toBe(false);
      }
    }
  });
});

describe("model resolver helpers", () => {
  test("nonEmpty treats empty and whitespace-only strings as absent", () => {
    expect(nonEmpty("")).toBeUndefined();
    expect(nonEmpty(" \t\n ")).toBeUndefined();
    expect(nonEmpty(undefined)).toBeUndefined();
    expect(nonEmpty("  model family  ")).toBe("model family");
  });

  test("offerIsLive compares expiry against an injected clock", () => {
    const now = 2_000_000;
    expect(offerIsLive({ kind: "promo", expiresAt: 1_999 }, now)).toBe(false);
    expect(offerIsLive({ kind: "promo", expiresAt: 2_001 }, now)).toBe(true);
    expect(offerIsLive({ kind: "included" }, now)).toBe(true);
    expect(offerIsLive(undefined, now)).toBe(false);
  });

  test("groupKeyOf prefers a non-empty label, then family, then wire id", () => {
    expect(groupKeyOf({ wireId: "wire", groupLabel: " Vendor label ", family: "family" })).toBe(
      "Vendor label"
    );
    expect(groupKeyOf({ wireId: "wire", groupLabel: "", family: " family " })).toBe("family");
    expect(groupKeyOf({ wireId: "wire", groupLabel: " ", family: "" })).toBe("wire");
  });
});