Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

10.2 KB
Raw Blame
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
import { describe, expect, it } from "vitest";
import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels";
import { buildModelAvailability, deploymentRefsFromModelInfo } from "@/lib/autorouter_presets";
import {
  buildAutomaticRouterConfig,
  buildPreferredTierModels,
  prefillEmptyTiers,
  type PreferredTierModels,
} from "./auto_setup";

const models = (...names: string[]) => names.map((model_group) => ({ model_group, mode: "chat" }));
const reasoningModel = (model_group: string, supported_reasoning_efforts: string[]) => ({
  model_group,
  mode: "chat",
  supports_reasoning: true,
  supported_reasoning_efforts,
});
const deployment = (model_name: string, model = model_name): AutoRouterDeployment => ({
  model_name,
  litellm_params: { model },
});
const tierModels = (config: ReturnType<typeof buildAutomaticRouterConfig>) =>
  config && Object.values(config.tiers).map((tier) => (typeof tier === "string" ? tier : tier[0]));

describe("buildPreferredTierModels", () => {
  it("recognizes curated models that are not in a preset", () => {
    const available = ["gpt-5.6-luna", "claude-sonnet-5", "grok-4.6", "deepseek-v4-pro"];
    const availability = buildModelAvailability(available, []);
    const preferred = buildPreferredTierModels([], availability);
    const expected: PreferredTierModels = {
      SIMPLE: ["gpt-5.6-luna"],
      MEDIUM: ["claude-sonnet-5"],
      COMPLEX: ["deepseek-v4-pro", "grok-4.6"],
      REASONING: ["deepseek-v4-pro", "grok-4.6"],
    };

    expect(preferred).toEqual(expected);
  });

  it("prefers the current model ladder over older preset entries", () => {
    const availability = buildModelAvailability(["gpt-5.6-luna", "gpt-4o-mini"], []);
    const preferred = buildPreferredTierModels(
      [
        {
          key: "old",
          label: "Old",
          description: "Old model",
          complexity_router_config: {
            tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] },
            classifier_type: "heuristic_v2",
          },
        },
      ],
      availability,
    );

    expect(preferred.SIMPLE).toEqual(["gpt-5.6-luna", "gpt-4o-mini"]);
  });
});

describe("buildAutomaticRouterConfig", () => {
  it("selects native Terra and Sol groups with their reasoning settings, retaining cloud fallback", () => {
    const modelNames = ["gpt-5.6-terra", "gpt-5.6-sol"];
    const deployments = modelNames.flatMap((model) => [
      deployment(model, `azure/${model}`),
      deployment(`z-native-${model}`, `openai/${model}`),
    ]);
    const available = deployments.map(({ model_name }) => reasoningModel(model_name!, ["none", "high"]));
    const availability = buildModelAvailability(
      available.map(({ model_group }) => model_group),
      deploymentRefsFromModelInfo(deployments),
    );
    const preferred = buildPreferredTierModels([], availability);
    const config = buildAutomaticRouterConfig(available, deployments, preferred);

    expect(tierModels(config)).toEqual([
      "z-native-gpt-5.6-terra",
      "z-native-gpt-5.6-terra",
      "z-native-gpt-5.6-sol",
      "z-native-gpt-5.6-sol",
    ]);
    expect(config?.tier_model_params).toEqual({
      REASONING: { "z-native-gpt-5.6-sol": { reasoning_effort: "high" } },
    });

    const cloudOnly = models(...modelNames);
    const cloudAvailability = buildModelAvailability(modelNames, deploymentRefsFromModelInfo(deployments));
    const cloudPreferred = buildPreferredTierModels([], cloudAvailability);

    expect(tierModels(buildAutomaticRouterConfig(cloudOnly, deployments, cloudPreferred))).toEqual([
      "gpt-5.6-terra",
      "gpt-5.6-terra",
      "gpt-5.6-sol",
      "gpt-5.6-sol",
    ]);
  });

  it("selects one preferred model for each tier", () => {
    const preferred: PreferredTierModels = {
      SIMPLE: ["simple"],
      MEDIUM: ["medium"],
      COMPLEX: ["complex"],
      REASONING: ["reasoning"],
    };

    expect(
      tierModels(buildAutomaticRouterConfig(models("simple", "medium", "complex", "reasoning"), [], preferred)),
    ).toEqual(["simple", "medium", "complex", "reasoning"]);
  });

  it.each([
    {
      provider: "OpenAI",
      available: ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-6-astra"],
      expected: ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-6-astra", "gpt-6-astra"],
      supportedEfforts: ["low", "medium", "high", "xhigh", "max"],
      effort: "max",
    },
    {
      provider: "Anthropic",
      available: ["claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5"],
      expected: ["claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5", "claude-opus-5"],
      supportedEfforts: ["low", "medium", "high", "max"],
      effort: "max",
    },
    {
      provider: "Google",
      available: ["gemini-3.5-flash-lite", "gemini-3.8-flash", "gemini-3.1-pro-preview"],
      expected: ["gemini-3.5-flash-lite", "gemini-3.8-flash", "gemini-3.1-pro-preview", "gemini-3.1-pro-preview"],
      supportedEfforts: ["low", "medium", "high"],
      effort: "high",
    },
    {
      provider: "DeepSeek",
      available: ["deepseek-v4-flash", "deepseek-v4-pro"],
      expected: ["deepseek-v4-flash", "deepseek-v4-flash", "deepseek-v4-pro", "deepseek-v4-pro"],
      supportedEfforts: ["none", "high"],
      effort: "high",
    },
    {
      provider: "xAI",
      available: ["grok-4.6"],
      expected: ["grok-4.6", "grok-4.6", "grok-4.6", "grok-4.6"],
      supportedEfforts: ["low", "medium", "high", "xhigh"],
      effort: "xhigh",
    },
  ])(
    "uses the current $provider ladder and strongest advertised reasoning effort",
    ({ available, expected, supportedEfforts, effort }) => {
      const availability = buildModelAvailability(available, []);
      const preferred = buildPreferredTierModels([], availability);
      const modelInfo = models(...available).map((model) =>
        model.model_group === expected[3] ? reasoningModel(model.model_group, supportedEfforts) : model,
      );

      const config = buildAutomaticRouterConfig(modelInfo, [], preferred);

      expect(tierModels(config)).toEqual(expected);
      expect(config?.tier_model_params).toEqual({
        REASONING: { [expected[3]]: { reasoning_effort: effort } },
      });
    },
  );

  it("never exceeds the selected model group's advertised reasoning efforts", () => {
    const available = ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-5.6-sol"];
    const availability = buildModelAvailability(available, []);
    const preferred = buildPreferredTierModels([], availability);
    const modelInfo = [
      ...models("gpt-5.6-luna", "gpt-5.6-terra"),
      reasoningModel("gpt-5.6-sol", ["none", "low", "medium", "high", "xhigh"]),
    ];

    const config = buildAutomaticRouterConfig(modelInfo, [], preferred);

    expect(config?.tier_model_params).toEqual({
      REASONING: { "gpt-5.6-sol": { reasoning_effort: "xhigh" } },
    });
  });

  it("leaves reasoning effort unset when the proxy does not report supported values", () => {
    const available = ["grok-4.6"];
    const availability = buildModelAvailability(available, []);
    const preferred = buildPreferredTierModels([], availability);

    const config = buildAutomaticRouterConfig(models(...available), [], preferred);

    expect(config?.tier_model_params).toBeUndefined();
  });

  it("reuses the closest available tier when a tier has no match", () => {
    const preferred: PreferredTierModels = {
      SIMPLE: ["simple"],
      MEDIUM: [],
      COMPLEX: ["complex"],
      REASONING: [],
    };

    expect(tierModels(buildAutomaticRouterConfig(models("simple", "complex"), [], preferred))).toEqual([
      "simple",
      "simple",
      "complex",
      "complex",
    ]);
  });

  it("returns null when none of the available models are recommended", () => {
    const preferred: PreferredTierModels = {
      SIMPLE: ["missing-simple"],
      MEDIUM: ["missing-medium"],
      COMPLEX: ["missing-complex"],
      REASONING: ["missing-reasoning"],
    };

    expect(buildAutomaticRouterConfig(models("unknown-model"), [], preferred)).toBeNull();
  });

  it("ignores non-chat models and existing auto routers", () => {
    const preferred: PreferredTierModels = {
      SIMPLE: ["gpt-4o-mini", "smart-router"],
      MEDIUM: [],
      COMPLEX: [],
      REASONING: [],
    };
    const available = [
      { model_group: "gpt-4o-mini", mode: "chat" },
      { model_group: "image-model", mode: "image_generation" },
      { model_group: "smart-router", mode: "chat" },
    ];

    expect(
      tierModels(
        buildAutomaticRouterConfig(available, [deployment("smart-router", "auto_router/complexity_router")], preferred),
      ),
    ).toEqual(["gpt-4o-mini", "gpt-4o-mini", "gpt-4o-mini", "gpt-4o-mini"]);
  });
});

describe("prefillEmptyTiers", () => {
  const empty: ComplexityRouterConfigValue = {
    classifier_type: "jev",
    jev_classifier_config: { model: "jev-custom", timeout_ms: 4200 },
    classification_mode: "user_turn",
    tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] },
  };
  const automatic: ComplexityRouterConfigValue = {
    classifier_type: "heuristic_v2",
    tiers: { SIMPLE: ["fast"], MEDIUM: ["mid"], COMPLEX: ["strong"], REASONING: ["strong"] },
    tier_model_params: { REASONING: { strong: { reasoning_effort: "high" } } },
  };
  it("fills tiers while preserving a classifier configured before models load", () => {
    expect(prefillEmptyTiers(empty, automatic)).toEqual({
      ...empty,
      tiers: automatic.tiers,
      tier_model_params: automatic.tier_model_params,
    });
  });
  it("leaves the draft alone when no recommended models are available", () => {
    expect(prefillEmptyTiers(empty, null)).toBe(empty);
  });
  it("preserves a partial manual assignment", () => {
    const partial = { ...empty, tiers: { ...empty.tiers, SIMPLE: ["manual"] } };
    expect(prefillEmptyTiers(partial, automatic)).toBe(partial);
  });
  it("preserves a custom tier set created before models load", () => {
    const custom = { ...empty, custom_tier_set: { tiers: [], fallback_tier_id: "" } };
    expect(prefillEmptyTiers(custom, automatic)).toBe(custom);
  });
  it.each(["capability", "llm_v2"] as const)("does not prefill %s solver pools", (classifier_type) => {
    const solver = { ...empty, classifier_type };
    expect(prefillEmptyTiers(solver, automatic)).toBe(solver);
  });
});