Skip to content

A local-first multi-agent app development platform, built on PenguinHarness

79.5 KB
Raw Blame
/**
 * A Project's model configuration: the built-in catalog as presets, one config file holding the
 * credentials, rekeying, pricing fields, the connectivity test, and what a save does to loaded
 * Sessions.
 *
 * - A new Project, and default_project when the seeded admin adopts it, is preset with every
 *   built-in model (provider and model_id stored as separate fields, the pair the key) and the
 *   default model, so a Session opens with no model ref; a default_project the CLI already
 *   configured is left alone.
 * - Credentials live inline in `.project_config.toml` (0600) and GET shows only a mask; the env
 *   fallback is reported and masked only where the entry is allowed one, never leaking a value.
 * - GET fills in display name and env key from the catalog; vision comes from the file, else the
 *   catalog default; a pre-rename `client_type = "openai"` reads as openai-chat; a model the
 *   catalog dropped still loads and stays usable; one model_id under two providers is two rows.
 * - PUT persists a custom model's vision flag and max_tokens (clearing and rejecting bad values),
 *   falls back to OPENAI_API_KEY for the openai protocol, requires a provider, refuses an id a
 *   vendor group cannot route when the request introduces it, and stores or clears a discount.
 * - renamedFrom (and a group change) migrates the credential, the config and the default and
 *   vision pointers; without it a rekey is delete-then-create; an invalid one is 400. A
 *   ModelScope settings save is serialized with a group credential write.
 * - The display name is persisted only when it differs from the catalog's; an absent one
 *   inherits, an empty one clears, and a cleared one is restorable.
 * - A scheduled row answers both rates from its one price on disk; a hand-edited price and a
 *   promotion apply in both tiers.
 * - The connectivity test sends the entry's upstream model_id, no tools, works for saved and
 *   draft models, converges to ok:false with no credential, tests the draft (clearApiKey) rather
 *   than the stored key, and refuses a keyless gateway row or a Penguin Go probe without reaching
 *   for vendor environment keys or the network.
 * - A PUT invalidates the Project's cached Session runtimes (reads do not), tells the Project's
 *   open Session channels only, and GET/PUT report the file's updatedAt.
 *
 * One app per describe; every case works in a Project of its own.
 */
import { createServer } from "node:http";
import type { AddressInfo } from "node:net";
import { readFile, stat, writeFile } from "node:fs/promises";
import path from "node:path";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import {
  MODEL_CATALOG,
  PENGUIN_GO_BASE_URL,
  catalogEntryFor,
  presetPromotions,
  userText,
} from "@adelie/core";
import type {
  ErrorBody,
  ModelsResponse,
  ModelTestResponse,
  ModelVisionDetectResponse,
  ProjectCreateResponse,
  SessionCreateResponse,
} from "../src/api/types.js";
import { ProjectConfigService } from "../src/services/project-config-service.js";
import type { RuntimeSession } from "../src/runtime/session-manager.js";
import type { ChannelEvent } from "../src/runtime/channel.js";
import { jsonResponse, stubFetch } from "./fixtures/fetch.js";
import { fakeSession, sessionRow } from "./fixtures/session.js";
import { apiClient, createTestApp, loginAdmin, provisionUser, waitFor } from "./helpers.js";
import type { TestApp } from "./helpers.js";
import { wire } from "@adelie/core/kernel";

/** Preset paired refs (config primary key = (provider, model_id)): retired catalog rows are not presets. */
const catalogPairs = MODEL_CATALOG.filter((m) => m.retired !== true).map(
  (m) => `${m.provider}\0${m.modelId}`,
);
/** Response row → comparable paired key (test-only comparison, not the storage format). */
const pairKey = (m: { provider: string; modelId: string }): string => `${m.provider}\0${m.modelId}`;
/** Fetch a row by its paired ref. */
const pick = (body: ModelsResponse, provider: string, modelId: string) =>
  body.models.find((m) => m.provider === provider && m.modelId === modelId)!;

/**
 * Writes rows straight into a Project's config file, as a table saved before a rule existed
 * would hold them: rows a built-in group no longer takes from a PUT (see "model_not_addable")
 * are only ever reached this way.
 */
async function storeRows(
  root: string,
  projectId: string,
  rows: Array<{ provider: string; modelId: string; clientType?: string }>,
): Promise<void> {
  const toml = rows.flatMap((r) => [
    "[[models]]",
    `provider = "${r.provider}"`,
    `model_id = "${r.modelId}"`,
    ...(r.clientType !== undefined ? [`client_type = "${r.clientType}"`] : []),
    "",
  ]);
  await writeFile(path.join(root, projectId, ".project_config.toml"), toml.join("\n"), "utf8");
}

describe("models preset & catalog enrichment", () => {
  let t: TestApp;
  let api: ReturnType<typeof apiClient>;
  let projectId: string;
  const url = () => `/api/projects/${projectId}/models`;

  beforeAll(async () => {
    t = await createTestApp();
    const { cookie } = await provisionUser(t.app, "alice");
    api = apiClient(t.app, cookie);
  });
  afterAll(async () => {
    await t.cleanup();
  });
  let projects = 0;
  beforeEach(async () => {
    projects += 1;
    const created = (await (
      await api.post("/api/projects", {
        projectId: `alice-preset_${projects}`,
        name: "Preset project",
      })
    ).json()) as ProjectCreateResponse;
    projectId = created.project.projectId;
  });

  it("credentials are inlined in one file: the apiKey from PUT lands in .project_config.toml (0600), GET returns only a mask", async () => {
    const put = await api.put(url(), {
      defaultModel: { provider: "custom", modelId: "m-inline" },
      models: [
        {
          provider: "custom",
          modelId: "m-inline",
          apiKey: "sk-server-inline-1",
          baseUrl: "https://inline.example/v1",
          clientType: "openai",
        },
      ],
    });
    expect(put.status).toBe(200);

    // GET: the masked key and base URL are both visible; plaintext is never sent.
    const body = (await (await api.get(url())).json()) as ModelsResponse;
    const m = pick(body, "custom", "m-inline");
    expect(m.credential?.baseUrl).toBe("https://inline.example/v1");
    expect(m.credential?.apiKeyMasked).toBe("sk-s…ne-1");
    expect(m.credential?.createdAt).toBeTruthy();
    expect(JSON.stringify(body)).not.toContain("sk-server-inline-1");

    // Secrets inlined in the single config file (persisted 0600); provider and model_id are separate columns, no concatenated string.
    const projectDir = path.join(t.root, projectId);
    const cfgFile = path.join(projectDir, ".project_config.toml");
    const cfgRaw = await readFile(cfgFile, "utf8");
    expect(cfgRaw).toContain("sk-server-inline-1");
    expect(cfgRaw).toContain('provider = "custom"');
    expect(cfgRaw).toContain('model_id = "m-inline"');
    expect(cfgRaw).not.toContain("custom/m-inline");
    // POSIX-only: Windows has no owner-only mode bits (chmod maps to the read-only attribute).
    if (process.platform !== "win32") {
      expect((await stat(cfgFile)).mode & 0o777).toBe(0o600);
    }
    // No more separate .credentials.toml / project_config.toml files.
    await expect(readFile(path.join(projectDir, ".credentials.toml"), "utf8")).rejects.toThrow();
    await expect(readFile(path.join(projectDir, "project_config.toml"), "utf8")).rejects.toThrow();
  });

  it("a preset a sync adds to a gateway group borrows the group's key instead of arriving unconfigured", async () => {
    const cfgFile = path.join(t.root, projectId, ".project_config.toml");
    const GATEWAY = "https://tokendance.space/gateway/v1";
    const row = (modelId: string, extra: Record<string, unknown> = {}) => ({
      provider: "tokendance",
      modelId,
      clientType: "openai-chat",
      baseUrl: GATEWAY,
      ...extra,
    });

    // The table before the sync: one TokenDance row, the key pasted once.
    const first = await api.put(url(), {
      defaultModel: { provider: "tokendance", modelId: "deepseek-v4-flash-0731" },
      models: [row("deepseek-v4-flash-0731", { apiKey: "sk-group-gateway-1" })],
    });
    expect(first.status).toBe(200);

    // What "sync presets" sends: the row that was there (restating no key at all) plus a preset
    // the catalog carries and this table does not — the shape that used to land keyless.
    const synced = await api.put(url(), {
      models: [row("deepseek-v4-flash-0731"), row("deepseek-v4-pro")],
    });
    expect(synced.status).toBe(200);
    const body = (await synced.json()) as ModelsResponse;
    const stored = pick(body, "tokendance", "deepseek-v4-flash-0731").credential?.apiKeyMasked;
    expect(stored).toBeTruthy();
    expect(pick(body, "tokendance", "deepseek-v4-pro").credential?.apiKeyMasked).toBe(stored);
    // Borrowed, never echoed: the plaintext stays on disk.
    expect(JSON.stringify(body)).not.toContain("sk-group-gateway-1");

    const cfgRaw = await readFile(cfgFile, "utf8");
    expect(cfgRaw.split('api_key = "sk-group-gateway-1"').length - 1).toBe(2);

    // The shape the rule exists for: a whole gateway group, the key pasted on one row of it.
    // One endpoint behind one login, so every preset the group carries is answered by that key.
    const group = MODEL_CATALOG.filter((m) => m.provider === "tokendance" && m.retired !== true);
    expect(group.length).toBeGreaterThan(4);
    const wholeGroup = await api.put(url(), {
      models: group.map((m, i) => ({
        provider: m.provider,
        modelId: m.modelId,
        ...(m.clientType !== undefined ? { clientType: m.clientType } : {}),
        ...(m.baseUrl !== undefined ? { baseUrl: m.baseUrl } : {}),
        ...(i === 0 ? { apiKey: "sk-group-gateway-1" } : {}),
      })),
    });
    expect(wholeGroup.status).toBe(200);
    const groupBody = (await wholeGroup.json()) as ModelsResponse;
    const firstMask = pick(groupBody, group[0]!.provider, group[0]!.modelId).credential
      ?.apiKeyMasked;
    expect(firstMask).toBeTruthy();
    for (const m of group) {
      expect(pick(groupBody, m.provider, m.modelId).credential?.apiKeyMasked, m.modelId).toBe(
        firstMask,
      );
    }
    const groupRaw = await readFile(cfgFile, "utf8");
    expect(groupRaw.split('api_key = "sk-group-gateway-1"').length - 1).toBe(group.length);
  });

  it("a group holding two different keys is left to decide: the new row arrives keyless", async () => {
    const GATEWAY = "https://tokendance.space/gateway/v1";
    const row = (modelId: string, extra: Record<string, unknown> = {}) => ({
      provider: "tokendance",
      modelId,
      clientType: "openai-chat",
      baseUrl: GATEWAY,
      ...extra,
    });
    const twoKeys = await api.put(url(), {
      models: [
        row("deepseek-v4-flash-0731", { apiKey: "sk-first-account" }),
        row("deepseek-v4-pro", { apiKey: "sk-second-account" }),
      ],
    });
    expect(twoKeys.status).toBe(200);

    const added = await api.put(url(), {
      models: [row("deepseek-v4-flash-0731"), row("deepseek-v4-pro"), row("deepseek-v4.1-flash")],
    });
    expect(added.status).toBe(200);
    // Two accounts under one group is a deliberate state: picking one of them would be a guess.
    expect(
      pick((await added.json()) as ModelsResponse, "tokendance", "deepseek-v4.1-flash").credential
        ?.apiKeyMasked,
    ).toBeUndefined();
  });

  it("a row the request clears stays keyless: a clear is not filled back in by the row beside it", async () => {
    const GATEWAY = "https://tokendance.space/gateway/v1";
    const row = (modelId: string, extra: Record<string, unknown> = {}) => ({
      provider: "tokendance",
      modelId,
      clientType: "openai-chat",
      baseUrl: GATEWAY,
      ...extra,
    });
    await api.put(url(), {
      models: [
        row("deepseek-v4-flash-0731", { apiKey: "sk-group-gateway-1" }),
        row("deepseek-v4-pro"),
      ],
    });
    // Both rows do have the key by now (the second borrowed it) …
    const before = (await (await api.get(url())).json()) as ModelsResponse;
    expect(pick(before, "tokendance", "deepseek-v4-pro").credential?.apiKeyMasked).toBeTruthy();

    // … and taking one of them back is what keeps it keyless.
    const cleared = await api.put(url(), {
      models: [row("deepseek-v4-flash-0731"), row("deepseek-v4-pro", { clearApiKey: true })],
    });
    expect(cleared.status).toBe(200);
    const body = (await cleared.json()) as ModelsResponse;
    expect(pick(body, "tokendance", "deepseek-v4-pro").credential?.apiKeyMasked).toBeUndefined();
    expect(
      pick(body, "tokendance", "deepseek-v4-flash-0731").credential?.apiKeyMasked,
    ).toBeTruthy();
  });

  it("a preset pointed at another endpoint is another service: nothing is borrowed across them", async () => {
    const GATEWAY = "https://tokendance.space/gateway/v1";
    const row = (modelId: string, extra: Record<string, unknown> = {}) => ({
      provider: "tokendance",
      modelId,
      clientType: "openai-chat",
      ...extra,
    });
    await api.put(url(), {
      models: [row("deepseek-v4-flash-0731", { apiKey: "sk-gateway-account", baseUrl: GATEWAY })],
    });

    // The next preset is answered by the user's own relay: the gateway key is not its key.
    const relayed = await api.put(url(), {
      models: [
        row("deepseek-v4-flash-0731", { baseUrl: GATEWAY }),
        row("deepseek-v4-pro", { baseUrl: "https://relay.example/v1" }),
      ],
    });
    expect(relayed.status).toBe(200);
    const body = (await relayed.json()) as ModelsResponse;
    expect(pick(body, "tokendance", "deepseek-v4-pro").credential?.apiKeyMasked).toBeUndefined();
    expect(
      pick(body, "tokendance", "deepseek-v4-flash-0731").credential?.apiKeyMasked,
    ).toBeTruthy();

    // A row added by hand is the user's own entry: replacing it without its key has always meant
    // the credential does not migrate, and the rows beside it do not change that.
    const custom = (modelId: string, baseUrl: string, extra: Record<string, unknown> = {}) => ({
      provider: "custom",
      modelId,
      clientType: "openai-chat",
      baseUrl,
      ...extra,
    });
    await api.put(url(), {
      models: [custom("self-hosted-a", "https://a.example/v1", { apiKey: "sk-service-a" })],
    });
    const plain = await api.put(url(), {
      models: [
        custom("self-hosted-a", "https://a.example/v1"),
        custom("self-hosted-b", "https://a.example/v1"),
      ],
    });
    expect(plain.status).toBe(200);
    const plainBody = (await plain.json()) as ModelsResponse;
    expect(pick(plainBody, "custom", "self-hosted-b").credential?.apiKeyMasked).toBeUndefined();
    // The row that carried the key keeps it on its own account, base URL and all.
    expect(pick(plainBody, "custom", "self-hosted-a").credential?.apiKeyMasked).toBeTruthy();
  });

  it("a new Project is preset with every built-in model (provider and model_id as separate fields + catalog info)", async () => {
    const res = await api.get(url());
    expect(res.status).toBe(200);
    const body = (await res.json()) as ModelsResponse;
    expect(body.defaultModel).toEqual({
      provider: "deepseek",
      modelId: "deepseek-flash",
    });
    expect(body.models.map(pairKey)).toEqual(catalogPairs);
    // A retired catalog row is kept only for the Projects that already carry it, so a new
    // Project gets none of them.
    const retired = MODEL_CATALOG.filter((m) => m.retired === true).map(pairKey);
    expect(retired.length).toBeGreaterThan(0);
    expect(body.models.map(pairKey).filter((key) => retired.includes(key))).toEqual([]);

    // Every row reads its display name, context window, list price, vision and pinned
    // protocol and endpoint off its catalog row; nothing of the vendor's is copied here.
    for (const row of body.models) {
      const entry = catalogEntryFor(row.provider, row.modelId)!;
      const key = `${row.provider}/${row.modelId}`;
      expect(row.displayName, key).toBe(entry.displayName);
      expect(row.contextWindow, key).toBe(entry.contextWindow);
      expect(row.vision, key).toBe(entry.supportsVision);
      expect(row.clientType, key).toBe(entry.clientType);
      expect(row.credential?.baseUrl, key).toBe(entry.baseUrl);
      // A preset carries no secret.
      expect(row.credential?.apiKeyMasked, key).toBeUndefined();
    }

    // A vendor's own row falls back to its client's environment variable; it has no stored
    // credential and no client_type (AgentHub routes it by its upstream id).
    const sonnet = pick(body, "anthropic", "claude-sonnet-4-6");
    expect(sonnet.isDefault).toBe(false);
    expect(sonnet.envKey).toBe("ANTHROPIC_API_KEY");
    expect(sonnet.credential).toBeUndefined();
    expect(sonnet.clientType).toBeUndefined();
    expect(pick(body, "deepseek", "deepseek-v4-pro").envKey).toBe("DEEPSEEK_API_KEY");
    expect(pick(body, "deepseek", "deepseek-flash").isDefault).toBe(true);

    // A gateway pins each row's protocol and endpoint, so one group can hold rows on different
    // clients (OpenCode Go: a Messages row and a Chat Completions row). Like every gateway's
    // rows, neither falls back to its client's environment variable — that holds the user's
    // vendor key, and the gateway is not the vendor. An upstream id with a `/` is a plain
    // string under column storage.
    const messagesRow = pick(body, "opencode-go", "qwen3.8-max");
    const chatRow = pick(body, "opencode-go", "glm-5.3");
    expect(messagesRow.clientType).toBe("ant-messages");
    expect(chatRow.clientType).toBe("openai-chat");
    expect(messagesRow.envKey).toBeUndefined();
    expect(chatRow.envKey).toBeUndefined();
    expect(pick(body, "openrouter", "xiaomi/mimo-v2.5").credential?.baseUrl).toBeDefined();

    // A catalog row on a promotion is preset at its LIST price like every other row, and the
    // promotion is seeded beside it in web.db: the rows reporting a discount are exactly the
    // catalog's flat promotions.
    const promoted = catalogEntryFor("tokendance", "glm-5.3-flash")!;
    expect(pick(body, "tokendance", "glm-5.3-flash")).toMatchObject({
      pricing: {
        cacheRead: promoted.pricing!.cache_read,
        cacheWrite: promoted.pricing!.cache_write,
        output: promoted.pricing!.output,
      },
      discount: promoted.discount,
    });
    expect(
      body.models
        .filter((m) => m.discount !== undefined)
        .map((m) => ({ provider: m.provider, modelId: m.modelId, discount: m.discount })),
    ).toEqual(presetPromotions());
    // An undiscounted row is preset at its list price, unchanged, and reports no discount.
    const plain = catalogEntryFor("tokendance", "hy4-preview")!;
    expect(pick(body, "tokendance", "hy4-preview").pricing).toEqual({
      cacheRead: plain.pricing!.cache_read,
      cacheWrite: plain.pricing!.cache_write,
      output: plain.pricing!.output,
    });
    expect(pick(body, "tokendance", "hy4-preview")).not.toHaveProperty("discount");
  });

  it("reports and masks the env fallback only where the entry is allowed one (the vendor's own endpoint, or the relay's own variable), and never leaks a value", async () => {
    const saved = {
      anthropic: process.env.ANTHROPIC_API_KEY,
      deepseek: process.env.DEEPSEEK_API_KEY,
      openai: process.env.OPENAI_API_KEY,
      penguin: process.env.PENGUIN_GO_API_KEY,
    };
    const anthropicValue = "sk-ant-test-secret-value-123456";
    const openaiValue = "sk-openai-test-secret-value-789";
    process.env.ANTHROPIC_API_KEY = anthropicValue;
    process.env.OPENAI_API_KEY = openaiValue;
    process.env.PENGUIN_GO_API_KEY = "sk-penguin-test-secret-value-456";
    // Empty counts as absent — it would not authenticate either.
    process.env.DEEPSEEK_API_KEY = "";
    // Two vendor-group rows no PUT may add any more, stored as a table from before would hold them.
    await storeRows(t.root, projectId, [
      { provider: "anthropic", modelId: "claude-sonnet-4-6-preview" },
      { provider: "anthropic", modelId: "claude-via-gateway", clientType: "openai" },
    ]);
    try {
      const put = await api.put(url(), {
        defaultModel: { provider: "anthropic", modelId: "claude-sonnet-4-6" },
        models: [
          // First-party catalog preset, variable set: masked preview.
          { provider: "anthropic", modelId: "claude-sonnet-4-6" },
          // First-party catalog preset, variable empty: no preview.
          { provider: "deepseek", modelId: "deepseek-v4-pro" },
          // Off-catalog id in a vendor group, pure auto-route: still first-party.
          { provider: "anthropic", modelId: "claude-sonnet-4-6-preview" },
          // Vendor group on a generic protocol with no base URL: the OpenAI client talks to
          // api.openai.com, so the OpenAI key may back it — whatever the group is called.
          { provider: "anthropic", modelId: "claude-via-gateway", clientType: "openai" },
          // A gateway row carries the gateway's endpoint: no fallback at all, so no envKey
          // and no preview, even with OPENAI_API_KEY set.
          {
            provider: "openrouter",
            modelId: "xiaomi/mimo-v2.5",
            clientType: "openai-chat",
            baseUrl: "https://openrouter.ai/api/v1",
          },
          // Custom rows are judged by their endpoint: none → the vendor's own (a fallback, but
          // not a preview — see modelEnvPreviewKey); a private server → nothing.
          { provider: "custom", modelId: "my-model", clientType: "openai" },
          {
            provider: "custom",
            modelId: "my-local",
            clientType: "openai",
            baseUrl: "http://127.0.0.1:8000/v1",
          },
          // A vLLM preset as seeded: no base URL, the vLLM adapter pinned. It falls back to
          // OPENAI_API_KEY under the rule, but showing the mask would list a self-hosted id in
          // the chat picker as configured and send the OpenAI key to api.openai.com.
          { provider: "vllm", modelId: "Qwen/Qwen3.8-27B", clientType: "openai-chat-vllm-adapter" },
          // Both protocols in the relay group resolve the same provider-scoped key.
          {
            provider: "penguin-go",
            modelId: "gemini-3.8-flash",
            clientType: "gemini-3.8",
            baseUrl: PENGUIN_GO_BASE_URL,
          },
          {
            provider: "penguin-go",
            modelId: "deepseek-flash",
            clientType: "deepseek-v4",
            baseUrl: PENGUIN_GO_BASE_URL,
          },
        ],
      });
      expect(put.status).toBe(200);
      // The masked preview follows maskApiKey; the plaintext must never be serialized.
      const text = await put.text();
      expect(text).not.toContain(anthropicValue);
      expect(text).not.toContain(openaiValue);
      const body = JSON.parse(text) as ModelsResponse;
      const masked = `${anthropicValue.slice(0, 4)}…${anthropicValue.slice(-4)}`;
      expect(pick(body, "anthropic", "claude-sonnet-4-6").envKeyMasked).toBe(masked);
      expect(pick(body, "anthropic", "claude-sonnet-4-6-preview").envKeyMasked).toBe(masked);
      expect(pick(body, "deepseek", "deepseek-v4-pro").envKeyMasked).toBeUndefined();
      const openaiMasked = `${openaiValue.slice(0, 4)}…${openaiValue.slice(-4)}`;
      const rePointed = pick(body, "anthropic", "claude-via-gateway");
      expect(rePointed.envKey).toBe("OPENAI_API_KEY");
      expect(rePointed.envKeyMasked).toBe(openaiMasked);
      const gateway = pick(body, "openrouter", "xiaomi/mimo-v2.5");
      expect(gateway.envKey).toBeUndefined();
      expect(gateway.envKeyMasked).toBeUndefined();
      expect(pick(body, "custom", "my-model").envKey).toBe("OPENAI_API_KEY");
      expect(pick(body, "custom", "my-model").envKeyMasked).toBeUndefined();
      expect(pick(body, "custom", "my-local").envKey).toBeUndefined();
      expect(pick(body, "custom", "my-local").envKeyMasked).toBeUndefined();
      const vllm = pick(body, "vllm", "Qwen/Qwen3.8-27B");
      expect(vllm.envKey).toBe("OPENAI_API_KEY");
      expect(vllm.envKeyMasked).toBeUndefined();
      const penguinMasked = "sk-p…-456";
      expect(pick(body, "penguin-go", "gemini-3.8-flash").envKey).toBe("PENGUIN_GO_API_KEY");
      expect(pick(body, "penguin-go", "gemini-3.8-flash").envKeyMasked).toBe(penguinMasked);
      expect(pick(body, "penguin-go", "deepseek-flash").envKey).toBe("PENGUIN_GO_API_KEY");
      expect(pick(body, "penguin-go", "deepseek-flash").envKeyMasked).toBe(penguinMasked);
    } finally {
      for (const [key, value] of [
        ["ANTHROPIC_API_KEY", saved.anthropic],
        ["DEEPSEEK_API_KEY", saved.deepseek],
        ["OPENAI_API_KEY", saved.openai],
        ["PENGUIN_GO_API_KEY", saved.penguin],
      ] as const) {
        if (value === undefined) delete process.env[key];
        else process.env[key] = value;
      }
    }
  });

  it("PUT a custom model: the vision flag persists; the openai protocol falls back to OPENAI_API_KEY; provider is required", async () => {
    // A vendor-group row on the openai protocol, as a group header's add once wrote it.
    await storeRows(t.root, projectId, [
      { provider: "anthropic", modelId: "claude-via-gateway", clientType: "openai" },
    ]);
    const put = await api.put(url(), {
      defaultModel: { provider: "custom", modelId: "my-model" },
      models: [
        { provider: "custom", modelId: "my-model", clientType: "openai", vision: false },
        { provider: "custom", modelId: "opaque-model" },
        { provider: "anthropic", modelId: "claude-via-gateway", clientType: "openai" },
      ],
    });
    expect(put.status).toBe(200);
    const body = (await put.json()) as ModelsResponse;

    const mine = pick(body, "custom", "my-model");
    expect(mine.displayName).toBeUndefined();
    expect(mine.vision).toBe(false);
    expect(mine.envKey).toBe("OPENAI_API_KEY");
    // The request carried the deprecated bare "openai" alias (pre-0.4.2 clients/configs):
    // it is normalized to the canonical "openai-chat" on write and reported canonically.
    expect(mine.clientType).toBe("openai-chat");

    // Off-catalog model without client_type: no env-var fallback; with no vision annotation the field is omitted (default = supported).
    const opaque = pick(body, "custom", "opaque-model");
    expect("vision" in opaque).toBe(false);
    expect(opaque.envKey).toBeUndefined();

    // Listed under one vendor's group but using the openai protocol (a model added at a group header):
    // AgentHub's openai client actually reads OPENAI_API_KEY, so the env fallback reports that, not the vendor's var name.
    expect(pick(body, "anthropic", "claude-via-gateway").envKey).toBe("OPENAI_API_KEY");

    // GET again: vision was persisted (not just echoed from the request body), and the disk
    // stores the canonical client-type spelling.
    const again = (await (await api.get(url())).json()) as ModelsResponse;
    expect(pick(again, "custom", "my-model").vision).toBe(false);
    expect(pick(again, "custom", "my-model").clientType).toBe("openai-chat");
    const toml = await readFile(path.join(t.root, projectId, ".project_config.toml"), "utf8");
    expect(toml).toContain('client_type = "openai-chat"');
    expect(toml).not.toContain('"openai"');

    // vision shape check: non-boolean → 400.
    const bad = await api.put(url(), {
      models: [{ provider: "custom", modelId: "my-model", vision: "yes" }],
    });
    expect(bad.status).toBe(400);

    // Missing provider → 400 (refs are always a pair; neither half may be omitted).
    const noProvider = await api.put(url(), { models: [{ modelId: "my-model" }] });
    expect(noProvider.status).toBe(400);
  });

  it("PUT refuses an id a vendor group cannot route when the request introduces it, and carries a stored one through", async () => {
    const cfgFile = path.join(t.root, projectId, ".project_config.toml");
    // A row in the shape a vendor-group add used to produce: no client_type, and an id
    // AgentHub places nowhere, so it fails at request time with its "is not supported"
    // sentence. Rows like this exist in configs written before the rule below.
    await writeFile(
      cfgFile,
      ["[[models]]", 'provider = "deepseek"', 'model_id = "qwen/qwen3.8-flash-next"'].join("\n"),
      "utf8",
    );

    // Already stored: the whole-table PUT writes it back untouched. The models page sends the
    // entire table on every save, so refusing it here would make one legacy row block every
    // later edit of every other row.
    const kept = await api.put(url(), {
      models: [
        { provider: "deepseek", modelId: "qwen/qwen3.8-flash-next", contextWindow: 65536 },
        { provider: "custom", modelId: "mine", clientType: "openai-chat" },
      ],
    });
    expect(kept.status).toBe(200);
    const keptBody = (await kept.json()) as ModelsResponse;
    expect(pick(keptBody, "deepseek", "qwen/qwen3.8-flash-next").contextWindow).toBe(65536);

    // New in this request: refused, by a code the frontend can localize and a message naming
    // the entry and the way out.
    const added = await api.put(url(), {
      models: [
        { provider: "deepseek", modelId: "qwen/qwen3.8-flash-next" },
        { provider: "deepseek", modelId: "another-fine-tune" },
      ],
    });
    expect(added.status).toBe(400);
    const error = ((await added.json()) as ErrorBody).error;
    expect(error.code).toBe("model_not_routable");
    expect(error.message).toContain("another-fine-tune");
    expect(error.message).toContain("custom group");

    // A changed id is a different entry, so the rename is judged like a new one.
    const renamed = await api.put(url(), {
      models: [
        {
          provider: "deepseek",
          modelId: "qwen/qwen3.8-flash-next-preview",
          renamedFrom: { provider: "deepseek", modelId: "qwen/qwen3.8-flash-next" },
        },
      ],
    });
    expect(renamed.status).toBe(400);
    expect(((await renamed.json()) as ErrorBody).error.code).toBe("model_not_routable");

    // So is moving a model into a vendor group: the group is the half of the key that changed.
    const moved = await api.put(url(), {
      models: [
        { provider: "deepseek", modelId: "qwen/qwen3.8-flash-next" },
        {
          provider: "deepseek",
          modelId: "mine",
          renamedFrom: { provider: "custom", modelId: "mine" },
        },
      ],
    });
    expect(moved.status).toBe(400);

    // Nothing above was written: the stored table is still the one the accepted PUT left.
    const stored = (await (await api.get(url())).json()) as ModelsResponse;
    const expected = ["custom\0mine", "deepseek\0qwen/qwen3.8-flash-next"];
    expect(stored.models.map(pairKey).sort()).toEqual(expected.sort());

    // The same id is accepted wherever the group answers the protocol question itself.
    const custom = await api.put(url(), {
      models: [
        { provider: "deepseek", modelId: "qwen/qwen3.8-flash-next" },
        {
          provider: "custom",
          modelId: "qwen/qwen3.8-flash-next",
          clientType: "openai-chat",
          baseUrl: "https://gateway.example/v1",
        },
      ],
    });
    expect(custom.status).toBe(200);
  });

  it("PUT refuses a new row that is no preset in a built-in group but custom and vLLM, and keeps every stored one", async () => {
    // A row a user added to a gateway group while gateways still took them.
    await storeRows(t.root, projectId, [
      { provider: "openrouter", modelId: "acme/legacy-pick", clientType: "openai-responses" },
      { provider: "custom", modelId: "mine", clientType: "openai-chat" },
    ]);
    const legacy = {
      provider: "openrouter",
      modelId: "acme/legacy-pick",
      clientType: "openai-responses",
    };
    const mine = { provider: "custom", modelId: "mine", clientType: "openai-chat" };

    // New in this request, in two gateways and in a vendor group — the last one an id the
    // vendor's client would route, so it is this rule and not routing that answers.
    for (const entry of [
      { provider: "openrouter", modelId: "acme/new-pick", clientType: "openai-responses" },
      { provider: "tokendance", modelId: "acme/new-pick", clientType: "openai-chat" },
      { provider: "deepseek", modelId: "deepseek-v4-my-tune" },
    ]) {
      const res = await api.put(url(), { models: [legacy, mine, entry] });
      expect(res.status, entry.provider).toBe(400);
      const error = ((await res.json()) as ErrorBody).error;
      expect(error.code, entry.provider).toBe("model_not_addable");
      expect(error.message).toContain(entry.modelId);
      expect(error.message).toContain("custom group");
    }

    // Moving a row in from another group is adding it to this one.
    const moved = await api.put(url(), {
      models: [
        legacy,
        {
          provider: "openrouter",
          modelId: "mine",
          clientType: "openai-responses",
          renamedFrom: { provider: "custom", modelId: "mine" },
        },
      ],
    });
    expect(moved.status).toBe(400);
    expect(((await moved.json()) as ErrorBody).error.code).toBe("model_not_addable");

    // Nothing above was written.
    const before = (await (await api.get(url())).json()) as ModelsResponse;
    expect(before.models.map(pairKey).sort()).toEqual(
      ["custom\0mine", "openrouter\0acme/legacy-pick"].sort(),
    );

    // The stored row keeps working: edited in place, then renamed inside its own group.
    const edited = await api.put(url(), { models: [{ ...legacy, contextWindow: 65536 }, mine] });
    expect(edited.status).toBe(200);
    const renamed = await api.put(url(), {
      models: [
        {
          provider: "openrouter",
          modelId: "acme/renamed-pick",
          clientType: "openai-responses",
          renamedFrom: { provider: "openrouter", modelId: "acme/legacy-pick" },
        },
        mine,
      ],
    });
    expect(renamed.status).toBe(200);
    const after = (await renamed.json()) as ModelsResponse;
    expect(after.models.map(pairKey).sort()).toEqual(
      ["custom\0mine", "openrouter\0acme/renamed-pick"].sort(),
    );

    // A group's own preset is welcome back, and custom, vLLM and user-defined groups take
    // anything.
    const open = await api.put(url(), {
      models: [
        { provider: "openrouter", modelId: "acme/renamed-pick", clientType: "openai-responses" },
        mine,
        { provider: "deepseek", modelId: "deepseek-v4-pro" },
        {
          provider: "vllm",
          modelId: "my-served-model",
          clientType: "openai-chat-vllm-adapter",
          baseUrl: "http://127.0.0.1:8000/v1",
        },
        { provider: "my-own-group", modelId: "anything", clientType: "openai-chat" },
      ],
    });
    expect(open.status).toBe(200);
  });

  it('a config stored before the AgentHub 0.4.2 rename (client_type = "openai") keeps working: GET reports the canonical openai-chat', async () => {
    // Simulate an existing user config written by an older harness version: the deprecated
    // bare "openai" spelling on disk. Reading must not error and must report the canonical
    // spelling (normalize-on-read, no disk rewrite needed until the next PUT).
    const cfgFile = path.join(t.root, projectId, ".project_config.toml");
    await writeFile(
      cfgFile,
      [
        "[[models]]",
        'provider = "custom"',
        'model_id = "legacy-openai-model"',
        'client_type = "openai"',
        'base_url = "https://legacy.example/v1"',
      ].join("\n"),
      "utf8",
    );
    const body = (await (await api.get(url())).json()) as ModelsResponse;
    const legacy = pick(body, "custom", "legacy-openai-model");
    expect(legacy.clientType).toBe("openai-chat");
    // Its own endpoint, so no environment fallback is reported for it.
    expect(legacy.envKey).toBeUndefined();
  });

  it("a configured model that has since been dropped from the built-in catalog still loads, keeps its data, and stays usable", async () => {
    // Migration guard for catalog removals (the 2026-08-18 inclusionai/ling-3.0-flash:free
    // delisting is the live example): a user who configured the preset before it was removed
    // keeps a row on disk that catalogEntryFor no longer matches. Everything presetModelEntries
    // persisted must survive — only display_name lived solely in the catalog — and the row must
    // stay a normal, selectable entry rather than being pruned, rejected or blanked.
    const cfgFile = path.join(t.root, projectId, ".project_config.toml");
    await writeFile(
      cfgFile,
      [
        "[[models]]",
        'provider = "openrouter"',
        'model_id = "vendor/removed-from-catalog:free"',
        "context_window = 262144",
        'client_type = "openai-chat"',
        'base_url = "https://openrouter.ai/api/v1"',
        'api_key = "sk-still-here"',
        "vision = false",
        "",
        "[models.pricing]",
        'unit = "usd_per_mtok"',
        "cache_read = 0.0",
        "cache_write = 0.0",
        "output = 0.0",
      ].join("\n"),
      "utf8",
    );
    // Sanity: the id really is absent from the built-in catalog, so this exercises the
    // off-catalog path rather than an accidental match.
    expect(
      MODEL_CATALOG.some(
        (m) => m.provider === "openrouter" && m.modelId === "vendor/removed-from-catalog:free",
      ),
    ).toBe(false);

    const body = (await (await api.get(url())).json()) as ModelsResponse;
    const orphan = pick(body, "openrouter", "vendor/removed-from-catalog:free");
    // Everything stored in TOML is read straight back — pricing and context window come from
    // the file, not the catalog, so a $0 free-tier row keeps costing 0 rather than going
    // "unknown".
    expect(orphan.contextWindow).toBe(262144);
    expect(orphan.pricing).toEqual({ cacheRead: 0, cacheWrite: 0, output: 0 });
    expect(orphan.vision).toBe(false);
    expect(orphan.clientType).toBe("openai-chat");
    // It still carries the gateway's endpoint, so it gets no environment fallback.
    expect(orphan.envKey).toBeUndefined();
    // The credential survives, masked; the base URL is still inlined on the entry.
    expect(orphan.credential?.baseUrl).toBe("https://openrouter.ai/api/v1");
    expect(orphan.credential?.apiKeyMasked).toBeTruthy();
    // The one real loss: display_name only ever lived in the catalog, so the UI falls back to
    // the raw upstream id.
    expect(orphan.displayName).toBeUndefined();

    // Still a legal default-model target — nothing validates the pair against the catalog.
    const put = await api.put(url(), {
      models: [
        {
          provider: "openrouter",
          modelId: "vendor/removed-from-catalog:free",
          contextWindow: 262144,
          clientType: "openai-chat",
        },
      ],
      defaultModel: { provider: "openrouter", modelId: "vendor/removed-from-catalog:free" },
    });
    expect(put.status).toBe(200);
    const after = (await (await api.get(url())).json()) as ModelsResponse;
    expect(pick(after, "openrouter", "vendor/removed-from-catalog:free").isDefault).toBe(true);
  });

  it("PUT maxTokens: persisted as max_tokens and read back through GET; omitting it table-wide clears it; 0 / negative / non-numeric 400", async () => {
    const put = await api.put(url(), {
      models: [
        { provider: "custom", modelId: "local-qwen", clientType: "openai", maxTokens: 8000 },
      ],
    });
    expect(put.status).toBe(200);
    expect(pick((await put.json()) as ModelsResponse, "custom", "local-qwen").maxTokens).toBe(8000);

    // Round-trips through disk (persisted as snake_case on the entry, not just echoed back).
    const again = (await (await api.get(url())).json()) as ModelsResponse;
    expect(pick(again, "custom", "local-qwen").maxTokens).toBe(8000);
    const toml = await readFile(path.join(t.root, projectId, ".project_config.toml"), "utf8");
    expect(toml).toContain("max_tokens = 8000");

    // Full-table PUT omitting the field clears the annotation (same replace semantics as vision/contextWindow).
    const cleared = await api.put(url(), {
      models: [{ provider: "custom", modelId: "local-qwen", clientType: "openai" }],
    });
    expect(cleared.status).toBe(200);
    const clearedRow = pick((await cleared.json()) as ModelsResponse, "custom", "local-qwen");
    expect("maxTokens" in clearedRow).toBe(false);
    expect(
      await readFile(path.join(t.root, projectId, ".project_config.toml"), "utf8"),
    ).not.toContain("max_tokens");

    // Not a positive integer → 400 with the field-labelled message (nothing written).
    for (const bad of [0, -5, 1.5, "8000"]) {
      const res = await api.put(url(), {
        models: [{ provider: "custom", modelId: "local-qwen", maxTokens: bad }],
      });
      expect(res.status).toBe(400);
      const body = (await res.json()) as { error: { message: string } };
      expect(body.error.message).toContain("models[0].maxTokens");
    }
  });

  it("the same model_id can coexist under different providers (paired keys, neither overwrites the other)", async () => {
    // The gateway's copy was added by hand, before gateways stopped taking such rows.
    await storeRows(t.root, projectId, [
      { provider: "siliconflow", modelId: "kimi-k2.6", clientType: "openai" },
    ]);
    const put = await api.put(url(), {
      models: [
        { provider: "moonshot", modelId: "kimi-k2.6", apiKey: "sk-official-aaaa1111" },
        {
          provider: "siliconflow",
          modelId: "kimi-k2.6",
          clientType: "openai",
          apiKey: "sk-gateway-bbbb2222",
        },
      ],
    });
    expect(put.status).toBe(200);
    const body = (await put.json()) as ModelsResponse;
    expect(body.models).toHaveLength(2);
    // The two entries are independent: credential and envKey don't cross over.
    expect(pick(body, "moonshot", "kimi-k2.6").credential?.apiKeyMasked).toBe("sk-o…1111");
    expect(pick(body, "moonshot", "kimi-k2.6").envKey).toBe("MOONSHOT_API_KEY");
    expect(pick(body, "siliconflow", "kimi-k2.6").credential?.apiKeyMasked).toBe("sk-g…2222");
    // Saved without the gateway's base URL, this row's client talks to api.openai.com — so
    // the OpenAI variable honestly is its fallback (the preset, which carries the gateway
    // endpoint, gets none; see the env-fallback test above).
    expect(pick(body, "siliconflow", "kimi-k2.6").envKey).toBe("OPENAI_API_KEY");

    // Round-trips through disk unchanged.
    const again = (await (await api.get(url())).json()) as ModelsResponse;
    expect(again.models.map(pairKey).sort()).toEqual(body.models.map(pairKey).sort());
  });
});

describe("default_project presets", () => {
  let t: TestApp;
  let prevKey: string | undefined;

  beforeEach(() => {
    // The default model (DeepSeek) uses the OpenAI protocol, whose SDK requires a credential at
    // **construction time** — this case creates a Session, so we stuff in a fake key (no real request is sent). CI has no keys.
    prevKey = process.env.OPENAI_API_KEY;
    process.env.OPENAI_API_KEY = "test-key-not-used";
  });
  afterEach(async () => {
    if (prevKey === undefined) delete process.env.OPENAI_API_KEY;
    else process.env.OPENAI_API_KEY = prevKey;
    await t.cleanup();
  });

  it("when the seeded admin adopts default_project, the preset models and the default model are filled in (no prior .project_config.toml)", async () => {
    // The default_project shared by admin seeding and the CLI (the dir already exists, so writeInitialConfig is skipped).
    t = await createTestApp();
    const { cookie } = await loginAdmin(t.app);
    const api = apiClient(t.app, cookie);
    const res = await api.get("/api/projects/default_project/models");
    expect(res.status).toBe(200);
    const body = (await res.json()) as ModelsResponse;
    expect(body.defaultModel).toEqual({
      provider: "deepseek",
      modelId: "deepseek-flash",
    });
    expect(body.models.map(pairKey)).toEqual(catalogPairs);
    // Presets backfilled here bring their promotions too, seeded once the Project row exists.
    expect(body.models.filter((m) => m.discount !== undefined)).toHaveLength(
      presetPromotions().length,
    );

    // The point of presets is "works out of the box": creating a Session should succeed without passing a model ref.
    const created = await api.post(
      "/api/projects/default_project/agents/default_agent/sessions",
      {},
    );
    expect(created.status).toBe(201);
    const { session } = (await created.json()) as SessionCreateResponse;
    expect(session.provider).toBe("deepseek");
    expect(session.modelId).toBe("deepseek-flash");
  });

  it("a default_project that already has models configured is left untouched (existing CLI config is not overwritten)", async () => {
    // First have the "CLI" write a config with a single custom model, then admin seeding adopts it.
    t = await createTestApp({
      beforeSeed: async (root) => {
        await wire(ProjectConfigService, { paths: { root } }).writeRaw("default_project", {
          default_model: { provider: "custom", model_id: "cli-model" },
          models: [{ provider: "custom", model_id: "cli-model", context_window: 1234 }],
        });
      },
    });
    const { cookie } = await loginAdmin(t.app);
    const api = apiClient(t.app, cookie);
    const body = (await (
      await api.get("/api/projects/default_project/models")
    ).json()) as ModelsResponse;
    expect(body.defaultModel).toEqual({ provider: "custom", modelId: "cli-model" });
    expect(body.models.map(pairKey)).toEqual(["custom\0cli-model"]);
    expect(body.models[0]!.contextWindow).toBe(1234);
  });
});

describe("model-reference rekeying and the connectivity test", () => {
  let t: TestApp;
  let api: ReturnType<typeof apiClient>;
  let projectId: string;
  const url = () => `/api/projects/${projectId}/models`;
  const testUrl = () => `${url()}/test`;

  beforeAll(async () => {
    t = await createTestApp();
    const { cookie } = await provisionUser(t.app, "carol");
    api = apiClient(t.app, cookie);
  });
  afterAll(async () => {
    await t.cleanup();
  });
  let projects = 0;
  beforeEach(async () => {
    projects += 1;
    const created = (await (
      await api.post("/api/projects", {
        projectId: `carol-rename_${projects}`,
        name: "Rename project",
      })
    ).json()) as ProjectCreateResponse;
    projectId = created.project.projectId;
  });

  it("renamedFrom migrates the credential and the config; the default / vision model pointers follow the rekey", async () => {
    await api.put(url(), {
      defaultModel: { provider: "custom", modelId: "old-id" },
      visionModel: { provider: "custom", modelId: "old-id" },
      models: [
        {
          provider: "custom",
          modelId: "old-id",
          contextWindow: 4096,
          apiKey: "sk-secret-abcd1234",
        },
      ],
    });

    const put = await api.put(url(), {
      models: [
        {
          provider: "custom",
          modelId: "new-id",
          renamedFrom: { provider: "custom", modelId: "old-id" },
          contextWindow: 4096,
        },
      ],
    });
    expect(put.status).toBe(200);
    const body = (await put.json()) as ModelsResponse;
    expect(body.models.map(pairKey)).toEqual(["custom\0new-id"]);
    // The credential migrates with the key change (masked value visible), and the pointer follows the new ref.
    expect(body.models[0]!.credential?.apiKeyMasked).toBe("sk-s…1234");
    expect(body.defaultModel).toEqual({ provider: "custom", modelId: "new-id" });
    expect(body.visionModel).toEqual({ provider: "custom", modelId: "new-id" });
    // Round-trips through disk unchanged (not just echoed).
    const again = (await (await api.get(url())).json()) as ModelsResponse;
    expect(again.models[0]!.credential?.apiKeyMasked).toBe("sk-s…1234");
    expect(again.defaultModel).toEqual({ provider: "custom", modelId: "new-id" });
  });

  it("a group change is a rekey too: the credential migrates with it; envKey resolves by client, not by group (PRN-021)", async () => {
    // Move the preset DeepSeek model to another group (changing provider is a key change; the paired renamedFrom migrates it).
    const put = await api.put(url(), {
      models: [
        {
          provider: "deepseek",
          modelId: "deepseek-v4-pro",
          apiKey: "sk-secret-abcd1234",
          vision: false,
        },
      ],
    });
    expect(put.status).toBe(200);
    expect(pick((await put.json()) as ModelsResponse, "deepseek", "deepseek-v4-pro").envKey).toBe(
      "DEEPSEEK_API_KEY",
    );

    const put2 = await api.put(url(), {
      models: [
        {
          provider: "custom",
          modelId: "deepseek-v4-pro",
          renamedFrom: { provider: "deepseek", modelId: "deepseek-v4-pro" },
          vision: false,
        },
      ],
    });
    expect(put2.status).toBe(200);
    const moved = ((await put2.json()) as ModelsResponse).models[0]!;
    expect(moved.provider).toBe("custom");
    expect(moved.modelId).toBe("deepseek-v4-pro");
    expect(moved.credential?.apiKeyMasked).toBe("sk-s…1234");
    // The env fallback follows client resolution — with no client_type on the entry,
    // AgentHub still routes by id to the DeepSeek client (reading DEEPSEEK_API_KEY), regardless of group membership.
    expect(moved.envKey).toBe("DEEPSEEK_API_KEY");
  });

  it("serializes a ModelScope settings save with a group credential write", async () => {
    const svc = wire(ProjectConfigService, { paths: { root: t.root } });
    const raceProject = "modelscope-save-refresh-race";
    await svc.writeRaw(raceProject, {
      models: [
        {
          provider: "modelscope",
          model_id: "Qwen/Qwen3.8-27B",
          api_key: "old-access",
          client_type: "openai-chat-vllm-adapter",
        },
      ],
    });

    const readRaw = svc.readRaw.bind(svc);
    let releaseRead!: () => void;
    const readGate = new Promise<void>((resolve) => (releaseRead = resolve));
    let announceRead!: () => void;
    const readStarted = new Promise<void>((resolve) => (announceRead = resolve));
    let firstRead = true;
    svc.readRaw = async (id: string) => {
      const raw = await readRaw(id);
      if (firstRead) {
        firstRead = false;
        announceRead();
        await readGate;
      }
      return raw;
    };

    const save = svc.updateModels(raceProject, {
      models: [
        {
          provider: "modelscope",
          modelId: "Qwen/Qwen3.8-27B",
          clientType: "openai-chat-vllm-adapter",
        },
      ],
    });
    await readStarted;
    let credentialWriteSettled = false;
    const credentialWrite = svc
      .setGroupApiKey(raceProject, "modelscope", "new-access")
      .finally(() => {
        credentialWriteSettled = true;
      });
    await Promise.resolve();
    expect(credentialWriteSettled).toBe(false);

    releaseRead();
    await Promise.all([save, credentialWrite]);
    const stored = await readRaw(raceProject);
    expect((stored.models as Array<Record<string, unknown>>)[0]?.api_key).toBe("new-access");
    expect(JSON.stringify(stored)).not.toContain("refreshToken");
    expect(JSON.stringify(stored)).not.toContain("accessTokenExpiresAt");
  });

  it("the display name is editable: not persisted when it matches the built-in catalog; provider is always persisted as an entry field", async () => {
    // Preset model: saved as-is → the display name falls back to the built-in catalog, and display_name isn't written to the config file.
    await api.put(url(), {
      models: [
        { provider: "openai", modelId: "gpt-5.5", displayName: "GPT-5.5" },
        { provider: "openai", modelId: "gpt-5.4", displayName: "My GPT" },
      ],
    });
    const body = (await (await api.get(url())).json()) as ModelsResponse;
    expect(pick(body, "openai", "gpt-5.5").displayName).toBe("GPT-5.5");
    // Edited one: the display name takes effect per the user's setting.
    expect(pick(body, "openai", "gpt-5.4").displayName).toBe("My GPT");

    // Clean on disk: unchanged preset models don't write display_name; provider is stored as a separate column, no concatenated string.
    const toml = await readFile(path.join(t.root, projectId, ".project_config.toml"), "utf8");
    expect(toml).not.toContain('display_name = "GPT-5.5"');
    expect(toml).toContain('display_name = "My GPT"');
    expect(toml).toContain('provider = "openai"');
    expect(toml).not.toContain("openai/gpt-5.5");
  });

  // 2026-08-31 is a Monday: 01:30Z is 09:30 in Beijing (peak), 12:00Z is 20:00 (off-peak).
  const PEAK = new Date("2026-08-31T01:30:00Z");
  const OFF_PEAK = new Date("2026-08-31T12:00:00Z");

  it("a scheduled row stores the one stable price, and the tier is derived from a Request's own instant", async () => {
    // The number on disk is the peak price whatever the hour; which tier a Request bills at is
    // decided by when it completed, so the same stored price bills twice as much at 09:30 as at
    // 20:00 on the same Monday.
    const svc = wire(ProjectConfigService, { paths: { root: t.root } });
    const peak = await svc.getPricing(projectId, "deepseek", "deepseek-flash", PEAK);
    const catalogPeak = catalogEntryFor("deepseek", "deepseek-flash")!.pricing!;
    expect(peak!.output).toBe(catalogPeak.output);
    expect(peak!.cacheRead).toBe(catalogPeak.cache_read);
    const off = await svc.getPricing(projectId, "deepseek", "deepseek-flash", OFF_PEAK);
    expect(off!.output).toBeCloseTo(catalogPeak.output / 2, 5);
    expect(off!.cacheRead).toBeCloseTo(catalogPeak.cache_read / 2, 6);
  });

  it("a hand-edited price on a scheduled row is billed as typed, in both tiers", async () => {
    const body = (await (await api.get(url())).json()) as ModelsResponse;
    const entries = body.models.map((m) => ({
      provider: m.provider,
      modelId: m.modelId,
      ...(m.pricing
        ? {
            pricing:
              m.provider === "deepseek" && m.modelId === "deepseek-flash"
                ? { cacheRead: 1, cacheWrite: 2, output: 3 }
                : {
                    cacheRead: m.pricing.cacheRead,
                    cacheWrite: m.pricing.cacheWrite,
                    output: m.pricing.output,
                  },
          }
        : {}),
    }));
    await api.put(url(), { models: entries });
    const svc = wire(ProjectConfigService, { paths: { root: t.root } });
    // Nothing here knows whether 3 is a peak rate, so halving it would invent a discount: the
    // typed number is what bills, whenever the Request runs.
    const typed = { cacheRead: 1, cacheWrite: 2, output: 3 };
    expect(await svc.getPricing(projectId, "deepseek", "deepseek-flash", PEAK)).toEqual(typed);
    expect(await svc.getPricing(projectId, "deepseek", "deepseek-flash", OFF_PEAK)).toEqual(typed);
  });

  it("a stored promotion takes its fraction off the list price on disk, multiplied with a scheduled row's off-peak tier", async () => {
    const svc = t.deps.projectConfigService;
    const off = (v: number, fraction: number): number => Math.round(v * (1 - fraction) * 1e6) / 1e6;
    // Seeded with the Project: TokenDance's glm-5.3-flash is on 10% off.
    const list = catalogEntryFor("tokendance", "glm-5.3-flash")!.pricing!;
    const promoted = {
      cacheRead: off(list.cache_read, 0.1),
      cacheWrite: off(list.cache_write, 0.1),
      output: off(list.output, 0.1),
    };
    expect(await svc.getPricing(projectId, "tokendance", "glm-5.3-flash", PEAK)).toEqual(promoted);

    // A scheduled row still at the catalog's peak price, with a promotion of its own: peak bills
    // the promotion alone, off-peak the half-price tier with the promotion taken off it too.
    const peak = catalogEntryFor("deepseek", "deepseek-v4-flash")!.pricing!;
    await svc.updateModels(projectId, {
      models: [
        {
          provider: "deepseek",
          modelId: "deepseek-v4-flash",
          pricing: {
            cacheRead: peak.cache_read,
            cacheWrite: peak.cache_write,
            output: peak.output,
          },
          discount: 0.2,
        },
      ],
    });
    const peakRates = (await svc.getPricing(projectId, "deepseek", "deepseek-v4-flash", PEAK))!;
    expect(peakRates.output).toBe(off(peak.output, 0.2));
    const offPeakRates = (await svc.getPricing(
      projectId,
      "deepseek",
      "deepseek-v4-flash",
      OFF_PEAK,
    ))!;
    expect(offPeakRates.output).toBe(off(off(peak.output, 0.5), 0.2));
  });

  it("PUT discount: a declared one is stored and null clears it; an omitted one survives only on an unrenamed row at an unchanged price", async () => {
    const price = { cacheRead: 0.1, cacheWrite: 1, output: 4 };
    const row = (modelId: string, extra: Record<string, unknown> = {}) => ({
      provider: "custom",
      modelId,
      pricing: price,
      ...extra,
    });
    const stored = () =>
      t.deps.db
        .prepare(
          "SELECT model_id, discount FROM model_promotions WHERE project_id = ? ORDER BY model_id",
        )
        .all(projectId);

    const declared = await api.put(url(), {
      models: ["kept", "repriced", "renamed", "cleared", "dropped"].map((id) =>
        row(id, { discount: 0.2 }),
      ),
    });
    expect(declared.status).toBe(200);
    expect(
      ((await declared.json()) as ModelsResponse).models.map((m) => [m.modelId, m.discount]),
    ).toEqual([
      ["kept", 0.2],
      ["repriced", 0.2],
      ["renamed", 0.2],
      ["cleared", 0.2],
      ["dropped", 0.2],
    ]);

    const next = await api.put(url(), {
      models: [
        row("kept"),
        row("repriced", { pricing: { ...price, output: 5 } }),
        row("renamed-to", { renamedFrom: { provider: "custom", modelId: "renamed" } }),
        row("cleared", { discount: null }),
      ],
    });
    expect(next.status).toBe(200);
    expect(
      ((await next.json()) as ModelsResponse).models.map((m) => [m.modelId, m.discount]),
    ).toEqual([
      ["kept", 0.2],
      ["repriced", undefined],
      ["renamed-to", undefined],
      ["cleared", undefined],
    ]);
    // The dropped row's promotion went with it rather than lingering unseen.
    expect(stored()).toEqual([{ model_id: "kept", discount: 0.2 }]);

    // Out of range or not a number: 400 before anything is written, so neither the file nor
    // the table moves — not even for the valid row beside it, whose new price would otherwise
    // have cleared its promotion.
    const cfgFile = path.join(t.root, projectId, ".project_config.toml");
    const before = await readFile(cfgFile, "utf8");
    for (const bad of [0, 1, -0.2, 1.5, "0.2"]) {
      const res = await api.put(url(), {
        models: [
          row("kept", { pricing: { ...price, output: 9 } }),
          row("other", { discount: bad }),
        ],
      });
      expect(res.status, String(bad)).toBe(400);
    }
    expect(await readFile(cfgFile, "utf8")).toBe(before);
    expect(stored()).toEqual([{ model_id: "kept", discount: 0.2 }]);
  });

  it("an absent display name inherits the catalog's; only an empty one clears it", async () => {
    // Two different requests, and reading them as one is how a client that simply has no name
    // for a model — one built from the catalog, which stores no name of its own — destroys the
    // name of every model it sends.
    const cfgFile = path.join(t.root, projectId, ".project_config.toml");
    await api.put(url(), { models: [{ provider: "openai", modelId: "gpt-5.5" }] });
    const inherited = (await (await api.get(url())).json()) as ModelsResponse;
    expect(pick(inherited, "openai", "gpt-5.5").displayName).toBe("GPT-5.5");
    expect(await readFile(cfgFile, "utf8")).not.toContain("display_name");

    // Cleared: the empty string records the deletion, so the name does not come back on the
    // next load, and the GET reports it as an empty name rather than as no name — the whole
    // table comes back on the next PUT, where "no name" would ask for the catalog's again.
    await api.put(url(), {
      models: [{ provider: "openai", modelId: "gpt-5.5", displayName: "" }],
    });
    const body = (await (await api.get(url())).json()) as ModelsResponse;
    expect(pick(body, "openai", "gpt-5.5").displayName).toBe("");
    expect(await readFile(cfgFile, "utf8")).toContain('display_name = ""');
    // And it survives that round trip, exactly as the page performs it.
    await api.put(url(), {
      models: [{ provider: "openai", modelId: "gpt-5.5", displayName: "" }],
    });
    const again = (await (await api.get(url())).json()) as ModelsResponse;
    expect(pick(again, "openai", "gpt-5.5").displayName).toBe("");
  });

  it("a cleared name is restorable, and clearing a model the catalog does not name writes nothing", async () => {
    await api.put(url(), {
      models: [{ provider: "openai", modelId: "gpt-5.5", displayName: "" }],
    });
    // Naming it again drops the marker rather than leaving both on disk.
    await api.put(url(), {
      models: [{ provider: "openai", modelId: "gpt-5.5", displayName: "Renamed" }],
    });
    const body = (await (await api.get(url())).json()) as ModelsResponse;
    expect(pick(body, "openai", "gpt-5.5").displayName).toBe("Renamed");
    let toml = await readFile(path.join(t.root, projectId, ".project_config.toml"), "utf8");
    expect(toml).not.toContain('display_name = ""');

    // A model outside the catalog has no name to inherit, so absence already says "no name"
    // and the marker would be noise in the file — an explicitly empty one included.
    await api.put(url(), {
      models: [
        { provider: "openai", modelId: "gpt-5.5", displayName: "Renamed" },
        { provider: "custom", modelId: "my-own-model", displayName: "" },
      ],
    });
    toml = await readFile(path.join(t.root, projectId, ".project_config.toml"), "utf8");
    expect(toml).not.toContain('display_name = ""');
    const after = (await (await api.get(url())).json()) as ModelsResponse;
    expect(pick(after, "custom", "my-own-model").displayName).toBeUndefined();
  });

  it("an invalid renamedFrom is 400; rekeying without renamedFrom equals delete-old-then-create-new (the credential does not migrate)", async () => {
    await api.put(url(), {
      models: [{ provider: "custom", modelId: "m-a", apiKey: "sk-secret-abcd1234" }],
    });
    // Invalid shape: renamedFrom must be a { provider, modelId } pair object; a string is always 400.
    const bad = await api.put(url(), {
      models: [{ provider: "custom", modelId: "m-b", renamedFrom: "custom/m-a" }],
    });
    expect(bad.status).toBe(400);
    // Giving only half a ref (missing provider) is also 400 — neither half of a ref may be omitted.
    const half = await api.put(url(), {
      models: [{ provider: "custom", modelId: "m-b", renamedFrom: { modelId: "m-a" } }],
    });
    expect(half.status).toBe(400);

    const plain = await api.put(url(), { models: [{ provider: "custom", modelId: "m-b" }] });
    const body = (await plain.json()) as ModelsResponse;
    expect(body.models.map(pairKey)).toEqual(["custom\0m-b"]);
    expect(body.models[0]!.credential).toBeUndefined();
  });

  it("the connectivity test sends the entry's upstream model_id (the reference pair travels with the request body)", async () => {
    // Local openai-compatible endpoint: records the model field from the request body, then always rejects with 401 (never hits the network).
    const seenModels: string[] = [];
    const server = createServer((req, res) => {
      let raw = "";
      req.on("data", (chunk: Buffer) => (raw += chunk.toString("utf8")));
      req.on("end", () => {
        try {
          seenModels.push((JSON.parse(raw) as { model?: string }).model ?? "");
        } catch {
          seenModels.push("");
        }
        res.statusCode = 401;
        res.setHeader("content-type", "application/json");
        res.end(JSON.stringify({ error: { message: "test-reject", type: "invalid_request" } }));
      });
    });
    await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
    const port = (server.address() as AddressInfo).port;
    try {
      await api.put(url(), {
        models: [
          {
            provider: "custom",
            modelId: "actual-upstream-model",
            clientType: "openai",
            apiKey: "sk-test-local",
            baseUrl: `http://127.0.0.1:${port}/v1`,
          },
        ],
      });
      const res = await api.post(testUrl(), {
        provider: "custom",
        modelId: "actual-upstream-model",
      });
      expect(res.status).toBe(200);
      const body = (await res.json()) as ModelTestResponse;
      expect(body.ok).toBe(false);
      // What's sent is exactly the entry's model_id (under column storage it's the upstream id itself, no concatenated string).
      expect(seenModels).toContain("actual-upstream-model");
      expect(seenModels).not.toContain("custom/actual-upstream-model");
    } finally {
      await new Promise<void>((resolve) => server.close(() => resolve()));
    }
  });

  it("the connectivity-test request body carries no tools or tool_choice (an empty tool list is omitted entirely, so strict endpoints such as vLLM no longer 400)", async () => {
    // The probe runs with an empty tool list. The wire body must omit `tools` entirely —
    // `tools: []` is rejected by strict OpenAI-compatible servers (vLLM: "tools must not be an
    // empty array") — and must never carry `tool_choice`.
    const bodies: Record<string, unknown>[] = [];
    const server = createServer((req, res) => {
      let raw = "";
      req.on("data", (chunk: Buffer) => (raw += chunk.toString("utf8")));
      req.on("end", () => {
        try {
          bodies.push(JSON.parse(raw) as Record<string, unknown>);
        } catch {
          bodies.push({});
        }
        res.statusCode = 401;
        res.setHeader("content-type", "application/json");
        res.end(JSON.stringify({ error: { message: "test-reject", type: "invalid_request" } }));
      });
    });
    await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
    const port = (server.address() as AddressInfo).port;
    try {
      const res = await api.post(testUrl(), {
        provider: "custom",
        modelId: "probe-wire-model",
        clientType: "openai",
        apiKey: "sk-test-local",
        baseUrl: `http://127.0.0.1:${port}/v1`,
      });
      expect(res.status).toBe(200);
      expect(bodies.length).toBeGreaterThan(0);
      for (const body of bodies) {
        expect("tools" in body).toBe(false);
        expect("tool_choice" in body).toBe(false);
      }
    } finally {
      await new Promise<void>((resolve) => server.close(() => resolve()));
    }
  });

  it("connectivity test: both a saved model and a **not-yet-saved** custom model can be tested (the LLM layer throws nothing, every outcome converges)", async () => {
    // Both endpoints refuse the key they are handed.
    const network = stubFetch(() =>
      jsonResponse(
        { error: { message: "Incorrect API key provided.", code: "invalid_api_key" } },
        401,
      ),
    );
    try {
      await api.put(url(), {
        models: [{ provider: "openai", modelId: "gpt-5.5", apiKey: "sk-invalid-key-for-test" }],
      });
      const saved = await api.post(testUrl(), { provider: "openai", modelId: "gpt-5.5" });
      expect(saved.status).toBe(200);
      const savedBody = (await saved.json()) as ModelTestResponse;
      expect(savedBody.ok).toBe(false);
      expect(typeof savedBody.message).toBe("string");

      // "Test before save" for adding a custom model: the model isn't in the config, so all params come from the request body.
      const unsaved = await api.post(testUrl(), {
        provider: "custom",
        modelId: "my-new-model",
        apiKey: "sk-invalid",
        baseUrl: "https://example.invalid/v1",
        clientType: "openai",
      });
      expect(unsaved.status).toBe(200);
      const unsavedBody = (await unsaved.json()) as ModelTestResponse;
      expect(unsavedBody.ok).toBe(false);
      expect(typeof unsavedBody.message).toBe("string");
      // Each probe went to its own model's endpoint, with its own key.
      const keyAt = (host: string) =>
        network.calls.find((c) => c.url.startsWith(host))?.headers.get("authorization");
      expect(keyAt("https://api.openai.com/")).toBe("Bearer sk-invalid-key-for-test");
      expect(keyAt("https://example.invalid/v1")).toBe("Bearer sk-invalid");
    } finally {
      vi.unstubAllGlobals();
    }
  });

  it("connectivity test: a model with no credential at all converges to ok:false instead of 500", async () => {
    // A model using the OpenAI protocol: the provider SDK throws at **client construction** because
    // the key is missing — if that construction were outside the try it would bubble up as a 500. Clear the env-var key so there's nowhere to get one (no real network request).
    const prev = process.env.OPENAI_API_KEY;
    delete process.env.OPENAI_API_KEY;
    try {
      await api.put(url(), {
        models: [{ provider: "custom", modelId: "no-key-openai", clientType: "openai" }],
      });
      const res = await api.post(testUrl(), { provider: "custom", modelId: "no-key-openai" });
      expect(res.status).toBe(200);
      const body = (await res.json()) as ModelTestResponse;
      expect(body.ok).toBe(false);
      expect(typeof body.message).toBe("string");
    } finally {
      if (prev !== undefined) process.env.OPENAI_API_KEY = prev;
    }
  });

  it("the connectivity test and the group speed test refuse a keyless gateway row without touching the environment", async () => {
    // The #786 review's finding: a keyless gateway row handed AgentHub no key, and its
    // generic client read OPENAI_API_KEY / ANTHROPIC_API_KEY itself — one speed test on a
    // gateway group sent the user's vendor keys to the gateway once per model. Both paths run
    // through testModel; both must refuse before any request leaves the process.
    const saved = { openai: process.env.OPENAI_API_KEY, anthropic: process.env.ANTHROPIC_API_KEY };
    process.env.OPENAI_API_KEY = "vendor-openai-must-not-cross";
    process.env.ANTHROPIC_API_KEY = "vendor-anthropic-must-not-cross";
    const network = stubFetch(() => {
      throw new Error("no request may leave the process for a refused probe");
    });
    try {
      // A gateway preset: the group's endpoint, no key. Both OpenAI-protocol gateways and the
      // Messages-protocol custom preset (Atria — the path that carried the Anthropic key in the
      // #786 finding) are refused, and the refusal is the harness's own sentence, never the
      // SDK's text with a vendor variable in it.
      for (const [provider, modelId] of [
        ["tokendance", "glm-5.3"],
        ["openrouter", "openai/gpt-5.5"],
        ["custom", "Atria-Dawn-Preview"],
      ] as const) {
        expect(catalogEntryFor(provider, modelId)).toBeDefined();
        for (const speed of [false, true]) {
          const res = await api.post(testUrl(), { provider, modelId, speed });
          expect(res.status).toBe(200);
          const body = (await res.json()) as ModelTestResponse;
          expect(body.ok).toBe(false);
          expect(body.message).toMatch(/has no API key/);
          expect(body.message).toMatch(/set the API key on the model entry/);
          expect(body.message).not.toMatch(/OPENAI_API_KEY|ANTHROPIC_API_KEY|must-not-cross/);
        }
      }
      // A not-yet-saved custom row pointed at a private server: same refusal.
      const draft = await api.post(testUrl(), {
        provider: "custom",
        modelId: "draft-model",
        clientType: "openai-chat",
        baseUrl: "http://127.0.0.1:9/v1",
      });
      expect(((await draft.json()) as ModelTestResponse).message).toMatch(/has no API key/);
      // The vision probe shares the rule.
      const vision = await api.post(`${url()}/detect-vision`, {
        provider: "tokendance",
        modelId: "glm-5.3",
      });
      expect((await vision.json()) as ModelVisionDetectResponse).toMatchObject({
        outcome: "failed",
        message: expect.stringMatching(/has no API key/) as string,
      });
      expect(network.calls).toEqual([]);
    } finally {
      vi.unstubAllGlobals();
      for (const [key, value] of [
        ["OPENAI_API_KEY", saved.openai],
        ["ANTHROPIC_API_KEY", saved.anthropic],
      ] as const) {
        if (value === undefined) delete process.env[key];
        else process.env[key] = value;
      }
    }
  });

  it("Penguin Go probes never substitute vendor environment keys", async () => {
    const saved = {
      gemini: process.env.GEMINI_API_KEY,
      openai: process.env.OPENAI_API_KEY,
      penguin: process.env.PENGUIN_GO_API_KEY,
    };
    process.env.GEMINI_API_KEY = "vendor-gemini-must-not-cross";
    process.env.OPENAI_API_KEY = "vendor-openai-must-not-cross";
    delete process.env.PENGUIN_GO_API_KEY;
    try {
      for (const modelId of ["gemini-3.8-flash", "deepseek-flash"]) {
        const tested = await api.post(testUrl(), { provider: "penguin-go", modelId });
        expect(tested.status).toBe(200);
        expect((await tested.json()) as ModelTestResponse).toMatchObject({
          ok: false,
          message: expect.stringMatching(/has no API key.*PENGUIN_GO_API_KEY/) as string,
        });

        const vision = await api.post(`${url()}/detect-vision`, {
          provider: "penguin-go",
          modelId,
        });
        expect(vision.status).toBe(200);
        expect((await vision.json()) as ModelVisionDetectResponse).toMatchObject({
          outcome: "failed",
          message: expect.stringMatching(/has no API key.*PENGUIN_GO_API_KEY/) as string,
        });
      }

      process.env.PENGUIN_GO_API_KEY = "penguin-go-key-without-endpoint";
      const noEndpoint = await api.post(testUrl(), {
        provider: "penguin-go",
        modelId: "gemini-3.8-flash",
        baseUrl: null,
      });
      expect((await noEndpoint.json()) as ModelTestResponse).toMatchObject({
        ok: false,
        message: expect.stringMatching(/has no base URL/) as string,
      });
    } finally {
      for (const [key, value] of [
        ["GEMINI_API_KEY", saved.gemini],
        ["OPENAI_API_KEY", saved.openai],
        ["PENGUIN_GO_API_KEY", saved.penguin],
      ] as const) {
        if (value === undefined) delete process.env[key];
        else process.env[key] = value;
      }
    }
  });

  it("connectivity test: clearApiKey does not fall back to the stored key (the current draft is what gets tested)", async () => {
    // A key is already saved, but the test request carries clearApiKey — the server must **not** use
    // the saved key. Clear the env var so "don't use the saved key" == no credential at all, so construction synchronously throws missing-credential (no network request, deterministic).
    const prev = process.env.OPENAI_API_KEY;
    delete process.env.OPENAI_API_KEY;
    try {
      await api.put(url(), {
        models: [
          {
            provider: "custom",
            modelId: "has-key-openai",
            clientType: "openai",
            apiKey: "sk-saved-key",
          },
        ],
      });
      // Without clearApiKey: use the saved key and construction succeeds (would hit the real network; its result isn't asserted here).
      // With clearApiKey: don't fall back to the saved key → no credential → synchronously resolves to "missing credential".
      const cleared = await api.post(testUrl(), {
        provider: "custom",
        modelId: "has-key-openai",
        clearApiKey: true,
      });
      expect(cleared.status).toBe(200);
      const body = (await cleared.json()) as ModelTestResponse;
      expect(body.ok).toBe(false);
      // Missing-credential is thrown synchronously at construction (message contains "credentials"); if the saved key were still used, it wouldn't be this message.
      expect(body.message ?? "").toMatch(/credential/i);
    } finally {
      if (prev !== undefined) process.env.OPENAI_API_KEY = prev;
    }
  });
});

describe("models update reaches loaded sessions (invalidation + live unlock)", () => {
  let t: TestApp;
  let api: ReturnType<typeof apiClient>;
  let projectId: string;
  /** Loader call count: how many times the manager (re)built a runtime from the index. */
  let loads: number;

  const insertSession = (sessionId: string, project: string): void => {
    t.deps.sessionsRepo.insert(sessionRow(sessionId, { projectId: project, workspace: t.root }));
  };

  const putModels = (apiKey: string) =>
    api.put(`/api/projects/${projectId}/models`, {
      defaultModel: { provider: "custom", modelId: "m-inv" },
      models: [{ provider: "custom", modelId: "m-inv", apiKey }],
    });

  beforeAll(async () => {
    // Every runtime the manager builds from the index is an idle fake: no LLM call is made.
    t = await createTestApp({
      loader: {
        load: async (row) => {
          loads++;
          return fakeSession(row.sessionId);
        },
      },
    });
    const { cookie } = await provisionUser(t.app, "inv_owner");
    api = apiClient(t.app, cookie);
  });
  afterAll(async () => {
    await t.cleanup();
  });
  let projects = 0;
  beforeEach(async () => {
    loads = 0;
    projects += 1;
    const created = (await (
      await api.post("/api/projects", {
        projectId: `inv_owner-models_${projects}`,
        name: "invalidation",
      })
    ).json()) as ProjectCreateResponse;
    projectId = created.project.projectId;
  });

  it("PUT invalidates the Project's cached Session runtimes: the next Task re-resumes with the new key", async () => {
    insertSession("models-sess-1", projectId);
    const idle = () => t.deps.manager.statusOf("models-sess-1") === "idle";

    // First Task builds the runtime (load #1); the second reuses the active-table entry.
    await t.deps.manager.startTask("models-sess-1", [userText("a")]);
    await waitFor(idle);
    await t.deps.manager.startTask("models-sess-1", [userText("b")]);
    await waitFor(idle);
    expect(loads).toBe(1);

    // Key update via the API: the cached runtime is stale, so the next Task re-resumes
    // (the loader re-reads the Project config — the new api_key reaches the next Task).
    expect((await putModels("sk-fresh-key-000111")).status).toBe(200);
    await t.deps.manager.startTask("models-sess-1", [userText("c")]);
    await waitFor(idle);
    expect(loads).toBe(2);

    // Reads don't invalidate: the rebuilt runtime is reused.
    expect((await api.get(`/api/projects/${projectId}/models`)).status).toBe(200);
    await t.deps.manager.startTask("models-sess-1", [userText("d")]);
    await waitFor(idle);
    expect(loads).toBe(2);
  });

  it("PUT publishes credentials_updated to the Project's existing session channels only", async () => {
    insertSession("models-sess-live", projectId);
    insertSession("models-sess-cold", projectId);
    // A session of ANOTHER project must not receive the event.
    const other = (await (
      await api.post("/api/projects", { projectId: "inv_owner-other", name: "other" })
    ).json()) as ProjectCreateResponse;
    insertSession("other-sess", other.project.projectId);

    // Only the "live" session has an open channel (a subscribed tab); "cold" has none.
    const events: ChannelEvent[] = [];
    t.deps.channels.get("models-sess-live").subscribe((e) => events.push(e));
    const otherEvents: ChannelEvent[] = [];
    t.deps.channels.get("other-sess").subscribe((e) => otherEvents.push(e));

    expect((await putModels("sk-live-key-000222")).status).toBe(200);

    const types = events
      .filter((e) => e.event === "server_event")
      .map((e) => (JSON.parse(e.data) as { type: string }).type);
    expect(types).toContain("credentials_updated");
    // Cross-project channels stay silent, and no channel is created for unsubscribed sessions.
    expect(
      otherEvents
        .filter((e) => e.event === "server_event")
        .map((e) => (JSON.parse(e.data) as { type: string }).type),
    ).not.toContain("credentials_updated");
    expect(t.deps.channels.peek("models-sess-cold")).toBeUndefined();
  });

  it("GET/PUT responses expose updatedAt (config file mtime) for the web's auth-dead gate", async () => {
    const before = Date.now() - 60_000;
    const put = await putModels("sk-ts-key-000333");
    expect(put.status).toBe(200);
    const putBody = (await put.json()) as ModelsResponse;
    expect(typeof putBody.updatedAt).toBe("string");
    expect(Date.parse(putBody.updatedAt!)).toBeGreaterThan(before);

    const get = await api.get(`/api/projects/${projectId}/models`);
    const getBody = (await get.json()) as ModelsResponse;
    expect(typeof getBody.updatedAt).toBe("string");
    // Reads don't bump it: still the PUT's write time (same file mtime).
    expect(Date.parse(getBody.updatedAt!)).toBeGreaterThanOrEqual(Date.parse(putBody.updatedAt!));
  });
});