A local-first multi-agent app development platform, built on PenguinHarness
/**
* A Project's model configuration: the built-in catalog as presets, one config file holding the
* credentials, rekeying, pricing fields, the connectivity test, and what a save does to loaded
* Sessions.
*
* - A new Project, and default_project when the seeded admin adopts it, is preset with every
* built-in model (provider and model_id stored as separate fields, the pair the key) and the
* default model, so a Session opens with no model ref; a default_project the CLI already
* configured is left alone.
* - Credentials live inline in `.project_config.toml` (0600) and GET shows only a mask; the env
* fallback is reported and masked only where the entry is allowed one, never leaking a value.
* - GET fills in display name and env key from the catalog; vision comes from the file, else the
* catalog default; a pre-rename `client_type = "openai"` reads as openai-chat; a model the
* catalog dropped still loads and stays usable; one model_id under two providers is two rows.
* - PUT persists a custom model's vision flag and max_tokens (clearing and rejecting bad values),
* falls back to OPENAI_API_KEY for the openai protocol, requires a provider, refuses an id a
* vendor group cannot route when the request introduces it, and stores or clears a discount.
* - renamedFrom (and a group change) migrates the credential, the config and the default and
* vision pointers; without it a rekey is delete-then-create; an invalid one is 400. A
* ModelScope settings save is serialized with a group credential write.
* - The display name is persisted only when it differs from the catalog's; an absent one
* inherits, an empty one clears, and a cleared one is restorable.
* - A scheduled row answers both rates from its one price on disk; a hand-edited price and a
* promotion apply in both tiers.
* - The connectivity test sends the entry's upstream model_id, no tools, works for saved and
* draft models, converges to ok:false with no credential, tests the draft (clearApiKey) rather
* than the stored key, and refuses a keyless gateway row or a Penguin Go probe without reaching
* for vendor environment keys or the network.
* - A PUT invalidates the Project's cached Session runtimes (reads do not), tells the Project's
* open Session channels only, and GET/PUT report the file's updatedAt.
*
* One app per describe; every case works in a Project of its own.
*/
import { createServer } from "node:http";
import type { AddressInfo } from "node:net";
import { readFile, stat, writeFile } from "node:fs/promises";
import path from "node:path";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import {
MODEL_CATALOG,
PENGUIN_GO_BASE_URL,
catalogEntryFor,
presetPromotions,
userText,
} from "@adelie/core";
import type {
ErrorBody,
ModelsResponse,
ModelTestResponse,
ModelVisionDetectResponse,
ProjectCreateResponse,
SessionCreateResponse,
} from "../src/api/types.js";
import { ProjectConfigService } from "../src/services/project-config-service.js";
import type { RuntimeSession } from "../src/runtime/session-manager.js";
import type { ChannelEvent } from "../src/runtime/channel.js";
import { jsonResponse, stubFetch } from "./fixtures/fetch.js";
import { fakeSession, sessionRow } from "./fixtures/session.js";
import { apiClient, createTestApp, loginAdmin, provisionUser, waitFor } from "./helpers.js";
import type { TestApp } from "./helpers.js";
import { wire } from "@adelie/core/kernel";
/** Preset paired refs (config primary key = (provider, model_id)): retired catalog rows are not presets. */
const catalogPairs = MODEL_CATALOG.filter((m) => m.retired !== true).map(
(m) => `${m.provider}\0${m.modelId}`,
);
/** Response row → comparable paired key (test-only comparison, not the storage format). */
const pairKey = (m: { provider: string; modelId: string }): string => `${m.provider}\0${m.modelId}`;
/** Fetch a row by its paired ref. */
const pick = (body: ModelsResponse, provider: string, modelId: string) =>
body.models.find((m) => m.provider === provider && m.modelId === modelId)!;
/**
* Writes rows straight into a Project's config file, as a table saved before a rule existed
* would hold them: rows a built-in group no longer takes from a PUT (see "model_not_addable")
* are only ever reached this way.
*/
async function storeRows(
root: string,
projectId: string,
rows: Array<{ provider: string; modelId: string; clientType?: string }>,
): Promise<void> {
const toml = rows.flatMap((r) => [
"[[models]]",
`provider = "${r.provider}"`,
`model_id = "${r.modelId}"`,
...(r.clientType !== undefined ? [`client_type = "${r.clientType}"`] : []),
"",
]);
await writeFile(path.join(root, projectId, ".project_config.toml"), toml.join("\n"), "utf8");
}
describe("models preset & catalog enrichment", () => {
let t: TestApp;
let api: ReturnType<typeof apiClient>;
let projectId: string;
const url = () => `/api/projects/${projectId}/models`;
beforeAll(async () => {
t = await createTestApp();
const { cookie } = await provisionUser(t.app, "alice");
api = apiClient(t.app, cookie);
});
afterAll(async () => {
await t.cleanup();
});
let projects = 0;
beforeEach(async () => {
projects += 1;
const created = (await (
await api.post("/api/projects", {
projectId: `alice-preset_${projects}`,
name: "Preset project",
})
).json()) as ProjectCreateResponse;
projectId = created.project.projectId;
});
it("credentials are inlined in one file: the apiKey from PUT lands in .project_config.toml (0600), GET returns only a mask", async () => {
const put = await api.put(url(), {
defaultModel: { provider: "custom", modelId: "m-inline" },
models: [
{
provider: "custom",
modelId: "m-inline",
apiKey: "sk-server-inline-1",
baseUrl: "https://inline.example/v1",
clientType: "openai",
},
],
});
expect(put.status).toBe(200);
// GET: the masked key and base URL are both visible; plaintext is never sent.
const body = (await (await api.get(url())).json()) as ModelsResponse;
const m = pick(body, "custom", "m-inline");
expect(m.credential?.baseUrl).toBe("https://inline.example/v1");
expect(m.credential?.apiKeyMasked).toBe("sk-s…ne-1");
expect(m.credential?.createdAt).toBeTruthy();
expect(JSON.stringify(body)).not.toContain("sk-server-inline-1");
// Secrets inlined in the single config file (persisted 0600); provider and model_id are separate columns, no concatenated string.
const projectDir = path.join(t.root, projectId);
const cfgFile = path.join(projectDir, ".project_config.toml");
const cfgRaw = await readFile(cfgFile, "utf8");
expect(cfgRaw).toContain("sk-server-inline-1");
expect(cfgRaw).toContain('provider = "custom"');
expect(cfgRaw).toContain('model_id = "m-inline"');
expect(cfgRaw).not.toContain("custom/m-inline");
// POSIX-only: Windows has no owner-only mode bits (chmod maps to the read-only attribute).
if (process.platform !== "win32") {
expect((await stat(cfgFile)).mode & 0o777).toBe(0o600);
}
// No more separate .credentials.toml / project_config.toml files.
await expect(readFile(path.join(projectDir, ".credentials.toml"), "utf8")).rejects.toThrow();
await expect(readFile(path.join(projectDir, "project_config.toml"), "utf8")).rejects.toThrow();
});
it("a preset a sync adds to a gateway group borrows the group's key instead of arriving unconfigured", async () => {
const cfgFile = path.join(t.root, projectId, ".project_config.toml");
const GATEWAY = "https://tokendance.space/gateway/v1";
const row = (modelId: string, extra: Record<string, unknown> = {}) => ({
provider: "tokendance",
modelId,
clientType: "openai-chat",
baseUrl: GATEWAY,
...extra,
});
// The table before the sync: one TokenDance row, the key pasted once.
const first = await api.put(url(), {
defaultModel: { provider: "tokendance", modelId: "deepseek-v4-flash-0731" },
models: [row("deepseek-v4-flash-0731", { apiKey: "sk-group-gateway-1" })],
});
expect(first.status).toBe(200);
// What "sync presets" sends: the row that was there (restating no key at all) plus a preset
// the catalog carries and this table does not — the shape that used to land keyless.
const synced = await api.put(url(), {
models: [row("deepseek-v4-flash-0731"), row("deepseek-v4-pro")],
});
expect(synced.status).toBe(200);
const body = (await synced.json()) as ModelsResponse;
const stored = pick(body, "tokendance", "deepseek-v4-flash-0731").credential?.apiKeyMasked;
expect(stored).toBeTruthy();
expect(pick(body, "tokendance", "deepseek-v4-pro").credential?.apiKeyMasked).toBe(stored);
// Borrowed, never echoed: the plaintext stays on disk.
expect(JSON.stringify(body)).not.toContain("sk-group-gateway-1");
const cfgRaw = await readFile(cfgFile, "utf8");
expect(cfgRaw.split('api_key = "sk-group-gateway-1"').length - 1).toBe(2);
// The shape the rule exists for: a whole gateway group, the key pasted on one row of it.
// One endpoint behind one login, so every preset the group carries is answered by that key.
const group = MODEL_CATALOG.filter((m) => m.provider === "tokendance" && m.retired !== true);
expect(group.length).toBeGreaterThan(4);
const wholeGroup = await api.put(url(), {
models: group.map((m, i) => ({
provider: m.provider,
modelId: m.modelId,
...(m.clientType !== undefined ? { clientType: m.clientType } : {}),
...(m.baseUrl !== undefined ? { baseUrl: m.baseUrl } : {}),
...(i === 0 ? { apiKey: "sk-group-gateway-1" } : {}),
})),
});
expect(wholeGroup.status).toBe(200);
const groupBody = (await wholeGroup.json()) as ModelsResponse;
const firstMask = pick(groupBody, group[0]!.provider, group[0]!.modelId).credential
?.apiKeyMasked;
expect(firstMask).toBeTruthy();
for (const m of group) {
expect(pick(groupBody, m.provider, m.modelId).credential?.apiKeyMasked, m.modelId).toBe(
firstMask,
);
}
const groupRaw = await readFile(cfgFile, "utf8");
expect(groupRaw.split('api_key = "sk-group-gateway-1"').length - 1).toBe(group.length);
});
it("a group holding two different keys is left to decide: the new row arrives keyless", async () => {
const GATEWAY = "https://tokendance.space/gateway/v1";
const row = (modelId: string, extra: Record<string, unknown> = {}) => ({
provider: "tokendance",
modelId,
clientType: "openai-chat",
baseUrl: GATEWAY,
...extra,
});
const twoKeys = await api.put(url(), {
models: [
row("deepseek-v4-flash-0731", { apiKey: "sk-first-account" }),
row("deepseek-v4-pro", { apiKey: "sk-second-account" }),
],
});
expect(twoKeys.status).toBe(200);
const added = await api.put(url(), {
models: [row("deepseek-v4-flash-0731"), row("deepseek-v4-pro"), row("deepseek-v4.1-flash")],
});
expect(added.status).toBe(200);
// Two accounts under one group is a deliberate state: picking one of them would be a guess.
expect(
pick((await added.json()) as ModelsResponse, "tokendance", "deepseek-v4.1-flash").credential
?.apiKeyMasked,
).toBeUndefined();
});
it("a row the request clears stays keyless: a clear is not filled back in by the row beside it", async () => {
const GATEWAY = "https://tokendance.space/gateway/v1";
const row = (modelId: string, extra: Record<string, unknown> = {}) => ({
provider: "tokendance",
modelId,
clientType: "openai-chat",
baseUrl: GATEWAY,
...extra,
});
await api.put(url(), {
models: [
row("deepseek-v4-flash-0731", { apiKey: "sk-group-gateway-1" }),
row("deepseek-v4-pro"),
],
});
// Both rows do have the key by now (the second borrowed it) …
const before = (await (await api.get(url())).json()) as ModelsResponse;
expect(pick(before, "tokendance", "deepseek-v4-pro").credential?.apiKeyMasked).toBeTruthy();
// … and taking one of them back is what keeps it keyless.
const cleared = await api.put(url(), {
models: [row("deepseek-v4-flash-0731"), row("deepseek-v4-pro", { clearApiKey: true })],
});
expect(cleared.status).toBe(200);
const body = (await cleared.json()) as ModelsResponse;
expect(pick(body, "tokendance", "deepseek-v4-pro").credential?.apiKeyMasked).toBeUndefined();
expect(
pick(body, "tokendance", "deepseek-v4-flash-0731").credential?.apiKeyMasked,
).toBeTruthy();
});
it("a preset pointed at another endpoint is another service: nothing is borrowed across them", async () => {
const GATEWAY = "https://tokendance.space/gateway/v1";
const row = (modelId: string, extra: Record<string, unknown> = {}) => ({
provider: "tokendance",
modelId,
clientType: "openai-chat",
...extra,
});
await api.put(url(), {
models: [row("deepseek-v4-flash-0731", { apiKey: "sk-gateway-account", baseUrl: GATEWAY })],
});
// The next preset is answered by the user's own relay: the gateway key is not its key.
const relayed = await api.put(url(), {
models: [
row("deepseek-v4-flash-0731", { baseUrl: GATEWAY }),
row("deepseek-v4-pro", { baseUrl: "https://relay.example/v1" }),
],
});
expect(relayed.status).toBe(200);
const body = (await relayed.json()) as ModelsResponse;
expect(pick(body, "tokendance", "deepseek-v4-pro").credential?.apiKeyMasked).toBeUndefined();
expect(
pick(body, "tokendance", "deepseek-v4-flash-0731").credential?.apiKeyMasked,
).toBeTruthy();
// A row added by hand is the user's own entry: replacing it without its key has always meant
// the credential does not migrate, and the rows beside it do not change that.
const custom = (modelId: string, baseUrl: string, extra: Record<string, unknown> = {}) => ({
provider: "custom",
modelId,
clientType: "openai-chat",
baseUrl,
...extra,
});
await api.put(url(), {
models: [custom("self-hosted-a", "https://a.example/v1", { apiKey: "sk-service-a" })],
});
const plain = await api.put(url(), {
models: [
custom("self-hosted-a", "https://a.example/v1"),
custom("self-hosted-b", "https://a.example/v1"),
],
});
expect(plain.status).toBe(200);
const plainBody = (await plain.json()) as ModelsResponse;
expect(pick(plainBody, "custom", "self-hosted-b").credential?.apiKeyMasked).toBeUndefined();
// The row that carried the key keeps it on its own account, base URL and all.
expect(pick(plainBody, "custom", "self-hosted-a").credential?.apiKeyMasked).toBeTruthy();
});
it("a new Project is preset with every built-in model (provider and model_id as separate fields + catalog info)", async () => {
const res = await api.get(url());
expect(res.status).toBe(200);
const body = (await res.json()) as ModelsResponse;
expect(body.defaultModel).toEqual({
provider: "deepseek",
modelId: "deepseek-flash",
});
expect(body.models.map(pairKey)).toEqual(catalogPairs);
// A retired catalog row is kept only for the Projects that already carry it, so a new
// Project gets none of them.
const retired = MODEL_CATALOG.filter((m) => m.retired === true).map(pairKey);
expect(retired.length).toBeGreaterThan(0);
expect(body.models.map(pairKey).filter((key) => retired.includes(key))).toEqual([]);
// Every row reads its display name, context window, list price, vision and pinned
// protocol and endpoint off its catalog row; nothing of the vendor's is copied here.
for (const row of body.models) {
const entry = catalogEntryFor(row.provider, row.modelId)!;
const key = `${row.provider}/${row.modelId}`;
expect(row.displayName, key).toBe(entry.displayName);
expect(row.contextWindow, key).toBe(entry.contextWindow);
expect(row.vision, key).toBe(entry.supportsVision);
expect(row.clientType, key).toBe(entry.clientType);
expect(row.credential?.baseUrl, key).toBe(entry.baseUrl);
// A preset carries no secret.
expect(row.credential?.apiKeyMasked, key).toBeUndefined();
}
// A vendor's own row falls back to its client's environment variable; it has no stored
// credential and no client_type (AgentHub routes it by its upstream id).
const sonnet = pick(body, "anthropic", "claude-sonnet-4-6");
expect(sonnet.isDefault).toBe(false);
expect(sonnet.envKey).toBe("ANTHROPIC_API_KEY");
expect(sonnet.credential).toBeUndefined();
expect(sonnet.clientType).toBeUndefined();
expect(pick(body, "deepseek", "deepseek-v4-pro").envKey).toBe("DEEPSEEK_API_KEY");
expect(pick(body, "deepseek", "deepseek-flash").isDefault).toBe(true);
// A gateway pins each row's protocol and endpoint, so one group can hold rows on different
// clients (OpenCode Go: a Messages row and a Chat Completions row). Like every gateway's
// rows, neither falls back to its client's environment variable — that holds the user's
// vendor key, and the gateway is not the vendor. An upstream id with a `/` is a plain
// string under column storage.
const messagesRow = pick(body, "opencode-go", "qwen3.8-max");
const chatRow = pick(body, "opencode-go", "glm-5.3");
expect(messagesRow.clientType).toBe("ant-messages");
expect(chatRow.clientType).toBe("openai-chat");
expect(messagesRow.envKey).toBeUndefined();
expect(chatRow.envKey).toBeUndefined();
expect(pick(body, "openrouter", "xiaomi/mimo-v2.5").credential?.baseUrl).toBeDefined();
// A catalog row on a promotion is preset at its LIST price like every other row, and the
// promotion is seeded beside it in web.db: the rows reporting a discount are exactly the
// catalog's flat promotions.
const promoted = catalogEntryFor("tokendance", "glm-5.3-flash")!;
expect(pick(body, "tokendance", "glm-5.3-flash")).toMatchObject({
pricing: {
cacheRead: promoted.pricing!.cache_read,
cacheWrite: promoted.pricing!.cache_write,
output: promoted.pricing!.output,
},
discount: promoted.discount,
});
expect(
body.models
.filter((m) => m.discount !== undefined)
.map((m) => ({ provider: m.provider, modelId: m.modelId, discount: m.discount })),
).toEqual(presetPromotions());
// An undiscounted row is preset at its list price, unchanged, and reports no discount.
const plain = catalogEntryFor("tokendance", "hy4-preview")!;
expect(pick(body, "tokendance", "hy4-preview").pricing).toEqual({
cacheRead: plain.pricing!.cache_read,
cacheWrite: plain.pricing!.cache_write,
output: plain.pricing!.output,
});
expect(pick(body, "tokendance", "hy4-preview")).not.toHaveProperty("discount");
});
it("reports and masks the env fallback only where the entry is allowed one (the vendor's own endpoint, or the relay's own variable), and never leaks a value", async () => {
const saved = {
anthropic: process.env.ANTHROPIC_API_KEY,
deepseek: process.env.DEEPSEEK_API_KEY,
openai: process.env.OPENAI_API_KEY,
penguin: process.env.PENGUIN_GO_API_KEY,
};
const anthropicValue = "sk-ant-test-secret-value-123456";
const openaiValue = "sk-openai-test-secret-value-789";
process.env.ANTHROPIC_API_KEY = anthropicValue;
process.env.OPENAI_API_KEY = openaiValue;
process.env.PENGUIN_GO_API_KEY = "sk-penguin-test-secret-value-456";
// Empty counts as absent — it would not authenticate either.
process.env.DEEPSEEK_API_KEY = "";
// Two vendor-group rows no PUT may add any more, stored as a table from before would hold them.
await storeRows(t.root, projectId, [
{ provider: "anthropic", modelId: "claude-sonnet-4-6-preview" },
{ provider: "anthropic", modelId: "claude-via-gateway", clientType: "openai" },
]);
try {
const put = await api.put(url(), {
defaultModel: { provider: "anthropic", modelId: "claude-sonnet-4-6" },
models: [
// First-party catalog preset, variable set: masked preview.
{ provider: "anthropic", modelId: "claude-sonnet-4-6" },
// First-party catalog preset, variable empty: no preview.
{ provider: "deepseek", modelId: "deepseek-v4-pro" },
// Off-catalog id in a vendor group, pure auto-route: still first-party.
{ provider: "anthropic", modelId: "claude-sonnet-4-6-preview" },
// Vendor group on a generic protocol with no base URL: the OpenAI client talks to
// api.openai.com, so the OpenAI key may back it — whatever the group is called.
{ provider: "anthropic", modelId: "claude-via-gateway", clientType: "openai" },
// A gateway row carries the gateway's endpoint: no fallback at all, so no envKey
// and no preview, even with OPENAI_API_KEY set.
{
provider: "openrouter",
modelId: "xiaomi/mimo-v2.5",
clientType: "openai-chat",
baseUrl: "https://openrouter.ai/api/v1",
},
// Custom rows are judged by their endpoint: none → the vendor's own (a fallback, but
// not a preview — see modelEnvPreviewKey); a private server → nothing.
{ provider: "custom", modelId: "my-model", clientType: "openai" },
{
provider: "custom",
modelId: "my-local",
clientType: "openai",
baseUrl: "http://127.0.0.1:8000/v1",
},
// A vLLM preset as seeded: no base URL, the vLLM adapter pinned. It falls back to
// OPENAI_API_KEY under the rule, but showing the mask would list a self-hosted id in
// the chat picker as configured and send the OpenAI key to api.openai.com.
{ provider: "vllm", modelId: "Qwen/Qwen3.8-27B", clientType: "openai-chat-vllm-adapter" },
// Both protocols in the relay group resolve the same provider-scoped key.
{
provider: "penguin-go",
modelId: "gemini-3.8-flash",
clientType: "gemini-3.8",
baseUrl: PENGUIN_GO_BASE_URL,
},
{
provider: "penguin-go",
modelId: "deepseek-flash",
clientType: "deepseek-v4",
baseUrl: PENGUIN_GO_BASE_URL,
},
],
});
expect(put.status).toBe(200);
// The masked preview follows maskApiKey; the plaintext must never be serialized.
const text = await put.text();
expect(text).not.toContain(anthropicValue);
expect(text).not.toContain(openaiValue);
const body = JSON.parse(text) as ModelsResponse;
const masked = `${anthropicValue.slice(0, 4)}…${anthropicValue.slice(-4)}`;
expect(pick(body, "anthropic", "claude-sonnet-4-6").envKeyMasked).toBe(masked);
expect(pick(body, "anthropic", "claude-sonnet-4-6-preview").envKeyMasked).toBe(masked);
expect(pick(body, "deepseek", "deepseek-v4-pro").envKeyMasked).toBeUndefined();
const openaiMasked = `${openaiValue.slice(0, 4)}…${openaiValue.slice(-4)}`;
const rePointed = pick(body, "anthropic", "claude-via-gateway");
expect(rePointed.envKey).toBe("OPENAI_API_KEY");
expect(rePointed.envKeyMasked).toBe(openaiMasked);
const gateway = pick(body, "openrouter", "xiaomi/mimo-v2.5");
expect(gateway.envKey).toBeUndefined();
expect(gateway.envKeyMasked).toBeUndefined();
expect(pick(body, "custom", "my-model").envKey).toBe("OPENAI_API_KEY");
expect(pick(body, "custom", "my-model").envKeyMasked).toBeUndefined();
expect(pick(body, "custom", "my-local").envKey).toBeUndefined();
expect(pick(body, "custom", "my-local").envKeyMasked).toBeUndefined();
const vllm = pick(body, "vllm", "Qwen/Qwen3.8-27B");
expect(vllm.envKey).toBe("OPENAI_API_KEY");
expect(vllm.envKeyMasked).toBeUndefined();
const penguinMasked = "sk-p…-456";
expect(pick(body, "penguin-go", "gemini-3.8-flash").envKey).toBe("PENGUIN_GO_API_KEY");
expect(pick(body, "penguin-go", "gemini-3.8-flash").envKeyMasked).toBe(penguinMasked);
expect(pick(body, "penguin-go", "deepseek-flash").envKey).toBe("PENGUIN_GO_API_KEY");
expect(pick(body, "penguin-go", "deepseek-flash").envKeyMasked).toBe(penguinMasked);
} finally {
for (const [key, value] of [
["ANTHROPIC_API_KEY", saved.anthropic],
["DEEPSEEK_API_KEY", saved.deepseek],
["OPENAI_API_KEY", saved.openai],
["PENGUIN_GO_API_KEY", saved.penguin],
] as const) {
if (value === undefined) delete process.env[key];
else process.env[key] = value;
}
}
});
it("PUT a custom model: the vision flag persists; the openai protocol falls back to OPENAI_API_KEY; provider is required", async () => {
// A vendor-group row on the openai protocol, as a group header's add once wrote it.
await storeRows(t.root, projectId, [
{ provider: "anthropic", modelId: "claude-via-gateway", clientType: "openai" },
]);
const put = await api.put(url(), {
defaultModel: { provider: "custom", modelId: "my-model" },
models: [
{ provider: "custom", modelId: "my-model", clientType: "openai", vision: false },
{ provider: "custom", modelId: "opaque-model" },
{ provider: "anthropic", modelId: "claude-via-gateway", clientType: "openai" },
],
});
expect(put.status).toBe(200);
const body = (await put.json()) as ModelsResponse;
const mine = pick(body, "custom", "my-model");
expect(mine.displayName).toBeUndefined();
expect(mine.vision).toBe(false);
expect(mine.envKey).toBe("OPENAI_API_KEY");
// The request carried the deprecated bare "openai" alias (pre-0.4.2 clients/configs):
// it is normalized to the canonical "openai-chat" on write and reported canonically.
expect(mine.clientType).toBe("openai-chat");
// Off-catalog model without client_type: no env-var fallback; with no vision annotation the field is omitted (default = supported).
const opaque = pick(body, "custom", "opaque-model");
expect("vision" in opaque).toBe(false);
expect(opaque.envKey).toBeUndefined();
// Listed under one vendor's group but using the openai protocol (a model added at a group header):
// AgentHub's openai client actually reads OPENAI_API_KEY, so the env fallback reports that, not the vendor's var name.
expect(pick(body, "anthropic", "claude-via-gateway").envKey).toBe("OPENAI_API_KEY");
// GET again: vision was persisted (not just echoed from the request body), and the disk
// stores the canonical client-type spelling.
const again = (await (await api.get(url())).json()) as ModelsResponse;
expect(pick(again, "custom", "my-model").vision).toBe(false);
expect(pick(again, "custom", "my-model").clientType).toBe("openai-chat");
const toml = await readFile(path.join(t.root, projectId, ".project_config.toml"), "utf8");
expect(toml).toContain('client_type = "openai-chat"');
expect(toml).not.toContain('"openai"');
// vision shape check: non-boolean → 400.
const bad = await api.put(url(), {
models: [{ provider: "custom", modelId: "my-model", vision: "yes" }],
});
expect(bad.status).toBe(400);
// Missing provider → 400 (refs are always a pair; neither half may be omitted).
const noProvider = await api.put(url(), { models: [{ modelId: "my-model" }] });
expect(noProvider.status).toBe(400);
});
it("PUT refuses an id a vendor group cannot route when the request introduces it, and carries a stored one through", async () => {
const cfgFile = path.join(t.root, projectId, ".project_config.toml");
// A row in the shape a vendor-group add used to produce: no client_type, and an id
// AgentHub places nowhere, so it fails at request time with its "is not supported"
// sentence. Rows like this exist in configs written before the rule below.
await writeFile(
cfgFile,
["[[models]]", 'provider = "deepseek"', 'model_id = "qwen/qwen3.8-flash-next"'].join("\n"),
"utf8",
);
// Already stored: the whole-table PUT writes it back untouched. The models page sends the
// entire table on every save, so refusing it here would make one legacy row block every
// later edit of every other row.
const kept = await api.put(url(), {
models: [
{ provider: "deepseek", modelId: "qwen/qwen3.8-flash-next", contextWindow: 65536 },
{ provider: "custom", modelId: "mine", clientType: "openai-chat" },
],
});
expect(kept.status).toBe(200);
const keptBody = (await kept.json()) as ModelsResponse;
expect(pick(keptBody, "deepseek", "qwen/qwen3.8-flash-next").contextWindow).toBe(65536);
// New in this request: refused, by a code the frontend can localize and a message naming
// the entry and the way out.
const added = await api.put(url(), {
models: [
{ provider: "deepseek", modelId: "qwen/qwen3.8-flash-next" },
{ provider: "deepseek", modelId: "another-fine-tune" },
],
});
expect(added.status).toBe(400);
const error = ((await added.json()) as ErrorBody).error;
expect(error.code).toBe("model_not_routable");
expect(error.message).toContain("another-fine-tune");
expect(error.message).toContain("custom group");
// A changed id is a different entry, so the rename is judged like a new one.
const renamed = await api.put(url(), {
models: [
{
provider: "deepseek",
modelId: "qwen/qwen3.8-flash-next-preview",
renamedFrom: { provider: "deepseek", modelId: "qwen/qwen3.8-flash-next" },
},
],
});
expect(renamed.status).toBe(400);
expect(((await renamed.json()) as ErrorBody).error.code).toBe("model_not_routable");
// So is moving a model into a vendor group: the group is the half of the key that changed.
const moved = await api.put(url(), {
models: [
{ provider: "deepseek", modelId: "qwen/qwen3.8-flash-next" },
{
provider: "deepseek",
modelId: "mine",
renamedFrom: { provider: "custom", modelId: "mine" },
},
],
});
expect(moved.status).toBe(400);
// Nothing above was written: the stored table is still the one the accepted PUT left.
const stored = (await (await api.get(url())).json()) as ModelsResponse;
const expected = ["custom\0mine", "deepseek\0qwen/qwen3.8-flash-next"];
expect(stored.models.map(pairKey).sort()).toEqual(expected.sort());
// The same id is accepted wherever the group answers the protocol question itself.
const custom = await api.put(url(), {
models: [
{ provider: "deepseek", modelId: "qwen/qwen3.8-flash-next" },
{
provider: "custom",
modelId: "qwen/qwen3.8-flash-next",
clientType: "openai-chat",
baseUrl: "https://gateway.example/v1",
},
],
});
expect(custom.status).toBe(200);
});
it("PUT refuses a new row that is no preset in a built-in group but custom and vLLM, and keeps every stored one", async () => {
// A row a user added to a gateway group while gateways still took them.
await storeRows(t.root, projectId, [
{ provider: "openrouter", modelId: "acme/legacy-pick", clientType: "openai-responses" },
{ provider: "custom", modelId: "mine", clientType: "openai-chat" },
]);
const legacy = {
provider: "openrouter",
modelId: "acme/legacy-pick",
clientType: "openai-responses",
};
const mine = { provider: "custom", modelId: "mine", clientType: "openai-chat" };
// New in this request, in two gateways and in a vendor group — the last one an id the
// vendor's client would route, so it is this rule and not routing that answers.
for (const entry of [
{ provider: "openrouter", modelId: "acme/new-pick", clientType: "openai-responses" },
{ provider: "tokendance", modelId: "acme/new-pick", clientType: "openai-chat" },
{ provider: "deepseek", modelId: "deepseek-v4-my-tune" },
]) {
const res = await api.put(url(), { models: [legacy, mine, entry] });
expect(res.status, entry.provider).toBe(400);
const error = ((await res.json()) as ErrorBody).error;
expect(error.code, entry.provider).toBe("model_not_addable");
expect(error.message).toContain(entry.modelId);
expect(error.message).toContain("custom group");
}
// Moving a row in from another group is adding it to this one.
const moved = await api.put(url(), {
models: [
legacy,
{
provider: "openrouter",
modelId: "mine",
clientType: "openai-responses",
renamedFrom: { provider: "custom", modelId: "mine" },
},
],
});
expect(moved.status).toBe(400);
expect(((await moved.json()) as ErrorBody).error.code).toBe("model_not_addable");
// Nothing above was written.
const before = (await (await api.get(url())).json()) as ModelsResponse;
expect(before.models.map(pairKey).sort()).toEqual(
["custom\0mine", "openrouter\0acme/legacy-pick"].sort(),
);
// The stored row keeps working: edited in place, then renamed inside its own group.
const edited = await api.put(url(), { models: [{ ...legacy, contextWindow: 65536 }, mine] });
expect(edited.status).toBe(200);
const renamed = await api.put(url(), {
models: [
{
provider: "openrouter",
modelId: "acme/renamed-pick",
clientType: "openai-responses",
renamedFrom: { provider: "openrouter", modelId: "acme/legacy-pick" },
},
mine,
],
});
expect(renamed.status).toBe(200);
const after = (await renamed.json()) as ModelsResponse;
expect(after.models.map(pairKey).sort()).toEqual(
["custom\0mine", "openrouter\0acme/renamed-pick"].sort(),
);
// A group's own preset is welcome back, and custom, vLLM and user-defined groups take
// anything.
const open = await api.put(url(), {
models: [
{ provider: "openrouter", modelId: "acme/renamed-pick", clientType: "openai-responses" },
mine,
{ provider: "deepseek", modelId: "deepseek-v4-pro" },
{
provider: "vllm",
modelId: "my-served-model",
clientType: "openai-chat-vllm-adapter",
baseUrl: "http://127.0.0.1:8000/v1",
},
{ provider: "my-own-group", modelId: "anything", clientType: "openai-chat" },
],
});
expect(open.status).toBe(200);
});
it('a config stored before the AgentHub 0.4.2 rename (client_type = "openai") keeps working: GET reports the canonical openai-chat', async () => {
// Simulate an existing user config written by an older harness version: the deprecated
// bare "openai" spelling on disk. Reading must not error and must report the canonical
// spelling (normalize-on-read, no disk rewrite needed until the next PUT).
const cfgFile = path.join(t.root, projectId, ".project_config.toml");
await writeFile(
cfgFile,
[
"[[models]]",
'provider = "custom"',
'model_id = "legacy-openai-model"',
'client_type = "openai"',
'base_url = "https://legacy.example/v1"',
].join("\n"),
"utf8",
);
const body = (await (await api.get(url())).json()) as ModelsResponse;
const legacy = pick(body, "custom", "legacy-openai-model");
expect(legacy.clientType).toBe("openai-chat");
// Its own endpoint, so no environment fallback is reported for it.
expect(legacy.envKey).toBeUndefined();
});
it("a configured model that has since been dropped from the built-in catalog still loads, keeps its data, and stays usable", async () => {
// Migration guard for catalog removals (the 2026-08-18 inclusionai/ling-3.0-flash:free
// delisting is the live example): a user who configured the preset before it was removed
// keeps a row on disk that catalogEntryFor no longer matches. Everything presetModelEntries
// persisted must survive — only display_name lived solely in the catalog — and the row must
// stay a normal, selectable entry rather than being pruned, rejected or blanked.
const cfgFile = path.join(t.root, projectId, ".project_config.toml");
await writeFile(
cfgFile,
[
"[[models]]",
'provider = "openrouter"',
'model_id = "vendor/removed-from-catalog:free"',
"context_window = 262144",
'client_type = "openai-chat"',
'base_url = "https://openrouter.ai/api/v1"',
'api_key = "sk-still-here"',
"vision = false",
"",
"[models.pricing]",
'unit = "usd_per_mtok"',
"cache_read = 0.0",
"cache_write = 0.0",
"output = 0.0",
].join("\n"),
"utf8",
);
// Sanity: the id really is absent from the built-in catalog, so this exercises the
// off-catalog path rather than an accidental match.
expect(
MODEL_CATALOG.some(
(m) => m.provider === "openrouter" && m.modelId === "vendor/removed-from-catalog:free",
),
).toBe(false);
const body = (await (await api.get(url())).json()) as ModelsResponse;
const orphan = pick(body, "openrouter", "vendor/removed-from-catalog:free");
// Everything stored in TOML is read straight back — pricing and context window come from
// the file, not the catalog, so a $0 free-tier row keeps costing 0 rather than going
// "unknown".
expect(orphan.contextWindow).toBe(262144);
expect(orphan.pricing).toEqual({ cacheRead: 0, cacheWrite: 0, output: 0 });
expect(orphan.vision).toBe(false);
expect(orphan.clientType).toBe("openai-chat");
// It still carries the gateway's endpoint, so it gets no environment fallback.
expect(orphan.envKey).toBeUndefined();
// The credential survives, masked; the base URL is still inlined on the entry.
expect(orphan.credential?.baseUrl).toBe("https://openrouter.ai/api/v1");
expect(orphan.credential?.apiKeyMasked).toBeTruthy();
// The one real loss: display_name only ever lived in the catalog, so the UI falls back to
// the raw upstream id.
expect(orphan.displayName).toBeUndefined();
// Still a legal default-model target — nothing validates the pair against the catalog.
const put = await api.put(url(), {
models: [
{
provider: "openrouter",
modelId: "vendor/removed-from-catalog:free",
contextWindow: 262144,
clientType: "openai-chat",
},
],
defaultModel: { provider: "openrouter", modelId: "vendor/removed-from-catalog:free" },
});
expect(put.status).toBe(200);
const after = (await (await api.get(url())).json()) as ModelsResponse;
expect(pick(after, "openrouter", "vendor/removed-from-catalog:free").isDefault).toBe(true);
});
it("PUT maxTokens: persisted as max_tokens and read back through GET; omitting it table-wide clears it; 0 / negative / non-numeric 400", async () => {
const put = await api.put(url(), {
models: [
{ provider: "custom", modelId: "local-qwen", clientType: "openai", maxTokens: 8000 },
],
});
expect(put.status).toBe(200);
expect(pick((await put.json()) as ModelsResponse, "custom", "local-qwen").maxTokens).toBe(8000);
// Round-trips through disk (persisted as snake_case on the entry, not just echoed back).
const again = (await (await api.get(url())).json()) as ModelsResponse;
expect(pick(again, "custom", "local-qwen").maxTokens).toBe(8000);
const toml = await readFile(path.join(t.root, projectId, ".project_config.toml"), "utf8");
expect(toml).toContain("max_tokens = 8000");
// Full-table PUT omitting the field clears the annotation (same replace semantics as vision/contextWindow).
const cleared = await api.put(url(), {
models: [{ provider: "custom", modelId: "local-qwen", clientType: "openai" }],
});
expect(cleared.status).toBe(200);
const clearedRow = pick((await cleared.json()) as ModelsResponse, "custom", "local-qwen");
expect("maxTokens" in clearedRow).toBe(false);
expect(
await readFile(path.join(t.root, projectId, ".project_config.toml"), "utf8"),
).not.toContain("max_tokens");
// Not a positive integer → 400 with the field-labelled message (nothing written).
for (const bad of [0, -5, 1.5, "8000"]) {
const res = await api.put(url(), {
models: [{ provider: "custom", modelId: "local-qwen", maxTokens: bad }],
});
expect(res.status).toBe(400);
const body = (await res.json()) as { error: { message: string } };
expect(body.error.message).toContain("models[0].maxTokens");
}
});
it("the same model_id can coexist under different providers (paired keys, neither overwrites the other)", async () => {
// The gateway's copy was added by hand, before gateways stopped taking such rows.
await storeRows(t.root, projectId, [
{ provider: "siliconflow", modelId: "kimi-k2.6", clientType: "openai" },
]);
const put = await api.put(url(), {
models: [
{ provider: "moonshot", modelId: "kimi-k2.6", apiKey: "sk-official-aaaa1111" },
{
provider: "siliconflow",
modelId: "kimi-k2.6",
clientType: "openai",
apiKey: "sk-gateway-bbbb2222",
},
],
});
expect(put.status).toBe(200);
const body = (await put.json()) as ModelsResponse;
expect(body.models).toHaveLength(2);
// The two entries are independent: credential and envKey don't cross over.
expect(pick(body, "moonshot", "kimi-k2.6").credential?.apiKeyMasked).toBe("sk-o…1111");
expect(pick(body, "moonshot", "kimi-k2.6").envKey).toBe("MOONSHOT_API_KEY");
expect(pick(body, "siliconflow", "kimi-k2.6").credential?.apiKeyMasked).toBe("sk-g…2222");
// Saved without the gateway's base URL, this row's client talks to api.openai.com — so
// the OpenAI variable honestly is its fallback (the preset, which carries the gateway
// endpoint, gets none; see the env-fallback test above).
expect(pick(body, "siliconflow", "kimi-k2.6").envKey).toBe("OPENAI_API_KEY");
// Round-trips through disk unchanged.
const again = (await (await api.get(url())).json()) as ModelsResponse;
expect(again.models.map(pairKey).sort()).toEqual(body.models.map(pairKey).sort());
});
});
describe("default_project presets", () => {
let t: TestApp;
let prevKey: string | undefined;
beforeEach(() => {
// The default model (DeepSeek) uses the OpenAI protocol, whose SDK requires a credential at
// **construction time** — this case creates a Session, so we stuff in a fake key (no real request is sent). CI has no keys.
prevKey = process.env.OPENAI_API_KEY;
process.env.OPENAI_API_KEY = "test-key-not-used";
});
afterEach(async () => {
if (prevKey === undefined) delete process.env.OPENAI_API_KEY;
else process.env.OPENAI_API_KEY = prevKey;
await t.cleanup();
});
it("when the seeded admin adopts default_project, the preset models and the default model are filled in (no prior .project_config.toml)", async () => {
// The default_project shared by admin seeding and the CLI (the dir already exists, so writeInitialConfig is skipped).
t = await createTestApp();
const { cookie } = await loginAdmin(t.app);
const api = apiClient(t.app, cookie);
const res = await api.get("/api/projects/default_project/models");
expect(res.status).toBe(200);
const body = (await res.json()) as ModelsResponse;
expect(body.defaultModel).toEqual({
provider: "deepseek",
modelId: "deepseek-flash",
});
expect(body.models.map(pairKey)).toEqual(catalogPairs);
// Presets backfilled here bring their promotions too, seeded once the Project row exists.
expect(body.models.filter((m) => m.discount !== undefined)).toHaveLength(
presetPromotions().length,
);
// The point of presets is "works out of the box": creating a Session should succeed without passing a model ref.
const created = await api.post(
"/api/projects/default_project/agents/default_agent/sessions",
{},
);
expect(created.status).toBe(201);
const { session } = (await created.json()) as SessionCreateResponse;
expect(session.provider).toBe("deepseek");
expect(session.modelId).toBe("deepseek-flash");
});
it("a default_project that already has models configured is left untouched (existing CLI config is not overwritten)", async () => {
// First have the "CLI" write a config with a single custom model, then admin seeding adopts it.
t = await createTestApp({
beforeSeed: async (root) => {
await wire(ProjectConfigService, { paths: { root } }).writeRaw("default_project", {
default_model: { provider: "custom", model_id: "cli-model" },
models: [{ provider: "custom", model_id: "cli-model", context_window: 1234 }],
});
},
});
const { cookie } = await loginAdmin(t.app);
const api = apiClient(t.app, cookie);
const body = (await (
await api.get("/api/projects/default_project/models")
).json()) as ModelsResponse;
expect(body.defaultModel).toEqual({ provider: "custom", modelId: "cli-model" });
expect(body.models.map(pairKey)).toEqual(["custom\0cli-model"]);
expect(body.models[0]!.contextWindow).toBe(1234);
});
});
describe("model-reference rekeying and the connectivity test", () => {
let t: TestApp;
let api: ReturnType<typeof apiClient>;
let projectId: string;
const url = () => `/api/projects/${projectId}/models`;
const testUrl = () => `${url()}/test`;
beforeAll(async () => {
t = await createTestApp();
const { cookie } = await provisionUser(t.app, "carol");
api = apiClient(t.app, cookie);
});
afterAll(async () => {
await t.cleanup();
});
let projects = 0;
beforeEach(async () => {
projects += 1;
const created = (await (
await api.post("/api/projects", {
projectId: `carol-rename_${projects}`,
name: "Rename project",
})
).json()) as ProjectCreateResponse;
projectId = created.project.projectId;
});
it("renamedFrom migrates the credential and the config; the default / vision model pointers follow the rekey", async () => {
await api.put(url(), {
defaultModel: { provider: "custom", modelId: "old-id" },
visionModel: { provider: "custom", modelId: "old-id" },
models: [
{
provider: "custom",
modelId: "old-id",
contextWindow: 4096,
apiKey: "sk-secret-abcd1234",
},
],
});
const put = await api.put(url(), {
models: [
{
provider: "custom",
modelId: "new-id",
renamedFrom: { provider: "custom", modelId: "old-id" },
contextWindow: 4096,
},
],
});
expect(put.status).toBe(200);
const body = (await put.json()) as ModelsResponse;
expect(body.models.map(pairKey)).toEqual(["custom\0new-id"]);
// The credential migrates with the key change (masked value visible), and the pointer follows the new ref.
expect(body.models[0]!.credential?.apiKeyMasked).toBe("sk-s…1234");
expect(body.defaultModel).toEqual({ provider: "custom", modelId: "new-id" });
expect(body.visionModel).toEqual({ provider: "custom", modelId: "new-id" });
// Round-trips through disk unchanged (not just echoed).
const again = (await (await api.get(url())).json()) as ModelsResponse;
expect(again.models[0]!.credential?.apiKeyMasked).toBe("sk-s…1234");
expect(again.defaultModel).toEqual({ provider: "custom", modelId: "new-id" });
});
it("a group change is a rekey too: the credential migrates with it; envKey resolves by client, not by group (PRN-021)", async () => {
// Move the preset DeepSeek model to another group (changing provider is a key change; the paired renamedFrom migrates it).
const put = await api.put(url(), {
models: [
{
provider: "deepseek",
modelId: "deepseek-v4-pro",
apiKey: "sk-secret-abcd1234",
vision: false,
},
],
});
expect(put.status).toBe(200);
expect(pick((await put.json()) as ModelsResponse, "deepseek", "deepseek-v4-pro").envKey).toBe(
"DEEPSEEK_API_KEY",
);
const put2 = await api.put(url(), {
models: [
{
provider: "custom",
modelId: "deepseek-v4-pro",
renamedFrom: { provider: "deepseek", modelId: "deepseek-v4-pro" },
vision: false,
},
],
});
expect(put2.status).toBe(200);
const moved = ((await put2.json()) as ModelsResponse).models[0]!;
expect(moved.provider).toBe("custom");
expect(moved.modelId).toBe("deepseek-v4-pro");
expect(moved.credential?.apiKeyMasked).toBe("sk-s…1234");
// The env fallback follows client resolution — with no client_type on the entry,
// AgentHub still routes by id to the DeepSeek client (reading DEEPSEEK_API_KEY), regardless of group membership.
expect(moved.envKey).toBe("DEEPSEEK_API_KEY");
});
it("serializes a ModelScope settings save with a group credential write", async () => {
const svc = wire(ProjectConfigService, { paths: { root: t.root } });
const raceProject = "modelscope-save-refresh-race";
await svc.writeRaw(raceProject, {
models: [
{
provider: "modelscope",
model_id: "Qwen/Qwen3.8-27B",
api_key: "old-access",
client_type: "openai-chat-vllm-adapter",
},
],
});
const readRaw = svc.readRaw.bind(svc);
let releaseRead!: () => void;
const readGate = new Promise<void>((resolve) => (releaseRead = resolve));
let announceRead!: () => void;
const readStarted = new Promise<void>((resolve) => (announceRead = resolve));
let firstRead = true;
svc.readRaw = async (id: string) => {
const raw = await readRaw(id);
if (firstRead) {
firstRead = false;
announceRead();
await readGate;
}
return raw;
};
const save = svc.updateModels(raceProject, {
models: [
{
provider: "modelscope",
modelId: "Qwen/Qwen3.8-27B",
clientType: "openai-chat-vllm-adapter",
},
],
});
await readStarted;
let credentialWriteSettled = false;
const credentialWrite = svc
.setGroupApiKey(raceProject, "modelscope", "new-access")
.finally(() => {
credentialWriteSettled = true;
});
await Promise.resolve();
expect(credentialWriteSettled).toBe(false);
releaseRead();
await Promise.all([save, credentialWrite]);
const stored = await readRaw(raceProject);
expect((stored.models as Array<Record<string, unknown>>)[0]?.api_key).toBe("new-access");
expect(JSON.stringify(stored)).not.toContain("refreshToken");
expect(JSON.stringify(stored)).not.toContain("accessTokenExpiresAt");
});
it("the display name is editable: not persisted when it matches the built-in catalog; provider is always persisted as an entry field", async () => {
// Preset model: saved as-is → the display name falls back to the built-in catalog, and display_name isn't written to the config file.
await api.put(url(), {
models: [
{ provider: "openai", modelId: "gpt-5.5", displayName: "GPT-5.5" },
{ provider: "openai", modelId: "gpt-5.4", displayName: "My GPT" },
],
});
const body = (await (await api.get(url())).json()) as ModelsResponse;
expect(pick(body, "openai", "gpt-5.5").displayName).toBe("GPT-5.5");
// Edited one: the display name takes effect per the user's setting.
expect(pick(body, "openai", "gpt-5.4").displayName).toBe("My GPT");
// Clean on disk: unchanged preset models don't write display_name; provider is stored as a separate column, no concatenated string.
const toml = await readFile(path.join(t.root, projectId, ".project_config.toml"), "utf8");
expect(toml).not.toContain('display_name = "GPT-5.5"');
expect(toml).toContain('display_name = "My GPT"');
expect(toml).toContain('provider = "openai"');
expect(toml).not.toContain("openai/gpt-5.5");
});
// 2026-08-31 is a Monday: 01:30Z is 09:30 in Beijing (peak), 12:00Z is 20:00 (off-peak).
const PEAK = new Date("2026-08-31T01:30:00Z");
const OFF_PEAK = new Date("2026-08-31T12:00:00Z");
it("a scheduled row stores the one stable price, and the tier is derived from a Request's own instant", async () => {
// The number on disk is the peak price whatever the hour; which tier a Request bills at is
// decided by when it completed, so the same stored price bills twice as much at 09:30 as at
// 20:00 on the same Monday.
const svc = wire(ProjectConfigService, { paths: { root: t.root } });
const peak = await svc.getPricing(projectId, "deepseek", "deepseek-flash", PEAK);
const catalogPeak = catalogEntryFor("deepseek", "deepseek-flash")!.pricing!;
expect(peak!.output).toBe(catalogPeak.output);
expect(peak!.cacheRead).toBe(catalogPeak.cache_read);
const off = await svc.getPricing(projectId, "deepseek", "deepseek-flash", OFF_PEAK);
expect(off!.output).toBeCloseTo(catalogPeak.output / 2, 5);
expect(off!.cacheRead).toBeCloseTo(catalogPeak.cache_read / 2, 6);
});
it("a hand-edited price on a scheduled row is billed as typed, in both tiers", async () => {
const body = (await (await api.get(url())).json()) as ModelsResponse;
const entries = body.models.map((m) => ({
provider: m.provider,
modelId: m.modelId,
...(m.pricing
? {
pricing:
m.provider === "deepseek" && m.modelId === "deepseek-flash"
? { cacheRead: 1, cacheWrite: 2, output: 3 }
: {
cacheRead: m.pricing.cacheRead,
cacheWrite: m.pricing.cacheWrite,
output: m.pricing.output,
},
}
: {}),
}));
await api.put(url(), { models: entries });
const svc = wire(ProjectConfigService, { paths: { root: t.root } });
// Nothing here knows whether 3 is a peak rate, so halving it would invent a discount: the
// typed number is what bills, whenever the Request runs.
const typed = { cacheRead: 1, cacheWrite: 2, output: 3 };
expect(await svc.getPricing(projectId, "deepseek", "deepseek-flash", PEAK)).toEqual(typed);
expect(await svc.getPricing(projectId, "deepseek", "deepseek-flash", OFF_PEAK)).toEqual(typed);
});
it("a stored promotion takes its fraction off the list price on disk, multiplied with a scheduled row's off-peak tier", async () => {
const svc = t.deps.projectConfigService;
const off = (v: number, fraction: number): number => Math.round(v * (1 - fraction) * 1e6) / 1e6;
// Seeded with the Project: TokenDance's glm-5.3-flash is on 10% off.
const list = catalogEntryFor("tokendance", "glm-5.3-flash")!.pricing!;
const promoted = {
cacheRead: off(list.cache_read, 0.1),
cacheWrite: off(list.cache_write, 0.1),
output: off(list.output, 0.1),
};
expect(await svc.getPricing(projectId, "tokendance", "glm-5.3-flash", PEAK)).toEqual(promoted);
// A scheduled row still at the catalog's peak price, with a promotion of its own: peak bills
// the promotion alone, off-peak the half-price tier with the promotion taken off it too.
const peak = catalogEntryFor("deepseek", "deepseek-v4-flash")!.pricing!;
await svc.updateModels(projectId, {
models: [
{
provider: "deepseek",
modelId: "deepseek-v4-flash",
pricing: {
cacheRead: peak.cache_read,
cacheWrite: peak.cache_write,
output: peak.output,
},
discount: 0.2,
},
],
});
const peakRates = (await svc.getPricing(projectId, "deepseek", "deepseek-v4-flash", PEAK))!;
expect(peakRates.output).toBe(off(peak.output, 0.2));
const offPeakRates = (await svc.getPricing(
projectId,
"deepseek",
"deepseek-v4-flash",
OFF_PEAK,
))!;
expect(offPeakRates.output).toBe(off(off(peak.output, 0.5), 0.2));
});
it("PUT discount: a declared one is stored and null clears it; an omitted one survives only on an unrenamed row at an unchanged price", async () => {
const price = { cacheRead: 0.1, cacheWrite: 1, output: 4 };
const row = (modelId: string, extra: Record<string, unknown> = {}) => ({
provider: "custom",
modelId,
pricing: price,
...extra,
});
const stored = () =>
t.deps.db
.prepare(
"SELECT model_id, discount FROM model_promotions WHERE project_id = ? ORDER BY model_id",
)
.all(projectId);
const declared = await api.put(url(), {
models: ["kept", "repriced", "renamed", "cleared", "dropped"].map((id) =>
row(id, { discount: 0.2 }),
),
});
expect(declared.status).toBe(200);
expect(
((await declared.json()) as ModelsResponse).models.map((m) => [m.modelId, m.discount]),
).toEqual([
["kept", 0.2],
["repriced", 0.2],
["renamed", 0.2],
["cleared", 0.2],
["dropped", 0.2],
]);
const next = await api.put(url(), {
models: [
row("kept"),
row("repriced", { pricing: { ...price, output: 5 } }),
row("renamed-to", { renamedFrom: { provider: "custom", modelId: "renamed" } }),
row("cleared", { discount: null }),
],
});
expect(next.status).toBe(200);
expect(
((await next.json()) as ModelsResponse).models.map((m) => [m.modelId, m.discount]),
).toEqual([
["kept", 0.2],
["repriced", undefined],
["renamed-to", undefined],
["cleared", undefined],
]);
// The dropped row's promotion went with it rather than lingering unseen.
expect(stored()).toEqual([{ model_id: "kept", discount: 0.2 }]);
// Out of range or not a number: 400 before anything is written, so neither the file nor
// the table moves — not even for the valid row beside it, whose new price would otherwise
// have cleared its promotion.
const cfgFile = path.join(t.root, projectId, ".project_config.toml");
const before = await readFile(cfgFile, "utf8");
for (const bad of [0, 1, -0.2, 1.5, "0.2"]) {
const res = await api.put(url(), {
models: [
row("kept", { pricing: { ...price, output: 9 } }),
row("other", { discount: bad }),
],
});
expect(res.status, String(bad)).toBe(400);
}
expect(await readFile(cfgFile, "utf8")).toBe(before);
expect(stored()).toEqual([{ model_id: "kept", discount: 0.2 }]);
});
it("an absent display name inherits the catalog's; only an empty one clears it", async () => {
// Two different requests, and reading them as one is how a client that simply has no name
// for a model — one built from the catalog, which stores no name of its own — destroys the
// name of every model it sends.
const cfgFile = path.join(t.root, projectId, ".project_config.toml");
await api.put(url(), { models: [{ provider: "openai", modelId: "gpt-5.5" }] });
const inherited = (await (await api.get(url())).json()) as ModelsResponse;
expect(pick(inherited, "openai", "gpt-5.5").displayName).toBe("GPT-5.5");
expect(await readFile(cfgFile, "utf8")).not.toContain("display_name");
// Cleared: the empty string records the deletion, so the name does not come back on the
// next load, and the GET reports it as an empty name rather than as no name — the whole
// table comes back on the next PUT, where "no name" would ask for the catalog's again.
await api.put(url(), {
models: [{ provider: "openai", modelId: "gpt-5.5", displayName: "" }],
});
const body = (await (await api.get(url())).json()) as ModelsResponse;
expect(pick(body, "openai", "gpt-5.5").displayName).toBe("");
expect(await readFile(cfgFile, "utf8")).toContain('display_name = ""');
// And it survives that round trip, exactly as the page performs it.
await api.put(url(), {
models: [{ provider: "openai", modelId: "gpt-5.5", displayName: "" }],
});
const again = (await (await api.get(url())).json()) as ModelsResponse;
expect(pick(again, "openai", "gpt-5.5").displayName).toBe("");
});
it("a cleared name is restorable, and clearing a model the catalog does not name writes nothing", async () => {
await api.put(url(), {
models: [{ provider: "openai", modelId: "gpt-5.5", displayName: "" }],
});
// Naming it again drops the marker rather than leaving both on disk.
await api.put(url(), {
models: [{ provider: "openai", modelId: "gpt-5.5", displayName: "Renamed" }],
});
const body = (await (await api.get(url())).json()) as ModelsResponse;
expect(pick(body, "openai", "gpt-5.5").displayName).toBe("Renamed");
let toml = await readFile(path.join(t.root, projectId, ".project_config.toml"), "utf8");
expect(toml).not.toContain('display_name = ""');
// A model outside the catalog has no name to inherit, so absence already says "no name"
// and the marker would be noise in the file — an explicitly empty one included.
await api.put(url(), {
models: [
{ provider: "openai", modelId: "gpt-5.5", displayName: "Renamed" },
{ provider: "custom", modelId: "my-own-model", displayName: "" },
],
});
toml = await readFile(path.join(t.root, projectId, ".project_config.toml"), "utf8");
expect(toml).not.toContain('display_name = ""');
const after = (await (await api.get(url())).json()) as ModelsResponse;
expect(pick(after, "custom", "my-own-model").displayName).toBeUndefined();
});
it("an invalid renamedFrom is 400; rekeying without renamedFrom equals delete-old-then-create-new (the credential does not migrate)", async () => {
await api.put(url(), {
models: [{ provider: "custom", modelId: "m-a", apiKey: "sk-secret-abcd1234" }],
});
// Invalid shape: renamedFrom must be a { provider, modelId } pair object; a string is always 400.
const bad = await api.put(url(), {
models: [{ provider: "custom", modelId: "m-b", renamedFrom: "custom/m-a" }],
});
expect(bad.status).toBe(400);
// Giving only half a ref (missing provider) is also 400 — neither half of a ref may be omitted.
const half = await api.put(url(), {
models: [{ provider: "custom", modelId: "m-b", renamedFrom: { modelId: "m-a" } }],
});
expect(half.status).toBe(400);
const plain = await api.put(url(), { models: [{ provider: "custom", modelId: "m-b" }] });
const body = (await plain.json()) as ModelsResponse;
expect(body.models.map(pairKey)).toEqual(["custom\0m-b"]);
expect(body.models[0]!.credential).toBeUndefined();
});
it("the connectivity test sends the entry's upstream model_id (the reference pair travels with the request body)", async () => {
// Local openai-compatible endpoint: records the model field from the request body, then always rejects with 401 (never hits the network).
const seenModels: string[] = [];
const server = createServer((req, res) => {
let raw = "";
req.on("data", (chunk: Buffer) => (raw += chunk.toString("utf8")));
req.on("end", () => {
try {
seenModels.push((JSON.parse(raw) as { model?: string }).model ?? "");
} catch {
seenModels.push("");
}
res.statusCode = 401;
res.setHeader("content-type", "application/json");
res.end(JSON.stringify({ error: { message: "test-reject", type: "invalid_request" } }));
});
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const port = (server.address() as AddressInfo).port;
try {
await api.put(url(), {
models: [
{
provider: "custom",
modelId: "actual-upstream-model",
clientType: "openai",
apiKey: "sk-test-local",
baseUrl: `http://127.0.0.1:${port}/v1`,
},
],
});
const res = await api.post(testUrl(), {
provider: "custom",
modelId: "actual-upstream-model",
});
expect(res.status).toBe(200);
const body = (await res.json()) as ModelTestResponse;
expect(body.ok).toBe(false);
// What's sent is exactly the entry's model_id (under column storage it's the upstream id itself, no concatenated string).
expect(seenModels).toContain("actual-upstream-model");
expect(seenModels).not.toContain("custom/actual-upstream-model");
} finally {
await new Promise<void>((resolve) => server.close(() => resolve()));
}
});
it("the connectivity-test request body carries no tools or tool_choice (an empty tool list is omitted entirely, so strict endpoints such as vLLM no longer 400)", async () => {
// The probe runs with an empty tool list. The wire body must omit `tools` entirely —
// `tools: []` is rejected by strict OpenAI-compatible servers (vLLM: "tools must not be an
// empty array") — and must never carry `tool_choice`.
const bodies: Record<string, unknown>[] = [];
const server = createServer((req, res) => {
let raw = "";
req.on("data", (chunk: Buffer) => (raw += chunk.toString("utf8")));
req.on("end", () => {
try {
bodies.push(JSON.parse(raw) as Record<string, unknown>);
} catch {
bodies.push({});
}
res.statusCode = 401;
res.setHeader("content-type", "application/json");
res.end(JSON.stringify({ error: { message: "test-reject", type: "invalid_request" } }));
});
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
const port = (server.address() as AddressInfo).port;
try {
const res = await api.post(testUrl(), {
provider: "custom",
modelId: "probe-wire-model",
clientType: "openai",
apiKey: "sk-test-local",
baseUrl: `http://127.0.0.1:${port}/v1`,
});
expect(res.status).toBe(200);
expect(bodies.length).toBeGreaterThan(0);
for (const body of bodies) {
expect("tools" in body).toBe(false);
expect("tool_choice" in body).toBe(false);
}
} finally {
await new Promise<void>((resolve) => server.close(() => resolve()));
}
});
it("connectivity test: both a saved model and a **not-yet-saved** custom model can be tested (the LLM layer throws nothing, every outcome converges)", async () => {
// Both endpoints refuse the key they are handed.
const network = stubFetch(() =>
jsonResponse(
{ error: { message: "Incorrect API key provided.", code: "invalid_api_key" } },
401,
),
);
try {
await api.put(url(), {
models: [{ provider: "openai", modelId: "gpt-5.5", apiKey: "sk-invalid-key-for-test" }],
});
const saved = await api.post(testUrl(), { provider: "openai", modelId: "gpt-5.5" });
expect(saved.status).toBe(200);
const savedBody = (await saved.json()) as ModelTestResponse;
expect(savedBody.ok).toBe(false);
expect(typeof savedBody.message).toBe("string");
// "Test before save" for adding a custom model: the model isn't in the config, so all params come from the request body.
const unsaved = await api.post(testUrl(), {
provider: "custom",
modelId: "my-new-model",
apiKey: "sk-invalid",
baseUrl: "https://example.invalid/v1",
clientType: "openai",
});
expect(unsaved.status).toBe(200);
const unsavedBody = (await unsaved.json()) as ModelTestResponse;
expect(unsavedBody.ok).toBe(false);
expect(typeof unsavedBody.message).toBe("string");
// Each probe went to its own model's endpoint, with its own key.
const keyAt = (host: string) =>
network.calls.find((c) => c.url.startsWith(host))?.headers.get("authorization");
expect(keyAt("https://api.openai.com/")).toBe("Bearer sk-invalid-key-for-test");
expect(keyAt("https://example.invalid/v1")).toBe("Bearer sk-invalid");
} finally {
vi.unstubAllGlobals();
}
});
it("connectivity test: a model with no credential at all converges to ok:false instead of 500", async () => {
// A model using the OpenAI protocol: the provider SDK throws at **client construction** because
// the key is missing — if that construction were outside the try it would bubble up as a 500. Clear the env-var key so there's nowhere to get one (no real network request).
const prev = process.env.OPENAI_API_KEY;
delete process.env.OPENAI_API_KEY;
try {
await api.put(url(), {
models: [{ provider: "custom", modelId: "no-key-openai", clientType: "openai" }],
});
const res = await api.post(testUrl(), { provider: "custom", modelId: "no-key-openai" });
expect(res.status).toBe(200);
const body = (await res.json()) as ModelTestResponse;
expect(body.ok).toBe(false);
expect(typeof body.message).toBe("string");
} finally {
if (prev !== undefined) process.env.OPENAI_API_KEY = prev;
}
});
it("the connectivity test and the group speed test refuse a keyless gateway row without touching the environment", async () => {
// The #786 review's finding: a keyless gateway row handed AgentHub no key, and its
// generic client read OPENAI_API_KEY / ANTHROPIC_API_KEY itself — one speed test on a
// gateway group sent the user's vendor keys to the gateway once per model. Both paths run
// through testModel; both must refuse before any request leaves the process.
const saved = { openai: process.env.OPENAI_API_KEY, anthropic: process.env.ANTHROPIC_API_KEY };
process.env.OPENAI_API_KEY = "vendor-openai-must-not-cross";
process.env.ANTHROPIC_API_KEY = "vendor-anthropic-must-not-cross";
const network = stubFetch(() => {
throw new Error("no request may leave the process for a refused probe");
});
try {
// A gateway preset: the group's endpoint, no key. Both OpenAI-protocol gateways and the
// Messages-protocol custom preset (Atria — the path that carried the Anthropic key in the
// #786 finding) are refused, and the refusal is the harness's own sentence, never the
// SDK's text with a vendor variable in it.
for (const [provider, modelId] of [
["tokendance", "glm-5.3"],
["openrouter", "openai/gpt-5.5"],
["custom", "Atria-Dawn-Preview"],
] as const) {
expect(catalogEntryFor(provider, modelId)).toBeDefined();
for (const speed of [false, true]) {
const res = await api.post(testUrl(), { provider, modelId, speed });
expect(res.status).toBe(200);
const body = (await res.json()) as ModelTestResponse;
expect(body.ok).toBe(false);
expect(body.message).toMatch(/has no API key/);
expect(body.message).toMatch(/set the API key on the model entry/);
expect(body.message).not.toMatch(/OPENAI_API_KEY|ANTHROPIC_API_KEY|must-not-cross/);
}
}
// A not-yet-saved custom row pointed at a private server: same refusal.
const draft = await api.post(testUrl(), {
provider: "custom",
modelId: "draft-model",
clientType: "openai-chat",
baseUrl: "http://127.0.0.1:9/v1",
});
expect(((await draft.json()) as ModelTestResponse).message).toMatch(/has no API key/);
// The vision probe shares the rule.
const vision = await api.post(`${url()}/detect-vision`, {
provider: "tokendance",
modelId: "glm-5.3",
});
expect((await vision.json()) as ModelVisionDetectResponse).toMatchObject({
outcome: "failed",
message: expect.stringMatching(/has no API key/) as string,
});
expect(network.calls).toEqual([]);
} finally {
vi.unstubAllGlobals();
for (const [key, value] of [
["OPENAI_API_KEY", saved.openai],
["ANTHROPIC_API_KEY", saved.anthropic],
] as const) {
if (value === undefined) delete process.env[key];
else process.env[key] = value;
}
}
});
it("Penguin Go probes never substitute vendor environment keys", async () => {
const saved = {
gemini: process.env.GEMINI_API_KEY,
openai: process.env.OPENAI_API_KEY,
penguin: process.env.PENGUIN_GO_API_KEY,
};
process.env.GEMINI_API_KEY = "vendor-gemini-must-not-cross";
process.env.OPENAI_API_KEY = "vendor-openai-must-not-cross";
delete process.env.PENGUIN_GO_API_KEY;
try {
for (const modelId of ["gemini-3.8-flash", "deepseek-flash"]) {
const tested = await api.post(testUrl(), { provider: "penguin-go", modelId });
expect(tested.status).toBe(200);
expect((await tested.json()) as ModelTestResponse).toMatchObject({
ok: false,
message: expect.stringMatching(/has no API key.*PENGUIN_GO_API_KEY/) as string,
});
const vision = await api.post(`${url()}/detect-vision`, {
provider: "penguin-go",
modelId,
});
expect(vision.status).toBe(200);
expect((await vision.json()) as ModelVisionDetectResponse).toMatchObject({
outcome: "failed",
message: expect.stringMatching(/has no API key.*PENGUIN_GO_API_KEY/) as string,
});
}
process.env.PENGUIN_GO_API_KEY = "penguin-go-key-without-endpoint";
const noEndpoint = await api.post(testUrl(), {
provider: "penguin-go",
modelId: "gemini-3.8-flash",
baseUrl: null,
});
expect((await noEndpoint.json()) as ModelTestResponse).toMatchObject({
ok: false,
message: expect.stringMatching(/has no base URL/) as string,
});
} finally {
for (const [key, value] of [
["GEMINI_API_KEY", saved.gemini],
["OPENAI_API_KEY", saved.openai],
["PENGUIN_GO_API_KEY", saved.penguin],
] as const) {
if (value === undefined) delete process.env[key];
else process.env[key] = value;
}
}
});
it("connectivity test: clearApiKey does not fall back to the stored key (the current draft is what gets tested)", async () => {
// A key is already saved, but the test request carries clearApiKey — the server must **not** use
// the saved key. Clear the env var so "don't use the saved key" == no credential at all, so construction synchronously throws missing-credential (no network request, deterministic).
const prev = process.env.OPENAI_API_KEY;
delete process.env.OPENAI_API_KEY;
try {
await api.put(url(), {
models: [
{
provider: "custom",
modelId: "has-key-openai",
clientType: "openai",
apiKey: "sk-saved-key",
},
],
});
// Without clearApiKey: use the saved key and construction succeeds (would hit the real network; its result isn't asserted here).
// With clearApiKey: don't fall back to the saved key → no credential → synchronously resolves to "missing credential".
const cleared = await api.post(testUrl(), {
provider: "custom",
modelId: "has-key-openai",
clearApiKey: true,
});
expect(cleared.status).toBe(200);
const body = (await cleared.json()) as ModelTestResponse;
expect(body.ok).toBe(false);
// Missing-credential is thrown synchronously at construction (message contains "credentials"); if the saved key were still used, it wouldn't be this message.
expect(body.message ?? "").toMatch(/credential/i);
} finally {
if (prev !== undefined) process.env.OPENAI_API_KEY = prev;
}
});
});
describe("models update reaches loaded sessions (invalidation + live unlock)", () => {
let t: TestApp;
let api: ReturnType<typeof apiClient>;
let projectId: string;
/** Loader call count: how many times the manager (re)built a runtime from the index. */
let loads: number;
const insertSession = (sessionId: string, project: string): void => {
t.deps.sessionsRepo.insert(sessionRow(sessionId, { projectId: project, workspace: t.root }));
};
const putModels = (apiKey: string) =>
api.put(`/api/projects/${projectId}/models`, {
defaultModel: { provider: "custom", modelId: "m-inv" },
models: [{ provider: "custom", modelId: "m-inv", apiKey }],
});
beforeAll(async () => {
// Every runtime the manager builds from the index is an idle fake: no LLM call is made.
t = await createTestApp({
loader: {
load: async (row) => {
loads++;
return fakeSession(row.sessionId);
},
},
});
const { cookie } = await provisionUser(t.app, "inv_owner");
api = apiClient(t.app, cookie);
});
afterAll(async () => {
await t.cleanup();
});
let projects = 0;
beforeEach(async () => {
loads = 0;
projects += 1;
const created = (await (
await api.post("/api/projects", {
projectId: `inv_owner-models_${projects}`,
name: "invalidation",
})
).json()) as ProjectCreateResponse;
projectId = created.project.projectId;
});
it("PUT invalidates the Project's cached Session runtimes: the next Task re-resumes with the new key", async () => {
insertSession("models-sess-1", projectId);
const idle = () => t.deps.manager.statusOf("models-sess-1") === "idle";
// First Task builds the runtime (load #1); the second reuses the active-table entry.
await t.deps.manager.startTask("models-sess-1", [userText("a")]);
await waitFor(idle);
await t.deps.manager.startTask("models-sess-1", [userText("b")]);
await waitFor(idle);
expect(loads).toBe(1);
// Key update via the API: the cached runtime is stale, so the next Task re-resumes
// (the loader re-reads the Project config — the new api_key reaches the next Task).
expect((await putModels("sk-fresh-key-000111")).status).toBe(200);
await t.deps.manager.startTask("models-sess-1", [userText("c")]);
await waitFor(idle);
expect(loads).toBe(2);
// Reads don't invalidate: the rebuilt runtime is reused.
expect((await api.get(`/api/projects/${projectId}/models`)).status).toBe(200);
await t.deps.manager.startTask("models-sess-1", [userText("d")]);
await waitFor(idle);
expect(loads).toBe(2);
});
it("PUT publishes credentials_updated to the Project's existing session channels only", async () => {
insertSession("models-sess-live", projectId);
insertSession("models-sess-cold", projectId);
// A session of ANOTHER project must not receive the event.
const other = (await (
await api.post("/api/projects", { projectId: "inv_owner-other", name: "other" })
).json()) as ProjectCreateResponse;
insertSession("other-sess", other.project.projectId);
// Only the "live" session has an open channel (a subscribed tab); "cold" has none.
const events: ChannelEvent[] = [];
t.deps.channels.get("models-sess-live").subscribe((e) => events.push(e));
const otherEvents: ChannelEvent[] = [];
t.deps.channels.get("other-sess").subscribe((e) => otherEvents.push(e));
expect((await putModels("sk-live-key-000222")).status).toBe(200);
const types = events
.filter((e) => e.event === "server_event")
.map((e) => (JSON.parse(e.data) as { type: string }).type);
expect(types).toContain("credentials_updated");
// Cross-project channels stay silent, and no channel is created for unsubscribed sessions.
expect(
otherEvents
.filter((e) => e.event === "server_event")
.map((e) => (JSON.parse(e.data) as { type: string }).type),
).not.toContain("credentials_updated");
expect(t.deps.channels.peek("models-sess-cold")).toBeUndefined();
});
it("GET/PUT responses expose updatedAt (config file mtime) for the web's auth-dead gate", async () => {
const before = Date.now() - 60_000;
const put = await putModels("sk-ts-key-000333");
expect(put.status).toBe(200);
const putBody = (await put.json()) as ModelsResponse;
expect(typeof putBody.updatedAt).toBe("string");
expect(Date.parse(putBody.updatedAt!)).toBeGreaterThan(before);
const get = await api.get(`/api/projects/${projectId}/models`);
const getBody = (await get.json()) as ModelsResponse;
expect(typeof getBody.updatedAt).toBe("string");
// Reads don't bump it: still the PUT's write time (same file mtime).
expect(Date.parse(getBody.updatedAt!)).toBeGreaterThanOrEqual(Date.parse(putBody.updatedAt!));
});
});