Skip to content

A local-first multi-agent app development platform, built on PenguinHarness

35.5 KB
Raw Blame
/**
 * Protocol auto-detection: which wire protocol a typed base URL speaks.
 *
 * - A probe's answer is classified by what it proves: 404/405 is a missing route by status
 *   alone; a 4xx in an API-error shape (auth failures included, a FastAPI `detail` only when it
 *   carries a message) is a served route; HTML, non-JSON, a 2xx without protocol markers and
 *   shapeless 4xx JSON are junk; a 5xx proves nothing.
 * - A typed URL expands to candidates covering one `/v1` too many, one too few and a whole
 *   endpoint URL pasted in (the longest endpoint path stripped, repeats collapsed, the rest kept
 *   exactly as typed), joined to protocol paths the way the wrapped SDKs do.
 * - Probes run openai-responses → ant-messages → openai-chat, candidate by candidate, stop at the
 *   first served protocol and report the base URL it was served on; a probe that fails or
 *   floods (abandoned at the body cap) does not stop the sequence.
 * - Auth headers mirror AgentHub's clients (Bearer for the OpenAI protocols; x-api-key, Bearer
 *   and anthropic-version for ant-messages); a keyless probe sends no credential; the key never
 *   reaches a URL or the reported probes.
 * - With no key typed or stored, each probe of a vendor's own URL borrows that protocol's
 *   environment variable (ANTHROPIC_* for ant-messages, OPENAI_* for the others), never for any
 *   other URL; unset and blank are absent, and a typed key wins.
 * - POST /api/projects/:p/models/detect validates the URL, falls back to the stored key of the
 *   paired reference unless clearApiKey says not to, never echoes the key, and never sends
 *   vendor environment keys to the Penguin Go relay.
 *
 * No test reaches the network except against a loopback server of its own: the probed endpoint
 * is the suite's fetch fake.
 */
import { createServer } from "node:http";
import type { AddressInfo } from "node:net";
import type { IncomingMessage, Server, ServerResponse } from "node:http";
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
import type { ModelProtocolDetectResponse, ModelsResponse } from "../src/api/types.js";
import {
  MAX_PROBE_BODY_BYTES,
  candidateBaseUrls,
  classifyProbeResponse,
  detectModelProtocol,
  envApiKeyForProtocol,
  isHttpUrl,
  probeUrl,
} from "../src/services/protocol-detect.js";
import { fakeFetch } from "./fixtures/fetch.js";
import { apiClient, createTestApp, provisionUser } from "./helpers.js";
import type { TestApp } from "./helpers.js";

// ---------------------------------------------------------------------------
// Pure classification
// ---------------------------------------------------------------------------

const OPENAI_ERROR = JSON.stringify({
  error: { message: "you must provide a model parameter", type: "invalid_request_error" },
});
const ANTHROPIC_ERROR = JSON.stringify({
  type: "error",
  error: { type: "invalid_request_error", message: "messages: Field required" },
});

describe("classifyProbeResponse", () => {
  it("404/405 mean route-missing by status alone — even in protocol-shaped JSON (OpenAI wraps unknown paths in its error shape; the {} probe carries no model id, so 404 can never mean model-not-found)", () => {
    expect(classifyProbeResponse("openai-responses", 404, OPENAI_ERROR)).toBe("route_missing");
    expect(classifyProbeResponse("ant-messages", 404, ANTHROPIC_ERROR)).toBe("route_missing");
    expect(classifyProbeResponse("openai-chat", 405, '{"detail":"Method Not Allowed"}')).toBe(
      "route_missing",
    );
  });

  it("4xx with an API-error JSON shape means the route exists (served) — auth failures included", () => {
    expect(classifyProbeResponse("openai-responses", 401, OPENAI_ERROR)).toBe("served");
    expect(classifyProbeResponse("openai-responses", 400, OPENAI_ERROR)).toBe("served");
    expect(classifyProbeResponse("ant-messages", 401, ANTHROPIC_ERROR)).toBe("served");
    expect(classifyProbeResponse("openai-chat", 403, OPENAI_ERROR)).toBe("served");
    expect(classifyProbeResponse("openai-chat", 429, OPENAI_ERROR)).toBe("served");
    // vLLM / FastAPI dialects: 422 {"detail": …} validation errors and the flat
    // {"object":"error"} shape are real API answers from an existing route.
    expect(classifyProbeResponse("openai-chat", 422, '{"detail":[{"loc":["body","model"]}]}')).toBe(
      "served",
    );
    expect(
      classifyProbeResponse(
        "openai-chat",
        400,
        '{"object":"error","message":"model is required","type":"BadRequestError"}',
      ),
    ).toBe("served");
    // Gateways answering a bare {"error": "..."} string still count.
    expect(classifyProbeResponse("openai-responses", 400, '{"error":"missing model"}')).toBe(
      "served",
    );
  });

  it("HTML and non-JSON bodies are junk regardless of status (edge/login walls are not the protocol route)", () => {
    expect(classifyProbeResponse("openai-responses", 200, "<!doctype html><html></html>")).toBe(
      "junk",
    );
    expect(
      classifyProbeResponse("openai-chat", 401, "<html>401 Authorization Required</html>"),
    ).toBe("junk");
    expect(classifyProbeResponse("ant-messages", 400, "Bad Request")).toBe("junk");
    expect(classifyProbeResponse("openai-chat", 200, '"just a string"')).toBe("junk");
  });

  it("2xx needs protocol success markers (catch-all gateways that 200 every path are junk)", () => {
    expect(classifyProbeResponse("openai-responses", 200, '{"ok":true}')).toBe("junk");
    expect(
      classifyProbeResponse("openai-responses", 200, '{"object":"response","output":[]}'),
    ).toBe("served");
    expect(classifyProbeResponse("openai-chat", 200, '{"object":"chat.completion"}')).toBe(
      "served",
    );
    expect(classifyProbeResponse("openai-chat", 200, '{"choices":[]}')).toBe("served");
    expect(
      classifyProbeResponse("ant-messages", 200, '{"type":"message","role":"assistant"}'),
    ).toBe("served");
    // Chat markers on the responses path do not make the Responses protocol served.
    expect(classifyProbeResponse("openai-responses", 200, '{"choices":[]}')).toBe("junk");
  });

  it("5xx proves nothing about the path (gateways emit it for any URL), so it is server_error, not served", () => {
    expect(classifyProbeResponse("openai-responses", 500, OPENAI_ERROR)).toBe("server_error");
    expect(classifyProbeResponse("openai-chat", 503, OPENAI_ERROR)).toBe("server_error");
    expect(classifyProbeResponse("ant-messages", 502, "<html>bad gateway</html>")).toBe("junk");
  });

  it("4xx JSON that matches no API-error shape is junk", () => {
    expect(classifyProbeResponse("openai-chat", 400, '{"status":"bad"}')).toBe("junk");
  });

  it("a FastAPI-style `detail` only proves the route when it carries a message (null / empty say nothing)", () => {
    expect(classifyProbeResponse("openai-chat", 400, '{"detail":"field required"}')).toBe("served");
    expect(classifyProbeResponse("openai-chat", 400, '{"detail":[{"loc":["body"]}]}')).toBe(
      "served",
    );
    expect(classifyProbeResponse("openai-chat", 400, '{"detail":null}')).toBe("junk");
    expect(classifyProbeResponse("openai-chat", 400, '{"detail":""}')).toBe("junk");
  });
});

describe("isHttpUrl / probeUrl", () => {
  it("accepts absolute http(s) URLs only", () => {
    expect(isHttpUrl("https://api.example.com/v1")).toBe(true);
    expect(isHttpUrl("http://127.0.0.1:8000/v1")).toBe(true);
    expect(isHttpUrl("ftp://example.com")).toBe(false);
    expect(isHttpUrl("api.example.com/v1")).toBe(false);
    expect(isHttpUrl("")).toBe(false);
  });

  it("joins base URL and protocol path the way the wrapped SDKs do (trailing slashes stripped)", () => {
    expect(probeUrl("https://api.example.com/v1", "/responses")).toBe(
      "https://api.example.com/v1/responses",
    );
    expect(probeUrl("https://api.example.com/v1/", "/responses")).toBe(
      "https://api.example.com/v1/responses",
    );
    expect(probeUrl("https://api.example.com/anthropic/", "/v1/messages")).toBe(
      "https://api.example.com/anthropic/v1/messages",
    );
  });
});

describe("candidateBaseUrls", () => {
  it("covers the three ways a base URL is mistyped: one /v1 too many, one too few, a whole endpoint URL", () => {
    expect(candidateBaseUrls("https://host/v1/v1")).toEqual(["https://host/v1", "https://host"]);
    expect(candidateBaseUrls("https://host")).toEqual(["https://host", "https://host/v1"]);
    expect(candidateBaseUrls("https://host/v1/chat/completions")).toEqual([
      "https://host/v1",
      "https://host",
    ]);
    expect(candidateBaseUrls("https://host/v1/responses")).toEqual([
      "https://host/v1",
      "https://host",
    ]);
    expect(candidateBaseUrls("https://host/v1/messages")).toEqual([
      "https://host",
      "https://host/v1",
    ]);
  });

  it("strips the longest endpoint path, not its tail (/chat/completions before /completions, /v1/messages before /messages)", () => {
    expect(candidateBaseUrls("https://host/v1/completions")).toEqual([
      "https://host/v1",
      "https://host",
    ]);
    expect(candidateBaseUrls("https://host/messages")).toEqual(["https://host", "https://host/v1"]);
    expect(candidateBaseUrls("https://host/api/chat/completions")).toEqual([
      "https://host/api",
      "https://host/api/v1",
    ]);
  });

  it("a URL that is already right still gets its neighbour, and a plain host gets /v1", () => {
    expect(candidateBaseUrls("https://api.example.com/v1")).toEqual([
      "https://api.example.com/v1",
      "https://api.example.com",
    ]);
    expect(candidateBaseUrls("http://127.0.0.1:8000")).toEqual([
      "http://127.0.0.1:8000",
      "http://127.0.0.1:8000/v1",
    ]);
  });

  it("trims whitespace, drops trailing slashes, query and fragment", () => {
    expect(candidateBaseUrls("  https://host/v1/  ")).toEqual(["https://host/v1", "https://host"]);
    expect(candidateBaseUrls("https://host/v1/chat/completions?key=abc")).toEqual([
      "https://host/v1",
      "https://host",
    ]);
    expect(candidateBaseUrls("https://host/v1#section")).toEqual([
      "https://host/v1",
      "https://host",
    ]);
  });

  it("collapses a run of repeated /v1 segments, endpoint path included", () => {
    expect(candidateBaseUrls("https://host/v1/v1/v1")).toEqual(["https://host/v1", "https://host"]);
    expect(candidateBaseUrls("https://host/v1/v1/responses")).toEqual([
      "https://host/v1",
      "https://host",
    ]);
  });

  it("leaves the host's case and every other path segment exactly as typed", () => {
    expect(candidateBaseUrls("https://API.Example.COM/v1")).toEqual([
      "https://API.Example.COM/v1",
      "https://API.Example.COM",
    ]);
    expect(candidateBaseUrls("https://host/OpenAI/Compat")).toEqual([
      "https://host/OpenAI/Compat",
      "https://host/OpenAI/Compat/v1",
    ]);
  });
});

// ---------------------------------------------------------------------------
// Probe ordering and header shapes (mocked fetch)
// ---------------------------------------------------------------------------

interface SeenCall {
  url: string;
  headers: Record<string, string>;
  body: string;
}

/**
 * The endpoint under probe, over the suite's fetch fake: fixed answers by pathname, and an
 * OpenAI-style JSON 404 for any other path (like real OpenAI-compatible servers). Each request
 * lands in `seen` with its headers as a lower-cased record, so an absent one reads undefined.
 */
function gateway(
  routes: Record<string, { status: number; body: string }>,
  seen?: SeenCall[],
): typeof fetch {
  return fakeFetch((call) => {
    seen?.push({ url: call.url, headers: Object.fromEntries(call.headers), body: call.body });
    const route = routes[new URL(call.url).pathname];
    return new Response(route?.body ?? OPENAI_ERROR, {
      status: route?.status ?? 404,
      headers: { "content-type": "application/json" },
    });
  }).fetch;
}

describe("detectModelProtocol (order and stop-at-first)", () => {
  const BASE = "https://gw.example.com/v1";

  it("openai-responses wins when served, and later probes never run", async () => {
    const seen: SeenCall[] = [];
    const res = await detectModelProtocol({
      baseUrl: BASE,
      apiKey: "sk-unit",
      fetchImpl: gateway(
        {
          "/v1/responses": { status: 401, body: OPENAI_ERROR },
          "/v1/v1/messages": { status: 401, body: ANTHROPIC_ERROR },
          "/v1/chat/completions": { status: 401, body: OPENAI_ERROR },
        },
        seen,
      ),
    });
    expect(res.detected).toBe("openai-responses");
    // A base URL that needed no fixing is still reported, so callers compare rather than guess.
    expect(res.baseUrl).toBe(BASE);
    expect(res.probes.map((p) => p.outcome)).toEqual(["served"]);
    expect(seen.map((c) => c.url)).toEqual(["https://gw.example.com/v1/responses"]);
  });

  it("falls through to ant-messages when /responses is missing", async () => {
    const res = await detectModelProtocol({
      baseUrl: BASE,
      fetchImpl: gateway({
        "/v1/v1/messages": { status: 401, body: ANTHROPIC_ERROR },
        "/v1/chat/completions": { status: 401, body: OPENAI_ERROR },
      }),
    });
    expect(res.detected).toBe("ant-messages");
    expect(res.probes.map((p) => [p.clientType, p.outcome])).toEqual([
      ["openai-responses", "route_missing"],
      ["ant-messages", "served"],
    ]);
    expect(res.probes[1]!.status).toBe(401);
  });

  it("openai-chat is the last resort", async () => {
    const res = await detectModelProtocol({
      baseUrl: BASE,
      fetchImpl: gateway({
        "/v1/chat/completions": { status: 400, body: OPENAI_ERROR },
      }),
    });
    expect(res.detected).toBe("openai-chat");
    expect(res.probes.map((p) => p.outcome)).toEqual(["route_missing", "route_missing", "served"]);
  });

  it("nothing matches: no detected protocol, no base URL, and every candidate's probes reported", async () => {
    const res = await detectModelProtocol({ baseUrl: BASE, fetchImpl: gateway({}) });
    expect(res.detected).toBeUndefined();
    expect(res.baseUrl).toBeUndefined();
    expect(res.probes).toHaveLength(candidateBaseUrls(BASE).length * 3);
    expect(res.probes.every((p) => p.outcome === "route_missing")).toBe(true);
    // The whole first candidate is exhausted before the second one is tried.
    expect(res.probes.map((p) => p.url)).toEqual([
      "https://gw.example.com/v1/responses",
      "https://gw.example.com/v1/v1/messages",
      "https://gw.example.com/v1/chat/completions",
      "https://gw.example.com/responses",
      "https://gw.example.com/v1/messages",
      "https://gw.example.com/chat/completions",
    ]);
  });

  it("a pasted endpoint URL detects on the base it strips down to, and reports that base back", async () => {
    const res = await detectModelProtocol({
      baseUrl: "https://gw.example.com/v1/chat/completions",
      fetchImpl: gateway({ "/v1/chat/completions": { status: 401, body: OPENAI_ERROR } }),
    });
    expect(res.detected).toBe("openai-chat");
    expect(res.baseUrl).toBe("https://gw.example.com/v1");
    // The two misses before it are the same candidate's earlier protocols, not another base.
    expect(res.probes.map((p) => [p.clientType, p.outcome, p.url])).toEqual([
      ["openai-responses", "route_missing", "https://gw.example.com/v1/responses"],
      ["ant-messages", "route_missing", "https://gw.example.com/v1/v1/messages"],
      ["openai-chat", "served", "https://gw.example.com/v1/chat/completions"],
    ]);
  });

  it("a base URL typed without /v1 falls through to the /v1 candidate and reports it", async () => {
    const res = await detectModelProtocol({
      baseUrl: "https://gw.example.com",
      fetchImpl: gateway({ "/v1/responses": { status: 400, body: OPENAI_ERROR } }),
    });
    expect(res.detected).toBe("openai-responses");
    expect(res.baseUrl).toBe("https://gw.example.com/v1");
    expect(res.probes.map((p) => p.outcome)).toEqual([
      "route_missing",
      "route_missing",
      "route_missing",
      "served",
    ]);
    expect(res.probes[3]!.url).toBe("https://gw.example.com/v1/responses");
  });

  it("probe failures do not stop the sequence: a timeout on /responses still finds chat completions", async () => {
    const timeoutOnResponses = fakeFetch((call) => {
      const path = new URL(call.url).pathname;
      if (path === "/v1/responses") {
        throw new DOMException("The operation timed out", "TimeoutError");
      }
      if (path === "/v1/v1/messages") {
        throw new TypeError("fetch failed"); // connection refused / DNS style failure
      }
      return new Response(OPENAI_ERROR, {
        status: 401,
        headers: { "content-type": "application/json" },
      });
    }).fetch;
    const res = await detectModelProtocol({ baseUrl: BASE, fetchImpl: timeoutOnResponses });
    expect(res.detected).toBe("openai-chat");
    expect(res.probes.map((p) => p.outcome)).toEqual(["timeout", "network_error", "served"]);
  });

  it("an oversized response body is abandoned at the cap and counts as junk, so a flooding endpoint neither detects nor buffers", async () => {
    // Streams well past MAX_PROBE_BODY_BYTES; the reader must stop at the cap rather than
    // materialize the whole thing, and the probe must not be credited as served.
    let pushed = 0;
    const flooder = fakeFetch((call) => {
      const path = new URL(call.url).pathname;
      if (path === "/v1/v1/messages") {
        return new Response(OPENAI_ERROR, {
          status: 404,
          headers: { "content-type": "application/json" },
        });
      }
      if (path !== "/v1/responses") {
        return new Response(OPENAI_ERROR, {
          status: 401,
          headers: { "content-type": "application/json" },
        });
      }
      const chunk = new Uint8Array(8 * 1024).fill(0x20); // 8 KiB of spaces per pull
      return new Response(
        new ReadableStream<Uint8Array>({
          pull(controller) {
            pushed += chunk.byteLength;
            // Far more than the cap if it were ever drained in full.
            if (pushed > MAX_PROBE_BODY_BYTES * 100) {
              controller.close();
              return;
            }
            controller.enqueue(chunk);
          },
        }),
        { status: 200, headers: { "content-type": "application/json" } },
      );
    }).fetch;
    const res = await detectModelProtocol({ baseUrl: BASE, fetchImpl: flooder });
    expect(res.probes[0]?.outcome).toBe("junk");
    expect(res.detected).toBe("openai-chat");
    // Read stopped near the cap instead of draining the stream (the producer runs a chunk
    // or two ahead of the reader, so this is "the cap plus slack", not the exact cap —
    // what matters is that it is nowhere near the 100x the stream would have emitted).
    expect(pushed).toBeLessThanOrEqual(MAX_PROBE_BODY_BYTES * 2);
  });

  it("auth headers mirror the AgentHub clients: Bearer for the OpenAI protocols; x-api-key + Bearer + anthropic-version for ant-messages; the {} body and key-free URLs everywhere", async () => {
    const seen: SeenCall[] = [];
    await detectModelProtocol({
      baseUrl: `${BASE}/`, // trailing slash must not double up
      apiKey: "sk-secret-key",
      fetchImpl: gateway({}, seen),
    });
    // Nothing serves, so both candidates are probed; the header shapes are asserted on the
    // first candidate's three calls.
    expect(seen).toHaveLength(6);
    const [responses, messages, chat] = seen as [SeenCall, SeenCall, SeenCall];
    expect(responses.url).toBe("https://gw.example.com/v1/responses");
    expect(messages.url).toBe("https://gw.example.com/v1/v1/messages");
    expect(chat.url).toBe("https://gw.example.com/v1/chat/completions");
    for (const call of seen) {
      expect(call.body).toBe("{}");
      expect(call.url).not.toContain("sk-secret-key");
      expect(call.headers.authorization).toBe("Bearer sk-secret-key");
    }
    expect(responses.headers["x-api-key"]).toBeUndefined();
    expect(chat.headers["x-api-key"]).toBeUndefined();
    expect(messages.headers["x-api-key"]).toBe("sk-secret-key");
    expect(messages.headers["anthropic-version"]).toBe("2023-06-01");
  });

  it("keyless probes send no credential headers (anthropic-version stays), so a 401 in protocol shape still detects", async () => {
    const seen: SeenCall[] = [];
    const res = await detectModelProtocol({
      baseUrl: BASE,
      // Empty env: no key from anywhere, so this stays the genuinely keyless case
      // regardless of what the machine running the suite happens to export.
      env: {},
      fetchImpl: gateway({ "/v1/responses": { status: 401, body: OPENAI_ERROR } }, seen),
    });
    expect(res.detected).toBe("openai-responses");
    expect(seen[0]!.headers.authorization).toBeUndefined();
    expect(seen[0]!.headers["x-api-key"]).toBeUndefined();
  });
});

// ---------------------------------------------------------------------------
// Environment-variable fallback (no key typed, none stored)
// ---------------------------------------------------------------------------

describe("envApiKeyForProtocol", () => {
  const OPENAI = "https://api.openai.com/v1";
  const ANTHROPIC = "https://api.anthropic.com";

  it("maps each protocol to the variable its AgentHub client actually reads, for the vendor's own endpoint", () => {
    const env = { OPENAI_API_KEY: "sk-openai", ANTHROPIC_API_KEY: "sk-anthropic" };
    expect(envApiKeyForProtocol("openai-responses", OPENAI, env)).toBe("sk-openai");
    expect(envApiKeyForProtocol("openai-chat", OPENAI, env)).toBe("sk-openai");
    // The whole point of resolving per protocol: this one is NOT an OPENAI_* reader.
    expect(envApiKeyForProtocol("ant-messages", ANTHROPIC, env)).toBe("sk-anthropic");
  });

  it("lends nothing to a URL that is not the vendor's own, even one OPENAI_BASE_URL names", () => {
    const env = { OPENAI_API_KEY: "sk-openai", ANTHROPIC_API_KEY: "sk-anthropic" };
    // A gateway, a private server: the user's vendor key must not be sent there.
    expect(envApiKeyForProtocol("openai-chat", "https://gw.example.com/v1", env)).toBeUndefined();
    expect(envApiKeyForProtocol("ant-messages", "http://127.0.0.1:8000", env)).toBeUndefined();
    // Environment keys are for official endpoints only: naming the URL in OPENAI_BASE_URL
    // does not make it one.
    expect(
      envApiKeyForProtocol("openai-chat", "https://gw.example.com/v1", {
        ...env,
        OPENAI_BASE_URL: "https://gw.example.com/v1/",
      }),
    ).toBeUndefined();
  });

  it("treats unset and blank as absent", () => {
    expect(envApiKeyForProtocol("openai-chat", OPENAI, {})).toBeUndefined();
    expect(envApiKeyForProtocol("openai-chat", OPENAI, { OPENAI_API_KEY: "   " })).toBeUndefined();
    expect(
      envApiKeyForProtocol("ant-messages", ANTHROPIC, { OPENAI_API_KEY: "sk-openai" }),
    ).toBeUndefined();
  });
});

describe("detectModelProtocol env fallback", () => {
  const BASE = "https://gw.example.com/v1";

  it("probes a URL that is not a vendor's own anonymously, whatever vendor keys are set", async () => {
    const seen: SeenCall[] = [];
    await detectModelProtocol({
      baseUrl: BASE,
      // OPENAI_BASE_URL naming the URL changes nothing: environment keys are for official
      // endpoints only.
      env: {
        OPENAI_API_KEY: "sk-openai",
        OPENAI_BASE_URL: BASE,
        ANTHROPIC_API_KEY: "sk-anthropic",
      },
      fetchImpl: gateway({}, seen), // every path 404s, so both candidates run all three probes
    });
    expect(seen).toHaveLength(6);
    for (const call of seen) {
      expect(call.headers.authorization).toBeUndefined();
      expect(call.headers["x-api-key"]).toBeUndefined();
    }
  });

  it("authenticates each probe of a vendor's own URL from the env var for the protocol THAT probe speaks", async () => {
    const seen: SeenCall[] = [];
    await detectModelProtocol({
      baseUrl: "https://api.openai.com/v1",
      env: { OPENAI_API_KEY: "sk-openai", ANTHROPIC_API_KEY: "sk-anthropic" },
      fetchImpl: gateway({}, seen), // every path 404s, so both candidates run all three probes
    });
    expect(seen).toHaveLength(6);
    const [responses, messages, chat] = seen as [SeenCall, SeenCall, SeenCall];
    expect(responses.headers.authorization).toBe("Bearer sk-openai");
    expect(chat.headers.authorization).toBe("Bearer sk-openai");
    // OpenAI's host is not Anthropic's endpoint: the Messages probe stays anonymous there.
    expect(messages.headers.authorization).toBeUndefined();
    expect(messages.headers["x-api-key"]).toBeUndefined();
    // The neighbouring /v1-less candidate is not the official endpoint either.
    const [responses2, messages2, chat2] = seen.slice(3) as [SeenCall, SeenCall, SeenCall];
    expect(responses2.headers.authorization).toBeUndefined();
    expect(messages2.headers["x-api-key"]).toBeUndefined();
    expect(chat2.headers.authorization).toBeUndefined();
    // Whatever the source, the credential never reaches a URL.
    for (const call of seen) {
      expect(call.url).not.toContain("sk-openai");
      expect(call.url).not.toContain("sk-anthropic");
    }
  });

  it("an explicit key wins over the environment for every probe", async () => {
    const seen: SeenCall[] = [];
    await detectModelProtocol({
      baseUrl: BASE,
      apiKey: "sk-typed",
      env: { OPENAI_API_KEY: "sk-openai", ANTHROPIC_API_KEY: "sk-anthropic" },
      fetchImpl: gateway({}, seen),
    });
    for (const call of seen) expect(call.headers.authorization).toBe("Bearer sk-typed");
    expect((seen[1] as SeenCall).headers["x-api-key"]).toBe("sk-typed");
  });

  it("falls back per protocol, so a half-configured environment still authenticates its own probe", async () => {
    const seen: SeenCall[] = [];
    await detectModelProtocol({
      baseUrl: "https://api.anthropic.com",
      env: { ANTHROPIC_API_KEY: "sk-anthropic" },
      fetchImpl: gateway({}, seen),
    });
    const [responses, messages, chat] = seen as [SeenCall, SeenCall, SeenCall];
    expect(responses.headers.authorization).toBeUndefined();
    expect(chat.headers.authorization).toBeUndefined();
    expect(messages.headers["x-api-key"]).toBe("sk-anthropic");
  });

  it("never echoes an env-sourced key into the reported probes", async () => {
    const res = await detectModelProtocol({
      baseUrl: "https://api.openai.com/v1",
      env: { OPENAI_API_KEY: "sk-openai", ANTHROPIC_API_KEY: "sk-anthropic" },
      fetchImpl: gateway({ "/v1/responses": { status: 401, body: OPENAI_ERROR } }),
    });
    expect(res.detected).toBe("openai-responses");
    expect(JSON.stringify(res)).not.toContain("sk-openai");
    expect(JSON.stringify(res)).not.toContain("sk-anthropic");
  });
});

// ---------------------------------------------------------------------------
// Route integration
// ---------------------------------------------------------------------------

/** Local endpoint serving only the Anthropic Messages path; records each request's path and auth headers. */
function antOnlyServer(seen: Array<{ path: string; auth?: string; xApiKey?: string }>): Server {
  return createServer((req: IncomingMessage, res: ServerResponse) => {
    req.on("data", () => {});
    req.on("end", () => {
      seen.push({
        path: req.url ?? "",
        ...(req.headers.authorization ? { auth: req.headers.authorization } : {}),
        ...(typeof req.headers["x-api-key"] === "string"
          ? { xApiKey: req.headers["x-api-key"] }
          : {}),
      });
      res.setHeader("content-type", "application/json");
      if (req.url === "/v1/messages") {
        res.statusCode = 401;
        res.end(ANTHROPIC_ERROR);
      } else {
        res.statusCode = 404;
        res.end(OPENAI_ERROR);
      }
    });
  });
}

describe("POST /api/projects/:p/models/detect", () => {
  let t: TestApp;
  let api: ReturnType<typeof apiClient>;
  let projectId: string;
  const detectUrl = () => `/api/projects/${projectId}/models/detect`;

  beforeAll(async () => {
    t = await createTestApp();
    const { cookie } = await provisionUser(t.app, "diana");
    api = apiClient(t.app, cookie);
  });
  afterAll(async () => {
    await t.cleanup();
  });

  // Every case works in a Project of its own.
  let projects = 0;
  beforeEach(async () => {
    projects += 1;
    const created = (await (
      await api.post("/api/projects", {
        projectId: `diana-detect_${projects}`,
        name: "Detect project",
      })
    ).json()) as { project: { projectId: string } };
    projectId = created.project.projectId;
  });

  it("validates baseUrl: required and absolute http(s)", async () => {
    expect((await api.post(detectUrl(), {})).status).toBe(400);
    expect((await api.post(detectUrl(), { baseUrl: "ftp://example.com" })).status).toBe(400);
    expect((await api.post(detectUrl(), { baseUrl: "not a url" })).status).toBe(400);
  });

  it("detects ant-messages on a live endpoint (404 on /responses, protocol-shaped 401 on /v1/messages) and never echoes the key", async () => {
    const seen: Array<{ path: string; auth?: string; xApiKey?: string }> = [];
    const server = antOnlyServer(seen);
    await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
    const port = (server.address() as AddressInfo).port;
    try {
      const res = await api.post(detectUrl(), {
        baseUrl: `http://127.0.0.1:${port}`,
        apiKey: "sk-local-detect",
      });
      expect(res.status).toBe(200);
      const body = (await res.json()) as ModelProtocolDetectResponse;
      expect(body.detected).toBe("ant-messages");
      expect(body.probes.map((p) => [p.clientType, p.outcome])).toEqual([
        ["openai-responses", "route_missing"],
        ["ant-messages", "served"],
      ]);
      expect(body.probes[1]!.status).toBe(401);
      // The key travels only in headers, and the response carries no secret.
      expect(JSON.stringify(body)).not.toContain("sk-local-detect");
      expect(seen.map((c) => c.path)).toEqual(["/responses", "/v1/messages"]);
      expect(seen[1]!.xApiKey).toBe("sk-local-detect");
      expect(seen[1]!.auth).toBe("Bearer sk-local-detect");
    } finally {
      await new Promise<void>((resolve) => server.close(() => resolve()));
    }
  });

  it("the paired reference falls back to the stored key (probe before the frontend ever sees a plaintext key); clearApiKey suppresses that fallback, and a private URL is not lent the env var", async () => {
    const seen: Array<{ path: string; auth?: string; xApiKey?: string }> = [];
    const server = antOnlyServer(seen);
    await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
    const port = (server.address() as AddressInfo).port;
    try {
      const put = await api.put(`/api/projects/${projectId}/models`, {
        models: [
          {
            provider: "custom",
            modelId: "stored-key-model",
            apiKey: "sk-stored-secret",
            baseUrl: `http://127.0.0.1:${port}`,
          },
        ],
      });
      expect(put.status).toBe(200);
      expect(((await put.json()) as ModelsResponse).models).toHaveLength(1);

      const res = await api.post(detectUrl(), {
        baseUrl: `http://127.0.0.1:${port}`,
        provider: "custom",
        modelId: "stored-key-model",
      });
      expect(res.status).toBe(200);
      expect(((await res.json()) as ModelProtocolDetectResponse).detected).toBe("ant-messages");
      const messagesProbe = seen.find((c) => c.path === "/v1/messages");
      expect(messagesProbe?.xApiKey).toBe("sk-stored-secret");

      // clearApiKey drops the STORED key. What backs the probe then is whatever the saved
      // model itself would run on — and a private server is allowed no env var — so this
      // asserts both halves, with the environment pinned (the real one must never reach a
      // probe, and the suite must not depend on what the developer exports).
      const savedAnthropic = process.env.ANTHROPIC_API_KEY;
      const savedOpenai = process.env.OPENAI_API_KEY;
      const savedAnthropicBase = process.env.ANTHROPIC_BASE_URL;
      try {
        delete process.env.ANTHROPIC_API_KEY;
        delete process.env.OPENAI_API_KEY;
        delete process.env.ANTHROPIC_BASE_URL;
        seen.length = 0;
        const cleared = await api.post(detectUrl(), {
          baseUrl: `http://127.0.0.1:${port}`,
          provider: "custom",
          modelId: "stored-key-model",
          clearApiKey: true,
        });
        expect(cleared.status).toBe(200);
        // Route-shape 401s still detect the protocol, but no credential was sent.
        expect(((await cleared.json()) as ModelProtocolDetectResponse).detected).toBe(
          "ant-messages",
        );
        const keyless = seen.find((c) => c.path === "/v1/messages");
        expect(keyless?.xApiKey).toBeUndefined();
        expect(keyless?.auth).toBeUndefined();

        // Same request with the env var present — and even with ANTHROPIC_BASE_URL naming
        // this very server: the stored key stays suppressed, and the local server is not
        // Anthropic's endpoint, so the Anthropic key is NOT lent to it and the probe stays
        // anonymous. Environment keys are for official endpoints only.
        process.env.ANTHROPIC_API_KEY = "sk-env-anthropic";
        process.env.ANTHROPIC_BASE_URL = `http://127.0.0.1:${port}`;
        seen.length = 0;
        const withheld = await api.post(detectUrl(), {
          baseUrl: `http://127.0.0.1:${port}`,
          provider: "custom",
          modelId: "stored-key-model",
          clearApiKey: true,
        });
        expect(withheld.status).toBe(200);
        expect(((await withheld.json()) as ModelProtocolDetectResponse).detected).toBe(
          "ant-messages",
        );
        const withheldProbe = seen.find((c) => c.path === "/v1/messages");
        expect(withheldProbe?.xApiKey).toBeUndefined();
        expect(withheldProbe?.auth).toBeUndefined();
      } finally {
        if (savedAnthropic === undefined) delete process.env.ANTHROPIC_API_KEY;
        else process.env.ANTHROPIC_API_KEY = savedAnthropic;
        if (savedOpenai === undefined) delete process.env.OPENAI_API_KEY;
        else process.env.OPENAI_API_KEY = savedOpenai;
        if (savedAnthropicBase === undefined) delete process.env.ANTHROPIC_BASE_URL;
        else process.env.ANTHROPIC_BASE_URL = savedAnthropicBase;
      }
    } finally {
      await new Promise<void>((resolve) => server.close(() => resolve()));
    }
  });

  it("a Penguin Go probe never sends vendor environment keys to the relay endpoint", async () => {
    const seen: Array<{ path: string; auth?: string; xApiKey?: string }> = [];
    const server = antOnlyServer(seen);
    await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
    const port = (server.address() as AddressInfo).port;
    const saved = {
      anthropic: process.env.ANTHROPIC_API_KEY,
      openai: process.env.OPENAI_API_KEY,
      penguin: process.env.PENGUIN_GO_API_KEY,
    };
    process.env.ANTHROPIC_API_KEY = "vendor-anthropic-must-not-cross";
    process.env.OPENAI_API_KEY = "vendor-openai-must-not-cross";
    delete process.env.PENGUIN_GO_API_KEY;
    try {
      const response = await api.post(detectUrl(), {
        baseUrl: `http://127.0.0.1:${port}`,
        provider: "penguin-go",
        modelId: "deepseek-flash",
      });
      expect(response.status).toBe(200);
      expect(((await response.json()) as ModelProtocolDetectResponse).detected).toBe(
        "ant-messages",
      );
      expect(seen.length).toBeGreaterThan(0);
      expect(seen.every((request) => request.auth === undefined)).toBe(true);
      expect(seen.every((request) => request.xApiKey === undefined)).toBe(true);
    } finally {
      if (saved.anthropic === undefined) delete process.env.ANTHROPIC_API_KEY;
      else process.env.ANTHROPIC_API_KEY = saved.anthropic;
      if (saved.openai === undefined) delete process.env.OPENAI_API_KEY;
      else process.env.OPENAI_API_KEY = saved.openai;
      if (saved.penguin === undefined) delete process.env.PENGUIN_GO_API_KEY;
      else process.env.PENGUIN_GO_API_KEY = saved.penguin;
      await new Promise<void>((resolve) => server.close(() => resolve()));
    }
  });
});