Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

test(ai-gateway): full end-to-end tests (#18802)

Tobias Wochingerd3f61320862d
diff --git a/web/AGENTS.md b/web/AGENTS.md
index 907eabc1ba..0890a14100 100644
--- a/web/AGENTS.md
+++ b/web/AGENTS.md
@@ -265,7 +265,7 @@ Sentry instrumentation skill first and decide whether it should capture at all
- In-source tests: `pnpm --filter web run test:in-source <args>`
- Client tests: `pnpm --filter web run test-client <args>`
- E2E tests: `pnpm --filter web run test:e2e`
-- AI Gateway E2E tests against the local configured stack (also run in CI for AI Gateway changes): `pnpm --filter web run test:e2e:ai-gateway`
+- AI Gateway E2E tests against the local configured stack (also run in CI for AI Gateway changes): `pnpm --filter web run test:e2e:ai-gateway`. Vitest starts the built `ai-gateway` binary (`pnpm --filter ai-gateway build`, or `LANGFUSE_AI_GATEWAY_BIN`) and sends the client traffic through it.
- Agent browser install to the default user-level Playwright cache: `pnpm run playwright:install`
- Build: `pnpm --filter web run build`
- Structure-RFC violation counts: `pnpm --filter web run structure:stats`
diff --git a/web/src/__e2e__/ai-gateway.gatewaye2e.ts b/web/src/__e2e__/ai-gateway.gatewaye2e.ts
index 2a7ef7233d..8fa3d7e356 100644
--- a/web/src/__e2e__/ai-gateway.gatewaye2e.ts
+++ b/web/src/__e2e__/ai-gateway.gatewaye2e.ts
@@ -1,15 +1,20 @@
import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";
+import { type ChildProcess, spawn } from "node:child_process";
import { createHash, randomBytes } from "node:crypto";
+import { existsSync } from "node:fs";
+import { createServer } from "node:net";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
import type { Session } from "next-auth";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import waitForExpect from "wait-for-expect";
+import { z } from "zod/v4";
import { env } from "@/src/env.mjs";
import { $root } from "@/src/pages/api/public/otel/otlp-proto/generated/root";
import {
type GatewayApiFormat,
- GatewayModelsResponseSchema,
GatewayResolveResponseSchema,
} from "@/src/features/ai-gateway/server/provider";
import type { GatewayProviderName } from "@/src/features/ai-gateway/server/provider/registry";
@@ -29,6 +34,7 @@ import {
getTraceByIdFromEventsTable,
queryClickhouse,
redis,
+ UsageDetails,
} from "@langfuse/shared/src/server";
import { getDisplaySecretKey } from "@langfuse/shared/src/server/auth/apiKeys";
@@ -39,7 +45,6 @@ const ORGANIZATION_ID =
process.env.LANGFUSE_AI_GATEWAY_E2E_ORGANIZATION_ID ?? "seed-org-id";
const PROJECT_ID_OVERRIDE = process.env.LANGFUSE_AI_GATEWAY_E2E_PROJECT_ID;
const DEFAULT_PROJECT_ID = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
-const MODELS_PATH = "/api/internal/ai-gateway/v1/models";
const RESOLVE_PATH = "/api/internal/ai-gateway/v1/resolve";
const TEST_TIMEOUT_MS = 120_000;
const PROVIDER_CREDENTIALS: Partial<Record<GatewayProviderName, string>> = {
@@ -48,30 +53,187 @@ const PROVIDER_CREDENTIALS: Partial<Record<GatewayProviderName, string>> = {
};
const ENABLED_PROVIDERS = Object.values(GatewayProvider);
-const PROVIDER_FORMATS = [
+// Public catalog routes. Chat completions has no separate models route; the
+// OpenAI route lists the responses catalog.
+const GATEWAY_MODEL_ROUTES = [
{
provider: GatewayProvider.OPENAI,
apiFormat: "openai.responses",
- baseUrl: "https://api.openai.com/v1",
- authType: "Bearer",
+ path: "/openai/v1/models",
+ },
+ {
+ provider: GatewayProvider.ANTHROPIC,
+ apiFormat: "anthropic.messages",
+ path: "/anthropic/v1/models",
+ },
+] as const satisfies ReadonlyArray<{
+ provider: GatewayProviderName;
+ apiFormat: GatewayApiFormat;
+ path: string;
+}>;
+
+const openAiModelListSchema = z
+ .object({
+ object: z.literal("list"),
+ data: z
+ .array(
+ z
+ .object({
+ id: z.string().min(1),
+ object: z.literal("model"),
+ created: z.number().int().nonnegative(),
+ owned_by: z.literal("openai"),
+ })
+ .strict(),
+ )
+ .min(1),
+ })
+ .strict();
+
+const anthropicModelListSchema = z
+ .object({
+ data: z
+ .array(
+ z
+ .object({
+ type: z.literal("model"),
+ id: z.string().min(1),
+ display_name: z.string().min(1),
+ created_at: z.iso.datetime(),
+ })
+ .strict(),
+ )
+ .min(1),
+ has_more: z.literal(false),
+ first_id: z.string().min(1),
+ last_id: z.string().min(1),
+ })
+ .strict();
+
+const UNAUTHENTICATED_REQUESTS = [
+ {
+ path: "/openai/v1/responses",
+ body: { model: "gpt-6-sol", input: "hello" },
+ error: {
+ error: {
+ message: "Invalid gateway credential",
+ type: "authentication_error",
+ param: null,
+ code: "invalid_api_key",
+ },
+ },
+ },
+ {
+ path: "/anthropic/v1/messages",
+ body: {
+ model: "claude-opus-5-5",
+ max_tokens: 8,
+ messages: [{ role: "user", content: "hello" }],
+ },
+ error: {
+ type: "error",
+ error: {
+ type: "authentication_error",
+ message: "Invalid gateway credential",
+ },
+ },
+ },
+] as const;
+
+const UNKNOWN_MODEL_REQUESTS = [
+ {
+ provider: GatewayProvider.OPENAI,
+ apiFormat: "openai.responses",
+ path: "/openai/v1/responses",
+ body: { model: "not-a-gateway-model", input: "hello" },
+ error: {
+ error: {
+ message: "Invalid gateway model",
+ type: "invalid_request_error",
+ param: null,
+ code: "invalid_model",
+ },
+ },
+ },
+ {
+ provider: GatewayProvider.ANTHROPIC,
+ apiFormat: "anthropic.messages",
+ path: "/anthropic/v1/messages",
+ body: {
+ model: "not-a-gateway-model",
+ max_tokens: 8,
+ messages: [{ role: "user", content: "hello" }],
+ },
+ error: {
+ type: "error",
+ error: {
+ type: "invalid_request_error",
+ message: "Invalid gateway model",
+ },
+ },
+ },
+] as const satisfies ReadonlyArray<{
+ provider: GatewayProviderName;
+ apiFormat: GatewayApiFormat;
+ path: string;
+ body: Record<string, unknown>;
+ error: object;
+}>;
+
+// The gateway never accepts a client span, so project spoofing stays a direct
+// resolve plus OTLP upload. One format is enough: ingestion ignores the span
+// project the same way for every provider.
+const SPOOFED_INGEST = {
+ provider: GatewayProvider.OPENAI,
+ apiFormat: "openai.responses",
+ baseUrl: "https://api.openai.com/v1",
+} as const;
+
+// Unprefixed provider model ids. A connection slug looks like "OpenAI:gpt-6-sol".
+const SLUGLESS_COMPLETIONS = [
+ {
+ provider: GatewayProvider.OPENAI,
+ apiFormat: "openai.responses",
+ model: "gpt-6-sol",
+ path: "/openai/v1/responses",
+ body: {
+ model: "gpt-6-sol",
+ input: "Reply with the word hello.",
+ max_output_tokens: 32,
+ },
+ headers: {},
},
{
provider: GatewayProvider.OPENAI,
apiFormat: "openai.chat-completions",
- baseUrl: "https://api.openai.com/v1",
- authType: "Bearer",
+ model: "gpt-6-sol",
+ path: "/openai/v1/chat/completions",
+ body: {
+ model: "gpt-6-sol",
+ messages: [{ role: "user", content: "Reply with the word hello." }],
+ max_completion_tokens: 32,
+ },
+ headers: {},
},
{
provider: GatewayProvider.ANTHROPIC,
apiFormat: "anthropic.messages",
- baseUrl: "https://api.anthropic.com/v1",
- authType: "x-api-key",
+ model: "claude-opus-5-5",
+ path: "/anthropic/v1/messages",
+ body: {
+ model: "claude-opus-5-5",
+ max_tokens: 32,
+ messages: [{ role: "user", content: "Reply with the word hello." }],
+ },
+ headers: { "anthropic-version": "2023-06-01" },
},
] as const satisfies ReadonlyArray<{
provider: GatewayProviderName;
apiFormat: GatewayApiFormat;
- baseUrl: string;
- authType: "Bearer" | "x-api-key";
+ model: string;
+ path: string;
+ body: Record<string, unknown>;
+ headers: Record<string, string>;
}>;
type ConnectionSnapshot = {
@@ -92,6 +254,8 @@ type Admin = Awaited<ReturnType<typeof createGatewayAdmin>>;
let admin: Admin;
let defaultProjectId: string;
+let gatewayBaseUrl = "";
+let stopGateway = async () => {};
let gatewayConfigSnapshot: GatewayConfigSnapshot | null = null;
let gatewayConfigPrepared = false;
let connectionSnapshots: ConnectionSnapshot[] = [];
@@ -243,55 +407,88 @@ describe("AI gateway live end-to-end", () => {
env.LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST.push(ORGANIZATION_ID);
}
admin = await createGatewayAdmin();
- }, 30_000);
+ const gateway = await startGatewayProcess();
+ gatewayBaseUrl = gateway.url;
+ stopGateway = gateway.stop;
+ }, 60_000);
afterAll(async () => {
try {
- await cleanUpTestArtifacts();
+ await stopGateway();
} finally {
try {
- await restoreProviderConnections();
+ await cleanUpTestArtifacts();
} finally {
try {
- await restoreGatewayConfig();
+ await restoreProviderConnections();
} finally {
- env.LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST.splice(
- 0,
- env.LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST.length,
- ...originalAllowlist,
- );
+ try {
+ await restoreGatewayConfig();
+ } finally {
+ env.LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST.splice(
+ 0,
+ env.LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST.length,
+ ...originalAllowlist,
+ );
+ }
}
}
}
- }, 30_000);
+ }, 50_000);
- describe("model discovery", () => {
- it.each(PROVIDER_FORMATS)(
- "$provider / $apiFormat returns canonical models",
- async ({ provider, apiFormat }) => {
+ describe("model catalog", () => {
+ it.each(GATEWAY_MODEL_ROUTES)(
+ "GET $path lists $provider models",
+ async ({ provider, apiFormat, path: requestPath }) => {
await withGatewayApiKey(
{ provider, apiFormat },
async ({ secretKey }) => {
- const modelsBody = await gatewayControlPlaneRequest({
- path: MODELS_PATH,
- apiFormat,
- virtualSecretKey: secretKey,
+ const response = await fetch(`${gatewayBaseUrl}${requestPath}`, {
+ headers: { Authorization: `Bearer ${secretKey}` },
});
- const models = GatewayModelsResponseSchema.parse(modelsBody);
- expect(models.data.length).toBeGreaterThan(0);
- expect(models.data).toEqual(
- expect.arrayContaining([
- expect.objectContaining({
- id: expect.stringMatching(/\S/),
- provider,
- canonicalSlug: expect.stringMatching(/\S/),
- displayName: expect.stringMatching(/\S/),
- createdAt: expect.stringMatching(
- /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/,
- ),
- }),
- ]),
- );
+ const body = await readJsonResponse(response, 200, requestPath);
+ if (provider === GatewayProvider.OPENAI) {
+ openAiModelListSchema.parse(body);
+ return;
+ }
+ const models = anthropicModelListSchema.parse(body);
+ expect(models.first_id).toBe(models.data[0]?.id);
+ expect(models.last_id).toBe(models.data.at(-1)?.id);
+ },
+ );
+ },
+ TEST_TIMEOUT_MS,
+ );
+ });
+
+ describe("client errors", () => {
+ it.each(UNAUTHENTICATED_REQUESTS)(
+ "POST $path without a credential is rejected",
+ async ({ path: requestPath, body, error }) => {
+ const response = await fetch(`${gatewayBaseUrl}${requestPath}`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify(body),
+ });
+ await expectGatewayError(response, 401, error);
+ },
+ );
+
+ it.each(UNKNOWN_MODEL_REQUESTS)(
+ "POST $path rejects an unknown model",
+ async ({ provider, apiFormat, path: requestPath, body, error }) => {
+ await withGatewayApiKey(
+ { provider, apiFormat },
+ async ({ secretKey }) => {
+ const response = await fetch(`${gatewayBaseUrl}${requestPath}`, {
+ method: "POST",
+ headers: {
+ Authorization: `Bearer ${secretKey}`,
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify(body),
+ });
+ await expectGatewayError(response, 400, error);
},
);
},
@@ -299,10 +496,91 @@ describe("AI gateway live end-to-end", () => {
);
});
- describe("gateway roundtrip", () => {
- it.each(PROVIDER_FORMATS)(
- "$provider / $apiFormat resolves credentials and ingests telemetry",
- async ({ provider, apiFormat, baseUrl, authType }) => {
+ describe("slugless completions", () => {
+ it.each(SLUGLESS_COMPLETIONS)(
+ "$provider / $apiFormat completes slugless $model",
+ async ({
+ provider,
+ apiFormat,
+ model,
+ path: requestPath,
+ body,
+ headers,
+ }) => {
+ await withGatewayApiKey(
+ { provider, apiFormat },
+ async ({ secretKey, providerConnectionId }) => {
+ await withAllModelsAllowed(providerConnectionId, async () => {
+ const response = await fetch(`${gatewayBaseUrl}${requestPath}`, {
+ method: "POST",
+ headers: {
+ Authorization: `Bearer ${secretKey}`,
+ "Content-Type": "application/json",
+ ...headers,
+ },
+ body: JSON.stringify(body),
+ });
+ const text = await response.text();
+ expect(response.status, text).toBe(200);
+ const payload = JSON.parse(text) as {
+ model?: string;
+ usage?: unknown;
+ };
+ expectRequestedProviderModel(payload.model, model);
+ expect(payload.model).not.toContain(":");
+ const expectedUsage = expectedProvidedUsage(
+ apiFormat,
+ payload.usage,
+ );
+
+ const traceId = response.headers.get("langfuse-trace-id");
+ const observationId = response.headers.get(
+ "langfuse-observation-id",
+ );
+ expect(traceId).toMatch(/^[0-9a-f]{32}$/);
+ expect(observationId).toMatch(/^[0-9a-f]{16}$/);
+
+ await waitForExpect(
+ async () => {
+ const observation = await getObservationByIdFromEventsTable({
+ id: observationId!,
+ projectId: defaultProjectId,
+ traceId: traceId!,
+ });
+ if (!observation) {
+ throw new Error(
+ `Observation ${observationId} was not found`,
+ );
+ }
+ expect(observation.projectId).toBe(defaultProjectId);
+ expect(observation.traceId).toBe(traceId);
+ expect(observation.type).toBe("GENERATION");
+ expectRequestedProviderModel(observation.model, model);
+ expect(observation.model).not.toContain(":");
+ expect(observation.providedUsageDetails).toEqual(
+ expectedUsage,
+ );
+ await expectStoredInOneProject({
+ observationId: observationId!,
+ traceId: traceId!,
+ });
+ },
+ 90_000,
+ 1_000,
+ );
+ });
+ },
+ );
+ },
+ 180_000,
+ );
+ });
+
+ describe("ingestion project binding", () => {
+ it(
+ "ignores a spoofed project id on a resolved span",
+ async () => {
+ const { provider, apiFormat, baseUrl } = SPOOFED_INGEST;
await withGatewayApiKey(
{ provider, apiFormat },
async ({
@@ -311,28 +589,20 @@ describe("AI gateway live end-to-end", () => {
providerConnectionId,
providerConnectionName,
}) => {
- const resolveBody = await gatewayControlPlaneRequest({
- path: RESOLVE_PATH,
- apiFormat,
- virtualSecretKey: secretKey,
- model: `${providerConnectionName}:ai-gateway-e2e`,
- });
- const resolved = GatewayResolveResponseSchema.parse(resolveBody);
+ const resolved = GatewayResolveResponseSchema.parse(
+ await gatewayControlPlaneRequest({
+ apiFormat,
+ virtualSecretKey: secretKey,
+ model: `${providerConnectionName}:ai-gateway-e2e`,
+ }),
+ );
expect(resolved.connection.api_format).toBe(apiFormat);
expect(resolved.connection.base_url).toBe(baseUrl);
- if (authType === "x-api-key") {
- expect(resolved.connection.auth).toMatchObject({
- type: "x-api-key",
- header: "x-api-key",
- value: expect.stringMatching(/\S/),
- });
- } else {
- expect(resolved.connection.auth).toMatchObject({
- type: "Bearer",
- token: expect.stringMatching(/\S/),
- });
- }
+ expect(resolved.connection.auth).toMatchObject({
+ type: "Bearer",
+ token: expect.stringMatching(/\S/),
+ });
expect(resolved.attribution).toEqual({
organization_id: ORGANIZATION_ID,
project_id: expect.stringMatching(/\S/),
@@ -350,14 +620,8 @@ describe("AI gateway live end-to-end", () => {
expires_at: expect.any(Number),
});
- const ingestionToken = resolved.ingestion?.access_token;
- if (!ingestionToken)
- throw new Error("Resolve did not return an ingestion token");
-
- await ingestTrace(ingestionToken, {
- provider,
- apiFormat,
- });
+ const ingestionToken = resolved.ingestion.access_token;
+ await ingestTrace(ingestionToken, { provider, apiFormat });
},
);
},
@@ -548,10 +812,9 @@ async function clearModelCaches(connectionIds: string[]) {
}
async function gatewayControlPlaneRequest(input: {
- path: string;
apiFormat: GatewayApiFormat;
virtualSecretKey: string;
- model?: string;
+ model: string;
}): Promise<unknown> {
const timestamp = Math.floor(Date.now() / 1000);
const canonicalMessage = [
@@ -563,22 +826,44 @@ async function gatewayControlPlaneRequest(input: {
canonicalMessage,
env.LANGFUSE_AI_GATEWAY_SERVICE_KEY!,
)}`;
- const isModelsRequest = input.path === MODELS_PATH;
- const url = new URL(`${BASE_URL}${input.path}`);
- if (isModelsRequest) url.searchParams.set("api_format", input.apiFormat);
- const response = await fetch(url, {
- method: isModelsRequest ? "GET" : "POST",
+ const response = await fetch(`${BASE_URL}${RESOLVE_PATH}`, {
+ method: "POST",
headers: {
Authorization: `Bearer ${input.virtualSecretKey}`,
- ...(!isModelsRequest ? { "Content-Type": "application/json" } : {}),
+ "Content-Type": "application/json",
"Langfuse-Gateway-Authorization": gatewayAuthorization,
},
- body: isModelsRequest
- ? undefined
- : JSON.stringify({ apiFormat: input.apiFormat, model: input.model }),
+ body: JSON.stringify({ apiFormat: input.apiFormat, model: input.model }),
});
- return readJsonResponse(response, 200, input.path);
+ return readJsonResponse(response, 200, RESOLVE_PATH);
+}
+
+async function expectGatewayError(
+ response: Response,
+ expectedStatus: number,
+ error: object,
+) {
+ const text = await response.text();
+ expect(response.status, text).toBe(expectedStatus);
+ expect(JSON.parse(text)).toMatchObject({
+ ...error,
+ request_id: expect.stringMatching(/\S/),
+ });
+}
+
+async function expectStoredInOneProject(input: {
+ observationId: string;
+ traceId: string;
+}) {
+ const projects = await queryClickhouse<{ project_id: string }>({
+ query: `SELECT DISTINCT project_id FROM events_core
+ WHERE span_id = {observationId: String} AND trace_id = {traceId: String}`,
+ params: input,
+ });
+ expect(projects.map(({ project_id }) => project_id)).toEqual([
+ defaultProjectId,
+ ]);
}
async function ingestTrace(
@@ -672,14 +957,7 @@ async function ingestTrace(
]);
expect(trace?.id).toBe(traceId);
expect(observation?.id).toBe(observationId);
- const projects = await queryClickhouse<{ project_id: string }>({
- query: `SELECT DISTINCT project_id FROM events_core
- WHERE span_id = {observationId: String} AND trace_id = {traceId: String}`,
- params: { observationId, traceId },
- });
- expect(projects.map(({ project_id }) => project_id)).toEqual([
- defaultProjectId,
- ]);
+ await expectStoredInOneProject({ observationId, traceId });
},
40_000,
1_000,
@@ -734,6 +1012,268 @@ function sha256Hex(value: string) {
return createHash("sha256").update(value, "utf8").digest("hex");
}
+function expectRequestedProviderModel(actual: unknown, requested: string) {
+ const value = typeof actual === "string" ? actual : "";
+ const matches =
+ value === requested ||
+ value.startsWith(`${requested}-`) ||
+ value.startsWith(`${requested}.`);
+ expect(
+ matches,
+ `expected ${JSON.stringify(actual)} to equal ${requested} or a dated snapshot of it`,
+ ).toBe(true);
+}
+
+function expectedProvidedUsage(
+ apiFormat: GatewayApiFormat,
+ usage: unknown,
+): Record<string, number> {
+ const projected = projectGatewayUsage(apiFormat, usage);
+ const parsed = projected ? UsageDetails.safeParse(projected) : null;
+ if (!parsed?.success || !parsed.data) {
+ throw new Error(
+ `Provider usage for ${apiFormat} did not convert: ${JSON.stringify(usage)}`,
+ );
+ }
+ if (!Object.values(parsed.data).some((value) => value > 0)) {
+ throw new Error(
+ `Provider usage for ${apiFormat} has no positive counters: ${JSON.stringify(usage)}`,
+ );
+ }
+ return parsed.data;
+}
+
+function projectGatewayUsage(apiFormat: GatewayApiFormat, usage: unknown) {
+ if (!usage || typeof usage !== "object" || Array.isArray(usage)) return null;
+ const record = usage as Record<string, unknown>;
+ if (apiFormat === "anthropic.messages") return record;
+ if (apiFormat === "openai.responses") {
+ return projectOpenAiUsage(
+ record,
+ ["input_tokens", "output_tokens", "total_tokens"],
+ ["input_tokens_details", "output_tokens_details"],
+ );
+ }
+ return projectOpenAiUsage(
+ record,
+ ["prompt_tokens", "completion_tokens", "total_tokens"],
+ ["prompt_tokens_details", "completion_tokens_details"],
+ );
+}
+
+function projectOpenAiUsage(
+ usage: Record<string, unknown>,
+ counters: readonly [string, string, string],
+ details: readonly [string, string],
+) {
+ const projected: Record<string, unknown> = {};
+ for (const key of counters) {
+ const value = usage[key];
+ if (!isCountedToken(value)) return null;
+ projected[key] = value;
+ }
+ for (const key of details) {
+ const value = usage[key];
+ if (value === null) {
+ projected[key] = null;
+ continue;
+ }
+ if (!value || typeof value !== "object" || Array.isArray(value)) continue;
+ const kept: Record<string, number | null> = {};
+ for (const [detailKey, detailValue] of Object.entries(value)) {
+ if (detailValue === null || isCountedToken(detailValue)) {
+ kept[detailKey] = detailValue;
+ }
+ }
+ projected[key] = kept;
+ }
+ return projected;
+}
+
+function isCountedToken(value: unknown): value is number {
+ return typeof value === "number" && Number.isInteger(value) && value >= 0;
+}
+
+async function withAllModelsAllowed(
+ connectionId: string,
+ run: () => Promise<void>,
+) {
+ const existing = await prisma.gatewayAiConnection.findUniqueOrThrow({
+ where: { id: connectionId },
+ select: { allowedModels: true, updatedAt: true },
+ });
+ const previous = Array.isArray(existing.allowedModels)
+ ? existing.allowedModels.filter(
+ (model): model is string => typeof model === "string",
+ )
+ : null;
+ const opened = await admin.caller.aiGateway.updateConnectionModels({
+ orgId: ORGANIZATION_ID,
+ id: connectionId,
+ allowedModels: null,
+ expectedUpdatedAt: existing.updatedAt,
+ });
+ try {
+ await run();
+ } finally {
+ await admin.caller.aiGateway.updateConnectionModels({
+ orgId: ORGANIZATION_ID,
+ id: connectionId,
+ allowedModels: previous,
+ expectedUpdatedAt: opened.updatedAt,
+ });
+ }
+}
+
+function resolveGatewayBinary() {
+ const override = process.env.LANGFUSE_AI_GATEWAY_BIN;
+ if (override) {
+ if (!existsSync(override)) {
+ throw new Error(`LANGFUSE_AI_GATEWAY_BIN does not exist: ${override}`);
+ }
+ return override;
+ }
+ const root = path.resolve(
+ path.dirname(fileURLToPath(import.meta.url)),
+ "../../..",
+ );
+ const binaryName =
+ process.platform === "win32" ? "ai-gateway.exe" : "ai-gateway";
+ for (const profile of ["release", "debug"]) {
+ const candidate = path.join(
+ root,
+ "ai-gateway",
+ "target",
+ profile,
+ binaryName,
+ );
+ if (existsSync(candidate)) return candidate;
+ }
+ throw new Error(
+ "Build the AI gateway before this suite (`pnpm --filter ai-gateway build`) or set LANGFUSE_AI_GATEWAY_BIN.",
+ );
+}
+
+function reserveLoopbackPort() {
+ return new Promise<number>((resolve, reject) => {
+ const server = createServer();
+ server.once("error", reject);
+ server.listen(0, "127.0.0.1", () => {
+ const address = server.address();
+ if (!address || typeof address === "string") {
+ server.close();
+ reject(new Error("Could not reserve a gateway port"));
+ return;
+ }
+ const { port } = address;
+ server.close((error) => {
+ if (error) reject(error);
+ else resolve(port);
+ });
+ });
+ });
+}
+
+async function startGatewayProcess() {
+ if (!env.LANGFUSE_AI_GATEWAY_SERVICE_KEY) {
+ throw new Error(
+ "Set LANGFUSE_AI_GATEWAY_SERVICE_KEY before starting the gateway process.",
+ );
+ }
+ const binary = resolveGatewayBinary();
+ const port = await reserveLoopbackPort();
+ const logs: string[] = [];
+ const appendLog = (chunk: string) => {
+ logs.push(chunk);
+ const joined = logs.join("");
+ if (joined.length > 16_000) {
+ logs.splice(0, logs.length, joined.slice(-12_000));
+ }
+ };
+ const child = spawn(binary, [], {
+ env: {
+ ...process.env,
+ LANGFUSE_AI_GATEWAY_WEB_URL: BASE_URL,
+ LANGFUSE_AI_GATEWAY_SERVICE_KEY: env.LANGFUSE_AI_GATEWAY_SERVICE_KEY,
+ LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS: `127.0.0.1:${port}`,
+ LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT: "false",
+ LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS: "5",
+ LANGFUSE_LOG_FORMAT: "json",
+ LANGFUSE_LOG_LEVEL: "info",
+ },
+ stdio: ["ignore", "pipe", "pipe"],
+ });
+ child.stdout?.setEncoding("utf8");
+ child.stderr?.setEncoding("utf8");
+ child.stdout?.on("data", appendLog);
+ child.stderr?.on("data", appendLog);
+ const url = `http://127.0.0.1:${port}`;
+ let startupError: Error | undefined;
+ child.once("error", (error) => {
+ startupError = error;
+ });
+ try {
+ await waitForGatewayReady(child, url, logs, () => startupError);
+ } catch (error) {
+ await stopGatewayProcess(child);
+ throw error;
+ }
+ return {
+ url,
+ stop: () => stopGatewayProcess(child),
+ };
+}
+
+async function waitForGatewayReady(
+ child: ChildProcess,
+ url: string,
+ logs: string[],
+ startupError: () => Error | undefined,
+) {
+ const deadline = Date.now() + 20_000;
+ let lastError = "gateway did not become ready";
+ while (Date.now() < deadline) {
+ const error = startupError();
+ if (error) {
+ throw new Error(
+ `AI gateway failed to start: ${error.message}\n${logs.join("")}`,
+ );
+ }
+ if (child.exitCode != null || child.signalCode != null) {
+ throw new Error(
+ `AI gateway exited ${child.exitCode ?? child.signalCode} before ready.\n${logs.join("")}`,
+ );
+ }
+ try {
+ const response = await fetch(`${url}/ready`, {
+ signal: AbortSignal.timeout(500),
+ });
+ if (response.ok) return;
+ lastError = `ready returned ${response.status}`;
+ } catch (error) {
+ lastError = error instanceof Error ? error.message : "connection failed";
+ }
+ await new Promise((resolve) => setTimeout(resolve, 200));
+ }
+ throw new Error(`${lastError}\n${logs.join("")}`);
+}
+
+function stopGatewayProcess(child: ChildProcess) {
+ if (child.exitCode != null || child.signalCode != null) {
+ return Promise.resolve();
+ }
+ return new Promise<void>((resolve) => {
+ const timer = setTimeout(() => {
+ child.kill("SIGKILL");
+ }, 15_000);
+ child.once("exit", () => {
+ clearTimeout(timer);
+ resolve();
+ });
+ child.kill("SIGTERM");
+ });
+}
+
function assertLocalTestTarget() {
const allowedHosts = new Set([
"localhost",
diff --git a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts b/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts
index 609144edcc..d5c78cb0b2 100644
--- a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts
+++ b/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts
@@ -863,25 +863,25 @@ describe("AI gateway control plane", () => {
};
await expect(
- authenticateRequest({ ...params, model: "gpt-4o" }),
+ authenticateRequest({ ...params, model: "gpt-6-sol" }),
).resolves.toMatchObject({
connection: {
id: openAiPrimary.id,
provider: "openai",
- provider_model: "gpt-4o",
+ provider_model: "gpt-6-sol",
},
});
await expect(
authenticateRequest({
...params,
apiFormat: "anthropic.messages",
- model: "claude-haiku-4-5-20251001",
+ model: "claude-opus-5-5",
}),
).resolves.toMatchObject({
connection: {
id: anthropic.id,
provider: "anthropic",
- provider_model: "claude-haiku-4-5-20251001",
+ provider_model: "claude-opus-5-5",
},
});
await expect(
diff --git a/web/src/features/ai-gateway/AGENTS.md b/web/src/features/ai-gateway/AGENTS.md
index d7fa3ac18e..5ea1cbfe74 100644
--- a/web/src/features/ai-gateway/AGENTS.md
+++ b/web/src/features/ai-gateway/AGENTS.md
@@ -8,4 +8,5 @@
### Testing
- When changing the core flow or model provider-specifics: ensure that the end-to-end test suite
- remains valid: `web/src/__e2e__/ai-gateway.gatewaye2e.ts`
+ remains valid: `web/src/__e2e__/ai-gateway.gatewaye2e.ts`. Vitest starts the built binary and
+ sends model listing, client errors, and slugless completions through it.