🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
test(ai-gateway): full end-to-end tests (#18802)
Tobias Wochingerd3f61320862d
diff --git a/web/AGENTS.md b/web/AGENTS.mdindex 907eabc1ba..0890a14100 100644--- a/web/AGENTS.md+++ b/web/AGENTS.md@@ -265,7 +265,7 @@ Sentry instrumentation skill first and decide whether it should capture at all- In-source tests: `pnpm --filter web run test:in-source <args>`- Client tests: `pnpm --filter web run test-client <args>`- E2E tests: `pnpm --filter web run test:e2e`-- AI Gateway E2E tests against the local configured stack (also run in CI for AI Gateway changes): `pnpm --filter web run test:e2e:ai-gateway`+- AI Gateway E2E tests against the local configured stack (also run in CI for AI Gateway changes): `pnpm --filter web run test:e2e:ai-gateway`. Vitest starts the built `ai-gateway` binary (`pnpm --filter ai-gateway build`, or `LANGFUSE_AI_GATEWAY_BIN`) and sends the client traffic through it.- Agent browser install to the default user-level Playwright cache: `pnpm run playwright:install`- Build: `pnpm --filter web run build`- Structure-RFC violation counts: `pnpm --filter web run structure:stats`diff --git a/web/src/__e2e__/ai-gateway.gatewaye2e.ts b/web/src/__e2e__/ai-gateway.gatewaye2e.tsindex 2a7ef7233d..8fa3d7e356 100644--- a/web/src/__e2e__/ai-gateway.gatewaye2e.ts+++ b/web/src/__e2e__/ai-gateway.gatewaye2e.ts@@ -1,15 +1,20 @@import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";+import { type ChildProcess, spawn } from "node:child_process";import { createHash, randomBytes } from "node:crypto";+import { existsSync } from "node:fs";+import { createServer } from "node:net";+import path from "node:path";+import { fileURLToPath } from "node:url";import type { Session } from "next-auth";import { afterAll, beforeAll, describe, expect, it } from "vitest";import waitForExpect from "wait-for-expect";+import { z } from "zod/v4";import { env } from "@/src/env.mjs";import { $root } from "@/src/pages/api/public/otel/otlp-proto/generated/root";import {type GatewayApiFormat,- GatewayModelsResponseSchema,GatewayResolveResponseSchema,} from "@/src/features/ai-gateway/server/provider";import type { GatewayProviderName } from "@/src/features/ai-gateway/server/provider/registry";@@ -29,6 +34,7 @@ import {getTraceByIdFromEventsTable,queryClickhouse,redis,+ UsageDetails,} from "@langfuse/shared/src/server";import { getDisplaySecretKey } from "@langfuse/shared/src/server/auth/apiKeys";@@ -39,7 +45,6 @@ const ORGANIZATION_ID =process.env.LANGFUSE_AI_GATEWAY_E2E_ORGANIZATION_ID ?? "seed-org-id";const PROJECT_ID_OVERRIDE = process.env.LANGFUSE_AI_GATEWAY_E2E_PROJECT_ID;const DEFAULT_PROJECT_ID = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";-const MODELS_PATH = "/api/internal/ai-gateway/v1/models";const RESOLVE_PATH = "/api/internal/ai-gateway/v1/resolve";const TEST_TIMEOUT_MS = 120_000;const PROVIDER_CREDENTIALS: Partial<Record<GatewayProviderName, string>> = {@@ -48,30 +53,187 @@ const PROVIDER_CREDENTIALS: Partial<Record<GatewayProviderName, string>> = {};const ENABLED_PROVIDERS = Object.values(GatewayProvider);-const PROVIDER_FORMATS = [+// Public catalog routes. Chat completions has no separate models route; the+// OpenAI route lists the responses catalog.+const GATEWAY_MODEL_ROUTES = [{provider: GatewayProvider.OPENAI,apiFormat: "openai.responses",- baseUrl: "https://api.openai.com/v1",- authType: "Bearer",+ path: "/openai/v1/models",+ },+ {+ provider: GatewayProvider.ANTHROPIC,+ apiFormat: "anthropic.messages",+ path: "/anthropic/v1/models",+ },+] as const satisfies ReadonlyArray<{+ provider: GatewayProviderName;+ apiFormat: GatewayApiFormat;+ path: string;+}>;++const openAiModelListSchema = z+ .object({+ object: z.literal("list"),+ data: z+ .array(+ z+ .object({+ id: z.string().min(1),+ object: z.literal("model"),+ created: z.number().int().nonnegative(),+ owned_by: z.literal("openai"),+ })+ .strict(),+ )+ .min(1),+ })+ .strict();++const anthropicModelListSchema = z+ .object({+ data: z+ .array(+ z+ .object({+ type: z.literal("model"),+ id: z.string().min(1),+ display_name: z.string().min(1),+ created_at: z.iso.datetime(),+ })+ .strict(),+ )+ .min(1),+ has_more: z.literal(false),+ first_id: z.string().min(1),+ last_id: z.string().min(1),+ })+ .strict();++const UNAUTHENTICATED_REQUESTS = [+ {+ path: "/openai/v1/responses",+ body: { model: "gpt-6-sol", input: "hello" },+ error: {+ error: {+ message: "Invalid gateway credential",+ type: "authentication_error",+ param: null,+ code: "invalid_api_key",+ },+ },+ },+ {+ path: "/anthropic/v1/messages",+ body: {+ model: "claude-opus-5-5",+ max_tokens: 8,+ messages: [{ role: "user", content: "hello" }],+ },+ error: {+ type: "error",+ error: {+ type: "authentication_error",+ message: "Invalid gateway credential",+ },+ },+ },+] as const;++const UNKNOWN_MODEL_REQUESTS = [+ {+ provider: GatewayProvider.OPENAI,+ apiFormat: "openai.responses",+ path: "/openai/v1/responses",+ body: { model: "not-a-gateway-model", input: "hello" },+ error: {+ error: {+ message: "Invalid gateway model",+ type: "invalid_request_error",+ param: null,+ code: "invalid_model",+ },+ },+ },+ {+ provider: GatewayProvider.ANTHROPIC,+ apiFormat: "anthropic.messages",+ path: "/anthropic/v1/messages",+ body: {+ model: "not-a-gateway-model",+ max_tokens: 8,+ messages: [{ role: "user", content: "hello" }],+ },+ error: {+ type: "error",+ error: {+ type: "invalid_request_error",+ message: "Invalid gateway model",+ },+ },+ },+] as const satisfies ReadonlyArray<{+ provider: GatewayProviderName;+ apiFormat: GatewayApiFormat;+ path: string;+ body: Record<string, unknown>;+ error: object;+}>;++// The gateway never accepts a client span, so project spoofing stays a direct+// resolve plus OTLP upload. One format is enough: ingestion ignores the span+// project the same way for every provider.+const SPOOFED_INGEST = {+ provider: GatewayProvider.OPENAI,+ apiFormat: "openai.responses",+ baseUrl: "https://api.openai.com/v1",+} as const;++// Unprefixed provider model ids. A connection slug looks like "OpenAI:gpt-6-sol".+const SLUGLESS_COMPLETIONS = [+ {+ provider: GatewayProvider.OPENAI,+ apiFormat: "openai.responses",+ model: "gpt-6-sol",+ path: "/openai/v1/responses",+ body: {+ model: "gpt-6-sol",+ input: "Reply with the word hello.",+ max_output_tokens: 32,+ },+ headers: {},},{provider: GatewayProvider.OPENAI,apiFormat: "openai.chat-completions",- baseUrl: "https://api.openai.com/v1",- authType: "Bearer",+ model: "gpt-6-sol",+ path: "/openai/v1/chat/completions",+ body: {+ model: "gpt-6-sol",+ messages: [{ role: "user", content: "Reply with the word hello." }],+ max_completion_tokens: 32,+ },+ headers: {},},{provider: GatewayProvider.ANTHROPIC,apiFormat: "anthropic.messages",- baseUrl: "https://api.anthropic.com/v1",- authType: "x-api-key",+ model: "claude-opus-5-5",+ path: "/anthropic/v1/messages",+ body: {+ model: "claude-opus-5-5",+ max_tokens: 32,+ messages: [{ role: "user", content: "Reply with the word hello." }],+ },+ headers: { "anthropic-version": "2023-06-01" },},] as const satisfies ReadonlyArray<{provider: GatewayProviderName;apiFormat: GatewayApiFormat;- baseUrl: string;- authType: "Bearer" | "x-api-key";+ model: string;+ path: string;+ body: Record<string, unknown>;+ headers: Record<string, string>;}>;type ConnectionSnapshot = {@@ -92,6 +254,8 @@ type Admin = Awaited<ReturnType<typeof createGatewayAdmin>>;let admin: Admin;let defaultProjectId: string;+let gatewayBaseUrl = "";+let stopGateway = async () => {};let gatewayConfigSnapshot: GatewayConfigSnapshot | null = null;let gatewayConfigPrepared = false;let connectionSnapshots: ConnectionSnapshot[] = [];@@ -243,55 +407,88 @@ describe("AI gateway live end-to-end", () => {env.LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST.push(ORGANIZATION_ID);}admin = await createGatewayAdmin();- }, 30_000);+ const gateway = await startGatewayProcess();+ gatewayBaseUrl = gateway.url;+ stopGateway = gateway.stop;+ }, 60_000);afterAll(async () => {try {- await cleanUpTestArtifacts();+ await stopGateway();} finally {try {- await restoreProviderConnections();+ await cleanUpTestArtifacts();} finally {try {- await restoreGatewayConfig();+ await restoreProviderConnections();} finally {- env.LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST.splice(- 0,- env.LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST.length,- ...originalAllowlist,- );+ try {+ await restoreGatewayConfig();+ } finally {+ env.LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST.splice(+ 0,+ env.LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST.length,+ ...originalAllowlist,+ );+ }}}}- }, 30_000);+ }, 50_000);- describe("model discovery", () => {- it.each(PROVIDER_FORMATS)(- "$provider / $apiFormat returns canonical models",- async ({ provider, apiFormat }) => {+ describe("model catalog", () => {+ it.each(GATEWAY_MODEL_ROUTES)(+ "GET $path lists $provider models",+ async ({ provider, apiFormat, path: requestPath }) => {await withGatewayApiKey({ provider, apiFormat },async ({ secretKey }) => {- const modelsBody = await gatewayControlPlaneRequest({- path: MODELS_PATH,- apiFormat,- virtualSecretKey: secretKey,+ const response = await fetch(`${gatewayBaseUrl}${requestPath}`, {+ headers: { Authorization: `Bearer ${secretKey}` },});- const models = GatewayModelsResponseSchema.parse(modelsBody);- expect(models.data.length).toBeGreaterThan(0);- expect(models.data).toEqual(- expect.arrayContaining([- expect.objectContaining({- id: expect.stringMatching(/\S/),- provider,- canonicalSlug: expect.stringMatching(/\S/),- displayName: expect.stringMatching(/\S/),- createdAt: expect.stringMatching(- /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/,- ),- }),- ]),- );+ const body = await readJsonResponse(response, 200, requestPath);+ if (provider === GatewayProvider.OPENAI) {+ openAiModelListSchema.parse(body);+ return;+ }+ const models = anthropicModelListSchema.parse(body);+ expect(models.first_id).toBe(models.data[0]?.id);+ expect(models.last_id).toBe(models.data.at(-1)?.id);+ },+ );+ },+ TEST_TIMEOUT_MS,+ );+ });++ describe("client errors", () => {+ it.each(UNAUTHENTICATED_REQUESTS)(+ "POST $path without a credential is rejected",+ async ({ path: requestPath, body, error }) => {+ const response = await fetch(`${gatewayBaseUrl}${requestPath}`, {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify(body),+ });+ await expectGatewayError(response, 401, error);+ },+ );++ it.each(UNKNOWN_MODEL_REQUESTS)(+ "POST $path rejects an unknown model",+ async ({ provider, apiFormat, path: requestPath, body, error }) => {+ await withGatewayApiKey(+ { provider, apiFormat },+ async ({ secretKey }) => {+ const response = await fetch(`${gatewayBaseUrl}${requestPath}`, {+ method: "POST",+ headers: {+ Authorization: `Bearer ${secretKey}`,+ "Content-Type": "application/json",+ },+ body: JSON.stringify(body),+ });+ await expectGatewayError(response, 400, error);},);},@@ -299,10 +496,91 @@ describe("AI gateway live end-to-end", () => {);});- describe("gateway roundtrip", () => {- it.each(PROVIDER_FORMATS)(- "$provider / $apiFormat resolves credentials and ingests telemetry",- async ({ provider, apiFormat, baseUrl, authType }) => {+ describe("slugless completions", () => {+ it.each(SLUGLESS_COMPLETIONS)(+ "$provider / $apiFormat completes slugless $model",+ async ({+ provider,+ apiFormat,+ model,+ path: requestPath,+ body,+ headers,+ }) => {+ await withGatewayApiKey(+ { provider, apiFormat },+ async ({ secretKey, providerConnectionId }) => {+ await withAllModelsAllowed(providerConnectionId, async () => {+ const response = await fetch(`${gatewayBaseUrl}${requestPath}`, {+ method: "POST",+ headers: {+ Authorization: `Bearer ${secretKey}`,+ "Content-Type": "application/json",+ ...headers,+ },+ body: JSON.stringify(body),+ });+ const text = await response.text();+ expect(response.status, text).toBe(200);+ const payload = JSON.parse(text) as {+ model?: string;+ usage?: unknown;+ };+ expectRequestedProviderModel(payload.model, model);+ expect(payload.model).not.toContain(":");+ const expectedUsage = expectedProvidedUsage(+ apiFormat,+ payload.usage,+ );++ const traceId = response.headers.get("langfuse-trace-id");+ const observationId = response.headers.get(+ "langfuse-observation-id",+ );+ expect(traceId).toMatch(/^[0-9a-f]{32}$/);+ expect(observationId).toMatch(/^[0-9a-f]{16}$/);++ await waitForExpect(+ async () => {+ const observation = await getObservationByIdFromEventsTable({+ id: observationId!,+ projectId: defaultProjectId,+ traceId: traceId!,+ });+ if (!observation) {+ throw new Error(+ `Observation ${observationId} was not found`,+ );+ }+ expect(observation.projectId).toBe(defaultProjectId);+ expect(observation.traceId).toBe(traceId);+ expect(observation.type).toBe("GENERATION");+ expectRequestedProviderModel(observation.model, model);+ expect(observation.model).not.toContain(":");+ expect(observation.providedUsageDetails).toEqual(+ expectedUsage,+ );+ await expectStoredInOneProject({+ observationId: observationId!,+ traceId: traceId!,+ });+ },+ 90_000,+ 1_000,+ );+ });+ },+ );+ },+ 180_000,+ );+ });++ describe("ingestion project binding", () => {+ it(+ "ignores a spoofed project id on a resolved span",+ async () => {+ const { provider, apiFormat, baseUrl } = SPOOFED_INGEST;await withGatewayApiKey({ provider, apiFormat },async ({@@ -311,28 +589,20 @@ describe("AI gateway live end-to-end", () => {providerConnectionId,providerConnectionName,}) => {- const resolveBody = await gatewayControlPlaneRequest({- path: RESOLVE_PATH,- apiFormat,- virtualSecretKey: secretKey,- model: `${providerConnectionName}:ai-gateway-e2e`,- });- const resolved = GatewayResolveResponseSchema.parse(resolveBody);+ const resolved = GatewayResolveResponseSchema.parse(+ await gatewayControlPlaneRequest({+ apiFormat,+ virtualSecretKey: secretKey,+ model: `${providerConnectionName}:ai-gateway-e2e`,+ }),+ );expect(resolved.connection.api_format).toBe(apiFormat);expect(resolved.connection.base_url).toBe(baseUrl);- if (authType === "x-api-key") {- expect(resolved.connection.auth).toMatchObject({- type: "x-api-key",- header: "x-api-key",- value: expect.stringMatching(/\S/),- });- } else {- expect(resolved.connection.auth).toMatchObject({- type: "Bearer",- token: expect.stringMatching(/\S/),- });- }+ expect(resolved.connection.auth).toMatchObject({+ type: "Bearer",+ token: expect.stringMatching(/\S/),+ });expect(resolved.attribution).toEqual({organization_id: ORGANIZATION_ID,project_id: expect.stringMatching(/\S/),@@ -350,14 +620,8 @@ describe("AI gateway live end-to-end", () => {expires_at: expect.any(Number),});- const ingestionToken = resolved.ingestion?.access_token;- if (!ingestionToken)- throw new Error("Resolve did not return an ingestion token");-- await ingestTrace(ingestionToken, {- provider,- apiFormat,- });+ const ingestionToken = resolved.ingestion.access_token;+ await ingestTrace(ingestionToken, { provider, apiFormat });},);},@@ -548,10 +812,9 @@ async function clearModelCaches(connectionIds: string[]) {}async function gatewayControlPlaneRequest(input: {- path: string;apiFormat: GatewayApiFormat;virtualSecretKey: string;- model?: string;+ model: string;}): Promise<unknown> {const timestamp = Math.floor(Date.now() / 1000);const canonicalMessage = [@@ -563,22 +826,44 @@ async function gatewayControlPlaneRequest(input: {canonicalMessage,env.LANGFUSE_AI_GATEWAY_SERVICE_KEY!,)}`;- const isModelsRequest = input.path === MODELS_PATH;- const url = new URL(`${BASE_URL}${input.path}`);- if (isModelsRequest) url.searchParams.set("api_format", input.apiFormat);- const response = await fetch(url, {- method: isModelsRequest ? "GET" : "POST",+ const response = await fetch(`${BASE_URL}${RESOLVE_PATH}`, {+ method: "POST",headers: {Authorization: `Bearer ${input.virtualSecretKey}`,- ...(!isModelsRequest ? { "Content-Type": "application/json" } : {}),+ "Content-Type": "application/json","Langfuse-Gateway-Authorization": gatewayAuthorization,},- body: isModelsRequest- ? undefined- : JSON.stringify({ apiFormat: input.apiFormat, model: input.model }),+ body: JSON.stringify({ apiFormat: input.apiFormat, model: input.model }),});- return readJsonResponse(response, 200, input.path);+ return readJsonResponse(response, 200, RESOLVE_PATH);+}++async function expectGatewayError(+ response: Response,+ expectedStatus: number,+ error: object,+) {+ const text = await response.text();+ expect(response.status, text).toBe(expectedStatus);+ expect(JSON.parse(text)).toMatchObject({+ ...error,+ request_id: expect.stringMatching(/\S/),+ });+}++async function expectStoredInOneProject(input: {+ observationId: string;+ traceId: string;+}) {+ const projects = await queryClickhouse<{ project_id: string }>({+ query: `SELECT DISTINCT project_id FROM events_core+ WHERE span_id = {observationId: String} AND trace_id = {traceId: String}`,+ params: input,+ });+ expect(projects.map(({ project_id }) => project_id)).toEqual([+ defaultProjectId,+ ]);}async function ingestTrace(@@ -672,14 +957,7 @@ async function ingestTrace(]);expect(trace?.id).toBe(traceId);expect(observation?.id).toBe(observationId);- const projects = await queryClickhouse<{ project_id: string }>({- query: `SELECT DISTINCT project_id FROM events_core- WHERE span_id = {observationId: String} AND trace_id = {traceId: String}`,- params: { observationId, traceId },- });- expect(projects.map(({ project_id }) => project_id)).toEqual([- defaultProjectId,- ]);+ await expectStoredInOneProject({ observationId, traceId });},40_000,1_000,@@ -734,6 +1012,268 @@ function sha256Hex(value: string) {return createHash("sha256").update(value, "utf8").digest("hex");}+function expectRequestedProviderModel(actual: unknown, requested: string) {+ const value = typeof actual === "string" ? actual : "";+ const matches =+ value === requested ||+ value.startsWith(`${requested}-`) ||+ value.startsWith(`${requested}.`);+ expect(+ matches,+ `expected ${JSON.stringify(actual)} to equal ${requested} or a dated snapshot of it`,+ ).toBe(true);+}++function expectedProvidedUsage(+ apiFormat: GatewayApiFormat,+ usage: unknown,+): Record<string, number> {+ const projected = projectGatewayUsage(apiFormat, usage);+ const parsed = projected ? UsageDetails.safeParse(projected) : null;+ if (!parsed?.success || !parsed.data) {+ throw new Error(+ `Provider usage for ${apiFormat} did not convert: ${JSON.stringify(usage)}`,+ );+ }+ if (!Object.values(parsed.data).some((value) => value > 0)) {+ throw new Error(+ `Provider usage for ${apiFormat} has no positive counters: ${JSON.stringify(usage)}`,+ );+ }+ return parsed.data;+}++function projectGatewayUsage(apiFormat: GatewayApiFormat, usage: unknown) {+ if (!usage || typeof usage !== "object" || Array.isArray(usage)) return null;+ const record = usage as Record<string, unknown>;+ if (apiFormat === "anthropic.messages") return record;+ if (apiFormat === "openai.responses") {+ return projectOpenAiUsage(+ record,+ ["input_tokens", "output_tokens", "total_tokens"],+ ["input_tokens_details", "output_tokens_details"],+ );+ }+ return projectOpenAiUsage(+ record,+ ["prompt_tokens", "completion_tokens", "total_tokens"],+ ["prompt_tokens_details", "completion_tokens_details"],+ );+}++function projectOpenAiUsage(+ usage: Record<string, unknown>,+ counters: readonly [string, string, string],+ details: readonly [string, string],+) {+ const projected: Record<string, unknown> = {};+ for (const key of counters) {+ const value = usage[key];+ if (!isCountedToken(value)) return null;+ projected[key] = value;+ }+ for (const key of details) {+ const value = usage[key];+ if (value === null) {+ projected[key] = null;+ continue;+ }+ if (!value || typeof value !== "object" || Array.isArray(value)) continue;+ const kept: Record<string, number | null> = {};+ for (const [detailKey, detailValue] of Object.entries(value)) {+ if (detailValue === null || isCountedToken(detailValue)) {+ kept[detailKey] = detailValue;+ }+ }+ projected[key] = kept;+ }+ return projected;+}++function isCountedToken(value: unknown): value is number {+ return typeof value === "number" && Number.isInteger(value) && value >= 0;+}++async function withAllModelsAllowed(+ connectionId: string,+ run: () => Promise<void>,+) {+ const existing = await prisma.gatewayAiConnection.findUniqueOrThrow({+ where: { id: connectionId },+ select: { allowedModels: true, updatedAt: true },+ });+ const previous = Array.isArray(existing.allowedModels)+ ? existing.allowedModels.filter(+ (model): model is string => typeof model === "string",+ )+ : null;+ const opened = await admin.caller.aiGateway.updateConnectionModels({+ orgId: ORGANIZATION_ID,+ id: connectionId,+ allowedModels: null,+ expectedUpdatedAt: existing.updatedAt,+ });+ try {+ await run();+ } finally {+ await admin.caller.aiGateway.updateConnectionModels({+ orgId: ORGANIZATION_ID,+ id: connectionId,+ allowedModels: previous,+ expectedUpdatedAt: opened.updatedAt,+ });+ }+}++function resolveGatewayBinary() {+ const override = process.env.LANGFUSE_AI_GATEWAY_BIN;+ if (override) {+ if (!existsSync(override)) {+ throw new Error(`LANGFUSE_AI_GATEWAY_BIN does not exist: ${override}`);+ }+ return override;+ }+ const root = path.resolve(+ path.dirname(fileURLToPath(import.meta.url)),+ "../../..",+ );+ const binaryName =+ process.platform === "win32" ? "ai-gateway.exe" : "ai-gateway";+ for (const profile of ["release", "debug"]) {+ const candidate = path.join(+ root,+ "ai-gateway",+ "target",+ profile,+ binaryName,+ );+ if (existsSync(candidate)) return candidate;+ }+ throw new Error(+ "Build the AI gateway before this suite (`pnpm --filter ai-gateway build`) or set LANGFUSE_AI_GATEWAY_BIN.",+ );+}++function reserveLoopbackPort() {+ return new Promise<number>((resolve, reject) => {+ const server = createServer();+ server.once("error", reject);+ server.listen(0, "127.0.0.1", () => {+ const address = server.address();+ if (!address || typeof address === "string") {+ server.close();+ reject(new Error("Could not reserve a gateway port"));+ return;+ }+ const { port } = address;+ server.close((error) => {+ if (error) reject(error);+ else resolve(port);+ });+ });+ });+}++async function startGatewayProcess() {+ if (!env.LANGFUSE_AI_GATEWAY_SERVICE_KEY) {+ throw new Error(+ "Set LANGFUSE_AI_GATEWAY_SERVICE_KEY before starting the gateway process.",+ );+ }+ const binary = resolveGatewayBinary();+ const port = await reserveLoopbackPort();+ const logs: string[] = [];+ const appendLog = (chunk: string) => {+ logs.push(chunk);+ const joined = logs.join("");+ if (joined.length > 16_000) {+ logs.splice(0, logs.length, joined.slice(-12_000));+ }+ };+ const child = spawn(binary, [], {+ env: {+ ...process.env,+ LANGFUSE_AI_GATEWAY_WEB_URL: BASE_URL,+ LANGFUSE_AI_GATEWAY_SERVICE_KEY: env.LANGFUSE_AI_GATEWAY_SERVICE_KEY,+ LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS: `127.0.0.1:${port}`,+ LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT: "false",+ LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS: "5",+ LANGFUSE_LOG_FORMAT: "json",+ LANGFUSE_LOG_LEVEL: "info",+ },+ stdio: ["ignore", "pipe", "pipe"],+ });+ child.stdout?.setEncoding("utf8");+ child.stderr?.setEncoding("utf8");+ child.stdout?.on("data", appendLog);+ child.stderr?.on("data", appendLog);+ const url = `http://127.0.0.1:${port}`;+ let startupError: Error | undefined;+ child.once("error", (error) => {+ startupError = error;+ });+ try {+ await waitForGatewayReady(child, url, logs, () => startupError);+ } catch (error) {+ await stopGatewayProcess(child);+ throw error;+ }+ return {+ url,+ stop: () => stopGatewayProcess(child),+ };+}++async function waitForGatewayReady(+ child: ChildProcess,+ url: string,+ logs: string[],+ startupError: () => Error | undefined,+) {+ const deadline = Date.now() + 20_000;+ let lastError = "gateway did not become ready";+ while (Date.now() < deadline) {+ const error = startupError();+ if (error) {+ throw new Error(+ `AI gateway failed to start: ${error.message}\n${logs.join("")}`,+ );+ }+ if (child.exitCode != null || child.signalCode != null) {+ throw new Error(+ `AI gateway exited ${child.exitCode ?? child.signalCode} before ready.\n${logs.join("")}`,+ );+ }+ try {+ const response = await fetch(`${url}/ready`, {+ signal: AbortSignal.timeout(500),+ });+ if (response.ok) return;+ lastError = `ready returned ${response.status}`;+ } catch (error) {+ lastError = error instanceof Error ? error.message : "connection failed";+ }+ await new Promise((resolve) => setTimeout(resolve, 200));+ }+ throw new Error(`${lastError}\n${logs.join("")}`);+}++function stopGatewayProcess(child: ChildProcess) {+ if (child.exitCode != null || child.signalCode != null) {+ return Promise.resolve();+ }+ return new Promise<void>((resolve) => {+ const timer = setTimeout(() => {+ child.kill("SIGKILL");+ }, 15_000);+ child.once("exit", () => {+ clearTimeout(timer);+ resolve();+ });+ child.kill("SIGTERM");+ });+}+function assertLocalTestTarget() {const allowedHosts = new Set(["localhost",diff --git a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts b/web/src/__tests__/server/ai-gateway-control-plane.servertest.tsindex 609144edcc..d5c78cb0b2 100644--- a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts+++ b/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts@@ -863,25 +863,25 @@ describe("AI gateway control plane", () => {};await expect(- authenticateRequest({ ...params, model: "gpt-4o" }),+ authenticateRequest({ ...params, model: "gpt-6-sol" }),).resolves.toMatchObject({connection: {id: openAiPrimary.id,provider: "openai",- provider_model: "gpt-4o",+ provider_model: "gpt-6-sol",},});await expect(authenticateRequest({...params,apiFormat: "anthropic.messages",- model: "claude-haiku-4-5-20251001",+ model: "claude-opus-5-5",}),).resolves.toMatchObject({connection: {id: anthropic.id,provider: "anthropic",- provider_model: "claude-haiku-4-5-20251001",+ provider_model: "claude-opus-5-5",},});await expect(diff --git a/web/src/features/ai-gateway/AGENTS.md b/web/src/features/ai-gateway/AGENTS.mdindex d7fa3ac18e..5ea1cbfe74 100644--- a/web/src/features/ai-gateway/AGENTS.md+++ b/web/src/features/ai-gateway/AGENTS.md@@ -8,4 +8,5 @@### Testing- When changing the core flow or model provider-specifics: ensure that the end-to-end test suite- remains valid: `web/src/__e2e__/ai-gateway.gatewaye2e.ts`+ remains valid: `web/src/__e2e__/ai-gateway.gatewaye2e.ts`. Vitest starts the built binary and+ sends model listing, client errors, and slugless completions through it.