🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(ai-gateway): always record gateway requests on Langfuse Cloud (#19505)
Hassieb Pakzadb7634d68dd26
diff --git a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts b/web/src/__tests__/server/ai-gateway-control-plane.servertest.tsindex ed2516830b..c40529f16e 100644--- a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts+++ b/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts@@ -719,11 +719,11 @@ describe("AI gateway control plane", () => {).resolves.toBe(0);});- it("requires an ingestion project unless ingestion mode is NONE, also on cloud", async () => {+ it("requires an ingestion project unless ingestion mode is NONE on self-hosted", async () => {const { caller, org } = await prepare();const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;try {- Object.assign(env, { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU" });+ Object.assign(env, { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined });await expect(caller.aiGateway.updateConfig({orgId: org.id,@@ -747,6 +747,50 @@ describe("AI gateway control plane", () => {}});+ it("rejects NONE on cloud and treats a stored NONE config as USAGE", async () => {+ const { caller, org, project } = await prepare();+ const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;+ try {+ Object.assign(env, { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU" });+ await expect(+ caller.aiGateway.updateConfig({+ orgId: org.id,+ defaultIngestionProjectId: project.id,+ ingestionMode: "NONE",+ }),+ ).rejects.toThrow("not available on Langfuse Cloud");++ await prisma.gatewayConfig.create({+ data: {+ organizationId: org.id,+ defaultIngestionProjectId: null,+ ingestionMode: "NONE",+ },+ });+ await expect(+ caller.aiGateway.getConfig({ orgId: org.id }),+ ).resolves.toMatchObject({ config: { ingestionMode: "USAGE" } });+ await expect(+ caller.aiGateway.getSetupStatus({ orgId: org.id }),+ ).resolves.toEqual({ isConfigured: false });++ await prisma.gatewayConfig.update({+ where: { organizationId: org.id },+ data: { defaultIngestionProjectId: project.id },+ });+ await expect(+ caller.projects.deletionProtection({ projectId: project.id }),+ ).resolves.toEqual({ isGatewayIngestionProject: true });+ await expect(+ caller.projects.delete({ projectId: project.id }),+ ).rejects.toThrow("Select another ingestion project before deleting it");+ } finally {+ Object.assign(env, {+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: originalCloudRegion,+ });+ }+ });+it.each([{ adminRole: Role.OWNER, readRole: Role.MEMBER },{ adminRole: Role.ADMIN, readRole: Role.VIEWER },@@ -2226,28 +2270,34 @@ describe("AI gateway control plane", () => {).resolves.toEqual({ isGatewayIngestionProject: true });await expect(caller.projects.delete({ projectId: project.id }),- ).rejects.toThrow(- "Select another ingestion project or set the ingestion mode to None before deleting it",- );+ ).rejects.toThrow("Select another ingestion project");await expect(prisma.project.findUnique({ where: { id: project.id } }),).resolves.toMatchObject({ deletedAt: null });});- it("allows deleting the linked ingestion project while ingestion is NONE", async () => {+ it("allows deleting the linked ingestion project while ingestion is NONE on self-hosted", async () => {const { caller, org, project } = await prepare();- await caller.aiGateway.updateConfig({- orgId: org.id,- defaultIngestionProjectId: project.id,- ingestionMode: "NONE",- });+ const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;+ try {+ Object.assign(env, { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined });+ await caller.aiGateway.updateConfig({+ orgId: org.id,+ defaultIngestionProjectId: project.id,+ ingestionMode: "NONE",+ });- await expect(- caller.projects.deletionProtection({ projectId: project.id }),- ).resolves.toEqual({ isGatewayIngestionProject: false });- await expect(- caller.projects.delete({ projectId: project.id }),- ).resolves.toBe(true);+ await expect(+ caller.projects.deletionProtection({ projectId: project.id }),+ ).resolves.toEqual({ isGatewayIngestionProject: false });+ await expect(+ caller.projects.delete({ projectId: project.id }),+ ).resolves.toBe(true);+ } finally {+ Object.assign(env, {+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: originalCloudRegion,+ });+ }await expect(prisma.project.findUnique({ where: { id: project.id } }),).resolves.toMatchObject({ deletedAt: expect.any(Date) });diff --git a/web/src/__tests__/server/apiKeyCacheInvalidation.servertest.ts b/web/src/__tests__/server/apiKeyCacheInvalidation.servertest.tsindex 66527118ec..f68789b6bc 100644--- a/web/src/__tests__/server/apiKeyCacheInvalidation.servertest.ts+++ b/web/src/__tests__/server/apiKeyCacheInvalidation.servertest.ts@@ -1,3 +1,4 @@+import { getGatewayIngestionProjectDeletionMessage } from "@/src/features/ai-gateway/fns/gatewayIngestionModePolicy";import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";import { LLMAdapter, type Plan, Role } from "@langfuse/shared";import { prisma } from "@langfuse/shared/src/db";@@ -389,8 +390,7 @@ describe("API-key cache invalidation on project/org lifecycle", () => {expect(res.statusCode).toBe(409);expect(res.body).toEqual({- message:- "This project is used as the AI Gateway ingestion project. Select another ingestion project or set the ingestion mode to None before deleting it.",+ message: getGatewayIngestionProjectDeletionMessage(),});expect(await survivingKeys(keys)).toEqual(keys);await expect(diff --git a/web/src/ee/features/admin-api/server/projects/projectById/index.ts b/web/src/ee/features/admin-api/server/projects/projectById/index.tsindex eb45ea770e..1308372f77 100644--- a/web/src/ee/features/admin-api/server/projects/projectById/index.ts+++ b/web/src/ee/features/admin-api/server/projects/projectById/index.ts@@ -1,3 +1,7 @@+import {+ getCapturingGatewayIngestionModes,+ getGatewayIngestionProjectDeletionMessage,+} from "@/src/features/ai-gateway/fns/gatewayIngestionModePolicy";import { type NextApiRequest, type NextApiResponse } from "next";import { prisma } from "@langfuse/shared/src/db";import {@@ -125,14 +129,13 @@ export async function handleDeleteProject(organizationId: scope.orgId,defaultIngestionProjectId: projectId,// With ingestion NONE the project is only remembered, not written to.- ingestionMode: { not: "NONE" },+ ingestionMode: { in: getCapturingGatewayIngestionModes() },},select: { organizationId: true },});if (gatewayConfig) {return res.status(409).json({- message:- "This project is used as the AI Gateway ingestion project. Select another ingestion project or set the ingestion mode to None before deleting it.",+ message: getGatewayIngestionProjectDeletionMessage(),});}diff --git a/web/src/features/ai-gateway/components/GatewaySetupPage/components/IngestionSettingsFields/IngestionSettingsFields.tsx b/web/src/features/ai-gateway/components/GatewaySetupPage/components/IngestionSettingsFields/IngestionSettingsFields.tsxindex ed0d5732f5..1e2fce6e04 100644--- a/web/src/features/ai-gateway/components/GatewaySetupPage/components/IngestionSettingsFields/IngestionSettingsFields.tsx+++ b/web/src/features/ai-gateway/components/GatewaySetupPage/components/IngestionSettingsFields/IngestionSettingsFields.tsx@@ -7,6 +7,8 @@ import type {GatewayProject,} from "@/src/features/ai-gateway/components/GatewaySetupPage/types/gatewaySetup";import { GATEWAY_INGESTION_MODES } from "@/src/features/ai-gateway/constants/gatewayIngestionModes";+import { isGatewayIngestionModeAvailable } from "@/src/features/ai-gateway/fns/gatewayIngestionModePolicy";+import { cn } from "@/src/utils/tailwind";export type IngestionSettingsFieldsProps = {settings: GatewayIngestionSettings;@@ -19,14 +21,23 @@ export function IngestionSettingsFields({projects,onChange,}: IngestionSettingsFieldsProps) {+ const modes = GATEWAY_INGESTION_MODES.filter((option) =>+ isGatewayIngestionModeAvailable(option.value),+ );+function handleModeSelect(ingestionMode: GatewayIngestionMode) {onChange({ ...settings, ingestionMode });}return (<div className="flex flex-col gap-4">- <div className="grid gap-2 md:grid-cols-3">- {GATEWAY_INGESTION_MODES.map((option) => (+ <div+ className={cn(+ "grid gap-2",+ modes.length > 2 ? "md:grid-cols-3" : "md:grid-cols-2",+ )}+ >+ {modes.map((option) => (<IngestionModeCardkey={option.value}option={option}diff --git a/web/src/features/ai-gateway/fns/gatewayIngestionModePolicy.ts b/web/src/features/ai-gateway/fns/gatewayIngestionModePolicy.tsnew file mode 100644index 0000000000..7eca928cba--- /dev/null+++ b/web/src/features/ai-gateway/fns/gatewayIngestionModePolicy.ts@@ -0,0 +1,37 @@+import type { GatewayIngestionMode } from "@langfuse/shared";++import { env } from "@/src/env.mjs";++const STORED_INGESTION_MODES = [+ "NONE",+ "USAGE",+ "FULL",+] as const satisfies readonly GatewayIngestionMode[];++/** Langfuse Cloud records a generation for every gateway request, so `NONE` is self-hosted only. */+export function isGatewayIngestionModeAvailable(mode: GatewayIngestionMode) {+ return mode !== "NONE" || !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;+}++/**+ * The mode a stored config runs with. A `NONE` config stored on Cloud+ * captures usage, the minimal level, so no request goes unrecorded.+ */+export function getEffectiveGatewayIngestionMode(+ mode: GatewayIngestionMode,+): GatewayIngestionMode {+ return isGatewayIngestionModeAvailable(mode) ? mode : "USAGE";+}++/** Stored modes under which the gateway writes into the ingestion project. */+export function getCapturingGatewayIngestionModes() {+ return STORED_INGESTION_MODES.filter(+ (mode) => getEffectiveGatewayIngestionMode(mode) !== "NONE",+ );+}++export function getGatewayIngestionProjectDeletionMessage() {+ return isGatewayIngestionModeAvailable("NONE")+ ? "This project is used as the AI Gateway ingestion project. Select another ingestion project or set the ingestion mode to None before deleting it."+ : "This project is used as the AI Gateway ingestion project. Select another ingestion project before deleting it.";+}diff --git a/web/src/features/ai-gateway/server/config/gatewayConfigService.ts b/web/src/features/ai-gateway/server/config/gatewayConfigService.tsindex 20dc8267a8..2fb60dd865 100644--- a/web/src/features/ai-gateway/server/config/gatewayConfigService.ts+++ b/web/src/features/ai-gateway/server/config/gatewayConfigService.ts@@ -7,6 +7,10 @@ import { auditLog } from "@/src/features/audit-logs/server";import type { OrgAuthedContext } from "@/src/server/api/trpc";import { invalidateGatewayResolveCacheForOrganization } from "@/src/features/ai-gateway/server/resolve/gatewayResolveCache";import { isGatewayIngestionConfigured } from "@/src/features/ai-gateway/fns/isGatewayIngestionConfigured";+import {+ getEffectiveGatewayIngestionMode,+ isGatewayIngestionModeAvailable,+} from "@/src/features/ai-gateway/fns/gatewayIngestionModePolicy";import { GatewayConfigRepository } from "./gatewayConfigRepository";export class GatewayConfigService {@@ -16,8 +20,14 @@ export class GatewayConfigService {this.repository = new GatewayConfigRepository(prisma);}- getConfig(organizationId: string) {- return this.repository.getConfig(organizationId);+ async getConfig(organizationId: string) {+ const config = await this.repository.getConfig(organizationId);+ return (+ config && {+ ...config,+ ingestionMode: getEffectiveGatewayIngestionMode(config.ingestionMode),+ }+ );}async getSetupStatus(organizationId: string) {@@ -38,6 +48,11 @@ export class GatewayConfigService {ingestionMode: GatewayIngestionMode;session: OrgAuthedContext["session"];}) {+ if (!isGatewayIngestionModeAvailable(params.ingestionMode)) {+ throw new InvalidRequestError(+ "Ingestion mode NONE is not available on Langfuse Cloud",+ );+ }if (!isGatewayIngestionConfigured({ingestionMode: params.ingestionMode,@@ -49,7 +64,7 @@ export class GatewayConfigService {);}- const before = await this.getConfig(params.organizationId);+ const before = await this.repository.getConfig(params.organizationId);if (params.defaultIngestionProjectId) {const project = await this.repository.getActiveOrganizationProject({diff --git a/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts b/web/src/features/ai-gateway/server/resolve/resolveService.servertest.tsindex fcb80b7bed..cbe1d88602 100644--- a/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts+++ b/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts@@ -1,5 +1,5 @@import type { PrismaClient } from "@langfuse/shared/src/db";-import { describe, expect, it, vi } from "vitest";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import { GatewayApiKeyAuthenticator } from "@/src/features/ai-gateway/server/auth/gatewayApiKeyAuthenticator";import type { GatewayApiFormat } from "@/src/features/ai-gateway/server/provider";@@ -17,6 +17,7 @@ const mockEnv = vi.hoisted(() => ({LANGFUSE_AI_GATEWAY_JWT_KEY_ID: "key-id" as string | undefined,LANGFUSE_AI_GATEWAY_JWT_ISSUER: "test-issuer",LANGFUSE_AI_GATEWAY_JWT_AUDIENCE: "test-audience",+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined as string | undefined,}));vi.mock("@/src/env.mjs", () => ({ env: mockEnv }));@@ -289,6 +290,45 @@ describe("GatewayResolveService", () => {},);+ describe("on Langfuse Cloud", () => {+ beforeEach(() => {+ mockEnv.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "EU";+ });+ afterEach(() => {+ mockEnv.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;+ });++ it("resolves a stored NONE mode as USAGE with an ingestion grant", async () => {+ const service = serviceWith(() =>+ Promise.resolve(row({ ingestionMode: "NONE" })),+ );++ await expect(+ service.resolve({+ fastHashedSecretKey: "hashed-secret-key",+ apiFormat: "openai.responses",+ }),+ ).resolves.toMatchObject({+ attribution: { project_id: "project-1" },+ ingestion_mode: "usage",+ ingestion: { access_token: "ingestion-token", token_type: "Bearer" },+ });+ });++ it("refuses to resolve a stored NONE mode without an ingestion project", async () => {+ const service = serviceWith(() =>+ Promise.resolve(row({ ingestionMode: "NONE", withoutProject: true })),+ );++ await expect(+ service.resolve({+ fastHashedSecretKey: "hashed-secret-key",+ apiFormat: "openai.responses",+ }),+ ).rejects.toMatchObject({ status: 403 });+ });+ });+it("rejects an unknown or expired gateway key", async () => {const service = serviceWith(() => Promise.resolve(null));diff --git a/web/src/features/ai-gateway/server/resolve/resolveService.ts b/web/src/features/ai-gateway/server/resolve/resolveService.tsindex f94352d227..2a79030a90 100644--- a/web/src/features/ai-gateway/server/resolve/resolveService.ts+++ b/web/src/features/ai-gateway/server/resolve/resolveService.ts@@ -8,6 +8,7 @@ import {type Es256JwtSigner,} from "@/src/server/utils/jwt";+import { getEffectiveGatewayIngestionMode } from "@/src/features/ai-gateway/fns/gatewayIngestionModePolicy";import type { GatewayApiKeyAuthContext } from "@/src/features/ai-gateway/server/auth/gatewayApiKeyAuthenticator";import { resolveGatewayJwtKeys } from "@/src/features/ai-gateway/server/auth/gatewayJwtKeys";import { GATEWAY_INGESTION_TOKEN_TTL_SECONDS } from "@/src/features/ai-gateway/server/auth/ingestionTokenVerifier";@@ -71,7 +72,12 @@ export class GatewayResolveService {params.context.organizationId,);const { context } = params;- const { ingestionMode, ingestionProjectId } = context;+ const { ingestionProjectId } = context;+ // Applied after the resolve cache, so an entry cached as NONE still+ // captures on Cloud.+ const ingestionMode =+ context.ingestionMode &&+ getEffectiveGatewayIngestionMode(context.ingestionMode);// NONE mode delivers nothing to Langfuse, so it needs neither an// ingestion project nor ingestion signing keys.const deliversToLangfuse = ingestionMode !== "NONE";diff --git a/web/src/features/projects/components/DeleteProjectDialogController/DeleteProjectDialogController.tsx b/web/src/features/projects/components/DeleteProjectDialogController/DeleteProjectDialogController.tsxindex 267d20321b..ea24240c38 100644--- a/web/src/features/projects/components/DeleteProjectDialogController/DeleteProjectDialogController.tsx+++ b/web/src/features/projects/components/DeleteProjectDialogController/DeleteProjectDialogController.tsx@@ -1,3 +1,4 @@+import { getGatewayIngestionProjectDeletionMessage } from "@/src/features/ai-gateway/fns/gatewayIngestionModePolicy";import { ExternalLink } from "@langfuse/icons";import { type ReactNode } from "react";import { useRouter } from "next/router";@@ -61,7 +62,7 @@ export function DeleteProjectDialogController({return (<Dialogtitle="Project cannot be deleted"- text="This project is used as the AI Gateway ingestion project. Select another ingestion project or set the ingestion mode to None before deleting it."+ text={getGatewayIngestionProjectDeletionMessage()}actions={[{label: "Open AI Gateway settings",diff --git a/web/src/features/projects/server/projectsRouter.ts b/web/src/features/projects/server/projectsRouter.tsindex 06c444c166..c2126e7816 100644--- a/web/src/features/projects/server/projectsRouter.ts+++ b/web/src/features/projects/server/projectsRouter.ts@@ -1,3 +1,7 @@+import {+ getCapturingGatewayIngestionModes,+ getGatewayIngestionProjectDeletionMessage,+} from "@/src/features/ai-gateway/fns/gatewayIngestionModePolicy";import {createTRPCRouter,protectedOrganizationProcedure,@@ -416,7 +420,7 @@ async function isGatewayIngestionProject(params: {organizationId: params.organizationId,defaultIngestionProjectId: params.projectId,// With ingestion NONE the project is only remembered, not written to.- ingestionMode: { not: "NONE" },+ ingestionMode: { in: getCapturingGatewayIngestionModes() },},select: { organizationId: true },});@@ -430,7 +434,7 @@ async function throwIfGatewayIngestionProject(params: {}) {if (await isGatewayIngestionProject(params)) {throw new LangfuseConflictError(- "This project is used as the AI Gateway ingestion project. Select another ingestion project or set the ingestion mode to None before deleting it.",+ getGatewayIngestionProjectDeletionMessage(),);}}