Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(ai-gateway): always record gateway requests on Langfuse Cloud (#19505)

Hassieb Pakzadb7634d68dd26
diff --git a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts b/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts
index ed2516830b..c40529f16e 100644
--- a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts
+++ b/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts
@@ -719,11 +719,11 @@ describe("AI gateway control plane", () => {
).resolves.toBe(0);
});
- it("requires an ingestion project unless ingestion mode is NONE, also on cloud", async () => {
+ it("requires an ingestion project unless ingestion mode is NONE on self-hosted", async () => {
const { caller, org } = await prepare();
const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
try {
- Object.assign(env, { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU" });
+ Object.assign(env, { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined });
await expect(
caller.aiGateway.updateConfig({
orgId: org.id,
@@ -747,6 +747,50 @@ describe("AI gateway control plane", () => {
}
});
+ it("rejects NONE on cloud and treats a stored NONE config as USAGE", async () => {
+ const { caller, org, project } = await prepare();
+ const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
+ try {
+ Object.assign(env, { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU" });
+ await expect(
+ caller.aiGateway.updateConfig({
+ orgId: org.id,
+ defaultIngestionProjectId: project.id,
+ ingestionMode: "NONE",
+ }),
+ ).rejects.toThrow("not available on Langfuse Cloud");
+
+ await prisma.gatewayConfig.create({
+ data: {
+ organizationId: org.id,
+ defaultIngestionProjectId: null,
+ ingestionMode: "NONE",
+ },
+ });
+ await expect(
+ caller.aiGateway.getConfig({ orgId: org.id }),
+ ).resolves.toMatchObject({ config: { ingestionMode: "USAGE" } });
+ await expect(
+ caller.aiGateway.getSetupStatus({ orgId: org.id }),
+ ).resolves.toEqual({ isConfigured: false });
+
+ await prisma.gatewayConfig.update({
+ where: { organizationId: org.id },
+ data: { defaultIngestionProjectId: project.id },
+ });
+ await expect(
+ caller.projects.deletionProtection({ projectId: project.id }),
+ ).resolves.toEqual({ isGatewayIngestionProject: true });
+ await expect(
+ caller.projects.delete({ projectId: project.id }),
+ ).rejects.toThrow("Select another ingestion project before deleting it");
+ } finally {
+ Object.assign(env, {
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: originalCloudRegion,
+ });
+ }
+ });
+
it.each([
{ adminRole: Role.OWNER, readRole: Role.MEMBER },
{ adminRole: Role.ADMIN, readRole: Role.VIEWER },
@@ -2226,28 +2270,34 @@ describe("AI gateway control plane", () => {
).resolves.toEqual({ isGatewayIngestionProject: true });
await expect(
caller.projects.delete({ projectId: project.id }),
- ).rejects.toThrow(
- "Select another ingestion project or set the ingestion mode to None before deleting it",
- );
+ ).rejects.toThrow("Select another ingestion project");
await expect(
prisma.project.findUnique({ where: { id: project.id } }),
).resolves.toMatchObject({ deletedAt: null });
});
- it("allows deleting the linked ingestion project while ingestion is NONE", async () => {
+ it("allows deleting the linked ingestion project while ingestion is NONE on self-hosted", async () => {
const { caller, org, project } = await prepare();
- await caller.aiGateway.updateConfig({
- orgId: org.id,
- defaultIngestionProjectId: project.id,
- ingestionMode: "NONE",
- });
+ const originalCloudRegion = env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
+ try {
+ Object.assign(env, { NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined });
+ await caller.aiGateway.updateConfig({
+ orgId: org.id,
+ defaultIngestionProjectId: project.id,
+ ingestionMode: "NONE",
+ });
- await expect(
- caller.projects.deletionProtection({ projectId: project.id }),
- ).resolves.toEqual({ isGatewayIngestionProject: false });
- await expect(
- caller.projects.delete({ projectId: project.id }),
- ).resolves.toBe(true);
+ await expect(
+ caller.projects.deletionProtection({ projectId: project.id }),
+ ).resolves.toEqual({ isGatewayIngestionProject: false });
+ await expect(
+ caller.projects.delete({ projectId: project.id }),
+ ).resolves.toBe(true);
+ } finally {
+ Object.assign(env, {
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: originalCloudRegion,
+ });
+ }
await expect(
prisma.project.findUnique({ where: { id: project.id } }),
).resolves.toMatchObject({ deletedAt: expect.any(Date) });
diff --git a/web/src/__tests__/server/apiKeyCacheInvalidation.servertest.ts b/web/src/__tests__/server/apiKeyCacheInvalidation.servertest.ts
index 66527118ec..f68789b6bc 100644
--- a/web/src/__tests__/server/apiKeyCacheInvalidation.servertest.ts
+++ b/web/src/__tests__/server/apiKeyCacheInvalidation.servertest.ts
@@ -1,3 +1,4 @@
+import { getGatewayIngestionProjectDeletionMessage } from "@/src/features/ai-gateway/fns/gatewayIngestionModePolicy";
import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";
import { LLMAdapter, type Plan, Role } from "@langfuse/shared";
import { prisma } from "@langfuse/shared/src/db";
@@ -389,8 +390,7 @@ describe("API-key cache invalidation on project/org lifecycle", () => {
expect(res.statusCode).toBe(409);
expect(res.body).toEqual({
- message:
- "This project is used as the AI Gateway ingestion project. Select another ingestion project or set the ingestion mode to None before deleting it.",
+ message: getGatewayIngestionProjectDeletionMessage(),
});
expect(await survivingKeys(keys)).toEqual(keys);
await expect(
diff --git a/web/src/ee/features/admin-api/server/projects/projectById/index.ts b/web/src/ee/features/admin-api/server/projects/projectById/index.ts
index eb45ea770e..1308372f77 100644
--- a/web/src/ee/features/admin-api/server/projects/projectById/index.ts
+++ b/web/src/ee/features/admin-api/server/projects/projectById/index.ts
@@ -1,3 +1,7 @@
+import {
+ getCapturingGatewayIngestionModes,
+ getGatewayIngestionProjectDeletionMessage,
+} from "@/src/features/ai-gateway/fns/gatewayIngestionModePolicy";
import { type NextApiRequest, type NextApiResponse } from "next";
import { prisma } from "@langfuse/shared/src/db";
import {
@@ -125,14 +129,13 @@ export async function handleDeleteProject(
organizationId: scope.orgId,
defaultIngestionProjectId: projectId,
// With ingestion NONE the project is only remembered, not written to.
- ingestionMode: { not: "NONE" },
+ ingestionMode: { in: getCapturingGatewayIngestionModes() },
},
select: { organizationId: true },
});
if (gatewayConfig) {
return res.status(409).json({
- message:
- "This project is used as the AI Gateway ingestion project. Select another ingestion project or set the ingestion mode to None before deleting it.",
+ message: getGatewayIngestionProjectDeletionMessage(),
});
}
diff --git a/web/src/features/ai-gateway/components/GatewaySetupPage/components/IngestionSettingsFields/IngestionSettingsFields.tsx b/web/src/features/ai-gateway/components/GatewaySetupPage/components/IngestionSettingsFields/IngestionSettingsFields.tsx
index ed0d5732f5..1e2fce6e04 100644
--- a/web/src/features/ai-gateway/components/GatewaySetupPage/components/IngestionSettingsFields/IngestionSettingsFields.tsx
+++ b/web/src/features/ai-gateway/components/GatewaySetupPage/components/IngestionSettingsFields/IngestionSettingsFields.tsx
@@ -7,6 +7,8 @@ import type {
GatewayProject,
} from "@/src/features/ai-gateway/components/GatewaySetupPage/types/gatewaySetup";
import { GATEWAY_INGESTION_MODES } from "@/src/features/ai-gateway/constants/gatewayIngestionModes";
+import { isGatewayIngestionModeAvailable } from "@/src/features/ai-gateway/fns/gatewayIngestionModePolicy";
+import { cn } from "@/src/utils/tailwind";
export type IngestionSettingsFieldsProps = {
settings: GatewayIngestionSettings;
@@ -19,14 +21,23 @@ export function IngestionSettingsFields({
projects,
onChange,
}: IngestionSettingsFieldsProps) {
+ const modes = GATEWAY_INGESTION_MODES.filter((option) =>
+ isGatewayIngestionModeAvailable(option.value),
+ );
+
function handleModeSelect(ingestionMode: GatewayIngestionMode) {
onChange({ ...settings, ingestionMode });
}
return (
<div className="flex flex-col gap-4">
- <div className="grid gap-2 md:grid-cols-3">
- {GATEWAY_INGESTION_MODES.map((option) => (
+ <div
+ className={cn(
+ "grid gap-2",
+ modes.length > 2 ? "md:grid-cols-3" : "md:grid-cols-2",
+ )}
+ >
+ {modes.map((option) => (
<IngestionModeCard
key={option.value}
option={option}
diff --git a/web/src/features/ai-gateway/fns/gatewayIngestionModePolicy.ts b/web/src/features/ai-gateway/fns/gatewayIngestionModePolicy.ts
new file mode 100644
index 0000000000..7eca928cba
--- /dev/null
+++ b/web/src/features/ai-gateway/fns/gatewayIngestionModePolicy.ts
@@ -0,0 +1,37 @@
+import type { GatewayIngestionMode } from "@langfuse/shared";
+
+import { env } from "@/src/env.mjs";
+
+const STORED_INGESTION_MODES = [
+ "NONE",
+ "USAGE",
+ "FULL",
+] as const satisfies readonly GatewayIngestionMode[];
+
+/** Langfuse Cloud records a generation for every gateway request, so `NONE` is self-hosted only. */
+export function isGatewayIngestionModeAvailable(mode: GatewayIngestionMode) {
+ return mode !== "NONE" || !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
+}
+
+/**
+ * The mode a stored config runs with. A `NONE` config stored on Cloud
+ * captures usage, the minimal level, so no request goes unrecorded.
+ */
+export function getEffectiveGatewayIngestionMode(
+ mode: GatewayIngestionMode,
+): GatewayIngestionMode {
+ return isGatewayIngestionModeAvailable(mode) ? mode : "USAGE";
+}
+
+/** Stored modes under which the gateway writes into the ingestion project. */
+export function getCapturingGatewayIngestionModes() {
+ return STORED_INGESTION_MODES.filter(
+ (mode) => getEffectiveGatewayIngestionMode(mode) !== "NONE",
+ );
+}
+
+export function getGatewayIngestionProjectDeletionMessage() {
+ return isGatewayIngestionModeAvailable("NONE")
+ ? "This project is used as the AI Gateway ingestion project. Select another ingestion project or set the ingestion mode to None before deleting it."
+ : "This project is used as the AI Gateway ingestion project. Select another ingestion project before deleting it.";
+}
diff --git a/web/src/features/ai-gateway/server/config/gatewayConfigService.ts b/web/src/features/ai-gateway/server/config/gatewayConfigService.ts
index 20dc8267a8..2fb60dd865 100644
--- a/web/src/features/ai-gateway/server/config/gatewayConfigService.ts
+++ b/web/src/features/ai-gateway/server/config/gatewayConfigService.ts
@@ -7,6 +7,10 @@ import { auditLog } from "@/src/features/audit-logs/server";
import type { OrgAuthedContext } from "@/src/server/api/trpc";
import { invalidateGatewayResolveCacheForOrganization } from "@/src/features/ai-gateway/server/resolve/gatewayResolveCache";
import { isGatewayIngestionConfigured } from "@/src/features/ai-gateway/fns/isGatewayIngestionConfigured";
+import {
+ getEffectiveGatewayIngestionMode,
+ isGatewayIngestionModeAvailable,
+} from "@/src/features/ai-gateway/fns/gatewayIngestionModePolicy";
import { GatewayConfigRepository } from "./gatewayConfigRepository";
export class GatewayConfigService {
@@ -16,8 +20,14 @@ export class GatewayConfigService {
this.repository = new GatewayConfigRepository(prisma);
}
- getConfig(organizationId: string) {
- return this.repository.getConfig(organizationId);
+ async getConfig(organizationId: string) {
+ const config = await this.repository.getConfig(organizationId);
+ return (
+ config && {
+ ...config,
+ ingestionMode: getEffectiveGatewayIngestionMode(config.ingestionMode),
+ }
+ );
}
async getSetupStatus(organizationId: string) {
@@ -38,6 +48,11 @@ export class GatewayConfigService {
ingestionMode: GatewayIngestionMode;
session: OrgAuthedContext["session"];
}) {
+ if (!isGatewayIngestionModeAvailable(params.ingestionMode)) {
+ throw new InvalidRequestError(
+ "Ingestion mode NONE is not available on Langfuse Cloud",
+ );
+ }
if (
!isGatewayIngestionConfigured({
ingestionMode: params.ingestionMode,
@@ -49,7 +64,7 @@ export class GatewayConfigService {
);
}
- const before = await this.getConfig(params.organizationId);
+ const before = await this.repository.getConfig(params.organizationId);
if (params.defaultIngestionProjectId) {
const project = await this.repository.getActiveOrganizationProject({
diff --git a/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts b/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts
index fcb80b7bed..cbe1d88602 100644
--- a/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts
+++ b/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts
@@ -1,5 +1,5 @@
import type { PrismaClient } from "@langfuse/shared/src/db";
-import { describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { GatewayApiKeyAuthenticator } from "@/src/features/ai-gateway/server/auth/gatewayApiKeyAuthenticator";
import type { GatewayApiFormat } from "@/src/features/ai-gateway/server/provider";
@@ -17,6 +17,7 @@ const mockEnv = vi.hoisted(() => ({
LANGFUSE_AI_GATEWAY_JWT_KEY_ID: "key-id" as string | undefined,
LANGFUSE_AI_GATEWAY_JWT_ISSUER: "test-issuer",
LANGFUSE_AI_GATEWAY_JWT_AUDIENCE: "test-audience",
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined as string | undefined,
}));
vi.mock("@/src/env.mjs", () => ({ env: mockEnv }));
@@ -289,6 +290,45 @@ describe("GatewayResolveService", () => {
},
);
+ describe("on Langfuse Cloud", () => {
+ beforeEach(() => {
+ mockEnv.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "EU";
+ });
+ afterEach(() => {
+ mockEnv.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = undefined;
+ });
+
+ it("resolves a stored NONE mode as USAGE with an ingestion grant", async () => {
+ const service = serviceWith(() =>
+ Promise.resolve(row({ ingestionMode: "NONE" })),
+ );
+
+ await expect(
+ service.resolve({
+ fastHashedSecretKey: "hashed-secret-key",
+ apiFormat: "openai.responses",
+ }),
+ ).resolves.toMatchObject({
+ attribution: { project_id: "project-1" },
+ ingestion_mode: "usage",
+ ingestion: { access_token: "ingestion-token", token_type: "Bearer" },
+ });
+ });
+
+ it("refuses to resolve a stored NONE mode without an ingestion project", async () => {
+ const service = serviceWith(() =>
+ Promise.resolve(row({ ingestionMode: "NONE", withoutProject: true })),
+ );
+
+ await expect(
+ service.resolve({
+ fastHashedSecretKey: "hashed-secret-key",
+ apiFormat: "openai.responses",
+ }),
+ ).rejects.toMatchObject({ status: 403 });
+ });
+ });
+
it("rejects an unknown or expired gateway key", async () => {
const service = serviceWith(() => Promise.resolve(null));
diff --git a/web/src/features/ai-gateway/server/resolve/resolveService.ts b/web/src/features/ai-gateway/server/resolve/resolveService.ts
index f94352d227..2a79030a90 100644
--- a/web/src/features/ai-gateway/server/resolve/resolveService.ts
+++ b/web/src/features/ai-gateway/server/resolve/resolveService.ts
@@ -8,6 +8,7 @@ import {
type Es256JwtSigner,
} from "@/src/server/utils/jwt";
+import { getEffectiveGatewayIngestionMode } from "@/src/features/ai-gateway/fns/gatewayIngestionModePolicy";
import type { GatewayApiKeyAuthContext } from "@/src/features/ai-gateway/server/auth/gatewayApiKeyAuthenticator";
import { resolveGatewayJwtKeys } from "@/src/features/ai-gateway/server/auth/gatewayJwtKeys";
import { GATEWAY_INGESTION_TOKEN_TTL_SECONDS } from "@/src/features/ai-gateway/server/auth/ingestionTokenVerifier";
@@ -71,7 +72,12 @@ export class GatewayResolveService {
params.context.organizationId,
);
const { context } = params;
- const { ingestionMode, ingestionProjectId } = context;
+ const { ingestionProjectId } = context;
+ // Applied after the resolve cache, so an entry cached as NONE still
+ // captures on Cloud.
+ const ingestionMode =
+ context.ingestionMode &&
+ getEffectiveGatewayIngestionMode(context.ingestionMode);
// NONE mode delivers nothing to Langfuse, so it needs neither an
// ingestion project nor ingestion signing keys.
const deliversToLangfuse = ingestionMode !== "NONE";
diff --git a/web/src/features/projects/components/DeleteProjectDialogController/DeleteProjectDialogController.tsx b/web/src/features/projects/components/DeleteProjectDialogController/DeleteProjectDialogController.tsx
index 267d20321b..ea24240c38 100644
--- a/web/src/features/projects/components/DeleteProjectDialogController/DeleteProjectDialogController.tsx
+++ b/web/src/features/projects/components/DeleteProjectDialogController/DeleteProjectDialogController.tsx
@@ -1,3 +1,4 @@
+import { getGatewayIngestionProjectDeletionMessage } from "@/src/features/ai-gateway/fns/gatewayIngestionModePolicy";
import { ExternalLink } from "@langfuse/icons";
import { type ReactNode } from "react";
import { useRouter } from "next/router";
@@ -61,7 +62,7 @@ export function DeleteProjectDialogController({
return (
<Dialog
title="Project cannot be deleted"
- text="This project is used as the AI Gateway ingestion project. Select another ingestion project or set the ingestion mode to None before deleting it."
+ text={getGatewayIngestionProjectDeletionMessage()}
actions={[
{
label: "Open AI Gateway settings",
diff --git a/web/src/features/projects/server/projectsRouter.ts b/web/src/features/projects/server/projectsRouter.ts
index 06c444c166..c2126e7816 100644
--- a/web/src/features/projects/server/projectsRouter.ts
+++ b/web/src/features/projects/server/projectsRouter.ts
@@ -1,3 +1,7 @@
+import {
+ getCapturingGatewayIngestionModes,
+ getGatewayIngestionProjectDeletionMessage,
+} from "@/src/features/ai-gateway/fns/gatewayIngestionModePolicy";
import {
createTRPCRouter,
protectedOrganizationProcedure,
@@ -416,7 +420,7 @@ async function isGatewayIngestionProject(params: {
organizationId: params.organizationId,
defaultIngestionProjectId: params.projectId,
// With ingestion NONE the project is only remembered, not written to.
- ingestionMode: { not: "NONE" },
+ ingestionMode: { in: getCapturingGatewayIngestionModes() },
},
select: { organizationId: true },
});
@@ -430,7 +434,7 @@ async function throwIfGatewayIngestionProject(params: {
}) {
if (await isGatewayIngestionProject(params)) {
throw new LangfuseConflictError(
- "This project is used as the AI Gateway ingestion project. Select another ingestion project or set the ingestion mode to None before deleting it.",
+ getGatewayIngestionProjectDeletionMessage(),
);
}
}