🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(web): gate self-hosted AI gateway on LANGFUSE_AI_GATEWAY_DEPLOYED (#19237)
Tobias Wochinger5264b13ee5d2
diff --git a/.env.dev.example b/.env.dev.exampleindex 69c503c75d..7969e54685 100644--- a/.env.dev.example+++ b/.env.dev.example@@ -235,8 +235,13 @@ ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000# Without a public key, Web derives it from the private key (PKCS#8 or SEC1 PEM).# Without a key ID, Web uses the public key's RFC 7638 JWK thumbprint (the# previous key ID likewise defaults from the previous public key).-# Set to true to enable the gateway for every organization, ignoring the allowlist.-# LANGFUSE_AI_GATEWAY_ENABLED=false+# Langfuse Cloud, development and preview deployments always count as having a gateway deployed;+# elsewhere Web turns the gateway off unless LANGFUSE_AI_GATEWAY_DEPLOYED=true.+# Development and preview enable it for every organization, ignoring the allowlist.+# LANGFUSE_AI_GATEWAY_DEPLOYED=false+# Unset enables a deployed gateway for every organization, except on Langfuse Cloud;+# false limits it to organizations in the allowlist.+# LANGFUSE_AI_GATEWAY_ENABLED=# LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST=# LANGFUSE_AI_GATEWAY_SERVICE_KEY=# LANGFUSE_AI_GATEWAY_SERVICE_KEY_PREVIOUS=diff --git a/.env.prod.example b/.env.prod.exampleindex 2c4aba1274..a72025b89f 100644--- a/.env.prod.example+++ b/.env.prod.example@@ -610,9 +610,12 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true# LANGFUSE_AI_GATEWAY_ALLOW_HTTP_WEB_URL=false# Same shared service key configured in Web. Provider keys are resolved through Web.# LANGFUSE_AI_GATEWAY_SERVICE_KEY=-# Web only: enable the gateway for every organization. When false, only-# organizations in LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST can use it.-# LANGFUSE_AI_GATEWAY_ENABLED=false+# Web only: set to true once a gateway is deployed next to Web. While false, every+# gateway surface in Web is off, whatever the two settings below say.+# LANGFUSE_AI_GATEWAY_DEPLOYED=false+# Web only: enable a deployed gateway for every organization (the default). When+# false, only organizations in LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST can use it.+# LANGFUSE_AI_GATEWAY_ENABLED=true# Web only: ES256 (P-256) private key, PKCS#8 or SEC1 PEM, that signs gateway# ingestion tokens. The public key and key ID default to values derived from it.# Set the key ID on every Web deployment or on none, so all of them agree on it.diff --git a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts b/web/src/__tests__/server/ai-gateway-control-plane.servertest.tsindex 4710e1ed89..d12df0cbee 100644--- a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts+++ b/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts@@ -269,6 +269,32 @@ describe("AI gateway control plane", () => {}});+ it.skipIf(env.NODE_ENV === "development")(+ "turns the gateway off on a self-hosted deployment without a gateway",+ async () => {+ const { caller, org } = await prepare();+ const original = {+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION:+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,+ LANGFUSE_AI_GATEWAY_DEPLOYED: env.LANGFUSE_AI_GATEWAY_DEPLOYED,+ LANGFUSE_AI_GATEWAY_ENABLED: env.LANGFUSE_AI_GATEWAY_ENABLED,+ };+ Object.assign(env, {+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,+ LANGFUSE_AI_GATEWAY_DEPLOYED: "false",+ LANGFUSE_AI_GATEWAY_ENABLED: "true",+ });++ try {+ await expect(+ caller.aiGateway.getConfig({ orgId: org.id }),+ ).rejects.toMatchObject({ code: "FORBIDDEN" });+ } finally {+ Object.assign(env, original);+ }+ },+ );+it("enforces admin scope and validates the ingestion project organization", async () => {const owner = await prepare();const member = await prepare(Role.MEMBER);diff --git a/web/src/env.mjs b/web/src/env.mjsindex 168c6a4774..1bc8fc6740 100644--- a/web/src/env.mjs+++ b/web/src/env.mjs@@ -368,9 +368,15 @@ export const env = createEnv({.optional(),LANGFUSE_AI_GATEWAY_SERVICE_KEY: z.string().min(1).optional(),LANGFUSE_AI_GATEWAY_SERVICE_KEY_PREVIOUS: z.string().min(1).optional(),- // Enables the gateway for every organization, e.g. on self-hosted- // deployments. When false, LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST applies.- LANGFUSE_AI_GATEWAY_ENABLED: z.enum(["true", "false"]).default("false"),+ // Whether an AI gateway is deployed next to Web. When false, every gateway+ // surface in Web is off regardless of the flags below. Langfuse Cloud (any+ // NEXT_PUBLIC_LANGFUSE_CLOUD_REGION), development and preview deployments+ // count as deployed.+ LANGFUSE_AI_GATEWAY_DEPLOYED: z.enum(["true", "false"]).default("false"),+ // Enables a deployed gateway for every organization. When false, only+ // organizations in LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST can use it.+ // Unset means true, except on Langfuse Cloud where it means false.+ LANGFUSE_AI_GATEWAY_ENABLED: z.enum(["true", "false"]).optional(),LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST: z.string().optional()@@ -1153,6 +1159,7 @@ export const env = createEnv({process.env.LANGFUSE_AI_GATEWAY_SERVICE_KEY,LANGFUSE_AI_GATEWAY_SERVICE_KEY_PREVIOUS:process.env.LANGFUSE_AI_GATEWAY_SERVICE_KEY_PREVIOUS,+ LANGFUSE_AI_GATEWAY_DEPLOYED: process.env.LANGFUSE_AI_GATEWAY_DEPLOYED,LANGFUSE_AI_GATEWAY_ENABLED: process.env.LANGFUSE_AI_GATEWAY_ENABLED,LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST:process.env.LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST,diff --git a/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.servertest.ts b/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.servertest.tsindex 00ae671487..125e3f4e4c 100644--- a/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.servertest.ts+++ b/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.servertest.ts@@ -18,6 +18,8 @@ vi.mock("@/src/env.mjs", () => ({env: {LANGFUSE_AI_GATEWAY_SERVICE_KEY: "current-service-secret",LANGFUSE_AI_GATEWAY_SERVICE_KEY_PREVIOUS: "previous-service-secret",+ LANGFUSE_AI_GATEWAY_DEPLOYED: "true",+ LANGFUSE_AI_GATEWAY_ENABLED: "false",LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST: ["org-1"],SALT: "test-salt",},diff --git a/web/src/features/ai-gateway/server/auth/ingestionTokenVerifier.servertest.ts b/web/src/features/ai-gateway/server/auth/ingestionTokenVerifier.servertest.tsindex cf86605175..78666ca5ee 100644--- a/web/src/features/ai-gateway/server/auth/ingestionTokenVerifier.servertest.ts+++ b/web/src/features/ai-gateway/server/auth/ingestionTokenVerifier.servertest.ts@@ -31,6 +31,8 @@ vi.mock("@/src/env.mjs", () => ({LANGFUSE_AI_GATEWAY_JWT_PREVIOUS_PUBLIC_KEY: undefined,LANGFUSE_AI_GATEWAY_JWT_ISSUER: "test-issuer",LANGFUSE_AI_GATEWAY_JWT_AUDIENCE: "test-audience",+ LANGFUSE_AI_GATEWAY_DEPLOYED: "true",+ LANGFUSE_AI_GATEWAY_ENABLED: "false",LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST: ["org-1"],LANGFUSE_AI_GATEWAY_SERVICE_KEY: "current-service-key",LANGFUSE_AI_GATEWAY_SERVICE_KEY_PREVIOUS: "previous-service-key",@@ -117,6 +119,8 @@ describe("verifyGatewayIngestionAuthorization with derived key config", () => {LANGFUSE_AI_GATEWAY_JWT_KEY_ID: undefined,LANGFUSE_AI_GATEWAY_JWT_ISSUER: "test-issuer",LANGFUSE_AI_GATEWAY_JWT_AUDIENCE: "test-audience",+ LANGFUSE_AI_GATEWAY_DEPLOYED: "true",+ LANGFUSE_AI_GATEWAY_ENABLED: "false",LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST: ["org-1"],LANGFUSE_AI_GATEWAY_SERVICE_KEY: "current-service-key",},@@ -171,6 +175,8 @@ describe("verifyGatewayIngestionAuthorization with derived key config", () => {LANGFUSE_AI_GATEWAY_JWT_PREVIOUS_PUBLIC_KEY: publicKey,LANGFUSE_AI_GATEWAY_JWT_ISSUER: "test-issuer",LANGFUSE_AI_GATEWAY_JWT_AUDIENCE: "test-audience",+ LANGFUSE_AI_GATEWAY_DEPLOYED: "true",+ LANGFUSE_AI_GATEWAY_ENABLED: "false",LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST: ["org-1"],LANGFUSE_AI_GATEWAY_SERVICE_KEY: "current-service-key",},@@ -213,6 +219,8 @@ describe("verifyGatewayIngestionAuthorization with derived key config", () => {LANGFUSE_AI_GATEWAY_JWT_PRIVATE_KEY: "not a pem",LANGFUSE_AI_GATEWAY_JWT_ISSUER: "test-issuer",LANGFUSE_AI_GATEWAY_JWT_AUDIENCE: "test-audience",+ LANGFUSE_AI_GATEWAY_DEPLOYED: "true",+ LANGFUSE_AI_GATEWAY_ENABLED: "false",LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST: ["org-1"],},}));diff --git a/web/src/features/ai-gateway/server/availability.servertest.ts b/web/src/features/ai-gateway/server/availability.servertest.tsindex 38e75f1326..56fe2bfc2c 100644--- a/web/src/features/ai-gateway/server/availability.servertest.ts+++ b/web/src/features/ai-gateway/server/availability.servertest.ts@@ -6,9 +6,19 @@ import {requireGatewayEnabledForOrganization,} from "./availability";+const PREVIEW_PR_URL = "https://github.com/langfuse/langfuse/pull/18561";++const deployedSelfHosted = {+ nodeEnv: "production",+ deployed: "true",+} as const;+describe("gateway organization allowlist", () => {it("matches organization IDs exactly", () => {- const environment = { nodeEnv: "production" };+ const environment = {+ ...deployedSelfHosted,+ enabledForAllOrganizations: "false",+ } as const;expect(isGatewayEnabledForOrganization("org-1", ["org-1", "org-2"], environment),@@ -16,27 +26,139 @@ describe("gateway organization allowlist", () => {expect(isGatewayEnabledForOrganization("org", ["org-1"], environment)).toBe(false,);- });-- it("keeps the allowlist active outside local development", () => {- expect(- isGatewayEnabledForOrganization("org-not-allowed", [], {- nodeEnv: "production",- }),- ).toBe(false);expect(() =>- requireGatewayEnabledForOrganization("org-not-allowed", [], {- nodeEnv: "production",- }),+ requireGatewayEnabledForOrganization("org", ["org-1"], environment),).toThrow(ForbiddenError);});+});- it("enables the gateway in preview deployments", () => {+describe("self-hosted deployments", () => {+ it("enables every organization of a deployed gateway unless disabled", () => {expect(- isGatewayEnabledForOrganization("org-not-allowed", [], {- nodeEnv: "production",- previewPrUrl: "https://github.com/langfuse/langfuse/pull/18561",+ isGatewayEnabledForOrganization("org-2", [], deployedSelfHosted),+ ).toBe(true);+ expect(+ isGatewayEnabledForOrganization("org-2", [], {+ ...deployedSelfHosted,+ enabledForAllOrganizations: "true",}),).toBe(true);+ expect(+ isGatewayEnabledForOrganization("org-2", [], {+ ...deployedSelfHosted,+ enabledForAllOrganizations: "false",+ }),+ ).toBe(false);});++ it.each([undefined, "false"] as const)(+ "turns the gateway off when deployed is %s, despite flags and allowlist",+ (deployed) => {+ const notDeployed = {+ nodeEnv: "production",+ deployed,+ enabledForAllOrganizations: "true",+ } as const;++ expect(+ isGatewayEnabledForOrganization("org-1", ["org-1"], notDeployed),+ ).toBe(false);+ expect(() =>+ requireGatewayEnabledForOrganization("org-1", ["org-1"], notDeployed),+ ).toThrow(ForbiddenError);+ },+ );++ it.each([+ { nodeEnv: "development" },+ { nodeEnv: "production", previewPrUrl: PREVIEW_PR_URL },+ ])(+ "enables every organization on $nodeEnv / preview $previewPrUrl",+ (environment) => {+ for (const enabledForAllOrganizations of [+ undefined,+ "true",+ "false",+ ] as const) {+ expect(+ isGatewayEnabledForOrganization("org-2", [], {+ ...environment,+ enabledForAllOrganizations,+ }),+ ).toBe(true);+ }+ },+ );+});++describe("Langfuse Cloud matches the previous availability rule", () => {+ // The rule before LANGFUSE_AI_GATEWAY_DEPLOYED existed, when+ // LANGFUSE_AI_GATEWAY_ENABLED defaulted to "false".+ const previousRule = (+ organizationId: string,+ allowedOrganizationIds: string[],+ environment: {+ nodeEnv: string;+ previewPrUrl?: string;+ enabledForAllOrganizations?: "true" | "false";+ },+ ) =>+ environment.enabledForAllOrganizations === "true" ||+ environment.nodeEnv === "development" ||+ environment.previewPrUrl !== undefined ||+ allowedOrganizationIds.includes(organizationId);++ const cases = ["US", "EU", "STAGING", "DEV", "HIPAA", "JP"].flatMap(+ (cloudRegion) =>+ [undefined, "true", "false"].flatMap((enabledForAllOrganizations) =>+ [true, false].flatMap((allowlisted) =>+ [+ { name: "production", nodeEnv: "production" },+ { name: "development", nodeEnv: "development" },+ {+ name: "preview",+ nodeEnv: "production",+ previewPrUrl: PREVIEW_PR_URL,+ },+ ].flatMap((runtime) =>+ [undefined, "true", "false"].map((deployed) => ({+ cloudRegion,+ enabledForAllOrganizations: enabledForAllOrganizations as+ | "true"+ | "false"+ | undefined,+ allowlisted,+ runtime,+ deployed: deployed as "true" | "false" | undefined,+ })),+ ),+ ),+ ),+ );++ it.each(cases)(+ "$cloudRegion, ENABLED=$enabledForAllOrganizations, allowlisted=$allowlisted, $runtime.name, DEPLOYED=$deployed",+ ({+ cloudRegion,+ enabledForAllOrganizations,+ allowlisted,+ runtime,+ deployed,+ }) => {+ const allowlist = allowlisted ? ["org-1"] : ["org-2"];+ const environment = {+ nodeEnv: runtime.nodeEnv,+ previewPrUrl: runtime.previewPrUrl,+ enabledForAllOrganizations,+ };++ expect(+ isGatewayEnabledForOrganization("org-1", allowlist, {+ ...environment,+ cloudRegion,+ deployed,+ }),+ ).toBe(previousRule("org-1", allowlist, environment));+ },+ );});diff --git a/web/src/features/ai-gateway/server/availability.ts b/web/src/features/ai-gateway/server/availability.tsindex aeb3665e24..08e911513e 100644--- a/web/src/features/ai-gateway/server/availability.ts+++ b/web/src/features/ai-gateway/server/availability.ts@@ -4,27 +4,52 @@ import { env } from "@/src/env.mjs";type GatewayAvailabilityEnvironment = {nodeEnv: string | undefined;+ cloudRegion?: string;previewPrUrl?: string;- enabledForAllOrganizations?: boolean;+ deployed?: "true" | "false";+ enabledForAllOrganizations?: "true" | "false";};const getGatewayAvailabilityEnvironment =(): GatewayAvailabilityEnvironment => ({nodeEnv: env.NODE_ENV,+ cloudRegion: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,previewPrUrl: env.NEXT_PUBLIC_PREVIEW_PR_URL,- enabledForAllOrganizations: env.LANGFUSE_AI_GATEWAY_ENABLED === "true",+ deployed: env.LANGFUSE_AI_GATEWAY_DEPLOYED,+ enabledForAllOrganizations: env.LANGFUSE_AI_GATEWAY_ENABLED,});+const isDevelopmentOrPreview = (environment: GatewayAvailabilityEnvironment) =>+ environment.nodeEnv === "development" ||+ environment.previewPrUrl !== undefined;++/**+ * Langfuse Cloud (every region), local development and preview deployments+ * always run a gateway next to Web.+ */+const isGatewayDeployed = (environment: GatewayAvailabilityEnvironment) =>+ environment.deployed === "true" ||+ Boolean(environment.cloudRegion) ||+ isDevelopmentOrPreview(environment);++/** Unset enables every organization, except on Langfuse Cloud. */+const isEnabledForAllOrganizations = (+ environment: GatewayAvailabilityEnvironment,+) =>+ environment.enabledForAllOrganizations === undefined+ ? !environment.cloudRegion+ : environment.enabledForAllOrganizations === "true";+export const isGatewayEnabledForOrganization = (organizationId: string,allowedOrganizationIds = env.LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST ??[],environment = getGatewayAvailabilityEnvironment(),): boolean =>- environment.enabledForAllOrganizations === true ||- environment.nodeEnv === "development" ||- environment.previewPrUrl !== undefined ||- allowedOrganizationIds.includes(organizationId);+ isGatewayDeployed(environment) &&+ (isEnabledForAllOrganizations(environment) ||+ isDevelopmentOrPreview(environment) ||+ allowedOrganizationIds.includes(organizationId));export const requireGatewayEnabledForOrganization = (organizationId: string,diff --git a/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts b/web/src/features/ai-gateway/server/resolve/resolveService.servertest.tsindex 378cc4183e..2c8dd9f60d 100644--- a/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts+++ b/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts@@ -6,6 +6,8 @@ import type { GatewayApiFormat } from "@/src/features/ai-gateway/server/providerimport { GatewayResolveService } from "./resolveService";const mockEnv = vi.hoisted(() => ({+ LANGFUSE_AI_GATEWAY_DEPLOYED: "true",+ LANGFUSE_AI_GATEWAY_ENABLED: "false",LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST: ["org-1"],// Caching is covered against real Redis in the control-plane servertest;// these cases are about what the service does with a database row.