Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(web): gate self-hosted AI gateway on LANGFUSE_AI_GATEWAY_DEPLOYED (#19237)

Tobias Wochinger5264b13ee5d2
diff --git a/.env.dev.example b/.env.dev.example
index 69c503c75d..7969e54685 100644
--- a/.env.dev.example
+++ b/.env.dev.example
@@ -235,8 +235,13 @@ ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000
# Without a public key, Web derives it from the private key (PKCS#8 or SEC1 PEM).
# Without a key ID, Web uses the public key's RFC 7638 JWK thumbprint (the
# previous key ID likewise defaults from the previous public key).
-# Set to true to enable the gateway for every organization, ignoring the allowlist.
-# LANGFUSE_AI_GATEWAY_ENABLED=false
+# Langfuse Cloud, development and preview deployments always count as having a gateway deployed;
+# elsewhere Web turns the gateway off unless LANGFUSE_AI_GATEWAY_DEPLOYED=true.
+# Development and preview enable it for every organization, ignoring the allowlist.
+# LANGFUSE_AI_GATEWAY_DEPLOYED=false
+# Unset enables a deployed gateway for every organization, except on Langfuse Cloud;
+# false limits it to organizations in the allowlist.
+# LANGFUSE_AI_GATEWAY_ENABLED=
# LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST=
# LANGFUSE_AI_GATEWAY_SERVICE_KEY=
# LANGFUSE_AI_GATEWAY_SERVICE_KEY_PREVIOUS=
diff --git a/.env.prod.example b/.env.prod.example
index 2c4aba1274..a72025b89f 100644
--- a/.env.prod.example
+++ b/.env.prod.example
@@ -610,9 +610,12 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true
# LANGFUSE_AI_GATEWAY_ALLOW_HTTP_WEB_URL=false
# Same shared service key configured in Web. Provider keys are resolved through Web.
# LANGFUSE_AI_GATEWAY_SERVICE_KEY=
-# Web only: enable the gateway for every organization. When false, only
-# organizations in LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST can use it.
-# LANGFUSE_AI_GATEWAY_ENABLED=false
+# Web only: set to true once a gateway is deployed next to Web. While false, every
+# gateway surface in Web is off, whatever the two settings below say.
+# LANGFUSE_AI_GATEWAY_DEPLOYED=false
+# Web only: enable a deployed gateway for every organization (the default). When
+# false, only organizations in LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST can use it.
+# LANGFUSE_AI_GATEWAY_ENABLED=true
# Web only: ES256 (P-256) private key, PKCS#8 or SEC1 PEM, that signs gateway
# ingestion tokens. The public key and key ID default to values derived from it.
# Set the key ID on every Web deployment or on none, so all of them agree on it.
diff --git a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts b/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts
index 4710e1ed89..d12df0cbee 100644
--- a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts
+++ b/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts
@@ -269,6 +269,32 @@ describe("AI gateway control plane", () => {
}
});
+ it.skipIf(env.NODE_ENV === "development")(
+ "turns the gateway off on a self-hosted deployment without a gateway",
+ async () => {
+ const { caller, org } = await prepare();
+ const original = {
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION:
+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,
+ LANGFUSE_AI_GATEWAY_DEPLOYED: env.LANGFUSE_AI_GATEWAY_DEPLOYED,
+ LANGFUSE_AI_GATEWAY_ENABLED: env.LANGFUSE_AI_GATEWAY_ENABLED,
+ };
+ Object.assign(env, {
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,
+ LANGFUSE_AI_GATEWAY_DEPLOYED: "false",
+ LANGFUSE_AI_GATEWAY_ENABLED: "true",
+ });
+
+ try {
+ await expect(
+ caller.aiGateway.getConfig({ orgId: org.id }),
+ ).rejects.toMatchObject({ code: "FORBIDDEN" });
+ } finally {
+ Object.assign(env, original);
+ }
+ },
+ );
+
it("enforces admin scope and validates the ingestion project organization", async () => {
const owner = await prepare();
const member = await prepare(Role.MEMBER);
diff --git a/web/src/env.mjs b/web/src/env.mjs
index 168c6a4774..1bc8fc6740 100644
--- a/web/src/env.mjs
+++ b/web/src/env.mjs
@@ -368,9 +368,15 @@ export const env = createEnv({
.optional(),
LANGFUSE_AI_GATEWAY_SERVICE_KEY: z.string().min(1).optional(),
LANGFUSE_AI_GATEWAY_SERVICE_KEY_PREVIOUS: z.string().min(1).optional(),
- // Enables the gateway for every organization, e.g. on self-hosted
- // deployments. When false, LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST applies.
- LANGFUSE_AI_GATEWAY_ENABLED: z.enum(["true", "false"]).default("false"),
+ // Whether an AI gateway is deployed next to Web. When false, every gateway
+ // surface in Web is off regardless of the flags below. Langfuse Cloud (any
+ // NEXT_PUBLIC_LANGFUSE_CLOUD_REGION), development and preview deployments
+ // count as deployed.
+ LANGFUSE_AI_GATEWAY_DEPLOYED: z.enum(["true", "false"]).default("false"),
+ // Enables a deployed gateway for every organization. When false, only
+ // organizations in LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST can use it.
+ // Unset means true, except on Langfuse Cloud where it means false.
+ LANGFUSE_AI_GATEWAY_ENABLED: z.enum(["true", "false"]).optional(),
LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST: z
.string()
.optional()
@@ -1153,6 +1159,7 @@ export const env = createEnv({
process.env.LANGFUSE_AI_GATEWAY_SERVICE_KEY,
LANGFUSE_AI_GATEWAY_SERVICE_KEY_PREVIOUS:
process.env.LANGFUSE_AI_GATEWAY_SERVICE_KEY_PREVIOUS,
+ LANGFUSE_AI_GATEWAY_DEPLOYED: process.env.LANGFUSE_AI_GATEWAY_DEPLOYED,
LANGFUSE_AI_GATEWAY_ENABLED: process.env.LANGFUSE_AI_GATEWAY_ENABLED,
LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST:
process.env.LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST,
diff --git a/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.servertest.ts b/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.servertest.ts
index 00ae671487..125e3f4e4c 100644
--- a/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.servertest.ts
+++ b/web/src/features/ai-gateway/server/auth/gatewayRequestSignatureVerifier.servertest.ts
@@ -18,6 +18,8 @@ vi.mock("@/src/env.mjs", () => ({
env: {
LANGFUSE_AI_GATEWAY_SERVICE_KEY: "current-service-secret",
LANGFUSE_AI_GATEWAY_SERVICE_KEY_PREVIOUS: "previous-service-secret",
+ LANGFUSE_AI_GATEWAY_DEPLOYED: "true",
+ LANGFUSE_AI_GATEWAY_ENABLED: "false",
LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST: ["org-1"],
SALT: "test-salt",
},
diff --git a/web/src/features/ai-gateway/server/auth/ingestionTokenVerifier.servertest.ts b/web/src/features/ai-gateway/server/auth/ingestionTokenVerifier.servertest.ts
index cf86605175..78666ca5ee 100644
--- a/web/src/features/ai-gateway/server/auth/ingestionTokenVerifier.servertest.ts
+++ b/web/src/features/ai-gateway/server/auth/ingestionTokenVerifier.servertest.ts
@@ -31,6 +31,8 @@ vi.mock("@/src/env.mjs", () => ({
LANGFUSE_AI_GATEWAY_JWT_PREVIOUS_PUBLIC_KEY: undefined,
LANGFUSE_AI_GATEWAY_JWT_ISSUER: "test-issuer",
LANGFUSE_AI_GATEWAY_JWT_AUDIENCE: "test-audience",
+ LANGFUSE_AI_GATEWAY_DEPLOYED: "true",
+ LANGFUSE_AI_GATEWAY_ENABLED: "false",
LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST: ["org-1"],
LANGFUSE_AI_GATEWAY_SERVICE_KEY: "current-service-key",
LANGFUSE_AI_GATEWAY_SERVICE_KEY_PREVIOUS: "previous-service-key",
@@ -117,6 +119,8 @@ describe("verifyGatewayIngestionAuthorization with derived key config", () => {
LANGFUSE_AI_GATEWAY_JWT_KEY_ID: undefined,
LANGFUSE_AI_GATEWAY_JWT_ISSUER: "test-issuer",
LANGFUSE_AI_GATEWAY_JWT_AUDIENCE: "test-audience",
+ LANGFUSE_AI_GATEWAY_DEPLOYED: "true",
+ LANGFUSE_AI_GATEWAY_ENABLED: "false",
LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST: ["org-1"],
LANGFUSE_AI_GATEWAY_SERVICE_KEY: "current-service-key",
},
@@ -171,6 +175,8 @@ describe("verifyGatewayIngestionAuthorization with derived key config", () => {
LANGFUSE_AI_GATEWAY_JWT_PREVIOUS_PUBLIC_KEY: publicKey,
LANGFUSE_AI_GATEWAY_JWT_ISSUER: "test-issuer",
LANGFUSE_AI_GATEWAY_JWT_AUDIENCE: "test-audience",
+ LANGFUSE_AI_GATEWAY_DEPLOYED: "true",
+ LANGFUSE_AI_GATEWAY_ENABLED: "false",
LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST: ["org-1"],
LANGFUSE_AI_GATEWAY_SERVICE_KEY: "current-service-key",
},
@@ -213,6 +219,8 @@ describe("verifyGatewayIngestionAuthorization with derived key config", () => {
LANGFUSE_AI_GATEWAY_JWT_PRIVATE_KEY: "not a pem",
LANGFUSE_AI_GATEWAY_JWT_ISSUER: "test-issuer",
LANGFUSE_AI_GATEWAY_JWT_AUDIENCE: "test-audience",
+ LANGFUSE_AI_GATEWAY_DEPLOYED: "true",
+ LANGFUSE_AI_GATEWAY_ENABLED: "false",
LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST: ["org-1"],
},
}));
diff --git a/web/src/features/ai-gateway/server/availability.servertest.ts b/web/src/features/ai-gateway/server/availability.servertest.ts
index 38e75f1326..56fe2bfc2c 100644
--- a/web/src/features/ai-gateway/server/availability.servertest.ts
+++ b/web/src/features/ai-gateway/server/availability.servertest.ts
@@ -6,9 +6,19 @@ import {
requireGatewayEnabledForOrganization,
} from "./availability";
+const PREVIEW_PR_URL = "https://github.com/langfuse/langfuse/pull/18561";
+
+const deployedSelfHosted = {
+ nodeEnv: "production",
+ deployed: "true",
+} as const;
+
describe("gateway organization allowlist", () => {
it("matches organization IDs exactly", () => {
- const environment = { nodeEnv: "production" };
+ const environment = {
+ ...deployedSelfHosted,
+ enabledForAllOrganizations: "false",
+ } as const;
expect(
isGatewayEnabledForOrganization("org-1", ["org-1", "org-2"], environment),
@@ -16,27 +26,139 @@ describe("gateway organization allowlist", () => {
expect(isGatewayEnabledForOrganization("org", ["org-1"], environment)).toBe(
false,
);
- });
-
- it("keeps the allowlist active outside local development", () => {
- expect(
- isGatewayEnabledForOrganization("org-not-allowed", [], {
- nodeEnv: "production",
- }),
- ).toBe(false);
expect(() =>
- requireGatewayEnabledForOrganization("org-not-allowed", [], {
- nodeEnv: "production",
- }),
+ requireGatewayEnabledForOrganization("org", ["org-1"], environment),
).toThrow(ForbiddenError);
});
+});
- it("enables the gateway in preview deployments", () => {
+describe("self-hosted deployments", () => {
+ it("enables every organization of a deployed gateway unless disabled", () => {
expect(
- isGatewayEnabledForOrganization("org-not-allowed", [], {
- nodeEnv: "production",
- previewPrUrl: "https://github.com/langfuse/langfuse/pull/18561",
+ isGatewayEnabledForOrganization("org-2", [], deployedSelfHosted),
+ ).toBe(true);
+ expect(
+ isGatewayEnabledForOrganization("org-2", [], {
+ ...deployedSelfHosted,
+ enabledForAllOrganizations: "true",
}),
).toBe(true);
+ expect(
+ isGatewayEnabledForOrganization("org-2", [], {
+ ...deployedSelfHosted,
+ enabledForAllOrganizations: "false",
+ }),
+ ).toBe(false);
});
+
+ it.each([undefined, "false"] as const)(
+ "turns the gateway off when deployed is %s, despite flags and allowlist",
+ (deployed) => {
+ const notDeployed = {
+ nodeEnv: "production",
+ deployed,
+ enabledForAllOrganizations: "true",
+ } as const;
+
+ expect(
+ isGatewayEnabledForOrganization("org-1", ["org-1"], notDeployed),
+ ).toBe(false);
+ expect(() =>
+ requireGatewayEnabledForOrganization("org-1", ["org-1"], notDeployed),
+ ).toThrow(ForbiddenError);
+ },
+ );
+
+ it.each([
+ { nodeEnv: "development" },
+ { nodeEnv: "production", previewPrUrl: PREVIEW_PR_URL },
+ ])(
+ "enables every organization on $nodeEnv / preview $previewPrUrl",
+ (environment) => {
+ for (const enabledForAllOrganizations of [
+ undefined,
+ "true",
+ "false",
+ ] as const) {
+ expect(
+ isGatewayEnabledForOrganization("org-2", [], {
+ ...environment,
+ enabledForAllOrganizations,
+ }),
+ ).toBe(true);
+ }
+ },
+ );
+});
+
+describe("Langfuse Cloud matches the previous availability rule", () => {
+ // The rule before LANGFUSE_AI_GATEWAY_DEPLOYED existed, when
+ // LANGFUSE_AI_GATEWAY_ENABLED defaulted to "false".
+ const previousRule = (
+ organizationId: string,
+ allowedOrganizationIds: string[],
+ environment: {
+ nodeEnv: string;
+ previewPrUrl?: string;
+ enabledForAllOrganizations?: "true" | "false";
+ },
+ ) =>
+ environment.enabledForAllOrganizations === "true" ||
+ environment.nodeEnv === "development" ||
+ environment.previewPrUrl !== undefined ||
+ allowedOrganizationIds.includes(organizationId);
+
+ const cases = ["US", "EU", "STAGING", "DEV", "HIPAA", "JP"].flatMap(
+ (cloudRegion) =>
+ [undefined, "true", "false"].flatMap((enabledForAllOrganizations) =>
+ [true, false].flatMap((allowlisted) =>
+ [
+ { name: "production", nodeEnv: "production" },
+ { name: "development", nodeEnv: "development" },
+ {
+ name: "preview",
+ nodeEnv: "production",
+ previewPrUrl: PREVIEW_PR_URL,
+ },
+ ].flatMap((runtime) =>
+ [undefined, "true", "false"].map((deployed) => ({
+ cloudRegion,
+ enabledForAllOrganizations: enabledForAllOrganizations as
+ | "true"
+ | "false"
+ | undefined,
+ allowlisted,
+ runtime,
+ deployed: deployed as "true" | "false" | undefined,
+ })),
+ ),
+ ),
+ ),
+ );
+
+ it.each(cases)(
+ "$cloudRegion, ENABLED=$enabledForAllOrganizations, allowlisted=$allowlisted, $runtime.name, DEPLOYED=$deployed",
+ ({
+ cloudRegion,
+ enabledForAllOrganizations,
+ allowlisted,
+ runtime,
+ deployed,
+ }) => {
+ const allowlist = allowlisted ? ["org-1"] : ["org-2"];
+ const environment = {
+ nodeEnv: runtime.nodeEnv,
+ previewPrUrl: runtime.previewPrUrl,
+ enabledForAllOrganizations,
+ };
+
+ expect(
+ isGatewayEnabledForOrganization("org-1", allowlist, {
+ ...environment,
+ cloudRegion,
+ deployed,
+ }),
+ ).toBe(previousRule("org-1", allowlist, environment));
+ },
+ );
});
diff --git a/web/src/features/ai-gateway/server/availability.ts b/web/src/features/ai-gateway/server/availability.ts
index aeb3665e24..08e911513e 100644
--- a/web/src/features/ai-gateway/server/availability.ts
+++ b/web/src/features/ai-gateway/server/availability.ts
@@ -4,27 +4,52 @@ import { env } from "@/src/env.mjs";
type GatewayAvailabilityEnvironment = {
nodeEnv: string | undefined;
+ cloudRegion?: string;
previewPrUrl?: string;
- enabledForAllOrganizations?: boolean;
+ deployed?: "true" | "false";
+ enabledForAllOrganizations?: "true" | "false";
};
const getGatewayAvailabilityEnvironment =
(): GatewayAvailabilityEnvironment => ({
nodeEnv: env.NODE_ENV,
+ cloudRegion: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,
previewPrUrl: env.NEXT_PUBLIC_PREVIEW_PR_URL,
- enabledForAllOrganizations: env.LANGFUSE_AI_GATEWAY_ENABLED === "true",
+ deployed: env.LANGFUSE_AI_GATEWAY_DEPLOYED,
+ enabledForAllOrganizations: env.LANGFUSE_AI_GATEWAY_ENABLED,
});
+const isDevelopmentOrPreview = (environment: GatewayAvailabilityEnvironment) =>
+ environment.nodeEnv === "development" ||
+ environment.previewPrUrl !== undefined;
+
+/**
+ * Langfuse Cloud (every region), local development and preview deployments
+ * always run a gateway next to Web.
+ */
+const isGatewayDeployed = (environment: GatewayAvailabilityEnvironment) =>
+ environment.deployed === "true" ||
+ Boolean(environment.cloudRegion) ||
+ isDevelopmentOrPreview(environment);
+
+/** Unset enables every organization, except on Langfuse Cloud. */
+const isEnabledForAllOrganizations = (
+ environment: GatewayAvailabilityEnvironment,
+) =>
+ environment.enabledForAllOrganizations === undefined
+ ? !environment.cloudRegion
+ : environment.enabledForAllOrganizations === "true";
+
export const isGatewayEnabledForOrganization = (
organizationId: string,
allowedOrganizationIds = env.LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST ??
[],
environment = getGatewayAvailabilityEnvironment(),
): boolean =>
- environment.enabledForAllOrganizations === true ||
- environment.nodeEnv === "development" ||
- environment.previewPrUrl !== undefined ||
- allowedOrganizationIds.includes(organizationId);
+ isGatewayDeployed(environment) &&
+ (isEnabledForAllOrganizations(environment) ||
+ isDevelopmentOrPreview(environment) ||
+ allowedOrganizationIds.includes(organizationId));
export const requireGatewayEnabledForOrganization = (
organizationId: string,
diff --git a/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts b/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts
index 378cc4183e..2c8dd9f60d 100644
--- a/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts
+++ b/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts
@@ -6,6 +6,8 @@ import type { GatewayApiFormat } from "@/src/features/ai-gateway/server/provider
import { GatewayResolveService } from "./resolveService";
const mockEnv = vi.hoisted(() => ({
+ LANGFUSE_AI_GATEWAY_DEPLOYED: "true",
+ LANGFUSE_AI_GATEWAY_ENABLED: "false",
LANGFUSE_AI_GATEWAY_ORGANIZATION_ID_ALLOWLIST: ["org-1"],
// Caching is covered against real Redis in the control-plane servertest;
// these cases are about what the service does with a database row.