🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(web): separate AI Gateway keys from organization API keys (#18936)
Tobias Wochinger2c313311618e
diff --git a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts b/web/src/__tests__/server/ai-gateway-control-plane.servertest.tsindex 122a0627aa..6d053a4643 100644--- a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts+++ b/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts@@ -36,7 +36,11 @@ import {import { createEs256JwtVerifier } from "@/src/server/utils/jwt";import { prisma, Role } from "@langfuse/shared/src/db";import { decrypt } from "@langfuse/shared/encryption";-import { createShaHash } from "@langfuse/shared/src/server/auth/apiKeys";+import {+ createApiKey,+ createShaHash,+} from "@langfuse/shared/src/server/auth/apiKeys";+import { OrganizationId, SystemRoleId, UserId } from "@langfuse/shared/rbac";const cleanupOrganizations: string[] = [];const cleanupUsers: string[] = [];@@ -1366,7 +1370,7 @@ describe("AI gateway control plane", () => {apiFormat: "openai.responses" as const,model: "OpenAI:gpt-4o",};- const lookup = vi.spyOn(prisma.gatewayApiKeyAssociation, "findFirst");+ const lookup = vi.spyOn(prisma.apiKey, "findFirst");for (let attempt = 0; attempt < 2; attempt++) {await expect(authenticateRequest(resolveParams)).resolves.toMatchObject({@@ -1413,7 +1417,7 @@ describe("AI gateway control plane", () => {apiFormat: "openai.responses" as const,model: "gpt-4o",};- const lookup = vi.spyOn(prisma.gatewayApiKeyAssociation, "findFirst");+ const lookup = vi.spyOn(prisma.apiKey, "findFirst");for (let attempt = 0; attempt < 3; attempt++) {await expect(authenticateRequest(resolveParams)).rejects.toEqual(@@ -1426,7 +1430,7 @@ describe("AI gateway control plane", () => {lookup.mockRestore();});- it("enforces gateway:invoke when API key role enforcement is on", async () => {+ it("enforces gateway:invoke regardless of API key role enforcement", async () => {const originalMigration = env.API_AUTH_MIGRATION;const originalCache = env.LANGFUSE_AI_GATEWAY_CACHE_RESOLVE_ENABLED;Object.assign(env, { LANGFUSE_AI_GATEWAY_CACHE_RESOLVE_ENABLED: "false" });@@ -1442,7 +1446,6 @@ describe("AI gateway control plane", () => {apiFormat: "openai.responses" as const,};- Object.assign(env, { API_AUTH_MIGRATION: "enforce" });await expect(authenticator.authenticateModelsRequest(request),).resolves.toMatchObject({@@ -1453,22 +1456,46 @@ describe("AI gateway control plane", () => {where: { apiKeyId: gatewayKey.id },data: { systemRole: "INGEST", roleId: "system/INGEST" },});- await expect(- authenticator.authenticateModelsRequest(request),- ).rejects.toMatchObject({- status: 403,- message: "API key is not allowed to invoke the gateway",+ for (const migration of ["enforce", "legacy"] as const) {+ Object.assign(env, { API_AUTH_MIGRATION: migration });+ await expect(+ authenticator.authenticateModelsRequest(request),+ ).rejects.toMatchObject({+ status: 403,+ message: "API key is not allowed to invoke the gateway",+ });+ }+ } finally {+ Object.assign(env, {+ API_AUTH_MIGRATION: originalMigration,+ LANGFUSE_AI_GATEWAY_CACHE_RESOLVE_ENABLED: originalCache,+ });+ }+ });++ it("accepts organization keys granted gateway:invoke without a gateway association", async () => {+ const originalCache = env.LANGFUSE_AI_GATEWAY_CACHE_RESOLVE_ENABLED;+ Object.assign(env, { LANGFUSE_AI_GATEWAY_CACHE_RESOLVE_ENABLED: "false" });+ try {+ const { org, user } = await prepare();+ const adminKey = await createApiKey(prisma, {+ owner: OrganizationId(org.id),+ role: SystemRoleId("ADMIN"),+ createdBy: UserId(user.id),});- Object.assign(env, { API_AUTH_MIGRATION: "legacy" });await expect(- authenticator.authenticateModelsRequest(request),+ new GatewayApiKeyAuthenticator(prisma).authenticateModelsRequest({+ fastHashedSecretKey: createShaHash(adminKey.secretKey, env.SALT),+ apiFormat: "openai.responses",+ }),).resolves.toMatchObject({organizationId: org.id,+ apiKeyId: adminKey.id,+ keyMetadata: {},});} finally {Object.assign(env, {- API_AUTH_MIGRATION: originalMigration,LANGFUSE_AI_GATEWAY_CACHE_RESOLVE_ENABLED: originalCache,});}diff --git a/web/src/__tests__/server/organizations-api-key-trpc.servertest.ts b/web/src/__tests__/server/organizations-api-key-trpc.servertest.tsindex 50c8ca90a3..e56db20949 100644--- a/web/src/__tests__/server/organizations-api-key-trpc.servertest.ts+++ b/web/src/__tests__/server/organizations-api-key-trpc.servertest.ts@@ -242,6 +242,29 @@ describe("organization API keys trpc", () => {);});+ it("filters AI Gateway keys", async () => {+ const gatewayKey = await createApiKey(prisma, {+ owner: OrganizationId(organizationId),+ role: SystemRoleId("AI_GATEWAY"),+ createdBy: UserId("user-1"),+ name: "Gateway key",+ });+ const ingestKey = await createApiKey(prisma, {+ owner: OrganizationId(organizationId),+ role: SystemRoleId("INGEST"),+ createdBy: UserId("user-1"),+ name: "Ingestion key",+ });++ const apiKeys = await ownerCaller.organizationApiKeys.byOrganizationId({+ orgId: organizationId,+ });+ const ids = apiKeys.map((key) => key.id);++ expect(ids).not.toContain(gatewayKey.id);+ expect(ids).toContain(ingestKey.id);+ });+it("regular member cannot fetch organization API keys", async () => {await expect(memberCaller.organizationApiKeys.byOrganizationId({diff --git a/web/src/features/ai-gateway/fns/separateGatewayApiKeys.ts b/web/src/features/ai-gateway/fns/separateGatewayApiKeys.tsnew file mode 100644index 0000000000..15aa2cd63e--- /dev/null+++ b/web/src/features/ai-gateway/fns/separateGatewayApiKeys.ts@@ -0,0 +1,16 @@+import { type SystemRole } from "@langfuse/shared/src/db";++/**+ * Gateway keys are created and listed only on the AI Gateway settings page.+ * Set to false to offer and list them in organization API keys again. The+ * public API still accepts and returns them either way.+ */+export const SEPARATE_GATEWAY_API_KEYS = true;++export const GATEWAY_ROLE: SystemRole = "AI_GATEWAY";++/** withoutGatewayRole drops the gateway role from the roles offered when creating an organization API key. */+export function withoutGatewayRole(roles: SystemRole[]): SystemRole[] {+ if (!SEPARATE_GATEWAY_API_KEYS) return roles;+ return roles.filter((role) => role !== GATEWAY_ROLE);+}diff --git a/web/src/features/ai-gateway/server/apiKey/excludeGatewayApiKeys.ts b/web/src/features/ai-gateway/server/apiKey/excludeGatewayApiKeys.tsnew file mode 100644index 0000000000..bf5d7dfcf6--- /dev/null+++ b/web/src/features/ai-gateway/server/apiKey/excludeGatewayApiKeys.ts@@ -0,0 +1,12 @@+import type { Prisma } from "@langfuse/shared/src/db";++import {+ GATEWAY_ROLE,+ SEPARATE_GATEWAY_API_KEYS,+} from "@/src/features/ai-gateway/fns/separateGatewayApiKeys";++/** excludeGatewayApiKeys filters gateway-role keys out of the organization API key settings list. */+export const excludeGatewayApiKeys: Prisma.ApiKeyWhereInput =+ SEPARATE_GATEWAY_API_KEYS+ ? { roleAssignments: { none: { systemRole: GATEWAY_ROLE } } }+ : {};diff --git a/web/src/features/ai-gateway/server/auth/gatewayApiKeyAuthenticator.ts b/web/src/features/ai-gateway/server/auth/gatewayApiKeyAuthenticator.tsindex 48aa006e11..ae5804ddbd 100644--- a/web/src/features/ai-gateway/server/auth/gatewayApiKeyAuthenticator.ts+++ b/web/src/features/ai-gateway/server/auth/gatewayApiKeyAuthenticator.ts@@ -1,7 +1,6 @@import type { GatewayProvider, PrismaClient } from "@langfuse/shared/src/db";import { ApiKeyId, OrganizationId } from "@langfuse/shared/rbac";-import { env } from "@/src/env.mjs";import { authorize } from "@/src/features/rbac/authorize";import { getRolesForPrincipal } from "@/src/features/rbac/getRolesForPrincipal";import { isGatewayEnabledForOrganization } from "@/src/features/ai-gateway/server/availability";@@ -129,8 +128,8 @@ export class GatewayApiKeyAuthenticator {providers: supportedProviders(params.apiFormat),});- const organizationId = row?.apiKey.orgId;- const organization = row?.apiKey.organization;+ const organizationId = row?.orgId;+ const organization = row?.organization;if (!row || !organizationId || !organization) {await this.cache.set({fastHashedSecretKey: params.fastHashedSecretKey,@@ -148,8 +147,8 @@ export class GatewayApiKeyAuthenticator {project?.orgId === organizationId;const context: CachedResolveContext = {organizationId,- apiKeyId: row.apiKeyId,- keyMetadata: toKeyMetadata(row.metadata),+ apiKeyId: row.id,+ keyMetadata: toKeyMetadata(row.gatewayAssociation?.metadata),ingestionProjectId: projectIsUsable? (gatewayConfig?.defaultIngestionProjectId ?? null): null,@@ -160,15 +159,15 @@ export class GatewayApiKeyAuthenticator {})),};- if (env.API_AUTH_MIGRATION === "enforce") {- await assertCanInvokeGateway(this.prisma, row.apiKeyId, organizationId);- }+ // Any organization key may call the gateway when its roles grant+ // gateway:invoke, so this check must not depend on API_AUTH_MIGRATION.+ await assertCanInvokeGateway(this.prisma, row.id, organizationId);await this.cache.set({fastHashedSecretKey: params.fastHashedSecretKey,apiFormat: params.apiFormat,context,- expiresAt: row.apiKey.expiresAt,+ expiresAt: row.expiresAt,});return context;}diff --git a/web/src/features/ai-gateway/server/resolve/gatewayResolveRepository.ts b/web/src/features/ai-gateway/server/resolve/gatewayResolveRepository.tsindex ff6443adb0..4b6c091a44 100644--- a/web/src/features/ai-gateway/server/resolve/gatewayResolveRepository.ts+++ b/web/src/features/ai-gateway/server/resolve/gatewayResolveRepository.ts@@ -8,50 +8,44 @@ export class GatewayResolveRepository {providers: GatewayProvider[];}) {const now = new Date();- return this.prisma.gatewayApiKeyAssociation.findFirst({+ return this.prisma.apiKey.findFirst({relationLoadStrategy: "join",where: {- apiKey: {- fastHashedSecretKey: params.fastHashedSecretKey,- scope: "ORGANIZATION",- orgId: { not: null },- OR: [{ expiresAt: null }, { expiresAt: { gt: now } }],- },+ fastHashedSecretKey: params.fastHashedSecretKey,+ scope: "ORGANIZATION",+ orgId: { not: null },+ OR: [{ expiresAt: null }, { expiresAt: { gt: now } }],},select: {- apiKeyId: true,- metadata: true,- apiKey: {+ id: true,+ orgId: true,+ expiresAt: true,+ gatewayAssociation: { select: { metadata: true } },+ organization: {select: {- orgId: true,- expiresAt: true,- organization: {+ gatewayConfig: {select: {- gatewayConfig: {- select: {- defaultIngestionProjectId: true,- ingestionMode: true,- defaultIngestionProject: {- select: { id: true, orgId: true, deletedAt: true },- },- },- },- gatewayAiConnections: {- where: {- provider: { in: params.providers },- status: "ENABLED",- },- orderBy: [{ createdAt: "asc" }, { id: "asc" }],- select: {- id: true,- name: true,- provider: true,- encryptedCredential: true,- allowedModels: true,- },+ defaultIngestionProjectId: true,+ ingestionMode: true,+ defaultIngestionProject: {+ select: { id: true, orgId: true, deletedAt: true },},},},+ gatewayAiConnections: {+ where: {+ provider: { in: params.providers },+ status: "ENABLED",+ },+ orderBy: [{ createdAt: "asc" }, { id: "asc" }],+ select: {+ id: true,+ name: true,+ provider: true,+ encryptedCredential: true,+ allowedModels: true,+ },+ },},},},diff --git a/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts b/web/src/features/ai-gateway/server/resolve/resolveService.servertest.tsindex 95e49dc9c1..47486b0f89 100644--- a/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts+++ b/web/src/features/ai-gateway/server/resolve/resolveService.servertest.ts@@ -30,9 +30,15 @@ vi.mock("@langfuse/shared/encryption", () => ({decrypt: vi.fn(() => "sk-test"),}));-const serviceWith = (findFirst: () => Promise<unknown>) => {+const serviceWith = (findFirst: () => Promise<unknown>, keyOrgId = "org-1") => {const prisma = {- gatewayApiKeyAssociation: { findFirst },+ apiKey: { findFirst },+ roleAssignment: {+ findMany: () =>+ Promise.resolve([+ { systemRole: "AI_GATEWAY", orgId: keyOrgId, projectId: null },+ ]),+ },} as unknown as PrismaClient;const contextResolver = new GatewayApiKeyAuthenticator(prisma);const service = new GatewayResolveService();@@ -56,30 +62,30 @@ const row = (overrides: {projectOrgId?: string;metadata?: unknown;}) => ({- apiKeyId: "key-1",- metadata: overrides.metadata,- apiKey: {- orgId: "org-1",- organization: {- gatewayConfig: {- defaultIngestionProjectId: "project-1",- ingestionMode: overrides.ingestionMode ?? "USAGE",- defaultIngestionProject: {- id: "project-1",- orgId: overrides.projectOrgId ?? "org-1",- deletedAt: null,- },+ id: "key-1",+ orgId: "org-1",+ expiresAt: null,+ gatewayAssociation:+ overrides.metadata === undefined ? null : { metadata: overrides.metadata },+ organization: {+ gatewayConfig: {+ defaultIngestionProjectId: "project-1",+ ingestionMode: overrides.ingestionMode ?? "USAGE",+ defaultIngestionProject: {+ id: "project-1",+ orgId: overrides.projectOrgId ?? "org-1",+ deletedAt: null,},- gatewayAiConnections: [- {- id: "connection-1",- name: "Production OpenAI",- provider: "OPENAI",- encryptedCredential: "encrypted-credential",- allowedModels: null,- },- ],},+ gatewayAiConnections: [+ {+ id: "connection-1",+ name: "Production OpenAI",+ provider: "OPENAI",+ encryptedCredential: "encrypted-credential",+ allowedModels: null,+ },+ ],},});@@ -146,6 +152,19 @@ describe("GatewayResolveService", () => {});});+ it("resolves a key without a gateway association with empty metadata", async () => {+ const service = serviceWith(() => Promise.resolve(row({})));++ await expect(+ service.resolve({+ fastHashedSecretKey: "hashed-secret-key",+ apiFormat: "openai.responses",+ }),+ ).resolves.toMatchObject({+ attribution: { key_id: "key-1", key_metadata: {} },+ });+ });+it("refuses to resolve when the ingestion project belongs to another organization", async () => {const service = serviceWith(() =>Promise.resolve(row({ projectOrgId: "org-other" })),@@ -171,14 +190,16 @@ describe("GatewayResolveService", () => {});it("rejects gateway keys from organizations outside the allowlist", async () => {- const service = serviceWith(() =>- Promise.resolve({- apiKeyId: "key-1",- apiKey: {+ const service = serviceWith(+ () =>+ Promise.resolve({+ id: "key-1",orgId: "org-not-allowed",+ expiresAt: null,+ gatewayAssociation: null,organization: { gatewayConfig: null, gatewayAiConnections: [] },- },- }),+ }),+ "org-not-allowed",);await expect(@@ -186,6 +207,9 @@ describe("GatewayResolveService", () => {fastHashedSecretKey: "hashed-secret-key",apiFormat: "openai.responses",}),- ).rejects.toMatchObject({ status: 403 });+ ).rejects.toMatchObject({+ status: 403,+ message: "Gateway is not enabled for this organization",+ });});});diff --git a/web/src/features/public-api/components/ApiKeyCreateDialogContent.tsx b/web/src/features/public-api/components/ApiKeyCreateDialogContent.tsxindex 46cb3f2648..9435f341b5 100644--- a/web/src/features/public-api/components/ApiKeyCreateDialogContent.tsx+++ b/web/src/features/public-api/components/ApiKeyCreateDialogContent.tsx@@ -23,6 +23,7 @@ import {SelectContent,SelectTrigger,} from "@/src/components/ui/select";+import { withoutGatewayRole } from "@/src/features/ai-gateway/fns/separateGatewayApiKeys";import {ApiKeyExpirationInput,type ApiKeyExpirationPreset,@@ -93,7 +94,7 @@ function ApiKeyCreateForm({onSubmit,isPending,}: Extract<ApiKeyCreateDialogContentProps, { type: "form" }>) {- const roles = apiKeyRolesForScope(scope);+ const roles = withoutGatewayRole(apiKeyRolesForScope(scope));const { data: session } = useSession();const roleSelectionEnabled =(scope === "project"diff --git a/web/src/features/public-api/server/organizationApiKeyRouter.ts b/web/src/features/public-api/server/organizationApiKeyRouter.tsindex b8155d03ba..f42c4c51e1 100644--- a/web/src/features/public-api/server/organizationApiKeyRouter.ts+++ b/web/src/features/public-api/server/organizationApiKeyRouter.ts@@ -12,6 +12,7 @@ import { LangfuseNotFoundError } from "@langfuse/shared";import { createApiKey } from "@langfuse/shared/src/server/auth/apiKeys";import { OrganizationId, SystemRoleId, UserId } from "@langfuse/shared/rbac";import { apiKeyCreationRoleSchema } from "@/src/features/public-api/server/apiKeyCreationRoleSchema";+import { excludeGatewayApiKeys } from "@/src/features/ai-gateway/server/apiKey/excludeGatewayApiKeys";export const organizationApiKeysRouter = createTRPCRouter({byOrganizationId: protectedOrganizationProcedure@@ -32,6 +33,7 @@ export const organizationApiKeysRouter = createTRPCRouter({orgId: input.orgId,scope: "ORGANIZATION",isInAppAgentKey: false,+ ...excludeGatewayApiKeys,},select: {id: true,