🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(gateway): let members discover models on a cold cache (#19609)
Tobias Wochingerec87d2e01402
diff --git a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts b/web/src/__tests__/server/ai-gateway-control-plane.servertest.tsindex 2ce83f3a8f..e66d49ed7c 100644--- a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts+++ b/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts@@ -73,7 +73,7 @@ import {} from "@/src/features/ai-gateway/server/provider";import { createEs256JwtVerifier } from "@/src/server/utils/jwt";import { prisma, Role } from "@langfuse/shared/src/db";-import { decrypt } from "@langfuse/shared/encryption";+import { decrypt, encrypt } from "@langfuse/shared/encryption";import {createApiKey,createShaHash,@@ -849,18 +849,20 @@ describe("AI gateway control plane", () => {await setup.caller.aiGateway.updateConnectionModels({orgId: setup.org.id,id: connection.id,- allowedModels: ["enabled-model"],+ allowedModels: ["enabled-model", "unlisted-model"],expectedUpdatedAt: connection.updatedAt,});const readCaller = await createCallerForOrganizationRole(setup, readRole);- const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue(- Response.json({- data: [- { id: "disabled-model", created: 10 },- { id: "enabled-model", created: 20 },- ],- }),- );+ const fetchSpy = vi+ .spyOn(globalThis, "fetch")+ .mockImplementation(async () =>+ Response.json({+ data: [+ { id: "disabled-model", created: 10 },+ { id: "enabled-model", created: 20 },+ ],+ }),+ );try {await expect(@@ -886,13 +888,98 @@ describe("AI gateway control plane", () => {models: ["enabled-model"],},]);- expect(fetchSpy).not.toHaveBeenCalled();+ expect(fetchSpy).toHaveBeenCalledTimes(1);} finally {fetchSpy.mockRestore();}},);+ it("calls a rejecting provider once per window for a MEMBER and shows the allowlist without marking ERROR", async () => {+ const setup = await prepare();+ const connection = await setup.caller.aiGateway.createConnection({+ orgId: setup.org.id,+ name: "OpenAI",+ provider: "OPENAI",+ credential: "sk-test-openai",+ });+ await setup.caller.aiGateway.updateConnectionModels({+ orgId: setup.org.id,+ id: connection.id,+ allowedModels: ["gpt-5"],+ expectedUpdatedAt: connection.updatedAt,+ });+ const memberCaller = await createCallerForOrganizationRole(+ setup,+ Role.MEMBER,+ );+ const fetchSpy = vi+ .spyOn(globalThis, "fetch")+ .mockResolvedValue(new Response(null, { status: 401 }));++ try {+ await new GatewayModelCatalogService(prisma).clearModelCache(+ setup.org.id,+ connection.id,+ );+ for (let load = 0; load < 2; load++) {+ await expect(+ memberCaller.aiGateway.refreshModels({ orgId: setup.org.id }),+ ).resolves.toEqual([+ { connectionId: connection.id, success: true, models: ["gpt-5"] },+ ]);+ }+ expect(fetchSpy).toHaveBeenCalledTimes(1);+ await expect(+ prisma.gatewayAiConnection.findUniqueOrThrow({+ where: { id: connection.id },+ select: { status: true },+ }),+ ).resolves.toEqual({ status: "ENABLED" });+ } finally {+ fetchSpy.mockRestore();+ }+ });++ it("keeps a connection ENABLED when its credential changes while a rejected discovery runs", async () => {+ const setup = await prepare();+ const connection = await setup.caller.aiGateway.createConnection({+ orgId: setup.org.id,+ name: "OpenAI",+ provider: "OPENAI",+ credential: "sk-revoked-openai",+ });+ const fetchSpy = vi+ .spyOn(globalThis, "fetch")+ .mockImplementation(async () => {+ await prisma.gatewayAiConnection.update({+ where: { id: connection.id },+ data: { encryptedCredential: encrypt("sk-replaced-openai") },+ });+ return new Response(null, { status: 401 });+ });++ try {+ await new GatewayModelCatalogService(prisma).clearModelCache(+ setup.org.id,+ connection.id,+ );+ await expect(+ setup.caller.aiGateway.refreshModels({ orgId: setup.org.id }),+ ).resolves.toEqual([+ { connectionId: connection.id, success: false, error: "unauthorized" },+ ]);+ await expect(+ prisma.gatewayAiConnection.findUniqueOrThrow({+ where: { id: connection.id },+ select: { status: true },+ }),+ ).resolves.toEqual({ status: "ENABLED" });+ } finally {+ fetchSpy.mockRestore();+ }+ });+it("blocks an auto-granted demo organization VIEWER from gateway catalog metadata", async () => {const setup = await prepare();const connection = await setup.caller.aiGateway.createConnection({diff --git a/web/src/features/ai-gateway/server/provider/connection/gatewayProviderRepository.ts b/web/src/features/ai-gateway/server/provider/connection/gatewayProviderRepository.tsindex ffb9470013..5eb60541b7 100644--- a/web/src/features/ai-gateway/server/provider/connection/gatewayProviderRepository.ts+++ b/web/src/features/ai-gateway/server/provider/connection/gatewayProviderRepository.ts@@ -168,16 +168,33 @@ export class GatewayProviderRepository {return toSafeConnection(connection);}+ /** Returns false when the connection changed since `expectedUpdatedAt`, leaving the newer state in place. */async updateConnectionStatus(params: {organizationId: string;id: string;status: GatewayConnectionStatus;+ expectedUpdatedAt: Date;}) {- const connection = await this.prisma.gatewayAiConnection.update({- where: { id: params.id, organizationId: params.organizationId },+ const result = await this.prisma.gatewayAiConnection.updateMany({+ where: {+ id: params.id,+ organizationId: params.organizationId,+ updatedAt: params.expectedUpdatedAt,+ },data: { status: params.status },- select: safeConnectionSelect,});- return toSafeConnection(connection);+ return result.count > 0;+ }++ async isConnectionUnchanged(params: {+ organizationId: string;+ id: string;+ updatedAt: Date;+ }) {+ const connection = await this.prisma.gatewayAiConnection.findFirst({+ where: params,+ select: { id: true },+ });+ return connection !== null;}}diff --git a/web/src/features/ai-gateway/server/provider/models/gatewayModelCatalogService.ts b/web/src/features/ai-gateway/server/provider/models/gatewayModelCatalogService.tsindex da2cd20446..329bc5d93a 100644--- a/web/src/features/ai-gateway/server/provider/models/gatewayModelCatalogService.ts+++ b/web/src/features/ai-gateway/server/provider/models/gatewayModelCatalogService.ts@@ -4,7 +4,7 @@ import type {} from "@langfuse/shared/src/db";import { LangfuseNotFoundError } from "@langfuse/shared";import { decrypt } from "@langfuse/shared/encryption";-import { redis as defaultRedis } from "@langfuse/shared/src/server";+import { logger, redis as defaultRedis } from "@langfuse/shared/src/server";import type { Cluster, Redis } from "ioredis";import { auditLog } from "@/src/features/audit-logs/server";@@ -42,7 +42,13 @@ export type ModelCatalogRefreshResult =| ModelRefreshError;const MODEL_CACHE_PREFIX = "ai-gateway:models";+const MODEL_CACHE_REFRESH_LOCK_PREFIX = "ai-gateway:models-refresh";const MODEL_CACHE_TTL_SECONDS = 5 * 60;+/**+ * A cached catalog this close to expiry is still served but refreshed in the background.+ * The refresh lock lives exactly as long, so each connection gets at most one refresh per window.+ */+const MODEL_CACHE_REFRESH_WINDOW_SECONDS = 30;/*** Codex sends its auto-review requests as `codex-auto-review`, which OpenAI serves but does not list in `/v1/models`.* Listing it here keeps it allowed by default on every OpenAI connection while admins can still disable it.@@ -77,31 +83,28 @@ export class GatewayModelCatalogService {connectionId: connection.id,forceRefresh,explicitRetry: false,+ readOnly: false,}),),);}- async listCachedModels(- organizationId: string,- ): Promise<ModelRefreshResult[]> {+ /** Discovers models without changing connection status and with bounded provider calls, so read-only roles may call it. */+ async listAllModels(organizationId: string): Promise<ModelRefreshResult[]> {const connections = await this.listAllConnections(organizationId,"ENABLED",);return Promise.all(- connections.map(async (connection) => {- const cachedModels = await this.getCachedModels(+ connections.map((connection) =>+ this.refreshConnectionModels({organizationId,- connection.id,- );- return {connectionId: connection.id,- success: true as const,- models:- cachedModels?.map(({ id }) => id) ?? connection.allowedModels ?? [],- };- }),+ forceRefresh: false,+ explicitRetry: false,+ readOnly: true,+ }),+ ),);}@@ -127,6 +130,7 @@ export class GatewayModelCatalogService {...params,forceRefresh: params.forceRefresh ?? false,explicitRetry: false,+ readOnly: false,});}@@ -140,6 +144,7 @@ export class GatewayModelCatalogService {connectionId: params.connectionId,forceRefresh: true,explicitRetry: true,+ readOnly: false,});if (result.success) {await auditLog(@@ -163,6 +168,7 @@ export class GatewayModelCatalogService {...params,forceRefresh: false,explicitRetry: false,+ readOnly: false,});}@@ -194,12 +200,9 @@ export class GatewayModelCatalogService {return connections;}- private async refreshConnectionModels(params: {- organizationId: string;- connectionId: string;- forceRefresh: boolean;- explicitRetry: boolean;- }): Promise<ModelRefreshResult> {+ private async refreshConnectionModels(+ params: CatalogRefreshParams,+ ): Promise<ModelRefreshResult> {const result = await this.refreshConnectionModelCatalog(params);return result.success? {@@ -210,34 +213,85 @@ export class GatewayModelCatalogService {: result;}- private async refreshConnectionModelCatalog(params: {- organizationId: string;- connectionId: string;- forceRefresh: boolean;- explicitRetry: boolean;- }): Promise<ModelCatalogRefreshResult> {- const connection = await this.repository.getConnectionWithCredential({- organizationId: params.organizationId,- id: params.connectionId,- });- if (!connection) throw new LangfuseNotFoundError("Provider credential");-+ private async refreshConnectionModelCatalog(+ params: CatalogRefreshParams,+ ): Promise<ModelCatalogRefreshResult> {if (params.forceRefresh) {await this.clearModelCache(params.organizationId, params.connectionId);} else {- const cachedModels = await this.getCachedModels(+ const cached = await this.getCachedModels(params.organizationId,params.connectionId,);- if (cachedModels) {+ if (cached) {+ if (cached.expiresInSeconds <= MODEL_CACHE_REFRESH_WINDOW_SECONDS) {+ this.refreshInBackground(params);+ }return {- connectionId: connection.id,+ connectionId: params.connectionId,success: true,- models: cachedModels,+ models: cached.models,};}}+ const connection = await this.repository.getConnectionWithCredential({+ organizationId: params.organizationId,+ id: params.connectionId,+ });+ if (!connection) throw new LangfuseNotFoundError("Provider credential");+ return params.readOnly+ ? this.discoverModelsReadOnly(connection, params)+ : this.discoverModels(connection, params);+ }++ /**+ * Read-only roles reach the provider at most once per refresh window, even when discovery fails,+ * because a failure keeps the connection ENABLED and is not cached. Otherwise they see the allowlist.+ */+ private async discoverModelsReadOnly(+ connection: ConnectionWithCredential,+ params: CatalogRefreshParams,+ ): Promise<ModelCatalogRefreshResult> {+ const allowedModels = {+ connectionId: connection.id,+ success: true as const,+ models: (connection.allowedModels ?? []).map((id) =>+ toCatalogEntry(connection.provider as GatewayProviderName, id),+ ),+ };+ if (!(await this.claimRefresh(params.organizationId, connection.id))) {+ return allowedModels;+ }+ const result = await this.discoverModels(connection, params);+ if (result.success) return result;+ logDiscoveryFailure(params, result.error);+ return allowedModels;+ }++ private async refreshInBackground(params: CatalogRefreshParams) {+ try {+ if (+ !(await this.claimRefresh(params.organizationId, params.connectionId))+ ) {+ return;+ }+ const connection = await this.repository.getConnectionWithCredential({+ organizationId: params.organizationId,+ id: params.connectionId,+ });+ if (!connection) return;+ const result = await this.discoverModels(connection, params);+ if (!result.success) logDiscoveryFailure(params, result.error);+ } catch (error) {+ logDiscoveryFailure(params, error);+ }+ }++ private async discoverModels(+ connection: ConnectionWithCredential,+ params: CatalogRefreshParams,+ ): Promise<ModelCatalogRefreshResult> {const provider = connection.provider as GatewayProviderName;const credential = decrypt(connection.encryptedCredential);const models = await getModelDiscoveryAdapter(provider).fetchCatalog({@@ -245,17 +299,24 @@ export class GatewayModelCatalogService {fetcher: this.fetcher,});if (!models.success) {- if (models.error === "unauthorized" && connection.status === "ENABLED") {- await this.repository.updateConnectionStatus({+ if (+ !params.readOnly &&+ models.error === "unauthorized" &&+ connection.status === "ENABLED"+ ) {+ const marked = await this.repository.updateConnectionStatus({organizationId: params.organizationId,id: connection.id,status: "ERROR",+ expectedUpdatedAt: connection.updatedAt,});// A connection that just failed authorization must stop being handed// to the data plane.- await invalidateGatewayResolveCacheForOrganization(- params.organizationId,- );+ if (marked) {+ await invalidateGatewayResolveCacheForOrganization(+ params.organizationId,+ );+ }}return {connectionId: connection.id,@@ -264,14 +325,31 @@ export class GatewayModelCatalogService {};}const catalog = addCustomModels(provider, models.models);- await this.cacheModels(params.organizationId, params.connectionId, catalog);- if (params.explicitRetry && connection.status === "ERROR") {- await this.repository.updateConnectionStatus({+ // Discovery can outlast a credential update, whose catalog must not be overwritten by the old one.+ const unchanged = await this.repository.isConnectionUnchanged({+ organizationId: params.organizationId,+ id: connection.id,+ updatedAt: connection.updatedAt,+ });+ if (unchanged) {+ await this.cacheModels(+ params.organizationId,+ params.connectionId,+ catalog,+ );+ }+ if (unchanged && params.explicitRetry && connection.status === "ERROR") {+ const enabled = await this.repository.updateConnectionStatus({organizationId: params.organizationId,id: connection.id,status: "ENABLED",+ expectedUpdatedAt: connection.updatedAt,});- await invalidateGatewayResolveCacheForOrganization(params.organizationId);+ if (enabled) {+ await invalidateGatewayResolveCacheForOrganization(+ params.organizationId,+ );+ }}return {connectionId: connection.id,@@ -284,23 +362,42 @@ export class GatewayModelCatalogService {return `${MODEL_CACHE_PREFIX}:${organizationId}:${connectionId}`;}- private async getCachedModels(- organizationId: string,- connectionId: string,- ): Promise<GatewayModelCatalogEntry[] | null> {+ private async getCachedModels(organizationId: string, connectionId: string) {if (!this.redis) return null;try {- const cached = await this.redis.get(- this.modelCacheKey(organizationId, connectionId),- );+ const key = this.modelCacheKey(organizationId, connectionId);+ const [cached, expiresInSeconds] = await Promise.all([+ this.redis.get(key),+ this.redis.ttl(key),+ ]);if (!cached) return null;const parsed: unknown = JSON.parse(cached);- return isGatewayModelCatalog(parsed) ? parsed : null;+ return isGatewayModelCatalog(parsed)+ ? { models: parsed, expiresInSeconds }+ : null;} catch {return null;}}+ /** Cross-instance guard so concurrent requests trigger a single provider call per refresh window. */+ private async claimRefresh(organizationId: string, connectionId: string) {+ if (!this.redis) return true;+ try {+ const claimed = await this.redis.set(+ `${MODEL_CACHE_REFRESH_LOCK_PREFIX}:${organizationId}:${connectionId}`,+ "1",+ "EX",+ MODEL_CACHE_REFRESH_WINDOW_SECONDS,+ "NX",+ );+ return claimed === "OK";+ } catch {+ // Redis is optional. Without it discovery proceeds without deduplication.+ return true;+ }+ }+private async cacheModels(organizationId: string,connectionId: string,@@ -326,16 +423,31 @@ function addCustomModels(const discoveredIds = new Set(models.map(({ id }) => id));const customModels = (CUSTOM_MODELS[provider] ?? []).filter((id) => !discoveredIds.has(id))- .map((id) => ({- id,- provider,- canonicalSlug: id,- displayName: id,- createdAt: new Date(0).toISOString(),- }));+ .map((id) => toCatalogEntry(provider, id));return models.concat(customModels);}+function logDiscoveryFailure(params: CatalogRefreshParams, error: unknown) {+ logger.warn("Failed to refresh AI Gateway model catalog", {+ organizationId: params.organizationId,+ connectionId: params.connectionId,+ error,+ });+}++function toCatalogEntry(+ provider: GatewayProviderName,+ id: string,+): GatewayModelCatalogEntry {+ return {+ id,+ provider,+ canonicalSlug: id,+ displayName: id,+ createdAt: new Date(0).toISOString(),+ };+}+function isGatewayModelCatalog(value: unknown,): value is GatewayModelCatalogEntry[] {@@ -358,3 +470,15 @@ function isGatewayModelCatalog());}++type CatalogRefreshParams = {+ organizationId: string;+ connectionId: string;+ forceRefresh: boolean;+ explicitRetry: boolean;+ readOnly: boolean;+};++type ConnectionWithCredential = NonNullable<+ Awaited<ReturnType<GatewayProviderRepository["getConnectionWithCredential"]>>+>;diff --git a/web/src/features/ai-gateway/server/provider/registry.servertest.ts b/web/src/features/ai-gateway/server/provider/registry.servertest.tsindex ade335ee7e..7c11582a4e 100644--- a/web/src/features/ai-gateway/server/provider/registry.servertest.ts+++ b/web/src/features/ai-gateway/server/provider/registry.servertest.ts@@ -72,6 +72,7 @@ describe("AI gateway provider registry", () => {},]),),+ ttl: vi.fn().mockResolvedValue(300),setex: vi.fn(),} as unknown as Redis;const prisma = {@@ -101,6 +102,123 @@ describe("AI gateway provider registry", () => {"ai-gateway:models:org-1:connection-1",);expect(fetcher).not.toHaveBeenCalled();+ expect(prisma.gatewayAiConnection.findFirst).not.toHaveBeenCalled();+ });++ it("does not cache a background refresh that finished after its connection changed", async () => {+ const fetcher = vi+ .fn<typeof fetch>()+ .mockResolvedValue(+ Response.json({ data: [{ id: "old-credential-model", created: 10 }] }),+ );+ const redis = {+ get: vi.fn().mockResolvedValue(+ JSON.stringify([+ {+ id: "gpt-5-mini",+ provider: "OPENAI",+ canonicalSlug: "gpt-5-mini",+ displayName: "gpt-5-mini",+ createdAt: "2026-01-01T00:00:00.000Z",+ },+ ]),+ ),+ ttl: vi.fn().mockResolvedValue(10),+ set: vi.fn().mockResolvedValue("OK"),+ setex: vi.fn().mockResolvedValue("OK"),+ } as unknown as Redis;+ const findFirst = vi+ .fn()+ .mockResolvedValueOnce({+ id: "connection-1",+ organizationId: "org-1",+ provider: "OPENAI",+ encryptedCredential: encrypt("sk-old"),+ status: "ENABLED",+ allowedModels: null,+ updatedAt: new Date(0),+ })+ .mockResolvedValueOnce(null);+ const prisma = {+ gatewayAiConnection: { findFirst },+ } as unknown as PrismaClient;+ const service = new GatewayModelCatalogService(prisma, fetcher, redis);++ await service.refreshModels({+ organizationId: "org-1",+ connectionId: "connection-1",+ });++ await vi.waitFor(() => expect(findFirst).toHaveBeenCalledTimes(2));+ expect(findFirst).toHaveBeenLastCalledWith({+ where: {+ organizationId: "org-1",+ id: "connection-1",+ updatedAt: new Date(0),+ },+ select: { id: true },+ });+ await new Promise((resolve) => setTimeout(resolve, 0));+ expect(redis.setex).not.toHaveBeenCalled();+ });++ it("serves a nearly expired model list while refreshing it once in the background", async () => {+ const fetcher = vi+ .fn<typeof fetch>()+ .mockResolvedValue(+ Response.json({ data: [{ id: "gpt-5", created: 10 }] }),+ );+ const redis = {+ get: vi.fn().mockResolvedValue(+ JSON.stringify([+ {+ id: "gpt-5-mini",+ provider: "OPENAI",+ canonicalSlug: "gpt-5-mini",+ displayName: "gpt-5-mini",+ createdAt: "2026-01-01T00:00:00.000Z",+ },+ ]),+ ),+ ttl: vi.fn().mockResolvedValue(10),+ set: vi.fn().mockResolvedValueOnce("OK").mockResolvedValue(null),+ setex: vi.fn().mockResolvedValue("OK"),+ } as unknown as Redis;+ const prisma = {+ gatewayAiConnection: {+ findFirst: vi.fn().mockResolvedValue({+ id: "connection-1",+ organizationId: "org-1",+ provider: "OPENAI",+ encryptedCredential: encrypt("sk-test"),+ status: "ENABLED",+ allowedModels: null,+ }),+ },+ } as unknown as PrismaClient;+ const service = new GatewayModelCatalogService(prisma, fetcher, redis);+ const request = {+ organizationId: "org-1",+ connectionId: "connection-1",+ };++ await expect(+ Promise.all([+ service.refreshModels(request),+ service.refreshModels(request),+ ]),+ ).resolves.toEqual([+ { connectionId: "connection-1", success: true, models: ["gpt-5-mini"] },+ { connectionId: "connection-1", success: true, models: ["gpt-5-mini"] },+ ]);+ await vi.waitFor(() =>+ expect(redis.setex).toHaveBeenCalledWith(+ "ai-gateway:models:org-1:connection-1",+ 300,+ expect.stringContaining('"id":"gpt-5"'),+ ),+ );+ expect(fetcher).toHaveBeenCalledTimes(1);});it("collects every Anthropic model page as canonical metadata", async () => {diff --git a/web/src/features/ai-gateway/server/router.ts b/web/src/features/ai-gateway/server/router.tsindex 7c5a1ff970..c4ae663be3 100644--- a/web/src/features/ai-gateway/server/router.ts+++ b/web/src/features/ai-gateway/server/router.ts@@ -397,7 +397,7 @@ export const aiGatewayRouter = createTRPCRouter({const catalogService = new GatewayModelCatalogService(ctx.prisma);const results = canManageGateway? await catalogService.refreshAllModels(input.orgId)- : await catalogService.listCachedModels(input.orgId);+ : await catalogService.listAllModels(input.orgId);if (canManageGateway) return results;const readableConnections = await catalogService.listModelConnections(