Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(gateway): let members discover models on a cold cache (#19609)

Tobias Wochingerec87d2e01402
diff --git a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts b/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts
index 2ce83f3a8f..e66d49ed7c 100644
--- a/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts
+++ b/web/src/__tests__/server/ai-gateway-control-plane.servertest.ts
@@ -73,7 +73,7 @@ import {
} from "@/src/features/ai-gateway/server/provider";
import { createEs256JwtVerifier } from "@/src/server/utils/jwt";
import { prisma, Role } from "@langfuse/shared/src/db";
-import { decrypt } from "@langfuse/shared/encryption";
+import { decrypt, encrypt } from "@langfuse/shared/encryption";
import {
createApiKey,
createShaHash,
@@ -849,18 +849,20 @@ describe("AI gateway control plane", () => {
await setup.caller.aiGateway.updateConnectionModels({
orgId: setup.org.id,
id: connection.id,
- allowedModels: ["enabled-model"],
+ allowedModels: ["enabled-model", "unlisted-model"],
expectedUpdatedAt: connection.updatedAt,
});
const readCaller = await createCallerForOrganizationRole(setup, readRole);
- const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue(
- Response.json({
- data: [
- { id: "disabled-model", created: 10 },
- { id: "enabled-model", created: 20 },
- ],
- }),
- );
+ const fetchSpy = vi
+ .spyOn(globalThis, "fetch")
+ .mockImplementation(async () =>
+ Response.json({
+ data: [
+ { id: "disabled-model", created: 10 },
+ { id: "enabled-model", created: 20 },
+ ],
+ }),
+ );
try {
await expect(
@@ -886,13 +888,98 @@ describe("AI gateway control plane", () => {
models: ["enabled-model"],
},
]);
- expect(fetchSpy).not.toHaveBeenCalled();
+ expect(fetchSpy).toHaveBeenCalledTimes(1);
} finally {
fetchSpy.mockRestore();
}
},
);
+ it("calls a rejecting provider once per window for a MEMBER and shows the allowlist without marking ERROR", async () => {
+ const setup = await prepare();
+ const connection = await setup.caller.aiGateway.createConnection({
+ orgId: setup.org.id,
+ name: "OpenAI",
+ provider: "OPENAI",
+ credential: "sk-test-openai",
+ });
+ await setup.caller.aiGateway.updateConnectionModels({
+ orgId: setup.org.id,
+ id: connection.id,
+ allowedModels: ["gpt-5"],
+ expectedUpdatedAt: connection.updatedAt,
+ });
+ const memberCaller = await createCallerForOrganizationRole(
+ setup,
+ Role.MEMBER,
+ );
+ const fetchSpy = vi
+ .spyOn(globalThis, "fetch")
+ .mockResolvedValue(new Response(null, { status: 401 }));
+
+ try {
+ await new GatewayModelCatalogService(prisma).clearModelCache(
+ setup.org.id,
+ connection.id,
+ );
+ for (let load = 0; load < 2; load++) {
+ await expect(
+ memberCaller.aiGateway.refreshModels({ orgId: setup.org.id }),
+ ).resolves.toEqual([
+ { connectionId: connection.id, success: true, models: ["gpt-5"] },
+ ]);
+ }
+ expect(fetchSpy).toHaveBeenCalledTimes(1);
+ await expect(
+ prisma.gatewayAiConnection.findUniqueOrThrow({
+ where: { id: connection.id },
+ select: { status: true },
+ }),
+ ).resolves.toEqual({ status: "ENABLED" });
+ } finally {
+ fetchSpy.mockRestore();
+ }
+ });
+
+ it("keeps a connection ENABLED when its credential changes while a rejected discovery runs", async () => {
+ const setup = await prepare();
+ const connection = await setup.caller.aiGateway.createConnection({
+ orgId: setup.org.id,
+ name: "OpenAI",
+ provider: "OPENAI",
+ credential: "sk-revoked-openai",
+ });
+ const fetchSpy = vi
+ .spyOn(globalThis, "fetch")
+ .mockImplementation(async () => {
+ await prisma.gatewayAiConnection.update({
+ where: { id: connection.id },
+ data: { encryptedCredential: encrypt("sk-replaced-openai") },
+ });
+ return new Response(null, { status: 401 });
+ });
+
+ try {
+ await new GatewayModelCatalogService(prisma).clearModelCache(
+ setup.org.id,
+ connection.id,
+ );
+ await expect(
+ setup.caller.aiGateway.refreshModels({ orgId: setup.org.id }),
+ ).resolves.toEqual([
+ { connectionId: connection.id, success: false, error: "unauthorized" },
+ ]);
+ await expect(
+ prisma.gatewayAiConnection.findUniqueOrThrow({
+ where: { id: connection.id },
+ select: { status: true },
+ }),
+ ).resolves.toEqual({ status: "ENABLED" });
+ } finally {
+ fetchSpy.mockRestore();
+ }
+ });
+
it("blocks an auto-granted demo organization VIEWER from gateway catalog metadata", async () => {
const setup = await prepare();
const connection = await setup.caller.aiGateway.createConnection({
diff --git a/web/src/features/ai-gateway/server/provider/connection/gatewayProviderRepository.ts b/web/src/features/ai-gateway/server/provider/connection/gatewayProviderRepository.ts
index ffb9470013..5eb60541b7 100644
--- a/web/src/features/ai-gateway/server/provider/connection/gatewayProviderRepository.ts
+++ b/web/src/features/ai-gateway/server/provider/connection/gatewayProviderRepository.ts
@@ -168,16 +168,33 @@ export class GatewayProviderRepository {
return toSafeConnection(connection);
}
+ /** Returns false when the connection changed since `expectedUpdatedAt`, leaving the newer state in place. */
async updateConnectionStatus(params: {
organizationId: string;
id: string;
status: GatewayConnectionStatus;
+ expectedUpdatedAt: Date;
}) {
- const connection = await this.prisma.gatewayAiConnection.update({
- where: { id: params.id, organizationId: params.organizationId },
+ const result = await this.prisma.gatewayAiConnection.updateMany({
+ where: {
+ id: params.id,
+ organizationId: params.organizationId,
+ updatedAt: params.expectedUpdatedAt,
+ },
data: { status: params.status },
- select: safeConnectionSelect,
});
- return toSafeConnection(connection);
+ return result.count > 0;
+ }
+
+ async isConnectionUnchanged(params: {
+ organizationId: string;
+ id: string;
+ updatedAt: Date;
+ }) {
+ const connection = await this.prisma.gatewayAiConnection.findFirst({
+ where: params,
+ select: { id: true },
+ });
+ return connection !== null;
}
}
diff --git a/web/src/features/ai-gateway/server/provider/models/gatewayModelCatalogService.ts b/web/src/features/ai-gateway/server/provider/models/gatewayModelCatalogService.ts
index da2cd20446..329bc5d93a 100644
--- a/web/src/features/ai-gateway/server/provider/models/gatewayModelCatalogService.ts
+++ b/web/src/features/ai-gateway/server/provider/models/gatewayModelCatalogService.ts
@@ -4,7 +4,7 @@ import type {
} from "@langfuse/shared/src/db";
import { LangfuseNotFoundError } from "@langfuse/shared";
import { decrypt } from "@langfuse/shared/encryption";
-import { redis as defaultRedis } from "@langfuse/shared/src/server";
+import { logger, redis as defaultRedis } from "@langfuse/shared/src/server";
import type { Cluster, Redis } from "ioredis";
import { auditLog } from "@/src/features/audit-logs/server";
@@ -42,7 +42,13 @@ export type ModelCatalogRefreshResult =
| ModelRefreshError;
const MODEL_CACHE_PREFIX = "ai-gateway:models";
+const MODEL_CACHE_REFRESH_LOCK_PREFIX = "ai-gateway:models-refresh";
const MODEL_CACHE_TTL_SECONDS = 5 * 60;
+/**
+ * A cached catalog this close to expiry is still served but refreshed in the background.
+ * The refresh lock lives exactly as long, so each connection gets at most one refresh per window.
+ */
+const MODEL_CACHE_REFRESH_WINDOW_SECONDS = 30;
/**
* Codex sends its auto-review requests as `codex-auto-review`, which OpenAI serves but does not list in `/v1/models`.
* Listing it here keeps it allowed by default on every OpenAI connection while admins can still disable it.
@@ -77,31 +83,28 @@ export class GatewayModelCatalogService {
connectionId: connection.id,
forceRefresh,
explicitRetry: false,
+ readOnly: false,
}),
),
);
}
- async listCachedModels(
- organizationId: string,
- ): Promise<ModelRefreshResult[]> {
+ /** Discovers models without changing connection status and with bounded provider calls, so read-only roles may call it. */
+ async listAllModels(organizationId: string): Promise<ModelRefreshResult[]> {
const connections = await this.listAllConnections(
organizationId,
"ENABLED",
);
return Promise.all(
- connections.map(async (connection) => {
- const cachedModels = await this.getCachedModels(
+ connections.map((connection) =>
+ this.refreshConnectionModels({
organizationId,
- connection.id,
- );
- return {
connectionId: connection.id,
- success: true as const,
- models:
- cachedModels?.map(({ id }) => id) ?? connection.allowedModels ?? [],
- };
- }),
+ forceRefresh: false,
+ explicitRetry: false,
+ readOnly: true,
+ }),
+ ),
);
}
@@ -127,6 +130,7 @@ export class GatewayModelCatalogService {
...params,
forceRefresh: params.forceRefresh ?? false,
explicitRetry: false,
+ readOnly: false,
});
}
@@ -140,6 +144,7 @@ export class GatewayModelCatalogService {
connectionId: params.connectionId,
forceRefresh: true,
explicitRetry: true,
+ readOnly: false,
});
if (result.success) {
await auditLog(
@@ -163,6 +168,7 @@ export class GatewayModelCatalogService {
...params,
forceRefresh: false,
explicitRetry: false,
+ readOnly: false,
});
}
@@ -194,12 +200,9 @@ export class GatewayModelCatalogService {
return connections;
}
- private async refreshConnectionModels(params: {
- organizationId: string;
- connectionId: string;
- forceRefresh: boolean;
- explicitRetry: boolean;
- }): Promise<ModelRefreshResult> {
+ private async refreshConnectionModels(
+ params: CatalogRefreshParams,
+ ): Promise<ModelRefreshResult> {
const result = await this.refreshConnectionModelCatalog(params);
return result.success
? {
@@ -210,34 +213,85 @@ export class GatewayModelCatalogService {
: result;
}
- private async refreshConnectionModelCatalog(params: {
- organizationId: string;
- connectionId: string;
- forceRefresh: boolean;
- explicitRetry: boolean;
- }): Promise<ModelCatalogRefreshResult> {
- const connection = await this.repository.getConnectionWithCredential({
- organizationId: params.organizationId,
- id: params.connectionId,
- });
- if (!connection) throw new LangfuseNotFoundError("Provider credential");
-
+ private async refreshConnectionModelCatalog(
+ params: CatalogRefreshParams,
+ ): Promise<ModelCatalogRefreshResult> {
if (params.forceRefresh) {
await this.clearModelCache(params.organizationId, params.connectionId);
} else {
- const cachedModels = await this.getCachedModels(
+ const cached = await this.getCachedModels(
params.organizationId,
params.connectionId,
);
- if (cachedModels) {
+ if (cached) {
+ if (cached.expiresInSeconds <= MODEL_CACHE_REFRESH_WINDOW_SECONDS) {
+ this.refreshInBackground(params);
+ }
return {
- connectionId: connection.id,
+ connectionId: params.connectionId,
success: true,
- models: cachedModels,
+ models: cached.models,
};
}
}
+ const connection = await this.repository.getConnectionWithCredential({
+ organizationId: params.organizationId,
+ id: params.connectionId,
+ });
+ if (!connection) throw new LangfuseNotFoundError("Provider credential");
+ return params.readOnly
+ ? this.discoverModelsReadOnly(connection, params)
+ : this.discoverModels(connection, params);
+ }
+
+ /**
+ * Read-only roles reach the provider at most once per refresh window, even when discovery fails,
+ * because a failure keeps the connection ENABLED and is not cached. Otherwise they see the allowlist.
+ */
+ private async discoverModelsReadOnly(
+ connection: ConnectionWithCredential,
+ params: CatalogRefreshParams,
+ ): Promise<ModelCatalogRefreshResult> {
+ const allowedModels = {
+ connectionId: connection.id,
+ success: true as const,
+ models: (connection.allowedModels ?? []).map((id) =>
+ toCatalogEntry(connection.provider as GatewayProviderName, id),
+ ),
+ };
+ if (!(await this.claimRefresh(params.organizationId, connection.id))) {
+ return allowedModels;
+ }
+ const result = await this.discoverModels(connection, params);
+ if (result.success) return result;
+ logDiscoveryFailure(params, result.error);
+ return allowedModels;
+ }
+
+ private async refreshInBackground(params: CatalogRefreshParams) {
+ try {
+ if (
+ !(await this.claimRefresh(params.organizationId, params.connectionId))
+ ) {
+ return;
+ }
+ const connection = await this.repository.getConnectionWithCredential({
+ organizationId: params.organizationId,
+ id: params.connectionId,
+ });
+ if (!connection) return;
+ const result = await this.discoverModels(connection, params);
+ if (!result.success) logDiscoveryFailure(params, result.error);
+ } catch (error) {
+ logDiscoveryFailure(params, error);
+ }
+ }
+
+ private async discoverModels(
+ connection: ConnectionWithCredential,
+ params: CatalogRefreshParams,
+ ): Promise<ModelCatalogRefreshResult> {
const provider = connection.provider as GatewayProviderName;
const credential = decrypt(connection.encryptedCredential);
const models = await getModelDiscoveryAdapter(provider).fetchCatalog({
@@ -245,17 +299,24 @@ export class GatewayModelCatalogService {
fetcher: this.fetcher,
});
if (!models.success) {
- if (models.error === "unauthorized" && connection.status === "ENABLED") {
- await this.repository.updateConnectionStatus({
+ if (
+ !params.readOnly &&
+ models.error === "unauthorized" &&
+ connection.status === "ENABLED"
+ ) {
+ const marked = await this.repository.updateConnectionStatus({
organizationId: params.organizationId,
id: connection.id,
status: "ERROR",
+ expectedUpdatedAt: connection.updatedAt,
});
// A connection that just failed authorization must stop being handed
// to the data plane.
- await invalidateGatewayResolveCacheForOrganization(
- params.organizationId,
- );
+ if (marked) {
+ await invalidateGatewayResolveCacheForOrganization(
+ params.organizationId,
+ );
+ }
}
return {
connectionId: connection.id,
@@ -264,14 +325,31 @@ export class GatewayModelCatalogService {
};
}
const catalog = addCustomModels(provider, models.models);
- await this.cacheModels(params.organizationId, params.connectionId, catalog);
- if (params.explicitRetry && connection.status === "ERROR") {
- await this.repository.updateConnectionStatus({
+ // Discovery can outlast a credential update, whose catalog must not be overwritten by the old one.
+ const unchanged = await this.repository.isConnectionUnchanged({
+ organizationId: params.organizationId,
+ id: connection.id,
+ updatedAt: connection.updatedAt,
+ });
+ if (unchanged) {
+ await this.cacheModels(
+ params.organizationId,
+ params.connectionId,
+ catalog,
+ );
+ }
+ if (unchanged && params.explicitRetry && connection.status === "ERROR") {
+ const enabled = await this.repository.updateConnectionStatus({
organizationId: params.organizationId,
id: connection.id,
status: "ENABLED",
+ expectedUpdatedAt: connection.updatedAt,
});
- await invalidateGatewayResolveCacheForOrganization(params.organizationId);
+ if (enabled) {
+ await invalidateGatewayResolveCacheForOrganization(
+ params.organizationId,
+ );
+ }
}
return {
connectionId: connection.id,
@@ -284,23 +362,42 @@ export class GatewayModelCatalogService {
return `${MODEL_CACHE_PREFIX}:${organizationId}:${connectionId}`;
}
- private async getCachedModels(
- organizationId: string,
- connectionId: string,
- ): Promise<GatewayModelCatalogEntry[] | null> {
+ private async getCachedModels(organizationId: string, connectionId: string) {
if (!this.redis) return null;
try {
- const cached = await this.redis.get(
- this.modelCacheKey(organizationId, connectionId),
- );
+ const key = this.modelCacheKey(organizationId, connectionId);
+ const [cached, expiresInSeconds] = await Promise.all([
+ this.redis.get(key),
+ this.redis.ttl(key),
+ ]);
if (!cached) return null;
const parsed: unknown = JSON.parse(cached);
- return isGatewayModelCatalog(parsed) ? parsed : null;
+ return isGatewayModelCatalog(parsed)
+ ? { models: parsed, expiresInSeconds }
+ : null;
} catch {
return null;
}
}
+ /** Cross-instance guard so concurrent requests trigger a single provider call per refresh window. */
+ private async claimRefresh(organizationId: string, connectionId: string) {
+ if (!this.redis) return true;
+ try {
+ const claimed = await this.redis.set(
+ `${MODEL_CACHE_REFRESH_LOCK_PREFIX}:${organizationId}:${connectionId}`,
+ "1",
+ "EX",
+ MODEL_CACHE_REFRESH_WINDOW_SECONDS,
+ "NX",
+ );
+ return claimed === "OK";
+ } catch {
+ // Redis is optional. Without it discovery proceeds without deduplication.
+ return true;
+ }
+ }
+
private async cacheModels(
organizationId: string,
connectionId: string,
@@ -326,16 +423,31 @@ function addCustomModels(
const discoveredIds = new Set(models.map(({ id }) => id));
const customModels = (CUSTOM_MODELS[provider] ?? [])
.filter((id) => !discoveredIds.has(id))
- .map((id) => ({
- id,
- provider,
- canonicalSlug: id,
- displayName: id,
- createdAt: new Date(0).toISOString(),
- }));
+ .map((id) => toCatalogEntry(provider, id));
return models.concat(customModels);
}
+function logDiscoveryFailure(params: CatalogRefreshParams, error: unknown) {
+ logger.warn("Failed to refresh AI Gateway model catalog", {
+ organizationId: params.organizationId,
+ connectionId: params.connectionId,
+ error,
+ });
+}
+
+function toCatalogEntry(
+ provider: GatewayProviderName,
+ id: string,
+): GatewayModelCatalogEntry {
+ return {
+ id,
+ provider,
+ canonicalSlug: id,
+ displayName: id,
+ createdAt: new Date(0).toISOString(),
+ };
+}
+
function isGatewayModelCatalog(
value: unknown,
): value is GatewayModelCatalogEntry[] {
@@ -358,3 +470,15 @@ function isGatewayModelCatalog(
)
);
}
+
+type CatalogRefreshParams = {
+ organizationId: string;
+ connectionId: string;
+ forceRefresh: boolean;
+ explicitRetry: boolean;
+ readOnly: boolean;
+};
+
+type ConnectionWithCredential = NonNullable<
+ Awaited<ReturnType<GatewayProviderRepository["getConnectionWithCredential"]>>
+>;
diff --git a/web/src/features/ai-gateway/server/provider/registry.servertest.ts b/web/src/features/ai-gateway/server/provider/registry.servertest.ts
index ade335ee7e..7c11582a4e 100644
--- a/web/src/features/ai-gateway/server/provider/registry.servertest.ts
+++ b/web/src/features/ai-gateway/server/provider/registry.servertest.ts
@@ -72,6 +72,7 @@ describe("AI gateway provider registry", () => {
},
]),
),
+ ttl: vi.fn().mockResolvedValue(300),
setex: vi.fn(),
} as unknown as Redis;
const prisma = {
@@ -101,6 +102,123 @@ describe("AI gateway provider registry", () => {
"ai-gateway:models:org-1:connection-1",
);
expect(fetcher).not.toHaveBeenCalled();
+ expect(prisma.gatewayAiConnection.findFirst).not.toHaveBeenCalled();
+ });
+
+ it("does not cache a background refresh that finished after its connection changed", async () => {
+ const fetcher = vi
+ .fn<typeof fetch>()
+ .mockResolvedValue(
+ Response.json({ data: [{ id: "old-credential-model", created: 10 }] }),
+ );
+ const redis = {
+ get: vi.fn().mockResolvedValue(
+ JSON.stringify([
+ {
+ id: "gpt-5-mini",
+ provider: "OPENAI",
+ canonicalSlug: "gpt-5-mini",
+ displayName: "gpt-5-mini",
+ createdAt: "2026-01-01T00:00:00.000Z",
+ },
+ ]),
+ ),
+ ttl: vi.fn().mockResolvedValue(10),
+ set: vi.fn().mockResolvedValue("OK"),
+ setex: vi.fn().mockResolvedValue("OK"),
+ } as unknown as Redis;
+ const findFirst = vi
+ .fn()
+ .mockResolvedValueOnce({
+ id: "connection-1",
+ organizationId: "org-1",
+ provider: "OPENAI",
+ encryptedCredential: encrypt("sk-old"),
+ status: "ENABLED",
+ allowedModels: null,
+ updatedAt: new Date(0),
+ })
+ .mockResolvedValueOnce(null);
+ const prisma = {
+ gatewayAiConnection: { findFirst },
+ } as unknown as PrismaClient;
+ const service = new GatewayModelCatalogService(prisma, fetcher, redis);
+
+ await service.refreshModels({
+ organizationId: "org-1",
+ connectionId: "connection-1",
+ });
+
+ await vi.waitFor(() => expect(findFirst).toHaveBeenCalledTimes(2));
+ expect(findFirst).toHaveBeenLastCalledWith({
+ where: {
+ organizationId: "org-1",
+ id: "connection-1",
+ updatedAt: new Date(0),
+ },
+ select: { id: true },
+ });
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ expect(redis.setex).not.toHaveBeenCalled();
+ });
+
+ it("serves a nearly expired model list while refreshing it once in the background", async () => {
+ const fetcher = vi
+ .fn<typeof fetch>()
+ .mockResolvedValue(
+ Response.json({ data: [{ id: "gpt-5", created: 10 }] }),
+ );
+ const redis = {
+ get: vi.fn().mockResolvedValue(
+ JSON.stringify([
+ {
+ id: "gpt-5-mini",
+ provider: "OPENAI",
+ canonicalSlug: "gpt-5-mini",
+ displayName: "gpt-5-mini",
+ createdAt: "2026-01-01T00:00:00.000Z",
+ },
+ ]),
+ ),
+ ttl: vi.fn().mockResolvedValue(10),
+ set: vi.fn().mockResolvedValueOnce("OK").mockResolvedValue(null),
+ setex: vi.fn().mockResolvedValue("OK"),
+ } as unknown as Redis;
+ const prisma = {
+ gatewayAiConnection: {
+ findFirst: vi.fn().mockResolvedValue({
+ id: "connection-1",
+ organizationId: "org-1",
+ provider: "OPENAI",
+ encryptedCredential: encrypt("sk-test"),
+ status: "ENABLED",
+ allowedModels: null,
+ }),
+ },
+ } as unknown as PrismaClient;
+ const service = new GatewayModelCatalogService(prisma, fetcher, redis);
+ const request = {
+ organizationId: "org-1",
+ connectionId: "connection-1",
+ };
+
+ await expect(
+ Promise.all([
+ service.refreshModels(request),
+ service.refreshModels(request),
+ ]),
+ ).resolves.toEqual([
+ { connectionId: "connection-1", success: true, models: ["gpt-5-mini"] },
+ { connectionId: "connection-1", success: true, models: ["gpt-5-mini"] },
+ ]);
+ await vi.waitFor(() =>
+ expect(redis.setex).toHaveBeenCalledWith(
+ "ai-gateway:models:org-1:connection-1",
+ 300,
+ expect.stringContaining('"id":"gpt-5"'),
+ ),
+ );
+ expect(fetcher).toHaveBeenCalledTimes(1);
});
it("collects every Anthropic model page as canonical metadata", async () => {
diff --git a/web/src/features/ai-gateway/server/router.ts b/web/src/features/ai-gateway/server/router.ts
index 7c5a1ff970..c4ae663be3 100644
--- a/web/src/features/ai-gateway/server/router.ts
+++ b/web/src/features/ai-gateway/server/router.ts
@@ -397,7 +397,7 @@ export const aiGatewayRouter = createTRPCRouter({
const catalogService = new GatewayModelCatalogService(ctx.prisma);
const results = canManageGateway
? await catalogService.refreshAllModels(input.orgId)
- : await catalogService.listCachedModels(input.orgId);
+ : await catalogService.listAllModels(input.orgId);
if (canManageGateway) return results;
const readableConnections = await catalogService.listModelConnections(