🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { z } from "zod/v4";
import type { Session } from "next-auth";
import {
hasOrganizationAccess,
throwIfNoOrganizationAccess,
} from "@/src/features/rbac/utils/checkOrganizationAccess";
import {
createTRPCRouter,
protectedOrganizationProcedure,
protectedOrganizationProcedureWithoutTracing,
type AuthedSession,
} from "@/src/server/api/trpc";
import { env } from "@/src/env.mjs";
import { canReadGatewayModels } from "@/src/features/ai-gateway/fns/canReadGatewayModels";
import { isGatewayModelAllowed } from "@/src/features/ai-gateway/fns/isGatewayModelAllowed";
import { getProductBaseUrl } from "@/src/utils/base-url";
import {
GatewayConnectionStatus,
GatewayIngestionMode,
GatewayProvider,
} from "@langfuse/shared/src/db";
import { redis } from "@langfuse/shared/src/server";
import { orderBy } from "@langfuse/shared";
import {
GatewayApiKeyService,
gatewayApiKeyCreatorScopes,
} from "./apiKey/gatewayApiKeyService";
import { requireGatewayEnabledForOrganization } from "./availability";
import { getGatewayBaseUrl } from "@/src/features/ai-gateway/fns/gatewayUrls/getGatewayBaseUrl";
import {
MAX_MEMBER_SEARCH_LENGTH,
MAX_TEAM_INVITES,
} from "@/src/features/ai-gateway/constants/gatewayTeamInvites";
import { GatewayAnalyticsLabelsRepository } from "./analytics/gatewayAnalyticsLabelsRepository";
import { GatewayConfigService } from "./config/gatewayConfigService";
import { GatewayTeamService } from "./team/gatewayTeamService";
import { GatewayMetadataSchema } from "@/src/features/ai-gateway/server/provider/registry";
import { GatewayModelCatalogService } from "@/src/features/ai-gateway/server/provider/models/gatewayModelCatalogService";
import { GatewayProviderService } from "@/src/features/ai-gateway/server/provider/connection/gatewayProviderService";
const organizationInput = z.object({ orgId: z.string() });
/** Naming rules live in `parseGatewayConnectionName` so the dialog can show them on the field. */
const connectionNameSchema = z.string().max(1_000);
const pageLimitSchema = z.number().int().min(1).max(100).default(50);
const paginatedOrganizationInput = organizationInput.extend({
cursor: z.string().optional(),
limit: pageLimitSchema,
});
const offsetPaginatedOrganizationInput = organizationInput.extend({
page: z.number().int().min(0).default(0),
limit: pageLimitSchema,
});
const connectionInput = organizationInput.extend({ id: z.string() });
const allowedModelsSchema = z
.array(z.string().trim().min(1).max(500))
.max(10_000)
.nullable();
function requireGatewayAdmin(params: { session: Session; orgId: string }) {
requireGatewayEnabledForOrganization(params.orgId);
throwIfNoOrganizationAccess({
session: params.session,
organizationId: params.orgId,
scope: "gateway:manage",
});
}
/** Admits gateway key owners; the returned `ownerUserId` limits non-managers to their own keys. */
function requireGatewayInvoke(params: {
session: AuthedSession;
orgId: string;
}) {
requireGatewayEnabledForOrganization(params.orgId);
throwIfNoOrganizationAccess({
session: params.session,
organizationId: params.orgId,
scope: "gateway:invoke",
});
const canManageGateway = hasOrganizationAccess({
session: params.session,
organizationId: params.orgId,
scope: "gateway:manage",
});
return { ownerUserId: canManageGateway ? undefined : params.session.user.id };
}
function requireGatewayRead(params: { session: Session; orgId: string }) {
requireGatewayEnabledForOrganization(params.orgId);
const canManageGateway = hasOrganizationAccess({
session: params.session,
organizationId: params.orgId,
scope: "gateway:manage",
});
const canReadGateway = hasOrganizationAccess({
session: params.session,
organizationId: params.orgId,
scope: "gateway:read",
});
if (
!canReadGatewayModels({
organizationId: params.orgId,
demoOrganizationId: env.NEXT_PUBLIC_DEMO_ORG_ID,
canReadGateway,
canManageGateway,
})
) {
throwIfNoOrganizationAccess({
session: params.session,
organizationId: params.orgId,
scope: "gateway:manage",
});
}
throwIfNoOrganizationAccess({
session: params.session,
organizationId: params.orgId,
scope: "gateway:read",
});
}
export const aiGatewayRouter = createTRPCRouter({
getConfig: protectedOrganizationProcedure
.input(organizationInput)
.query(async ({ input, ctx }) => {
requireGatewayAdmin({ session: ctx.session, orgId: input.orgId });
return {
config: await new GatewayConfigService(ctx.prisma).getConfig(
input.orgId,
),
};
}),
/** Readable by every gateway reader and key owner; exposes only whether setup is done, not the config itself. */
getSetupStatus: protectedOrganizationProcedure
.input(organizationInput)
.query(async ({ input, ctx }) => {
const canInvokeGateway = hasOrganizationAccess({
session: ctx.session,
organizationId: input.orgId,
scope: "gateway:invoke",
});
if (canInvokeGateway)
requireGatewayInvoke({ session: ctx.session, orgId: input.orgId });
else requireGatewayRead({ session: ctx.session, orgId: input.orgId });
return new GatewayConfigService(ctx.prisma).getSetupStatus(input.orgId);
}),
/** Names for the API key and user IDs that gateway analytics break down by. */
getAnalyticsLabels: protectedOrganizationProcedure
.input(organizationInput)
.query(async ({ input, ctx }) => {
requireGatewayAdmin({ session: ctx.session, orgId: input.orgId });
return new GatewayAnalyticsLabelsRepository(ctx.prisma).getLabels({
organizationId: input.orgId,
includePlainOrganizationKeys: hasOrganizationAccess({
session: ctx.session,
organizationId: input.orgId,
scope: "organization:CRUD_apiKeys",
}),
});
}),
/** What a key owner needs to point a client at the gateway; no connection details. */
getClientSetup: protectedOrganizationProcedure
.input(organizationInput)
.query(async ({ input, ctx }) => {
requireGatewayInvoke({ session: ctx.session, orgId: input.orgId });
const connections = await new GatewayProviderService(ctx.prisma).listAll(
input.orgId,
GatewayConnectionStatus.ENABLED,
);
return {
gatewayBaseUrl: getGatewayBaseUrl(
getProductBaseUrl(),
env.LANGFUSE_AI_GATEWAY_PUBLIC_URL,
),
providers: [
...new Set(connections.map((connection) => connection.provider)),
],
};
}),
updateConfig: protectedOrganizationProcedure
.input(
organizationInput.extend({
defaultIngestionProjectId: z.string().nullable(),
ingestionMode: z.enum(GatewayIngestionMode),
}),
)
.mutation(async ({ input, ctx }) => {
requireGatewayAdmin({ session: ctx.session, orgId: input.orgId });
return new GatewayConfigService(ctx.prisma).updateConfig({
organizationId: input.orgId,
defaultIngestionProjectId: input.defaultIngestionProjectId,
ingestionMode: input.ingestionMode,
session: ctx.session,
});
}),
getNotifiableMembers: protectedOrganizationProcedure
.input(organizationInput)
.query(async ({ input, ctx }) => {
requireGatewayAdmin({ session: ctx.session, orgId: input.orgId });
return new GatewayTeamService(ctx.prisma).getNotifiableMembers({
organizationId: input.orgId,
currentUserId: ctx.session.user.id,
});
}),
getMemberSuggestions: protectedOrganizationProcedure
.input(
organizationInput.extend({
search: z.string().max(MAX_MEMBER_SEARCH_LENGTH),
excludeEmails: z.array(z.string()).max(MAX_TEAM_INVITES),
}),
)
.query(async ({ input, ctx }) => {
requireGatewayAdmin({ session: ctx.session, orgId: input.orgId });
return new GatewayTeamService(ctx.prisma).getMemberSuggestions({
organizationId: input.orgId,
currentUserId: ctx.session.user.id,
search: input.search,
excludeEmails: input.excludeEmails,
});
}),
getInviteeMembers: protectedOrganizationProcedure
.input(
organizationInput.extend({
emails: z.array(z.email()).max(MAX_TEAM_INVITES),
}),
)
.query(async ({ input, ctx }) => {
requireGatewayAdmin({ session: ctx.session, orgId: input.orgId });
return new GatewayTeamService(ctx.prisma).getInviteeMembers({
organizationId: input.orgId,
currentUserId: ctx.session.user.id,
emails: input.emails,
});
}),
/** Invites new people with the AI Gateway role and optionally tells existing members. */
inviteTeam: protectedOrganizationProcedure
.input(
organizationInput.extend({
emails: z.array(z.email()).max(MAX_TEAM_INVITES),
notifyMembers: z.boolean(),
}),
)
.mutation(async ({ input, ctx }) => {
requireGatewayAdmin({ session: ctx.session, orgId: input.orgId });
return new GatewayTeamService(ctx.prisma).inviteTeam({
ctx,
organizationId: input.orgId,
emails: input.emails,
notifyMembers: input.notifyMembers,
});
}),
listConnections: protectedOrganizationProcedure
.input(paginatedOrganizationInput)
.query(async ({ input, ctx }) => {
requireGatewayAdmin({ session: ctx.session, orgId: input.orgId });
return new GatewayProviderService(ctx.prisma).list({
organizationId: input.orgId,
cursor: input.cursor,
limit: input.limit,
});
}),
getConnection: protectedOrganizationProcedure
.input(connectionInput)
.query(async ({ input, ctx }) => {
requireGatewayAdmin({ session: ctx.session, orgId: input.orgId });
return new GatewayProviderService(ctx.prisma).get({
organizationId: input.orgId,
id: input.id,
});
}),
testConnection: protectedOrganizationProcedureWithoutTracing
.input(
organizationInput.extend({
provider: z.enum(GatewayProvider),
credential: z.string().min(1),
}),
)
.mutation(async ({ input, ctx }) => {
requireGatewayAdmin({ session: ctx.session, orgId: input.orgId });
await new GatewayProviderService(ctx.prisma).testCredential({
provider: input.provider,
credential: input.credential,
});
return { success: true };
}),
createConnection: protectedOrganizationProcedureWithoutTracing
.input(
organizationInput.extend({
name: connectionNameSchema,
provider: z.enum(GatewayProvider),
credential: z.string().min(1),
}),
)
.mutation(async ({ input, ctx }) => {
requireGatewayAdmin({ session: ctx.session, orgId: input.orgId });
const connection = await new GatewayProviderService(ctx.prisma).create({
organizationId: input.orgId,
name: input.name,
provider: input.provider,
credential: input.credential,
session: ctx.session,
});
return connection;
}),
updateConnection: protectedOrganizationProcedureWithoutTracing
.input(
organizationInput.extend({
id: z.string(),
name: connectionNameSchema.optional(),
credential: z.string().min(1).optional(),
status: z
.enum([
GatewayConnectionStatus.ENABLED,
GatewayConnectionStatus.DISABLED,
])
.optional(),
}),
)
.mutation(async ({ input, ctx }) => {
requireGatewayAdmin({ session: ctx.session, orgId: input.orgId });
return new GatewayProviderService(ctx.prisma).update({
organizationId: input.orgId,
id: input.id,
name: input.name,
credential: input.credential,
status: input.status,
session: ctx.session,
});
}),
deleteConnection: protectedOrganizationProcedure
.input(connectionInput)
.mutation(async ({ input, ctx }) => {
requireGatewayAdmin({ session: ctx.session, orgId: input.orgId });
await new GatewayProviderService(ctx.prisma).delete({
organizationId: input.orgId,
id: input.id,
session: ctx.session,
});
return { success: true };
}),
updateConnectionModels: protectedOrganizationProcedure
.input(
connectionInput.extend({
allowedModels: allowedModelsSchema,
expectedUpdatedAt: z.date(),
}),
)
.mutation(async ({ input, ctx }) => {
requireGatewayAdmin({ session: ctx.session, orgId: input.orgId });
return new GatewayProviderService(ctx.prisma).updateAllowedModels({
organizationId: input.orgId,
id: input.id,
allowedModels: input.allowedModels,
expectedUpdatedAt: input.expectedUpdatedAt,
session: ctx.session,
});
}),
listModelConnections: protectedOrganizationProcedure
.input(organizationInput)
.query(async ({ input, ctx }) => {
requireGatewayRead({ session: ctx.session, orgId: input.orgId });
return new GatewayModelCatalogService(ctx.prisma).listModelConnections(
input.orgId,
);
}),
refreshModels: protectedOrganizationProcedure
.input(organizationInput)
.query(async ({ input, ctx }) => {
requireGatewayRead({
session: ctx.session,
orgId: input.orgId,
});
const canManageGateway = hasOrganizationAccess({
session: ctx.session,
organizationId: input.orgId,
scope: "gateway:manage",
});
const catalogService = new GatewayModelCatalogService(ctx.prisma);
const results = canManageGateway
? await catalogService.refreshAllModels(input.orgId)
: await catalogService.listAllModels(input.orgId);
if (canManageGateway) return results;
const readableConnections = await catalogService.listModelConnections(
input.orgId,
);
const allowedModelsByConnectionId = new Map(
readableConnections.map((connection) => [
connection.id,
connection.allowedModels,
]),
);
return results.map((result) =>
result.success
? {
...result,
models: result.models.filter((model) => {
const allowedModels = allowedModelsByConnectionId.get(
result.connectionId,
);
return isGatewayModelAllowed(
allowedModels === undefined ? [] : allowedModels,
model,
);
}),
}
: result,
);
}),
getConnectionModels: protectedOrganizationProcedure
.input(connectionInput)
.query(async ({ input, ctx }) => {
requireGatewayAdmin({ session: ctx.session, orgId: input.orgId });
return new GatewayModelCatalogService(ctx.prisma).refreshModels({
organizationId: input.orgId,
connectionId: input.id,
});
}),
syncModels: protectedOrganizationProcedure
.input(organizationInput)
.mutation(async ({ input, ctx }) => {
requireGatewayAdmin({ session: ctx.session, orgId: input.orgId });
return new GatewayModelCatalogService(ctx.prisma).refreshAllModels(
input.orgId,
true,
);
}),
retryConnection: protectedOrganizationProcedure
.input(connectionInput)
.mutation(async ({ input, ctx }) => {
requireGatewayAdmin({ session: ctx.session, orgId: input.orgId });
return new GatewayModelCatalogService(ctx.prisma).retryConnection({
organizationId: input.orgId,
connectionId: input.id,
session: ctx.session,
});
}),
listApiKeys: protectedOrganizationProcedure
.input(
offsetPaginatedOrganizationInput.extend({
searchQuery: z.string().trim().optional(),
orderBy: orderBy.optional(),
}),
)
.query(async ({ input, ctx }) => {
const { ownerUserId } = requireGatewayInvoke({
session: ctx.session,
orgId: input.orgId,
});
return new GatewayApiKeyService(ctx.prisma, redis).list({
organizationId: input.orgId,
ownerUserId,
searchQuery: input.searchQuery,
orderBy: input.orderBy,
page: input.page,
limit: input.limit,
});
}),
/** Loaded after listApiKeys for the visible page, so ClickHouse never delays the key list. */
getApiKeysLastUsed: protectedOrganizationProcedure
.input(
organizationInput.extend({
apiKeyIds: z.array(z.string()).max(100),
}),
)
.query(async ({ input, ctx }) => {
const { ownerUserId } = requireGatewayInvoke({
session: ctx.session,
orgId: input.orgId,
});
return new GatewayApiKeyService(ctx.prisma, redis).getLastUsed({
organizationId: input.orgId,
ownerUserId,
apiKeyIds: input.apiKeyIds,
});
}),
// Returns the plaintext secret key once, so the procedure must not be traced.
createApiKey: protectedOrganizationProcedureWithoutTracing
.input(
organizationInput.extend({
name: z.string().trim().min(1, "Gateway keys need a name").max(500),
metadata: GatewayMetadataSchema.default({}),
expiresAt: z
.date()
.nullish()
.refine((date) => date == null || date.getTime() > Date.now(), {
message: "Expiration date must be in the future",
}),
issuedAs: z.enum(["user", "serviceAccount"]).default("user"),
}),
)
.mutation(async ({ input, ctx }) => {
// Legacy public API auth ignores key roles, so a gateway key passes as an
// organization admin key there; only managers may mint one outside enforce.
if (env.API_AUTH_MIGRATION === "enforce" && input.issuedAs === "user") {
requireGatewayInvoke({ session: ctx.session, orgId: input.orgId });
} else {
requireGatewayAdmin({ session: ctx.session, orgId: input.orgId });
}
for (const scope of gatewayApiKeyCreatorScopes(input.issuedAs)) {
throwIfNoOrganizationAccess({
session: ctx.session,
organizationId: input.orgId,
scope,
});
}
const key = await new GatewayApiKeyService(ctx.prisma, redis).create({
organizationId: input.orgId,
name: input.name,
metadata: input.metadata,
expiresAt: input.expiresAt,
issuedAs: input.issuedAs,
session: ctx.session,
});
return key;
}),
revokeApiKey: protectedOrganizationProcedure
.input(connectionInput)
.mutation(async ({ input, ctx }) => {
const { ownerUserId } = requireGatewayInvoke({
session: ctx.session,
orgId: input.orgId,
});
await new GatewayApiKeyService(ctx.prisma, redis).revoke({
organizationId: input.orgId,
ownerUserId,
apiKeyId: input.id,
session: ctx.session,
});
return { success: true };
}),
});