🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(topics): resolve LLM connections by provider like evals (#19642)
Lysander Kiesel23622970ae6b
diff --git a/packages/shared/src/server/topics/model-config.test.ts b/packages/shared/src/server/topics/model-config.test.tsindex f628dabc9d..1336638b16 100644--- a/packages/shared/src/server/topics/model-config.test.ts+++ b/packages/shared/src/server/topics/model-config.test.ts@@ -9,6 +9,7 @@ import {const mocks = vi.hoisted(() => ({keys: vi.fn(),+ candidates: vi.fn(),projects: vi.fn(),configs: vi.fn(),current: vi.fn(),@@ -29,7 +30,7 @@ vi.mock("./processing-gate", () => ({vi.mock("../../db", () => {const tx = {llmApiKeys: { findMany: mocks.keys },- project: { findMany: mocks.projects },+ project: { findMany: mocks.projects, findUnique: mocks.candidates },topicConfig: {findMany: mocks.configs,findUnique: mocks.current,@@ -51,13 +52,13 @@ vi.mock("../../db", () => {const connection = (id: string, adapter: LLMAdapter) => ({id,adapter,- provider: `${adapter}-connection`,+ provider: id,secretKey: "encrypted",extraHeaders: null,baseURL: null,config: null,});-const slot = (llmApiKeyId: string, model: string) => ({ llmApiKeyId, model });+const slot = (provider: string, model: string) => ({ provider, model });const settings = {summary: slot("openai", "gpt-6-luna"),embedding: slot("openai", "text-embedding-3-small"),@@ -76,10 +77,13 @@ const saveTopicsModelSettings = async (beforeEach(() => {vi.resetAllMocks();- mocks.keys.mockResolvedValue([- connection("openai", LLMAdapter.OpenAI),- connection("anthropic", LLMAdapter.Anthropic),- ]);+ mocks.candidates.mockResolvedValue({+ LlmApiKeys: [+ connection("openai", LLMAdapter.OpenAI),+ connection("anthropic", LLMAdapter.Anthropic),+ ],+ organization: { organizationLlmApiKeys: [] },+ });mocks.current.mockResolvedValue(null);mocks.projects.mockResolvedValue([{ id: "project", orgId: "target-org" }]);mocks.hasEmbeddings.mockResolvedValue(false);@@ -97,35 +101,6 @@ beforeEach(() => {});describe("prepareTopicsModelSettings", () => {- it("accepts an organization connection inherited by the project", async () => {- mocks.keys.mockResolvedValue([- {- ...connection("openai", LLMAdapter.OpenAI),- projectId: null,- organizationId: "organization",- },- ]);-- await saveTopicsModelSettings("project", settings);-- expect(mocks.keys).toHaveBeenCalledWith({- where: {- id: { in: ["openai", "openai", "openai"] },- OR: [- { projectId: "project" },- {- organization: {- projects: {- some: { id: "project", deletedAt: null },- },- },- },- ],- },- });- expect(mocks.upsert).toHaveBeenCalledOnce();- });-it("validates text slots through the Topics transport before saving", async () => {await saveTopicsModelSettings("project", {...settings,@@ -319,14 +294,15 @@ describe("getEnabledTopicsModels", () => {projectId: "complete",project: { orgId: "organization" },enabled: true,- summaryLlmApiKey: openai,+ summaryLlmApiKey: { provider: "openai" },summaryModel: "gpt-6-luna",- embeddingLlmApiKey: openai,+ embeddingLlmApiKey: { provider: "openai" },embeddingModel: "text-embedding-3-small",embeddingDimensions: 512,- clusteringLlmApiKey: openai,+ clusteringLlmApiKey: { provider: "openai" },clusteringModel: "gpt-5.6-terra",};+ mocks.keys.mockResolvedValue([openai]);mocks.configs.mockResolvedValue([row,// A deleted connection leaves its slot empty.diff --git a/packages/shared/src/server/topics/model-config.ts b/packages/shared/src/server/topics/model-config.tsindex f71f08049a..e8f2ff772d 100644--- a/packages/shared/src/server/topics/model-config.ts+++ b/packages/shared/src/server/topics/model-config.ts@@ -12,6 +12,7 @@ import {} from "../llm/llmText";import { LLMAdapter } from "../llm/types";import { findCompatibleLlmApiKeysForProjects } from "../services/LLMApiKeys/findCompatibleLlmApiKeysForProjects";+import { findProjectLlmApiKeyCandidates } from "../services/LLMApiKeys/LlmApiKeyRepository";import {TOPICS_MODEL_SLOT_DETAILS,TOPICS_MODEL_SLOTS,@@ -41,18 +42,47 @@ export type TopicsModels = {clustering: TopicsModel;};-const withConnections = {+/** Settings after each provider resolved to the project's effective connection. */+type ConnectionSlot = { llmApiKeyId: string; model: string } | null;+type ResolvedModelSettings = {+ summary: ConnectionSlot;+ embedding: ConnectionSlot;+ embeddingDimensions: number;+ clustering: ConnectionSlot;+};++type LlmApiKeyCandidates = {+ projectConnections: LlmApiKeys[];+ organizationConnections: LlmApiKeys[];+};++/** Like evals: a project connection overrides its organization's connection of the same provider. */+function effectiveConnection(+ candidates: LlmApiKeyCandidates | null,+ provider: string,+): LlmApiKeys | null {+ const byProvider = (key: LlmApiKeys) => key.provider === provider;+ return (+ candidates?.projectConnections.find(byProvider) ??+ candidates?.organizationConnections.find(byProvider) ??+ null+ );+}++const storedProvider = { select: { provider: true } } as const;+const withStoredProviders = {project: { select: { orgId: true } },- summaryLlmApiKey: true,- embeddingLlmApiKey: true,- clusteringLlmApiKey: true,+ summaryLlmApiKey: storedProvider,+ embeddingLlmApiKey: storedProvider,+ clusteringLlmApiKey: storedProvider,} as const;-type ConfigWithConnections = TopicConfig & {+type StoredConnection = { provider: string } | null;+type ConfigWithProviders = TopicConfig & {project: { orgId: string };- summaryLlmApiKey: LlmApiKeys | null;- embeddingLlmApiKey: LlmApiKeys | null;- clusteringLlmApiKey: LlmApiKeys | null;+ summaryLlmApiKey: StoredConnection;+ embeddingLlmApiKey: StoredConnection;+ clusteringLlmApiKey: StoredConnection;};function resolveSlot(@@ -60,15 +90,8 @@ function resolveSlot(key: LlmApiKeys | null,model: string | null,projectId?: string,- organizationId?: string,): TopicsModel | null {if (!key || !model) return null;- if (- projectId &&- key.projectId !== projectId &&- key.organizationId !== organizationId- )- return null;return {slot,provider: key.provider,@@ -84,27 +107,32 @@ function resolveSlot(};}-function resolveModels(row: ConfigWithConnections): TopicsModels | null {- const summary = resolveSlot(- "summary",- row.summaryLlmApiKey,- row.summaryModel,- row.projectId,- row.project.orgId,- );- const embedding = resolveSlot(+/** The stored connection only names the provider; it resolves again on every run, like evals. */+function resolveModels(+ row: ConfigWithProviders,+ candidates: LlmApiKeyCandidates,+): TopicsModels | null {+ const slot = (+ name: TopicsModelSlotName,+ stored: StoredConnection,+ model: string | null,+ ) =>+ resolveSlot(+ name,+ stored && effectiveConnection(candidates, stored.provider),+ model,+ row.projectId,+ );+ const summary = slot("summary", row.summaryLlmApiKey, row.summaryModel);+ const embedding = slot("embedding",row.embeddingLlmApiKey,row.embeddingModel,- row.projectId,- row.project.orgId,);- const clustering = resolveSlot(+ const clustering = slot("clustering",row.clusteringLlmApiKey,row.clusteringModel,- row.projectId,- row.project.orgId,);if (!summary || !embedding || !clustering) return null;return {@@ -116,32 +144,64 @@ function resolveModels(row: ConfigWithConnections): TopicsModels | null {};}+/** Like evals, loads only the connections of the stored providers, in one query for all configs. */+async function resolveConfigs(+ rows: ConfigWithProviders[],+): Promise<TopicsModels[]> {+ const providers = [+ ...new Set(+ rows.flatMap((row) =>+ [row.summaryLlmApiKey, row.embeddingLlmApiKey, row.clusteringLlmApiKey]+ .filter((stored) => stored !== null)+ .map(({ provider }) => provider),+ ),+ ),+ ];+ const keys = providers.length+ ? await prisma.llmApiKeys.findMany({+ where: {+ provider: { in: providers },+ OR: [+ { projectId: { in: rows.map((row) => row.projectId) } },+ { organizationId: { in: rows.map((row) => row.project.orgId) } },+ ],+ },+ })+ : [];+ return rows.flatMap((row) => {+ const resolved = resolveModels(row, {+ projectConnections: keys.filter((key) => key.projectId === row.projectId),+ organizationConnections: keys.filter(+ (key) => key.organizationId === row.project.orgId,+ ),+ });+ return resolved ? [resolved] : [];+ });+}+/** Returns the configured models of a project, or null while any slot is missing. */export async function getTopicsModels(projectId: string,): Promise<TopicsModels | null> {const row = await prisma.topicConfig.findUnique({where: { projectId },- include: withConnections,+ include: withStoredProviders,});- return row ? resolveModels(row) : null;+ return row ? ((await resolveConfigs([row]))[0] ?? null) : null;}-/** Loads the enabled, fully configured projects of a trace batch in one query. */+/** Loads the enabled, fully configured projects of a trace batch in two queries. */export async function getEnabledTopicsModels(projectIds: string[],): Promise<Map<string, TopicsModels>> {if (!projectIds.length) return new Map();const rows = await prisma.topicConfig.findMany({where: { projectId: { in: projectIds }, enabled: true },- include: withConnections,+ include: withStoredProviders,});- const models = new Map<string, TopicsModels>();- for (const row of rows) {- const resolved = resolveModels(row);- if (resolved) models.set(row.projectId, resolved);- }- return models;+ return new Map(+ (await resolveConfigs(rows)).map((models) => [models.projectId, models]),+ );}/** Turns automatic processing off and records why, for the Topics page. */@@ -336,18 +396,23 @@ export async function reconcileTopicsConnectionsForProject(export async function readTopicsModelSettings(projectId: string) {const row = await prisma.topicConfig.findUnique({where: { projectId },+ include: {+ summaryLlmApiKey: storedProvider,+ embeddingLlmApiKey: storedProvider,+ clusteringLlmApiKey: storedProvider,+ },});- const slot = (llmApiKeyId: string | null, model: string | null) =>- llmApiKeyId && model ? { llmApiKeyId, model } : null;+ const slot = (connection: StoredConnection, model: string | null) =>+ connection && model ? { provider: connection.provider, model } : null;return {- summary: slot(row?.summaryLlmApiKeyId ?? null, row?.summaryModel ?? null),+ summary: slot(row?.summaryLlmApiKey ?? null, row?.summaryModel ?? null),embedding: slot(- row?.embeddingLlmApiKeyId ?? null,+ row?.embeddingLlmApiKey ?? null,row?.embeddingModel ?? null,),embeddingDimensions: row?.embeddingDimensions ?? 1024,clustering: slot(- row?.clusteringLlmApiKeyId ?? null,+ row?.clusteringLlmApiKey ?? null,row?.clusteringModel ?? null,),enabled: row ? row.enabled : null,@@ -357,35 +422,34 @@ export async function readTopicsModelSettings(projectId: string) {};}-async function loadTopicsModelKeys(+/** Resolves each slot's provider to the project's effective connection, or throws. */+async function resolveModelSettings(projectId: string,settings: TopicsModelSettings,-) {- const ids = [- settings.summary,- settings.embedding,- settings.clustering,- ].flatMap((slot) => (slot ? [slot.llmApiKeyId] : []));- const keys = await prisma.llmApiKeys.findMany({- where: {- id: { in: ids },- OR: [- { projectId },- {- organization: {- projects: { some: { id: projectId, deletedAt: null } },- },- },- ],- },- });- for (const id of ids) {- if (!keys.some((key) => key.id === id))+): Promise<{ settings: ResolvedModelSettings; keys: LlmApiKeys[] }> {+ const candidates = await findProjectLlmApiKeyCandidates({ projectId });+ const keys: LlmApiKeys[] = [];+ const slot = (+ requested: TopicsModelSettings[TopicsModelSlotName],+ ): ConnectionSlot => {+ if (!requested) return null;+ const key = effectiveConnection(candidates, requested.provider);+ if (!key)throw new InvalidRequestError("LLM connection not found in this project.",);- }- return keys;+ keys.push(key);+ return { llmApiKeyId: key.id, model: requested.model };+ };+ return {+ settings: {+ summary: slot(settings.summary),+ embedding: slot(settings.embedding),+ embeddingDimensions: settings.embeddingDimensions,+ clustering: slot(settings.clustering),+ },+ keys,+ };}/** Real call per configured slot. The message is the provider error for that row. */@@ -393,10 +457,8 @@ export async function checkTopicsModelSettings(projectId: string,settings: TopicsModelSettings,): Promise<Partial<Record<TopicsModelSlotName, string>>> {- return testTopicsModelSlots(- settings,- await loadTopicsModelKeys(projectId, settings),- );+ const resolved = await resolveModelSettings(projectId, settings);+ return testTopicsModelSlots(resolved.settings, resolved.keys);}/**@@ -408,7 +470,10 @@ export async function prepareTopicsModelSettings(projectId: string,settings: TopicsModelSettings,) {- const keys = await loadTopicsModelKeys(projectId, settings);+ const { settings: resolved, keys } = await resolveModelSettings(+ projectId,+ settings,+ );const current = await prisma.topicConfig.findUnique({where: { projectId },});@@ -421,7 +486,7 @@ export async function prepareTopicsModelSettings("Choose a summary, embedding, and clustering model while automatic processing is on, or turn it off first.",);- const slots = resolveSavedSlots(settings, current);+ const slots = resolveSavedSlots(resolved, current);const embeddingChanged =current?.embeddingModel &&(slots.embedding.model !== current.embeddingModel ||@@ -433,7 +498,7 @@ export async function prepareTopicsModelSettings(// Only changed models are tested: the dialog tests each pick, turning// processing on tests all three, and a broken stored connection pauses Topics.- await testTopicsModels(changedSlots(settings, current), keys);+ await testTopicsModels(changedSlots(resolved, current), keys);const data = {summaryLlmApiKeyId: slots.summary.llmApiKeyId,@@ -457,7 +522,7 @@ type StoredTopicConfig = Awaited<* deleted, so saving never discards a model.*/function resolveSavedSlots(- settings: TopicsModelSettings,+ settings: ResolvedModelSettings,current: StoredTopicConfig,) {const keep = current && !current.enabled ? current : null;@@ -490,11 +555,11 @@ function resolveSavedSlots(}function changedSlots(- settings: TopicsModelSettings,+ settings: ResolvedModelSettings,current: StoredTopicConfig,-): TopicsModelSettings {+): ResolvedModelSettings {const changed = (- slot: TopicsModelSettings[TopicsModelSlotName],+ slot: ConnectionSlot,llmApiKeyId: string | null | undefined,model: string | null | undefined,extraChange = false,@@ -531,24 +596,15 @@ function changedSlots(export async function checkStoredTopicsModels(config: NonNullable<StoredTopicConfig>,): Promise<void> {- const slot = (llmApiKeyId: string | null, model: string | null) =>- llmApiKeyId && model ? { llmApiKeyId, model } : null;- const settings = {- summary: slot(config.summaryLlmApiKeyId, config.summaryModel),- embedding: slot(config.embeddingLlmApiKeyId, config.embeddingModel),- embeddingDimensions: config.embeddingDimensions,- clustering: slot(config.clusteringLlmApiKeyId, config.clusteringModel),- };- if (!settings.summary || !settings.embedding || !settings.clustering)+ const stored = await readTopicsModelSettings(config.projectId);+ if (!stored.summary || !stored.embedding || !stored.clustering)throw new InvalidRequestError("Choose a summary, embedding, and clustering model in Configure Topics before turning on automatic processing.",);// Saved models were tested on save; a pause means a connection failed since.if (!config.blockedAt) return;- await testTopicsModels(- settings,- await loadTopicsModelKeys(config.projectId, settings),- );+ const resolved = await resolveModelSettings(config.projectId, stored);+ await testTopicsModels(resolved.settings, resolved.keys);}export async function writeTopicsModelSettings(@@ -580,7 +636,7 @@ const TEST_SCHEMAS = {* length must match the configured dimensions. A slot with no error is omitted.*/async function testTopicsModelSlots(- settings: TopicsModelSettings,+ settings: ResolvedModelSettings,keys: LlmApiKeys[],): Promise<Partial<Record<TopicsModelSlotName, string>>> {const errors: Partial<Record<TopicsModelSlotName, string>> = {};@@ -648,7 +704,7 @@ async function testTopicsModelSlots(}async function testTopicsModels(- settings: TopicsModelSettings,+ settings: ResolvedModelSettings,keys: LlmApiKeys[],): Promise<void> {const errors = await testTopicsModelSlots(settings, keys);diff --git a/packages/shared/src/server/topics/setup.test.ts b/packages/shared/src/server/topics/setup.test.tsindex bd3c90c382..e40b18c30b 100644--- a/packages/shared/src/server/topics/setup.test.ts+++ b/packages/shared/src/server/topics/setup.test.ts@@ -44,10 +44,10 @@ const facets = DEFAULT_TOPIC_FACETS.map((facet, index) => ({}));const setup = {- summary: { llmApiKeyId: "key", model: "gpt" },- embedding: { llmApiKeyId: "key", model: "embed" },+ summary: { provider: "key", model: "gpt" },+ embedding: { provider: "key", model: "embed" },embeddingDimensions: 1024,- clustering: { llmApiKeyId: "key", model: "cluster" },+ clustering: { provider: "key", model: "cluster" },facets: facets.map((facet) => ({name: facet.name,enabled: facet.enabled,diff --git a/packages/shared/src/topics/index.ts b/packages/shared/src/topics/index.tsindex fd03cae3bf..1fc78084aa 100644--- a/packages/shared/src/topics/index.ts+++ b/packages/shared/src/topics/index.ts@@ -59,8 +59,9 @@ export const TOPICS_MODEL_SLOT_DETAILS: Record<"Names and describes each topic when topics are updated. Use a stronger model such as GPT-5.6 Terra, or Claude Sonnet through Amazon Bedrock.",},};+// Like evals: the provider name picks the project's connection, else its organization's.const topicsModelSlotSchema = z.object({- llmApiKeyId: z.string().min(1),+ provider: z.string().min(1),model: z.string().trim().min(1).max(256),});export const topicsModelSettingsSchema = z.object({diff --git a/web/src/__tests__/server/topics-setup.servertest.ts b/web/src/__tests__/server/topics-setup.servertest.tsindex bab2a845d9..f5256da795 100644--- a/web/src/__tests__/server/topics-setup.servertest.ts+++ b/web/src/__tests__/server/topics-setup.servertest.ts@@ -47,7 +47,7 @@ async function topicsProject(displaySecretKey: "...key",},});- const slot = (model: string) => ({ llmApiKeyId: connection.id, model });+ const slot = (model: string) => ({ provider: connection.provider, model });await prisma.topicConfig.create({data: {projectId,diff --git a/web/src/__tests__/server/topics-trpc-access.servertest.ts b/web/src/__tests__/server/topics-trpc-access.servertest.tsindex 9acd4b3f4e..be1bd8ecb7 100644--- a/web/src/__tests__/server/topics-trpc-access.servertest.ts+++ b/web/src/__tests__/server/topics-trpc-access.servertest.ts@@ -1,6 +1,6 @@import { randomUUID } from "node:crypto";import type { Session } from "next-auth";-import { afterAll, beforeAll, describe, expect, it } from "vitest";+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";import { LLMAdapter } from "@langfuse/shared";import { env as sharedEnv } from "@langfuse/shared/src/env";import { prisma } from "@langfuse/shared/src/db";@@ -16,6 +16,17 @@ import {createTopicRun,getTopicsModels,} from "@langfuse/shared/topics/server";+// Only the LLM transport is faked, to see which connection a test call used.+const generateTopicText = vi.hoisted(() =>+ vi.fn(async () => ({+ output: { summary: "Account access", status: "applicable" },+ usage: {},+ })),+);+vi.mock("@langfuse/shared/src/server/topics/text", () => ({+ generateTopicText,+}));+import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";import { appRouter } from "@/src/server/api/root";import { createInnerTRPCContext } from "@/src/server/api/trpc";@@ -44,11 +55,13 @@ function caller(userId: string, memberships: [string, string, Role][]) {function llmConnection(owner: { projectId: string } | { organizationId: string },+ provider = `openai-${randomUUID()}`,) {return prisma.llmApiKeys.create({data: {...owner,- provider: `openai-${randomUUID()}`,+ provider,+ baseURL: `https://${randomUUID()}.example.com`,adapter: LLMAdapter.OpenAI,secretKey: "encrypted",displaySecretKey: "...key",@@ -250,33 +263,42 @@ describe("Topics tRPC access", () => {).toBe(0);});- it("uses only LLM connections of the project or its organization", async () => {+ it("resolves LLM connections by provider within the project, then its organization", async () => {const admin = caller(userId, [[a.orgId, a.projectId, "ADMIN"],[b.orgId, b.projectId, "ADMIN"],]);const projectId = a.projectId;- const ownOrgKey = await llmConnection({ organizationId: a.orgId });+ const shared = `openai-${randomUUID()}`;+ const overriddenOrgKey = await llmConnection(+ { organizationId: a.orgId },+ shared,+ );+ const overridingProjectKey = await llmConnection({ projectId }, shared);+ const inheritedOrgKey = await llmConnection({ organizationId: a.orgId });const otherOrgKey = await llmConnection({ organizationId: b.orgId });+ const otherProjectKey = await prisma.llmApiKeys.findUniqueOrThrow({+ where: { id: b.llmKeyId },+ });const config = await prisma.topicConfig.findUniqueOrThrow({where: { projectId },});- const settings = (llmApiKeyId: string) => ({+ const settings = (provider: string) => ({projectId,- summary: { llmApiKeyId, model: "summary-model" },+ summary: { provider, model: "summary-model" },embedding: null,embeddingDimensions: 1024,clustering: null,});- // Saving or testing: another project's or another org's connection is refused.- for (const foreignKeyId of [b.llmKeyId, otherOrgKey.id]) {- await expect(- admin.testModelSettings(settings(foreignKeyId)),- ).rejects.toThrow("LLM connection not found in this project.");+ // Saving or testing: providers only another project or org has are refused.+ for (const { provider } of [otherProjectKey, otherOrgKey]) {+ await expect(admin.testModelSettings(settings(provider))).rejects.toThrow(+ "LLM connection not found in this project.",+ );await expect(admin.saveSetup({- ...settings(foreignKeyId),+ ...settings(provider),facets: DEFAULT_TOPIC_FACETS.map(({ name }) => ({name,enabled: true,@@ -290,17 +312,34 @@ describe("Topics tRPC access", () => {await prisma.topicConfig.findUniqueOrThrow({ where: { projectId } }),).toEqual(config);- // Running: a stored foreign connection resolves to no models; the own org's does.- const resolveWithSummaryKey = async (summaryLlmApiKeyId: string) => {+ // The project's connection overrides its organization's for the same provider.+ await admin.testModelSettings(settings(shared));+ expect(generateTopicText).toHaveBeenLastCalledWith(+ expect.objectContaining({+ model: expect.objectContaining({+ connection: expect.objectContaining({+ baseURL: overridingProjectKey.baseURL,+ }),+ }),+ }),+ );++ // Running: the stored connection's provider resolves again in this project.+ const summaryBaseUrl = async (summaryLlmApiKeyId: string) => {await prisma.topicConfig.update({where: { projectId },data: { summaryLlmApiKeyId },});- return getTopicsModels(projectId);+ return (await getTopicsModels(projectId))?.summary.connection.baseURL;};- expect(await resolveWithSummaryKey(ownOrgKey.id)).not.toBeNull();- expect(await resolveWithSummaryKey(otherOrgKey.id)).toBeNull();- expect(await resolveWithSummaryKey(b.llmKeyId)).toBeNull();+ expect(await summaryBaseUrl(overriddenOrgKey.id)).toBe(+ overridingProjectKey.baseURL,+ );+ expect(await summaryBaseUrl(inheritedOrgKey.id)).toBe(+ inheritedOrgKey.baseURL,+ );+ expect(await summaryBaseUrl(otherOrgKey.id)).toBeUndefined();+ expect(await summaryBaseUrl(otherProjectKey.id)).toBeUndefined();await prisma.topicConfig.update({where: { projectId },data: { summaryLlmApiKeyId: config.summaryLlmApiKeyId },diff --git a/web/src/__tests__/server/unit/topics.servertest.ts b/web/src/__tests__/server/unit/topics.servertest.tsindex 1ad0b23f34..3490b08a50 100644--- a/web/src/__tests__/server/unit/topics.servertest.ts+++ b/web/src/__tests__/server/unit/topics.servertest.ts@@ -1459,7 +1459,7 @@ describe("Topics model settings", () => {const settings = {projectId,summary: null,- embedding: { llmApiKeyId: "key", model: "text-embedding-3-small" },+ embedding: { provider: "key", model: "text-embedding-3-small" },embeddingDimensions: 1024,clustering: null,};@@ -1492,10 +1492,10 @@ describe("Topics model settings", () => {describe("Topics setup", () => {const setup = {projectId,- summary: { llmApiKeyId: "key", model: "gpt" },- embedding: { llmApiKeyId: "key", model: "embed" },+ summary: { provider: "key", model: "gpt" },+ embedding: { provider: "key", model: "embed" },embeddingDimensions: 1024,- clustering: { llmApiKeyId: "key", model: "cluster" },+ clustering: { provider: "key", model: "cluster" },facets: DEFAULT_TOPIC_FACETS.map((facet) => ({name: facet.name,enabled: facet.name !== "Sentiment",@@ -1621,7 +1621,7 @@ describe("Topics write access", () => {description: "",prompt: "Describe the original request.",};- const slot = { llmApiKeyId: "key", model: "gpt" };+ const slot = { provider: "key", model: "gpt" };const settings = {summary: slot,embedding: slot,diff --git a/web/src/features/topics/ConfigureTopicsDialogController.tsx b/web/src/features/topics/ConfigureTopicsDialogController.tsxindex 9173c3605a..0d8bf5855e 100644--- a/web/src/features/topics/ConfigureTopicsDialogController.tsx+++ b/web/src/features/topics/ConfigureTopicsDialogController.tsx@@ -22,9 +22,9 @@ import {} from "./ConfigureTopicsDialog";import { type ConfigureTopicsSettingsInput } from "./TopicModelSettings";import { topicsSetupPayload } from "./topics-setup";+import { resolveTopicModelSlot } from "./topic-setup-flow";type Connection = RouterOutputs["llmApiKey"]["effective"]["data"][number];-type SlotValue = { llmApiKeyId: string; model: string } | null;type ConfigureTopicsDialogControllerProps = {projectId: string;@@ -99,16 +99,16 @@ function ConfigureTopicsSettings({modelsFor(connection,"embedding",- stored.embedding?.llmApiKeyId === connection.id+ stored.embedding?.provider === connection.provider? stored.embedding.model: null,),]);const draft = {- summary: judgeModel(stored.summary, connections),- embedding: judgeModel(stored.embedding, connections),+ summary: stored.summary,+ embedding: stored.embedding,embeddingDimensions: String(stored.embeddingDimensions),- clustering: judgeModel(stored.clustering, connections),+ clustering: stored.clustering,facets: facetDrafts(facets, rule ? new Set(rule.facetIds) : null),filters: [],idleSeconds: String((traceIdleMs ?? 600_000) / 1000),@@ -146,19 +146,13 @@ function ConfigureTopicsSettings({}));return;}- const preferred = {- summary: stored.summary?.llmApiKeyId,- embedding: stored.embedding?.llmApiKeyId,- clustering: stored.clustering?.llmApiKeyId,- }[slot];- const llmApiKeyId = connectionId(+ const value = resolveTopicModelSlot(model,connections,- preferred,slot === "embedding" ? "embedding" : "chat",- slot === "embedding" ? stored.embedding?.model : null,+ slot === "embedding" ? stored.embedding : null,);- if (!llmApiKeyId) {+ if (!value) {setChecks((current) => ({...current,[slot]: {@@ -170,7 +164,6 @@ function ConfigureTopicsSettings({}const request = ++requests.current[slot];setChecks((current) => ({ ...current, [slot]: { status: "testing" } }));- const value = { llmApiKeyId, model: model.model };test.mutateAsync({projectId,@@ -213,23 +206,16 @@ function ConfigureTopicsSettings({parsedDimensions.error.issues[0]?.message ??"Check the embedding dimensions.",);- const summary = slotValue(- value.summary,- connections,- stored.summary?.llmApiKeyId,- "chat",- );- const embedding = slotValue(+ const summary = resolveTopicModelSlot(value.summary, connections, "chat");+ const embedding = resolveTopicModelSlot(value.embedding,connections,- stored.embedding?.llmApiKeyId,"embedding",- stored.embedding?.model,+ stored.embedding,);- const clustering = slotValue(+ const clustering = resolveTopicModelSlot(value.clustering,connections,- stored.clustering?.llmApiKeyId,"chat",);if (stored.enabled && (!summary || !embedding || !clustering))@@ -332,59 +318,6 @@ function modelsFor();}-function judgeModel(- slot: SlotValue,- connections: Connection[],-): JudgeModel | null {- if (!slot) return null;- const connection = connections.find((item) => item.id === slot.llmApiKeyId);- return {- provider: connection?.provider ?? "Missing connection",- model: slot.model,- };-}--function connectionId(- model: JudgeModel,- connections: Connection[],- preferredId: string | undefined,- kind: "chat" | "embedding",- savedModel?: string | null,-) {- const matches = connections.filter(- (connection) =>- connection.provider === model.provider &&- modelsFor(- connection,- kind,- connection.id === preferredId ? savedModel : null,- ).includes(model.model),- );- return (- matches.find((connection) => connection.id === preferredId)?.id ??- matches[0]?.id ??- null- );-}--function slotValue(- model: JudgeModel | null,- connections: Connection[],- preferredId: string | undefined,- kind: "chat" | "embedding",- savedModel?: string | null,-) {- if (!model) return null;- const llmApiKeyId = connectionId(- model,- connections,- preferredId,- kind,- savedModel,- );- return llmApiKeyId ? { llmApiKeyId, model: model.model } : null;-}-function facetDrafts(facets: TopicFacet[],enabledIds: ReadonlySet<string> | null,diff --git a/web/src/features/topics/ManualTopicsFormContent.tsx b/web/src/features/topics/ManualTopicsFormContent.tsxindex 2edbf78f54..ff459e54cc 100644--- a/web/src/features/topics/ManualTopicsFormContent.tsx+++ b/web/src/features/topics/ManualTopicsFormContent.tsx@@ -136,10 +136,7 @@ export function ManualTopicsFormContent({const value = settings?.stored[slot];return {model: value?.model ?? "Not configured",- connection:- settings?.connections.find(- (connection) => connection.id === value?.llmApiKeyId,- )?.provider ?? "Project connection",+ connection: value?.provider ?? "Project connection",};};async function submit() {diff --git a/web/src/features/topics/TopicSetupFlowController.tsx b/web/src/features/topics/TopicSetupFlowController.tsxindex 124e5477ba..7d9d0dce77 100644--- a/web/src/features/topics/TopicSetupFlowController.tsx+++ b/web/src/features/topics/TopicSetupFlowController.tsx@@ -157,23 +157,16 @@ function TopicSetupSession({);async function saveQuestions() {- const summary = resolveTopicModelSlot(- draft.summary,- connections,- settings.stored.summary?.llmApiKeyId,- "chat",- );+ const summary = resolveTopicModelSlot(draft.summary, connections, "chat");const embedding = resolveTopicModelSlot(draft.embedding,connections,- settings.stored.embedding?.llmApiKeyId,"embedding",- settings.stored.embedding?.model,+ settings.stored.embedding,);const clustering = resolveTopicModelSlot(draft.clustering,connections,- settings.stored.clustering?.llmApiKeyId,"chat",);if (!summary || !embedding || !clustering)diff --git a/web/src/features/topics/topic-setup-flow.clienttest.ts b/web/src/features/topics/topic-setup-flow.clienttest.tsindex 6696fc2849..a88c784eba 100644--- a/web/src/features/topics/topic-setup-flow.clienttest.ts+++ b/web/src/features/topics/topic-setup-flow.clienttest.ts@@ -84,7 +84,7 @@ it("keeps a stored 10 minute idle time and a stored model", () => {const draft = initialTopicSetupDraft({stored: {summary: {- llmApiKeyId: "bedrock",+ provider: "Amazon Bedrock",model: "eu.anthropic.claude-sonnet-4-5",},embedding: null,diff --git a/web/src/features/topics/topic-setup-flow.ts b/web/src/features/topics/topic-setup-flow.tsindex f6e5074a0a..33c556bc91 100644--- a/web/src/features/topics/topic-setup-flow.ts+++ b/web/src/features/topics/topic-setup-flow.ts@@ -154,47 +154,29 @@ function compareModelVersions(a: string, b: string) {return 0;}-function topicJudgeModel(- slot: { llmApiKeyId: string; model: string } | null,- connections: Array<{ id: string; provider: string }>,-): JudgeModel | null {- if (!slot) return null;- const connection = connections.find((item) => item.id === slot.llmApiKeyId);- return {- provider: connection?.provider ?? "Missing connection",- model: slot.model,- };-}-+/** The slot to save when the project's connection for the provider offers the model; `saved` keeps a stored model selectable. */export function resolveTopicModelSlot(model: JudgeModel | null,connections: TopicSetupConnection[],- preferredId: string | undefined,kind: "chat" | "embedding",- savedModel?: string | null,-) {+ saved?: JudgeModel | null,+): JudgeModel | null {if (!model) return null;- const matches = connections.filter(- (connection) =>- connection.provider === model.provider &&- topicModelsFor(- connection,- kind,- connection.id === preferredId ? savedModel : null,- ).includes(model.model),+ const connection = connections.find(+ (item) => item.provider === model.provider,);- const llmApiKeyId =- matches.find((connection) => connection.id === preferredId)?.id ??- matches[0]?.id ??- null;- return llmApiKeyId ? { llmApiKeyId, model: model.model } : null;+ const savedModel = saved?.provider === model.provider ? saved.model : null;+ return connection &&+ topicModelsFor(connection, kind, savedModel).includes(model.model)+ ? { provider: model.provider, model: model.model }+ : null;}export function initialTopicSetupDraft(input: {stored: {- summary: { llmApiKeyId: string; model: string } | null;- embedding: { llmApiKeyId: string; model: string } | null;- clustering: { llmApiKeyId: string; model: string } | null;+ summary: JudgeModel | null;+ embedding: JudgeModel | null;+ clustering: JudgeModel | null;};connections: TopicSetupConnection[];facets: Array<{@@ -239,13 +221,13 @@ export function initialTopicSetupDraft(input: {};}),summary:- topicJudgeModel(input.stored.summary, input.connections) ??+ input.stored.summary ??pickRecommendedTopicModel("summary", summaryGroups),embedding:- topicJudgeModel(input.stored.embedding, input.connections) ??+ input.stored.embedding ??pickRecommendedTopicModel("embedding", embeddingGroups),clustering:- topicJudgeModel(input.stored.clustering, input.connections) ??+ input.stored.clustering ??pickRecommendedTopicModel("clustering", clusteringGroups),waitMinutes: setupWaitMinutes(input.traceIdleMs),advancedOpen: false,diff --git a/web/src/features/topics/topics-setup.clienttest.ts b/web/src/features/topics/topics-setup.clienttest.tsindex 959ed05198..442351e227 100644--- a/web/src/features/topics/topics-setup.clienttest.ts+++ b/web/src/features/topics/topics-setup.clienttest.ts@@ -4,10 +4,10 @@ import type { ConfigureTopicsSaveDraft } from "./ConfigureTopicsDialog";import { topicsSetupPayload } from "./topics-setup";const slots = {- summary: { llmApiKeyId: "key", model: "gpt" },- embedding: { llmApiKeyId: "key", model: "embed" },+ summary: { provider: "key", model: "gpt" },+ embedding: { provider: "key", model: "embed" },embeddingDimensions: 1024,- clustering: { llmApiKeyId: "key", model: "cluster" },+ clustering: { provider: "key", model: "cluster" },};const draft: ConfigureTopicsSaveDraft = {