Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(topics): resolve LLM connections by provider like evals (#19642)

Lysander Kiesel23622970ae6b
diff --git a/packages/shared/src/server/topics/model-config.test.ts b/packages/shared/src/server/topics/model-config.test.ts
index f628dabc9d..1336638b16 100644
--- a/packages/shared/src/server/topics/model-config.test.ts
+++ b/packages/shared/src/server/topics/model-config.test.ts
@@ -9,6 +9,7 @@ import {
const mocks = vi.hoisted(() => ({
keys: vi.fn(),
+ candidates: vi.fn(),
projects: vi.fn(),
configs: vi.fn(),
current: vi.fn(),
@@ -29,7 +30,7 @@ vi.mock("./processing-gate", () => ({
vi.mock("../../db", () => {
const tx = {
llmApiKeys: { findMany: mocks.keys },
- project: { findMany: mocks.projects },
+ project: { findMany: mocks.projects, findUnique: mocks.candidates },
topicConfig: {
findMany: mocks.configs,
findUnique: mocks.current,
@@ -51,13 +52,13 @@ vi.mock("../../db", () => {
const connection = (id: string, adapter: LLMAdapter) => ({
id,
adapter,
- provider: `${adapter}-connection`,
+ provider: id,
secretKey: "encrypted",
extraHeaders: null,
baseURL: null,
config: null,
});
-const slot = (llmApiKeyId: string, model: string) => ({ llmApiKeyId, model });
+const slot = (provider: string, model: string) => ({ provider, model });
const settings = {
summary: slot("openai", "gpt-6-luna"),
embedding: slot("openai", "text-embedding-3-small"),
@@ -76,10 +77,13 @@ const saveTopicsModelSettings = async (
beforeEach(() => {
vi.resetAllMocks();
- mocks.keys.mockResolvedValue([
- connection("openai", LLMAdapter.OpenAI),
- connection("anthropic", LLMAdapter.Anthropic),
- ]);
+ mocks.candidates.mockResolvedValue({
+ LlmApiKeys: [
+ connection("openai", LLMAdapter.OpenAI),
+ connection("anthropic", LLMAdapter.Anthropic),
+ ],
+ organization: { organizationLlmApiKeys: [] },
+ });
mocks.current.mockResolvedValue(null);
mocks.projects.mockResolvedValue([{ id: "project", orgId: "target-org" }]);
mocks.hasEmbeddings.mockResolvedValue(false);
@@ -97,35 +101,6 @@ beforeEach(() => {
});
describe("prepareTopicsModelSettings", () => {
- it("accepts an organization connection inherited by the project", async () => {
- mocks.keys.mockResolvedValue([
- {
- ...connection("openai", LLMAdapter.OpenAI),
- projectId: null,
- organizationId: "organization",
- },
- ]);
-
- await saveTopicsModelSettings("project", settings);
-
- expect(mocks.keys).toHaveBeenCalledWith({
- where: {
- id: { in: ["openai", "openai", "openai"] },
- OR: [
- { projectId: "project" },
- {
- organization: {
- projects: {
- some: { id: "project", deletedAt: null },
- },
- },
- },
- ],
- },
- });
- expect(mocks.upsert).toHaveBeenCalledOnce();
- });
-
it("validates text slots through the Topics transport before saving", async () => {
await saveTopicsModelSettings("project", {
...settings,
@@ -319,14 +294,15 @@ describe("getEnabledTopicsModels", () => {
projectId: "complete",
project: { orgId: "organization" },
enabled: true,
- summaryLlmApiKey: openai,
+ summaryLlmApiKey: { provider: "openai" },
summaryModel: "gpt-6-luna",
- embeddingLlmApiKey: openai,
+ embeddingLlmApiKey: { provider: "openai" },
embeddingModel: "text-embedding-3-small",
embeddingDimensions: 512,
- clusteringLlmApiKey: openai,
+ clusteringLlmApiKey: { provider: "openai" },
clusteringModel: "gpt-5.6-terra",
};
+ mocks.keys.mockResolvedValue([openai]);
mocks.configs.mockResolvedValue([
row,
// A deleted connection leaves its slot empty.
diff --git a/packages/shared/src/server/topics/model-config.ts b/packages/shared/src/server/topics/model-config.ts
index f71f08049a..e8f2ff772d 100644
--- a/packages/shared/src/server/topics/model-config.ts
+++ b/packages/shared/src/server/topics/model-config.ts
@@ -12,6 +12,7 @@ import {
} from "../llm/llmText";
import { LLMAdapter } from "../llm/types";
import { findCompatibleLlmApiKeysForProjects } from "../services/LLMApiKeys/findCompatibleLlmApiKeysForProjects";
+import { findProjectLlmApiKeyCandidates } from "../services/LLMApiKeys/LlmApiKeyRepository";
import {
TOPICS_MODEL_SLOT_DETAILS,
TOPICS_MODEL_SLOTS,
@@ -41,18 +42,47 @@ export type TopicsModels = {
clustering: TopicsModel;
};
-const withConnections = {
+/** Settings after each provider resolved to the project's effective connection. */
+type ConnectionSlot = { llmApiKeyId: string; model: string } | null;
+type ResolvedModelSettings = {
+ summary: ConnectionSlot;
+ embedding: ConnectionSlot;
+ embeddingDimensions: number;
+ clustering: ConnectionSlot;
+};
+
+type LlmApiKeyCandidates = {
+ projectConnections: LlmApiKeys[];
+ organizationConnections: LlmApiKeys[];
+};
+
+/** Like evals: a project connection overrides its organization's connection of the same provider. */
+function effectiveConnection(
+ candidates: LlmApiKeyCandidates | null,
+ provider: string,
+): LlmApiKeys | null {
+ const byProvider = (key: LlmApiKeys) => key.provider === provider;
+ return (
+ candidates?.projectConnections.find(byProvider) ??
+ candidates?.organizationConnections.find(byProvider) ??
+ null
+ );
+}
+
+const storedProvider = { select: { provider: true } } as const;
+const withStoredProviders = {
project: { select: { orgId: true } },
- summaryLlmApiKey: true,
- embeddingLlmApiKey: true,
- clusteringLlmApiKey: true,
+ summaryLlmApiKey: storedProvider,
+ embeddingLlmApiKey: storedProvider,
+ clusteringLlmApiKey: storedProvider,
} as const;
-type ConfigWithConnections = TopicConfig & {
+type StoredConnection = { provider: string } | null;
+type ConfigWithProviders = TopicConfig & {
project: { orgId: string };
- summaryLlmApiKey: LlmApiKeys | null;
- embeddingLlmApiKey: LlmApiKeys | null;
- clusteringLlmApiKey: LlmApiKeys | null;
+ summaryLlmApiKey: StoredConnection;
+ embeddingLlmApiKey: StoredConnection;
+ clusteringLlmApiKey: StoredConnection;
};
function resolveSlot(
@@ -60,15 +90,8 @@ function resolveSlot(
key: LlmApiKeys | null,
model: string | null,
projectId?: string,
- organizationId?: string,
): TopicsModel | null {
if (!key || !model) return null;
- if (
- projectId &&
- key.projectId !== projectId &&
- key.organizationId !== organizationId
- )
- return null;
return {
slot,
provider: key.provider,
@@ -84,27 +107,32 @@ function resolveSlot(
};
}
-function resolveModels(row: ConfigWithConnections): TopicsModels | null {
- const summary = resolveSlot(
- "summary",
- row.summaryLlmApiKey,
- row.summaryModel,
- row.projectId,
- row.project.orgId,
- );
- const embedding = resolveSlot(
+/** The stored connection only names the provider; it resolves again on every run, like evals. */
+function resolveModels(
+ row: ConfigWithProviders,
+ candidates: LlmApiKeyCandidates,
+): TopicsModels | null {
+ const slot = (
+ name: TopicsModelSlotName,
+ stored: StoredConnection,
+ model: string | null,
+ ) =>
+ resolveSlot(
+ name,
+ stored && effectiveConnection(candidates, stored.provider),
+ model,
+ row.projectId,
+ );
+ const summary = slot("summary", row.summaryLlmApiKey, row.summaryModel);
+ const embedding = slot(
"embedding",
row.embeddingLlmApiKey,
row.embeddingModel,
- row.projectId,
- row.project.orgId,
);
- const clustering = resolveSlot(
+ const clustering = slot(
"clustering",
row.clusteringLlmApiKey,
row.clusteringModel,
- row.projectId,
- row.project.orgId,
);
if (!summary || !embedding || !clustering) return null;
return {
@@ -116,32 +144,64 @@ function resolveModels(row: ConfigWithConnections): TopicsModels | null {
};
}
+/** Like evals, loads only the connections of the stored providers, in one query for all configs. */
+async function resolveConfigs(
+ rows: ConfigWithProviders[],
+): Promise<TopicsModels[]> {
+ const providers = [
+ ...new Set(
+ rows.flatMap((row) =>
+ [row.summaryLlmApiKey, row.embeddingLlmApiKey, row.clusteringLlmApiKey]
+ .filter((stored) => stored !== null)
+ .map(({ provider }) => provider),
+ ),
+ ),
+ ];
+ const keys = providers.length
+ ? await prisma.llmApiKeys.findMany({
+ where: {
+ provider: { in: providers },
+ OR: [
+ { projectId: { in: rows.map((row) => row.projectId) } },
+ { organizationId: { in: rows.map((row) => row.project.orgId) } },
+ ],
+ },
+ })
+ : [];
+ return rows.flatMap((row) => {
+ const resolved = resolveModels(row, {
+ projectConnections: keys.filter((key) => key.projectId === row.projectId),
+ organizationConnections: keys.filter(
+ (key) => key.organizationId === row.project.orgId,
+ ),
+ });
+ return resolved ? [resolved] : [];
+ });
+}
+
/** Returns the configured models of a project, or null while any slot is missing. */
export async function getTopicsModels(
projectId: string,
): Promise<TopicsModels | null> {
const row = await prisma.topicConfig.findUnique({
where: { projectId },
- include: withConnections,
+ include: withStoredProviders,
});
- return row ? resolveModels(row) : null;
+ return row ? ((await resolveConfigs([row]))[0] ?? null) : null;
}
-/** Loads the enabled, fully configured projects of a trace batch in one query. */
+/** Loads the enabled, fully configured projects of a trace batch in two queries. */
export async function getEnabledTopicsModels(
projectIds: string[],
): Promise<Map<string, TopicsModels>> {
if (!projectIds.length) return new Map();
const rows = await prisma.topicConfig.findMany({
where: { projectId: { in: projectIds }, enabled: true },
- include: withConnections,
+ include: withStoredProviders,
});
- const models = new Map<string, TopicsModels>();
- for (const row of rows) {
- const resolved = resolveModels(row);
- if (resolved) models.set(row.projectId, resolved);
- }
- return models;
+ return new Map(
+ (await resolveConfigs(rows)).map((models) => [models.projectId, models]),
+ );
}
/** Turns automatic processing off and records why, for the Topics page. */
@@ -336,18 +396,23 @@ export async function reconcileTopicsConnectionsForProject(
export async function readTopicsModelSettings(projectId: string) {
const row = await prisma.topicConfig.findUnique({
where: { projectId },
+ include: {
+ summaryLlmApiKey: storedProvider,
+ embeddingLlmApiKey: storedProvider,
+ clusteringLlmApiKey: storedProvider,
+ },
});
- const slot = (llmApiKeyId: string | null, model: string | null) =>
- llmApiKeyId && model ? { llmApiKeyId, model } : null;
+ const slot = (connection: StoredConnection, model: string | null) =>
+ connection && model ? { provider: connection.provider, model } : null;
return {
- summary: slot(row?.summaryLlmApiKeyId ?? null, row?.summaryModel ?? null),
+ summary: slot(row?.summaryLlmApiKey ?? null, row?.summaryModel ?? null),
embedding: slot(
- row?.embeddingLlmApiKeyId ?? null,
+ row?.embeddingLlmApiKey ?? null,
row?.embeddingModel ?? null,
),
embeddingDimensions: row?.embeddingDimensions ?? 1024,
clustering: slot(
- row?.clusteringLlmApiKeyId ?? null,
+ row?.clusteringLlmApiKey ?? null,
row?.clusteringModel ?? null,
),
enabled: row ? row.enabled : null,
@@ -357,35 +422,34 @@ export async function readTopicsModelSettings(projectId: string) {
};
}
-async function loadTopicsModelKeys(
+/** Resolves each slot's provider to the project's effective connection, or throws. */
+async function resolveModelSettings(
projectId: string,
settings: TopicsModelSettings,
-) {
- const ids = [
- settings.summary,
- settings.embedding,
- settings.clustering,
- ].flatMap((slot) => (slot ? [slot.llmApiKeyId] : []));
- const keys = await prisma.llmApiKeys.findMany({
- where: {
- id: { in: ids },
- OR: [
- { projectId },
- {
- organization: {
- projects: { some: { id: projectId, deletedAt: null } },
- },
- },
- ],
- },
- });
- for (const id of ids) {
- if (!keys.some((key) => key.id === id))
+): Promise<{ settings: ResolvedModelSettings; keys: LlmApiKeys[] }> {
+ const candidates = await findProjectLlmApiKeyCandidates({ projectId });
+ const keys: LlmApiKeys[] = [];
+ const slot = (
+ requested: TopicsModelSettings[TopicsModelSlotName],
+ ): ConnectionSlot => {
+ if (!requested) return null;
+ const key = effectiveConnection(candidates, requested.provider);
+ if (!key)
throw new InvalidRequestError(
"LLM connection not found in this project.",
);
- }
- return keys;
+ keys.push(key);
+ return { llmApiKeyId: key.id, model: requested.model };
+ };
+ return {
+ settings: {
+ summary: slot(settings.summary),
+ embedding: slot(settings.embedding),
+ embeddingDimensions: settings.embeddingDimensions,
+ clustering: slot(settings.clustering),
+ },
+ keys,
+ };
}
/** Real call per configured slot. The message is the provider error for that row. */
@@ -393,10 +457,8 @@ export async function checkTopicsModelSettings(
projectId: string,
settings: TopicsModelSettings,
): Promise<Partial<Record<TopicsModelSlotName, string>>> {
- return testTopicsModelSlots(
- settings,
- await loadTopicsModelKeys(projectId, settings),
- );
+ const resolved = await resolveModelSettings(projectId, settings);
+ return testTopicsModelSlots(resolved.settings, resolved.keys);
}
/**
@@ -408,7 +470,10 @@ export async function prepareTopicsModelSettings(
projectId: string,
settings: TopicsModelSettings,
) {
- const keys = await loadTopicsModelKeys(projectId, settings);
+ const { settings: resolved, keys } = await resolveModelSettings(
+ projectId,
+ settings,
+ );
const current = await prisma.topicConfig.findUnique({
where: { projectId },
});
@@ -421,7 +486,7 @@ export async function prepareTopicsModelSettings(
"Choose a summary, embedding, and clustering model while automatic processing is on, or turn it off first.",
);
- const slots = resolveSavedSlots(settings, current);
+ const slots = resolveSavedSlots(resolved, current);
const embeddingChanged =
current?.embeddingModel &&
(slots.embedding.model !== current.embeddingModel ||
@@ -433,7 +498,7 @@ export async function prepareTopicsModelSettings(
// Only changed models are tested: the dialog tests each pick, turning
// processing on tests all three, and a broken stored connection pauses Topics.
- await testTopicsModels(changedSlots(settings, current), keys);
+ await testTopicsModels(changedSlots(resolved, current), keys);
const data = {
summaryLlmApiKeyId: slots.summary.llmApiKeyId,
@@ -457,7 +522,7 @@ type StoredTopicConfig = Awaited<
* deleted, so saving never discards a model.
*/
function resolveSavedSlots(
- settings: TopicsModelSettings,
+ settings: ResolvedModelSettings,
current: StoredTopicConfig,
) {
const keep = current && !current.enabled ? current : null;
@@ -490,11 +555,11 @@ function resolveSavedSlots(
}
function changedSlots(
- settings: TopicsModelSettings,
+ settings: ResolvedModelSettings,
current: StoredTopicConfig,
-): TopicsModelSettings {
+): ResolvedModelSettings {
const changed = (
- slot: TopicsModelSettings[TopicsModelSlotName],
+ slot: ConnectionSlot,
llmApiKeyId: string | null | undefined,
model: string | null | undefined,
extraChange = false,
@@ -531,24 +596,15 @@ function changedSlots(
export async function checkStoredTopicsModels(
config: NonNullable<StoredTopicConfig>,
): Promise<void> {
- const slot = (llmApiKeyId: string | null, model: string | null) =>
- llmApiKeyId && model ? { llmApiKeyId, model } : null;
- const settings = {
- summary: slot(config.summaryLlmApiKeyId, config.summaryModel),
- embedding: slot(config.embeddingLlmApiKeyId, config.embeddingModel),
- embeddingDimensions: config.embeddingDimensions,
- clustering: slot(config.clusteringLlmApiKeyId, config.clusteringModel),
- };
- if (!settings.summary || !settings.embedding || !settings.clustering)
+ const stored = await readTopicsModelSettings(config.projectId);
+ if (!stored.summary || !stored.embedding || !stored.clustering)
throw new InvalidRequestError(
"Choose a summary, embedding, and clustering model in Configure Topics before turning on automatic processing.",
);
// Saved models were tested on save; a pause means a connection failed since.
if (!config.blockedAt) return;
- await testTopicsModels(
- settings,
- await loadTopicsModelKeys(config.projectId, settings),
- );
+ const resolved = await resolveModelSettings(config.projectId, stored);
+ await testTopicsModels(resolved.settings, resolved.keys);
}
export async function writeTopicsModelSettings(
@@ -580,7 +636,7 @@ const TEST_SCHEMAS = {
* length must match the configured dimensions. A slot with no error is omitted.
*/
async function testTopicsModelSlots(
- settings: TopicsModelSettings,
+ settings: ResolvedModelSettings,
keys: LlmApiKeys[],
): Promise<Partial<Record<TopicsModelSlotName, string>>> {
const errors: Partial<Record<TopicsModelSlotName, string>> = {};
@@ -648,7 +704,7 @@ async function testTopicsModelSlots(
}
async function testTopicsModels(
- settings: TopicsModelSettings,
+ settings: ResolvedModelSettings,
keys: LlmApiKeys[],
): Promise<void> {
const errors = await testTopicsModelSlots(settings, keys);
diff --git a/packages/shared/src/server/topics/setup.test.ts b/packages/shared/src/server/topics/setup.test.ts
index bd3c90c382..e40b18c30b 100644
--- a/packages/shared/src/server/topics/setup.test.ts
+++ b/packages/shared/src/server/topics/setup.test.ts
@@ -44,10 +44,10 @@ const facets = DEFAULT_TOPIC_FACETS.map((facet, index) => ({
}));
const setup = {
- summary: { llmApiKeyId: "key", model: "gpt" },
- embedding: { llmApiKeyId: "key", model: "embed" },
+ summary: { provider: "key", model: "gpt" },
+ embedding: { provider: "key", model: "embed" },
embeddingDimensions: 1024,
- clustering: { llmApiKeyId: "key", model: "cluster" },
+ clustering: { provider: "key", model: "cluster" },
facets: facets.map((facet) => ({
name: facet.name,
enabled: facet.enabled,
diff --git a/packages/shared/src/topics/index.ts b/packages/shared/src/topics/index.ts
index fd03cae3bf..1fc78084aa 100644
--- a/packages/shared/src/topics/index.ts
+++ b/packages/shared/src/topics/index.ts
@@ -59,8 +59,9 @@ export const TOPICS_MODEL_SLOT_DETAILS: Record<
"Names and describes each topic when topics are updated. Use a stronger model such as GPT-5.6 Terra, or Claude Sonnet through Amazon Bedrock.",
},
};
+// Like evals: the provider name picks the project's connection, else its organization's.
const topicsModelSlotSchema = z.object({
- llmApiKeyId: z.string().min(1),
+ provider: z.string().min(1),
model: z.string().trim().min(1).max(256),
});
export const topicsModelSettingsSchema = z.object({
diff --git a/web/src/__tests__/server/topics-setup.servertest.ts b/web/src/__tests__/server/topics-setup.servertest.ts
index bab2a845d9..f5256da795 100644
--- a/web/src/__tests__/server/topics-setup.servertest.ts
+++ b/web/src/__tests__/server/topics-setup.servertest.ts
@@ -47,7 +47,7 @@ async function topicsProject(
displaySecretKey: "...key",
},
});
- const slot = (model: string) => ({ llmApiKeyId: connection.id, model });
+ const slot = (model: string) => ({ provider: connection.provider, model });
await prisma.topicConfig.create({
data: {
projectId,
diff --git a/web/src/__tests__/server/topics-trpc-access.servertest.ts b/web/src/__tests__/server/topics-trpc-access.servertest.ts
index 9acd4b3f4e..be1bd8ecb7 100644
--- a/web/src/__tests__/server/topics-trpc-access.servertest.ts
+++ b/web/src/__tests__/server/topics-trpc-access.servertest.ts
@@ -1,6 +1,6 @@
import { randomUUID } from "node:crypto";
import type { Session } from "next-auth";
-import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { LLMAdapter } from "@langfuse/shared";
import { env as sharedEnv } from "@langfuse/shared/src/env";
import { prisma } from "@langfuse/shared/src/db";
@@ -16,6 +16,17 @@ import {
createTopicRun,
getTopicsModels,
} from "@langfuse/shared/topics/server";
+// Only the LLM transport is faked, to see which connection a test call used.
+const generateTopicText = vi.hoisted(() =>
+ vi.fn(async () => ({
+ output: { summary: "Account access", status: "applicable" },
+ usage: {},
+ })),
+);
+vi.mock("@langfuse/shared/src/server/topics/text", () => ({
+ generateTopicText,
+}));
+
import { testFeatureFlags } from "@/src/__tests__/fixtures/feature-flags";
import { appRouter } from "@/src/server/api/root";
import { createInnerTRPCContext } from "@/src/server/api/trpc";
@@ -44,11 +55,13 @@ function caller(userId: string, memberships: [string, string, Role][]) {
function llmConnection(
owner: { projectId: string } | { organizationId: string },
+ provider = `openai-${randomUUID()}`,
) {
return prisma.llmApiKeys.create({
data: {
...owner,
- provider: `openai-${randomUUID()}`,
+ provider,
+ baseURL: `https://${randomUUID()}.example.com`,
adapter: LLMAdapter.OpenAI,
secretKey: "encrypted",
displaySecretKey: "...key",
@@ -250,33 +263,42 @@ describe("Topics tRPC access", () => {
).toBe(0);
});
- it("uses only LLM connections of the project or its organization", async () => {
+ it("resolves LLM connections by provider within the project, then its organization", async () => {
const admin = caller(userId, [
[a.orgId, a.projectId, "ADMIN"],
[b.orgId, b.projectId, "ADMIN"],
]);
const projectId = a.projectId;
- const ownOrgKey = await llmConnection({ organizationId: a.orgId });
+ const shared = `openai-${randomUUID()}`;
+ const overriddenOrgKey = await llmConnection(
+ { organizationId: a.orgId },
+ shared,
+ );
+ const overridingProjectKey = await llmConnection({ projectId }, shared);
+ const inheritedOrgKey = await llmConnection({ organizationId: a.orgId });
const otherOrgKey = await llmConnection({ organizationId: b.orgId });
+ const otherProjectKey = await prisma.llmApiKeys.findUniqueOrThrow({
+ where: { id: b.llmKeyId },
+ });
const config = await prisma.topicConfig.findUniqueOrThrow({
where: { projectId },
});
- const settings = (llmApiKeyId: string) => ({
+ const settings = (provider: string) => ({
projectId,
- summary: { llmApiKeyId, model: "summary-model" },
+ summary: { provider, model: "summary-model" },
embedding: null,
embeddingDimensions: 1024,
clustering: null,
});
- // Saving or testing: another project's or another org's connection is refused.
- for (const foreignKeyId of [b.llmKeyId, otherOrgKey.id]) {
- await expect(
- admin.testModelSettings(settings(foreignKeyId)),
- ).rejects.toThrow("LLM connection not found in this project.");
+ // Saving or testing: providers only another project or org has are refused.
+ for (const { provider } of [otherProjectKey, otherOrgKey]) {
+ await expect(admin.testModelSettings(settings(provider))).rejects.toThrow(
+ "LLM connection not found in this project.",
+ );
await expect(
admin.saveSetup({
- ...settings(foreignKeyId),
+ ...settings(provider),
facets: DEFAULT_TOPIC_FACETS.map(({ name }) => ({
name,
enabled: true,
@@ -290,17 +312,34 @@ describe("Topics tRPC access", () => {
await prisma.topicConfig.findUniqueOrThrow({ where: { projectId } }),
).toEqual(config);
- // Running: a stored foreign connection resolves to no models; the own org's does.
- const resolveWithSummaryKey = async (summaryLlmApiKeyId: string) => {
+ // The project's connection overrides its organization's for the same provider.
+ await admin.testModelSettings(settings(shared));
+ expect(generateTopicText).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ model: expect.objectContaining({
+ connection: expect.objectContaining({
+ baseURL: overridingProjectKey.baseURL,
+ }),
+ }),
+ }),
+ );
+
+ // Running: the stored connection's provider resolves again in this project.
+ const summaryBaseUrl = async (summaryLlmApiKeyId: string) => {
await prisma.topicConfig.update({
where: { projectId },
data: { summaryLlmApiKeyId },
});
- return getTopicsModels(projectId);
+ return (await getTopicsModels(projectId))?.summary.connection.baseURL;
};
- expect(await resolveWithSummaryKey(ownOrgKey.id)).not.toBeNull();
- expect(await resolveWithSummaryKey(otherOrgKey.id)).toBeNull();
- expect(await resolveWithSummaryKey(b.llmKeyId)).toBeNull();
+ expect(await summaryBaseUrl(overriddenOrgKey.id)).toBe(
+ overridingProjectKey.baseURL,
+ );
+ expect(await summaryBaseUrl(inheritedOrgKey.id)).toBe(
+ inheritedOrgKey.baseURL,
+ );
+ expect(await summaryBaseUrl(otherOrgKey.id)).toBeUndefined();
+ expect(await summaryBaseUrl(otherProjectKey.id)).toBeUndefined();
await prisma.topicConfig.update({
where: { projectId },
data: { summaryLlmApiKeyId: config.summaryLlmApiKeyId },
diff --git a/web/src/__tests__/server/unit/topics.servertest.ts b/web/src/__tests__/server/unit/topics.servertest.ts
index 1ad0b23f34..3490b08a50 100644
--- a/web/src/__tests__/server/unit/topics.servertest.ts
+++ b/web/src/__tests__/server/unit/topics.servertest.ts
@@ -1459,7 +1459,7 @@ describe("Topics model settings", () => {
const settings = {
projectId,
summary: null,
- embedding: { llmApiKeyId: "key", model: "text-embedding-3-small" },
+ embedding: { provider: "key", model: "text-embedding-3-small" },
embeddingDimensions: 1024,
clustering: null,
};
@@ -1492,10 +1492,10 @@ describe("Topics model settings", () => {
describe("Topics setup", () => {
const setup = {
projectId,
- summary: { llmApiKeyId: "key", model: "gpt" },
- embedding: { llmApiKeyId: "key", model: "embed" },
+ summary: { provider: "key", model: "gpt" },
+ embedding: { provider: "key", model: "embed" },
embeddingDimensions: 1024,
- clustering: { llmApiKeyId: "key", model: "cluster" },
+ clustering: { provider: "key", model: "cluster" },
facets: DEFAULT_TOPIC_FACETS.map((facet) => ({
name: facet.name,
enabled: facet.name !== "Sentiment",
@@ -1621,7 +1621,7 @@ describe("Topics write access", () => {
description: "",
prompt: "Describe the original request.",
};
- const slot = { llmApiKeyId: "key", model: "gpt" };
+ const slot = { provider: "key", model: "gpt" };
const settings = {
summary: slot,
embedding: slot,
diff --git a/web/src/features/topics/ConfigureTopicsDialogController.tsx b/web/src/features/topics/ConfigureTopicsDialogController.tsx
index 9173c3605a..0d8bf5855e 100644
--- a/web/src/features/topics/ConfigureTopicsDialogController.tsx
+++ b/web/src/features/topics/ConfigureTopicsDialogController.tsx
@@ -22,9 +22,9 @@ import {
} from "./ConfigureTopicsDialog";
import { type ConfigureTopicsSettingsInput } from "./TopicModelSettings";
import { topicsSetupPayload } from "./topics-setup";
+import { resolveTopicModelSlot } from "./topic-setup-flow";
type Connection = RouterOutputs["llmApiKey"]["effective"]["data"][number];
-type SlotValue = { llmApiKeyId: string; model: string } | null;
type ConfigureTopicsDialogControllerProps = {
projectId: string;
@@ -99,16 +99,16 @@ function ConfigureTopicsSettings({
modelsFor(
connection,
"embedding",
- stored.embedding?.llmApiKeyId === connection.id
+ stored.embedding?.provider === connection.provider
? stored.embedding.model
: null,
),
]);
const draft = {
- summary: judgeModel(stored.summary, connections),
- embedding: judgeModel(stored.embedding, connections),
+ summary: stored.summary,
+ embedding: stored.embedding,
embeddingDimensions: String(stored.embeddingDimensions),
- clustering: judgeModel(stored.clustering, connections),
+ clustering: stored.clustering,
facets: facetDrafts(facets, rule ? new Set(rule.facetIds) : null),
filters: [],
idleSeconds: String((traceIdleMs ?? 600_000) / 1000),
@@ -146,19 +146,13 @@ function ConfigureTopicsSettings({
}));
return;
}
- const preferred = {
- summary: stored.summary?.llmApiKeyId,
- embedding: stored.embedding?.llmApiKeyId,
- clustering: stored.clustering?.llmApiKeyId,
- }[slot];
- const llmApiKeyId = connectionId(
+ const value = resolveTopicModelSlot(
model,
connections,
- preferred,
slot === "embedding" ? "embedding" : "chat",
- slot === "embedding" ? stored.embedding?.model : null,
+ slot === "embedding" ? stored.embedding : null,
);
- if (!llmApiKeyId) {
+ if (!value) {
setChecks((current) => ({
...current,
[slot]: {
@@ -170,7 +164,6 @@ function ConfigureTopicsSettings({
}
const request = ++requests.current[slot];
setChecks((current) => ({ ...current, [slot]: { status: "testing" } }));
- const value = { llmApiKeyId, model: model.model };
test
.mutateAsync({
projectId,
@@ -213,23 +206,16 @@ function ConfigureTopicsSettings({
parsedDimensions.error.issues[0]?.message ??
"Check the embedding dimensions.",
);
- const summary = slotValue(
- value.summary,
- connections,
- stored.summary?.llmApiKeyId,
- "chat",
- );
- const embedding = slotValue(
+ const summary = resolveTopicModelSlot(value.summary, connections, "chat");
+ const embedding = resolveTopicModelSlot(
value.embedding,
connections,
- stored.embedding?.llmApiKeyId,
"embedding",
- stored.embedding?.model,
+ stored.embedding,
);
- const clustering = slotValue(
+ const clustering = resolveTopicModelSlot(
value.clustering,
connections,
- stored.clustering?.llmApiKeyId,
"chat",
);
if (stored.enabled && (!summary || !embedding || !clustering))
@@ -332,59 +318,6 @@ function modelsFor(
);
}
-function judgeModel(
- slot: SlotValue,
- connections: Connection[],
-): JudgeModel | null {
- if (!slot) return null;
- const connection = connections.find((item) => item.id === slot.llmApiKeyId);
- return {
- provider: connection?.provider ?? "Missing connection",
- model: slot.model,
- };
-}
-
-function connectionId(
- model: JudgeModel,
- connections: Connection[],
- preferredId: string | undefined,
- kind: "chat" | "embedding",
- savedModel?: string | null,
-) {
- const matches = connections.filter(
- (connection) =>
- connection.provider === model.provider &&
- modelsFor(
- connection,
- kind,
- connection.id === preferredId ? savedModel : null,
- ).includes(model.model),
- );
- return (
- matches.find((connection) => connection.id === preferredId)?.id ??
- matches[0]?.id ??
- null
- );
-}
-
-function slotValue(
- model: JudgeModel | null,
- connections: Connection[],
- preferredId: string | undefined,
- kind: "chat" | "embedding",
- savedModel?: string | null,
-) {
- if (!model) return null;
- const llmApiKeyId = connectionId(
- model,
- connections,
- preferredId,
- kind,
- savedModel,
- );
- return llmApiKeyId ? { llmApiKeyId, model: model.model } : null;
-}
-
function facetDrafts(
facets: TopicFacet[],
enabledIds: ReadonlySet<string> | null,
diff --git a/web/src/features/topics/ManualTopicsFormContent.tsx b/web/src/features/topics/ManualTopicsFormContent.tsx
index 2edbf78f54..ff459e54cc 100644
--- a/web/src/features/topics/ManualTopicsFormContent.tsx
+++ b/web/src/features/topics/ManualTopicsFormContent.tsx
@@ -136,10 +136,7 @@ export function ManualTopicsFormContent({
const value = settings?.stored[slot];
return {
model: value?.model ?? "Not configured",
- connection:
- settings?.connections.find(
- (connection) => connection.id === value?.llmApiKeyId,
- )?.provider ?? "Project connection",
+ connection: value?.provider ?? "Project connection",
};
};
async function submit() {
diff --git a/web/src/features/topics/TopicSetupFlowController.tsx b/web/src/features/topics/TopicSetupFlowController.tsx
index 124e5477ba..7d9d0dce77 100644
--- a/web/src/features/topics/TopicSetupFlowController.tsx
+++ b/web/src/features/topics/TopicSetupFlowController.tsx
@@ -157,23 +157,16 @@ function TopicSetupSession({
);
async function saveQuestions() {
- const summary = resolveTopicModelSlot(
- draft.summary,
- connections,
- settings.stored.summary?.llmApiKeyId,
- "chat",
- );
+ const summary = resolveTopicModelSlot(draft.summary, connections, "chat");
const embedding = resolveTopicModelSlot(
draft.embedding,
connections,
- settings.stored.embedding?.llmApiKeyId,
"embedding",
- settings.stored.embedding?.model,
+ settings.stored.embedding,
);
const clustering = resolveTopicModelSlot(
draft.clustering,
connections,
- settings.stored.clustering?.llmApiKeyId,
"chat",
);
if (!summary || !embedding || !clustering)
diff --git a/web/src/features/topics/topic-setup-flow.clienttest.ts b/web/src/features/topics/topic-setup-flow.clienttest.ts
index 6696fc2849..a88c784eba 100644
--- a/web/src/features/topics/topic-setup-flow.clienttest.ts
+++ b/web/src/features/topics/topic-setup-flow.clienttest.ts
@@ -84,7 +84,7 @@ it("keeps a stored 10 minute idle time and a stored model", () => {
const draft = initialTopicSetupDraft({
stored: {
summary: {
- llmApiKeyId: "bedrock",
+ provider: "Amazon Bedrock",
model: "eu.anthropic.claude-sonnet-4-5",
},
embedding: null,
diff --git a/web/src/features/topics/topic-setup-flow.ts b/web/src/features/topics/topic-setup-flow.ts
index f6e5074a0a..33c556bc91 100644
--- a/web/src/features/topics/topic-setup-flow.ts
+++ b/web/src/features/topics/topic-setup-flow.ts
@@ -154,47 +154,29 @@ function compareModelVersions(a: string, b: string) {
return 0;
}
-function topicJudgeModel(
- slot: { llmApiKeyId: string; model: string } | null,
- connections: Array<{ id: string; provider: string }>,
-): JudgeModel | null {
- if (!slot) return null;
- const connection = connections.find((item) => item.id === slot.llmApiKeyId);
- return {
- provider: connection?.provider ?? "Missing connection",
- model: slot.model,
- };
-}
-
+/** The slot to save when the project's connection for the provider offers the model; `saved` keeps a stored model selectable. */
export function resolveTopicModelSlot(
model: JudgeModel | null,
connections: TopicSetupConnection[],
- preferredId: string | undefined,
kind: "chat" | "embedding",
- savedModel?: string | null,
-) {
+ saved?: JudgeModel | null,
+): JudgeModel | null {
if (!model) return null;
- const matches = connections.filter(
- (connection) =>
- connection.provider === model.provider &&
- topicModelsFor(
- connection,
- kind,
- connection.id === preferredId ? savedModel : null,
- ).includes(model.model),
+ const connection = connections.find(
+ (item) => item.provider === model.provider,
);
- const llmApiKeyId =
- matches.find((connection) => connection.id === preferredId)?.id ??
- matches[0]?.id ??
- null;
- return llmApiKeyId ? { llmApiKeyId, model: model.model } : null;
+ const savedModel = saved?.provider === model.provider ? saved.model : null;
+ return connection &&
+ topicModelsFor(connection, kind, savedModel).includes(model.model)
+ ? { provider: model.provider, model: model.model }
+ : null;
}
export function initialTopicSetupDraft(input: {
stored: {
- summary: { llmApiKeyId: string; model: string } | null;
- embedding: { llmApiKeyId: string; model: string } | null;
- clustering: { llmApiKeyId: string; model: string } | null;
+ summary: JudgeModel | null;
+ embedding: JudgeModel | null;
+ clustering: JudgeModel | null;
};
connections: TopicSetupConnection[];
facets: Array<{
@@ -239,13 +221,13 @@ export function initialTopicSetupDraft(input: {
};
}),
summary:
- topicJudgeModel(input.stored.summary, input.connections) ??
+ input.stored.summary ??
pickRecommendedTopicModel("summary", summaryGroups),
embedding:
- topicJudgeModel(input.stored.embedding, input.connections) ??
+ input.stored.embedding ??
pickRecommendedTopicModel("embedding", embeddingGroups),
clustering:
- topicJudgeModel(input.stored.clustering, input.connections) ??
+ input.stored.clustering ??
pickRecommendedTopicModel("clustering", clusteringGroups),
waitMinutes: setupWaitMinutes(input.traceIdleMs),
advancedOpen: false,
diff --git a/web/src/features/topics/topics-setup.clienttest.ts b/web/src/features/topics/topics-setup.clienttest.ts
index 959ed05198..442351e227 100644
--- a/web/src/features/topics/topics-setup.clienttest.ts
+++ b/web/src/features/topics/topics-setup.clienttest.ts
@@ -4,10 +4,10 @@ import type { ConfigureTopicsSaveDraft } from "./ConfigureTopicsDialog";
import { topicsSetupPayload } from "./topics-setup";
const slots = {
- summary: { llmApiKeyId: "key", model: "gpt" },
- embedding: { llmApiKeyId: "key", model: "embed" },
+ summary: { provider: "key", model: "gpt" },
+ embedding: { provider: "key", model: "embed" },
embeddingDimensions: 1024,
- clustering: { llmApiKeyId: "key", model: "cluster" },
+ clustering: { provider: "key", model: "cluster" },
};
const draft: ConfigureTopicsSaveDraft = {