Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(topics): configure Topics on the project's own LLM connections (#18414)

Lysander Kiesel93217ac3c5da
diff --git a/.env.dev.example b/.env.dev.example
index fc1bf5937a..72ab869c2d 100644
--- a/.env.dev.example
+++ b/.env.dev.example
@@ -77,14 +77,7 @@ NEXTAUTH_SECRET="secret"
# LANGFUSE_TOPICS_ENABLED_PROJECT_IDS=project-a,project-b
# Topics summaries awaiting embedding expire from Redis after 3 hours.
# LANGFUSE_TOPICS_REDIS_TTL_SECONDS=10800
-# Internal Topics PoC only: both model IDs are required and have no defaults.
-# Set on web and worker to Bedrock model IDs; leave unset to disable model processing.
-# LANGFUSE_TOPICS_SUMMARY_MODEL=
-# LANGFUSE_TOPICS_EMBEDDING_MODEL=
-# All Topics calls use LANGFUSE_AI_AWS_BEDROCK_REGION (e.g. eu-west-1).
-# Topic naming currently uses us.openai.gpt-5.6-terra.
-# For Cohere Embed v4, configure eu.cohere.embed-v4:0 or us.cohere.embed-v4:0.
-# Uses the shared LANGFUSE_AI_FEATURES_AWS_PROFILE configured below.
+# Topics models come from each project's LLM connections (Topics page > Models).
# Internal PoC integration: trace-batch settings below feed automatic Topics processing.
# Jobs are discarded unless the cloud region above is set. Leave the four enablement flags commented until you opt in.
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.md
index d9f4dbaec9..9cfdd34018 100644
--- a/packages/shared/AGENTS.md
+++ b/packages/shared/AGENTS.md
@@ -90,8 +90,16 @@
payloads through assignment and save terminal job state before cleanup.
Summary references carry facet/version/source fields within project and
execution scope; exact storage reads also require facet/version scope.
- - `text.ts` and `embeddings.ts`: Bedrock model transport using the shared AI SDK;
- worker model calls own usage, cost and vector validation.
+ - `text.ts` and `embeddings.ts`: model transport through the project's LLM
+ connections (`generateLLMText` / `embedLLMText`); worker model calls own
+ usage, cost and vector validation.
+ - `model-config.ts`: per-project `topic_configs` (three connection +
+ model slots and the automatic-processing switch). Saving tests every slot
+ with a real call; Anthropic works for summaries and clustering but fails the
+ embedding test. The embedding model and dimensions are locked once embeddings
+ exist. `setup.ts` tests the models, then commits the single Topics rule,
+ models, and per-project trace idle time together. Refresh the Redis readiness
+ setting only after that transaction commits.
- `loadTopicTranscript`: shared in-memory source assembly for worker and inspector.
Returns the shared `Transcript | null`, capped at 10,000 serialized characters.
Historical reuse must match `TOPICS_TRANSCRIPT_VERSION`; accepted Redis results
@@ -100,9 +108,6 @@
including cleanup. Processing also requires
`LANGFUSE_TOPICS_ENABLED_PROJECT_IDS` (empty by default); reads/configuration
remain feature-flag/RBAC controlled.
- The internal Topics PoC model IDs come from the default-free
- `LANGFUSE_TOPICS_SUMMARY_MODEL` and `LANGFUSE_TOPICS_EMBEDDING_MODEL` env vars;
- keep them out of the production env template.
- `src/server/transcript`: order minimal `TranscriptObservation` inputs with
`orderObservations`, then pass the enriched result and optional
@@ -120,8 +125,8 @@
- `@langfuse/shared/src/server` via `src/server/index.ts`: server-only barrel
for shared backend services, repositories, queue helpers/contracts, Redis and
ClickHouse helpers, auth helpers, logger/instrumentation, ingestion helpers,
- AI SDK-native LLM execution helpers (`generateLLMText` and
- `streamLLMText`), Bedrock default-credential provider auth
+ AI SDK-native LLM execution helpers (`generateLLMText`, `streamLLMText`, and
+ `embedLLMText`), Bedrock default-credential provider auth
(`createDefaultBedrockProviderAuth`), and server test utilities. Langfuse AI
callers select their local profile through `getLangfuseAIAwsProfile`;
generic Bedrock auth only uses an explicitly supplied profile or the normal
diff --git a/packages/shared/prisma/migrations/20261007142940_add_topic_configs/migration.sql b/packages/shared/prisma/migrations/20261007142940_add_topic_configs/migration.sql
new file mode 100644
index 0000000000..eb9a57eca8
--- /dev/null
+++ b/packages/shared/prisma/migrations/20261007142940_add_topic_configs/migration.sql
@@ -0,0 +1,53 @@
+-- CreateTable
+CREATE TABLE "topic_configs" (
+ "id" TEXT NOT NULL,
+ "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ "created_by" TEXT,
+ "updated_by" TEXT,
+ "project_id" TEXT NOT NULL,
+ "enabled" BOOLEAN NOT NULL DEFAULT false,
+ -- Same columns as Evaluator, set when a configured connection is no longer valid.
+ "blocked_at" TIMESTAMP(3),
+ "block_reason" "EvaluatorBlockReason",
+ "block_message" TEXT,
+ "summary_llm_api_key_id" TEXT,
+ "summary_model" TEXT,
+ "embedding_llm_api_key_id" TEXT,
+ "embedding_model" TEXT,
+ "embedding_dimensions" INTEGER NOT NULL DEFAULT 1024,
+ "clustering_llm_api_key_id" TEXT,
+ "clustering_model" TEXT,
+
+ CONSTRAINT "topic_configs_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "topic_configs_project_id_key" ON "topic_configs"("project_id");
+
+-- CreateIndex
+CREATE INDEX "topic_configs_summary_llm_api_key_id_idx" ON "topic_configs"("summary_llm_api_key_id");
+
+-- CreateIndex
+CREATE INDEX "topic_configs_embedding_llm_api_key_id_idx" ON "topic_configs"("embedding_llm_api_key_id");
+
+-- CreateIndex
+CREATE INDEX "topic_configs_clustering_llm_api_key_id_idx" ON "topic_configs"("clustering_llm_api_key_id");
+
+-- AddForeignKey
+ALTER TABLE "topic_configs" ADD CONSTRAINT "topic_configs_created_by_fkey" FOREIGN KEY ("created_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "topic_configs" ADD CONSTRAINT "topic_configs_updated_by_fkey" FOREIGN KEY ("updated_by") REFERENCES "users"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "topic_configs" ADD CONSTRAINT "topic_configs_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "topic_configs" ADD CONSTRAINT "topic_configs_summary_llm_api_key_id_fkey" FOREIGN KEY ("summary_llm_api_key_id") REFERENCES "llm_api_keys"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "topic_configs" ADD CONSTRAINT "topic_configs_embedding_llm_api_key_id_fkey" FOREIGN KEY ("embedding_llm_api_key_id") REFERENCES "llm_api_keys"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "topic_configs" ADD CONSTRAINT "topic_configs_clustering_llm_api_key_id_fkey" FOREIGN KEY ("clustering_llm_api_key_id") REFERENCES "llm_api_keys"("id") ON DELETE SET NULL ON UPDATE CASCADE;
diff --git a/packages/shared/prisma/schema.prisma b/packages/shared/prisma/schema.prisma
index 21ecab1c93..03d5e99236 100644
--- a/packages/shared/prisma/schema.prisma
+++ b/packages/shared/prisma/schema.prisma
@@ -77,6 +77,8 @@ model User {
tableViewPresetUpdated TableViewPreset[] @relation("UpdatedByUser")
monitorsCreated Monitor[] @relation("CreatedByUser")
monitorsUpdated Monitor[] @relation("UpdatedByUser")
+ topicConfigsCreated TopicConfig[] @relation("CreatedByUser")
+ topicConfigsUpdated TopicConfig[] @relation("UpdatedByUser")
tracingReadinessCreated TracingReadinessSettings[] @relation("CreatedByUser")
tracingReadinessUpdated TracingReadinessSettings[] @relation("UpdatedByUser")
annotationQueueAssignment AnnotationQueueAssignment[]
@@ -201,6 +203,7 @@ model Project {
Automation Automation[]
monitors Monitor[]
DefaultLlmModel DefaultLlmModel[]
+ TopicConfig TopicConfig?
Price Price[]
SlackIntegration SlackIntegration?
PendingDeletion PendingDeletion[]
@@ -505,6 +508,9 @@ model LlmApiKeys {
projectId String @map("project_id")
project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
DefaultLlmModel DefaultLlmModel[] @relation("LlmApiKeyId")
+ topicSummaryConfigs TopicConfig[] @relation("TopicConfigSummaryLlmApiKey")
+ topicEmbeddingConfigs TopicConfig[] @relation("TopicConfigEmbeddingLlmApiKey")
+ topicClusteringConfigs TopicConfig[] @relation("TopicConfigClusteringLlmApiKey")
@@unique([projectId, provider])
@@map("llm_api_keys")
@@ -1363,6 +1369,48 @@ model DefaultLlmModel {
@@map("default_llm_models")
}
+// Per-project Topics configuration: the switch, pause state, and the three model
+// slots, each referencing one of the project's LLM connections.
+model TopicConfig {
+ id String @id @default(cuid())
+ createdAt DateTime @default(now()) @map("created_at")
+ updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+ createdBy String? @map("created_by")
+ createdByUser User? @relation("CreatedByUser", fields: [createdBy], references: [id], onDelete: SetNull)
+ updatedBy String? @map("updated_by")
+ updatedByUser User? @relation("UpdatedByUser", fields: [updatedBy], references: [id], onDelete: SetNull)
+
+ projectId String @unique @map("project_id")
+ project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+ // Kill switch for automatic trace processing; facet rules are kept while off.
+ enabled Boolean @default(false)
+ // Same columns as Evaluator, set when a configured connection is no longer valid.
+ blockedAt DateTime? @map("blocked_at")
+ blockReason EvaluatorBlockReason? @map("block_reason")
+ blockMessage String? @map("block_message")
+
+ summaryLlmApiKeyId String? @map("summary_llm_api_key_id")
+ summaryLlmApiKey LlmApiKeys? @relation("TopicConfigSummaryLlmApiKey", fields: [summaryLlmApiKeyId], references: [id], onDelete: SetNull)
+ summaryModel String? @map("summary_model")
+
+ embeddingLlmApiKeyId String? @map("embedding_llm_api_key_id")
+ embeddingLlmApiKey LlmApiKeys? @relation("TopicConfigEmbeddingLlmApiKey", fields: [embeddingLlmApiKeyId], references: [id], onDelete: SetNull)
+ embeddingModel String? @map("embedding_model")
+ embeddingDimensions Int @default(1024) @map("embedding_dimensions")
+
+ // Names and describes each cluster during Update topics.
+ clusteringLlmApiKeyId String? @map("clustering_llm_api_key_id")
+ clusteringLlmApiKey LlmApiKeys? @relation("TopicConfigClusteringLlmApiKey", fields: [clusteringLlmApiKeyId], references: [id], onDelete: SetNull)
+ clusteringModel String? @map("clustering_model")
+
+ @@index([summaryLlmApiKeyId])
+ @@index([embeddingLlmApiKeyId])
+ @@index([clusteringLlmApiKeyId])
+ @@map("topic_configs")
+}
+
// Single Sign-On configuration for a domain
// This feature is part of the Enterprise Edition
model SsoConfig {
diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.ts
index 0664fddf5e..41d73a2ba0 100644
--- a/packages/shared/src/env.ts
+++ b/packages/shared/src/env.ts
@@ -44,9 +44,6 @@ const EnvSchema = z.object({
NEXTAUTH_URL: z.url().optional(),
// Internal Topics PoC controls; not a supported self-hosting configuration.
LANGFUSE_TOPICS_ENABLED: z.enum(["true", "false"]).default("false"),
- // Unset model IDs make Topics model processing unavailable.
- LANGFUSE_TOPICS_SUMMARY_MODEL: z.string().trim().min(1).optional(),
- LANGFUSE_TOPICS_EMBEDDING_MODEL: z.string().trim().min(1).optional(),
LANGFUSE_TOPICS_ENABLED_PROJECT_IDS: z
.string()
.default("")
diff --git a/packages/shared/src/server/llm/ai-sdk/providers/azure.ts b/packages/shared/src/server/llm/ai-sdk/providers/azure.ts
index ebdba123fc..5fde858d24 100644
--- a/packages/shared/src/server/llm/ai-sdk/providers/azure.ts
+++ b/packages/shared/src/server/llm/ai-sdk/providers/azure.ts
@@ -1,5 +1,5 @@
import { createAzure } from "@ai-sdk/azure";
-import type { LanguageModel } from "ai";
+import type { EmbeddingModel, LanguageModel } from "ai";
import { trimTrailingSlashes } from "./utils";
@@ -50,13 +50,12 @@ export function translateAzureBaseURL(
return { ok: true, value: trimmed };
}
-export function buildAzureModel(params: {
- modelId: string;
+function createAzureProvider(params: {
apiKey: string;
baseURL?: string | null;
extraHeaders?: Record<string, string>;
fetch: typeof fetch;
-}): LanguageModel {
+}) {
const baseUrlTranslation = translateAzureBaseURL(params.baseURL);
if (!baseUrlTranslation.ok) {
// Configuration validation runs before model construction; keep this
@@ -64,7 +63,7 @@ export function buildAzureModel(params: {
throw new Error(baseUrlTranslation.reason);
}
- const provider = createAzure({
+ return createAzure({
apiKey: params.apiKey,
baseURL: baseUrlTranslation.value,
apiVersion: AZURE_OPENAI_API_VERSION,
@@ -72,7 +71,25 @@ export function buildAzureModel(params: {
headers: params.extraHeaders,
fetch: params.fetch,
});
+}
+export function buildAzureModel(params: {
+ modelId: string;
+ apiKey: string;
+ baseURL?: string | null;
+ extraHeaders?: Record<string, string>;
+ fetch: typeof fetch;
+}): LanguageModel {
// Azure connections use Chat Completions; the model name is the deployment.
- return provider.chat(params.modelId);
+ return createAzureProvider(params).chat(params.modelId);
+}
+
+export function buildAzureEmbeddingModel(params: {
+ modelId: string;
+ apiKey: string;
+ baseURL?: string | null;
+ extraHeaders?: Record<string, string>;
+ fetch: typeof fetch;
+}): EmbeddingModel {
+ return createAzureProvider(params).embedding(params.modelId);
}
diff --git a/packages/shared/src/server/llm/ai-sdk/providers/bedrock.ts b/packages/shared/src/server/llm/ai-sdk/providers/bedrock.ts
index d6dd5d0e48..25ab6e71c6 100644
--- a/packages/shared/src/server/llm/ai-sdk/providers/bedrock.ts
+++ b/packages/shared/src/server/llm/ai-sdk/providers/bedrock.ts
@@ -1,6 +1,6 @@
import { createAmazonBedrock } from "@ai-sdk/amazon-bedrock";
import { fromNodeProviderChain } from "@aws-sdk/credential-providers";
-import { type LanguageModel } from "ai";
+import { type EmbeddingModel, type LanguageModel } from "ai";
import { env } from "../../../../env";
import {
@@ -118,11 +118,19 @@ export function createDefaultBedrockProviderAuth(params?: {
};
}
-/**
- * Resolves the persisted Bedrock credential contract: the decrypted secret is
- * either an authorized default-credentials sentinel, AWS access key JSON, or a Bedrock API key
- * used as a bearer token.
- */
+/** Cloud uses the existing project allowlist for runtime credentials. */
+export function canUseBedrockDefaultCredentials(projectId?: string): boolean {
+ if (!env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) return true;
+ const allowedProjects =
+ env.LANGFUSE_BEDROCK_LLM_CONNECTION_RUNTIME_CREDENTIALS_ALLOWED_PROJECT_IDS;
+ return Boolean(
+ projectId &&
+ Array.isArray(allowedProjects) &&
+ allowedProjects.includes(projectId),
+ );
+}
+
+/** Resolves a default-credential sentinel, AWS access key JSON, or Bedrock API key. */
export function resolveBedrockProviderAuth(params: {
secretKey: string;
allowDefaultCredentials: boolean;
@@ -172,14 +180,13 @@ export function resolveBedrockProviderAuth(params: {
* Bedrock through VPC endpoints resolving to private IPs, which the secure
* LLM fetch would block. Requests therefore use the provider's AWS transport.
*/
-export function buildBedrockModel(params: {
+function createBedrockProvider(params: {
projectId?: string;
- modelId: string;
apiKey: string;
config?: LLMConnectionConfig | null;
credentialSource: LLMCredentialSource;
-}): LanguageModel {
- const { modelId, apiKey, config, credentialSource } = params;
+}) {
+ const { apiKey, config, credentialSource } = params;
const shouldUseLangfuseAPIKey = credentialSource === "langfuse";
const { region } = shouldUseLangfuseAPIKey
@@ -187,24 +194,36 @@ export function buildBedrockModel(params: {
: BedrockConfigSchema.parse(config);
assertValidBedrockRegion(region);
- const isSelfHosted = !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION;
- const allowedProjects =
- env.LANGFUSE_BEDROCK_LLM_CONNECTION_RUNTIME_CREDENTIALS_ALLOWED_PROJECT_IDS;
const auth = resolveBedrockProviderAuth({
secretKey: apiKey,
allowDefaultCredentials:
- isSelfHosted ||
shouldUseLangfuseAPIKey ||
- (!!params.projectId &&
- Array.isArray(allowedProjects) &&
- allowedProjects.includes(params.projectId)),
+ canUseBedrockDefaultCredentials(params.projectId),
profile: shouldUseLangfuseAPIKey ? getLangfuseAIAwsProfile() : undefined,
});
- const provider = createAmazonBedrock({
+ return createAmazonBedrock({
region,
...auth,
});
+}
- return provider(modelId);
+export function buildBedrockModel(params: {
+ projectId?: string;
+ modelId: string;
+ apiKey: string;
+ config?: LLMConnectionConfig | null;
+ credentialSource: LLMCredentialSource;
+}): LanguageModel {
+ return createBedrockProvider(params)(params.modelId);
+}
+
+export function buildBedrockEmbeddingModel(params: {
+ projectId?: string;
+ modelId: string;
+ apiKey: string;
+ config?: LLMConnectionConfig | null;
+ credentialSource: LLMCredentialSource;
+}): EmbeddingModel {
+ return createBedrockProvider(params).embeddingModel(params.modelId);
}
diff --git a/packages/shared/src/server/llm/ai-sdk/providers/google.ts b/packages/shared/src/server/llm/ai-sdk/providers/google.ts
index 55a2c8cf98..0b930c5927 100644
--- a/packages/shared/src/server/llm/ai-sdk/providers/google.ts
+++ b/packages/shared/src/server/llm/ai-sdk/providers/google.ts
@@ -1,5 +1,5 @@
import { createGoogleGenerativeAI } from "@ai-sdk/google";
-import type { LanguageModel } from "ai";
+import type { EmbeddingModel, LanguageModel } from "ai";
import type { TranslatedProviderOptions } from "./types";
import { ensureBaseURLSuffix, isPlainObject } from "./utils";
@@ -32,6 +32,19 @@ export function buildGoogleAIStudioModel(params: {
return provider(params.modelId);
}
+export function buildGoogleAIStudioEmbeddingModel(params: {
+ modelId: string;
+ apiKey: string;
+ baseURL?: string | null;
+ fetch: typeof fetch;
+}): EmbeddingModel {
+ return createGoogleGenerativeAI({
+ apiKey: params.apiKey,
+ baseURL: toGoogleAIStudioBaseURL(params.baseURL),
+ fetch: params.fetch,
+ }).embedding(params.modelId);
+}
+
const GOOGLE_THINKING_LEVELS = new Set(["minimal", "low", "medium", "high"]);
/**
diff --git a/packages/shared/src/server/llm/ai-sdk/providers/index.ts b/packages/shared/src/server/llm/ai-sdk/providers/index.ts
index d116efb8ca..825527762e 100644
--- a/packages/shared/src/server/llm/ai-sdk/providers/index.ts
+++ b/packages/shared/src/server/llm/ai-sdk/providers/index.ts
@@ -1,15 +1,22 @@
-import type { LanguageModel } from "ai";
+import type { EmbeddingModel, LanguageModel } from "ai";
import type { LLMConnectionConfig } from "../../../../interfaces/customLLMProviderConfigSchemas";
import { LLMAdapter } from "../../types";
import type { AiSdkModelConfig } from "../resolveAiSdkModelConfig";
import { buildAnthropicModel } from "./anthropic";
-import { buildAzureModel } from "./azure";
-import { buildBedrockModel } from "./bedrock";
-import { buildGoogleAIStudioModel } from "./google";
-import { buildOpenAIModel } from "./openai";
+import { buildAzureEmbeddingModel, buildAzureModel } from "./azure";
+import { buildBedrockEmbeddingModel, buildBedrockModel } from "./bedrock";
+import {
+ buildGoogleAIStudioEmbeddingModel,
+ buildGoogleAIStudioModel,
+} from "./google";
+import { buildOpenAIEmbeddingModel, buildOpenAIModel } from "./openai";
import type { LLMCredentialSource } from "./types";
-import { buildVertexModel, isClaudeModel } from "./vertex";
+import {
+ buildVertexEmbeddingModel,
+ buildVertexModel,
+ isClaudeModel,
+} from "./vertex";
const AZURE_OPENAI_API_KEY_HEADER = "api-key";
const ANTHROPIC_API_KEY_HEADER = "x-api-key";
@@ -131,3 +138,83 @@ export async function buildAiSdkModel(params: {
}
}
}
+
+/**
+ * Builds the AI SDK `EmbeddingModel` for a persisted connection. Anthropic and
+ * TypeSafe have no embeddings API.
+ */
+export async function buildAiSdkEmbeddingModel(params: {
+ projectId?: string;
+ model: { adapter: LLMAdapter; id: string };
+ apiKey: string;
+ baseURL?: string | null;
+ extraHeaders?: Record<string, string>;
+ config?: LLMConnectionConfig | null;
+ credentialSource: LLMCredentialSource;
+ createFetch: CreateSecureFetch;
+}): Promise<EmbeddingModel> {
+ const { model, apiKey, baseURL, extraHeaders, config, credentialSource } =
+ params;
+
+ switch (model.adapter) {
+ case LLMAdapter.OpenAI:
+ return buildOpenAIEmbeddingModel({
+ modelId: model.id,
+ apiKey,
+ baseURL,
+ extraHeaders,
+ fetch: params.createFetch("OpenAI LLM base URL"),
+ });
+
+ case LLMAdapter.Azure:
+ return buildAzureEmbeddingModel({
+ modelId: model.id,
+ apiKey,
+ baseURL,
+ extraHeaders,
+ fetch: params.createFetch("Azure OpenAI LLM base URL", [
+ AZURE_OPENAI_API_KEY_HEADER,
+ ]),
+ });
+
+ case LLMAdapter.Bedrock:
+ return buildBedrockEmbeddingModel({
+ projectId: params.projectId,
+ modelId: model.id,
+ apiKey,
+ config,
+ credentialSource,
+ });
+
+ case LLMAdapter.GoogleAIStudio:
+ return buildGoogleAIStudioEmbeddingModel({
+ modelId: model.id,
+ apiKey,
+ baseURL,
+ fetch: params.createFetch("Google AI Studio LLM base URL", [
+ GOOGLE_API_KEY_HEADER,
+ ]),
+ });
+
+ case LLMAdapter.VertexAI:
+ return buildVertexEmbeddingModel({
+ modelId: model.id,
+ apiKey,
+ config,
+ extraHeaders,
+ credentialSource,
+ fetch: params.createFetch("Vertex AI LLM endpoint", [
+ VERTEX_AI_AUTH_HEADER,
+ ]),
+ });
+
+ case LLMAdapter.Anthropic:
+ case LLMAdapter.TypeSafe:
+ throw new Error(`${model.adapter} connections do not support embeddings`);
+
+ default: {
+ const _exhaustiveCheck: never = model.adapter;
+ throw new Error(`AI SDK adapter is not supported: ${_exhaustiveCheck}`);
+ }
+ }
+}
diff --git a/packages/shared/src/server/llm/ai-sdk/providers/openai.ts b/packages/shared/src/server/llm/ai-sdk/providers/openai.ts
index fce954fbb3..52ff889fb5 100644
--- a/packages/shared/src/server/llm/ai-sdk/providers/openai.ts
+++ b/packages/shared/src/server/llm/ai-sdk/providers/openai.ts
@@ -1,6 +1,6 @@
import { createOpenAI } from "@ai-sdk/openai";
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
-import type { LanguageModel } from "ai";
+import type { EmbeddingModel, LanguageModel } from "ai";
import { parseOutboundUrl } from "../../../outbound-url/validation";
import { processOpenAIBaseURL } from "../../utils";
@@ -55,6 +55,36 @@ export function buildOpenAIModel(params: {
: provider.chat(modelId);
}
+export function buildOpenAIEmbeddingModel(params: {
+ modelId: string;
+ apiKey: string;
+ baseURL?: string | null;
+ extraHeaders?: Record<string, string>;
+ fetch: typeof fetch;
+}): EmbeddingModel {
+ const { apiKey, baseURL, extraHeaders, modelId } = params;
+ const processedBaseURL = processOpenAIBaseURL({
+ url: baseURL,
+ modelName: modelId,
+ });
+
+ if (isOpenAICompatibleEndpoint(processedBaseURL))
+ return createOpenAICompatible({
+ name: "openai",
+ apiKey,
+ baseURL: processedBaseURL,
+ headers: extraHeaders,
+ fetch: params.fetch,
+ }).embeddingModel(modelId);
+
+ return createOpenAI({
+ apiKey,
+ baseURL: processedBaseURL ?? undefined,
+ headers: extraHeaders,
+ fetch: params.fetch,
+ }).embedding(modelId);
+}
+
export function isOpenAICompatibleEndpoint(
baseURL: string | null | undefined,
): baseURL is string {
diff --git a/packages/shared/src/server/llm/ai-sdk/providers/vertex.ts b/packages/shared/src/server/llm/ai-sdk/providers/vertex.ts
index 871789bb9a..d0da206c74 100644
--- a/packages/shared/src/server/llm/ai-sdk/providers/vertex.ts
+++ b/packages/shared/src/server/llm/ai-sdk/providers/vertex.ts
@@ -1,6 +1,6 @@
import { createVertex } from "@ai-sdk/google-vertex";
import { createVertexAnthropic } from "@ai-sdk/google-vertex/anthropic";
-import type { LanguageModel } from "ai";
+import type { EmbeddingModel, LanguageModel } from "ai";
import { type GoogleAuthOptions } from "google-auth-library";
import { env } from "../../../../env";
@@ -64,15 +64,12 @@ export function getLangfuseAIVertexLocation(): string | undefined {
* allowed for self-hosted deployments and for Langfuse-operated AI, in which
* case the project is resolved from the default credential chain.
*/
-export async function buildVertexModel(params: {
- modelId: string;
+async function resolveVertexConnection(params: {
apiKey: string;
config?: LLMConnectionConfig | null;
- extraHeaders?: Record<string, string>;
credentialSource: LLMCredentialSource;
- fetch: typeof fetch;
-}): Promise<LanguageModel> {
- const { modelId, apiKey, config, extraHeaders, credentialSource } = params;
+}) {
+ const { apiKey, config, credentialSource } = params;
const shouldUseLangfuseAPIKey = credentialSource === "langfuse";
// Langfuse-operated AI carries no persisted connection, so its location comes
@@ -108,15 +105,25 @@ export async function buildVertexModel(params: {
serviceAccountKey?.project_id ?? (await resolveVertexProjectIdFromADC());
// Existing connections default the location to "global" for both families.
- const resolvedLocation = location ?? "global";
+ return { project, location: location ?? "global", googleAuthOptions };
+}
+
+export async function buildVertexModel(params: {
+ modelId: string;
+ apiKey: string;
+ config?: LLMConnectionConfig | null;
+ extraHeaders?: Record<string, string>;
+ credentialSource: LLMCredentialSource;
+ fetch: typeof fetch;
+}): Promise<LanguageModel> {
+ const { modelId, extraHeaders } = params;
+ const connection = await resolveVertexConnection(params);
if (isClaudeModel(modelId)) {
assertValidAnthropicVertexModelName(modelId);
const provider = createVertexAnthropic({
- project,
- location: resolvedLocation,
- googleAuthOptions,
+ ...connection,
headers: extraHeaders,
fetch: params.fetch,
});
@@ -125,12 +132,25 @@ export async function buildVertexModel(params: {
}
const provider = createVertex({
- project,
- location: resolvedLocation,
- googleAuthOptions,
+ ...connection,
headers: extraHeaders,
fetch: params.fetch,
});
return provider(modelId);
}
+
+export async function buildVertexEmbeddingModel(params: {
+ modelId: string;
+ apiKey: string;
+ config?: LLMConnectionConfig | null;
+ extraHeaders?: Record<string, string>;
+ credentialSource: LLMCredentialSource;
+ fetch: typeof fetch;
+}): Promise<EmbeddingModel> {
+ return createVertex({
+ ...(await resolveVertexConnection(params)),
+ headers: params.extraHeaders,
+ fetch: params.fetch,
+ }).embeddingModel(params.modelId);
+}
diff --git a/packages/shared/src/server/llm/llmText.ts b/packages/shared/src/server/llm/llmText.ts
index 371a8e4148..55153911fa 100644
--- a/packages/shared/src/server/llm/llmText.ts
+++ b/packages/shared/src/server/llm/llmText.ts
@@ -2,6 +2,7 @@ import { type ZodType } from "zod";
import {
Output,
+ embed,
generateText,
jsonSchema,
streamText,
@@ -10,6 +11,7 @@ import {
type GenerateTextOnAbortCallback,
type GenerateTextOnEndCallback,
type GenerateTextResult,
+ type EmbedResult,
type Experimental_DownloadFunction,
type JSONValue,
type LanguageModelCallOptions,
@@ -26,7 +28,7 @@ import type { LLMConnectionConfig } from "../../interfaces/customLLMProviderConf
import { LLMValidationError } from "./errors";
import { resolveEvaluatorMediaTransport } from "./mediaMessages";
import { mapChatMessagesToModelMessages } from "./ai-sdk/messages";
-import { buildAiSdkModel } from "./ai-sdk/providers";
+import { buildAiSdkEmbeddingModel, buildAiSdkModel } from "./ai-sdk/providers";
import { translateAnthropicProviderOptions } from "./ai-sdk/providers/anthropic";
import { translateBedrockProviderOptions } from "./ai-sdk/providers/bedrock";
import { translateGoogleProviderOptions } from "./ai-sdk/providers/google";
@@ -307,25 +309,10 @@ async function prepareLLMTextCall<
trace: options.trace,
});
- const apiKey = decrypt(options.connection.secretKey);
- const extraHeaders = decryptAndParseExtraHeaders(
- options.connection.extraHeaders,
+ const { apiKey, extraHeaders, createFetch } = openLLMConnection(
+ options.connection,
);
- const createFetch = (
- logContext: string,
- additionalSensitiveHeaders?: string[],
- ) =>
- createSecureLlmFetch({
- logContext,
- // Connection-specific headers are encrypted at rest and can contain
- // gateway credentials. Keep them on same-origin redirects, but strip
- // them alongside provider auth headers when the origin changes.
- additionalSensitiveHeaders: (additionalSensitiveHeaders ?? []).concat(
- Object.keys(extraHeaders ?? {}),
- ),
- });
-
const languageModel = await buildAiSdkModel({
model: options.model,
modelConfig,
@@ -371,6 +358,62 @@ async function prepareLLMTextCall<
};
}
+function openLLMConnection(connection: LLMConnection) {
+ const apiKey = decrypt(connection.secretKey);
+ const extraHeaders = decryptAndParseExtraHeaders(connection.extraHeaders);
+ const createFetch = (
+ logContext: string,
+ additionalSensitiveHeaders?: string[],
+ ) =>
+ createSecureLlmFetch({
+ logContext,
+ // Connection-specific headers are encrypted at rest and can contain
+ // gateway credentials. Keep them on same-origin redirects, but strip
+ // them alongside provider auth headers when the origin changes.
+ additionalSensitiveHeaders: (additionalSensitiveHeaders ?? []).concat(
+ Object.keys(extraHeaders ?? {}),
+ ),
+ });
+ return { apiKey, extraHeaders, createFetch };
+}
+
+/** Embeds one value with a persisted connection; calls are not traced. */
+export async function embedLLMText(options: {
+ model: LLMModelRef;
+ connection: LLMConnection;
+ value: string;
+ providerOptions?: ProviderOptions;
+ maxRetries?: number;
+ abortSignal?: AbortSignal;
+}): Promise<EmbedResult> {
+ resolveAiSdkModelConfig({
+ model: options.model,
+ connectionConfig: options.connection.config,
+ baseURL: options.connection.baseURL,
+ credentialSource: "user",
+ });
+ const { apiKey, extraHeaders, createFetch } = openLLMConnection(
+ options.connection,
+ );
+ const embeddingModel = await buildAiSdkEmbeddingModel({
+ model: options.model,
+ projectId: options.connection.projectId,
+ apiKey,
+ baseURL: options.connection.baseURL,
+ extraHeaders,
+ config: options.connection.config,
+ credentialSource: "user",
+ createFetch,
+ });
+ return embed({
+ model: embeddingModel,
+ value: options.value,
+ providerOptions: options.providerOptions,
+ maxRetries: options.maxRetries,
+ abortSignal: options.abortSignal,
+ });
+}
+
function addOpenRouterInternalTraceProvenance(params: {
model: LLMModelRef;
baseURL?: string | null;
diff --git a/packages/shared/src/server/topics/clickhouse.ts b/packages/shared/src/server/topics/clickhouse.ts
index 64ef7490d5..283768f5a1 100644
--- a/packages/shared/src/server/topics/clickhouse.ts
+++ b/packages/shared/src/server/topics/clickhouse.ts
@@ -411,6 +411,17 @@ export async function getTopicClusteringSummaries(
return rows.map(summaryResult);
}
+export async function hasTopicEmbeddings(projectId: string): Promise<boolean> {
+ const rows = await queryClickhouse<{ found: number }>({
+ query: `SELECT 1 AS found FROM topic_facet_summaries
+ WHERE project_id = {projectId:String} AND processing_state = 'complete'
+ LIMIT 1`,
+ params: { projectId },
+ tags: { route: "topics-has-embeddings", projectId },
+ });
+ return rows.length > 0;
+}
+
/** Count current compatible summaries without loading summary text or vectors. */
export async function getTopicSummaryCounts(
projectId: string,
diff --git a/packages/shared/src/server/topics/config.ts b/packages/shared/src/server/topics/config.ts
index 89604dff0d..9fc1b0b0fb 100644
--- a/packages/shared/src/server/topics/config.ts
+++ b/packages/shared/src/server/topics/config.ts
@@ -4,13 +4,6 @@ export function isTopicsEnabled(): boolean {
return env.LANGFUSE_TOPICS_ENABLED === "true";
}
-export function getTopicsModelConfig() {
- return {
- summaryModel: env.LANGFUSE_TOPICS_SUMMARY_MODEL,
- embeddingModel: env.LANGFUSE_TOPICS_EMBEDDING_MODEL,
- };
-}
-
export function isTopicsProjectEnabled(projectId: string): boolean {
return (
isTopicsEnabled() &&
diff --git a/packages/shared/src/server/topics/embeddings.ts b/packages/shared/src/server/topics/embeddings.ts
index 461df26671..50b06471d8 100644
--- a/packages/shared/src/server/topics/embeddings.ts
+++ b/packages/shared/src/server/topics/embeddings.ts
@@ -1,44 +1,61 @@
-import { createAmazonBedrock } from "@ai-sdk/amazon-bedrock";
-import { embed } from "ai";
+import { embedLLMText } from "../llm/llmText";
+import { LLMAdapter } from "../llm/types";
import { topicEmbeddingConfigSchema } from "../../topics";
-import {
- assertValidBedrockRegion,
- createDefaultBedrockProviderAuth,
-} from "../llm/ai-sdk/providers/bedrock";
+import type { TopicsModel } from "./model-config";
-/** Uses the shared AI SDK version for the Topics worker's embedding call. */
+// The AI SDK validates the whole Bedrock options object, and each family names
+// its size option differently, so only the family's own fields are sent.
+function bedrockEmbeddingOptions(
+ modelId: string,
+ dimensions: number,
+): Record<string, string | number> {
+ if (modelId.includes("nova"))
+ return { embeddingPurpose: "CLUSTERING", embeddingDimension: dimensions };
+ if (modelId.includes("cohere"))
+ return {
+ inputType: "clustering",
+ truncate: "NONE",
+ outputDimension: dimensions,
+ };
+ return { dimensions };
+}
+
+/** Embeds one summary through the project's LLM connection. */
export async function generateTopicEmbedding(params: {
- model: string;
+ model: TopicsModel;
summary: string;
dimensions: number;
- region: string;
- profile?: string;
}) {
- const config = topicEmbeddingConfigSchema.parse({
- embeddingModel: params.model,
+ const { embeddingDimensions } = topicEmbeddingConfigSchema.parse({
embeddingDimensions: params.dimensions,
});
- if (!config.embeddingModel)
- throw new Error("Topics embedding model is not configured.");
- assertValidBedrockRegion(params.region);
- const provider = createAmazonBedrock({
- region: params.region,
- ...createDefaultBedrockProviderAuth({ profile: params.profile }),
- });
- const result = await embed({
- model: provider.embeddingModel(params.model),
+ const result = await embedLLMText({
+ model: { adapter: params.model.adapter, id: params.model.model },
+ connection: params.model.connection,
value: params.summary,
- providerOptions: {
- amazonBedrock: {
- // Discovery and later centroid assignment must share one vector space.
- inputType: "clustering",
- outputDimension: config.embeddingDimensions,
- truncate: "NONE",
- },
- },
- maxRetries: 0,
+ // Discovery and later centroid assignment must share one vector space, so
+ // every provider is asked for clustering vectors of the configured size.
+ providerOptions:
+ params.model.adapter === LLMAdapter.Bedrock
+ ? {
+ amazonBedrock: bedrockEmbeddingOptions(
+ params.model.model,
+ embeddingDimensions,
+ ),
+ }
+ : {
+ openai: { dimensions: embeddingDimensions },
+ google: {
+ taskType: "CLUSTERING",
+ outputDimensionality: embeddingDimensions,
+ },
+ vertex: {
+ taskType: "CLUSTERING",
+ outputDimensionality: embeddingDimensions,
+ },
+ },
+ maxRetries: 2,
abortSignal: AbortSignal.timeout(60_000),
});
- // Cohere defaults to float output; the SDK accepts only float vectors.
return { embedding: result.embedding, tokens: result.usage.tokens };
}
diff --git a/packages/shared/src/server/topics/index.ts b/packages/shared/src/server/topics/index.ts
index 98b809b190..f43942a76d 100644
--- a/packages/shared/src/server/topics/index.ts
+++ b/packages/shared/src/server/topics/index.ts
@@ -7,5 +7,7 @@ export { emitTopicStagingMetrics } from "./staging-metrics";
export * from "./embeddings";
export * from "./text";
export * from "./config";
+export * from "./model-config";
+export * from "./setup";
export * from "./trace-input";
export * from "./trace-selection";
diff --git a/packages/shared/src/server/topics/model-config.test.ts b/packages/shared/src/server/topics/model-config.test.ts
new file mode 100644
index 0000000000..51db82ba95
--- /dev/null
+++ b/packages/shared/src/server/topics/model-config.test.ts
@@ -0,0 +1,180 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { LLMAdapter } from "../llm/types";
+import {
+ getEnabledTopicsModels,
+ prepareTopicsModelSettings,
+ writeTopicsModelSettings,
+} from "./model-config";
+
+const mocks = vi.hoisted(() => ({
+ keys: vi.fn(),
+ configs: vi.fn(),
+ current: vi.fn(),
+ upsert: vi.fn(),
+ hasEmbeddings: vi.fn(),
+ testCall: vi.fn(),
+ embed: vi.fn(),
+}));
+vi.mock("./clickhouse", () => ({ hasTopicEmbeddings: mocks.hasEmbeddings }));
+vi.mock("../llm/testModelCall", () => ({ testModelCall: mocks.testCall }));
+vi.mock("./embeddings", () => ({ generateTopicEmbedding: mocks.embed }));
+vi.mock("../../db", () => ({
+ prisma: {
+ llmApiKeys: { findMany: mocks.keys },
+ topicConfig: {
+ findMany: mocks.configs,
+ findUnique: mocks.current,
+ upsert: mocks.upsert,
+ },
+ },
+}));
+
+const connection = (id: string, adapter: LLMAdapter) => ({
+ id,
+ adapter,
+ provider: `${adapter}-connection`,
+ secretKey: "encrypted",
+ extraHeaders: null,
+ baseURL: null,
+ config: null,
+});
+const slot = (llmApiKeyId: string, model: string) => ({ llmApiKeyId, model });
+const settings = {
+ summary: slot("openai", "gpt-6-luna"),
+ embedding: slot("openai", "text-embedding-3-small"),
+ embeddingDimensions: 1024 as const,
+ clustering: slot("openai", "gpt-5.6-terra"),
+ enabled: true,
+};
+
+const saveTopicsModelSettings = async (
+ projectId: string,
+ value: Parameters<typeof prepareTopicsModelSettings>[1],
+) =>
+ writeTopicsModelSettings(
+ projectId,
+ await prepareTopicsModelSettings(projectId, value),
+ );
+
+beforeEach(() => {
+ vi.resetAllMocks();
+ mocks.keys.mockResolvedValue([
+ connection("openai", LLMAdapter.OpenAI),
+ connection("anthropic", LLMAdapter.Anthropic),
+ ]);
+ mocks.current.mockResolvedValue(null);
+ mocks.hasEmbeddings.mockResolvedValue(false);
+ mocks.testCall.mockResolvedValue(undefined);
+ mocks.embed.mockImplementation(async ({ dimensions }) => ({
+ embedding: Array(dimensions).fill(0.1),
+ }));
+});
+
+describe("prepareTopicsModelSettings", () => {
+ it("saves an Anthropic connection when its test call succeeds", async () => {
+ await saveTopicsModelSettings("project", {
+ ...settings,
+ clustering: slot("anthropic", "claude-sonnet"),
+ });
+ expect(mocks.testCall).toHaveBeenCalledWith(
+ expect.objectContaining({ model: "claude-sonnet" }),
+ );
+ expect(mocks.upsert).toHaveBeenCalledOnce();
+ });
+
+ it("saves a partial setup but enables Topics only with all three models", async () => {
+ await saveTopicsModelSettings("project", {
+ ...settings,
+ clustering: null,
+ enabled: false,
+ });
+ expect(mocks.upsert).toHaveBeenCalledOnce();
+ await expect(
+ saveTopicsModelSettings("project", { ...settings, clustering: null }),
+ ).rejects.toThrow("before turning Topics on");
+ expect(mocks.upsert).toHaveBeenCalledOnce();
+ });
+
+ it("tests every configured slot and names the ones that fail", async () => {
+ mocks.testCall.mockImplementation(async ({ model }) => {
+ if (model === "gpt-5.6-terra")
+ throw new Error("Model not found (HTTP 404)");
+ });
+ mocks.embed.mockResolvedValue({ embedding: Array(1536).fill(0.1) });
+
+ await expect(saveTopicsModelSettings("project", settings)).rejects.toThrow(
+ "Embeddings (text-embedding-3-small): The model returned 1536 dimensions instead of 1024. Choose a size this model supports.; Topic clustering (gpt-5.6-terra): Model not found (HTTP 404)",
+ );
+ expect(mocks.testCall).toHaveBeenCalledTimes(2);
+ expect(mocks.embed).toHaveBeenCalledWith(
+ expect.objectContaining({ dimensions: 1024 }),
+ );
+ expect(mocks.upsert).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ ["model", { embedding: slot("openai", "text-embedding-3-large") }],
+ ["dimensions", { embeddingDimensions: 512 as const }],
+ ])(
+ "locks the embedding %s once summaries are embedded",
+ async (_, change) => {
+ mocks.current.mockResolvedValue({
+ embeddingModel: "text-embedding-3-small",
+ embeddingDimensions: 1024,
+ });
+ mocks.hasEmbeddings.mockResolvedValue(true);
+ await expect(
+ saveTopicsModelSettings("project", { ...settings, ...change }),
+ ).rejects.toThrow("cannot change after summaries were embedded");
+ // Other slots stay editable; the embedding connection may change too.
+ await saveTopicsModelSettings("project", {
+ ...settings,
+ summary: slot("openai", "gpt-6-luna-mini"),
+ });
+ expect(mocks.upsert).toHaveBeenCalledOnce();
+ },
+ );
+});
+
+describe("getEnabledTopicsModels", () => {
+ it("returns only enabled projects whose three connections are usable", async () => {
+ const openai = {
+ ...connection("openai", LLMAdapter.OpenAI),
+ projectId: "complete",
+ };
+ const row = {
+ projectId: "complete",
+ enabled: true,
+ summaryLlmApiKey: openai,
+ summaryModel: "gpt-6-luna",
+ embeddingLlmApiKey: openai,
+ embeddingModel: "text-embedding-3-small",
+ embeddingDimensions: 512,
+ clusteringLlmApiKey: openai,
+ clusteringModel: "gpt-5.6-terra",
+ };
+ mocks.configs.mockResolvedValue([
+ row,
+ // A deleted connection leaves its slot empty.
+ { ...row, projectId: "missing-slot", clusteringLlmApiKey: null },
+ ]);
+
+ const models = await getEnabledTopicsModels(["complete", "missing-slot"]);
+
+ expect(mocks.configs).toHaveBeenCalledWith(
+ expect.objectContaining({
+ where: {
+ projectId: { in: ["complete", "missing-slot"] },
+ enabled: true,
+ },
+ }),
+ );
+ expect([...models.keys()]).toEqual(["complete"]);
+ expect(models.get("complete")?.embedding).toMatchObject({
+ adapter: LLMAdapter.OpenAI,
+ model: "text-embedding-3-small",
+ dimensions: 512,
+ connection: { projectId: "complete" },
+ });
+ });
+});
diff --git a/packages/shared/src/server/topics/model-config.ts b/packages/shared/src/server/topics/model-config.ts
new file mode 100644
index 0000000000..bfd6f77282
--- /dev/null
+++ b/packages/shared/src/server/topics/model-config.ts
@@ -0,0 +1,370 @@
+import {
+ EvaluatorBlockReason,
+ type LlmApiKeys,
+ type Prisma,
+ type TopicConfig,
+} from "@prisma/client";
+import { prisma } from "../../db";
+import { InvalidRequestError } from "../../errors";
+import {
+ getClientInitiatedNonStreamingLlmTimeoutMs,
+ type LLMConnection,
+} from "../llm/llmText";
+import { testModelCall } from "../llm/testModelCall";
+import { LLMAdapter, type LLMApiKeySchema } from "../llm/types";
+import {
+ TOPICS_MODEL_SLOT_DETAILS,
+ TOPICS_MODEL_SLOTS,
+ type TopicsModelSettings,
+ type TopicsModelSlotName,
+} from "../../topics";
+import { hasTopicEmbeddings } from "./clickhouse";
+import { generateTopicEmbedding } from "./embeddings";
+import { z } from "zod";
+
+export type TopicsModel = {
+ slot: TopicsModelSlotName;
+ provider: string;
+ adapter: LLMAdapter;
+ model: string;
+ connection: LLMConnection;
+};
+
+/** Resolved models of a project whose three slots are all configured. */
+export type TopicsModels = {
+ projectId: string;
+ enabled: boolean;
+ summary: TopicsModel;
+ embedding: TopicsModel & { dimensions: number };
+ clustering: TopicsModel;
+};
+
+const withConnections = {
+ summaryLlmApiKey: true,
+ embeddingLlmApiKey: true,
+ clusteringLlmApiKey: true,
+} as const;
+
+type ConfigWithConnections = TopicConfig & {
+ summaryLlmApiKey: LlmApiKeys | null;
+ embeddingLlmApiKey: LlmApiKeys | null;
+ clusteringLlmApiKey: LlmApiKeys | null;
+};
+
+function resolveSlot(
+ slot: TopicsModelSlotName,
+ key: LlmApiKeys | null,
+ model: string | null,
+): TopicsModel | null {
+ if (!key || !model) return null;
+ return {
+ slot,
+ provider: key.provider,
+ adapter: key.adapter as LLMAdapter,
+ model,
+ connection: {
+ projectId: key.projectId,
+ secretKey: key.secretKey,
+ extraHeaders: key.extraHeaders,
+ baseURL: key.baseURL,
+ config: key.config as LLMConnection["config"],
+ },
+ };
+}
+
+function resolveModels(row: ConfigWithConnections): TopicsModels | null {
+ const summary = resolveSlot(
+ "summary",
+ row.summaryLlmApiKey,
+ row.summaryModel,
+ );
+ const embedding = resolveSlot(
+ "embedding",
+ row.embeddingLlmApiKey,
+ row.embeddingModel,
+ );
+ const clustering = resolveSlot(
+ "clustering",
+ row.clusteringLlmApiKey,
+ row.clusteringModel,
+ );
+ if (!summary || !embedding || !clustering) return null;
+ return {
+ projectId: row.projectId,
+ enabled: row.enabled,
+ summary,
+ embedding: { ...embedding, dimensions: row.embeddingDimensions },
+ clustering,
+ };
+}
+
+/** Returns the configured models of a project, or null while any slot is missing. */
+export async function getTopicsModels(
+ projectId: string,
+): Promise<TopicsModels | null> {
+ const row = await prisma.topicConfig.findUnique({
+ where: { projectId },
+ include: withConnections,
+ });
+ return row ? resolveModels(row) : null;
+}
+
+/** Loads the enabled, fully configured projects of a trace batch in one query. */
+export async function getEnabledTopicsModels(
+ projectIds: string[],
+): Promise<Map<string, TopicsModels>> {
+ if (!projectIds.length) return new Map();
+ const rows = await prisma.topicConfig.findMany({
+ where: { projectId: { in: projectIds }, enabled: true },
+ include: withConnections,
+ });
+ const models = new Map<string, TopicsModels>();
+ for (const row of rows) {
+ const resolved = resolveModels(row);
+ if (resolved) models.set(row.projectId, resolved);
+ }
+ return models;
+}
+
+/** Turns automatic processing off and records why, for the Topics page. */
+export async function pauseTopicsModels(
+ projectId: string,
+ block: { blockReason: EvaluatorBlockReason; blockMessage: string },
+): Promise<void> {
+ await prisma.topicConfig.updateMany({
+ where: { projectId, enabled: true },
+ data: {
+ enabled: false,
+ blockedAt: new Date(),
+ blockReason: block.blockReason,
+ blockMessage: block.blockMessage,
+ },
+ });
+}
+
+/** Pauses Topics in every config that uses a connection about to be deleted. */
+export async function pauseTopicsModelsUsingConnection(
+ params: { projectId: string; llmApiKeyId: string; provider: string },
+ tx: Prisma.TransactionClient = prisma,
+): Promise<void> {
+ const { projectId, llmApiKeyId, provider } = params;
+ await tx.topicConfig.updateMany({
+ where: {
+ projectId,
+ OR: [
+ { summaryLlmApiKeyId: llmApiKeyId },
+ { embeddingLlmApiKeyId: llmApiKeyId },
+ { clusteringLlmApiKeyId: llmApiKeyId },
+ ],
+ },
+ data: {
+ enabled: false,
+ blockedAt: new Date(),
+ blockReason: EvaluatorBlockReason.LLM_CONNECTION_MISSING,
+ blockMessage: `The LLM connection "${provider}" was deleted. Choose another connection and turn Topics on again.`,
+ },
+ });
+}
+
+/** Settings shown in the Topics page; never includes secrets. */
+export async function readTopicsModelSettings(projectId: string) {
+ const row = await prisma.topicConfig.findUnique({
+ where: { projectId },
+ });
+ const slot = (llmApiKeyId: string | null, model: string | null) =>
+ llmApiKeyId && model ? { llmApiKeyId, model } : null;
+ return {
+ summary: slot(row?.summaryLlmApiKeyId ?? null, row?.summaryModel ?? null),
+ embedding: slot(
+ row?.embeddingLlmApiKeyId ?? null,
+ row?.embeddingModel ?? null,
+ ),
+ embeddingDimensions: row?.embeddingDimensions ?? 1024,
+ clustering: slot(
+ row?.clusteringLlmApiKeyId ?? null,
+ row?.clusteringModel ?? null,
+ ),
+ enabled: row?.enabled ?? false,
+ blockedAt: row?.blockedAt ?? null,
+ blockReason: row?.blockReason ?? null,
+ blockMessage: row?.blockMessage ?? null,
+ };
+}
+
+async function loadTopicsModelKeys(
+ projectId: string,
+ settings: TopicsModelSettings,
+) {
+ const ids = [
+ settings.summary,
+ settings.embedding,
+ settings.clustering,
+ ].flatMap((slot) => (slot ? [slot.llmApiKeyId] : []));
+ const keys = await prisma.llmApiKeys.findMany({
+ where: { projectId, id: { in: ids } },
+ });
+ for (const id of ids) {
+ if (!keys.some((key) => key.id === id))
+ throw new InvalidRequestError(
+ "LLM connection not found in this project.",
+ );
+ }
+ return keys;
+}
+
+/** Real call per configured slot. The message is the provider error for that row. */
+export async function checkTopicsModelSettings(
+ projectId: string,
+ settings: TopicsModelSettings,
+): Promise<Partial<Record<TopicsModelSlotName, string>>> {
+ return testTopicsModelSlots(
+ settings,
+ await loadTopicsModelKeys(projectId, settings),
+ );
+}
+
+/**
+ * Validates and tests the settings without writing them, so callers can save
+ * related configuration first and commit the models last.
+ */
+export async function prepareTopicsModelSettings(
+ projectId: string,
+ settings: TopicsModelSettings,
+) {
+ const keys = await loadTopicsModelKeys(projectId, settings);
+ if (
+ settings.enabled &&
+ (!settings.summary || !settings.embedding || !settings.clustering)
+ )
+ throw new InvalidRequestError(
+ "Choose a summary, embedding, and clustering model before turning Topics on.",
+ );
+
+ const current = await prisma.topicConfig.findUnique({
+ where: { projectId },
+ });
+ const embeddingChanged =
+ current?.embeddingModel &&
+ (settings.embedding?.model !== current.embeddingModel ||
+ settings.embeddingDimensions !== current.embeddingDimensions);
+ if (embeddingChanged && (await hasTopicEmbeddings(projectId)))
+ throw new InvalidRequestError(
+ "The embedding model and dimensions cannot change after summaries were embedded; existing vectors could no longer be clustered together.",
+ );
+
+ await testTopicsModels(settings, keys);
+
+ const data = {
+ enabled: settings.enabled,
+ // Saving the settings acknowledges the last pause.
+ blockedAt: null,
+ blockReason: null,
+ blockMessage: null,
+ summaryLlmApiKeyId: settings.summary?.llmApiKeyId ?? null,
+ summaryModel: settings.summary?.model ?? null,
+ embeddingLlmApiKeyId: settings.embedding?.llmApiKeyId ?? null,
+ embeddingModel: settings.embedding?.model ?? null,
+ embeddingDimensions: settings.embeddingDimensions,
+ clusteringLlmApiKeyId: settings.clustering?.llmApiKeyId ?? null,
+ clusteringModel: settings.clustering?.model ?? null,
+ };
+ return data;
+}
+
+export async function writeTopicsModelSettings(
+ projectId: string,
+ data: Awaited<ReturnType<typeof prepareTopicsModelSettings>>,
+ {
+ userId = null,
+ tx = prisma,
+ }: { userId?: string | null; tx?: Prisma.TransactionClient } = {},
+): Promise<void> {
+ await tx.topicConfig.upsert({
+ where: { projectId },
+ create: { projectId, ...data, createdBy: userId, updatedBy: userId },
+ update: { ...data, updatedBy: userId },
+ });
+}
+
+const TEST_SCHEMAS = {
+ summary: z.object({
+ summary: z.string(),
+ status: z.enum(["applicable", "not_applicable", "insufficient_input"]),
+ }),
+ clustering: z.object({ name: z.string(), description: z.string() }),
+};
+
+/**
+ * Makes one real call per configured slot, like the evaluator model check:
+ * structured output for summaries and clustering, and an embedding whose
+ * length must match the configured dimensions. A slot with no error is omitted.
+ */
+async function testTopicsModelSlots(
+ settings: TopicsModelSettings,
+ keys: LlmApiKeys[],
+): Promise<Partial<Record<TopicsModelSlotName, string>>> {
+ const errors: Partial<Record<TopicsModelSlotName, string>> = {};
+ await Promise.all(
+ TOPICS_MODEL_SLOTS.map(async (name) => {
+ const slot = settings[name];
+ if (!slot) return;
+ const llmApiKey = keys.find(
+ (candidate) => candidate.id === slot.llmApiKeyId,
+ );
+ if (!llmApiKey) {
+ errors[name] = "LLM connection not found in this project.";
+ return;
+ }
+ try {
+ if (name !== "embedding") {
+ await testModelCall({
+ provider: llmApiKey.provider,
+ model: slot.model,
+ apiKey: llmApiKey as z.infer<typeof LLMApiKeySchema>,
+ structuredOutputSchema: TEST_SCHEMAS[name],
+ timeout: getClientInitiatedNonStreamingLlmTimeoutMs(),
+ });
+ return;
+ }
+ const resolved = resolveSlot(name, llmApiKey, slot.model);
+ if (!resolved) {
+ errors[name] = "LLM connection not found in this project.";
+ return;
+ }
+ const { embedding } = await generateTopicEmbedding({
+ model: resolved,
+ summary: "Topics model check",
+ dimensions: settings.embeddingDimensions,
+ });
+ if (embedding.length !== settings.embeddingDimensions)
+ throw new Error(
+ `The model returned ${embedding.length} dimensions instead of ${settings.embeddingDimensions}. Choose a size this model supports.`,
+ );
+ } catch (error) {
+ errors[name] =
+ error instanceof Error
+ ? error.message.slice(0, 300)
+ : "The test call failed.";
+ }
+ }),
+ );
+ return errors;
+}
+
+async function testTopicsModels(
+ settings: TopicsModelSettings,
+ keys: LlmApiKeys[],
+): Promise<void> {
+ const errors = await testTopicsModelSlots(settings, keys);
+ const failures = TOPICS_MODEL_SLOTS.flatMap((name) => {
+ const message = errors[name];
+ const slot = settings[name];
+ return message && slot
+ ? [`${TOPICS_MODEL_SLOT_DETAILS[name].label} (${slot.model}): ${message}`]
+ : [];
+ });
+ if (failures.length)
+ throw new InvalidRequestError(
+ `The test call failed for ${failures.join("; ")}`,
+ );
+}
diff --git a/packages/shared/src/server/topics/postgres.ts b/packages/shared/src/server/topics/postgres.ts
index 63fd9d8443..105784bbc2 100644
--- a/packages/shared/src/server/topics/postgres.ts
+++ b/packages/shared/src/server/topics/postgres.ts
@@ -1,6 +1,7 @@
import { prisma, Prisma } from "../../db";
import {
- topicRuleConfigSchema,
+ DEFAULT_TOPIC_FACETS,
+ topicRuleSettingsSchema,
topicTimeRangeSchema,
sameTopicGeometry,
type TopicFacet,
@@ -127,117 +128,7 @@ export async function createTopicFacetVersion(
export async function ensureDefaultTopicFacets(
projectId: string,
): Promise<TopicFacet[]> {
- const presets = [
- {
- name: "Intent",
- description: "What the run was asked to do.",
- prompt: `Describe what the user, or the calling application, wanted from this run as a whole.
-
-Format: an imperative verb phrase stating the goal, such as "Find…", "Fix…", "Summarize…".
-
-- Describe the goal of the whole run, not its last step. Follow-up requests to verify, save, show, fix, or reformat earlier work belong to the goal they serve.
-- A run can contain several unrelated requests, for example a user who finishes one job and then starts another. Name each of them in a few words, in the order they were asked, joined with "and". Describing only the latest request misses the earlier ones.
-- Earlier conversation can clarify the goals of this run but does not add goals of its own.
-- In an automated run with no human author (extraction, classification, routing, templated generation), describe the job the input sets. Do not invent a person asking for it.
-- When the user asks the assistant to work on supplied material, describe that work and name the material's subject in a few words.
-- Keep the user's own verb and object. "Fix the date filter in a SQL query" must not become "Improve a data workflow".
-- Describe goals only. Leave out how the assistant approached them, what went wrong, and whether they succeeded.
-- Social or casual messages have a goal too; state it plainly.
-
-Status: applicable whenever a goal can be identified, even if the run failed or produced nothing. insufficient_input when no request or input survives. Do not use not_applicable.
-
-Examples:
-- Export last quarter's orders to CSV grouped by region.
-- Classify an inbound support email by product area and urgency.
-- Fix a failing nightly data import and draft a welcome email for new customers.
-- Chat about plans for the weekend.`,
- },
- {
- name: "Sentiment",
- description: "How the end user felt about the interaction.",
- prompt: `Describe how the end user felt about this interaction and what the feeling was directed at.
-
-Format: "<Label>: <cue and its target>", where Label is one of:
-- Positive: thanks, praise, relief, or warm engagement.
-- Negative: frustration, annoyance, distrust, or giving up.
-- Mixed: both are substantial, for example annoyance that turns into relief.
-- Neutral: factual or procedural messages with no emotional signal.
-
-- Judge only text the end user wrote. Assistant, tool, and system text shows what the user reacts to, never how the user feels.
-- Judge the attitude toward the interaction, not the subject. A calm report of a bug, a failed test, or a personal hardship is not negative by itself.
-- Weight sustained tone and how the user ends over a single remark. Terse or strict instructions are Neutral; so is routine politeness.
-- Name the target in terms of the assistant's behavior (wrong answers, slow progress, ignored corrections, a working fix), not the user's private circumstances.
-
-Status: not_applicable when the run has no end-user text, as in automated pipelines. insufficient_input when user text exists but is unreadable.
-
-Examples:
-- Negative: user repeated the same formatting correction and said the assistant keeps ignoring it.
-- Positive: user thanked the assistant after a working fix for a failing deployment script.
-- Mixed: user was annoyed by two wrong answers, then relieved when the third one worked.
-- Neutral: user gave step-by-step instructions without emotional cues.`,
- },
- {
- name: "Outcome",
- description: "Where the run ended and whether that is confirmed.",
- prompt: `Describe where this run ended: what was delivered or done, and whether the transcript confirms it.
-
-Format: "<State>: <the concrete result>", where State is one of:
-- Completed: the requested answer was given, or a tool result confirms the requested action.
-- Partial: part of the request was delivered and part was not.
-- Unconfirmed: the assistant says an action happened, but no result confirms it.
-- Needs input: the run ends waiting on the user, such as a clarifying question or an approval.
-- Not completed: the run stopped, was blocked, or was declined without delivering the request.
-
-- Start from the end of this run: the last assistant message and the last tool results decide the state.
-- Base the state on results, not on the assistant's words. A tool result confirms an action; a message saying "done" does not. A returned draft is a delivered draft, not a sent message.
-- For questions, explanations, and analyses, the delivered answer is the result.
-- Name the concrete deliverable or stopping point, not just the state. Mention remaining work only when the transcript shows it.
-- Say where the run stopped, not why something failed, unless the reason is itself the result, as with a declined request.
-
-Status: applicable whenever the run has a final response or result, including failures. insufficient_input when no response or result survives. Do not use not_applicable.
-
-Examples:
-- Completed: returned a SQL query that filters orders by signup month.
-- Unconfirmed: said the meeting was booked, but no calendar tool result confirms it.
-- Needs input: asked which of two accounts the transfer should come from.
-- Not completed: stopped after the payments tool failed, without answering the user.`,
- },
- {
- name: "Issues",
- description: "The main problem in how the run was handled.",
- prompt: `Describe the problem that did the most damage to the result of this run: the mistake that made the final answer or action wrong, unsupported, or missing.
-
-Format: one sentence that names the mistake, the kind of step where it happened, and its consequence, for example "Answered from memory instead of querying the dataset, so the reported value is unverified."
-- Describe the mechanism in plain, generic words: what the assistant did wrong and how, such as answering from memory, inventing data, claiming a check that never ran, choosing the wrong tool, breaking a required format, or stopping before the answer. Use the words that fit this run.
-- Name the kind of step (a web search, a file read, a calculation, a code change, the final answer), not the task's subject. "Skipped the file read and guessed the count" is right; "gave a wrong count of crustacean slides" is not.
-- Do not start with a label or category.
-
-How to find it:
-- Start from the end. Check the final answer or action against the request: is it on target, complete, and backed by what the run actually retrieved or did? Then work back to the step that caused the gap.
-- Report the cause with its consequence, not the symptom. When a tool fails and the assistant then answers anyway from guesses or invented data, the problem is the unsupported answer, not the tool failure.
-- When several problems occur, report the one with the biggest effect on the result. Prefer, in this order: an answer or reported result that is wrong or not backed by the run; a task left unfinished or answered off target; a wrong tool, approach, or argument; a broken instruction or format; a tool failure that blocked the result. A problem the assistant fully recovered from counts only when nothing worse happened.
-- Keep the symptoms of one problem together; do not list separate problems.
-
-Before you return not_applicable, check these four points. Return not_applicable only when all of them hold:
-1. When the request depends on data, files, tools, or current facts that the run had to look up, every such fact, number, or result in the final answer comes from a tool result or user input in this run, not from memory or assumption. General knowledge, explanations, and writing do not need a lookup.
-2. Every tool call used a tool suited to its input, such as a file tool on a local file rather than a web address, with valid arguments.
-3. The run followed the output format, tags, and steps that the system prompt or the assistant's own plan required.
-4. The final answer addresses exactly what was asked: the right quantity, unit, entity, and scope.
-
-- Only problems in this run count. An error the user pastes for explanation, a problem in earlier conversation, or a complaint about something outside the run is not an issue here.
-- A clarifying question, a justified refusal, a short answer, and a retry that succeeds are not issues.
-- Check the end of this run before calling it unfinished: if a later assistant message delivers an answer or fallback, the run is finished.
-- Report only what the transcript shows directly. A run-specific fact that needed a lookup but has none is shown directly. A problem that is only a possibility is not.
-
-Status: not_applicable when the run is complete enough to judge and shows no problem. insufficient_input when too much is missing to judge, for example only the first request survives.
-
-Examples:
-- Answered from a made-up example after the data file could not be read, so the reported result is invented.
-- Reported the total instead of the requested minimum, so the answer misses the question.
-- Searched the web for figures the attached spreadsheet contained, so the answer used outdated data.
-- Every inventory lookup timed out, so no quote could be produced and the user was asked to retry later.`,
- },
- ];
+ const presets = DEFAULT_TOPIC_FACETS;
const facets = await listTopicFacets(projectId);
if (
presets.every((preset) =>
@@ -276,7 +167,10 @@ const topicRule = (row: RuleRow): TopicRule => ({
id: row.id,
projectId: row.projectId,
name: row.name,
- ...topicRuleConfigSchema.parse(row),
+ ...topicRuleSettingsSchema.parse({
+ filter: row.filter,
+ sampling: Number(row.sampling),
+ }),
facetIds: row.assignments.map((assignment) => assignment.evaluatorId),
updatedAt: row.updatedAt.toISOString(),
});
@@ -313,62 +207,99 @@ export async function getTopicRule(
return row ? topicRule(row) : null;
}
-export async function saveTopicRule(
- input: Omit<TopicRule, "id" | "updatedAt"> & { id?: string },
+/**
+ * A project has at most one Topics rule: its trace filter, sampling, and idle
+ * time apply to every facet it runs, because trace readiness keeps one idle
+ * time per trace. Postgres has no constraint for this, so writers hold this
+ * per-project lock while they look up the rule; otherwise two first saves
+ * could each find none and create a rule.
+ */
+async function lockProjectTopicRule(
+ tx: Prisma.TransactionClient,
+ projectId: string,
+) {
+ await tx.$executeRaw`SELECT pg_advisory_xact_lock(hashtextextended(${`topics-rule:${projectId}`}, 0))`;
+}
+
+export type TopicRuleInput = Omit<
+ TopicRule,
+ "id" | "name" | "updatedAt" | "sampling"
+> &
+ Partial<Pick<TopicRule, "name" | "sampling">> & {
+ id?: string;
+ /** The user saving the rule; recorded when it is created. */
+ userId?: string | null;
+ };
+
+export async function saveTopicRule(input: TopicRuleInput): Promise<TopicRule> {
+ return prisma.$transaction((tx) => writeTopicRule(tx, input));
+}
+
+/** Writes the project's rule inside the caller's transaction, under its lock. */
+export async function writeTopicRule(
+ tx: Prisma.TransactionClient,
+ input: TopicRuleInput,
): Promise<TopicRule> {
const facetIds = [...new Set(input.facetIds)];
- return prisma.$transaction(async (tx) => {
- const facetCount = await tx.evaluator.count({
- where: {
- projectId: input.projectId,
- type: "FACET",
- id: { in: facetIds },
- },
- });
- if (!facetIds.length || facetCount !== facetIds.length)
- throw new InvalidRequestError("Select facets from this project.");
- if (
- input.id &&
- !(await tx.evaluationRule.findFirst({
- where: { ...topicRuleWhere(input.projectId), id: input.id },
- }))
- )
- throw new InvalidRequestError("Topic rule not found in this project.");
- const config = topicRuleConfigSchema.parse(input);
- const data = {
- name: input.name.trim(),
- filter: JSON.parse(
- JSON.stringify(config.filter),
- ) as Prisma.InputJsonValue,
- sampling: 1,
- delay: 0,
- targetObject: EvalTargetObject.TRACE,
- timeScope: ["NEW"],
- status: "ACTIVE" as const,
- };
- const assignments = facetIds.map((evaluatorId) => ({
+ await lockProjectTopicRule(tx, input.projectId);
+ const facetCount = await tx.evaluator.count({
+ where: {
projectId: input.projectId,
- evaluatorId,
- }));
- const row = input.id
- ? await tx.evaluationRule.update({
- where: { ...topicRuleWhere(input.projectId), id: input.id },
- data: {
- ...data,
- assignments: { deleteMany: {}, create: assignments },
- },
- include: { assignments: true },
- })
- : await tx.evaluationRule.create({
- data: {
- ...data,
- projectId: input.projectId,
- assignments: { create: assignments },
- },
- include: { assignments: true },
- });
- return topicRule(row);
+ type: "FACET",
+ id: { in: facetIds },
+ },
});
+ if (!facetIds.length || facetCount !== facetIds.length)
+ throw new InvalidRequestError("Select facets from this project.");
+ const rules = await tx.evaluationRule.findMany({
+ where: topicRuleWhere(input.projectId),
+ select: { id: true, name: true },
+ });
+ if (rules.length > 1)
+ throw new InvalidRequestError(
+ "This project has more than one Topics rule.",
+ );
+ const ruleId = rules[0]?.id;
+ const name = input.name?.trim() || rules[0]?.name || "Topics";
+ if (ruleId && !input.id)
+ throw new InvalidRequestError(
+ "This project already has a Topics rule. Reload and save that rule instead.",
+ );
+ if (input.id && input.id !== ruleId)
+ throw new InvalidRequestError("Topic rule not found in this project.");
+ const config = topicRuleSettingsSchema.parse(input);
+ const data = {
+ name,
+ filter: JSON.parse(JSON.stringify(config.filter)) as Prisma.InputJsonValue,
+ sampling: config.sampling,
+ delay: 0,
+ targetObject: EvalTargetObject.TRACE,
+ timeScope: ["NEW"],
+ status: "ACTIVE" as const,
+ };
+ const assignments = facetIds.map((evaluatorId) => ({
+ projectId: input.projectId,
+ evaluatorId,
+ }));
+ const row = ruleId
+ ? await tx.evaluationRule.update({
+ where: { ...topicRuleWhere(input.projectId), id: ruleId },
+ data: {
+ ...data,
+ assignments: { deleteMany: {}, create: assignments },
+ },
+ include: { assignments: true },
+ })
+ : await tx.evaluationRule.create({
+ data: {
+ ...data,
+ projectId: input.projectId,
+ createdByUserId: input.userId ?? null,
+ assignments: { create: assignments },
+ },
+ include: { assignments: true },
+ });
+ return topicRule(row);
}
function runResult(
diff --git a/packages/shared/src/server/topics/setup.test.ts b/packages/shared/src/server/topics/setup.test.ts
new file mode 100644
index 0000000000..1a6ceaa840
--- /dev/null
+++ b/packages/shared/src/server/topics/setup.test.ts
@@ -0,0 +1,156 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { InvalidRequestError } from "../../errors";
+import { DEFAULT_TOPIC_FACETS } from "../../topics";
+
+const mocks = vi.hoisted(() => ({
+ prepareModels: vi.fn(),
+ writeModels: vi.fn(),
+ ensureFacets: vi.fn(),
+ saveRule: vi.fn(),
+ writeReadiness: vi.fn(),
+ cacheReadiness: vi.fn(),
+}));
+
+vi.mock("./model-config", () => ({
+ prepareTopicsModelSettings: mocks.prepareModels,
+ writeTopicsModelSettings: mocks.writeModels,
+}));
+vi.mock("./postgres", () => ({
+ ensureDefaultTopicFacets: mocks.ensureFacets,
+ writeTopicRule: mocks.saveRule,
+}));
+vi.mock("../tracingReadinessSettings", () => ({
+ writeTracingReadinessSettings: mocks.writeReadiness,
+ cacheTracingReadinessSettings: mocks.cacheReadiness,
+}));
+vi.mock("../../db", () => ({
+ prisma: {
+ $transaction: (callback: (tx: string) => Promise<unknown>) =>
+ callback("tx"),
+ },
+}));
+
+import { saveTopicsSetup } from "./setup";
+
+const projectId = "project-a";
+const facets = DEFAULT_TOPIC_FACETS.map((facet, index) => ({
+ id: facet.name.toLowerCase(),
+ projectId,
+ name: facet.name,
+ description: facet.description,
+ isBuiltIn: true,
+ versions: [],
+ enabled: index !== 1,
+}));
+
+const setup = {
+ summary: { llmApiKeyId: "key", model: "gpt" },
+ embedding: { llmApiKeyId: "key", model: "embed" },
+ embeddingDimensions: 1024,
+ clustering: { llmApiKeyId: "key", model: "cluster" },
+ enabled: true,
+ facets: facets.map((facet) => ({
+ name: facet.name,
+ enabled: facet.enabled,
+ })),
+ filter: [{ column: "name", type: "string", operator: "=", value: "billing" }],
+ sampling: 0.25,
+ traceIdleMs: 120_000,
+} satisfies Parameters<typeof saveTopicsSetup>[1];
+
+describe("Topics setup", () => {
+ beforeEach(() => {
+ vi.resetAllMocks();
+ mocks.ensureFacets.mockResolvedValue(facets);
+ mocks.saveRule.mockResolvedValue({ id: "rule-a" });
+ mocks.prepareModels.mockResolvedValue({ enabled: true });
+ });
+
+ it("creates the built-in facets and one rule for the enabled ones", async () => {
+ await saveTopicsSetup(projectId, setup, "user-a");
+
+ expect(mocks.prepareModels).toHaveBeenCalledWith(projectId, {
+ summary: setup.summary,
+ embedding: setup.embedding,
+ embeddingDimensions: 1024,
+ clustering: setup.clustering,
+ enabled: true,
+ });
+ expect(mocks.ensureFacets).toHaveBeenCalledWith(projectId);
+ expect(mocks.saveRule).toHaveBeenCalledWith("tx", {
+ projectId,
+ filter: setup.filter,
+ sampling: 0.25,
+ facetIds: facets.flatMap((facet) => (facet.enabled ? [facet.id] : [])),
+ userId: "user-a",
+ });
+ expect(mocks.writeModels).toHaveBeenCalledWith(
+ projectId,
+ { enabled: true },
+ { userId: "user-a", tx: "tx" },
+ );
+ expect(mocks.writeReadiness).toHaveBeenCalledWith(
+ "tx",
+ projectId,
+ { traceIdleMs: 120_000 },
+ "user-a",
+ );
+ expect(mocks.cacheReadiness).toHaveBeenCalledWith(projectId, {
+ traceIdleMs: 120_000,
+ });
+ });
+
+ it("enables the models only after the rule is saved", async () => {
+ mocks.saveRule.mockRejectedValue(
+ new InvalidRequestError("This project has more than one Topics rule."),
+ );
+
+ await expect(saveTopicsSetup(projectId, setup)).rejects.toThrow(
+ "more than one Topics rule",
+ );
+ expect(mocks.prepareModels).toHaveBeenCalledOnce();
+ expect(mocks.writeModels).not.toHaveBeenCalled();
+ expect(mocks.writeReadiness).not.toHaveBeenCalled();
+ });
+
+ it("keeps enabled custom facets assigned to the rule", async () => {
+ await saveTopicsSetup(projectId, {
+ ...setup,
+ customFacetIds: ["custom-language"],
+ });
+
+ expect(mocks.saveRule).toHaveBeenCalledWith(
+ "tx",
+ expect.objectContaining({
+ facetIds: expect.arrayContaining(["intent", "custom-language"]),
+ }),
+ );
+ });
+
+ it("rejects a setup with every facet turned off", async () => {
+ await expect(
+ saveTopicsSetup(projectId, {
+ ...setup,
+ facets: setup.facets.map((facet) => ({ ...facet, enabled: false })),
+ }),
+ ).rejects.toThrow(/at least one facet/);
+ expect(mocks.prepareModels).not.toHaveBeenCalled();
+ });
+
+ it("rejects a setup that leaves out a built-in facet", async () => {
+ await expect(
+ saveTopicsSetup(projectId, {
+ ...setup,
+ facets: setup.facets.slice(1),
+ }),
+ ).rejects.toThrow(/each built-in facet/);
+ expect(mocks.prepareModels).not.toHaveBeenCalled();
+ });
+
+ it("rejects an idle time below 60 seconds before testing models", async () => {
+ await expect(
+ saveTopicsSetup(projectId, { ...setup, traceIdleMs: 59_000 }),
+ ).rejects.toThrow(/at least 60 seconds/);
+ expect(mocks.prepareModels).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/shared/src/server/topics/setup.ts b/packages/shared/src/server/topics/setup.ts
new file mode 100644
index 0000000000..7e5e307016
--- /dev/null
+++ b/packages/shared/src/server/topics/setup.ts
@@ -0,0 +1,89 @@
+import type { z } from "zod";
+import { InvalidRequestError } from "../../errors";
+import { topicsSetupSchema, type TopicRule } from "../../topics";
+import {
+ prepareTopicsModelSettings,
+ writeTopicsModelSettings,
+} from "./model-config";
+import { prisma } from "../../db";
+import {
+ cacheTracingReadinessSettings,
+ writeTracingReadinessSettings,
+} from "../tracingReadinessSettings";
+import {
+ ensureDefaultTopicFacets,
+ writeTopicRule,
+ type TopicRuleInput,
+} from "./postgres";
+
+/** One Topics setup: models, the four built-in facets, and a single rule. */
+export async function saveTopicsSetup(
+ projectId: string,
+ input: z.input<typeof topicsSetupSchema>,
+ userId: string | null = null,
+): Promise<TopicRule> {
+ const setup = topicsSetupSchema.parse(input);
+ const facets = await ensureDefaultTopicFacets(projectId);
+ const facetIdByName = new Map(
+ facets
+ .filter((facet) => facet.isBuiltIn)
+ .map((facet) => [facet.name, facet.id]),
+ );
+ const facetIds = setup.facets.flatMap((facet) => {
+ if (!facet.enabled) return [];
+ const id = facetIdByName.get(facet.name);
+ if (!id)
+ throw new InvalidRequestError(
+ `Built-in facet ${facet.name} was not created.`,
+ );
+ return [id];
+ });
+ facetIds.push(...setup.customFacetIds);
+ // The test calls hit the network, so they run before the transaction.
+ const models = await prepareTopicsModelSettings(projectId, {
+ summary: setup.summary,
+ embedding: setup.embedding,
+ embeddingDimensions: setup.embeddingDimensions,
+ clustering: setup.clustering,
+ enabled: setup.enabled,
+ });
+ return saveTopicRuleAndModels(
+ {
+ id: setup.ruleId,
+ projectId,
+ userId,
+ filter: setup.filter,
+ sampling: setup.sampling,
+ facetIds,
+ },
+ models,
+ setup.traceIdleMs,
+ );
+}
+
+/**
+ * Writes the rule and the model settings together: a failed model write must
+ * not leave a rule without models, and a failed rule write must not enable them.
+ */
+export async function saveTopicRuleAndModels(
+ rule: TopicRuleInput,
+ models: Parameters<typeof writeTopicsModelSettings>[1],
+ traceIdleMs: number,
+): Promise<TopicRule> {
+ const saved = await prisma.$transaction(async (tx) => {
+ const saved = await writeTopicRule(tx, rule);
+ await writeTopicsModelSettings(rule.projectId, models, {
+ userId: rule.userId,
+ tx,
+ });
+ await writeTracingReadinessSettings(
+ tx,
+ rule.projectId,
+ { traceIdleMs },
+ rule.userId,
+ );
+ return saved;
+ });
+ await cacheTracingReadinessSettings(rule.projectId, { traceIdleMs });
+ return saved;
+}
diff --git a/packages/shared/src/server/topics/text.test.ts b/packages/shared/src/server/topics/text.test.ts
index c5037f2201..5c6db7e120 100644
--- a/packages/shared/src/server/topics/text.test.ts
+++ b/packages/shared/src/server/topics/text.test.ts
@@ -1,113 +1,81 @@
-import { fromNodeProviderChain } from "@aws-sdk/credential-providers";
-import { afterEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { z } from "zod";
+import { encrypt } from "../../encryption";
+import { env } from "../../env";
+import { BEDROCK_USE_DEFAULT_CREDENTIALS } from "../../interfaces/customLLMProviderConfigSchemas";
+import { LLMAdapter } from "../llm/types";
+import { generateTopicEmbedding } from "./embeddings";
+import type { TopicsModel } from "./model-config";
import { generateTopicText } from "./text";
-vi.mock("@aws-sdk/credential-providers", () => ({
- fromNodeProviderChain: vi.fn(() => async () => ({
- accessKeyId: "topics-test-key",
- secretAccessKey: "topics-test-secret",
- sessionToken: "topics-test-session",
- })),
-}));
-
afterEach(() => {
vi.unstubAllGlobals();
vi.unstubAllEnvs();
- vi.clearAllMocks();
});
+function bedrockModel(slot: TopicsModel["slot"], model: string): TopicsModel {
+ return {
+ slot,
+ provider: "bedrock-topics",
+ adapter: LLMAdapter.Bedrock,
+ model,
+ connection: {
+ secretKey: encrypt(
+ JSON.stringify({ accessKeyId: "AKIATOPICS", secretAccessKey: "s" }),
+ ),
+ config: { region: "eu-central-1" },
+ },
+ };
+}
+
describe("generateTopicText", () => {
- it.each([
- "us.openai.gpt-5.6-luna",
- "us.openai.gpt-5.6-terra",
- "us.openai.gpt-6-luna",
- "global.openai.gpt-6-luna",
- ])(
- "serializes uncached %s structured output through Converse with the Topics profile",
- async (model) => {
- const schema = z.object({ summary: z.string() });
- const output = { summary: "Account access" };
- const fetch = vi.fn<typeof globalThis.fetch>(async () =>
- Response.json({
- output: {
- message: {
- role: "assistant",
- content: [
- {
- toolUse: { toolUseId: "json-1", name: "json", input: output },
- },
- ],
- },
+ it("sends structured output through the project's Bedrock connection", async () => {
+ const output = { summary: "Account access" };
+ const fetch = vi.fn<typeof globalThis.fetch>(async () =>
+ Response.json({
+ output: {
+ message: {
+ role: "assistant",
+ content: [
+ { toolUse: { toolUseId: "json-1", name: "json", input: output } },
+ ],
},
- stopReason: "tool_use",
- usage: { inputTokens: 12, outputTokens: 4, totalTokens: 16 },
- metrics: { latencyMs: 1 },
- }),
- );
- vi.stubGlobal("fetch", fetch);
- vi.stubEnv("AWS_BEARER_TOKEN_BEDROCK", "ambient-bearer-token");
-
- const result = await generateTopicText({
- model,
- system: [{ text: "Summarize the user's request." }],
- input: "I cannot sign in.",
- schema,
- maxOutputTokens: 256,
- region: "eu-central-1",
- profile: "topics-local",
- });
-
- expect(fromNodeProviderChain).toHaveBeenCalledWith({
- profile: "topics-local",
- });
- expect(fetch).toHaveBeenCalledTimes(1);
- const request = new Request(...fetch.mock.calls[0]);
- expect(request.url).toBe(
- `https://bedrock-runtime.eu-central-1.amazonaws.com/model/${model}/converse`,
- );
- expect(request.headers.get("authorization")).toMatch(
- /^AWS4-HMAC-SHA256 Credential=topics-test-key\//,
- );
- expect(request.headers.get("x-amz-security-token")).toBe(
- "topics-test-session",
- );
- const body = await request.json();
- expect(body).toMatchObject({
- system: [{ text: "Summarize the user's request." }],
- messages: [{ role: "user", content: [{ text: "I cannot sign in." }] }],
- inferenceConfig: { maxTokens: 256 },
- additionalModelRequestFields: { reasoning: { effort: "none" } },
- toolConfig: {
- tools: [
- {
- toolSpec: {
- name: "json",
- inputSchema: {
- json: {
- type: "object",
- properties: { summary: { type: "string" } },
- required: ["summary"],
- additionalProperties: false,
- },
- },
- },
- },
- ],
- toolChoice: { any: {} },
},
- });
- expect(body).not.toHaveProperty("inferenceConfig.temperature");
- expect(result.output).toEqual(output);
- expect(result.usage).toMatchObject({
- inputTokens: 12,
- outputTokens: 4,
- totalTokens: 16,
- });
- },
- );
+ stopReason: "tool_use",
+ usage: { inputTokens: 12, outputTokens: 4, totalTokens: 16 },
+ metrics: { latencyMs: 1 },
+ }),
+ );
+ vi.stubGlobal("fetch", fetch);
+ // Connection credentials must win over ambient worker credentials.
+ vi.stubEnv("AWS_BEARER_TOKEN_BEDROCK", "ambient-bearer-token");
+
+ const result = await generateTopicText({
+ model: bedrockModel("summary", "us.openai.gpt-6-luna"),
+ system: [{ text: "Summarize the user's request." }],
+ input: "I cannot sign in.",
+ schema: z.object({ summary: z.string() }),
+ maxOutputTokens: 256,
+ });
- it("sends cached prompts through InvokeModel at the configured region and inference profile", async () => {
+ expect(fetch).toHaveBeenCalledTimes(1);
+ const request = new Request(...fetch.mock.calls[0]);
+ expect(request.url).toBe(
+ "https://bedrock-runtime.eu-central-1.amazonaws.com/model/us.openai.gpt-6-luna/converse",
+ );
+ expect(request.headers.get("authorization")).toMatch(
+ /^AWS4-HMAC-SHA256 Credential=AKIATOPICS\//,
+ );
+ expect(await request.json()).toMatchObject({
+ system: [{ text: "Summarize the user's request." }],
+ inferenceConfig: { maxTokens: 256 },
+ additionalModelRequestFields: { reasoning: { effort: "none" } },
+ toolConfig: { toolChoice: { any: {} } },
+ });
+ expect(result.output).toEqual(output);
+ });
+
+ it("sends cached prompts through InvokeModel with the connection's credentials and region", async () => {
const fetch = vi.fn<typeof globalThis.fetch>(async () =>
Response.json({
choices: [
@@ -127,7 +95,7 @@ describe("generateTopicText", () => {
vi.stubGlobal("fetch", fetch);
const result = await generateTopicText({
- model: "us.openai.gpt-6-luna",
+ model: bedrockModel("summary", "us.openai.gpt-6-luna"),
system: [
{ text: "System prompt and built-in facets.", cache: true },
{ text: "Custom facets.", cache: true },
@@ -135,16 +103,14 @@ describe("generateTopicText", () => {
input: "I cannot sign in.",
schema: z.object({ summary: z.string() }),
maxOutputTokens: 256,
- region: "us-west-2",
- profile: "topics-local",
});
const request = new Request(...fetch.mock.calls[0]);
expect(request.url).toBe(
- "https://bedrock-runtime.us-west-2.amazonaws.com/model/us.openai.gpt-6-luna/invoke",
+ "https://bedrock-runtime.eu-central-1.amazonaws.com/model/us.openai.gpt-6-luna/invoke",
);
expect(request.headers.get("authorization")).toMatch(
- /^AWS4-HMAC-SHA256 Credential=topics-test-key\/\d{8}\/us-west-2\/bedrock\//,
+ /^AWS4-HMAC-SHA256 Credential=AKIATOPICS\/\d{8}\/eu-central-1\/bedrock\//,
);
const body = (await request.json()) as {
messages: { content: unknown }[];
@@ -166,15 +132,12 @@ describe("generateTopicText", () => {
{ role: "user", content: [{ type: "text", text: "I cannot sign in." }] },
]);
expect(body.prompt_cache_options).toEqual({ mode: "explicit", ttl: "30m" });
- expect(result).toEqual({
- output: { summary: "Account access" },
- usage: {
- inputTokens: 3653,
- outputTokens: 275,
- totalTokens: 3928,
- cacheReadTokens: 2840,
- cacheWriteTokens: 0,
- },
+ expect(result.usage).toEqual({
+ inputTokens: 3653,
+ outputTokens: 275,
+ totalTokens: 3928,
+ cacheReadTokens: 2840,
+ cacheWriteTokens: 0,
});
});
@@ -185,31 +148,214 @@ describe("generateTopicText", () => {
);
await expect(
generateTopicText({
- model: "us.openai.gpt-6-luna",
+ model: bedrockModel("summary", "us.openai.gpt-6-luna"),
system: [{ text: "Summarize.", cache: true }],
input: "Hi",
schema: z.object({ summary: z.string() }),
maxOutputTokens: 256,
- region: "us-west-2",
}),
).rejects.toMatchObject({ status: 429 });
});
- it("rejects a host-reshaping region before resolving credentials or sending a request", async () => {
- const fetch = vi.fn();
+ it("reports cached prompt tokens from providers that cache automatically", async () => {
+ vi.stubGlobal(
+ "fetch",
+ vi.fn(async () =>
+ Response.json({
+ id: "chatcmpl-1",
+ object: "chat.completion",
+ created: 1,
+ model: "gpt-6-luna",
+ choices: [
+ {
+ index: 0,
+ finish_reason: "stop",
+ message: {
+ role: "assistant",
+ content: JSON.stringify({ summary: "Account access" }),
+ },
+ },
+ ],
+ usage: {
+ prompt_tokens: 3000,
+ completion_tokens: 200,
+ total_tokens: 3200,
+ prompt_tokens_details: { cached_tokens: 2560 },
+ },
+ }),
+ ),
+ );
+
+ const result = await generateTopicText({
+ model: {
+ slot: "summary",
+ provider: "openai-topics",
+ adapter: LLMAdapter.OpenAI,
+ model: "gpt-6-luna",
+ connection: { secretKey: encrypt("sk-test") },
+ },
+ system: [{ text: "System prompt and facets.", cache: true }],
+ input: "I cannot sign in.",
+ schema: z.object({ summary: z.string() }),
+ maxOutputTokens: 256,
+ });
+
+ expect(result.usage).toMatchObject({
+ inputTokens: 3000,
+ cacheReadTokens: 2560,
+ });
+ });
+});
+
+describe("generateTopicEmbedding", () => {
+ it("requests clustering vectors of the configured size from Cohere on Bedrock", async () => {
+ const vector = Array.from({ length: 512 }, (_, index) => index + 1);
+ const fetch = vi.fn<typeof globalThis.fetch>(async () =>
+ Response.json(
+ { embeddings: { float: [vector] }, texts: ["Account access"] },
+ { headers: { "x-amzn-bedrock-input-token-count": "3" } },
+ ),
+ );
+ vi.stubGlobal("fetch", fetch);
+
+ const result = await generateTopicEmbedding({
+ model: bedrockModel("embedding", "eu.cohere.embed-v4:0"),
+ summary: "Account access",
+ dimensions: 512,
+ });
+
+ const request = new Request(...fetch.mock.calls[0]);
+ expect(decodeURIComponent(request.url)).toBe(
+ "https://bedrock-runtime.eu-central-1.amazonaws.com/model/eu.cohere.embed-v4:0/invoke",
+ );
+ expect(await request.json()).toEqual({
+ input_type: "clustering",
+ texts: ["Account access"],
+ truncate: "NONE",
+ output_dimension: 512,
+ });
+ expect(result.embedding).toEqual(vector);
+ });
+
+ it("asks OpenAI for any configured size", async () => {
+ const vector = Array.from({ length: 768 }, () => 0.5);
+ const fetch = vi.fn<typeof globalThis.fetch>(async () =>
+ Response.json({
+ data: [{ embedding: vector, index: 0 }],
+ usage: { prompt_tokens: 3 },
+ }),
+ );
vi.stubGlobal("fetch", fetch);
+ const result = await generateTopicEmbedding({
+ model: {
+ slot: "embedding",
+ provider: "openai-topics",
+ adapter: LLMAdapter.OpenAI,
+ model: "text-embedding-3-small",
+ connection: { secretKey: encrypt("sk-test") },
+ },
+ summary: "Account access",
+ dimensions: 768,
+ });
+
+ const request = new Request(...fetch.mock.calls[0]);
+ expect(request.url).toBe("https://api.openai.com/v1/embeddings");
+ expect(await request.json()).toMatchObject({
+ model: "text-embedding-3-small",
+ input: ["Account access"],
+ dimensions: 768,
+ });
+ expect(result.embedding).toHaveLength(768);
+ });
+});
+
+describe("allowlisted Cloud Bedrock runtime credentials", () => {
+ const originalEnv = {
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION,
+ LANGFUSE_BEDROCK_LLM_CONNECTION_RUNTIME_CREDENTIALS_ALLOWED_PROJECT_IDS:
+ env.LANGFUSE_BEDROCK_LLM_CONNECTION_RUNTIME_CREDENTIALS_ALLOWED_PROJECT_IDS,
+ };
+
+ beforeEach(() => {
+ env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION = "eu";
+ env.LANGFUSE_BEDROCK_LLM_CONNECTION_RUNTIME_CREDENTIALS_ALLOWED_PROJECT_IDS =
+ ["allowed"];
+ vi.stubEnv("AWS_PROFILE", undefined);
+ vi.stubEnv("AWS_ACCESS_KEY_ID", "AKIARUNTIME");
+ vi.stubEnv("AWS_SECRET_ACCESS_KEY", "runtime-secret");
+ });
+
+ afterEach(() => Object.assign(env, originalEnv));
+
+ const runtimeModel = (slot: TopicsModel["slot"], projectId: string) => {
+ const model = bedrockModel(
+ slot,
+ slot === "embedding" ? "eu.cohere.embed-v4:0" : "us.openai.gpt-6-luna",
+ );
+ model.connection = {
+ ...model.connection,
+ projectId,
+ secretKey: encrypt(BEDROCK_USE_DEFAULT_CREDENTIALS),
+ };
+ return model;
+ };
+
+ it("uses the existing project allowlist for cached text", async () => {
+ const fetch = vi.fn<typeof globalThis.fetch>(async () =>
+ Response.json({
+ choices: [
+ {
+ finish_reason: "stop",
+ message: { content: JSON.stringify({ summary: "Account access" }) },
+ },
+ ],
+ usage: { prompt_tokens: 10, completion_tokens: 3, total_tokens: 13 },
+ }),
+ );
+ vi.stubGlobal("fetch", fetch);
+ const input = {
+ system: [{ text: "Summarize.", cache: true }],
+ input: "I cannot sign in.",
+ schema: z.object({ summary: z.string() }),
+ maxOutputTokens: 256,
+ };
+
+ await generateTopicText({
+ ...input,
+ model: runtimeModel("summary", "allowed"),
+ });
+ expect(
+ new Request(...fetch.mock.calls[0]).headers.get("authorization"),
+ ).toContain("Credential=AKIARUNTIME/");
await expect(
- generateTopicText({
- model: "us.openai.gpt-5.6-luna",
- system: [{ text: "Summarize.", cache: true }],
- input: "Hi",
- schema: z.object({ summary: z.string() }),
- maxOutputTokens: 256,
- region: "eu-central-1.attacker.test",
+ generateTopicText({ ...input, model: runtimeModel("summary", "other") }),
+ ).rejects.toThrow("Invalid Bedrock credentials");
+ expect(fetch).toHaveBeenCalledTimes(1);
+ });
+
+ it("uses the existing project allowlist for embeddings", async () => {
+ const vector = Array.from({ length: 512 }, () => 0.1);
+ const fetch = vi.fn<typeof globalThis.fetch>(async () =>
+ Response.json(
+ { embeddings: { float: [vector] }, texts: ["Account access"] },
+ { headers: { "x-amzn-bedrock-input-token-count": "3" } },
+ ),
+ );
+ vi.stubGlobal("fetch", fetch);
+ const input = { summary: "Account access", dimensions: 512 };
+
+ await generateTopicEmbedding({
+ ...input,
+ model: runtimeModel("embedding", "allowed"),
+ });
+ expect(fetch).toHaveBeenCalledTimes(1);
+ await expect(
+ generateTopicEmbedding({
+ ...input,
+ model: runtimeModel("embedding", "other"),
}),
- ).rejects.toThrow("Invalid Bedrock region");
- expect(fromNodeProviderChain).not.toHaveBeenCalled();
- expect(fetch).not.toHaveBeenCalled();
+ ).rejects.toThrow("Invalid Bedrock credentials");
+ expect(fetch).toHaveBeenCalledTimes(1);
});
});
diff --git a/packages/shared/src/server/topics/text.ts b/packages/shared/src/server/topics/text.ts
index db0209f8f3..81294b07a8 100644
--- a/packages/shared/src/server/topics/text.ts
+++ b/packages/shared/src/server/topics/text.ts
@@ -1,13 +1,17 @@
-import { createAmazonBedrock } from "@ai-sdk/amazon-bedrock";
-import { fromNodeProviderChain } from "@aws-sdk/credential-providers";
import { Sha256 } from "@aws-crypto/sha256-js";
import { SignatureV4 } from "@smithy/signature-v4";
-import { generateText, Output } from "ai";
+import { Output } from "ai";
import { z } from "zod";
+import { decrypt } from "../../encryption";
+import { BedrockConfigSchema } from "../../interfaces/customLLMProviderConfigSchemas";
import {
assertValidBedrockRegion,
- createDefaultBedrockProviderAuth,
+ canUseBedrockDefaultCredentials,
+ resolveBedrockProviderAuth,
} from "../llm/ai-sdk/providers/bedrock";
+import { generateLLMText } from "../llm/llmText";
+import { LLMAdapter } from "../llm/types";
+import type { TopicsModel } from "./model-config";
/** A system prompt segment; `cache` ends a reusable prefix with a cache breakpoint. */
export type TopicPromptPart = { text: string; cache?: boolean };
@@ -25,31 +29,30 @@ type TopicTextResult = {
const TIMEOUT_MS = 60_000;
+const isBedrockOpenAI = (model: TopicsModel) =>
+ model.adapter === LLMAdapter.Bedrock && /(^|\.)openai\./.test(model.model);
+
/**
- * Uses AWS credentials for Topics structured text. GPT-5.6+ models on Bedrock
- * reuse a shared prompt prefix only with explicit cache breakpoints, which
- * Converse does not accept for them, so cached prompts on OpenAI models use
- * InvokeModel at the same regional endpoint and model ID; others use Converse.
+ * Structured Topics text through the project's LLM connection. GPT-5.6+ models
+ * on Bedrock reuse a shared prompt prefix only with explicit cache breakpoints,
+ * which Converse does not accept for them, so cached prompts on those models
+ * use InvokeModel at the same regional endpoint and model ID.
*/
export async function generateTopicText<T>(params: {
- model: string;
+ model: TopicsModel;
system: TopicPromptPart[];
input: string;
schema: z.ZodType<T>;
maxOutputTokens: number;
- region: string;
- profile?: string;
}): Promise<TopicTextResult> {
- assertValidBedrockRegion(params.region);
- if (/(^|\.)openai\./.test(params.model) && params.system.some((p) => p.cache))
+ if (isBedrockOpenAI(params.model) && params.system.some((p) => p.cache))
return invokeWithCache(params);
- const provider = createAmazonBedrock({
- region: params.region,
- ...createDefaultBedrockProviderAuth({ profile: params.profile }),
- });
- const result = await generateText({
- model: provider(params.model),
+ const result = await generateLLMText({
+ model: { adapter: params.model.adapter, id: params.model.model },
+ connection: params.model.connection,
+ // The static system prompt comes first so providers with automatic prefix
+ // caching can reuse it across traces.
messages: [
{
role: "system",
@@ -57,15 +60,22 @@ export async function generateTopicText<T>(params: {
},
{ role: "user", content: params.input },
],
- allowSystemInMessages: true,
output: Output.object({ schema: params.schema }),
maxOutputTokens: params.maxOutputTokens,
- providerOptions: {
- bedrock: {
- additionalModelRequestFields: { reasoning: { effort: "none" } },
- },
- },
- maxRetries: 0,
+ reasoning: "none",
+ // Bedrock ignores portable reasoning for OpenAI models; their default effort
+ // is not none.
+ ...(isBedrockOpenAI(params.model)
+ ? {
+ providerOptions: {
+ bedrock: {
+ additionalModelRequestFields: { reasoning: { effort: "none" } },
+ },
+ },
+ }
+ : {}),
+ // The AI SDK retries 429 and 5xx responses with exponential backoff.
+ maxRetries: 2,
timeout: TIMEOUT_MS,
});
return {
@@ -74,22 +84,87 @@ export async function generateTopicText<T>(params: {
inputTokens: result.usage.inputTokens,
outputTokens: result.usage.outputTokens,
totalTokens: result.usage.totalTokens,
+ cacheReadTokens: result.usage.inputTokenDetails?.cacheReadTokens,
+ cacheWriteTokens: result.usage.inputTokenDetails?.cacheWriteTokens,
},
};
}
+function signingCredentials(
+ auth: ReturnType<typeof resolveBedrockProviderAuth>,
+) {
+ const { credentialProvider } = auth;
+ if (credentialProvider)
+ return async () => {
+ const resolved = await credentialProvider();
+ return {
+ accessKeyId: resolved.accessKeyId,
+ secretAccessKey: resolved.secretAccessKey,
+ sessionToken: resolved.sessionToken,
+ };
+ };
+ if (auth.accessKeyId && auth.secretAccessKey)
+ return {
+ accessKeyId: auth.accessKeyId,
+ secretAccessKey: auth.secretAccessKey,
+ };
+ return undefined;
+}
+
+/** SigV4 or bearer-token auth from the connection's stored Bedrock credential. */
+async function bedrockRequestHeaders(
+ model: TopicsModel,
+ url: URL,
+ region: string,
+ body: string,
+): Promise<Record<string, string>> {
+ const headers = {
+ host: url.hostname,
+ "content-type": "application/json",
+ accept: "application/json",
+ };
+ const auth = resolveBedrockProviderAuth({
+ secretKey: decrypt(model.connection.secretKey),
+ allowDefaultCredentials: canUseBedrockDefaultCredentials(
+ model.connection.projectId,
+ ),
+ });
+ if (auth.apiKey)
+ return { ...headers, authorization: `Bearer ${auth.apiKey}` };
+ const credentials = signingCredentials(auth);
+ if (!credentials)
+ throw new Error(
+ "Invalid Bedrock credentials. Expected AWS access key JSON or a Bedrock API key.",
+ );
+ // AWS's own SigV4 signer, as used by the AWS SDK.
+ const signed = await new SignatureV4({
+ credentials,
+ region,
+ service: "bedrock",
+ sha256: Sha256,
+ }).sign({
+ method: "POST",
+ protocol: url.protocol,
+ hostname: url.hostname,
+ path: url.pathname,
+ headers,
+ body,
+ });
+ return signed.headers;
+}
+
async function invokeWithCache(params: {
- model: string;
+ model: TopicsModel;
system: TopicPromptPart[];
input: string;
schema: z.ZodType;
maxOutputTokens: number;
- region: string;
- profile?: string;
}): Promise<TopicTextResult> {
+ const { region } = BedrockConfigSchema.parse(params.model.connection.config);
+ assertValidBedrockRegion(region);
const { $schema: _, ...schema } = z.toJSONSchema(params.schema);
const url = new URL(
- `https://bedrock-runtime.${params.region}.amazonaws.com/model/${encodeURIComponent(params.model)}/invoke`,
+ `https://bedrock-runtime.${region}.amazonaws.com/model/${encodeURIComponent(params.model.model)}/invoke`,
);
const body = JSON.stringify({
messages: [
@@ -113,30 +188,10 @@ async function invokeWithCache(params: {
json_schema: { name: "topics", schema, strict: true },
},
});
- // AWS's own SigV4 signer (as used by the AWS SDK), with the default credential chain.
- const signed = await new SignatureV4({
- credentials: fromNodeProviderChain(
- params.profile ? { profile: params.profile } : {},
- ),
- region: params.region,
- service: "bedrock",
- sha256: Sha256,
- }).sign({
- method: "POST",
- protocol: url.protocol,
- hostname: url.hostname,
- path: url.pathname,
- headers: {
- host: url.hostname,
- "content-type": "application/json",
- accept: "application/json",
- },
- body,
- });
- // A single attempt, like the Converse path (maxRetries: 0).
+ // A single attempt; the host is derived from the validated region.
const response = await fetch(url, {
method: "POST",
- headers: signed.headers,
+ headers: await bedrockRequestHeaders(params.model, url, region, body),
body,
signal: AbortSignal.timeout(TIMEOUT_MS),
});
diff --git a/packages/shared/src/server/topics/topic-rule.test.ts b/packages/shared/src/server/topics/topic-rule.test.ts
new file mode 100644
index 0000000000..830df78188
--- /dev/null
+++ b/packages/shared/src/server/topics/topic-rule.test.ts
@@ -0,0 +1,129 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+ lock: vi.fn(),
+ facetCount: vi.fn(),
+ findRules: vi.fn(),
+ create: vi.fn(),
+ update: vi.fn(),
+}));
+
+vi.mock("./clickhouse", () => ({
+ getTopicDefinitions: vi.fn(),
+ writeTopicDefinitions: vi.fn(),
+}));
+vi.mock("../../db", () => ({
+ Prisma: {},
+ prisma: {
+ $transaction: (callback: (tx: unknown) => Promise<unknown>) =>
+ callback({
+ $executeRaw: mocks.lock,
+ evaluator: { count: mocks.facetCount },
+ evaluationRule: {
+ findMany: mocks.findRules,
+ create: mocks.create,
+ update: mocks.update,
+ },
+ }),
+ },
+}));
+
+import { saveTopicRule } from "./postgres";
+
+const projectId = "project-a";
+const savedRow = {
+ id: "rule-a",
+ projectId,
+ name: "Topics",
+ filter: [],
+ sampling: 0.25,
+ updatedAt: new Date("2026-10-08T00:00:00Z"),
+ assignments: [{ evaluatorId: "intent" }],
+};
+
+describe("saveTopicRule", () => {
+ beforeEach(() => {
+ vi.resetAllMocks();
+ mocks.facetCount.mockResolvedValue(1);
+ mocks.findRules.mockResolvedValue([]);
+ mocks.create.mockResolvedValue(savedRow);
+ mocks.update.mockResolvedValue(savedRow);
+ });
+
+ it("stores the sampling rate on the single rule", async () => {
+ const rule = await saveTopicRule({
+ projectId,
+ name: "Topics",
+ filter: [],
+ facetIds: ["intent"],
+ sampling: 0.25,
+ });
+
+ expect(mocks.create).toHaveBeenCalledWith(
+ expect.objectContaining({
+ data: expect.objectContaining({
+ sampling: 0.25,
+ projectId,
+ }),
+ }),
+ );
+ expect(rule).toMatchObject({
+ sampling: 0.25,
+ facetIds: ["intent"],
+ });
+ });
+
+ it("rejects a save without an id once the project has a rule", async () => {
+ mocks.findRules.mockResolvedValue([{ id: "rule-a", name: "Topics" }]);
+ const input = {
+ projectId,
+ filter: [],
+ facetIds: ["intent"],
+ sampling: 0.5,
+ };
+
+ // The rule is looked up under the project lock, so a racing first save
+ // sees the rule the other one created and is rejected.
+ await expect(saveTopicRule(input)).rejects.toThrow(
+ "This project already has a Topics rule.",
+ );
+ expect(mocks.lock.mock.invocationCallOrder[0]).toBeLessThan(
+ mocks.findRules.mock.invocationCallOrder[0],
+ );
+ expect(mocks.update).not.toHaveBeenCalled();
+
+ await saveTopicRule({ ...input, id: "rule-a" });
+ expect(mocks.create).not.toHaveBeenCalled();
+ expect(mocks.update).toHaveBeenCalledWith(
+ expect.objectContaining({
+ where: expect.objectContaining({ id: "rule-a" }),
+ data: expect.objectContaining({
+ name: "Topics",
+ sampling: 0.5,
+ }),
+ }),
+ );
+ });
+
+ it("rejects another project's rule id and projects with several rules", async () => {
+ mocks.findRules.mockResolvedValue([{ id: "rule-a", name: "Topics" }]);
+ await expect(
+ saveTopicRule({
+ id: "rule-b",
+ projectId,
+ filter: [],
+ facetIds: ["intent"],
+ }),
+ ).rejects.toThrow("Topic rule not found in this project.");
+
+ mocks.findRules.mockResolvedValue([
+ { id: "rule-a", name: "Topics" },
+ { id: "rule-b", name: "Old" },
+ ]);
+ await expect(
+ saveTopicRule({ projectId, filter: [], facetIds: ["intent"] }),
+ ).rejects.toThrow("This project has more than one Topics rule.");
+ expect(mocks.create).not.toHaveBeenCalled();
+ expect(mocks.update).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/shared/src/server/topics/trace-selection.ts b/packages/shared/src/server/topics/trace-selection.ts
index 2bf5f68fb6..4f3cf274d1 100644
--- a/packages/shared/src/server/topics/trace-selection.ts
+++ b/packages/shared/src/server/topics/trace-selection.ts
@@ -36,15 +36,24 @@ export async function selectTopicTraceRows<Mode extends keyof SelectionRows>(
projectId: input.projectId,
prisma,
objectType: "OBSERVATION",
- filterState: input.filter.map((filter) =>
- filter.column === "tags" ? { ...filter, column: "traceTags" } : filter,
- ),
+ filterState: input.filter.flatMap((filter) => {
+ if (filter.column === "isRootObservation") return [];
+ if (filter.column === "tags")
+ return [{ ...filter, column: "traceTags" as const }];
+ return [filter];
+ }),
});
if (hasNoMatches) return [];
const { queryBuilder } = buildEventsObservationRowSelection({
projectId: input.projectId,
filter: filterState.concat([
+ {
+ column: "isRootObservation",
+ type: "boolean",
+ operator: "=",
+ value: true,
+ },
{
column: "startTime",
type: "datetime",
diff --git a/packages/shared/src/server/tracingReadinessSettings.ts b/packages/shared/src/server/tracingReadinessSettings.ts
index 23a04dd1f5..babb45c934 100644
--- a/packages/shared/src/server/tracingReadinessSettings.ts
+++ b/packages/shared/src/server/tracingReadinessSettings.ts
@@ -1,5 +1,6 @@
import { z } from "zod";
import { prisma } from "../db";
+import type { Prisma } from "../db";
import { logger } from "./logger";
import { redis } from "./redis/redis";
@@ -77,13 +78,36 @@ export async function saveTracingReadinessSettings(
{ userId = null }: { userId?: string | null } = {},
): Promise<TracingReadinessSettings> {
const data = tracingReadinessSettingsSchema.parse(settings);
- const row = await prisma.tracingReadinessSettings.upsert({
+ const saved = await writeTracingReadinessSettings(
+ prisma,
+ projectId,
+ data,
+ userId,
+ );
+ await cacheTracingReadinessSettings(projectId, saved);
+ return saved;
+}
+
+export async function writeTracingReadinessSettings(
+ tx: Prisma.TransactionClient,
+ projectId: string,
+ settings: TracingReadinessSettings,
+ userId: string | null = null,
+): Promise<TracingReadinessSettings> {
+ const data = tracingReadinessSettingsSchema.parse(settings);
+ const row = await tx.tracingReadinessSettings.upsert({
where: { projectId },
create: { projectId, ...data, createdBy: userId, updatedBy: userId },
update: { ...data, updatedBy: userId },
select: { traceIdleMs: true },
});
- const saved = { traceIdleMs: row.traceIdleMs };
+ return { traceIdleMs: row.traceIdleMs };
+}
+
+export async function cacheTracingReadinessSettings(
+ projectId: string,
+ saved: TracingReadinessSettings,
+): Promise<void> {
try {
await redis?.set(
cacheKey(projectId),
@@ -97,7 +121,6 @@ export async function saveTracingReadinessSettings(
error,
});
}
- return saved;
}
async function readCache(
diff --git a/packages/shared/src/topics/default-facets.ts b/packages/shared/src/topics/default-facets.ts
new file mode 100644
index 0000000000..539708b230
--- /dev/null
+++ b/packages/shared/src/topics/default-facets.ts
@@ -0,0 +1,111 @@
+export const DEFAULT_TOPIC_FACETS = [
+ {
+ name: "Intent",
+ description: "What the run was asked to do.",
+ prompt: `Describe what the user, or the calling application, wanted from this run as a whole.
+
+Format: an imperative verb phrase stating the goal, such as "Find…", "Fix…", "Summarize…".
+
+- Describe the goal of the whole run, not its last step. Follow-up requests to verify, save, show, fix, or reformat earlier work belong to the goal they serve.
+- A run can contain several unrelated requests, for example a user who finishes one job and then starts another. Name each of them in a few words, in the order they were asked, joined with "and". Describing only the latest request misses the earlier ones.
+- Earlier conversation can clarify the goals of this run but does not add goals of its own.
+- In an automated run with no human author (extraction, classification, routing, templated generation), describe the job the input sets. Do not invent a person asking for it.
+- When the user asks the assistant to work on supplied material, describe that work and name the material's subject in a few words.
+- Keep the user's own verb and object. "Fix the date filter in a SQL query" must not become "Improve a data workflow".
+- Describe goals only. Leave out how the assistant approached them, what went wrong, and whether they succeeded.
+- Social or casual messages have a goal too; state it plainly.
+
+Status: applicable whenever a goal can be identified, even if the run failed or produced nothing. insufficient_input when no request or input survives. Do not use not_applicable.
+
+Examples:
+- Export last quarter's orders to CSV grouped by region.
+- Classify an inbound support email by product area and urgency.
+- Fix a failing nightly data import and draft a welcome email for new customers.
+- Chat about plans for the weekend.`,
+ },
+ {
+ name: "Sentiment",
+ description: "How the end user felt about the interaction.",
+ prompt: `Describe how the end user felt about this interaction and what the feeling was directed at.
+
+Format: "<Label>: <cue and its target>", where Label is one of:
+- Positive: thanks, praise, relief, or warm engagement.
+- Negative: frustration, annoyance, distrust, or giving up.
+- Mixed: both are substantial, for example annoyance that turns into relief.
+- Neutral: factual or procedural messages with no emotional signal.
+
+- Judge only text the end user wrote. Assistant, tool, and system text shows what the user reacts to, never how the user feels.
+- Judge the attitude toward the interaction, not the subject. A calm report of a bug, a failed test, or a personal hardship is not negative by itself.
+- Weight sustained tone and how the user ends over a single remark. Terse or strict instructions are Neutral; so is routine politeness.
+- Name the target in terms of the assistant's behavior (wrong answers, slow progress, ignored corrections, a working fix), not the user's private circumstances.
+
+Status: not_applicable when the run has no end-user text, as in automated pipelines. insufficient_input when user text exists but is unreadable.
+
+Examples:
+- Negative: user repeated the same formatting correction and said the assistant keeps ignoring it.
+- Positive: user thanked the assistant after a working fix for a failing deployment script.
+- Mixed: user was annoyed by two wrong answers, then relieved when the third one worked.
+- Neutral: user gave step-by-step instructions without emotional cues.`,
+ },
+ {
+ name: "Outcome",
+ description: "Where the run ended and whether that is confirmed.",
+ prompt: `Describe where this run ended: what was delivered or done, and whether the transcript confirms it.
+
+Format: "<State>: <the concrete result>", where State is one of:
+- Completed: the requested answer was given, or a tool result confirms the requested action.
+- Partial: part of the request was delivered and part was not.
+- Unconfirmed: the assistant says an action happened, but no result confirms it.
+- Needs input: the run ends waiting on the user, such as a clarifying question or an approval.
+- Not completed: the run stopped, was blocked, or was declined without delivering the request.
+
+- Start from the end of this run: the last assistant message and the last tool results decide the state.
+- Base the state on results, not on the assistant's words. A tool result confirms an action; a message saying "done" does not. A returned draft is a delivered draft, not a sent message.
+- For questions, explanations, and analyses, the delivered answer is the result.
+- Name the concrete deliverable or stopping point, not just the state. Mention remaining work only when the transcript shows it.
+- Say where the run stopped, not why something failed, unless the reason is itself the result, as with a declined request.
+
+Status: applicable whenever the run has a final response or result, including failures. insufficient_input when no response or result survives. Do not use not_applicable.
+
+Examples:
+- Completed: returned a SQL query that filters orders by signup month.
+- Unconfirmed: said the meeting was booked, but no calendar tool result confirms it.
+- Needs input: asked which of two accounts the transfer should come from.
+- Not completed: stopped after the payments tool failed, without answering the user.`,
+ },
+ {
+ name: "Issues",
+ description: "The main problem in how the run was handled.",
+ prompt: `Describe the problem that did the most damage to the result of this run: the mistake that made the final answer or action wrong, unsupported, or missing.
+
+Format: one sentence that names the mistake, the kind of step where it happened, and its consequence, for example "Answered from memory instead of querying the dataset, so the reported value is unverified."
+- Describe the mechanism in plain, generic words: what the assistant did wrong and how, such as answering from memory, inventing data, claiming a check that never ran, choosing the wrong tool, breaking a required format, or stopping before the answer. Use the words that fit this run.
+- Name the kind of step (a web search, a file read, a calculation, a code change, the final answer), not the task's subject. "Skipped the file read and guessed the count" is right; "gave a wrong count of crustacean slides" is not.
+- Do not start with a label or category.
+
+How to find it:
+- Start from the end. Check the final answer or action against the request: is it on target, complete, and backed by what the run actually retrieved or did? Then work back to the step that caused the gap.
+- Report the cause with its consequence, not the symptom. When a tool fails and the assistant then answers anyway from guesses or invented data, the problem is the unsupported answer, not the tool failure.
+- When several problems occur, report the one with the biggest effect on the result. Prefer, in this order: an answer or reported result that is wrong or not backed by the run; a task left unfinished or answered off target; a wrong tool, approach, or argument; a broken instruction or format; a tool failure that blocked the result. A problem the assistant fully recovered from counts only when nothing worse happened.
+- Keep the symptoms of one problem together; do not list separate problems.
+
+Before you return not_applicable, check these four points. Return not_applicable only when all of them hold:
+1. When the request depends on data, files, tools, or current facts that the run had to look up, every such fact, number, or result in the final answer comes from a tool result or user input in this run, not from memory or assumption. General knowledge, explanations, and writing do not need a lookup.
+2. Every tool call used a tool suited to its input, such as a file tool on a local file rather than a web address, with valid arguments.
+3. The run followed the output format, tags, and steps that the system prompt or the assistant's own plan required.
+4. The final answer addresses exactly what was asked: the right quantity, unit, entity, and scope.
+
+- Only problems in this run count. An error the user pastes for explanation, a problem in earlier conversation, or a complaint about something outside the run is not an issue here.
+- A clarifying question, a justified refusal, a short answer, and a retry that succeeds are not issues.
+- Check the end of this run before calling it unfinished: if a later assistant message delivers an answer or fallback, the run is finished.
+- Report only what the transcript shows directly. A run-specific fact that needed a lookup but has none is shown directly. A problem that is only a possibility is not.
+
+Status: not_applicable when the run is complete enough to judge and shows no problem. insufficient_input when too much is missing to judge, for example only the first request survives.
+
+Examples:
+- Answered from a made-up example after the data file could not be read, so the reported result is invented.
+- Reported the total instead of the requested minimum, so the answer misses the question.
+- Searched the web for figures the attached spreadsheet contained, so the answer used outdated data.
+- Every inventory lookup timed out, so no quote could be produced and the user was asked to retry later.`,
+ },
+];
diff --git a/packages/shared/src/topics/index.ts b/packages/shared/src/topics/index.ts
index fa623b6238..061a0b7e6e 100644
--- a/packages/shared/src/topics/index.ts
+++ b/packages/shared/src/topics/index.ts
@@ -1,5 +1,8 @@
import { z } from "zod";
import { singleFilterList } from "../interfaces/filters";
+import { DEFAULT_TOPIC_FACETS } from "./default-facets";
+
+export { DEFAULT_TOPIC_FACETS };
export const TOPICS_BEDROCK_ACCESS_ERROR =
"Unable to invoke LLM model (HTTP 403), please contact Langfuse Support";
@@ -21,12 +24,52 @@ export const topicEmbeddingConfigSchema = z.object({
embeddingModel: z.string().trim().min(1).optional(),
embeddingDimensions: z
.number()
- .refine((value) => [256, 512, 1024, 1536].includes(value), {
- message: "Choose 256, 512, 1024, or 1536 embedding dimensions.",
- })
+ .int("Embedding dimensions must be a whole number.")
+ .min(64, "Use at least 64 embedding dimensions.")
+ .max(4096, "Use at most 4,096 embedding dimensions.")
.default(1024),
});
export type TopicEmbeddingConfig = z.infer<typeof topicEmbeddingConfigSchema>;
+
+export const TOPICS_MODEL_SLOTS = [
+ "summary",
+ "embedding",
+ "clustering",
+] as const;
+export type TopicsModelSlotName = (typeof TOPICS_MODEL_SLOTS)[number];
+export const TOPICS_MODEL_SLOT_DETAILS: Record<
+ TopicsModelSlotName,
+ { label: string; recommendation: string }
+> = {
+ summary: {
+ label: "Facet summaries",
+ recommendation:
+ "A small, fast model such as GPT-6 Luna. It runs once per trace.",
+ },
+ embedding: {
+ label: "Embeddings",
+ recommendation:
+ "An embedding model such as text-embedding-3-small or Cohere Embed v4. It cannot be changed once summaries are embedded.",
+ },
+ clustering: {
+ label: "Topic clustering",
+ recommendation:
+ "Names and describes each topic when topics are updated. Use a stronger model such as GPT-5.6 Terra, or Claude Sonnet through Amazon Bedrock.",
+ },
+};
+const topicsModelSlotSchema = z.object({
+ llmApiKeyId: z.string().min(1),
+ model: z.string().trim().min(1).max(256),
+});
+export const topicsModelSettingsSchema = z.object({
+ summary: topicsModelSlotSchema.nullable(),
+ embedding: topicsModelSlotSchema.nullable(),
+ embeddingDimensions:
+ topicEmbeddingConfigSchema.shape.embeddingDimensions.unwrap(),
+ clustering: topicsModelSlotSchema.nullable(),
+ enabled: z.boolean(),
+});
+export type TopicsModelSettings = z.infer<typeof topicsModelSettingsSchema>;
export const topicProcessingConfigSchema = z.object({
summaryModel: z.string().trim().min(1).optional(),
maxInputTokens: z.number().int().min(256).max(120_000).default(120_000),
@@ -63,7 +106,54 @@ export const topicTraceSelectionSnapshotSchema =
topicTraceSelectionCriteriaSchema.safeExtend({
excludedTraceIds: z.array(topicTraceIdSchema).default([]),
});
-export type TopicRule = z.infer<typeof topicRuleConfigSchema> & {
+export const topicRuleSettingsSchema = topicRuleConfigSchema.extend({
+ sampling: z.number().min(0).max(1).default(1),
+});
+
+export const topicsSetupSchema = topicsModelSettingsSchema.extend({
+ traceIdleMs: z
+ .number()
+ .int()
+ .multipleOf(1000, "Use whole seconds for the idle time.")
+ .min(60_000, "Wait at least 60 seconds.")
+ .max(3_600_000, "Wait at most 3,600 seconds."),
+ // The project's existing rule; omit only when the project has none yet.
+ ruleId: topicIdSchema.optional(),
+ facets: z
+ .array(
+ z.object({
+ name: z.string(),
+ enabled: z.boolean(),
+ }),
+ )
+ .superRefine((facets, ctx) => {
+ const expected = DEFAULT_TOPIC_FACETS.map((facet) => facet.name);
+ const names = facets.map((facet) => facet.name);
+ if (
+ names.length !== expected.length ||
+ new Set(names).size !== names.length ||
+ expected.some((name) => !names.includes(name))
+ ) {
+ ctx.addIssue({
+ code: "custom",
+ message: "Include each built-in facet once.",
+ });
+ }
+ if (!facets.some((facet) => facet.enabled)) {
+ ctx.addIssue({
+ code: "custom",
+ message: "Turn on at least one facet.",
+ });
+ }
+ }),
+ // Enabled custom facets keep their rule assignment; they are edited elsewhere.
+ customFacetIds: z.array(topicIdSchema).default([]),
+ filter: topicRuleConfigSchema.shape.filter,
+ sampling: z.number().min(0).max(1),
+});
+export type TopicsSetup = z.infer<typeof topicsSetupSchema>;
+
+export type TopicRule = z.infer<typeof topicRuleSettingsSchema> & {
id: string;
projectId: string;
name: string;
diff --git a/web/AGENTS.md b/web/AGENTS.md
index 5d162bd8f7..d3e41d396a 100644
--- a/web/AGENTS.md
+++ b/web/AGENTS.md
@@ -30,9 +30,11 @@
use the `langfuseTopics` flag and project permissions. Storage, retry and setup
details: `../worker/src/features/topics/README.md`. Preview and trigger
selection share `topics/server`'s bounded trace selector with worker
- backfills; preserve sampling and the frozen submission range. Internal PoC
- model selection requires `LANGFUSE_TOPICS_SUMMARY_MODEL` and
- `LANGFUSE_TOPICS_EMBEDDING_MODEL` on both web and worker, with no defaults.
+ backfills; preserve sampling and the frozen submission range. Models come
+ from the project's LLM connections (`TopicModelSettings.tsx`,
+ `topics.modelSettings` / `saveSetup`). Configure Topics loads and saves idle time
+ through the project's tracing readiness setting; triggers stamp the configured
+ summary and embedding models into the frozen request.
The source inspector returns `loadTopicTranscript`'s shared `Transcript | null`
directly, using the same character cap as worker inference.
- Reusable UI components: `src/components/*`
diff --git a/web/src/__tests__/server/llm-api-key.servertest.ts b/web/src/__tests__/server/llm-api-key.servertest.ts
index f156cac9db..1fe1135fe1 100644
--- a/web/src/__tests__/server/llm-api-key.servertest.ts
+++ b/web/src/__tests__/server/llm-api-key.servertest.ts
@@ -1823,5 +1823,47 @@ describe("llmApiKey.all RPC", () => {
});
}
});
+
+ it("turns Topics off and records why when its connection is deleted", async () => {
+ await caller.llmApiKey.create({
+ projectId,
+ secretKey: "test-secret",
+ provider: PROVIDER,
+ adapter: LLMAdapter.OpenAI,
+ customModels: [],
+ withDefaultModels: true,
+ });
+ const connection = await prisma.llmApiKeys.findFirstOrThrow({
+ where: { projectId, provider: PROVIDER },
+ });
+ await prisma.topicConfig.create({
+ data: {
+ projectId,
+ enabled: true,
+ summaryLlmApiKeyId: connection.id,
+ summaryModel: "gpt-6-luna",
+ embeddingLlmApiKeyId: connection.id,
+ embeddingModel: "text-embedding-3-small",
+ clusteringLlmApiKeyId: connection.id,
+ clusteringModel: "gpt-5.6-terra",
+ },
+ });
+
+ await caller.llmApiKey.delete({ projectId, id: connection.id });
+
+ expect(
+ await prisma.topicConfig.findUniqueOrThrow({
+ where: { projectId },
+ }),
+ ).toMatchObject({
+ enabled: false,
+ blockReason: "LLM_CONNECTION_MISSING",
+ blockMessage: expect.stringContaining(`"${PROVIDER}" was deleted`),
+ blockedAt: expect.any(Date),
+ summaryLlmApiKeyId: null,
+ embeddingLlmApiKeyId: null,
+ clusteringLlmApiKeyId: null,
+ });
+ });
});
});
diff --git a/web/src/__tests__/server/unit/topics.servertest.ts b/web/src/__tests__/server/unit/topics.servertest.ts
index b5fe985703..ca90e0b2d8 100644
--- a/web/src/__tests__/server/unit/topics.servertest.ts
+++ b/web/src/__tests__/server/unit/topics.servertest.ts
@@ -5,6 +5,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import {
topicProcessingConfigSchema,
topicEmbeddingConfigSchema,
+ DEFAULT_TOPIC_FACETS,
type TopicExecution,
type TopicExecutionInput,
} from "@langfuse/shared/topics";
@@ -34,10 +35,13 @@ const mocks = vi.hoisted(() => ({
loadTopicTranscript: vi.fn<typeof topicsServer.loadTopicTranscript>(),
isTopicsEnabled: vi.fn(),
isTopicsProjectEnabled: vi.fn(),
- getTopicsModelConfig: vi.fn(),
+ getTopicsModels: vi.fn(),
+ checkTopicsModelSettings: vi.fn(),
+ saveTopicsSetup: vi.fn(),
enqueueTopicExecution: vi.fn(),
getTopicExecutionQueueState: vi.fn(),
queryClickhouse: vi.fn(),
+ auditLog: vi.fn(),
}));
vi.mock("@langfuse/shared/topics/server", async (importOriginal) => {
const { selectTopicTraceRows } = await importOriginal<typeof topicsServer>();
@@ -50,6 +54,9 @@ vi.mock(
queryClickhouse: mocks.queryClickhouse,
}),
);
+vi.mock("@/src/features/audit-logs/auditLog", () => ({
+ auditLog: mocks.auditLog,
+}));
vi.mock("@/src/features/posthog-analytics/server/backendActivity", () => ({
recordBackendActivity: vi.fn(async () => undefined),
}));
@@ -187,10 +194,10 @@ beforeEach(() => {
mocks.queryClickhouse.mockResolvedValue([]);
mocks.isTopicsEnabled.mockReturnValue(true);
mocks.isTopicsProjectEnabled.mockReturnValue(true);
- mocks.getTopicsModelConfig.mockReturnValue({
- summaryModel: "us.openai.gpt-5.6-luna",
- embeddingModel: "eu.cohere.embed-v4:0",
- });
+ mocks.getTopicsModels.mockResolvedValue({
+ summary: { model: "us.openai.gpt-5.6-luna" },
+ embedding: { model: "eu.cohere.embed-v4:0", dimensions: 1024 },
+ } as Awaited<ReturnType<typeof topicsServer.getTopicsModels>>);
mocks.getTopicFacetVersion.mockResolvedValue({
facetId,
version: facetVersion,
@@ -722,6 +729,15 @@ describe("Topics published scatter map", () => {
});
describe("Topics local execution access and publication", () => {
+ it("requires the project's Topics models before creating work", async () => {
+ mocks.getTopicsModels.mockResolvedValue(null);
+ await expect(caller().trigger(input)).rejects.toThrow(
+ "Choose summary, embedding, and clustering models",
+ );
+ expect(mocks.createTopicExecution).not.toHaveBeenCalled();
+ expect(mocks.enqueueTopicExecution).not.toHaveBeenCalled();
+ });
+
it("rejects a foreign facet before creating or enqueuing work", async () => {
mocks.getTopicFacetVersion.mockResolvedValue(null);
await expect(caller().trigger(input)).rejects.toMatchObject({
@@ -764,7 +780,6 @@ describe("Topics local execution access and publication", () => {
const request = {
projectId,
facets: selectedFacets,
- embeddingConfig: input.embeddingConfig,
timeRange,
};
mocks.getTopicSummaryCounts.mockResolvedValue([
@@ -1265,3 +1280,186 @@ describe("Topics current results", () => {
},
);
});
+
+describe("Topics model settings", () => {
+ const settings = {
+ projectId,
+ summary: null,
+ embedding: { llmApiKeyId: "key", model: "text-embedding-3-small" },
+ embeddingDimensions: 1024,
+ clustering: null,
+ enabled: false,
+ };
+
+ it("returns the slot check to a writer and rejects a viewer before testing", async () => {
+ const errors = {
+ embedding: "You are not allowed to generate embeddings from this model",
+ };
+ mocks.checkTopicsModelSettings.mockResolvedValue(errors);
+
+ await expect(
+ caller("VIEWER").testModelSettings(settings),
+ ).rejects.toMatchObject({ code: "FORBIDDEN" });
+ await expect(
+ caller().testModelSettings({ ...settings, projectId: "foreign" }),
+ ).rejects.toMatchObject({ code: "UNAUTHORIZED" });
+ expect(mocks.checkTopicsModelSettings).not.toHaveBeenCalled();
+
+ await expect(caller().testModelSettings(settings)).resolves.toEqual(errors);
+ expect(mocks.checkTopicsModelSettings).toHaveBeenCalledOnce();
+ expect(mocks.checkTopicsModelSettings).toHaveBeenCalledWith(projectId, {
+ summary: null,
+ embedding: settings.embedding,
+ embeddingDimensions: 1024,
+ clustering: null,
+ enabled: false,
+ });
+ });
+});
+
+describe("Topics setup", () => {
+ const setup = {
+ projectId,
+ summary: { llmApiKeyId: "key", model: "gpt" },
+ embedding: { llmApiKeyId: "key", model: "embed" },
+ embeddingDimensions: 1024,
+ clustering: { llmApiKeyId: "key", model: "cluster" },
+ enabled: true,
+ facets: DEFAULT_TOPIC_FACETS.map((facet) => ({
+ name: facet.name,
+ enabled: facet.name !== "Sentiment",
+ })),
+ filter: [
+ {
+ column: "name",
+ type: "string" as const,
+ operator: "=" as const,
+ value: "billing",
+ },
+ ],
+ sampling: 0.25,
+ traceIdleMs: 120_000,
+ };
+
+ it("saves setup for a writer and rejects a viewer before saving", async () => {
+ mocks.saveTopicsSetup.mockResolvedValue({ id: "rule-a" });
+
+ await expect(caller("VIEWER").saveSetup(setup)).rejects.toMatchObject({
+ code: "FORBIDDEN",
+ });
+ await expect(
+ caller().saveSetup({ ...setup, projectId: "foreign" }),
+ ).rejects.toMatchObject({ code: "UNAUTHORIZED" });
+ expect(mocks.saveTopicsSetup).not.toHaveBeenCalled();
+
+ await expect(caller().saveSetup(setup)).resolves.toEqual({ id: "rule-a" });
+ expect(mocks.saveTopicsSetup).toHaveBeenCalledOnce();
+ expect(mocks.saveTopicsSetup).toHaveBeenCalledWith(
+ projectId,
+ {
+ summary: setup.summary,
+ embedding: setup.embedding,
+ embeddingDimensions: 1024,
+ clustering: setup.clustering,
+ enabled: true,
+ facets: setup.facets,
+ customFacetIds: [],
+ filter: setup.filter,
+ sampling: 0.25,
+ traceIdleMs: 120_000,
+ },
+ "user-a",
+ );
+ expect(mocks.auditLog).toHaveBeenCalledWith(
+ expect.objectContaining({
+ resourceType: "job",
+ resourceId: "rule-a",
+ action: "create",
+ session: expect.objectContaining({
+ user: expect.objectContaining({ id: "user-a" }),
+ }),
+ }),
+ );
+ });
+
+ it("rejects a setup with every facet turned off", async () => {
+ await expect(
+ caller().saveSetup({
+ ...setup,
+ facets: setup.facets.map((facet) => ({ ...facet, enabled: false })),
+ }),
+ ).rejects.toMatchObject({ code: "BAD_REQUEST" });
+ expect(mocks.saveTopicsSetup).not.toHaveBeenCalled();
+ });
+});
+
+describe("Topics write access", () => {
+ const facetDraft = {
+ name: "Intent",
+ description: "",
+ prompt: "Describe the original request.",
+ };
+ const slot = { llmApiKeyId: "key", model: "gpt" };
+ const settings = {
+ summary: slot,
+ embedding: slot,
+ embeddingDimensions: 1024,
+ clustering: slot,
+ enabled: true,
+ };
+ const writes: Record<
+ string,
+ (api: ReturnType<typeof caller>, projectId: string) => Promise<unknown>
+ > = {
+ initialize: (api, projectId) => api.initialize({ projectId }),
+ saveFacet: (api, projectId) => api.saveFacet({ projectId, ...facetDraft }),
+ saveRule: (api, projectId) =>
+ api.saveRule({
+ projectId,
+ name: "Topics",
+ filter: [],
+ facetIds: [facetId],
+ }),
+ testModelSettings: (api, projectId) =>
+ api.testModelSettings({ projectId, ...settings }),
+ saveSetup: (api, projectId) =>
+ api.saveSetup({
+ projectId,
+ ...settings,
+ facets: DEFAULT_TOPIC_FACETS.map(({ name }) => ({
+ name,
+ enabled: true,
+ })),
+ filter: [],
+ sampling: 1,
+ traceIdleMs: 120_000,
+ }),
+ trigger: (api, projectId) => api.trigger({ ...input, projectId }),
+ retry: (api, projectId) =>
+ api.retry({ projectId, executionId: "execution-a" }),
+ };
+ const writeMocks = [
+ mocks.ensureDefaultTopicFacets,
+ mocks.createTopicFacet,
+ mocks.createTopicFacetVersion,
+ mocks.saveTopicRule,
+ mocks.checkTopicsModelSettings,
+ mocks.saveTopicsSetup,
+ mocks.createTopicExecution,
+ mocks.enqueueTopicExecution,
+ mocks.auditLog,
+ ];
+
+ it.each(Object.entries(writes))(
+ "rejects %s for viewers and other projects before any write",
+ async (_, write) => {
+ await expect(write(caller("VIEWER"), projectId)).rejects.toMatchObject({
+ code: "FORBIDDEN",
+ });
+ await expect(write(caller(), "foreign")).rejects.toMatchObject({
+ code: "UNAUTHORIZED",
+ });
+ for (const mock of writeMocks) expect(mock).not.toHaveBeenCalled();
+ },
+ );
+});
diff --git a/web/src/components/ui/sheet.tsx b/web/src/components/ui/sheet.tsx
index c88baa5d95..fcfb7c4ce2 100644
--- a/web/src/components/ui/sheet.tsx
+++ b/web/src/components/ui/sheet.tsx
@@ -65,27 +65,34 @@ const sheetVariants = cva(
interface SheetContentProps
extends
React.ComponentPropsWithoutRef<typeof SheetPrimitive.Content>,
- VariantProps<typeof sheetVariants> {}
+ VariantProps<typeof sheetVariants> {
+ overlayClassName?: string;
+}
const SheetContent = React.forwardRef<
React.ComponentRef<typeof SheetPrimitive.Content>,
SheetContentProps
->(({ side = "right", className, children, ...props }, ref) => (
- <SheetPortal>
- <SheetOverlay />
- <SheetPrimitive.Content
- ref={ref}
- className={cn(sheetVariants({ side }), className)}
- {...props}
- >
- {children}
- <SheetPrimitive.Close className="ring-offset-background focus:ring-ring data-[state=open]:bg-secondary absolute top-2 right-2 inline-flex size-8 items-center justify-center rounded-sm opacity-70 transition-opacity hover:opacity-100 focus:ring-2 focus:ring-offset-2 focus:outline-hidden disabled:pointer-events-none">
- <X className="icon-base" />
- <span className="sr-only">Close</span>
- </SheetPrimitive.Close>
- </SheetPrimitive.Content>
- </SheetPortal>
-));
+>(
+ (
+ { side = "right", className, children, overlayClassName, ...props },
+ ref,
+ ) => (
+ <SheetPortal>
+ <SheetOverlay className={overlayClassName} />
+ <SheetPrimitive.Content
+ ref={ref}
+ className={cn(sheetVariants({ side }), className)}
+ {...props}
+ >
+ {children}
+ <SheetPrimitive.Close className="ring-offset-background focus:ring-ring data-[state=open]:bg-secondary absolute top-2 right-2 inline-flex size-8 items-center justify-center rounded-sm opacity-70 transition-opacity hover:opacity-100 focus:ring-2 focus:ring-offset-2 focus:outline-hidden disabled:pointer-events-none">
+ <X className="icon-base" />
+ <span className="sr-only">Close</span>
+ </SheetPrimitive.Close>
+ </SheetPrimitive.Content>
+ </SheetPortal>
+ ),
+);
SheetContent.displayName = SheetPrimitive.Content.displayName;
const SheetHeader = ({
diff --git a/web/src/env.mjs b/web/src/env.mjs
index 75c2fdba82..0078289cc6 100644
--- a/web/src/env.mjs
+++ b/web/src/env.mjs
@@ -96,9 +96,6 @@ export const env = createEnv({
LANGFUSE_ADMIN_ACCESS_WEBHOOK: z.url().optional(),
// Add `.min(1) on ID and SECRET if you want to make sure they're not empty
LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: z.enum(["true", "false"]).optional(),
- // Internal Topics PoC model selection; not a supported self-hosting setting.
- LANGFUSE_TOPICS_SUMMARY_MODEL: z.string().trim().min(1).optional(),
- LANGFUSE_TOPICS_EMBEDDING_MODEL: z.string().trim().min(1).optional(),
SALT: z.string({
error: (issue) =>
issue.input === undefined
@@ -804,9 +801,6 @@ export const env = createEnv({
process.env.NEXT_PUBLIC_PREVIEW_DEMO_AUTO_SIGN_IN,
LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:
process.env.LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES,
- LANGFUSE_TOPICS_SUMMARY_MODEL: process.env.LANGFUSE_TOPICS_SUMMARY_MODEL,
- LANGFUSE_TOPICS_EMBEDDING_MODEL:
- process.env.LANGFUSE_TOPICS_EMBEDDING_MODEL,
AWS_ACCESS_KEY_ID: process.env.AWS_ACCESS_KEY_ID,
AWS_SECRET_ACCESS_KEY: process.env.AWS_SECRET_ACCESS_KEY,
LANGFUSE_IN_APP_AGENT_MAX_ACTIVE_RUNS_PER_USER:
diff --git a/web/src/features/audit-logs/auditLog.ts b/web/src/features/audit-logs/auditLog.ts
index 6f51397ce9..58a067b1d6 100644
--- a/web/src/features/audit-logs/auditLog.ts
+++ b/web/src/features/audit-logs/auditLog.ts
@@ -58,6 +58,7 @@ type AuditableResource =
| "ssoConfig"
| "gatewayConfig"
| "gatewayAiConnection"
+ | "topicConfig"
// legacy resources
| "membership";
diff --git a/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleSamplingSection.tsx b/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleSamplingSection.tsx
index 6d2c17caeb..2756699a7c 100644
--- a/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleSamplingSection.tsx
+++ b/web/src/features/evals/v2/components/Rules/RuleSetup/components/RuleSamplingSection.tsx
@@ -8,7 +8,15 @@ import {
} from "@/src/features/evals/v2/constants/ruleSampling";
import type { RuleSetupStore } from "@/src/features/evals/v2/types/rules";
-export function RuleSamplingSection({ store }: { store: RuleSetupStore }) {
+export function RuleSamplingSection({
+ store,
+ description = "Set the percentage of matching observations this rule evaluates.",
+ tooltip = "The percentage of matching observations that will be evaluated. Lower sampling rates reduce evaluation volume and cost.",
+}: {
+ store: RuleSetupStore;
+ description?: string;
+ tooltip?: string;
+}) {
const sampling = useStore(store, (state) => state.sampling);
const setSampling = useStore(store, (state) => state.actions.setSampling);
@@ -17,14 +25,9 @@ export function RuleSamplingSection({ store }: { store: RuleSetupStore }) {
<div>
<div className="flex items-center gap-1.5">
<h3 className="text-sm font-bold">Sampling rate</h3>
- <InfoTooltip label="About sampling rate">
- The percentage of matching observations that will be evaluated.
- Lower sampling rates reduce evaluation volume and cost.
- </InfoTooltip>
+ <InfoTooltip label="About sampling rate">{tooltip}</InfoTooltip>
</div>
- <p className="text-muted-foreground text-sm">
- Set the percentage of matching observations this rule evaluates.
- </p>
+ <p className="text-muted-foreground text-sm">{description}</p>
</div>
<Slider
min={SAMPLING_SLIDER_MIN}
diff --git a/web/src/features/llm-api-key/server/router.ts b/web/src/features/llm-api-key/server/router.ts
index 4d71613733..60754d28b6 100644
--- a/web/src/features/llm-api-key/server/router.ts
+++ b/web/src/features/llm-api-key/server/router.ts
@@ -51,6 +51,7 @@ import {
validateLlmConnectionBaseURL,
} from "@langfuse/shared/src/server";
import { env } from "@/src/env.mjs";
+import { pauseTopicsModelsUsingConnection } from "@langfuse/shared/topics/server";
import { TRPCError } from "@trpc/server";
export function getDisplaySecretKey(secretKey: string) {
@@ -456,6 +457,15 @@ export const llmApiKeyRouter = createTRPCRouter({
})
: EMPTY_EVALUATOR_BLOCK;
+ await pauseTopicsModelsUsingConnection(
+ {
+ projectId: input.projectId,
+ llmApiKeyId: llmApiKey.id,
+ provider: llmApiKey.provider,
+ },
+ tx,
+ );
+
await tx.llmApiKeys.delete({
where: {
id: input.id,
diff --git a/web/src/features/topics/ConfigureTopicsDialog.stories.tsx b/web/src/features/topics/ConfigureTopicsDialog.stories.tsx
new file mode 100644
index 0000000000..4fe1701ac2
--- /dev/null
+++ b/web/src/features/topics/ConfigureTopicsDialog.stories.tsx
@@ -0,0 +1,141 @@
+import { expect, fn, userEvent, within } from "storybook/test";
+
+import preview from "../../../.storybook/preview";
+import {
+ ConfigureTopicsDialog,
+ type ConfigureTopicsDraft,
+} from "./ConfigureTopicsDialog";
+
+const providerGroups: Array<[string, string[]]> = [
+ [
+ "Amazon Bedrock",
+ ["us.openai.gpt-6-luna", "us.cohere.embed-v4:0", "us.openai.gpt-5.6-terra"],
+ ],
+ ["OpenAI", ["gpt-6-luna", "text-embedding-3-small", "gpt-5.6-terra"]],
+ ["Anthropic", ["claude-sonnet-4-5"]],
+];
+
+const builtInFacets = [
+ {
+ id: "intent",
+ name: "Intent",
+ question: "What is the user trying to do?",
+ builtIn: true,
+ enabled: true,
+ },
+ {
+ id: "outcome",
+ name: "Outcome",
+ question: "Did the assistant resolve the request?",
+ builtIn: true,
+ enabled: true,
+ },
+];
+
+const emptyDraft = {
+ summary: null,
+ embedding: null,
+ embeddingDimensions: "1024",
+ clustering: null,
+ facets: builtInFacets,
+ filters: [],
+ sampling: 1,
+ idleSeconds: "600",
+ embeddingLocked: false,
+} satisfies ConfigureTopicsDraft;
+
+const filledDraft = {
+ summary: { provider: "Amazon Bedrock", model: "us.openai.gpt-6-luna" },
+ embedding: { provider: "Amazon Bedrock", model: "us.cohere.embed-v4:0" },
+ embeddingDimensions: "1024",
+ clustering: { provider: "Amazon Bedrock", model: "us.openai.gpt-5.6-terra" },
+ facets: [
+ ...builtInFacets,
+ {
+ id: "frustration",
+ name: "Frustration",
+ question: "Is the user frustrated?",
+ builtIn: false,
+ enabled: false,
+ },
+ ],
+ idleSeconds: "600",
+ filters: [
+ { column: "name", type: "string", operator: "=", value: "billing" },
+ ],
+ sampling: 0.4,
+ embeddingLocked: true,
+} satisfies ConfigureTopicsDraft;
+
+const meta = preview.meta({
+ component: ConfigureTopicsDialog,
+ parameters: { layout: "fullscreen" },
+ args: {
+ providerGroups,
+ onConfigureProviders: fn(),
+ onSave: fn(),
+ },
+});
+
+export const NeedsSetup = meta.story({
+ args: {
+ draft: emptyDraft,
+ defaultOpen: true,
+ notice: "none",
+ triggerVariant: "primary",
+ },
+});
+
+export const Configured = meta.story({
+ args: {
+ draft: filledDraft,
+ defaultOpen: true,
+ notice: "none",
+ triggerVariant: "secondary",
+ },
+});
+
+export const SavesTheSheet = meta.story({
+ name: "(Test) Save sends facets, filter, idle time, and sampling",
+ args: {
+ draft: filledDraft,
+ defaultOpen: true,
+ notice: "none",
+ triggerVariant: "secondary",
+ },
+ play: async ({ args }) => {
+ const body = within(document.body);
+ const idle = body.getByRole("spinbutton", {
+ name: "Seconds to wait after the last observation",
+ });
+ await userEvent.clear(idle);
+ await userEvent.type(idle, "90");
+ await userEvent.click(body.getByRole("tab", { name: "Outcome" }));
+ await userEvent.click(body.getByRole("switch", { name: "Enable Outcome" }));
+ await userEvent.click(body.getByRole("button", { name: "Save" }));
+ await expect(args.onSave).toHaveBeenCalledWith(
+ expect.objectContaining({
+ idleSeconds: "90",
+ sampling: 0.4,
+ filters: filledDraft.filters,
+ facets: expect.arrayContaining([
+ expect.objectContaining({ name: "Intent", enabled: true }),
+ expect.objectContaining({ name: "Outcome", enabled: false }),
+ ]),
+ }),
+ );
+ },
+});
+
+export const Paused = meta.story({
+ args: {
+ draft: filledDraft,
+ defaultOpen: true,
+ notice: "paused",
+ shortLabel: "LLM connection missing",
+ pausedAgo: "3 days ago",
+ message:
+ "No LLM connection found for the provider used by facet summaries. Add or restore the LLM connection, then reactivate Topics.",
+ triggerVariant: "primary",
+ },
+});
diff --git a/web/src/features/topics/ConfigureTopicsDialog.tsx b/web/src/features/topics/ConfigureTopicsDialog.tsx
new file mode 100644
index 0000000000..f2ab90b324
--- /dev/null
+++ b/web/src/features/topics/ConfigureTopicsDialog.tsx
@@ -0,0 +1,1093 @@
+import { useMemo, useState } from "react";
+import {
+ AlertTriangle,
+ ExternalLinkIcon,
+ Info,
+ RefreshCcw,
+} from "lucide-react";
+import { Alert } from "@/src/components/design-system/Alert/Alert";
+import { Button } from "@/src/components/design-system/Button/Button";
+import { Input } from "@/src/components/design-system/Input/Input";
+import { Switch } from "@/src/components/design-system/Switch/Switch";
+import { Button as LegacyButton } from "@/src/components/ui/button";
+import { PopoverTrigger } from "@/src/components/ui/popover";
+import {
+ Sheet,
+ SheetContent,
+ SheetDescription,
+ SheetHeader,
+ SheetTitle,
+} from "@/src/components/ui/sheet";
+import {
+ Tooltip,
+ TooltipContent,
+ TooltipTrigger,
+} from "@/src/components/ui/tooltip";
+import { ObservationFilterBuilder } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder";
+import { FilterModeToggle } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/FilterModeToggle";
+import { RULE_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/ruleSearchRegistry";
+import {
+ EventsSearchBarRow,
+ toObservedOptions,
+ useEventsSearchBar,
+} from "@/src/features/search-bar";
+import { SectionHeader } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SectionHeader/SectionHeader";
+import { Tabs } from "@/src/components/design-system/Tabs/Tabs";
+import { Badge } from "@/src/components/design-system/Badge/Badge";
+import { RuleSamplingSection } from "@/src/features/evals/v2/components/Rules/RuleSetup/components/RuleSamplingSection";
+import { createRuleSetupStore } from "@/src/features/evals/v2/stores/createRuleSetupStore";
+import type { RuleSetupStore } from "@/src/features/evals/v2/types/rules";
+import { Stepper } from "@/src/features/evals/v2/components/Stepper/Stepper";
+import type { EvaluatorFilterExperience } from "@/src/features/evals/v2/types/evaluatorFilterExperience";
+import {
+ JudgeModelPicker,
+ JudgeModelPickerTrigger,
+} from "@/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker";
+import { type JudgeModel } from "@/src/features/evals/v2/judgeModel";
+import { format } from "date-fns";
+import {
+ eventsEvalFilterColumns,
+ experimentEvalFilterColsWithOptions,
+ observationEvalFilterColsWithOptions,
+ type FilterState,
+ type TimeFilter,
+ type TracingSearchType,
+} from "@langfuse/shared";
+import { useEventsFilterOptions } from "@/src/features/events/hooks/useEventsFilterOptions";
+import { api, sendAsPostOption } from "@/src/utils/api";
+import { relativeTopicTimeRange } from "./time-range";
+
+type FacetDraft = {
+ id: string;
+ name: string;
+ question: string;
+ builtIn: boolean;
+ enabled: boolean;
+};
+
+const MAX_ENABLED_FACETS = 5;
+const ADD_FACET_TAB = "add-facet";
+
+export type TopicSlotCheck = {
+ status: "idle" | "testing" | "ok" | "error";
+ message?: string;
+};
+
+export type ConfigureTopicsSaveDraft = {
+ summary: JudgeModel | null;
+ embedding: JudgeModel | null;
+ embeddingDimensions: string;
+ clustering: JudgeModel | null;
+ facets: FacetDraft[];
+ filters: FilterState;
+ sampling: number;
+ idleSeconds: string;
+};
+
+export type ConfigureTopicsDraft = {
+ summary: JudgeModel | null;
+ embedding: JudgeModel | null;
+ embeddingDimensions: string;
+ clustering: JudgeModel | null;
+ facets: FacetDraft[];
+ filters: FilterState;
+ sampling: number;
+ idleSeconds: string;
+ embeddingLocked: boolean;
+};
+
+type ConfigureTopicsDialogProps = {
+ providerGroups: Array<[string, string[]]>;
+ embeddingProviderGroups?: Array<[string, string[]]>;
+ draft: ConfigureTopicsDraft;
+ defaultOpen: boolean;
+ onConfigureProviders: () => void;
+ onSave: (draft: ConfigureTopicsSaveDraft) => void | Promise<void>;
+ onSlotCommit?: (
+ slot: "summary" | "embedding" | "clustering",
+ model: JudgeModel,
+ dimensions: string,
+ ) => void;
+ slotChecks?: Partial<
+ Record<"summary" | "embedding" | "clustering", TopicSlotCheck>
+ >;
+ canWrite?: boolean;
+ saving?: boolean;
+ saveError?: string | null;
+ projectId?: string;
+ /** Storybook keeps the sample filter. The project page chooses traces when a run starts. */
+ sampleScope?: boolean;
+ triggerVariant: "primary" | "secondary";
+} & (
+ | { notice: "none" }
+ | {
+ notice: "paused";
+ shortLabel: string;
+ pausedAgo: string;
+ message: string;
+ }
+);
+
+const SLOT_COPY = {
+ summary: {
+ label: "Facet summaries",
+ hint: "A small, fast model. It runs once per trace.",
+ },
+ embedding: {
+ label: "Embeddings",
+ hint: "Locked after the first summaries are embedded.",
+ },
+ clustering: {
+ label: "Topic clustering",
+ hint: "Names each topic when topics are updated.",
+ },
+} as const;
+
+export function ConfigureTopicsDialog(props: ConfigureTopicsDialogProps) {
+ const {
+ defaultOpen,
+ draft,
+ canWrite = true,
+ onConfigureProviders,
+ onSave,
+ onSlotCommit,
+ projectId,
+ providerGroups,
+ embeddingProviderGroups = providerGroups,
+ sampleScope = true,
+ saveError = null,
+ saving = false,
+ slotChecks,
+ triggerVariant,
+ } = props;
+ const paused =
+ props.notice === "paused"
+ ? {
+ shortLabel: props.shortLabel,
+ pausedAgo: props.pausedAgo,
+ message: props.message,
+ }
+ : null;
+ const [open, setOpen] = useState(defaultOpen);
+ const [summary, setSummary] = useState(draft.summary);
+ const [embedding, setEmbedding] = useState(draft.embedding);
+ const [dimensions, setDimensions] = useState(draft.embeddingDimensions);
+ const [clustering, setClustering] = useState(draft.clustering);
+ const [facets, setFacets] = useState(draft.facets);
+ const [browsing, setBrowsing] = useState(facets[0]?.id ?? ADD_FACET_TAB);
+ const [idleSeconds, setIdleSeconds] = useState(draft.idleSeconds);
+ const [filters, setFilters] = useState(draft.filters);
+ const [samplingStore] = useState(() =>
+ createRuleSetupStore({
+ name: "",
+ filter: [],
+ sampling: draft.sampling,
+ assignments: [],
+ }),
+ );
+ const modelsReady = Boolean(summary && embedding && clustering);
+ const enabledFacets = facets.filter((facet) => facet.enabled);
+ const testsReady =
+ !slotChecks ||
+ (["summary", "embedding", "clustering"] as const).every(
+ (slot) => slotChecks[slot]?.status === "ok",
+ );
+ const canSave =
+ canWrite &&
+ !saving &&
+ modelsReady &&
+ enabledFacets.length > 0 &&
+ testsReady;
+ const browsed = facets.find((facet) => facet.id === browsing) ?? null;
+ const atFacetCap = enabledFacets.length >= MAX_ENABLED_FACETS;
+ const commitSlot = (
+ slot: "summary" | "embedding" | "clustering",
+ model: JudgeModel,
+ nextDimensions = dimensions,
+ ) => {
+ onSlotCommit?.(slot, model, nextDimensions);
+ };
+ const setFacetEnabled = (id: string, enabled: boolean) => {
+ setFacets((current) => {
+ const enabledCount = current.filter((facet) => facet.enabled).length;
+ const target = current.find((facet) => facet.id === id);
+ if (
+ enabled &&
+ target &&
+ !target.enabled &&
+ enabledCount >= MAX_ENABLED_FACETS
+ )
+ return current;
+ return current.map((facet) =>
+ facet.id === id ? { ...facet, enabled } : facet,
+ );
+ });
+ };
+
+ return (
+ <div className="flex flex-col items-start gap-4">
+ <Button
+ text="Configure Topics"
+ variant={triggerVariant}
+ size="sm"
+ onClick={() => setOpen(true)}
+ />
+ <Sheet open={open} onOpenChange={setOpen}>
+ <SheetContent
+ overlayClassName="bg-transparent"
+ className="flex flex-col gap-0 overflow-hidden p-0 sm:max-w-4xl"
+ >
+ <SheetHeader className="items-start space-y-1 p-6 pb-4 text-left">
+ <SheetTitle>Configure Topics</SheetTitle>
+ <SheetDescription>
+ Models, what to look for, and which traces to run.
+ </SheetDescription>
+ </SheetHeader>
+ <form
+ className="flex min-h-0 flex-1 flex-col"
+ onSubmit={(event) => {
+ event.preventDefault();
+ Promise.resolve(
+ onSave({
+ summary,
+ embedding,
+ embeddingDimensions: dimensions,
+ clustering,
+ facets,
+ filters,
+ sampling: samplingStore.getState().sampling,
+ idleSeconds,
+ }),
+ )
+ .then(() => setOpen(false))
+ .catch(() => undefined);
+ }}
+ >
+ <div className="flex min-h-0 flex-1 flex-col gap-4 overflow-y-auto px-6 pb-4">
+ <Alert variant="info" icon={Info}>
+ <Alert.Description>
+ Incoming traces are processed automatically only when
+ ingestion is v4-ready. Legacy ingestion is not included.
+ </Alert.Description>
+ </Alert>
+ {paused ? (
+ <PausedNotice
+ canReactivate={canSave}
+ shortLabel={paused.shortLabel}
+ pausedAgo={paused.pausedAgo}
+ message={paused.message}
+ onConfigureProviders={onConfigureProviders}
+ />
+ ) : null}
+ <Stepper
+ number={1}
+ title="Models"
+ description="The connections and models used for summaries, embeddings, and clustering."
+ compactBottomSpacing
+ >
+ <ModelSlot
+ name="summary"
+ model={summary}
+ locked={false}
+ disabled={!canWrite}
+ check={slotChecks?.summary}
+ connectionMissing={paused !== null}
+ providerGroups={providerGroups}
+ onConfigureProviders={onConfigureProviders}
+ onSelect={(model) => {
+ setSummary(model);
+ commitSlot("summary", model);
+ }}
+ />
+ <ModelSlot
+ name="embedding"
+ model={embedding}
+ locked={draft.embeddingLocked}
+ disabled={!canWrite}
+ check={slotChecks?.embedding}
+ dimensions={dimensions}
+ onDimensionsChange={(value) => {
+ setDimensions(value);
+ if (embedding) commitSlot("embedding", embedding, value);
+ }}
+ providerGroups={embeddingProviderGroups}
+ onConfigureProviders={onConfigureProviders}
+ onSelect={(model) => {
+ setEmbedding(model);
+ commitSlot("embedding", model);
+ }}
+ />
+ <ModelSlot
+ name="clustering"
+ model={clustering}
+ locked={false}
+ disabled={!canWrite}
+ check={slotChecks?.clustering}
+ providerGroups={providerGroups}
+ onConfigureProviders={onConfigureProviders}
+ onSelect={(model) => {
+ setClustering(model);
+ commitSlot("clustering", model);
+ }}
+ />
+ </Stepper>
+ <Stepper
+ number={2}
+ title="Insights"
+ description="Choose which questions you want answered. Five can be on."
+ compactBottomSpacing
+ >
+ <FacetPicker
+ facets={facets}
+ browsing={browsing}
+ browsed={browsed}
+ atCap={atFacetCap}
+ canWrite={canWrite}
+ onBrowse={setBrowsing}
+ onEnabledChange={setFacetEnabled}
+ />
+ </Stepper>
+ <Stepper
+ number={3}
+ title="Run settings"
+ description="Choose which traces to include and set the idle time."
+ compactBottomSpacing
+ >
+ <div className="flex items-center gap-2">
+ <div className="w-24">
+ <Input
+ aria-label="Seconds to wait after the last observation"
+ type="number"
+ inputMode="numeric"
+ min={60}
+ value={idleSeconds}
+ onChange={(event) => setIdleSeconds(event.target.value)}
+ />
+ </div>
+ <p className="text-muted-foreground text-sm">
+ seconds to wait after the last observation before
+ considering the trace done and summarizing it.
+ </p>
+ </div>
+ {sampleScope ? (
+ <RuleScope
+ projectId={projectId}
+ filters={filters}
+ onFiltersChange={setFilters}
+ samplingStore={samplingStore}
+ />
+ ) : null}
+ </Stepper>
+ </div>
+ {saveError ? (
+ <p className="text-destructive px-6 pb-2 text-sm">{saveError}</p>
+ ) : null}
+ <div className="mt-auto flex items-center justify-end gap-2 border-t p-4">
+ <div className="flex gap-2">
+ <LegacyButton
+ type="button"
+ variant="outline"
+ onClick={() => setOpen(false)}
+ >
+ Cancel
+ </LegacyButton>
+ <LegacyButton type="submit" disabled={!canSave}>
+ {saving ? "Saving…" : "Save"}
+ </LegacyButton>
+ </div>
+ </div>
+ </form>
+ </SheetContent>
+ </Sheet>
+ </div>
+ );
+}
+
+function ModelSlot({
+ check,
+ connectionMissing = false,
+ dimensions,
+ disabled = false,
+ locked,
+ model,
+ name,
+ onConfigureProviders,
+ onDimensionsChange,
+ onSelect,
+ providerGroups,
+}: {
+ check?: TopicSlotCheck;
+ connectionMissing?: boolean;
+ dimensions?: string;
+ disabled?: boolean;
+ locked: boolean;
+ model: JudgeModel | null;
+ name: keyof typeof SLOT_COPY;
+ onConfigureProviders: () => void;
+ onDimensionsChange?: (dimensions: string) => void;
+ onSelect: (model: JudgeModel) => void;
+ providerGroups: Array<[string, string[]]>;
+}) {
+ const [open, setOpen] = useState(false);
+ const copy = SLOT_COPY[name];
+ const inactive = locked || disabled;
+ const controls = (
+ <>
+ <div
+ className={`min-w-0 flex-1 [&_button]:w-full ${inactive ? "[&_button]:pointer-events-none" : ""}`}
+ >
+ <JudgeModelPicker
+ purpose="decision"
+ open={open && !inactive}
+ onOpenChange={setOpen}
+ providerGroups={providerGroups}
+ selectedModel={model}
+ onSelect={onSelect}
+ onConfigureProviders={onConfigureProviders}
+ >
+ <PopoverTrigger asChild>
+ <JudgeModelPickerTrigger
+ mode={model ? "custom" : "default"}
+ selectedModel={model}
+ disabled={inactive}
+ modelAvailability={connectionMissing ? "missing" : "available"}
+ />
+ </PopoverTrigger>
+ </JudgeModelPicker>
+ </div>
+ {dimensions !== undefined ? (
+ <div
+ className={`w-20 shrink-0 ${inactive ? "pointer-events-none" : ""}`}
+ >
+ <Input
+ aria-label="Embedding dimensions"
+ type="number"
+ inputMode="numeric"
+ value={dimensions}
+ disabled={inactive}
+ onChange={(event) => onDimensionsChange?.(event.target.value)}
+ />
+ </div>
+ ) : null}
+ </>
+ );
+ const fields = (
+ <div className="flex min-w-0 flex-1 items-center gap-2">{controls}</div>
+ );
+ return (
+ <div className="flex items-start gap-2">
+ <p
+ className="flex h-8 w-36 shrink-0 items-center text-sm font-bold"
+ title={copy.hint}
+ >
+ {copy.label}
+ </p>
+ <div className="flex min-w-0 flex-1 flex-col gap-1">
+ {locked ? (
+ <Tooltip>
+ <TooltipTrigger asChild>{fields}</TooltipTrigger>
+ <TooltipContent side="bottom">{copy.hint}</TooltipContent>
+ </Tooltip>
+ ) : (
+ fields
+ )}
+ {check?.status === "testing" ? (
+ <p className="text-muted-foreground text-sm">Testing this model…</p>
+ ) : null}
+ {check?.status === "error" && check.message ? (
+ <p className="text-destructive text-sm">{check.message}</p>
+ ) : null}
+ </div>
+ </div>
+ );
+}
+
+function FacetPicker({
+ atCap,
+ browsed,
+ browsing,
+ canWrite,
+ facets,
+ onBrowse,
+ onEnabledChange,
+}: {
+ atCap: boolean;
+ browsed: FacetDraft | null;
+ browsing: string;
+ canWrite: boolean;
+ facets: FacetDraft[];
+ onBrowse: (id: string) => void;
+ onEnabledChange: (id: string, enabled: boolean) => void;
+}) {
+ const enabledCount = facets.filter((facet) => facet.enabled).length;
+ return (
+ <div className="flex flex-col gap-3">
+ <div className="max-w-full overflow-x-auto">
+ <Tabs value={browsing} onValueChange={onBrowse}>
+ <Tabs.List
+ aria-label="Facets"
+ variant="inset"
+ size="sm"
+ layout="packed"
+ >
+ {facets.map((facet) => (
+ <Tabs.Trigger key={facet.id} value={facet.id} title={facet.name}>
+ <span
+ className={`size-1.5 shrink-0 rounded-full ${facet.enabled ? "bg-dark-green" : "bg-border"}`}
+ />
+ <span className="truncate" title={facet.name}>
+ {facet.name}
+ </span>
+ </Tabs.Trigger>
+ ))}
+ <Tabs.Trigger value={ADD_FACET_TAB} label="Add my own" />
+ </Tabs.List>
+ </Tabs>
+ </div>
+ {browsed ? (
+ <div className="flex flex-col gap-2">
+ <p className="text-sm">{browsed.question}</p>
+ <label className="flex items-center gap-2 text-sm">
+ <Switch
+ checked={browsed.enabled}
+ disabled={!canWrite || (atCap && !browsed.enabled)}
+ aria-label={`Enable ${browsed.name}`}
+ onCheckedChange={(checked) =>
+ onEnabledChange(browsed.id, checked)
+ }
+ />
+ {browsed.enabled ? "On" : "Off"}
+ {browsed.builtIn ? (
+ <span className="text-muted-foreground text-xs">Built-in</span>
+ ) : null}
+ </label>
+ {atCap && !browsed.enabled ? (
+ <p className="text-muted-foreground text-xs">
+ Five facets can be on. Turn one off to enable another.
+ </p>
+ ) : (
+ <p className="text-muted-foreground text-xs">
+ {enabledCount} of {MAX_ENABLED_FACETS} on
+ </p>
+ )}
+ </div>
+ ) : (
+ <p className="text-muted-foreground text-sm">
+ Custom questions are added on their own page. This setup uses the
+ facets already in the project.
+ </p>
+ )}
+ </div>
+ );
+}
+
+const SAMPLE_TRACES_IN_SCOPE = [
+ {
+ start: "2026-10-08 15:58:44",
+ type: "GENERATION",
+ name: "billing-question",
+ input: '{"role":"user","content":"My subscription invoice..."}',
+ },
+ {
+ start: "2026-10-08 15:58:43",
+ type: "GENERATION",
+ name: "weather-question",
+ input: '{"role":"user","content":"I\'m visiting Lisbon this..."}',
+ },
+ {
+ start: "2026-10-08 15:54:02",
+ type: "SPAN",
+ name: "weather-question",
+ input: '{"role":"user","content":"What is the weather in..."}',
+ },
+];
+
+const topicsFilterRegistry = {
+ ...RULE_FIELD_REGISTRY,
+ aiFilterPrompt: true,
+ fields: RULE_FIELD_REGISTRY.fields.filter(
+ (field) => field.id !== "isRootObservation",
+ ),
+ columns: RULE_FIELD_REGISTRY.columns.filter(
+ (column) => column.id !== "isRootObservation",
+ ),
+};
+
+const queryOnlyColumnIds = topicsFilterRegistry.fields
+ .filter((field) => field.directFilter === false)
+ .map((field) => field.filterColumn ?? field.id);
+
+const BUILDER_OPTION_COLUMNS = [
+ "environment",
+ "name",
+ "traceTags",
+ "traceName",
+ "calledToolNames",
+] as const;
+
+function authoredTraceFilters(filters: FilterState): FilterState {
+ return filters.flatMap((filter) => {
+ if (filter.column === "isRootObservation") return [];
+ if (filter.column === "tags")
+ return [{ ...filter, column: "traceTags" as const }];
+ return [filter];
+ });
+}
+
+function matchCountLabel(count: number | null) {
+ if (count === null) return null;
+ const noun = count === 1 ? "match" : "matches";
+ return `(${count} ${noun})`;
+}
+
+function previewStatus(
+ pending: boolean,
+ error: string | undefined,
+ rowCount: number,
+) {
+ if (pending) return "Loading traces…";
+ if (error) return error;
+ if (rowCount === 0) return "No traces match this filter in the last 7 days.";
+ return null;
+}
+
+function RuleScope({
+ projectId,
+ filters,
+ onFiltersChange,
+ samplingStore,
+}: {
+ projectId?: string;
+ filters: FilterState;
+ onFiltersChange: (filters: FilterState) => void;
+ samplingStore: RuleSetupStore;
+}) {
+ if (!projectId)
+ return (
+ <SampleTraceScope
+ filters={filters}
+ onFiltersChange={onFiltersChange}
+ samplingStore={samplingStore}
+ />
+ );
+ return (
+ <ProjectTraceScope
+ projectId={projectId}
+ filters={filters}
+ onFiltersChange={onFiltersChange}
+ samplingStore={samplingStore}
+ />
+ );
+}
+
+function SampleTraceScope({
+ filters,
+ onFiltersChange,
+ samplingStore,
+}: {
+ filters: FilterState;
+ onFiltersChange: (filters: FilterState) => void;
+ samplingStore: RuleSetupStore;
+}) {
+ return (
+ <TraceScopeFrame
+ projectId={undefined}
+ filters={filters}
+ onFiltersChange={onFiltersChange}
+ samplingStore={samplingStore}
+ countLabel="Sample"
+ rows={SAMPLE_TRACES_IN_SCOPE}
+ />
+ );
+}
+
+function ProjectTraceScope({
+ projectId,
+ filters,
+ onFiltersChange,
+ samplingStore,
+}: {
+ projectId: string;
+ filters: FilterState;
+ onFiltersChange: (filters: FilterState) => void;
+ samplingStore: RuleSetupStore;
+}) {
+ const [range] = useState(() => relativeTopicTimeRange(7));
+ const [mode, setMode] = useState<EvaluatorFilterExperience>("query");
+ const [searchQuery, setSearchQuery] = useState<string | null>(null);
+ const [searchType, setSearchType] = useState<TracingSearchType[]>([]);
+ const startTimeFilter = useMemo<TimeFilter[]>(
+ () => [
+ {
+ column: "startTime",
+ type: "datetime",
+ operator: ">=",
+ value: range.from,
+ },
+ {
+ column: "startTime",
+ type: "datetime",
+ operator: "<",
+ value: range.to,
+ },
+ ],
+ [range],
+ );
+ const authored = useMemo(() => authoredTraceFilters(filters), [filters]);
+ const options = useEventsFilterOptions({
+ projectId,
+ startTimeFilter,
+ refiningFilter: authored,
+ isRootObservation: true,
+ includeApproxCount: true,
+ lazy: mode === "query",
+ columns: mode === "query" ? undefined : [...BUILDER_OPTION_COLUMNS],
+ enabled: true,
+ });
+ const observed = useMemo(
+ () =>
+ toObservedOptions(options.filterOptions, options.isFilterOptionsPending),
+ [options.filterOptions, options.isFilterOptionsPending],
+ );
+ const previewFilters = useMemo<FilterState>(
+ () => [
+ ...authored,
+ {
+ column: "isRootObservation",
+ type: "boolean",
+ operator: "=",
+ value: true,
+ },
+ ...startTimeFilter,
+ ],
+ [authored, startTimeFilter],
+ );
+ const preview = api.events.listCursor.useQuery({
+ projectId,
+ filter: previewFilters,
+ searchQuery,
+ searchType,
+ limit: 10,
+ });
+ const observations = preview.data?.observations ?? [];
+ const ioTargets = observations.flatMap((observation) =>
+ observation.traceId
+ ? [{ id: observation.id, traceId: observation.traceId }]
+ : [],
+ );
+ const startTimes = observations.map((observation) =>
+ observation.startTime.getTime(),
+ );
+ const io = api.events.batchIO.useQuery(
+ {
+ projectId,
+ observations: ioTargets,
+ minStartTime: new Date(
+ startTimes.length > 0 ? Math.min(...startTimes) : 0,
+ ),
+ maxStartTime: new Date(
+ startTimes.length > 0 ? Math.max(...startTimes) : 0,
+ ),
+ },
+ {
+ ...sendAsPostOption,
+ enabled: preview.isSuccess && ioTargets.length > 0,
+ },
+ );
+ const inputById = new Map((io.data ?? []).map((row) => [row.id, row.input]));
+ const builderColumns = useMemo(
+ () =>
+ experimentEvalFilterColsWithOptions(
+ options.filterOptions,
+ observationEvalFilterColsWithOptions(
+ {
+ ...options.filterOptions,
+ tags: options.filterOptions?.traceTags,
+ },
+ [...eventsEvalFilterColumns],
+ ),
+ ).flatMap((column) =>
+ column.id === "isRootObservation" ? [] : [column],
+ ),
+ [options.filterOptions],
+ );
+ const rows = observations.map((observation) => ({
+ id: observation.id,
+ start: format(observation.startTime, "yyyy-MM-dd HH:mm:ss"),
+ type: observation.type,
+ name: observation.name ?? "—",
+ input: inputById.get(observation.id) ?? (io.isPending ? "…" : ""),
+ }));
+ const status = previewStatus(
+ preview.isPending,
+ preview.error?.message,
+ rows.length,
+ );
+
+ return (
+ <TraceScopeFrame
+ projectId={projectId}
+ mode={mode}
+ filters={filters}
+ searchQuery={searchQuery}
+ observed={observed}
+ builderColumns={builderColumns}
+ erroredColumns={options.erroredColumns}
+ onRequestColumns={options.requestColumns}
+ countLabel={matchCountLabel(options.approxTotalCount)}
+ description="Root observations from the last 7 days. Topics summarizes the whole trace."
+ tooltip="The same filter Topics uses when it selects traces, limited to the last 7 days. Only root observations are shown."
+ rows={rows}
+ status={status}
+ searchType={searchType}
+ onModeChange={setMode}
+ onFiltersChange={onFiltersChange}
+ samplingStore={samplingStore}
+ onSearchQueryChange={setSearchQuery}
+ onSearchTypeChange={setSearchType}
+ />
+ );
+}
+
+function TraceScopeFrame({
+ projectId,
+ mode: modeProp,
+ filters: filtersProp,
+ searchQuery: searchQueryProp,
+ searchType: searchTypeProp,
+ observed,
+ builderColumns,
+ erroredColumns,
+ onRequestColumns,
+ countLabel,
+ description,
+ tooltip,
+ rows,
+ status,
+ samplingStore,
+ onModeChange,
+ onFiltersChange,
+ onSearchQueryChange,
+ onSearchTypeChange,
+}: {
+ projectId?: string;
+ mode?: EvaluatorFilterExperience;
+ filters?: FilterState;
+ searchQuery?: string | null;
+ searchType?: TracingSearchType[];
+ observed?: ReturnType<typeof toObservedOptions>;
+ builderColumns?: ReturnType<typeof experimentEvalFilterColsWithOptions>;
+ erroredColumns?: ReadonlySet<string>;
+ onRequestColumns?: (columns: readonly string[]) => void;
+ countLabel?: string | null;
+ description?: string;
+ tooltip?: string;
+ rows: Array<{
+ id?: string;
+ start: string;
+ type: string;
+ name: string;
+ input: string;
+ }>;
+ status?: string | null;
+ samplingStore: RuleSetupStore;
+ onModeChange?: (mode: EvaluatorFilterExperience) => void;
+ onFiltersChange?: (filters: FilterState) => void;
+ onSearchQueryChange?: (query: string | null) => void;
+ onSearchTypeChange?: (searchType: TracingSearchType[]) => void;
+}) {
+ const [localMode, setLocalMode] =
+ useState<EvaluatorFilterExperience>("query");
+ const [localFilters, setLocalFilters] = useState<FilterState>([]);
+ const [localSearchQuery, setLocalSearchQuery] = useState<string | null>(null);
+ const [localSearchType, setLocalSearchType] = useState<TracingSearchType[]>(
+ [],
+ );
+ const mode = modeProp ?? localMode;
+ const filters = filtersProp ?? localFilters;
+ const searchQuery = searchQueryProp ?? localSearchQuery;
+ const searchType = searchTypeProp ?? localSearchType;
+ const setMode = onModeChange ?? setLocalMode;
+ const setFilters = onFiltersChange ?? setLocalFilters;
+ const setSearchQuery = onSearchQueryChange ?? setLocalSearchQuery;
+ const setSearchType = onSearchTypeChange ?? setLocalSearchType;
+ const search = useEventsSearchBar({
+ projectId,
+ tableName: "topics-setup",
+ enabled: true,
+ filterState: filters,
+ searchQuery,
+ searchType,
+ observed,
+ setFilterState: setFilters,
+ setSearchQuery,
+ setSearchType,
+ registry: topicsFilterRegistry,
+ });
+
+ return (
+ <div className="flex flex-col gap-4">
+ <section className="flex flex-col gap-2">
+ <SectionHeader
+ title="Filter traces"
+ meta={<Badge text="Coming soon" color="blue" size="sm" />}
+ description="Topics processes every trace for now; custom trace filters are coming soon."
+ tooltip="Only traces matching this filter will be summarized. Topics always runs on whole traces."
+ trailing={
+ <div inert className="opacity-50">
+ <FilterModeToggle
+ mode={mode}
+ onChange={(next) => {
+ if (next === "builder" && searchQuery !== null)
+ search.applyFilters(filters);
+ setMode(next);
+ }}
+ />
+ </div>
+ }
+ />
+ <div inert aria-disabled className="opacity-50">
+ {mode === "query" ? (
+ <EventsSearchBarRow
+ projectId={projectId}
+ tableName="topics-setup"
+ store={search.store}
+ commit={search.commit}
+ observed={observed}
+ erroredColumns={erroredColumns}
+ registry={topicsFilterRegistry}
+ onApplyFilters={search.applyFilters}
+ onRequestColumns={onRequestColumns}
+ className="p-0"
+ />
+ ) : (
+ <ObservationFilterBuilder
+ columns={builderColumns ?? [...topicsFilterRegistry.columns]}
+ filterState={filters}
+ onChange={setFilters}
+ queryOnlyColumnIds={queryOnlyColumnIds}
+ />
+ )}
+ </div>
+ </section>
+ <section className="flex flex-col gap-2">
+ <SectionHeader
+ title="Matching traces"
+ meta={
+ countLabel ? (
+ <span className="text-muted-foreground text-sm">
+ {countLabel}
+ </span>
+ ) : null
+ }
+ description={
+ description ?? "A sample of traces this setup would include."
+ }
+ tooltip={
+ tooltip ??
+ "A fixed sample. The filter is applied when traces are processed, not in this table."
+ }
+ trailing={null}
+ />
+ <div className="max-h-72 overflow-auto rounded-md border">
+ <table className="w-full text-left text-sm">
+ <thead className="text-muted-foreground bg-background sticky top-0">
+ <tr className="border-b">
+ <th className="px-3 py-2 font-normal">Start time</th>
+ <th className="px-3 py-2 font-normal">Type</th>
+ <th className="px-3 py-2 font-normal">Name</th>
+ <th className="px-3 py-2 font-normal">Input</th>
+ </tr>
+ </thead>
+ <tbody>
+ {status ? (
+ <tr>
+ <td className="text-muted-foreground px-3 py-2" colSpan={4}>
+ {status}
+ </td>
+ </tr>
+ ) : (
+ rows.map((row) => (
+ <tr
+ key={row.id ?? `${row.start}-${row.name}`}
+ className="border-b last:border-b-0"
+ >
+ <td className="px-3 py-2 whitespace-nowrap">{row.start}</td>
+ <td className="px-3 py-2">{row.type}</td>
+ <td className="px-3 py-2">{row.name}</td>
+ <td
+ className="text-muted-foreground max-w-48 truncate px-3 py-2"
+ title={row.input}
+ >
+ {row.input}
+ </td>
+ </tr>
+ ))
+ )}
+ </tbody>
+ </table>
+ </div>
+ </section>
+ <RuleSamplingSection
+ store={samplingStore}
+ description="Set the percentage of matching traces Topics processes."
+ tooltip="Topics processes this share of the traces that match the filter. A lower rate processes fewer traces."
+ />
+ </div>
+ );
+}
+
+function PausedNotice({
+ canReactivate,
+ message,
+ onConfigureProviders,
+ pausedAgo,
+ shortLabel,
+}: {
+ canReactivate: boolean;
+ message: string;
+ onConfigureProviders: () => void;
+ pausedAgo: string;
+ shortLabel: string;
+}) {
+ return (
+ <section
+ role="alert"
+ className="border-light-yellow bg-light-yellow rounded-lg border"
+ >
+ <div className="flex gap-3 p-4">
+ <AlertTriangle className="text-dark-yellow icon-base mt-0.5 shrink-0" />
+ <div className="min-w-0 flex-1">
+ <h2 className="text-foreground text-base font-bold">Topics paused</h2>
+ <div className="text-muted-foreground mt-1 flex flex-wrap items-center gap-2 text-sm">
+ <span className="font-bold">{shortLabel}</span>
+ <span className="bg-border h-1 w-1 rounded-full" />
+ <span>Paused {pausedAgo}</span>
+ </div>
+ <p className="text-muted-foreground mt-2 text-sm">{message}</p>
+ <div className="mt-3 flex flex-wrap items-center gap-2">
+ <LegacyButton
+ type="button"
+ variant="outline"
+ size="sm"
+ onClick={onConfigureProviders}
+ >
+ <ExternalLinkIcon className="icon-base text-icon-foreground mr-1.5" />
+ Open LLM connections
+ </LegacyButton>
+ {/* Saving re-runs the model tests and clears the pause. */}
+ <LegacyButton
+ type="submit"
+ variant="outline"
+ size="sm"
+ disabled={!canReactivate}
+ >
+ <RefreshCcw className="icon-base text-icon-foreground mr-1.5" />
+ Reactivate
+ </LegacyButton>
+ </div>
+ </div>
+ </div>
+ </section>
+ );
+}
diff --git a/web/src/features/topics/README.md b/web/src/features/topics/README.md
index e856443de0..c37c2280d6 100644
--- a/web/src/features/topics/README.md
+++ b/web/src/features/topics/README.md
@@ -23,16 +23,19 @@ project permissions. Processing additionally requires the project allowlist.
in-flight poll cannot satisfy completion. The query retains the tRPC prefix
for mutation and manual invalidation.
The polling/completion race is covered directly by a frontend hook test.
- Configure topics opens a centered dialog; Process traces or Update topics
- submits the retained configuration. Overlay owners stay outside the responsive
- header menu. Facet versions contain only prompts; processing settings and
- embedding dimensions are frozen on executions. Built-in facets are read-only;
- custom facets can be revised by creating a new prompt version.
-- `useTopicPipelineForm` owns the configuration dialog, operation, facet versions
- and submission. The workspace renders its header actions and dialog separately;
- configuration changes preserve the current results and selection. Its state
- stays mounted when the dialog closes. Preview trace clicks close
- configuration before opening the trace peek in the panel layer.
+ Configure Topics opens a side sheet for models, facets, and which traces to run.
+ Saving it stores the models, creates the four built-in facets as evaluators
+ with a first version, and saves one rule for the filter, idle time, and
+ sampling rate. A project can have only one Topics rule.
+ Process traces or Update topics opens its own run dialog. Overlay owners stay
+ outside the responsive header menu. Facet versions contain only prompts;
+ processing settings and embedding dimensions are frozen on executions.
+ Built-in facets are read-only; custom facets are added on their own page.
+- `useTopicPipelineForm` owns the run dialog, operation, facet versions and
+ submission. The workspace renders its header action and dialog separately;
+ closing the dialog preserves the current results and selection. Its state
+ stays mounted when the dialog closes. Preview trace clicks close the run
+ dialog before opening the trace peek in the panel layer.
Every available facet starts selected. Process traces is the default operation:
it summarizes, embeds, and assigns only the selected batch to current topics.
Without a compatible map, summaries wait for an explicit Update topics run.
@@ -44,17 +47,14 @@ project permissions. Processing additionally requires the project allowlist.
IDs, and per-trace errors, using aggregate counts for progress.
Expanding Trace errors loads failed trace IDs and reasons on demand from
retained queue state. When it expires, the UI says so; permanent counts remain.
- Saved topic rules hold reusable filters and stable facet IDs. Selecting a rule
- loads its filters and each facet's latest prompt version; editing filters or
- facets becomes an ad hoc run until explicitly saved. Sampling and its limit stay
- specific to each execution and are retained when switching saved rules. Rules
- do not create separate maps or invalidate summaries and embeddings.
- These local PoC operations and rule saving add no product analytics event.
+ The run dialog chooses facets and traces for one run. It does not save that
+ choice. Sampling and its limit stay specific to each execution.
+ These local PoC operations add no product analytics event.
- `useTopicTraceSelector` reuses the eval filter builder and query editor, with
a time range, all matching traces selected by default, and optional random/latest
- sampling with a user-chosen size. Selecting a rule or changing the operation
- resets the trace-selection draft while retaining sampling settings; dates stay
- specific to each execution.
+ sampling with a user-chosen size. Changing the operation resets the
+ trace-selection draft while retaining sampling settings; dates stay specific
+ to each execution.
The hook supplies selection, criteria and controls so the dialog can unmount
without losing the reviewed cohort. Explicit preview counts the cohort;
changing criteria invalidates it. Rows can be excluded across preview pages.
@@ -139,7 +139,7 @@ project permissions. Processing additionally requires the project allowlist.
discovery cohort and coordinates while summary text reflects the latest stored
result. These reads survive transient job state expiry.
Resume requeues retained job inputs; if they have expired, start a new run.
- Facet prompt edits create versions independently of saved selection rules.
+ Facet prompt edits are a separate page from this setup.
- `parse-trace-input.ts` validates pasted trace IDs and links without fetching.
Trace IDs are opaque data; URL path segments are decoded once, while internal
execution identifiers retain the restricted ID format.
diff --git a/web/src/features/topics/TopicModelSettings.tsx b/web/src/features/topics/TopicModelSettings.tsx
new file mode 100644
index 0000000000..67d1b9043d
--- /dev/null
+++ b/web/src/features/topics/TopicModelSettings.tsx
@@ -0,0 +1,482 @@
+import { useRef, useState } from "react";
+import { formatDistanceToNow } from "date-fns";
+import { Alert } from "@/src/components/design-system/Alert/Alert";
+import { env } from "@/src/env.mjs";
+import { type JudgeModel } from "@/src/features/evals/v2/judgeModel";
+import { api, type RouterOutputs } from "@/src/utils/api";
+import {
+ getEvaluatorBlockMetadata,
+ LLMAdapter,
+ supportedModels,
+} from "@langfuse/shared";
+import {
+ DEFAULT_TOPIC_FACETS,
+ topicsModelSettingsSchema,
+ type TopicFacet,
+ type TopicRule,
+ type TopicsModelSlotName,
+} from "@langfuse/shared/topics";
+import {
+ ConfigureTopicsDialog,
+ type ConfigureTopicsSaveDraft,
+ type TopicSlotCheck,
+} from "./ConfigureTopicsDialog";
+import { topicsSetupPayload } from "./topics-setup";
+
+type StoredSettings = RouterOutputs["topics"]["modelSettings"];
+type Connection = RouterOutputs["llmApiKey"]["all"]["data"][number];
+type SlotValue = { llmApiKeyId: string; model: string } | null;
+
+/** Topics setup: one sheet for models, facets, and which traces to include. */
+export function useTopicModelSettings({
+ projectId,
+ canWrite,
+ facets,
+ facetsReady,
+}: {
+ projectId: string;
+ canWrite: boolean;
+ facets: TopicFacet[];
+ facetsReady: boolean;
+}) {
+ const settings = api.topics.modelSettings.useQuery({ projectId });
+ const readinessSettings = api.topics.tracingReadinessSettings.useQuery({
+ projectId,
+ });
+ const rules = api.topics.rules.useQuery({ projectId });
+ const connections = api.llmApiKey.all.useQuery({ projectId });
+ const configured = Boolean(
+ settings.data?.summary &&
+ settings.data.embedding &&
+ settings.data.clustering,
+ );
+
+ let notice = null;
+ if (settings.data?.blockMessage)
+ notice = (
+ <Alert variant="warning">
+ <Alert.Title>Automatic Topics processing is paused</Alert.Title>
+ <Alert.Description>
+ <p className="break-words">{settings.data.blockMessage}</p>
+ </Alert.Description>
+ </Alert>
+ );
+ else if (settings.data && !configured)
+ notice = (
+ <Alert variant="info">
+ <Alert.Title>Choose the models Topics runs on</Alert.Title>
+ <Alert.Description>
+ <p>
+ Topics uses your LLM connections for three jobs: facet summaries,
+ embeddings, and topic clustering. Open Configure Topics to set all
+ three.
+ </p>
+ </Alert.Description>
+ </Alert>
+ );
+
+ const ready = Boolean(
+ settings.data &&
+ readinessSettings.data &&
+ connections.data &&
+ rules.isSuccess &&
+ facetsReady,
+ );
+ const rule = rules.data?.length === 1 ? rules.data[0] : null;
+ const action = ready ? (
+ <ConfigureTopicsSettings
+ key={`${settings.dataUpdatedAt}-${rules.dataUpdatedAt}-${readinessSettings.dataUpdatedAt}`}
+ projectId={projectId}
+ canWrite={canWrite}
+ stored={settings.data!}
+ traceIdleMs={readinessSettings.data!.traceIdleMs}
+ connections={connections.data!.data}
+ facets={facets}
+ rule={rule}
+ />
+ ) : (
+ <ConfigureTopicsDialog
+ projectId={projectId}
+ providerGroups={[]}
+ draft={{
+ summary: null,
+ embedding: null,
+ embeddingDimensions: "1024",
+ clustering: null,
+ facets: [],
+ filters: [],
+ sampling: 1,
+ idleSeconds: "600",
+ embeddingLocked: false,
+ }}
+ defaultOpen={false}
+ onConfigureProviders={() => undefined}
+ onSave={() => undefined}
+ canWrite={false}
+ triggerVariant="secondary"
+ notice="none"
+ />
+ );
+
+ return { action, notice };
+}
+
+function ConfigureTopicsSettings({
+ projectId,
+ canWrite,
+ stored,
+ traceIdleMs,
+ connections,
+ facets,
+ rule,
+}: {
+ projectId: string;
+ canWrite: boolean;
+ stored: StoredSettings;
+ traceIdleMs: number | null;
+ connections: Connection[];
+ facets: TopicFacet[];
+ rule: TopicRule | null;
+}) {
+ const utils = api.useUtils();
+ const requests = useRef({ summary: 0, embedding: 0, clustering: 0 });
+ const [checks, setChecks] = useState<
+ Record<TopicsModelSlotName, TopicSlotCheck>
+ >(() => ({
+ summary: stored.summary ? { status: "ok" } : { status: "idle" },
+ embedding: stored.embedding ? { status: "ok" } : { status: "idle" },
+ clustering: stored.clustering ? { status: "ok" } : { status: "idle" },
+ }));
+ const [saveError, setSaveError] = useState<string | null>(null);
+ const test = api.topics.testModelSettings.useMutation();
+ const save = api.topics.saveSetup.useMutation({
+ onSuccess: () => {
+ utils.topics.modelSettings.invalidate({ projectId });
+ utils.topics.facets.invalidate({ projectId });
+ utils.topics.rules.invalidate({ projectId });
+ utils.topics.tracingReadinessSettings.invalidate({ projectId });
+ },
+ });
+ const providerGroups = connections.map<[string, string[]]>((connection) => [
+ connection.provider,
+ modelsFor(connection, "chat"),
+ ]);
+ const embeddingGroups = connections.map<[string, string[]]>((connection) => [
+ connection.provider,
+ modelsFor(
+ connection,
+ "embedding",
+ stored.embedding?.llmApiKeyId === connection.id
+ ? stored.embedding.model
+ : null,
+ ),
+ ]);
+ const draft = {
+ summary: judgeModel(stored.summary, connections),
+ embedding: judgeModel(stored.embedding, connections),
+ embeddingDimensions: String(stored.embeddingDimensions),
+ clustering: judgeModel(stored.clustering, connections),
+ facets: facetDrafts(facets, rule ? new Set(rule.facetIds) : null),
+ filters: [],
+ sampling: rule?.sampling ?? 1,
+ idleSeconds: String((traceIdleMs ?? 600_000) / 1000),
+ embeddingLocked: false,
+ };
+ const paused = stored.blockMessage
+ ? {
+ shortLabel: stored.blockReason
+ ? getEvaluatorBlockMetadata(stored.blockReason).shortLabel
+ : "Paused",
+ pausedAgo: stored.blockedAt
+ ? formatDistanceToNow(new Date(stored.blockedAt), { addSuffix: true })
+ : "just now",
+ message: stored.blockMessage,
+ }
+ : null;
+
+ const commitSlot = (
+ slot: TopicsModelSlotName,
+ model: JudgeModel,
+ dimensions: string,
+ ) => {
+ const parsedDimensions =
+ topicsModelSettingsSchema.shape.embeddingDimensions.safeParse(
+ Number(dimensions),
+ );
+ if (slot === "embedding" && !parsedDimensions.success) {
+ setChecks((current) => ({
+ ...current,
+ embedding: {
+ status: "error",
+ message: parsedDimensions.error.issues[0]?.message,
+ },
+ }));
+ return;
+ }
+ const preferred = {
+ summary: stored.summary?.llmApiKeyId,
+ embedding: stored.embedding?.llmApiKeyId,
+ clustering: stored.clustering?.llmApiKeyId,
+ }[slot];
+ const llmApiKeyId = connectionId(
+ model,
+ connections,
+ preferred,
+ slot === "embedding" ? "embedding" : "chat",
+ slot === "embedding" ? stored.embedding?.model : null,
+ );
+ if (!llmApiKeyId) {
+ setChecks((current) => ({
+ ...current,
+ [slot]: {
+ status: "error",
+ message: "Choose a model from a connection in this project.",
+ },
+ }));
+ return;
+ }
+ const request = ++requests.current[slot];
+ setChecks((current) => ({ ...current, [slot]: { status: "testing" } }));
+ const value = { llmApiKeyId, model: model.model };
+ test
+ .mutateAsync({
+ projectId,
+ summary: slot === "summary" ? value : null,
+ embedding: slot === "embedding" ? value : null,
+ embeddingDimensions: parsedDimensions.success
+ ? parsedDimensions.data
+ : stored.embeddingDimensions,
+ clustering: slot === "clustering" ? value : null,
+ enabled: false,
+ })
+ .then((errors) => {
+ if (requests.current[slot] !== request) return;
+ const message = errors[slot];
+ setChecks((current) => ({
+ ...current,
+ [slot]: message ? { status: "error", message } : { status: "ok" },
+ }));
+ })
+ .catch((error: unknown) => {
+ if (requests.current[slot] !== request) return;
+ setChecks((current) => ({
+ ...current,
+ [slot]: {
+ status: "error",
+ message:
+ error instanceof Error ? error.message : "The test call failed.",
+ },
+ }));
+ });
+ };
+
+ const onSave = async (value: ConfigureTopicsSaveDraft) => {
+ setSaveError(null);
+ const parsedDimensions =
+ topicsModelSettingsSchema.shape.embeddingDimensions.safeParse(
+ Number(value.embeddingDimensions),
+ );
+ if (!parsedDimensions.success)
+ throw new Error(
+ parsedDimensions.error.issues[0]?.message ??
+ "Check the embedding dimensions.",
+ );
+ const summary = slotValue(
+ value.summary,
+ connections,
+ stored.summary?.llmApiKeyId,
+ "chat",
+ );
+ const embedding = slotValue(
+ value.embedding,
+ connections,
+ stored.embedding?.llmApiKeyId,
+ "embedding",
+ stored.embedding?.model,
+ );
+ const clustering = slotValue(
+ value.clustering,
+ connections,
+ stored.clustering?.llmApiKeyId,
+ "chat",
+ );
+ if (!summary || !embedding || !clustering)
+ throw new Error("Choose a model from a connection in this project.");
+ try {
+ await save.mutateAsync({
+ projectId,
+ ...topicsSetupPayload(value, {
+ summary,
+ embedding,
+ embeddingDimensions: parsedDimensions.data,
+ clustering,
+ }),
+ ruleId: rule?.id,
+ });
+ } catch (error) {
+ const message =
+ error instanceof Error ? error.message : "Could not save Topics.";
+ setSaveError(message);
+ throw error;
+ }
+ };
+
+ return (
+ <ConfigureTopicsDialog
+ projectId={projectId}
+ providerGroups={providerGroups}
+ embeddingProviderGroups={embeddingGroups}
+ draft={draft}
+ defaultOpen={false}
+ canWrite={canWrite}
+ saving={save.isPending}
+ saveError={saveError}
+ slotChecks={checks}
+ onSlotCommit={commitSlot}
+ onConfigureProviders={() => {
+ window.open(
+ `${env.NEXT_PUBLIC_BASE_PATH ?? ""}/project/${projectId}/settings/llm-connections`,
+ "_blank",
+ "noopener,noreferrer",
+ );
+ }}
+ onSave={onSave}
+ triggerVariant={configured(stored) ? "secondary" : "primary"}
+ {...(paused
+ ? { notice: "paused" as const, ...paused }
+ : { notice: "none" as const })}
+ />
+ );
+}
+
+function configured(stored: StoredSettings) {
+ return Boolean(stored.summary && stored.embedding && stored.clustering);
+}
+
+const OPENAI_EMBEDDING_MODELS = [
+ "text-embedding-3-small",
+ "text-embedding-3-large",
+ "text-embedding-ada-002",
+];
+
+const GOOGLE_EMBEDDING_MODELS = [
+ "textembedding-gecko",
+ "textembedding-gecko-multilingual",
+];
+
+function embeddingModelsFor(adapter: string) {
+ if (adapter === LLMAdapter.OpenAI || adapter === LLMAdapter.Azure)
+ return OPENAI_EMBEDDING_MODELS;
+ if (adapter === LLMAdapter.GoogleAIStudio || adapter === LLMAdapter.VertexAI)
+ return GOOGLE_EMBEDDING_MODELS;
+ return [];
+}
+
+function modelsFor(
+ connection: Connection,
+ kind: "chat" | "embedding",
+ savedModel?: string | null,
+) {
+ const chat = connection.withDefaultModels
+ ? (supportedModels[connection.adapter as keyof typeof supportedModels] ??
+ [])
+ : [];
+ const embedding =
+ kind === "embedding" && connection.withDefaultModels
+ ? embeddingModelsFor(connection.adapter)
+ : [];
+ return Array.from(
+ new Set(
+ [savedModel, ...embedding, ...connection.customModels, ...chat].filter(
+ (model): model is string => Boolean(model),
+ ),
+ ),
+ );
+}
+
+function judgeModel(
+ slot: SlotValue,
+ connections: Connection[],
+): JudgeModel | null {
+ if (!slot) return null;
+ const connection = connections.find((item) => item.id === slot.llmApiKeyId);
+ return {
+ provider: connection?.provider ?? "Missing connection",
+ model: slot.model,
+ };
+}
+
+function connectionId(
+ model: JudgeModel,
+ connections: Connection[],
+ preferredId: string | undefined,
+ kind: "chat" | "embedding",
+ savedModel?: string | null,
+) {
+ const matches = connections.filter(
+ (connection) =>
+ connection.provider === model.provider &&
+ modelsFor(
+ connection,
+ kind,
+ connection.id === preferredId ? savedModel : null,
+ ).includes(model.model),
+ );
+ return (
+ matches.find((connection) => connection.id === preferredId)?.id ??
+ matches[0]?.id ??
+ null
+ );
+}
+
+function slotValue(
+ model: JudgeModel | null,
+ connections: Connection[],
+ preferredId: string | undefined,
+ kind: "chat" | "embedding",
+ savedModel?: string | null,
+) {
+ if (!model) return null;
+ const llmApiKeyId = connectionId(
+ model,
+ connections,
+ preferredId,
+ kind,
+ savedModel,
+ );
+ return llmApiKeyId ? { llmApiKeyId, model: model.model } : null;
+}
+
+function facetDrafts(
+ facets: TopicFacet[],
+ enabledIds: ReadonlySet<string> | null,
+) {
+ const byName = new Map(facets.map((facet) => [facet.name, facet]));
+ const builtIn = DEFAULT_TOPIC_FACETS.map((preset) => {
+ const existing = byName.get(preset.name);
+ return {
+ id: existing?.id ?? preset.name,
+ name: preset.name,
+ question: existing?.versions[0]?.prompt ?? preset.prompt,
+ builtIn: true,
+ enabled: enabledIds
+ ? Boolean(existing && enabledIds.has(existing.id))
+ : true,
+ };
+ });
+ const custom = facets.flatMap((facet) => {
+ if (facet.isBuiltIn) return [];
+ const question = facet.versions[0]?.prompt;
+ if (!question) return [];
+ return [
+ {
+ id: facet.id,
+ name: facet.name,
+ question,
+ builtIn: false,
+ enabled: enabledIds?.has(facet.id) ?? false,
+ },
+ ];
+ });
+ return [...builtIn, ...custom];
+}
diff --git a/web/src/features/topics/TopicPipelineForm.tsx b/web/src/features/topics/TopicPipelineForm.tsx
index bd8531aad0..d06a42074a 100644
--- a/web/src/features/topics/TopicPipelineForm.tsx
+++ b/web/src/features/topics/TopicPipelineForm.tsx
@@ -1,23 +1,14 @@
import { Alert } from "@/src/components/design-system/Alert/Alert";
-import { type ReactNode, useRef, useState } from "react";
+import { useRef, useState } from "react";
import { skipToken } from "@tanstack/react-query";
import * as DialogPrimitive from "@radix-ui/react-dialog";
import { Button } from "@/src/components/design-system/Button/Button";
import { Dialog } from "@/src/components/design-system/Dialog/Dialog";
-import { Input } from "@/src/components/design-system/Input/Input";
import { Input as NumericInput } from "@/src/components/ui/input";
import { Checkbox } from "@/src/components/design-system/Checkbox/Checkbox";
import { SelectInput } from "@/src/components/design-system/SelectInput/SelectInput";
-import {
- Select,
- SelectContent,
- SelectItem,
- SelectTrigger,
- SelectValue,
-} from "@/src/components/ui/select";
import { api } from "@/src/utils/api";
import {
- topicEmbeddingConfigSchema,
topicMinimumTraceCountSchema,
type TopicFacet,
type TopicOperation,
@@ -30,43 +21,26 @@ export function useTopicPipelineForm({
facets,
canWrite,
onTriggered,
- facetEditor,
timeRange,
}: {
projectId: string;
facets: TopicFacet[];
canWrite: boolean;
onTriggered: (id: string) => void;
- facetEditor: ReactNode;
timeRange: TopicTimeRange | null;
}) {
- const [configurationOpen, setConfigurationOpen] = useState(false);
- function openConfiguration() {
- setConfigurationOpen(true);
- }
- function closeConfiguration() {
- setConfigurationOpen(false);
- }
+ const [runOpen, setRunOpen] = useState(false);
const [operation, setOperation] = useState<TopicOperation>("process");
const [reuseExistingSummaries, setReuseExistingSummaries] = useState(false);
- const rules = api.topics.rules.useQuery(
- { projectId },
- { enabled: facets.length > 0 },
- );
- const [ruleId, setRuleId] = useState<string | null>(null);
- const selectedRule = rules.data?.find((rule) => rule.id === ruleId);
- const saveRuleLabel = selectedRule ? "Update rule" : "Save rule";
- const [ruleName, setRuleName] = useState("");
const {
selection,
- criteria,
controls: traceControls,
reset: resetTraceSelection,
} = useTopicTraceSelector({
projectId,
enabled: facets.length > 0 && operation === "process",
- filterOptionsEnabled: configurationOpen,
- onOpenTrace: closeConfiguration,
+ filterOptionsEnabled: runOpen,
+ onOpenTrace: () => setRunOpen(false),
});
const [selectedFacetIds, setSelectedFacetIds] = useState<string[]>(() =>
facets
@@ -88,26 +62,10 @@ export function useTopicPipelineForm({
const minimumTraceCountResult = topicMinimumTraceCountSchema.safeParse(
Number(minimumTraceCountValue),
);
- const [dimensions, setDimensions] = useState(() =>
- String(topicEmbeddingConfigSchema.parse({}).embeddingDimensions),
- );
- const embeddingConfig = topicEmbeddingConfigSchema.safeParse({
- embeddingDimensions: Number(dimensions),
- });
const [error, setError] = useState<string | null>(null);
const request = useRef<{ key: string; id: string } | null>(null);
const trigger = api.topics.trigger.useMutation();
const utils = api.useUtils();
- const saveRule = api.topics.saveRule.useMutation({
- onSuccess: (rule) => {
- utils.topics.rules.setData({ projectId }, (current) => [
- rule,
- ...(current ?? []).filter((item) => item.id !== rule.id),
- ]);
- setRuleId(rule.id);
- setRuleName(rule.name);
- },
- });
const facetChoices = facets.flatMap((facet) => {
const version =
facet.versions.find(
@@ -123,28 +81,18 @@ export function useTopicPipelineForm({
facetId: choice.facet.id,
version: choice.version.version,
}));
- const activeFacetIds = selectedFacets.map(({ facetId }) => facetId);
- const matchesRule =
- selectedRule &&
- criteria &&
- JSON.stringify(criteria) === JSON.stringify(selectedRule.filter) &&
- activeFacetIds.length === selectedRule.facetIds.length &&
- activeFacetIds.every((id) => selectedRule.facetIds.includes(id));
const summaryCounts = api.topics.summaryCounts.useQuery(
timeRange
? {
projectId,
facets: selectedFacets,
timeRange,
- embeddingConfig:
- embeddingConfig.data ?? topicEmbeddingConfigSchema.parse({}),
}
: skipToken,
{
enabled:
operation === "update" &&
timeRange !== null &&
- embeddingConfig.success &&
selectedFacets.length > 0,
},
);
@@ -167,14 +115,10 @@ export function useTopicPipelineForm({
async function submit() {
setError(null);
try {
- if (selectedFacets.length === 0)
- throw new Error("Select at least one facet.");
+ if (!selectedFacets.length) throw new Error("Select at least one facet.");
const base = {
projectId,
facets: selectedFacets,
- embeddingConfig: topicEmbeddingConfigSchema.parse({
- embeddingDimensions: Number(dimensions),
- }),
};
const values = (() => {
if (operation === "update") {
@@ -201,7 +145,6 @@ export function useTopicPipelineForm({
operation,
reuseExistingSummaries,
...traceInput,
- ...(selectedRule && matchesRule ? { ruleId: selectedRule.id } : {}),
};
})();
const key = JSON.stringify(values);
@@ -212,7 +155,7 @@ export function useTopicPipelineForm({
requestId: request.current.id,
});
request.current = null;
- setConfigurationOpen(false);
+ setRunOpen(false);
onTriggered(result.id);
} catch (cause) {
setError(
@@ -236,7 +179,7 @@ export function useTopicPipelineForm({
value={operation}
onValueChange={(value) => {
setOperation(value as TopicOperation);
- resetTraceSelection(selectedRule?.filter);
+ resetTraceSelection();
setError(null);
if (value === "update")
utils.topics.summaryCounts.invalidate({ projectId });
@@ -244,54 +187,6 @@ export function useTopicPipelineForm({
/>
</div>
</div>
- {operation === "process" && (
- <label className="flex flex-col gap-1 text-sm">
- Saved configuration
- <Select
- value={selectedRule?.id ?? "adhoc"}
- disabled={rules.isLoading || saveRule.isPending}
- onValueChange={(value) => {
- const rule = rules.data?.find((item) => item.id === value);
- setRuleId(rule?.id ?? null);
- setRuleName(rule?.name ?? "");
- setSelectedFacetIds(
- rule?.facetIds ??
- facets
- .filter((facet) => facet.versions.length > 0)
- .map((facet) => facet.id),
- );
- setFacetVersions(
- Object.fromEntries(
- facets.map((facet) => [
- facet.id,
- facet.versions[0]?.version ?? 0,
- ]),
- ),
- );
- resetTraceSelection(rule?.filter);
- saveRule.reset();
- }}
- >
- <SelectTrigger className="w-64" aria-label="Saved configuration">
- <SelectValue />
- </SelectTrigger>
- <SelectContent className="ph-no-capture">
- <SelectItem value="adhoc">Custom configuration</SelectItem>
- {rules.data?.map((rule) => (
- <SelectItem key={rule.id} value={rule.id}>
- {rule.name}
- </SelectItem>
- ))}
- </SelectContent>
- </Select>
- {rules.error && (
- <span role="alert" className="text-destructive text-sm">
- Could not load saved rules. Custom configuration is still
- available.
- </span>
- )}
- </label>
- )}
</>
);
let actionLabel = "Update topics";
@@ -299,21 +194,21 @@ export function useTopicPipelineForm({
actionLabel = selection?.count
? `Process ${selection.count.toLocaleString()} traces`
: "Process traces";
+ const canSubmit =
+ canWrite &&
+ !trigger.isPending &&
+ selectedFacets.length > 0 &&
+ (operation === "update"
+ ? Boolean(timeRange) &&
+ minimumTraceCountResult.success &&
+ !summaryCounts.isFetching &&
+ !summaryCounts.error &&
+ hasStoredSummaries
+ : Boolean(selection?.count));
const triggerAction = {
- label: trigger.isPending ? "Starting…" : actionLabel,
- disabled:
- !canWrite ||
- !embeddingConfig.success ||
- (operation === "update" &&
- (!timeRange || !minimumTraceCountResult.success)) ||
- trigger.isPending ||
- selectedFacets.length === 0 ||
- (operation === "update"
- ? summaryCounts.isFetching ||
- !!summaryCounts.error ||
- !hasStoredSummaries
- : !selection?.count),
- onSelect: submit,
+ label: operation === "process" ? "Process traces" : "Update topics",
+ disabled: !canWrite,
+ onSelect: () => setRunOpen(true),
};
const primaryAction = (
<Button
@@ -324,11 +219,12 @@ export function useTopicPipelineForm({
/>
);
const configuration = (
- <DialogPrimitive.Root
- open={configurationOpen}
- onOpenChange={setConfigurationOpen}
- >
- <Dialog title="Configure topics" size="xxl" closeOnInteractionOutside>
+ <DialogPrimitive.Root open={runOpen} onOpenChange={setRunOpen}>
+ <Dialog
+ title={operation === "process" ? "Process traces" : "Update topics"}
+ size="xxl"
+ closeOnInteractionOutside
+ >
<Dialog.Body>
<div className="ph-no-capture flex flex-col gap-6">
<p className="text-muted-foreground">
@@ -384,55 +280,6 @@ export function useTopicPipelineForm({
</div>
))}
</fieldset>
- {operation === "process" && criteria && (
- <div className="flex flex-col gap-2">
- <div className="flex flex-wrap items-end gap-2">
- <label className="flex w-64 max-w-full flex-col gap-1 text-sm">
- Rule name
- <Input
- aria-label="Saved configuration name"
- placeholder="Save these filters and facets"
- value={ruleName}
- onChange={(event) => setRuleName(event.target.value)}
- />
- </label>
- <Button
- text={saveRule.isPending ? "Saving…" : saveRuleLabel}
- type="button"
- variant="secondary"
- disabled={
- !canWrite ||
- !ruleName.trim() ||
- activeFacetIds.length === 0 ||
- saveRule.isPending
- }
- onClick={() =>
- saveRule.mutate({
- projectId,
- ...(selectedRule ? { id: selectedRule.id } : {}),
- name: ruleName,
- filter: criteria,
- facetIds: activeFacetIds,
- })
- }
- />
- </div>
- <p className="text-muted-foreground text-xs">
- Rules save filters and selected facets. Choose the time range,
- sampling and summary reuse for each run.
- {selectedRule && !matchesRule
- ? " Unsaved changes apply only to this run until you update the rule."
- : ""}
- </p>
- {saveRule.error && (
- <Alert variant="destructive" size="sm">
- <Alert.Description>
- <p className="break-words">{saveRule.error.message}</p>
- </Alert.Description>
- </Alert>
- )}
- </div>
- )}
<div className="flex flex-wrap items-end gap-5">
{operation === "update" && (
<label className="flex flex-col gap-1 text-sm">
@@ -451,19 +298,6 @@ export function useTopicPipelineForm({
/>
</label>
)}
- <label className="flex flex-col gap-1 text-sm">
- Embedding dimensions
- <SelectInput
- aria-label="Embedding dimensions"
- placeholder="Embedding dimensions"
- value={dimensions}
- onValueChange={setDimensions}
- options={[256, 512, 1024, 1536].map((value) => ({
- value: String(value),
- label: String(value),
- }))}
- />
- </label>
{operation === "update" && (
<label className="flex items-center gap-2 text-sm">
<Checkbox
@@ -497,8 +331,8 @@ export function useTopicPipelineForm({
Process {selection.count.toLocaleString()} traces across{" "}
{selectedFacets.length} facets.{" "}
{reuseExistingSummaries
- ? "Matching stored summaries and embeddings are reused. Missing summaries use OpenAI; embeddings use Cohere on Amazon Bedrock."
- : "Generate fresh summaries with OpenAI and embeddings with Cohere on Amazon Bedrock."}
+ ? "Matching stored summaries and embeddings are reused. Missing ones use the project's Topics models."
+ : "Generate fresh summaries and embeddings with the project's Topics models."}
</p>
) : null}
{operation === "process" && (
@@ -523,14 +357,25 @@ export function useTopicPipelineForm({
traces.
</p>
)}
- {facetEditor}
+ {error && (
+ <Alert variant="destructive" size="sm">
+ <Alert.Description>
+ <p className="break-words">{error}</p>
+ </Alert.Description>
+ </Alert>
+ )}
</div>
</Dialog.Body>
- <div className="flex shrink-0 justify-end p-4">
+ <div className="flex shrink-0 justify-end gap-2 p-4">
<Button
- text="Done"
+ text="Cancel"
variant="secondary"
- onClick={closeConfiguration}
+ onClick={() => setRunOpen(false)}
+ />
+ <Button
+ text={trigger.isPending ? "Starting…" : actionLabel}
+ disabled={!canSubmit}
+ onClick={submit}
/>
</div>
</Dialog>
@@ -539,8 +384,6 @@ export function useTopicPipelineForm({
return {
primaryAction: facets.length > 0 ? primaryAction : null,
triggerAction: facets.length > 0 ? triggerAction : null,
- openConfiguration,
- error: facets.length > 0 ? error : null,
configuration: facets.length > 0 ? configuration : null,
};
}
diff --git a/web/src/features/topics/TopicTraceSelector.clienttest.tsx b/web/src/features/topics/TopicTraceSelector.clienttest.tsx
index 45ffc2324d..7f3df9de9a 100644
--- a/web/src/features/topics/TopicTraceSelector.clienttest.tsx
+++ b/web/src/features/topics/TopicTraceSelector.clienttest.tsx
@@ -42,7 +42,7 @@ vi.mock("@/src/features/events/hooks/useEventsFilterOptions", () => ({
vi.mock("@/src/features/search-bar", () => ({
TableSearchBar: () => <div>Query editor</div>,
toObservedOptions: () => ({}),
- fieldRegistryFromColumns: () => ({ fields: [] }),
+ fieldRegistryFromColumns: () => ({ fields: [], columns: [] }),
}));
vi.mock(
"@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder",
diff --git a/web/src/features/topics/TopicTraceSelector.tsx b/web/src/features/topics/TopicTraceSelector.tsx
index de2b57ec27..f6dcb2bc5e 100644
--- a/web/src/features/topics/TopicTraceSelector.tsx
+++ b/web/src/features/topics/TopicTraceSelector.tsx
@@ -46,7 +46,16 @@ type TopicTraceSelection = { count: number } & (
>["selection"];
}
);
-const registry = { ...RULE_FIELD_REGISTRY, aiFilterPrompt: false };
+const registry = {
+ ...RULE_FIELD_REGISTRY,
+ aiFilterPrompt: false,
+ fields: RULE_FIELD_REGISTRY.fields.filter(
+ (field) => field.id !== "isRootObservation",
+ ),
+ columns: RULE_FIELD_REGISTRY.columns.filter(
+ (column) => column.id !== "isRootObservation",
+ ),
+};
export function useTopicTraceSelector({
projectId,
@@ -169,11 +178,12 @@ export function useTopicTraceSelector({
{ ...options.filterOptions, tags: options.filterOptions?.traceTags },
[...eventsEvalFilterColumns],
),
- ).map((column) =>
- column.id === "experimentId" && column.type === "stringOptions"
- ? { ...column, options: options.filterOptions?.experimentId ?? [] }
- : column,
- );
+ ).flatMap((column) => {
+ if (column.id === "isRootObservation") return [];
+ return column.id === "experimentId" && column.type === "stringOptions"
+ ? [{ ...column, options: options.filterOptions?.experimentId ?? [] }]
+ : [column];
+ });
const previewTraces = () => {
setExcluded([]);
setRequest({
diff --git a/web/src/features/topics/TopicsActionsMenu.tsx b/web/src/features/topics/TopicsActionsMenu.tsx
index c2cd379b7c..a55e66105c 100644
--- a/web/src/features/topics/TopicsActionsMenu.tsx
+++ b/web/src/features/topics/TopicsActionsMenu.tsx
@@ -1,13 +1,11 @@
import { type ReactNode } from "react";
-import { History, Play, RefreshCw, Settings2 } from "lucide-react";
+import { History, Play, RefreshCw } from "lucide-react";
import { HeaderActionMenuRow } from "@/src/components/HeaderActionMenuRow";
export function TopicsActionsMenu({
children,
- hasFacets,
triggerAction,
closeMenu,
- onOpenConfiguration,
onOpenHistory,
onRefresh,
}: TopicsActionsMenuProps) {
@@ -15,10 +13,6 @@ export function TopicsActionsMenu({
closeMenu();
triggerAction?.onSelect();
}
- function configureTopics() {
- closeMenu({ handoffFocus: true });
- onOpenConfiguration();
- }
function openHistory() {
closeMenu({ handoffFocus: true });
onOpenHistory();
@@ -38,13 +32,6 @@ export function TopicsActionsMenu({
onClick={processTraces}
/>
)}
- {hasFacets && (
- <HeaderActionMenuRow
- label="Configure topics"
- icon={<Settings2 className="icon-base text-icon-foreground" />}
- onClick={configureTopics}
- />
- )}
<HeaderActionMenuRow
label="History"
icon={<History className="icon-base text-icon-foreground" />}
@@ -61,14 +48,12 @@ export function TopicsActionsMenu({
type TopicsActionsMenuProps = {
children?: ReactNode;
- hasFacets: boolean;
triggerAction: {
label: string;
disabled: boolean;
onSelect: () => void;
} | null;
closeMenu: (options?: { handoffFocus?: boolean }) => void;
- onOpenConfiguration: () => void;
onOpenHistory: () => void;
onRefresh: () => void;
};
diff --git a/web/src/features/topics/TopicsPage.clienttest.tsx b/web/src/features/topics/TopicsPage.clienttest.tsx
index f6716d8f68..ecf657a95b 100644
--- a/web/src/features/topics/TopicsPage.clienttest.tsx
+++ b/web/src/features/topics/TopicsPage.clienttest.tsx
@@ -64,11 +64,14 @@ vi.mock("./TopicPipelineForm", () => ({
state.pipeline(input);
return {
primaryAction: null,
- openConfiguration: vi.fn(),
+ triggerAction: null,
configuration: null,
};
},
}));
+vi.mock("./TopicModelSettings", () => ({
+ useTopicModelSettings: () => ({ action: null, notice: null }),
+}));
vi.mock("./CurrentTopics", () => ({
useCurrentTopics: (input: {
running: boolean;
diff --git a/web/src/features/topics/TopicsPage.tsx b/web/src/features/topics/TopicsPage.tsx
index 9741121d66..36573edcfc 100644
--- a/web/src/features/topics/TopicsPage.tsx
+++ b/web/src/features/topics/TopicsPage.tsx
@@ -25,19 +25,10 @@ import {
SheetTitle,
SheetDescription,
} from "@/src/components/ui/sheet";
-import { Input } from "@/src/components/design-system/Input/Input";
import { DropdownMenu } from "@/src/components/design-system/DropdownMenu/DropdownMenu";
import { IconButton } from "@/src/components/design-system/IconButton/IconButton";
-import { Textarea } from "@/src/components/ui/textarea";
import { PopoverController } from "@/src/components/ui/popover";
import { Badge } from "@/src/components/design-system/Badge/Badge";
-import {
- Select,
- SelectContent,
- SelectItem,
- SelectTrigger,
- SelectValue,
-} from "@/src/components/ui/select";
import { api, type RouterOutputs } from "@/src/utils/api";
import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
import useIsFeatureEnabled from "@/src/features/feature-flags/hooks/useIsFeatureEnabled";
@@ -48,6 +39,7 @@ import {
type TopicOperation,
} from "@langfuse/shared/topics";
import { useTopicPipelineForm } from "./TopicPipelineForm";
+import { useTopicModelSettings } from "./TopicModelSettings";
import { CurrentTopics, useCurrentTopics } from "./CurrentTopics";
import { TopicsFilters } from "./TopicsFilters";
import { TopicsActionsMenu } from "./TopicsActionsMenu";
@@ -260,40 +252,27 @@ function TopicsWorkspaceView({
function openHistory() {
setHistoryOpen(true);
}
- const {
- primaryAction,
- triggerAction,
- openConfiguration,
- configuration,
- error,
- } = useTopicPipelineForm({
+ const { primaryAction, triggerAction, configuration } = useTopicPipelineForm({
projectId,
facets: facets.data ?? [],
canWrite,
onTriggered: openExecution,
timeRange: validTimeRange ? currentTimeRange : null,
- facetEditor:
- canWrite && (facets.data?.length ?? 0) > 0 ? (
- <FacetEditor projectId={projectId} facets={facets.data ?? []} />
- ) : null,
+ });
+ const modelSettings = useTopicModelSettings({
+ projectId,
+ canWrite,
+ facets: facets.data ?? [],
+ facetsReady: facets.isSuccess,
});
const actions = (
<div className="ph-no-capture flex items-center justify-end gap-1">
+ {modelSettings.action}
{primaryAction}
<DropdownMenu
ariaLabel="Topics actions"
placement="bottom-end"
items={[
- ...((facets.data?.length ?? 0) > 0
- ? [
- {
- id: "configure",
- type: "item" as const,
- title: "Configure topics",
- onClick: openConfiguration,
- },
- ]
- : []),
{
id: "history",
type: "item",
@@ -330,10 +309,8 @@ function TopicsWorkspaceView({
actionButtonsRight: actions,
actionButtonsMenu: ({ closeMenu }) => (
<TopicsActionsMenu
- hasFacets={(facets.data?.length ?? 0) > 0}
triggerAction={triggerAction}
closeMenu={closeMenu}
- onOpenConfiguration={openConfiguration}
onOpenHistory={openHistory}
onRefresh={refreshResults}
/>
@@ -362,7 +339,7 @@ function TopicsWorkspaceView({
{!validTimeRange && (
<ErrorMessage message="Select a time range of at most 90 days." />
)}
- {error && <ErrorMessage message={error} />}
+ {modelSettings.notice}
{configuration}
<Sheet
open={historyOpen || executionId !== null}
@@ -482,99 +459,6 @@ function TopicsWorkspaceView({
);
}
-function FacetEditor({
- projectId,
- facets,
-}: {
- projectId: string;
- facets: TopicFacet[];
-}) {
- const utils = api.useUtils();
- const [facetId, setFacetId] = useState("new");
- const [name, setName] = useState("");
- const [prompt, setPrompt] = useState("");
- const isBuiltIn = facets.some(
- (facet) => facet.id === facetId && facet.isBuiltIn,
- );
- const save = api.topics.saveFacet.useMutation({
- onSuccess: () => utils.topics.facets.invalidate({ projectId }),
- });
- return (
- <details className="border-t pt-6">
- <summary className="cursor-pointer font-bold">
- Add a facet or revise a question
- </summary>
- <div className="mt-4 flex flex-col gap-3">
- <Select
- value={facetId}
- onValueChange={(value) => {
- setFacetId(value);
- const facet = facets.find((f) => f.id === value);
- setName(facet?.name ?? "");
- setPrompt(facet?.versions[0]?.prompt ?? "");
- }}
- >
- <SelectTrigger aria-label="Facet to edit">
- <SelectValue />
- </SelectTrigger>
- <SelectContent className="ph-no-capture">
- <SelectItem value="new">New facet</SelectItem>
- {facets.map((facet) => (
- <SelectItem
- key={facet.id}
- value={facet.id}
- disabled={facet.isBuiltIn}
- >
- {facet.name} · {facet.isBuiltIn ? "built-in" : "new version"}
- </SelectItem>
- ))}
- </SelectContent>
- </Select>
- <Input
- aria-label="Facet name"
- placeholder="Facet name"
- value={name}
- disabled={facetId !== "new"}
- onChange={(event) => setName(event.target.value)}
- />
- <Textarea
- aria-label="Facet question"
- placeholder="What should each trace summary describe?"
- value={prompt}
- disabled={isBuiltIn}
- onChange={(event) => setPrompt(event.target.value)}
- />
- <p className="text-muted-foreground text-xs">
- Built-in questions are read-only. Revising a custom question creates
- an immutable version; existing summaries and maps retain their
- original question.
- </p>
- <div className="self-start">
- <Button
- text="Save facet"
- disabled={
- isBuiltIn ||
- save.isPending ||
- !name.trim() ||
- prompt.trim().length < 10
- }
- onClick={() =>
- save.mutate({
- projectId,
- ...(facetId === "new" ? {} : { facetId }),
- name,
- prompt,
- })
- }
- />
- </div>
- {save.error && <ErrorMessage message={save.error.message} />}
- {save.isSuccess && <p className="text-sm">Facet saved.</p>}
- </div>
- </details>
- );
-}
-
function ExecutionPanel({
projectId,
executionId,
diff --git a/web/src/features/topics/server/topicsRouter.ts b/web/src/features/topics/server/topicsRouter.ts
index bf3c505340..bb632c1844 100644
--- a/web/src/features/topics/server/topicsRouter.ts
+++ b/web/src/features/topics/server/topicsRouter.ts
@@ -1,7 +1,10 @@
import { z } from "zod";
import { createHash } from "node:crypto";
import { InvalidRequestError, LangfuseNotFoundError } from "@langfuse/shared";
-import type { Transcript } from "@langfuse/shared/src/server";
+import {
+ getTracingReadinessSettings,
+ type Transcript,
+} from "@langfuse/shared/src/server";
import {
topicEmbeddingConfigSchema,
topicExecutionInputSchema,
@@ -10,6 +13,8 @@ import {
topicTraceIdSchema,
topicRuleConfigSchema,
topicTimeRangeSchema,
+ topicsModelSettingsSchema,
+ topicsSetupSchema,
type TopicTimeRange,
type TopicExecutionSummary,
type TopicFacetRef,
@@ -36,15 +41,21 @@ import {
loadTopicTranscript,
isTopicsEnabled,
isTopicsProjectEnabled,
- getTopicsModelConfig,
+ getTopicsModels,
+ readTopicsModelSettings,
+ checkTopicsModelSettings,
+ saveTopicsSetup,
enqueueTopicExecution,
getTopicExecutionQueueState,
} from "@langfuse/shared/topics/server";
import {
createTRPCRouter,
protectedProjectProcedureWithoutTracing,
+ type ProjectAuthedContext,
} from "@/src/server/api/trpc";
import { throwIfNoProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { auditLog } from "@/src/features/audit-logs/auditLog";
+import { JOB_CONFIGURATION_AUDIT_LOG_RESOURCE_TYPE } from "@/src/features/evals/server/audit-log-resource-types";
import { getContextualFeatureFlags } from "@/src/features/feature-flags/utils";
import {
previewTopicTraces,
@@ -90,6 +101,40 @@ const topicsWriteProcedure = topicsProcedure.use(({ ctx, input, next }) => {
return next();
});
+function readTopicConfig(ctx: ProjectAuthedContext, projectId: string) {
+ return ctx.prisma.topicConfig.findUnique({ where: { projectId } });
+}
+
+/** Saves the project's Topics config and records the change in the audit log. */
+async function auditedTopicConfigSave<T>(
+ ctx: ProjectAuthedContext,
+ projectId: string,
+ save: () => Promise<T>,
+): Promise<T> {
+ const before = await readTopicConfig(ctx, projectId);
+ const result = await save();
+ const after = await readTopicConfig(ctx, projectId);
+ if (after)
+ await auditLog({
+ session: ctx.session,
+ resourceType: "topicConfig",
+ resourceId: after.id,
+ action: before ? "update" : "create",
+ before,
+ after,
+ });
+ return result;
+}
+
+async function requireTopicsModels(projectId: string) {
+ const models = await getTopicsModels(projectId);
+ if (!models)
+ throw new InvalidRequestError(
+ "Choose summary, embedding, and clustering models in the Topics model settings first.",
+ );
+ return models;
+}
+
async function requireFacetVersions(
projectId: string,
facets: TopicFacetRef[],
@@ -308,27 +353,61 @@ export const topicsRouter = createTRPCRouter({
facetIds: z.array(topicIdSchema).min(1),
}),
)
- .mutation(({ input }) => saveTopicRule(input)),
+ .mutation(async ({ input, ctx }) => {
+ const rule = await saveTopicRule({
+ ...input,
+ userId: ctx.session.user.id,
+ });
+ await auditLog({
+ session: ctx.session,
+ resourceType: JOB_CONFIGURATION_AUDIT_LOG_RESOURCE_TYPE,
+ resourceId: rule.id,
+ action: input.id ? "update" : "create",
+ after: rule,
+ });
+ return rule;
+ }),
+ modelSettings: topicsProcedure.query(({ input }) =>
+ readTopicsModelSettings(input.projectId),
+ ),
+ tracingReadinessSettings: topicsProcedure.query(({ input }) =>
+ getTracingReadinessSettings(input.projectId),
+ ),
+ testModelSettings: topicsWriteProcedure
+ .input(topicsModelSettingsSchema.extend({ projectId: topicIdSchema }))
+ .mutation(({ input: { projectId, ...settings } }) =>
+ checkTopicsModelSettings(projectId, settings),
+ ),
+ saveSetup: topicsWriteProcedure
+ .input(topicsSetupSchema.extend({ projectId: topicIdSchema }))
+ .mutation(async ({ input: { projectId, ...setup }, ctx }) => {
+ const rule = await auditedTopicConfigSave(ctx, projectId, () =>
+ saveTopicsSetup(projectId, setup, ctx.session.user.id),
+ );
+ await auditLog({
+ session: ctx.session,
+ resourceType: JOB_CONFIGURATION_AUDIT_LOG_RESOURCE_TYPE,
+ resourceId: rule.id,
+ action: setup.ruleId ? "update" : "create",
+ after: rule,
+ });
+ return rule;
+ }),
summaryCounts: topicsProcedure
.input(
resultInput.extend({
facets: z.array(topicFacetRefSchema).min(1),
- embeddingConfig: topicEmbeddingConfigSchema,
}),
)
.query(async ({ input }) => {
await requireFacetVersions(input.projectId, input.facets);
- const { embeddingModel } = getTopicsModelConfig();
- if (!embeddingModel)
- throw new InvalidRequestError(
- "Set LANGFUSE_TOPICS_EMBEDDING_MODEL on web and worker to use Topics.",
- );
+ const { embedding } = await requireTopicsModels(input.projectId);
return getTopicSummaryCounts(
input.projectId,
input.facets,
topicEmbeddingConfigSchema.parse({
- ...input.embeddingConfig,
- embeddingModel,
+ embeddingModel: embedding.model,
+ embeddingDimensions: embedding.dimensions,
}),
input.timeRange,
);
@@ -360,11 +439,12 @@ export const topicsRouter = createTRPCRouter({
throw new InvalidRequestError(
"Topics processing is not enabled for this project.",
);
- const models = getTopicsModelConfig();
- if (!models.summaryModel || !models.embeddingModel)
- throw new InvalidRequestError(
- "Set LANGFUSE_TOPICS_SUMMARY_MODEL and LANGFUSE_TOPICS_EMBEDDING_MODEL on web and worker to run Topics.",
- );
+ const { summary, embedding } = await requireTopicsModels(input.projectId);
+ const models = {
+ summaryModel: summary.model,
+ embeddingModel: embedding.model,
+ embeddingDimensions: embedding.dimensions,
+ };
const requestHash = createHash("sha256")
.update(JSON.stringify({ input, models }))
.digest("hex");
@@ -410,8 +490,8 @@ export const topicsRouter = createTRPCRouter({
const configuredInput = topicExecutionInputSchema.parse({
...resolvedInput,
embeddingConfig: {
- ...resolvedInput.embeddingConfig,
embeddingModel: models.embeddingModel,
+ embeddingDimensions: models.embeddingDimensions,
},
...(resolvedInput.operation === "process"
? {
diff --git a/web/src/features/topics/topics-setup.clienttest.ts b/web/src/features/topics/topics-setup.clienttest.ts
new file mode 100644
index 0000000000..ba655e5df8
--- /dev/null
+++ b/web/src/features/topics/topics-setup.clienttest.ts
@@ -0,0 +1,73 @@
+import { DEFAULT_TOPIC_FACETS } from "@langfuse/shared/topics";
+import { describe, expect, it } from "vitest";
+import type { ConfigureTopicsSaveDraft } from "./ConfigureTopicsDialog";
+import { topicsSetupPayload } from "./topics-setup";
+
+const slots = {
+ summary: { llmApiKeyId: "key", model: "gpt" },
+ embedding: { llmApiKeyId: "key", model: "embed" },
+ embeddingDimensions: 1024,
+ clustering: { llmApiKeyId: "key", model: "cluster" },
+};
+
+const draft: ConfigureTopicsSaveDraft = {
+ summary: null,
+ embedding: null,
+ embeddingDimensions: "1024",
+ clustering: null,
+ idleSeconds: "120",
+ sampling: 0.25,
+ filters: [
+ { column: "name", type: "string", operator: "=", value: "billing" },
+ ],
+ facets: [
+ ...DEFAULT_TOPIC_FACETS.map((facet, index) => ({
+ id: facet.name,
+ name: facet.name,
+ question: facet.prompt,
+ builtIn: true,
+ enabled: index !== 1,
+ })),
+ {
+ id: "custom",
+ name: "Frustration",
+ question: "Is the user frustrated?",
+ builtIn: false,
+ enabled: true,
+ },
+ ],
+};
+
+describe("topicsSetupPayload", () => {
+ it("keeps the filter, sampling, and built-in enabled state", () => {
+ expect(topicsSetupPayload(draft, slots)).toMatchObject({
+ enabled: true,
+ embeddingDimensions: 1024,
+ sampling: 0.25,
+ traceIdleMs: 120_000,
+ filter: draft.filters,
+ facets: DEFAULT_TOPIC_FACETS.map((facet, index) => ({
+ name: facet.name,
+ enabled: index !== 1,
+ })),
+ });
+ });
+
+ it("rejects a setup with every built-in facet turned off", () => {
+ expect(() =>
+ topicsSetupPayload(
+ {
+ ...draft,
+ facets: draft.facets.map((facet) => ({ ...facet, enabled: false })),
+ },
+ slots,
+ ),
+ ).toThrow(/at least one facet/);
+ });
+
+ it("rejects an idle time below 60 seconds", () => {
+ expect(() =>
+ topicsSetupPayload({ ...draft, idleSeconds: "59" }, slots),
+ ).toThrow();
+ });
+});
diff --git a/web/src/features/topics/topics-setup.ts b/web/src/features/topics/topics-setup.ts
new file mode 100644
index 0000000000..a564743be7
--- /dev/null
+++ b/web/src/features/topics/topics-setup.ts
@@ -0,0 +1,31 @@
+import { topicsSetupSchema, type TopicsSetup } from "@langfuse/shared/topics";
+import type { ConfigureTopicsSaveDraft } from "./ConfigureTopicsDialog";
+
+type Slots = Pick<
+ TopicsSetup,
+ "summary" | "embedding" | "embeddingDimensions" | "clustering"
+>;
+
+export function topicsSetupPayload(
+ value: ConfigureTopicsSaveDraft,
+ slots: Slots,
+): TopicsSetup {
+ const parsed = topicsSetupSchema.safeParse({
+ ...slots,
+ enabled: true,
+ facets: value.facets
+ .filter((facet) => facet.builtIn)
+ .map((facet) => ({ name: facet.name, enabled: facet.enabled })),
+ customFacetIds: value.facets
+ .filter((facet) => !facet.builtIn && facet.enabled)
+ .map((facet) => facet.id),
+ filter: value.filters,
+ sampling: value.sampling,
+ traceIdleMs: Number(value.idleSeconds) * 1000,
+ });
+ if (!parsed.success)
+ throw new Error(
+ parsed.error.issues[0]?.message ?? "Could not save Topics.",
+ );
+ return parsed.data;
+}
diff --git a/web/src/pages/api/public/llm-connections/[id].ts b/web/src/pages/api/public/llm-connections/[id].ts
index d9ff1a5908..b0352aa22a 100644
--- a/web/src/pages/api/public/llm-connections/[id].ts
+++ b/web/src/pages/api/public/llm-connections/[id].ts
@@ -6,6 +6,7 @@ import {
DeleteLlmConnectionV1Response,
} from "@/src/features/public-api/types/llm-connections";
import { auditLog } from "@/src/features/audit-logs/server";
+import { pauseTopicsModelsUsingConnection } from "@langfuse/shared/topics/server";
import { EvaluatorBlockReason, LangfuseNotFoundError } from "@langfuse/shared";
import {
blockEvaluatorsUsingDefaultModel,
@@ -55,6 +56,15 @@ export default withMiddlewares({
? await blockEvaluatorsUsingDefaultModel({ tx, projectId })
: EMPTY_EVALUATOR_BLOCK;
+ await pauseTopicsModelsUsingConnection(
+ {
+ projectId,
+ llmApiKeyId: llmApiKey.id,
+ provider: llmApiKey.provider,
+ },
+ tx,
+ );
+
await tx.llmApiKeys.delete({
where: { id: llmApiKey.id, projectId },
});
diff --git a/worker/AGENTS.md b/worker/AGENTS.md
index 625810de06..fa762dc2d6 100644
--- a/worker/AGENTS.md
+++ b/worker/AGENTS.md
@@ -31,12 +31,12 @@
`Transcript | null`; serialize it for inference and skip inference on null.
Token counting and its WASM dependency stay in `src/features/topics/models.ts`.
Model calls use `generateTopicText` and `generateTopicEmbedding` from
- `@langfuse/shared/topics/server` to keep Bedrock transport on shared's AI SDK
- version. Internal PoC model selection uses required, default-free
- `LANGFUSE_TOPICS_SUMMARY_MODEL` and `LANGFUSE_TOPICS_EMBEDDING_MODEL` values
- shared by web and worker; topic naming still uses the fixed Terra profile.
- Local AWS auth uses shared `LANGFUSE_AI_FEATURES_AWS_PROFILE`; region/setup
- details live in the Topics README.
+ `@langfuse/shared/topics/server` with the project's resolved `TopicsModels`
+ (summary, embedding, clustering LLM connections from `topic_configs`).
+ Pass models explicitly; never put them on execution or batch state, which is
+ persisted. Trace-batch jobs load enabled settings for all allowlisted batch
+ projects in one query (`getEnabledTopicsModels`) and pause a project on
+ authentication errors (`pauseTopicsModels`).
- OTEL event processing:
`src/features/otel-ingestion/processOtelEvents.ts`; the OTEL queue calls this
after its legacy persistence path for event normalization, evaluation
diff --git a/worker/src/env.ts b/worker/src/env.ts
index 8d7aa3dc5a..a1e6dfe7ac 100644
--- a/worker/src/env.ts
+++ b/worker/src/env.ts
@@ -366,9 +366,6 @@ const EnvSchema = z.object({
LANGFUSE_IN_APP_AGENT_INTEGRITY_RUNNER_ENABLED: z
.enum(["true", "false"])
.optional(),
- // Internal Topics PoC model selection; not a supported self-hosting setting.
- LANGFUSE_TOPICS_SUMMARY_MODEL: z.string().trim().min(1).optional(),
- LANGFUSE_TOPICS_EMBEDDING_MODEL: z.string().trim().min(1).optional(),
LANGFUSE_IN_APP_AGENT_SANDBOX_PROVIDER: z
.enum(["dangerous-docker", "lambda-microvm"])
.optional(),
diff --git a/worker/src/features/topics/README.md b/worker/src/features/topics/README.md
index cb31080d15..32bbb19809 100644
--- a/worker/src/features/topics/README.md
+++ b/worker/src/features/topics/README.md
@@ -22,7 +22,9 @@ Topics reuses the evaluator and evaluation-rule tables:
original request, including its selection criteria. Retries never re-evaluate a rule.
Postgres uses normal Prisma migrations; `topic_clustering_runs` is the only new
-Topics table. Idle time is per project, not per rule: see
+Topics table. Configure Topics upserts the per-project idle time in
+`tracing_readiness_settings`; ingestion uses it to schedule trace readiness.
+Idle time is per project, not per rule: see
`tracing_readiness_settings` in `worker/src/features/traceBatching/README.md`.
## Setup
@@ -55,25 +57,31 @@ dev commands also build it):
pnpm --filter @langfuse/native run build
```
-The worker reads `LANGFUSE_AI_AWS_BEDROCK_REGION` for Topics model calls.
-Set `LANGFUSE_TOPICS_SUMMARY_MODEL` and `LANGFUSE_TOPICS_EMBEDDING_MODEL` on
-both web and worker to Bedrock model IDs. Neither has a default; Topics
-processing is unavailable until both are set. Changing either requires a new
-execution, and changing the embedding model requires rebuilding the map from
-the new vectors. These are internal Topics PoC settings, not part of the
-self-hosted configuration surface. Topic naming continues to use
-`us.openai.gpt-5.6-terra`.
-
-Bedrock uses the default AWS credential chain; the worker role needs
-`bedrock:InvokeModel` access to the selected summary and embedding models,
-`us.openai.gpt-5.6-terra` for topic naming, and any routed foundation models.
-Usage is retained for custom models, but cost estimates are only available for
-the built-in summary, naming and embedding model IDs. Locally, set
-`LANGFUSE_AI_FEATURES_AWS_PROFILE=playground` to use the SSO profile without changing
-credentials for local object storage. `AWS_PROFILE` takes precedence when set.
-Assistant and Ask AI use the same profile; `LANGFUSE_IN_APP_AGENT_AWS_PROFILE`
-remains a fallback for existing local setups.
-Restart web and worker after changing these values.
+Topics runs on the project's own LLM connections (the same connections evals
+and the playground use). Each project stores one `topic_configs` row with
+three slots, each a connection plus a model ID: facet summaries, embeddings (plus
+dimensions), and topic clustering (the model that names and describes topics). Set them on the Topics page under **Models**.
+Processing is unavailable until all three are set. Any connection whose save-time
+test call succeeds can serve summaries and clustering, including Anthropic.
+Embeddings need OpenAI (including OpenAI-compatible base URLs), Azure OpenAI,
+Amazon Bedrock, Google AI Studio or Vertex AI; Anthropic has no embeddings API,
+so its embedding test call fails. Summary and clustering calls go through `generateLLMText`;
+embeddings go through `embedLLMText`. Both retry 429 and 5xx responses twice
+with backoff. On Cloud, Bedrock and Vertex connections need explicit
+credentials; the default credential chain is self-hosted only.
+
+Saving the settings makes one real call per configured slot (structured output
+for summaries and clustering, one embedding whose length must equal the
+configured dimensions) and rejects the save if any call fails. Saves record the
+user in `created_by` / `updated_by` and write `topicConfig` (and, for the rule,
+`job`) audit log entries.
+The embedding model and dimensions cannot change once the project has embedded
+summaries, so existing vectors stay comparable. Summary and clustering model changes apply
+to new work only; an execution frozen on another summary or embedding model
+fails and must be started again. Deleting a connection that a slot uses turns
+automatic processing off and records why. Usage is retained for any model, but
+cost estimates are only available for the model IDs priced in
+[models.ts](models.ts).
Numerical fitting uses the worker's existing Node runtime and compiled
`@langfuse/native` addon, with no extra runtime or service.
Summary and assignment records use `trace_id` as their source when present;
@@ -88,37 +96,34 @@ Summaries snapshot source environment and trace name; assignments copy that
snapshot, including empty environment strings. Reprocessing refreshes metadata
even when text is reused; session results have no trace name.
-The summary model is selected with `LANGFUSE_TOPICS_SUMMARY_MODEL`. Use
-`us.openai.gpt-6-luna` for US routing or `global.openai.gpt-6-luna` for global
-routing; GPT-6 Luna has no EU geographic profile. Both support the existing
-Bedrock Converse tool-based structured output with reasoning disabled.
+For Bedrock summary models, use `us.openai.gpt-6-luna` for US routing or
+`global.openai.gpt-6-luna` for global routing; GPT-6 Luna has no EU geographic
+profile. Both support Bedrock Converse tool-based structured output; OpenAI
+models on Bedrock are called with reasoning effort `none`.
Summary cost estimates include both GPT-6 Luna profiles and the existing
`us.openai.gpt-5.6-luna` profile. See the
[GPT-6 Luna model card](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-luna.html)
-for supported source regions and pricing. Topic
-naming uses `us.openai.gpt-5.6-terra` through Bedrock Converse with reasoning
-disabled. The AWS model cards for [GPT-5.6 Luna](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html)
+for supported source regions and pricing. The
+recommended Bedrock clustering model is `us.openai.gpt-5.6-terra`. The AWS model cards for [GPT-5.6 Luna](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html)
and [GPT-5.6 Terra](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html)
currently list `us.` geographic inference profiles in commercial Regions, but
no `eu.` profiles; these profiles route within the US geography, including when
invoked from `eu-west-1`.
-Embeddings use a regional Cohere Embed v4 inference profile on Amazon Bedrock:
-set `LANGFUSE_TOPICS_EMBEDDING_MODEL` to `eu.cohere.embed-v4:0` for EU routing
-or `us.cohere.embed-v4:0` for US routing. Avoid the global profile when
-geographic residency matters. Calls use float output, `clustering` input type
-for both discovery and assignment, and truncation disabled. Default: 1,024
-dimensions; supported choices: 256, 512, 1,024, 1,536. Calls embed one summary at a
+On Bedrock, Cohere Embed v4 uses `eu.cohere.embed-v4:0` for EU routing or
+`us.cohere.embed-v4:0` for US routing. Avoid the global profile when geographic
+residency matters. Every provider is asked for clustering vectors of the
+configured size (Cohere `clustering` input type with truncation disabled, Google
+`CLUSTERING` task type, OpenAI `dimensions`). Default: 1,024
+dimensions; any whole number from 64 to 4,096 is accepted, and a provider that
+returns a vector of another length fails the call. Calls embed one summary at a
time, retaining per-summary Redis checkpointing.
Embedding settings are independent of immutable facet versions; changing them
with stored-summary reuse enabled regenerates vectors without summarization inference. This PoC does
not use `aiEmbed` or change ingestion. It accepts traces already stored in v4
events; legacy-only traces are unsupported.
-After changing embedding models, start a new **Process traces** execution with
-**Reuse stored summaries** to replace vectors without repeating summary inference,
-then **Update topics** to rebuild the map. Old vectors/maps cannot match the new
-configuration. Drain existing work before deploying; old-model staged jobs cannot
-resume under the new schema. Old execution history remains readable.
+Drain existing work before deploying model changes; old-model staged jobs cannot
+resume. Old execution history remains readable.
After changing the summary model, start a new **Process traces** execution;
stored-summary reuse cannot reuse results from a different summary model.
Existing topic names remain until their definitions change during **Update topics**.
@@ -136,8 +141,13 @@ skipped on retry. A failing trace does not stop the batch: every trace is
processed and outcomes are counted once per job (`langfuse.topics.trace_outcomes`
by outcome and reason). Failed traces are not retried, because a retry would
read the shared batch from ClickHouse again. Only a failed batch read fails the
-job. Summaries and embeddings use the same Bedrock models as manual
-processing.
+job. Summaries and embeddings use the same project models as manual
+processing. While the read starts, one Postgres query loads the enabled model
+settings of every allowlisted project in the batch; facets load once per
+project. Projects whose settings are off or incomplete are skipped, and new
+traces that arrive meanwhile are not queued. An authentication or billing error
+(HTTP 401, 402, 403) turns the project's automatic processing off with the
+failing slot as the reason, and the rest of the batch skips that project.
Required for a local run, in addition to Postgres, ClickHouse, and Redis:
@@ -150,10 +160,6 @@ Required for a local run, in addition to Postgres, ClickHouse, and Redis:
| `LANGFUSE_TRACE_BATCH_READ_ENABLED` | Allow the ClickHouse read. Default off. |
| `LANGFUSE_TRACE_BATCH_IDLE_MS` | Idle time before a trace is ready. Unset is 2 minutes on `DEV` and 10 minutes otherwise. |
| `LANGFUSE_TOPICS_ENABLED_PROJECT_IDS` | Same allowlist on web and worker. Unset defaults to the demo project. These projects bypass trace-batch sampling at ingestion, so every trace is summarized automatically; set `LANGFUSE_TRACE_BATCH_SAMPLING_RATE=0` to run the flow for them only. |
-| `LANGFUSE_AI_AWS_BEDROCK_REGION` | Bedrock region for summaries, naming, and embeddings. |
-| `LANGFUSE_TOPICS_SUMMARY_MODEL` | Required internal PoC setting; Bedrock model ID used for trace summaries. |
-| `LANGFUSE_TOPICS_EMBEDDING_MODEL` | Required internal PoC setting; Bedrock embedding model ID. |
-| `LANGFUSE_AI_FEATURES_AWS_PROFILE` | Optional shared local AI profile. `AWS_PROFILE` takes precedence; falls back to `LANGFUSE_IN_APP_AGENT_AWS_PROFILE`. |
## Run the experiment
@@ -401,7 +407,7 @@ Provider usage and calculated model costs use the events-style
`provided_usage_details`, `usage_details`, `provided_cost_details` and `cost_details`
maps. Summary and embedding keys are prefixed by stage; effective maps include
combined totals.
-Global Bedrock text rates per million input/output tokens are $0.20/$1.20 for
+Priced model IDs: global Bedrock text rates per million input/output tokens are $0.20/$1.20 for
[Luna](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html)
and $2/$12 for
[Terra](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html).
@@ -411,7 +417,7 @@ is logged and omitted from usage/cost maps, never estimated with an OpenAI token
Model prices and token budgets live in [models.ts](models.ts) and the shared
Topics contracts. Summary input defaults to 120,000 tokens. Oversized naming
input fails before calling the provider; member summaries are never silently discarded.
-Provider SDK retries are disabled. Embedding queue jobs retry transient failures
+Provider calls retry 429 and 5xx responses twice. Embedding queue jobs retry transient failures
up to three attempts with exponential backoff; authentication and invalid
input/output failures stop immediately. A manual resume can retry a failed batch
while its Redis payload still exists; each manual resume resets the three-attempt
diff --git a/worker/src/features/topics/models.test.ts b/worker/src/features/topics/models.test.ts
index 4d9fca1f3f..11bff393d5 100644
--- a/worker/src/features/topics/models.test.ts
+++ b/worker/src/features/topics/models.test.ts
@@ -4,20 +4,12 @@ const state = vi.hoisted(() => ({
call: vi.fn(),
embed: vi.fn(),
increment: vi.fn(),
- region: vi.fn(),
- summaryModel: vi.fn(),
}));
vi.mock("@langfuse/shared/src/server", () => ({
- getLangfuseAIAwsProfile: () => "ai-test",
- getLangfuseAIBedrockRegion: () => state.region(),
logger: { warn: vi.fn() },
recordIncrement: state.increment,
}));
vi.mock("@langfuse/shared/topics/server", () => ({
- getTopicsModelConfig: () => ({
- summaryModel: state.summaryModel(),
- embeddingModel: "eu.cohere.embed-v4:0",
- }),
generateTopicText: (...args: unknown[]) => state.call(...args),
generateTopicEmbedding: (...args: unknown[]) => state.embed(...args),
}));
@@ -29,6 +21,31 @@ import {
summarizeTopicTraceFacets,
} from "./models";
import { topicProcessingConfigSchema } from "@langfuse/shared/topics";
+import { LLMAdapter } from "@langfuse/shared";
+import type { TopicsModel, TopicsModels } from "@langfuse/shared/topics/server";
+
+function slot(slot: TopicsModel["slot"], model: string): TopicsModel {
+ return {
+ slot,
+ provider: "bedrock",
+ adapter: LLMAdapter.Bedrock,
+ model,
+ connection: { secretKey: "encrypted", config: { region: "eu-west-1" } },
+ };
+}
+
+function topicsModels(summaryModel = "us.openai.gpt-5.6-luna"): TopicsModels {
+ return {
+ projectId: "project",
+ enabled: true,
+ summary: slot("summary", summaryModel),
+ embedding: {
+ ...slot("embedding", "eu.cohere.embed-v4:0"),
+ dimensions: 256,
+ },
+ clustering: slot("clustering", "us.openai.gpt-5.6-terra"),
+ };
+}
const facet = {
projectId: "project",
@@ -50,8 +67,6 @@ beforeEach(() => {
state.call.mockReset();
state.embed.mockReset();
state.increment.mockReset();
- state.region.mockReset().mockReturnValue("eu-west-1");
- state.summaryModel.mockReset().mockReturnValue("us.openai.gpt-5.6-luna");
});
describe("Topics naming boundary", () => {
@@ -62,12 +77,12 @@ describe("Topics naming boundary", () => {
])(
"summarizes a trace and prices %s usage",
async (model, inputCost, outputCost) => {
- state.summaryModel.mockReturnValue(model);
state.call.mockResolvedValue({
output: { summary: "A billing request.", status: "applicable" },
usage: { inputTokens: 100, outputTokens: 30 },
});
const result = await summarizeTopicTrace(
+ topicsModels(model),
facet,
"RAW_TRANSCRIPT_SENTINEL",
topicProcessingConfigSchema.parse({
@@ -92,11 +107,7 @@ describe("Topics naming boundary", () => {
expect(result.costDetails.summary_output).toBeCloseTo(outputCost, 10);
expect(result.costDetails.total).toBeCloseTo(inputCost + outputCost, 10);
const request = state.call.mock.calls[0][0];
- expect(request).toMatchObject({
- model,
- region: "eu-west-1",
- profile: "ai-test",
- });
+ expect(request.model).toMatchObject({ slot: "summary", model });
expect(request.system).toEqual([
{ text: expect.stringContaining(facet.prompt), cache: true },
]);
@@ -106,17 +117,17 @@ describe("Topics naming boundary", () => {
},
);
- it("rejects missing Bedrock configuration before calling the provider", async () => {
- state.region.mockReturnValue(undefined);
+ it("rejects a request frozen on another summary model before calling the provider", async () => {
await expect(
summarizeTopicTrace(
+ topicsModels("us.openai.gpt-6-luna"),
facet,
"Trace evidence.",
topicProcessingConfigSchema.parse({
summaryModel: "us.openai.gpt-5.6-luna",
}),
),
- ).rejects.toMatchObject({ reason: "authentication" });
+ ).rejects.toMatchObject({ reason: "configuration" });
expect(state.call).not.toHaveBeenCalled();
});
@@ -129,6 +140,7 @@ describe("Topics naming boundary", () => {
summaryModel: "us.openai.gpt-5.6-luna",
});
const result = await summarizeTopicTrace(
+ topicsModels(),
facet,
"A request for an invoice.",
config,
@@ -151,6 +163,7 @@ describe("Topics naming boundary", () => {
it("rejects an oversized shared transcript before calling the provider", async () => {
await expect(
summarizeTopicTrace(
+ topicsModels(),
facet,
"Trace evidence. ".repeat(1000),
topicProcessingConfigSchema.parse({
@@ -165,7 +178,6 @@ describe("Topics naming boundary", () => {
});
it("prices cache reads and writes at their cache rates", async () => {
- state.summaryModel.mockReturnValue("us.openai.gpt-6-luna");
state.call.mockResolvedValue({
output: { summary: "A billing request.", status: "applicable" },
usage: {
@@ -177,6 +189,7 @@ describe("Topics naming boundary", () => {
},
});
const result = await summarizeTopicTrace(
+ topicsModels("us.openai.gpt-6-luna"),
facet,
"RAW_TRANSCRIPT_SENTINEL",
topicProcessingConfigSchema.parse({
@@ -219,7 +232,12 @@ describe("Topics naming boundary", () => {
{ key: "intent_1", facet, builtIn: true },
{ key: "issues_2", facet: issues, builtIn: false },
];
- await summarizeTopicTraceFacets(facets, "RAW_TRANSCRIPT_SENTINEL", config);
+ await summarizeTopicTraceFacets(
+ topicsModels(),
+ facets,
+ "RAW_TRANSCRIPT_SENTINEL",
+ config,
+ );
expect(state.call).toHaveBeenCalledOnce();
const request = state.call.mock.calls[0][0];
// Built-in facets end the first cached prefix, custom facets the second; the transcript is uncached.
@@ -252,6 +270,7 @@ describe("Topics naming boundary", () => {
state.call.mockClear();
await expect(
summarizeTopicTraceFacets(
+ topicsModels(),
facets,
"Trace evidence. ".repeat(1000),
topicProcessingConfigSchema.parse({
@@ -276,13 +295,19 @@ describe("Topics naming boundary", () => {
},
usage: { inputTokens: 16000, outputTokens: 30 },
});
- const result = await nameTopicGroup({ members, contrasts: [] });
+ const result = await nameTopicGroup(topicsModels(), {
+ members,
+ contrasts: [],
+ });
expect(result.output).toMatchObject({
name: "Invoice assistance",
evidenceSummaryIds: [members[399].id],
});
expect(state.call).toHaveBeenCalledOnce();
- expect(state.call.mock.calls[0][0].model).toBe("us.openai.gpt-5.6-terra");
+ expect(state.call.mock.calls[0][0].model).toMatchObject({
+ slot: "clustering",
+ model: "us.openai.gpt-5.6-terra",
+ });
expect(result.costDetails.total).toBeCloseTo(0.03236, 10);
const submitted = JSON.parse(state.call.mock.calls[0][0].input);
expect(submitted.members).toEqual(members);
@@ -299,7 +324,7 @@ describe("Topics naming boundary", () => {
},
usage: { inputTokens: 100, outputTokens: 30 },
});
- await expect(nameTopicGroup(evidence)).rejects.toThrow();
+ await expect(nameTopicGroup(topicsModels(), evidence)).rejects.toThrow();
// Provider work is billable even when local output validation rejects it.
expect(state.increment.mock.calls).toEqual([
[
@@ -319,7 +344,7 @@ describe("Topics naming boundary", () => {
it("records embedding input usage even when the returned vector is invalid", async () => {
state.embed.mockResolvedValue({ embedding: [], tokens: 12 });
await expect(
- embedTopicSummary("An invoice request.", 256, "eu.cohere.embed-v4:0"),
+ embedTopicSummary(topicsModels().embedding, "An invoice request.", 256),
).rejects.toMatchObject({
reason: "invalid_output",
});
@@ -340,9 +365,9 @@ describe("Topics naming boundary", () => {
tokens,
});
await embedTopicSummary(
+ topicsModels().embedding,
"An invoice request.",
256,
- "eu.cohere.embed-v4:0",
);
expect(state.increment.mock.calls).toEqual([
[
diff --git a/worker/src/features/topics/models.ts b/worker/src/features/topics/models.ts
index 2fc4e6d523..6026f13c50 100644
--- a/worker/src/features/topics/models.ts
+++ b/worker/src/features/topics/models.ts
@@ -1,15 +1,12 @@
import { get_encoding } from "tiktoken";
import { z } from "zod";
-import {
- logger,
- getLangfuseAIAwsProfile,
- getLangfuseAIBedrockRegion,
-} from "@langfuse/shared/src/server";
+import { logger } from "@langfuse/shared/src/server";
import {
generateTopicEmbedding,
generateTopicText,
- getTopicsModelConfig,
type TopicPromptPart,
+ type TopicsModel,
+ type TopicsModels,
} from "@langfuse/shared/topics/server";
import {
topicEmbeddingConfigSchema,
@@ -23,7 +20,6 @@ import {
topicProviderError,
} from "./provider-error";
-export const TOPICS_NAMING_MODEL = "us.openai.gpt-5.6-terra";
type Rates = {
input: number;
output: number;
@@ -31,54 +27,35 @@ type Rates = {
cacheWrite?: number;
};
// USD per million tokens. Summary inputs are capped below the 272k-token long-context pricing threshold.
-const TOPICS_SUMMARY_RATES: Record<string, Rates> = {
- "us.openai.gpt-5.6-luna": {
- input: 0.2,
- output: 1.2,
- cacheRead: 0.02,
- cacheWrite: 0.25,
- },
- "us.openai.gpt-6-luna": {
- input: 0.11,
- output: 0.55,
- cacheRead: 0.011,
- cacheWrite: 0.1375,
- },
- "global.openai.gpt-6-luna": {
- input: 0.1,
- output: 0.5,
- cacheRead: 0.01,
- cacheWrite: 0.125,
- },
-};
+const TOPICS_SUMMARY_RATES = new Map<string, Rates>([
+ [
+ "us.openai.gpt-5.6-luna",
+ { input: 0.2, output: 1.2, cacheRead: 0.02, cacheWrite: 0.25 },
+ ],
+ [
+ "us.openai.gpt-6-luna",
+ { input: 0.11, output: 0.55, cacheRead: 0.011, cacheWrite: 0.1375 },
+ ],
+ [
+ "global.openai.gpt-6-luna",
+ { input: 0.1, output: 0.5, cacheRead: 0.01, cacheWrite: 0.125 },
+ ],
+]);
+// Bedrock rates apply to the entire request above 272k input tokens.
+const TOPICS_NAMING_RATES = new Map([
+ [
+ "us.openai.gpt-5.6-terra",
+ (tokens: number): Rates => ({
+ input: tokens > 272_000 ? 4 : 2,
+ output: tokens > 272_000 ? 18 : 12,
+ }),
+ ],
+]);
const TOPICS_EMBEDDING_COST_MODELS = new Set([
"us.cohere.embed-v4:0",
"eu.cohere.embed-v4:0",
]);
-export function requireTopicsModelConfig() {
- const models = getTopicsModelConfig();
- if (!models.summaryModel || !models.embeddingModel)
- throw new TopicsProviderUnavailable(
- "Configure LANGFUSE_TOPICS_SUMMARY_MODEL and LANGFUSE_TOPICS_EMBEDDING_MODEL before running Topics.",
- "authentication",
- );
- return models;
-}
-
-function bedrockConfig() {
- const region = getLangfuseAIBedrockRegion();
- if (!region)
- throw new TopicsProviderUnavailable(
- "LANGFUSE_AI_AWS_BEDROCK_REGION is required for Topics models. Configure the worker's Bedrock region before resuming.",
- "authentication",
- );
- return {
- region,
- profile: getLangfuseAIAwsProfile(),
- };
-}
-
function countTopicRequestTokens(
system: TopicPromptPart[],
input: string,
@@ -111,19 +88,14 @@ async function structuredCall<T>(
inputLimit: number,
outputLimit: number,
stage: "summary" | "naming",
- model: string,
+ model: TopicsModel,
): Promise<ModelResult<T>> {
- const connection = bedrockConfig();
- // Bedrock global rates apply to the entire request above 272k input tokens.
+ // Unknown model IDs record usage without cost.
const rates = (tokens: number): Rates | undefined =>
stage === "naming"
- ? {
- input: tokens > 272_000 ? 4 : 2,
- output: tokens > 272_000 ? 18 : 12,
- }
- : TOPICS_SUMMARY_RATES[model];
+ ? TOPICS_NAMING_RATES.get(model.model)?.(tokens)
+ : TOPICS_SUMMARY_RATES.get(model.model);
const result = await generateTopicText({
- ...connection,
model,
system,
input,
@@ -131,10 +103,11 @@ async function structuredCall<T>(
maxOutputTokens: outputLimit,
}).catch((error: unknown) => {
logger.warn("Topics model request failed", {
- model,
+ model: model.model,
+ adapter: model.adapter,
errorType: error instanceof Error ? error.name : "unknown",
});
- throw topicProviderError(error);
+ throw topicProviderError(error, model);
});
const actualRates = rates(result.usage.inputTokens ?? inputLimit);
recordTopicTokenUsage(stage, {
@@ -213,18 +186,24 @@ Recordings come from many frameworks and can be messy. Repeated, partial, or pas
Status says whether the facet applies, not whether the run succeeded.
</status>`;
+function requireFrozenSummaryModel(
+ models: TopicsModels,
+ config: TopicProcessingConfig,
+) {
+ if (config.summaryModel !== models.summary.model)
+ throw new TopicsProviderUnavailable(
+ "The project's summary model changed after this request was created. Start a new Topics execution.",
+ "configuration",
+ );
+}
+
export async function summarizeTopicTrace(
+ models: TopicsModels,
facet: TopicFacetVersion,
text: string,
config: TopicProcessingConfig,
) {
- const models = requireTopicsModelConfig();
- const model = config.summaryModel;
- if (!model || model !== models.summaryModel)
- throw new TopicsProviderUnavailable(
- "Configure the matching LANGFUSE_TOPICS_SUMMARY_MODEL on web and worker before running Topics.",
- "authentication",
- );
+ requireFrozenSummaryModel(models, config);
const system = [
{
text: `${SUMMARY_SYSTEM_PROMPT}\n\n<facet>\n${facet.prompt}\n</facet>`,
@@ -247,7 +226,7 @@ export async function summarizeTopicTrace(
config.maxInputTokens,
config.maxOutputTokens,
"summary",
- model,
+ models.summary,
);
}
@@ -259,17 +238,12 @@ export async function summarizeTopicTrace(
* must pass facets in a stable order with built-ins first.
*/
export async function summarizeTopicTraceFacets(
+ models: TopicsModels,
facets: { key: string; facet: TopicFacetVersion; builtIn: boolean }[],
text: string,
config: TopicProcessingConfig,
) {
- const models = requireTopicsModelConfig();
- const model = config.summaryModel;
- if (!model || model !== models.summaryModel)
- throw new TopicsProviderUnavailable(
- "Configure the matching LANGFUSE_TOPICS_SUMMARY_MODEL on web and worker before running Topics.",
- "authentication",
- );
+ requireFrozenSummaryModel(models, config);
const entry = z.object({
notes: z
.string()
@@ -319,15 +293,17 @@ ${facetBlock(true)}`,
config.maxInputTokens,
config.maxOutputTokens * facets.length,
"summary",
- model,
+ models.summary,
);
}
-export async function nameTopicGroup(group: {
- members: { id: string; summary: string }[];
- contrasts: { id: string; summary: string }[];
-}) {
- requireTopicsModelConfig();
+export async function nameTopicGroup(
+ models: TopicsModels,
+ group: {
+ members: { id: string; summary: string }[];
+ contrasts: { id: string; summary: string }[];
+ },
+) {
if (!group.members.length)
throw new Error("Naming requires one non-empty effective group.");
const memberIds = new Set(group.members.map((member) => member.id));
@@ -368,38 +344,30 @@ export async function nameTopicGroup(group: {
inputLimit,
1000,
"naming",
- TOPICS_NAMING_MODEL,
+ models.clustering,
);
}
export async function embedTopicSummary(
+ model: TopicsModel,
summary: string,
dimensions: number,
- model: string,
): Promise<TopicModelUsage & { embedding: number[] }> {
- const models = requireTopicsModelConfig();
- if (!model.trim() || model !== models.embeddingModel)
- throw new TopicsProviderUnavailable(
- "Configure the matching LANGFUSE_TOPICS_EMBEDDING_MODEL on web and worker before running Topics.",
- "authentication",
- );
- const connection = bedrockConfig();
if (
!summary.trim() ||
!topicEmbeddingConfigSchema.safeParse({ embeddingDimensions: dimensions })
.success
)
throw new TopicsProviderUnavailable(
- "Topics embeddings require non-empty text and 256, 512, 1024, or 1536 dimensions.",
+ "Topics embeddings require non-empty text and 64 to 4,096 dimensions.",
"invalid_input",
);
const result = await generateTopicEmbedding({
- ...connection,
model,
summary,
dimensions,
}).catch((error: unknown) => {
- throw topicProviderError(error);
+ throw topicProviderError(error, model);
});
recordTopicTokenUsage("embedding", { input: result.tokens });
// ClickHouse stores Float32; calibration and future classification must use those same vectors.
@@ -417,12 +385,12 @@ export async function embedTopicSummary(
const costDetails: Record<string, number> = {};
if (Number.isSafeInteger(result.tokens) && result.tokens >= 0) {
usageDetails.embedding_input = usageDetails.total = result.tokens;
- if (TOPICS_EMBEDDING_COST_MODELS.has(model))
+ if (TOPICS_EMBEDDING_COST_MODELS.has(model.model))
costDetails.embedding_input = costDetails.total =
(result.tokens * 0.12) / 1_000_000;
} else {
logger.warn("Topics embedding response omitted token usage", {
- model,
+ model: model.model,
});
}
return {
diff --git a/worker/src/features/topics/processTopicEmbeddingBatch.test.ts b/worker/src/features/topics/processTopicEmbeddingBatch.test.ts
index 606856647c..6cd81853b8 100644
--- a/worker/src/features/topics/processTopicEmbeddingBatch.test.ts
+++ b/worker/src/features/topics/processTopicEmbeddingBatch.test.ts
@@ -12,24 +12,36 @@ const mocks = vi.hoisted(() => ({
staged: vi.fn(),
update: vi.fn(),
embed: vi.fn(),
+ pause: vi.fn(),
+ embeddingModel: "eu.cohere.embed-v4:0",
}));
vi.mock("@langfuse/shared/topics/server", () => ({
+ pauseTopicsModels: mocks.pause,
writeTopicSummaries: mocks.write,
readStagedTopicSummary: mocks.staged,
updateStagedTopicSummary: mocks.update,
+ getTopicsModels: async () => ({
+ projectId: "project",
+ enabled: true,
+ embedding: {
+ slot: "embedding",
+ provider: "bedrock",
+ adapter: "bedrock",
+ model: mocks.embeddingModel,
+ connection: { secretKey: "encrypted" },
+ dimensions: 256,
+ },
+ }),
TOPIC_EMBEDDING_EXPIRED_ERROR:
"Topics staged results expired before processing completed. Start a new execution with stored-summary reuse to recover persisted results.",
}));
vi.mock("@langfuse/shared/src/server", () => ({
+ logger: { error: vi.fn() },
recordIncrement: vi.fn(),
recordDistribution: vi.fn(),
}));
vi.mock("./models", () => ({
embedTopicSummary: mocks.embed,
- requireTopicsModelConfig: () => ({
- summaryModel: "us.openai.gpt-5.6-luna",
- embeddingModel: "eu.cohere.embed-v4:0",
- }),
}));
import { processTopicEmbeddingBatch } from "./processTopicEmbeddingBatch";
@@ -222,6 +234,22 @@ describe("Topics embedding handoff", () => {
);
expect(mocks.write).not.toHaveBeenCalled();
expect(staged.get(topicSourceKey(row))?.state).toBe("summarized");
+ expect(mocks.pause).toHaveBeenCalledExactlyOnceWith("project", {
+ blockReason: "LLM_CONNECTION_AUTH_INVALID",
+ blockMessage: expect.stringContaining("Check worker credentials."),
+ });
+ });
+
+ it("does not pause Topics when the saved embedding model changed", async () => {
+ const row = summary();
+ staged.set(topicSourceKey(row), row);
+ mocks.embeddingModel = "text-embedding-3-small";
+ await expect(processTopicEmbeddingBatch(batch(row))).rejects.toThrow(
+ UnrecoverableError,
+ );
+ mocks.embeddingModel = "eu.cohere.embed-v4:0";
+ expect(mocks.embed).not.toHaveBeenCalled();
+ expect(mocks.pause).not.toHaveBeenCalled();
});
it("fences an embedding when its Redis payload disappears during the call", async () => {
diff --git a/worker/src/features/topics/processTopicEmbeddingBatch.ts b/worker/src/features/topics/processTopicEmbeddingBatch.ts
index 1aab65c19f..ee6ec65c69 100644
--- a/worker/src/features/topics/processTopicEmbeddingBatch.ts
+++ b/worker/src/features/topics/processTopicEmbeddingBatch.ts
@@ -1,16 +1,19 @@
import { UnrecoverableError } from "bullmq";
import type { TopicSummary } from "@langfuse/shared/topics";
import {
+ getTopicsModels,
readStagedTopicSummary,
TOPIC_EMBEDDING_EXPIRED_ERROR,
updateStagedTopicSummary,
writeTopicSummaries,
type TopicEmbeddingBatch,
} from "@langfuse/shared/topics/server";
-import { embedTopicSummary, requireTopicsModelConfig } from "./models";
+import { embedTopicSummary } from "./models";
import { TopicMetrics } from "./metrics";
import { mergeTopicModelUsage } from "./summaryResult";
import {
+ isRejectedConnection,
+ pauseForRejectedConnection,
topicProviderError,
TopicsProviderUnavailable,
} from "./provider-error";
@@ -22,28 +25,33 @@ export async function processTopicEmbeddingBatch(
const metrics = new TopicMetrics();
const pending: TopicSummary[] = [];
try {
- const models = requireTopicsModelConfig();
+ const models = await getTopicsModels(batch.projectId);
+ if (!models)
+ throw new TopicsProviderUnavailable(
+ "Choose Topics models for this project, then start a new Topics execution.",
+ "configuration",
+ );
for (const ref of batch.summaries) {
const staged = await readStagedTopicSummary(batch, ref);
if (!staged) throw new UnrecoverableError(TOPIC_EMBEDDING_EXPIRED_ERROR);
- if (staged.embeddingConfig.embeddingModel !== models.embeddingModel)
+ if (staged.embeddingConfig.embeddingModel !== models.embedding.model)
throw new TopicsProviderUnavailable(
- "LANGFUSE_TOPICS_EMBEDDING_MODEL changed after this batch was created. Start a new Topics execution.",
- "authentication",
+ "The project's embedding model changed after this batch was created. Start a new Topics execution.",
+ "configuration",
);
let { summary } = staged;
if (summary.state === "summarized") {
const result = await metrics.measure("embedding", async () => {
try {
return await embedTopicSummary(
+ models.embedding,
summary.summary,
staged.embeddingConfig.embeddingDimensions,
- staged.embeddingConfig.embeddingModel!,
);
} catch (error) {
throw error instanceof TopicsProviderUnavailable
? error
- : topicProviderError(error);
+ : topicProviderError(error, models.embedding);
}
});
summary = {
@@ -65,11 +73,16 @@ export async function processTopicEmbeddingBatch(
}
}
} catch (error) {
+ if (isRejectedConnection(error))
+ await pauseForRejectedConnection(batch.projectId, error);
if (
error instanceof TopicsProviderUnavailable &&
- ["authentication", "invalid_input", "invalid_output"].includes(
- error.reason,
- )
+ [
+ "authentication",
+ "configuration",
+ "invalid_input",
+ "invalid_output",
+ ].includes(error.reason)
)
throw new UnrecoverableError(error.message);
throw error;
diff --git a/worker/src/features/topics/processTopicsExecution.test.ts b/worker/src/features/topics/processTopicsExecution.test.ts
index 9488b67337..670d6908dd 100644
--- a/worker/src/features/topics/processTopicsExecution.test.ts
+++ b/worker/src/features/topics/processTopicsExecution.test.ts
@@ -72,10 +72,25 @@ vi.mock("@langfuse/shared/topics/server", async (importOriginal) => {
await importOriginal<typeof import("@langfuse/shared/topics/server")>();
return {
isTopicsEnabled: () => true,
- getTopicsModelConfig: () => ({
- summaryModel: "gpt-4.1-nano",
- embeddingModel: "eu.cohere.embed-v4:0",
- }),
+ getTopicsModels: async () => {
+ const slot = (name: string, model: string) => ({
+ slot: name,
+ provider: "openai",
+ adapter: "openai",
+ model,
+ connection: { secretKey: "encrypted" },
+ });
+ return {
+ projectId: "project",
+ enabled: true,
+ summary: slot("summary", "gpt-4.1-nano"),
+ embedding: {
+ ...slot("embedding", "eu.cohere.embed-v4:0"),
+ dimensions: 256,
+ },
+ clustering: slot("clustering", "gpt-5.6-terra"),
+ };
+ },
TOPICS_TRANSCRIPT_VERSION,
TOPIC_EMBEDDING_EXPIRED_ERROR,
stageTopicSummary: async (
@@ -298,14 +313,11 @@ vi.mock("@langfuse/shared/topics/server", async (importOriginal) => {
};
});
vi.mock("./models", () => ({
- TOPICS_NAMING_MODEL: "us.openai.gpt-5.6-terra",
- requireTopicsModelConfig: () => ({
- summaryModel: "gpt-4.1-nano",
- embeddingModel: "eu.cohere.embed-v4:0",
- }),
- summarizeTopicTrace: (...args: unknown[]) => state.summarize(...args),
- embedTopicSummary: (...args: unknown[]) => state.embed(...args),
- nameTopicGroup: (...args: unknown[]) => state.name(...args),
+ summarizeTopicTrace: (_models: unknown, ...args: unknown[]) =>
+ state.summarize(...args),
+ embedTopicSummary: (_model: unknown, ...args: unknown[]) =>
+ state.embed(...args),
+ nameTopicGroup: (_models: unknown, ...args: unknown[]) => state.name(...args),
}));
vi.mock("./numeric", async (importOriginal) => ({
...(await importOriginal<typeof import("./numeric")>()),
@@ -797,7 +809,12 @@ describe("Topics execution", () => {
const original = state.summarize.getMockImplementation()!;
state.summarize
.mockImplementationOnce(original)
- .mockRejectedValueOnce(topicProviderError({ statusCode: 503 }));
+ .mockRejectedValueOnce(
+ topicProviderError(
+ { statusCode: 503 },
+ { slot: "summary", provider: "openai", model: "gpt-4.1-nano" },
+ ),
+ );
await processSelection("partial", 3);
expect(state.batches.get("partial")?.summaries).toHaveLength(1);
expect(state.executions.get("partial")?.status).toBe("failed");
diff --git a/worker/src/features/topics/processTopicsExecution.ts b/worker/src/features/topics/processTopicsExecution.ts
index bde3e03bb5..c5ca4da92d 100644
--- a/worker/src/features/topics/processTopicsExecution.ts
+++ b/worker/src/features/topics/processTopicsExecution.ts
@@ -22,8 +22,9 @@ import {
enqueueTopicEmbeddingBatch,
TOPIC_EMBEDDING_EXPIRED_ERROR,
TOPICS_TRANSCRIPT_VERSION,
- getTopicsModelConfig,
+ getTopicsModels,
type TopicEmbeddingRef,
+ type TopicsModels,
} from "@langfuse/shared/topics/server";
import {
type TopicExecution,
@@ -38,12 +39,12 @@ import {
sameTopicGeometry,
topicTimeRangeSchema,
} from "@langfuse/shared/topics";
+import { summarizeTopicTrace, nameTopicGroup } from "./models";
import {
- summarizeTopicTrace,
- nameTopicGroup,
- TOPICS_NAMING_MODEL,
-} from "./models";
-import { TopicsProviderUnavailable } from "./provider-error";
+ isRejectedConnection,
+ pauseForRejectedConnection,
+ TopicsProviderUnavailable,
+} from "./provider-error";
import {
buildTopicPrototypes,
buildNamingEvidence,
@@ -101,6 +102,7 @@ async function saveProgress(
async function summarizeTrace(
metrics: TopicMetrics,
+ models: TopicsModels,
execution: ProcessExecution,
facet: TopicFacetVersion,
traceId: string,
@@ -185,6 +187,7 @@ async function summarizeTrace(
} else if (transcript !== null) {
const result = await metrics.measure("summary", async () => {
const result = await summarizeTopicTrace(
+ models,
facet,
JSON.stringify(transcript),
execution.input.processingConfig,
@@ -416,6 +419,7 @@ async function assignSummaries(
async function clusterFacet(
metrics: TopicMetrics,
+ models: TopicsModels,
execution: UpdateExecution,
facet: TopicFacetVersion,
progress: TopicFacetProgress,
@@ -640,7 +644,7 @@ async function clusterFacet(
const members = new Map(
group.members.map((summary, index) => [`m${index + 1}`, summary]),
);
- const { output: label } = await nameTopicGroup({
+ const { output: label } = await nameTopicGroup(models, {
members: Array.from(members, ([id, { summary }]) => ({
id,
summary,
@@ -690,7 +694,7 @@ async function clusterFacet(
metadata: {
...candidate.metadata,
effectiveMemberCount: group.count,
- namingModel: TOPICS_NAMING_MODEL,
+ namingModel: models.clustering.model,
},
});
metrics.result("naming", "generated");
@@ -764,6 +768,7 @@ async function pendingFacets(
/** Processes one bounded queue batch; the job owns retry state until completion. */
async function processTraces(
metrics: TopicMetrics,
+ models: TopicsModels,
execution: ProcessExecution,
state: TopicProcessBatchState,
batchId: string,
@@ -846,6 +851,7 @@ async function processTraces(
try {
await summarizeTrace(
metrics,
+ models,
execution,
facet,
traceId,
@@ -957,7 +963,11 @@ async function processTraces(
}
/** Each unpublished attempt fits the current compatible summaries from scratch. */
-async function updateTopics(metrics: TopicMetrics, execution: UpdateExecution) {
+async function updateTopics(
+ metrics: TopicMetrics,
+ models: TopicsModels,
+ execution: UpdateExecution,
+) {
for (const { facet, progress } of await pendingFacets(execution)) {
try {
const accepted = progress.runId
@@ -1010,6 +1020,7 @@ async function updateTopics(metrics: TopicMetrics, execution: UpdateExecution) {
progress.counts.requested = summaries.length;
await clusterFacet(
metrics,
+ models,
execution,
facet,
progress,
@@ -1132,27 +1143,30 @@ export async function processTopicsExecution({
const phase = execution.phase === "embedding" ? "embedding" : "summarizing";
await save(processing ? phase : "selecting");
try {
- const configuredModels = getTopicsModelConfig();
- if (!configuredModels.summaryModel || !configuredModels.embeddingModel)
+ const models = await getTopicsModels(projectId);
+ if (!models)
throw new TopicsProviderUnavailable(
- "Configure LANGFUSE_TOPICS_SUMMARY_MODEL and LANGFUSE_TOPICS_EMBEDDING_MODEL before running Topics.",
- "authentication",
+ "Choose Topics models for this project before running Topics.",
+ "configuration",
+ );
+ if (
+ execution.input.embeddingConfig.embeddingModel !== models.embedding.model
+ )
+ throw new TopicsProviderUnavailable(
+ "The project's embedding model changed after this execution was created; start a new Topics execution.",
+ "configuration",
);
if (execution.input.operation === "process") {
if (
- !execution.input.processingConfig.summaryModel ||
- !execution.input.embeddingConfig.embeddingModel ||
- execution.input.processingConfig.summaryModel !==
- configuredModels.summaryModel ||
- execution.input.embeddingConfig.embeddingModel !==
- configuredModels.embeddingModel
+ execution.input.processingConfig.summaryModel !== models.summary.model
)
throw new TopicsProviderUnavailable(
- "Configure matching LANGFUSE_TOPICS_SUMMARY_MODEL and LANGFUSE_TOPICS_EMBEDDING_MODEL values on web and worker, then start a new Topics execution.",
- "authentication",
+ "The project's summary model changed after this execution was created; start a new Topics execution.",
+ "configuration",
);
await processTraces(
metrics,
+ models,
execution as ProcessExecution,
state,
batchId!,
@@ -1160,15 +1174,7 @@ export async function processTopicsExecution({
save,
);
} else {
- if (
- execution.input.embeddingConfig.embeddingModel !==
- configuredModels.embeddingModel
- )
- throw new TopicsProviderUnavailable(
- "LANGFUSE_TOPICS_EMBEDDING_MODEL changed after this execution was created. Start a new Topics execution.",
- "authentication",
- );
- await updateTopics(metrics, execution as UpdateExecution);
+ await updateTopics(metrics, models, execution as UpdateExecution);
}
execution.status = execution.facets.some(
(facet) => facet.outcome === "failed" || facet.counts.failed > 0,
@@ -1183,6 +1189,8 @@ export async function processTopicsExecution({
await save("embedding");
return;
}
+ if (isRejectedConnection(error))
+ await pauseForRejectedConnection(projectId, error);
metrics.error("execution", error);
metrics.execution("failed");
execution.status = "failed";
diff --git a/worker/src/features/topics/provider-error.ts b/worker/src/features/topics/provider-error.ts
index ee382a5822..9ea7e98989 100644
--- a/worker/src/features/topics/provider-error.ts
+++ b/worker/src/features/topics/provider-error.ts
@@ -1,7 +1,18 @@
-import { TOPICS_BEDROCK_ACCESS_ERROR } from "@langfuse/shared/topics";
+import {
+ TOPICS_MODEL_SLOT_DETAILS,
+ type TopicsModelSlotName,
+} from "@langfuse/shared/topics";
+import { EvaluatorBlockReason } from "@prisma/client";
+import { logger } from "@langfuse/shared/src/server";
+import {
+ pauseTopicsModels,
+ type TopicsModel,
+} from "@langfuse/shared/topics/server";
type TopicsProviderErrorReason =
| "authentication"
+ // The project's saved models are missing or differ from a frozen request.
+ | "configuration"
| "rate_limit"
| "timeout"
| "invalid_output"
@@ -12,13 +23,17 @@ export class TopicsProviderUnavailable extends Error {
constructor(
message: string,
readonly reason: TopicsProviderErrorReason = "provider_error",
+ readonly slot?: TopicsModelSlotName,
) {
super(message);
}
}
/** Provider bodies and exception messages can echo credentials or trace content. */
-export function topicProviderError(error: unknown): TopicsProviderUnavailable {
+export function topicProviderError(
+ error: unknown,
+ model: Pick<TopicsModel, "slot" | "provider" | "model">,
+): TopicsProviderUnavailable {
let current = error;
let status: number | undefined;
let reason: TopicsProviderErrorReason = "provider_error";
@@ -31,7 +46,8 @@ export function topicProviderError(error: unknown): TopicsProviderUnavailable {
const candidate = record.statusCode ?? record.status;
if (typeof candidate === "number" && candidate >= 400 && candidate <= 599) {
status = candidate;
- if (status === 401 || status === 403) reason = "authentication";
+ if (status === 401 || status === 402 || status === 403)
+ reason = "authentication";
else if (status === 400 || status === 413 || status === 422)
reason = "invalid_input";
else if (status === 429) reason = "rate_limit";
@@ -53,9 +69,41 @@ export function topicProviderError(error: unknown): TopicsProviderUnavailable {
current = record.cause;
}
return new TopicsProviderUnavailable(
- status === 403
- ? TOPICS_BEDROCK_ACCESS_ERROR
- : `Topics provider call failed${status ? ` (HTTP ${status})` : ""}. Processing stopped. Check worker credentials or provider availability, then resume the execution.`,
+ `The ${TOPICS_MODEL_SLOT_DETAILS[model.slot].label.toLowerCase()} model ${model.model} on LLM connection "${model.provider}" failed${status ? ` (HTTP ${status})` : ""}. Check the connection's credentials, the model ID, and provider availability.`,
reason,
+ model.slot,
);
}
+
+/** The provider rejected the connection's credentials or billing. */
+export function isRejectedConnection(
+ error: unknown,
+): error is TopicsProviderUnavailable {
+ return (
+ error instanceof TopicsProviderUnavailable &&
+ error.reason === "authentication"
+ );
+}
+
+function rejectedConnectionBlock(error: TopicsProviderUnavailable) {
+ return {
+ blockReason: EvaluatorBlockReason.LLM_CONNECTION_AUTH_INVALID,
+ blockMessage: `${error.message} Topics was turned off; fix the connection and turn it on again.`,
+ };
+}
+
+/** Turns automatic processing off; a failed write is logged, not rethrown. */
+export async function pauseForRejectedConnection(
+ projectId: string,
+ error: TopicsProviderUnavailable,
+): Promise<void> {
+ try {
+ await pauseTopicsModels(projectId, rejectedConnectionBlock(error));
+ } catch (pauseError) {
+ logger.error("Failed to pause Topics after a rejected connection", {
+ projectId,
+ error:
+ pauseError instanceof Error ? pauseError.message : String(pauseError),
+ });
+ }
+}
diff --git a/worker/src/features/topics/storage.integration.test.ts b/worker/src/features/topics/storage.integration.test.ts
index bda474bea3..27853b45f0 100644
--- a/worker/src/features/topics/storage.integration.test.ts
+++ b/worker/src/features/topics/storage.integration.test.ts
@@ -23,6 +23,7 @@ import {
getTopicRule,
listTopicRules,
saveTopicRule,
+ saveTopicRuleAndModels,
createTopicExecution,
readTopicExecutionSummary,
writeTopicExecution,
@@ -37,6 +38,10 @@ describe("Topics eval-backed configuration", () => {
const orgId = randomUUID();
const projectId = randomUUID();
const otherProjectId = randomUUID();
+ const ruleProjectId = randomUUID();
+ const rollbackProjectId = randomUUID();
+ const auditProjectId = randomUUID();
+ const userIds = [randomUUID(), randomUUID()];
beforeAll(async () => {
await prisma.organization.create({
@@ -47,14 +52,21 @@ describe("Topics eval-backed configuration", () => {
create: [
{ id: projectId, name: "Topics" },
{ id: otherProjectId, name: "Other project" },
+ { id: ruleProjectId, name: "Topics rule" },
+ { id: rollbackProjectId, name: "Topics rollback" },
+ { id: auditProjectId, name: "Topics audit" },
],
},
},
});
+ await prisma.user.createMany({
+ data: userIds.map((id) => ({ id, email: `${id}@example.com` })),
+ });
});
afterAll(async () => {
await prisma.organization.deleteMany({ where: { id: orgId } });
+ await prisma.user.deleteMany({ where: { id: { in: userIds } } });
});
it("initializes built-in FACET evaluators once under concurrent requests", async () => {
@@ -90,6 +102,122 @@ describe("Topics eval-backed configuration", () => {
).rejects.toThrow(/built-in/i);
});
+ it("keeps one Topics rule per project when first saves race", async () => {
+ const facets = await ensureDefaultTopicFacets(ruleProjectId);
+ const facetIds = facets.slice(0, 2).map((facet) => facet.id);
+ const saved = await Promise.allSettled(
+ [0.25, 0.5, 0.75].map((sampling) =>
+ saveTopicRule({
+ projectId: ruleProjectId,
+ filter: [],
+ facetIds,
+ sampling,
+ }),
+ ),
+ );
+
+ // One save creates the rule; the others see it and are rejected.
+ expect(
+ saved.filter((result) => result.status === "fulfilled"),
+ ).toHaveLength(1);
+ for (const result of saved.filter((r) => r.status === "rejected"))
+ expect(String(result.reason)).toMatch(/already has a Topics rule/);
+ const rules = await listTopicRules(ruleProjectId);
+ expect(rules).toHaveLength(1);
+ expect(rules[0].facetIds.sort()).toEqual([...facetIds].sort());
+ });
+
+ it("leaves no rule behind when the model settings cannot be written", async () => {
+ const facets = await ensureDefaultTopicFacets(rollbackProjectId);
+ await expect(
+ saveTopicRuleAndModels(
+ {
+ projectId: rollbackProjectId,
+ filter: [],
+ facetIds: [facets[0].id],
+ },
+ {
+ enabled: true,
+ blockedAt: null,
+ blockReason: null,
+ blockMessage: null,
+ // No such connection: the foreign key rejects the model write.
+ summaryLlmApiKeyId: "missing-connection",
+ summaryModel: "gpt-6-luna",
+ embeddingLlmApiKeyId: null,
+ embeddingModel: null,
+ embeddingDimensions: 1024,
+ clusteringLlmApiKeyId: null,
+ clusteringModel: null,
+ },
+ 120_000,
+ ),
+ ).rejects.toThrow();
+
+ expect(await listTopicRules(rollbackProjectId)).toEqual([]);
+ expect(
+ await prisma.topicConfig.findUnique({
+ where: { projectId: rollbackProjectId },
+ }),
+ ).toBeNull();
+ });
+
+ it("records who created and last updated the Topics configuration", async () => {
+ const facets = await ensureDefaultTopicFacets(auditProjectId);
+ const models = {
+ enabled: false,
+ blockedAt: null,
+ blockReason: null,
+ blockMessage: null,
+ summaryLlmApiKeyId: null,
+ summaryModel: null,
+ embeddingLlmApiKeyId: null,
+ embeddingModel: null,
+ embeddingDimensions: 1024,
+ clusteringLlmApiKeyId: null,
+ clusteringModel: null,
+ };
+ const rule = await saveTopicRuleAndModels(
+ {
+ projectId: auditProjectId,
+ filter: [],
+ facetIds: [facets[0].id],
+ userId: userIds[0],
+ },
+ models,
+ 120_000,
+ );
+ await saveTopicRuleAndModels(
+ {
+ id: rule.id,
+ projectId: auditProjectId,
+ filter: [],
+ facetIds: [facets[0].id],
+ userId: userIds[1],
+ },
+ models,
+ 180_000,
+ );
+
+ expect(
+ await prisma.topicConfig.findUnique({
+ where: { projectId: auditProjectId },
+ }),
+ ).toMatchObject({ createdBy: userIds[0], updatedBy: userIds[1] });
+ expect(
+ await prisma.tracingReadinessSettings.findUnique({
+ where: { projectId: auditProjectId },
+ }),
+ ).toMatchObject({
+ traceIdleMs: 180_000,
+ createdBy: userIds[0],
+ updatedBy: userIds[1],
+ });
+ expect(
+ await prisma.evaluationRule.findUnique({ where: { id: rule.id } }),
+ ).toMatchObject({ createdByUserId: userIds[0] });
+ });
+
it("keeps versions and rule assignments scoped to this project's FACET evaluators", async () => {
const facet = await createTopicFacet({
projectId,
@@ -150,6 +278,17 @@ describe("Topics eval-backed configuration", () => {
facetIds: [facet.id, facet.id],
});
expect(rule.facetIds).toEqual([facet.id]);
+ const second = { projectId, filter: [], facetIds: [facet.id] };
+ await expect(saveTopicRule(second)).rejects.toThrow(
+ /already has a Topics rule/,
+ );
+ expect(
+ await saveTopicRule({
+ ...second,
+ id: rule.id,
+ name: "Second selection",
+ }),
+ ).toMatchObject({ id: rule.id, name: "Second selection" });
expect(
await prisma.evaluationRule.findUnique({ where: { id: rule.id } }),
).toMatchObject({
diff --git a/worker/src/features/topics/summarizeAssembledTrace.test.ts b/worker/src/features/topics/summarizeAssembledTrace.test.ts
index b4f8328c1b..9eb6adaa5e 100644
--- a/worker/src/features/topics/summarizeAssembledTrace.test.ts
+++ b/worker/src/features/topics/summarizeAssembledTrace.test.ts
@@ -5,9 +5,9 @@ import {
type Transcript,
} from "@langfuse/shared/src/server";
import type { TopicFacet } from "@langfuse/shared/topics";
+import type { TopicsModels } from "@langfuse/shared/topics/server";
const state = vi.hoisted(() => ({
- enabled: true,
facets: vi.fn(),
stored: vi.fn(),
write: vi.fn(),
@@ -17,12 +17,6 @@ const state = vi.hoisted(() => ({
}));
vi.mock("@langfuse/shared/topics/server", () => ({
- isTopicsProjectEnabled: () => state.enabled,
- getTopicsModelConfig: () => ({
- summaryModel: "us.openai.gpt-5.6-luna",
- embeddingModel: "eu.cohere.embed-v4:0",
- }),
- ensureDefaultTopicFacets: (...args: unknown[]) => state.facets(...args),
listTopicSummaries: (...args: unknown[]) => state.stored(...args),
writeTopicSummaries: (...args: unknown[]) => state.write(...args),
TOPICS_TRANSCRIPT_VERSION: "shared-transcript-v3",
@@ -85,6 +79,21 @@ const facet: TopicFacet = {
],
};
+const slot = (name: "summary" | "embedding" | "clustering", model: string) => ({
+ slot: name,
+ provider: "bedrock",
+ adapter: "bedrock",
+ model,
+ connection: { secretKey: "encrypted" },
+});
+const models = {
+ projectId: "project-a",
+ enabled: true,
+ summary: slot("summary", "us.openai.gpt-5.6-luna"),
+ embedding: { ...slot("embedding", "eu.cohere.embed-v4:0"), dimensions: 1024 },
+ clustering: slot("clustering", "us.openai.gpt-5.6-terra"),
+} as TopicsModels;
+
const usage = {
providedUsageDetails: { summary_input: 20 },
usageDetails: { summary_input: 20, summary_output: 8, total: 28 },
@@ -99,14 +108,13 @@ const usage = {
beforeEach(() => {
vi.mocked(recordDistribution).mockClear();
vi.mocked(recordIncrement).mockClear();
- state.enabled = true;
state.facets.mockReset();
state.stored.mockReset();
state.write.mockReset();
state.summarize.mockReset();
state.embed.mockReset();
state.setAttribute.mockClear();
- state.facets.mockResolvedValue([facet]);
+ state.facets.mockReturnValue([facet]);
state.stored.mockResolvedValue([]);
state.write.mockResolvedValue(undefined);
state.summarize.mockResolvedValue({
@@ -125,21 +133,6 @@ beforeEach(() => {
});
describe("summarizeAssembledTrace", () => {
- it("leaves other projects untouched", async () => {
- state.enabled = false;
- await summarizeAssembledTrace({
- projectId: "project-b",
- traceId: "trace-1",
- traceTimestamp: "2026-09-22T12:00:00.000Z",
- environment: "default",
- traceName: "agent-turn",
- transcript,
- });
- expect(state.facets).not.toHaveBeenCalled();
- expect(state.summarize).not.toHaveBeenCalled();
- expect(state.write).not.toHaveBeenCalled();
- });
-
it("summarizes the assembled transcript once per current facet and skips a finished retry", async () => {
await summarizeAssembledTrace({
projectId: "project-a",
@@ -148,15 +141,17 @@ describe("summarizeAssembledTrace", () => {
environment: "default",
traceName: "agent-turn",
transcript,
+ models,
+ facets: state.facets(),
});
- expect(state.facets).toHaveBeenCalledWith("project-a");
expect(state.summarize).toHaveBeenCalledTimes(1);
const submitted = state.summarize.mock.calls[0];
- expect(submitted[0]).toEqual([
+ expect(submitted[0]).toBe(models);
+ expect(submitted[1]).toEqual([
{ key: "intent_1", facet: facet.versions[0], builtIn: true },
]);
- expect(submitted[1]).toContain("Export monthly sales");
- expect(submitted[1]).not.toContain("observation");
+ expect(submitted[2]).toContain("Export monthly sales");
+ expect(submitted[2]).not.toContain("observation");
const written = state.write.mock.calls[0][0][0];
expect(written).toMatchObject({
projectId: "project-a",
@@ -203,6 +198,8 @@ describe("summarizeAssembledTrace", () => {
environment: "default",
traceName: "agent-turn",
transcript,
+ models,
+ facets: state.facets(),
});
expect(state.summarize).not.toHaveBeenCalled();
expect(state.write).not.toHaveBeenCalled();
@@ -213,6 +210,8 @@ describe("summarizeAssembledTrace", () => {
environment: "default",
traceName: "agent-turn",
transcript,
+ models,
+ facets: state.facets(),
});
expect(state.summarize).toHaveBeenCalledOnce();
expect(state.write.mock.calls[0][0][0].unitStartTime).toBe(
@@ -227,7 +226,7 @@ describe("summarizeAssembledTrace", () => {
name: "Issues",
versions: [{ ...facet.versions[0], facetId: "facet-2", version: 1 }],
};
- state.facets.mockResolvedValue([facet, issues]);
+ state.facets.mockReturnValue([facet, issues]);
state.summarize.mockResolvedValue({
output: {
intent_1: { summary: "Export monthly sales.", status: "applicable" },
@@ -242,6 +241,8 @@ describe("summarizeAssembledTrace", () => {
environment: "default",
traceName: "agent-turn",
transcript,
+ models,
+ facets: state.facets(),
});
expect(state.summarize).toHaveBeenCalledOnce();
const [intent, issue] = state.write.mock.calls[0][0];
@@ -268,6 +269,8 @@ describe("summarizeAssembledTrace", () => {
environment: "default",
traceName: "agent-turn",
transcript,
+ models,
+ facets: state.facets(),
});
const [written] = state.write.mock.calls[0][0];
expect(written).toMatchObject({ state: "not_applicable" });
@@ -282,7 +285,7 @@ describe("summarizeAssembledTrace", () => {
name: "Issues",
versions: [{ ...facet.versions[0], facetId: "facet-2", version: 1 }],
};
- state.facets.mockResolvedValue([facet, issues]);
+ state.facets.mockReturnValue([facet, issues]);
state.summarize.mockResolvedValue({
output: {
intent_1: { summary: "", status: "applicable" },
@@ -298,6 +301,8 @@ describe("summarizeAssembledTrace", () => {
environment: "default",
traceName: "agent-turn",
transcript,
+ models,
+ facets: state.facets(),
}),
).rejects.toThrow(
"Applicable facet summary must contain a concise summary.",
@@ -338,7 +343,7 @@ describe("summarizeAssembledTrace", () => {
isBuiltIn: false,
versions: [{ ...facet.versions[0], facetId: "facet-custom", version: 1 }],
};
- state.facets.mockResolvedValue([custom, facet]);
+ state.facets.mockReturnValue([custom, facet]);
state.summarize.mockResolvedValue({
output: {
intent_1: { summary: "Export monthly sales.", status: "applicable" },
@@ -353,15 +358,17 @@ describe("summarizeAssembledTrace", () => {
environment: "default",
traceName: "agent-turn",
transcript,
+ models,
+ facets: state.facets(),
});
- expect(state.summarize.mock.calls[0][0]).toEqual([
+ expect(state.summarize.mock.calls[0][1]).toEqual([
{ key: "intent_1", facet: facet.versions[0], builtIn: true },
{ key: "language_2", facet: custom.versions[0], builtIn: false },
]);
});
it("persists successful facets and attributes an embedding failure once", async () => {
- state.facets.mockResolvedValue([
+ state.facets.mockReturnValue([
facet,
{
...facet,
@@ -387,6 +394,8 @@ describe("summarizeAssembledTrace", () => {
environment: "default",
traceName: "agent-turn",
transcript,
+ models,
+ facets: state.facets(),
}),
).rejects.toBe(error);
expect(state.summarize).toHaveBeenCalledOnce();
@@ -427,6 +436,8 @@ describe("summarizeAssembledTrace", () => {
environment: "default",
traceName: "agent-turn",
transcript,
+ models,
+ facets: state.facets(),
}),
).rejects.toBe(error);
expect(state.setAttribute).toHaveBeenCalledWith(
@@ -462,6 +473,8 @@ describe("summarizeAssembledTrace", () => {
environment: "default",
traceName: "agent-turn",
transcript,
+ models,
+ facets: state.facets(),
}),
).rejects.toMatchObject({ reason: "invalid_output" });
expect(state.embed).not.toHaveBeenCalled();
diff --git a/worker/src/features/topics/summarizeAssembledTrace.ts b/worker/src/features/topics/summarizeAssembledTrace.ts
index 96f8e65dfa..ca63f351c6 100644
--- a/worker/src/features/topics/summarizeAssembledTrace.ts
+++ b/worker/src/features/topics/summarizeAssembledTrace.ts
@@ -1,13 +1,12 @@
import {
- ensureDefaultTopicFacets,
- isTopicsProjectEnabled,
listTopicSummaries,
TOPICS_TRANSCRIPT_VERSION,
writeTopicSummaries,
+ type TopicsModels,
} from "@langfuse/shared/topics/server";
import {
- topicEmbeddingConfigSchema,
topicProcessingConfigSchema,
+ type TopicFacet,
type TopicFacetVersion,
type TopicSummary,
} from "@langfuse/shared/topics";
@@ -25,7 +24,6 @@ import {
topicSummaryOutputError,
type TopicModelUsage,
} from "./summaryResult";
-import { getTopicsModelConfig } from "@langfuse/shared/topics/server";
type FacetOutput = {
summary: string;
@@ -45,6 +43,8 @@ type AssembledTraceInput = {
environment: string;
traceName: string;
transcript: Transcript | null;
+ models: TopicsModels;
+ facets: TopicFacet[];
};
/**
@@ -54,8 +54,7 @@ type AssembledTraceInput = {
*/
export async function summarizeAssembledTrace(
input: AssembledTraceInput,
-): Promise<"disabled" | "unchanged" | "summarized"> {
- if (!isTopicsProjectEnabled(input.projectId)) return "disabled";
+): Promise<"unchanged" | "summarized"> {
return instrumentAsync(
{ name: "topics-trace-summary", traceScope: "topics" },
async (span) => {
@@ -82,19 +81,10 @@ async function summarizeEnabledTrace(
input: AssembledTraceInput,
metrics: TopicMetrics,
): Promise<"unchanged" | "summarized"> {
- const models = getTopicsModelConfig();
- if (!models.summaryModel || !models.embeddingModel)
- throw new TopicsProviderUnavailable(
- "Configure LANGFUSE_TOPICS_SUMMARY_MODEL and LANGFUSE_TOPICS_EMBEDDING_MODEL before processing Topics traces.",
- "authentication",
- );
- const { embeddingModel } = models;
- const facets = await ensureDefaultTopicFacets(input.projectId);
+ const { models, facets } = input;
+ const embeddingModel = models.embedding.model;
const config = topicProcessingConfigSchema.parse({
- summaryModel: models.summaryModel,
- });
- const embeddingConfig = topicEmbeddingConfigSchema.parse({
- embeddingModel,
+ summaryModel: models.summary.model,
});
const timestamp = Date.parse(input.traceTimestamp);
const timeRange = {
@@ -147,6 +137,7 @@ async function summarizeEnabledTrace(
const result = await metrics
.measure("summary", async () => {
generated = await summarizeTopicTraceFacets(
+ models,
keyed,
prepared.text,
config,
@@ -191,8 +182,7 @@ async function summarizeEnabledTrace(
outputs[key],
// One call serves every facet of the trace; record its usage once.
rows.length ? NO_USAGE : usage,
- embeddingConfig.embeddingDimensions,
- embeddingModel,
+ models,
metrics,
),
);
@@ -248,8 +238,7 @@ async function finalizeSummary(
base: TopicSummary,
output: FacetOutput,
usage: TopicModelUsage,
- dimensions: number,
- embeddingModel: string,
+ models: TopicsModels,
metrics: TopicMetrics,
): Promise<TopicSummary> {
if (output.status !== "applicable")
@@ -263,7 +252,7 @@ async function finalizeSummary(
};
const summary = output.summary.trim();
const embedded = await metrics.measure("embedding", () =>
- embedTopicSummary(summary, dimensions, embeddingModel),
+ embedTopicSummary(models.embedding, summary, models.embedding.dimensions),
);
return {
...base,
diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.ts
index 2e9ca3720c..014a143efd 100644
--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts
+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts
@@ -30,6 +30,14 @@ import { tokenCountAsync } from "../../features/tokenisation/async-usage";
import { tokenCount } from "../../features/tokenisation/usage";
import { TopicsProviderUnavailable } from "../../features/topics/provider-error";
import { summarizeAssembledTrace } from "../../features/topics/summarizeAssembledTrace";
+import {
+ getEnabledTopicsModels,
+ listTopicFacets,
+ listTopicRules,
+ pauseTopicsModels,
+ type TopicsModels,
+} from "@langfuse/shared/topics/server";
+import type { TopicFacet, TopicRule } from "@langfuse/shared/topics";
import { recordTraceBatchTranscript } from "../../features/traceBatching/traceBatchTranscript";
import * as traceBatchTranscript from "../../features/traceBatching/traceBatchTranscript";
import {
@@ -47,7 +55,18 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
}));
vi.mock("../../features/topics/summarizeAssembledTrace", () => ({
- summarizeAssembledTrace: vi.fn(async () => "disabled"),
+ summarizeAssembledTrace: vi.fn(async () => "unchanged"),
+}));
+
+vi.mock("@langfuse/shared/topics/server", () => ({
+ isTopicsProjectEnabled: vi.fn(() => true),
+ getEnabledTopicsModels: vi.fn(
+ async (projectIds: string[]) =>
+ new Map(projectIds.map((projectId) => [projectId, { projectId }])),
+ ),
+ listTopicFacets: vi.fn(async () => []),
+ listTopicRules: vi.fn(async () => []),
+ pauseTopicsModels: vi.fn(),
}));
// Exercise real tokenization without starting the compiled worker-thread pool.
@@ -664,7 +683,7 @@ describe("trace batch queue", () => {
resolveSecond();
await processing;
env.LANGFUSE_TRACE_BATCH_MAX_PENDING_SUMMARY_BYTES = originalCap;
- vi.mocked(summarizeAssembledTrace).mockResolvedValue("disabled");
+ vi.mocked(summarizeAssembledTrace).mockResolvedValue("unchanged");
}
});
it("finishes the batch past a failed trace, reports outcomes and does not re-enqueue the failure", async () => {
@@ -736,7 +755,167 @@ describe("trace batch queue", () => {
);
expect(add).not.toHaveBeenCalled();
vi.mocked(summarizeAssembledTrace).mockImplementation(
- async () => "disabled",
+ async () => "unchanged",
+ );
+ });
+
+ it("summarizes only the saved rule's facets and skips traces outside its sampling", async () => {
+ vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {
+ for (const projectId of ["picked", "sampled-out"])
+ yield {
+ project_id: projectId,
+ trace_id: "trace",
+ environment: "default",
+ span_id: "span",
+ parent_span_id: null,
+ start_time: "2026-09-11 00:00:00.000000",
+ event_ts: "2026-09-11 00:00:00.000000",
+ type: "GENERATION",
+ name: "generation",
+ input: JSON.stringify([{ role: "user", content: projectId }]),
+ output: JSON.stringify({ role: "assistant", content: "answer" }),
+ metadata: {},
+ level: "DEFAULT",
+ status_message: null,
+ tool_definitions: {},
+ tool_calls: [],
+ tool_call_names: [],
+ };
+ });
+ const facet = (id: string) => ({ id }) as TopicFacet;
+ vi.mocked(listTopicFacets).mockResolvedValue([
+ facet("intent"),
+ facet("issues"),
+ ]);
+ vi.mocked(listTopicRules).mockImplementation(async (projectId) => [
+ {
+ facetIds: ["issues"],
+ sampling: projectId === "picked" ? 1 : 0,
+ } as TopicRule,
+ ]);
+ const job = {
+ data: {
+ id: "rule",
+ name: QueueJobs.TraceBatch,
+ timestamp: new Date(),
+ payload: {
+ traces: ["picked", "sampled-out"].map((projectId) => ({
+ projectId,
+ traceId: "trace",
+ minStart: 0,
+ maxStart: 1,
+ revision: "r",
+ })),
+ },
+ },
+ } as Job<TQueueJobTypes[QueueName.TraceBatch]>;
+
+ await traceBatchQueueProcessor(job, undefined);
+
+ expect(
+ vi
+ .mocked(summarizeAssembledTrace)
+ .mock.calls.map(([input]) => [
+ input.projectId,
+ input.facets.map(({ id }) => id),
+ ]),
+ ).toEqual([["picked", ["issues"]]]);
+ vi.mocked(listTopicFacets).mockResolvedValue([]);
+ vi.mocked(listTopicRules).mockResolvedValue([]);
+ });
+
+ it("skips projects without enabled Topics models and pauses a project whose connection is rejected", async () => {
+ const rows = [
+ ["off", "trace-1"],
+ ["on", "trace-1"],
+ ["on", "trace-2"],
+ ["on", "trace-3"],
+ ] as const;
+ vi.mocked(getTraceBatchEventStream).mockImplementation(async function* () {
+ for (const [projectId, traceId] of rows)
+ yield {
+ project_id: projectId,
+ trace_id: traceId,
+ environment: "default",
+ span_id: "span",
+ parent_span_id: null,
+ start_time: "2026-09-11 00:00:00.000000",
+ event_ts: "2026-09-11 00:00:00.000000",
+ type: "GENERATION",
+ name: "generation",
+ input: JSON.stringify([{ role: "user", content: traceId }]),
+ output: JSON.stringify({ role: "assistant", content: "answer" }),
+ metadata: {},
+ level: "DEFAULT",
+ status_message: null,
+ tool_definitions: {},
+ tool_calls: [],
+ tool_call_names: [],
+ };
+ });
+ const models = { projectId: "on" } as TopicsModels;
+ vi.mocked(getEnabledTopicsModels).mockResolvedValueOnce(
+ new Map([["on", models]]),
+ );
+ vi.mocked(summarizeAssembledTrace).mockImplementation(
+ async ({ traceId }) => {
+ if (traceId === "trace-2")
+ throw new TopicsProviderUnavailable(
+ "Connection rejected (HTTP 401).",
+ "authentication",
+ "summary",
+ );
+ return "summarized";
+ },
+ );
+ const job = {
+ data: {
+ id: "kill-switch",
+ name: QueueJobs.TraceBatch,
+ timestamp: new Date(),
+ payload: {
+ traces: rows.map(([projectId, traceId]) => ({
+ projectId,
+ traceId,
+ minStart: 0,
+ maxStart: 1,
+ revision: "r",
+ })),
+ },
+ },
+ } as Job<TQueueJobTypes[QueueName.TraceBatch]>;
+
+ await traceBatchQueueProcessor(job, undefined);
+
+ // One Postgres read for the whole batch, deduplicated by project.
+ expect(getEnabledTopicsModels).toHaveBeenCalledExactlyOnceWith([
+ "off",
+ "on",
+ ]);
+ expect(listTopicFacets).toHaveBeenCalledExactlyOnceWith("on");
+ expect(
+ vi
+ .mocked(summarizeAssembledTrace)
+ .mock.calls.map(([input]) => [
+ input.projectId,
+ input.traceId,
+ input.models,
+ ]),
+ ).toEqual([
+ ["on", "trace-1", models],
+ ["on", "trace-2", models],
+ ]);
+ expect(pauseTopicsModels).toHaveBeenCalledExactlyOnceWith("on", {
+ blockReason: "LLM_CONNECTION_AUTH_INVALID",
+ blockMessage: expect.stringContaining("Connection rejected (HTTP 401)."),
+ });
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.topics.trace_outcomes",
+ 2,
+ { outcome: "disabled" },
+ );
+ vi.mocked(summarizeAssembledTrace).mockImplementation(
+ async () => "unchanged",
);
});
diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.ts
index 4715ef3c16..6c2423ad95 100644
--- a/worker/src/queues/traceBatchQueue.ts
+++ b/worker/src/queues/traceBatchQueue.ts
@@ -13,8 +13,24 @@ import {
type QueueName,
type TQueueJobTypes,
} from "@langfuse/shared/src/server";
+import {
+ getEnabledTopicsModels,
+ isTopicsProjectEnabled,
+ listTopicFacets,
+ listTopicRules,
+ type TopicsModels,
+} from "@langfuse/shared/topics/server";
+import type { TopicFacet } from "@langfuse/shared/topics";
import { env } from "../env";
-import { TopicsProviderUnavailable } from "../features/topics/provider-error";
+import {
+ getDeterministicSamplingValue,
+ shouldSampleEvaluation,
+} from "../features/evaluation/deterministicSampling";
+import {
+ isRejectedConnection,
+ pauseForRejectedConnection,
+ TopicsProviderUnavailable,
+} from "../features/topics/provider-error";
import { summarizeAssembledTrace } from "../features/topics/summarizeAssembledTrace";
import { recordTraceBatchTranscript } from "../features/traceBatching/traceBatchTranscript";
@@ -22,8 +38,65 @@ type TraceOutcome =
| { outcome: "disabled" | "unchanged" | "summarized" }
| { outcome: "failed"; reason: string };
+/**
+ * Topics settings of one batch, loaded with one query while the read starts.
+ * A project paused mid-batch is skipped for the rest of the batch.
+ */
+class TopicsBatchProjects {
+ private readonly models: Promise<Map<string, TopicsModels>>;
+ private readonly settings = new Map<
+ string,
+ Promise<{ facets: TopicFacet[]; sampling: number }>
+ >();
+ private readonly paused = new Set<string>();
+
+ constructor(projectIds: Iterable<string>) {
+ const allowed = [...new Set(projectIds)].filter(isTopicsProjectEnabled);
+ this.models = getEnabledTopicsModels(allowed);
+ // A failed load fails each trace that reads it, not the whole batch.
+ this.models.catch(() => {});
+ }
+
+ async get(projectId: string) {
+ if (this.paused.has(projectId)) return null;
+ const models = (await this.models).get(projectId);
+ if (!models) return null;
+ let settings = this.settings.get(projectId);
+ if (!settings) {
+ settings = loadProjectRule(projectId);
+ this.settings.set(projectId, settings);
+ // A failed load fails this trace only; the next trace loads again.
+ settings.catch(() => this.settings.delete(projectId));
+ }
+ return { models, ...(await settings) };
+ }
+
+ async pause(projectId: string, error: TopicsProviderUnavailable) {
+ if (this.paused.has(projectId)) return;
+ // The project is skipped for this batch even if recording the pause fails.
+ this.paused.add(projectId);
+ await pauseForRejectedConnection(projectId, error);
+ }
+}
+
+/** The saved rule's facets and sampling; without a rule, every facet runs. */
+async function loadProjectRule(projectId: string) {
+ const [facets, rules] = await Promise.all([
+ listTopicFacets(projectId),
+ listTopicRules(projectId),
+ ]);
+ const rule = rules[0];
+ if (!rule) return { facets, sampling: 1 };
+ const selected = new Set(rule.facetIds);
+ return {
+ facets: facets.filter((facet) => selected.has(facet.id)),
+ sampling: rule.sampling,
+ };
+}
+
async function summarizeTraceBatch(
observations: Observation[],
+ topics: TopicsBatchProjects,
): Promise<TraceOutcome> {
const first = observations[0];
const traceId = first?.traceId;
@@ -43,6 +116,15 @@ async function summarizeTraceBatch(
let outcome: TraceOutcome = { outcome: "disabled" };
await recordTraceBatchTranscript(observations, async (transcript) => {
try {
+ const project = await topics.get(first.projectId);
+ if (
+ !project ||
+ !shouldSampleEvaluation({
+ samplingValue: getDeterministicSamplingValue(traceId),
+ samplingRate: project.sampling,
+ })
+ )
+ return;
outcome = {
outcome: await summarizeAssembledTrace({
projectId: first.projectId,
@@ -51,12 +133,16 @@ async function summarizeTraceBatch(
environment: first.environment,
traceName: first.name ?? "",
transcript,
+ models: project.models,
+ facets: project.facets,
}),
};
} catch (error) {
// One trace's failure must not fail or re-read the shared batch.
const reason =
error instanceof TopicsProviderUnavailable ? error.reason : "other";
+ if (isRejectedConnection(error))
+ await topics.pause(first.projectId, error);
outcome = { outcome: "failed", reason };
logger.warn("Topics summary failed for trace", {
projectId: first.projectId,
@@ -154,6 +240,9 @@ export const traceBatchQueueProcessor: Processor<
}
const batch = event.payload;
const topicsOutcomes: TraceOutcome[] = [];
+ const topics = new TopicsBatchProjects(
+ batch.traces.map((trace) => trace.projectId),
+ );
const queryOptions = {
maxThreads: env.LANGFUSE_TRACE_BATCH_MAX_THREADS,
maxBlockSize: env.LANGFUSE_TRACE_BATCH_MAX_BLOCK_SIZE,
@@ -226,7 +315,9 @@ export const traceBatchQueueProcessor: Processor<
performance.now() - enqueuedAt,
);
if (!summaryFailed)
- topicsOutcomes.push(await summarizeTraceBatch(observations));
+ topicsOutcomes.push(
+ await summarizeTraceBatch(observations, topics),
+ );
})
.catch((error: unknown) => {
if (!summaryFailed) summaryError = error;