Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(topics): guided first run with combined backfill and progress (#19088)

Lysander Kieselcde1f5252cb5
diff --git a/packages/shared/src/server/repositories/index.ts b/packages/shared/src/server/repositories/index.ts
index 7dc92c8788..706f905cfd 100644
--- a/packages/shared/src/server/repositories/index.ts
+++ b/packages/shared/src/server/repositories/index.ts
@@ -30,3 +30,4 @@ export * from "./dataset-runs";
export * from "./score-configs";
export * from "./auditLogs";
export * from "./billable-events";
+export * from "./topic-setup-events";
diff --git a/packages/shared/src/server/repositories/topic-setup-events.ts b/packages/shared/src/server/repositories/topic-setup-events.ts
new file mode 100644
index 0000000000..7a6e3912a5
--- /dev/null
+++ b/packages/shared/src/server/repositories/topic-setup-events.ts
@@ -0,0 +1,93 @@
+import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import {
+ INTERNAL_EVAL_ENVIRONMENT_PREFIX,
+ PUBLIC_LLM_JUDGE_ENVIRONMENT,
+} from "../llm/isInternalEvalEnvironment";
+import { queryClickhouse } from "./clickhouse";
+
+// Finished traces: nothing ingested within the project's idle wait (event_ts, as
+// trace-batch uses). Grouped by trace_id, so traces without a root observation count.
+const topicSetupTracesSql = (days: "1" | "7", sampled: boolean) => `
+ SELECT trace_id, max(event_ts) AS last_ts
+ FROM events_core
+ WHERE project_id = {projectId: String}
+ AND start_time >= {now: DateTime64(3)} - INTERVAL ${days} DAY
+ AND trace_id != ''
+ AND is_deleted = 0
+ AND NOT startsWith(environment, {internalEnvironmentPrefix: String})
+ AND environment != {publicLlmJudgeEnvironment: String}
+ ${
+ sampled
+ ? // Mirrors worker deterministicSampling: sha256(domain + traceId), top 53 bits.
+ "AND bitShiftRight(reinterpretAsUInt64(reverse(substring(SHA256(concat('langfuse:evaluation-sampling:v1', char(0), trace_id)), 1, 8))), 11) < {sampling: Float64} * 9007199254740992"
+ : ""
+ }
+ GROUP BY trace_id
+ HAVING last_ts < {now: DateTime64(3)} - toIntervalMillisecond({idleMs: UInt32})`;
+
+const topicSetupParams = (projectId: string, idleMs: number, now: Date) => ({
+ projectId,
+ idleMs,
+ now: convertDateToClickhouseDateTime(now),
+ internalEnvironmentPrefix: INTERNAL_EVAL_ENVIRONMENT_PREFIX,
+ publicLlmJudgeEnvironment: PUBLIC_LLM_JUDGE_ENVIRONMENT,
+});
+
+// Setup reads must never hang the wizard; a timeout surfaces as a recoverable error.
+const TOPIC_SETUP_COUNT_TIMEOUT_SECONDS = 10;
+const TOPIC_SETUP_SELECTION_TIMEOUT_SECONDS = 30;
+
+/** Unsampled finished-trace counts over at most 7 days for the Topics setup estimate. */
+export async function getTopicSetupTraceCounts(input: {
+ projectId: string;
+ idleMs: number;
+ now: Date;
+}) {
+ const rows = await queryClickhouse<{
+ finished1d: string;
+ finished7d: string;
+ }>({
+ query: `
+ SELECT
+ countIf(last_ts >= {now: DateTime64(3)} - INTERVAL 1 DAY) AS finished1d,
+ count() AS finished7d
+ FROM (${topicSetupTracesSql("7", false)})`,
+ params: topicSetupParams(input.projectId, input.idleMs, input.now),
+ tags: { route: "topics-setup-counts", projectId: input.projectId },
+ preferredClickhouseService: "EventsReadOnly",
+ clickhouseSettings: {
+ max_execution_time: TOPIC_SETUP_COUNT_TIMEOUT_SECONDS,
+ },
+ });
+ return {
+ finished1d: Number(rows[0]?.finished1d ?? 0),
+ finished7d: Number(rows[0]?.finished7d ?? 0),
+ };
+}
+
+/** The most recent finished, sampled traces in one window for the Topics setup backfill. */
+export async function getTopicSetupTraceIds(input: {
+ projectId: string;
+ idleMs: number;
+ now: Date;
+ days: 1 | 7;
+ sampling: number;
+ limit: number;
+}) {
+ const rows = await queryClickhouse<{ trace_id: string }>({
+ query: `${topicSetupTracesSql(input.days === 1 ? "1" : "7", input.sampling < 1)}
+ ORDER BY last_ts DESC, trace_id
+ LIMIT {limit: UInt32}`,
+ params: {
+ ...topicSetupParams(input.projectId, input.idleMs, input.now),
+ sampling: input.sampling,
+ limit: input.limit,
+ },
+ tags: { route: "topics-setup-traces", projectId: input.projectId },
+ preferredClickhouseService: "EventsReadOnly",
+ clickhouseSettings: {
+ max_execution_time: TOPIC_SETUP_SELECTION_TIMEOUT_SECONDS,
+ },
+ });
+ return rows.map((row) => row.trace_id);
+}
diff --git a/packages/shared/src/server/topics/execution-store.ts b/packages/shared/src/server/topics/execution-store.ts
index 7a26102fff..6bbacf98e4 100644
--- a/packages/shared/src/server/topics/execution-store.ts
+++ b/packages/shared/src/server/topics/execution-store.ts
@@ -81,6 +81,24 @@ function checkRequest(
);
}
+// JSON storage turns time ranges into strings; restore them as dates.
+function storedInput(
+ settings: BatchMetadata["inputSettings"],
+): TopicExecutionSummary["input"] {
+ if (settings.operation === "update")
+ return {
+ ...settings,
+ timeRange: topicTimeRangeSchema.parse(settings.timeRange),
+ };
+ if (!settings.thenUpdate) return settings;
+ return {
+ ...settings,
+ thenUpdate: {
+ timeRange: topicTimeRangeSchema.parse(settings.thenUpdate.timeRange),
+ },
+ };
+}
+
function summaryResult(row: BatchRow): TopicExecutionSummary {
const state = batchMetadata(row);
const status = (
@@ -95,15 +113,7 @@ function summaryResult(row: BatchRow): TopicExecutionSummary {
createdAt: row.createdAt.toISOString(),
updatedAt: row.updatedAt.toISOString(),
error: row.log,
- input:
- state.inputSettings.operation === "update"
- ? {
- ...state.inputSettings,
- timeRange: topicTimeRangeSchema.parse(
- state.inputSettings.timeRange,
- ),
- }
- : state.inputSettings,
+ input: storedInput(state.inputSettings),
facets: state.inputSettings.facets.map(({ facetId, version }) => {
const facet = state.facets.find(
(candidate) =>
@@ -116,6 +126,15 @@ function summaryResult(row: BatchRow): TopicExecutionSummary {
};
}
+/** The user who started an execution, so follow-up runs stay in their History. */
+export async function readTopicExecutionUserId(
+ projectId: string,
+ executionId: string,
+): Promise<string | null> {
+ const row = await storedExecution(projectId, executionId);
+ return row?.userId ?? null;
+}
+
/** Postgres owns compact execution settings and progress, never per-trace payloads. */
export async function createTopicExecution(
rawInput: TopicExecutionInput,
diff --git a/packages/shared/src/server/topics/queue.test.ts b/packages/shared/src/server/topics/queue.test.ts
index fe656109bf..7c6c6ca355 100644
--- a/packages/shared/src/server/topics/queue.test.ts
+++ b/packages/shared/src/server/topics/queue.test.ts
@@ -30,10 +30,14 @@ const mocks = vi.hoisted(() => ({
hget: vi.fn(),
exists: vi.fn(),
get: vi.fn(),
+ create: vi.fn(),
+ userId: vi.fn(),
}));
vi.mock("./execution-store", () => ({
readTopicExecutionSummary: mocks.read,
writeTopicExecution: mocks.write,
+ createTopicExecution: mocks.create,
+ readTopicExecutionUserId: mocks.userId,
}));
vi.mock("../redis/redis", () => ({
createBullMQQueueOptionsWithRedis: () => null,
@@ -446,6 +450,7 @@ describe("Topics execution queue state", () => {
expect(mocks.write.mock.calls.map(([, version]) => version)).toEqual([
1, 2, 3, 4, 5,
]);
+ expect(mocks.create).not.toHaveBeenCalled();
for (const key of keys) expect(await client.ttl(key)).toBeGreaterThan(0);
await client.del(...keys);
mocks.write.mockClear();
@@ -460,6 +465,73 @@ describe("Topics execution queue state", () => {
});
});
+describe("Topics setup follow-up", () => {
+ it("generates topics once all batches finish, with standard clustering", async () => {
+ const client = new Redis({
+ lazyConnect: true,
+ maxRetriesPerRequest: 0,
+ host: env.REDIS_HOST ?? "localhost",
+ port: env.REDIS_PORT ?? 6379,
+ password: env.REDIS_AUTH ?? undefined,
+ username: env.REDIS_USERNAME ?? undefined,
+ });
+ const keys = new Set<string>();
+ mocks.exists.mockImplementation((...keys: string[]) =>
+ client.exists(...keys),
+ );
+ mocks.eval.mockImplementation(
+ async (script: string, count: number, ...args: (string | number)[]) => {
+ args.slice(0, count).forEach((key) => keys.add(String(key)));
+ return client.eval(script, count, ...args);
+ },
+ );
+ const timeRange = {
+ from: new Date("2026-10-08T00:00:00.000Z"),
+ to: new Date("2026-10-22T00:00:00.000Z"),
+ };
+ const parent = { ...execution("process", 60), id: randomUUID() };
+ parent.input = topicExecutionInputSchema.parse({
+ ...parent.input,
+ thenUpdate: { timeRange },
+ });
+ mocks.read.mockImplementation(async () => structuredClone(parent));
+ mocks.write.mockImplementation(async (state: TopicExecution) => {
+ Object.assign(parent, structuredClone(state));
+ });
+ mocks.userId.mockResolvedValue("user-1");
+ mocks.create.mockResolvedValue({ id: "update", status: "completed" });
+ const local = structuredClone(parent);
+ local.status = "completed";
+ local.facets[0].counts.complete = 60;
+ local.facets[0].outcome = "awaiting_topics";
+ const batch: TopicProcessBatchState = {
+ execution: local,
+ summaries: [],
+ failedTraceIds: [],
+ summarized: true,
+ };
+ try {
+ await recordTopicProcessBatchProgress("0", batch);
+ await recordTopicProcessBatchProgress("0", batch);
+ expect(mocks.create).toHaveBeenCalledTimes(2);
+ expect(mocks.create.mock.calls[0]).toEqual(mocks.create.mock.calls[1]);
+ expect(mocks.create.mock.calls[0]![0].exploratory).toBe(false);
+ expect(mocks.create.mock.calls[0]).toEqual([
+ expect.objectContaining({
+ operation: "update",
+ requestId: `${parent.id}-topics`,
+ timeRange,
+ }),
+ undefined,
+ "user-1",
+ ]);
+ } finally {
+ if (keys.size) await client.del(...keys);
+ client.disconnect();
+ }
+ });
+});
+
describe("Topics embedding queue handoff", () => {
const batch = {
projectId: "project-a",
diff --git a/packages/shared/src/server/topics/queue.ts b/packages/shared/src/server/topics/queue.ts
index 7bb8626edc..2af1a1b1d7 100644
--- a/packages/shared/src/server/topics/queue.ts
+++ b/packages/shared/src/server/topics/queue.ts
@@ -5,7 +5,9 @@ import { createBullMQQueueOptionsWithRedis, redis } from "../redis/redis";
import { logger } from "../logger";
import { isTopicsEnabled } from "./config";
import {
+ createTopicExecution,
readTopicExecutionSummary,
+ readTopicExecutionUserId,
writeTopicExecution,
} from "./execution-store";
import type {
@@ -344,6 +346,33 @@ export async function recordTopicProcessBatchProgress(
execution.phase = done ? execution.status : local.phase;
execution.error = failed ? (local.error ?? execution.error) : null;
await writeTopicExecution(execution, totals.get("version"));
+ if (done && !failed) await startFollowUpTopicUpdate(execution);
+}
+
+/** Idempotent: the update's request ID derives from the process execution. */
+async function startFollowUpTopicUpdate(execution: TopicExecutionSummary) {
+ const { input } = execution;
+ if (input.operation !== "process" || !input.thenUpdate) return;
+ const userId = await readTopicExecutionUserId(
+ execution.projectId,
+ execution.id,
+ );
+ if (!userId) return;
+ const update = await createTopicExecution(
+ {
+ operation: "update",
+ projectId: execution.projectId,
+ requestId: `${execution.id}-topics`,
+ facets: input.facets,
+ embeddingConfig: input.embeddingConfig,
+ timeRange: input.thenUpdate.timeRange,
+ exploratory: false,
+ },
+ undefined,
+ userId,
+ );
+ if (update.status === "queued")
+ await enqueueTopicExecution(execution.projectId, update.id);
}
/** Error details have the queue's retention; counts remain in Postgres. */
diff --git a/packages/shared/src/topics/index.ts b/packages/shared/src/topics/index.ts
index 26bbf608e7..d5f4b87ab9 100644
--- a/packages/shared/src/topics/index.ts
+++ b/packages/shared/src/topics/index.ts
@@ -196,6 +196,15 @@ export const topicExecutionInputSchema = z.discriminatedUnion("operation", [
processingConfig: topicProcessingConfigSchema.default(() =>
topicProcessingConfigSchema.parse({}),
),
+ // Generate topics over this range once every batch has finished.
+ thenUpdate: z
+ .object({
+ timeRange: topicTimeRangeSchema.refine(
+ ({ from, to }) => to.getTime() - from.getTime() <= 93 * 86_400_000,
+ "Select a time range of at most 93 days.",
+ ),
+ })
+ .optional(),
})
.strict(),
z
diff --git a/web/AGENTS.md b/web/AGENTS.md
index 5454540ede..945e9ec84c 100644
--- a/web/AGENTS.md
+++ b/web/AGENTS.md
@@ -32,7 +32,10 @@
selection share `topics/server`'s bounded trace selector with worker
backfills; preserve sampling and the frozen submission range. Models come
from the project's LLM connections (`TopicModelSettings.tsx`,
- `topics.modelSettings` / `saveSetup`). Configure Topics loads and saves idle time
+ `topics.modelSettings` / `saveSetup`). The first-run wizard
+ (`TopicSetupFlowController.tsx`) saves models, facets, and idle time as you
+ leave each step. Start analyzing turns automatic processing on at the chosen
+ sample rate. The trace filter applies only to the optional past-trace batch. Configure Topics loads and saves idle time
through the project's tracing readiness setting; triggers stamp the configured
summary and embedding models into the frozen request.
- Reusable UI components: `src/components/*`
diff --git a/web/src/__tests__/server/unit/topics.servertest.ts b/web/src/__tests__/server/unit/topics.servertest.ts
index e15a636df6..80a46a4c0f 100644
--- a/web/src/__tests__/server/unit/topics.servertest.ts
+++ b/web/src/__tests__/server/unit/topics.servertest.ts
@@ -186,6 +186,7 @@ const summary = {
beforeEach(() => {
vi.resetAllMocks();
mocks.isTopicsProjectEnabled.mockReturnValue(true);
+ mocks.listTopicRules.mockResolvedValue([]);
mocks.queryClickhouse.mockResolvedValue([]);
mocks.getTopicsModels.mockResolvedValue({
summary: { model: "us.openai.gpt-5.6-luna" },
@@ -1130,6 +1131,29 @@ describe("Topics current results", () => {
);
});
+ it("lists only the facets enabled on the project's rule", async () => {
+ mocks.listTopicFacets.mockResolvedValue([
+ { id: facetId, name: "Intent", versions: [{ version: facetVersion }] },
+ {
+ id: "other-facet",
+ name: "Sentiment",
+ versions: [{ version: facetVersion }],
+ },
+ ]);
+ mocks.listTopicRules.mockResolvedValue([{ facetIds: [facetId] }] as Awaited<
+ ReturnType<typeof topicsServer.listTopicRules>
+ >);
+ mocks.getLatestFacetSummaries.mockResolvedValue([]);
+ mocks.readLatestTopicAssignments.mockResolvedValue([]);
+ mocks.getPublishedTopicRun.mockResolvedValue(null);
+
+ const result = await caller("VIEWER").currentResults({
+ projectId,
+ timeRange,
+ });
+ expect(result.facets.map((facet) => facet.facetId)).toEqual([facetId]);
+ });
+
it("retains published map descriptions when reprocessed traces await assignment", async () => {
mocks.listTopicFacets.mockResolvedValue([
{ id: facetId, name: "Intent", versions: [{ version: facetVersion }] },
diff --git a/web/src/features/posthog-analytics/usePostHogClientCapture.ts b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
index 0b094e5697..51094f5230 100644
--- a/web/src/features/posthog-analytics/usePostHogClientCapture.ts
+++ b/web/src/features/posthog-analytics/usePostHogClientCapture.ts
@@ -447,7 +447,12 @@ const events = {
"message_sent",
"community_hours_click",
], // also used on landing page for consistency
- topics: ["results_view_select"],
+ topics: [
+ "results_view_select",
+ "setup_opened",
+ "setup_step_saved",
+ "setup_started",
+ ],
in_app_agent: [
"activity_opened",
"entry_point_click",
diff --git a/web/src/features/topics/TopicAutomaticTraceScope.tsx b/web/src/features/topics/TopicAutomaticTraceScope.tsx
index e7347a2736..68015810de 100644
--- a/web/src/features/topics/TopicAutomaticTraceScope.tsx
+++ b/web/src/features/topics/TopicAutomaticTraceScope.tsx
@@ -1,14 +1,6 @@
import { useMemo, useState } from "react";
import { format } from "date-fns";
-import { Badge } from "@/src/components/design-system/Badge/Badge";
-import { ObservationFilterBuilder } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder";
-import { FilterModeToggle } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/FilterModeToggle";
-import { RULE_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/ruleSearchRegistry";
-import {
- EventsSearchBarRow,
- toObservedOptions,
- useEventsSearchBar,
-} from "@/src/features/search-bar";
+import { toObservedOptions } from "@/src/features/search-bar";
import { SectionHeader } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SectionHeader/SectionHeader";
import type { EvaluatorFilterExperience } from "@/src/features/evals/v2/types/evaluatorFilterExperience";
import {
@@ -22,6 +14,10 @@ import {
import { useEventsFilterOptions } from "@/src/features/events/hooks/useEventsFilterOptions";
import { api, sendAsPostOption } from "@/src/utils/api";
import { relativeTopicTimeRange } from "./time-range";
+import {
+ TopicTraceFilterSection,
+ type TopicTraceFilterSectionProps,
+} from "./TopicTraceFilterSection";
const SAMPLE_TRACES_IN_SCOPE = [
{
@@ -44,21 +40,6 @@ const SAMPLE_TRACES_IN_SCOPE = [
},
];
-const topicsFilterRegistry = {
- ...RULE_FIELD_REGISTRY,
- aiFilterPrompt: true,
- fields: RULE_FIELD_REGISTRY.fields.filter(
- (field) => field.id !== "isRootObservation",
- ),
- columns: RULE_FIELD_REGISTRY.columns.filter(
- (column) => column.id !== "isRootObservation",
- ),
-};
-
-const queryOnlyColumnIds = topicsFilterRegistry.fields
- .filter((field) => field.directFilter === false)
- .map((field) => field.filterColumn ?? field.id);
-
const BUILDER_OPTION_COLUMNS = [
"environment",
"name",
@@ -275,34 +256,13 @@ function ProjectTraceScope({
}
function TraceScopeFrame({
- projectId,
- mode: modeProp,
- filters: filtersProp,
- searchQuery: searchQueryProp,
- searchType: searchTypeProp,
- observed,
- builderColumns,
- erroredColumns,
- onRequestColumns,
countLabel,
description,
tooltip,
rows,
status,
- onModeChange,
- onFiltersChange,
- onSearchQueryChange,
- onSearchTypeChange,
-}: {
- projectId?: string;
- mode?: EvaluatorFilterExperience;
- filters?: FilterState;
- searchQuery?: string | null;
- searchType?: TracingSearchType[];
- observed?: ReturnType<typeof toObservedOptions>;
- builderColumns?: ReturnType<typeof experimentEvalFilterColsWithOptions>;
- erroredColumns?: ReadonlySet<string>;
- onRequestColumns?: (columns: readonly string[]) => void;
+ ...filterProps
+}: Omit<TopicTraceFilterSectionProps, "description"> & {
countLabel?: string | null;
description?: string;
tooltip?: string;
@@ -314,85 +274,10 @@ function TraceScopeFrame({
input: string;
}>;
status?: string | null;
- onModeChange?: (mode: EvaluatorFilterExperience) => void;
- onFiltersChange?: (filters: FilterState) => void;
- onSearchQueryChange?: (query: string | null) => void;
- onSearchTypeChange?: (searchType: TracingSearchType[]) => void;
}) {
- const [localMode, setLocalMode] =
- useState<EvaluatorFilterExperience>("query");
- const [localFilters, setLocalFilters] = useState<FilterState>([]);
- const [localSearchQuery, setLocalSearchQuery] = useState<string | null>(null);
- const [localSearchType, setLocalSearchType] = useState<TracingSearchType[]>(
- [],
- );
- const mode = modeProp ?? localMode;
- const filters = filtersProp ?? localFilters;
- const searchQuery = searchQueryProp ?? localSearchQuery;
- const searchType = searchTypeProp ?? localSearchType;
- const setMode = onModeChange ?? setLocalMode;
- const setFilters = onFiltersChange ?? setLocalFilters;
- const setSearchQuery = onSearchQueryChange ?? setLocalSearchQuery;
- const setSearchType = onSearchTypeChange ?? setLocalSearchType;
- const search = useEventsSearchBar({
- projectId,
- tableName: "topics-automatic-processing",
- enabled: true,
- filterState: filters,
- searchQuery,
- searchType,
- observed,
- setFilterState: setFilters,
- setSearchQuery,
- setSearchType,
- registry: topicsFilterRegistry,
- });
-
return (
<div className="flex flex-col gap-4">
- <section className="flex flex-col gap-2">
- <SectionHeader
- title="Filter traces"
- meta={<Badge text="Coming soon" color="blue" size="sm" />}
- description="The sampling rate applies to all incoming traces. Custom trace filters are coming soon."
- tooltip="Only traces matching this filter will be summarized. Topics always runs on whole traces."
- trailing={
- <div inert className="opacity-50">
- <FilterModeToggle
- mode={mode}
- onChange={(next) => {
- if (next === "builder" && searchQuery !== null)
- search.applyFilters(filters);
- setMode(next);
- }}
- />
- </div>
- }
- />
- <div inert aria-disabled className="opacity-50">
- {mode === "query" ? (
- <EventsSearchBarRow
- projectId={projectId}
- tableName="topics-automatic-processing"
- store={search.store}
- commit={search.commit}
- observed={observed}
- erroredColumns={erroredColumns}
- registry={topicsFilterRegistry}
- onApplyFilters={search.applyFilters}
- onRequestColumns={onRequestColumns}
- className="p-0"
- />
- ) : (
- <ObservationFilterBuilder
- columns={builderColumns ?? [...topicsFilterRegistry.columns]}
- filterState={filters}
- onChange={setFilters}
- queryOnlyColumnIds={queryOnlyColumnIds}
- />
- )}
- </div>
- </section>
+ <TopicTraceFilterSection {...filterProps} />
<section className="flex flex-col gap-2">
<SectionHeader
title="Matching traces"
diff --git a/web/src/features/topics/TopicSetupFlowController.tsx b/web/src/features/topics/TopicSetupFlowController.tsx
new file mode 100644
index 0000000000..7446a785b0
--- /dev/null
+++ b/web/src/features/topics/TopicSetupFlowController.tsx
@@ -0,0 +1,362 @@
+import { useEffect, useState, type ReactNode } from "react";
+import { type FilterState } from "@langfuse/shared";
+import { DialogController } from "@/src/components/design-system/DialogController/DialogController";
+import { env } from "@/src/env.mjs";
+import { type JudgeModel } from "@/src/features/evals/v2/judgeModel";
+import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
+import { api } from "@/src/utils/api";
+import { type ConfigureTopicsSettingsInput } from "./TopicModelSettings";
+import {
+ TopicSetupFlowDialog,
+ type TopicSetupPastCount,
+} from "./TopicSetupFlowDialog";
+import {
+ TOPIC_SETUP_BACKFILL_DAYS,
+ TOPIC_SETUP_BACKFILL_LIMIT,
+ followsTopicRecommendations,
+ initialTopicSetupDraft,
+ resolveTopicModelSlot,
+ topicConnectionCaption,
+ topicProviderGroups,
+ type TopicSetupConnection,
+ type TopicSetupDraft,
+ type TopicSetupQuestion,
+} from "./topic-setup-flow";
+
+export function TopicSetupFlowController({
+ canWrite,
+ children,
+ onSaveSampling,
+ projectId,
+ settings,
+}: {
+ projectId: string;
+ canWrite: boolean;
+ settings: ConfigureTopicsSettingsInput | null;
+ onSaveSampling: (input: {
+ enabled: boolean;
+ sampling: number;
+ }) => Promise<unknown>;
+ children: (control: { openSetup: () => void }) => ReactNode;
+}) {
+ // Opens only from Start analyzing, never on its own, so the landing shows first.
+ const [open, setOpen] = useState(false);
+ function openSetup() {
+ setOpen(true);
+ }
+ function closeSetup() {
+ setOpen(false);
+ }
+ return (
+ <>
+ {children({ openSetup })}
+ {open && settings ? (
+ <DialogController<null>
+ initialState={() => null}
+ onDismiss={closeSetup}
+ renderDialog={({ closeDialog }) => (
+ <TopicSetupSession
+ projectId={projectId}
+ canWrite={canWrite}
+ settings={settings}
+ onSaveSampling={onSaveSampling}
+ closeDialog={closeDialog}
+ />
+ )}
+ >
+ {() => null}
+ </DialogController>
+ ) : null}
+ </>
+ );
+}
+
+function TopicSetupSession({
+ canWrite,
+ closeDialog,
+ onSaveSampling,
+ projectId,
+ settings,
+}: {
+ projectId: string;
+ canWrite: boolean;
+ settings: ConfigureTopicsSettingsInput;
+ onSaveSampling: (input: {
+ enabled: boolean;
+ sampling: number;
+ }) => Promise<unknown>;
+ closeDialog: () => void;
+}) {
+ const capture = usePostHogClientCapture();
+ const utils = api.useUtils();
+ const save = api.topics.saveSetup.useMutation({
+ onSuccess: () => {
+ utils.topics.modelSettings.invalidate({ projectId });
+ utils.topics.facets.invalidate({ projectId });
+ utils.topics.rules.invalidate({ projectId });
+ utils.topics.tracingReadinessSettings.invalidate({ projectId });
+ },
+ });
+ const startSetup = api.topics.startSetup.useMutation();
+ const connections = settings.connections.map(
+ (connection): TopicSetupConnection => ({
+ id: connection.id,
+ provider: connection.provider,
+ adapter: connection.adapter,
+ customModels: connection.customModels,
+ withDefaultModels: connection.withDefaultModels,
+ }),
+ );
+ const [draft, setDraft] = useState(() =>
+ initialTopicSetupDraft({
+ stored: settings.stored,
+ connections,
+ facets: settings.facets,
+ enabledFacetIds: settings.rule ? new Set(settings.rule.facetIds) : null,
+ traceIdleMs: settings.traceIdleMs,
+ }),
+ );
+ const [error, setError] = useState<string | null>(null);
+ // One request per session, so a repeated Start resolves to the same backfill.
+ const [requestId] = useState(() => crypto.randomUUID());
+ const [pending, setPending] = useState(false);
+ useEffect(() => {
+ capture("topics:setup_opened", { source: "start_analyzing" });
+ }, [capture]);
+ // Counted once per step 2 visit; the slider only rescales the result.
+ const counts = api.topics.setupTraceCounts.useQuery(
+ { projectId },
+ { enabled: draft.step !== "questions", retry: false },
+ );
+ const pastCount = pastTraceCount(
+ counts.data,
+ draft.samplePercent,
+ counts.error?.message ?? null,
+ counts.isFetching,
+ );
+
+ async function saveQuestions() {
+ const summary = resolveTopicModelSlot(
+ draft.summary,
+ connections,
+ settings.stored.summary?.llmApiKeyId,
+ "chat",
+ );
+ const embedding = resolveTopicModelSlot(
+ draft.embedding,
+ connections,
+ settings.stored.embedding?.llmApiKeyId,
+ "embedding",
+ settings.stored.embedding?.model,
+ );
+ const clustering = resolveTopicModelSlot(
+ draft.clustering,
+ connections,
+ settings.stored.clustering?.llmApiKeyId,
+ "chat",
+ );
+ if (!summary || !embedding || !clustering)
+ throw new Error("Choose a model from a connection in this project.");
+ const filter: FilterState = [];
+ await save.mutateAsync({
+ projectId,
+ summary,
+ embedding,
+ embeddingDimensions: settings.stored.embeddingDimensions,
+ clustering,
+ facets: draft.questions.map((question) => ({
+ name: question.name,
+ enabled: question.enabled,
+ })),
+ customFacetIds: settings.facets.flatMap((facet) =>
+ !facet.isBuiltIn && settings.rule?.facetIds.includes(facet.id)
+ ? [facet.id]
+ : [],
+ ),
+ filter,
+ traceIdleMs: draft.waitMinutes * 60_000,
+ ...(settings.rule ? { ruleId: settings.rule.id } : {}),
+ });
+ capture("topics:setup_step_saved", {
+ step: "questions",
+ questionCount: draft.questions.filter((question) => question.enabled)
+ .length,
+ followsRecommendation: followsTopicRecommendations(draft),
+ waitMinutes: draft.waitMinutes,
+ });
+ }
+
+ async function persistSampling(enabled: boolean) {
+ await onSaveSampling({
+ enabled,
+ sampling: draft.samplePercent / 100,
+ });
+ }
+
+ async function startAnalyzing() {
+ await persistSampling(true);
+ const facets = await utils.topics.facets.fetch({ projectId });
+ const enabled = new Set<string>(
+ draft.questions
+ .filter((question) => question.enabled)
+ .map((question) => question.name),
+ );
+ const refs = facets.flatMap((facet) => {
+ const version = facet.versions[0]?.version;
+ if (!facet.isBuiltIn || !enabled.has(facet.name) || !version) return [];
+ return [{ facetId: facet.id, version }];
+ });
+ if (refs.length === 0)
+ throw new Error("Save at least one question before starting.");
+ const { traceCount: backfillCount } = await startSetup.mutateAsync({
+ projectId,
+ requestId,
+ sampling: draft.samplePercent / 100,
+ facets: refs,
+ });
+ await utils.topics.executions.invalidate({ projectId });
+ capture("topics:setup_started", {
+ backfillCount,
+ samplePercent: draft.samplePercent,
+ });
+ closeDialog();
+ }
+
+ async function handleNext() {
+ if (pending) return;
+ setPending(true);
+ setError(null);
+ try {
+ if (draft.step === "questions") {
+ await saveQuestions();
+ setDraft((current) => ({ ...current, step: "traces" }));
+ return;
+ }
+ if (draft.step === "traces") {
+ await persistSampling(false);
+ capture("topics:setup_step_saved", {
+ step: "traces",
+ samplePercent: draft.samplePercent,
+ });
+ setDraft((current) => ({ ...current, step: "review" }));
+ return;
+ }
+ await startAnalyzing();
+ } catch (cause) {
+ setError(
+ cause instanceof Error ? cause.message : "Could not save Topics.",
+ );
+ } finally {
+ setPending(false);
+ }
+ }
+
+ return (
+ <TopicSetupFlowDialog
+ step={draft.step}
+ busy={pending}
+ canWrite={canWrite}
+ error={error}
+ questions={draft.questions}
+ summary={draft.summary}
+ embedding={draft.embedding}
+ clustering={draft.clustering}
+ providerGroups={topicProviderGroups(
+ connections,
+ "chat",
+ settings.stored.summary?.model,
+ )}
+ embeddingProviderGroups={topicProviderGroups(
+ connections,
+ "embedding",
+ settings.stored.embedding?.model,
+ )}
+ connectionCaption={topicConnectionCaption(connections)}
+ advancedOpen={draft.advancedOpen}
+ waitMinutes={draft.waitMinutes}
+ samplePercent={draft.samplePercent}
+ pastCount={pastCount}
+ projectId={projectId}
+ onQuestionEnabled={(name, enabled) =>
+ setDraft((current) => ({
+ ...current,
+ questions: setQuestionEnabled(current.questions, name, enabled),
+ }))
+ }
+ onSummaryChange={(model) => updateModel(setDraft, "summary", model)}
+ onEmbeddingChange={(model) => updateModel(setDraft, "embedding", model)}
+ onClusteringChange={(model) => updateModel(setDraft, "clustering", model)}
+ onConfigureProviders={() => {
+ window.open(
+ `${env.NEXT_PUBLIC_BASE_PATH ?? ""}/project/${projectId}/settings/llm-connections`,
+ "_blank",
+ "noopener,noreferrer",
+ );
+ }}
+ onAdvancedOpenChange={(advancedOpen) =>
+ setDraft((current) => ({ ...current, advancedOpen }))
+ }
+ onWaitMinutesChange={(waitMinutes) =>
+ setDraft((current) => ({ ...current, waitMinutes }))
+ }
+ onSamplePercentChange={(samplePercent) =>
+ setDraft((current) => ({ ...current, samplePercent }))
+ }
+ onEdit={(step) => setDraft((current) => ({ ...current, step }))}
+ onBack={() =>
+ setDraft((current) => ({
+ ...current,
+ step: current.step === "review" ? "traces" : "questions",
+ }))
+ }
+ onNext={() => {
+ handleNext().catch(() => undefined);
+ }}
+ />
+ );
+}
+
+function pastTraceCount(
+ counts: { finished1d: number; finished7d: number } | undefined,
+ samplePercent: number,
+ error: string | null,
+ fetching: boolean,
+): TopicSetupPastCount {
+ if (counts) {
+ // Mirrors the server: 1 day when it fills the batch, else 7 days.
+ const rate = samplePercent / 100;
+ const window =
+ counts.finished1d * rate >= TOPIC_SETUP_BACKFILL_LIMIT
+ ? { days: 1, count: counts.finished1d }
+ : { days: TOPIC_SETUP_BACKFILL_DAYS, count: counts.finished7d };
+ return {
+ status: "ready",
+ days: window.days,
+ matched: Math.min(
+ TOPIC_SETUP_BACKFILL_LIMIT,
+ Math.round(window.count * rate),
+ ),
+ dailyTraces: counts.finished7d / TOPIC_SETUP_BACKFILL_DAYS,
+ };
+ }
+ if (error && !fetching) return { status: "error", message: error };
+ return { status: "loading" };
+}
+
+function setQuestionEnabled(
+ questions: TopicSetupQuestion[],
+ name: TopicSetupQuestion["name"],
+ enabled: boolean,
+) {
+ return questions.map((question) =>
+ question.name === name ? { ...question, enabled } : question,
+ );
+}
+
+function updateModel(
+ setDraft: (update: (current: TopicSetupDraft) => TopicSetupDraft) => void,
+ slot: "summary" | "embedding" | "clustering",
+ model: JudgeModel,
+) {
+ setDraft((current) => ({ ...current, [slot]: model }));
+}
diff --git a/web/src/features/topics/TopicSetupFlowDialog.stories.tsx b/web/src/features/topics/TopicSetupFlowDialog.stories.tsx
new file mode 100644
index 0000000000..b76919c436
--- /dev/null
+++ b/web/src/features/topics/TopicSetupFlowDialog.stories.tsx
@@ -0,0 +1,126 @@
+import { useState } from "react";
+import { DEFAULT_TOPIC_FACETS } from "@langfuse/shared/topics";
+import { expect, fn, userEvent, within } from "storybook/test";
+
+import preview from "../../../.storybook/preview";
+import { DialogController } from "@/src/components/design-system/DialogController/DialogController";
+import { TopicSetupFlowDialog } from "./TopicSetupFlowDialog";
+import {
+ TOPIC_SETUP_QUESTIONS,
+ type TopicSetupQuestion,
+} from "./topic-setup-flow";
+
+const providerGroups: Array<[string, string[]]> = [
+ [
+ "Amazon Bedrock",
+ [
+ "us.anthropic.claude-haiku-5-5",
+ "us.anthropic.claude-sonnet-5-5",
+ "us.cohere.embed-v4:0",
+ ],
+ ],
+];
+
+const questions: TopicSetupQuestion[] = TOPIC_SETUP_QUESTIONS.map(
+ (question) => ({
+ name: question.name,
+ description:
+ DEFAULT_TOPIC_FACETS.find((facet) => facet.name === question.name)
+ ?.description ?? "",
+ enabled: true,
+ }),
+);
+
+function SetupStory({ step }: { step: "questions" | "traces" | "review" }) {
+ const [selected, setSelected] = useState(questions);
+ return (
+ <TopicSetupFlowDialog
+ advancedOpen={false}
+ busy={false}
+ canWrite
+ clustering={{
+ provider: "Amazon Bedrock",
+ model: "us.anthropic.claude-sonnet-5-5",
+ }}
+ connectionCaption="Amazon Bedrock"
+ embedding={{
+ provider: "Amazon Bedrock",
+ model: "us.cohere.embed-v4:0",
+ }}
+ embeddingProviderGroups={providerGroups}
+ error={null}
+ onAdvancedOpenChange={fn()}
+ onBack={fn()}
+ onClusteringChange={fn()}
+ onConfigureProviders={fn()}
+ onEdit={fn()}
+ onEmbeddingChange={fn()}
+ onNext={fn()}
+ onQuestionEnabled={(name, enabled) =>
+ setSelected((current) =>
+ current.map((question) =>
+ question.name === name ? { ...question, enabled } : question,
+ ),
+ )
+ }
+ onSamplePercentChange={fn()}
+ onSummaryChange={fn()}
+ onWaitMinutesChange={fn()}
+ pastCount={{ status: "ready", matched: 412, days: 14, dailyTraces: 180 }}
+ providerGroups={providerGroups}
+ questions={selected}
+ samplePercent={100}
+ step={step}
+ summary={{
+ provider: "Amazon Bedrock",
+ model: "us.anthropic.claude-haiku-5-5",
+ }}
+ waitMinutes={5}
+ />
+ );
+}
+
+const meta = preview.meta({
+ component: SetupStory,
+ parameters: { layout: "fullscreen" },
+ decorators: [
+ (Story) => (
+ <DialogController<null>
+ initialState={() => null}
+ renderDialog={() => <Story />}
+ >
+ {() => null}
+ </DialogController>
+ ),
+ ],
+ args: {
+ step: "questions" as const,
+ },
+});
+
+export const Questions = meta.story({
+ args: { step: "questions" },
+});
+
+export const Traces = meta.story({
+ args: { step: "traces" },
+});
+
+export const Review = meta.story({
+ args: { step: "review" },
+});
+
+export const RequiresAQuestion = meta.story({
+ name: "(Test) Next stays disabled when every question is off",
+ args: { step: "questions" },
+ play: async () => {
+ const body = within(document.body);
+ for (const question of TOPIC_SETUP_QUESTIONS)
+ await userEvent.click(
+ body.getByRole("checkbox", { name: question.name }),
+ );
+ expect(body.getByRole("button", { name: "Next" })).toBeDisabled();
+ await userEvent.click(body.getByRole("checkbox", { name: "Intent" }));
+ expect(body.getByRole("button", { name: "Next" })).toBeEnabled();
+ },
+});
diff --git a/web/src/features/topics/TopicSetupFlowDialog.tsx b/web/src/features/topics/TopicSetupFlowDialog.tsx
new file mode 100644
index 0000000000..3398b5570b
--- /dev/null
+++ b/web/src/features/topics/TopicSetupFlowDialog.tsx
@@ -0,0 +1,528 @@
+import { useState } from "react";
+import { ChevronDown, ChevronRight, Clock, RefreshCw } from "@langfuse/icons";
+import { Alert } from "@/src/components/design-system/Alert/Alert";
+import { Badge } from "@/src/components/design-system/Badge/Badge";
+import { Button } from "@/src/components/design-system/Button/Button";
+import { Checkbox } from "@/src/components/design-system/Checkbox/Checkbox";
+import { Dialog } from "@/src/components/design-system/Dialog/Dialog";
+import { Input } from "@/src/components/design-system/Input/Input";
+import { PopoverTrigger } from "@/src/components/ui/popover";
+import { JudgeModelPicker } from "@/src/features/evals/v2/components/Evaluators/JudgeModelPicker/JudgeModelPicker";
+import { type JudgeModel } from "@/src/features/evals/v2/judgeModel";
+import {
+ TOPIC_SETUP_BACKFILL_DAYS,
+ TOPIC_SETUP_WAIT_MINUTES,
+ canLeaveTopicQuestions,
+ isRecommendedTopicModel,
+ backfillNotice,
+ type TopicSetupQuestion,
+ type TopicSetupStep,
+ type TopicSetupWaitMinutes,
+} from "./topic-setup-flow";
+import { TopicTraceFilterSection } from "./TopicTraceFilterSection";
+
+const STEP_TITLES: Record<TopicSetupStep, string> = {
+ questions: "Set up topics",
+ traces: "Select traces to analyze",
+ review: "Review",
+};
+
+const STEP_NUMBERS: Record<TopicSetupStep, string> = {
+ questions: "1 / 3",
+ traces: "2 / 3",
+ review: "3 / 3",
+};
+
+const MODEL_ROWS = [
+ { slot: "summary", label: "Summaries" },
+ { slot: "embedding", label: "Embeddings" },
+ { slot: "clustering", label: "Grouping" },
+] as const;
+
+export type TopicSetupPastCount =
+ | { status: "loading" }
+ | { status: "ready"; matched: number; days: number; dailyTraces: number }
+ | { status: "error"; message: string };
+
+export type TopicSetupFlowDialogProps = {
+ advancedOpen: boolean;
+ busy: boolean;
+ canWrite: boolean;
+ clustering: JudgeModel | null;
+ connectionCaption: string;
+ embedding: JudgeModel | null;
+ embeddingProviderGroups: Array<[string, string[]]>;
+ error: string | null;
+ onAdvancedOpenChange: (open: boolean) => void;
+ onBack: () => void;
+ onClusteringChange: (model: JudgeModel) => void;
+ onConfigureProviders: () => void;
+ onEdit: (step: "questions" | "traces") => void;
+ onEmbeddingChange: (model: JudgeModel) => void;
+ onNext: () => void;
+ onQuestionEnabled: (
+ name: TopicSetupQuestion["name"],
+ enabled: boolean,
+ ) => void;
+ onSamplePercentChange: (percent: number) => void;
+ onSummaryChange: (model: JudgeModel) => void;
+ onWaitMinutesChange: (minutes: TopicSetupWaitMinutes) => void;
+ pastCount: TopicSetupPastCount;
+ projectId?: string;
+ providerGroups: Array<[string, string[]]>;
+ questions: TopicSetupQuestion[];
+ samplePercent: number;
+ step: TopicSetupStep;
+ summary: JudgeModel | null;
+ waitMinutes: TopicSetupWaitMinutes;
+};
+
+export function TopicSetupFlowDialog({
+ advancedOpen,
+ busy,
+ canWrite,
+ clustering,
+ connectionCaption,
+ embedding,
+ embeddingProviderGroups,
+ error,
+ onAdvancedOpenChange,
+ onBack,
+ onClusteringChange,
+ onConfigureProviders,
+ onEdit,
+ onEmbeddingChange,
+ onNext,
+ onQuestionEnabled,
+ onSamplePercentChange,
+ onSummaryChange,
+ onWaitMinutesChange,
+ pastCount,
+ projectId,
+ providerGroups,
+ questions,
+ samplePercent,
+ step,
+ summary,
+ waitMinutes,
+}: TopicSetupFlowDialogProps) {
+ const models = { summary, embedding, clustering };
+ const onModelChange = {
+ summary: onSummaryChange,
+ embedding: onEmbeddingChange,
+ clustering: onClusteringChange,
+ };
+ const groups = {
+ summary: providerGroups,
+ embedding: embeddingProviderGroups,
+ clustering: providerGroups,
+ };
+ const canLeaveQuestions = canLeaveTopicQuestions({
+ canWrite,
+ questions,
+ summary,
+ embedding,
+ clustering,
+ });
+ // The server picks its own traces, so a failed estimate must not block Start.
+ const canStart = canWrite && !busy && pastCount.status !== "loading";
+ const nextDisabled = nextStepDisabled({
+ step,
+ busy,
+ canWrite,
+ canLeaveQuestions,
+ canStart,
+ });
+ const notice =
+ pastCount.status === "ready" ? backfillNotice(pastCount.matched) : null;
+
+ return (
+ <Dialog title={`${STEP_TITLES[step]} · ${STEP_NUMBERS[step]}`}>
+ <div className="flex min-h-0 flex-1 flex-col">
+ <div className="flex min-h-0 flex-1 flex-col gap-5 overflow-y-auto px-4 pb-4">
+ {step === "questions" ? (
+ <>
+ <section className="flex flex-col gap-2">
+ <h2 className="text-sm font-bold">What to look for</h2>
+ <div className="divide-y rounded-md border">
+ {questions.map((question) => (
+ <div
+ key={question.name}
+ className="grid grid-cols-[16px_96px_minmax(0,1fr)] items-center gap-2.5 px-3 py-2.5"
+ >
+ <Checkbox
+ checked={question.enabled}
+ disabled={!canWrite}
+ aria-label={question.name}
+ onCheckedChange={(checked) =>
+ onQuestionEnabled(question.name, checked === true)
+ }
+ />
+ <span className="text-sm font-bold">{question.name}</span>
+ <span className="text-muted-foreground text-sm">
+ {question.description}
+ </span>
+ </div>
+ ))}
+ </div>
+ </section>
+ <section className="flex flex-col gap-2">
+ <div className="flex items-baseline justify-between gap-3">
+ <h2 className="text-sm font-bold">
+ Select models for analysis
+ </h2>
+ <span className="text-muted-foreground text-xs">
+ {connectionCaption}
+ </span>
+ </div>
+ <div className="divide-y rounded-md border">
+ {MODEL_ROWS.map((row) => {
+ const model = models[row.slot];
+ return (
+ <ModelRow
+ key={row.slot}
+ label={row.label}
+ model={model}
+ recommended={
+ model
+ ? isRecommendedTopicModel(row.slot, model.model)
+ : false
+ }
+ disabled={!canWrite}
+ providerGroups={groups[row.slot]}
+ onConfigureProviders={onConfigureProviders}
+ onSelect={onModelChange[row.slot]}
+ />
+ );
+ })}
+ </div>
+ {error ? (
+ <p className="text-destructive text-sm">{error}</p>
+ ) : null}
+ </section>
+ <div className="flex flex-col gap-3">
+ <button
+ type="button"
+ className="flex w-fit items-center gap-1 text-sm font-bold"
+ aria-expanded={advancedOpen}
+ onClick={() => onAdvancedOpenChange(!advancedOpen)}
+ >
+ Advanced
+ {advancedOpen ? (
+ <ChevronDown size="base" />
+ ) : (
+ <ChevronRight size="base" />
+ )}
+ </button>
+ {advancedOpen ? (
+ <div className="flex flex-wrap items-center gap-3">
+ <span className="text-sm">Wait before summarizing</span>
+ <div className="flex gap-1.5">
+ {TOPIC_SETUP_WAIT_MINUTES.map((minutes) => {
+ const selected = waitMinutes === minutes;
+ return (
+ <button
+ key={minutes}
+ type="button"
+ className={
+ selected
+ ? "bg-primary text-primary-foreground border-primary rounded-full border px-3 py-1 text-sm"
+ : "rounded-full border px-3 py-1 text-sm"
+ }
+ aria-pressed={selected}
+ onClick={() => onWaitMinutesChange(minutes)}
+ >
+ {minutes} min
+ </button>
+ );
+ })}
+ </div>
+ </div>
+ ) : null}
+ </div>
+ </>
+ ) : null}
+ {step === "traces" ? (
+ <div className="flex flex-col gap-4">
+ <TopicTraceFilterSection
+ projectId={projectId}
+ description="Custom trace filters are coming soon. For now, the sample rate applies to all new traces."
+ />
+ <div className="grid grid-cols-[150px_minmax(0,1fr)] items-center gap-3">
+ <span className="text-sm font-bold">Sample rate</span>
+ <div className="flex flex-wrap items-center gap-3">
+ <input
+ type="range"
+ min={1}
+ max={100}
+ value={samplePercent}
+ aria-label="Sample rate"
+ className="accent-primary w-40"
+ disabled={!canWrite}
+ onChange={(event) =>
+ onSamplePercentChange(Number(event.target.value))
+ }
+ />
+ <div className="w-28">
+ <Input
+ type="number"
+ min={1}
+ inputMode="numeric"
+ aria-label="Sample rate percent"
+ value={String(samplePercent)}
+ disabled={!canWrite}
+ onChange={(event) => {
+ const percent = samplePercentFromInput(
+ event.target.value,
+ );
+ if (percent !== null) onSamplePercentChange(percent);
+ }}
+ />
+ </div>
+ <span className="text-sm">%</span>
+ </div>
+ </div>
+ <p className="text-muted-foreground text-xs">
+ {pastCount.status === "ready"
+ ? `~${dailyEstimate(pastCount.dailyTraces, samplePercent)} new traces a day, based on the last ${TOPIC_SETUP_BACKFILL_DAYS} days.`
+ : "Share of new traces analyzed once automatic processing is on."}
+ </p>
+ </div>
+ ) : null}
+ {step === "review" ? (
+ <div className="flex flex-col gap-2.5">
+ <ReviewRow
+ label="Look for"
+ onEdit={() => onEdit("questions")}
+ lines={[
+ {
+ value: questions
+ .filter((question) => question.enabled)
+ .map((question) => question.name)
+ .join(" · "),
+ },
+ ]}
+ />
+ <ReviewRow
+ label="Models"
+ onEdit={() => onEdit("questions")}
+ lines={[
+ {
+ key: "Summaries",
+ value: summary?.model ?? "—",
+ mono: true,
+ },
+ {
+ key: "Embeddings",
+ value: embedding?.model ?? "—",
+ mono: true,
+ },
+ {
+ key: "Grouping",
+ value: clustering?.model ?? "—",
+ mono: true,
+ },
+ ]}
+ />
+ <ReviewRow
+ label="Traces"
+ onEdit={() => onEdit("traces")}
+ lines={[
+ {
+ value:
+ pastCount.status === "ready"
+ ? `${samplePercent}% sampling rate · ~${dailyEstimate(pastCount.dailyTraces, samplePercent)} new / day`
+ : `${samplePercent}% sampling rate`,
+ },
+ { value: backfillLine(pastCount) },
+ ]}
+ />
+ {notice ? (
+ <Alert variant="warning" size="sm">
+ <Alert.Description>
+ <p>{notice}</p>
+ </Alert.Description>
+ </Alert>
+ ) : null}
+ <div className="bg-light-blue text-dark-blue mt-1 flex flex-col gap-1.5 rounded-md px-4 py-2.5 text-sm">
+ {pastCount.status !== "ready" || pastCount.matched > 0 ? (
+ <span className="flex items-center gap-2">
+ <Clock size="base" aria-hidden="true" />
+ First results from past traces in ~3–5 min
+ </span>
+ ) : null}
+ <span className="flex items-center gap-2">
+ <RefreshCw size="base" aria-hidden="true" />
+ New traces show up {waitMinutes} min after their last activity
+ </span>
+ </div>
+ </div>
+ ) : null}
+ {step !== "questions" && error ? (
+ <p className="text-destructive text-sm">{error}</p>
+ ) : null}
+ </div>
+ <div className="flex items-center justify-between border-t px-4 py-3">
+ {step === "questions" ? (
+ <span />
+ ) : (
+ <Button
+ text="Back"
+ variant="ghost"
+ disabled={busy}
+ onClick={onBack}
+ />
+ )}
+ <Button
+ text={step === "review" ? "Start analyzing" : "Next"}
+ variant="primary"
+ loading={busy}
+ disabled={nextDisabled}
+ onClick={onNext}
+ />
+ </div>
+ </div>
+ </Dialog>
+ );
+}
+
+function dailyEstimate(dailyTraces: number, samplePercent: number) {
+ const perDay = (dailyTraces * samplePercent) / 100;
+ if (perDay > 0 && perDay < 1) return "<1";
+ return Math.round(perDay).toLocaleString("en-US");
+}
+
+function backfillLine(count: TopicSetupPastCount) {
+ if (count.status === "loading") return "Counting past traces…";
+ if (count.status === "error") return "Could not count past traces";
+ if (count.matched === 0)
+ return `No past traces in the last ${TOPIC_SETUP_BACKFILL_DAYS} days`;
+ const window = count.days === 1 ? "day" : `${count.days} days`;
+ return `Include ~${count.matched.toLocaleString("en-US")} past traces from the last ${window}`;
+}
+
+function samplePercentFromInput(value: string) {
+ if (!/^\d+$/.test(value)) return null;
+ const percent = Number(value);
+ if (percent < 1 || percent > 100) return null;
+ return percent;
+}
+
+function nextStepDisabled({
+ busy,
+ canLeaveQuestions,
+ canStart,
+ canWrite,
+ step,
+}: {
+ step: TopicSetupStep;
+ busy: boolean;
+ canWrite: boolean;
+ canLeaveQuestions: boolean;
+ canStart: boolean;
+}) {
+ if (step === "questions") return busy || !canLeaveQuestions;
+ if (step === "review") return !canStart;
+ return busy || !canWrite;
+}
+
+function ModelRow({
+ disabled,
+ label,
+ model,
+ onConfigureProviders,
+ onSelect,
+ providerGroups,
+ recommended,
+}: {
+ disabled: boolean;
+ label: string;
+ model: JudgeModel | null;
+ onConfigureProviders: () => void;
+ onSelect: (model: JudgeModel) => void;
+ providerGroups: Array<[string, string[]]>;
+ recommended: boolean;
+}) {
+ const [open, setOpen] = useState(false);
+ return (
+ <div className="grid grid-cols-[112px_minmax(0,1fr)] items-center gap-2 py-1.5 pr-1.5 pl-3">
+ <span className="text-sm">{label}</span>
+ <JudgeModelPicker
+ purpose="decision"
+ open={open && !disabled}
+ onOpenChange={setOpen}
+ providerGroups={providerGroups}
+ selectedModel={model}
+ onSelect={onSelect}
+ onConfigureProviders={onConfigureProviders}
+ >
+ <PopoverTrigger asChild>
+ <button
+ type="button"
+ disabled={disabled}
+ aria-label={`${label} model`}
+ title={model ? `${model.provider} / ${model.model}` : undefined}
+ className="hover:bg-muted data-[state=open]:bg-muted flex h-8 min-w-0 items-center justify-between gap-2 rounded-md px-2.5 disabled:cursor-not-allowed disabled:opacity-50"
+ >
+ <span className="flex min-w-0 items-center gap-2">
+ <span
+ title={model?.model}
+ className={
+ model
+ ? "truncate font-mono text-xs"
+ : "text-muted-foreground truncate text-sm"
+ }
+ >
+ {model?.model ?? "Select a model"}
+ </span>
+ {recommended ? (
+ <Badge text="Recommended" color="green" size="sm" />
+ ) : null}
+ </span>
+ <ChevronDown
+ size="base"
+ __rawClassName="text-muted-foreground shrink-0"
+ />
+ </button>
+ </PopoverTrigger>
+ </JudgeModelPicker>
+ </div>
+ );
+}
+
+function ReviewRow({
+ label,
+ lines,
+ onEdit,
+}: {
+ label: string;
+ lines: Array<{ key?: string; value: string; mono?: boolean }>;
+ onEdit: () => void;
+}) {
+ return (
+ <div className="grid grid-cols-[120px_minmax(0,1fr)_auto] items-center gap-3 rounded-md border px-3.5 py-3">
+ <span className="text-sm font-bold">{label}</span>
+ <div className="flex min-w-0 flex-col gap-1.5">
+ {lines.map((line) => (
+ <div
+ key={`${line.key ?? ""}${line.value}`}
+ className="flex min-w-0 items-baseline gap-1.5"
+ >
+ {line.key ? (
+ <span className="text-muted-foreground shrink-0 text-xs">
+ {line.key}
+ </span>
+ ) : null}
+ <span
+ className={line.mono ? "truncate font-mono text-xs" : "text-sm"}
+ title={line.mono ? line.value : undefined}
+ >
+ {line.value}
+ </span>
+ </div>
+ ))}
+ </div>
+ <Button text="Edit" variant="ghost" size="sm" onClick={onEdit} />
+ </div>
+ );
+}
diff --git a/web/src/features/topics/TopicSetupNotice.stories.tsx b/web/src/features/topics/TopicSetupNotice.stories.tsx
index 161400393e..a840248090 100644
--- a/web/src/features/topics/TopicSetupNotice.stories.tsx
+++ b/web/src/features/topics/TopicSetupNotice.stories.tsx
@@ -4,47 +4,14 @@ import { TopicSetupNotice } from "./TopicSetupNotice";
const meta = preview.meta({ component: TopicSetupNotice });
-export const Setup = meta.story({
- args: {
- notice: { status: { kind: "setup" } },
- canWrite: true,
- canConfigure: true,
- onConfigure: fn(),
- onKeepOff: fn(),
- onTurnBackOn: fn(),
- },
-});
-
-export const ReadyToProcess = meta.story({
- args: {
- canWrite: true,
- canConfigure: true,
- onConfigure: fn(),
- onKeepOff: fn(),
- onTurnBackOn: fn(),
- notice: { status: null },
- showOnboarding: true,
- configured: true,
- canProcess: true,
- onAutomaticProcessing: fn(),
- onManualProcessing: fn(),
- },
-});
-
-export const Processing = meta.story({
+export const Landing = meta.story({
args: {
notice: { status: null },
canWrite: true,
canConfigure: true,
- onConfigure: fn(),
+ showOnboarding: true,
+ onStart: fn(),
onKeepOff: fn(),
onTurnBackOn: fn(),
- showOnboarding: true,
- configured: true,
- canProcess: true,
- onAutomaticProcessing: fn(),
- onManualProcessing: fn(),
- automaticEnabled: true,
- running: true,
},
});
diff --git a/web/src/features/topics/TopicSetupNotice.tsx b/web/src/features/topics/TopicSetupNotice.tsx
index 27023abe2d..15ebbbe79b 100644
--- a/web/src/features/topics/TopicSetupNotice.tsx
+++ b/web/src/features/topics/TopicSetupNotice.tsx
@@ -1,24 +1,18 @@
-import { Sparkles } from "@langfuse/icons";
+import { ArrowRight } from "@langfuse/icons";
import { Alert } from "@/src/components/design-system/Alert/Alert";
import { Button } from "@/src/components/design-system/Button/Button";
-import { StatusBadge } from "@/src/components/ui/StatusBadge/StatusBadge";
-import { SplashScreen } from "@/src/components/ui/SplashScreen";
import { type TopicSetupNoticeState } from "./TopicModelSettings";
type TopicSetupNoticeProps = {
notice: TopicSetupNoticeState;
canWrite: boolean;
canConfigure: boolean;
- onConfigure: () => void;
+ onStart: () => void;
+ // Set when analysis is already running, so setup cannot start a second backfill.
+ startBlockedReason?: string | null;
onKeepOff: () => void;
onTurnBackOn: () => void;
showOnboarding?: boolean;
- configured?: boolean;
- canProcess?: boolean;
- automaticEnabled?: boolean;
- running?: boolean;
- onAutomaticProcessing?: () => void;
- onManualProcessing?: () => void;
};
/** Guides projects to their first summaries and surfaces paused processing. */
@@ -26,24 +20,13 @@ export function TopicSetupNotice({
notice,
canWrite,
canConfigure,
- onConfigure,
+ onStart,
+ startBlockedReason = null,
onKeepOff,
onTurnBackOn,
showOnboarding = notice.status?.kind === "setup",
- configured = false,
- canProcess = false,
- automaticEnabled = false,
- running = false,
- onAutomaticProcessing,
- onManualProcessing,
}: TopicSetupNoticeProps) {
const { status } = notice;
- const processingChoiceDescription = automaticEnabled
- ? "Automatic processing is on for new traces. You can also summarize existing traces in manual batches."
- : "Enable automatic processing for incoming traces, or summarize existing traces in manual batches of up to 1,000.";
- const processingDescription = running
- ? "Your traces are being processed. Summaries will appear here as they become available."
- : processingChoiceDescription;
return (
<>
{status?.kind === "paused" ? (
@@ -71,79 +54,106 @@ export function TopicSetupNotice({
</Alert>
) : null}
{showOnboarding ? (
- <div className="mx-auto flex w-full max-w-xl flex-col items-center overflow-y-auto pt-12">
- <Sparkles aria-hidden="true" size="xl" color="muted" />
- <SplashScreen
- title={
- configured
- ? "Start analyzing your traces"
- : "Find patterns across your traces"
- }
- description="Topics analyzes each trace along the questions you choose, called facets, and clusters similar answers, so you can see what users ask for, how runs end, and which issues come up along the way."
- steps={[
- {
- title: "Configure Topics",
- badge: configured ? (
- <StatusBadge type="completed" showText={false} isLive={false}>
- Done
- </StatusBadge>
- ) : undefined,
- description: configured
- ? "Your project models and facets are ready. You can adjust them at any time."
- : "Pick the models Topics runs on and the facets to analyze, such as intent, outcome, and issues.",
- content: (
- <Button
- text="Configure Topics"
- size="sm"
- variant={configured ? "secondary" : "primary"}
- disabled={!canWrite || !canConfigure}
- onClick={onConfigure}
- />
- ),
- },
- {
- title: "Analyze traces",
- description: processingDescription,
- content: (
- <div className="flex flex-wrap gap-2">
- <Button
- text="Automatic processing"
- variant={
- configured && canProcess && !automaticEnabled
- ? "primary"
- : "secondary"
- }
- size="sm"
- disabled={!configured || !canProcess}
- onClick={onAutomaticProcessing}
- />
- <Button
- text="Summarize traces & embed"
- variant="secondary"
- size="sm"
- disabled={!configured || !canProcess || running}
- onClick={onManualProcessing}
- />
- </div>
- ),
- },
- {
- title: "Gather insights",
- description:
- "Once summaries are available, generate topics to discover recurring patterns across your traces.",
- content: (
- <Button
- text="Generate topics"
- variant="secondary"
- size="sm"
- disabled
- />
- ),
- },
- ]}
- />
+ <div className="mx-auto flex w-full max-w-4xl flex-col items-center gap-8 overflow-y-auto px-4 pt-12">
+ <SortingIllustration />
+ <h2 className="text-center text-2xl font-bold">
+ Thousands of traces, identify recurring topics & issues
+ </h2>
+ {startBlockedReason ? (
+ <p role="status" className="text-muted-foreground text-sm">
+ {startBlockedReason}
+ </p>
+ ) : (
+ <Button
+ text="Start analyzing"
+ disabled={!canWrite || !canConfigure}
+ onClick={onStart}
+ />
+ )}
</div>
) : null}
</>
);
}
+
+// Decorative: a messy pile of traces sorted into three example topics.
+const MESSY_CARDS = [
+ [10, 24, -12],
+ [78, 0, 7],
+ [150, 34, -4],
+ [24, 96, 9],
+ [112, 84, -15],
+ [166, 118, 11],
+ [0, 160, -6],
+ [86, 156, 4],
+ [150, 176, -9],
+ [62, 48, 17],
+] as const;
+const SLAB_JITTER = [0, 3, -2, 4, -3, 1, -4, 2, -1, 3, 0];
+const PILES = [
+ {
+ name: "Refund requests",
+ count: 214,
+ slabs: 11,
+ className: "bg-dark-violet/25 border-dark-violet/40",
+ },
+ {
+ name: "Order status",
+ count: 156,
+ slabs: 8,
+ className: "bg-dark-blue/20 border-dark-blue/35",
+ },
+ {
+ name: "Cancel subscription",
+ count: 61,
+ slabs: 4,
+ className: "bg-dark-teal/20 border-dark-teal/35",
+ },
+];
+
+function SortingIllustration() {
+ return (
+ <div
+ aria-hidden="true"
+ className="bg-light-violet flex w-full items-center justify-center gap-8 overflow-hidden rounded-2xl px-6 py-10"
+ >
+ <div className="relative hidden h-60 w-72 shrink-0 sm:block">
+ {MESSY_CARDS.map(([left, top, rotate]) => (
+ <div
+ key={`${left}-${top}`}
+ className="bg-card absolute flex h-16 w-28 flex-col gap-2 rounded-lg border p-2.5 shadow-sm"
+ style={{ left, top, transform: `rotate(${rotate}deg)` }}
+ >
+ <span className="bg-muted size-2 rounded-full" />
+ <span className="bg-muted h-1.5 w-4/5 rounded" />
+ <span className="bg-muted h-1.5 w-3/5 rounded" />
+ </div>
+ ))}
+ </div>
+ <ArrowRight
+ size="lg"
+ color="muted"
+ __rawClassName="hidden shrink-0 sm:block"
+ />
+ <div className="flex items-end gap-5">
+ {PILES.map((pile) => (
+ <div key={pile.name} className="flex flex-col items-center gap-1">
+ <div className="flex flex-col gap-1">
+ {SLAB_JITTER.slice(0, pile.slabs).map((offset, index) => (
+ <span
+ key={index}
+ className={`h-3.5 w-28 rounded border ${pile.className}`}
+ style={{ transform: `translateX(${offset}px)` }}
+ />
+ ))}
+ </div>
+ <span className="mt-2 text-sm font-bold whitespace-nowrap">
+ {pile.name}
+ </span>
+ <span className="text-muted-foreground text-sm">{pile.count}</span>
+ </div>
+ ))}
+ </div>
+ </div>
+ );
+}
diff --git a/web/src/features/topics/TopicSetupProgressView.stories.tsx b/web/src/features/topics/TopicSetupProgressView.stories.tsx
new file mode 100644
index 0000000000..bb45f1d2f4
--- /dev/null
+++ b/web/src/features/topics/TopicSetupProgressView.stories.tsx
@@ -0,0 +1,47 @@
+import { fn } from "storybook/test";
+import preview from "../../../.storybook/preview";
+import { TopicSetupProgressView } from "./TopicSetupProgressView";
+
+const facets = ["Intent", "Sentiment", "Outcome", "Issues"].map((name) => ({
+ key: name,
+ name,
+ complete: 200,
+ requested: 500,
+ outcome: null as string | null,
+ topicCount: null as number | null,
+}));
+
+const meta = preview.meta({
+ component: TopicSetupProgressView,
+ args: {
+ stage: "summarizing" as const,
+ facets,
+ retrying: false,
+ canRetry: true,
+ onDisplayResults: fn(),
+ onRetry: fn(),
+ },
+});
+
+export const Summarizing = meta.story({});
+
+export const Naming = meta.story({
+ args: {
+ stage: "naming" as const,
+ facets: facets.map((facet) => ({ ...facet, complete: 500 })),
+ },
+});
+
+export const Done = meta.story({
+ args: {
+ stage: "done" as const,
+ facets: facets.map((facet, index) => ({
+ ...facet,
+ complete: 500,
+ outcome: index === 3 ? "insufficient_data" : "published",
+ topicCount: index === 3 ? null : 4 + index,
+ })),
+ },
+});
+
+export const Failed = meta.story({ args: { stage: "failed" as const } });
diff --git a/web/src/features/topics/TopicSetupProgressView.tsx b/web/src/features/topics/TopicSetupProgressView.tsx
new file mode 100644
index 0000000000..fb111f700e
--- /dev/null
+++ b/web/src/features/topics/TopicSetupProgressView.tsx
@@ -0,0 +1,211 @@
+import { Check, Sparkles } from "@langfuse/icons";
+import { Button } from "@/src/components/design-system/Button/Button";
+import { Progress } from "@/src/components/design-system/Progress/Progress";
+import { Spinner } from "@/src/components/design-system/Spinner/Spinner";
+import { type TopicSetupStage } from "./topic-setup-flow";
+
+export type TopicSetupProgressFacet = {
+ key: string;
+ name: string;
+ complete: number;
+ requested: number;
+ outcome: string | null;
+ topicCount: number | null;
+};
+
+const RUNNING = {
+ title: "Analyzing your traces",
+ detail: "This runs in the background. You can leave this page.",
+};
+const COPY: Record<TopicSetupStage, { title: string; detail: string }> = {
+ summarizing: RUNNING,
+ grouping: RUNNING,
+ naming: RUNNING,
+ done: {
+ title: "Your first topics are ready",
+ detail: "Display the results to explore them.",
+ },
+ failed: {
+ title: "The first analysis did not finish",
+ detail: "Retry picks up where it stopped.",
+ },
+};
+const TOPIC_STEPS = [
+ { stage: "grouping", label: "Identifying clusters" },
+ { stage: "naming", label: "Generating & refining topics" },
+] as const;
+const NO_TOPICS = new Set([
+ "no_topics",
+ "insufficient_data",
+ "no_applicable_summaries",
+]);
+
+/** First-run progress in two phases: summaries, then topics. */
+export function TopicSetupProgressView({
+ canRetry,
+ facets,
+ onViewDetails,
+ onDisplayResults,
+ onRetry,
+ retrying,
+ stage,
+}: {
+ facets: TopicSetupProgressFacet[];
+ stage: TopicSetupStage;
+ retrying: boolean;
+ canRetry: boolean;
+ onDisplayResults: () => void;
+ // Opens the run in History; set when a run or one of its facets failed.
+ onViewDetails?: () => void;
+ onRetry: () => void;
+}) {
+ const requested = Math.max(0, ...facets.map((facet) => facet.requested));
+ const complete = Math.min(
+ requested,
+ ...facets.map((facet) => facet.complete),
+ );
+ const summarizing = stage === "summarizing";
+ const summariesDone =
+ stage === "grouping" || stage === "naming" || stage === "done";
+ const percent =
+ summariesDone || requested === 0
+ ? 100
+ : Math.round((complete / requested) * 100);
+ const topicIndex = ["grouping", "naming", "done"].indexOf(stage);
+ return (
+ <div className="mx-auto flex w-full max-w-2xl flex-col items-center gap-6 overflow-y-auto px-4 pt-12">
+ <div className="flex flex-col items-center gap-2 text-center">
+ <Sparkles aria-hidden="true" size="xl" color="muted" />
+ <h2 className="text-2xl font-bold">{COPY[stage].title}</h2>
+ <p className="text-muted-foreground text-sm">
+ {stage === "failed" && !canRetry
+ ? "Topics could not start. Use Generate topics in the menu."
+ : COPY[stage].detail}
+ </p>
+ </div>
+ <div className="grid w-full gap-4 sm:grid-cols-2">
+ <section className="bg-card flex flex-col gap-3 rounded-xl border p-5">
+ <div className="flex items-center gap-2">
+ <PhaseMarker done={summariesDone} active={summarizing} index={1} />
+ <h3 className="text-sm font-bold">
+ Analyzing traces & generating summaries
+ </h3>
+ </div>
+ <div className="flex items-center gap-3">
+ <div className="flex-1">
+ <Progress value={percent} />
+ </div>
+ <span className="text-muted-foreground w-10 text-right font-mono text-xs">
+ {percent}%
+ </span>
+ </div>
+ <span className="text-muted-foreground text-xs">
+ {summariesDone
+ ? `${requested.toLocaleString("en-US")} traces summarized`
+ : `${complete.toLocaleString("en-US")} of ${requested.toLocaleString("en-US")} traces`}
+ </span>
+ </section>
+ <section className="bg-card flex flex-col gap-3 rounded-xl border p-5">
+ <div className="flex items-center gap-2">
+ <PhaseMarker
+ done={stage === "done"}
+ active={stage === "grouping" || stage === "naming"}
+ index={2}
+ />
+ <h3 className="text-sm font-bold">
+ Identifying clusters & generating topics
+ </h3>
+ </div>
+ {topicIndex < 0 ? (
+ <span className="text-muted-foreground text-xs">
+ Starts when summaries are done.
+ </span>
+ ) : null}
+ {topicIndex >= 0 && stage !== "done" ? (
+ <ul className="flex flex-col gap-1.5">
+ {TOPIC_STEPS.map((step, index) => (
+ <li
+ key={step.stage}
+ className="text-muted-foreground flex items-center gap-2 text-xs"
+ >
+ <StepMarker state={stepState(index, topicIndex)} />
+ {step.label}
+ </li>
+ ))}
+ </ul>
+ ) : null}
+ {stage === "done" ? (
+ <ul className="flex flex-col gap-1.5">
+ {facets.map((facet) => (
+ <li
+ key={facet.key}
+ className="flex items-center justify-between gap-2 text-xs"
+ >
+ <span>{facet.name}</span>
+ <span className="text-muted-foreground">
+ {facetResult(facet)}
+ </span>
+ </li>
+ ))}
+ </ul>
+ ) : null}
+ </section>
+ </div>
+ {stage === "done" ? (
+ <Button text="Display results" onClick={onDisplayResults} />
+ ) : null}
+ {stage === "failed" && canRetry ? (
+ <Button text="Retry" loading={retrying} onClick={onRetry} />
+ ) : null}
+ {onViewDetails ? (
+ <Button
+ text="See run details"
+ variant="ghost"
+ onClick={onViewDetails}
+ />
+ ) : null}
+ </div>
+ );
+}
+
+function stepState(index: number, current: number) {
+ if (index < current) return "done";
+ if (index === current) return "active";
+ return "waiting";
+}
+
+function facetResult(facet: TopicSetupProgressFacet) {
+ if (facet.outcome === "failed") return "Failed";
+ if (NO_TOPICS.has(facet.outcome ?? "")) return "Not enough data yet";
+ if (facet.topicCount === null) return "Topics ready";
+ return `${facet.topicCount} ${facet.topicCount === 1 ? "topic" : "topics"}`;
+}
+
+function PhaseMarker({
+ active,
+ done,
+ index,
+}: {
+ active: boolean;
+ done: boolean;
+ index: number;
+}) {
+ if (done)
+ return (
+ <span className="bg-dark-green flex size-5 shrink-0 items-center justify-center rounded-full text-white">
+ <Check size="sm" aria-hidden="true" />
+ </span>
+ );
+ if (active) return <Spinner size="sm" />;
+ return (
+ <span className="text-muted-foreground flex size-5 shrink-0 items-center justify-center rounded-full border text-xs">
+ {index}
+ </span>
+ );
+}
+
+function StepMarker({ state }: { state: "done" | "active" | "waiting" }) {
+ if (state === "done") return <Check size="sm" aria-hidden="true" />;
+ if (state === "active") return <Spinner size="xxs" />;
+ return <span className="size-3 rounded-full border" />;
+}
diff --git a/web/src/features/topics/TopicTraceFilterSection.tsx b/web/src/features/topics/TopicTraceFilterSection.tsx
new file mode 100644
index 0000000000..a59d450530
--- /dev/null
+++ b/web/src/features/topics/TopicTraceFilterSection.tsx
@@ -0,0 +1,142 @@
+import { useState } from "react";
+import { Badge } from "@/src/components/design-system/Badge/Badge";
+import { ObservationFilterBuilder } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/ObservationFilterBuilder/ObservationFilterBuilder";
+import { FilterModeToggle } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SampleObservationSelectorBase/components/FilterModeToggle";
+import { RULE_FIELD_REGISTRY } from "@/src/features/evals/v2/constants/ruleSearchRegistry";
+import {
+ EventsSearchBarRow,
+ type toObservedOptions,
+ useEventsSearchBar,
+} from "@/src/features/search-bar";
+import { SectionHeader } from "@/src/features/evals/v2/components/Evaluators/Testing/components/SectionHeader/SectionHeader";
+import type { EvaluatorFilterExperience } from "@/src/features/evals/v2/types/evaluatorFilterExperience";
+import {
+ type experimentEvalFilterColsWithOptions,
+ type FilterState,
+ type TracingSearchType,
+} from "@langfuse/shared";
+
+const topicsFilterRegistry = {
+ ...RULE_FIELD_REGISTRY,
+ aiFilterPrompt: true,
+ fields: RULE_FIELD_REGISTRY.fields.filter(
+ (field) => field.id !== "isRootObservation",
+ ),
+ columns: RULE_FIELD_REGISTRY.columns.filter(
+ (column) => column.id !== "isRootObservation",
+ ),
+};
+
+const queryOnlyColumnIds = topicsFilterRegistry.fields
+ .filter((field) => field.directFilter === false)
+ .map((field) => field.filterColumn ?? field.id);
+
+export type TopicTraceFilterSectionProps = {
+ projectId?: string;
+ mode?: EvaluatorFilterExperience;
+ filters?: FilterState;
+ searchQuery?: string | null;
+ searchType?: TracingSearchType[];
+ observed?: ReturnType<typeof toObservedOptions>;
+ builderColumns?: ReturnType<typeof experimentEvalFilterColsWithOptions>;
+ erroredColumns?: ReadonlySet<string>;
+ onRequestColumns?: (columns: readonly string[]) => void;
+ onModeChange?: (mode: EvaluatorFilterExperience) => void;
+ onFiltersChange?: (filters: FilterState) => void;
+ onSearchQueryChange?: (query: string | null) => void;
+ onSearchTypeChange?: (searchType: TracingSearchType[]) => void;
+ description?: string;
+};
+
+/** The trace filter, shown disabled until custom filters ship. */
+export function TopicTraceFilterSection({
+ projectId,
+ mode: modeProp,
+ filters: filtersProp,
+ searchQuery: searchQueryProp,
+ searchType: searchTypeProp,
+ observed,
+ builderColumns,
+ erroredColumns,
+ onRequestColumns,
+ onModeChange,
+ onFiltersChange,
+ onSearchQueryChange,
+ onSearchTypeChange,
+ description = "The sampling rate applies to all incoming traces. Custom trace filters are coming soon.",
+}: TopicTraceFilterSectionProps) {
+ const [localMode, setLocalMode] =
+ useState<EvaluatorFilterExperience>("query");
+ const [localFilters, setLocalFilters] = useState<FilterState>([]);
+ const [localSearchQuery, setLocalSearchQuery] = useState<string | null>(null);
+ const [localSearchType, setLocalSearchType] = useState<TracingSearchType[]>(
+ [],
+ );
+ const mode = modeProp ?? localMode;
+ const filters = filtersProp ?? localFilters;
+ const searchQuery = searchQueryProp ?? localSearchQuery;
+ const searchType = searchTypeProp ?? localSearchType;
+ const setMode = onModeChange ?? setLocalMode;
+ const setFilters = onFiltersChange ?? setLocalFilters;
+ const setSearchQuery = onSearchQueryChange ?? setLocalSearchQuery;
+ const setSearchType = onSearchTypeChange ?? setLocalSearchType;
+ const search = useEventsSearchBar({
+ projectId,
+ tableName: "topics-automatic-processing",
+ enabled: true,
+ filterState: filters,
+ searchQuery,
+ searchType,
+ observed,
+ setFilterState: setFilters,
+ setSearchQuery,
+ setSearchType,
+ registry: topicsFilterRegistry,
+ });
+
+ return (
+ <section className="flex flex-col gap-2">
+ <SectionHeader
+ title="Filter traces"
+ meta={<Badge text="Coming soon" color="blue" size="sm" />}
+ description={description}
+ tooltip="Only traces matching this filter will be summarized. Topics always runs on whole traces."
+ trailing={
+ <div inert className="opacity-50">
+ <FilterModeToggle
+ mode={mode}
+ onChange={(next) => {
+ if (next === "builder" && searchQuery !== null)
+ search.applyFilters(filters);
+ setMode(next);
+ }}
+ />
+ </div>
+ }
+ />
+ <div inert aria-disabled className="opacity-50">
+ {mode === "query" ? (
+ <EventsSearchBarRow
+ projectId={projectId}
+ tableName="topics-automatic-processing"
+ store={search.store}
+ commit={search.commit}
+ observed={observed}
+ erroredColumns={erroredColumns}
+ registry={topicsFilterRegistry}
+ onApplyFilters={search.applyFilters}
+ onRequestColumns={onRequestColumns}
+ className="p-0"
+ />
+ ) : (
+ <ObservationFilterBuilder
+ columns={builderColumns ?? [...topicsFilterRegistry.columns]}
+ filterState={filters}
+ onChange={setFilters}
+ queryOnlyColumnIds={queryOnlyColumnIds}
+ />
+ )}
+ </div>
+ </section>
+ );
+}
diff --git a/web/src/features/topics/TopicsPage.clienttest.tsx b/web/src/features/topics/TopicsPage.clienttest.tsx
index 001213720d..2c6064918f 100644
--- a/web/src/features/topics/TopicsPage.clienttest.tsx
+++ b/web/src/features/topics/TopicsPage.clienttest.tsx
@@ -142,6 +142,13 @@ vi.mock("./ConfigureTopicsDialogController", () => ({
children: (control: { openDialog: () => void }) => ReactNode;
}) => children({ openDialog: state.openConfigure }),
}));
+vi.mock("./TopicSetupFlowController", () => ({
+ TopicSetupFlowController: ({
+ children,
+ }: {
+ children: (control: { openSetup: () => void }) => ReactNode;
+ }) => children({ openSetup: () => {} }),
+}));
vi.mock("./AutomaticProcessingDialogController", () => ({
AutomaticProcessingDialogController: ({
children,
@@ -223,6 +230,9 @@ vi.mock("@/src/utils/api", () => ({
: null,
}),
},
+ setupTopicCounts: {
+ useQuery: () => ({ data: undefined }),
+ },
setAutomaticProcessing: {
useMutation: () => ({
isPending: false,
@@ -1039,30 +1049,15 @@ it.each([
},
);
-it("guides a configured project through processing until its first stored summary exists", () => {
+it("shows the setup landing until the project's first stored summary exists", () => {
state.hasStoredSummaries = false;
const view = render(<TopicsPage />);
- expect(
- screen.getByRole("button", { name: "Summarize traces & embed" }),
- ).toBeEnabled();
+ expect(screen.getByRole("button", { name: "Start analyzing" })).toBeEnabled();
expect(screen.queryByTestId("current-topics")).toBeNull();
- fireEvent.click(
- screen.getAllByRole("button", { name: "Automatic processing" }).at(-1)!,
- );
- expect(state.openAutomaticProcessing).toHaveBeenCalledWith(false);
- fireEvent.click(
- screen.getByRole("button", { name: "Summarize traces & embed" }),
- );
- expect(
- screen.getByRole("dialog", { name: "Manual processing" }),
- ).toBeVisible();
- fireEvent.click(screen.getByRole("button", { name: "Cancel" }));
state.hasStoredSummaries = true;
view.rerender(<TopicsPage />);
expect(screen.getByTestId("current-topics")).toBeVisible();
- expect(
- screen.queryByRole("button", { name: "Summarize traces & embed" }),
- ).toBeNull();
+ expect(screen.queryByRole("button", { name: "Start analyzing" })).toBeNull();
});
it("opens generation directly from the empty topics panel", () => {
@@ -1090,5 +1085,9 @@ it("keeps date controls reachable when a disabled results query has no data", ()
expect(
screen.getByRole("button", { name: "Select last 30 days" }),
).toBeVisible();
- expect(screen.queryByText("Start analyzing your traces")).toBeNull();
+ expect(
+ screen.queryByText(
+ "Thousands of traces, identify recurring topics & issues",
+ ),
+ ).toBeNull();
});
diff --git a/web/src/features/topics/TopicsPage.tsx b/web/src/features/topics/TopicsPage.tsx
index a128c096a4..54c1b63b00 100644
--- a/web/src/features/topics/TopicsPage.tsx
+++ b/web/src/features/topics/TopicsPage.tsx
@@ -55,7 +55,10 @@ import { useRegenerateTopics } from "./useRegenerateTopics";
import { useTopicModelSettings } from "./TopicModelSettings";
import { AutomaticProcessingDialogController } from "./AutomaticProcessingDialogController";
import { ConfigureTopicsDialogController } from "./ConfigureTopicsDialogController";
+import { TopicSetupFlowController } from "./TopicSetupFlowController";
import { TopicSetupNotice } from "./TopicSetupNotice";
+import { topicSetupRun, topicSetupStage } from "./topic-setup-flow";
+import { TopicSetupProgressView } from "./TopicSetupProgressView";
import { CurrentTopics, useCurrentTopics } from "./CurrentTopics";
import { TopicsActionsMenu } from "./TopicsActionsMenu";
import { TopicsWorkspaceGate } from "./TopicsWorkspaceGate";
@@ -125,13 +128,32 @@ export default function TopicsPage() {
function TopicsWorkspace({ projectId }: { projectId: string }) {
const facets = api.topics.facets.useQuery({ projectId });
+ const canWrite = useHasProjectAccess({ projectId, scope: "topics:CUD" });
+ const modelSettings = useTopicModelSettings({
+ projectId,
+ canWrite,
+ facets: facets.data ?? [],
+ facetsReady: facets.isSuccess,
+ });
return (
<TopicsWorkspaceGate isLoading={facets.isLoading} fallback={loadingPage}>
- <TopicsWorkspaceView
- key={(facets.data?.length ?? 0) > 0 ? "configured" : "empty"}
+ <TopicSetupFlowController
projectId={projectId}
- facets={facets}
- />
+ canWrite={canWrite}
+ settings={modelSettings.configure.settings}
+ onSaveSampling={modelSettings.automaticProcessing.save}
+ >
+ {({ openSetup }) => (
+ <TopicsWorkspaceView
+ key={(facets.data?.length ?? 0) > 0 ? "configured" : "empty"}
+ projectId={projectId}
+ facets={facets}
+ canWrite={canWrite}
+ modelSettings={modelSettings}
+ openSetup={openSetup}
+ />
+ )}
+ </TopicSetupFlowController>
</TopicsWorkspaceGate>
);
}
@@ -139,20 +161,22 @@ function TopicsWorkspace({ projectId }: { projectId: string }) {
type TopicsWorkspaceViewProps = {
projectId: string;
facets: UseQueryResult<TopicFacet[], { message: string }>;
+ canWrite: boolean;
+ modelSettings: ReturnType<typeof useTopicModelSettings>;
+ openSetup: () => void;
};
/**
* Wraps the workspace in the dialog controllers whose dialogs open from more
* than one place, and hands their `openDialog` to the content.
*/
-function TopicsWorkspaceView({ projectId, facets }: TopicsWorkspaceViewProps) {
- const canWrite = useHasProjectAccess({ projectId, scope: "topics:CUD" });
- const modelSettings = useTopicModelSettings({
- projectId,
- canWrite,
- facets: facets.data ?? [],
- facetsReady: facets.isSuccess,
- });
+function TopicsWorkspaceView({
+ projectId,
+ facets,
+ canWrite,
+ modelSettings,
+ openSetup,
+}: TopicsWorkspaceViewProps) {
return (
<ConfigureTopicsDialogController
projectId={projectId}
@@ -169,6 +193,7 @@ function TopicsWorkspaceView({ projectId, facets }: TopicsWorkspaceViewProps) {
canWrite={canWrite}
modelSettings={modelSettings}
openConfigure={openConfigure}
+ openSetup={openSetup}
openAutomaticProcessing={openAutomaticProcessing}
/>
)}
@@ -179,8 +204,6 @@ function TopicsWorkspaceView({ projectId, facets }: TopicsWorkspaceViewProps) {
}
type TopicsWorkspaceContentProps = TopicsWorkspaceViewProps & {
- canWrite: boolean;
- modelSettings: ReturnType<typeof useTopicModelSettings>;
openConfigure: () => void;
openAutomaticProcessing: (initialEnabled: boolean) => void;
};
@@ -191,6 +214,7 @@ function TopicsWorkspaceContent({
canWrite,
modelSettings,
openConfigure,
+ openSetup,
openAutomaticProcessing,
}: TopicsWorkspaceContentProps) {
const [historyOpen, setHistoryOpen] = useState(false);
@@ -231,12 +255,66 @@ function TopicsWorkspaceContent({
const executions = api.topics.executions.useQuery(
{ projectId },
{
- refetchInterval: (query) =>
- query.state.data?.some((execution) => busy(execution.status))
+ refetchInterval: (query) => {
+ const data = query.state.data ?? [];
+ if (data.some((execution) => busy(execution.status))) return 3000;
+ // Keep polling through the hand-off from summaries to the topic update.
+ const run = topicSetupRun(data);
+ return run &&
+ !run.update &&
+ topicSetupStage(run, Date.now()).stage === "grouping"
? 3000
- : false,
+ : false;
+ },
},
);
+ const [mountedAt] = useState(Date.now);
+ const [acknowledgedSetupId, setAcknowledgedSetupId] = useState<string | null>(
+ null,
+ );
+ const setupRun = topicSetupRun(executions.data ?? []);
+ const setup = setupRun ? topicSetupStage(setupRun, Date.now()) : null;
+ // A run that finished before this visit is not replayed; results show instead.
+ const showSetupProgress =
+ setupRun !== null &&
+ setup !== null &&
+ setupRun.process.id !== acknowledgedSetupId &&
+ (["summarizing", "grouping", "naming"].includes(setup.stage) ||
+ Date.parse((setupRun.update ?? setupRun.process).updatedAt) > mountedAt);
+ const setupRunIds =
+ setupRun?.update?.facets.flatMap((facet) =>
+ facet.runId ? [facet.runId] : [],
+ ) ?? [];
+ const setupTopicCounts = api.topics.setupTopicCounts.useQuery(
+ { projectId, runIds: setupRunIds },
+ { enabled: setup?.stage === "done" && setupRunIds.length > 0 },
+ );
+ const setupRetry = api.topics.retry.useMutation({
+ onSuccess: () => utils.topics.executions.invalidate({ projectId }),
+ });
+ // A run worth inspecting: the one that failed, or a topic run with failed facets.
+ const setupDetailsId =
+ setupRun && setup ? detailsRunId(setupRun, setup) : null;
+ const setupFacets =
+ setupRun?.process.facets.map((facet) => {
+ const update = setupRun.update?.facets.find(
+ (item) =>
+ item.facetId === facet.facetId &&
+ item.facetVersion === facet.facetVersion,
+ );
+ return {
+ key: `${facet.facetId}:${facet.facetVersion}`,
+ name:
+ facets.data?.find((item) => item.id === facet.facetId)?.name ??
+ "Facet",
+ complete: facet.counts.complete,
+ requested: facet.counts.requested,
+ outcome: update?.outcome ?? null,
+ topicCount:
+ setupTopicCounts.data?.find((count) => count.runId === update?.runId)
+ ?.topicCount ?? null,
+ };
+ }) ?? [];
const executionId =
typeof router.query.executionId === "string"
? router.query.executionId
@@ -295,6 +373,10 @@ function TopicsWorkspaceContent({
const running =
Boolean(executions.data?.some((execution) => busy(execution.status))) ||
busy(selectedExecution.data?.status ?? "");
+ const setupBlockedReason = startBlockedReason(
+ running,
+ modelSettings.automaticProcessing.state.status === "on",
+ );
const results = useCurrentTopics({
projectId,
running,
@@ -397,12 +479,13 @@ function TopicsWorkspaceContent({
results.data?.facets ?? [],
selectedFacetId,
);
- const showOnboarding =
+ const showLanding =
validTimeRange &&
results.data !== undefined &&
!results.isLoading &&
!results.error &&
!hasStoredSummaries;
+ const showOnboarding = showSetupProgress || showLanding;
return (
<Page
headerProps={{
@@ -434,7 +517,7 @@ function TopicsWorkspaceContent({
manualAvailable={pipeline.available}
openManual={openManualProcessing}
onAutomaticProcessing={openAutomaticProcessingFromHeader}
- onConfigure={openConfigure}
+ onConfigure={configured ? openConfigure : openSetup}
onHistory={openHistory}
onRefresh={refreshResults}
/>
@@ -507,17 +590,36 @@ function TopicsWorkspaceContent({
notice={modelSettings.notice}
canWrite={canWrite}
canConfigure={modelSettings.configure.settings !== null}
- onConfigure={openConfigure}
+ onStart={openSetup}
+ startBlockedReason={setupBlockedReason}
onKeepOff={automaticProcessing.clearPause}
onTurnBackOn={turnAutomaticProcessingBackOn}
- showOnboarding={showOnboarding}
- configured={configured}
- canProcess={canWrite}
- automaticEnabled={automaticProcessing.state.status === "on"}
- running={running}
- onAutomaticProcessing={openAutomaticProcessingFromHeader}
- onManualProcessing={openManualProcessing}
+ showOnboarding={showLanding && !showSetupProgress}
/>
+ {showSetupProgress && setup ? (
+ <TopicSetupProgressView
+ stage={setup.stage}
+ facets={setupFacets}
+ retrying={setupRetry.isPending}
+ canRetry={setup.retryId !== null}
+ onViewDetails={
+ setupDetailsId
+ ? () => openExecution(setupDetailsId)
+ : undefined
+ }
+ onRetry={() => {
+ if (setup.retryId)
+ setupRetry.mutate({
+ projectId,
+ executionId: setup.retryId,
+ });
+ }}
+ onDisplayResults={() => {
+ setAcknowledgedSetupId(setupRun.process.id);
+ return utils.topics.invalidate();
+ }}
+ />
+ ) : null}
<Sheet
open={historyOpen || executionId !== null}
onOpenChange={showExecutionList}
@@ -929,3 +1031,25 @@ function ErrorMessage({ message }: { message: string }) {
</Alert>
);
}
+
+function startBlockedReason(running: boolean, automaticOn: boolean) {
+ if (running)
+ return "Analysis is running. Topics appear here when it finishes.";
+ if (automaticOn)
+ return "Automatic processing is on. Topics appear here as traces are summarized.";
+ return null;
+}
+
+function detailsRunId(
+ run: {
+ process: { id: string };
+ update: { id: string; facets: Array<{ outcome: string }> } | null;
+ },
+ setup: { stage: string; retryId: string | null },
+) {
+ if (setup.stage === "failed")
+ return setup.retryId ?? run.update?.id ?? run.process.id;
+ return run.update?.facets.some((facet) => facet.outcome === "failed")
+ ? run.update.id
+ : null;
+}
diff --git a/web/src/features/topics/server/currentResults.ts b/web/src/features/topics/server/currentResults.ts
index d462e88c16..794dd03ac5 100644
--- a/web/src/features/topics/server/currentResults.ts
+++ b/web/src/features/topics/server/currentResults.ts
@@ -9,6 +9,7 @@ import {
getTopicRuns,
hasStoredTopicSummaries,
listTopicFacets,
+ listTopicRules,
readLatestTopicAssignments,
} from "@langfuse/shared/topics/server";
@@ -50,123 +51,128 @@ export async function currentTopicResults(
const hasStoredSummaries =
facets.length > 0 && (await hasStoredTopicSummaries(projectId));
if (!hasStoredSummaries) return { hasStoredSummaries, facets: [] };
+ // Only facets enabled on the project's rule get tabs; without a rule, all show.
+ const [rule] = await listTopicRules(projectId);
+ const enabledFacetIds = rule ? new Set(rule.facetIds) : null;
const currentFacets = await Promise.all(
- facets.map(async (facet) => {
- const version = facet.versions[0];
- const [storedAssignments, storedSummaries, run] = await Promise.all([
- readLatestTopicAssignments(projectId, facet.id, timeRange),
- getLatestFacetSummaries(projectId, facet.id, undefined, timeRange),
- getPublishedTopicRun(projectId, facet.id),
- ]);
- const latestSummaries = storedSummaries.filter(
- (row) => row.traceId !== null,
- );
- const assignmentByTrace = new Map(
- storedAssignments.map((row) => [row.traceId, row]),
- );
- const current = latestSummaries
- .sort(
- (a, b) =>
- b.unitStartTime.localeCompare(a.unitStartTime) ||
- a.traceId.localeCompare(b.traceId),
- )
- .map((summary) => ({
- summary,
- ...resolveTopicResult(
+ facets
+ .filter((facet) => !enabledFacetIds || enabledFacetIds.has(facet.id))
+ .map(async (facet) => {
+ const version = facet.versions[0];
+ const [storedAssignments, storedSummaries, run] = await Promise.all([
+ readLatestTopicAssignments(projectId, facet.id, timeRange),
+ getLatestFacetSummaries(projectId, facet.id, undefined, timeRange),
+ getPublishedTopicRun(projectId, facet.id),
+ ]);
+ const latestSummaries = storedSummaries.filter(
+ (row) => row.traceId !== null,
+ );
+ const assignmentByTrace = new Map(
+ storedAssignments.map((row) => [row.traceId, row]),
+ );
+ const current = latestSummaries
+ .sort(
+ (a, b) =>
+ b.unitStartTime.localeCompare(a.unitStartTime) ||
+ a.traceId.localeCompare(b.traceId),
+ )
+ .map((summary) => ({
summary,
- assignmentByTrace.get(summary.traceId),
- ),
- }));
- const assignmentRuns = await getTopicRuns(projectId, [
- ...new Set(
- current.flatMap(({ assignment }) =>
- assignment?.topicId &&
- assignment.runId &&
- assignment.runId !== run?.id
- ? [assignment.runId]
- : [],
+ ...resolveTopicResult(
+ summary,
+ assignmentByTrace.get(summary.traceId),
+ ),
+ }));
+ const assignmentRuns = await getTopicRuns(projectId, [
+ ...new Set(
+ current.flatMap(({ assignment }) =>
+ assignment?.topicId &&
+ assignment.runId &&
+ assignment.runId !== run?.id
+ ? [assignment.runId]
+ : [],
+ ),
),
- ),
- ]);
- const definitions = [
- ...(run?.topics ?? []),
- ...assignmentRuns.flatMap((item) => item.topics),
- ];
- const topicByVersion = new Map(
- definitions.map((topic) => [topic.topicVersionId, topic]),
- );
- const topics = new Map<
- string,
- { assignment: TopicAssignment; count: number }
- >();
- const rows = current.map(({ summary, assignment, outcome }) => {
- const definition = assignment?.topicVersionId
- ? topicByVersion.get(assignment.topicVersionId)
- : undefined;
- if (assignment?.topicId) {
- const topic = topics.get(assignment.topicId);
- topics.set(assignment.topicId, {
- assignment:
- !topic || assignment.assignedAt > topic.assignment.assignedAt
- ? assignment
- : topic.assignment,
- count: (topic?.count ?? 0) + 1,
- });
- }
+ ]);
+ const definitions = [
+ ...(run?.topics ?? []),
+ ...assignmentRuns.flatMap((item) => item.topics),
+ ];
+ const topicByVersion = new Map(
+ definitions.map((topic) => [topic.topicVersionId, topic]),
+ );
+ const topics = new Map<
+ string,
+ { assignment: TopicAssignment; count: number }
+ >();
+ const rows = current.map(({ summary, assignment, outcome }) => {
+ const definition = assignment?.topicVersionId
+ ? topicByVersion.get(assignment.topicVersionId)
+ : undefined;
+ if (assignment?.topicId) {
+ const topic = topics.get(assignment.topicId);
+ topics.set(assignment.topicId, {
+ assignment:
+ !topic || assignment.assignedAt > topic.assignment.assignedAt
+ ? assignment
+ : topic.assignment,
+ count: (topic?.count ?? 0) + 1,
+ });
+ }
+ return {
+ facetVersion: summary.facetVersion,
+ traceId: summary.traceId,
+ unitStartTime: summary.unitStartTime,
+ summary: summary.summary,
+ outcome,
+ topicId: assignment?.topicId ?? null,
+ topicName: definition?.name ?? null,
+ };
+ });
+ // Current map names take precedence; retired IDs keep their latest referenced name.
+ const currentTopics = new Map(
+ run?.topics.map((topic) => [topic.topicId, topic]),
+ );
return {
- facetVersion: summary.facetVersion,
- traceId: summary.traceId,
- unitStartTime: summary.unitStartTime,
- summary: summary.summary,
- outcome,
- topicId: assignment?.topicId ?? null,
- topicName: definition?.name ?? null,
+ facetId: facet.id,
+ name: facet.name,
+ facetVersion: version?.version ?? null,
+ usableCount: latestSummaries.filter(
+ (row) =>
+ row.facetVersion === version?.version && row.state === "complete",
+ ).length,
+ awaitingCount: rows.filter((row) => row.outcome === "awaiting_map")
+ .length,
+ topics: [...topics]
+ .sort(
+ ([, a], [, b]) =>
+ b.count - a.count ||
+ b.assignment.assignedAt.localeCompare(a.assignment.assignedAt),
+ )
+ .map(([id, topic]) => {
+ const definition =
+ currentTopics.get(id) ??
+ topicByVersion.get(topic.assignment.topicVersionId ?? "");
+ return {
+ id,
+ name: definition?.name ?? "Unavailable topic",
+ description: definition?.description ?? "",
+ count: topic.count,
+ };
+ }),
+ rows,
+ map: run
+ ? {
+ runId: run.id,
+ topics: run.topics.map((topic) => ({
+ id: topic.topicId,
+ name: topic.name,
+ description: topic.description,
+ })),
+ }
+ : null,
};
- });
- // Current map names take precedence; retired IDs keep their latest referenced name.
- const currentTopics = new Map(
- run?.topics.map((topic) => [topic.topicId, topic]),
- );
- return {
- facetId: facet.id,
- name: facet.name,
- facetVersion: version?.version ?? null,
- usableCount: latestSummaries.filter(
- (row) =>
- row.facetVersion === version?.version && row.state === "complete",
- ).length,
- awaitingCount: rows.filter((row) => row.outcome === "awaiting_map")
- .length,
- topics: [...topics]
- .sort(
- ([, a], [, b]) =>
- b.count - a.count ||
- b.assignment.assignedAt.localeCompare(a.assignment.assignedAt),
- )
- .map(([id, topic]) => {
- const definition =
- currentTopics.get(id) ??
- topicByVersion.get(topic.assignment.topicVersionId ?? "");
- return {
- id,
- name: definition?.name ?? "Unavailable topic",
- description: definition?.description ?? "",
- count: topic.count,
- };
- }),
- rows,
- map: run
- ? {
- runId: run.id,
- topics: run.topics.map((topic) => ({
- id: topic.topicId,
- name: topic.name,
- description: topic.description,
- })),
- }
- : null,
- };
- }),
+ }),
);
return { hasStoredSummaries, facets: currentFacets };
}
diff --git a/web/src/features/topics/server/topicsRouter.ts b/web/src/features/topics/server/topicsRouter.ts
index 371e42c38d..087ddaef59 100644
--- a/web/src/features/topics/server/topicsRouter.ts
+++ b/web/src/features/topics/server/topicsRouter.ts
@@ -1,12 +1,16 @@
import { z } from "zod";
import { createHash } from "node:crypto";
import { InvalidRequestError, LangfuseNotFoundError } from "@langfuse/shared";
-import { getTracingReadinessSettings } from "@langfuse/shared/src/server";
+import {
+ getTopicSetupTraceCounts,
+ getTracingReadinessSettings,
+} from "@langfuse/shared/src/server";
import {
topicEmbeddingConfigSchema,
topicExecutionInputSchema,
topicIdSchema,
topicFacetRefSchema,
+ topicProcessingConfigSchema,
topicRuleConfigSchema,
topicTimeRangeSchema,
topicsAutomaticProcessingSchema,
@@ -55,13 +59,18 @@ import { canViewTopics } from "@/src/features/topics/topicsAccess";
import {
previewTopicTraces,
resolveTopicTraceSelection,
+ selectTopicSetupTraces,
+ topicSetupIdleMs,
topicTraceSelectionSchema,
topicTriggerInputSchema,
} from "./traceSelection";
import { currentTopicResults } from "./currentResults";
+import { TOPIC_SETUP_BACKFILL_LIMIT } from "../topic-setup-flow";
const projectInput = z.object({ projectId: topicIdSchema });
+// Covers the gap between a trace's first and last event (p99 is about 4 minutes).
+const SETUP_TRACE_SPAN_MARGIN_MS = 5 * 60_000;
const executionInput = projectInput.extend({ executionId: topicIdSchema });
const resultInput = projectInput.extend({
timeRange: topicTimeRangeSchema.refine(
@@ -455,6 +464,101 @@ export const topicsRouter = createTRPCRouter({
if (!execution) throw new LangfuseNotFoundError("Execution not found.");
return readTopicExecutionTraceErrors(execution);
}),
+ /** Topics each grouping run published, for the setup progress view. */
+ setupTopicCounts: topicsProcedure
+ .input(projectInput.extend({ runIds: z.array(topicIdSchema).max(20) }))
+ .query(async ({ input, ctx }) => {
+ const runs = await ctx.prisma.topicClusteringRun.findMany({
+ where: { projectId: input.projectId, id: { in: input.runIds } },
+ select: { id: true, topicVersionIds: true },
+ });
+ return runs.map((run) => ({
+ runId: run.id,
+ topicCount: run.topicVersionIds.length,
+ }));
+ }),
+ setupTraceCounts: topicsProcedure.query(async ({ input }) =>
+ getTopicSetupTraceCounts({
+ projectId: input.projectId,
+ idleMs: await topicSetupIdleMs(input.projectId),
+ now: new Date(),
+ }),
+ ),
+ /** Summarizes recent finished traces, then generates topics over the same window. */
+ startSetup: topicsWriteProcedure
+ .input(
+ projectInput.extend({
+ requestId: topicIdSchema,
+ sampling: z.number().gt(0).max(1),
+ facets: z.array(topicFacetRefSchema).min(1),
+ }),
+ )
+ .mutation(async ({ input, ctx }) => {
+ const { summary, embedding } = await requireTopicsModels(input.projectId);
+ // Hash only the caller's input: a retry must find this run, not re-select traces.
+ const requestHash = createHash("sha256")
+ .update(
+ JSON.stringify({
+ input,
+ summaryModel: summary.model,
+ embeddingModel: embedding.model,
+ }),
+ )
+ .digest("hex");
+ const existing = await readTopicExecutionForRequest(
+ input.projectId,
+ input.requestId,
+ requestHash,
+ );
+ if (existing) {
+ if (existing.status === "queued")
+ await enqueueTopicExecution(input.projectId, existing.id);
+ return {
+ id: existing.id,
+ traceCount: existing.facets[0]?.counts.requested ?? 0,
+ };
+ }
+ await requireFacetVersions(input.projectId, input.facets);
+ const now = new Date();
+ const { traceIds, days } = await selectTopicSetupTraces({
+ projectId: input.projectId,
+ sampling: input.sampling,
+ limit: TOPIC_SETUP_BACKFILL_LIMIT,
+ now,
+ });
+ if (traceIds.length === 0) return { id: null, traceCount: 0 };
+ const execution = await createTopicExecution(
+ {
+ operation: "process",
+ projectId: input.projectId,
+ requestId: input.requestId,
+ facets: input.facets,
+ traceIds,
+ reuseExistingSummaries: true,
+ embeddingConfig: {
+ embeddingModel: embedding.model,
+ embeddingDimensions: embedding.dimensions,
+ },
+ processingConfig: topicProcessingConfigSchema.parse({
+ summaryModel: summary.model,
+ }),
+ thenUpdate: {
+ // Summaries date from a trace's first event, which can precede the window.
+ timeRange: {
+ from: new Date(
+ now.getTime() - days * 86_400_000 - SETUP_TRACE_SPAN_MARGIN_MS,
+ ),
+ to: now,
+ },
+ },
+ },
+ requestHash,
+ ctx.session.user.id,
+ );
+ if (execution.status === "queued")
+ await enqueueTopicExecution(input.projectId, execution.id, traceIds);
+ return { id: execution.id, traceCount: traceIds.length };
+ }),
trigger: topicsWriteProcedure
.input(topicTriggerInputSchema)
.mutation(async ({ input, ctx }) => {
diff --git a/web/src/features/topics/server/traceSelection.ts b/web/src/features/topics/server/traceSelection.ts
index f6ad40bca0..d8b1f746a0 100644
--- a/web/src/features/topics/server/traceSelection.ts
+++ b/web/src/features/topics/server/traceSelection.ts
@@ -9,6 +9,10 @@ import {
} from "@langfuse/shared/topics";
import type { PrismaClient } from "@langfuse/shared/src/db";
import { selectTopicTraceRows } from "@langfuse/shared/topics/server";
+import {
+ getTopicSetupTraceIds,
+ getTracingReadinessSettings,
+} from "@langfuse/shared/src/server";
export const topicTraceSelectionSchema =
topicTraceSelectionCriteriaSchema.safeExtend({
@@ -93,3 +97,29 @@ export async function resolveTopicTraceSelection(
traceIds,
});
}
+
+// Fallback when a project has no stored idle time; the setup flow always saves one.
+const SETUP_DEFAULT_IDLE_MS = 600_000;
+const SETUP_WINDOWS = [1, 7] as const;
+
+export async function topicSetupIdleMs(projectId: string) {
+ const { traceIdleMs } = await getTracingReadinessSettings(projectId);
+ return traceIdleMs ?? SETUP_DEFAULT_IDLE_MS;
+}
+
+/** Widens from 1 to 7 days until the limit is met, so busy projects scan one day. */
+export async function selectTopicSetupTraces(input: {
+ projectId: string;
+ sampling: number;
+ limit: number;
+ now: Date;
+}) {
+ const idleMs = await topicSetupIdleMs(input.projectId);
+ let traceIds: string[] = [];
+ let days: (typeof SETUP_WINDOWS)[number] = 1;
+ for (days of SETUP_WINDOWS) {
+ traceIds = await getTopicSetupTraceIds({ ...input, idleMs, days });
+ if (traceIds.length >= input.limit) break;
+ }
+ return { traceIds, days };
+}
diff --git a/web/src/features/topics/topic-setup-flow.clienttest.ts b/web/src/features/topics/topic-setup-flow.clienttest.ts
new file mode 100644
index 0000000000..6696fc2849
--- /dev/null
+++ b/web/src/features/topics/topic-setup-flow.clienttest.ts
@@ -0,0 +1,196 @@
+import { expect, it } from "vitest";
+import {
+ canLeaveTopicQuestions,
+ followsTopicRecommendations,
+ initialTopicSetupDraft,
+ isRecommendedTopicModel,
+ pickRecommendedTopicModel,
+ setupWaitMinutes,
+ topicSetupRun,
+ topicSetupStage,
+ topicConnectionCaption,
+} from "./topic-setup-flow";
+
+const connections = [
+ {
+ id: "bedrock",
+ provider: "Amazon Bedrock",
+ adapter: "bedrock",
+ customModels: [
+ "us.anthropic.claude-haiku-5-5",
+ "us.anthropic.claude-sonnet-5-5",
+ "us.cohere.embed-v4:0",
+ ],
+ withDefaultModels: false,
+ },
+];
+
+it("recommends only the listed models for each job", () => {
+ expect(isRecommendedTopicModel("summary", "us.openai.gpt-6-luna")).toBe(true);
+ expect(
+ isRecommendedTopicModel("summary", "eu.anthropic.claude-haiku-4-5"),
+ ).toBe(false);
+ expect(isRecommendedTopicModel("summary", "gemini-2.5-pro")).toBe(false);
+ expect(isRecommendedTopicModel("embedding", "us.cohere.embed-v4:0")).toBe(
+ true,
+ );
+ expect(
+ isRecommendedTopicModel("embedding", "amazon.titan-embed-text-v2:0"),
+ ).toBe(false);
+ expect(
+ isRecommendedTopicModel("clustering", "us.anthropic.claude-sonnet-5-5"),
+ ).toBe(true);
+ expect(
+ isRecommendedTopicModel("clustering", "us.anthropic.claude-opus-5-5"),
+ ).toBe(false);
+});
+
+it("preselects the highest-ranked recommended model across connections", () => {
+ expect(
+ pickRecommendedTopicModel("summary", [
+ ["Anthropic", ["claude-sonnet-5-5", "claude-haiku-5-5"]],
+ ["Amazon Bedrock", ["us.openai.gpt-6-luna"]],
+ ]),
+ ).toEqual({ provider: "Amazon Bedrock", model: "us.openai.gpt-6-luna" });
+ expect(
+ pickRecommendedTopicModel("clustering", [
+ ["OpenAI", ["gpt-6-sol", "gpt-6.1-sol", "claude-sonnet-5-5"]],
+ ]),
+ ).toEqual({ provider: "OpenAI", model: "gpt-6.1-sol" });
+});
+
+it("starts from recommended models and a 5 minute wait when the project has no topics setup", () => {
+ const draft = initialTopicSetupDraft({
+ stored: { summary: null, embedding: null, clustering: null },
+ connections,
+ facets: [],
+ enabledFacetIds: null,
+ traceIdleMs: null,
+ });
+ expect(draft.questions.every((question) => question.enabled)).toBe(true);
+ expect(
+ draft.questions.find((question) => question.name === "Intent")?.description,
+ ).toBe("What the run was asked to do.");
+ expect(draft.summary?.model).toBe("us.anthropic.claude-haiku-5-5");
+ expect(draft.embedding?.model).toBe("us.cohere.embed-v4:0");
+ expect(draft.clustering?.model).toBe("us.anthropic.claude-sonnet-5-5");
+ expect(draft.waitMinutes).toBe(5);
+ expect(followsTopicRecommendations(draft)).toBe(true);
+});
+
+it("keeps a stored 10 minute idle time and a stored model", () => {
+ expect(setupWaitMinutes(600_000)).toBe(10);
+ expect(setupWaitMinutes(90_000)).toBe(5);
+ const draft = initialTopicSetupDraft({
+ stored: {
+ summary: {
+ llmApiKeyId: "bedrock",
+ model: "eu.anthropic.claude-sonnet-4-5",
+ },
+ embedding: null,
+ clustering: null,
+ },
+ connections,
+ facets: [
+ {
+ id: "intent",
+ name: "Intent",
+ description: "Custom intent description.",
+ isBuiltIn: true,
+ },
+ ],
+ enabledFacetIds: new Set(["intent"]),
+ traceIdleMs: 600_000,
+ });
+ expect(draft.summary?.model).toBe("eu.anthropic.claude-sonnet-4-5");
+ expect(draft.waitMinutes).toBe(10);
+ expect(
+ draft.questions.find((question) => question.name === "Intent")?.description,
+ ).toBe("Custom intent description.");
+ expect(
+ draft.questions.find((question) => question.name === "Sentiment")?.enabled,
+ ).toBe(false);
+});
+
+it("requires one question and all three models before the first step can save", () => {
+ const model = {
+ provider: "Amazon Bedrock",
+ model: "eu.anthropic.claude-haiku-4-5",
+ };
+ expect(
+ canLeaveTopicQuestions({
+ canWrite: true,
+ questions: [
+ { enabled: false },
+ { enabled: false },
+ { enabled: false },
+ { enabled: false },
+ ],
+ summary: model,
+ embedding: model,
+ clustering: model,
+ }),
+ ).toBe(false);
+ expect(
+ canLeaveTopicQuestions({
+ canWrite: true,
+ questions: [{ enabled: true }],
+ summary: model,
+ embedding: null,
+ clustering: model,
+ }),
+ ).toBe(false);
+});
+
+it("follows a setup run from summaries through its topic update", () => {
+ const now = Date.parse("2026-10-22T08:00:00Z");
+ const process = (status: string, updatedAt = now) => ({
+ id: "setup",
+ status,
+ phase: status,
+ updatedAt: new Date(updatedAt).toISOString(),
+ input: {
+ operation: "process" as const,
+ requestId: "r",
+ thenUpdate: { timeRange: {} },
+ },
+ });
+ const update = (status: string, phase = status) => ({
+ id: "topics",
+ status,
+ phase,
+ updatedAt: new Date(now).toISOString(),
+ input: { operation: "update" as const, requestId: "setup-topics" },
+ });
+ const stage = (executions: Parameters<typeof topicSetupRun>[0]) =>
+ topicSetupStage(topicSetupRun(executions)!, now);
+ expect(topicSetupRun([update("completed")])).toBeNull();
+ expect(stage([process("running")]).stage).toBe("summarizing");
+ expect(stage([process("completed")]).stage).toBe("grouping");
+ expect(stage([process("completed", now - 5 * 60_000)])).toEqual({
+ stage: "failed",
+ retryId: null,
+ });
+ expect(
+ stage([update("running", "clustering"), process("completed")]).stage,
+ ).toBe("grouping");
+ expect(stage([update("running", "naming"), process("completed")]).stage).toBe(
+ "naming",
+ );
+ expect(
+ stage([update("completed"), process("completed_with_errors")]),
+ ).toEqual({ stage: "done", retryId: null });
+ expect(stage([process("failed")])).toEqual({
+ stage: "failed",
+ retryId: "setup",
+ });
+ expect(stage([update("failed"), process("completed")])).toEqual({
+ stage: "failed",
+ retryId: "topics",
+ });
+});
+
+it("names the connection when the project has one provider", () => {
+ expect(topicConnectionCaption(connections)).toBe("Amazon Bedrock");
+ expect(topicConnectionCaption([])).toBe("No AI connection");
+});
diff --git a/web/src/features/topics/topic-setup-flow.ts b/web/src/features/topics/topic-setup-flow.ts
new file mode 100644
index 0000000000..f6e5074a0a
--- /dev/null
+++ b/web/src/features/topics/topic-setup-flow.ts
@@ -0,0 +1,369 @@
+import { LLMAdapter, supportedModels } from "@langfuse/shared";
+import { DEFAULT_TOPIC_FACETS } from "@langfuse/shared/topics";
+import { type JudgeModel } from "@/src/features/evals/v2/judgeModel";
+
+export const TOPIC_SETUP_QUESTIONS = [
+ { name: "Intent" },
+ { name: "Sentiment" },
+ { name: "Outcome" },
+ { name: "Issues" },
+] as const;
+
+export const TOPIC_SETUP_WAIT_MINUTES = [2, 5, 10] as const;
+const TOPIC_SETUP_DEFAULT_WAIT_MINUTES = 5;
+// Every setup summarizes up to this many recent traces so first topics form quickly.
+export const TOPIC_SETUP_BACKFILL_DAYS = 7;
+export const TOPIC_SETUP_BACKFILL_LIMIT = 500;
+const TOPIC_SETUP_MIN_PAST_TRACES = 100;
+
+export type TopicSetupStep = "questions" | "traces" | "review";
+export type TopicSetupWaitMinutes = (typeof TOPIC_SETUP_WAIT_MINUTES)[number];
+export type TopicSetupQuestion = {
+ name: (typeof TOPIC_SETUP_QUESTIONS)[number]["name"];
+ description: string;
+ enabled: boolean;
+};
+
+export type TopicSetupConnection = {
+ id: string;
+ provider: string;
+ adapter: string;
+ customModels: string[];
+ withDefaultModels: boolean;
+};
+
+export type TopicSetupDraft = {
+ step: TopicSetupStep;
+ questions: TopicSetupQuestion[];
+ summary: JudgeModel | null;
+ embedding: JudgeModel | null;
+ clustering: JudgeModel | null;
+ waitMinutes: TopicSetupWaitMinutes;
+ advancedOpen: boolean;
+ samplePercent: number;
+};
+
+// Best family first; within a family the highest version wins (6.1 over 6).
+const RECOMMENDED_MODELS: Record<
+ "summary" | "embedding" | "clustering",
+ RegExp[]
+> = {
+ summary: [/luna/i, /haiku-5[-.]5/i],
+ embedding: [/text-embedding-3-small/i, /embed-v4/i],
+ clustering: [/terra/i, /\bsol\b/i, /sonnet-5[-.]5/i],
+};
+
+const OPENAI_EMBEDDING_MODELS = [
+ "text-embedding-3-small",
+ "text-embedding-3-large",
+ "text-embedding-ada-002",
+];
+const GOOGLE_EMBEDDING_MODELS = [
+ "textembedding-gecko",
+ "textembedding-gecko-multilingual",
+];
+
+export function isRecommendedTopicModel(
+ slot: "summary" | "embedding" | "clustering",
+ model: string,
+) {
+ return RECOMMENDED_MODELS[slot].some((pattern) => pattern.test(model));
+}
+
+export function topicConnectionCaption(
+ connections: Array<{ provider: string }>,
+) {
+ const providers = [
+ ...new Set(connections.map((connection) => connection.provider)),
+ ];
+ if (providers.length === 0) return "No AI connection";
+ if (providers.length === 1) return providers[0]!;
+ return "Project connections";
+}
+
+export function setupWaitMinutes(
+ traceIdleMs: number | null,
+): TopicSetupWaitMinutes {
+ if (traceIdleMs == null) return TOPIC_SETUP_DEFAULT_WAIT_MINUTES;
+ const minutes = traceIdleMs / 60_000;
+ if (minutes === 2 || minutes === 5 || minutes === 10) return minutes;
+ return TOPIC_SETUP_DEFAULT_WAIT_MINUTES;
+}
+
+function topicModelsFor(
+ connection: TopicSetupConnection,
+ kind: "chat" | "embedding",
+ savedModel?: string | null,
+) {
+ const chat = connection.withDefaultModels
+ ? (supportedModels[connection.adapter as keyof typeof supportedModels] ??
+ [])
+ : [];
+ const embedding =
+ kind === "embedding" && connection.withDefaultModels
+ ? embeddingModelsFor(connection.adapter)
+ : [];
+ return Array.from(
+ new Set(
+ [savedModel, ...embedding, ...connection.customModels, ...chat].filter(
+ (model): model is string => Boolean(model),
+ ),
+ ),
+ );
+}
+
+export function topicProviderGroups(
+ connections: TopicSetupConnection[],
+ kind: "chat" | "embedding",
+ savedModel?: string | null,
+) {
+ return connections.map(
+ (connection) =>
+ [connection.provider, topicModelsFor(connection, kind, savedModel)] as [
+ string,
+ string[],
+ ],
+ );
+}
+
+export function pickRecommendedTopicModel(
+ slot: "summary" | "embedding" | "clustering",
+ providerGroups: Array<[string, string[]]>,
+): JudgeModel | null {
+ for (const pattern of RECOMMENDED_MODELS[slot]) {
+ const matches = providerGroups.flatMap(([provider, models]) =>
+ models
+ .filter((model) => pattern.test(model))
+ .map((model) => ({ provider, model })),
+ );
+ if (matches.length > 0)
+ return matches.reduce((best, item) =>
+ compareModelVersions(item.model, best.model) > 0 ? item : best,
+ );
+ }
+ return null;
+}
+
+function compareModelVersions(a: string, b: string) {
+ const left = (a.match(/\d+/g) ?? []).map(Number);
+ const right = (b.match(/\d+/g) ?? []).map(Number);
+ for (let i = 0; i < Math.max(left.length, right.length); i++) {
+ const diff = (left[i] ?? -1) - (right[i] ?? -1);
+ if (diff !== 0) return diff;
+ }
+ return 0;
+}
+
+function topicJudgeModel(
+ slot: { llmApiKeyId: string; model: string } | null,
+ connections: Array<{ id: string; provider: string }>,
+): JudgeModel | null {
+ if (!slot) return null;
+ const connection = connections.find((item) => item.id === slot.llmApiKeyId);
+ return {
+ provider: connection?.provider ?? "Missing connection",
+ model: slot.model,
+ };
+}
+
+export function resolveTopicModelSlot(
+ model: JudgeModel | null,
+ connections: TopicSetupConnection[],
+ preferredId: string | undefined,
+ kind: "chat" | "embedding",
+ savedModel?: string | null,
+) {
+ if (!model) return null;
+ const matches = connections.filter(
+ (connection) =>
+ connection.provider === model.provider &&
+ topicModelsFor(
+ connection,
+ kind,
+ connection.id === preferredId ? savedModel : null,
+ ).includes(model.model),
+ );
+ const llmApiKeyId =
+ matches.find((connection) => connection.id === preferredId)?.id ??
+ matches[0]?.id ??
+ null;
+ return llmApiKeyId ? { llmApiKeyId, model: model.model } : null;
+}
+
+export function initialTopicSetupDraft(input: {
+ stored: {
+ summary: { llmApiKeyId: string; model: string } | null;
+ embedding: { llmApiKeyId: string; model: string } | null;
+ clustering: { llmApiKeyId: string; model: string } | null;
+ };
+ connections: TopicSetupConnection[];
+ facets: Array<{
+ id: string;
+ name: string;
+ description?: string;
+ isBuiltIn: boolean;
+ }>;
+ enabledFacetIds: ReadonlySet<string> | null;
+ traceIdleMs: number | null;
+}): TopicSetupDraft {
+ const summaryGroups = topicProviderGroups(
+ input.connections,
+ "chat",
+ input.stored.summary?.model,
+ );
+ const embeddingGroups = topicProviderGroups(
+ input.connections,
+ "embedding",
+ input.stored.embedding?.model,
+ );
+ const clusteringGroups = topicProviderGroups(
+ input.connections,
+ "chat",
+ input.stored.clustering?.model,
+ );
+ return {
+ step: "questions",
+ questions: TOPIC_SETUP_QUESTIONS.map((question) => {
+ const existing = input.facets.find(
+ (facet) => facet.isBuiltIn && facet.name === question.name,
+ );
+ return {
+ name: question.name,
+ description: topicQuestionDescription(
+ question.name,
+ existing?.description,
+ ),
+ enabled: input.enabledFacetIds
+ ? Boolean(existing && input.enabledFacetIds.has(existing.id))
+ : true,
+ };
+ }),
+ summary:
+ topicJudgeModel(input.stored.summary, input.connections) ??
+ pickRecommendedTopicModel("summary", summaryGroups),
+ embedding:
+ topicJudgeModel(input.stored.embedding, input.connections) ??
+ pickRecommendedTopicModel("embedding", embeddingGroups),
+ clustering:
+ topicJudgeModel(input.stored.clustering, input.connections) ??
+ pickRecommendedTopicModel("clustering", clusteringGroups),
+ waitMinutes: setupWaitMinutes(input.traceIdleMs),
+ advancedOpen: false,
+ samplePercent: 100,
+ };
+}
+
+export function canLeaveTopicQuestions(input: {
+ canWrite: boolean;
+ questions: Array<{ enabled: boolean }>;
+ summary: JudgeModel | null;
+ embedding: JudgeModel | null;
+ clustering: JudgeModel | null;
+}) {
+ return (
+ input.canWrite &&
+ input.questions.some((question) => question.enabled) &&
+ Boolean(input.summary && input.embedding && input.clustering)
+ );
+}
+
+export function followsTopicRecommendations(input: {
+ summary: JudgeModel | null;
+ embedding: JudgeModel | null;
+ clustering: JudgeModel | null;
+}) {
+ return (
+ Boolean(
+ input.summary && isRecommendedTopicModel("summary", input.summary.model),
+ ) &&
+ Boolean(
+ input.embedding &&
+ isRecommendedTopicModel("embedding", input.embedding.model),
+ ) &&
+ Boolean(
+ input.clustering &&
+ isRecommendedTopicModel("clustering", input.clustering.model),
+ )
+ );
+}
+
+export function backfillNotice(matched: number) {
+ if (matched >= TOPIC_SETUP_MIN_PAST_TRACES) return null;
+ return `Only ${matched.toLocaleString("en-US")} traces from the last ${TOPIC_SETUP_BACKFILL_DAYS} days. Topics need about ${TOPIC_SETUP_MIN_PAST_TRACES} to form, so the first topics may take longer.`;
+}
+
+export type TopicSetupStage =
+ | "summarizing"
+ | "grouping"
+ | "naming"
+ | "done"
+ | "failed";
+
+type SetupExecution = {
+ id: string;
+ status: string;
+ phase: string;
+ updatedAt: string;
+ input:
+ | { operation: "process"; requestId: string; thenUpdate?: unknown }
+ | { operation: "update"; requestId: string };
+};
+
+/** The newest setup backfill and the topic update it starts once summaries finish. */
+export function topicSetupRun<T extends SetupExecution>(executions: T[]) {
+ const process = executions.find(
+ (execution) =>
+ execution.input.operation === "process" &&
+ Boolean(execution.input.thenUpdate),
+ );
+ if (!process) return null;
+ const update =
+ executions.find(
+ (execution) =>
+ execution.input.operation === "update" &&
+ execution.input.requestId === `${process.id}-topics`,
+ ) ?? null;
+ return { process, update };
+}
+
+// The worker starts the topic update right after the last batch; longer means it failed to start.
+const FOLLOW_UP_GRACE_MS = 2 * 60_000;
+
+export function topicSetupStage(
+ run: { process: SetupExecution; update: SetupExecution | null },
+ now: number,
+): { stage: TopicSetupStage; retryId: string | null } {
+ const { process, update } = run;
+ if (process.status === "failed")
+ return { stage: "failed", retryId: process.id };
+ if (process.status === "queued" || process.status === "running")
+ return { stage: "summarizing", retryId: null };
+ if (!update && now - Date.parse(process.updatedAt) > FOLLOW_UP_GRACE_MS)
+ return { stage: "failed", retryId: null };
+ if (!update || update.status === "queued")
+ return { stage: "grouping", retryId: null };
+ if (update.status === "failed")
+ return { stage: "failed", retryId: update.id };
+ if (update.status === "running")
+ return {
+ stage: update.phase === "clustering" ? "grouping" : "naming",
+ retryId: null,
+ };
+ return { stage: "done", retryId: null };
+}
+
+function topicQuestionDescription(name: string, stored: string | undefined) {
+ const description = stored?.trim();
+ if (description) return description;
+ return (
+ DEFAULT_TOPIC_FACETS.find((facet) => facet.name === name)?.description ?? ""
+ );
+}
+
+function embeddingModelsFor(adapter: string) {
+ if (adapter === LLMAdapter.OpenAI || adapter === LLMAdapter.Azure)
+ return OPENAI_EMBEDDING_MODELS;
+ if (adapter === LLMAdapter.GoogleAIStudio || adapter === LLMAdapter.VertexAI)
+ return GOOGLE_EMBEDDING_MODELS;
+ return [];
+}