🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(topics): show cost of runs (#18668)
Nimar564f800fb530
diff --git a/packages/shared/src/server/topics/clickhouse.test.ts b/packages/shared/src/server/topics/clickhouse.test.tsindex f4504096f1..9815b657e1 100644--- a/packages/shared/src/server/topics/clickhouse.test.ts+++ b/packages/shared/src/server/topics/clickhouse.test.ts@@ -15,6 +15,7 @@ import {readLatestTopicAssignments,getLatestFacetSummaries,getTopicSummaryCounts,+ getTopicExecutionCosts,getTopicClusteringSummaries,writeTopicSummaries,getTopicDefinitions,@@ -57,6 +58,15 @@ beforeEach(() => {mocks.cacheGet.mockResolvedValue(null);});+describe("Topics execution estimates", () => {+ it("keeps history readable when ClickHouse cannot provide an estimate", async () => {+ mocks.query.mockRejectedValue(new Error("ClickHouse unavailable"));+ expect(await getTopicExecutionCosts("project-a", ["execution-a"])).toEqual(+ {},+ );+ });+});+describe("Topics summary readiness", () => {it("returns ClickHouse readiness even when Redis reads and writes fail", async () => {mocks.cacheGet.mockRejectedValue(new Error("Redis unavailable"));diff --git a/packages/shared/src/server/topics/clickhouse.ts b/packages/shared/src/server/topics/clickhouse.tsindex 9908cee2b9..9b77803d63 100644--- a/packages/shared/src/server/topics/clickhouse.ts+++ b/packages/shared/src/server/topics/clickhouse.ts@@ -522,6 +522,35 @@ export async function hasStoredTopicSummaries(return hasSummaries;}+/** Approximate costs; retry inserts can overcount until background merges deduplicate them. */+export async function getTopicExecutionCosts(+ projectId: string,+ executionIds: string[],+): Promise<Record<string, number>> {+ if (executionIds.length === 0) return {};+ const rows = await queryClickhouse<{ executionId: string; cost: string }>({+ query: `SELECT JSONExtractString(metadata, 'executionId') AS executionId,+ sum(calculated_total_cost) AS cost+ FROM topic_facet_summaries+ WHERE project_id = {projectId:String}+ AND executionId IN {executionIds:Array(String)}+ AND notEmpty(cost_details)+ GROUP BY executionId`,+ params: { projectId, executionIds },+ tags: { route: "topics-execution-costs", projectId },+ }).catch((error: unknown) => {+ traceException(error);+ logger.warn("Failed to estimate Topics execution costs", {+ projectId,+ error,+ });+ return [];+ });+ return Object.fromEntries(+ rows.map((row) => [row.executionId, Number(row.cost)]),+ );+}+/** Count current compatible summaries without loading summary text or vectors. */export async function getTopicSummaryCounts(projectId: string,diff --git a/web/src/__tests__/server/unit/topics.servertest.ts b/web/src/__tests__/server/unit/topics.servertest.tsindex e32013038b..f7651ed717 100644--- a/web/src/__tests__/server/unit/topics.servertest.ts+++ b/web/src/__tests__/server/unit/topics.servertest.ts@@ -14,6 +14,7 @@ const mocks = vi.hoisted(() => ({createTopicExecution: vi.fn(),readTopicExecutionSummary: vi.fn(),readTopicExecutionTraceErrors: vi.fn(),+ getTopicExecutionCosts: vi.fn(),getTopicSummaryCounts: vi.fn(),readTopicExecutionForRequest: vi.fn(),listTopicExecutions: vi.fn(),@@ -197,6 +198,7 @@ beforeEach(() => {projectId,});mocks.readTopicExecutionSummary.mockResolvedValue(execution());+ mocks.getTopicExecutionCosts.mockResolvedValue({});mocks.getLatestFacetSummaries.mockResolvedValue([summary]);mocks.hasStoredTopicSummaries.mockResolvedValue(false);mocks.readTopicExecutionTraceErrors.mockResolvedValue({@@ -868,6 +870,36 @@ describe("Topics local execution access and publication", () => {expect(mocks.readTopicExecutionTraceErrors).not.toHaveBeenCalled();});+ it("projects cost estimates into history and execution polling", async () => {+ const priced = { ...execution(), id: "priced", input };+ mocks.listTopicExecutions.mockResolvedValue([+ priced,+ { ...execution(), id: "zero", input },+ { ...execution(), id: "historical", input },+ execution(),+ ]);+ mocks.getTopicExecutionCosts.mockResolvedValue({ priced: 0.012, zero: 0 });++ expect(await caller("VIEWER").executions({ projectId })).toMatchObject([+ { id: "priced", estimatedCost: 0.012 },+ { id: "zero", estimatedCost: 0 },+ { id: "historical", estimatedCost: null },+ { id: "execution-a", estimatedCost: null },+ ]);+ expect(mocks.getTopicExecutionCosts).toHaveBeenCalledExactlyOnceWith(+ projectId,+ ["priced", "zero", "historical"],+ );++ mocks.readTopicExecutionSummary.mockResolvedValue(priced);+ expect(+ await caller("VIEWER").execution({ projectId, executionId: priced.id }),+ ).toMatchObject({ estimatedCost: 0.012 });+ expect(mocks.getTopicExecutionCosts).toHaveBeenLastCalledWith(projectId, [+ priced.id,+ ]);+ });+it("reconciles interrupted history and resumes published facets after their queue owner is gone", async () => {const interrupted = execution();interrupted.status = "running";diff --git a/web/src/features/topics/TopicsPage.tsx b/web/src/features/topics/TopicsPage.tsxindex c659439ca8..64b5fe8346 100644--- a/web/src/features/topics/TopicsPage.tsx+++ b/web/src/features/topics/TopicsPage.tsx@@ -29,6 +29,7 @@ import {import { PopoverController } from "@/src/components/ui/popover";import { Badge } from "@/src/components/design-system/Badge/Badge";import { api, type RouterOutputs } from "@/src/utils/api";+import { costFormatter } from "@/src/utils/numbers";import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";import useIsFeatureEnabled from "@/src/features/feature-flags/hooks/useIsFeatureEnabled";import {@@ -499,7 +500,8 @@ function TopicsWorkspaceContent({{executionId ? "Run status" : "Past executions"}</SheetTitle><SheetDescription>- Review progress, errors, and retry interrupted runs.+ Review progress, costs, errors, and retry interrupted+ runs.</SheetDescription></SheetHeader><div className="flex min-h-0 flex-1 flex-col gap-3 overflow-y-auto">@@ -552,6 +554,14 @@ function TopicsWorkspaceContent({</span><span>{execution.facets.length} facets</span><Badge text={executionLabels[execution.status]} />+ {execution.input.operation === "process" && (+ <span className="text-muted-foreground w-full tabular-nums">+ Estimated cost:{" "}+ {estimatedExecutionCost(+ execution.estimatedCost,+ )}+ </span>+ )}</button>))}</>@@ -730,6 +740,21 @@ function ExecutionPanel({</p>)}</div>+ {!isUpdate && (+ <section className="flex flex-col gap-2 rounded-md border p-3 text-sm">+ <p className="flex items-center justify-between gap-3">+ <span className="font-bold">Estimated cost</span>+ <span className="tabular-nums">+ {estimatedExecutionCost(execution.estimatedCost)}+ </span>+ </p>+ <p className="text-muted-foreground text-xs">+ Estimated cost for this run’s currently stored summaries and+ embeddings. Costs may decrease when summaries are reused,+ reprocessed, or deleted.+ </p>+ </section>+ )}{execution.error && <ErrorMessage message={execution.error} />}{execution.facets.some((facet) => facet.counts.failed > 0) && (<details@@ -865,6 +890,10 @@ function ExecutionPanel({);}+function estimatedExecutionCost(cost: number | null) {+ return cost != null ? `~${costFormatter(cost)}` : "Unavailable";+}+function ErrorMessage({ message }: { message: string }) {return (<Alert variant="destructive">diff --git a/web/src/features/topics/server/topicsRouter.ts b/web/src/features/topics/server/topicsRouter.tsindex e2cbe50608..c80adcee29 100644--- a/web/src/features/topics/server/topicsRouter.ts+++ b/web/src/features/topics/server/topicsRouter.ts@@ -21,6 +21,7 @@ import {readTopicExecutionForRequest,readTopicExecutionSummary,readTopicExecutionTraceErrors,+ getTopicExecutionCosts,getTopicSummaryCounts,listTopicExecutions,listTopicFacets,@@ -429,16 +430,32 @@ export const topicsRouter = createTRPCRouter({input.timeRange,);}),- executions: topicsProcedure.query(async ({ input }) =>- Promise.all(+ executions: topicsProcedure.query(async ({ input }) => {+ const executions = await Promise.all((await listTopicExecutions(input.projectId)).map(recoverExecutionState),- ),- ),- execution: topicsProcedure- .input(executionInput)- .query(({ input }) =>- executionWithRecovery(input.projectId, input.executionId),- ),+ );+ const costs = await getTopicExecutionCosts(+ input.projectId,+ executions+ .filter((execution) => execution.input.operation === "process")+ .map((execution) => execution.id),+ );+ return executions.map((execution) => ({+ ...execution,+ estimatedCost: costs[execution.id] ?? null,+ }));+ }),+ execution: topicsProcedure.input(executionInput).query(async ({ input }) => {+ const execution = await executionWithRecovery(+ input.projectId,+ input.executionId,+ );+ const costs = await getTopicExecutionCosts(+ input.projectId,+ execution.input.operation === "process" ? [execution.id] : [],+ );+ return { ...execution, estimatedCost: costs[execution.id] ?? null };+ }),traceErrors: topicsProcedure.input(executionInput).query(async ({ input }) => {diff --git a/worker/src/features/topics/README.md b/worker/src/features/topics/README.mdindex 76e15a8adf..ba033b8b25 100644--- a/worker/src/features/topics/README.md+++ b/worker/src/features/topics/README.md@@ -459,6 +459,9 @@ Provider usage and calculated model costs use the events-style`provided_usage_details`, `usage_details`, `provided_cost_details` and `cost_details`maps. Summary and embedding keys are prefixed by stage; effective maps includecombined totals.+Manual processing summaries carry their execution ID in metadata. Run estimates+sum retained summary and embedding costs with available pricing. They exclude+topic naming and failed provider calls, and can change after reprocessing or deletion.Priced model IDs: global Bedrock text rates per million input/output tokens are $0.20/$1.20 for[Luna](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html)and $2/$12 fordiff --git a/worker/src/features/topics/processTopicsExecution.test.ts b/worker/src/features/topics/processTopicsExecution.test.tsindex 6ae1eddee7..4eba4999a1 100644--- a/worker/src/features/topics/processTopicsExecution.test.ts+++ b/worker/src/features/topics/processTopicsExecution.test.ts@@ -845,6 +845,9 @@ describe("Topics execution", () => {"reuses only compatible historical summaries ($version)",async ({ version, calls, usageDetails }) => {await processSelection("first", 1);+ expect([...state.summaries.values()]).toMatchObject([+ { metadata: { executionId: "first" } },+ ]);for (const summary of state.summaries.values())summary.transcriptVersion = version;state.loadTranscript.mockImplementation(async ({ traceId }) => ({@@ -860,6 +863,7 @@ describe("Topics execution", () => {environment: "staging",traceName: "Updated support",transcriptVersion: TOPICS_TRANSCRIPT_VERSION,+ metadata: { executionId: "second" },},]);expect([...state.summaries.values()][0].usageDetails).toEqual(diff --git a/worker/src/features/topics/processTopicsExecution.ts b/worker/src/features/topics/processTopicsExecution.tsindex 761b803112..86aa1aa4bf 100644--- a/worker/src/features/topics/processTopicsExecution.ts+++ b/worker/src/features/topics/processTopicsExecution.ts@@ -232,7 +232,10 @@ async function summarizeTrace(} else {metrics.result("summary", "insufficient_input");}- return stageSummary(metrics, execution, summary);+ return stageSummary(metrics, execution, {+ ...summary,+ metadata: { ...summary.metadata, executionId: execution.id },+ });}async function stageSummary(diff --git a/worker/src/features/topics/storage.integration.test.ts b/worker/src/features/topics/storage.integration.test.tsindex d8f8df089e..24c3052531 100644--- a/worker/src/features/topics/storage.integration.test.ts+++ b/worker/src/features/topics/storage.integration.test.ts@@ -15,6 +15,7 @@ import {getLatestFacetSummaries,getTopicClusteringSummaries,getTopicSummaryCounts,+ getTopicExecutionCosts,createTopicFacet,createTopicFacetVersion,ensureDefaultTopicFacets,@@ -385,6 +386,31 @@ describe("Topics definition persistence", () => {const projectId = randomUUID();const otherProjectId = randomUUID();+ const summary: TopicSummary = {+ projectId,+ facetId: "facet",+ facetVersion: 1,+ traceId: randomUUID(),+ sessionId: null,+ triggerType: "manual_poc",+ environment: "default",+ traceName: "trace",+ unitStartTime: "2026-08-31T23:59:59.999Z",+ state: "complete",+ summary: "old",+ embedding: [1, ...Array<number>(255).fill(0)],+ transcriptId: "poc",+ transcriptVersion: "poc",+ summaryModel: "test",+ embeddingModel: "eu.cohere.embed-v4:0",+ providedUsageDetails: {},+ usageDetails: {},+ providedCostDetails: {},+ costDetails: {},+ processedAt: "2026-09-01T00:00:00.000Z",+ metadata: {},+ };+afterAll(async () => {for (const table of ["topics",@@ -399,30 +425,6 @@ describe("Topics definition persistence", () => {});it("partitions by source time while preserving replacement and published map identities", async () => {- const summary: TopicSummary = {- projectId,- facetId: "facet",- facetVersion: 1,- traceId: randomUUID(),- sessionId: null,- triggerType: "manual_poc",- environment: "default",- traceName: "trace",- unitStartTime: "2026-08-31T23:59:59.999Z",- state: "complete",- summary: "old",- embedding: [1, ...Array<number>(255).fill(0)],- transcriptId: "poc",- transcriptVersion: "poc",- summaryModel: "test",- embeddingModel: "eu.cohere.embed-v4:0",- providedUsageDetails: {},- usageDetails: {},- providedCostDetails: {},- costDetails: {},- processedAt: "2026-09-01T00:00:00.000Z",- metadata: {},- };const latest = {...summary,summary: "latest",@@ -677,6 +679,42 @@ describe("Topics definition persistence", () => {]);});+ it("sums priced summary and embedding costs scoped to the project and executions", async () => {+ const priced: TopicSummary = {+ ...summary,+ facetId: randomUUID(),+ costDetails: { summary_input: 0.1, summary_output: 0.2, total: 0.3 },+ metadata: { executionId: "priced" },+ };+ await writeTopicSummaries([+ priced,+ { ...priced, projectId: otherProjectId },+ { ...priced, traceId: "other-run", metadata: { executionId: "other" } },+ {+ ...priced,+ facetId: randomUUID(),+ costDetails: { embedding_input: 0.05, total: 0.05 },+ },+ {+ ...priced,+ traceId: "unpriced",+ costDetails: {},+ metadata: { executionId: "unpriced" },+ },+ {+ ...priced,+ traceId: "zero",+ costDetails: { total: 0 },+ metadata: { executionId: "zero" },+ },+ ]);+ const requested = ["priced", "unpriced", "zero"];+ expect(await getTopicExecutionCosts(projectId, requested)).toEqual({+ priced: 0.35,+ zero: 0,+ });+ });+it("round-trips Float64 geometry and normalized source references through ClickHouse", async () => {const topic: TopicDefinition = {projectId,