Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(topics): show cost of runs (#18668)

Nimar564f800fb530
diff --git a/packages/shared/src/server/topics/clickhouse.test.ts b/packages/shared/src/server/topics/clickhouse.test.ts
index f4504096f1..9815b657e1 100644
--- a/packages/shared/src/server/topics/clickhouse.test.ts
+++ b/packages/shared/src/server/topics/clickhouse.test.ts
@@ -15,6 +15,7 @@ import {
readLatestTopicAssignments,
getLatestFacetSummaries,
getTopicSummaryCounts,
+ getTopicExecutionCosts,
getTopicClusteringSummaries,
writeTopicSummaries,
getTopicDefinitions,
@@ -57,6 +58,15 @@ beforeEach(() => {
mocks.cacheGet.mockResolvedValue(null);
});
+describe("Topics execution estimates", () => {
+ it("keeps history readable when ClickHouse cannot provide an estimate", async () => {
+ mocks.query.mockRejectedValue(new Error("ClickHouse unavailable"));
+ expect(await getTopicExecutionCosts("project-a", ["execution-a"])).toEqual(
+ {},
+ );
+ });
+});
+
describe("Topics summary readiness", () => {
it("returns ClickHouse readiness even when Redis reads and writes fail", async () => {
mocks.cacheGet.mockRejectedValue(new Error("Redis unavailable"));
diff --git a/packages/shared/src/server/topics/clickhouse.ts b/packages/shared/src/server/topics/clickhouse.ts
index 9908cee2b9..9b77803d63 100644
--- a/packages/shared/src/server/topics/clickhouse.ts
+++ b/packages/shared/src/server/topics/clickhouse.ts
@@ -522,6 +522,35 @@ export async function hasStoredTopicSummaries(
return hasSummaries;
}
+/** Approximate costs; retry inserts can overcount until background merges deduplicate them. */
+export async function getTopicExecutionCosts(
+ projectId: string,
+ executionIds: string[],
+): Promise<Record<string, number>> {
+ if (executionIds.length === 0) return {};
+ const rows = await queryClickhouse<{ executionId: string; cost: string }>({
+ query: `SELECT JSONExtractString(metadata, 'executionId') AS executionId,
+ sum(calculated_total_cost) AS cost
+ FROM topic_facet_summaries
+ WHERE project_id = {projectId:String}
+ AND executionId IN {executionIds:Array(String)}
+ AND notEmpty(cost_details)
+ GROUP BY executionId`,
+ params: { projectId, executionIds },
+ tags: { route: "topics-execution-costs", projectId },
+ }).catch((error: unknown) => {
+ traceException(error);
+ logger.warn("Failed to estimate Topics execution costs", {
+ projectId,
+ error,
+ });
+ return [];
+ });
+ return Object.fromEntries(
+ rows.map((row) => [row.executionId, Number(row.cost)]),
+ );
+}
+
/** Count current compatible summaries without loading summary text or vectors. */
export async function getTopicSummaryCounts(
projectId: string,
diff --git a/web/src/__tests__/server/unit/topics.servertest.ts b/web/src/__tests__/server/unit/topics.servertest.ts
index e32013038b..f7651ed717 100644
--- a/web/src/__tests__/server/unit/topics.servertest.ts
+++ b/web/src/__tests__/server/unit/topics.servertest.ts
@@ -14,6 +14,7 @@ const mocks = vi.hoisted(() => ({
createTopicExecution: vi.fn(),
readTopicExecutionSummary: vi.fn(),
readTopicExecutionTraceErrors: vi.fn(),
+ getTopicExecutionCosts: vi.fn(),
getTopicSummaryCounts: vi.fn(),
readTopicExecutionForRequest: vi.fn(),
listTopicExecutions: vi.fn(),
@@ -197,6 +198,7 @@ beforeEach(() => {
projectId,
});
mocks.readTopicExecutionSummary.mockResolvedValue(execution());
+ mocks.getTopicExecutionCosts.mockResolvedValue({});
mocks.getLatestFacetSummaries.mockResolvedValue([summary]);
mocks.hasStoredTopicSummaries.mockResolvedValue(false);
mocks.readTopicExecutionTraceErrors.mockResolvedValue({
@@ -868,6 +870,36 @@ describe("Topics local execution access and publication", () => {
expect(mocks.readTopicExecutionTraceErrors).not.toHaveBeenCalled();
});
+ it("projects cost estimates into history and execution polling", async () => {
+ const priced = { ...execution(), id: "priced", input };
+ mocks.listTopicExecutions.mockResolvedValue([
+ priced,
+ { ...execution(), id: "zero", input },
+ { ...execution(), id: "historical", input },
+ execution(),
+ ]);
+ mocks.getTopicExecutionCosts.mockResolvedValue({ priced: 0.012, zero: 0 });
+
+ expect(await caller("VIEWER").executions({ projectId })).toMatchObject([
+ { id: "priced", estimatedCost: 0.012 },
+ { id: "zero", estimatedCost: 0 },
+ { id: "historical", estimatedCost: null },
+ { id: "execution-a", estimatedCost: null },
+ ]);
+ expect(mocks.getTopicExecutionCosts).toHaveBeenCalledExactlyOnceWith(
+ projectId,
+ ["priced", "zero", "historical"],
+ );
+
+ mocks.readTopicExecutionSummary.mockResolvedValue(priced);
+ expect(
+ await caller("VIEWER").execution({ projectId, executionId: priced.id }),
+ ).toMatchObject({ estimatedCost: 0.012 });
+ expect(mocks.getTopicExecutionCosts).toHaveBeenLastCalledWith(projectId, [
+ priced.id,
+ ]);
+ });
+
it("reconciles interrupted history and resumes published facets after their queue owner is gone", async () => {
const interrupted = execution();
interrupted.status = "running";
diff --git a/web/src/features/topics/TopicsPage.tsx b/web/src/features/topics/TopicsPage.tsx
index c659439ca8..64b5fe8346 100644
--- a/web/src/features/topics/TopicsPage.tsx
+++ b/web/src/features/topics/TopicsPage.tsx
@@ -29,6 +29,7 @@ import {
import { PopoverController } from "@/src/components/ui/popover";
import { Badge } from "@/src/components/design-system/Badge/Badge";
import { api, type RouterOutputs } from "@/src/utils/api";
+import { costFormatter } from "@/src/utils/numbers";
import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
import useIsFeatureEnabled from "@/src/features/feature-flags/hooks/useIsFeatureEnabled";
import {
@@ -499,7 +500,8 @@ function TopicsWorkspaceContent({
{executionId ? "Run status" : "Past executions"}
</SheetTitle>
<SheetDescription>
- Review progress, errors, and retry interrupted runs.
+ Review progress, costs, errors, and retry interrupted
+ runs.
</SheetDescription>
</SheetHeader>
<div className="flex min-h-0 flex-1 flex-col gap-3 overflow-y-auto">
@@ -552,6 +554,14 @@ function TopicsWorkspaceContent({
</span>
<span>{execution.facets.length} facets</span>
<Badge text={executionLabels[execution.status]} />
+ {execution.input.operation === "process" && (
+ <span className="text-muted-foreground w-full tabular-nums">
+ Estimated cost:{" "}
+ {estimatedExecutionCost(
+ execution.estimatedCost,
+ )}
+ </span>
+ )}
</button>
))}
</>
@@ -730,6 +740,21 @@ function ExecutionPanel({
</p>
)}
</div>
+ {!isUpdate && (
+ <section className="flex flex-col gap-2 rounded-md border p-3 text-sm">
+ <p className="flex items-center justify-between gap-3">
+ <span className="font-bold">Estimated cost</span>
+ <span className="tabular-nums">
+ {estimatedExecutionCost(execution.estimatedCost)}
+ </span>
+ </p>
+ <p className="text-muted-foreground text-xs">
+ Estimated cost for this run’s currently stored summaries and
+ embeddings. Costs may decrease when summaries are reused,
+ reprocessed, or deleted.
+ </p>
+ </section>
+ )}
{execution.error && <ErrorMessage message={execution.error} />}
{execution.facets.some((facet) => facet.counts.failed > 0) && (
<details
@@ -865,6 +890,10 @@ function ExecutionPanel({
);
}
+function estimatedExecutionCost(cost: number | null) {
+ return cost != null ? `~${costFormatter(cost)}` : "Unavailable";
+}
+
function ErrorMessage({ message }: { message: string }) {
return (
<Alert variant="destructive">
diff --git a/web/src/features/topics/server/topicsRouter.ts b/web/src/features/topics/server/topicsRouter.ts
index e2cbe50608..c80adcee29 100644
--- a/web/src/features/topics/server/topicsRouter.ts
+++ b/web/src/features/topics/server/topicsRouter.ts
@@ -21,6 +21,7 @@ import {
readTopicExecutionForRequest,
readTopicExecutionSummary,
readTopicExecutionTraceErrors,
+ getTopicExecutionCosts,
getTopicSummaryCounts,
listTopicExecutions,
listTopicFacets,
@@ -429,16 +430,32 @@ export const topicsRouter = createTRPCRouter({
input.timeRange,
);
}),
- executions: topicsProcedure.query(async ({ input }) =>
- Promise.all(
+ executions: topicsProcedure.query(async ({ input }) => {
+ const executions = await Promise.all(
(await listTopicExecutions(input.projectId)).map(recoverExecutionState),
- ),
- ),
- execution: topicsProcedure
- .input(executionInput)
- .query(({ input }) =>
- executionWithRecovery(input.projectId, input.executionId),
- ),
+ );
+ const costs = await getTopicExecutionCosts(
+ input.projectId,
+ executions
+ .filter((execution) => execution.input.operation === "process")
+ .map((execution) => execution.id),
+ );
+ return executions.map((execution) => ({
+ ...execution,
+ estimatedCost: costs[execution.id] ?? null,
+ }));
+ }),
+ execution: topicsProcedure.input(executionInput).query(async ({ input }) => {
+ const execution = await executionWithRecovery(
+ input.projectId,
+ input.executionId,
+ );
+ const costs = await getTopicExecutionCosts(
+ input.projectId,
+ execution.input.operation === "process" ? [execution.id] : [],
+ );
+ return { ...execution, estimatedCost: costs[execution.id] ?? null };
+ }),
traceErrors: topicsProcedure
.input(executionInput)
.query(async ({ input }) => {
diff --git a/worker/src/features/topics/README.md b/worker/src/features/topics/README.md
index 76e15a8adf..ba033b8b25 100644
--- a/worker/src/features/topics/README.md
+++ b/worker/src/features/topics/README.md
@@ -459,6 +459,9 @@ Provider usage and calculated model costs use the events-style
`provided_usage_details`, `usage_details`, `provided_cost_details` and `cost_details`
maps. Summary and embedding keys are prefixed by stage; effective maps include
combined totals.
+Manual processing summaries carry their execution ID in metadata. Run estimates
+sum retained summary and embedding costs with available pricing. They exclude
+topic naming and failed provider calls, and can change after reprocessing or deletion.
Priced model IDs: global Bedrock text rates per million input/output tokens are $0.20/$1.20 for
[Luna](https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html)
and $2/$12 for
diff --git a/worker/src/features/topics/processTopicsExecution.test.ts b/worker/src/features/topics/processTopicsExecution.test.ts
index 6ae1eddee7..4eba4999a1 100644
--- a/worker/src/features/topics/processTopicsExecution.test.ts
+++ b/worker/src/features/topics/processTopicsExecution.test.ts
@@ -845,6 +845,9 @@ describe("Topics execution", () => {
"reuses only compatible historical summaries ($version)",
async ({ version, calls, usageDetails }) => {
await processSelection("first", 1);
+ expect([...state.summaries.values()]).toMatchObject([
+ { metadata: { executionId: "first" } },
+ ]);
for (const summary of state.summaries.values())
summary.transcriptVersion = version;
state.loadTranscript.mockImplementation(async ({ traceId }) => ({
@@ -860,6 +863,7 @@ describe("Topics execution", () => {
environment: "staging",
traceName: "Updated support",
transcriptVersion: TOPICS_TRANSCRIPT_VERSION,
+ metadata: { executionId: "second" },
},
]);
expect([...state.summaries.values()][0].usageDetails).toEqual(
diff --git a/worker/src/features/topics/processTopicsExecution.ts b/worker/src/features/topics/processTopicsExecution.ts
index 761b803112..86aa1aa4bf 100644
--- a/worker/src/features/topics/processTopicsExecution.ts
+++ b/worker/src/features/topics/processTopicsExecution.ts
@@ -232,7 +232,10 @@ async function summarizeTrace(
} else {
metrics.result("summary", "insufficient_input");
}
- return stageSummary(metrics, execution, summary);
+ return stageSummary(metrics, execution, {
+ ...summary,
+ metadata: { ...summary.metadata, executionId: execution.id },
+ });
}
async function stageSummary(
diff --git a/worker/src/features/topics/storage.integration.test.ts b/worker/src/features/topics/storage.integration.test.ts
index d8f8df089e..24c3052531 100644
--- a/worker/src/features/topics/storage.integration.test.ts
+++ b/worker/src/features/topics/storage.integration.test.ts
@@ -15,6 +15,7 @@ import {
getLatestFacetSummaries,
getTopicClusteringSummaries,
getTopicSummaryCounts,
+ getTopicExecutionCosts,
createTopicFacet,
createTopicFacetVersion,
ensureDefaultTopicFacets,
@@ -385,6 +386,31 @@ describe("Topics definition persistence", () => {
const projectId = randomUUID();
const otherProjectId = randomUUID();
+ const summary: TopicSummary = {
+ projectId,
+ facetId: "facet",
+ facetVersion: 1,
+ traceId: randomUUID(),
+ sessionId: null,
+ triggerType: "manual_poc",
+ environment: "default",
+ traceName: "trace",
+ unitStartTime: "2026-08-31T23:59:59.999Z",
+ state: "complete",
+ summary: "old",
+ embedding: [1, ...Array<number>(255).fill(0)],
+ transcriptId: "poc",
+ transcriptVersion: "poc",
+ summaryModel: "test",
+ embeddingModel: "eu.cohere.embed-v4:0",
+ providedUsageDetails: {},
+ usageDetails: {},
+ providedCostDetails: {},
+ costDetails: {},
+ processedAt: "2026-09-01T00:00:00.000Z",
+ metadata: {},
+ };
+
afterAll(async () => {
for (const table of [
"topics",
@@ -399,30 +425,6 @@ describe("Topics definition persistence", () => {
});
it("partitions by source time while preserving replacement and published map identities", async () => {
- const summary: TopicSummary = {
- projectId,
- facetId: "facet",
- facetVersion: 1,
- traceId: randomUUID(),
- sessionId: null,
- triggerType: "manual_poc",
- environment: "default",
- traceName: "trace",
- unitStartTime: "2026-08-31T23:59:59.999Z",
- state: "complete",
- summary: "old",
- embedding: [1, ...Array<number>(255).fill(0)],
- transcriptId: "poc",
- transcriptVersion: "poc",
- summaryModel: "test",
- embeddingModel: "eu.cohere.embed-v4:0",
- providedUsageDetails: {},
- usageDetails: {},
- providedCostDetails: {},
- costDetails: {},
- processedAt: "2026-09-01T00:00:00.000Z",
- metadata: {},
- };
const latest = {
...summary,
summary: "latest",
@@ -677,6 +679,42 @@ describe("Topics definition persistence", () => {
]);
});
+ it("sums priced summary and embedding costs scoped to the project and executions", async () => {
+ const priced: TopicSummary = {
+ ...summary,
+ facetId: randomUUID(),
+ costDetails: { summary_input: 0.1, summary_output: 0.2, total: 0.3 },
+ metadata: { executionId: "priced" },
+ };
+ await writeTopicSummaries([
+ priced,
+ { ...priced, projectId: otherProjectId },
+ { ...priced, traceId: "other-run", metadata: { executionId: "other" } },
+ {
+ ...priced,
+ facetId: randomUUID(),
+ costDetails: { embedding_input: 0.05, total: 0.05 },
+ },
+ {
+ ...priced,
+ traceId: "unpriced",
+ costDetails: {},
+ metadata: { executionId: "unpriced" },
+ },
+ {
+ ...priced,
+ traceId: "zero",
+ costDetails: { total: 0 },
+ metadata: { executionId: "zero" },
+ },
+ ]);
+ const requested = ["priced", "unpriced", "zero"];
+ expect(await getTopicExecutionCosts(projectId, requested)).toEqual({
+ priced: 0.35,
+ zero: 0,
+ });
+ });
+
it("round-trips Float64 geometry and normalized source references through ClickHouse", async () => {
const topic: TopicDefinition = {
projectId,