🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(topics): trace pipeline failures and outcomes in Datadog (#19022)
Lysander Kiesel491e58ba5591
diff --git a/packages/shared/src/server/topics/text.ts b/packages/shared/src/server/topics/text.tsindex e38d8e1b35..c615f8713b 100644--- a/packages/shared/src/server/topics/text.ts+++ b/packages/shared/src/server/topics/text.ts@@ -127,7 +127,11 @@ function topicTraceError(failure: unknown) {};}-function topicMessages(params: { system: TopicPromptPart[]; input: string }) {+/** The provider-bound messages of a Topics request; also used to measure its egress. */+export function topicMessages(params: {+ system: TopicPromptPart[];+ input: string;+}) {return [{role: "system" as const,diff --git a/worker/src/features/evaluation/evalExecutionDeps.ts b/worker/src/features/evaluation/evalExecutionDeps.tsindex 4ef05e6653..6ad86930d8 100644--- a/worker/src/features/evaluation/evalExecutionDeps.ts+++ b/worker/src/features/evaluation/evalExecutionDeps.ts@@ -25,7 +25,10 @@ import {import { UnrecoverableError } from "../../errors/UnrecoverableError";import { getEvalScoreS3StorageClient } from "./s3StorageClient";import { createInternalEventsWriter } from "../internal-tracing/createInternalEventsWriter";-import { recordExportVolume } from "../../services/exportVolumeMetric";+import {+ llmRequestEgressBytes,+ recordExportVolume,+} from "../../services/exportVolumeMetric";type StructuredOutputSchema = z.ZodObject<{reasoning: z.ZodString;@@ -159,21 +162,6 @@ export interface EvalExecutionDeps {writeInternalTrace: InternalTraceWriter;}-// Measure the schema as the JSON Schema LangChain ships, not Zod's _def.-function serializeSchemaForEgress(schema: unknown): string {- try {- return JSON.stringify(z.toJSONSchema(schema as z.ZodType));- } catch {- return JSON.stringify(schema);- }-}--function serializeProviderMessagesForEgress(messages: unknown): string {- return JSON.stringify(messages, (_key, value) =>- value instanceof Uint8Array ? Buffer.from(value).toString("base64") : value,- );-}-/*** Creates the production implementation of eval execution dependencies.* This is the default implementation used in production code.@@ -277,15 +265,10 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {// send, like the other integrations.recordExportVolume({integration: "llmaj",- bytes:- Buffer.byteLength(- serializeProviderMessagesForEgress(result.providerMessages),- "utf8",- ) +- Buffer.byteLength(- serializeSchemaForEgress(result.modelFacingOutputSchema),- "utf8",- ),+ bytes: llmRequestEgressBytes(+ result.providerMessages,+ result.modelFacingOutputSchema,+ ),projectId: params.traceSinkParams.targetProjectId,});diff --git a/worker/src/features/topics/README.md b/worker/src/features/topics/README.mdindex 2ddf850a9c..aaceb9e0cc 100644--- a/worker/src/features/topics/README.md+++ b/worker/src/features/topics/README.md@@ -515,6 +515,46 @@ entry from each waiting, paused, delayed and active state without loading jobpayloads. Retries can re-enter behind newer jobs, so this is head age, not theoldest creation time anywhere in the backlog.+Summary `.results` carry `source:model|rule` and `facet` (the built-in facet's+name, or `custom`): the model decides `applicable`, `not_applicable` or+`insufficient_input`; local rules decide `insufficient_input` (no transcript+content), `invalid_output` and `input_too_large`.++Every model call counts once in `.llm_calls{stage,adapter,status}`: `ok`,+`400`, `401`, `403`, `429`, `4xx` or `5xx` for any other code (the exact code is+on the span), or the failure reason when there is no HTTP status. One+embedding call can fan out into several provider requests (single-text Bedrock+models, the retry without dimensions), and AI SDK retries are not counted+separately. Each successful call adds its egress to the shared+`langfuse.export.serialized_bytes{integration:topics}`, measured exactly like+LLM-as-a-judge (`llmRequestEgressBytes`: serialized provider messages plus the+output JSON Schema, uncompressed); responses (mostly embedding vectors) and+failed or retried requests are not included.++### Datadog spans and logs++Spans carry tenant and request detail that metrics must not:++- `Worker.run *`: `langfuse.project.id` for every queue; Topics jobs add the+ execution ID, operation, status and phase. Trace-batch jobs add+ `langfuse.topics.traces.<outcome>` totals.+- `topics-trace-summary`: one per summarized trace (both paths), with facet+ result counts under `langfuse.topics.facets.*`.+- `topics-facet-update`: one per facet of an update, with its outcome. Children:+ `topics.clustering` (vectors, dimensions, status), `topics.naming` (one per+ group) and `topics.assignment` (also used when manual processing assigns).+ A failed facet marks its enclosing span as an error, although the run handles it.+- `topics.llm.summary|embedding|naming`: one per provider request, with model,+ adapter and prompt bytes; on failure, the HTTP status and the provider's error+ message, type, code and param.++A failed provider request logs one `Topics model request failed` warning with the+same fields. Like evals, only the AI SDK's parsed provider error (message cut to 500+characters, type, code, param) is kept; request and response bodies can contain+trace content and are never logged, so errors without that structure, such as+the direct Bedrock call, log the HTTP status only. A failed facet logs `Topics facet failed`: a warning for provider or+configuration errors, an error otherwise.+`langfuse.topics.tokens{stage:summary|embedding|naming,direction:input|output}`counts available provider-reported usage immediately after each model response,before local validation or persistence. Embeddings have input tokens only.diff --git a/worker/src/features/topics/metrics.ts b/worker/src/features/topics/metrics.tsindex ef37e8df6b..6ebc703d78 100644--- a/worker/src/features/topics/metrics.ts+++ b/worker/src/features/topics/metrics.ts@@ -14,11 +14,14 @@ type TopicStage =| "naming"| "assignment"| "storage";-type TopicResult =+export type TopicResult =| "generated"| "cached"+ | "applicable"| "not_applicable"| "insufficient_input"+ | "invalid_output"+ | "input_too_large"| "topics_found"| "no_topics"| "insufficient_data"@@ -35,6 +38,9 @@ type TopicErrorReason =| "numerical"| "unknown";+// Statuses with their own operational meaning; exact codes are on the span.+const COUNTED_HTTP_STATUSES = new Set([400, 401, 403, 429]);+/** Provider usage is counted per call, before local validation or persistence. */export function recordTopicTokenUsage(stage: "summary" | "embedding" | "naming",@@ -52,6 +58,20 @@ export function recordTopicTokenUsage(}}+/** Counts every provider request with a bounded status tag. */+export function recordTopicModelCall(+ stage: "summary" | "embedding" | "naming",+ adapter: string,+ outcome: { httpStatus?: number; reason?: string },+): void {+ const { httpStatus = 0, reason } = outcome;+ let status = reason ?? "ok";+ if (COUNTED_HTTP_STATUSES.has(httpStatus)) status = String(httpStatus);+ else if (httpStatus >= 400 && httpStatus < 500) status = "4xx";+ else if (httpStatus >= 500 && httpStatus < 600) status = "5xx";+ recordIncrement("langfuse.topics.llm_calls", 1, { stage, adapter, status });+}+/** One instance per execution attempt prevents nested catches counting an error twice. */export class TopicMetrics {private readonly reportedErrors = new WeakSet<object>();@@ -62,9 +82,19 @@ export class TopicMetrics {recordIncrement("langfuse.topics.executions", 1, { outcome });}- result(stage: TopicStage, result: TopicResult, count = 1): void {+ result(+ stage: TopicStage,+ result: TopicResult,+ count = 1,+ // Summary outcomes say whether the model or a local rule decided, and for which facet.+ tags: { source?: "model" | "rule"; facet?: string } = {},+ ): void {if (count > 0)- recordIncrement("langfuse.topics.results", count, { stage, result });+ recordIncrement("langfuse.topics.results", count, {+ stage,+ result,+ ...tags,+ });}clusteringWorkload(vectors: number, dimensions: number): void {diff --git a/worker/src/features/topics/models.test.ts b/worker/src/features/topics/models.test.tsindex f67a61f296..d6cd823438 100644--- a/worker/src/features/topics/models.test.ts+++ b/worker/src/features/topics/models.test.ts@@ -10,7 +10,8 @@ const state = vi.hoisted(() => ({traceSink: vi.fn(),}));vi.mock("@langfuse/shared/src/env", () => ({ env: state.env }));-vi.mock("@langfuse/shared/src/server", () => ({+vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({+ ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),getLangfuseAITraceSinkParams: state.traceSink,LangfuseInternalTraceEnvironment: { Topics: "langfuse-topics" },logger: { warn: vi.fn() },@@ -19,11 +20,20 @@ vi.mock("@langfuse/shared/src/server", () => ({vi.mock("@langfuse/shared/src/db", () => ({prisma: { project: { findFirst: state.project } },}));-vi.mock("@langfuse/shared/topics/server", () => ({+vi.mock("@langfuse/shared/topics/server", async (importOriginal) => ({+ topicMessages: (+ await importOriginal<typeof import("@langfuse/shared/topics/server")>()+ ).topicMessages,generateTopicText: (...args: unknown[]) => state.call(...args),generateTopicEmbeddings: (...args: unknown[]) => state.embed(...args),}));+// Token counters only; every call also records call and request-byte counters.+const tokenCalls = () =>+ state.increment.mock.calls.filter(([name]) =>+ String(name).startsWith("langfuse.topics.token"),+ );+import {embedTopicSummaries,nameTopicGroup,@@ -34,6 +44,8 @@ import {type TopicProcessingConfig,} from "@langfuse/shared/topics";import { LLMAdapter } from "@langfuse/shared";+import { logger } from "@langfuse/shared/src/server";+import { recordTopicModelCall } from "./metrics";import type { TopicsModel, TopicsModels } from "@langfuse/shared/topics/server";function slot(slot: TopicsModel["slot"], model: string): TopicsModel {@@ -272,6 +284,100 @@ describe("Topics naming boundary", () => {expect(state.call).not.toHaveBeenCalled();});+ it("logs the provider's error fields for a rejected request, never its bodies", async () => {+ // The AI SDK identifies an APICallError by these markers.+ const rejection = Object.assign(+ new Error(+ "Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead.",+ ),+ {+ [Symbol.for("vercel.ai.error")]: true,+ [Symbol.for("vercel.ai.error.AI_APICallError")]: true,+ name: "AI_APICallError",+ statusCode: 400,+ isRetryable: false,+ requestBodyValues: { messages: "secret transcript" },+ responseBody: '{"error":{"message":"secret echo"}}',+ data: {+ error: {+ type: "invalid_request_error",+ code: "unsupported_parameter",+ param: "max_tokens",+ },+ },+ },+ );+ state.call.mockRejectedValue(rejection);+ await expect(+ summarizeOne(+ topicsModels(),+ "Trace evidence.",+ topicProcessingConfigSchema.parse({+ summaryModel: "us.openai.gpt-5.6-luna",+ }),+ ),+ ).rejects.toMatchObject({ reason: "invalid_input" });+ const [message, fields] = vi.mocked(logger.warn).mock.calls.at(-1)!;+ expect(message).toBe("Topics model request failed");+ expect(fields).toMatchObject({+ "http.status_code": 400,+ "langfuse.topics.provider.message": rejection.message,+ "langfuse.topics.provider.code": "unsupported_parameter",+ "langfuse.topics.provider.param": "max_tokens",+ });+ expect(JSON.stringify(fields)).not.toMatch(/secret/);+ expect(state.increment).toHaveBeenCalledWith(+ "langfuse.topics.llm_calls",+ 1,+ {+ stage: "summary",+ adapter: LLMAdapter.Bedrock,+ status: "400",+ },+ );+ });++ it.each([+ [{}, "ok"],+ [{ httpStatus: 429, reason: "rate_limit" }, "429"],+ [{ httpStatus: 404, reason: "invalid_input" }, "4xx"],+ [{ httpStatus: 503, reason: "provider_error" }, "5xx"],+ [{ reason: "timeout" }, "timeout"],+ ])("tags a model call outcome %o as %s", (outcome, status) => {+ recordTopicModelCall("summary", LLMAdapter.OpenAI, outcome);+ expect(state.increment).toHaveBeenCalledWith(+ "langfuse.topics.llm_calls",+ 1,+ {+ stage: "summary",+ adapter: LLMAdapter.OpenAI,+ status,+ },+ );+ });++ it("logs only the status for an error whose message carries the response body", async () => {+ // The direct Bedrock call throws a plain error with the response text.+ state.call.mockRejectedValue(+ Object.assign(+ new Error('Bedrock InvokeModel failed with 400: {"secret": "echo"}'),+ { status: 400 },+ ),+ );+ await expect(+ summarizeOne(+ topicsModels(),+ "Trace evidence.",+ topicProcessingConfigSchema.parse({+ summaryModel: "us.openai.gpt-5.6-luna",+ }),+ ),+ ).rejects.toMatchObject({ reason: "invalid_input" });+ const [, fields] = vi.mocked(logger.warn).mock.calls.at(-1)!;+ expect(fields).toMatchObject({ "http.status_code": 400 });+ expect(JSON.stringify(fields)).not.toMatch(/secret/);+ });+it("keeps fallback token counts out of provider-reported usage", async () => {state.call.mockResolvedValue({output: keyed({ summary: "A billing request.", status: "applicable" }),@@ -286,7 +392,7 @@ describe("Topics naming boundary", () => {config,);expect(result.providedUsageDetails).toEqual({ summary_input: 100 });- expect(state.increment.mock.calls).toEqual([+ expect(tokenCalls()).toEqual([["langfuse.topics.tokens", 100, { stage: "summary", direction: "input" }],["langfuse.topics.token_usage_missing",@@ -534,7 +640,7 @@ describe("Topics naming boundary", () => {});await expect(nameTopicGroup(topicsModels(), evidence)).rejects.toThrow();// Provider work is billable even when local output validation rejects it.- expect(state.increment.mock.calls).toEqual([+ expect(tokenCalls()).toEqual([["langfuse.topics.tokens",100,@@ -643,11 +749,12 @@ describe("Topics naming boundary", () => {topicsModels().embedding,["An invoice request."],256,+ "project",),).rejects.toMatchObject({reason: "invalid_output",});- expect(state.increment.mock.calls).toEqual([+ expect(tokenCalls()).toEqual([["langfuse.topics.tokens",12,@@ -667,8 +774,9 @@ describe("Topics naming boundary", () => {topicsModels().embedding,["An invoice request."],256,+ "project",);- expect(state.increment.mock.calls).toEqual([+ expect(tokenCalls()).toEqual([["langfuse.topics.token_usage_missing",1,diff --git a/worker/src/features/topics/models.ts b/worker/src/features/topics/models.tsindex 20ba7987e4..a5777f2d5f 100644--- a/worker/src/features/topics/models.ts+++ b/worker/src/features/topics/models.ts@@ -5,11 +5,13 @@ import { env } from "@langfuse/shared/src/env";import {logger,getLangfuseAITraceSinkParams,+ instrumentAsync,LangfuseInternalTraceEnvironment,} from "@langfuse/shared/src/server";import {generateTopicEmbeddings,generateTopicText,+ topicMessages,type TopicPromptPart,type TopicsModel,type TopicsModels,@@ -19,7 +21,11 @@ import {type TopicFacetVersion,type TopicProcessingConfig,} from "@langfuse/shared/topics";-import { recordTopicTokenUsage } from "./metrics";+import { recordTopicModelCall, recordTopicTokenUsage } from "./metrics";+import {+ llmRequestEgressBytes,+ recordExportVolume,+} from "../../services/exportVolumeMetric";import type { TopicModelUsage } from "./summaryResult";import {TopicsProviderUnavailable,@@ -81,6 +87,66 @@ function countTopicRequestTokens(}}+/** The input exceeds the run's token limit; no model call was made. */+export class TopicInputTooLarge extends Error {}++/** One span, call counter and failure log per provider request. */+async function topicModelCall<T>(+ stage: "summary" | "embedding" | "naming",+ model: TopicsModel,+ projectId: string,+ requestBytes: number,+ context: Record<string, unknown>,+ call: () => Promise<T>,+): Promise<T> {+ return instrumentAsync(+ { name: `topics.llm.${stage}`, traceScope: "topics" },+ async (span) => {+ const fields: Record<string, string | number> = {+ "langfuse.topics.stage": stage,+ "langfuse.topics.model": model.model,+ "langfuse.topics.adapter": model.adapter,+ "langfuse.topics.request_bytes": requestBytes,+ };+ for (const [key, value] of Object.entries(context))+ if (typeof value === "string") fields[`langfuse.topics.${key}`] = value;+ span.setAttributes(fields);+ try {+ const result = await call();+ recordTopicModelCall(stage, model.adapter, {});+ // Shared egress metric; like llmaj, counted after a successful send.+ recordExportVolume({+ integration: "topics",+ bytes: requestBytes,+ projectId,+ });+ return result;+ } catch (error) {+ const failure = topicProviderError(error, model);+ const { httpStatus, message, type, code, param } = failure.details;+ recordTopicModelCall(stage, model.adapter, {+ httpStatus,+ reason: failure.reason,+ });+ const failed = {+ ...fields,+ "langfuse.topics.error.reason": failure.reason,+ "langfuse.topics.error.type":+ error instanceof Error ? error.name : "unknown",+ ...(httpStatus && { "http.status_code": httpStatus }),+ ...(message && { "langfuse.topics.provider.message": message }),+ ...(type && { "langfuse.topics.provider.type": type }),+ ...(code && { "langfuse.topics.provider.code": code }),+ ...(param && { "langfuse.topics.provider.param": param }),+ };+ span.setAttributes(failed);+ logger.warn("Topics model request failed", failed);+ throw failure;+ }+ },+ );+}+const summarySchema = z.object({summary: z.string(),status: z.enum(["applicable", "not_applicable", "insufficient_input"]),@@ -104,21 +170,26 @@ async function structuredCall<T>(stage === "naming"? TOPICS_NAMING_RATES.get(model.model)?.(tokens): TOPICS_SUMMARY_RATES.get(model.model);- const result = await generateTopicText({+ const trace = await topicTraceSink(traceName, projectId, metadata);+ const result = await topicModelCall(+ stage,model,- system,- input,- schema,- maxOutputTokens: outputLimit,- trace: await topicTraceSink(traceName, projectId, metadata),- }).catch((error: unknown) => {- logger.warn("Topics model request failed", {- model: model.model,- adapter: model.adapter,- errorType: error instanceof Error ? error.name : "unknown",- });- throw topicProviderError(error, model);- });+ projectId,+ llmRequestEgressBytes(topicMessages({ system, input }), schema),+ {+ trace_id: metadata?.source_trace_id,+ execution_id: metadata?.execution_id,+ },+ () =>+ generateTopicText({+ model,+ system,+ input,+ schema,+ maxOutputTokens: outputLimit,+ trace,+ }),+ );const actualRates = rates(result.usage.inputTokens ?? inputLimit);recordTopicTokenUsage(stage, {input: result.usage.inputTokens,@@ -300,7 +371,7 @@ ${facetBlock(true)}`,const countedInputTokens =countTopicRequestTokens(system, input, schema) + 256;if (countedInputTokens > config.maxInputTokens)- throw new Error(+ throw new TopicInputTooLarge(`The shared trace transcript and instructions are ${countedInputTokens} tokens, above this run's ${config.maxInputTokens}-token input limit. No model call was made; the transcript is never shortened per facet.`,);return structuredCall(@@ -391,6 +462,7 @@ export async function embedTopicSummaries(model: TopicsModel,summaries: string[],dimensions: number,+ projectId: string,): Promise<TopicModelUsage & { embeddings: number[][] }> {if (summaries.length === 0 ||@@ -402,13 +474,14 @@ export async function embedTopicSummaries("Topics embeddings require non-empty text and 64 to 4,096 dimensions.","invalid_input",);- const result = await generateTopicEmbeddings({+ const result = await topicModelCall(+ "embedding",model,- summaries,- dimensions,- }).catch((error: unknown) => {- throw topicProviderError(error, model);- });+ projectId,+ llmRequestEgressBytes(summaries),+ {},+ () => generateTopicEmbeddings({ model, summaries, dimensions }),+ );recordTopicTokenUsage("embedding", { input: result.tokens });// ClickHouse stores Float32; calibration and future classification must use those same vectors.const embeddings = result.embeddings.map((vector) =>diff --git a/worker/src/features/topics/numeric.ts b/worker/src/features/topics/numeric.tsindex 5d01e9af3a..b6aebae695 100644--- a/worker/src/features/topics/numeric.ts+++ b/worker/src/features/topics/numeric.ts@@ -78,14 +78,19 @@ export async function runTopicClustering(clearTimeout(timeout);reject(error);});- child.on("close", (code) => {+ child.on("close", (code, signal) => {clearTimeout(timeout);if (outputExceeded) {reject(new Error("Topics numerical fit returned oversized output"));return;}if (code !== 0) {- reject(new Error(`Topics numerical fit failed: ${errorOutput}`));+ // A kill (e.g. out of memory) leaves no stderr; the signal names it.+ reject(+ new Error(+ `Topics numerical fit failed (exit code ${code}, signal ${signal}): ${errorOutput}`,+ ),+ );return;}try {diff --git a/worker/src/features/topics/processTopicEmbeddingBatch.test.ts b/worker/src/features/topics/processTopicEmbeddingBatch.test.tsindex ec7901e951..9f8de7d1a0 100644--- a/worker/src/features/topics/processTopicEmbeddingBatch.test.ts+++ b/worker/src/features/topics/processTopicEmbeddingBatch.test.ts@@ -39,6 +39,7 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {const { isInternalEvalEnvironment } =await importOriginal<typeof import("@langfuse/shared/src/server")>();return {+ ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),isInternalEvalEnvironment,logger: { error: vi.fn() },recordIncrement: vi.fn(),@@ -146,6 +147,7 @@ describe("Topics embedding handoff", () => {expect.anything(),[allowed.summary],256,+ "project",);expect(mocks.update).toHaveBeenCalledTimes(1);expect(mocks.write).toHaveBeenCalledExactlyOnceWith([diff --git a/worker/src/features/topics/processTopicEmbeddingBatch.ts b/worker/src/features/topics/processTopicEmbeddingBatch.tsindex b085d7f8fe..d0ea496ee1 100644--- a/worker/src/features/topics/processTopicEmbeddingBatch.ts+++ b/worker/src/features/topics/processTopicEmbeddingBatch.ts@@ -70,6 +70,7 @@ export async function processTopicEmbeddingBatch(models.embedding,traceRows.map(({ summary }) => summary.summary),dimensions,+ models.projectId,);} catch (error) {throw error instanceof TopicsProviderUnavailablediff --git a/worker/src/features/topics/processTopicsExecution.test.ts b/worker/src/features/topics/processTopicsExecution.test.tsindex 92042f0ad4..0c29af5026 100644--- a/worker/src/features/topics/processTopicsExecution.test.ts+++ b/worker/src/features/topics/processTopicsExecution.test.ts@@ -55,6 +55,7 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {const { assembleTranscript, isInternalEvalEnvironment } =await importOriginal<typeof import("@langfuse/shared/src/server")>();return {+ ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),assembleTranscript,isInternalEvalEnvironment,recordIncrement: vi.fn(),@@ -324,6 +325,7 @@ vi.mock("@langfuse/shared/topics/server", async (importOriginal) => {};});vi.mock("./models", () => ({+ TopicInputTooLarge: class extends Error {},// One request per trace; `state.summarize` answers each facet, and one failure fails the request.summarizeTopicTraceFacets: async (_models: unknown,diff --git a/worker/src/features/topics/processTopicsExecution.ts b/worker/src/features/topics/processTopicsExecution.tsindex 4e553d4593..0156782650 100644--- a/worker/src/features/topics/processTopicsExecution.ts+++ b/worker/src/features/topics/processTopicsExecution.ts@@ -1,4 +1,10 @@-import { isInternalEvalEnvironment, logger } from "@langfuse/shared/src/server";+import {+ getCurrentSpan,+ instrumentAsync,+ isInternalEvalEnvironment,+ logger,+ traceException,+} from "@langfuse/shared/src/server";import {isTopicsEnabled,readTopicExecutionSummary,@@ -156,6 +162,13 @@ async function summarizeTrace(} catch (error) {if (isExecutionFailure(error) && !isTraceModelFailure(error)) throw error;metrics.error("summary", error);+ logger.warn("Topics summary failed for trace", {+ "langfuse.trace.id": traceId,+ "langfuse.topics.execution_id": execution.id,+ "langfuse.topics.error.reason":+ error instanceof TopicsProviderUnavailable ? error.reason : "other",+ error: errorMessage(error),+ });for (const facet of unresolved()) results.set(facet, error as Error);}}@@ -274,12 +287,6 @@ async function summarizePending(continue;}const output = result?.output;- if (!output) metrics.result("summary", "insufficient_input");- else- metrics.result(- "summary",- output.status === "applicable" ? "generated" : output.status,- );rows.push({facet,summary: {@@ -413,55 +420,64 @@ async function assignSummaries(coordinates: Map<string, [number, number]> = new Map(),assignedAt = new Date().toISOString(),) {- return metrics.measure("assignment", async () => {- const rows = topicAssignmentRows({- projectId: execution.projectId,- facet: { facetId: facet.facetId, version: facet.version },- embeddingConfig: execution.input.embeddingConfig,- summaries,- run,- origin: execution.input.operation === "process" ? "online" : "initial",- assignedAt,- coordinates,- });- await metrics.measure(- "storage",- () => writeTopicAssignments(rows),- "storage",- );- const visible =- execution.input.operation === "update" && rows.length- ? await readTopicAssignments(- execution.projectId,- { facetId: facet.facetId, version: facet.version },- summaries,- run.id,- summaryTimeRange(summaries),- )- : [];- const visibleByTrace = new Map(visible.map((row) => [row.traceId, row]));- if (- execution.input.operation === "update" &&- rows.some((row) => {- const persisted = visibleByTrace.get(row.traceId);- return (- !persisted ||- persisted.topicId !== row.topicId ||- persisted.topicVersionId !== row.topicVersionId ||- persisted.summaryProcessedAt !== row.summaryProcessedAt+ return instrumentAsync(+ { name: "topics.assignment", traceScope: "topics" },+ (span) => {+ span.setAttribute("langfuse.topics.summaries", summaries.length);+ return metrics.measure("assignment", async () => {+ const rows = topicAssignmentRows({+ projectId: execution.projectId,+ facet: { facetId: facet.facetId, version: facet.version },+ embeddingConfig: execution.input.embeddingConfig,+ summaries,+ run,+ origin:+ execution.input.operation === "process" ? "online" : "initial",+ assignedAt,+ coordinates,+ });+ await metrics.measure(+ "storage",+ () => writeTopicAssignments(rows),+ "storage",);- })- )- throw new Error(- "Topic assignments are not fully visible; map publication is deferred.",- );- progress.counts.assigned = rows.filter(- (row) => row.topicId !== null,- ).length;- progress.counts.outlier = rows.length - progress.counts.assigned;- metrics.result("assignment", "assigned", progress.counts.assigned);- metrics.result("assignment", "outlier", progress.counts.outlier);- });+ const visible =+ execution.input.operation === "update" && rows.length+ ? await readTopicAssignments(+ execution.projectId,+ { facetId: facet.facetId, version: facet.version },+ summaries,+ run.id,+ summaryTimeRange(summaries),+ )+ : [];+ const visibleByTrace = new Map(+ visible.map((row) => [row.traceId, row]),+ );+ if (+ execution.input.operation === "update" &&+ rows.some((row) => {+ const persisted = visibleByTrace.get(row.traceId);+ return (+ !persisted ||+ persisted.topicId !== row.topicId ||+ persisted.topicVersionId !== row.topicVersionId ||+ persisted.summaryProcessedAt !== row.summaryProcessedAt+ );+ })+ )+ throw new Error(+ "Topic assignments are not fully visible; map publication is deferred.",+ );+ progress.counts.assigned = rows.filter(+ (row) => row.topicId !== null,+ ).length;+ progress.counts.outlier = rows.length - progress.counts.assigned;+ metrics.result("assignment", "assigned", progress.counts.assigned);+ metrics.result("assignment", "outlier", progress.counts.outlier);+ });+ },+ );}async function clusterFacet(@@ -529,14 +545,26 @@ async function clusterFacet(summaries.length,execution.input.embeddingConfig.embeddingDimensions,);- const numeric = await metrics.measure(- "clustering",- () =>- runTopicClustering(- summaries.map((row) => row.embedding),- numericConfig,- ),- "numerical",+ const numeric = await instrumentAsync(+ { name: "topics.clustering", traceScope: "topics" },+ async (span) => {+ span.setAttributes({+ "langfuse.topics.clustering.vectors": summaries.length,+ "langfuse.topics.clustering.dimensions":+ execution.input.embeddingConfig.embeddingDimensions,+ });+ const result = await metrics.measure(+ "clustering",+ () =>+ runTopicClustering(+ summaries.map((row) => row.embedding),+ numericConfig,+ ),+ "numerical",+ );+ span.setAttribute("langfuse.topics.clustering.status", result.status);+ return result;+ },);const coordinates = new Map(summaries@@ -692,47 +720,56 @@ async function clusterFacet(topics.push(retained);continue;}- const label = await metrics.measure("naming", async () => {- const members = new Map(- group.members.map((summary, index) => [`m${index + 1}`, summary]),- );- const label = await nameGroupAvoidingConflicts(- metrics,- models,- named,- {- members: Array.from(members, ([id, { summary }]) => ({- id,- summary,- })),- contrasts: group.contrasts.map((member, index) => ({- id: `c${index + 1}`,- summary: member.summary,- })),- },- {- facet_id: facet.facetId,- facet_name: facetName,- facet_version: facet.version,- execution_id: execution.id,- run_id: run.id,- group_id: group.id,- },- );- return {- ...label,- representativeSummaries: label.evidenceSummaryIds.map((id) => {- const { facetId, facetVersion, traceId, sessionId } =- members.get(id)!;+ const label = await instrumentAsync(+ { name: "topics.naming", traceScope: "topics" },+ (span) => {+ span.setAttributes({+ "langfuse.topics.group_id": group.id,+ "langfuse.topics.group_members": group.members.length,+ });+ return metrics.measure("naming", async () => {+ const members = new Map(+ group.members.map((summary, index) => [`m${index + 1}`, summary]),+ );+ const label = await nameGroupAvoidingConflicts(+ metrics,+ models,+ named,+ {+ members: Array.from(members, ([id, { summary }]) => ({+ id,+ summary,+ })),+ contrasts: group.contrasts.map((member, index) => ({+ id: `c${index + 1}`,+ summary: member.summary,+ })),+ },+ {+ facet_id: facet.facetId,+ facet_name: facetName,+ facet_version: facet.version,+ execution_id: execution.id,+ run_id: run.id,+ group_id: group.id,+ },+ );return {- facetId,- facetVersion,- traceId,- sessionId: traceId === null ? sessionId : null,+ ...label,+ representativeSummaries: label.evidenceSummaryIds.map((id) => {+ const { facetId, facetVersion, traceId, sessionId } =+ members.get(id)!;+ return {+ facetId,+ facetVersion,+ traceId,+ sessionId: traceId === null ? sessionId : null,+ };+ }),};- }),- };- });+ });+ },+ );const candidate = candidates.find((topic) => topic.topicVersionId === group.id,)!;@@ -970,18 +1007,34 @@ async function processTraces(if (todo.length === 0) return;let outcomes: (TopicSummary | Error)[];try {- outcomes = await summarizeTrace(- metrics,- models,- execution,- todo.map(({ facet, name, builtIn }) => ({- version: facet,- name,- builtIn,- })),- traceId,- cached,- getTranscript,+ outcomes = await instrumentAsync(+ { name: "topics-trace-summary", traceScope: "topics" },+ async (span) => {+ span.setAttributes({+ "langfuse.trace.id": traceId,+ "langfuse.topics.execution_id": execution.id,+ });+ const results = await summarizeTrace(+ metrics,+ models,+ execution,+ todo.map(({ facet, name, builtIn }) => ({+ version: facet,+ name,+ builtIn,+ })),+ traceId,+ cached,+ getTranscript,+ );+ span.setAttribute(+ "langfuse.topics.trace_outcome",+ results.some((result) => result instanceof Error)+ ? "failed"+ : "summarized",+ );+ return results;+ },);} catch (error) {metrics.error("summary", error);@@ -1086,10 +1139,7 @@ async function processTraces(progress.outcome = "assigned";}} catch (error) {- metrics.error("execution", error);- progress.outcome = "failed";- progress.error = errorMessage(error);- if (isExecutionFailure(error)) throw error;+ failFacet(metrics, execution, facet, progress, error);}await save("processing");}@@ -1132,76 +1182,115 @@ async function updateTopics(]),);for (const { facet, progress } of await pendingFacets(execution)) {- try {- const accepted = progress.runId- ? await getTopicRun(execution.projectId, progress.runId)- : null;- if (- accepted &&- (accepted.projectId !== execution.projectId ||- accepted.facetId !== facet.facetId ||- accepted.facetVersion !== facet.version)- )- throw new Error("Clustering run does not match this facet version.");- if (accepted?.status === "completed") {- const assignments = await readTopicMapAssignments(- execution.projectId,- { facetId: facet.facetId, version: facet.version },- accepted.id,- topicTimeRangeSchema.parse(accepted.config.timeRange),- );- progress.counts.requested = assignments.length;- progress.counts.complete = assignments.length;- progress.counts.assigned = assignments.filter(- (row) => row.topicId !== null,- ).length;- progress.counts.outlier = assignments.length - progress.counts.assigned;- progress.outcome = accepted.topics.length ? "published" : "no_topics";- await saveProgress(metrics, execution, "processing");- continue;- }- if (accepted?.status === "skipped") {- progress.outcome = progress.counts.requested- ? "insufficient_data"- : "no_applicable_summaries";+ await instrumentAsync(+ { name: "topics-facet-update", traceScope: "topics" },+ async (span) => {+ span.setAttributes({+ "langfuse.topics.execution_id": execution.id,+ "langfuse.topics.facet_id": facet.facetId,+ "langfuse.topics.facet_version": facet.version,+ });+ try {+ const accepted = progress.runId+ ? await getTopicRun(execution.projectId, progress.runId)+ : null;+ if (+ accepted &&+ (accepted.projectId !== execution.projectId ||+ accepted.facetId !== facet.facetId ||+ accepted.facetVersion !== facet.version)+ )+ throw new Error(+ "Clustering run does not match this facet version.",+ );+ if (accepted?.status === "completed") {+ const assignments = await readTopicMapAssignments(+ execution.projectId,+ { facetId: facet.facetId, version: facet.version },+ accepted.id,+ topicTimeRangeSchema.parse(accepted.config.timeRange),+ );+ progress.counts.requested = assignments.length;+ progress.counts.complete = assignments.length;+ progress.counts.assigned = assignments.filter(+ (row) => row.topicId !== null,+ ).length;+ progress.counts.outlier =+ assignments.length - progress.counts.assigned;+ progress.outcome = accepted.topics.length+ ? "published"+ : "no_topics";+ await saveProgress(metrics, execution, "processing");+ return;+ }+ if (accepted?.status === "skipped") {+ progress.outcome = progress.counts.requested+ ? "insufficient_data"+ : "no_applicable_summaries";+ await saveProgress(metrics, execution, "processing");+ return;+ }+ const current = await getPublishedTopicRun(+ execution.projectId,+ facet.facetId,+ );+ const previous =+ current?.facetVersion === facet.version ? current : null;+ const summaries = await getTopicClusteringSummaries(+ execution.projectId,+ facet.facetId,+ facet.version,+ execution.input.embeddingConfig,+ execution.input.timeRange,+ );+ countSummaries(progress, summaries);+ progress.counts.requested = summaries.length;+ await clusterFacet(+ metrics,+ models,+ execution,+ facet,+ facetNames.get(facet.facetId),+ progress,+ summaries,+ accepted,+ previous,+ );+ } catch (error) {+ failFacet(metrics, execution, facet, progress, error);+ }await saveProgress(metrics, execution, "processing");- continue;- }- const current = await getPublishedTopicRun(- execution.projectId,- facet.facetId,- );- const previous = current?.facetVersion === facet.version ? current : null;- const summaries = await getTopicClusteringSummaries(- execution.projectId,- facet.facetId,- facet.version,- execution.input.embeddingConfig,- execution.input.timeRange,- );- countSummaries(progress, summaries);- progress.counts.requested = summaries.length;- await clusterFacet(- metrics,- models,- execution,- facet,- facetNames.get(facet.facetId),- progress,- summaries,- accepted,- previous,- );- } catch (error) {- metrics.error("execution", error);- progress.outcome = "failed";- progress.error = errorMessage(error);- if (isExecutionFailure(error)) throw error;- }- await saveProgress(metrics, execution, "processing");+ span.setAttribute("langfuse.topics.facet_outcome", progress.outcome);+ },+ );}}+/** Provider and configuration failures are the customer's to fix; anything else is ours. */+function failFacet(+ metrics: TopicMetrics,+ execution: TopicExecution,+ facet: TopicFacetVersion,+ progress: TopicFacetProgress,+ error: unknown,+) {+ metrics.error("execution", error);+ progress.outcome = "failed";+ progress.error = errorMessage(error);+ const log =+ error instanceof TopicsProviderUnavailable ? logger.warn : logger.error;+ log("Topics facet failed", {+ "langfuse.topics.execution_id": execution.id,+ "langfuse.topics.operation": execution.input.operation,+ "langfuse.topics.facet_id": facet.facetId,+ "langfuse.topics.facet_version": facet.version,+ error: progress.error,+ });+ if (isExecutionFailure(error)) throw error;+ // Handled, but still a failed facet for the user; the enclosing span reports it.+ traceException(error);+}+/** Queue batches own trace retry state; Postgres owns compact update progress. */export async function processTopicsExecution({projectId,@@ -1360,5 +1449,13 @@ export async function processTopicsExecution({execution.status = "failed";execution.error = errorMessage(error);await save("failed");+ } finally {+ getCurrentSpan()?.setAttributes({+ "langfuse.project.id": projectId,+ "langfuse.topics.execution_id": executionId,+ "langfuse.topics.operation": execution.input.operation,+ "langfuse.topics.execution_status": execution.status,+ "langfuse.topics.execution_phase": execution.phase,+ });}}diff --git a/worker/src/features/topics/provider-error.ts b/worker/src/features/topics/provider-error.tsindex 777d155270..398b17deab 100644--- a/worker/src/features/topics/provider-error.ts+++ b/worker/src/features/topics/provider-error.ts@@ -3,7 +3,7 @@ import {type TopicsModelSlotName,} from "@langfuse/shared/topics";import { EvaluatorBlockReason } from "@prisma/client";-import { logger } from "@langfuse/shared/src/server";+import { getLLMErrorInfo, logger } from "@langfuse/shared/src/server";import {pauseTopicsModels,type TopicsModel,@@ -19,11 +19,21 @@ type TopicsProviderErrorReason =| "invalid_input"| "provider_error";+/** Telemetry only; the user-facing message stays generic. */+export type TopicProviderErrorDetails = {+ httpStatus?: number;+ message?: string;+ type?: string;+ code?: string;+ param?: string;+};+export class TopicsProviderUnavailable extends Error {constructor(message: string,readonly reason: TopicsProviderErrorReason = "provider_error",readonly slot?: TopicsModelSlotName,+ readonly details: TopicProviderErrorDetails = {},) {super(message);}@@ -73,9 +83,38 @@ export function topicProviderError(`The ${TOPICS_MODEL_SLOT_DETAILS[model.slot].label.toLowerCase()} model ${model.model} on LLM connection "${model.provider}" failed${status ? ` (HTTP ${status})` : ""}. Check the connection's credentials, the model ID, and provider availability.`,reason,model.slot,+ providerErrorDetails(error, status),);}+/**+ * Like evals, keeps only the AI SDK's parsed provider error fields; plain errors+ * (e.g. the direct Bedrock call) carry the response body, so they keep the status only.+ */+function providerErrorDetails(+ error: unknown,+ status: number | undefined,+): TopicProviderErrorDetails {+ const provider = getLLMErrorInfo(error)?.providerError;+ const httpStatus = provider?.statusCode ?? status;+ if (!httpStatus) return {};+ if (!provider) return { httpStatus };+ const record = (value: unknown) =>+ value && typeof value === "object"+ ? (value as Record<string, unknown>)+ : undefined;+ const data = record(record(provider.data)?.error) ?? record(provider.data);+ const text = (value: unknown) =>+ typeof value === "string" ? value.slice(0, 500) : undefined;+ return {+ httpStatus,+ message: text(provider.message),+ type: text(data?.type),+ code: text(data?.code),+ param: text(data?.param),+ };+}+/** The provider rejected the connection's credentials or billing. */export function isRejectedConnection(error: unknown,diff --git a/worker/src/features/topics/summarizeAssembledTrace.test.ts b/worker/src/features/topics/summarizeAssembledTrace.test.tsindex 85ec59a3ee..3274c6da34 100644--- a/worker/src/features/topics/summarizeAssembledTrace.test.ts+++ b/worker/src/features/topics/summarizeAssembledTrace.test.ts@@ -20,6 +20,7 @@ vi.mock("@langfuse/shared/topics/server", () => ({TOPICS_TRANSCRIPT_VERSION: "shared-transcript-v3",}));vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({+ ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),isInternalEvalEnvironment: (await importOriginal<typeof import("@langfuse/shared/src/server")>()).isInternalEvalEnvironment,@@ -34,11 +35,13 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({) => action({ setAttributes: vi.fn(), setAttribute: state.setAttribute }),}));vi.mock("./models", () => ({+ TopicInputTooLarge: class extends Error {},summarizeTopicTraceFacets: (...args: unknown[]) => state.summarize(...args),embedTopicSummaries: (...args: unknown[]) => state.embed(...args),}));import { summarizeAssembledTrace } from "./summarizeAssembledTrace";+import { TopicsProviderUnavailable } from "./provider-error";const transcript: Transcript = {threads: [@@ -394,6 +397,33 @@ describe("summarizeAssembledTrace", () => {{ stage: "summary", reason: "invalid_output" },],]);+ // Rejected output is a local rule's decision; accepted output is the model's.+ expect(+ vi+ .mocked(recordIncrement)+ .mock.calls.filter(([name]) => name === "langfuse.topics.results"),+ ).toEqual([+ [+ "langfuse.topics.results",+ 1,+ {+ stage: "summary",+ result: "invalid_output",+ source: "rule",+ facet: "intent",+ },+ ],+ [+ "langfuse.topics.results",+ 1,+ {+ stage: "summary",+ result: "applicable",+ source: "model",+ facet: "issues",+ },+ ],+ ]);expect(state.setAttribute).toHaveBeenCalledWith("langfuse.topics.trace_outcome","failed",@@ -493,6 +523,41 @@ describe("summarizeAssembledTrace", () => {);});+ it("counts an unusable response as a rule rejection of every facet", async () => {+ state.summarize.mockRejectedValue(+ new TopicsProviderUnavailable("unparsable", "invalid_output"),+ );+ await expect(+ summarize({+ projectId: "project-a",+ traceId: "trace-1",+ traceTimestamp: "2026-09-22T12:00:00.000Z",+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),+ environment: "default",+ traceName: "agent-turn",+ transcript,+ models,+ facets: state.facets(),+ }),+ ).rejects.toMatchObject({ reason: "invalid_output" });+ expect(+ vi+ .mocked(recordIncrement)+ .mock.calls.filter(([name]) => name === "langfuse.topics.results"),+ ).toEqual([+ [+ "langfuse.topics.results",+ 1,+ {+ stage: "summary",+ result: "invalid_output",+ source: "rule",+ facet: "intent",+ },+ ],+ ]);+ });+it("propagates a provider failure for the trace outcome", async () => {const error = new Error("provider unavailable");state.summarize.mockRejectedValue(error);diff --git a/worker/src/features/topics/summarizeAssembledTrace.ts b/worker/src/features/topics/summarizeAssembledTrace.tsindex 4cc408a040..ee98ea7478 100644--- a/worker/src/features/topics/summarizeAssembledTrace.ts+++ b/worker/src/features/topics/summarizeAssembledTrace.ts@@ -4,6 +4,7 @@ import {type TopicsModels,} from "@langfuse/shared/topics/server";import {+ DEFAULT_TOPIC_FACETS,topicProcessingConfigSchema,type TopicFacet,type TopicProcessingConfig,@@ -11,13 +12,20 @@ import {type TopicSummary,} from "@langfuse/shared/topics";import {+ contextWithLangfuseProps,+ getCurrentSpan,instrumentAsync,isInternalEvalEnvironment,type Transcript,} from "@langfuse/shared/src/server";+import { context as otelContext } from "@opentelemetry/api";import { prepareAssembledTopicTranscript } from "./assembledTranscript";-import { TopicMetrics } from "./metrics";-import { embedTopicSummaries, summarizeTopicTraceFacets } from "./models";+import { TopicMetrics, type TopicResult } from "./metrics";+import {+ embedTopicSummaries,+ summarizeTopicTraceFacets,+ TopicInputTooLarge,+} from "./models";import { TopicsProviderUnavailable } from "./provider-error";import {mergeTopicModelUsage,@@ -37,6 +45,10 @@ const NO_USAGE: TopicModelUsage = {costDetails: {},};+const BUILT_IN_FACETS = new Set(+ DEFAULT_TOPIC_FACETS.map((facet) => facet.name),+);+// Rows from this transcript producer are interchangeable between trace-batch and manual processing.export const ASSEMBLED_TRANSCRIPT_ID = "trace-batch";@@ -80,29 +92,32 @@ export async function summarizeAssembledTrace(): Promise<AssembledTraceResult> {if (isInternalEvalEnvironment(input.environment))return { outcome: "disabled", rows: [] };- return instrumentAsync(- { name: "topics-trace-summary", traceScope: "topics" },- async (span) => {- span.setAttributes({- "langfuse.project.id": input.projectId,- "langfuse.trace.id": input.traceId,- });- const metrics = new TopicMetrics();- let result: AssembledTraceResult | undefined;- try {- await metrics.measure("trace", async () => {- result = await summarizeEnabledTrace(input, metrics);- if (result.failure !== undefined) throw result.failure;- });- span.setAttribute("langfuse.topics.trace_outcome", result!.outcome);- return result!;- } catch (error) {- span.setAttribute("langfuse.topics.trace_outcome", "failed");- // A partial failure still returns the valid facets' rows.- if (result && result.failure === error) return result;- throw error;- }- },+ // Trace-batch jobs span projects; the baggage tags this trace's spans and logs.+ const projectContext = contextWithLangfuseProps({+ projectId: input.projectId,+ });+ return otelContext.with(projectContext, () =>+ instrumentAsync(+ { name: "topics-trace-summary", traceScope: "topics" },+ async (span) => {+ span.setAttribute("langfuse.trace.id", input.traceId);+ const metrics = new TopicMetrics();+ let result: AssembledTraceResult | undefined;+ try {+ await metrics.measure("trace", async () => {+ result = await summarizeEnabledTrace(input, metrics);+ if (result.failure !== undefined) throw result.failure;+ });+ span.setAttribute("langfuse.topics.trace_outcome", result!.outcome);+ return result!;+ } catch (error) {+ span.setAttribute("langfuse.topics.trace_outcome", "failed");+ // A partial failure still returns the valid facets' rows.+ if (result && result.failure === error) return result;+ throw error;+ }+ },+ ),);}@@ -197,6 +212,7 @@ async function summarizeEnabledTrace(models.embedding,applicable,models.embedding.dimensions,+ models.projectId,),);} catch (error) {@@ -248,8 +264,16 @@ export async function summarizeTraceFacets(params: {{ results: TraceFacetResult[]; usage: TopicModelUsage } | undefined> {const { models, facets, config, metrics } = params;+ const outcome = (+ facet: TraceFacet,+ result: TopicResult,+ source: "model" | "rule",+ ) => metrics.result("summary", result, 1, { source, facet: facetTag(facet) });const prepared = prepareAssembledTopicTranscript(params.transcript);- if (!prepared.hasContent) return undefined;+ if (!prepared.hasContent) {+ for (const facet of facets) outcome(facet, "insufficient_input", "rule");+ return undefined;+ }// Built-in facets first, each group in listing order, so the cached prompt prefix stays stable.const ordered = [...facets.filter((facet) => facet.builtIn),@@ -287,18 +311,37 @@ export async function summarizeTraceFacets(params: {if (first) throw first;}).catch((error: unknown) => {+ if (error instanceof TopicInputTooLarge)+ for (const facet of facets) outcome(facet, "input_too_large", "rule");+ // An unusable response fails every facet before the per-facet checks run.+ else if (+ (error instanceof TopicsProviderUnavailable+ ? error.reason === "invalid_output"+ : error instanceof Error && error.name === "ZodError") &&+ ![...errors.values()].includes(error as never)+ )+ for (const facet of facets) outcome(facet, "invalid_output", "rule");// Valid facets can still be persisted after summary validation fails.if (!generated || ![...errors.values()].includes(error as never))throw error;});const result = generated!;const outputs = result.output as Record<string, FacetOutput>;+ const results = facets.map((facet): TraceFacetResult => {+ const { key } = keyed[ordered.indexOf(facet)];+ const error = errors.get(key);+ if (error) outcome(facet, "invalid_output", "rule");+ else outcome(facet, outputs[key].status, "model");+ return error ? { error } : { output: outputs[key] };+ });+ const counts: Record<string, number> = {};+ for (const item of results) {+ const key = `langfuse.topics.facets.${"error" in item ? "invalid_output" : item.output.status}`;+ counts[key] = (counts[key] ?? 0) + 1;+ }+ getCurrentSpan()?.setAttributes(counts);return {- results: facets.map((facet) => {- const { key } = keyed[ordered.indexOf(facet)];- const error = errors.get(key);- return error ? { error } : { output: outputs[key] };- }),+ results,usage: {providedUsageDetails: result.providedUsageDetails,usageDetails: result.usageDetails,@@ -308,6 +351,13 @@ export async function summarizeTraceFacets(params: {};}+/** Built-in facet names are a fixed set; custom names would be unbounded metric tags. */+function facetTag(facet: TraceFacet): string {+ return facet.builtIn && BUILT_IN_FACETS.has(facet.name)+ ? facet.name.toLowerCase()+ : "custom";+}+/** The summary row both processing paths start from, before any model call. */export function baseTopicSummary(params: {projectId: string;diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.tsindex 484b9522b6..5b3b8f41a6 100644--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts@@ -1903,7 +1903,10 @@ describe("trace batch queue", () => {metadataBytes: 0,ioMetadataBytes: hasRows ? 10 : 0,});- expect(processingSpan.setAttributes).not.toHaveBeenCalled();+ // Only Topics batch totals; no experiment attributes without an experiment.+ expect(vi.mocked(processingSpan.setAttributes).mock.calls).toEqual(+ hasRows ? [[{ "langfuse.topics.traces.unchanged": 1 }]] : [],+ );expect(recordDistribution).toHaveBeenCalledWith("langfuse.trace_batch.input_bytes",hasRows ? 5 : 0,diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.tsindex a7e52a5d64..43786635aa 100644--- a/worker/src/queues/traceBatchQueue.ts+++ b/worker/src/queues/traceBatchQueue.ts@@ -189,6 +189,7 @@ async function summarizeTraceBatch(projectId: first.projectId,traceId,reason,+ error: error instanceof Error ? error.message : String(error),});return { outcome: "failed", reason, rows };}@@ -241,13 +242,20 @@ function recordTopicsOutcomes(outcomes: TraceOutcome[]): void {]);counts.set(key, (counts.get(key) ?? 0) + 1);}+ const perOutcome: Record<string, number> = {};for (const [key, count] of counts) {const [outcome, reason] = JSON.parse(key) as [string, string | null];recordIncrement("langfuse.topics.trace_outcomes", count, {outcome,...(reason ? { reason } : {}),});+ if (outcome === "disabled") continue;+ const attribute = `langfuse.topics.traces.${outcome}`;+ perOutcome[attribute] = (perOutcome[attribute] ?? 0) + count;}+ // Batch totals of Topics work on the job span; each trace has its own child span.+ if (Object.keys(perOutcome).length > 0)+ getCurrentSpan()?.setAttributes(perOutcome);}const JOB_MAX_AGE_MS = 2 * 60 * 60_000;diff --git a/worker/src/services/exportVolumeMetric.ts b/worker/src/services/exportVolumeMetric.tsindex 543accf9c1..18ebba4c01 100644--- a/worker/src/services/exportVolumeMetric.ts+++ b/worker/src/services/exportVolumeMetric.ts@@ -1,16 +1,22 @@+import { z } from "zod";import { recordIncrement } from "@langfuse/shared/src/server";export const EXPORT_VOLUME_METRIC = "langfuse.export.serialized_bytes";-type ExportIntegration = "blob_storage" | "posthog" | "mixpanel" | "llmaj";+type ExportIntegration =+ | "blob_storage"+ | "posthog"+ | "mixpanel"+ | "llmaj"+ | "topics";type ExportVolume = {integration: ExportIntegration;// On-wire egress bytes shipped this run: gzipped for blob/posthog/mixpanel,- // uncompressed for llmaj (LLM requests aren't gzipped).+ // uncompressed for llmaj and topics (LLM requests aren't gzipped).bytes: number;// Only tagged for blob_storage (per-project egress cost attribution); the- // API-based integrations (posthog/mixpanel/llmaj) omit it to keep custom+ // API-based integrations (posthog/mixpanel/llmaj/topics) omit it to keep custom// metric cardinality down.projectId: string;// Egress-cost classification; blob only (S3 / S3_COMPATIBLE / AZURE_*).@@ -44,3 +50,32 @@ export const recordExportVolume = ({if (path !== undefined) tags.path = path;recordIncrement(EXPORT_VOLUME_METRIC, bytes, tags);};++/**+ * Egress bytes of an LLM request as LLM-as-a-judge and Topics count them: the+ * provider-bound messages plus the JSON Schema of the requested output, uncompressed.+ */+export const llmRequestEgressBytes = (+ messages: unknown,+ schema?: unknown,+): number =>+ Buffer.byteLength(+ JSON.stringify(messages, (_key, value) =>+ value instanceof Uint8Array+ ? Buffer.from(value).toString("base64")+ : value,+ ),+ "utf8",+ ) ++ (schema === undefined+ ? 0+ : Buffer.byteLength(serializeSchemaForEgress(schema), "utf8"));++// Measure the schema as the JSON Schema LangChain ships, not Zod's _def.+function serializeSchemaForEgress(schema: unknown): string {+ try {+ return JSON.stringify(z.toJSONSchema(schema as z.ZodType));+ } catch {+ return JSON.stringify(schema);+ }+}