Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(topics): trace pipeline failures and outcomes in Datadog (#19022)

Lysander Kiesel491e58ba5591
diff --git a/packages/shared/src/server/topics/text.ts b/packages/shared/src/server/topics/text.ts
index e38d8e1b35..c615f8713b 100644
--- a/packages/shared/src/server/topics/text.ts
+++ b/packages/shared/src/server/topics/text.ts
@@ -127,7 +127,11 @@ function topicTraceError(failure: unknown) {
};
}
-function topicMessages(params: { system: TopicPromptPart[]; input: string }) {
+/** The provider-bound messages of a Topics request; also used to measure its egress. */
+export function topicMessages(params: {
+ system: TopicPromptPart[];
+ input: string;
+}) {
return [
{
role: "system" as const,
diff --git a/worker/src/features/evaluation/evalExecutionDeps.ts b/worker/src/features/evaluation/evalExecutionDeps.ts
index 4ef05e6653..6ad86930d8 100644
--- a/worker/src/features/evaluation/evalExecutionDeps.ts
+++ b/worker/src/features/evaluation/evalExecutionDeps.ts
@@ -25,7 +25,10 @@ import {
import { UnrecoverableError } from "../../errors/UnrecoverableError";
import { getEvalScoreS3StorageClient } from "./s3StorageClient";
import { createInternalEventsWriter } from "../internal-tracing/createInternalEventsWriter";
-import { recordExportVolume } from "../../services/exportVolumeMetric";
+import {
+ llmRequestEgressBytes,
+ recordExportVolume,
+} from "../../services/exportVolumeMetric";
type StructuredOutputSchema = z.ZodObject<{
reasoning: z.ZodString;
@@ -159,21 +162,6 @@ export interface EvalExecutionDeps {
writeInternalTrace: InternalTraceWriter;
}
-// Measure the schema as the JSON Schema LangChain ships, not Zod's _def.
-function serializeSchemaForEgress(schema: unknown): string {
- try {
- return JSON.stringify(z.toJSONSchema(schema as z.ZodType));
- } catch {
- return JSON.stringify(schema);
- }
-}
-
-function serializeProviderMessagesForEgress(messages: unknown): string {
- return JSON.stringify(messages, (_key, value) =>
- value instanceof Uint8Array ? Buffer.from(value).toString("base64") : value,
- );
-}
-
/**
* Creates the production implementation of eval execution dependencies.
* This is the default implementation used in production code.
@@ -277,15 +265,10 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {
// send, like the other integrations.
recordExportVolume({
integration: "llmaj",
- bytes:
- Buffer.byteLength(
- serializeProviderMessagesForEgress(result.providerMessages),
- "utf8",
- ) +
- Buffer.byteLength(
- serializeSchemaForEgress(result.modelFacingOutputSchema),
- "utf8",
- ),
+ bytes: llmRequestEgressBytes(
+ result.providerMessages,
+ result.modelFacingOutputSchema,
+ ),
projectId: params.traceSinkParams.targetProjectId,
});
diff --git a/worker/src/features/topics/README.md b/worker/src/features/topics/README.md
index 2ddf850a9c..aaceb9e0cc 100644
--- a/worker/src/features/topics/README.md
+++ b/worker/src/features/topics/README.md
@@ -515,6 +515,46 @@ entry from each waiting, paused, delayed and active state without loading job
payloads. Retries can re-enter behind newer jobs, so this is head age, not the
oldest creation time anywhere in the backlog.
+Summary `.results` carry `source:model|rule` and `facet` (the built-in facet's
+name, or `custom`): the model decides `applicable`, `not_applicable` or
+`insufficient_input`; local rules decide `insufficient_input` (no transcript
+content), `invalid_output` and `input_too_large`.
+
+Every model call counts once in `.llm_calls{stage,adapter,status}`: `ok`,
+`400`, `401`, `403`, `429`, `4xx` or `5xx` for any other code (the exact code is
+on the span), or the failure reason when there is no HTTP status. One
+embedding call can fan out into several provider requests (single-text Bedrock
+models, the retry without dimensions), and AI SDK retries are not counted
+separately. Each successful call adds its egress to the shared
+`langfuse.export.serialized_bytes{integration:topics}`, measured exactly like
+LLM-as-a-judge (`llmRequestEgressBytes`: serialized provider messages plus the
+output JSON Schema, uncompressed); responses (mostly embedding vectors) and
+failed or retried requests are not included.
+
+### Datadog spans and logs
+
+Spans carry tenant and request detail that metrics must not:
+
+- `Worker.run *`: `langfuse.project.id` for every queue; Topics jobs add the
+ execution ID, operation, status and phase. Trace-batch jobs add
+ `langfuse.topics.traces.<outcome>` totals.
+- `topics-trace-summary`: one per summarized trace (both paths), with facet
+ result counts under `langfuse.topics.facets.*`.
+- `topics-facet-update`: one per facet of an update, with its outcome. Children:
+ `topics.clustering` (vectors, dimensions, status), `topics.naming` (one per
+ group) and `topics.assignment` (also used when manual processing assigns).
+ A failed facet marks its enclosing span as an error, although the run handles it.
+- `topics.llm.summary|embedding|naming`: one per provider request, with model,
+ adapter and prompt bytes; on failure, the HTTP status and the provider's error
+ message, type, code and param.
+
+A failed provider request logs one `Topics model request failed` warning with the
+same fields. Like evals, only the AI SDK's parsed provider error (message cut to 500
+characters, type, code, param) is kept; request and response bodies can contain
+trace content and are never logged, so errors without that structure, such as
+the direct Bedrock call, log the HTTP status only. A failed facet logs `Topics facet failed`: a warning for provider or
+configuration errors, an error otherwise.
+
`langfuse.topics.tokens{stage:summary|embedding|naming,direction:input|output}`
counts available provider-reported usage immediately after each model response,
before local validation or persistence. Embeddings have input tokens only.
diff --git a/worker/src/features/topics/metrics.ts b/worker/src/features/topics/metrics.ts
index ef37e8df6b..6ebc703d78 100644
--- a/worker/src/features/topics/metrics.ts
+++ b/worker/src/features/topics/metrics.ts
@@ -14,11 +14,14 @@ type TopicStage =
| "naming"
| "assignment"
| "storage";
-type TopicResult =
+export type TopicResult =
| "generated"
| "cached"
+ | "applicable"
| "not_applicable"
| "insufficient_input"
+ | "invalid_output"
+ | "input_too_large"
| "topics_found"
| "no_topics"
| "insufficient_data"
@@ -35,6 +38,9 @@ type TopicErrorReason =
| "numerical"
| "unknown";
+// Statuses with their own operational meaning; exact codes are on the span.
+const COUNTED_HTTP_STATUSES = new Set([400, 401, 403, 429]);
+
/** Provider usage is counted per call, before local validation or persistence. */
export function recordTopicTokenUsage(
stage: "summary" | "embedding" | "naming",
@@ -52,6 +58,20 @@ export function recordTopicTokenUsage(
}
}
+/** Counts every provider request with a bounded status tag. */
+export function recordTopicModelCall(
+ stage: "summary" | "embedding" | "naming",
+ adapter: string,
+ outcome: { httpStatus?: number; reason?: string },
+): void {
+ const { httpStatus = 0, reason } = outcome;
+ let status = reason ?? "ok";
+ if (COUNTED_HTTP_STATUSES.has(httpStatus)) status = String(httpStatus);
+ else if (httpStatus >= 400 && httpStatus < 500) status = "4xx";
+ else if (httpStatus >= 500 && httpStatus < 600) status = "5xx";
+ recordIncrement("langfuse.topics.llm_calls", 1, { stage, adapter, status });
+}
+
/** One instance per execution attempt prevents nested catches counting an error twice. */
export class TopicMetrics {
private readonly reportedErrors = new WeakSet<object>();
@@ -62,9 +82,19 @@ export class TopicMetrics {
recordIncrement("langfuse.topics.executions", 1, { outcome });
}
- result(stage: TopicStage, result: TopicResult, count = 1): void {
+ result(
+ stage: TopicStage,
+ result: TopicResult,
+ count = 1,
+ // Summary outcomes say whether the model or a local rule decided, and for which facet.
+ tags: { source?: "model" | "rule"; facet?: string } = {},
+ ): void {
if (count > 0)
- recordIncrement("langfuse.topics.results", count, { stage, result });
+ recordIncrement("langfuse.topics.results", count, {
+ stage,
+ result,
+ ...tags,
+ });
}
clusteringWorkload(vectors: number, dimensions: number): void {
diff --git a/worker/src/features/topics/models.test.ts b/worker/src/features/topics/models.test.ts
index f67a61f296..d6cd823438 100644
--- a/worker/src/features/topics/models.test.ts
+++ b/worker/src/features/topics/models.test.ts
@@ -10,7 +10,8 @@ const state = vi.hoisted(() => ({
traceSink: vi.fn(),
}));
vi.mock("@langfuse/shared/src/env", () => ({ env: state.env }));
-vi.mock("@langfuse/shared/src/server", () => ({
+vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),
getLangfuseAITraceSinkParams: state.traceSink,
LangfuseInternalTraceEnvironment: { Topics: "langfuse-topics" },
logger: { warn: vi.fn() },
@@ -19,11 +20,20 @@ vi.mock("@langfuse/shared/src/server", () => ({
vi.mock("@langfuse/shared/src/db", () => ({
prisma: { project: { findFirst: state.project } },
}));
-vi.mock("@langfuse/shared/topics/server", () => ({
+vi.mock("@langfuse/shared/topics/server", async (importOriginal) => ({
+ topicMessages: (
+ await importOriginal<typeof import("@langfuse/shared/topics/server")>()
+ ).topicMessages,
generateTopicText: (...args: unknown[]) => state.call(...args),
generateTopicEmbeddings: (...args: unknown[]) => state.embed(...args),
}));
+// Token counters only; every call also records call and request-byte counters.
+const tokenCalls = () =>
+ state.increment.mock.calls.filter(([name]) =>
+ String(name).startsWith("langfuse.topics.token"),
+ );
+
import {
embedTopicSummaries,
nameTopicGroup,
@@ -34,6 +44,8 @@ import {
type TopicProcessingConfig,
} from "@langfuse/shared/topics";
import { LLMAdapter } from "@langfuse/shared";
+import { logger } from "@langfuse/shared/src/server";
+import { recordTopicModelCall } from "./metrics";
import type { TopicsModel, TopicsModels } from "@langfuse/shared/topics/server";
function slot(slot: TopicsModel["slot"], model: string): TopicsModel {
@@ -272,6 +284,100 @@ describe("Topics naming boundary", () => {
expect(state.call).not.toHaveBeenCalled();
});
+ it("logs the provider's error fields for a rejected request, never its bodies", async () => {
+ // The AI SDK identifies an APICallError by these markers.
+ const rejection = Object.assign(
+ new Error(
+ "Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead.",
+ ),
+ {
+ [Symbol.for("vercel.ai.error")]: true,
+ [Symbol.for("vercel.ai.error.AI_APICallError")]: true,
+ name: "AI_APICallError",
+ statusCode: 400,
+ isRetryable: false,
+ requestBodyValues: { messages: "secret transcript" },
+ responseBody: '{"error":{"message":"secret echo"}}',
+ data: {
+ error: {
+ type: "invalid_request_error",
+ code: "unsupported_parameter",
+ param: "max_tokens",
+ },
+ },
+ },
+ );
+ state.call.mockRejectedValue(rejection);
+ await expect(
+ summarizeOne(
+ topicsModels(),
+ "Trace evidence.",
+ topicProcessingConfigSchema.parse({
+ summaryModel: "us.openai.gpt-5.6-luna",
+ }),
+ ),
+ ).rejects.toMatchObject({ reason: "invalid_input" });
+ const [message, fields] = vi.mocked(logger.warn).mock.calls.at(-1)!;
+ expect(message).toBe("Topics model request failed");
+ expect(fields).toMatchObject({
+ "http.status_code": 400,
+ "langfuse.topics.provider.message": rejection.message,
+ "langfuse.topics.provider.code": "unsupported_parameter",
+ "langfuse.topics.provider.param": "max_tokens",
+ });
+ expect(JSON.stringify(fields)).not.toMatch(/secret/);
+ expect(state.increment).toHaveBeenCalledWith(
+ "langfuse.topics.llm_calls",
+ 1,
+ {
+ stage: "summary",
+ adapter: LLMAdapter.Bedrock,
+ status: "400",
+ },
+ );
+ });
+
+ it.each([
+ [{}, "ok"],
+ [{ httpStatus: 429, reason: "rate_limit" }, "429"],
+ [{ httpStatus: 404, reason: "invalid_input" }, "4xx"],
+ [{ httpStatus: 503, reason: "provider_error" }, "5xx"],
+ [{ reason: "timeout" }, "timeout"],
+ ])("tags a model call outcome %o as %s", (outcome, status) => {
+ recordTopicModelCall("summary", LLMAdapter.OpenAI, outcome);
+ expect(state.increment).toHaveBeenCalledWith(
+ "langfuse.topics.llm_calls",
+ 1,
+ {
+ stage: "summary",
+ adapter: LLMAdapter.OpenAI,
+ status,
+ },
+ );
+ });
+
+ it("logs only the status for an error whose message carries the response body", async () => {
+ // The direct Bedrock call throws a plain error with the response text.
+ state.call.mockRejectedValue(
+ Object.assign(
+ new Error('Bedrock InvokeModel failed with 400: {"secret": "echo"}'),
+ { status: 400 },
+ ),
+ );
+ await expect(
+ summarizeOne(
+ topicsModels(),
+ "Trace evidence.",
+ topicProcessingConfigSchema.parse({
+ summaryModel: "us.openai.gpt-5.6-luna",
+ }),
+ ),
+ ).rejects.toMatchObject({ reason: "invalid_input" });
+ const [, fields] = vi.mocked(logger.warn).mock.calls.at(-1)!;
+ expect(fields).toMatchObject({ "http.status_code": 400 });
+ expect(JSON.stringify(fields)).not.toMatch(/secret/);
+ });
+
it("keeps fallback token counts out of provider-reported usage", async () => {
state.call.mockResolvedValue({
output: keyed({ summary: "A billing request.", status: "applicable" }),
@@ -286,7 +392,7 @@ describe("Topics naming boundary", () => {
config,
);
expect(result.providedUsageDetails).toEqual({ summary_input: 100 });
- expect(state.increment.mock.calls).toEqual([
+ expect(tokenCalls()).toEqual([
["langfuse.topics.tokens", 100, { stage: "summary", direction: "input" }],
[
"langfuse.topics.token_usage_missing",
@@ -534,7 +640,7 @@ describe("Topics naming boundary", () => {
});
await expect(nameTopicGroup(topicsModels(), evidence)).rejects.toThrow();
// Provider work is billable even when local output validation rejects it.
- expect(state.increment.mock.calls).toEqual([
+ expect(tokenCalls()).toEqual([
[
"langfuse.topics.tokens",
100,
@@ -643,11 +749,12 @@ describe("Topics naming boundary", () => {
topicsModels().embedding,
["An invoice request."],
256,
+ "project",
),
).rejects.toMatchObject({
reason: "invalid_output",
});
- expect(state.increment.mock.calls).toEqual([
+ expect(tokenCalls()).toEqual([
[
"langfuse.topics.tokens",
12,
@@ -667,8 +774,9 @@ describe("Topics naming boundary", () => {
topicsModels().embedding,
["An invoice request."],
256,
+ "project",
);
- expect(state.increment.mock.calls).toEqual([
+ expect(tokenCalls()).toEqual([
[
"langfuse.topics.token_usage_missing",
1,
diff --git a/worker/src/features/topics/models.ts b/worker/src/features/topics/models.ts
index 20ba7987e4..a5777f2d5f 100644
--- a/worker/src/features/topics/models.ts
+++ b/worker/src/features/topics/models.ts
@@ -5,11 +5,13 @@ import { env } from "@langfuse/shared/src/env";
import {
logger,
getLangfuseAITraceSinkParams,
+ instrumentAsync,
LangfuseInternalTraceEnvironment,
} from "@langfuse/shared/src/server";
import {
generateTopicEmbeddings,
generateTopicText,
+ topicMessages,
type TopicPromptPart,
type TopicsModel,
type TopicsModels,
@@ -19,7 +21,11 @@ import {
type TopicFacetVersion,
type TopicProcessingConfig,
} from "@langfuse/shared/topics";
-import { recordTopicTokenUsage } from "./metrics";
+import { recordTopicModelCall, recordTopicTokenUsage } from "./metrics";
+import {
+ llmRequestEgressBytes,
+ recordExportVolume,
+} from "../../services/exportVolumeMetric";
import type { TopicModelUsage } from "./summaryResult";
import {
TopicsProviderUnavailable,
@@ -81,6 +87,66 @@ function countTopicRequestTokens(
}
}
+/** The input exceeds the run's token limit; no model call was made. */
+export class TopicInputTooLarge extends Error {}
+
+/** One span, call counter and failure log per provider request. */
+async function topicModelCall<T>(
+ stage: "summary" | "embedding" | "naming",
+ model: TopicsModel,
+ projectId: string,
+ requestBytes: number,
+ context: Record<string, unknown>,
+ call: () => Promise<T>,
+): Promise<T> {
+ return instrumentAsync(
+ { name: `topics.llm.${stage}`, traceScope: "topics" },
+ async (span) => {
+ const fields: Record<string, string | number> = {
+ "langfuse.topics.stage": stage,
+ "langfuse.topics.model": model.model,
+ "langfuse.topics.adapter": model.adapter,
+ "langfuse.topics.request_bytes": requestBytes,
+ };
+ for (const [key, value] of Object.entries(context))
+ if (typeof value === "string") fields[`langfuse.topics.${key}`] = value;
+ span.setAttributes(fields);
+ try {
+ const result = await call();
+ recordTopicModelCall(stage, model.adapter, {});
+ // Shared egress metric; like llmaj, counted after a successful send.
+ recordExportVolume({
+ integration: "topics",
+ bytes: requestBytes,
+ projectId,
+ });
+ return result;
+ } catch (error) {
+ const failure = topicProviderError(error, model);
+ const { httpStatus, message, type, code, param } = failure.details;
+ recordTopicModelCall(stage, model.adapter, {
+ httpStatus,
+ reason: failure.reason,
+ });
+ const failed = {
+ ...fields,
+ "langfuse.topics.error.reason": failure.reason,
+ "langfuse.topics.error.type":
+ error instanceof Error ? error.name : "unknown",
+ ...(httpStatus && { "http.status_code": httpStatus }),
+ ...(message && { "langfuse.topics.provider.message": message }),
+ ...(type && { "langfuse.topics.provider.type": type }),
+ ...(code && { "langfuse.topics.provider.code": code }),
+ ...(param && { "langfuse.topics.provider.param": param }),
+ };
+ span.setAttributes(failed);
+ logger.warn("Topics model request failed", failed);
+ throw failure;
+ }
+ },
+ );
+}
+
const summarySchema = z.object({
summary: z.string(),
status: z.enum(["applicable", "not_applicable", "insufficient_input"]),
@@ -104,21 +170,26 @@ async function structuredCall<T>(
stage === "naming"
? TOPICS_NAMING_RATES.get(model.model)?.(tokens)
: TOPICS_SUMMARY_RATES.get(model.model);
- const result = await generateTopicText({
+ const trace = await topicTraceSink(traceName, projectId, metadata);
+ const result = await topicModelCall(
+ stage,
model,
- system,
- input,
- schema,
- maxOutputTokens: outputLimit,
- trace: await topicTraceSink(traceName, projectId, metadata),
- }).catch((error: unknown) => {
- logger.warn("Topics model request failed", {
- model: model.model,
- adapter: model.adapter,
- errorType: error instanceof Error ? error.name : "unknown",
- });
- throw topicProviderError(error, model);
- });
+ projectId,
+ llmRequestEgressBytes(topicMessages({ system, input }), schema),
+ {
+ trace_id: metadata?.source_trace_id,
+ execution_id: metadata?.execution_id,
+ },
+ () =>
+ generateTopicText({
+ model,
+ system,
+ input,
+ schema,
+ maxOutputTokens: outputLimit,
+ trace,
+ }),
+ );
const actualRates = rates(result.usage.inputTokens ?? inputLimit);
recordTopicTokenUsage(stage, {
input: result.usage.inputTokens,
@@ -300,7 +371,7 @@ ${facetBlock(true)}`,
const countedInputTokens =
countTopicRequestTokens(system, input, schema) + 256;
if (countedInputTokens > config.maxInputTokens)
- throw new Error(
+ throw new TopicInputTooLarge(
`The shared trace transcript and instructions are ${countedInputTokens} tokens, above this run's ${config.maxInputTokens}-token input limit. No model call was made; the transcript is never shortened per facet.`,
);
return structuredCall(
@@ -391,6 +462,7 @@ export async function embedTopicSummaries(
model: TopicsModel,
summaries: string[],
dimensions: number,
+ projectId: string,
): Promise<TopicModelUsage & { embeddings: number[][] }> {
if (
summaries.length === 0 ||
@@ -402,13 +474,14 @@ export async function embedTopicSummaries(
"Topics embeddings require non-empty text and 64 to 4,096 dimensions.",
"invalid_input",
);
- const result = await generateTopicEmbeddings({
+ const result = await topicModelCall(
+ "embedding",
model,
- summaries,
- dimensions,
- }).catch((error: unknown) => {
- throw topicProviderError(error, model);
- });
+ projectId,
+ llmRequestEgressBytes(summaries),
+ {},
+ () => generateTopicEmbeddings({ model, summaries, dimensions }),
+ );
recordTopicTokenUsage("embedding", { input: result.tokens });
// ClickHouse stores Float32; calibration and future classification must use those same vectors.
const embeddings = result.embeddings.map((vector) =>
diff --git a/worker/src/features/topics/numeric.ts b/worker/src/features/topics/numeric.ts
index 5d01e9af3a..b6aebae695 100644
--- a/worker/src/features/topics/numeric.ts
+++ b/worker/src/features/topics/numeric.ts
@@ -78,14 +78,19 @@ export async function runTopicClustering(
clearTimeout(timeout);
reject(error);
});
- child.on("close", (code) => {
+ child.on("close", (code, signal) => {
clearTimeout(timeout);
if (outputExceeded) {
reject(new Error("Topics numerical fit returned oversized output"));
return;
}
if (code !== 0) {
- reject(new Error(`Topics numerical fit failed: ${errorOutput}`));
+ // A kill (e.g. out of memory) leaves no stderr; the signal names it.
+ reject(
+ new Error(
+ `Topics numerical fit failed (exit code ${code}, signal ${signal}): ${errorOutput}`,
+ ),
+ );
return;
}
try {
diff --git a/worker/src/features/topics/processTopicEmbeddingBatch.test.ts b/worker/src/features/topics/processTopicEmbeddingBatch.test.ts
index ec7901e951..9f8de7d1a0 100644
--- a/worker/src/features/topics/processTopicEmbeddingBatch.test.ts
+++ b/worker/src/features/topics/processTopicEmbeddingBatch.test.ts
@@ -39,6 +39,7 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
const { isInternalEvalEnvironment } =
await importOriginal<typeof import("@langfuse/shared/src/server")>();
return {
+ ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),
isInternalEvalEnvironment,
logger: { error: vi.fn() },
recordIncrement: vi.fn(),
@@ -146,6 +147,7 @@ describe("Topics embedding handoff", () => {
expect.anything(),
[allowed.summary],
256,
+ "project",
);
expect(mocks.update).toHaveBeenCalledTimes(1);
expect(mocks.write).toHaveBeenCalledExactlyOnceWith([
diff --git a/worker/src/features/topics/processTopicEmbeddingBatch.ts b/worker/src/features/topics/processTopicEmbeddingBatch.ts
index b085d7f8fe..d0ea496ee1 100644
--- a/worker/src/features/topics/processTopicEmbeddingBatch.ts
+++ b/worker/src/features/topics/processTopicEmbeddingBatch.ts
@@ -70,6 +70,7 @@ export async function processTopicEmbeddingBatch(
models.embedding,
traceRows.map(({ summary }) => summary.summary),
dimensions,
+ models.projectId,
);
} catch (error) {
throw error instanceof TopicsProviderUnavailable
diff --git a/worker/src/features/topics/processTopicsExecution.test.ts b/worker/src/features/topics/processTopicsExecution.test.ts
index 92042f0ad4..0c29af5026 100644
--- a/worker/src/features/topics/processTopicsExecution.test.ts
+++ b/worker/src/features/topics/processTopicsExecution.test.ts
@@ -55,6 +55,7 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
const { assembleTranscript, isInternalEvalEnvironment } =
await importOriginal<typeof import("@langfuse/shared/src/server")>();
return {
+ ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),
assembleTranscript,
isInternalEvalEnvironment,
recordIncrement: vi.fn(),
@@ -324,6 +325,7 @@ vi.mock("@langfuse/shared/topics/server", async (importOriginal) => {
};
});
vi.mock("./models", () => ({
+ TopicInputTooLarge: class extends Error {},
// One request per trace; `state.summarize` answers each facet, and one failure fails the request.
summarizeTopicTraceFacets: async (
_models: unknown,
diff --git a/worker/src/features/topics/processTopicsExecution.ts b/worker/src/features/topics/processTopicsExecution.ts
index 4e553d4593..0156782650 100644
--- a/worker/src/features/topics/processTopicsExecution.ts
+++ b/worker/src/features/topics/processTopicsExecution.ts
@@ -1,4 +1,10 @@
-import { isInternalEvalEnvironment, logger } from "@langfuse/shared/src/server";
+import {
+ getCurrentSpan,
+ instrumentAsync,
+ isInternalEvalEnvironment,
+ logger,
+ traceException,
+} from "@langfuse/shared/src/server";
import {
isTopicsEnabled,
readTopicExecutionSummary,
@@ -156,6 +162,13 @@ async function summarizeTrace(
} catch (error) {
if (isExecutionFailure(error) && !isTraceModelFailure(error)) throw error;
metrics.error("summary", error);
+ logger.warn("Topics summary failed for trace", {
+ "langfuse.trace.id": traceId,
+ "langfuse.topics.execution_id": execution.id,
+ "langfuse.topics.error.reason":
+ error instanceof TopicsProviderUnavailable ? error.reason : "other",
+ error: errorMessage(error),
+ });
for (const facet of unresolved()) results.set(facet, error as Error);
}
}
@@ -274,12 +287,6 @@ async function summarizePending(
continue;
}
const output = result?.output;
- if (!output) metrics.result("summary", "insufficient_input");
- else
- metrics.result(
- "summary",
- output.status === "applicable" ? "generated" : output.status,
- );
rows.push({
facet,
summary: {
@@ -413,55 +420,64 @@ async function assignSummaries(
coordinates: Map<string, [number, number]> = new Map(),
assignedAt = new Date().toISOString(),
) {
- return metrics.measure("assignment", async () => {
- const rows = topicAssignmentRows({
- projectId: execution.projectId,
- facet: { facetId: facet.facetId, version: facet.version },
- embeddingConfig: execution.input.embeddingConfig,
- summaries,
- run,
- origin: execution.input.operation === "process" ? "online" : "initial",
- assignedAt,
- coordinates,
- });
- await metrics.measure(
- "storage",
- () => writeTopicAssignments(rows),
- "storage",
- );
- const visible =
- execution.input.operation === "update" && rows.length
- ? await readTopicAssignments(
- execution.projectId,
- { facetId: facet.facetId, version: facet.version },
- summaries,
- run.id,
- summaryTimeRange(summaries),
- )
- : [];
- const visibleByTrace = new Map(visible.map((row) => [row.traceId, row]));
- if (
- execution.input.operation === "update" &&
- rows.some((row) => {
- const persisted = visibleByTrace.get(row.traceId);
- return (
- !persisted ||
- persisted.topicId !== row.topicId ||
- persisted.topicVersionId !== row.topicVersionId ||
- persisted.summaryProcessedAt !== row.summaryProcessedAt
+ return instrumentAsync(
+ { name: "topics.assignment", traceScope: "topics" },
+ (span) => {
+ span.setAttribute("langfuse.topics.summaries", summaries.length);
+ return metrics.measure("assignment", async () => {
+ const rows = topicAssignmentRows({
+ projectId: execution.projectId,
+ facet: { facetId: facet.facetId, version: facet.version },
+ embeddingConfig: execution.input.embeddingConfig,
+ summaries,
+ run,
+ origin:
+ execution.input.operation === "process" ? "online" : "initial",
+ assignedAt,
+ coordinates,
+ });
+ await metrics.measure(
+ "storage",
+ () => writeTopicAssignments(rows),
+ "storage",
);
- })
- )
- throw new Error(
- "Topic assignments are not fully visible; map publication is deferred.",
- );
- progress.counts.assigned = rows.filter(
- (row) => row.topicId !== null,
- ).length;
- progress.counts.outlier = rows.length - progress.counts.assigned;
- metrics.result("assignment", "assigned", progress.counts.assigned);
- metrics.result("assignment", "outlier", progress.counts.outlier);
- });
+ const visible =
+ execution.input.operation === "update" && rows.length
+ ? await readTopicAssignments(
+ execution.projectId,
+ { facetId: facet.facetId, version: facet.version },
+ summaries,
+ run.id,
+ summaryTimeRange(summaries),
+ )
+ : [];
+ const visibleByTrace = new Map(
+ visible.map((row) => [row.traceId, row]),
+ );
+ if (
+ execution.input.operation === "update" &&
+ rows.some((row) => {
+ const persisted = visibleByTrace.get(row.traceId);
+ return (
+ !persisted ||
+ persisted.topicId !== row.topicId ||
+ persisted.topicVersionId !== row.topicVersionId ||
+ persisted.summaryProcessedAt !== row.summaryProcessedAt
+ );
+ })
+ )
+ throw new Error(
+ "Topic assignments are not fully visible; map publication is deferred.",
+ );
+ progress.counts.assigned = rows.filter(
+ (row) => row.topicId !== null,
+ ).length;
+ progress.counts.outlier = rows.length - progress.counts.assigned;
+ metrics.result("assignment", "assigned", progress.counts.assigned);
+ metrics.result("assignment", "outlier", progress.counts.outlier);
+ });
+ },
+ );
}
async function clusterFacet(
@@ -529,14 +545,26 @@ async function clusterFacet(
summaries.length,
execution.input.embeddingConfig.embeddingDimensions,
);
- const numeric = await metrics.measure(
- "clustering",
- () =>
- runTopicClustering(
- summaries.map((row) => row.embedding),
- numericConfig,
- ),
- "numerical",
+ const numeric = await instrumentAsync(
+ { name: "topics.clustering", traceScope: "topics" },
+ async (span) => {
+ span.setAttributes({
+ "langfuse.topics.clustering.vectors": summaries.length,
+ "langfuse.topics.clustering.dimensions":
+ execution.input.embeddingConfig.embeddingDimensions,
+ });
+ const result = await metrics.measure(
+ "clustering",
+ () =>
+ runTopicClustering(
+ summaries.map((row) => row.embedding),
+ numericConfig,
+ ),
+ "numerical",
+ );
+ span.setAttribute("langfuse.topics.clustering.status", result.status);
+ return result;
+ },
);
const coordinates = new Map(
summaries
@@ -692,47 +720,56 @@ async function clusterFacet(
topics.push(retained);
continue;
}
- const label = await metrics.measure("naming", async () => {
- const members = new Map(
- group.members.map((summary, index) => [`m${index + 1}`, summary]),
- );
- const label = await nameGroupAvoidingConflicts(
- metrics,
- models,
- named,
- {
- members: Array.from(members, ([id, { summary }]) => ({
- id,
- summary,
- })),
- contrasts: group.contrasts.map((member, index) => ({
- id: `c${index + 1}`,
- summary: member.summary,
- })),
- },
- {
- facet_id: facet.facetId,
- facet_name: facetName,
- facet_version: facet.version,
- execution_id: execution.id,
- run_id: run.id,
- group_id: group.id,
- },
- );
- return {
- ...label,
- representativeSummaries: label.evidenceSummaryIds.map((id) => {
- const { facetId, facetVersion, traceId, sessionId } =
- members.get(id)!;
+ const label = await instrumentAsync(
+ { name: "topics.naming", traceScope: "topics" },
+ (span) => {
+ span.setAttributes({
+ "langfuse.topics.group_id": group.id,
+ "langfuse.topics.group_members": group.members.length,
+ });
+ return metrics.measure("naming", async () => {
+ const members = new Map(
+ group.members.map((summary, index) => [`m${index + 1}`, summary]),
+ );
+ const label = await nameGroupAvoidingConflicts(
+ metrics,
+ models,
+ named,
+ {
+ members: Array.from(members, ([id, { summary }]) => ({
+ id,
+ summary,
+ })),
+ contrasts: group.contrasts.map((member, index) => ({
+ id: `c${index + 1}`,
+ summary: member.summary,
+ })),
+ },
+ {
+ facet_id: facet.facetId,
+ facet_name: facetName,
+ facet_version: facet.version,
+ execution_id: execution.id,
+ run_id: run.id,
+ group_id: group.id,
+ },
+ );
return {
- facetId,
- facetVersion,
- traceId,
- sessionId: traceId === null ? sessionId : null,
+ ...label,
+ representativeSummaries: label.evidenceSummaryIds.map((id) => {
+ const { facetId, facetVersion, traceId, sessionId } =
+ members.get(id)!;
+ return {
+ facetId,
+ facetVersion,
+ traceId,
+ sessionId: traceId === null ? sessionId : null,
+ };
+ }),
};
- }),
- };
- });
+ });
+ },
+ );
const candidate = candidates.find(
(topic) => topic.topicVersionId === group.id,
)!;
@@ -970,18 +1007,34 @@ async function processTraces(
if (todo.length === 0) return;
let outcomes: (TopicSummary | Error)[];
try {
- outcomes = await summarizeTrace(
- metrics,
- models,
- execution,
- todo.map(({ facet, name, builtIn }) => ({
- version: facet,
- name,
- builtIn,
- })),
- traceId,
- cached,
- getTranscript,
+ outcomes = await instrumentAsync(
+ { name: "topics-trace-summary", traceScope: "topics" },
+ async (span) => {
+ span.setAttributes({
+ "langfuse.trace.id": traceId,
+ "langfuse.topics.execution_id": execution.id,
+ });
+ const results = await summarizeTrace(
+ metrics,
+ models,
+ execution,
+ todo.map(({ facet, name, builtIn }) => ({
+ version: facet,
+ name,
+ builtIn,
+ })),
+ traceId,
+ cached,
+ getTranscript,
+ );
+ span.setAttribute(
+ "langfuse.topics.trace_outcome",
+ results.some((result) => result instanceof Error)
+ ? "failed"
+ : "summarized",
+ );
+ return results;
+ },
);
} catch (error) {
metrics.error("summary", error);
@@ -1086,10 +1139,7 @@ async function processTraces(
progress.outcome = "assigned";
}
} catch (error) {
- metrics.error("execution", error);
- progress.outcome = "failed";
- progress.error = errorMessage(error);
- if (isExecutionFailure(error)) throw error;
+ failFacet(metrics, execution, facet, progress, error);
}
await save("processing");
}
@@ -1132,76 +1182,115 @@ async function updateTopics(
]),
);
for (const { facet, progress } of await pendingFacets(execution)) {
- try {
- const accepted = progress.runId
- ? await getTopicRun(execution.projectId, progress.runId)
- : null;
- if (
- accepted &&
- (accepted.projectId !== execution.projectId ||
- accepted.facetId !== facet.facetId ||
- accepted.facetVersion !== facet.version)
- )
- throw new Error("Clustering run does not match this facet version.");
- if (accepted?.status === "completed") {
- const assignments = await readTopicMapAssignments(
- execution.projectId,
- { facetId: facet.facetId, version: facet.version },
- accepted.id,
- topicTimeRangeSchema.parse(accepted.config.timeRange),
- );
- progress.counts.requested = assignments.length;
- progress.counts.complete = assignments.length;
- progress.counts.assigned = assignments.filter(
- (row) => row.topicId !== null,
- ).length;
- progress.counts.outlier = assignments.length - progress.counts.assigned;
- progress.outcome = accepted.topics.length ? "published" : "no_topics";
- await saveProgress(metrics, execution, "processing");
- continue;
- }
- if (accepted?.status === "skipped") {
- progress.outcome = progress.counts.requested
- ? "insufficient_data"
- : "no_applicable_summaries";
+ await instrumentAsync(
+ { name: "topics-facet-update", traceScope: "topics" },
+ async (span) => {
+ span.setAttributes({
+ "langfuse.topics.execution_id": execution.id,
+ "langfuse.topics.facet_id": facet.facetId,
+ "langfuse.topics.facet_version": facet.version,
+ });
+ try {
+ const accepted = progress.runId
+ ? await getTopicRun(execution.projectId, progress.runId)
+ : null;
+ if (
+ accepted &&
+ (accepted.projectId !== execution.projectId ||
+ accepted.facetId !== facet.facetId ||
+ accepted.facetVersion !== facet.version)
+ )
+ throw new Error(
+ "Clustering run does not match this facet version.",
+ );
+ if (accepted?.status === "completed") {
+ const assignments = await readTopicMapAssignments(
+ execution.projectId,
+ { facetId: facet.facetId, version: facet.version },
+ accepted.id,
+ topicTimeRangeSchema.parse(accepted.config.timeRange),
+ );
+ progress.counts.requested = assignments.length;
+ progress.counts.complete = assignments.length;
+ progress.counts.assigned = assignments.filter(
+ (row) => row.topicId !== null,
+ ).length;
+ progress.counts.outlier =
+ assignments.length - progress.counts.assigned;
+ progress.outcome = accepted.topics.length
+ ? "published"
+ : "no_topics";
+ await saveProgress(metrics, execution, "processing");
+ return;
+ }
+ if (accepted?.status === "skipped") {
+ progress.outcome = progress.counts.requested
+ ? "insufficient_data"
+ : "no_applicable_summaries";
+ await saveProgress(metrics, execution, "processing");
+ return;
+ }
+ const current = await getPublishedTopicRun(
+ execution.projectId,
+ facet.facetId,
+ );
+ const previous =
+ current?.facetVersion === facet.version ? current : null;
+ const summaries = await getTopicClusteringSummaries(
+ execution.projectId,
+ facet.facetId,
+ facet.version,
+ execution.input.embeddingConfig,
+ execution.input.timeRange,
+ );
+ countSummaries(progress, summaries);
+ progress.counts.requested = summaries.length;
+ await clusterFacet(
+ metrics,
+ models,
+ execution,
+ facet,
+ facetNames.get(facet.facetId),
+ progress,
+ summaries,
+ accepted,
+ previous,
+ );
+ } catch (error) {
+ failFacet(metrics, execution, facet, progress, error);
+ }
await saveProgress(metrics, execution, "processing");
- continue;
- }
- const current = await getPublishedTopicRun(
- execution.projectId,
- facet.facetId,
- );
- const previous = current?.facetVersion === facet.version ? current : null;
- const summaries = await getTopicClusteringSummaries(
- execution.projectId,
- facet.facetId,
- facet.version,
- execution.input.embeddingConfig,
- execution.input.timeRange,
- );
- countSummaries(progress, summaries);
- progress.counts.requested = summaries.length;
- await clusterFacet(
- metrics,
- models,
- execution,
- facet,
- facetNames.get(facet.facetId),
- progress,
- summaries,
- accepted,
- previous,
- );
- } catch (error) {
- metrics.error("execution", error);
- progress.outcome = "failed";
- progress.error = errorMessage(error);
- if (isExecutionFailure(error)) throw error;
- }
- await saveProgress(metrics, execution, "processing");
+ span.setAttribute("langfuse.topics.facet_outcome", progress.outcome);
+ },
+ );
}
}
+/** Provider and configuration failures are the customer's to fix; anything else is ours. */
+function failFacet(
+ metrics: TopicMetrics,
+ execution: TopicExecution,
+ facet: TopicFacetVersion,
+ progress: TopicFacetProgress,
+ error: unknown,
+) {
+ metrics.error("execution", error);
+ progress.outcome = "failed";
+ progress.error = errorMessage(error);
+ const log =
+ error instanceof TopicsProviderUnavailable ? logger.warn : logger.error;
+ log("Topics facet failed", {
+ "langfuse.topics.execution_id": execution.id,
+ "langfuse.topics.operation": execution.input.operation,
+ "langfuse.topics.facet_id": facet.facetId,
+ "langfuse.topics.facet_version": facet.version,
+ error: progress.error,
+ });
+ if (isExecutionFailure(error)) throw error;
+ // Handled, but still a failed facet for the user; the enclosing span reports it.
+ traceException(error);
+}
+
/** Queue batches own trace retry state; Postgres owns compact update progress. */
export async function processTopicsExecution({
projectId,
@@ -1360,5 +1449,13 @@ export async function processTopicsExecution({
execution.status = "failed";
execution.error = errorMessage(error);
await save("failed");
+ } finally {
+ getCurrentSpan()?.setAttributes({
+ "langfuse.project.id": projectId,
+ "langfuse.topics.execution_id": executionId,
+ "langfuse.topics.operation": execution.input.operation,
+ "langfuse.topics.execution_status": execution.status,
+ "langfuse.topics.execution_phase": execution.phase,
+ });
}
}
diff --git a/worker/src/features/topics/provider-error.ts b/worker/src/features/topics/provider-error.ts
index 777d155270..398b17deab 100644
--- a/worker/src/features/topics/provider-error.ts
+++ b/worker/src/features/topics/provider-error.ts
@@ -3,7 +3,7 @@ import {
type TopicsModelSlotName,
} from "@langfuse/shared/topics";
import { EvaluatorBlockReason } from "@prisma/client";
-import { logger } from "@langfuse/shared/src/server";
+import { getLLMErrorInfo, logger } from "@langfuse/shared/src/server";
import {
pauseTopicsModels,
type TopicsModel,
@@ -19,11 +19,21 @@ type TopicsProviderErrorReason =
| "invalid_input"
| "provider_error";
+/** Telemetry only; the user-facing message stays generic. */
+export type TopicProviderErrorDetails = {
+ httpStatus?: number;
+ message?: string;
+ type?: string;
+ code?: string;
+ param?: string;
+};
+
export class TopicsProviderUnavailable extends Error {
constructor(
message: string,
readonly reason: TopicsProviderErrorReason = "provider_error",
readonly slot?: TopicsModelSlotName,
+ readonly details: TopicProviderErrorDetails = {},
) {
super(message);
}
@@ -73,9 +83,38 @@ export function topicProviderError(
`The ${TOPICS_MODEL_SLOT_DETAILS[model.slot].label.toLowerCase()} model ${model.model} on LLM connection "${model.provider}" failed${status ? ` (HTTP ${status})` : ""}. Check the connection's credentials, the model ID, and provider availability.`,
reason,
model.slot,
+ providerErrorDetails(error, status),
);
}
+/**
+ * Like evals, keeps only the AI SDK's parsed provider error fields; plain errors
+ * (e.g. the direct Bedrock call) carry the response body, so they keep the status only.
+ */
+function providerErrorDetails(
+ error: unknown,
+ status: number | undefined,
+): TopicProviderErrorDetails {
+ const provider = getLLMErrorInfo(error)?.providerError;
+ const httpStatus = provider?.statusCode ?? status;
+ if (!httpStatus) return {};
+ if (!provider) return { httpStatus };
+ const record = (value: unknown) =>
+ value && typeof value === "object"
+ ? (value as Record<string, unknown>)
+ : undefined;
+ const data = record(record(provider.data)?.error) ?? record(provider.data);
+ const text = (value: unknown) =>
+ typeof value === "string" ? value.slice(0, 500) : undefined;
+ return {
+ httpStatus,
+ message: text(provider.message),
+ type: text(data?.type),
+ code: text(data?.code),
+ param: text(data?.param),
+ };
+}
+
/** The provider rejected the connection's credentials or billing. */
export function isRejectedConnection(
error: unknown,
diff --git a/worker/src/features/topics/summarizeAssembledTrace.test.ts b/worker/src/features/topics/summarizeAssembledTrace.test.ts
index 85ec59a3ee..3274c6da34 100644
--- a/worker/src/features/topics/summarizeAssembledTrace.test.ts
+++ b/worker/src/features/topics/summarizeAssembledTrace.test.ts
@@ -20,6 +20,7 @@ vi.mock("@langfuse/shared/topics/server", () => ({
TOPICS_TRANSCRIPT_VERSION: "shared-transcript-v3",
}));
vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),
isInternalEvalEnvironment: (
await importOriginal<typeof import("@langfuse/shared/src/server")>()
).isInternalEvalEnvironment,
@@ -34,11 +35,13 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
) => action({ setAttributes: vi.fn(), setAttribute: state.setAttribute }),
}));
vi.mock("./models", () => ({
+ TopicInputTooLarge: class extends Error {},
summarizeTopicTraceFacets: (...args: unknown[]) => state.summarize(...args),
embedTopicSummaries: (...args: unknown[]) => state.embed(...args),
}));
import { summarizeAssembledTrace } from "./summarizeAssembledTrace";
+import { TopicsProviderUnavailable } from "./provider-error";
const transcript: Transcript = {
threads: [
@@ -394,6 +397,33 @@ describe("summarizeAssembledTrace", () => {
{ stage: "summary", reason: "invalid_output" },
],
]);
+ // Rejected output is a local rule's decision; accepted output is the model's.
+ expect(
+ vi
+ .mocked(recordIncrement)
+ .mock.calls.filter(([name]) => name === "langfuse.topics.results"),
+ ).toEqual([
+ [
+ "langfuse.topics.results",
+ 1,
+ {
+ stage: "summary",
+ result: "invalid_output",
+ source: "rule",
+ facet: "intent",
+ },
+ ],
+ [
+ "langfuse.topics.results",
+ 1,
+ {
+ stage: "summary",
+ result: "applicable",
+ source: "model",
+ facet: "issues",
+ },
+ ],
+ ]);
expect(state.setAttribute).toHaveBeenCalledWith(
"langfuse.topics.trace_outcome",
"failed",
@@ -493,6 +523,41 @@ describe("summarizeAssembledTrace", () => {
);
});
+ it("counts an unusable response as a rule rejection of every facet", async () => {
+ state.summarize.mockRejectedValue(
+ new TopicsProviderUnavailable("unparsable", "invalid_output"),
+ );
+ await expect(
+ summarize({
+ projectId: "project-a",
+ traceId: "trace-1",
+ traceTimestamp: "2026-09-22T12:00:00.000Z",
+ jobEnqueuedAt: new Date("2026-09-22T12:01:00.000Z"),
+ environment: "default",
+ traceName: "agent-turn",
+ transcript,
+ models,
+ facets: state.facets(),
+ }),
+ ).rejects.toMatchObject({ reason: "invalid_output" });
+ expect(
+ vi
+ .mocked(recordIncrement)
+ .mock.calls.filter(([name]) => name === "langfuse.topics.results"),
+ ).toEqual([
+ [
+ "langfuse.topics.results",
+ 1,
+ {
+ stage: "summary",
+ result: "invalid_output",
+ source: "rule",
+ facet: "intent",
+ },
+ ],
+ ]);
+ });
+
it("propagates a provider failure for the trace outcome", async () => {
const error = new Error("provider unavailable");
state.summarize.mockRejectedValue(error);
diff --git a/worker/src/features/topics/summarizeAssembledTrace.ts b/worker/src/features/topics/summarizeAssembledTrace.ts
index 4cc408a040..ee98ea7478 100644
--- a/worker/src/features/topics/summarizeAssembledTrace.ts
+++ b/worker/src/features/topics/summarizeAssembledTrace.ts
@@ -4,6 +4,7 @@ import {
type TopicsModels,
} from "@langfuse/shared/topics/server";
import {
+ DEFAULT_TOPIC_FACETS,
topicProcessingConfigSchema,
type TopicFacet,
type TopicProcessingConfig,
@@ -11,13 +12,20 @@ import {
type TopicSummary,
} from "@langfuse/shared/topics";
import {
+ contextWithLangfuseProps,
+ getCurrentSpan,
instrumentAsync,
isInternalEvalEnvironment,
type Transcript,
} from "@langfuse/shared/src/server";
+import { context as otelContext } from "@opentelemetry/api";
import { prepareAssembledTopicTranscript } from "./assembledTranscript";
-import { TopicMetrics } from "./metrics";
-import { embedTopicSummaries, summarizeTopicTraceFacets } from "./models";
+import { TopicMetrics, type TopicResult } from "./metrics";
+import {
+ embedTopicSummaries,
+ summarizeTopicTraceFacets,
+ TopicInputTooLarge,
+} from "./models";
import { TopicsProviderUnavailable } from "./provider-error";
import {
mergeTopicModelUsage,
@@ -37,6 +45,10 @@ const NO_USAGE: TopicModelUsage = {
costDetails: {},
};
+const BUILT_IN_FACETS = new Set(
+ DEFAULT_TOPIC_FACETS.map((facet) => facet.name),
+);
+
// Rows from this transcript producer are interchangeable between trace-batch and manual processing.
export const ASSEMBLED_TRANSCRIPT_ID = "trace-batch";
@@ -80,29 +92,32 @@ export async function summarizeAssembledTrace(
): Promise<AssembledTraceResult> {
if (isInternalEvalEnvironment(input.environment))
return { outcome: "disabled", rows: [] };
- return instrumentAsync(
- { name: "topics-trace-summary", traceScope: "topics" },
- async (span) => {
- span.setAttributes({
- "langfuse.project.id": input.projectId,
- "langfuse.trace.id": input.traceId,
- });
- const metrics = new TopicMetrics();
- let result: AssembledTraceResult | undefined;
- try {
- await metrics.measure("trace", async () => {
- result = await summarizeEnabledTrace(input, metrics);
- if (result.failure !== undefined) throw result.failure;
- });
- span.setAttribute("langfuse.topics.trace_outcome", result!.outcome);
- return result!;
- } catch (error) {
- span.setAttribute("langfuse.topics.trace_outcome", "failed");
- // A partial failure still returns the valid facets' rows.
- if (result && result.failure === error) return result;
- throw error;
- }
- },
+ // Trace-batch jobs span projects; the baggage tags this trace's spans and logs.
+ const projectContext = contextWithLangfuseProps({
+ projectId: input.projectId,
+ });
+ return otelContext.with(projectContext, () =>
+ instrumentAsync(
+ { name: "topics-trace-summary", traceScope: "topics" },
+ async (span) => {
+ span.setAttribute("langfuse.trace.id", input.traceId);
+ const metrics = new TopicMetrics();
+ let result: AssembledTraceResult | undefined;
+ try {
+ await metrics.measure("trace", async () => {
+ result = await summarizeEnabledTrace(input, metrics);
+ if (result.failure !== undefined) throw result.failure;
+ });
+ span.setAttribute("langfuse.topics.trace_outcome", result!.outcome);
+ return result!;
+ } catch (error) {
+ span.setAttribute("langfuse.topics.trace_outcome", "failed");
+ // A partial failure still returns the valid facets' rows.
+ if (result && result.failure === error) return result;
+ throw error;
+ }
+ },
+ ),
);
}
@@ -197,6 +212,7 @@ async function summarizeEnabledTrace(
models.embedding,
applicable,
models.embedding.dimensions,
+ models.projectId,
),
);
} catch (error) {
@@ -248,8 +264,16 @@ export async function summarizeTraceFacets(params: {
{ results: TraceFacetResult[]; usage: TopicModelUsage } | undefined
> {
const { models, facets, config, metrics } = params;
+ const outcome = (
+ facet: TraceFacet,
+ result: TopicResult,
+ source: "model" | "rule",
+ ) => metrics.result("summary", result, 1, { source, facet: facetTag(facet) });
const prepared = prepareAssembledTopicTranscript(params.transcript);
- if (!prepared.hasContent) return undefined;
+ if (!prepared.hasContent) {
+ for (const facet of facets) outcome(facet, "insufficient_input", "rule");
+ return undefined;
+ }
// Built-in facets first, each group in listing order, so the cached prompt prefix stays stable.
const ordered = [
...facets.filter((facet) => facet.builtIn),
@@ -287,18 +311,37 @@ export async function summarizeTraceFacets(params: {
if (first) throw first;
})
.catch((error: unknown) => {
+ if (error instanceof TopicInputTooLarge)
+ for (const facet of facets) outcome(facet, "input_too_large", "rule");
+ // An unusable response fails every facet before the per-facet checks run.
+ else if (
+ (error instanceof TopicsProviderUnavailable
+ ? error.reason === "invalid_output"
+ : error instanceof Error && error.name === "ZodError") &&
+ ![...errors.values()].includes(error as never)
+ )
+ for (const facet of facets) outcome(facet, "invalid_output", "rule");
// Valid facets can still be persisted after summary validation fails.
if (!generated || ![...errors.values()].includes(error as never))
throw error;
});
const result = generated!;
const outputs = result.output as Record<string, FacetOutput>;
+ const results = facets.map((facet): TraceFacetResult => {
+ const { key } = keyed[ordered.indexOf(facet)];
+ const error = errors.get(key);
+ if (error) outcome(facet, "invalid_output", "rule");
+ else outcome(facet, outputs[key].status, "model");
+ return error ? { error } : { output: outputs[key] };
+ });
+ const counts: Record<string, number> = {};
+ for (const item of results) {
+ const key = `langfuse.topics.facets.${"error" in item ? "invalid_output" : item.output.status}`;
+ counts[key] = (counts[key] ?? 0) + 1;
+ }
+ getCurrentSpan()?.setAttributes(counts);
return {
- results: facets.map((facet) => {
- const { key } = keyed[ordered.indexOf(facet)];
- const error = errors.get(key);
- return error ? { error } : { output: outputs[key] };
- }),
+ results,
usage: {
providedUsageDetails: result.providedUsageDetails,
usageDetails: result.usageDetails,
@@ -308,6 +351,13 @@ export async function summarizeTraceFacets(params: {
};
}
+/** Built-in facet names are a fixed set; custom names would be unbounded metric tags. */
+function facetTag(facet: TraceFacet): string {
+ return facet.builtIn && BUILT_IN_FACETS.has(facet.name)
+ ? facet.name.toLowerCase()
+ : "custom";
+}
+
/** The summary row both processing paths start from, before any model call. */
export function baseTopicSummary(params: {
projectId: string;
diff --git a/worker/src/queues/__tests__/traceBatchQueue.test.ts b/worker/src/queues/__tests__/traceBatchQueue.test.ts
index 484b9522b6..5b3b8f41a6 100644
--- a/worker/src/queues/__tests__/traceBatchQueue.test.ts
+++ b/worker/src/queues/__tests__/traceBatchQueue.test.ts
@@ -1903,7 +1903,10 @@ describe("trace batch queue", () => {
metadataBytes: 0,
ioMetadataBytes: hasRows ? 10 : 0,
});
- expect(processingSpan.setAttributes).not.toHaveBeenCalled();
+ // Only Topics batch totals; no experiment attributes without an experiment.
+ expect(vi.mocked(processingSpan.setAttributes).mock.calls).toEqual(
+ hasRows ? [[{ "langfuse.topics.traces.unchanged": 1 }]] : [],
+ );
expect(recordDistribution).toHaveBeenCalledWith(
"langfuse.trace_batch.input_bytes",
hasRows ? 5 : 0,
diff --git a/worker/src/queues/traceBatchQueue.ts b/worker/src/queues/traceBatchQueue.ts
index a7e52a5d64..43786635aa 100644
--- a/worker/src/queues/traceBatchQueue.ts
+++ b/worker/src/queues/traceBatchQueue.ts
@@ -189,6 +189,7 @@ async function summarizeTraceBatch(
projectId: first.projectId,
traceId,
reason,
+ error: error instanceof Error ? error.message : String(error),
});
return { outcome: "failed", reason, rows };
}
@@ -241,13 +242,20 @@ function recordTopicsOutcomes(outcomes: TraceOutcome[]): void {
]);
counts.set(key, (counts.get(key) ?? 0) + 1);
}
+ const perOutcome: Record<string, number> = {};
for (const [key, count] of counts) {
const [outcome, reason] = JSON.parse(key) as [string, string | null];
recordIncrement("langfuse.topics.trace_outcomes", count, {
outcome,
...(reason ? { reason } : {}),
});
+ if (outcome === "disabled") continue;
+ const attribute = `langfuse.topics.traces.${outcome}`;
+ perOutcome[attribute] = (perOutcome[attribute] ?? 0) + count;
}
+ // Batch totals of Topics work on the job span; each trace has its own child span.
+ if (Object.keys(perOutcome).length > 0)
+ getCurrentSpan()?.setAttributes(perOutcome);
}
const JOB_MAX_AGE_MS = 2 * 60 * 60_000;
diff --git a/worker/src/services/exportVolumeMetric.ts b/worker/src/services/exportVolumeMetric.ts
index 543accf9c1..18ebba4c01 100644
--- a/worker/src/services/exportVolumeMetric.ts
+++ b/worker/src/services/exportVolumeMetric.ts
@@ -1,16 +1,22 @@
+import { z } from "zod";
import { recordIncrement } from "@langfuse/shared/src/server";
export const EXPORT_VOLUME_METRIC = "langfuse.export.serialized_bytes";
-type ExportIntegration = "blob_storage" | "posthog" | "mixpanel" | "llmaj";
+type ExportIntegration =
+ | "blob_storage"
+ | "posthog"
+ | "mixpanel"
+ | "llmaj"
+ | "topics";
type ExportVolume = {
integration: ExportIntegration;
// On-wire egress bytes shipped this run: gzipped for blob/posthog/mixpanel,
- // uncompressed for llmaj (LLM requests aren't gzipped).
+ // uncompressed for llmaj and topics (LLM requests aren't gzipped).
bytes: number;
// Only tagged for blob_storage (per-project egress cost attribution); the
- // API-based integrations (posthog/mixpanel/llmaj) omit it to keep custom
+ // API-based integrations (posthog/mixpanel/llmaj/topics) omit it to keep custom
// metric cardinality down.
projectId: string;
// Egress-cost classification; blob only (S3 / S3_COMPATIBLE / AZURE_*).
@@ -44,3 +50,32 @@ export const recordExportVolume = ({
if (path !== undefined) tags.path = path;
recordIncrement(EXPORT_VOLUME_METRIC, bytes, tags);
};
+
+/**
+ * Egress bytes of an LLM request as LLM-as-a-judge and Topics count them: the
+ * provider-bound messages plus the JSON Schema of the requested output, uncompressed.
+ */
+export const llmRequestEgressBytes = (
+ messages: unknown,
+ schema?: unknown,
+): number =>
+ Buffer.byteLength(
+ JSON.stringify(messages, (_key, value) =>
+ value instanceof Uint8Array
+ ? Buffer.from(value).toString("base64")
+ : value,
+ ),
+ "utf8",
+ ) +
+ (schema === undefined
+ ? 0
+ : Buffer.byteLength(serializeSchemaForEgress(schema), "utf8"));
+
+// Measure the schema as the JSON Schema LangChain ships, not Zod's _def.
+function serializeSchemaForEgress(schema: unknown): string {
+ try {
+ return JSON.stringify(z.toJSONSchema(schema as z.ZodType));
+ } catch {
+ return JSON.stringify(schema);
+ }
+}