Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(topics): add more clustering diagnostics (#19050)

Nimarbde307305a1b
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.md
index eda86e1457..e4ed4aece7 100644
--- a/packages/shared/AGENTS.md
+++ b/packages/shared/AGENTS.md
@@ -118,6 +118,10 @@
saved evaluator routing independent. Text accepts optional generation tracing;
worker model calls own tracing allowlisting, organization consent, usage,
cost and vector validation.
+ Failed text calls retain bounded stop-reason/count diagnostics through
+ `getTopicTextResponseMetadata(error)` without changing the original error.
+ Keep response content out of these diagnostics; opted-in generation traces
+ remain the content-bearing path.
- `model-config.ts`: per-project `topic_configs` (three connection +
model slots and the automatic-processing state). Saving keeps the stored
state and pause (a first save starts off) and tests every slot with a real
diff --git a/packages/shared/src/server/topics/text.test.ts b/packages/shared/src/server/topics/text.test.ts
index 90c9d8f56c..5eaf98b2b1 100644
--- a/packages/shared/src/server/topics/text.test.ts
+++ b/packages/shared/src/server/topics/text.test.ts
@@ -4,10 +4,11 @@ import { encrypt } from "../../encryption";
import { env } from "../../env";
import { BEDROCK_USE_DEFAULT_CREDENTIALS } from "../../interfaces/customLLMProviderConfigSchemas";
import { LLMAdapter, type TraceSinkParams } from "../llm/types";
+import { logger } from "../logger";
import type { ResourceSpan } from "../otel/OtelIngestionProcessor";
import { generateTopicEmbeddings } from "./embeddings";
import type { TopicsModel } from "./model-config";
-import { generateTopicText } from "./text";
+import { generateTopicText, getTopicTextResponseMetadata } from "./text";
const { publishToOtelIngestionQueue, processorConstructor } = vi.hoisted(
() => ({
@@ -31,6 +32,10 @@ vi.mock("../llm/secureLlmFetch", () => ({
createSecureLlmFetch: () => globalThis.fetch,
}));
+vi.mock("../logger", () => ({
+ logger: { error: vi.fn(), warn: vi.fn(), info: vi.fn(), debug: vi.fn() },
+}));
+
const trace: TraceSinkParams = {
targetProjectId: "ai-features-project",
traceId: "0af7651916cd43dd8448eb211c80319c",
@@ -97,6 +102,69 @@ function bedrockModel(slot: TopicsModel["slot"], model: string): TopicsModel {
}
describe("generateTopicText", () => {
+ it.each([
+ ["max_tokens", "length", "", "AI_NoOutputGeneratedError"],
+ ["refusal", "other", "", "AI_NoOutputGeneratedError"],
+ ["guardrail_intervened", "content-filter", "", "AI_NoOutputGeneratedError"],
+ ["end_turn", "stop", "private invalid JSON", "AI_NoObjectGeneratedError"],
+ ["private provider reason", "other", "", "AI_NoOutputGeneratedError"],
+ ])(
+ "retains bounded response metadata before structured output fails: %s",
+ async (stopReason, finishReason, text, errorName) => {
+ vi.stubGlobal(
+ "fetch",
+ vi.fn(async () =>
+ Response.json({
+ output: {
+ message: {
+ role: "assistant",
+ content: [
+ {
+ reasoningContent: {
+ reasoningText: {
+ text: "private reasoning",
+ signature: "private signature",
+ },
+ },
+ },
+ ...(text ? [{ text }] : []),
+ ],
+ },
+ },
+ stopReason,
+ usage: { inputTokens: 12, outputTokens: 4, totalTokens: 16 },
+ metrics: { latencyMs: 1 },
+ }),
+ ),
+ );
+ const error = await generateTopicText({
+ model: bedrockModel("clustering", "eu.anthropic.claude-sonnet-5-5"),
+ system: [{ text: "private system prompt" }],
+ input: "private input",
+ schema: z.object({ name: z.string() }),
+ maxOutputTokens: 256,
+ }).catch((error: unknown) => error);
+
+ expect(error).toMatchObject({ name: errorName });
+ const metadata = getTopicTextResponseMetadata(error);
+ expect(metadata).toEqual({
+ "langfuse.topics.response.finish_reason": finishReason,
+ "langfuse.topics.response.raw_finish_reason": stopReason.startsWith(
+ "private",
+ )
+ ? "other"
+ : stopReason,
+ "langfuse.topics.response.text_characters": 0,
+ "langfuse.topics.response.text_part_count": text ? 1 : 0,
+ "langfuse.topics.response.reasoning_part_count": 1,
+ "langfuse.topics.response.output_tokens": 4,
+ "langfuse.topics.response.max_output_tokens": 256,
+ });
+ expect(JSON.stringify(metadata)).not.toContain("private");
+ expect(publishToOtelIngestionQueue).not.toHaveBeenCalled();
+ },
+ );
+
it("clusters with Bedrock Sonnet 5.5 without forcing tool use", async () => {
const output = { name: "Account access" };
const fetch = vi.fn<typeof globalThis.fetch>(async (...args) => {
@@ -349,7 +417,10 @@ describe("generateTopicText", () => {
"generates and traces Topics on %s / %s",
async (baseURL, model, reasoning, completionTokens) => {
publishToOtelIngestionQueue.mockRejectedValueOnce(
- new Error("queue down"),
+ Object.assign(new Error("queue rejected secret model input/output"), {
+ body: "secret model input/output",
+ cause: new Error("secret model input/output"),
+ }),
);
vi.stubGlobal(
"fetch",
@@ -425,6 +496,13 @@ describe("generateTopicText", () => {
inputTokens: 3000,
cacheReadTokens: 2560,
});
+ expect(logger.error).toHaveBeenCalledWith(
+ "Failed to publish Topics model trace",
+ expect.objectContaining({ traceId: trace.traceId }),
+ );
+ expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain(
+ "secret",
+ );
},
);
it("retains SDK usage when no output is generated", async () => {
diff --git a/packages/shared/src/server/topics/text.ts b/packages/shared/src/server/topics/text.ts
index c615f8713b..daa3c980dd 100644
--- a/packages/shared/src/server/topics/text.ts
+++ b/packages/shared/src/server/topics/text.ts
@@ -1,6 +1,12 @@
import { Sha256 } from "@aws-crypto/sha256-js";
import { SignatureV4 } from "@smithy/signature-v4";
-import { NoObjectGeneratedError, Output, type LanguageModelUsage } from "ai";
+import {
+ NoObjectGeneratedError,
+ Output,
+ type LanguageModelUsage,
+ type StepResult,
+ type ToolSet,
+} from "ai";
import { z } from "zod";
import { decrypt } from "../../encryption";
import { BedrockConfigSchema } from "../../interfaces/customLLMProviderConfigSchemas";
@@ -18,6 +24,35 @@ import type { TopicsModel } from "./model-config";
const TIMEOUT_MS = 60_000;
+// Error identity and lifetime are preserved, including for frozen SDK errors.
+const responseMetadataByError = new WeakMap<
+ object,
+ TopicTextResponseMetadata
+>();
+
+const RAW_FINISH_REASONS = new Set([
+ "stop",
+ "length",
+ "content_filter",
+ "tool_calls",
+ "function_call",
+ "end_turn",
+ "max_tokens",
+ "stop_sequence",
+ "tool_use",
+ "refusal",
+ "guardrail_intervened",
+ "content_filtered",
+ "model_context_window_exceeded",
+ "malformed_model_output",
+ "pause_turn",
+ "STOP",
+ "MAX_TOKENS",
+ "SAFETY",
+ "RECITATION",
+ "OTHER",
+]);
+
/**
* Structured Topics text through the project's LLM connection. GPT-5.6+ models
* on Bedrock reuse a shared prompt prefix only with explicit cache breakpoints,
@@ -29,9 +64,14 @@ export async function generateTopicText<T>(
): Promise<TopicTextResult> {
const startTimeISO = new Date().toISOString();
let result: TopicTextResult | undefined;
+ let responseMetadata: TopicTextResponseMetadata | undefined;
let failure: unknown;
- const recordResponse = (response: TopicTextResult) => {
+ const recordResponse = (
+ response: TopicTextResult,
+ metadata?: TopicTextResponseMetadata,
+ ) => {
result = response;
+ responseMetadata = metadata;
};
try {
result =
@@ -41,6 +81,9 @@ export async function generateTopicText<T>(
return result;
} catch (error) {
failure = error;
+ if (responseMetadata && error !== null && typeof error === "object") {
+ responseMetadataByError.set(error, responseMetadata);
+ }
if (NoObjectGeneratedError.isInstance(error)) {
result = { output: error.text, usage: topicUsage(error.usage) };
}
@@ -50,6 +93,13 @@ export async function generateTopicText<T>(
}
}
+/** Bounded response diagnostics for the existing Topics failure log; no content. */
+export function getTopicTextResponseMetadata(error: unknown) {
+ return error !== null && typeof error === "object"
+ ? responseMetadataByError.get(error)
+ : undefined;
+}
+
async function writeTopicTrace<T>(
params: TopicTextParams<T>,
startTimeISO: string,
@@ -86,9 +136,8 @@ async function writeTopicTrace<T>(
},
],
});
- } catch (error) {
+ } catch {
logger.error("Failed to publish Topics model trace", {
- error,
traceId: trace.traceId,
});
}
@@ -155,7 +204,10 @@ function topicUsage(
async function generateWithConnection<T>(
params: TopicTextParams<T>,
- recordResponse: (result: TopicTextResult) => void,
+ recordResponse: (
+ result: TopicTextResult,
+ metadata: TopicTextResponseMetadata,
+ ) => void,
): Promise<TopicTextResult> {
const nativeOpenAI = isFirstPartyOpenAI(params.model);
const result = await generateLLMText({
@@ -182,16 +234,48 @@ async function generateWithConnection<T>(
// The AI SDK retries 429 and 5xx responses with exponential backoff.
maxRetries: 2,
timeout: params.timeoutMs ?? TIMEOUT_MS,
+ // onEnd runs before structured-output parsing and the output getter can throw.
+ onEnd: ({ finalStep }) => {
+ const text = finalStep.text;
+ recordResponse(
+ { output: text, usage: topicUsage(finalStep.usage) },
+ responseAttributes(finalStep, text.length, params.maxOutputTokens),
+ );
+ },
});
const usage = topicUsage(result.usage);
- // The SDK's output getter can throw after a response with billable usage.
- recordResponse({ output: result.text, usage });
return {
output: result.output,
usage,
};
}
+function responseAttributes(
+ step: StepResult<ToolSet>,
+ textCharacters: number,
+ maxOutputTokens: number,
+): TopicTextResponseMetadata {
+ return {
+ "langfuse.topics.response.finish_reason": step.finishReason,
+ "langfuse.topics.response.raw_finish_reason": RAW_FINISH_REASONS.has(
+ step.rawFinishReason ?? "",
+ )
+ ? step.rawFinishReason!
+ : "other",
+ "langfuse.topics.response.text_characters": textCharacters,
+ "langfuse.topics.response.text_part_count": step.content.filter(
+ (part) => part.type === "text",
+ ).length,
+ "langfuse.topics.response.reasoning_part_count": step.content.filter(
+ (part) => part.type === "reasoning",
+ ).length,
+ "langfuse.topics.response.max_output_tokens": maxOutputTokens,
+ ...(step.usage.outputTokens !== undefined && {
+ "langfuse.topics.response.output_tokens": step.usage.outputTokens,
+ }),
+ };
+}
+
/** Topics owns its OpenAI request options; saved evaluator options keep their routing. */
function isFirstPartyOpenAI(model: TopicsModel): boolean {
if (model.adapter !== LLMAdapter.OpenAI) return false;
@@ -389,6 +473,8 @@ async function invokeWithCache(
/** A system prompt segment; `cache` ends a reusable prefix with a cache breakpoint. */
export type TopicPromptPart = { text: string; cache?: boolean };
+type TopicTextResponseMetadata = Record<string, string | number>;
+
type TopicTextResult = {
output: unknown;
usage: {
diff --git a/worker/AGENTS.md b/worker/AGENTS.md
index 74c3fea71c..1f9d585813 100644
--- a/worker/AGENTS.md
+++ b/worker/AGENTS.md
@@ -38,6 +38,8 @@
organization `aiFeaturesEnabled` and `aiTelemetryEnabled` in `models.ts`;
embeddings are untraced. Use `isInternalEvalEnvironment`
to exclude internal sources.
+ Model failures add shared response diagnostics to the existing warning/span;
+ preserve one warning per failed call and bounded metric tags.
Pass models explicitly; never put them on execution or batch state, which is
persisted. Trace-batch jobs load enabled settings for all allowlisted batch
projects in one query (`getEnabledTopicsModels`) and pause a project on
diff --git a/worker/src/features/topics/models.test.ts b/worker/src/features/topics/models.test.ts
index 0802c64e61..48010d3e24 100644
--- a/worker/src/features/topics/models.test.ts
+++ b/worker/src/features/topics/models.test.ts
@@ -8,6 +8,8 @@ const state = vi.hoisted(() => ({
increment: vi.fn(),
project: vi.fn(),
traceSink: vi.fn(),
+ responseMetadata: vi.fn(),
+ spanAttributes: vi.fn(),
encoding: null as {
encode: ReturnType<typeof vi.fn>;
free: ReturnType<typeof vi.fn>;
@@ -28,6 +30,10 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
LangfuseInternalTraceEnvironment: { Topics: "langfuse-topics" },
logger: { warn: vi.fn() },
recordIncrement: state.increment,
+ instrumentAsync: async (
+ _options: unknown,
+ call: (span: { setAttributes: typeof state.spanAttributes }) => unknown,
+ ) => call({ setAttributes: state.spanAttributes }),
}));
vi.mock("@langfuse/shared/src/db", () => ({
prisma: { project: { findFirst: state.project } },
@@ -38,6 +44,8 @@ vi.mock("@langfuse/shared/topics/server", async (importOriginal) => ({
).topicMessages,
generateTopicText: (...args: unknown[]) => state.call(...args),
generateTopicEmbeddings: (...args: unknown[]) => state.embed(...args),
+ getTopicTextResponseMetadata: (error: unknown) =>
+ state.responseMetadata(error),
}));
// Token counters only; every call also records call and request-byte counters.
@@ -115,6 +123,9 @@ beforeEach(() => {
state.call.mockReset();
state.embed.mockReset();
state.increment.mockReset();
+ state.responseMetadata.mockReset();
+ state.spanAttributes.mockReset();
+ vi.mocked(logger.warn).mockClear();
state.project.mockReset().mockResolvedValue({
organization: { aiFeaturesEnabled: true, aiTelemetryEnabled: true },
});
@@ -308,12 +319,10 @@ describe("Topics naming boundary", () => {
expect(state.call).not.toHaveBeenCalled();
});
- it("logs the provider's error fields for a rejected request, never its bodies", async () => {
+ it("excludes provider content from failure logs and spans, including echoed error fields", async () => {
// The AI SDK identifies an APICallError by these markers.
const rejection = Object.assign(
- new Error(
- "Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead.",
- ),
+ new Error("Invalid input: secret transcript"),
{
[Symbol.for("vercel.ai.error")]: true,
[Symbol.for("vercel.ai.error.AI_APICallError")]: true,
@@ -324,9 +333,9 @@ describe("Topics naming boundary", () => {
responseBody: '{"error":{"message":"secret echo"}}',
data: {
error: {
- type: "invalid_request_error",
- code: "unsupported_parameter",
- param: "max_tokens",
+ type: "secret error type",
+ code: "secret error code",
+ param: "secret input value",
},
},
},
@@ -345,11 +354,11 @@ describe("Topics naming boundary", () => {
expect(message).toBe("Topics model request failed");
expect(fields).toMatchObject({
"http.status_code": 400,
- "langfuse.topics.provider.message": rejection.message,
- "langfuse.topics.provider.code": "unsupported_parameter",
- "langfuse.topics.provider.param": "max_tokens",
});
expect(JSON.stringify(fields)).not.toMatch(/secret/);
+ expect(JSON.stringify(state.spanAttributes.mock.calls)).not.toMatch(
+ /secret/,
+ );
expect(state.increment).toHaveBeenCalledWith(
"langfuse.topics.llm_calls",
1,
@@ -361,6 +370,55 @@ describe("Topics naming boundary", () => {
);
});
+ it("reports unusable naming output without logging provider content", async () => {
+ const rejection = Object.assign(new Error("secret response text"), {
+ name: "AI_NoOutputGeneratedError",
+ responseBody: "secret response body",
+ text: "secret model output",
+ });
+ const response = {
+ "langfuse.topics.response.finish_reason": "other",
+ "langfuse.topics.response.raw_finish_reason": "refusal",
+ "langfuse.topics.response.text_characters": 0,
+ "langfuse.topics.response.output_tokens": 0,
+ };
+ state.responseMetadata.mockImplementation((error) =>
+ error === rejection ? response : undefined,
+ );
+ state.call.mockRejectedValue(rejection);
+
+ const failure = await nameTopicGroup(topicsModels(), evidence).catch(
+ (error: unknown) => error,
+ );
+ expect(failure).toMatchObject({
+ reason: "invalid_output",
+ message: expect.stringContaining("returned no usable structured output"),
+ });
+ expect((failure as Error).message).not.toMatch(/credentials|secret/);
+ expect(logger.warn).toHaveBeenCalledOnce();
+ const [message, fields] = vi.mocked(logger.warn).mock.calls[0];
+ expect(message).toBe("Topics model request failed");
+ expect(fields).toMatchObject({
+ ...response,
+ "langfuse.topics.error.reason": "invalid_output",
+ "langfuse.topics.error.type": "AI_NoOutputGeneratedError",
+ });
+ expect(state.spanAttributes).toHaveBeenCalledWith(fields);
+ expect(JSON.stringify(fields)).not.toMatch(/secret/);
+ expect(JSON.stringify(state.spanAttributes.mock.calls)).not.toMatch(
+ /secret/,
+ );
+ expect(state.increment).toHaveBeenCalledWith(
+ "langfuse.topics.llm_calls",
+ 1,
+ {
+ stage: "naming",
+ adapter: LLMAdapter.Bedrock,
+ status: "invalid_output",
+ },
+ );
+ });
+
it.each([
[{}, "ok"],
[{ httpStatus: 429, reason: "rate_limit" }, "429"],
diff --git a/worker/src/features/topics/models.ts b/worker/src/features/topics/models.ts
index 634c2a249e..b7799a3fe8 100644
--- a/worker/src/features/topics/models.ts
+++ b/worker/src/features/topics/models.ts
@@ -12,6 +12,7 @@ import {
import {
generateTopicEmbeddings,
generateTopicText,
+ getTopicTextResponseMetadata,
topicMessages,
type TopicPromptPart,
type TopicsModel,
@@ -125,7 +126,7 @@ async function topicModelCall<T>(
return result;
} catch (error) {
const failure = topicProviderError(error, model);
- const { httpStatus, message, type, code, param } = failure.details;
+ const { httpStatus } = failure.details;
recordTopicModelCall(stage, model.adapter, {
httpStatus,
reason: failure.reason,
@@ -136,10 +137,7 @@ async function topicModelCall<T>(
"langfuse.topics.error.type":
error instanceof Error ? error.name : "unknown",
...(httpStatus && { "http.status_code": httpStatus }),
- ...(message && { "langfuse.topics.provider.message": message }),
- ...(type && { "langfuse.topics.provider.type": type }),
- ...(code && { "langfuse.topics.provider.code": code }),
- ...(param && { "langfuse.topics.provider.param": param }),
+ ...getTopicTextResponseMetadata(error),
};
span.setAttributes(failed);
logger.warn("Topics model request failed", failed);
diff --git a/worker/src/features/topics/processTopicEmbeddingBatch.test.ts b/worker/src/features/topics/processTopicEmbeddingBatch.test.ts
index 9f8de7d1a0..8430f75d29 100644
--- a/worker/src/features/topics/processTopicEmbeddingBatch.test.ts
+++ b/worker/src/features/topics/processTopicEmbeddingBatch.test.ts
@@ -16,6 +16,7 @@ const mocks = vi.hoisted(() => ({
embeddingModel: "eu.cohere.embed-v4:0",
}));
vi.mock("@langfuse/shared/topics/server", () => ({
+ getTopicTextResponseMetadata: () => undefined,
pauseTopicsModels: mocks.pause,
writeTopicSummaries: mocks.write,
readStagedTopicSummary: mocks.staged,
diff --git a/worker/src/features/topics/processTopicsExecution.test.ts b/worker/src/features/topics/processTopicsExecution.test.ts
index ed0d15575b..8522956a2f 100644
--- a/worker/src/features/topics/processTopicsExecution.test.ts
+++ b/worker/src/features/topics/processTopicsExecution.test.ts
@@ -76,6 +76,7 @@ vi.mock("@langfuse/shared/topics/server", async (importOriginal) => {
const { TOPICS_TRANSCRIPT_VERSION, TOPIC_EMBEDDING_EXPIRED_ERROR } =
await importOriginal<typeof import("@langfuse/shared/topics/server")>();
return {
+ getTopicTextResponseMetadata: () => undefined,
isTopicsEnabled: () => true,
getTopicsModels: async () => {
const slot = (name: string, model: string) => ({
diff --git a/worker/src/features/topics/provider-error.ts b/worker/src/features/topics/provider-error.ts
index 398b17deab..0b2d7d2a21 100644
--- a/worker/src/features/topics/provider-error.ts
+++ b/worker/src/features/topics/provider-error.ts
@@ -22,10 +22,6 @@ type TopicsProviderErrorReason =
/** Telemetry only; the user-facing message stays generic. */
export type TopicProviderErrorDetails = {
httpStatus?: number;
- message?: string;
- type?: string;
- code?: string;
- param?: string;
};
export class TopicsProviderUnavailable extends Error {
@@ -79,40 +75,25 @@ export function topicProviderError(
reason = "invalid_output";
current = record.cause;
}
+ const modelDescription = `The ${TOPICS_MODEL_SLOT_DETAILS[model.slot].label.toLowerCase()} model ${model.model} on LLM connection "${model.provider}"`;
return new TopicsProviderUnavailable(
- `The ${TOPICS_MODEL_SLOT_DETAILS[model.slot].label.toLowerCase()} model ${model.model} on LLM connection "${model.provider}" failed${status ? ` (HTTP ${status})` : ""}. Check the connection's credentials, the model ID, and provider availability.`,
+ reason === "invalid_output"
+ ? `${modelDescription} returned no usable structured output. Resume interrupted stages to retry, or choose another model.`
+ : `${modelDescription} failed${status ? ` (HTTP ${status})` : ""}. Check the connection's credentials, the model ID, and provider availability.`,
reason,
model.slot,
providerErrorDetails(error, status),
);
}
-/**
- * Like evals, keeps only the AI SDK's parsed provider error fields; plain errors
- * (e.g. the direct Bedrock call) carry the response body, so they keep the status only.
- */
+/** Provider error strings can echo request/response content; retain only status. */
function providerErrorDetails(
error: unknown,
status: number | undefined,
): TopicProviderErrorDetails {
const provider = getLLMErrorInfo(error)?.providerError;
const httpStatus = provider?.statusCode ?? status;
- if (!httpStatus) return {};
- if (!provider) return { httpStatus };
- const record = (value: unknown) =>
- value && typeof value === "object"
- ? (value as Record<string, unknown>)
- : undefined;
- const data = record(record(provider.data)?.error) ?? record(provider.data);
- const text = (value: unknown) =>
- typeof value === "string" ? value.slice(0, 500) : undefined;
- return {
- httpStatus,
- message: text(provider.message),
- type: text(data?.type),
- code: text(data?.code),
- param: text(data?.param),
- };
+ return httpStatus ? { httpStatus } : {};
}
/** The provider rejected the connection's credentials or billing. */