🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(topics): add more clustering diagnostics (#19050)
Nimarbde307305a1b
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.mdindex eda86e1457..e4ed4aece7 100644--- a/packages/shared/AGENTS.md+++ b/packages/shared/AGENTS.md@@ -118,6 +118,10 @@saved evaluator routing independent. Text accepts optional generation tracing;worker model calls own tracing allowlisting, organization consent, usage,cost and vector validation.+ Failed text calls retain bounded stop-reason/count diagnostics through+ `getTopicTextResponseMetadata(error)` without changing the original error.+ Keep response content out of these diagnostics; opted-in generation traces+ remain the content-bearing path.- `model-config.ts`: per-project `topic_configs` (three connection +model slots and the automatic-processing state). Saving keeps the storedstate and pause (a first save starts off) and tests every slot with a realdiff --git a/packages/shared/src/server/topics/text.test.ts b/packages/shared/src/server/topics/text.test.tsindex 90c9d8f56c..5eaf98b2b1 100644--- a/packages/shared/src/server/topics/text.test.ts+++ b/packages/shared/src/server/topics/text.test.ts@@ -4,10 +4,11 @@ import { encrypt } from "../../encryption";import { env } from "../../env";import { BEDROCK_USE_DEFAULT_CREDENTIALS } from "../../interfaces/customLLMProviderConfigSchemas";import { LLMAdapter, type TraceSinkParams } from "../llm/types";+import { logger } from "../logger";import type { ResourceSpan } from "../otel/OtelIngestionProcessor";import { generateTopicEmbeddings } from "./embeddings";import type { TopicsModel } from "./model-config";-import { generateTopicText } from "./text";+import { generateTopicText, getTopicTextResponseMetadata } from "./text";const { publishToOtelIngestionQueue, processorConstructor } = vi.hoisted(() => ({@@ -31,6 +32,10 @@ vi.mock("../llm/secureLlmFetch", () => ({createSecureLlmFetch: () => globalThis.fetch,}));+vi.mock("../logger", () => ({+ logger: { error: vi.fn(), warn: vi.fn(), info: vi.fn(), debug: vi.fn() },+}));+const trace: TraceSinkParams = {targetProjectId: "ai-features-project",traceId: "0af7651916cd43dd8448eb211c80319c",@@ -97,6 +102,69 @@ function bedrockModel(slot: TopicsModel["slot"], model: string): TopicsModel {}describe("generateTopicText", () => {+ it.each([+ ["max_tokens", "length", "", "AI_NoOutputGeneratedError"],+ ["refusal", "other", "", "AI_NoOutputGeneratedError"],+ ["guardrail_intervened", "content-filter", "", "AI_NoOutputGeneratedError"],+ ["end_turn", "stop", "private invalid JSON", "AI_NoObjectGeneratedError"],+ ["private provider reason", "other", "", "AI_NoOutputGeneratedError"],+ ])(+ "retains bounded response metadata before structured output fails: %s",+ async (stopReason, finishReason, text, errorName) => {+ vi.stubGlobal(+ "fetch",+ vi.fn(async () =>+ Response.json({+ output: {+ message: {+ role: "assistant",+ content: [+ {+ reasoningContent: {+ reasoningText: {+ text: "private reasoning",+ signature: "private signature",+ },+ },+ },+ ...(text ? [{ text }] : []),+ ],+ },+ },+ stopReason,+ usage: { inputTokens: 12, outputTokens: 4, totalTokens: 16 },+ metrics: { latencyMs: 1 },+ }),+ ),+ );+ const error = await generateTopicText({+ model: bedrockModel("clustering", "eu.anthropic.claude-sonnet-5-5"),+ system: [{ text: "private system prompt" }],+ input: "private input",+ schema: z.object({ name: z.string() }),+ maxOutputTokens: 256,+ }).catch((error: unknown) => error);++ expect(error).toMatchObject({ name: errorName });+ const metadata = getTopicTextResponseMetadata(error);+ expect(metadata).toEqual({+ "langfuse.topics.response.finish_reason": finishReason,+ "langfuse.topics.response.raw_finish_reason": stopReason.startsWith(+ "private",+ )+ ? "other"+ : stopReason,+ "langfuse.topics.response.text_characters": 0,+ "langfuse.topics.response.text_part_count": text ? 1 : 0,+ "langfuse.topics.response.reasoning_part_count": 1,+ "langfuse.topics.response.output_tokens": 4,+ "langfuse.topics.response.max_output_tokens": 256,+ });+ expect(JSON.stringify(metadata)).not.toContain("private");+ expect(publishToOtelIngestionQueue).not.toHaveBeenCalled();+ },+ );+it("clusters with Bedrock Sonnet 5.5 without forcing tool use", async () => {const output = { name: "Account access" };const fetch = vi.fn<typeof globalThis.fetch>(async (...args) => {@@ -349,7 +417,10 @@ describe("generateTopicText", () => {"generates and traces Topics on %s / %s",async (baseURL, model, reasoning, completionTokens) => {publishToOtelIngestionQueue.mockRejectedValueOnce(- new Error("queue down"),+ Object.assign(new Error("queue rejected secret model input/output"), {+ body: "secret model input/output",+ cause: new Error("secret model input/output"),+ }),);vi.stubGlobal("fetch",@@ -425,6 +496,13 @@ describe("generateTopicText", () => {inputTokens: 3000,cacheReadTokens: 2560,});+ expect(logger.error).toHaveBeenCalledWith(+ "Failed to publish Topics model trace",+ expect.objectContaining({ traceId: trace.traceId }),+ );+ expect(JSON.stringify(vi.mocked(logger.error).mock.calls)).not.toContain(+ "secret",+ );},);it("retains SDK usage when no output is generated", async () => {diff --git a/packages/shared/src/server/topics/text.ts b/packages/shared/src/server/topics/text.tsindex c615f8713b..daa3c980dd 100644--- a/packages/shared/src/server/topics/text.ts+++ b/packages/shared/src/server/topics/text.ts@@ -1,6 +1,12 @@import { Sha256 } from "@aws-crypto/sha256-js";import { SignatureV4 } from "@smithy/signature-v4";-import { NoObjectGeneratedError, Output, type LanguageModelUsage } from "ai";+import {+ NoObjectGeneratedError,+ Output,+ type LanguageModelUsage,+ type StepResult,+ type ToolSet,+} from "ai";import { z } from "zod";import { decrypt } from "../../encryption";import { BedrockConfigSchema } from "../../interfaces/customLLMProviderConfigSchemas";@@ -18,6 +24,35 @@ import type { TopicsModel } from "./model-config";const TIMEOUT_MS = 60_000;+// Error identity and lifetime are preserved, including for frozen SDK errors.+const responseMetadataByError = new WeakMap<+ object,+ TopicTextResponseMetadata+>();++const RAW_FINISH_REASONS = new Set([+ "stop",+ "length",+ "content_filter",+ "tool_calls",+ "function_call",+ "end_turn",+ "max_tokens",+ "stop_sequence",+ "tool_use",+ "refusal",+ "guardrail_intervened",+ "content_filtered",+ "model_context_window_exceeded",+ "malformed_model_output",+ "pause_turn",+ "STOP",+ "MAX_TOKENS",+ "SAFETY",+ "RECITATION",+ "OTHER",+]);+/*** Structured Topics text through the project's LLM connection. GPT-5.6+ models* on Bedrock reuse a shared prompt prefix only with explicit cache breakpoints,@@ -29,9 +64,14 @@ export async function generateTopicText<T>(): Promise<TopicTextResult> {const startTimeISO = new Date().toISOString();let result: TopicTextResult | undefined;+ let responseMetadata: TopicTextResponseMetadata | undefined;let failure: unknown;- const recordResponse = (response: TopicTextResult) => {+ const recordResponse = (+ response: TopicTextResult,+ metadata?: TopicTextResponseMetadata,+ ) => {result = response;+ responseMetadata = metadata;};try {result =@@ -41,6 +81,9 @@ export async function generateTopicText<T>(return result;} catch (error) {failure = error;+ if (responseMetadata && error !== null && typeof error === "object") {+ responseMetadataByError.set(error, responseMetadata);+ }if (NoObjectGeneratedError.isInstance(error)) {result = { output: error.text, usage: topicUsage(error.usage) };}@@ -50,6 +93,13 @@ export async function generateTopicText<T>(}}+/** Bounded response diagnostics for the existing Topics failure log; no content. */+export function getTopicTextResponseMetadata(error: unknown) {+ return error !== null && typeof error === "object"+ ? responseMetadataByError.get(error)+ : undefined;+}+async function writeTopicTrace<T>(params: TopicTextParams<T>,startTimeISO: string,@@ -86,9 +136,8 @@ async function writeTopicTrace<T>(},],});- } catch (error) {+ } catch {logger.error("Failed to publish Topics model trace", {- error,traceId: trace.traceId,});}@@ -155,7 +204,10 @@ function topicUsage(async function generateWithConnection<T>(params: TopicTextParams<T>,- recordResponse: (result: TopicTextResult) => void,+ recordResponse: (+ result: TopicTextResult,+ metadata: TopicTextResponseMetadata,+ ) => void,): Promise<TopicTextResult> {const nativeOpenAI = isFirstPartyOpenAI(params.model);const result = await generateLLMText({@@ -182,16 +234,48 @@ async function generateWithConnection<T>(// The AI SDK retries 429 and 5xx responses with exponential backoff.maxRetries: 2,timeout: params.timeoutMs ?? TIMEOUT_MS,+ // onEnd runs before structured-output parsing and the output getter can throw.+ onEnd: ({ finalStep }) => {+ const text = finalStep.text;+ recordResponse(+ { output: text, usage: topicUsage(finalStep.usage) },+ responseAttributes(finalStep, text.length, params.maxOutputTokens),+ );+ },});const usage = topicUsage(result.usage);- // The SDK's output getter can throw after a response with billable usage.- recordResponse({ output: result.text, usage });return {output: result.output,usage,};}+function responseAttributes(+ step: StepResult<ToolSet>,+ textCharacters: number,+ maxOutputTokens: number,+): TopicTextResponseMetadata {+ return {+ "langfuse.topics.response.finish_reason": step.finishReason,+ "langfuse.topics.response.raw_finish_reason": RAW_FINISH_REASONS.has(+ step.rawFinishReason ?? "",+ )+ ? step.rawFinishReason!+ : "other",+ "langfuse.topics.response.text_characters": textCharacters,+ "langfuse.topics.response.text_part_count": step.content.filter(+ (part) => part.type === "text",+ ).length,+ "langfuse.topics.response.reasoning_part_count": step.content.filter(+ (part) => part.type === "reasoning",+ ).length,+ "langfuse.topics.response.max_output_tokens": maxOutputTokens,+ ...(step.usage.outputTokens !== undefined && {+ "langfuse.topics.response.output_tokens": step.usage.outputTokens,+ }),+ };+}+/** Topics owns its OpenAI request options; saved evaluator options keep their routing. */function isFirstPartyOpenAI(model: TopicsModel): boolean {if (model.adapter !== LLMAdapter.OpenAI) return false;@@ -389,6 +473,8 @@ async function invokeWithCache(/** A system prompt segment; `cache` ends a reusable prefix with a cache breakpoint. */export type TopicPromptPart = { text: string; cache?: boolean };+type TopicTextResponseMetadata = Record<string, string | number>;+type TopicTextResult = {output: unknown;usage: {diff --git a/worker/AGENTS.md b/worker/AGENTS.mdindex 74c3fea71c..1f9d585813 100644--- a/worker/AGENTS.md+++ b/worker/AGENTS.md@@ -38,6 +38,8 @@organization `aiFeaturesEnabled` and `aiTelemetryEnabled` in `models.ts`;embeddings are untraced. Use `isInternalEvalEnvironment`to exclude internal sources.+ Model failures add shared response diagnostics to the existing warning/span;+ preserve one warning per failed call and bounded metric tags.Pass models explicitly; never put them on execution or batch state, which ispersisted. Trace-batch jobs load enabled settings for all allowlisted batchprojects in one query (`getEnabledTopicsModels`) and pause a project ondiff --git a/worker/src/features/topics/models.test.ts b/worker/src/features/topics/models.test.tsindex 0802c64e61..48010d3e24 100644--- a/worker/src/features/topics/models.test.ts+++ b/worker/src/features/topics/models.test.ts@@ -8,6 +8,8 @@ const state = vi.hoisted(() => ({increment: vi.fn(),project: vi.fn(),traceSink: vi.fn(),+ responseMetadata: vi.fn(),+ spanAttributes: vi.fn(),encoding: null as {encode: ReturnType<typeof vi.fn>;free: ReturnType<typeof vi.fn>;@@ -28,6 +30,10 @@ vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({LangfuseInternalTraceEnvironment: { Topics: "langfuse-topics" },logger: { warn: vi.fn() },recordIncrement: state.increment,+ instrumentAsync: async (+ _options: unknown,+ call: (span: { setAttributes: typeof state.spanAttributes }) => unknown,+ ) => call({ setAttributes: state.spanAttributes }),}));vi.mock("@langfuse/shared/src/db", () => ({prisma: { project: { findFirst: state.project } },@@ -38,6 +44,8 @@ vi.mock("@langfuse/shared/topics/server", async (importOriginal) => ({).topicMessages,generateTopicText: (...args: unknown[]) => state.call(...args),generateTopicEmbeddings: (...args: unknown[]) => state.embed(...args),+ getTopicTextResponseMetadata: (error: unknown) =>+ state.responseMetadata(error),}));// Token counters only; every call also records call and request-byte counters.@@ -115,6 +123,9 @@ beforeEach(() => {state.call.mockReset();state.embed.mockReset();state.increment.mockReset();+ state.responseMetadata.mockReset();+ state.spanAttributes.mockReset();+ vi.mocked(logger.warn).mockClear();state.project.mockReset().mockResolvedValue({organization: { aiFeaturesEnabled: true, aiTelemetryEnabled: true },});@@ -308,12 +319,10 @@ describe("Topics naming boundary", () => {expect(state.call).not.toHaveBeenCalled();});- it("logs the provider's error fields for a rejected request, never its bodies", async () => {+ it("excludes provider content from failure logs and spans, including echoed error fields", async () => {// The AI SDK identifies an APICallError by these markers.const rejection = Object.assign(- new Error(- "Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead.",- ),+ new Error("Invalid input: secret transcript"),{[Symbol.for("vercel.ai.error")]: true,[Symbol.for("vercel.ai.error.AI_APICallError")]: true,@@ -324,9 +333,9 @@ describe("Topics naming boundary", () => {responseBody: '{"error":{"message":"secret echo"}}',data: {error: {- type: "invalid_request_error",- code: "unsupported_parameter",- param: "max_tokens",+ type: "secret error type",+ code: "secret error code",+ param: "secret input value",},},},@@ -345,11 +354,11 @@ describe("Topics naming boundary", () => {expect(message).toBe("Topics model request failed");expect(fields).toMatchObject({"http.status_code": 400,- "langfuse.topics.provider.message": rejection.message,- "langfuse.topics.provider.code": "unsupported_parameter",- "langfuse.topics.provider.param": "max_tokens",});expect(JSON.stringify(fields)).not.toMatch(/secret/);+ expect(JSON.stringify(state.spanAttributes.mock.calls)).not.toMatch(+ /secret/,+ );expect(state.increment).toHaveBeenCalledWith("langfuse.topics.llm_calls",1,@@ -361,6 +370,55 @@ describe("Topics naming boundary", () => {);});+ it("reports unusable naming output without logging provider content", async () => {+ const rejection = Object.assign(new Error("secret response text"), {+ name: "AI_NoOutputGeneratedError",+ responseBody: "secret response body",+ text: "secret model output",+ });+ const response = {+ "langfuse.topics.response.finish_reason": "other",+ "langfuse.topics.response.raw_finish_reason": "refusal",+ "langfuse.topics.response.text_characters": 0,+ "langfuse.topics.response.output_tokens": 0,+ };+ state.responseMetadata.mockImplementation((error) =>+ error === rejection ? response : undefined,+ );+ state.call.mockRejectedValue(rejection);++ const failure = await nameTopicGroup(topicsModels(), evidence).catch(+ (error: unknown) => error,+ );+ expect(failure).toMatchObject({+ reason: "invalid_output",+ message: expect.stringContaining("returned no usable structured output"),+ });+ expect((failure as Error).message).not.toMatch(/credentials|secret/);+ expect(logger.warn).toHaveBeenCalledOnce();+ const [message, fields] = vi.mocked(logger.warn).mock.calls[0];+ expect(message).toBe("Topics model request failed");+ expect(fields).toMatchObject({+ ...response,+ "langfuse.topics.error.reason": "invalid_output",+ "langfuse.topics.error.type": "AI_NoOutputGeneratedError",+ });+ expect(state.spanAttributes).toHaveBeenCalledWith(fields);+ expect(JSON.stringify(fields)).not.toMatch(/secret/);+ expect(JSON.stringify(state.spanAttributes.mock.calls)).not.toMatch(+ /secret/,+ );+ expect(state.increment).toHaveBeenCalledWith(+ "langfuse.topics.llm_calls",+ 1,+ {+ stage: "naming",+ adapter: LLMAdapter.Bedrock,+ status: "invalid_output",+ },+ );+ });+it.each([[{}, "ok"],[{ httpStatus: 429, reason: "rate_limit" }, "429"],diff --git a/worker/src/features/topics/models.ts b/worker/src/features/topics/models.tsindex 634c2a249e..b7799a3fe8 100644--- a/worker/src/features/topics/models.ts+++ b/worker/src/features/topics/models.ts@@ -12,6 +12,7 @@ import {import {generateTopicEmbeddings,generateTopicText,+ getTopicTextResponseMetadata,topicMessages,type TopicPromptPart,type TopicsModel,@@ -125,7 +126,7 @@ async function topicModelCall<T>(return result;} catch (error) {const failure = topicProviderError(error, model);- const { httpStatus, message, type, code, param } = failure.details;+ const { httpStatus } = failure.details;recordTopicModelCall(stage, model.adapter, {httpStatus,reason: failure.reason,@@ -136,10 +137,7 @@ async function topicModelCall<T>("langfuse.topics.error.type":error instanceof Error ? error.name : "unknown",...(httpStatus && { "http.status_code": httpStatus }),- ...(message && { "langfuse.topics.provider.message": message }),- ...(type && { "langfuse.topics.provider.type": type }),- ...(code && { "langfuse.topics.provider.code": code }),- ...(param && { "langfuse.topics.provider.param": param }),+ ...getTopicTextResponseMetadata(error),};span.setAttributes(failed);logger.warn("Topics model request failed", failed);diff --git a/worker/src/features/topics/processTopicEmbeddingBatch.test.ts b/worker/src/features/topics/processTopicEmbeddingBatch.test.tsindex 9f8de7d1a0..8430f75d29 100644--- a/worker/src/features/topics/processTopicEmbeddingBatch.test.ts+++ b/worker/src/features/topics/processTopicEmbeddingBatch.test.ts@@ -16,6 +16,7 @@ const mocks = vi.hoisted(() => ({embeddingModel: "eu.cohere.embed-v4:0",}));vi.mock("@langfuse/shared/topics/server", () => ({+ getTopicTextResponseMetadata: () => undefined,pauseTopicsModels: mocks.pause,writeTopicSummaries: mocks.write,readStagedTopicSummary: mocks.staged,diff --git a/worker/src/features/topics/processTopicsExecution.test.ts b/worker/src/features/topics/processTopicsExecution.test.tsindex ed0d15575b..8522956a2f 100644--- a/worker/src/features/topics/processTopicsExecution.test.ts+++ b/worker/src/features/topics/processTopicsExecution.test.ts@@ -76,6 +76,7 @@ vi.mock("@langfuse/shared/topics/server", async (importOriginal) => {const { TOPICS_TRANSCRIPT_VERSION, TOPIC_EMBEDDING_EXPIRED_ERROR } =await importOriginal<typeof import("@langfuse/shared/topics/server")>();return {+ getTopicTextResponseMetadata: () => undefined,isTopicsEnabled: () => true,getTopicsModels: async () => {const slot = (name: string, model: string) => ({diff --git a/worker/src/features/topics/provider-error.ts b/worker/src/features/topics/provider-error.tsindex 398b17deab..0b2d7d2a21 100644--- a/worker/src/features/topics/provider-error.ts+++ b/worker/src/features/topics/provider-error.ts@@ -22,10 +22,6 @@ type TopicsProviderErrorReason =/** Telemetry only; the user-facing message stays generic. */export type TopicProviderErrorDetails = {httpStatus?: number;- message?: string;- type?: string;- code?: string;- param?: string;};export class TopicsProviderUnavailable extends Error {@@ -79,40 +75,25 @@ export function topicProviderError(reason = "invalid_output";current = record.cause;}+ const modelDescription = `The ${TOPICS_MODEL_SLOT_DETAILS[model.slot].label.toLowerCase()} model ${model.model} on LLM connection "${model.provider}"`;return new TopicsProviderUnavailable(- `The ${TOPICS_MODEL_SLOT_DETAILS[model.slot].label.toLowerCase()} model ${model.model} on LLM connection "${model.provider}" failed${status ? ` (HTTP ${status})` : ""}. Check the connection's credentials, the model ID, and provider availability.`,+ reason === "invalid_output"+ ? `${modelDescription} returned no usable structured output. Resume interrupted stages to retry, or choose another model.`+ : `${modelDescription} failed${status ? ` (HTTP ${status})` : ""}. Check the connection's credentials, the model ID, and provider availability.`,reason,model.slot,providerErrorDetails(error, status),);}-/**- * Like evals, keeps only the AI SDK's parsed provider error fields; plain errors- * (e.g. the direct Bedrock call) carry the response body, so they keep the status only.- */+/** Provider error strings can echo request/response content; retain only status. */function providerErrorDetails(error: unknown,status: number | undefined,): TopicProviderErrorDetails {const provider = getLLMErrorInfo(error)?.providerError;const httpStatus = provider?.statusCode ?? status;- if (!httpStatus) return {};- if (!provider) return { httpStatus };- const record = (value: unknown) =>- value && typeof value === "object"- ? (value as Record<string, unknown>)- : undefined;- const data = record(record(provider.data)?.error) ?? record(provider.data);- const text = (value: unknown) =>- typeof value === "string" ? value.slice(0, 500) : undefined;- return {- httpStatus,- message: text(provider.message),- type: text(data?.type),- code: text(data?.code),- param: text(data?.param),- };+ return httpStatus ? { httpStatus } : {};}/** The provider rejected the connection's credentials or billing. */