🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(topics): reason-specific provider errors and size-limit error reasons (#19045)
Lysander Kieselafbc88d3ff17
diff --git a/packages/shared/src/server/topics/trace-input.ts b/packages/shared/src/server/topics/trace-input.tsindex f51bd6fc13..29439d7e0c 100644--- a/packages/shared/src/server/topics/trace-input.ts+++ b/packages/shared/src/server/topics/trace-input.ts@@ -17,6 +17,14 @@ export const TOPICS_TRANSCRIPT_VERSION = "shared-transcript-v3";const MAX_OBSERVATIONS = 2_000;const MAX_SNAPSHOT_BYTES = 10 * 1024 * 1024;+/** The trace is above the transcript limits, whether ClickHouse or the reader stopped it. */+export class TopicTraceTooLarge extends Error {+ readonly name = "TopicTraceTooLarge";+ constructor() {+ super("Trace exceeds the Topics PoC limit (2,000 observations or 10 MiB)");+ }+}+type SnapshotRow = {project_id: string;trace_id: string;@@ -121,53 +129,55 @@ export async function loadTopicTranscript(params: {let traceName = "";let rootName = "";let bytes = 0;- for await (const row of queryClickhouseStream<SnapshotRow>({- query,- params: queryParams,- preferredClickhouseService: "EventsReadOnly",- tags: { projectId },- clickhouseSettings: {- max_threads: 2,- max_execution_time: 30,- timeout_overflow_mode: "throw",- max_result_bytes: String(MAX_SNAPSHOT_BYTES),- result_overflow_mode: "throw",- },- })) {- if (row.project_id !== projectId || row.trace_id !== traceId)- throw new Error("Trace snapshot scope mismatch");- if (!observations.length) environment = row.environment;- // Rows arrive newest first; retain the latest non-empty session and name metadata.- if (!sessionId && row.session_id) sessionId = row.session_id;- if (!traceName && row.trace_name) traceName = row.trace_name;- if (- !rootName &&- row.name &&- isRootObservation({- parentObservationId: row.parent_span_id,- isAppRoot: row.is_app_root,- })- )- rootName = row.name;- bytes += Buffer.byteLength(JSON.stringify(row));- if (bytes > MAX_SNAPSHOT_BYTES || observations.length >= MAX_OBSERVATIONS) {- throw new Error(- "Trace exceeds the Topics PoC limit (2,000 observations or 10 MiB)",- );+ try {+ for await (const row of queryClickhouseStream<SnapshotRow>({+ query,+ params: queryParams,+ preferredClickhouseService: "EventsReadOnly",+ tags: { projectId },+ clickhouseSettings: {+ max_threads: 2,+ max_execution_time: 30,+ timeout_overflow_mode: "throw",+ max_result_bytes: String(MAX_SNAPSHOT_BYTES),+ result_overflow_mode: "throw",+ },+ })) {+ if (row.project_id !== projectId || row.trace_id !== traceId)+ throw new Error("Trace snapshot scope mismatch");+ if (!observations.length) environment = row.environment;+ // Rows arrive newest first; retain the latest non-empty session and name metadata.+ if (!sessionId && row.session_id) sessionId = row.session_id;+ if (!traceName && row.trace_name) traceName = row.trace_name;+ if (+ !rootName &&+ row.name &&+ isRootObservation({+ parentObservationId: row.parent_span_id,+ isAppRoot: row.is_app_root,+ })+ )+ rootName = row.name;+ bytes += Buffer.byteLength(JSON.stringify(row));+ if (bytes > MAX_SNAPSHOT_BYTES || observations.length >= MAX_OBSERVATIONS)+ throw new TopicTraceTooLarge();+ observations.push({+ id: row.span_id,+ traceId,+ parentObservationId: row.parent_span_id || null,+ startTime: new Date(utcTimestamp(row.start_time)),+ endTime:+ row.end_time === null ? null : new Date(utcTimestamp(row.end_time)),+ type: row.type,+ name: row.name,+ input: row.input,+ output: row.output,+ metadata: row.metadata,+ });}- observations.push({- id: row.span_id,- traceId,- parentObservationId: row.parent_span_id || null,- startTime: new Date(utcTimestamp(row.start_time)),- endTime:- row.end_time === null ? null : new Date(utcTimestamp(row.end_time)),- type: row.type,- name: row.name,- input: row.input,- output: row.output,- metadata: row.metadata,- });+ } catch (error) {+ if (isResultOverflow(error)) throw new TopicTraceTooLarge();+ throw error;}if (!observations.length)throw new Error(@@ -191,3 +201,13 @@ export async function loadTopicTranscript(params: {transcript: assembleTranscript(orderObservations(observations)),};}++/** ClickHouse stops oversized results before the reader can count them. */+function isResultOverflow(error: unknown): boolean {+ if (typeof error !== "object" || error === null) return false;+ return (+ ("type" in error && error.type === "TOO_MANY_ROWS_OR_BYTES") ||+ (error instanceof Error &&+ error.message.includes("Limit for result exceeded"))+ );+}diff --git a/worker/src/features/topics/README.md b/worker/src/features/topics/README.mdindex 481ca5ebd0..7e5460b67f 100644--- a/worker/src/features/topics/README.md+++ b/worker/src/features/topics/README.md@@ -552,6 +552,9 @@ entry from each waiting, paused, delayed and active state without loading jobpayloads. Retries can re-enter behind newer jobs, so this is head age, not theoldest creation time anywhere in the backlog.+`.errors` reasons include `input_too_large` (transcript above the summary input+limit) and `trace_too_large` (trace above the transcript load limits).+Summary `.results` carry `source:model|rule` and `facet` (the built-in facet'sname, or `custom`): the model decides `applicable`, `not_applicable` or`insufficient_input`; local rules decide `insufficient_input` (no transcript@@ -582,14 +585,14 @@ Spans carry tenant and request detail that metrics must not:group) and `topics.assignment` (also used when manual processing assigns).A failed facet marks its enclosing span as an error, although the run handles it.- `topics.llm.summary|embedding|naming`: one per provider request, with model,- adapter and prompt bytes; on failure, the HTTP status and the provider's error- message, type, code and param.+ adapter and prompt bytes; on failure, the HTTP status and, when the provider+ answered, bounded response metadata (`langfuse.topics.response.*`: finish+ reason, raw finish reason from a fixed list, text length, text and reasoning part counts,+ output tokens and the output-token cap; never the response text).A failed provider request logs one `Topics model request failed` warning with the-same fields. Like evals, only the AI SDK's parsed provider error (message cut to 500-characters, type, code, param) is kept; request and response bodies can contain-trace content and are never logged, so errors without that structure, such as-the direct Bedrock call, log the HTTP status only. A failed facet logs `Topics facet failed`: a warning for provider or+same fields. Provider error strings, codes and parameters can echo request or+response content, so only the HTTP status is kept. A failed facet logs `Topics facet failed`: a warning for provider orconfiguration errors, an error otherwise.`langfuse.topics.tokens{stage:summary|embedding|naming,direction:input|output}`diff --git a/worker/src/features/topics/metrics.ts b/worker/src/features/topics/metrics.tsindex 6ebc703d78..bef2494253 100644--- a/worker/src/features/topics/metrics.ts+++ b/worker/src/features/topics/metrics.ts@@ -36,6 +36,8 @@ type TopicErrorReason =| "trace_load"| "storage"| "numerical"+ | "input_too_large"+ | "trace_too_large"| "unknown";// Statuses with their own operational meaning; exact codes are on the span.@@ -149,6 +151,8 @@ export class TopicMetrics {reason = "invalid_output";if (["AbortError", "TimeoutError"].includes(error.name))reason = "timeout";+ if (error.name === "TopicInputTooLarge") reason = "input_too_large";+ if (error.name === "TopicTraceTooLarge") reason = "trace_too_large";if (["PrismaClientKnownRequestError",diff --git a/worker/src/features/topics/models.ts b/worker/src/features/topics/models.tsindex b7799a3fe8..c19e0b2bce 100644--- a/worker/src/features/topics/models.ts+++ b/worker/src/features/topics/models.ts@@ -91,7 +91,9 @@ function countTopicRequestTokens(}/** The input exceeds the run's token limit; no model call was made. */-export class TopicInputTooLarge extends Error {}+export class TopicInputTooLarge extends Error {+ readonly name = "TopicInputTooLarge";+}/** One span, call counter and failure log per provider request. */async function topicModelCall<T>(diff --git a/worker/src/features/topics/provider-error.ts b/worker/src/features/topics/provider-error.tsindex 0b2d7d2a21..04f373b12a 100644--- a/worker/src/features/topics/provider-error.ts+++ b/worker/src/features/topics/provider-error.ts@@ -75,11 +75,20 @@ export function topicProviderError(reason = "invalid_output";current = record.cause;}- const modelDescription = `The ${TOPICS_MODEL_SLOT_DETAILS[model.slot].label.toLowerCase()} model ${model.model} on LLM connection "${model.provider}"`;+ const subject = `The ${TOPICS_MODEL_SLOT_DETAILS[model.slot].label.toLowerCase()} model ${model.model} on LLM connection "${model.provider}"`;+ const failed = `${subject} failed${status ? ` (HTTP ${status})` : ""}.`;+ // The message names the actual failure; a generic credentials hint misleads for output or rate-limit errors.+ const message = {+ authentication: `${failed} Check the connection's credentials and billing.`,+ invalid_input: `${failed} The provider rejected the request; check that the model ID supports it.`,+ rate_limit: `${failed} The provider rate-limited the request. Try again later.`,+ timeout: `${failed} The request timed out. Try again later.`,+ invalid_output: `${subject} returned no usable structured output. Resume interrupted stages to retry, or choose another model.`,+ configuration: `${failed} Check the Topics model settings.`,+ provider_error: `${failed} Check the model ID and provider availability.`,+ }[reason];return new TopicsProviderUnavailable(- reason === "invalid_output"- ? `${modelDescription} returned no usable structured output. Resume interrupted stages to retry, or choose another model.`- : `${modelDescription} failed${status ? ` (HTTP ${status})` : ""}. Check the connection's credentials, the model ID, and provider availability.`,+ message,reason,model.slot,providerErrorDetails(error, status),