Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(topics): reason-specific provider errors and size-limit error reasons (#19045)

Lysander Kieselafbc88d3ff17
diff --git a/packages/shared/src/server/topics/trace-input.ts b/packages/shared/src/server/topics/trace-input.ts
index f51bd6fc13..29439d7e0c 100644
--- a/packages/shared/src/server/topics/trace-input.ts
+++ b/packages/shared/src/server/topics/trace-input.ts
@@ -17,6 +17,14 @@ export const TOPICS_TRANSCRIPT_VERSION = "shared-transcript-v3";
const MAX_OBSERVATIONS = 2_000;
const MAX_SNAPSHOT_BYTES = 10 * 1024 * 1024;
+/** The trace is above the transcript limits, whether ClickHouse or the reader stopped it. */
+export class TopicTraceTooLarge extends Error {
+ readonly name = "TopicTraceTooLarge";
+ constructor() {
+ super("Trace exceeds the Topics PoC limit (2,000 observations or 10 MiB)");
+ }
+}
+
type SnapshotRow = {
project_id: string;
trace_id: string;
@@ -121,53 +129,55 @@ export async function loadTopicTranscript(params: {
let traceName = "";
let rootName = "";
let bytes = 0;
- for await (const row of queryClickhouseStream<SnapshotRow>({
- query,
- params: queryParams,
- preferredClickhouseService: "EventsReadOnly",
- tags: { projectId },
- clickhouseSettings: {
- max_threads: 2,
- max_execution_time: 30,
- timeout_overflow_mode: "throw",
- max_result_bytes: String(MAX_SNAPSHOT_BYTES),
- result_overflow_mode: "throw",
- },
- })) {
- if (row.project_id !== projectId || row.trace_id !== traceId)
- throw new Error("Trace snapshot scope mismatch");
- if (!observations.length) environment = row.environment;
- // Rows arrive newest first; retain the latest non-empty session and name metadata.
- if (!sessionId && row.session_id) sessionId = row.session_id;
- if (!traceName && row.trace_name) traceName = row.trace_name;
- if (
- !rootName &&
- row.name &&
- isRootObservation({
- parentObservationId: row.parent_span_id,
- isAppRoot: row.is_app_root,
- })
- )
- rootName = row.name;
- bytes += Buffer.byteLength(JSON.stringify(row));
- if (bytes > MAX_SNAPSHOT_BYTES || observations.length >= MAX_OBSERVATIONS) {
- throw new Error(
- "Trace exceeds the Topics PoC limit (2,000 observations or 10 MiB)",
- );
+ try {
+ for await (const row of queryClickhouseStream<SnapshotRow>({
+ query,
+ params: queryParams,
+ preferredClickhouseService: "EventsReadOnly",
+ tags: { projectId },
+ clickhouseSettings: {
+ max_threads: 2,
+ max_execution_time: 30,
+ timeout_overflow_mode: "throw",
+ max_result_bytes: String(MAX_SNAPSHOT_BYTES),
+ result_overflow_mode: "throw",
+ },
+ })) {
+ if (row.project_id !== projectId || row.trace_id !== traceId)
+ throw new Error("Trace snapshot scope mismatch");
+ if (!observations.length) environment = row.environment;
+ // Rows arrive newest first; retain the latest non-empty session and name metadata.
+ if (!sessionId && row.session_id) sessionId = row.session_id;
+ if (!traceName && row.trace_name) traceName = row.trace_name;
+ if (
+ !rootName &&
+ row.name &&
+ isRootObservation({
+ parentObservationId: row.parent_span_id,
+ isAppRoot: row.is_app_root,
+ })
+ )
+ rootName = row.name;
+ bytes += Buffer.byteLength(JSON.stringify(row));
+ if (bytes > MAX_SNAPSHOT_BYTES || observations.length >= MAX_OBSERVATIONS)
+ throw new TopicTraceTooLarge();
+ observations.push({
+ id: row.span_id,
+ traceId,
+ parentObservationId: row.parent_span_id || null,
+ startTime: new Date(utcTimestamp(row.start_time)),
+ endTime:
+ row.end_time === null ? null : new Date(utcTimestamp(row.end_time)),
+ type: row.type,
+ name: row.name,
+ input: row.input,
+ output: row.output,
+ metadata: row.metadata,
+ });
}
- observations.push({
- id: row.span_id,
- traceId,
- parentObservationId: row.parent_span_id || null,
- startTime: new Date(utcTimestamp(row.start_time)),
- endTime:
- row.end_time === null ? null : new Date(utcTimestamp(row.end_time)),
- type: row.type,
- name: row.name,
- input: row.input,
- output: row.output,
- metadata: row.metadata,
- });
+ } catch (error) {
+ if (isResultOverflow(error)) throw new TopicTraceTooLarge();
+ throw error;
}
if (!observations.length)
throw new Error(
@@ -191,3 +201,13 @@ export async function loadTopicTranscript(params: {
transcript: assembleTranscript(orderObservations(observations)),
};
}
+
+/** ClickHouse stops oversized results before the reader can count them. */
+function isResultOverflow(error: unknown): boolean {
+ if (typeof error !== "object" || error === null) return false;
+ return (
+ ("type" in error && error.type === "TOO_MANY_ROWS_OR_BYTES") ||
+ (error instanceof Error &&
+ error.message.includes("Limit for result exceeded"))
+ );
+}
diff --git a/worker/src/features/topics/README.md b/worker/src/features/topics/README.md
index 481ca5ebd0..7e5460b67f 100644
--- a/worker/src/features/topics/README.md
+++ b/worker/src/features/topics/README.md
@@ -552,6 +552,9 @@ entry from each waiting, paused, delayed and active state without loading job
payloads. Retries can re-enter behind newer jobs, so this is head age, not the
oldest creation time anywhere in the backlog.
+`.errors` reasons include `input_too_large` (transcript above the summary input
+limit) and `trace_too_large` (trace above the transcript load limits).
+
Summary `.results` carry `source:model|rule` and `facet` (the built-in facet's
name, or `custom`): the model decides `applicable`, `not_applicable` or
`insufficient_input`; local rules decide `insufficient_input` (no transcript
@@ -582,14 +585,14 @@ Spans carry tenant and request detail that metrics must not:
group) and `topics.assignment` (also used when manual processing assigns).
A failed facet marks its enclosing span as an error, although the run handles it.
- `topics.llm.summary|embedding|naming`: one per provider request, with model,
- adapter and prompt bytes; on failure, the HTTP status and the provider's error
- message, type, code and param.
+ adapter and prompt bytes; on failure, the HTTP status and, when the provider
+ answered, bounded response metadata (`langfuse.topics.response.*`: finish
+ reason, raw finish reason from a fixed list, text length, text and reasoning part counts,
+ output tokens and the output-token cap; never the response text).
A failed provider request logs one `Topics model request failed` warning with the
-same fields. Like evals, only the AI SDK's parsed provider error (message cut to 500
-characters, type, code, param) is kept; request and response bodies can contain
-trace content and are never logged, so errors without that structure, such as
-the direct Bedrock call, log the HTTP status only. A failed facet logs `Topics facet failed`: a warning for provider or
+same fields. Provider error strings, codes and parameters can echo request or
+response content, so only the HTTP status is kept. A failed facet logs `Topics facet failed`: a warning for provider or
configuration errors, an error otherwise.
`langfuse.topics.tokens{stage:summary|embedding|naming,direction:input|output}`
diff --git a/worker/src/features/topics/metrics.ts b/worker/src/features/topics/metrics.ts
index 6ebc703d78..bef2494253 100644
--- a/worker/src/features/topics/metrics.ts
+++ b/worker/src/features/topics/metrics.ts
@@ -36,6 +36,8 @@ type TopicErrorReason =
| "trace_load"
| "storage"
| "numerical"
+ | "input_too_large"
+ | "trace_too_large"
| "unknown";
// Statuses with their own operational meaning; exact codes are on the span.
@@ -149,6 +151,8 @@ export class TopicMetrics {
reason = "invalid_output";
if (["AbortError", "TimeoutError"].includes(error.name))
reason = "timeout";
+ if (error.name === "TopicInputTooLarge") reason = "input_too_large";
+ if (error.name === "TopicTraceTooLarge") reason = "trace_too_large";
if (
[
"PrismaClientKnownRequestError",
diff --git a/worker/src/features/topics/models.ts b/worker/src/features/topics/models.ts
index b7799a3fe8..c19e0b2bce 100644
--- a/worker/src/features/topics/models.ts
+++ b/worker/src/features/topics/models.ts
@@ -91,7 +91,9 @@ function countTopicRequestTokens(
}
/** The input exceeds the run's token limit; no model call was made. */
-export class TopicInputTooLarge extends Error {}
+export class TopicInputTooLarge extends Error {
+ readonly name = "TopicInputTooLarge";
+}
/** One span, call counter and failure log per provider request. */
async function topicModelCall<T>(
diff --git a/worker/src/features/topics/provider-error.ts b/worker/src/features/topics/provider-error.ts
index 0b2d7d2a21..04f373b12a 100644
--- a/worker/src/features/topics/provider-error.ts
+++ b/worker/src/features/topics/provider-error.ts
@@ -75,11 +75,20 @@ export function topicProviderError(
reason = "invalid_output";
current = record.cause;
}
- const modelDescription = `The ${TOPICS_MODEL_SLOT_DETAILS[model.slot].label.toLowerCase()} model ${model.model} on LLM connection "${model.provider}"`;
+ const subject = `The ${TOPICS_MODEL_SLOT_DETAILS[model.slot].label.toLowerCase()} model ${model.model} on LLM connection "${model.provider}"`;
+ const failed = `${subject} failed${status ? ` (HTTP ${status})` : ""}.`;
+ // The message names the actual failure; a generic credentials hint misleads for output or rate-limit errors.
+ const message = {
+ authentication: `${failed} Check the connection's credentials and billing.`,
+ invalid_input: `${failed} The provider rejected the request; check that the model ID supports it.`,
+ rate_limit: `${failed} The provider rate-limited the request. Try again later.`,
+ timeout: `${failed} The request timed out. Try again later.`,
+ invalid_output: `${subject} returned no usable structured output. Resume interrupted stages to retry, or choose another model.`,
+ configuration: `${failed} Check the Topics model settings.`,
+ provider_error: `${failed} Check the model ID and provider availability.`,
+ }[reason];
return new TopicsProviderUnavailable(
- reason === "invalid_output"
- ? `${modelDescription} returned no usable structured output. Resume interrupted stages to retry, or choose another model.`
- : `${modelDescription} failed${status ? ` (HTTP ${status})` : ""}. Check the connection's credentials, the model ID, and provider availability.`,
+ message,
reason,
model.slot,
providerErrorDetails(error, status),