🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { ClickHouseError, parseError } from "@clickhouse/client";
import { recordDistribution, recordIncrement } from "../instrumentation";
import { type NormalizedClickHouseQueryTags } from "./queryTags";
import { type ClickhouseService } from "./client";
/**
* Terminal outcome of one logical ClickHouse query, counted once per query
* after retries are exhausted.
*
* Resource limits surface to public API callers as HTTP 422 and to tRPC
* callers as UNPROCESSABLE_CONTENT, so status-code-derived signals cannot
* distinguish them from a caller's malformed request. This metric is the
* signal that can: it is emitted where the resource error is classified.
*/
export const CLICKHOUSE_QUERY_OUTCOME_METRIC =
"langfuse.clickhouse.query.outcome";
/**
* Per-query runtime distribution, read from the `x-clickhouse-summary` response
* header and emitted unsampled with the outcome metric's tags. APM keeps only a
* small sample of query spans, so this is the series to track query runtime
* over time per route, table, shape and ClickHouse service.
*/
export const CLICKHOUSE_QUERY_ELAPSED_MS_METRIC =
"langfuse.clickhouse.query.elapsed_ms";
/**
* `client_closed` is not a ClickHouse failure: the caller aborted its signal for
* a reason other than a deadline, e.g. a browser closing a dashboard stream. A
* deadline abort is an `error` with `error_code:client_timeout`.
*/
export type ClickHouseQueryOutcome =
| "success"
| "client_closed"
| "timeout"
| "memory_limit"
| "overcommit"
| "error";
/**
* Maps `ClickHouseResourceError.errorType` onto outcomes. Keyed by the error
* type literals rather than importing the type, so this module keeps no
* dependency on the query paths that call it. Indexing it with `errorType`
* makes the compiler reject a new error type that is not mapped here.
*/
export const CLICKHOUSE_RESOURCE_ERROR_OUTCOMES = {
TIMEOUT: "timeout",
MEMORY_LIMIT: "memory_limit",
OVERCOMMIT: "overcommit",
} as const satisfies Record<string, ClickHouseQueryOutcome>;
/**
* Metric tags must be bounded, so a route only gets its own label when its
* value set is fixed by code:
*
* - Bare routes (`events.all`, `listObservations`, `langfuse.queue.trace_delete`)
* are tRPC procedure paths, MCP tool names and worker routes. Each is set only
* after the procedure, tool or queue resolved, so they pass through.
* - REST routes carry their page template (`GET /api/public/traces/[traceId]`),
* taken from a list of the dynamic API routes in web. As a guard against a
* route missing from that list, a segment that is neither a word, a version
* (`v2`) nor a `[param]` placeholder drops the route to `other`.
*/
const BARE_ROUTE_PATTERN = /^[A-Za-z][\w.]{0,99}$/;
const REST_PATH_PREFIX = "/api/";
const REST_STATIC_SEGMENT_PATTERN = /^(?:[A-Za-z][A-Za-z_-]*|v\d+|\[\w+\])$/;
const MAX_REST_SEGMENTS = 10;
const OTHER_ROUTE_LABEL = "other";
/**
* Renders a route as a Datadog tag value, matching the `resource_name`
* convention APM uses for the same route (`get_/api/public/v2/metrics`), so
* SLO and dashboard queries can filter both signals the same way. Datadog tag
* values cannot contain spaces or brackets, so the method is folded in with an
* underscore and a `[param]` placeholder is rendered as `_param_`.
*/
export function clickHouseQueryOutcomeRouteLabel(route?: string): string {
if (!route) return OTHER_ROUTE_LABEL;
const collapsed = route.trim();
const separatorIndex = collapsed.indexOf(" ");
if (separatorIndex === -1) {
return BARE_ROUTE_PATTERN.test(collapsed) ? collapsed : OTHER_ROUTE_LABEL;
}
const method = collapsed.slice(0, separatorIndex);
const path = collapsed.slice(separatorIndex + 1);
const normalizedPath =
path.length > 1 && path.endsWith("/") ? path.slice(0, -1) : path;
if (!/^[A-Z]+$/.test(method) || !normalizedPath.startsWith(REST_PATH_PREFIX))
return OTHER_ROUTE_LABEL;
const segments = normalizedPath.slice(1).split("/");
if (
segments.length > MAX_REST_SEGMENTS ||
!segments.every((segment) => REST_STATIC_SEGMENT_PATTERN.test(segment))
) {
return OTHER_ROUTE_LABEL;
}
return `${method.toLowerCase()}_${normalizedPath.replace(/\[(\w+)\]/g, "_$1_")}`;
}
/**
* Bounded table label so timeouts can be isolated to the table under load — the
* events read path (`events_full`/`events_core`) is the current driver, and the
* counter is blind to it without this tag.
*/
export type ClickHouseQueryTable =
| "events_full"
| "events_core"
| "traces"
| "observations"
| "scores"
| "dataset_run_items"
| "blob_storage_file_log"
| "other";
/**
* Derived from the query text because threading a table through every call site
* is impractical. Anchored to the `FROM` table (with an optional `db.` prefix)
* so a lightweight join partner never outranks the table under load — the Scores
* UI reads `FROM scores ... LEFT JOIN traces`, and a bare `traces` scan would
* otherwise steal the label. Still best-effort: a subquery/CTE reading a
* different table in its own `FROM` can win by priority order (first match
* wins). The events read path is single-table (`FROM events_full`/`events_core`,
* no v3 joins), so those labels are reliable. `dataset_run_items` ranks above
* `scores` because dataset run tables filtered by a score read scores in a CTE.
*/
const TABLE_LABEL_PATTERNS: ReadonlyArray<[ClickHouseQueryTable, RegExp]> = [
["events_full", /\bfrom\s+(?:\w+\.)?events_full\b/i],
["events_core", /\bfrom\s+(?:\w+\.)?events_core\b/i],
["observations", /\bfrom\s+(?:\w+\.)?observations\b/i],
["traces", /\bfrom\s+(?:\w+\.)?traces\b/i],
["dataset_run_items", /\bfrom\s+(?:\w+\.)?dataset_run_items_rmt\b/i],
["scores", /\bfrom\s+(?:\w+\.)?scores\b/i],
["blob_storage_file_log", /\bfrom\s+(?:\w+\.)?blob_storage_file_log\b/i],
];
const OTHER_TABLE_LABEL = "other" as const;
export function clickHouseQueryTableLabel(query: string): ClickHouseQueryTable {
for (const [label, pattern] of TABLE_LABEL_PATTERNS) {
if (pattern.test(query)) return label;
}
return OTHER_TABLE_LABEL;
}
/**
* Bounded filter-shape label so a timeout can be attributed to the predicate
* shape driving it, not just the table. Without it, one `events_full` timeout
* cannot be told from another on the metric, and per-shape rates have to be
* estimated from ~10%-sampled APM query text. This makes the shape a
* first-class, unsampled dimension on both the outcome metric and the query
* span.
*
* One value per query: the shapes are ranked by how strongly each drives scan
* cost, and the first match wins, so it adds a single bounded dimension rather
* than a boolean per shape. A query carrying several predicates (e.g. a
* by-`span_id` lookup that also bounds `trace_id`) is attributed to its
* highest-ranked shape.
*
* Scores reads get their own shapes for the dedup strategy, which decides their
* cost more than the filter predicate does. They rank first, but only when the
* query's table label is `scores`.
*/
export type ClickHouseQueryShape =
| "scores_final"
| "scores_join_seek"
| "scores_join"
| "io_content"
| "metadata_content"
| "id_or_ilike"
| "by_span_id"
| "by_trace_id"
| "other";
const OTHER_SHAPE_LABEL = "other" as const;
/**
* Derived from the query text (like the table label) because threading the
* filter shape through every call site is impractical. Patterns match the SQL
* the filter compilers emit:
*
* - `io_content`: a substring/token/prefix/(I)LIKE search over the `input`/
* `output` columns — the largest `events_full` columns and the confirmed
* full-scan class. Anchored to `input`/`output` so a metadata search never
* counts.
* - `metadata_content`: the same search family over the `metadata_names`/
* `metadata_values` arrays, distinguished by the array-column suffix and the
* `has(names, key)` / `indexOf(names, key)` key-lookup wrapper. Only filter
* functions are matched (`has`/`position`/`hasAllTokens`/`startsWith`/…), not
* the `mapFromArrays`/`argMax`/`arrayReverse` projection helpers.
* - `id_or_ilike`: the OR-of-`ILIKE` id search arm, matched on its distinctive
* `searchString` parameter.
* - `by_span_id`: a `span_id = {…}` point lookup (`getObservationById`) or a
* `span_id IN {…}` batch lookup. Ranked above `by_trace_id` so a span lookup
* that also bounds `trace_id` counts as the span lookup.
* - `by_trace_id`: a `trace_id IN (…)` / `trace_id IN {…}` / `trace_id = {…}`
* lookup — the unbounded variant is a distinct timeout class. The `IN` form
* appears both parenthesized (inline list) and bare (`{param: Array(...)}`).
*/
const IO_CONTENT_FUNCTION_PATTERN =
/\b(?:position(?:caseinsensitive)?|hasalltokens|hasanytokens|hastoken|startswith|endswith)\s*\(\s*(?:lower\s*\(\s*)?(?:\w+\.)?(?:input|output)\b/i;
const IO_CONTENT_LIKE_PATTERN =
/\b(?:\w+\.)?(?:input|output)\s+(?:not\s+)?i?like\b/i;
const METADATA_CONTENT_PATTERN =
/\b(?:has|position(?:caseinsensitive)?|hasalltokens|hasanytokens|hastoken|startswith|endswith)\s*\(\s*(?:lower\s*\(\s*)?(?:\w+\.)?metadata_(?:names|values)\b/i;
const ID_OR_ILIKE_PATTERN = /\bilike\s*\{\s*searchString\b/i;
const BY_SPAN_ID_PATTERN = /\bspan_id\s*=\s*\{|\bspan_id\s+in\s*(?:\(\s*)?\{/i;
const BY_TRACE_ID_PATTERN =
/\btrace_id\s*=\s*\{|\btrace_id\s+in\s*(?:\(\s*)?\{/i;
const QUERY_SHAPE_PATTERNS: ReadonlyArray<[ClickHouseQueryShape, RegExp]> = [
["io_content", IO_CONTENT_FUNCTION_PATTERN],
["io_content", IO_CONTENT_LIKE_PATTERN],
["metadata_content", METADATA_CONTENT_PATTERN],
["id_or_ilike", ID_OR_ILIKE_PATTERN],
["by_span_id", BY_SPAN_ID_PATTERN],
["by_trace_id", BY_TRACE_ID_PATTERN],
];
/**
* Scores dedup strategies, matching the SQL the scores UI read emits:
*
* - `scores_final`: `FROM scores [s] FINAL`.
* - `scores_join_seek`: rows joined back to each key's `max(event_ts)`, with
* the key set narrowed first by a `SELECT DISTINCT` seek.
* - `scores_join`: the same join without the seek.
*/
const SCORES_FINAL_PATTERN =
/\bfrom\s+(?:\w+\.)?scores\s+(?:(?:as\s+)?\w+\s+)?final\b/i;
const SCORES_LATEST_JOIN_PATTERN =
/\bmax\s*\(\s*(?:\w+\.)?event_ts\s*\)\s+as\s+latest_event_ts\b/i;
const SELECT_DISTINCT_PATTERN = /\bselect\s+distinct\b/i;
function scoresQueryShape(query: string): ClickHouseQueryShape | undefined {
if (SCORES_FINAL_PATTERN.test(query)) return "scores_final";
if (SCORES_LATEST_JOIN_PATTERN.test(query)) {
return SELECT_DISTINCT_PATTERN.test(query)
? "scores_join_seek"
: "scores_join";
}
return undefined;
}
export function clickHouseQueryShape(
query: string,
table: ClickHouseQueryTable = clickHouseQueryTableLabel(query),
): ClickHouseQueryShape {
if (table === "scores") {
const scoresShape = scoresQueryShape(query);
if (scoresShape) return scoresShape;
}
for (const [shape, pattern] of QUERY_SHAPE_PATTERNS) {
if (pattern.test(query)) return shape;
}
return OTHER_SHAPE_LABEL;
}
function clickHouseQueryMetricTags(
tags: NormalizedClickHouseQueryTags,
table: ClickHouseQueryTable,
shape: ClickHouseQueryShape,
clickhouseService: ClickhouseService,
) {
return {
surface: tags.surface,
route: clickHouseQueryOutcomeRouteLabel(tags.route),
table,
query_shape: shape,
clickhouse_service: clickhouseService,
};
}
/**
* Transport failures the query path retries. Shared with the retry predicate so
* an exhausted retry is labelled `network` on the outcome metric.
*/
export const CLICKHOUSE_NETWORK_ERROR_PATTERNS = [
"socket hang up",
"broken pipe",
"connection reset",
"econnreset",
"network_error",
"etimedout",
"econnrefused",
] as const;
/**
* ClickHouse error codes that get their own `error_code` label, keyed by the
* numeric code. Together they cover >99.9% of failed Langfuse reads; the rest is
* a long tail reported as `other` and recoverable from `system.query_log`.
* Matching on the number rather than the name keeps the label set closed: the
* name is parsed from the message text, which can echo caller input.
*/
const LABELLED_CLICKHOUSE_ERROR_CODES: Record<string, string> = {
"60": "unknown_table",
"159": "timeout_exceeded",
"202": "too_many_simultaneous_queries",
"209": "socket_timeout",
"210": "network_error",
"241": "memory_limit_exceeded",
"734": "cannot_write_after_buffer_canceled",
"735": "query_was_cancelled_by_client",
};
const OTHER_ERROR_CODE_LABEL = "other";
/**
* Bounded cause of a failed query, so `error` outcomes are not opaque and a
* server-side `TIMEOUT_EXCEEDED` can be told from a client request timeout.
*
* - A labelled ClickHouse error code (`timeout_exceeded`, `unknown_table`, …),
* from a thrown `ClickHouseError` or an exception row's `Code: N.` prefix.
* Any other ClickHouse code is `other`.
* - `network` for a transport failure that outlived the retries.
* - `client_timeout` for the client's own request timeout, which carries no
* ClickHouse code.
* - `other` for anything else.
*/
export function clickHouseQueryErrorCode(error: unknown): string {
const root =
error instanceof Error && error.cause instanceof Error
? error.cause
: error;
if (!(root instanceof Error)) return OTHER_ERROR_CODE_LABEL;
const parsed = root instanceof ClickHouseError ? root : parseError(root);
if (parsed instanceof ClickHouseError) {
return (
LABELLED_CLICKHOUSE_ERROR_CODES[parsed.code] ?? OTHER_ERROR_CODE_LABEL
);
}
const message = (root.message ?? "").toLowerCase();
if (CLICKHOUSE_NETWORK_ERROR_PATTERNS.some((p) => message.includes(p))) {
return "network";
}
if (message.includes("timeout") || message.includes("timed out")) {
return "client_timeout";
}
return OTHER_ERROR_CODE_LABEL;
}
/**
* `error` is the thrown error for a failed query and omitted on success, where
* `error_code` is `none`.
*/
export function recordClickHouseQueryOutcome(
outcome: ClickHouseQueryOutcome,
tags: NormalizedClickHouseQueryTags,
table: ClickHouseQueryTable,
shape: ClickHouseQueryShape,
clickhouseService: ClickhouseService,
error?: unknown,
): void {
recordIncrement(CLICKHOUSE_QUERY_OUTCOME_METRIC, 1, {
outcome,
error_code:
outcome === "success" ? "none" : clickHouseQueryErrorCode(error),
...clickHouseQueryMetricTags(tags, table, shape, clickhouseService),
});
}
/**
* Emits the runtime distribution from a parsed `x-clickhouse-summary` header.
* ClickHouse sends `elapsed_ns` as a numeric string; an older server that does
* not send it emits nothing.
*/
export function recordClickHouseQueryElapsed(
summary: Record<string, unknown>,
tags: NormalizedClickHouseQueryTags,
table: ClickHouseQueryTable,
shape: ClickHouseQueryShape,
clickhouseService: ClickhouseService,
): void {
if (summary.elapsed_ns === undefined) return;
const elapsedNs = Number(summary.elapsed_ns);
if (!Number.isFinite(elapsedNs)) return;
recordDistribution(
CLICKHOUSE_QUERY_ELAPSED_MS_METRIC,
elapsedNs / 1e6,
clickHouseQueryMetricTags(tags, table, shape, clickhouseService),
);
}