Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(clickhouse): attribute query outcomes to every route and error code (#18634)

Niklas Semmler054b543deaaf
diff --git a/packages/shared/src/server/clickhouse/queryOutcome.test.ts b/packages/shared/src/server/clickhouse/queryOutcome.test.ts
index f241a74906..72e22883e0 100644
--- a/packages/shared/src/server/clickhouse/queryOutcome.test.ts
+++ b/packages/shared/src/server/clickhouse/queryOutcome.test.ts
@@ -1,9 +1,11 @@
+import { ClickHouseError } from "@clickhouse/client";
import { beforeEach, describe, expect, it, vi } from "vitest";
import {
CLICKHOUSE_QUERY_OUTCOME_METRIC,
CLICKHOUSE_QUERY_ELAPSED_MS_METRIC,
CLICKHOUSE_RESOURCE_ERROR_OUTCOMES,
+ clickHouseQueryErrorCode,
clickHouseQueryOutcomeRouteLabel,
clickHouseQueryShape,
clickHouseQueryTableLabel,
@@ -40,45 +42,106 @@ describe("ClickHouse query outcome metric", () => {
).toBe("get_/api/public/v3/scores");
});
- // The route tag is derived from the request path, so caller-controlled
- // segments would otherwise become unbounded metric tag values.
+ it("renders a templated REST route with its parameter placeholders", () => {
+ expect(
+ clickHouseQueryOutcomeRouteLabel("GET /api/public/traces/[traceId]"),
+ ).toBe("get_/api/public/traces/_traceId_");
+ });
+
+ // REST routes are templated at the source; a segment the templating missed
+ // is caller-controlled and would make the tag unbounded.
it.each([
"GET /api/public/traces/123e4567-e89b-12d3-a456-426614174000",
"GET /api/public/v2/prompts/dashboard.status_headline.low.de",
- "GET /api/public/v2/observations/some-observation-id",
- "HEAD /api/public/v2/observations",
- "POST /api/public/v2/metrics",
+ `GET /api${"/a".repeat(10)}`,
+ "GET /health/[x]",
+ "get /api/public/v2/observations",
])("counts %s under the catch-all label", (route) => {
expect(clickHouseQueryOutcomeRouteLabel(route)).toBe("other");
});
- it.each([undefined, "", " ", "no-method-separator"])(
+ it("labels a REST route at the segment limit", () => {
+ const route = `GET /api${"/a".repeat(9)}`;
+ expect(clickHouseQueryOutcomeRouteLabel(route)).toBe(
+ `get_/api${"/a".repeat(9)}`,
+ );
+ });
+
+ it("labels a bare route at the length limit and rejects a longer one", () => {
+ const atLimit = `a${"b".repeat(99)}`;
+ expect(clickHouseQueryOutcomeRouteLabel(atLimit)).toBe(atLimit);
+ expect(clickHouseQueryOutcomeRouteLabel(`${atLimit}c`)).toBe("other");
+ });
+
+ it.each([undefined, "", " ", "no-method-separator!", "1.leading.digit"])(
"falls back to the catch-all label for %s",
(route) => {
expect(clickHouseQueryOutcomeRouteLabel(route)).toBe("other");
},
);
- // tRPC procedure paths and MCP tool names carry no method prefix and are
- // code-defined, so they are matched verbatim against the bare allowlist.
+ // tRPC procedure paths, MCP tool names and worker routes carry no method
+ // prefix and are code-defined, so they pass through.
it.each([
- ["events.all", "events.all"],
- ["events.batchIO", "events.batchIO"],
- ["events.experimentBatchIO", "events.experimentBatchIO"],
- ["events.sessionBatchIO", "events.sessionBatchIO"],
- ["events.filterOptions", "events.filterOptions"],
- ["experiments.batchIO", "experiments.batchIO"],
- ["scores.allFromEvents", "scores.allFromEvents"],
- ["scores.countAllFromEvents", "scores.countAllFromEvents"],
- ["traces.byId", "traces.byId"],
- ["listObservations", "listObservations"],
- ["trace_redirect", "trace_redirect"],
- ])("labels the bare route %s verbatim", (route, expected) => {
- expect(clickHouseQueryOutcomeRouteLabel(route)).toBe(expected);
+ "events.all",
+ "traces.byIdWithObservationsAndScores",
+ "listObservations",
+ "trace_redirect",
+ "langfuse.queue.trace_delete",
+ "langfuse.periodic_runner.batch_trace_deletion_cleaner",
+ ])("labels the bare route %s verbatim", (route) => {
+ expect(clickHouseQueryOutcomeRouteLabel(route)).toBe(route);
+ });
+ });
+
+ describe("error codes", () => {
+ it("labels a thrown ClickHouseError by its code", () => {
+ const error = new ClickHouseError({
+ message: "Unknown table expression identifier 'topic_assignments'",
+ code: "60",
+ type: "UNKNOWN_TABLE",
+ });
+ expect(clickHouseQueryErrorCode(error)).toBe("unknown_table");
+ });
+
+ it("parses the code from an exception row message", () => {
+ const error = new Error(
+ "Code: 159. DB::Exception: Timeout exceeded: elapsed 30.1 seconds, maximum: 30. (TIMEOUT_EXCEEDED) (version 26.4.1.1)",
+ );
+ expect(clickHouseQueryErrorCode(error)).toBe("timeout_exceeded");
+ });
+
+ it("reads through a wrapping error to its cause", () => {
+ const cause = new ClickHouseError({
+ message: "Memory limit (total) exceeded",
+ code: "241",
+ type: "MEMORY_LIMIT_EXCEEDED",
+ });
+ expect(
+ clickHouseQueryErrorCode(new Error(cause.message, { cause })),
+ ).toBe("memory_limit_exceeded");
+ });
+
+ // The exception name is parsed from message text that can echo caller
+ // input, so only the numeric code decides the label.
+ it("reports an unlabelled code as other, ignoring names in the message", () => {
+ const error = new Error(
+ "Code: 6. DB::Exception: Cannot parse string 'a (CALLER_CHOSEN) b' as Int64. (CANNOT_PARSE_TEXT) (version 26.4.1.1)",
+ );
+ expect(clickHouseQueryErrorCode(error)).toBe("other");
+ });
+
+ it.each([
+ ["socket hang up", "network"],
+ ["read ECONNRESET", "network"],
+ ["Timeout error.", "client_timeout"],
+ ["Unexpected end of JSON input", "other"],
+ ])("classifies a client-side %s as %s", (message, expected) => {
+ expect(clickHouseQueryErrorCode(new Error(message))).toBe(expected);
});
- it("counts an unlisted bare route under the catch-all label", () => {
- expect(clickHouseQueryOutcomeRouteLabel("prompts.all")).toBe("other");
+ it("labels a non-Error throw as other", () => {
+ expect(clickHouseQueryErrorCode("boom")).toBe("other");
});
});
@@ -315,6 +378,9 @@ describe("ClickHouse query outcome metric", () => {
"events_full",
"io_content",
"events_read_replica",
+ new Error(
+ "Code: 159. DB::Exception: Timeout exceeded. (TIMEOUT_EXCEEDED) (version 26.4.1.1)",
+ ),
);
expect(recordIncrement).toHaveBeenCalledTimes(1);
@@ -323,6 +389,7 @@ describe("ClickHouse query outcome metric", () => {
1,
{
outcome: "timeout",
+ error_code: "timeout_exceeded",
surface: "publicapi",
route: "get_/api/public/v2/observations",
table: "events_full",
@@ -398,6 +465,7 @@ describe("ClickHouse query outcome metric", () => {
1,
{
outcome: "success",
+ error_code: "none",
surface: "unknown",
route: "other",
table: "other",
diff --git a/packages/shared/src/server/clickhouse/queryOutcome.ts b/packages/shared/src/server/clickhouse/queryOutcome.ts
index c9bad8217a..0761aa02f2 100644
--- a/packages/shared/src/server/clickhouse/queryOutcome.ts
+++ b/packages/shared/src/server/clickhouse/queryOutcome.ts
@@ -1,3 +1,4 @@
+import { ClickHouseError, parseError } from "@clickhouse/client";
import { recordDistribution, recordIncrement } from "../instrumentation";
import { type NormalizedClickHouseQueryTags } from "./queryTags";
import { type ClickhouseService } from "./client";
@@ -43,41 +44,21 @@ export const CLICKHOUSE_RESOURCE_ERROR_OUTCOMES = {
} as const satisfies Record<string, ClickHouseQueryOutcome>;
/**
- * REST routes whose outcomes are reported under their own `route` tag.
- * Everything else is counted under `other`.
+ * Metric tags must be bounded, so a route only gets its own label when its
+ * value set is fixed by code:
*
- * `route` on the query tags is derived from the request path, so it carries
- * caller-controlled segments (trace ids, prompt names) and is unbounded. Metric
- * tags must be bounded, so only explicitly listed routes get a label. Add a
- * route here when it gains an SLO.
+ * - Bare routes (`events.all`, `listObservations`, `langfuse.queue.trace_delete`)
+ * are tRPC procedure paths, MCP tool names and worker routes. Each is set only
+ * after the procedure, tool or queue resolved, so they pass through.
+ * - REST routes carry their page template (`GET /api/public/traces/[traceId]`),
+ * taken from a list of the dynamic API routes in web. As a guard against a
+ * route missing from that list, a segment that is neither a word, a version
+ * (`v2`) nor a `[param]` placeholder drops the route to `other`.
*/
-const LABELLED_ROUTES = new Set([
- "GET /api/public/experiments",
- "GET /api/public/experiment-items",
- "GET /api/public/v2/observations",
- "GET /api/public/v2/metrics",
- "GET /api/public/v3/scores",
-]);
-
-/**
- * Non-REST routes matched verbatim: tRPC procedure paths (`events.all`) and MCP
- * tool names (`listObservations`) are code-defined and already bounded, so they
- * do not need the method/path parsing REST routes get. Add one when it gains an
- * SLO or drives a meaningful share of timeouts.
- */
-const LABELLED_BARE_ROUTES = new Set([
- "events.all",
- "events.batchIO",
- "events.experimentBatchIO",
- "events.sessionBatchIO",
- "events.filterOptions",
- "experiments.batchIO",
- "scores.allFromEvents",
- "scores.countAllFromEvents",
- "traces.byId",
- "listObservations",
- "trace_redirect",
-]);
+const BARE_ROUTE_PATTERN = /^[A-Za-z][\w.]{0,99}$/;
+const REST_PATH_PREFIX = "/api/";
+const REST_STATIC_SEGMENT_PATTERN = /^(?:[A-Za-z][A-Za-z_-]*|v\d+|\[\w+\])$/;
+const MAX_REST_SEGMENTS = 10;
const OTHER_ROUTE_LABEL = "other";
@@ -85,7 +66,8 @@ const OTHER_ROUTE_LABEL = "other";
* Renders a route as a Datadog tag value, matching the `resource_name`
* convention APM uses for the same route (`get_/api/public/v2/metrics`), so
* SLO and dashboard queries can filter both signals the same way. Datadog tag
- * values cannot contain spaces, so the method is folded in with an underscore.
+ * values cannot contain spaces or brackets, so the method is folded in with an
+ * underscore and a `[param]` placeholder is rendered as `_param_`.
*/
export function clickHouseQueryOutcomeRouteLabel(route?: string): string {
if (!route) return OTHER_ROUTE_LABEL;
@@ -93,19 +75,25 @@ export function clickHouseQueryOutcomeRouteLabel(route?: string): string {
const collapsed = route.trim();
const separatorIndex = collapsed.indexOf(" ");
if (separatorIndex === -1) {
- return LABELLED_BARE_ROUTES.has(collapsed) ? collapsed : OTHER_ROUTE_LABEL;
+ return BARE_ROUTE_PATTERN.test(collapsed) ? collapsed : OTHER_ROUTE_LABEL;
}
const method = collapsed.slice(0, separatorIndex);
const path = collapsed.slice(separatorIndex + 1);
const normalizedPath =
path.length > 1 && path.endsWith("/") ? path.slice(0, -1) : path;
+ if (!/^[A-Z]+$/.test(method) || !normalizedPath.startsWith(REST_PATH_PREFIX))
+ return OTHER_ROUTE_LABEL;
- if (!LABELLED_ROUTES.has(`${method} ${normalizedPath}`)) {
+ const segments = normalizedPath.slice(1).split("/");
+ if (
+ segments.length > MAX_REST_SEGMENTS ||
+ !segments.every((segment) => REST_STATIC_SEGMENT_PATTERN.test(segment))
+ ) {
return OTHER_ROUTE_LABEL;
}
- return `${method.toLowerCase()}_${normalizedPath}`;
+ return `${method.toLowerCase()}_${normalizedPath.replace(/\[(\w+)\]/g, "_$1_")}`;
}
/**
@@ -287,15 +275,92 @@ function clickHouseQueryMetricTags(
};
}
+/**
+ * Transport failures the query path retries. Shared with the retry predicate so
+ * an exhausted retry is labelled `network` on the outcome metric.
+ */
+export const CLICKHOUSE_NETWORK_ERROR_PATTERNS = [
+ "socket hang up",
+ "broken pipe",
+ "connection reset",
+ "econnreset",
+ "network_error",
+ "etimedout",
+ "econnrefused",
+] as const;
+
+/**
+ * ClickHouse error codes that get their own `error_code` label, keyed by the
+ * numeric code. Together they cover >99.9% of failed Langfuse reads; the rest is
+ * a long tail reported as `other` and recoverable from `system.query_log`.
+ * Matching on the number rather than the name keeps the label set closed: the
+ * name is parsed from the message text, which can echo caller input.
+ */
+const LABELLED_CLICKHOUSE_ERROR_CODES: Record<string, string> = {
+ "60": "unknown_table",
+ "159": "timeout_exceeded",
+ "202": "too_many_simultaneous_queries",
+ "209": "socket_timeout",
+ "210": "network_error",
+ "241": "memory_limit_exceeded",
+ "734": "cannot_write_after_buffer_canceled",
+ "735": "query_was_cancelled_by_client",
+};
+
+const OTHER_ERROR_CODE_LABEL = "other";
+
+/**
+ * Bounded cause of a failed query, so `error` outcomes are not opaque and a
+ * server-side `TIMEOUT_EXCEEDED` can be told from a client request timeout.
+ *
+ * - A labelled ClickHouse error code (`timeout_exceeded`, `unknown_table`, …),
+ * from a thrown `ClickHouseError` or an exception row's `Code: N.` prefix.
+ * Any other ClickHouse code is `other`.
+ * - `network` for a transport failure that outlived the retries.
+ * - `client_timeout` for the client's own request timeout, which carries no
+ * ClickHouse code.
+ * - `other` for anything else.
+ */
+export function clickHouseQueryErrorCode(error: unknown): string {
+ const root =
+ error instanceof Error && error.cause instanceof Error
+ ? error.cause
+ : error;
+ if (!(root instanceof Error)) return OTHER_ERROR_CODE_LABEL;
+
+ const parsed = root instanceof ClickHouseError ? root : parseError(root);
+ if (parsed instanceof ClickHouseError) {
+ return (
+ LABELLED_CLICKHOUSE_ERROR_CODES[parsed.code] ?? OTHER_ERROR_CODE_LABEL
+ );
+ }
+
+ const message = (root.message ?? "").toLowerCase();
+ if (CLICKHOUSE_NETWORK_ERROR_PATTERNS.some((p) => message.includes(p))) {
+ return "network";
+ }
+ if (message.includes("timeout") || message.includes("timed out")) {
+ return "client_timeout";
+ }
+ return OTHER_ERROR_CODE_LABEL;
+}
+
+/**
+ * `error` is the thrown error for a failed query and omitted on success, where
+ * `error_code` is `none`.
+ */
export function recordClickHouseQueryOutcome(
outcome: ClickHouseQueryOutcome,
tags: NormalizedClickHouseQueryTags,
table: ClickHouseQueryTable,
shape: ClickHouseQueryShape,
clickhouseService: ClickhouseService,
+ error?: unknown,
): void {
recordIncrement(CLICKHOUSE_QUERY_OUTCOME_METRIC, 1, {
outcome,
+ error_code:
+ outcome === "success" ? "none" : clickHouseQueryErrorCode(error),
...clickHouseQueryMetricTags(tags, table, shape, clickhouseService),
});
}
diff --git a/packages/shared/src/server/clickhouse/queryTags.test.ts b/packages/shared/src/server/clickhouse/queryTags.test.ts
index 70da06656d..8933fac5d1 100644
--- a/packages/shared/src/server/clickhouse/queryTags.test.ts
+++ b/packages/shared/src/server/clickhouse/queryTags.test.ts
@@ -4,6 +4,7 @@ import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-ho
import {
buildClickHouseLogComment,
CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS,
+ CLICKHOUSE_ROUTE_CONTEXT_KEY,
} from "./queryTags";
describe("ClickHouse query tags", () => {
@@ -11,18 +12,17 @@ describe("ClickHouse query tags", () => {
context.setGlobalContextManager(
new AsyncLocalStorageContextManager().enable(),
);
- const entrypoint = propagation.setBaggage(
- context.active(),
- propagation.createBaggage({
- [CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.surface]: { value: "worker" },
- [CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.route]: {
- value: "langfuse.queue.trace_batch",
- },
- [CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.projectId]: {
- value: "producer-project",
- },
- }),
- );
+ const entrypoint = propagation
+ .setBaggage(
+ context.active(),
+ propagation.createBaggage({
+ [CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.surface]: { value: "worker" },
+ [CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.projectId]: {
+ value: "producer-project",
+ },
+ }),
+ )
+ .setValue(CLICKHOUSE_ROUTE_CONTEXT_KEY, "langfuse.queue.trace_batch");
try {
const tags = context.with(entrypoint, () =>
JSON.parse(
diff --git a/packages/shared/src/server/clickhouse/queryTags.ts b/packages/shared/src/server/clickhouse/queryTags.ts
index 50ebfd0469..0e116709ff 100644
--- a/packages/shared/src/server/clickhouse/queryTags.ts
+++ b/packages/shared/src/server/clickhouse/queryTags.ts
@@ -1,4 +1,4 @@
-import { context, propagation } from "@opentelemetry/api";
+import { context, createContextKey, propagation } from "@opentelemetry/api";
export const CLICKHOUSE_QUERY_TAG_SCHEMA_VERSION = "1" as const;
const UNKNOWN_CLICKHOUSE_QUERY_TAG_VALUE = "unknown" as const;
@@ -43,6 +43,17 @@ export const CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS = {
userAgent: "langfuse.clickhouse.user_agent",
} as const;
+/**
+ * Holds the route the entrypoint set. The route is mirrored into baggage for
+ * span attributes but never read back from it: inbound requests can carry
+ * `langfuse.clickhouse.*` baggage, and a caller-chosen route would make the
+ * outcome metric's `route` tag unbounded. Inbound headers cannot set a context
+ * key.
+ */
+export const CLICKHOUSE_ROUTE_CONTEXT_KEY = createContextKey(
+ "langfuse.clickhouse.route",
+);
+
const surfaceSet = new Set<string>(clickHouseQuerySurfaces);
const isClickHouseQuerySurface = (
@@ -60,7 +71,7 @@ export function normalizeClickHouseQueryTags(
baggage?.getEntry(CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.surface)?.value;
const route =
providedTags?.route ??
- baggage?.getEntry(CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.route)?.value;
+ context.active().getValue(CLICKHOUSE_ROUTE_CONTEXT_KEY);
const projectId =
providedTags?.projectId ??
baggage?.getEntry(CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.projectId)?.value;
diff --git a/packages/shared/src/server/headerPropagation.ts b/packages/shared/src/server/headerPropagation.ts
index ca80a54168..639b1ef463 100644
--- a/packages/shared/src/server/headerPropagation.ts
+++ b/packages/shared/src/server/headerPropagation.ts
@@ -3,6 +3,7 @@ import type { IncomingHttpHeaders } from "http";
import { env } from "../env";
import {
CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS,
+ CLICKHOUSE_ROUTE_CONTEXT_KEY,
type ClickHouseQuerySurface,
} from "./clickhouse/queryTags";
import { extractPublicApiCallerAttribution } from "./ingestion/ingestionAttribution";
@@ -120,5 +121,9 @@ export const contextWithLangfuseProps = (
}
}
- return opentelemetry.propagation.setBaggage(ctx, baggage);
+ const withBaggage = opentelemetry.propagation.setBaggage(ctx, baggage);
+ const route = props.clickhouse?.route?.trim();
+ return route
+ ? withBaggage.setValue(CLICKHOUSE_ROUTE_CONTEXT_KEY, route)
+ : withBaggage;
};
diff --git a/packages/shared/src/server/instrumentation/index.test.ts b/packages/shared/src/server/instrumentation/index.test.ts
index 69019cdb77..5108036153 100644
--- a/packages/shared/src/server/instrumentation/index.test.ts
+++ b/packages/shared/src/server/instrumentation/index.test.ts
@@ -133,6 +133,26 @@ describe("instrumentation baggage propagation", () => {
expect(tags).not.toHaveProperty("userAgent");
});
+ // A route the entrypoint never set must not come from the caller: the route
+ // is a tag on the ClickHouse outcome metric.
+ it("ignores a ClickHouse route inherited from inbound baggage", () => {
+ const inboundContext = propagation.setBaggage(
+ ROOT_CONTEXT,
+ propagation.createBaggage({
+ [CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.surface]: { value: "trpc" },
+ [CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.route]: {
+ value: "attackerChosenRoute",
+ },
+ }),
+ );
+
+ const tags = context.with(inboundContext, () =>
+ normalizeClickHouseQueryTags(),
+ );
+
+ expect(tags).not.toHaveProperty("route");
+ });
+
it("does not add user emails to span attributes or baggage", () => {
const span = {
setAttribute: vi.fn(),
diff --git a/packages/shared/src/server/instrumentation/index.ts b/packages/shared/src/server/instrumentation/index.ts
index 5617b5c5c1..cae9ee3504 100644
--- a/packages/shared/src/server/instrumentation/index.ts
+++ b/packages/shared/src/server/instrumentation/index.ts
@@ -7,6 +7,7 @@ import * as opentelemetry from "@opentelemetry/api";
import * as dd from "dd-trace";
import { env } from "../../env";
import { logger } from "../logger";
+import { CLICKHOUSE_ROUTE_CONTEXT_KEY } from "../clickhouse/queryTags";
export type TCarrier = {
traceparent?: string;
@@ -30,12 +31,7 @@ export async function instrumentAsync<T>(
callback: AsyncCallbackFn<T>,
): Promise<T> {
const activeContext = ctx.startNewTrace
- ? // Sever the parent trace but carry baggage onto the new root.
- opentelemetry.propagation.setBaggage(
- opentelemetry.ROOT_CONTEXT,
- opentelemetry.propagation.getBaggage(opentelemetry.context.active()) ??
- opentelemetry.propagation.createBaggage(),
- )
+ ? newTraceRootContext()
: ctx.traceContext
? opentelemetry.propagation.extract(
opentelemetry.context.active(),
@@ -72,6 +68,23 @@ export async function instrumentAsync<T>(
);
}
+/**
+ * Severs the parent trace but carries baggage and the ClickHouse route onto the
+ * new root, so queries in the new trace keep their attribution.
+ */
+const newTraceRootContext = () => {
+ const active = opentelemetry.context.active();
+ const root = opentelemetry.propagation.setBaggage(
+ opentelemetry.ROOT_CONTEXT,
+ opentelemetry.propagation.getBaggage(active) ??
+ opentelemetry.propagation.createBaggage(),
+ );
+ const route = active.getValue(CLICKHOUSE_ROUTE_CONTEXT_KEY);
+ return route === undefined
+ ? root
+ : root.setValue(CLICKHOUSE_ROUTE_CONTEXT_KEY, route);
+};
+
type SyncCallbackFn<T> = (span: opentelemetry.Span) => T;
/** instrumentSync runs a callback inside a fresh OTel span. */
@@ -80,12 +93,7 @@ export function instrumentSync<T>(
callback: SyncCallbackFn<T>,
): T {
const activeContext = ctx.startNewTrace
- ? // Sever the parent trace but carry baggage onto the new root.
- opentelemetry.propagation.setBaggage(
- opentelemetry.ROOT_CONTEXT,
- opentelemetry.propagation.getBaggage(opentelemetry.context.active()) ??
- opentelemetry.propagation.createBaggage(),
- )
+ ? newTraceRootContext()
: ctx.traceContext
? opentelemetry.propagation.extract(
opentelemetry.context.active(),
diff --git a/packages/shared/src/server/repositories/clickhouse.ts b/packages/shared/src/server/repositories/clickhouse.ts
index 15e5b6e6f2..d1485a3d4f 100644
--- a/packages/shared/src/server/repositories/clickhouse.ts
+++ b/packages/shared/src/server/repositories/clickhouse.ts
@@ -31,6 +31,7 @@ import {
type NormalizedClickHouseQueryTags,
} from "../clickhouse/queryTags";
import {
+ CLICKHOUSE_NETWORK_ERROR_PATTERNS,
CLICKHOUSE_RESOURCE_ERROR_OUTCOMES,
clickHouseQueryShape,
clickHouseQueryTableLabel,
@@ -688,17 +689,9 @@ function isRetryableError(error: unknown): boolean {
const errorMessage = (error as Error).message?.toLowerCase() || "";
// Check for socket hang up and other network-related errors
- const retryablePatterns = [
- "socket hang up",
- "broken pipe",
- "connection reset",
- "econnreset",
- "network_error",
- "etimedout",
- "econnrefused",
- ];
-
- return retryablePatterns.some((pattern) => errorMessage.includes(pattern));
+ return CLICKHOUSE_NETWORK_ERROR_PATTERNS.some((pattern) =>
+ errorMessage.includes(pattern),
+ );
}
export async function queryClickhouse<T>(
@@ -777,6 +770,7 @@ export async function queryClickhouse<T>(
table,
shape,
clickhouseService,
+ error,
);
throw wrapped;
});
diff --git a/web/src/__tests__/server/withMiddlewares.servertest.ts b/web/src/__tests__/server/withMiddlewares.servertest.ts
index 5e0bd01163..32f31d19a8 100644
--- a/web/src/__tests__/server/withMiddlewares.servertest.ts
+++ b/web/src/__tests__/server/withMiddlewares.servertest.ts
@@ -3,7 +3,11 @@ import {
LEGACY_PUBLIC_API_METRICS_CLICKHOUSE_RESOURCE_ERROR_MESSAGE,
withMiddlewares,
} from "@/src/features/public-api/server/withMiddlewares";
-import { clickHouseRouteForRequest } from "@/src/features/public-api/server/clickHouseRequestTags";
+import {
+ clickHouseRouteForRequest,
+ DYNAMIC_API_ROUTE_TEMPLATES,
+} from "@/src/features/public-api/server/clickHouseRequestTags";
+import { readdirSync, readFileSync } from "fs";
import {
BaseError,
LangfuseNotFoundError,
@@ -54,6 +58,49 @@ describe("clickHouseRouteForRequest", () => {
).toBe("POST http://[::1");
});
+ it("reports a dynamic route as its template", () => {
+ expect(
+ clickHouseRouteForRequest(
+ request(
+ "GET",
+ "/api/public/v2/prompts/team%2Fsupport/versions/3?label=prod",
+ ),
+ ),
+ ).toBe("GET /api/public/v2/prompts/[promptName]/versions/[promptVersion]");
+ });
+
+ // A caller-chosen id must not survive into the route: the route tag feeds a
+ // metric, so every distinct id would add a series.
+ it.each([
+ "/api/public/traces/abcdefgh?traceId=x",
+ "/api/public/traces/api",
+ "/api/public/traces/traces/",
+ ])("templates %s regardless of the id's value", (url) => {
+ expect(clickHouseRouteForRequest(request("GET", url))).toBe(
+ "GET /api/public/traces/[traceId]",
+ );
+ });
+
+ it("lists every dynamic API route that tags its ClickHouse queries", () => {
+ const apiDir = `${__dirname}/../../pages/api`;
+ const tagsQueries = (file: string) =>
+ file.startsWith("public/") ||
+ /\b(withMiddlewares|createAuthedProjectAPIRoute|clickHouseRouteForRequest)\(/.test(
+ readFileSync(`${apiDir}/${file}`, "utf8"),
+ );
+ const dynamicTemplates = readdirSync(apiDir, { recursive: true })
+ .map(String)
+ .filter((file) => file.endsWith(".ts") && file.includes("["))
+ .filter(tagsQueries)
+ .map(
+ (file) => "/api/" + file.replace(/\.ts$/, "").replace(/\/index$/, ""),
+ );
+
+ expect([...DYNAMIC_API_ROUTE_TEMPLATES].sort()).toEqual(
+ [...new Set(dynamicTemplates)].sort(),
+ );
+ });
+
it("falls back to UNKNOWN method for missing methods", () => {
expect(
clickHouseRouteForRequest(request(undefined, "/api/public/health")),
diff --git a/web/src/features/public-api/server/clickHouseRequestTags.ts b/web/src/features/public-api/server/clickHouseRequestTags.ts
index 65749d353e..479b9e930c 100644
--- a/web/src/features/public-api/server/clickHouseRequestTags.ts
+++ b/web/src/features/public-api/server/clickHouseRequestTags.ts
@@ -1,8 +1,64 @@
import { type NextApiRequest } from "next";
-const stripSearchAndHash = (url: string) => url.split(/[?#]/, 1)[0] ?? "";
+/**
+ * Every dynamic API route that tags its ClickHouse queries, as its Next.js page
+ * path. A request path matching one of these is reported as the template, so
+ * the route names the endpoint rather than the entity and stays bounded.
+ * Static routes need no entry. `withMiddlewares.servertest.ts` keeps this
+ * list in sync with `web/src/pages/api`.
+ */
+export const DYNAMIC_API_ROUTE_TEMPLATES = [
+ "/api/public/annotation-queues/[queueId]",
+ "/api/public/annotation-queues/[queueId]/assignments",
+ "/api/public/annotation-queues/[queueId]/items",
+ "/api/public/annotation-queues/[queueId]/items/[itemId]",
+ "/api/public/comments/[commentId]",
+ "/api/public/dataset-items/[datasetItemId]",
+ "/api/public/datasets/[name]",
+ "/api/public/datasets/[name]/runs",
+ "/api/public/datasets/[name]/runs/[runName]",
+ "/api/public/integrations/blob-storage/[id]",
+ "/api/public/llm-connections/[id]",
+ "/api/public/media/[mediaId]",
+ "/api/public/models/[modelId]",
+ "/api/public/observations/[observationId]",
+ "/api/public/projects/[projectId]",
+ "/api/public/projects/[projectId]/apiKeys",
+ "/api/public/projects/[projectId]/apiKeys/[apiKeyId]",
+ "/api/public/projects/[projectId]/memberships",
+ "/api/public/scim/Users/[id]",
+ "/api/public/score-configs/[configId]",
+ "/api/public/scores/[scoreId]",
+ "/api/public/sessions/[sessionId]",
+ "/api/public/traces/[traceId]",
+ "/api/public/unstable/dashboard-widgets/[widgetId]",
+ "/api/public/unstable/dashboards/[dashboardId]",
+ "/api/public/unstable/dashboards/[dashboardId]/placements",
+ "/api/public/unstable/dashboards/[dashboardId]/placements/[placementId]",
+ "/api/public/unstable/skills/[skillName]",
+ "/api/public/unstable/skills/[skillName]/versions/[skillVersion]",
+ "/api/public/v2/datasets/[datasetName]",
+ "/api/public/v2/evaluation-rules/[evaluationRuleId]",
+ "/api/public/v2/evaluators/[evaluatorId]",
+ "/api/public/v2/evaluators/[evaluatorId]/versions",
+ "/api/public/v2/prompts/[promptName]",
+ "/api/public/v2/prompts/[promptName]/versions/[promptVersion]",
+ "/api/public/v2/scores/[scoreId]",
+ "/api/traces/[traceId]/download",
+ "/api/traces/[traceId]/observations/[observationId]/io/[field]",
+] as const;
+
+const TEMPLATE_SEGMENTS = DYNAMIC_API_ROUTE_TEMPLATES.map((template) =>
+ template.split("/"),
+);
export const clickHouseRouteForRequest = (req: NextApiRequest) => {
+ const { method, pathname } = requestPathForRequest(req);
+ return `${method} ${templatePath(pathname)}`;
+};
+
+/** The request method and path without search params or fragment. */
+export const requestPathForRequest = (req: NextApiRequest) => {
const method = req.method ?? "UNKNOWN";
const rawUrl = req.url ?? "";
@@ -10,8 +66,35 @@ export const clickHouseRouteForRequest = (req: NextApiRequest) => {
const pathname = rawUrl
? new URL(rawUrl, "http://langfuse.local").pathname
: "";
- return `${method} ${pathname}`;
+ return { method, pathname };
} catch {
- return `${method} ${stripSearchAndHash(rawUrl)}`;
+ return { method, pathname: stripSearchAndHash(rawUrl) };
}
};
+
+/**
+ * Only dynamic positions are replaced, so a value that equals a fixed segment
+ * (`/api/public/traces/api`) keeps the endpoint intact. No two templates match
+ * the same path and none overlaps a static route, so the first match is the
+ * page Next.js routed to.
+ */
+const templatePath = (pathname: string) => {
+ const normalized =
+ pathname.length > 1 && pathname.endsWith("/")
+ ? pathname.slice(0, -1)
+ : pathname;
+ const segments = normalized.split("/");
+ const match = TEMPLATE_SEGMENTS.find(
+ (template) =>
+ template.length === segments.length &&
+ template.every(
+ (segment, index) =>
+ isDynamicSegment(segment) || segment === segments[index],
+ ),
+ );
+ return match ? match.join("/") : pathname;
+};
+
+const isDynamicSegment = (segment: string) => segment.startsWith("[");
+
+const stripSearchAndHash = (url: string) => url.split(/[?#]/, 1)[0] ?? "";
diff --git a/web/src/features/public-api/server/legacyApiOrganizationCutoff.ts b/web/src/features/public-api/server/legacyApiOrganizationCutoff.ts
index 627ae50b7b..93c245b01f 100644
--- a/web/src/features/public-api/server/legacyApiOrganizationCutoff.ts
+++ b/web/src/features/public-api/server/legacyApiOrganizationCutoff.ts
@@ -8,7 +8,7 @@ import {
} from "@langfuse/shared/src/server";
import { env } from "@/src/env.mjs";
-import { clickHouseRouteForRequest } from "@/src/features/public-api/server/clickHouseRequestTags";
+import { requestPathForRequest } from "@/src/features/public-api/server/clickHouseRequestTags";
import { attachDeprecation } from "@/src/features/public-api/server/deprecations";
const utcDateFormatter = new Intl.DateTimeFormat("en-US", {
@@ -41,7 +41,8 @@ export function applyLegacyApiOrganizationCutoff(params: {
return null;
}
- const apiPath = clickHouseRouteForRequest(params.req);
+ const { method, pathname } = requestPathForRequest(params.req);
+ const apiPath = `${method} ${pathname}`;
const callerAttribution = extractPublicApiCallerAttribution(
params.req.headers,
);
diff --git a/worker/src/__tests__/periodicRunner.test.ts b/worker/src/__tests__/periodicRunner.test.ts
index bde6c5319d..1fe649bd6b 100644
--- a/worker/src/__tests__/periodicRunner.test.ts
+++ b/worker/src/__tests__/periodicRunner.test.ts
@@ -1,6 +1,8 @@
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
+import { context as otelContext, type Context } from "@opentelemetry/api";
const telemetry = vi.hoisted(() => ({
+ contextWithLangfuseProps: vi.fn(),
recordDistribution: vi.fn(),
recordGauge: vi.fn(),
recordIncrement: vi.fn(),
@@ -9,6 +11,7 @@ const telemetry = vi.hoisted(() => ({
}));
vi.mock("@langfuse/shared/src/server", () => ({
+ contextWithLangfuseProps: telemetry.contextWithLangfuseProps,
getCurrentSpan: vi.fn(() => telemetry.span),
instrumentAsync: vi.fn(
(
@@ -32,6 +35,20 @@ vi.mock("@langfuse/shared/src/server", () => ({
import { PeriodicRunner } from "../utils/PeriodicRunner";
import { PeriodicExclusiveRunner } from "../utils/PeriodicExclusiveRunner";
+// Tracks the context `otelContext.with` activates, so a test can tell whether
+// `execute()` runs inside it.
+const activeContexts: Context[] = [];
+vi.spyOn(otelContext, "with").mockImplementation(
+ (ctx, fn, thisArg, ...args) => {
+ activeContexts.push(ctx);
+ try {
+ return fn.call(thisArg, ...args);
+ } finally {
+ activeContexts.pop();
+ }
+ },
+);
+
/**
* Helper to flush microtasks without advancing timers
*
@@ -69,6 +86,7 @@ class TestRunner extends PeriodicRunner {
public shouldThrow = false;
public returnInterval: number | undefined = undefined;
public initialDelayMsForTest = 0;
+ public executeContext: Context | undefined;
constructor() {
super("test_runner");
@@ -88,6 +106,7 @@ class TestRunner extends PeriodicRunner {
protected async execute(): Promise<number | void> {
this.callCount++;
+ this.executeContext = activeContexts.at(-1);
if (this.shouldThrow) {
this.shouldThrow = false; // Only throw once
throw new Error("Test error");
@@ -154,12 +173,21 @@ describe("PeriodicRunner", () => {
});
it("should call execute on start and record canonical metrics", async () => {
+ const clickHouseContext = {} as Context;
+ telemetry.contextWithLangfuseProps.mockReturnValue(clickHouseContext);
const runner = new TestRunner();
runner.start();
await flushMicrotasks();
expect(runner.callCount).toBe(1);
+ expect(runner.executeContext).toBe(clickHouseContext);
+ expect(telemetry.contextWithLangfuseProps).toHaveBeenCalledWith({
+ clickhouse: {
+ surface: "worker",
+ route: "langfuse.periodic_runner.test_runner",
+ },
+ });
expect(telemetry.recordIncrement).toHaveBeenCalledWith(
"langfuse.periodic_runner.started",
1,
diff --git a/worker/src/__tests__/workerManagerMetrics.test.ts b/worker/src/__tests__/workerManagerMetrics.test.ts
index 5df2f329f1..d03eb47976 100644
--- a/worker/src/__tests__/workerManagerMetrics.test.ts
+++ b/worker/src/__tests__/workerManagerMetrics.test.ts
@@ -36,6 +36,7 @@ vi.mock("@opentelemetry/api", () => ({
context: {
with: (_context: unknown, callback: () => unknown) => callback(),
},
+ createContextKey: (description: string) => Symbol.for(description),
}));
vi.mock("@langfuse/shared/src/server", async () => {
diff --git a/worker/src/features/queue-metrics-runner/index.test.ts b/worker/src/features/queue-metrics-runner/index.test.ts
index 7064bb6eb6..3e6e976b58 100644
--- a/worker/src/features/queue-metrics-runner/index.test.ts
+++ b/worker/src/features/queue-metrics-runner/index.test.ts
@@ -15,6 +15,7 @@ vi.mock("@langfuse/shared/src/server", () => ({
QueueName: {
TraceDelete: "trace-delete",
},
+ contextWithLangfuseProps: vi.fn(),
convertQueueNameToMetricName: (queueName: string) =>
`langfuse.queue.${queueName.replace(/-/g, "_").replace(/_queue$/, "")}`,
flushMetricsToCloudWatch: mocks.flushMetricsToCloudWatch,
diff --git a/worker/src/utils/PeriodicRunner.ts b/worker/src/utils/PeriodicRunner.ts
index 69cb3737be..0f716091cd 100644
--- a/worker/src/utils/PeriodicRunner.ts
+++ b/worker/src/utils/PeriodicRunner.ts
@@ -1,4 +1,5 @@
import {
+ contextWithLangfuseProps,
instrumentAsync,
logger,
recordDistribution,
@@ -6,7 +7,7 @@ import {
recordIncrement,
traceException,
} from "@langfuse/shared/src/server";
-import { SpanKind } from "@opentelemetry/api";
+import { context as otelContext, SpanKind } from "@opentelemetry/api";
const METRIC_PREFIX = "langfuse.periodic_runner";
type RunOutcome = "success" | "failed" | "skipped";
@@ -94,7 +95,15 @@ export abstract class PeriodicRunner {
span.setAttribute("runner.name", this.name);
span.setAttribute("runner.interval_ms", this.defaultIntervalMs);
- const result = await this.execute();
+ const clickHouseCtx = contextWithLangfuseProps({
+ clickhouse: {
+ surface: "worker",
+ route: `${METRIC_PREFIX}.${this.metricName}`,
+ },
+ });
+ const result = await otelContext.with(clickHouseCtx, () =>
+ this.execute(),
+ );
if (typeof result === "number") {
nextDelayMs = result;
span.setAttribute("runner.next_delay_ms", nextDelayMs);