🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(clickhouse): attribute query outcomes to every route and error code (#18634)
Niklas Semmler054b543deaaf
diff --git a/packages/shared/src/server/clickhouse/queryOutcome.test.ts b/packages/shared/src/server/clickhouse/queryOutcome.test.tsindex f241a74906..72e22883e0 100644--- a/packages/shared/src/server/clickhouse/queryOutcome.test.ts+++ b/packages/shared/src/server/clickhouse/queryOutcome.test.ts@@ -1,9 +1,11 @@+import { ClickHouseError } from "@clickhouse/client";import { beforeEach, describe, expect, it, vi } from "vitest";import {CLICKHOUSE_QUERY_OUTCOME_METRIC,CLICKHOUSE_QUERY_ELAPSED_MS_METRIC,CLICKHOUSE_RESOURCE_ERROR_OUTCOMES,+ clickHouseQueryErrorCode,clickHouseQueryOutcomeRouteLabel,clickHouseQueryShape,clickHouseQueryTableLabel,@@ -40,45 +42,106 @@ describe("ClickHouse query outcome metric", () => {).toBe("get_/api/public/v3/scores");});- // The route tag is derived from the request path, so caller-controlled- // segments would otherwise become unbounded metric tag values.+ it("renders a templated REST route with its parameter placeholders", () => {+ expect(+ clickHouseQueryOutcomeRouteLabel("GET /api/public/traces/[traceId]"),+ ).toBe("get_/api/public/traces/_traceId_");+ });++ // REST routes are templated at the source; a segment the templating missed+ // is caller-controlled and would make the tag unbounded.it.each(["GET /api/public/traces/123e4567-e89b-12d3-a456-426614174000","GET /api/public/v2/prompts/dashboard.status_headline.low.de",- "GET /api/public/v2/observations/some-observation-id",- "HEAD /api/public/v2/observations",- "POST /api/public/v2/metrics",+ `GET /api${"/a".repeat(10)}`,+ "GET /health/[x]",+ "get /api/public/v2/observations",])("counts %s under the catch-all label", (route) => {expect(clickHouseQueryOutcomeRouteLabel(route)).toBe("other");});- it.each([undefined, "", " ", "no-method-separator"])(+ it("labels a REST route at the segment limit", () => {+ const route = `GET /api${"/a".repeat(9)}`;+ expect(clickHouseQueryOutcomeRouteLabel(route)).toBe(+ `get_/api${"/a".repeat(9)}`,+ );+ });++ it("labels a bare route at the length limit and rejects a longer one", () => {+ const atLimit = `a${"b".repeat(99)}`;+ expect(clickHouseQueryOutcomeRouteLabel(atLimit)).toBe(atLimit);+ expect(clickHouseQueryOutcomeRouteLabel(`${atLimit}c`)).toBe("other");+ });++ it.each([undefined, "", " ", "no-method-separator!", "1.leading.digit"])("falls back to the catch-all label for %s",(route) => {expect(clickHouseQueryOutcomeRouteLabel(route)).toBe("other");},);- // tRPC procedure paths and MCP tool names carry no method prefix and are- // code-defined, so they are matched verbatim against the bare allowlist.+ // tRPC procedure paths, MCP tool names and worker routes carry no method+ // prefix and are code-defined, so they pass through.it.each([- ["events.all", "events.all"],- ["events.batchIO", "events.batchIO"],- ["events.experimentBatchIO", "events.experimentBatchIO"],- ["events.sessionBatchIO", "events.sessionBatchIO"],- ["events.filterOptions", "events.filterOptions"],- ["experiments.batchIO", "experiments.batchIO"],- ["scores.allFromEvents", "scores.allFromEvents"],- ["scores.countAllFromEvents", "scores.countAllFromEvents"],- ["traces.byId", "traces.byId"],- ["listObservations", "listObservations"],- ["trace_redirect", "trace_redirect"],- ])("labels the bare route %s verbatim", (route, expected) => {- expect(clickHouseQueryOutcomeRouteLabel(route)).toBe(expected);+ "events.all",+ "traces.byIdWithObservationsAndScores",+ "listObservations",+ "trace_redirect",+ "langfuse.queue.trace_delete",+ "langfuse.periodic_runner.batch_trace_deletion_cleaner",+ ])("labels the bare route %s verbatim", (route) => {+ expect(clickHouseQueryOutcomeRouteLabel(route)).toBe(route);+ });+ });++ describe("error codes", () => {+ it("labels a thrown ClickHouseError by its code", () => {+ const error = new ClickHouseError({+ message: "Unknown table expression identifier 'topic_assignments'",+ code: "60",+ type: "UNKNOWN_TABLE",+ });+ expect(clickHouseQueryErrorCode(error)).toBe("unknown_table");+ });++ it("parses the code from an exception row message", () => {+ const error = new Error(+ "Code: 159. DB::Exception: Timeout exceeded: elapsed 30.1 seconds, maximum: 30. (TIMEOUT_EXCEEDED) (version 26.4.1.1)",+ );+ expect(clickHouseQueryErrorCode(error)).toBe("timeout_exceeded");+ });++ it("reads through a wrapping error to its cause", () => {+ const cause = new ClickHouseError({+ message: "Memory limit (total) exceeded",+ code: "241",+ type: "MEMORY_LIMIT_EXCEEDED",+ });+ expect(+ clickHouseQueryErrorCode(new Error(cause.message, { cause })),+ ).toBe("memory_limit_exceeded");+ });++ // The exception name is parsed from message text that can echo caller+ // input, so only the numeric code decides the label.+ it("reports an unlabelled code as other, ignoring names in the message", () => {+ const error = new Error(+ "Code: 6. DB::Exception: Cannot parse string 'a (CALLER_CHOSEN) b' as Int64. (CANNOT_PARSE_TEXT) (version 26.4.1.1)",+ );+ expect(clickHouseQueryErrorCode(error)).toBe("other");+ });++ it.each([+ ["socket hang up", "network"],+ ["read ECONNRESET", "network"],+ ["Timeout error.", "client_timeout"],+ ["Unexpected end of JSON input", "other"],+ ])("classifies a client-side %s as %s", (message, expected) => {+ expect(clickHouseQueryErrorCode(new Error(message))).toBe(expected);});- it("counts an unlisted bare route under the catch-all label", () => {- expect(clickHouseQueryOutcomeRouteLabel("prompts.all")).toBe("other");+ it("labels a non-Error throw as other", () => {+ expect(clickHouseQueryErrorCode("boom")).toBe("other");});});@@ -315,6 +378,9 @@ describe("ClickHouse query outcome metric", () => {"events_full","io_content","events_read_replica",+ new Error(+ "Code: 159. DB::Exception: Timeout exceeded. (TIMEOUT_EXCEEDED) (version 26.4.1.1)",+ ),);expect(recordIncrement).toHaveBeenCalledTimes(1);@@ -323,6 +389,7 @@ describe("ClickHouse query outcome metric", () => {1,{outcome: "timeout",+ error_code: "timeout_exceeded",surface: "publicapi",route: "get_/api/public/v2/observations",table: "events_full",@@ -398,6 +465,7 @@ describe("ClickHouse query outcome metric", () => {1,{outcome: "success",+ error_code: "none",surface: "unknown",route: "other",table: "other",diff --git a/packages/shared/src/server/clickhouse/queryOutcome.ts b/packages/shared/src/server/clickhouse/queryOutcome.tsindex c9bad8217a..0761aa02f2 100644--- a/packages/shared/src/server/clickhouse/queryOutcome.ts+++ b/packages/shared/src/server/clickhouse/queryOutcome.ts@@ -1,3 +1,4 @@+import { ClickHouseError, parseError } from "@clickhouse/client";import { recordDistribution, recordIncrement } from "../instrumentation";import { type NormalizedClickHouseQueryTags } from "./queryTags";import { type ClickhouseService } from "./client";@@ -43,41 +44,21 @@ export const CLICKHOUSE_RESOURCE_ERROR_OUTCOMES = {} as const satisfies Record<string, ClickHouseQueryOutcome>;/**- * REST routes whose outcomes are reported under their own `route` tag.- * Everything else is counted under `other`.+ * Metric tags must be bounded, so a route only gets its own label when its+ * value set is fixed by code:*- * `route` on the query tags is derived from the request path, so it carries- * caller-controlled segments (trace ids, prompt names) and is unbounded. Metric- * tags must be bounded, so only explicitly listed routes get a label. Add a- * route here when it gains an SLO.+ * - Bare routes (`events.all`, `listObservations`, `langfuse.queue.trace_delete`)+ * are tRPC procedure paths, MCP tool names and worker routes. Each is set only+ * after the procedure, tool or queue resolved, so they pass through.+ * - REST routes carry their page template (`GET /api/public/traces/[traceId]`),+ * taken from a list of the dynamic API routes in web. As a guard against a+ * route missing from that list, a segment that is neither a word, a version+ * (`v2`) nor a `[param]` placeholder drops the route to `other`.*/-const LABELLED_ROUTES = new Set([- "GET /api/public/experiments",- "GET /api/public/experiment-items",- "GET /api/public/v2/observations",- "GET /api/public/v2/metrics",- "GET /api/public/v3/scores",-]);--/**- * Non-REST routes matched verbatim: tRPC procedure paths (`events.all`) and MCP- * tool names (`listObservations`) are code-defined and already bounded, so they- * do not need the method/path parsing REST routes get. Add one when it gains an- * SLO or drives a meaningful share of timeouts.- */-const LABELLED_BARE_ROUTES = new Set([- "events.all",- "events.batchIO",- "events.experimentBatchIO",- "events.sessionBatchIO",- "events.filterOptions",- "experiments.batchIO",- "scores.allFromEvents",- "scores.countAllFromEvents",- "traces.byId",- "listObservations",- "trace_redirect",-]);+const BARE_ROUTE_PATTERN = /^[A-Za-z][\w.]{0,99}$/;+const REST_PATH_PREFIX = "/api/";+const REST_STATIC_SEGMENT_PATTERN = /^(?:[A-Za-z][A-Za-z_-]*|v\d+|\[\w+\])$/;+const MAX_REST_SEGMENTS = 10;const OTHER_ROUTE_LABEL = "other";@@ -85,7 +66,8 @@ const OTHER_ROUTE_LABEL = "other";* Renders a route as a Datadog tag value, matching the `resource_name`* convention APM uses for the same route (`get_/api/public/v2/metrics`), so* SLO and dashboard queries can filter both signals the same way. Datadog tag- * values cannot contain spaces, so the method is folded in with an underscore.+ * values cannot contain spaces or brackets, so the method is folded in with an+ * underscore and a `[param]` placeholder is rendered as `_param_`.*/export function clickHouseQueryOutcomeRouteLabel(route?: string): string {if (!route) return OTHER_ROUTE_LABEL;@@ -93,19 +75,25 @@ export function clickHouseQueryOutcomeRouteLabel(route?: string): string {const collapsed = route.trim();const separatorIndex = collapsed.indexOf(" ");if (separatorIndex === -1) {- return LABELLED_BARE_ROUTES.has(collapsed) ? collapsed : OTHER_ROUTE_LABEL;+ return BARE_ROUTE_PATTERN.test(collapsed) ? collapsed : OTHER_ROUTE_LABEL;}const method = collapsed.slice(0, separatorIndex);const path = collapsed.slice(separatorIndex + 1);const normalizedPath =path.length > 1 && path.endsWith("/") ? path.slice(0, -1) : path;+ if (!/^[A-Z]+$/.test(method) || !normalizedPath.startsWith(REST_PATH_PREFIX))+ return OTHER_ROUTE_LABEL;- if (!LABELLED_ROUTES.has(`${method} ${normalizedPath}`)) {+ const segments = normalizedPath.slice(1).split("/");+ if (+ segments.length > MAX_REST_SEGMENTS ||+ !segments.every((segment) => REST_STATIC_SEGMENT_PATTERN.test(segment))+ ) {return OTHER_ROUTE_LABEL;}- return `${method.toLowerCase()}_${normalizedPath}`;+ return `${method.toLowerCase()}_${normalizedPath.replace(/\[(\w+)\]/g, "_$1_")}`;}/**@@ -287,15 +275,92 @@ function clickHouseQueryMetricTags(};}+/**+ * Transport failures the query path retries. Shared with the retry predicate so+ * an exhausted retry is labelled `network` on the outcome metric.+ */+export const CLICKHOUSE_NETWORK_ERROR_PATTERNS = [+ "socket hang up",+ "broken pipe",+ "connection reset",+ "econnreset",+ "network_error",+ "etimedout",+ "econnrefused",+] as const;++/**+ * ClickHouse error codes that get their own `error_code` label, keyed by the+ * numeric code. Together they cover >99.9% of failed Langfuse reads; the rest is+ * a long tail reported as `other` and recoverable from `system.query_log`.+ * Matching on the number rather than the name keeps the label set closed: the+ * name is parsed from the message text, which can echo caller input.+ */+const LABELLED_CLICKHOUSE_ERROR_CODES: Record<string, string> = {+ "60": "unknown_table",+ "159": "timeout_exceeded",+ "202": "too_many_simultaneous_queries",+ "209": "socket_timeout",+ "210": "network_error",+ "241": "memory_limit_exceeded",+ "734": "cannot_write_after_buffer_canceled",+ "735": "query_was_cancelled_by_client",+};++const OTHER_ERROR_CODE_LABEL = "other";++/**+ * Bounded cause of a failed query, so `error` outcomes are not opaque and a+ * server-side `TIMEOUT_EXCEEDED` can be told from a client request timeout.+ *+ * - A labelled ClickHouse error code (`timeout_exceeded`, `unknown_table`, …),+ * from a thrown `ClickHouseError` or an exception row's `Code: N.` prefix.+ * Any other ClickHouse code is `other`.+ * - `network` for a transport failure that outlived the retries.+ * - `client_timeout` for the client's own request timeout, which carries no+ * ClickHouse code.+ * - `other` for anything else.+ */+export function clickHouseQueryErrorCode(error: unknown): string {+ const root =+ error instanceof Error && error.cause instanceof Error+ ? error.cause+ : error;+ if (!(root instanceof Error)) return OTHER_ERROR_CODE_LABEL;++ const parsed = root instanceof ClickHouseError ? root : parseError(root);+ if (parsed instanceof ClickHouseError) {+ return (+ LABELLED_CLICKHOUSE_ERROR_CODES[parsed.code] ?? OTHER_ERROR_CODE_LABEL+ );+ }++ const message = (root.message ?? "").toLowerCase();+ if (CLICKHOUSE_NETWORK_ERROR_PATTERNS.some((p) => message.includes(p))) {+ return "network";+ }+ if (message.includes("timeout") || message.includes("timed out")) {+ return "client_timeout";+ }+ return OTHER_ERROR_CODE_LABEL;+}++/**+ * `error` is the thrown error for a failed query and omitted on success, where+ * `error_code` is `none`.+ */export function recordClickHouseQueryOutcome(outcome: ClickHouseQueryOutcome,tags: NormalizedClickHouseQueryTags,table: ClickHouseQueryTable,shape: ClickHouseQueryShape,clickhouseService: ClickhouseService,+ error?: unknown,): void {recordIncrement(CLICKHOUSE_QUERY_OUTCOME_METRIC, 1, {outcome,+ error_code:+ outcome === "success" ? "none" : clickHouseQueryErrorCode(error),...clickHouseQueryMetricTags(tags, table, shape, clickhouseService),});}diff --git a/packages/shared/src/server/clickhouse/queryTags.test.ts b/packages/shared/src/server/clickhouse/queryTags.test.tsindex 70da06656d..8933fac5d1 100644--- a/packages/shared/src/server/clickhouse/queryTags.test.ts+++ b/packages/shared/src/server/clickhouse/queryTags.test.ts@@ -4,6 +4,7 @@ import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hoimport {buildClickHouseLogComment,CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS,+ CLICKHOUSE_ROUTE_CONTEXT_KEY,} from "./queryTags";describe("ClickHouse query tags", () => {@@ -11,18 +12,17 @@ describe("ClickHouse query tags", () => {context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable(),);- const entrypoint = propagation.setBaggage(- context.active(),- propagation.createBaggage({- [CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.surface]: { value: "worker" },- [CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.route]: {- value: "langfuse.queue.trace_batch",- },- [CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.projectId]: {- value: "producer-project",- },- }),- );+ const entrypoint = propagation+ .setBaggage(+ context.active(),+ propagation.createBaggage({+ [CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.surface]: { value: "worker" },+ [CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.projectId]: {+ value: "producer-project",+ },+ }),+ )+ .setValue(CLICKHOUSE_ROUTE_CONTEXT_KEY, "langfuse.queue.trace_batch");try {const tags = context.with(entrypoint, () =>JSON.parse(diff --git a/packages/shared/src/server/clickhouse/queryTags.ts b/packages/shared/src/server/clickhouse/queryTags.tsindex 50ebfd0469..0e116709ff 100644--- a/packages/shared/src/server/clickhouse/queryTags.ts+++ b/packages/shared/src/server/clickhouse/queryTags.ts@@ -1,4 +1,4 @@-import { context, propagation } from "@opentelemetry/api";+import { context, createContextKey, propagation } from "@opentelemetry/api";export const CLICKHOUSE_QUERY_TAG_SCHEMA_VERSION = "1" as const;const UNKNOWN_CLICKHOUSE_QUERY_TAG_VALUE = "unknown" as const;@@ -43,6 +43,17 @@ export const CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS = {userAgent: "langfuse.clickhouse.user_agent",} as const;+/**+ * Holds the route the entrypoint set. The route is mirrored into baggage for+ * span attributes but never read back from it: inbound requests can carry+ * `langfuse.clickhouse.*` baggage, and a caller-chosen route would make the+ * outcome metric's `route` tag unbounded. Inbound headers cannot set a context+ * key.+ */+export const CLICKHOUSE_ROUTE_CONTEXT_KEY = createContextKey(+ "langfuse.clickhouse.route",+);+const surfaceSet = new Set<string>(clickHouseQuerySurfaces);const isClickHouseQuerySurface = (@@ -60,7 +71,7 @@ export function normalizeClickHouseQueryTags(baggage?.getEntry(CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.surface)?.value;const route =providedTags?.route ??- baggage?.getEntry(CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.route)?.value;+ context.active().getValue(CLICKHOUSE_ROUTE_CONTEXT_KEY);const projectId =providedTags?.projectId ??baggage?.getEntry(CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.projectId)?.value;diff --git a/packages/shared/src/server/headerPropagation.ts b/packages/shared/src/server/headerPropagation.tsindex ca80a54168..639b1ef463 100644--- a/packages/shared/src/server/headerPropagation.ts+++ b/packages/shared/src/server/headerPropagation.ts@@ -3,6 +3,7 @@ import type { IncomingHttpHeaders } from "http";import { env } from "../env";import {CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS,+ CLICKHOUSE_ROUTE_CONTEXT_KEY,type ClickHouseQuerySurface,} from "./clickhouse/queryTags";import { extractPublicApiCallerAttribution } from "./ingestion/ingestionAttribution";@@ -120,5 +121,9 @@ export const contextWithLangfuseProps = (}}- return opentelemetry.propagation.setBaggage(ctx, baggage);+ const withBaggage = opentelemetry.propagation.setBaggage(ctx, baggage);+ const route = props.clickhouse?.route?.trim();+ return route+ ? withBaggage.setValue(CLICKHOUSE_ROUTE_CONTEXT_KEY, route)+ : withBaggage;};diff --git a/packages/shared/src/server/instrumentation/index.test.ts b/packages/shared/src/server/instrumentation/index.test.tsindex 69019cdb77..5108036153 100644--- a/packages/shared/src/server/instrumentation/index.test.ts+++ b/packages/shared/src/server/instrumentation/index.test.ts@@ -133,6 +133,26 @@ describe("instrumentation baggage propagation", () => {expect(tags).not.toHaveProperty("userAgent");});+ // A route the entrypoint never set must not come from the caller: the route+ // is a tag on the ClickHouse outcome metric.+ it("ignores a ClickHouse route inherited from inbound baggage", () => {+ const inboundContext = propagation.setBaggage(+ ROOT_CONTEXT,+ propagation.createBaggage({+ [CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.surface]: { value: "trpc" },+ [CLICKHOUSE_QUERY_TAG_BAGGAGE_KEYS.route]: {+ value: "attackerChosenRoute",+ },+ }),+ );++ const tags = context.with(inboundContext, () =>+ normalizeClickHouseQueryTags(),+ );++ expect(tags).not.toHaveProperty("route");+ });+it("does not add user emails to span attributes or baggage", () => {const span = {setAttribute: vi.fn(),diff --git a/packages/shared/src/server/instrumentation/index.ts b/packages/shared/src/server/instrumentation/index.tsindex 5617b5c5c1..cae9ee3504 100644--- a/packages/shared/src/server/instrumentation/index.ts+++ b/packages/shared/src/server/instrumentation/index.ts@@ -7,6 +7,7 @@ import * as opentelemetry from "@opentelemetry/api";import * as dd from "dd-trace";import { env } from "../../env";import { logger } from "../logger";+import { CLICKHOUSE_ROUTE_CONTEXT_KEY } from "../clickhouse/queryTags";export type TCarrier = {traceparent?: string;@@ -30,12 +31,7 @@ export async function instrumentAsync<T>(callback: AsyncCallbackFn<T>,): Promise<T> {const activeContext = ctx.startNewTrace- ? // Sever the parent trace but carry baggage onto the new root.- opentelemetry.propagation.setBaggage(- opentelemetry.ROOT_CONTEXT,- opentelemetry.propagation.getBaggage(opentelemetry.context.active()) ??- opentelemetry.propagation.createBaggage(),- )+ ? newTraceRootContext(): ctx.traceContext? opentelemetry.propagation.extract(opentelemetry.context.active(),@@ -72,6 +68,23 @@ export async function instrumentAsync<T>();}+/**+ * Severs the parent trace but carries baggage and the ClickHouse route onto the+ * new root, so queries in the new trace keep their attribution.+ */+const newTraceRootContext = () => {+ const active = opentelemetry.context.active();+ const root = opentelemetry.propagation.setBaggage(+ opentelemetry.ROOT_CONTEXT,+ opentelemetry.propagation.getBaggage(active) ??+ opentelemetry.propagation.createBaggage(),+ );+ const route = active.getValue(CLICKHOUSE_ROUTE_CONTEXT_KEY);+ return route === undefined+ ? root+ : root.setValue(CLICKHOUSE_ROUTE_CONTEXT_KEY, route);+};+type SyncCallbackFn<T> = (span: opentelemetry.Span) => T;/** instrumentSync runs a callback inside a fresh OTel span. */@@ -80,12 +93,7 @@ export function instrumentSync<T>(callback: SyncCallbackFn<T>,): T {const activeContext = ctx.startNewTrace- ? // Sever the parent trace but carry baggage onto the new root.- opentelemetry.propagation.setBaggage(- opentelemetry.ROOT_CONTEXT,- opentelemetry.propagation.getBaggage(opentelemetry.context.active()) ??- opentelemetry.propagation.createBaggage(),- )+ ? newTraceRootContext(): ctx.traceContext? opentelemetry.propagation.extract(opentelemetry.context.active(),diff --git a/packages/shared/src/server/repositories/clickhouse.ts b/packages/shared/src/server/repositories/clickhouse.tsindex 15e5b6e6f2..d1485a3d4f 100644--- a/packages/shared/src/server/repositories/clickhouse.ts+++ b/packages/shared/src/server/repositories/clickhouse.ts@@ -31,6 +31,7 @@ import {type NormalizedClickHouseQueryTags,} from "../clickhouse/queryTags";import {+ CLICKHOUSE_NETWORK_ERROR_PATTERNS,CLICKHOUSE_RESOURCE_ERROR_OUTCOMES,clickHouseQueryShape,clickHouseQueryTableLabel,@@ -688,17 +689,9 @@ function isRetryableError(error: unknown): boolean {const errorMessage = (error as Error).message?.toLowerCase() || "";// Check for socket hang up and other network-related errors- const retryablePatterns = [- "socket hang up",- "broken pipe",- "connection reset",- "econnreset",- "network_error",- "etimedout",- "econnrefused",- ];-- return retryablePatterns.some((pattern) => errorMessage.includes(pattern));+ return CLICKHOUSE_NETWORK_ERROR_PATTERNS.some((pattern) =>+ errorMessage.includes(pattern),+ );}export async function queryClickhouse<T>(@@ -777,6 +770,7 @@ export async function queryClickhouse<T>(table,shape,clickhouseService,+ error,);throw wrapped;});diff --git a/web/src/__tests__/server/withMiddlewares.servertest.ts b/web/src/__tests__/server/withMiddlewares.servertest.tsindex 5e0bd01163..32f31d19a8 100644--- a/web/src/__tests__/server/withMiddlewares.servertest.ts+++ b/web/src/__tests__/server/withMiddlewares.servertest.ts@@ -3,7 +3,11 @@ import {LEGACY_PUBLIC_API_METRICS_CLICKHOUSE_RESOURCE_ERROR_MESSAGE,withMiddlewares,} from "@/src/features/public-api/server/withMiddlewares";-import { clickHouseRouteForRequest } from "@/src/features/public-api/server/clickHouseRequestTags";+import {+ clickHouseRouteForRequest,+ DYNAMIC_API_ROUTE_TEMPLATES,+} from "@/src/features/public-api/server/clickHouseRequestTags";+import { readdirSync, readFileSync } from "fs";import {BaseError,LangfuseNotFoundError,@@ -54,6 +58,49 @@ describe("clickHouseRouteForRequest", () => {).toBe("POST http://[::1");});+ it("reports a dynamic route as its template", () => {+ expect(+ clickHouseRouteForRequest(+ request(+ "GET",+ "/api/public/v2/prompts/team%2Fsupport/versions/3?label=prod",+ ),+ ),+ ).toBe("GET /api/public/v2/prompts/[promptName]/versions/[promptVersion]");+ });++ // A caller-chosen id must not survive into the route: the route tag feeds a+ // metric, so every distinct id would add a series.+ it.each([+ "/api/public/traces/abcdefgh?traceId=x",+ "/api/public/traces/api",+ "/api/public/traces/traces/",+ ])("templates %s regardless of the id's value", (url) => {+ expect(clickHouseRouteForRequest(request("GET", url))).toBe(+ "GET /api/public/traces/[traceId]",+ );+ });++ it("lists every dynamic API route that tags its ClickHouse queries", () => {+ const apiDir = `${__dirname}/../../pages/api`;+ const tagsQueries = (file: string) =>+ file.startsWith("public/") ||+ /\b(withMiddlewares|createAuthedProjectAPIRoute|clickHouseRouteForRequest)\(/.test(+ readFileSync(`${apiDir}/${file}`, "utf8"),+ );+ const dynamicTemplates = readdirSync(apiDir, { recursive: true })+ .map(String)+ .filter((file) => file.endsWith(".ts") && file.includes("["))+ .filter(tagsQueries)+ .map(+ (file) => "/api/" + file.replace(/\.ts$/, "").replace(/\/index$/, ""),+ );++ expect([...DYNAMIC_API_ROUTE_TEMPLATES].sort()).toEqual(+ [...new Set(dynamicTemplates)].sort(),+ );+ });+it("falls back to UNKNOWN method for missing methods", () => {expect(clickHouseRouteForRequest(request(undefined, "/api/public/health")),diff --git a/web/src/features/public-api/server/clickHouseRequestTags.ts b/web/src/features/public-api/server/clickHouseRequestTags.tsindex 65749d353e..479b9e930c 100644--- a/web/src/features/public-api/server/clickHouseRequestTags.ts+++ b/web/src/features/public-api/server/clickHouseRequestTags.ts@@ -1,8 +1,64 @@import { type NextApiRequest } from "next";-const stripSearchAndHash = (url: string) => url.split(/[?#]/, 1)[0] ?? "";+/**+ * Every dynamic API route that tags its ClickHouse queries, as its Next.js page+ * path. A request path matching one of these is reported as the template, so+ * the route names the endpoint rather than the entity and stays bounded.+ * Static routes need no entry. `withMiddlewares.servertest.ts` keeps this+ * list in sync with `web/src/pages/api`.+ */+export const DYNAMIC_API_ROUTE_TEMPLATES = [+ "/api/public/annotation-queues/[queueId]",+ "/api/public/annotation-queues/[queueId]/assignments",+ "/api/public/annotation-queues/[queueId]/items",+ "/api/public/annotation-queues/[queueId]/items/[itemId]",+ "/api/public/comments/[commentId]",+ "/api/public/dataset-items/[datasetItemId]",+ "/api/public/datasets/[name]",+ "/api/public/datasets/[name]/runs",+ "/api/public/datasets/[name]/runs/[runName]",+ "/api/public/integrations/blob-storage/[id]",+ "/api/public/llm-connections/[id]",+ "/api/public/media/[mediaId]",+ "/api/public/models/[modelId]",+ "/api/public/observations/[observationId]",+ "/api/public/projects/[projectId]",+ "/api/public/projects/[projectId]/apiKeys",+ "/api/public/projects/[projectId]/apiKeys/[apiKeyId]",+ "/api/public/projects/[projectId]/memberships",+ "/api/public/scim/Users/[id]",+ "/api/public/score-configs/[configId]",+ "/api/public/scores/[scoreId]",+ "/api/public/sessions/[sessionId]",+ "/api/public/traces/[traceId]",+ "/api/public/unstable/dashboard-widgets/[widgetId]",+ "/api/public/unstable/dashboards/[dashboardId]",+ "/api/public/unstable/dashboards/[dashboardId]/placements",+ "/api/public/unstable/dashboards/[dashboardId]/placements/[placementId]",+ "/api/public/unstable/skills/[skillName]",+ "/api/public/unstable/skills/[skillName]/versions/[skillVersion]",+ "/api/public/v2/datasets/[datasetName]",+ "/api/public/v2/evaluation-rules/[evaluationRuleId]",+ "/api/public/v2/evaluators/[evaluatorId]",+ "/api/public/v2/evaluators/[evaluatorId]/versions",+ "/api/public/v2/prompts/[promptName]",+ "/api/public/v2/prompts/[promptName]/versions/[promptVersion]",+ "/api/public/v2/scores/[scoreId]",+ "/api/traces/[traceId]/download",+ "/api/traces/[traceId]/observations/[observationId]/io/[field]",+] as const;++const TEMPLATE_SEGMENTS = DYNAMIC_API_ROUTE_TEMPLATES.map((template) =>+ template.split("/"),+);export const clickHouseRouteForRequest = (req: NextApiRequest) => {+ const { method, pathname } = requestPathForRequest(req);+ return `${method} ${templatePath(pathname)}`;+};++/** The request method and path without search params or fragment. */+export const requestPathForRequest = (req: NextApiRequest) => {const method = req.method ?? "UNKNOWN";const rawUrl = req.url ?? "";@@ -10,8 +66,35 @@ export const clickHouseRouteForRequest = (req: NextApiRequest) => {const pathname = rawUrl? new URL(rawUrl, "http://langfuse.local").pathname: "";- return `${method} ${pathname}`;+ return { method, pathname };} catch {- return `${method} ${stripSearchAndHash(rawUrl)}`;+ return { method, pathname: stripSearchAndHash(rawUrl) };}};++/**+ * Only dynamic positions are replaced, so a value that equals a fixed segment+ * (`/api/public/traces/api`) keeps the endpoint intact. No two templates match+ * the same path and none overlaps a static route, so the first match is the+ * page Next.js routed to.+ */+const templatePath = (pathname: string) => {+ const normalized =+ pathname.length > 1 && pathname.endsWith("/")+ ? pathname.slice(0, -1)+ : pathname;+ const segments = normalized.split("/");+ const match = TEMPLATE_SEGMENTS.find(+ (template) =>+ template.length === segments.length &&+ template.every(+ (segment, index) =>+ isDynamicSegment(segment) || segment === segments[index],+ ),+ );+ return match ? match.join("/") : pathname;+};++const isDynamicSegment = (segment: string) => segment.startsWith("[");++const stripSearchAndHash = (url: string) => url.split(/[?#]/, 1)[0] ?? "";diff --git a/web/src/features/public-api/server/legacyApiOrganizationCutoff.ts b/web/src/features/public-api/server/legacyApiOrganizationCutoff.tsindex 627ae50b7b..93c245b01f 100644--- a/web/src/features/public-api/server/legacyApiOrganizationCutoff.ts+++ b/web/src/features/public-api/server/legacyApiOrganizationCutoff.ts@@ -8,7 +8,7 @@ import {} from "@langfuse/shared/src/server";import { env } from "@/src/env.mjs";-import { clickHouseRouteForRequest } from "@/src/features/public-api/server/clickHouseRequestTags";+import { requestPathForRequest } from "@/src/features/public-api/server/clickHouseRequestTags";import { attachDeprecation } from "@/src/features/public-api/server/deprecations";const utcDateFormatter = new Intl.DateTimeFormat("en-US", {@@ -41,7 +41,8 @@ export function applyLegacyApiOrganizationCutoff(params: {return null;}- const apiPath = clickHouseRouteForRequest(params.req);+ const { method, pathname } = requestPathForRequest(params.req);+ const apiPath = `${method} ${pathname}`;const callerAttribution = extractPublicApiCallerAttribution(params.req.headers,);diff --git a/worker/src/__tests__/periodicRunner.test.ts b/worker/src/__tests__/periodicRunner.test.tsindex bde6c5319d..1fe649bd6b 100644--- a/worker/src/__tests__/periodicRunner.test.ts+++ b/worker/src/__tests__/periodicRunner.test.ts@@ -1,6 +1,8 @@import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";+import { context as otelContext, type Context } from "@opentelemetry/api";const telemetry = vi.hoisted(() => ({+ contextWithLangfuseProps: vi.fn(),recordDistribution: vi.fn(),recordGauge: vi.fn(),recordIncrement: vi.fn(),@@ -9,6 +11,7 @@ const telemetry = vi.hoisted(() => ({}));vi.mock("@langfuse/shared/src/server", () => ({+ contextWithLangfuseProps: telemetry.contextWithLangfuseProps,getCurrentSpan: vi.fn(() => telemetry.span),instrumentAsync: vi.fn((@@ -32,6 +35,20 @@ vi.mock("@langfuse/shared/src/server", () => ({import { PeriodicRunner } from "../utils/PeriodicRunner";import { PeriodicExclusiveRunner } from "../utils/PeriodicExclusiveRunner";+// Tracks the context `otelContext.with` activates, so a test can tell whether+// `execute()` runs inside it.+const activeContexts: Context[] = [];+vi.spyOn(otelContext, "with").mockImplementation(+ (ctx, fn, thisArg, ...args) => {+ activeContexts.push(ctx);+ try {+ return fn.call(thisArg, ...args);+ } finally {+ activeContexts.pop();+ }+ },+);+/*** Helper to flush microtasks without advancing timers*@@ -69,6 +86,7 @@ class TestRunner extends PeriodicRunner {public shouldThrow = false;public returnInterval: number | undefined = undefined;public initialDelayMsForTest = 0;+ public executeContext: Context | undefined;constructor() {super("test_runner");@@ -88,6 +106,7 @@ class TestRunner extends PeriodicRunner {protected async execute(): Promise<number | void> {this.callCount++;+ this.executeContext = activeContexts.at(-1);if (this.shouldThrow) {this.shouldThrow = false; // Only throw oncethrow new Error("Test error");@@ -154,12 +173,21 @@ describe("PeriodicRunner", () => {});it("should call execute on start and record canonical metrics", async () => {+ const clickHouseContext = {} as Context;+ telemetry.contextWithLangfuseProps.mockReturnValue(clickHouseContext);const runner = new TestRunner();runner.start();await flushMicrotasks();expect(runner.callCount).toBe(1);+ expect(runner.executeContext).toBe(clickHouseContext);+ expect(telemetry.contextWithLangfuseProps).toHaveBeenCalledWith({+ clickhouse: {+ surface: "worker",+ route: "langfuse.periodic_runner.test_runner",+ },+ });expect(telemetry.recordIncrement).toHaveBeenCalledWith("langfuse.periodic_runner.started",1,diff --git a/worker/src/__tests__/workerManagerMetrics.test.ts b/worker/src/__tests__/workerManagerMetrics.test.tsindex 5df2f329f1..d03eb47976 100644--- a/worker/src/__tests__/workerManagerMetrics.test.ts+++ b/worker/src/__tests__/workerManagerMetrics.test.ts@@ -36,6 +36,7 @@ vi.mock("@opentelemetry/api", () => ({context: {with: (_context: unknown, callback: () => unknown) => callback(),},+ createContextKey: (description: string) => Symbol.for(description),}));vi.mock("@langfuse/shared/src/server", async () => {diff --git a/worker/src/features/queue-metrics-runner/index.test.ts b/worker/src/features/queue-metrics-runner/index.test.tsindex 7064bb6eb6..3e6e976b58 100644--- a/worker/src/features/queue-metrics-runner/index.test.ts+++ b/worker/src/features/queue-metrics-runner/index.test.ts@@ -15,6 +15,7 @@ vi.mock("@langfuse/shared/src/server", () => ({QueueName: {TraceDelete: "trace-delete",},+ contextWithLangfuseProps: vi.fn(),convertQueueNameToMetricName: (queueName: string) =>`langfuse.queue.${queueName.replace(/-/g, "_").replace(/_queue$/, "")}`,flushMetricsToCloudWatch: mocks.flushMetricsToCloudWatch,diff --git a/worker/src/utils/PeriodicRunner.ts b/worker/src/utils/PeriodicRunner.tsindex 69cb3737be..0f716091cd 100644--- a/worker/src/utils/PeriodicRunner.ts+++ b/worker/src/utils/PeriodicRunner.ts@@ -1,4 +1,5 @@import {+ contextWithLangfuseProps,instrumentAsync,logger,recordDistribution,@@ -6,7 +7,7 @@ import {recordIncrement,traceException,} from "@langfuse/shared/src/server";-import { SpanKind } from "@opentelemetry/api";+import { context as otelContext, SpanKind } from "@opentelemetry/api";const METRIC_PREFIX = "langfuse.periodic_runner";type RunOutcome = "success" | "failed" | "skipped";@@ -94,7 +95,15 @@ export abstract class PeriodicRunner {span.setAttribute("runner.name", this.name);span.setAttribute("runner.interval_ms", this.defaultIntervalMs);- const result = await this.execute();+ const clickHouseCtx = contextWithLangfuseProps({+ clickhouse: {+ surface: "worker",+ route: `${METRIC_PREFIX}.${this.metricName}`,+ },+ });+ const result = await otelContext.with(clickHouseCtx, () =>+ this.execute(),+ );if (typeof result === "number") {nextDelayMs = result;span.setAttribute("runner.next_delay_ms", nextDelayMs);