🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(shared): count ClickHouse query outcomes for read SLIs (#16777)
Steffen Schmitzcd5ad52dfa7e
diff --git a/packages/shared/src/server/clickhouse/queryOutcome.test.ts b/packages/shared/src/server/clickhouse/queryOutcome.test.tsnew file mode 100644index 0000000000..b4adcfe10a--- /dev/null+++ b/packages/shared/src/server/clickhouse/queryOutcome.test.ts@@ -0,0 +1,99 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++import {+ CLICKHOUSE_QUERY_OUTCOME_METRIC,+ CLICKHOUSE_RESOURCE_ERROR_OUTCOMES,+ clickHouseQueryOutcomeRouteLabel,+ recordClickHouseQueryOutcome,+} from "./queryOutcome";++const recordIncrement = vi.fn();++vi.mock("../instrumentation", () => ({+ recordIncrement: (...args: unknown[]) => recordIncrement(...args),+}));++describe("ClickHouse query outcome metric", () => {+ beforeEach(() => {+ recordIncrement.mockClear();+ });++ describe("route labels", () => {+ it.each([+ ["GET /api/public/v2/observations", "get_/api/public/v2/observations"],+ ["GET /api/public/v2/metrics", "get_/api/public/v2/metrics"],+ ["GET /api/public/v3/scores", "get_/api/public/v3/scores"],+ ])("labels %s as the APM resource name", (route, expected) => {+ expect(clickHouseQueryOutcomeRouteLabel(route)).toBe(expected);+ });++ it("normalizes a trailing slash before matching", () => {+ expect(+ clickHouseQueryOutcomeRouteLabel("GET /api/public/v3/scores/"),+ ).toBe("get_/api/public/v3/scores");+ });++ // The route tag is derived from the request path, so caller-controlled+ // segments would otherwise become unbounded metric tag values.+ it.each([+ "GET /api/public/traces/123e4567-e89b-12d3-a456-426614174000",+ "GET /api/public/v2/prompts/dashboard.status_headline.low.de",+ "GET /api/public/v2/observations/some-observation-id",+ "HEAD /api/public/v2/observations",+ "POST /api/public/v2/metrics",+ ])("counts %s under the catch-all label", (route) => {+ expect(clickHouseQueryOutcomeRouteLabel(route)).toBe("other");+ });++ it.each([undefined, "", " ", "no-method-separator"])(+ "falls back to the catch-all label for %s",+ (route) => {+ expect(clickHouseQueryOutcomeRouteLabel(route)).toBe("other");+ },+ );+ });++ it("maps every ClickHouse resource error type to an outcome", () => {+ expect(CLICKHOUSE_RESOURCE_ERROR_OUTCOMES).toEqual({+ TIMEOUT: "timeout",+ MEMORY_LIMIT: "memory_limit",+ OVERCOMMIT: "overcommit",+ });+ });++ it("emits the outcome with bounded tags", () => {+ recordClickHouseQueryOutcome("timeout", {+ tag_schema_version: "1",+ surface: "publicapi",+ route: "GET /api/public/v2/observations",+ projectId: "project-1",+ sdkName: "python",+ sdkVersion: "3.12.0",+ userAgent: "python-httpx/0.28.1",+ });++ expect(recordIncrement).toHaveBeenCalledTimes(1);+ expect(recordIncrement).toHaveBeenCalledWith(+ CLICKHOUSE_QUERY_OUTCOME_METRIC,+ 1,+ {+ outcome: "timeout",+ surface: "publicapi",+ route: "get_/api/public/v2/observations",+ },+ );+ });++ it("keeps unknown surfaces and unlabelled routes queryable", () => {+ recordClickHouseQueryOutcome("success", {+ tag_schema_version: "1",+ surface: "unknown",+ });++ expect(recordIncrement).toHaveBeenCalledWith(+ CLICKHOUSE_QUERY_OUTCOME_METRIC,+ 1,+ { outcome: "success", surface: "unknown", route: "other" },+ );+ });+});diff --git a/packages/shared/src/server/clickhouse/queryOutcome.ts b/packages/shared/src/server/clickhouse/queryOutcome.tsnew file mode 100644index 0000000000..71d0e6ff19--- /dev/null+++ b/packages/shared/src/server/clickhouse/queryOutcome.ts@@ -0,0 +1,86 @@+import { recordIncrement } from "../instrumentation";+import { type NormalizedClickHouseQueryTags } from "./queryTags";++/**+ * Terminal outcome of one logical ClickHouse query, counted once per query+ * after retries are exhausted.+ *+ * Resource limits surface to public API callers as HTTP 422 and to tRPC+ * callers as UNPROCESSABLE_CONTENT, so status-code-derived signals cannot+ * distinguish them from a caller's malformed request. This metric is the+ * signal that can: it is emitted where the resource error is classified.+ */+export const CLICKHOUSE_QUERY_OUTCOME_METRIC =+ "langfuse.clickhouse.query.outcome";++export type ClickHouseQueryOutcome =+ | "success"+ | "timeout"+ | "memory_limit"+ | "overcommit"+ | "error";++/**+ * Maps `ClickHouseResourceError.errorType` onto outcomes. Keyed by the error+ * type literals rather than importing the type, so this module keeps no+ * dependency on the query paths that call it. Indexing it with `errorType`+ * makes the compiler reject a new error type that is not mapped here.+ */+export const CLICKHOUSE_RESOURCE_ERROR_OUTCOMES = {+ TIMEOUT: "timeout",+ MEMORY_LIMIT: "memory_limit",+ OVERCOMMIT: "overcommit",+} as const satisfies Record<string, ClickHouseQueryOutcome>;++/**+ * Routes whose outcomes are reported under their own `route` tag. Everything+ * else is counted under `other`.+ *+ * `route` on the query tags is derived from the request path, so it carries+ * caller-controlled segments (trace ids, prompt names) and is unbounded. Metric+ * tags must be bounded, so only explicitly listed routes get a label. Add a+ * route here when it gains an SLO.+ */+const LABELLED_ROUTES = new Set([+ "GET /api/public/v2/observations",+ "GET /api/public/v2/metrics",+ "GET /api/public/v3/scores",+]);++const OTHER_ROUTE_LABEL = "other";++/**+ * Renders a route as a Datadog tag value, matching the `resource_name`+ * convention APM uses for the same route (`get_/api/public/v2/metrics`), so+ * SLO and dashboard queries can filter both signals the same way. Datadog tag+ * values cannot contain spaces, so the method is folded in with an underscore.+ */+export function clickHouseQueryOutcomeRouteLabel(route?: string): string {+ if (!route) return OTHER_ROUTE_LABEL;++ const collapsed = route.trim();+ const separatorIndex = collapsed.indexOf(" ");+ if (separatorIndex === -1) return OTHER_ROUTE_LABEL;++ const method = collapsed.slice(0, separatorIndex);+ const path = collapsed.slice(separatorIndex + 1);+ const normalizedPath =+ path.length > 1 && path.endsWith("/") ? path.slice(0, -1) : path;++ if (!LABELLED_ROUTES.has(`${method} ${normalizedPath}`)) {+ return OTHER_ROUTE_LABEL;+ }++ return `${method.toLowerCase()}_${normalizedPath}`;+}++export function recordClickHouseQueryOutcome(+ outcome: ClickHouseQueryOutcome,+ tags: NormalizedClickHouseQueryTags,+): void {+ recordIncrement(CLICKHOUSE_QUERY_OUTCOME_METRIC, 1, {+ outcome,+ surface: tags.surface,+ route: clickHouseQueryOutcomeRouteLabel(tags.route),+ });+}diff --git a/packages/shared/src/server/repositories/clickhouse.ts b/packages/shared/src/server/repositories/clickhouse.tsindex 55314bbf18..adbe2d2265 100644--- a/packages/shared/src/server/repositories/clickhouse.ts+++ b/packages/shared/src/server/repositories/clickhouse.ts@@ -31,6 +31,10 @@ import {type ClickHouseQueryTags,type NormalizedClickHouseQueryTags,} from "../clickhouse/queryTags";+import {+ CLICKHOUSE_RESOURCE_ERROR_OUTCOMES,+ recordClickHouseQueryOutcome,+} from "../clickhouse/queryOutcome";/*** Re-exported so callers can build `Array(Tuple(...))` query parameters without@@ -685,7 +689,7 @@ export async function queryClickhouse<T>(async (span) => {setSpanQueryAttributes(span, opts.query);- return await backOff(+ const rows = await backOff(async () => {const res = await sendClickhouseQuery({...opts,@@ -732,11 +736,21 @@ export async function queryClickhouse<T>(maxDelay: 100,},).catch((error) => {- throw ClickHouseResourceError.wrapIfResourceError(+ const wrapped = ClickHouseResourceError.wrapIfResourceError(error as Error,normalizedTags,);+ recordClickHouseQueryOutcome(+ wrapped instanceof ClickHouseResourceError+ ? CLICKHOUSE_RESOURCE_ERROR_OUTCOMES[wrapped.errorType]+ : "error",+ normalizedTags,+ );+ throw wrapped;});++ recordClickHouseQueryOutcome("success", normalizedTags);+ return rows;},);}