Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(shared): count ClickHouse query outcomes for read SLIs (#16777)

Steffen Schmitzcd5ad52dfa7e
diff --git a/packages/shared/src/server/clickhouse/queryOutcome.test.ts b/packages/shared/src/server/clickhouse/queryOutcome.test.ts
new file mode 100644
index 0000000000..b4adcfe10a
--- /dev/null
+++ b/packages/shared/src/server/clickhouse/queryOutcome.test.ts
@@ -0,0 +1,99 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import {
+ CLICKHOUSE_QUERY_OUTCOME_METRIC,
+ CLICKHOUSE_RESOURCE_ERROR_OUTCOMES,
+ clickHouseQueryOutcomeRouteLabel,
+ recordClickHouseQueryOutcome,
+} from "./queryOutcome";
+
+const recordIncrement = vi.fn();
+
+vi.mock("../instrumentation", () => ({
+ recordIncrement: (...args: unknown[]) => recordIncrement(...args),
+}));
+
+describe("ClickHouse query outcome metric", () => {
+ beforeEach(() => {
+ recordIncrement.mockClear();
+ });
+
+ describe("route labels", () => {
+ it.each([
+ ["GET /api/public/v2/observations", "get_/api/public/v2/observations"],
+ ["GET /api/public/v2/metrics", "get_/api/public/v2/metrics"],
+ ["GET /api/public/v3/scores", "get_/api/public/v3/scores"],
+ ])("labels %s as the APM resource name", (route, expected) => {
+ expect(clickHouseQueryOutcomeRouteLabel(route)).toBe(expected);
+ });
+
+ it("normalizes a trailing slash before matching", () => {
+ expect(
+ clickHouseQueryOutcomeRouteLabel("GET /api/public/v3/scores/"),
+ ).toBe("get_/api/public/v3/scores");
+ });
+
+ // The route tag is derived from the request path, so caller-controlled
+ // segments would otherwise become unbounded metric tag values.
+ it.each([
+ "GET /api/public/traces/123e4567-e89b-12d3-a456-426614174000",
+ "GET /api/public/v2/prompts/dashboard.status_headline.low.de",
+ "GET /api/public/v2/observations/some-observation-id",
+ "HEAD /api/public/v2/observations",
+ "POST /api/public/v2/metrics",
+ ])("counts %s under the catch-all label", (route) => {
+ expect(clickHouseQueryOutcomeRouteLabel(route)).toBe("other");
+ });
+
+ it.each([undefined, "", " ", "no-method-separator"])(
+ "falls back to the catch-all label for %s",
+ (route) => {
+ expect(clickHouseQueryOutcomeRouteLabel(route)).toBe("other");
+ },
+ );
+ });
+
+ it("maps every ClickHouse resource error type to an outcome", () => {
+ expect(CLICKHOUSE_RESOURCE_ERROR_OUTCOMES).toEqual({
+ TIMEOUT: "timeout",
+ MEMORY_LIMIT: "memory_limit",
+ OVERCOMMIT: "overcommit",
+ });
+ });
+
+ it("emits the outcome with bounded tags", () => {
+ recordClickHouseQueryOutcome("timeout", {
+ tag_schema_version: "1",
+ surface: "publicapi",
+ route: "GET /api/public/v2/observations",
+ projectId: "project-1",
+ sdkName: "python",
+ sdkVersion: "3.12.0",
+ userAgent: "python-httpx/0.28.1",
+ });
+
+ expect(recordIncrement).toHaveBeenCalledTimes(1);
+ expect(recordIncrement).toHaveBeenCalledWith(
+ CLICKHOUSE_QUERY_OUTCOME_METRIC,
+ 1,
+ {
+ outcome: "timeout",
+ surface: "publicapi",
+ route: "get_/api/public/v2/observations",
+ },
+ );
+ });
+
+ it("keeps unknown surfaces and unlabelled routes queryable", () => {
+ recordClickHouseQueryOutcome("success", {
+ tag_schema_version: "1",
+ surface: "unknown",
+ });
+
+ expect(recordIncrement).toHaveBeenCalledWith(
+ CLICKHOUSE_QUERY_OUTCOME_METRIC,
+ 1,
+ { outcome: "success", surface: "unknown", route: "other" },
+ );
+ });
+});
diff --git a/packages/shared/src/server/clickhouse/queryOutcome.ts b/packages/shared/src/server/clickhouse/queryOutcome.ts
new file mode 100644
index 0000000000..71d0e6ff19
--- /dev/null
+++ b/packages/shared/src/server/clickhouse/queryOutcome.ts
@@ -0,0 +1,86 @@
+import { recordIncrement } from "../instrumentation";
+import { type NormalizedClickHouseQueryTags } from "./queryTags";
+
+/**
+ * Terminal outcome of one logical ClickHouse query, counted once per query
+ * after retries are exhausted.
+ *
+ * Resource limits surface to public API callers as HTTP 422 and to tRPC
+ * callers as UNPROCESSABLE_CONTENT, so status-code-derived signals cannot
+ * distinguish them from a caller's malformed request. This metric is the
+ * signal that can: it is emitted where the resource error is classified.
+ */
+export const CLICKHOUSE_QUERY_OUTCOME_METRIC =
+ "langfuse.clickhouse.query.outcome";
+
+export type ClickHouseQueryOutcome =
+ | "success"
+ | "timeout"
+ | "memory_limit"
+ | "overcommit"
+ | "error";
+
+/**
+ * Maps `ClickHouseResourceError.errorType` onto outcomes. Keyed by the error
+ * type literals rather than importing the type, so this module keeps no
+ * dependency on the query paths that call it. Indexing it with `errorType`
+ * makes the compiler reject a new error type that is not mapped here.
+ */
+export const CLICKHOUSE_RESOURCE_ERROR_OUTCOMES = {
+ TIMEOUT: "timeout",
+ MEMORY_LIMIT: "memory_limit",
+ OVERCOMMIT: "overcommit",
+} as const satisfies Record<string, ClickHouseQueryOutcome>;
+
+/**
+ * Routes whose outcomes are reported under their own `route` tag. Everything
+ * else is counted under `other`.
+ *
+ * `route` on the query tags is derived from the request path, so it carries
+ * caller-controlled segments (trace ids, prompt names) and is unbounded. Metric
+ * tags must be bounded, so only explicitly listed routes get a label. Add a
+ * route here when it gains an SLO.
+ */
+const LABELLED_ROUTES = new Set([
+ "GET /api/public/v2/observations",
+ "GET /api/public/v2/metrics",
+ "GET /api/public/v3/scores",
+]);
+
+const OTHER_ROUTE_LABEL = "other";
+
+/**
+ * Renders a route as a Datadog tag value, matching the `resource_name`
+ * convention APM uses for the same route (`get_/api/public/v2/metrics`), so
+ * SLO and dashboard queries can filter both signals the same way. Datadog tag
+ * values cannot contain spaces, so the method is folded in with an underscore.
+ */
+export function clickHouseQueryOutcomeRouteLabel(route?: string): string {
+ if (!route) return OTHER_ROUTE_LABEL;
+
+ const collapsed = route.trim();
+ const separatorIndex = collapsed.indexOf(" ");
+ if (separatorIndex === -1) return OTHER_ROUTE_LABEL;
+
+ const method = collapsed.slice(0, separatorIndex);
+ const path = collapsed.slice(separatorIndex + 1);
+ const normalizedPath =
+ path.length > 1 && path.endsWith("/") ? path.slice(0, -1) : path;
+
+ if (!LABELLED_ROUTES.has(`${method} ${normalizedPath}`)) {
+ return OTHER_ROUTE_LABEL;
+ }
+
+ return `${method.toLowerCase()}_${normalizedPath}`;
+}
+
+export function recordClickHouseQueryOutcome(
+ outcome: ClickHouseQueryOutcome,
+ tags: NormalizedClickHouseQueryTags,
+): void {
+ recordIncrement(CLICKHOUSE_QUERY_OUTCOME_METRIC, 1, {
+ outcome,
+ surface: tags.surface,
+ route: clickHouseQueryOutcomeRouteLabel(tags.route),
+ });
+}
diff --git a/packages/shared/src/server/repositories/clickhouse.ts b/packages/shared/src/server/repositories/clickhouse.ts
index 55314bbf18..adbe2d2265 100644
--- a/packages/shared/src/server/repositories/clickhouse.ts
+++ b/packages/shared/src/server/repositories/clickhouse.ts
@@ -31,6 +31,10 @@ import {
type ClickHouseQueryTags,
type NormalizedClickHouseQueryTags,
} from "../clickhouse/queryTags";
+import {
+ CLICKHOUSE_RESOURCE_ERROR_OUTCOMES,
+ recordClickHouseQueryOutcome,
+} from "../clickhouse/queryOutcome";
/**
* Re-exported so callers can build `Array(Tuple(...))` query parameters without
@@ -685,7 +689,7 @@ export async function queryClickhouse<T>(
async (span) => {
setSpanQueryAttributes(span, opts.query);
- return await backOff(
+ const rows = await backOff(
async () => {
const res = await sendClickhouseQuery({
...opts,
@@ -732,11 +736,21 @@ export async function queryClickhouse<T>(
maxDelay: 100,
},
).catch((error) => {
- throw ClickHouseResourceError.wrapIfResourceError(
+ const wrapped = ClickHouseResourceError.wrapIfResourceError(
error as Error,
normalizedTags,
);
+ recordClickHouseQueryOutcome(
+ wrapped instanceof ClickHouseResourceError
+ ? CLICKHOUSE_RESOURCE_ERROR_OUTCOMES[wrapped.errorType]
+ : "error",
+ normalizedTags,
+ );
+ throw wrapped;
});
+
+ recordClickHouseQueryOutcome("success", normalizedTags);
+ return rows;
},
);
}