Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(shared): tag ClickHouse query outcome metric by table and label events routes (#17073)

Niklas Semmlerd9e1e073a89d
diff --git a/packages/shared/src/server/clickhouse/queryOutcome.test.ts b/packages/shared/src/server/clickhouse/queryOutcome.test.ts
index b4adcfe10a..9506ee160f 100644
--- a/packages/shared/src/server/clickhouse/queryOutcome.test.ts
+++ b/packages/shared/src/server/clickhouse/queryOutcome.test.ts
@@ -4,6 +4,7 @@ import {
CLICKHOUSE_QUERY_OUTCOME_METRIC,
CLICKHOUSE_RESOURCE_ERROR_OUTCOMES,
clickHouseQueryOutcomeRouteLabel,
+ clickHouseQueryTableLabel,
recordClickHouseQueryOutcome,
} from "./queryOutcome";
@@ -51,6 +52,74 @@ describe("ClickHouse query outcome metric", () => {
expect(clickHouseQueryOutcomeRouteLabel(route)).toBe("other");
},
);
+
+ // tRPC procedure paths and MCP tool names carry no method prefix and are
+ // code-defined, so they are matched verbatim against the bare allowlist.
+ it.each([
+ ["events.all", "events.all"],
+ ["listObservations", "listObservations"],
+ ])("labels the bare route %s verbatim", (route, expected) => {
+ expect(clickHouseQueryOutcomeRouteLabel(route)).toBe(expected);
+ });
+
+ it("counts an unlisted bare route under the catch-all label", () => {
+ expect(clickHouseQueryOutcomeRouteLabel("traces.byId")).toBe("other");
+ });
+ });
+
+ describe("table labels", () => {
+ it.each([
+ [
+ "SELECT * FROM events_full WHERE project_id = {p:String}",
+ "events_full",
+ ],
+ ["SELECT count() FROM events_core FINAL", "events_core"],
+ [
+ "SELECT * FROM observations o WHERE o.project_id = {p:String}",
+ "observations",
+ ],
+ ["SELECT * FROM traces WHERE id = {id:String}", "traces"],
+ ["SELECT * FROM scores WHERE project_id = {p:String}", "scores"],
+ ])("labels %s as %s", (query, expected) => {
+ expect(clickHouseQueryTableLabel(query)).toBe(expected);
+ });
+
+ it("prefers the events tables when several tables appear", () => {
+ expect(
+ clickHouseQueryTableLabel(
+ "SELECT * FROM events_full LEFT JOIN traces USING (id)",
+ ),
+ ).toBe("events_full");
+ });
+
+ // The Scores UI reads FROM scores and LEFT JOINs traces; the label must
+ // follow the table under load, not the join partner.
+ it("labels a FROM table over its join partner", () => {
+ expect(
+ clickHouseQueryTableLabel(
+ "SELECT * FROM scores s FINAL LEFT JOIN traces t ON s.trace_id = t.id",
+ ),
+ ).toBe("scores");
+ });
+
+ it("matches a schema-qualified FROM table", () => {
+ expect(
+ clickHouseQueryTableLabel("SELECT * FROM default.events_full"),
+ ).toBe("events_full");
+ });
+
+ it("does not match a table name inside a longer identifier", () => {
+ expect(
+ clickHouseQueryTableLabel("SELECT * FROM observations_batch_staging"),
+ ).toBe("other");
+ expect(clickHouseQueryTableLabel("SELECT * FROM traces_null")).toBe(
+ "other",
+ );
+ });
+
+ it("falls back to other for an unrecognised table", () => {
+ expect(clickHouseQueryTableLabel("SELECT 1")).toBe("other");
+ });
});
it("maps every ClickHouse resource error type to an outcome", () => {
@@ -62,15 +131,19 @@ describe("ClickHouse query outcome metric", () => {
});
it("emits the outcome with bounded tags", () => {
- recordClickHouseQueryOutcome("timeout", {
- tag_schema_version: "1",
- surface: "publicapi",
- route: "GET /api/public/v2/observations",
- projectId: "project-1",
- sdkName: "python",
- sdkVersion: "3.12.0",
- userAgent: "python-httpx/0.28.1",
- });
+ recordClickHouseQueryOutcome(
+ "timeout",
+ {
+ tag_schema_version: "1",
+ surface: "publicapi",
+ route: "GET /api/public/v2/observations",
+ projectId: "project-1",
+ sdkName: "python",
+ sdkVersion: "3.12.0",
+ userAgent: "python-httpx/0.28.1",
+ },
+ "events_full",
+ );
expect(recordIncrement).toHaveBeenCalledTimes(1);
expect(recordIncrement).toHaveBeenCalledWith(
@@ -80,20 +153,30 @@ describe("ClickHouse query outcome metric", () => {
outcome: "timeout",
surface: "publicapi",
route: "get_/api/public/v2/observations",
+ table: "events_full",
},
);
});
it("keeps unknown surfaces and unlabelled routes queryable", () => {
- recordClickHouseQueryOutcome("success", {
- tag_schema_version: "1",
- surface: "unknown",
- });
+ recordClickHouseQueryOutcome(
+ "success",
+ {
+ tag_schema_version: "1",
+ surface: "unknown",
+ },
+ "other",
+ );
expect(recordIncrement).toHaveBeenCalledWith(
CLICKHOUSE_QUERY_OUTCOME_METRIC,
1,
- { outcome: "success", surface: "unknown", route: "other" },
+ {
+ outcome: "success",
+ surface: "unknown",
+ route: "other",
+ table: "other",
+ },
);
});
});
diff --git a/packages/shared/src/server/clickhouse/queryOutcome.ts b/packages/shared/src/server/clickhouse/queryOutcome.ts
index 71d0e6ff19..308f7ec6cf 100644
--- a/packages/shared/src/server/clickhouse/queryOutcome.ts
+++ b/packages/shared/src/server/clickhouse/queryOutcome.ts
@@ -33,8 +33,8 @@ export const CLICKHOUSE_RESOURCE_ERROR_OUTCOMES = {
} as const satisfies Record<string, ClickHouseQueryOutcome>;
/**
- * Routes whose outcomes are reported under their own `route` tag. Everything
- * else is counted under `other`.
+ * REST routes whose outcomes are reported under their own `route` tag.
+ * Everything else is counted under `other`.
*
* `route` on the query tags is derived from the request path, so it carries
* caller-controlled segments (trace ids, prompt names) and is unbounded. Metric
@@ -47,6 +47,14 @@ const LABELLED_ROUTES = new Set([
"GET /api/public/v3/scores",
]);
+/**
+ * Non-REST routes matched verbatim: tRPC procedure paths (`events.all`) and MCP
+ * tool names (`listObservations`) are code-defined and already bounded, so they
+ * do not need the method/path parsing REST routes get. Add one when it gains an
+ * SLO or drives a meaningful share of timeouts.
+ */
+const LABELLED_BARE_ROUTES = new Set(["events.all", "listObservations"]);
+
const OTHER_ROUTE_LABEL = "other";
/**
@@ -60,7 +68,9 @@ export function clickHouseQueryOutcomeRouteLabel(route?: string): string {
const collapsed = route.trim();
const separatorIndex = collapsed.indexOf(" ");
- if (separatorIndex === -1) return OTHER_ROUTE_LABEL;
+ if (separatorIndex === -1) {
+ return LABELLED_BARE_ROUTES.has(collapsed) ? collapsed : OTHER_ROUTE_LABEL;
+ }
const method = collapsed.slice(0, separatorIndex);
const path = collapsed.slice(separatorIndex + 1);
@@ -74,13 +84,55 @@ export function clickHouseQueryOutcomeRouteLabel(route?: string): string {
return `${method.toLowerCase()}_${normalizedPath}`;
}
+/**
+ * Bounded table label so timeouts can be isolated to the table under load — the
+ * events read path (`events_full`/`events_core`) is the current driver, and the
+ * counter is blind to it without this tag.
+ */
+export type ClickHouseQueryTable =
+ | "events_full"
+ | "events_core"
+ | "traces"
+ | "observations"
+ | "scores"
+ | "other";
+
+/**
+ * Derived from the query text because threading a table through every call site
+ * is impractical. Anchored to the `FROM` table (with an optional `db.` prefix)
+ * so a lightweight join partner never outranks the table under load — the Scores
+ * UI reads `FROM scores ... LEFT JOIN traces`, and a bare `traces` scan would
+ * otherwise steal the label. Still best-effort: a subquery/CTE reading a
+ * different table in its own `FROM` can win by priority order (first match
+ * wins). The events read path is single-table (`FROM events_full`/`events_core`,
+ * no v3 joins), so those labels are reliable.
+ */
+const TABLE_LABEL_PATTERNS: ReadonlyArray<[ClickHouseQueryTable, RegExp]> = [
+ ["events_full", /\bfrom\s+(?:\w+\.)?events_full\b/i],
+ ["events_core", /\bfrom\s+(?:\w+\.)?events_core\b/i],
+ ["observations", /\bfrom\s+(?:\w+\.)?observations\b/i],
+ ["traces", /\bfrom\s+(?:\w+\.)?traces\b/i],
+ ["scores", /\bfrom\s+(?:\w+\.)?scores\b/i],
+];
+
+const OTHER_TABLE_LABEL = "other" as const;
+
+export function clickHouseQueryTableLabel(query: string): ClickHouseQueryTable {
+ for (const [label, pattern] of TABLE_LABEL_PATTERNS) {
+ if (pattern.test(query)) return label;
+ }
+ return OTHER_TABLE_LABEL;
+}
+
export function recordClickHouseQueryOutcome(
outcome: ClickHouseQueryOutcome,
tags: NormalizedClickHouseQueryTags,
+ table: ClickHouseQueryTable,
): void {
recordIncrement(CLICKHOUSE_QUERY_OUTCOME_METRIC, 1, {
outcome,
surface: tags.surface,
route: clickHouseQueryOutcomeRouteLabel(tags.route),
+ table,
});
}
diff --git a/packages/shared/src/server/repositories/clickhouse.ts b/packages/shared/src/server/repositories/clickhouse.ts
index adbe2d2265..97743be895 100644
--- a/packages/shared/src/server/repositories/clickhouse.ts
+++ b/packages/shared/src/server/repositories/clickhouse.ts
@@ -33,6 +33,7 @@ import {
} from "../clickhouse/queryTags";
import {
CLICKHOUSE_RESOURCE_ERROR_OUTCOMES,
+ clickHouseQueryTableLabel,
recordClickHouseQueryOutcome,
} from "../clickhouse/queryOutcome";
@@ -684,6 +685,7 @@ export async function queryClickhouse<T>(
): Promise<T[]> {
if (!opts.allowLegacyEventsRead) assertNoLegacyEventsRead(opts.query);
const normalizedTags = normalizeClickHouseQueryTags(opts.tags);
+ const table = clickHouseQueryTableLabel(opts.query);
return await instrumentAsync(
{ name: "clickhouse-query", spanKind: SpanKind.CLIENT },
async (span) => {
@@ -745,11 +747,12 @@ export async function queryClickhouse<T>(
? CLICKHOUSE_RESOURCE_ERROR_OUTCOMES[wrapped.errorType]
: "error",
normalizedTags,
+ table,
);
throw wrapped;
});
- recordClickHouseQueryOutcome("success", normalizedTags);
+ recordClickHouseQueryOutcome("success", normalizedTags, table);
return rows;
},
);