🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(shared): tag ClickHouse query outcome metric by table and label events routes (#17073)
Niklas Semmlerd9e1e073a89d
diff --git a/packages/shared/src/server/clickhouse/queryOutcome.test.ts b/packages/shared/src/server/clickhouse/queryOutcome.test.tsindex b4adcfe10a..9506ee160f 100644--- a/packages/shared/src/server/clickhouse/queryOutcome.test.ts+++ b/packages/shared/src/server/clickhouse/queryOutcome.test.ts@@ -4,6 +4,7 @@ import {CLICKHOUSE_QUERY_OUTCOME_METRIC,CLICKHOUSE_RESOURCE_ERROR_OUTCOMES,clickHouseQueryOutcomeRouteLabel,+ clickHouseQueryTableLabel,recordClickHouseQueryOutcome,} from "./queryOutcome";@@ -51,6 +52,74 @@ describe("ClickHouse query outcome metric", () => {expect(clickHouseQueryOutcomeRouteLabel(route)).toBe("other");},);++ // tRPC procedure paths and MCP tool names carry no method prefix and are+ // code-defined, so they are matched verbatim against the bare allowlist.+ it.each([+ ["events.all", "events.all"],+ ["listObservations", "listObservations"],+ ])("labels the bare route %s verbatim", (route, expected) => {+ expect(clickHouseQueryOutcomeRouteLabel(route)).toBe(expected);+ });++ it("counts an unlisted bare route under the catch-all label", () => {+ expect(clickHouseQueryOutcomeRouteLabel("traces.byId")).toBe("other");+ });+ });++ describe("table labels", () => {+ it.each([+ [+ "SELECT * FROM events_full WHERE project_id = {p:String}",+ "events_full",+ ],+ ["SELECT count() FROM events_core FINAL", "events_core"],+ [+ "SELECT * FROM observations o WHERE o.project_id = {p:String}",+ "observations",+ ],+ ["SELECT * FROM traces WHERE id = {id:String}", "traces"],+ ["SELECT * FROM scores WHERE project_id = {p:String}", "scores"],+ ])("labels %s as %s", (query, expected) => {+ expect(clickHouseQueryTableLabel(query)).toBe(expected);+ });++ it("prefers the events tables when several tables appear", () => {+ expect(+ clickHouseQueryTableLabel(+ "SELECT * FROM events_full LEFT JOIN traces USING (id)",+ ),+ ).toBe("events_full");+ });++ // The Scores UI reads FROM scores and LEFT JOINs traces; the label must+ // follow the table under load, not the join partner.+ it("labels a FROM table over its join partner", () => {+ expect(+ clickHouseQueryTableLabel(+ "SELECT * FROM scores s FINAL LEFT JOIN traces t ON s.trace_id = t.id",+ ),+ ).toBe("scores");+ });++ it("matches a schema-qualified FROM table", () => {+ expect(+ clickHouseQueryTableLabel("SELECT * FROM default.events_full"),+ ).toBe("events_full");+ });++ it("does not match a table name inside a longer identifier", () => {+ expect(+ clickHouseQueryTableLabel("SELECT * FROM observations_batch_staging"),+ ).toBe("other");+ expect(clickHouseQueryTableLabel("SELECT * FROM traces_null")).toBe(+ "other",+ );+ });++ it("falls back to other for an unrecognised table", () => {+ expect(clickHouseQueryTableLabel("SELECT 1")).toBe("other");+ });});it("maps every ClickHouse resource error type to an outcome", () => {@@ -62,15 +131,19 @@ describe("ClickHouse query outcome metric", () => {});it("emits the outcome with bounded tags", () => {- recordClickHouseQueryOutcome("timeout", {- tag_schema_version: "1",- surface: "publicapi",- route: "GET /api/public/v2/observations",- projectId: "project-1",- sdkName: "python",- sdkVersion: "3.12.0",- userAgent: "python-httpx/0.28.1",- });+ recordClickHouseQueryOutcome(+ "timeout",+ {+ tag_schema_version: "1",+ surface: "publicapi",+ route: "GET /api/public/v2/observations",+ projectId: "project-1",+ sdkName: "python",+ sdkVersion: "3.12.0",+ userAgent: "python-httpx/0.28.1",+ },+ "events_full",+ );expect(recordIncrement).toHaveBeenCalledTimes(1);expect(recordIncrement).toHaveBeenCalledWith(@@ -80,20 +153,30 @@ describe("ClickHouse query outcome metric", () => {outcome: "timeout",surface: "publicapi",route: "get_/api/public/v2/observations",+ table: "events_full",},);});it("keeps unknown surfaces and unlabelled routes queryable", () => {- recordClickHouseQueryOutcome("success", {- tag_schema_version: "1",- surface: "unknown",- });+ recordClickHouseQueryOutcome(+ "success",+ {+ tag_schema_version: "1",+ surface: "unknown",+ },+ "other",+ );expect(recordIncrement).toHaveBeenCalledWith(CLICKHOUSE_QUERY_OUTCOME_METRIC,1,- { outcome: "success", surface: "unknown", route: "other" },+ {+ outcome: "success",+ surface: "unknown",+ route: "other",+ table: "other",+ },);});});diff --git a/packages/shared/src/server/clickhouse/queryOutcome.ts b/packages/shared/src/server/clickhouse/queryOutcome.tsindex 71d0e6ff19..308f7ec6cf 100644--- a/packages/shared/src/server/clickhouse/queryOutcome.ts+++ b/packages/shared/src/server/clickhouse/queryOutcome.ts@@ -33,8 +33,8 @@ export const CLICKHOUSE_RESOURCE_ERROR_OUTCOMES = {} as const satisfies Record<string, ClickHouseQueryOutcome>;/**- * Routes whose outcomes are reported under their own `route` tag. Everything- * else is counted under `other`.+ * REST routes whose outcomes are reported under their own `route` tag.+ * Everything else is counted under `other`.** `route` on the query tags is derived from the request path, so it carries* caller-controlled segments (trace ids, prompt names) and is unbounded. Metric@@ -47,6 +47,14 @@ const LABELLED_ROUTES = new Set(["GET /api/public/v3/scores",]);+/**+ * Non-REST routes matched verbatim: tRPC procedure paths (`events.all`) and MCP+ * tool names (`listObservations`) are code-defined and already bounded, so they+ * do not need the method/path parsing REST routes get. Add one when it gains an+ * SLO or drives a meaningful share of timeouts.+ */+const LABELLED_BARE_ROUTES = new Set(["events.all", "listObservations"]);+const OTHER_ROUTE_LABEL = "other";/**@@ -60,7 +68,9 @@ export function clickHouseQueryOutcomeRouteLabel(route?: string): string {const collapsed = route.trim();const separatorIndex = collapsed.indexOf(" ");- if (separatorIndex === -1) return OTHER_ROUTE_LABEL;+ if (separatorIndex === -1) {+ return LABELLED_BARE_ROUTES.has(collapsed) ? collapsed : OTHER_ROUTE_LABEL;+ }const method = collapsed.slice(0, separatorIndex);const path = collapsed.slice(separatorIndex + 1);@@ -74,13 +84,55 @@ export function clickHouseQueryOutcomeRouteLabel(route?: string): string {return `${method.toLowerCase()}_${normalizedPath}`;}+/**+ * Bounded table label so timeouts can be isolated to the table under load — the+ * events read path (`events_full`/`events_core`) is the current driver, and the+ * counter is blind to it without this tag.+ */+export type ClickHouseQueryTable =+ | "events_full"+ | "events_core"+ | "traces"+ | "observations"+ | "scores"+ | "other";++/**+ * Derived from the query text because threading a table through every call site+ * is impractical. Anchored to the `FROM` table (with an optional `db.` prefix)+ * so a lightweight join partner never outranks the table under load — the Scores+ * UI reads `FROM scores ... LEFT JOIN traces`, and a bare `traces` scan would+ * otherwise steal the label. Still best-effort: a subquery/CTE reading a+ * different table in its own `FROM` can win by priority order (first match+ * wins). The events read path is single-table (`FROM events_full`/`events_core`,+ * no v3 joins), so those labels are reliable.+ */+const TABLE_LABEL_PATTERNS: ReadonlyArray<[ClickHouseQueryTable, RegExp]> = [+ ["events_full", /\bfrom\s+(?:\w+\.)?events_full\b/i],+ ["events_core", /\bfrom\s+(?:\w+\.)?events_core\b/i],+ ["observations", /\bfrom\s+(?:\w+\.)?observations\b/i],+ ["traces", /\bfrom\s+(?:\w+\.)?traces\b/i],+ ["scores", /\bfrom\s+(?:\w+\.)?scores\b/i],+];++const OTHER_TABLE_LABEL = "other" as const;++export function clickHouseQueryTableLabel(query: string): ClickHouseQueryTable {+ for (const [label, pattern] of TABLE_LABEL_PATTERNS) {+ if (pattern.test(query)) return label;+ }+ return OTHER_TABLE_LABEL;+}+export function recordClickHouseQueryOutcome(outcome: ClickHouseQueryOutcome,tags: NormalizedClickHouseQueryTags,+ table: ClickHouseQueryTable,): void {recordIncrement(CLICKHOUSE_QUERY_OUTCOME_METRIC, 1, {outcome,surface: tags.surface,route: clickHouseQueryOutcomeRouteLabel(tags.route),+ table,});}diff --git a/packages/shared/src/server/repositories/clickhouse.ts b/packages/shared/src/server/repositories/clickhouse.tsindex adbe2d2265..97743be895 100644--- a/packages/shared/src/server/repositories/clickhouse.ts+++ b/packages/shared/src/server/repositories/clickhouse.ts@@ -33,6 +33,7 @@ import {} from "../clickhouse/queryTags";import {CLICKHOUSE_RESOURCE_ERROR_OUTCOMES,+ clickHouseQueryTableLabel,recordClickHouseQueryOutcome,} from "../clickhouse/queryOutcome";@@ -684,6 +685,7 @@ export async function queryClickhouse<T>(): Promise<T[]> {if (!opts.allowLegacyEventsRead) assertNoLegacyEventsRead(opts.query);const normalizedTags = normalizeClickHouseQueryTags(opts.tags);+ const table = clickHouseQueryTableLabel(opts.query);return await instrumentAsync({ name: "clickhouse-query", spanKind: SpanKind.CLIENT },async (span) => {@@ -745,11 +747,12 @@ export async function queryClickhouse<T>(? CLICKHOUSE_RESOURCE_ERROR_OUTCOMES[wrapped.errorType]: "error",normalizedTags,+ table,);throw wrapped;});- recordClickHouseQueryOutcome("success", normalizedTags);+ recordClickHouseQueryOutcome("success", normalizedTags, table);return rows;},);