Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(clickhouse): emit unsampled per-query cost distributions (#18486)

Niklas Semmler36928100c0db
diff --git a/packages/shared/src/server/clickhouse/queryOutcome.test.ts b/packages/shared/src/server/clickhouse/queryOutcome.test.ts
index 4b94486ff1..fa6ebad9aa 100644
--- a/packages/shared/src/server/clickhouse/queryOutcome.test.ts
+++ b/packages/shared/src/server/clickhouse/queryOutcome.test.ts
@@ -2,22 +2,27 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import {
CLICKHOUSE_QUERY_OUTCOME_METRIC,
+ CLICKHOUSE_QUERY_PERFORMANCE_METRICS,
CLICKHOUSE_RESOURCE_ERROR_OUTCOMES,
clickHouseQueryOutcomeRouteLabel,
clickHouseQueryShape,
clickHouseQueryTableLabel,
recordClickHouseQueryOutcome,
+ recordClickHouseQueryPerformance,
} from "./queryOutcome";
const recordIncrement = vi.fn();
+const recordDistribution = vi.fn();
vi.mock("../instrumentation", () => ({
recordIncrement: (...args: unknown[]) => recordIncrement(...args),
+ recordDistribution: (...args: unknown[]) => recordDistribution(...args),
}));
describe("ClickHouse query outcome metric", () => {
beforeEach(() => {
recordIncrement.mockClear();
+ recordDistribution.mockClear();
});
describe("route labels", () => {
@@ -58,6 +63,14 @@ describe("ClickHouse query outcome metric", () => {
// code-defined, so they are matched verbatim against the bare allowlist.
it.each([
["events.all", "events.all"],
+ ["events.batchIO", "events.batchIO"],
+ ["events.experimentBatchIO", "events.experimentBatchIO"],
+ ["events.sessionBatchIO", "events.sessionBatchIO"],
+ ["events.filterOptions", "events.filterOptions"],
+ ["experiments.batchIO", "experiments.batchIO"],
+ ["scores.allFromEvents", "scores.allFromEvents"],
+ ["scores.countAllFromEvents", "scores.countAllFromEvents"],
+ ["traces.byId", "traces.byId"],
["listObservations", "listObservations"],
["trace_redirect", "trace_redirect"],
])("labels the bare route %s verbatim", (route, expected) => {
@@ -65,7 +78,7 @@ describe("ClickHouse query outcome metric", () => {
});
it("counts an unlisted bare route under the catch-all label", () => {
- expect(clickHouseQueryOutcomeRouteLabel("traces.byId")).toBe("other");
+ expect(clickHouseQueryOutcomeRouteLabel("prompts.all")).toBe("other");
});
});
@@ -114,6 +127,16 @@ describe("ClickHouse query outcome metric", () => {
).toBe("scores");
});
+ // Dataset run tables filtered by a score read scores in a CTE; the scan
+ // under load is dataset_run_items_rmt, and the scores dedup shapes must not
+ // claim it.
+ it("labels a dataset run items query with a scores filter CTE as dataset_run_items", () => {
+ const query =
+ "WITH scores_aggregated AS (SELECT s.trace_id FROM scores s FINAL WHERE s.project_id = {projectId: String}) SELECT dri.id FROM dataset_run_items_rmt dri LEFT JOIN scores_aggregated sa ON dri.trace_id = sa.trace_id";
+ expect(clickHouseQueryTableLabel(query)).toBe("dataset_run_items");
+ expect(clickHouseQueryShape(query)).toBe("other");
+ });
+
it("matches a schema-qualified FROM table", () => {
expect(
clickHouseQueryTableLabel("SELECT * FROM default.events_full"),
@@ -211,6 +234,62 @@ describe("ClickHouse query outcome metric", () => {
),
).toBe("id_or_ilike");
});
+
+ describe("scores dedup strategies", () => {
+ const latestJoin = `
+ SELECT s.id FROM (
+ SELECT s.* FROM scores s
+ INNER JOIN (
+ SELECT s.id AS latest_id, max(s.event_ts) AS latest_event_ts
+ FROM scores s
+ WHERE s.project_id = {projectId: String}
+ %SEEK%
+ GROUP BY s.project_id, toDate(s.timestamp), s.name, s.id
+ ) latest ON s.id = latest.latest_id AND s.event_ts = latest.latest_event_ts
+ WHERE s.trace_id = {traceId: String}
+ LIMIT 1 BY s.project_id, toDate(s.timestamp), s.name, s.id
+ ) s`;
+
+ it.each([
+ "SELECT s.id FROM (SELECT s.* FROM scores s FINAL WHERE s.project_id = {projectId: String}) s",
+ "SELECT * FROM scores FINAL WHERE project_id = {projectId: String}",
+ ])("labels a FINAL read as scores_final: %s", (query) => {
+ expect(clickHouseQueryShape(query)).toBe("scores_final");
+ });
+
+ // The seek's trace_id predicate would otherwise make it by_trace_id.
+ it("labels the latest-version join with a seek as scores_join_seek", () => {
+ const query = latestJoin.replace(
+ "%SEEK%",
+ "AND (s.project_id, toDate(s.timestamp), s.name, s.id) IN (SELECT DISTINCT s.project_id, toDate(s.timestamp), s.name, s.id FROM scores s WHERE s.trace_id = {traceId: String})",
+ );
+ expect(clickHouseQueryShape(query)).toBe("scores_join_seek");
+ });
+
+ it("labels the latest-version join without a seek as scores_join", () => {
+ expect(clickHouseQueryShape(latestJoin.replace("%SEEK%", ""))).toBe(
+ "scores_join",
+ );
+ });
+
+ it("does not label FINAL on another table as a scores shape", () => {
+ expect(
+ clickHouseQueryShape(
+ "SELECT * FROM traces t FINAL WHERE t.id = {id: String}",
+ ),
+ ).toBe("other");
+ });
+
+ // A scores FINAL subquery under an events read keeps the events table
+ // label, so the scores shapes do not apply.
+ it("applies the scores shapes only when the table label is scores", () => {
+ const query =
+ "SELECT * FROM events_full e WHERE e.trace_id = {traceId: String} AND e.span_id IN (SELECT observation_id FROM scores s FINAL)";
+ expect(clickHouseQueryTableLabel(query)).toBe("events_full");
+ expect(clickHouseQueryShape(query)).toBe("by_trace_id");
+ expect(clickHouseQueryShape(query, "scores")).toBe("scores_final");
+ });
+ });
});
it("maps every ClickHouse resource error type to an outcome", () => {
@@ -253,6 +332,58 @@ describe("ClickHouse query outcome metric", () => {
);
});
+ it("emits the summary header as distributions with the outcome tags", () => {
+ recordClickHouseQueryPerformance(
+ {
+ read_rows: "1200",
+ read_bytes: "524288",
+ written_rows: "0",
+ elapsed_ns: "37500000",
+ memory_usage: "8388608",
+ },
+ {
+ tag_schema_version: "1",
+ surface: "trpc",
+ route: "scores.allFromEvents",
+ projectId: "project-1",
+ },
+ "scores",
+ "scores_final",
+ "main",
+ );
+
+ const tags = {
+ surface: "trpc",
+ route: "scores.allFromEvents",
+ table: "scores",
+ query_shape: "scores_final",
+ clickhouse_service: "main",
+ };
+ expect(recordDistribution.mock.calls).toEqual([
+ [CLICKHOUSE_QUERY_PERFORMANCE_METRICS.elapsedMs, 37.5, tags],
+ [CLICKHOUSE_QUERY_PERFORMANCE_METRICS.readBytes, 524288, tags],
+ [CLICKHOUSE_QUERY_PERFORMANCE_METRICS.readRows, 1200, tags],
+ [CLICKHOUSE_QUERY_PERFORMANCE_METRICS.memoryBytes, 8388608, tags],
+ ]);
+ });
+
+ it("skips summary keys that are missing or not numeric", () => {
+ recordClickHouseQueryPerformance(
+ { read_rows: "10", read_bytes: "n/a" },
+ { tag_schema_version: "1", surface: "unknown" },
+ "other",
+ "other",
+ "main",
+ );
+
+ expect(recordDistribution).toHaveBeenCalledTimes(1);
+ expect(recordDistribution).toHaveBeenCalledWith(
+ CLICKHOUSE_QUERY_PERFORMANCE_METRICS.readRows,
+ 10,
+ expect.any(Object),
+ );
+ });
+
it("keeps unknown surfaces and unlabelled routes queryable", () => {
recordClickHouseQueryOutcome(
"success",
diff --git a/packages/shared/src/server/clickhouse/queryOutcome.ts b/packages/shared/src/server/clickhouse/queryOutcome.ts
index c0847f1167..9a7cb9ad37 100644
--- a/packages/shared/src/server/clickhouse/queryOutcome.ts
+++ b/packages/shared/src/server/clickhouse/queryOutcome.ts
@@ -1,4 +1,4 @@
-import { recordIncrement } from "../instrumentation";
+import { recordDistribution, recordIncrement } from "../instrumentation";
import { type NormalizedClickHouseQueryTags } from "./queryTags";
import { type ClickhouseService } from "./client";
@@ -14,6 +14,19 @@ import { type ClickhouseService } from "./client";
export const CLICKHOUSE_QUERY_OUTCOME_METRIC =
"langfuse.clickhouse.query.outcome";
+/**
+ * Per-query cost distributions, read from the `x-clickhouse-summary` response
+ * header and emitted unsampled with the outcome metric's tags. APM keeps only a
+ * small sample of query spans, so these are the series to track query cost over
+ * time per route, table, shape and ClickHouse service.
+ */
+export const CLICKHOUSE_QUERY_PERFORMANCE_METRICS = {
+ elapsedMs: "langfuse.clickhouse.query.elapsed_ms",
+ readBytes: "langfuse.clickhouse.query.read_bytes",
+ readRows: "langfuse.clickhouse.query.read_rows",
+ memoryBytes: "langfuse.clickhouse.query.memory_bytes",
+} as const;
+
export type ClickHouseQueryOutcome =
| "success"
| "timeout"
@@ -58,6 +71,14 @@ const LABELLED_ROUTES = new Set([
*/
const LABELLED_BARE_ROUTES = new Set([
"events.all",
+ "events.batchIO",
+ "events.experimentBatchIO",
+ "events.sessionBatchIO",
+ "events.filterOptions",
+ "experiments.batchIO",
+ "scores.allFromEvents",
+ "scores.countAllFromEvents",
+ "traces.byId",
"listObservations",
"trace_redirect",
]);
@@ -114,15 +135,16 @@ export type ClickHouseQueryTable =
* otherwise steal the label. Still best-effort: a subquery/CTE reading a
* different table in its own `FROM` can win by priority order (first match
* wins). The events read path is single-table (`FROM events_full`/`events_core`,
- * no v3 joins), so those labels are reliable.
+ * no v3 joins), so those labels are reliable. `dataset_run_items` ranks above
+ * `scores` because dataset run tables filtered by a score read scores in a CTE.
*/
const TABLE_LABEL_PATTERNS: ReadonlyArray<[ClickHouseQueryTable, RegExp]> = [
["events_full", /\bfrom\s+(?:\w+\.)?events_full\b/i],
["events_core", /\bfrom\s+(?:\w+\.)?events_core\b/i],
["observations", /\bfrom\s+(?:\w+\.)?observations\b/i],
["traces", /\bfrom\s+(?:\w+\.)?traces\b/i],
- ["scores", /\bfrom\s+(?:\w+\.)?scores\b/i],
["dataset_run_items", /\bfrom\s+(?:\w+\.)?dataset_run_items_rmt\b/i],
+ ["scores", /\bfrom\s+(?:\w+\.)?scores\b/i],
["blob_storage_file_log", /\bfrom\s+(?:\w+\.)?blob_storage_file_log\b/i],
];
@@ -148,8 +170,15 @@ export function clickHouseQueryTableLabel(query: string): ClickHouseQueryTable {
* than a boolean per shape. A query carrying several predicates (e.g. a
* by-`span_id` lookup that also bounds `trace_id`) is attributed to its
* highest-ranked shape.
+ *
+ * Scores reads get their own shapes for the dedup strategy, which decides their
+ * cost more than the filter predicate does. They rank first, but only when the
+ * query's table label is `scores`.
*/
export type ClickHouseQueryShape =
+ | "scores_final"
+ | "scores_join_seek"
+ | "scores_join"
| "io_content"
| "metadata_content"
| "id_or_ilike"
@@ -207,26 +236,121 @@ const QUERY_SHAPE_PATTERNS: ReadonlyArray<[ClickHouseQueryShape, RegExp]> = [
["by_trace_id", BY_TRACE_ID_PATTERN],
];
-export function clickHouseQueryShape(query: string): ClickHouseQueryShape {
+/**
+ * Scores dedup strategies, matching the SQL the scores UI read emits:
+ *
+ * - `scores_final`: `FROM scores [s] FINAL`.
+ * - `scores_join_seek`: rows joined back to each key's `max(event_ts)`, with
+ * the key set narrowed first by a `SELECT DISTINCT` seek.
+ * - `scores_join`: the same join without the seek.
+ */
+const SCORES_FINAL_PATTERN =
+ /\bfrom\s+(?:\w+\.)?scores\s+(?:(?:as\s+)?\w+\s+)?final\b/i;
+
+const SCORES_LATEST_JOIN_PATTERN =
+ /\bmax\s*\(\s*(?:\w+\.)?event_ts\s*\)\s+as\s+latest_event_ts\b/i;
+
+const SELECT_DISTINCT_PATTERN = /\bselect\s+distinct\b/i;
+
+function scoresQueryShape(query: string): ClickHouseQueryShape | undefined {
+ if (SCORES_FINAL_PATTERN.test(query)) return "scores_final";
+ if (SCORES_LATEST_JOIN_PATTERN.test(query)) {
+ return SELECT_DISTINCT_PATTERN.test(query)
+ ? "scores_join_seek"
+ : "scores_join";
+ }
+ return undefined;
+}
+
+export function clickHouseQueryShape(
+ query: string,
+ table: ClickHouseQueryTable = clickHouseQueryTableLabel(query),
+): ClickHouseQueryShape {
+ if (table === "scores") {
+ const scoresShape = scoresQueryShape(query);
+ if (scoresShape) return scoresShape;
+ }
for (const [shape, pattern] of QUERY_SHAPE_PATTERNS) {
if (pattern.test(query)) return shape;
}
return OTHER_SHAPE_LABEL;
}
-export function recordClickHouseQueryOutcome(
- outcome: ClickHouseQueryOutcome,
+function clickHouseQueryMetricTags(
tags: NormalizedClickHouseQueryTags,
table: ClickHouseQueryTable,
shape: ClickHouseQueryShape,
clickhouseService: ClickhouseService,
-): void {
- recordIncrement(CLICKHOUSE_QUERY_OUTCOME_METRIC, 1, {
- outcome,
+) {
+ return {
surface: tags.surface,
route: clickHouseQueryOutcomeRouteLabel(tags.route),
table,
query_shape: shape,
clickhouse_service: clickhouseService,
+ };
+}
+
+export function recordClickHouseQueryOutcome(
+ outcome: ClickHouseQueryOutcome,
+ tags: NormalizedClickHouseQueryTags,
+ table: ClickHouseQueryTable,
+ shape: ClickHouseQueryShape,
+ clickhouseService: ClickhouseService,
+): void {
+ recordIncrement(CLICKHOUSE_QUERY_OUTCOME_METRIC, 1, {
+ outcome,
+ ...clickHouseQueryMetricTags(tags, table, shape, clickhouseService),
});
}
+
+function summaryNumber(
+ summary: Record<string, unknown>,
+ key: string,
+): number | undefined {
+ if (summary[key] === undefined) return undefined;
+ const value = Number(summary[key]);
+ return Number.isFinite(value) ? value : undefined;
+}
+
+/**
+ * Emits the cost distributions from a parsed `x-clickhouse-summary` header.
+ * ClickHouse sends the values as numeric strings; keys an older server does not
+ * send are skipped.
+ */
+export function recordClickHouseQueryPerformance(
+ summary: Record<string, unknown>,
+ tags: NormalizedClickHouseQueryTags,
+ table: ClickHouseQueryTable,
+ shape: ClickHouseQueryShape,
+ clickhouseService: ClickhouseService,
+): void {
+ const metricTags = clickHouseQueryMetricTags(
+ tags,
+ table,
+ shape,
+ clickhouseService,
+ );
+ const elapsedNs = summaryNumber(summary, "elapsed_ns");
+ const values: Array<[string, number | undefined]> = [
+ [
+ CLICKHOUSE_QUERY_PERFORMANCE_METRICS.elapsedMs,
+ elapsedNs === undefined ? undefined : elapsedNs / 1e6,
+ ],
+ [
+ CLICKHOUSE_QUERY_PERFORMANCE_METRICS.readBytes,
+ summaryNumber(summary, "read_bytes"),
+ ],
+ [
+ CLICKHOUSE_QUERY_PERFORMANCE_METRICS.readRows,
+ summaryNumber(summary, "read_rows"),
+ ],
+ [
+ CLICKHOUSE_QUERY_PERFORMANCE_METRICS.memoryBytes,
+ summaryNumber(summary, "memory_usage"),
+ ],
+ ];
+ for (const [metric, value] of values) {
+ if (value !== undefined) recordDistribution(metric, value, metricTags);
+ }
+}
diff --git a/packages/shared/src/server/repositories/clickhouse.ts b/packages/shared/src/server/repositories/clickhouse.ts
index 2c6981c7f2..557ac36d9b 100644
--- a/packages/shared/src/server/repositories/clickhouse.ts
+++ b/packages/shared/src/server/repositories/clickhouse.ts
@@ -35,6 +35,7 @@ import {
clickHouseQueryShape,
clickHouseQueryTableLabel,
recordClickHouseQueryOutcome,
+ recordClickHouseQueryPerformance,
} from "../clickhouse/queryOutcome";
/**
@@ -596,24 +597,36 @@ export type ClickhouseQueryOpts = {
allowLegacyEventsRead?: boolean;
};
-function recordSummaryOnSpan(
- span: Span,
+/**
+ * ClickHouse writes this header when the response starts, so for a result
+ * larger than its output buffer the values cover only the work done until then.
+ */
+function parseClickHouseSummary(
responseHeaders: Record<string, string | string[] | undefined>,
-): void {
+): Record<string, string> | undefined {
const summaryHeader = responseHeaders["x-clickhouse-summary"];
- if (!summaryHeader) return;
+ if (!summaryHeader) return undefined;
try {
- const summary = Array.isArray(summaryHeader)
+ return Array.isArray(summaryHeader)
? JSON.parse(summaryHeader[0])
: JSON.parse(summaryHeader);
- for (const key in summary) {
- span.setAttribute(`ch.${key}`, summary[key]);
- }
} catch (error) {
logger.debug(
`Failed to parse clickhouse summary header ${summaryHeader}`,
error,
);
+ return undefined;
+ }
+}
+
+function recordSummaryOnSpan(
+ span: Span,
+ responseHeaders: Record<string, string | string[] | undefined>,
+): void {
+ const summary = parseClickHouseSummary(responseHeaders);
+ if (!summary) return;
+ for (const key in summary) {
+ span.setAttribute(`ch.${key}`, summary[key]);
}
}
@@ -694,7 +707,7 @@ export async function queryClickhouse<T>(
if (!opts.allowLegacyEventsRead) assertNoLegacyEventsRead(opts.query);
const normalizedTags = normalizeClickHouseQueryTags(opts.tags);
const table = clickHouseQueryTableLabel(opts.query);
- const shape = clickHouseQueryShape(opts.query);
+ const shape = clickHouseQueryShape(opts.query, table);
const clickhouseService = resolveClickhouseService(
opts.preferredClickhouseService,
);
@@ -703,6 +716,7 @@ export async function queryClickhouse<T>(
async (span) => {
setSpanQueryAttributes(span, opts.query);
+ let summary: Record<string, string> | undefined;
const rows = await backOff(
async () => {
const res = await sendClickhouseQuery({
@@ -715,6 +729,7 @@ export async function queryClickhouse<T>(
format: "JSONEachRow",
span,
});
+ summary = parseClickHouseSummary(res.response_headers);
return (await res.json<T>()).map(handleExceptionRow);
},
{
@@ -773,6 +788,15 @@ export async function queryClickhouse<T>(
shape,
clickhouseService,
);
+ if (summary) {
+ recordClickHouseQueryPerformance(
+ summary,
+ normalizedTags,
+ table,
+ shape,
+ clickhouseService,
+ );
+ }
return rows;
},
);