🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(clickhouse): emit unsampled per-query cost distributions (#18486)
Niklas Semmler36928100c0db
diff --git a/packages/shared/src/server/clickhouse/queryOutcome.test.ts b/packages/shared/src/server/clickhouse/queryOutcome.test.tsindex 4b94486ff1..fa6ebad9aa 100644--- a/packages/shared/src/server/clickhouse/queryOutcome.test.ts+++ b/packages/shared/src/server/clickhouse/queryOutcome.test.ts@@ -2,22 +2,27 @@ import { beforeEach, describe, expect, it, vi } from "vitest";import {CLICKHOUSE_QUERY_OUTCOME_METRIC,+ CLICKHOUSE_QUERY_PERFORMANCE_METRICS,CLICKHOUSE_RESOURCE_ERROR_OUTCOMES,clickHouseQueryOutcomeRouteLabel,clickHouseQueryShape,clickHouseQueryTableLabel,recordClickHouseQueryOutcome,+ recordClickHouseQueryPerformance,} from "./queryOutcome";const recordIncrement = vi.fn();+const recordDistribution = vi.fn();vi.mock("../instrumentation", () => ({recordIncrement: (...args: unknown[]) => recordIncrement(...args),+ recordDistribution: (...args: unknown[]) => recordDistribution(...args),}));describe("ClickHouse query outcome metric", () => {beforeEach(() => {recordIncrement.mockClear();+ recordDistribution.mockClear();});describe("route labels", () => {@@ -58,6 +63,14 @@ describe("ClickHouse query outcome metric", () => {// code-defined, so they are matched verbatim against the bare allowlist.it.each([["events.all", "events.all"],+ ["events.batchIO", "events.batchIO"],+ ["events.experimentBatchIO", "events.experimentBatchIO"],+ ["events.sessionBatchIO", "events.sessionBatchIO"],+ ["events.filterOptions", "events.filterOptions"],+ ["experiments.batchIO", "experiments.batchIO"],+ ["scores.allFromEvents", "scores.allFromEvents"],+ ["scores.countAllFromEvents", "scores.countAllFromEvents"],+ ["traces.byId", "traces.byId"],["listObservations", "listObservations"],["trace_redirect", "trace_redirect"],])("labels the bare route %s verbatim", (route, expected) => {@@ -65,7 +78,7 @@ describe("ClickHouse query outcome metric", () => {});it("counts an unlisted bare route under the catch-all label", () => {- expect(clickHouseQueryOutcomeRouteLabel("traces.byId")).toBe("other");+ expect(clickHouseQueryOutcomeRouteLabel("prompts.all")).toBe("other");});});@@ -114,6 +127,16 @@ describe("ClickHouse query outcome metric", () => {).toBe("scores");});+ // Dataset run tables filtered by a score read scores in a CTE; the scan+ // under load is dataset_run_items_rmt, and the scores dedup shapes must not+ // claim it.+ it("labels a dataset run items query with a scores filter CTE as dataset_run_items", () => {+ const query =+ "WITH scores_aggregated AS (SELECT s.trace_id FROM scores s FINAL WHERE s.project_id = {projectId: String}) SELECT dri.id FROM dataset_run_items_rmt dri LEFT JOIN scores_aggregated sa ON dri.trace_id = sa.trace_id";+ expect(clickHouseQueryTableLabel(query)).toBe("dataset_run_items");+ expect(clickHouseQueryShape(query)).toBe("other");+ });+it("matches a schema-qualified FROM table", () => {expect(clickHouseQueryTableLabel("SELECT * FROM default.events_full"),@@ -211,6 +234,62 @@ describe("ClickHouse query outcome metric", () => {),).toBe("id_or_ilike");});++ describe("scores dedup strategies", () => {+ const latestJoin = `+ SELECT s.id FROM (+ SELECT s.* FROM scores s+ INNER JOIN (+ SELECT s.id AS latest_id, max(s.event_ts) AS latest_event_ts+ FROM scores s+ WHERE s.project_id = {projectId: String}+ %SEEK%+ GROUP BY s.project_id, toDate(s.timestamp), s.name, s.id+ ) latest ON s.id = latest.latest_id AND s.event_ts = latest.latest_event_ts+ WHERE s.trace_id = {traceId: String}+ LIMIT 1 BY s.project_id, toDate(s.timestamp), s.name, s.id+ ) s`;++ it.each([+ "SELECT s.id FROM (SELECT s.* FROM scores s FINAL WHERE s.project_id = {projectId: String}) s",+ "SELECT * FROM scores FINAL WHERE project_id = {projectId: String}",+ ])("labels a FINAL read as scores_final: %s", (query) => {+ expect(clickHouseQueryShape(query)).toBe("scores_final");+ });++ // The seek's trace_id predicate would otherwise make it by_trace_id.+ it("labels the latest-version join with a seek as scores_join_seek", () => {+ const query = latestJoin.replace(+ "%SEEK%",+ "AND (s.project_id, toDate(s.timestamp), s.name, s.id) IN (SELECT DISTINCT s.project_id, toDate(s.timestamp), s.name, s.id FROM scores s WHERE s.trace_id = {traceId: String})",+ );+ expect(clickHouseQueryShape(query)).toBe("scores_join_seek");+ });++ it("labels the latest-version join without a seek as scores_join", () => {+ expect(clickHouseQueryShape(latestJoin.replace("%SEEK%", ""))).toBe(+ "scores_join",+ );+ });++ it("does not label FINAL on another table as a scores shape", () => {+ expect(+ clickHouseQueryShape(+ "SELECT * FROM traces t FINAL WHERE t.id = {id: String}",+ ),+ ).toBe("other");+ });++ // A scores FINAL subquery under an events read keeps the events table+ // label, so the scores shapes do not apply.+ it("applies the scores shapes only when the table label is scores", () => {+ const query =+ "SELECT * FROM events_full e WHERE e.trace_id = {traceId: String} AND e.span_id IN (SELECT observation_id FROM scores s FINAL)";+ expect(clickHouseQueryTableLabel(query)).toBe("events_full");+ expect(clickHouseQueryShape(query)).toBe("by_trace_id");+ expect(clickHouseQueryShape(query, "scores")).toBe("scores_final");+ });+ });});it("maps every ClickHouse resource error type to an outcome", () => {@@ -253,6 +332,58 @@ describe("ClickHouse query outcome metric", () => {);});+ it("emits the summary header as distributions with the outcome tags", () => {+ recordClickHouseQueryPerformance(+ {+ read_rows: "1200",+ read_bytes: "524288",+ written_rows: "0",+ elapsed_ns: "37500000",+ memory_usage: "8388608",+ },+ {+ tag_schema_version: "1",+ surface: "trpc",+ route: "scores.allFromEvents",+ projectId: "project-1",+ },+ "scores",+ "scores_final",+ "main",+ );++ const tags = {+ surface: "trpc",+ route: "scores.allFromEvents",+ table: "scores",+ query_shape: "scores_final",+ clickhouse_service: "main",+ };+ expect(recordDistribution.mock.calls).toEqual([+ [CLICKHOUSE_QUERY_PERFORMANCE_METRICS.elapsedMs, 37.5, tags],+ [CLICKHOUSE_QUERY_PERFORMANCE_METRICS.readBytes, 524288, tags],+ [CLICKHOUSE_QUERY_PERFORMANCE_METRICS.readRows, 1200, tags],+ [CLICKHOUSE_QUERY_PERFORMANCE_METRICS.memoryBytes, 8388608, tags],+ ]);+ });++ it("skips summary keys that are missing or not numeric", () => {+ recordClickHouseQueryPerformance(+ { read_rows: "10", read_bytes: "n/a" },+ { tag_schema_version: "1", surface: "unknown" },+ "other",+ "other",+ "main",+ );++ expect(recordDistribution).toHaveBeenCalledTimes(1);+ expect(recordDistribution).toHaveBeenCalledWith(+ CLICKHOUSE_QUERY_PERFORMANCE_METRICS.readRows,+ 10,+ expect.any(Object),+ );+ });+it("keeps unknown surfaces and unlabelled routes queryable", () => {recordClickHouseQueryOutcome("success",diff --git a/packages/shared/src/server/clickhouse/queryOutcome.ts b/packages/shared/src/server/clickhouse/queryOutcome.tsindex c0847f1167..9a7cb9ad37 100644--- a/packages/shared/src/server/clickhouse/queryOutcome.ts+++ b/packages/shared/src/server/clickhouse/queryOutcome.ts@@ -1,4 +1,4 @@-import { recordIncrement } from "../instrumentation";+import { recordDistribution, recordIncrement } from "../instrumentation";import { type NormalizedClickHouseQueryTags } from "./queryTags";import { type ClickhouseService } from "./client";@@ -14,6 +14,19 @@ import { type ClickhouseService } from "./client";export const CLICKHOUSE_QUERY_OUTCOME_METRIC ="langfuse.clickhouse.query.outcome";+/**+ * Per-query cost distributions, read from the `x-clickhouse-summary` response+ * header and emitted unsampled with the outcome metric's tags. APM keeps only a+ * small sample of query spans, so these are the series to track query cost over+ * time per route, table, shape and ClickHouse service.+ */+export const CLICKHOUSE_QUERY_PERFORMANCE_METRICS = {+ elapsedMs: "langfuse.clickhouse.query.elapsed_ms",+ readBytes: "langfuse.clickhouse.query.read_bytes",+ readRows: "langfuse.clickhouse.query.read_rows",+ memoryBytes: "langfuse.clickhouse.query.memory_bytes",+} as const;+export type ClickHouseQueryOutcome =| "success"| "timeout"@@ -58,6 +71,14 @@ const LABELLED_ROUTES = new Set([*/const LABELLED_BARE_ROUTES = new Set(["events.all",+ "events.batchIO",+ "events.experimentBatchIO",+ "events.sessionBatchIO",+ "events.filterOptions",+ "experiments.batchIO",+ "scores.allFromEvents",+ "scores.countAllFromEvents",+ "traces.byId","listObservations","trace_redirect",]);@@ -114,15 +135,16 @@ export type ClickHouseQueryTable =* otherwise steal the label. Still best-effort: a subquery/CTE reading a* different table in its own `FROM` can win by priority order (first match* wins). The events read path is single-table (`FROM events_full`/`events_core`,- * no v3 joins), so those labels are reliable.+ * no v3 joins), so those labels are reliable. `dataset_run_items` ranks above+ * `scores` because dataset run tables filtered by a score read scores in a CTE.*/const TABLE_LABEL_PATTERNS: ReadonlyArray<[ClickHouseQueryTable, RegExp]> = [["events_full", /\bfrom\s+(?:\w+\.)?events_full\b/i],["events_core", /\bfrom\s+(?:\w+\.)?events_core\b/i],["observations", /\bfrom\s+(?:\w+\.)?observations\b/i],["traces", /\bfrom\s+(?:\w+\.)?traces\b/i],- ["scores", /\bfrom\s+(?:\w+\.)?scores\b/i],["dataset_run_items", /\bfrom\s+(?:\w+\.)?dataset_run_items_rmt\b/i],+ ["scores", /\bfrom\s+(?:\w+\.)?scores\b/i],["blob_storage_file_log", /\bfrom\s+(?:\w+\.)?blob_storage_file_log\b/i],];@@ -148,8 +170,15 @@ export function clickHouseQueryTableLabel(query: string): ClickHouseQueryTable {* than a boolean per shape. A query carrying several predicates (e.g. a* by-`span_id` lookup that also bounds `trace_id`) is attributed to its* highest-ranked shape.+ *+ * Scores reads get their own shapes for the dedup strategy, which decides their+ * cost more than the filter predicate does. They rank first, but only when the+ * query's table label is `scores`.*/export type ClickHouseQueryShape =+ | "scores_final"+ | "scores_join_seek"+ | "scores_join"| "io_content"| "metadata_content"| "id_or_ilike"@@ -207,26 +236,121 @@ const QUERY_SHAPE_PATTERNS: ReadonlyArray<[ClickHouseQueryShape, RegExp]> = [["by_trace_id", BY_TRACE_ID_PATTERN],];-export function clickHouseQueryShape(query: string): ClickHouseQueryShape {+/**+ * Scores dedup strategies, matching the SQL the scores UI read emits:+ *+ * - `scores_final`: `FROM scores [s] FINAL`.+ * - `scores_join_seek`: rows joined back to each key's `max(event_ts)`, with+ * the key set narrowed first by a `SELECT DISTINCT` seek.+ * - `scores_join`: the same join without the seek.+ */+const SCORES_FINAL_PATTERN =+ /\bfrom\s+(?:\w+\.)?scores\s+(?:(?:as\s+)?\w+\s+)?final\b/i;++const SCORES_LATEST_JOIN_PATTERN =+ /\bmax\s*\(\s*(?:\w+\.)?event_ts\s*\)\s+as\s+latest_event_ts\b/i;++const SELECT_DISTINCT_PATTERN = /\bselect\s+distinct\b/i;++function scoresQueryShape(query: string): ClickHouseQueryShape | undefined {+ if (SCORES_FINAL_PATTERN.test(query)) return "scores_final";+ if (SCORES_LATEST_JOIN_PATTERN.test(query)) {+ return SELECT_DISTINCT_PATTERN.test(query)+ ? "scores_join_seek"+ : "scores_join";+ }+ return undefined;+}++export function clickHouseQueryShape(+ query: string,+ table: ClickHouseQueryTable = clickHouseQueryTableLabel(query),+): ClickHouseQueryShape {+ if (table === "scores") {+ const scoresShape = scoresQueryShape(query);+ if (scoresShape) return scoresShape;+ }for (const [shape, pattern] of QUERY_SHAPE_PATTERNS) {if (pattern.test(query)) return shape;}return OTHER_SHAPE_LABEL;}-export function recordClickHouseQueryOutcome(- outcome: ClickHouseQueryOutcome,+function clickHouseQueryMetricTags(tags: NormalizedClickHouseQueryTags,table: ClickHouseQueryTable,shape: ClickHouseQueryShape,clickhouseService: ClickhouseService,-): void {- recordIncrement(CLICKHOUSE_QUERY_OUTCOME_METRIC, 1, {- outcome,+) {+ return {surface: tags.surface,route: clickHouseQueryOutcomeRouteLabel(tags.route),table,query_shape: shape,clickhouse_service: clickhouseService,+ };+}++export function recordClickHouseQueryOutcome(+ outcome: ClickHouseQueryOutcome,+ tags: NormalizedClickHouseQueryTags,+ table: ClickHouseQueryTable,+ shape: ClickHouseQueryShape,+ clickhouseService: ClickhouseService,+): void {+ recordIncrement(CLICKHOUSE_QUERY_OUTCOME_METRIC, 1, {+ outcome,+ ...clickHouseQueryMetricTags(tags, table, shape, clickhouseService),});}++function summaryNumber(+ summary: Record<string, unknown>,+ key: string,+): number | undefined {+ if (summary[key] === undefined) return undefined;+ const value = Number(summary[key]);+ return Number.isFinite(value) ? value : undefined;+}++/**+ * Emits the cost distributions from a parsed `x-clickhouse-summary` header.+ * ClickHouse sends the values as numeric strings; keys an older server does not+ * send are skipped.+ */+export function recordClickHouseQueryPerformance(+ summary: Record<string, unknown>,+ tags: NormalizedClickHouseQueryTags,+ table: ClickHouseQueryTable,+ shape: ClickHouseQueryShape,+ clickhouseService: ClickhouseService,+): void {+ const metricTags = clickHouseQueryMetricTags(+ tags,+ table,+ shape,+ clickhouseService,+ );+ const elapsedNs = summaryNumber(summary, "elapsed_ns");+ const values: Array<[string, number | undefined]> = [+ [+ CLICKHOUSE_QUERY_PERFORMANCE_METRICS.elapsedMs,+ elapsedNs === undefined ? undefined : elapsedNs / 1e6,+ ],+ [+ CLICKHOUSE_QUERY_PERFORMANCE_METRICS.readBytes,+ summaryNumber(summary, "read_bytes"),+ ],+ [+ CLICKHOUSE_QUERY_PERFORMANCE_METRICS.readRows,+ summaryNumber(summary, "read_rows"),+ ],+ [+ CLICKHOUSE_QUERY_PERFORMANCE_METRICS.memoryBytes,+ summaryNumber(summary, "memory_usage"),+ ],+ ];+ for (const [metric, value] of values) {+ if (value !== undefined) recordDistribution(metric, value, metricTags);+ }+}diff --git a/packages/shared/src/server/repositories/clickhouse.ts b/packages/shared/src/server/repositories/clickhouse.tsindex 2c6981c7f2..557ac36d9b 100644--- a/packages/shared/src/server/repositories/clickhouse.ts+++ b/packages/shared/src/server/repositories/clickhouse.ts@@ -35,6 +35,7 @@ import {clickHouseQueryShape,clickHouseQueryTableLabel,recordClickHouseQueryOutcome,+ recordClickHouseQueryPerformance,} from "../clickhouse/queryOutcome";/**@@ -596,24 +597,36 @@ export type ClickhouseQueryOpts = {allowLegacyEventsRead?: boolean;};-function recordSummaryOnSpan(- span: Span,+/**+ * ClickHouse writes this header when the response starts, so for a result+ * larger than its output buffer the values cover only the work done until then.+ */+function parseClickHouseSummary(responseHeaders: Record<string, string | string[] | undefined>,-): void {+): Record<string, string> | undefined {const summaryHeader = responseHeaders["x-clickhouse-summary"];- if (!summaryHeader) return;+ if (!summaryHeader) return undefined;try {- const summary = Array.isArray(summaryHeader)+ return Array.isArray(summaryHeader)? JSON.parse(summaryHeader[0]): JSON.parse(summaryHeader);- for (const key in summary) {- span.setAttribute(`ch.${key}`, summary[key]);- }} catch (error) {logger.debug(`Failed to parse clickhouse summary header ${summaryHeader}`,error,);+ return undefined;+ }+}++function recordSummaryOnSpan(+ span: Span,+ responseHeaders: Record<string, string | string[] | undefined>,+): void {+ const summary = parseClickHouseSummary(responseHeaders);+ if (!summary) return;+ for (const key in summary) {+ span.setAttribute(`ch.${key}`, summary[key]);}}@@ -694,7 +707,7 @@ export async function queryClickhouse<T>(if (!opts.allowLegacyEventsRead) assertNoLegacyEventsRead(opts.query);const normalizedTags = normalizeClickHouseQueryTags(opts.tags);const table = clickHouseQueryTableLabel(opts.query);- const shape = clickHouseQueryShape(opts.query);+ const shape = clickHouseQueryShape(opts.query, table);const clickhouseService = resolveClickhouseService(opts.preferredClickhouseService,);@@ -703,6 +716,7 @@ export async function queryClickhouse<T>(async (span) => {setSpanQueryAttributes(span, opts.query);+ let summary: Record<string, string> | undefined;const rows = await backOff(async () => {const res = await sendClickhouseQuery({@@ -715,6 +729,7 @@ export async function queryClickhouse<T>(format: "JSONEachRow",span,});+ summary = parseClickHouseSummary(res.response_headers);return (await res.json<T>()).map(handleExceptionRow);},{@@ -773,6 +788,15 @@ export async function queryClickhouse<T>(shape,clickhouseService,);+ if (summary) {+ recordClickHouseQueryPerformance(+ summary,+ normalizedTags,+ table,+ shape,+ clickhouseService,+ );+ }return rows;},);