🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(clickhouse): attribute query outcomes and spans by filter shape (#17469)
Niklas Semmlereae3221536ae
diff --git a/packages/shared/src/server/clickhouse/queryOutcome.test.ts b/packages/shared/src/server/clickhouse/queryOutcome.test.tsindex 550c172c1a..43b374fcf3 100644--- a/packages/shared/src/server/clickhouse/queryOutcome.test.ts+++ b/packages/shared/src/server/clickhouse/queryOutcome.test.ts@@ -4,6 +4,7 @@ import {CLICKHOUSE_QUERY_OUTCOME_METRIC,CLICKHOUSE_RESOURCE_ERROR_OUTCOMES,clickHouseQueryOutcomeRouteLabel,+ clickHouseQueryShape,clickHouseQueryTableLabel,recordClickHouseQueryOutcome,} from "./queryOutcome";@@ -133,6 +134,85 @@ describe("ClickHouse query outcome metric", () => {});});+ describe("query shape labels", () => {+ // The two shapes the FTS compiler emits for an input/output search, plus a+ // bare (I)LIKE fallback. See fts.ts.+ it.each([+ "SELECT id FROM events_full e WHERE position(lower(e.output), lower({p: String})) > 0",+ "SELECT id FROM events_full e WHERE hasAllTokens(lower(e.input), arraySlice(arrayDistinct(tokens(lower({p: String}))), 1, 64))",+ "SELECT id FROM observations o WHERE o.input ILIKE {p: String}",+ "SELECT id FROM events_full e WHERE output LIKE {p: String}",+ // "starts with" / "ends with" UI filters on the string input/output cols.+ "SELECT id FROM events_full e WHERE startsWith(e.input, {p: String})",+ "SELECT id FROM events_full e WHERE endsWith(e.output, {p: String})",+ ])("labels an input/output content search as io_content: %s", (query) => {+ expect(clickHouseQueryShape(query)).toBe("io_content");+ });++ // Metadata search runs the same functions over the _names/_values arrays.+ it.each([+ "SELECT id FROM events_full e WHERE has(e.metadata_names, {k: String}) AND (position(e.metadata_values[indexOf(e.metadata_names, {k: String})], {v: String}) > 0)",+ "SELECT id FROM events_full e WHERE hasAllTokens(e.metadata_values, arraySlice({t: Array(String)}, 1, 64))",+ ])("labels a metadata content search as metadata_content: %s", (query) => {+ expect(clickHouseQueryShape(query)).toBe("metadata_content");+ });++ it("labels the OR-of-ILIKE id search arm as id_or_ilike", () => {+ expect(+ clickHouseQueryShape(+ "SELECT id FROM events_full e WHERE e.project_id = {p: String} AND (e.span_id ILIKE {searchString: String} OR e.trace_id ILIKE {searchString: String})",+ ),+ ).toBe("id_or_ilike");+ });++ it.each([+ "SELECT * FROM events_full e WHERE e.project_id = {p: String} AND span_id = {id: String}",+ // Batch IO fetch uses the paren-less IN {param} form.+ "SELECT * FROM events_full e WHERE e.span_id IN {observationIds: Array(String)}",+ ])("labels a span_id lookup as by_span_id: %s", (query) => {+ expect(clickHouseQueryShape(query)).toBe("by_span_id");+ });++ // A span lookup that also bounds trace_id is a span lookup, not a+ // by_trace_id scan: by_span_id outranks by_trace_id.+ it("labels a span lookup that also bounds trace_id as by_span_id", () => {+ expect(+ clickHouseQueryShape(+ "SELECT * FROM events_full e WHERE span_id = {id: String} AND trace_id = {traceId: String}",+ ),+ ).toBe("by_span_id");+ });++ it.each([+ "SELECT * FROM events_full e WHERE trace_id IN ({traceIds: Array(String)})",+ // Batch/event queries emit the paren-less IN {param} form.+ "SELECT * FROM events_full e WHERE trace_id IN {traceIds: Array(String)}",+ "SELECT * FROM events_full e WHERE trace_id = {traceId: String}",+ ])("labels a trace_id lookup as by_trace_id: %s", (query) => {+ expect(clickHouseQueryShape(query)).toBe("by_trace_id");+ });++ // A metadata search must not be counted as io_content, and an exact+ // equality / unrelated query has no shape.+ it.each([+ "SELECT id FROM events_full e WHERE e.input = {p: String}",+ "SELECT id FROM events_full e WHERE e.project_id = {p: String}",+ "SELECT count() FROM traces",+ ])("labels %s as other", (query) => {+ expect(clickHouseQueryShape(query)).toBe("other");+ });++ // trace_id in the OR-ILIKE arm is a search predicate, not a by_trace_id+ // lookup; the whole query is id_or_ilike.+ it("does not mistake an ILIKE trace_id arm for a by_trace_id lookup", () => {+ expect(+ clickHouseQueryShape(+ "SELECT id FROM events_full e WHERE (e.trace_id ILIKE {searchString: String})",+ ),+ ).toBe("id_or_ilike");+ });+ });+it("maps every ClickHouse resource error type to an outcome", () => {expect(CLICKHOUSE_RESOURCE_ERROR_OUTCOMES).toEqual({TIMEOUT: "timeout",@@ -154,6 +234,7 @@ describe("ClickHouse query outcome metric", () => {userAgent: "python-httpx/0.28.1",},"events_full",+ "io_content",);expect(recordIncrement).toHaveBeenCalledTimes(1);@@ -165,6 +246,7 @@ describe("ClickHouse query outcome metric", () => {surface: "publicapi",route: "get_/api/public/v2/observations",table: "events_full",+ query_shape: "io_content",},);});@@ -177,6 +259,7 @@ describe("ClickHouse query outcome metric", () => {surface: "unknown",},"other",+ "other",);expect(recordIncrement).toHaveBeenCalledWith(@@ -187,6 +270,7 @@ describe("ClickHouse query outcome metric", () => {surface: "unknown",route: "other",table: "other",+ query_shape: "other",},);});diff --git a/packages/shared/src/server/clickhouse/queryOutcome.ts b/packages/shared/src/server/clickhouse/queryOutcome.tsindex 2a7883e8f0..6e1f61cfcb 100644--- a/packages/shared/src/server/clickhouse/queryOutcome.ts+++ b/packages/shared/src/server/clickhouse/queryOutcome.ts@@ -132,15 +132,96 @@ export function clickHouseQueryTableLabel(query: string): ClickHouseQueryTable {return OTHER_TABLE_LABEL;}+/**+ * Bounded filter-shape label so a timeout can be attributed to the predicate+ * shape driving it, not just the table. Without it, one `events_full` timeout+ * cannot be told from another on the metric, and per-shape rates have to be+ * estimated from ~10%-sampled APM query text. This makes the shape a+ * first-class, unsampled dimension on both the outcome metric and the query+ * span.+ *+ * One value per query: the shapes are ranked by how strongly each drives scan+ * cost, and the first match wins, so it adds a single bounded dimension rather+ * than a boolean per shape. A query carrying several predicates (e.g. a+ * by-`span_id` lookup that also bounds `trace_id`) is attributed to its+ * highest-ranked shape.+ */+export type ClickHouseQueryShape =+ | "io_content"+ | "metadata_content"+ | "id_or_ilike"+ | "by_span_id"+ | "by_trace_id"+ | "other";++const OTHER_SHAPE_LABEL = "other" as const;++/**+ * Derived from the query text (like the table label) because threading the+ * filter shape through every call site is impractical. Patterns match the SQL+ * the filter compilers emit:+ *+ * - `io_content`: a substring/token/prefix/(I)LIKE search over the `input`/+ * `output` columns — the largest `events_full` columns and the confirmed+ * full-scan class. Anchored to `input`/`output` so a metadata search never+ * counts.+ * - `metadata_content`: the same search family over the `metadata_names`/+ * `metadata_values` arrays, distinguished by the array-column suffix and the+ * `has(names, key)` / `indexOf(names, key)` key-lookup wrapper. Only filter+ * functions are matched (`has`/`position`/`hasAllTokens`/`startsWith`/…), not+ * the `mapFromArrays`/`argMax`/`arrayReverse` projection helpers.+ * - `id_or_ilike`: the OR-of-`ILIKE` id search arm, matched on its distinctive+ * `searchString` parameter.+ * - `by_span_id`: a `span_id = {…}` point lookup (`getObservationById`) or a+ * `span_id IN {…}` batch lookup. Ranked above `by_trace_id` so a span lookup+ * that also bounds `trace_id` counts as the span lookup.+ * - `by_trace_id`: a `trace_id IN (…)` / `trace_id IN {…}` / `trace_id = {…}`+ * lookup — the unbounded variant is a distinct timeout class. The `IN` form+ * appears both parenthesized (inline list) and bare (`{param: Array(...)}`).+ */+const IO_CONTENT_FUNCTION_PATTERN =+ /\b(?:position(?:caseinsensitive)?|hasalltokens|hasanytokens|hastoken|startswith|endswith)\s*\(\s*(?:lower\s*\(\s*)?(?:\w+\.)?(?:input|output)\b/i;++const IO_CONTENT_LIKE_PATTERN =+ /\b(?:\w+\.)?(?:input|output)\s+(?:not\s+)?i?like\b/i;++const METADATA_CONTENT_PATTERN =+ /\b(?:has|position(?:caseinsensitive)?|hasalltokens|hasanytokens|hastoken|startswith|endswith)\s*\(\s*(?:lower\s*\(\s*)?(?:\w+\.)?metadata_(?:names|values)\b/i;++const ID_OR_ILIKE_PATTERN = /\bilike\s*\{\s*searchString\b/i;++const BY_SPAN_ID_PATTERN = /\bspan_id\s*=\s*\{|\bspan_id\s+in\s*(?:\(\s*)?\{/i;++const BY_TRACE_ID_PATTERN =+ /\btrace_id\s*=\s*\{|\btrace_id\s+in\s*(?:\(\s*)?\{/i;++const QUERY_SHAPE_PATTERNS: ReadonlyArray<[ClickHouseQueryShape, RegExp]> = [+ ["io_content", IO_CONTENT_FUNCTION_PATTERN],+ ["io_content", IO_CONTENT_LIKE_PATTERN],+ ["metadata_content", METADATA_CONTENT_PATTERN],+ ["id_or_ilike", ID_OR_ILIKE_PATTERN],+ ["by_span_id", BY_SPAN_ID_PATTERN],+ ["by_trace_id", BY_TRACE_ID_PATTERN],+];++export function clickHouseQueryShape(query: string): ClickHouseQueryShape {+ for (const [shape, pattern] of QUERY_SHAPE_PATTERNS) {+ if (pattern.test(query)) return shape;+ }+ return OTHER_SHAPE_LABEL;+}+export function recordClickHouseQueryOutcome(outcome: ClickHouseQueryOutcome,tags: NormalizedClickHouseQueryTags,table: ClickHouseQueryTable,+ shape: ClickHouseQueryShape,): void {recordIncrement(CLICKHOUSE_QUERY_OUTCOME_METRIC, 1, {outcome,surface: tags.surface,route: clickHouseQueryOutcomeRouteLabel(tags.route),table,+ query_shape: shape,});}diff --git a/packages/shared/src/server/repositories/clickhouse.ts b/packages/shared/src/server/repositories/clickhouse.tsindex 7d7e6f5d10..2a8aea8f41 100644--- a/packages/shared/src/server/repositories/clickhouse.ts+++ b/packages/shared/src/server/repositories/clickhouse.ts@@ -33,6 +33,7 @@ import {} from "../clickhouse/queryTags";import {CLICKHOUSE_RESOURCE_ERROR_OUTCOMES,+ clickHouseQueryShape,clickHouseQueryTableLabel,recordClickHouseQueryOutcome,} from "../clickhouse/queryOutcome";@@ -628,6 +629,7 @@ function recordSummaryOnSpan(function setSpanQueryAttributes(span: Span, query: string): void {span.setAttribute("ch.query.text", query);+ span.setAttribute("ch.query.shape", clickHouseQueryShape(query));span.setAttribute("db.system", "clickhouse");span.setAttribute("db.query.text", query);span.setAttribute("db.operation.name", "SELECT");@@ -702,6 +704,7 @@ export async function queryClickhouse<T>(if (!opts.allowLegacyEventsRead) assertNoLegacyEventsRead(opts.query);const normalizedTags = normalizeClickHouseQueryTags(opts.tags);const table = clickHouseQueryTableLabel(opts.query);+ const shape = clickHouseQueryShape(opts.query);return await instrumentAsync({ name: "clickhouse-query", spanKind: SpanKind.CLIENT },async (span) => {@@ -764,11 +767,12 @@ export async function queryClickhouse<T>(: "error",normalizedTags,table,+ shape,);throw wrapped;});- recordClickHouseQueryOutcome("success", normalizedTags, table);+ recordClickHouseQueryOutcome("success", normalizedTags, table, shape);return rows;},);