Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(clickhouse): attribute query outcomes and spans by filter shape (#17469)

Niklas Semmlereae3221536ae
diff --git a/packages/shared/src/server/clickhouse/queryOutcome.test.ts b/packages/shared/src/server/clickhouse/queryOutcome.test.ts
index 550c172c1a..43b374fcf3 100644
--- a/packages/shared/src/server/clickhouse/queryOutcome.test.ts
+++ b/packages/shared/src/server/clickhouse/queryOutcome.test.ts
@@ -4,6 +4,7 @@ import {
CLICKHOUSE_QUERY_OUTCOME_METRIC,
CLICKHOUSE_RESOURCE_ERROR_OUTCOMES,
clickHouseQueryOutcomeRouteLabel,
+ clickHouseQueryShape,
clickHouseQueryTableLabel,
recordClickHouseQueryOutcome,
} from "./queryOutcome";
@@ -133,6 +134,85 @@ describe("ClickHouse query outcome metric", () => {
});
});
+ describe("query shape labels", () => {
+ // The two shapes the FTS compiler emits for an input/output search, plus a
+ // bare (I)LIKE fallback. See fts.ts.
+ it.each([
+ "SELECT id FROM events_full e WHERE position(lower(e.output), lower({p: String})) > 0",
+ "SELECT id FROM events_full e WHERE hasAllTokens(lower(e.input), arraySlice(arrayDistinct(tokens(lower({p: String}))), 1, 64))",
+ "SELECT id FROM observations o WHERE o.input ILIKE {p: String}",
+ "SELECT id FROM events_full e WHERE output LIKE {p: String}",
+ // "starts with" / "ends with" UI filters on the string input/output cols.
+ "SELECT id FROM events_full e WHERE startsWith(e.input, {p: String})",
+ "SELECT id FROM events_full e WHERE endsWith(e.output, {p: String})",
+ ])("labels an input/output content search as io_content: %s", (query) => {
+ expect(clickHouseQueryShape(query)).toBe("io_content");
+ });
+
+ // Metadata search runs the same functions over the _names/_values arrays.
+ it.each([
+ "SELECT id FROM events_full e WHERE has(e.metadata_names, {k: String}) AND (position(e.metadata_values[indexOf(e.metadata_names, {k: String})], {v: String}) > 0)",
+ "SELECT id FROM events_full e WHERE hasAllTokens(e.metadata_values, arraySlice({t: Array(String)}, 1, 64))",
+ ])("labels a metadata content search as metadata_content: %s", (query) => {
+ expect(clickHouseQueryShape(query)).toBe("metadata_content");
+ });
+
+ it("labels the OR-of-ILIKE id search arm as id_or_ilike", () => {
+ expect(
+ clickHouseQueryShape(
+ "SELECT id FROM events_full e WHERE e.project_id = {p: String} AND (e.span_id ILIKE {searchString: String} OR e.trace_id ILIKE {searchString: String})",
+ ),
+ ).toBe("id_or_ilike");
+ });
+
+ it.each([
+ "SELECT * FROM events_full e WHERE e.project_id = {p: String} AND span_id = {id: String}",
+ // Batch IO fetch uses the paren-less IN {param} form.
+ "SELECT * FROM events_full e WHERE e.span_id IN {observationIds: Array(String)}",
+ ])("labels a span_id lookup as by_span_id: %s", (query) => {
+ expect(clickHouseQueryShape(query)).toBe("by_span_id");
+ });
+
+ // A span lookup that also bounds trace_id is a span lookup, not a
+ // by_trace_id scan: by_span_id outranks by_trace_id.
+ it("labels a span lookup that also bounds trace_id as by_span_id", () => {
+ expect(
+ clickHouseQueryShape(
+ "SELECT * FROM events_full e WHERE span_id = {id: String} AND trace_id = {traceId: String}",
+ ),
+ ).toBe("by_span_id");
+ });
+
+ it.each([
+ "SELECT * FROM events_full e WHERE trace_id IN ({traceIds: Array(String)})",
+ // Batch/event queries emit the paren-less IN {param} form.
+ "SELECT * FROM events_full e WHERE trace_id IN {traceIds: Array(String)}",
+ "SELECT * FROM events_full e WHERE trace_id = {traceId: String}",
+ ])("labels a trace_id lookup as by_trace_id: %s", (query) => {
+ expect(clickHouseQueryShape(query)).toBe("by_trace_id");
+ });
+
+ // A metadata search must not be counted as io_content, and an exact
+ // equality / unrelated query has no shape.
+ it.each([
+ "SELECT id FROM events_full e WHERE e.input = {p: String}",
+ "SELECT id FROM events_full e WHERE e.project_id = {p: String}",
+ "SELECT count() FROM traces",
+ ])("labels %s as other", (query) => {
+ expect(clickHouseQueryShape(query)).toBe("other");
+ });
+
+ // trace_id in the OR-ILIKE arm is a search predicate, not a by_trace_id
+ // lookup; the whole query is id_or_ilike.
+ it("does not mistake an ILIKE trace_id arm for a by_trace_id lookup", () => {
+ expect(
+ clickHouseQueryShape(
+ "SELECT id FROM events_full e WHERE (e.trace_id ILIKE {searchString: String})",
+ ),
+ ).toBe("id_or_ilike");
+ });
+ });
+
it("maps every ClickHouse resource error type to an outcome", () => {
expect(CLICKHOUSE_RESOURCE_ERROR_OUTCOMES).toEqual({
TIMEOUT: "timeout",
@@ -154,6 +234,7 @@ describe("ClickHouse query outcome metric", () => {
userAgent: "python-httpx/0.28.1",
},
"events_full",
+ "io_content",
);
expect(recordIncrement).toHaveBeenCalledTimes(1);
@@ -165,6 +246,7 @@ describe("ClickHouse query outcome metric", () => {
surface: "publicapi",
route: "get_/api/public/v2/observations",
table: "events_full",
+ query_shape: "io_content",
},
);
});
@@ -177,6 +259,7 @@ describe("ClickHouse query outcome metric", () => {
surface: "unknown",
},
"other",
+ "other",
);
expect(recordIncrement).toHaveBeenCalledWith(
@@ -187,6 +270,7 @@ describe("ClickHouse query outcome metric", () => {
surface: "unknown",
route: "other",
table: "other",
+ query_shape: "other",
},
);
});
diff --git a/packages/shared/src/server/clickhouse/queryOutcome.ts b/packages/shared/src/server/clickhouse/queryOutcome.ts
index 2a7883e8f0..6e1f61cfcb 100644
--- a/packages/shared/src/server/clickhouse/queryOutcome.ts
+++ b/packages/shared/src/server/clickhouse/queryOutcome.ts
@@ -132,15 +132,96 @@ export function clickHouseQueryTableLabel(query: string): ClickHouseQueryTable {
return OTHER_TABLE_LABEL;
}
+/**
+ * Bounded filter-shape label so a timeout can be attributed to the predicate
+ * shape driving it, not just the table. Without it, one `events_full` timeout
+ * cannot be told from another on the metric, and per-shape rates have to be
+ * estimated from ~10%-sampled APM query text. This makes the shape a
+ * first-class, unsampled dimension on both the outcome metric and the query
+ * span.
+ *
+ * One value per query: the shapes are ranked by how strongly each drives scan
+ * cost, and the first match wins, so it adds a single bounded dimension rather
+ * than a boolean per shape. A query carrying several predicates (e.g. a
+ * by-`span_id` lookup that also bounds `trace_id`) is attributed to its
+ * highest-ranked shape.
+ */
+export type ClickHouseQueryShape =
+ | "io_content"
+ | "metadata_content"
+ | "id_or_ilike"
+ | "by_span_id"
+ | "by_trace_id"
+ | "other";
+
+const OTHER_SHAPE_LABEL = "other" as const;
+
+/**
+ * Derived from the query text (like the table label) because threading the
+ * filter shape through every call site is impractical. Patterns match the SQL
+ * the filter compilers emit:
+ *
+ * - `io_content`: a substring/token/prefix/(I)LIKE search over the `input`/
+ * `output` columns — the largest `events_full` columns and the confirmed
+ * full-scan class. Anchored to `input`/`output` so a metadata search never
+ * counts.
+ * - `metadata_content`: the same search family over the `metadata_names`/
+ * `metadata_values` arrays, distinguished by the array-column suffix and the
+ * `has(names, key)` / `indexOf(names, key)` key-lookup wrapper. Only filter
+ * functions are matched (`has`/`position`/`hasAllTokens`/`startsWith`/…), not
+ * the `mapFromArrays`/`argMax`/`arrayReverse` projection helpers.
+ * - `id_or_ilike`: the OR-of-`ILIKE` id search arm, matched on its distinctive
+ * `searchString` parameter.
+ * - `by_span_id`: a `span_id = {…}` point lookup (`getObservationById`) or a
+ * `span_id IN {…}` batch lookup. Ranked above `by_trace_id` so a span lookup
+ * that also bounds `trace_id` counts as the span lookup.
+ * - `by_trace_id`: a `trace_id IN (…)` / `trace_id IN {…}` / `trace_id = {…}`
+ * lookup — the unbounded variant is a distinct timeout class. The `IN` form
+ * appears both parenthesized (inline list) and bare (`{param: Array(...)}`).
+ */
+const IO_CONTENT_FUNCTION_PATTERN =
+ /\b(?:position(?:caseinsensitive)?|hasalltokens|hasanytokens|hastoken|startswith|endswith)\s*\(\s*(?:lower\s*\(\s*)?(?:\w+\.)?(?:input|output)\b/i;
+
+const IO_CONTENT_LIKE_PATTERN =
+ /\b(?:\w+\.)?(?:input|output)\s+(?:not\s+)?i?like\b/i;
+
+const METADATA_CONTENT_PATTERN =
+ /\b(?:has|position(?:caseinsensitive)?|hasalltokens|hasanytokens|hastoken|startswith|endswith)\s*\(\s*(?:lower\s*\(\s*)?(?:\w+\.)?metadata_(?:names|values)\b/i;
+
+const ID_OR_ILIKE_PATTERN = /\bilike\s*\{\s*searchString\b/i;
+
+const BY_SPAN_ID_PATTERN = /\bspan_id\s*=\s*\{|\bspan_id\s+in\s*(?:\(\s*)?\{/i;
+
+const BY_TRACE_ID_PATTERN =
+ /\btrace_id\s*=\s*\{|\btrace_id\s+in\s*(?:\(\s*)?\{/i;
+
+const QUERY_SHAPE_PATTERNS: ReadonlyArray<[ClickHouseQueryShape, RegExp]> = [
+ ["io_content", IO_CONTENT_FUNCTION_PATTERN],
+ ["io_content", IO_CONTENT_LIKE_PATTERN],
+ ["metadata_content", METADATA_CONTENT_PATTERN],
+ ["id_or_ilike", ID_OR_ILIKE_PATTERN],
+ ["by_span_id", BY_SPAN_ID_PATTERN],
+ ["by_trace_id", BY_TRACE_ID_PATTERN],
+];
+
+export function clickHouseQueryShape(query: string): ClickHouseQueryShape {
+ for (const [shape, pattern] of QUERY_SHAPE_PATTERNS) {
+ if (pattern.test(query)) return shape;
+ }
+ return OTHER_SHAPE_LABEL;
+}
+
export function recordClickHouseQueryOutcome(
outcome: ClickHouseQueryOutcome,
tags: NormalizedClickHouseQueryTags,
table: ClickHouseQueryTable,
+ shape: ClickHouseQueryShape,
): void {
recordIncrement(CLICKHOUSE_QUERY_OUTCOME_METRIC, 1, {
outcome,
surface: tags.surface,
route: clickHouseQueryOutcomeRouteLabel(tags.route),
table,
+ query_shape: shape,
});
}
diff --git a/packages/shared/src/server/repositories/clickhouse.ts b/packages/shared/src/server/repositories/clickhouse.ts
index 7d7e6f5d10..2a8aea8f41 100644
--- a/packages/shared/src/server/repositories/clickhouse.ts
+++ b/packages/shared/src/server/repositories/clickhouse.ts
@@ -33,6 +33,7 @@ import {
} from "../clickhouse/queryTags";
import {
CLICKHOUSE_RESOURCE_ERROR_OUTCOMES,
+ clickHouseQueryShape,
clickHouseQueryTableLabel,
recordClickHouseQueryOutcome,
} from "../clickhouse/queryOutcome";
@@ -628,6 +629,7 @@ function recordSummaryOnSpan(
function setSpanQueryAttributes(span: Span, query: string): void {
span.setAttribute("ch.query.text", query);
+ span.setAttribute("ch.query.shape", clickHouseQueryShape(query));
span.setAttribute("db.system", "clickhouse");
span.setAttribute("db.query.text", query);
span.setAttribute("db.operation.name", "SELECT");
@@ -702,6 +704,7 @@ export async function queryClickhouse<T>(
if (!opts.allowLegacyEventsRead) assertNoLegacyEventsRead(opts.query);
const normalizedTags = normalizeClickHouseQueryTags(opts.tags);
const table = clickHouseQueryTableLabel(opts.query);
+ const shape = clickHouseQueryShape(opts.query);
return await instrumentAsync(
{ name: "clickhouse-query", spanKind: SpanKind.CLIENT },
async (span) => {
@@ -764,11 +767,12 @@ export async function queryClickhouse<T>(
: "error",
normalizedTags,
table,
+ shape,
);
throw wrapped;
});
- recordClickHouseQueryOutcome("success", normalizedTags, table);
+ recordClickHouseQueryOutcome("success", normalizedTags, table, shape);
return rows;
},
);