Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

perf(clickhouse): use events_full ngram indexes for name/user_id/session_id filters (#17577)

Niklas Semmler2aea85c699d6
diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-ngram-columns.test.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-ngram-columns.test.ts
new file mode 100644
index 0000000000..a7c61a5175
--- /dev/null
+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-ngram-columns.test.ts
@@ -0,0 +1,112 @@
+import { describe, expect, it } from "vitest";
+
+import { StringFilter, StringOptionsFilter } from "./clickhouse-filter";
+
+// events_full carries ngrambf_v1 skip indexes on lower(name)/lower(user_id)/
+// lower(session_id). These tests pin that the filter classes emit an
+// index-usable lower() predicate for those columns on events-family tables,
+// while keeping the exact (case-sensitive) predicate for correctness.
+
+const stringFilter = (
+ field: string,
+ operator: StringFilter["operator"],
+ value: string,
+ clickhouseTable = "events_proto",
+) =>
+ new StringFilter({
+ clickhouseTable,
+ field,
+ operator,
+ value,
+ tablePrefix: "e",
+ }).apply();
+
+describe("StringFilter ngram column acceleration (events_full)", () => {
+ it("wraps `=` with a lower() equality conjunct plus the exact match", () => {
+ const { query } = stringFilter("name", "=", "MyName");
+ expect(query).toContain("lower(e.name) = lower({");
+ expect(query).toContain("e.name = {");
+ });
+
+ it("adds a lower() LIKE prefilter for `contains`", () => {
+ const { query, params } = stringFilter("user_id", "contains", "abc");
+ expect(query).toContain("lower(e.user_id) LIKE lower({");
+ expect(query).toContain("position(e.user_id, {");
+ expect(Object.values(params)).toContain("%abc%");
+ });
+
+ it("uses a suffix wildcard for `starts with`", () => {
+ const { query, params } = stringFilter("session_id", "starts with", "sess");
+ expect(query).toContain("lower(e.session_id) LIKE lower({");
+ expect(query).toContain("startsWith(e.session_id, {");
+ expect(Object.values(params)).toContain("sess%");
+ });
+
+ it("uses a prefix wildcard for `ends with`", () => {
+ const { query, params } = stringFilter("name", "ends with", "tail");
+ expect(query).toContain("lower(e.name) LIKE lower({");
+ expect(query).toContain("endsWith(e.name, {");
+ expect(Object.values(params)).toContain("%tail");
+ });
+
+ it("escapes LIKE wildcards in the ngram pattern", () => {
+ const { params } = stringFilter("name", "contains", "50%_off");
+ expect(Object.values(params)).toContain("%50\\%\\_off%");
+ });
+
+ it("does not accelerate `does not contain` (bloom cannot prune negation)", () => {
+ const { query } = stringFilter("name", "does not contain", "abc");
+ expect(query).not.toContain("lower(");
+ });
+
+ it("leaves non-events tables untouched", () => {
+ const { query } = stringFilter("name", "=", "MyName", "traces");
+ expect(query).not.toContain("lower(");
+ expect(query).toContain("e.name = {");
+ });
+
+ it("leaves the trace_name COALESCE column untouched (handled separately)", () => {
+ // trace_name maps to a COALESCE expression, not a bare column; a simple
+ // lower() wrap cannot use the index, so it is deliberately excluded here.
+ const { query } = stringFilter("trace_name", "=", "MyTrace");
+ expect(query).not.toContain("lower(");
+ });
+});
+
+describe("StringOptionsFilter ngram column acceleration (events_full)", () => {
+ it("adds a lowered IN prefilter for `any of`", () => {
+ const { query, params } = new StringOptionsFilter({
+ clickhouseTable: "events_proto",
+ field: "name",
+ operator: "any of",
+ values: ["Foo", "BAR"],
+ tablePrefix: "e",
+ }).apply();
+ expect(query).toContain("lower(e.name) IN ({");
+ expect(query).toContain("e.name IN ({");
+ expect(Object.values(params)).toContainEqual(["foo", "bar"]);
+ });
+
+ it("does not accelerate `none of` (NOT IN cannot be pruned)", () => {
+ const { query } = new StringOptionsFilter({
+ clickhouseTable: "events_proto",
+ field: "name",
+ operator: "none of",
+ values: ["Foo"],
+ tablePrefix: "e",
+ }).apply();
+ expect(query).not.toContain("lower(");
+ });
+
+ it("leaves non-events tables untouched", () => {
+ const { query } = new StringOptionsFilter({
+ clickhouseTable: "traces",
+ field: "name",
+ operator: "any of",
+ values: ["Foo"],
+ tablePrefix: "t",
+ }).apply();
+ expect(query).not.toContain("lower(");
+ expect(query).toMatch(/^t\.name IN \(\{/);
+ });
+});
diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts
index 8c3de3dde2..a1082f819e 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts
@@ -14,6 +14,7 @@ import {
isFtsMetadataField,
isFtsTextField,
isFtsTextTarget,
+ isNgramSubstringTarget,
} from "./fts";
export type ClickhouseOperator =
@@ -36,6 +37,13 @@ const NGRAM_ACCELERATED_METADATA_OPERATORS = new Set<
(typeof filterOperators)["stringObject"][number]
>(["contains", "starts with", "ends with"]);
+// ClickHouse `lower()` (used by the events_full ngram indexes) lowercases ASCII
+// A-Z only, byte-wise, leaving multi-byte UTF-8 untouched. Mirror that exactly
+// so a JS-lowered IN set matches the index and never prunes a real match. Full
+// String.toLowerCase() would diverge on non-ASCII and is unsafe here.
+const clickhouseAsciiLower = (value: string): string =>
+ value.replace(/[A-Z]/g, (c) => c.toLowerCase());
+
// Substring operators with an empty value skip the ngram prefilter and degrade
// to a full-scan "does this key exist" over the whole time window — a confirmed
// events_full timeout vector. Callers who want key existence must use the
@@ -75,6 +83,20 @@ export class StringFilter implements Filter {
const fieldWithPrefix = `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}`;
+ // events_full carries an ngrambf_v1 skip index on lower(<col>) for these
+ // columns. Emit a lower() conjunct so the index can prune; the exact
+ // predicate below keeps the original (case-sensitive) semantics.
+ const ngramTarget = isNgramSubstringTarget(
+ this.clickhouseTable,
+ this.field,
+ );
+ const ngramParams: Record<string, string> = {};
+ const ngramLikeConjunct = (pattern: string): string => {
+ const p = `stringFilterNgram${clickhouseCompliantRandomCharacters()}`;
+ ngramParams[p] = pattern;
+ return `lower(${fieldWithPrefix}) LIKE lower({${p}: String}) AND `;
+ };
+
// '' ≡ NULL: when filtering with empty value, match both '' and NULL.
// ClickHouse functions like startsWith/endsWith/position return NULL (not true)
// for NULL inputs, so we need an explicit OR IS NULL guard.
@@ -102,19 +124,30 @@ export class StringFilter implements Filter {
`{${varName}: String}`,
query,
);
+ } else if (ngramTarget) {
+ query = `(lower(${fieldWithPrefix}) = lower({${varName}: String}) AND ${query})`;
}
break;
case "contains":
query = `position(${fieldWithPrefix}, {${varName}: String}) > 0`;
+ if (ngramTarget && this.value.length > 0) {
+ query = `(${ngramLikeConjunct(`%${escapeSqlLikePattern(this.value)}%`)}${query})`;
+ }
break;
case "does not contain":
query = `position(${fieldWithPrefix}, {${varName}: String}) = 0`;
break;
case "starts with":
query = `startsWith(${fieldWithPrefix}, {${varName}: String})`;
+ if (ngramTarget && this.value.length > 0) {
+ query = `(${ngramLikeConjunct(`${escapeSqlLikePattern(this.value)}%`)}${query})`;
+ }
break;
case "ends with":
query = `endsWith(${fieldWithPrefix}, {${varName}: String})`;
+ if (ngramTarget && this.value.length > 0) {
+ query = `(${ngramLikeConjunct(`%${escapeSqlLikePattern(this.value)}`)}${query})`;
+ }
break;
case "is not empty":
query = `(${fieldWithPrefix} != '' AND ${fieldWithPrefix} IS NOT NULL)`;
@@ -145,7 +178,10 @@ export class StringFilter implements Filter {
return {
query,
- params: this.operator === "is not empty" ? {} : { [varName]: this.value },
+ params:
+ this.operator === "is not empty"
+ ? {}
+ : { [varName]: this.value, ...ngramParams },
};
}
}
@@ -259,9 +295,25 @@ export class StringOptionsFilter implements Filter {
const fieldWithPrefix = `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}`;
const hasEmpty = this.emptyEqualsNull && this.values.includes("");
+ // events_full ngram index prefilter for the positive (any of) set. `none of`
+ // (NOT IN) cannot be pruned by a presence bloom, so it is left untouched.
+ const ngramParams: Record<string, string[]> = {};
+ let ngramConjunct = "";
+ if (
+ this.operator === "any of" &&
+ isNgramSubstringTarget(this.clickhouseTable, this.field) &&
+ this.values.length > 0
+ ) {
+ const loweredVar = `stringOptionsFilterNgram${clickhouseCompliantRandomCharacters()}`;
+ ngramParams[loweredVar] = this.values.map(clickhouseAsciiLower);
+ ngramConjunct = `lower(${fieldWithPrefix}) IN ({${loweredVar}: Array(String)}) AND `;
+ }
+
let query =
this.operator === "any of"
- ? `${fieldWithPrefix} IN ({${varName}: Array(String)})`
+ ? ngramConjunct
+ ? `(${ngramConjunct}${fieldWithPrefix} IN ({${varName}: Array(String)}))`
+ : `${fieldWithPrefix} IN ({${varName}: Array(String)})`
: `${fieldWithPrefix} NOT IN ({${varName}: Array(String)})`;
if (hasEmpty && this.operator === "any of") {
@@ -277,7 +329,7 @@ export class StringOptionsFilter implements Filter {
return {
query,
- params: { [varName]: this.values },
+ params: { [varName]: this.values, ...ngramParams },
};
}
}
diff --git a/packages/shared/src/server/queries/clickhouse-sql/fts.ts b/packages/shared/src/server/queries/clickhouse-sql/fts.ts
index ae8060a82c..268d44e9e9 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/fts.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/fts.ts
@@ -29,6 +29,20 @@ export const FTS_TEXT_FIELDS: ReadonlySet<string> = new Set([
]);
export const FTS_METADATA_FIELD = "metadata";
+// events_full carries ngrambf_v1 skip indexes on lower(<col>) for these
+// human-readable substring-search columns. A predicate uses them only when
+// lower(col) appears directly under a supported function (=, LIKE, IN), so
+// StringFilter/StringOptionsFilter emit a lower() conjunct for these fields.
+// trace_name is intentionally excluded: it maps to a COALESCE expression, not a
+// bare column, and needs a branch-aware rewrite. span_id/trace_id match by
+// equality against their bloom_filter/primary key (handled by the search
+// id-lane), so they need no ngram treatment here.
+const FTS_NGRAM_SUBSTRING_FIELDS: ReadonlySet<string> = new Set([
+ "name",
+ "user_id",
+ "session_id",
+]);
+
// Column mappings may carry a table prefix (e.g. "e.input"); strip to the bare
// field name before set lookup.
export const bareFtsField = (field: string): string => {
@@ -67,6 +81,19 @@ export const isFtsMetadataTarget = (
field: string,
): boolean => isFtsEventsTable(clickhouseTable) && isFtsMetadataField(field);
+const isNgramSubstringField = (field: string): boolean =>
+ FTS_NGRAM_SUBSTRING_FIELDS.has(bareFtsField(field));
+
+// True when a filter targets an events-family table column backed by an
+// events_full lower(col) ngram index. The filter layer cannot tell events_core
+// from events_full (physical selection happens later in EventsQueryBuilder); on
+// events_core the emitted lower() conjunct is simply an unindexed, correct, and
+// cheap extra comparison on these small columns.
+export const isNgramSubstringTarget = (
+ clickhouseTable: string,
+ field: string,
+): boolean => isFtsEventsTable(clickhouseTable) && isNgramSubstringField(field);
+
export const isFtsAcceleratedIoOperator = (operator: string): boolean =>
FTS_TEXT_OPERATORS.has(operator as FtsStringOperator);