🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(clickhouse): use events_full ngram indexes for name/user_id/session_id filters (#17577)
Niklas Semmler2aea85c699d6
diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-ngram-columns.test.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-ngram-columns.test.tsnew file mode 100644index 0000000000..a7c61a5175--- /dev/null+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-ngram-columns.test.ts@@ -0,0 +1,112 @@+import { describe, expect, it } from "vitest";++import { StringFilter, StringOptionsFilter } from "./clickhouse-filter";++// events_full carries ngrambf_v1 skip indexes on lower(name)/lower(user_id)/+// lower(session_id). These tests pin that the filter classes emit an+// index-usable lower() predicate for those columns on events-family tables,+// while keeping the exact (case-sensitive) predicate for correctness.++const stringFilter = (+ field: string,+ operator: StringFilter["operator"],+ value: string,+ clickhouseTable = "events_proto",+) =>+ new StringFilter({+ clickhouseTable,+ field,+ operator,+ value,+ tablePrefix: "e",+ }).apply();++describe("StringFilter ngram column acceleration (events_full)", () => {+ it("wraps `=` with a lower() equality conjunct plus the exact match", () => {+ const { query } = stringFilter("name", "=", "MyName");+ expect(query).toContain("lower(e.name) = lower({");+ expect(query).toContain("e.name = {");+ });++ it("adds a lower() LIKE prefilter for `contains`", () => {+ const { query, params } = stringFilter("user_id", "contains", "abc");+ expect(query).toContain("lower(e.user_id) LIKE lower({");+ expect(query).toContain("position(e.user_id, {");+ expect(Object.values(params)).toContain("%abc%");+ });++ it("uses a suffix wildcard for `starts with`", () => {+ const { query, params } = stringFilter("session_id", "starts with", "sess");+ expect(query).toContain("lower(e.session_id) LIKE lower({");+ expect(query).toContain("startsWith(e.session_id, {");+ expect(Object.values(params)).toContain("sess%");+ });++ it("uses a prefix wildcard for `ends with`", () => {+ const { query, params } = stringFilter("name", "ends with", "tail");+ expect(query).toContain("lower(e.name) LIKE lower({");+ expect(query).toContain("endsWith(e.name, {");+ expect(Object.values(params)).toContain("%tail");+ });++ it("escapes LIKE wildcards in the ngram pattern", () => {+ const { params } = stringFilter("name", "contains", "50%_off");+ expect(Object.values(params)).toContain("%50\\%\\_off%");+ });++ it("does not accelerate `does not contain` (bloom cannot prune negation)", () => {+ const { query } = stringFilter("name", "does not contain", "abc");+ expect(query).not.toContain("lower(");+ });++ it("leaves non-events tables untouched", () => {+ const { query } = stringFilter("name", "=", "MyName", "traces");+ expect(query).not.toContain("lower(");+ expect(query).toContain("e.name = {");+ });++ it("leaves the trace_name COALESCE column untouched (handled separately)", () => {+ // trace_name maps to a COALESCE expression, not a bare column; a simple+ // lower() wrap cannot use the index, so it is deliberately excluded here.+ const { query } = stringFilter("trace_name", "=", "MyTrace");+ expect(query).not.toContain("lower(");+ });+});++describe("StringOptionsFilter ngram column acceleration (events_full)", () => {+ it("adds a lowered IN prefilter for `any of`", () => {+ const { query, params } = new StringOptionsFilter({+ clickhouseTable: "events_proto",+ field: "name",+ operator: "any of",+ values: ["Foo", "BAR"],+ tablePrefix: "e",+ }).apply();+ expect(query).toContain("lower(e.name) IN ({");+ expect(query).toContain("e.name IN ({");+ expect(Object.values(params)).toContainEqual(["foo", "bar"]);+ });++ it("does not accelerate `none of` (NOT IN cannot be pruned)", () => {+ const { query } = new StringOptionsFilter({+ clickhouseTable: "events_proto",+ field: "name",+ operator: "none of",+ values: ["Foo"],+ tablePrefix: "e",+ }).apply();+ expect(query).not.toContain("lower(");+ });++ it("leaves non-events tables untouched", () => {+ const { query } = new StringOptionsFilter({+ clickhouseTable: "traces",+ field: "name",+ operator: "any of",+ values: ["Foo"],+ tablePrefix: "t",+ }).apply();+ expect(query).not.toContain("lower(");+ expect(query).toMatch(/^t\.name IN \(\{/);+ });+});diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.tsindex 8c3de3dde2..a1082f819e 100644--- a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts@@ -14,6 +14,7 @@ import {isFtsMetadataField,isFtsTextField,isFtsTextTarget,+ isNgramSubstringTarget,} from "./fts";export type ClickhouseOperator =@@ -36,6 +37,13 @@ const NGRAM_ACCELERATED_METADATA_OPERATORS = new Set<(typeof filterOperators)["stringObject"][number]>(["contains", "starts with", "ends with"]);+// ClickHouse `lower()` (used by the events_full ngram indexes) lowercases ASCII+// A-Z only, byte-wise, leaving multi-byte UTF-8 untouched. Mirror that exactly+// so a JS-lowered IN set matches the index and never prunes a real match. Full+// String.toLowerCase() would diverge on non-ASCII and is unsafe here.+const clickhouseAsciiLower = (value: string): string =>+ value.replace(/[A-Z]/g, (c) => c.toLowerCase());+// Substring operators with an empty value skip the ngram prefilter and degrade// to a full-scan "does this key exist" over the whole time window — a confirmed// events_full timeout vector. Callers who want key existence must use the@@ -75,6 +83,20 @@ export class StringFilter implements Filter {const fieldWithPrefix = `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}`;+ // events_full carries an ngrambf_v1 skip index on lower(<col>) for these+ // columns. Emit a lower() conjunct so the index can prune; the exact+ // predicate below keeps the original (case-sensitive) semantics.+ const ngramTarget = isNgramSubstringTarget(+ this.clickhouseTable,+ this.field,+ );+ const ngramParams: Record<string, string> = {};+ const ngramLikeConjunct = (pattern: string): string => {+ const p = `stringFilterNgram${clickhouseCompliantRandomCharacters()}`;+ ngramParams[p] = pattern;+ return `lower(${fieldWithPrefix}) LIKE lower({${p}: String}) AND `;+ };+// '' ≡ NULL: when filtering with empty value, match both '' and NULL.// ClickHouse functions like startsWith/endsWith/position return NULL (not true)// for NULL inputs, so we need an explicit OR IS NULL guard.@@ -102,19 +124,30 @@ export class StringFilter implements Filter {`{${varName}: String}`,query,);+ } else if (ngramTarget) {+ query = `(lower(${fieldWithPrefix}) = lower({${varName}: String}) AND ${query})`;}break;case "contains":query = `position(${fieldWithPrefix}, {${varName}: String}) > 0`;+ if (ngramTarget && this.value.length > 0) {+ query = `(${ngramLikeConjunct(`%${escapeSqlLikePattern(this.value)}%`)}${query})`;+ }break;case "does not contain":query = `position(${fieldWithPrefix}, {${varName}: String}) = 0`;break;case "starts with":query = `startsWith(${fieldWithPrefix}, {${varName}: String})`;+ if (ngramTarget && this.value.length > 0) {+ query = `(${ngramLikeConjunct(`${escapeSqlLikePattern(this.value)}%`)}${query})`;+ }break;case "ends with":query = `endsWith(${fieldWithPrefix}, {${varName}: String})`;+ if (ngramTarget && this.value.length > 0) {+ query = `(${ngramLikeConjunct(`%${escapeSqlLikePattern(this.value)}`)}${query})`;+ }break;case "is not empty":query = `(${fieldWithPrefix} != '' AND ${fieldWithPrefix} IS NOT NULL)`;@@ -145,7 +178,10 @@ export class StringFilter implements Filter {return {query,- params: this.operator === "is not empty" ? {} : { [varName]: this.value },+ params:+ this.operator === "is not empty"+ ? {}+ : { [varName]: this.value, ...ngramParams },};}}@@ -259,9 +295,25 @@ export class StringOptionsFilter implements Filter {const fieldWithPrefix = `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}`;const hasEmpty = this.emptyEqualsNull && this.values.includes("");+ // events_full ngram index prefilter for the positive (any of) set. `none of`+ // (NOT IN) cannot be pruned by a presence bloom, so it is left untouched.+ const ngramParams: Record<string, string[]> = {};+ let ngramConjunct = "";+ if (+ this.operator === "any of" &&+ isNgramSubstringTarget(this.clickhouseTable, this.field) &&+ this.values.length > 0+ ) {+ const loweredVar = `stringOptionsFilterNgram${clickhouseCompliantRandomCharacters()}`;+ ngramParams[loweredVar] = this.values.map(clickhouseAsciiLower);+ ngramConjunct = `lower(${fieldWithPrefix}) IN ({${loweredVar}: Array(String)}) AND `;+ }+let query =this.operator === "any of"- ? `${fieldWithPrefix} IN ({${varName}: Array(String)})`+ ? ngramConjunct+ ? `(${ngramConjunct}${fieldWithPrefix} IN ({${varName}: Array(String)}))`+ : `${fieldWithPrefix} IN ({${varName}: Array(String)})`: `${fieldWithPrefix} NOT IN ({${varName}: Array(String)})`;if (hasEmpty && this.operator === "any of") {@@ -277,7 +329,7 @@ export class StringOptionsFilter implements Filter {return {query,- params: { [varName]: this.values },+ params: { [varName]: this.values, ...ngramParams },};}}diff --git a/packages/shared/src/server/queries/clickhouse-sql/fts.ts b/packages/shared/src/server/queries/clickhouse-sql/fts.tsindex ae8060a82c..268d44e9e9 100644--- a/packages/shared/src/server/queries/clickhouse-sql/fts.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/fts.ts@@ -29,6 +29,20 @@ export const FTS_TEXT_FIELDS: ReadonlySet<string> = new Set([]);export const FTS_METADATA_FIELD = "metadata";+// events_full carries ngrambf_v1 skip indexes on lower(<col>) for these+// human-readable substring-search columns. A predicate uses them only when+// lower(col) appears directly under a supported function (=, LIKE, IN), so+// StringFilter/StringOptionsFilter emit a lower() conjunct for these fields.+// trace_name is intentionally excluded: it maps to a COALESCE expression, not a+// bare column, and needs a branch-aware rewrite. span_id/trace_id match by+// equality against their bloom_filter/primary key (handled by the search+// id-lane), so they need no ngram treatment here.+const FTS_NGRAM_SUBSTRING_FIELDS: ReadonlySet<string> = new Set([+ "name",+ "user_id",+ "session_id",+]);+// Column mappings may carry a table prefix (e.g. "e.input"); strip to the bare// field name before set lookup.export const bareFtsField = (field: string): string => {@@ -67,6 +81,19 @@ export const isFtsMetadataTarget = (field: string,): boolean => isFtsEventsTable(clickhouseTable) && isFtsMetadataField(field);+const isNgramSubstringField = (field: string): boolean =>+ FTS_NGRAM_SUBSTRING_FIELDS.has(bareFtsField(field));++// True when a filter targets an events-family table column backed by an+// events_full lower(col) ngram index. The filter layer cannot tell events_core+// from events_full (physical selection happens later in EventsQueryBuilder); on+// events_core the emitted lower() conjunct is simply an unindexed, correct, and+// cheap extra comparison on these small columns.+export const isNgramSubstringTarget = (+ clickhouseTable: string,+ field: string,+): boolean => isFtsEventsTable(clickhouseTable) && isNgramSubstringField(field);+export const isFtsAcceleratedIoOperator = (operator: string): boolean =>FTS_TEXT_OPERATORS.has(operator as FtsStringOperator);