🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(filter-options): accelerate metadata equality with a text-index prefilter (#17353)
Niklas Semmlere9d67b7eabef
diff --git a/packages/shared/src/server/queries/clickhouse-sql/fts.ts b/packages/shared/src/server/queries/clickhouse-sql/fts.tsindex 8df554f50c..ae8060a82c 100644--- a/packages/shared/src/server/queries/clickhouse-sql/fts.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/fts.ts@@ -109,14 +109,23 @@ const ftsTextIndexedSubstringCondition = (): string =>`(position(${normalizeFtsTextExpr(fieldExpr)}, ${normalizeFtsTextExpr(valueParam)}) > 0 AND ${ftsTextTokenPredicate(fieldExpr, valueParam)})`;-const ftsMetadataArrayHas = (arrayExpr: string, valueParam: string): string =>- `has(${arrayExpr}, ${valueParam})`;-const ftsMetadataArrayTokenConjunct = (arrayExpr: string,valueParam: string,): string => ftsTokenPrefilterPredicate(arrayExpr, valueParam, false);+// Text-index prefilter for exact metadata equality. The stored value equals the+// search value, so it trivially contains all of the value's tokens: a+// correctness-safe superset that engages `idx_fts_metadata_values`. Guard against+// tokenless values (e.g. "!!!"), where `tokens()` is empty and the prefilter+// would otherwise exclude legitimate matches; the exact `= value` check remains+// the precise post-filter. Metadata is case-sensitive, so tokens are not lowered.+const ftsMetadataArrayEqualityTokenConjunct = (+ arrayExpr: string,+ valueParam: string,+): string =>+ `(empty(${ftsSearchTokensExpr(valueParam, false)}) OR ${ftsMetadataArrayTokenConjunct(arrayExpr, valueParam)})`;+type FtsMetadataArrayConditionContext = {hasKey: string;valuesColumn: string;@@ -155,7 +164,7 @@ export const FTS_OPERATOR_DESCRIPTORS = {valueAccessor,valueParam,}) =>- `${hasKey} AND ${ftsMetadataArrayHas(valuesColumn, valueParam)} AND (${valueAccessor} = ${valueParam})`,+ `${hasKey} AND ${ftsMetadataArrayEqualityTokenConjunct(valuesColumn, valueParam)} AND (${valueAccessor} = ${valueParam})`,},[FTS_MATCH_OPERATOR]: {textCondition: (fieldExpr, valueParam, _exactCondition) =>diff --git a/web/src/__tests__/server/createFilterFromFilterState-filterTypeValidation.servertest.ts b/web/src/__tests__/server/createFilterFromFilterState-filterTypeValidation.servertest.tsindex 9208865671..29a777f7ed 100644--- a/web/src/__tests__/server/createFilterFromFilterState-filterTypeValidation.servertest.ts+++ b/web/src/__tests__/server/createFilterFromFilterState-filterTypeValidation.servertest.ts@@ -516,6 +516,37 @@ describe("createFilterFromFilterState filter type validation", () => {expect(Object.values(params)).toEqual(["source", "needle"]);});+ it("accelerates event metadata equality with a text-index prefilter", () => {+ const filters = [+ {+ column: "metadata",+ type: "stringObject",+ operator: "=",+ key: "source",+ value: "needle",+ },+ ] satisfies EventsTableFilterState;++ const [result] = createFilterFromFilterState(+ filters,+ [mappings.eventMetadata],+ columnDefinitions,+ );++ const { query, params } = result.apply();++ expect(query).toContain("has(e.metadata_names,");+ // `hasAllTokens` engages idx_fts_metadata_values as a correctness-safe superset.+ expect(query).toContain("hasAllTokens(e.metadata_values,");+ // Exact equality remains the precise post-filter.+ expect(query).toContain("e.metadata_values[indexOf(e.metadata_names,");+ // Equality is served by the index alone; no non-indexed array scan.+ expect(query).not.toMatch(/has\(e\.metadata_values,/);+ // Metadata is case-sensitive: tokens are not lowered.+ expect(query).not.toContain("lower(");+ expect(Object.values(params)).toEqual(["source", "needle"]);+ });+it("adds ngram prefilter params for event metadata substring filters", () => {const filters = [{diff --git a/web/src/__tests__/server/fts-rewrites.servertest.ts b/web/src/__tests__/server/fts-rewrites.servertest.tsindex 25e56aa26d..30b86e137e 100644--- a/web/src/__tests__/server/fts-rewrites.servertest.ts+++ b/web/src/__tests__/server/fts-rewrites.servertest.ts@@ -217,7 +217,9 @@ maybeEventsTable("FTS filter rewrites", () => {}).apply();if (operator === "=") {- expect(rewritten.query).toContain("has(e.metadata_values,");+ // Equality prefilters with the text index; no non-indexed array scan.+ expect(rewritten.query).toContain("hasAllTokens(e.metadata_values,");+ expect(rewritten.query).not.toMatch(/has\(e\.metadata_values,/);} else if (NGRAM_METADATA_OPERATORS.has(operator)) {expect(rewritten.query).toContain("like(arrayStringConcat(e.metadata_values),",