Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

perf(filter-options): accelerate metadata equality with a text-index prefilter (#17353)

Niklas Semmlere9d67b7eabef
diff --git a/packages/shared/src/server/queries/clickhouse-sql/fts.ts b/packages/shared/src/server/queries/clickhouse-sql/fts.ts
index 8df554f50c..ae8060a82c 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/fts.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/fts.ts
@@ -109,14 +109,23 @@ const ftsTextIndexedSubstringCondition = (
): string =>
`(position(${normalizeFtsTextExpr(fieldExpr)}, ${normalizeFtsTextExpr(valueParam)}) > 0 AND ${ftsTextTokenPredicate(fieldExpr, valueParam)})`;
-const ftsMetadataArrayHas = (arrayExpr: string, valueParam: string): string =>
- `has(${arrayExpr}, ${valueParam})`;
-
const ftsMetadataArrayTokenConjunct = (
arrayExpr: string,
valueParam: string,
): string => ftsTokenPrefilterPredicate(arrayExpr, valueParam, false);
+// Text-index prefilter for exact metadata equality. The stored value equals the
+// search value, so it trivially contains all of the value's tokens: a
+// correctness-safe superset that engages `idx_fts_metadata_values`. Guard against
+// tokenless values (e.g. "!!!"), where `tokens()` is empty and the prefilter
+// would otherwise exclude legitimate matches; the exact `= value` check remains
+// the precise post-filter. Metadata is case-sensitive, so tokens are not lowered.
+const ftsMetadataArrayEqualityTokenConjunct = (
+ arrayExpr: string,
+ valueParam: string,
+): string =>
+ `(empty(${ftsSearchTokensExpr(valueParam, false)}) OR ${ftsMetadataArrayTokenConjunct(arrayExpr, valueParam)})`;
+
type FtsMetadataArrayConditionContext = {
hasKey: string;
valuesColumn: string;
@@ -155,7 +164,7 @@ export const FTS_OPERATOR_DESCRIPTORS = {
valueAccessor,
valueParam,
}) =>
- `${hasKey} AND ${ftsMetadataArrayHas(valuesColumn, valueParam)} AND (${valueAccessor} = ${valueParam})`,
+ `${hasKey} AND ${ftsMetadataArrayEqualityTokenConjunct(valuesColumn, valueParam)} AND (${valueAccessor} = ${valueParam})`,
},
[FTS_MATCH_OPERATOR]: {
textCondition: (fieldExpr, valueParam, _exactCondition) =>
diff --git a/web/src/__tests__/server/createFilterFromFilterState-filterTypeValidation.servertest.ts b/web/src/__tests__/server/createFilterFromFilterState-filterTypeValidation.servertest.ts
index 9208865671..29a777f7ed 100644
--- a/web/src/__tests__/server/createFilterFromFilterState-filterTypeValidation.servertest.ts
+++ b/web/src/__tests__/server/createFilterFromFilterState-filterTypeValidation.servertest.ts
@@ -516,6 +516,37 @@ describe("createFilterFromFilterState filter type validation", () => {
expect(Object.values(params)).toEqual(["source", "needle"]);
});
+ it("accelerates event metadata equality with a text-index prefilter", () => {
+ const filters = [
+ {
+ column: "metadata",
+ type: "stringObject",
+ operator: "=",
+ key: "source",
+ value: "needle",
+ },
+ ] satisfies EventsTableFilterState;
+
+ const [result] = createFilterFromFilterState(
+ filters,
+ [mappings.eventMetadata],
+ columnDefinitions,
+ );
+
+ const { query, params } = result.apply();
+
+ expect(query).toContain("has(e.metadata_names,");
+ // `hasAllTokens` engages idx_fts_metadata_values as a correctness-safe superset.
+ expect(query).toContain("hasAllTokens(e.metadata_values,");
+ // Exact equality remains the precise post-filter.
+ expect(query).toContain("e.metadata_values[indexOf(e.metadata_names,");
+ // Equality is served by the index alone; no non-indexed array scan.
+ expect(query).not.toMatch(/has\(e\.metadata_values,/);
+ // Metadata is case-sensitive: tokens are not lowered.
+ expect(query).not.toContain("lower(");
+ expect(Object.values(params)).toEqual(["source", "needle"]);
+ });
+
it("adds ngram prefilter params for event metadata substring filters", () => {
const filters = [
{
diff --git a/web/src/__tests__/server/fts-rewrites.servertest.ts b/web/src/__tests__/server/fts-rewrites.servertest.ts
index 25e56aa26d..30b86e137e 100644
--- a/web/src/__tests__/server/fts-rewrites.servertest.ts
+++ b/web/src/__tests__/server/fts-rewrites.servertest.ts
@@ -217,7 +217,9 @@ maybeEventsTable("FTS filter rewrites", () => {
}).apply();
if (operator === "=") {
- expect(rewritten.query).toContain("has(e.metadata_values,");
+ // Equality prefilters with the text index; no non-indexed array scan.
+ expect(rewritten.query).toContain("hasAllTokens(e.metadata_values,");
+ expect(rewritten.query).not.toMatch(/has\(e\.metadata_values,/);
} else if (NGRAM_METADATA_OPERATORS.has(operator)) {
expect(rewritten.query).toContain(
"like(arrayStringConcat(e.metadata_values),",