Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

perf(fts): decide the token prefilter at build time, not with an OR (#17724)

Niklas Semmler25543e6f2c22
diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts
index b36ee0a75a..dae050a76c 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts
@@ -12,6 +12,7 @@ import {
assertValidFtsMatchFilter,
bareFtsField,
FTS_OPERATOR_DESCRIPTORS,
+ hasFtsSearchToken,
isFtsEventsTable,
isFtsMetadataField,
isFtsTextField,
@@ -134,6 +135,7 @@ export class StringFilter implements Filter {
fieldWithPrefix,
`{${varName}: String}`,
query,
+ hasFtsSearchToken(this.value),
);
} else if (ngramTarget) {
query = `(lower(${fieldWithPrefix}) = lower({${varName}: String}) AND ${query})`;
@@ -174,8 +176,10 @@ export class StringFilter implements Filter {
fieldWithPrefix,
`{${varName}: String}`,
// `matches` shares the descriptor signature with exact filters but
- // does not need a base exact predicate.
+ // does not need a base exact predicate, and always has a token
+ // (guaranteed by assertValidFtsMatchFilter above).
"",
+ true,
);
break;
default:
@@ -479,6 +483,7 @@ export class StringObjectFilter implements Filter {
valuesColumn,
valueAccessor,
valueParam,
+ hasToken: hasFtsSearchToken(this.value),
});
break;
case "contains":
@@ -513,6 +518,7 @@ export class StringObjectFilter implements Filter {
valuesColumn,
valueAccessor,
valueParam,
+ hasToken: true,
});
break;
default:
diff --git a/packages/shared/src/server/queries/clickhouse-sql/fts.ts b/packages/shared/src/server/queries/clickhouse-sql/fts.ts
index 268d44e9e9..7a54d40851 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/fts.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/fts.ts
@@ -121,14 +121,15 @@ const ftsTokenPrefilterPredicate = (
): string =>
`hasAllTokens(${fieldExpr}, ${ftsSearchTokenPrefilterExpr(valueParam, normalizeValue)})`;
-const ftsTextTokenPredicate = (fieldExpr: string, valueParam: string): string =>
- ftsTokenPrefilterPredicate(normalizeFtsTextExpr(fieldExpr), valueParam, true);
-
-export const ftsTextTokenConjunct = (
+// Bare `hasAllTokens` index prefilter. Callers include it only when the search
+// value yields at least one token (`hasFtsSearchToken`): a tokenless value
+// makes `tokens()` empty, leaving the prefilter a no-op that only costs
+// skip-index pruning on the surrounding OR.
+export const ftsTextTokenPredicate = (
fieldExpr: string,
valueParam: string,
): string =>
- `(empty(${ftsSearchTokensExpr(valueParam, true)}) OR ${ftsTextTokenPredicate(fieldExpr, valueParam)})`;
+ ftsTokenPrefilterPredicate(normalizeFtsTextExpr(fieldExpr), valueParam, true);
const ftsTextIndexedSubstringCondition = (
fieldExpr: string,
@@ -141,23 +142,14 @@ const ftsMetadataArrayTokenConjunct = (
valueParam: string,
): string => ftsTokenPrefilterPredicate(arrayExpr, valueParam, false);
-// Text-index prefilter for exact metadata equality. The stored value equals the
-// search value, so it trivially contains all of the value's tokens: a
-// correctness-safe superset that engages `idx_fts_metadata_values`. Guard against
-// tokenless values (e.g. "!!!"), where `tokens()` is empty and the prefilter
-// would otherwise exclude legitimate matches; the exact `= value` check remains
-// the precise post-filter. Metadata is case-sensitive, so tokens are not lowered.
-const ftsMetadataArrayEqualityTokenConjunct = (
- arrayExpr: string,
- valueParam: string,
-): string =>
- `(empty(${ftsSearchTokensExpr(valueParam, false)}) OR ${ftsMetadataArrayTokenConjunct(arrayExpr, valueParam)})`;
-
type FtsMetadataArrayConditionContext = {
hasKey: string;
valuesColumn: string;
valueAccessor: string;
valueParam: string;
+ // Whether the search value yields at least one token; gates the text-index
+ // prefilter so tokenless values fall back to the exact check alone.
+ hasToken: boolean;
};
type FtsOperatorDescriptor = {
@@ -165,6 +157,7 @@ type FtsOperatorDescriptor = {
fieldExpr: string,
valueParam: string,
exactCondition: string,
+ hasToken: boolean,
) => string;
metadataArrayCondition: (ctx: FtsMetadataArrayConditionContext) => string;
};
@@ -183,18 +176,23 @@ type FtsOperatorDescriptors = {
export const FTS_OPERATOR_DESCRIPTORS = {
"=": {
- textCondition: (fieldExpr, valueParam, exactCondition) =>
- `(${exactCondition} AND ${ftsTextTokenConjunct(fieldExpr, valueParam)})`,
+ textCondition: (fieldExpr, valueParam, exactCondition, hasToken) =>
+ hasToken
+ ? `(${exactCondition} AND ${ftsTextTokenPredicate(fieldExpr, valueParam)})`
+ : exactCondition,
metadataArrayCondition: ({
hasKey,
valuesColumn,
valueAccessor,
valueParam,
+ hasToken,
}) =>
- `${hasKey} AND ${ftsMetadataArrayEqualityTokenConjunct(valuesColumn, valueParam)} AND (${valueAccessor} = ${valueParam})`,
+ hasToken
+ ? `${hasKey} AND ${ftsMetadataArrayTokenConjunct(valuesColumn, valueParam)} AND (${valueAccessor} = ${valueParam})`
+ : `${hasKey} AND (${valueAccessor} = ${valueParam})`,
},
[FTS_MATCH_OPERATOR]: {
- textCondition: (fieldExpr, valueParam, _exactCondition) =>
+ textCondition: (fieldExpr, valueParam, _exactCondition, _hasToken) =>
ftsTextIndexedSubstringCondition(fieldExpr, valueParam),
metadataArrayCondition: ftsMetadataArrayIndexedSubstringCondition,
},
diff --git a/packages/shared/src/server/queries/clickhouse-sql/search.ts b/packages/shared/src/server/queries/clickhouse-sql/search.ts
index b5e4b4915b..a65219e290 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/search.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/search.ts
@@ -5,7 +5,7 @@ import {
TRACING_SEARCH_TYPE_REQUIRED_MESSAGE,
type TracingSearchType,
} from "../../../interfaces/search";
-import { bareFtsField, ftsTextTokenConjunct } from "./fts";
+import { bareFtsField, ftsTextTokenPredicate, hasFtsSearchToken } from "./fts";
const regexIndefiniteCharacters = "%";
@@ -78,12 +78,18 @@ export const clickhouseSearchCondition = ({
const prefix = tablePrefix ? `${tablePrefix}.` : "";
- const ilikeWithPrefilter = (col: string, param = "{searchString: String}") =>
+ const ilikeWithPrefilter = (
+ col: string,
+ param = "{searchString: String}",
+ hasToken = false,
+ ) =>
// Fast-mode UI search intentionally narrows IO search to token matches
// before applying ILIKE. This gives ClickHouse an inverted-index lookup,
- // but drops embedded-word substring matches like "foobarneedle".
- useEventsTablePath
- ? `(${col} ILIKE ${param} AND ${ftsTextTokenConjunct(col, param)})`
+ // but drops embedded-word substring matches like "foobarneedle". The token
+ // prefilter is only meaningful when the value yields tokens; a tokenless
+ // value leans on ILIKE alone.
+ useEventsTablePath && hasToken
+ ? `(${col} ILIKE ${param} AND ${ftsTextTokenPredicate(col, param)})`
: `${col} ILIKE ${param}`;
const idLaneMatch = (col: string) => {
@@ -120,13 +126,24 @@ export const clickhouseSearchCondition = ({
const escapedQuery = query ? toJsonUnicodeEscaped(query) : undefined;
const hasEscapedVariant = !!query && escapedQuery !== query;
+ // `hasFtsSearchToken` mirrors ClickHouse's `tokens()` emptiness, so the token
+ // prefilter can be gated per search value at build time.
+ const searchStringHasToken = !!query && hasFtsSearchToken(query);
+ const searchStringEscapedHasToken =
+ hasEscapedVariant && hasFtsSearchToken(escapedQuery!);
+
const ioColumnMatch = (col: string) =>
hasEscapedVariant
- ? `${ilikeWithPrefilter(col)} OR ${ilikeWithPrefilter(
+ ? `${ilikeWithPrefilter(
+ col,
+ "{searchString: String}",
+ searchStringHasToken,
+ )} OR ${ilikeWithPrefilter(
col,
"{searchStringEscaped: String}",
+ searchStringEscapedHasToken,
)}`
- : ilikeWithPrefilter(col);
+ : ilikeWithPrefilter(col, "{searchString: String}", searchStringHasToken);
// The default cols include t.user_id for callers querying via traces CTE (traces.ts, observations.ts).
const conditions = [
diff --git a/web/src/__tests__/server/fts-rewrites.servertest.ts b/web/src/__tests__/server/fts-rewrites.servertest.ts
index 30b86e137e..6c678fcff1 100644
--- a/web/src/__tests__/server/fts-rewrites.servertest.ts
+++ b/web/src/__tests__/server/fts-rewrites.servertest.ts
@@ -6,6 +6,7 @@ import {
FTS_TEXT_OPERATORS,
StringFilter,
StringObjectFilter,
+ hasFtsSearchToken,
queryClickhouse,
} from "@langfuse/shared/src/server";
@@ -175,8 +176,11 @@ maybeEventsTable("FTS filter rewrites", () => {
value,
}).apply();
+ // The token prefilter is emitted only for FTS-accelerated operators whose
+ // value actually yields tokens; a tokenless value (e.g. "!!!") leans on
+ // the exact predicate alone.
expect(rewritten.query.includes("hasAllTokens")).toBe(
- FTS_TEXT_OPERATORS.has(operator),
+ FTS_TEXT_OPERATORS.has(operator) && hasFtsSearchToken(value),
);
await expect(matchingIds(rewritten)).resolves.toEqual(
await matchingIds(baseline),
@@ -189,6 +193,7 @@ maybeEventsTable("FTS filter rewrites", () => {
(
[
{ operator: "=", value: "alpha" },
+ { operator: "=", value: "!!!" },
{ operator: "contains", value: "alpha" },
{ operator: "starts with", value: "alpha" },
{ operator: "ends with", value: "alpha" },
@@ -217,8 +222,12 @@ maybeEventsTable("FTS filter rewrites", () => {
}).apply();
if (operator === "=") {
- // Equality prefilters with the text index; no non-indexed array scan.
- expect(rewritten.query).toContain("hasAllTokens(e.metadata_values,");
+ // Equality prefilters with the text index when the value yields tokens;
+ // a tokenless value keeps the exact check alone. Neither path adds a
+ // non-indexed array scan.
+ expect(
+ rewritten.query.includes("hasAllTokens(e.metadata_values,"),
+ ).toBe(hasFtsSearchToken(value));
expect(rewritten.query).not.toMatch(/has\(e\.metadata_values,/);
} else if (NGRAM_METADATA_OPERATORS.has(operator)) {
expect(rewritten.query).toContain(