🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(fts): decide the token prefilter at build time, not with an OR (#17724)
Niklas Semmler25543e6f2c22
diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.tsindex b36ee0a75a..dae050a76c 100644--- a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts@@ -12,6 +12,7 @@ import {assertValidFtsMatchFilter,bareFtsField,FTS_OPERATOR_DESCRIPTORS,+ hasFtsSearchToken,isFtsEventsTable,isFtsMetadataField,isFtsTextField,@@ -134,6 +135,7 @@ export class StringFilter implements Filter {fieldWithPrefix,`{${varName}: String}`,query,+ hasFtsSearchToken(this.value),);} else if (ngramTarget) {query = `(lower(${fieldWithPrefix}) = lower({${varName}: String}) AND ${query})`;@@ -174,8 +176,10 @@ export class StringFilter implements Filter {fieldWithPrefix,`{${varName}: String}`,// `matches` shares the descriptor signature with exact filters but- // does not need a base exact predicate.+ // does not need a base exact predicate, and always has a token+ // (guaranteed by assertValidFtsMatchFilter above)."",+ true,);break;default:@@ -479,6 +483,7 @@ export class StringObjectFilter implements Filter {valuesColumn,valueAccessor,valueParam,+ hasToken: hasFtsSearchToken(this.value),});break;case "contains":@@ -513,6 +518,7 @@ export class StringObjectFilter implements Filter {valuesColumn,valueAccessor,valueParam,+ hasToken: true,});break;default:diff --git a/packages/shared/src/server/queries/clickhouse-sql/fts.ts b/packages/shared/src/server/queries/clickhouse-sql/fts.tsindex 268d44e9e9..7a54d40851 100644--- a/packages/shared/src/server/queries/clickhouse-sql/fts.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/fts.ts@@ -121,14 +121,15 @@ const ftsTokenPrefilterPredicate = (): string =>`hasAllTokens(${fieldExpr}, ${ftsSearchTokenPrefilterExpr(valueParam, normalizeValue)})`;-const ftsTextTokenPredicate = (fieldExpr: string, valueParam: string): string =>- ftsTokenPrefilterPredicate(normalizeFtsTextExpr(fieldExpr), valueParam, true);--export const ftsTextTokenConjunct = (+// Bare `hasAllTokens` index prefilter. Callers include it only when the search+// value yields at least one token (`hasFtsSearchToken`): a tokenless value+// makes `tokens()` empty, leaving the prefilter a no-op that only costs+// skip-index pruning on the surrounding OR.+export const ftsTextTokenPredicate = (fieldExpr: string,valueParam: string,): string =>- `(empty(${ftsSearchTokensExpr(valueParam, true)}) OR ${ftsTextTokenPredicate(fieldExpr, valueParam)})`;+ ftsTokenPrefilterPredicate(normalizeFtsTextExpr(fieldExpr), valueParam, true);const ftsTextIndexedSubstringCondition = (fieldExpr: string,@@ -141,23 +142,14 @@ const ftsMetadataArrayTokenConjunct = (valueParam: string,): string => ftsTokenPrefilterPredicate(arrayExpr, valueParam, false);-// Text-index prefilter for exact metadata equality. The stored value equals the-// search value, so it trivially contains all of the value's tokens: a-// correctness-safe superset that engages `idx_fts_metadata_values`. Guard against-// tokenless values (e.g. "!!!"), where `tokens()` is empty and the prefilter-// would otherwise exclude legitimate matches; the exact `= value` check remains-// the precise post-filter. Metadata is case-sensitive, so tokens are not lowered.-const ftsMetadataArrayEqualityTokenConjunct = (- arrayExpr: string,- valueParam: string,-): string =>- `(empty(${ftsSearchTokensExpr(valueParam, false)}) OR ${ftsMetadataArrayTokenConjunct(arrayExpr, valueParam)})`;-type FtsMetadataArrayConditionContext = {hasKey: string;valuesColumn: string;valueAccessor: string;valueParam: string;+ // Whether the search value yields at least one token; gates the text-index+ // prefilter so tokenless values fall back to the exact check alone.+ hasToken: boolean;};type FtsOperatorDescriptor = {@@ -165,6 +157,7 @@ type FtsOperatorDescriptor = {fieldExpr: string,valueParam: string,exactCondition: string,+ hasToken: boolean,) => string;metadataArrayCondition: (ctx: FtsMetadataArrayConditionContext) => string;};@@ -183,18 +176,23 @@ type FtsOperatorDescriptors = {export const FTS_OPERATOR_DESCRIPTORS = {"=": {- textCondition: (fieldExpr, valueParam, exactCondition) =>- `(${exactCondition} AND ${ftsTextTokenConjunct(fieldExpr, valueParam)})`,+ textCondition: (fieldExpr, valueParam, exactCondition, hasToken) =>+ hasToken+ ? `(${exactCondition} AND ${ftsTextTokenPredicate(fieldExpr, valueParam)})`+ : exactCondition,metadataArrayCondition: ({hasKey,valuesColumn,valueAccessor,valueParam,+ hasToken,}) =>- `${hasKey} AND ${ftsMetadataArrayEqualityTokenConjunct(valuesColumn, valueParam)} AND (${valueAccessor} = ${valueParam})`,+ hasToken+ ? `${hasKey} AND ${ftsMetadataArrayTokenConjunct(valuesColumn, valueParam)} AND (${valueAccessor} = ${valueParam})`+ : `${hasKey} AND (${valueAccessor} = ${valueParam})`,},[FTS_MATCH_OPERATOR]: {- textCondition: (fieldExpr, valueParam, _exactCondition) =>+ textCondition: (fieldExpr, valueParam, _exactCondition, _hasToken) =>ftsTextIndexedSubstringCondition(fieldExpr, valueParam),metadataArrayCondition: ftsMetadataArrayIndexedSubstringCondition,},diff --git a/packages/shared/src/server/queries/clickhouse-sql/search.ts b/packages/shared/src/server/queries/clickhouse-sql/search.tsindex b5e4b4915b..a65219e290 100644--- a/packages/shared/src/server/queries/clickhouse-sql/search.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/search.ts@@ -5,7 +5,7 @@ import {TRACING_SEARCH_TYPE_REQUIRED_MESSAGE,type TracingSearchType,} from "../../../interfaces/search";-import { bareFtsField, ftsTextTokenConjunct } from "./fts";+import { bareFtsField, ftsTextTokenPredicate, hasFtsSearchToken } from "./fts";const regexIndefiniteCharacters = "%";@@ -78,12 +78,18 @@ export const clickhouseSearchCondition = ({const prefix = tablePrefix ? `${tablePrefix}.` : "";- const ilikeWithPrefilter = (col: string, param = "{searchString: String}") =>+ const ilikeWithPrefilter = (+ col: string,+ param = "{searchString: String}",+ hasToken = false,+ ) =>// Fast-mode UI search intentionally narrows IO search to token matches// before applying ILIKE. This gives ClickHouse an inverted-index lookup,- // but drops embedded-word substring matches like "foobarneedle".- useEventsTablePath- ? `(${col} ILIKE ${param} AND ${ftsTextTokenConjunct(col, param)})`+ // but drops embedded-word substring matches like "foobarneedle". The token+ // prefilter is only meaningful when the value yields tokens; a tokenless+ // value leans on ILIKE alone.+ useEventsTablePath && hasToken+ ? `(${col} ILIKE ${param} AND ${ftsTextTokenPredicate(col, param)})`: `${col} ILIKE ${param}`;const idLaneMatch = (col: string) => {@@ -120,13 +126,24 @@ export const clickhouseSearchCondition = ({const escapedQuery = query ? toJsonUnicodeEscaped(query) : undefined;const hasEscapedVariant = !!query && escapedQuery !== query;+ // `hasFtsSearchToken` mirrors ClickHouse's `tokens()` emptiness, so the token+ // prefilter can be gated per search value at build time.+ const searchStringHasToken = !!query && hasFtsSearchToken(query);+ const searchStringEscapedHasToken =+ hasEscapedVariant && hasFtsSearchToken(escapedQuery!);+const ioColumnMatch = (col: string) =>hasEscapedVariant- ? `${ilikeWithPrefilter(col)} OR ${ilikeWithPrefilter(+ ? `${ilikeWithPrefilter(+ col,+ "{searchString: String}",+ searchStringHasToken,+ )} OR ${ilikeWithPrefilter(col,"{searchStringEscaped: String}",+ searchStringEscapedHasToken,)}`- : ilikeWithPrefilter(col);+ : ilikeWithPrefilter(col, "{searchString: String}", searchStringHasToken);// The default cols include t.user_id for callers querying via traces CTE (traces.ts, observations.ts).const conditions = [diff --git a/web/src/__tests__/server/fts-rewrites.servertest.ts b/web/src/__tests__/server/fts-rewrites.servertest.tsindex 30b86e137e..6c678fcff1 100644--- a/web/src/__tests__/server/fts-rewrites.servertest.ts+++ b/web/src/__tests__/server/fts-rewrites.servertest.ts@@ -6,6 +6,7 @@ import {FTS_TEXT_OPERATORS,StringFilter,StringObjectFilter,+ hasFtsSearchToken,queryClickhouse,} from "@langfuse/shared/src/server";@@ -175,8 +176,11 @@ maybeEventsTable("FTS filter rewrites", () => {value,}).apply();+ // The token prefilter is emitted only for FTS-accelerated operators whose+ // value actually yields tokens; a tokenless value (e.g. "!!!") leans on+ // the exact predicate alone.expect(rewritten.query.includes("hasAllTokens")).toBe(- FTS_TEXT_OPERATORS.has(operator),+ FTS_TEXT_OPERATORS.has(operator) && hasFtsSearchToken(value),);await expect(matchingIds(rewritten)).resolves.toEqual(await matchingIds(baseline),@@ -189,6 +193,7 @@ maybeEventsTable("FTS filter rewrites", () => {([{ operator: "=", value: "alpha" },+ { operator: "=", value: "!!!" },{ operator: "contains", value: "alpha" },{ operator: "starts with", value: "alpha" },{ operator: "ends with", value: "alpha" },@@ -217,8 +222,12 @@ maybeEventsTable("FTS filter rewrites", () => {}).apply();if (operator === "=") {- // Equality prefilters with the text index; no non-indexed array scan.- expect(rewritten.query).toContain("hasAllTokens(e.metadata_values,");+ // Equality prefilters with the text index when the value yields tokens;+ // a tokenless value keeps the exact check alone. Neither path adds a+ // non-indexed array scan.+ expect(+ rewritten.query.includes("hasAllTokens(e.metadata_values,"),+ ).toBe(hasFtsSearchToken(value));expect(rewritten.query).not.toMatch(/has\(e\.metadata_values,/);} else if (NGRAM_METADATA_OPERATORS.has(operator)) {expect(rewritten.query).toContain(