🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
/* eslint-disable @typescript-eslint/no-use-before-define */
import {
FTS_MATCH_OPERATOR,
type FtsMatchOperator,
filterOperators,
} from "../../../interfaces/filters";
import { InvalidRequestError } from "../../../errors/InvalidRequestError";
import { EVENTS_TABLE_NAMES } from "../../clickhouse/schema";
export { FTS_MATCH_OPERATOR } from "../../../interfaces/filters";
type StringOperator = (typeof filterOperators)["string"][number];
export type FtsStringOperator = StringOperator | FtsMatchOperator;
type FtsAcceleratedStringOperator = "=" | FtsMatchOperator;
const FTS_MATCH_TOKEN_ERROR = "`matches` requires at least one search token.";
const FTS_MATCH_TARGET_ERROR =
"`matches` is only supported for input, output, and metadata filters.";
const FTS_TEXT_NORMALIZER = "lower";
const FTS_HAS_ALL_TOKENS_MAX_SEARCH_TOKENS = 64;
export const FTS_EVENTS_TABLES: ReadonlySet<string> = new Set(
EVENTS_TABLE_NAMES,
);
export const FTS_TEXT_FIELDS: ReadonlySet<string> = new Set([
"input",
"output",
]);
export const FTS_METADATA_FIELD = "metadata";
// events_full carries ngrambf_v1 skip indexes on lower(<col>) for these
// human-readable substring-search columns. A predicate uses them only when
// lower(col) appears directly under a supported function (=, LIKE, IN), so
// StringFilter/StringOptionsFilter emit a lower() conjunct for these fields.
// trace_name is intentionally excluded: it maps to a COALESCE expression, not a
// bare column, and needs a branch-aware rewrite. span_id/trace_id match by
// equality against their bloom_filter/primary key (handled by the search
// id-lane), so they need no ngram treatment here.
const FTS_NGRAM_SUBSTRING_FIELDS: ReadonlySet<string> = new Set([
"name",
"user_id",
"session_id",
]);
// Column mappings may carry a table prefix (e.g. "e.input"); strip to the bare
// field name before set lookup.
export const bareFtsField = (field: string): string => {
const dot = field.lastIndexOf(".");
if (dot === -1) return field;
const tail = field.slice(dot + 1);
return tail.replace(/^"(.*)"$/, "$1");
};
export const isFtsEventsTable = (
clickhouseTable: string | undefined,
): boolean =>
clickhouseTable !== undefined && FTS_EVENTS_TABLES.has(clickhouseTable);
export const isFtsMatchOperator = (
operator: string | undefined,
): operator is FtsMatchOperator => operator === FTS_MATCH_OPERATOR;
export const isFtsTextField = (field: string): boolean =>
FTS_TEXT_FIELDS.has(bareFtsField(field));
export const isFtsMetadataField = (field: string): boolean =>
bareFtsField(field) === FTS_METADATA_FIELD;
export const isFtsTextTarget = (
clickhouseTable: string,
field: string,
operator: FtsStringOperator,
): boolean =>
isFtsEventsTable(clickhouseTable) &&
isFtsTextField(field) &&
FTS_TEXT_OPERATORS.has(operator);
export const isFtsMetadataTarget = (
clickhouseTable: string,
field: string,
): boolean => isFtsEventsTable(clickhouseTable) && isFtsMetadataField(field);
const isNgramSubstringField = (field: string): boolean =>
FTS_NGRAM_SUBSTRING_FIELDS.has(bareFtsField(field));
// True when a filter targets an events-family table column backed by an
// events_full lower(col) ngram index. The filter layer cannot tell events_core
// from events_full (physical selection happens later in EventsQueryBuilder); on
// events_core the emitted lower() conjunct is simply an unindexed, correct, and
// cheap extra comparison on these small columns.
export const isNgramSubstringTarget = (
clickhouseTable: string,
field: string,
): boolean => isFtsEventsTable(clickhouseTable) && isNgramSubstringField(field);
export const isFtsAcceleratedIoOperator = (operator: string): boolean =>
FTS_TEXT_OPERATORS.has(operator as FtsStringOperator);
export const hasFtsSearchToken = (value: string): boolean =>
/[\p{L}\p{N}]/u.test(value);
const normalizeFtsTextExpr = (expr: string): string =>
`${FTS_TEXT_NORMALIZER}(${expr})`;
const ftsSearchTokensExpr = (valueParam: string, normalize: boolean): string =>
`arrayDistinct(tokens(${normalize ? normalizeFtsTextExpr(valueParam) : valueParam}))`;
const ftsSearchTokenPrefilterExpr = (
valueParam: string,
normalize: boolean,
): string =>
`arraySlice(${ftsSearchTokensExpr(valueParam, normalize)}, 1, ${FTS_HAS_ALL_TOKENS_MAX_SEARCH_TOKENS})`;
// `hasAllTokens` can only accept up to 64 search tokens. This predicate is only
// an index prefilter; exact equality, ILIKE, or position() enforces semantics.
const ftsTokenPrefilterPredicate = (
fieldExpr: string,
valueParam: string,
normalizeValue: boolean,
): string =>
`hasAllTokens(${fieldExpr}, ${ftsSearchTokenPrefilterExpr(valueParam, normalizeValue)})`;
// Bare `hasAllTokens` index prefilter. Callers include it only when the search
// value yields at least one token (`hasFtsSearchToken`): a tokenless value
// makes `tokens()` empty, leaving the prefilter a no-op that only costs
// skip-index pruning on the surrounding OR.
export const ftsTextTokenPredicate = (
fieldExpr: string,
valueParam: string,
): string =>
ftsTokenPrefilterPredicate(normalizeFtsTextExpr(fieldExpr), valueParam, true);
const ftsTextIndexedSubstringCondition = (
fieldExpr: string,
valueParam: string,
): string =>
`(position(${normalizeFtsTextExpr(fieldExpr)}, ${normalizeFtsTextExpr(valueParam)}) > 0 AND ${ftsTextTokenPredicate(fieldExpr, valueParam)})`;
const ftsMetadataArrayTokenConjunct = (
arrayExpr: string,
valueParam: string,
): string => ftsTokenPrefilterPredicate(arrayExpr, valueParam, false);
type FtsMetadataArrayConditionContext = {
hasKey: string;
valuesColumn: string;
valueAccessor: string;
valueParam: string;
// Whether the search value yields at least one token; gates the text-index
// prefilter so tokenless values fall back to the exact check alone.
hasToken: boolean;
};
type FtsOperatorDescriptor = {
textCondition: (
fieldExpr: string,
valueParam: string,
exactCondition: string,
hasToken: boolean,
) => string;
metadataArrayCondition: (ctx: FtsMetadataArrayConditionContext) => string;
};
const ftsMetadataArrayIndexedSubstringCondition = ({
hasKey,
valuesColumn,
valueAccessor,
valueParam,
}: FtsMetadataArrayConditionContext): string =>
`${hasKey} AND ${ftsMetadataArrayTokenConjunct(valuesColumn, valueParam)} AND (position(${valueAccessor}, ${valueParam}) > 0)`;
type FtsOperatorDescriptors = {
[operator in FtsAcceleratedStringOperator]: FtsOperatorDescriptor;
};
export const FTS_OPERATOR_DESCRIPTORS = {
"=": {
textCondition: (fieldExpr, valueParam, exactCondition, hasToken) =>
hasToken
? `(${exactCondition} AND ${ftsTextTokenPredicate(fieldExpr, valueParam)})`
: exactCondition,
metadataArrayCondition: ({
hasKey,
valuesColumn,
valueAccessor,
valueParam,
hasToken,
}) =>
hasToken
? `${hasKey} AND ${ftsMetadataArrayTokenConjunct(valuesColumn, valueParam)} AND (${valueAccessor} = ${valueParam})`
: `${hasKey} AND (${valueAccessor} = ${valueParam})`,
},
[FTS_MATCH_OPERATOR]: {
textCondition: (fieldExpr, valueParam, _exactCondition, _hasToken) =>
ftsTextIndexedSubstringCondition(fieldExpr, valueParam),
metadataArrayCondition: ftsMetadataArrayIndexedSubstringCondition,
},
} satisfies FtsOperatorDescriptors;
// StringFilter rewrites must preserve filter API semantics. Limit transparent
// text-index rewrites to equality because substring filters are expected to
// match inside larger tokens. `matches` is an explicit indexed literal-search
// operator.
export const FTS_TEXT_OPERATORS: ReadonlySet<FtsStringOperator> = new Set(
Object.keys(FTS_OPERATOR_DESCRIPTORS) as FtsAcceleratedStringOperator[],
);
export const assertValidFtsMatchFilter = (opts: {
filterType: "string" | "stringObject";
clickhouseTable: string;
field: string;
value: string;
}) => {
if (!hasFtsSearchToken(opts.value)) {
throw new InvalidRequestError(FTS_MATCH_TOKEN_ERROR);
}
if (
(opts.filterType === "string" &&
isFtsTextTarget(opts.clickhouseTable, opts.field, FTS_MATCH_OPERATOR)) ||
(opts.filterType === "stringObject" &&
isFtsMetadataTarget(opts.clickhouseTable, opts.field))
) {
return;
}
throw new InvalidRequestError(FTS_MATCH_TARGET_ERROR);
};