Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

8.1 KB
Raw Blame
/* eslint-disable @typescript-eslint/no-use-before-define */
import {
  FTS_MATCH_OPERATOR,
  type FtsMatchOperator,
  filterOperators,
} from "../../../interfaces/filters";
import { InvalidRequestError } from "../../../errors/InvalidRequestError";
import { EVENTS_TABLE_NAMES } from "../../clickhouse/schema";

export { FTS_MATCH_OPERATOR } from "../../../interfaces/filters";

type StringOperator = (typeof filterOperators)["string"][number];
export type FtsStringOperator = StringOperator | FtsMatchOperator;
type FtsAcceleratedStringOperator = "=" | FtsMatchOperator;

const FTS_MATCH_TOKEN_ERROR = "`matches` requires at least one search token.";
const FTS_MATCH_TARGET_ERROR =
  "`matches` is only supported for input, output, and metadata filters.";

const FTS_TEXT_NORMALIZER = "lower";
const FTS_HAS_ALL_TOKENS_MAX_SEARCH_TOKENS = 64;

export const FTS_EVENTS_TABLES: ReadonlySet<string> = new Set(
  EVENTS_TABLE_NAMES,
);

export const FTS_TEXT_FIELDS: ReadonlySet<string> = new Set([
  "input",
  "output",
]);
export const FTS_METADATA_FIELD = "metadata";

// events_full carries ngrambf_v1 skip indexes on lower(<col>) for these
// human-readable substring-search columns. A predicate uses them only when
// lower(col) appears directly under a supported function (=, LIKE, IN), so
// StringFilter/StringOptionsFilter emit a lower() conjunct for these fields.
// trace_name is intentionally excluded: it maps to a COALESCE expression, not a
// bare column, and needs a branch-aware rewrite. span_id/trace_id match by
// equality against their bloom_filter/primary key (handled by the search
// id-lane), so they need no ngram treatment here.
const FTS_NGRAM_SUBSTRING_FIELDS: ReadonlySet<string> = new Set([
  "name",
  "user_id",
  "session_id",
]);

// Column mappings may carry a table prefix (e.g. "e.input"); strip to the bare
// field name before set lookup.
export const bareFtsField = (field: string): string => {
  const dot = field.lastIndexOf(".");
  if (dot === -1) return field;
  const tail = field.slice(dot + 1);
  return tail.replace(/^"(.*)"$/, "$1");
};

export const isFtsEventsTable = (
  clickhouseTable: string | undefined,
): boolean =>
  clickhouseTable !== undefined && FTS_EVENTS_TABLES.has(clickhouseTable);

export const isFtsMatchOperator = (
  operator: string | undefined,
): operator is FtsMatchOperator => operator === FTS_MATCH_OPERATOR;

export const isFtsTextField = (field: string): boolean =>
  FTS_TEXT_FIELDS.has(bareFtsField(field));

export const isFtsMetadataField = (field: string): boolean =>
  bareFtsField(field) === FTS_METADATA_FIELD;

export const isFtsTextTarget = (
  clickhouseTable: string,
  field: string,
  operator: FtsStringOperator,
): boolean =>
  isFtsEventsTable(clickhouseTable) &&
  isFtsTextField(field) &&
  FTS_TEXT_OPERATORS.has(operator);

export const isFtsMetadataTarget = (
  clickhouseTable: string,
  field: string,
): boolean => isFtsEventsTable(clickhouseTable) && isFtsMetadataField(field);

const isNgramSubstringField = (field: string): boolean =>
  FTS_NGRAM_SUBSTRING_FIELDS.has(bareFtsField(field));

// True when a filter targets an events-family table column backed by an
// events_full lower(col) ngram index. The filter layer cannot tell events_core
// from events_full (physical selection happens later in EventsQueryBuilder); on
// events_core the emitted lower() conjunct is simply an unindexed, correct, and
// cheap extra comparison on these small columns.
export const isNgramSubstringTarget = (
  clickhouseTable: string,
  field: string,
): boolean => isFtsEventsTable(clickhouseTable) && isNgramSubstringField(field);

export const isFtsAcceleratedIoOperator = (operator: string): boolean =>
  FTS_TEXT_OPERATORS.has(operator as FtsStringOperator);

export const hasFtsSearchToken = (value: string): boolean =>
  /[\p{L}\p{N}]/u.test(value);

const normalizeFtsTextExpr = (expr: string): string =>
  `${FTS_TEXT_NORMALIZER}(${expr})`;

const ftsSearchTokensExpr = (valueParam: string, normalize: boolean): string =>
  `arrayDistinct(tokens(${normalize ? normalizeFtsTextExpr(valueParam) : valueParam}))`;

const ftsSearchTokenPrefilterExpr = (
  valueParam: string,
  normalize: boolean,
): string =>
  `arraySlice(${ftsSearchTokensExpr(valueParam, normalize)}, 1, ${FTS_HAS_ALL_TOKENS_MAX_SEARCH_TOKENS})`;

// `hasAllTokens` can only accept up to 64 search tokens. This predicate is only
// an index prefilter; exact equality, ILIKE, or position() enforces semantics.
const ftsTokenPrefilterPredicate = (
  fieldExpr: string,
  valueParam: string,
  normalizeValue: boolean,
): string =>
  `hasAllTokens(${fieldExpr}, ${ftsSearchTokenPrefilterExpr(valueParam, normalizeValue)})`;

// Bare `hasAllTokens` index prefilter. Callers include it only when the search
// value yields at least one token (`hasFtsSearchToken`): a tokenless value
// makes `tokens()` empty, leaving the prefilter a no-op that only costs
// skip-index pruning on the surrounding OR.
export const ftsTextTokenPredicate = (
  fieldExpr: string,
  valueParam: string,
): string =>
  ftsTokenPrefilterPredicate(normalizeFtsTextExpr(fieldExpr), valueParam, true);

const ftsTextIndexedSubstringCondition = (
  fieldExpr: string,
  valueParam: string,
): string =>
  `(position(${normalizeFtsTextExpr(fieldExpr)}, ${normalizeFtsTextExpr(valueParam)}) > 0 AND ${ftsTextTokenPredicate(fieldExpr, valueParam)})`;

const ftsMetadataArrayTokenConjunct = (
  arrayExpr: string,
  valueParam: string,
): string => ftsTokenPrefilterPredicate(arrayExpr, valueParam, false);

type FtsMetadataArrayConditionContext = {
  hasKey: string;
  valuesColumn: string;
  valueAccessor: string;
  valueParam: string;
  // Whether the search value yields at least one token; gates the text-index
  // prefilter so tokenless values fall back to the exact check alone.
  hasToken: boolean;
};

type FtsOperatorDescriptor = {
  textCondition: (
    fieldExpr: string,
    valueParam: string,
    exactCondition: string,
    hasToken: boolean,
  ) => string;
  metadataArrayCondition: (ctx: FtsMetadataArrayConditionContext) => string;
};

const ftsMetadataArrayIndexedSubstringCondition = ({
  hasKey,
  valuesColumn,
  valueAccessor,
  valueParam,
}: FtsMetadataArrayConditionContext): string =>
  `${hasKey} AND ${ftsMetadataArrayTokenConjunct(valuesColumn, valueParam)} AND (position(${valueAccessor}, ${valueParam}) > 0)`;

type FtsOperatorDescriptors = {
  [operator in FtsAcceleratedStringOperator]: FtsOperatorDescriptor;
};

export const FTS_OPERATOR_DESCRIPTORS = {
  "=": {
    textCondition: (fieldExpr, valueParam, exactCondition, hasToken) =>
      hasToken
        ? `(${exactCondition} AND ${ftsTextTokenPredicate(fieldExpr, valueParam)})`
        : exactCondition,
    metadataArrayCondition: ({
      hasKey,
      valuesColumn,
      valueAccessor,
      valueParam,
      hasToken,
    }) =>
      hasToken
        ? `${hasKey} AND ${ftsMetadataArrayTokenConjunct(valuesColumn, valueParam)} AND (${valueAccessor} = ${valueParam})`
        : `${hasKey} AND (${valueAccessor} = ${valueParam})`,
  },
  [FTS_MATCH_OPERATOR]: {
    textCondition: (fieldExpr, valueParam, _exactCondition, _hasToken) =>
      ftsTextIndexedSubstringCondition(fieldExpr, valueParam),
    metadataArrayCondition: ftsMetadataArrayIndexedSubstringCondition,
  },
} satisfies FtsOperatorDescriptors;

// StringFilter rewrites must preserve filter API semantics. Limit transparent
// text-index rewrites to equality because substring filters are expected to
// match inside larger tokens. `matches` is an explicit indexed literal-search
// operator.
export const FTS_TEXT_OPERATORS: ReadonlySet<FtsStringOperator> = new Set(
  Object.keys(FTS_OPERATOR_DESCRIPTORS) as FtsAcceleratedStringOperator[],
);

export const assertValidFtsMatchFilter = (opts: {
  filterType: "string" | "stringObject";
  clickhouseTable: string;
  field: string;
  value: string;
}) => {
  if (!hasFtsSearchToken(opts.value)) {
    throw new InvalidRequestError(FTS_MATCH_TOKEN_ERROR);
  }

  if (
    (opts.filterType === "string" &&
      isFtsTextTarget(opts.clickhouseTable, opts.field, FTS_MATCH_OPERATOR)) ||
    (opts.filterType === "stringObject" &&
      isFtsMetadataTarget(opts.clickhouseTable, opts.field))
  ) {
    return;
  }

  throw new InvalidRequestError(FTS_MATCH_TARGET_ERROR);
};