Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

6.7 KB
Raw Blame
/* eslint-disable @repo/no-exotic-operators */
import { InvalidRequestError } from "../../../errors/InvalidRequestError";
import {
  hasValidTracingSearchTypes,
  TRACING_SEARCH_TYPE_REQUIRED_MESSAGE,
  type TracingSearchType,
} from "../../../interfaces/search";
import { escapeSqlLikePattern } from "../../utils/sqlLike";
import { bareFtsField, ftsTextTokenPredicate, hasFtsSearchToken } from "./fts";

const regexIndefiniteCharacters = "%";

// events_full ngrambf_v1 skip indexes are defined on lower(<col>). ILIKE on the
// raw column cannot use them; the query must match lower(col) LIKE lower(...).
// Every leading-wildcard disjunct in the search OR must be index-usable or
// ClickHouse discards the input/output text indexes for the whole query.
const NGRAM_SUBSTRING_COLUMNS = new Set([
  "name",
  "trace_name",
  "user_id",
  "session_id",
]);

// span_id/trace_id are UUID-shaped: an ngrambf on hex+hyphen prunes almost
// nothing, so we match by equality against their existing bloom_filter (and,
// for span_id, the primary key) instead of adding an index. Exact-match and
// case-sensitive on the events path; callers paste whole ids.
const EQUALITY_ID_COLUMNS = new Set(["span_id", "trace_id"]);

/**
 * Re-encodes a string the way a JSON serializer with `ensure_ascii=True` does (e.g. Python's
 * `json.dumps`, used by the Langfuse Python SDK's EventSerializer / the OpenTelemetry ingestion
 * path): every code point >= U+0080 becomes a `\uXXXX` escape (astral code points become a
 * UTF-16 surrogate pair). ASCII is left untouched.
 *
 * Trace / observation `input` and `output` ingested through that path is persisted in ClickHouse
 * verbatim in this escaped form (e.g. `你好` is stored as the literal `\u4f60\u597d`), so a
 * substring search for the raw, human-readable text would never match it. By also searching for
 * this escaped form we recover full-text search for non-English content. See issue #11538.
 */
const toJsonUnicodeEscaped = (value: string): string => {
  let out = "";
  for (const ch of value) {
    const cp = ch.codePointAt(0)!;
    if (cp < 0x80) {
      out += ch;
    } else if (cp <= 0xffff) {
      out += "\\u" + cp.toString(16).padStart(4, "0");
    } else {
      const v = cp - 0x10000;
      out +=
        "\\u" +
        (0xd800 + (v >> 10)).toString(16).padStart(4, "0") +
        "\\u" +
        (0xdc00 + (v & 0x3ff)).toString(16).padStart(4, "0");
    }
  }
  return out;
};

export type ClickhouseSearchConditionOptions = {
  query?: string;
  searchType?: TracingSearchType[];
  tablePrefix?: string;
  searchColumns?: readonly string[];
  useEventsTablePath?: boolean;
};

export const clickhouseSearchCondition = ({
  query,
  searchType,
  tablePrefix,
  searchColumns,
  useEventsTablePath = false,
}: ClickhouseSearchConditionOptions) => {
  if (!hasValidTracingSearchTypes({ searchQuery: query, searchType })) {
    throw new InvalidRequestError(TRACING_SEARCH_TYPE_REQUIRED_MESSAGE);
  }

  const prefix = tablePrefix ? `${tablePrefix}.` : "";

  const ilikeWithPrefilter = (
    col: string,
    param = "{searchString: String}",
    hasToken = false,
  ) =>
    // Fast-mode UI search intentionally narrows IO search to token matches
    // before applying ILIKE. This gives ClickHouse an inverted-index lookup,
    // but drops embedded-word substring matches like "foobarneedle". The token
    // prefilter is only meaningful when the value yields tokens; a tokenless
    // value leans on ILIKE alone.
    useEventsTablePath && hasToken
      ? `(${col} ILIKE ${param} AND ${ftsTextTokenPredicate(col, param)})`
      : `${col} ILIKE ${param}`;

  const idLaneMatch = (col: string) => {
    if (useEventsTablePath) {
      const bare = bareFtsField(col);
      if (EQUALITY_ID_COLUMNS.has(bare)) {
        return `${col} = {searchStringExact: String}`;
      }
      if (NGRAM_SUBSTRING_COLUMNS.has(bare)) {
        return `lower(${col}) LIKE lower({searchString: String})`;
      }
    }
    return `${col} ILIKE {searchString: String}`;
  };

  const defaultCols = [`${prefix}id`, `t.user_id`, `${prefix}name`];
  const cols = (searchColumns ?? defaultCols).map((col) =>
    col.includes(".") ? col : `${prefix}${col}`,
  );
  const inputCol = `${prefix}input`;
  const outputCol = `${prefix}output`;
  const requiresEventsFull =
    Boolean(query) &&
    Boolean(
      searchType?.includes("content") ||
      searchType?.includes("input") ||
      searchType?.includes("output"),
    );

  // For input/output (which can hold JSON written by an `ensure_ascii=True` serializer), also
  // match the `\uXXXX`-escaped form of the query. ASCII-only queries are unaffected: the escaped
  // form is identical, so no extra parameter or clause is emitted (keeps existing behaviour and
  // query plans for the common case). See issue #11538 and `toJsonUnicodeEscaped` above.
  const escapedQuery = query ? toJsonUnicodeEscaped(query) : undefined;
  const hasEscapedVariant = !!query && escapedQuery !== query;

  // `hasFtsSearchToken` mirrors ClickHouse's `tokens()` emptiness, so the token
  // prefilter can be gated per search value at build time.
  const searchStringHasToken = !!query && hasFtsSearchToken(query);
  const searchStringEscapedHasToken =
    hasEscapedVariant && hasFtsSearchToken(escapedQuery!);

  const ioColumnMatch = (col: string) =>
    hasEscapedVariant
      ? `${ilikeWithPrefilter(
          col,
          "{searchString: String}",
          searchStringHasToken,
        )} OR ${ilikeWithPrefilter(
          col,
          "{searchStringEscaped: String}",
          searchStringEscapedHasToken,
        )}`
      : ilikeWithPrefilter(col, "{searchString: String}", searchStringHasToken);

  // The default cols include t.user_id for callers querying via traces CTE (traces.ts, observations.ts).
  const conditions = [
    !searchType || searchType.includes("id")
      ? cols.map((col) => idLaneMatch(col)).join(" OR ")
      : null,
    searchType && searchType.includes("content")
      ? `${ioColumnMatch(inputCol)} OR ${ioColumnMatch(outputCol)}`
      : null,
    searchType && searchType.includes("input") ? ioColumnMatch(inputCol) : null,
    searchType && searchType.includes("output")
      ? ioColumnMatch(outputCol)
      : null,
  ].filter(Boolean);

  return {
    query: query ? `AND (${conditions.join(" OR ")})` : "",
    requiresEventsFull,
    params: query
      ? {
          searchString: `${regexIndefiniteCharacters}${escapeSqlLikePattern(query)}${regexIndefiniteCharacters}`,
          // Unwrapped for the events-path equality columns (span_id/trace_id).
          ...(useEventsTablePath ? { searchStringExact: query } : {}),
          ...(hasEscapedVariant
            ? {
                searchStringEscaped: `${regexIndefiniteCharacters}${escapeSqlLikePattern(escapedQuery!)}${regexIndefiniteCharacters}`,
              }
            : {}),
        }
      : {},
  };
};