🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
/* eslint-disable @repo/no-exotic-operators */
import { InvalidRequestError } from "../../../errors/InvalidRequestError";
import {
hasValidTracingSearchTypes,
TRACING_SEARCH_TYPE_REQUIRED_MESSAGE,
type TracingSearchType,
} from "../../../interfaces/search";
import { escapeSqlLikePattern } from "../../utils/sqlLike";
import { bareFtsField, ftsTextTokenPredicate, hasFtsSearchToken } from "./fts";
const regexIndefiniteCharacters = "%";
// events_full ngrambf_v1 skip indexes are defined on lower(<col>). ILIKE on the
// raw column cannot use them; the query must match lower(col) LIKE lower(...).
// Every leading-wildcard disjunct in the search OR must be index-usable or
// ClickHouse discards the input/output text indexes for the whole query.
const NGRAM_SUBSTRING_COLUMNS = new Set([
"name",
"trace_name",
"user_id",
"session_id",
]);
// span_id/trace_id are UUID-shaped: an ngrambf on hex+hyphen prunes almost
// nothing, so we match by equality against their existing bloom_filter (and,
// for span_id, the primary key) instead of adding an index. Exact-match and
// case-sensitive on the events path; callers paste whole ids.
const EQUALITY_ID_COLUMNS = new Set(["span_id", "trace_id"]);
/**
* Re-encodes a string the way a JSON serializer with `ensure_ascii=True` does (e.g. Python's
* `json.dumps`, used by the Langfuse Python SDK's EventSerializer / the OpenTelemetry ingestion
* path): every code point >= U+0080 becomes a `\uXXXX` escape (astral code points become a
* UTF-16 surrogate pair). ASCII is left untouched.
*
* Trace / observation `input` and `output` ingested through that path is persisted in ClickHouse
* verbatim in this escaped form (e.g. `ä½ å¥½` is stored as the literal `\u4f60\u597d`), so a
* substring search for the raw, human-readable text would never match it. By also searching for
* this escaped form we recover full-text search for non-English content. See issue #11538.
*/
const toJsonUnicodeEscaped = (value: string): string => {
let out = "";
for (const ch of value) {
const cp = ch.codePointAt(0)!;
if (cp < 0x80) {
out += ch;
} else if (cp <= 0xffff) {
out += "\\u" + cp.toString(16).padStart(4, "0");
} else {
const v = cp - 0x10000;
out +=
"\\u" +
(0xd800 + (v >> 10)).toString(16).padStart(4, "0") +
"\\u" +
(0xdc00 + (v & 0x3ff)).toString(16).padStart(4, "0");
}
}
return out;
};
export type ClickhouseSearchConditionOptions = {
query?: string;
searchType?: TracingSearchType[];
tablePrefix?: string;
searchColumns?: readonly string[];
useEventsTablePath?: boolean;
};
export const clickhouseSearchCondition = ({
query,
searchType,
tablePrefix,
searchColumns,
useEventsTablePath = false,
}: ClickhouseSearchConditionOptions) => {
if (!hasValidTracingSearchTypes({ searchQuery: query, searchType })) {
throw new InvalidRequestError(TRACING_SEARCH_TYPE_REQUIRED_MESSAGE);
}
const prefix = tablePrefix ? `${tablePrefix}.` : "";
const ilikeWithPrefilter = (
col: string,
param = "{searchString: String}",
hasToken = false,
) =>
// Fast-mode UI search intentionally narrows IO search to token matches
// before applying ILIKE. This gives ClickHouse an inverted-index lookup,
// but drops embedded-word substring matches like "foobarneedle". The token
// prefilter is only meaningful when the value yields tokens; a tokenless
// value leans on ILIKE alone.
useEventsTablePath && hasToken
? `(${col} ILIKE ${param} AND ${ftsTextTokenPredicate(col, param)})`
: `${col} ILIKE ${param}`;
const idLaneMatch = (col: string) => {
if (useEventsTablePath) {
const bare = bareFtsField(col);
if (EQUALITY_ID_COLUMNS.has(bare)) {
return `${col} = {searchStringExact: String}`;
}
if (NGRAM_SUBSTRING_COLUMNS.has(bare)) {
return `lower(${col}) LIKE lower({searchString: String})`;
}
}
return `${col} ILIKE {searchString: String}`;
};
const defaultCols = [`${prefix}id`, `t.user_id`, `${prefix}name`];
const cols = (searchColumns ?? defaultCols).map((col) =>
col.includes(".") ? col : `${prefix}${col}`,
);
const inputCol = `${prefix}input`;
const outputCol = `${prefix}output`;
const requiresEventsFull =
Boolean(query) &&
Boolean(
searchType?.includes("content") ||
searchType?.includes("input") ||
searchType?.includes("output"),
);
// For input/output (which can hold JSON written by an `ensure_ascii=True` serializer), also
// match the `\uXXXX`-escaped form of the query. ASCII-only queries are unaffected: the escaped
// form is identical, so no extra parameter or clause is emitted (keeps existing behaviour and
// query plans for the common case). See issue #11538 and `toJsonUnicodeEscaped` above.
const escapedQuery = query ? toJsonUnicodeEscaped(query) : undefined;
const hasEscapedVariant = !!query && escapedQuery !== query;
// `hasFtsSearchToken` mirrors ClickHouse's `tokens()` emptiness, so the token
// prefilter can be gated per search value at build time.
const searchStringHasToken = !!query && hasFtsSearchToken(query);
const searchStringEscapedHasToken =
hasEscapedVariant && hasFtsSearchToken(escapedQuery!);
const ioColumnMatch = (col: string) =>
hasEscapedVariant
? `${ilikeWithPrefilter(
col,
"{searchString: String}",
searchStringHasToken,
)} OR ${ilikeWithPrefilter(
col,
"{searchStringEscaped: String}",
searchStringEscapedHasToken,
)}`
: ilikeWithPrefilter(col, "{searchString: String}", searchStringHasToken);
// The default cols include t.user_id for callers querying via traces CTE (traces.ts, observations.ts).
const conditions = [
!searchType || searchType.includes("id")
? cols.map((col) => idLaneMatch(col)).join(" OR ")
: null,
searchType && searchType.includes("content")
? `${ioColumnMatch(inputCol)} OR ${ioColumnMatch(outputCol)}`
: null,
searchType && searchType.includes("input") ? ioColumnMatch(inputCol) : null,
searchType && searchType.includes("output")
? ioColumnMatch(outputCol)
: null,
].filter(Boolean);
return {
query: query ? `AND (${conditions.join(" OR ")})` : "",
requiresEventsFull,
params: query
? {
searchString: `${regexIndefiniteCharacters}${escapeSqlLikePattern(query)}${regexIndefiniteCharacters}`,
// Unwrapped for the events-path equality columns (span_id/trace_id).
...(useEventsTablePath ? { searchStringExact: query } : {}),
...(hasEscapedVariant
? {
searchStringEscaped: `${regexIndefiniteCharacters}${escapeSqlLikePattern(escapedQuery!)}${regexIndefiniteCharacters}`,
}
: {}),
}
: {},
};
};