Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

perf(query): keep truncation-safe metadata filters on events_core (#16894)

Niklas Semmler0be2152b9000
diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.ts
new file mode 100644
index 0000000000..daec275c62
--- /dev/null
+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.ts
@@ -0,0 +1,201 @@
+import { describe, expect, it } from "vitest";
+
+import {
+ BooleanObjectFilter,
+ FilterList,
+ filtersRequireEventsFull,
+ metadataFilterIsEventsCoreSafe,
+ NullFilter,
+ NumberObjectFilter,
+ StringFilter,
+ StringObjectFilter,
+} from "./clickhouse-filter";
+
+// events_core_mv truncates metadata values to leftUTF8(v, 200), so the safety
+// boundary is 200 Unicode code points.
+const under200 = "a".repeat(199);
+const exactly200 = "a".repeat(200);
+const over200 = "a".repeat(500);
+
+const metadataString = (
+ operator: StringObjectFilter["operator"],
+ value: string,
+) =>
+ new StringObjectFilter({
+ clickhouseTable: "events_core",
+ field: "metadata",
+ key: "some_key",
+ operator,
+ value,
+ });
+
+describe("metadataFilterIsEventsCoreSafe", () => {
+ it("keeps `=` on a sub-200 value on events_core", () => {
+ expect(metadataFilterIsEventsCoreSafe("=", under200)).toBe(true);
+ });
+
+ it("routes `=` at exactly 200 to events_full (a >200 stored value shares the first 200 chars)", () => {
+ expect(metadataFilterIsEventsCoreSafe("=", exactly200)).toBe(false);
+ });
+
+ it("routes `=` above 200 to events_full", () => {
+ expect(metadataFilterIsEventsCoreSafe("=", over200)).toBe(false);
+ });
+
+ it("keeps `starts with` up to and including 200 on events_core", () => {
+ expect(metadataFilterIsEventsCoreSafe("starts with", exactly200)).toBe(
+ true,
+ );
+ });
+
+ it("routes `starts with` above 200 to events_full", () => {
+ expect(metadataFilterIsEventsCoreSafe("starts with", over200)).toBe(false);
+ });
+
+ it.each(["contains", "does not contain", "ends with", "matches"] as const)(
+ "routes truncation-sensitive `%s` to events_full regardless of value",
+ (operator) => {
+ expect(metadataFilterIsEventsCoreSafe(operator, "x")).toBe(false);
+ },
+ );
+
+ // Fail-safe default: the classifier is an allow-list, so any operator not
+ // explicitly whitelisted must route to events_full. `!=` and `any of` are
+ // ClickhouseOperator members that are not on the allow-list.
+ it.each(["!=", "any of"] as const)(
+ "routes non-allow-listed `%s` to events_full by default",
+ (operator) => {
+ expect(metadataFilterIsEventsCoreSafe(operator, "x")).toBe(false);
+ },
+ );
+
+ it("keeps numeric metadata comparisons on events_core", () => {
+ expect(metadataFilterIsEventsCoreSafe("=", 42)).toBe(true);
+ expect(metadataFilterIsEventsCoreSafe(">", 42)).toBe(true);
+ });
+
+ it("keeps boolean metadata comparisons on events_core", () => {
+ expect(metadataFilterIsEventsCoreSafe("=", true)).toBe(true);
+ expect(metadataFilterIsEventsCoreSafe("<>", false)).toBe(true);
+ });
+
+ it("keeps null / existence checks on events_core", () => {
+ expect(metadataFilterIsEventsCoreSafe("is null", undefined)).toBe(true);
+ expect(metadataFilterIsEventsCoreSafe("is not null", undefined)).toBe(true);
+ });
+
+ it("counts by code point, not UTF-16 unit (astral chars near the boundary)", () => {
+ // 100 astral code points = 200 UTF-16 units but only 100 code points, so an
+ // `=` filter stays on events_core.
+ const astral = "\u{1F600}".repeat(100);
+ expect(astral.length).toBe(200); // UTF-16 units
+ expect(metadataFilterIsEventsCoreSafe("=", astral)).toBe(true);
+ });
+});
+
+describe("filtersRequireEventsFull metadata routing", () => {
+ const requires = (filter: StringObjectFilter | NumberObjectFilter) =>
+ filtersRequireEventsFull(new FilterList([filter]));
+
+ it("keeps a metadata `=` with a sub-200 param on events_core", () => {
+ // Regression: a >200-char stored value truncates to 200 code points and can
+ // never equal this <200 param, so events_core stays correct.
+ expect(requires(metadataString("=", under200))).toBe(false);
+ });
+
+ it("keeps metadata `starts with` (<=200), numeric, and boolean filters on events_core", () => {
+ expect(requires(metadataString("starts with", exactly200))).toBe(false);
+ expect(
+ requires(
+ new NumberObjectFilter({
+ clickhouseTable: "events_core",
+ field: "metadata",
+ key: "n",
+ operator: "=",
+ value: 5,
+ }),
+ ),
+ ).toBe(false);
+ expect(
+ filtersRequireEventsFull(
+ new FilterList([
+ new BooleanObjectFilter({
+ clickhouseTable: "events_core",
+ field: "metadata",
+ key: "b",
+ operator: "=",
+ value: true,
+ }),
+ ]),
+ ),
+ ).toBe(false);
+ expect(
+ filtersRequireEventsFull(
+ new FilterList([
+ new NullFilter({
+ clickhouseTable: "events_core",
+ field: "metadata",
+ operator: "is not null",
+ }),
+ ]),
+ ),
+ ).toBe(false);
+ });
+
+ it("routes a metadata `contains` to events_full (match may sit past char 200)", () => {
+ expect(requires(metadataString("contains", "needle"))).toBe(true);
+ });
+
+ it("routes metadata `ends with`, `does not contain`, and `=` >=200 to events_full", () => {
+ expect(requires(metadataString("ends with", "x"))).toBe(true);
+ expect(requires(metadataString("does not contain", "x"))).toBe(true);
+ expect(requires(metadataString("=", exactly200))).toBe(true);
+ expect(requires(metadataString("starts with", over200))).toBe(true);
+ });
+});
+
+describe("filtersRequireEventsFull input/output routing", () => {
+ const ioFilter = (field: "input" | "output") =>
+ new StringFilter({
+ clickhouseTable: "events_core",
+ field,
+ operator: "=",
+ value: "short",
+ });
+
+ it("always forces events_full for input/output filters (truncated + no I/O FTS index)", () => {
+ expect(filtersRequireEventsFull(new FilterList([ioFilter("input")]))).toBe(
+ true,
+ );
+ expect(filtersRequireEventsFull(new FilterList([ioFilter("output")]))).toBe(
+ true,
+ );
+ });
+
+ it("ignores filters on non-events tables", () => {
+ expect(
+ filtersRequireEventsFull(
+ new FilterList([
+ new StringObjectFilter({
+ clickhouseTable: "traces",
+ field: "metadata",
+ key: "k",
+ operator: "contains",
+ value: "x",
+ }),
+ ]),
+ ),
+ ).toBe(false);
+ });
+
+ it("forces events_full if any filter in the list is truncation-sensitive", () => {
+ expect(
+ filtersRequireEventsFull(
+ new FilterList([
+ metadataString("=", under200),
+ metadataString("contains", "needle"),
+ ]),
+ ),
+ ).toBe(true);
+ });
+});
diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts
index 71d8aded1a..b44752ad1d 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts
@@ -742,12 +742,105 @@ export class FilterList {
}
}
+// events_core_mv truncates each metadata_values element (and input/output) to
+// the first 200 UTF-8 code points via leftUTF8(v, 200). A metadata filter is
+// only safe to answer against events_core when 200-char truncation cannot
+// change its result.
+const EVENTS_CORE_TRUNCATION_LIMIT = 200;
+
+// Metadata operators that can be answered correctly against the truncated
+// events_core copy — an allow-list, so any operator not named here defaults to
+// events_full. That default is the safe one: routing a decidable filter to
+// events_full only costs performance, whereas routing a truncation-sensitive
+// filter to events_core silently drops matches past code point 200. When a new
+// ClickhouseOperator is added it must be reviewed and added here explicitly
+// before it can use events_core.
+// - `=` / `starts with`: string ops, subject to the length guard below.
+// - `>` `<` `>=` `<=`: numeric comparisons — values are inherently short.
+// - `<>`: boolean not-equal — value is inherently short.
+// - `is null` / `is not null`: truncation-invariant (metadata_names is not
+// truncated; emptiness is unaffected).
+// Deliberately excluded (match can live past code point 200): `contains`,
+// `does not contain`, `ends with`, and the FTS `matches` operator (which also
+// needs the events_full-only index).
+const EVENTS_CORE_SAFE_METADATA_OPERATORS = new Set<ClickhouseOperator>([
+ "=",
+ "starts with",
+ ">",
+ "<",
+ ">=",
+ "<=",
+ "<>",
+ "is null",
+ "is not null",
+]);
+
+// Count Unicode code points to mirror ClickHouse leftUTF8, which truncates by
+// code point rather than UTF-16 unit or byte.
+const codePointLength = (value: string): number => Array.from(value).length;
+
+/**
+ * Truncation-safety classifier: can a single metadata filter be answered
+ * correctly against the truncated events_core copy, or must it read events_full?
+ *
+ * events_core keeps only the first {@link EVENTS_CORE_TRUNCATION_LIMIT} code
+ * points of each metadata value (metadata_names is not truncated). This is an
+ * allow-list: an operator is events_core-safe only if it is named in
+ * {@link EVENTS_CORE_SAFE_METADATA_OPERATORS} and, for the two length-sensitive
+ * string operators, its value fits within the retained prefix:
+ * - `starts with`, value length <= 200 — the prefix lives within retained chars.
+ * - `=`, value length < 200 — a stored value longer than 200 truncates to 200
+ * code points and can never equal a sub-200 value. Strict `<` avoids the
+ * exact-200 false positive where a >200 value shares the first 200 chars.
+ * - `is null` / `is not null` — truncation invariant (metadata_names is
+ * untruncated; emptiness is unaffected).
+ * - numeric / boolean metadata comparisons — the value is inherently short,
+ * so its truncated copy is complete.
+ * Anything else (including any newly added operator) is routed to events_full,
+ * because over-routing only costs performance while under-routing silently
+ * drops matches past the truncation boundary.
+ *
+ * Single source of truth for metadata truncation routing. Do not reuse
+ * NGRAM_ACCELERATED_METADATA_OPERATORS or FTS_TEXT_OPERATORS: those classify
+ * ngram/FTS index eligibility, a different axis (the ngram set includes
+ * truncation-unsafe `contains`/`ends with` and excludes truncation-safe `=`).
+ */
+export const metadataFilterIsEventsCoreSafe = (
+ operator: ClickhouseOperator,
+ value: unknown,
+): boolean => {
+ if (!EVENTS_CORE_SAFE_METADATA_OPERATORS.has(operator)) {
+ return false;
+ }
+ if (typeof value === "string") {
+ if (operator === "starts with") {
+ return codePointLength(value) <= EVENTS_CORE_TRUNCATION_LIMIT;
+ }
+ if (operator === "=") {
+ return codePointLength(value) < EVENTS_CORE_TRUNCATION_LIMIT;
+ }
+ }
+ return true;
+};
+
// events_core stores input/output/metadata_values truncated to 200 chars
-// (events_core_mv); filters on these fields must run against events_full or
-// matches beyond the truncation point are silently dropped.
+// (events_core_mv). A filter must read events_full when truncation could change
+// its result: input/output always (truncated, and events_core lacks the I/O FTS
+// indices); metadata only when the operator/value is truncation-sensitive.
+const filterRequiresEventsFull = (filter: Filter): boolean => {
+ if (!isFtsEventsTable(filter.clickhouseTable)) {
+ return false;
+ }
+ if (isFtsTextField(filter.field)) {
+ return true;
+ }
+ if (isFtsMetadataField(filter.field)) {
+ const value =
+ "value" in filter ? (filter as { value: unknown }).value : undefined;
+ return !metadataFilterIsEventsCoreSafe(filter.operator, value);
+ }
+ return false;
+};
+
export const filtersRequireEventsFull = (filters: FilterList): boolean =>
- filters.some(
- (f) =>
- isFtsEventsTable(f.clickhouseTable) &&
- (isFtsTextField(f.field) || isFtsMetadataField(f.field)),
- );
+ filters.some(filterRequiresEventsFull);
diff --git a/packages/shared/src/server/queries/index.ts b/packages/shared/src/server/queries/index.ts
index 5a35767abb..780cab0d25 100644
--- a/packages/shared/src/server/queries/index.ts
+++ b/packages/shared/src/server/queries/index.ts
@@ -20,6 +20,7 @@ export {
NullFilter,
encodeBooleanScoreEntry,
filtersRequireEventsFull,
+ metadataFilterIsEventsCoreSafe,
type ClickhouseOperator,
} from "./clickhouse-sql/clickhouse-filter";
export {
diff --git a/web/src/__tests__/server/event-query-builder.servertest.ts b/web/src/__tests__/server/event-query-builder.servertest.ts
index adf6894f0d..c9b80eb83c 100644
--- a/web/src/__tests__/server/event-query-builder.servertest.ts
+++ b/web/src/__tests__/server/event-query-builder.servertest.ts
@@ -241,8 +241,11 @@ describe("buildEventsFilterOptionsForColumnsQuery", () => {
projectId: "test-project",
filter: [
{
+ // `contains` is truncation-unsafe (a match can sit past char 200),
+ // so it must force full-table routing — which is what this test
+ // asserts. A short `=`/`starts with` value stays on events_core.
column: "metadata",
- operator: "=",
+ operator: "contains",
key: "region",
value: "eu",
type: "stringObject",