🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
perf(query): keep truncation-safe metadata filters on events_core (#16894)
Niklas Semmler0be2152b9000
diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.tsnew file mode 100644index 0000000000..daec275c62--- /dev/null+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.ts@@ -0,0 +1,201 @@+import { describe, expect, it } from "vitest";++import {+ BooleanObjectFilter,+ FilterList,+ filtersRequireEventsFull,+ metadataFilterIsEventsCoreSafe,+ NullFilter,+ NumberObjectFilter,+ StringFilter,+ StringObjectFilter,+} from "./clickhouse-filter";++// events_core_mv truncates metadata values to leftUTF8(v, 200), so the safety+// boundary is 200 Unicode code points.+const under200 = "a".repeat(199);+const exactly200 = "a".repeat(200);+const over200 = "a".repeat(500);++const metadataString = (+ operator: StringObjectFilter["operator"],+ value: string,+) =>+ new StringObjectFilter({+ clickhouseTable: "events_core",+ field: "metadata",+ key: "some_key",+ operator,+ value,+ });++describe("metadataFilterIsEventsCoreSafe", () => {+ it("keeps `=` on a sub-200 value on events_core", () => {+ expect(metadataFilterIsEventsCoreSafe("=", under200)).toBe(true);+ });++ it("routes `=` at exactly 200 to events_full (a >200 stored value shares the first 200 chars)", () => {+ expect(metadataFilterIsEventsCoreSafe("=", exactly200)).toBe(false);+ });++ it("routes `=` above 200 to events_full", () => {+ expect(metadataFilterIsEventsCoreSafe("=", over200)).toBe(false);+ });++ it("keeps `starts with` up to and including 200 on events_core", () => {+ expect(metadataFilterIsEventsCoreSafe("starts with", exactly200)).toBe(+ true,+ );+ });++ it("routes `starts with` above 200 to events_full", () => {+ expect(metadataFilterIsEventsCoreSafe("starts with", over200)).toBe(false);+ });++ it.each(["contains", "does not contain", "ends with", "matches"] as const)(+ "routes truncation-sensitive `%s` to events_full regardless of value",+ (operator) => {+ expect(metadataFilterIsEventsCoreSafe(operator, "x")).toBe(false);+ },+ );++ // Fail-safe default: the classifier is an allow-list, so any operator not+ // explicitly whitelisted must route to events_full. `!=` and `any of` are+ // ClickhouseOperator members that are not on the allow-list.+ it.each(["!=", "any of"] as const)(+ "routes non-allow-listed `%s` to events_full by default",+ (operator) => {+ expect(metadataFilterIsEventsCoreSafe(operator, "x")).toBe(false);+ },+ );++ it("keeps numeric metadata comparisons on events_core", () => {+ expect(metadataFilterIsEventsCoreSafe("=", 42)).toBe(true);+ expect(metadataFilterIsEventsCoreSafe(">", 42)).toBe(true);+ });++ it("keeps boolean metadata comparisons on events_core", () => {+ expect(metadataFilterIsEventsCoreSafe("=", true)).toBe(true);+ expect(metadataFilterIsEventsCoreSafe("<>", false)).toBe(true);+ });++ it("keeps null / existence checks on events_core", () => {+ expect(metadataFilterIsEventsCoreSafe("is null", undefined)).toBe(true);+ expect(metadataFilterIsEventsCoreSafe("is not null", undefined)).toBe(true);+ });++ it("counts by code point, not UTF-16 unit (astral chars near the boundary)", () => {+ // 100 astral code points = 200 UTF-16 units but only 100 code points, so an+ // `=` filter stays on events_core.+ const astral = "\u{1F600}".repeat(100);+ expect(astral.length).toBe(200); // UTF-16 units+ expect(metadataFilterIsEventsCoreSafe("=", astral)).toBe(true);+ });+});++describe("filtersRequireEventsFull metadata routing", () => {+ const requires = (filter: StringObjectFilter | NumberObjectFilter) =>+ filtersRequireEventsFull(new FilterList([filter]));++ it("keeps a metadata `=` with a sub-200 param on events_core", () => {+ // Regression: a >200-char stored value truncates to 200 code points and can+ // never equal this <200 param, so events_core stays correct.+ expect(requires(metadataString("=", under200))).toBe(false);+ });++ it("keeps metadata `starts with` (<=200), numeric, and boolean filters on events_core", () => {+ expect(requires(metadataString("starts with", exactly200))).toBe(false);+ expect(+ requires(+ new NumberObjectFilter({+ clickhouseTable: "events_core",+ field: "metadata",+ key: "n",+ operator: "=",+ value: 5,+ }),+ ),+ ).toBe(false);+ expect(+ filtersRequireEventsFull(+ new FilterList([+ new BooleanObjectFilter({+ clickhouseTable: "events_core",+ field: "metadata",+ key: "b",+ operator: "=",+ value: true,+ }),+ ]),+ ),+ ).toBe(false);+ expect(+ filtersRequireEventsFull(+ new FilterList([+ new NullFilter({+ clickhouseTable: "events_core",+ field: "metadata",+ operator: "is not null",+ }),+ ]),+ ),+ ).toBe(false);+ });++ it("routes a metadata `contains` to events_full (match may sit past char 200)", () => {+ expect(requires(metadataString("contains", "needle"))).toBe(true);+ });++ it("routes metadata `ends with`, `does not contain`, and `=` >=200 to events_full", () => {+ expect(requires(metadataString("ends with", "x"))).toBe(true);+ expect(requires(metadataString("does not contain", "x"))).toBe(true);+ expect(requires(metadataString("=", exactly200))).toBe(true);+ expect(requires(metadataString("starts with", over200))).toBe(true);+ });+});++describe("filtersRequireEventsFull input/output routing", () => {+ const ioFilter = (field: "input" | "output") =>+ new StringFilter({+ clickhouseTable: "events_core",+ field,+ operator: "=",+ value: "short",+ });++ it("always forces events_full for input/output filters (truncated + no I/O FTS index)", () => {+ expect(filtersRequireEventsFull(new FilterList([ioFilter("input")]))).toBe(+ true,+ );+ expect(filtersRequireEventsFull(new FilterList([ioFilter("output")]))).toBe(+ true,+ );+ });++ it("ignores filters on non-events tables", () => {+ expect(+ filtersRequireEventsFull(+ new FilterList([+ new StringObjectFilter({+ clickhouseTable: "traces",+ field: "metadata",+ key: "k",+ operator: "contains",+ value: "x",+ }),+ ]),+ ),+ ).toBe(false);+ });++ it("forces events_full if any filter in the list is truncation-sensitive", () => {+ expect(+ filtersRequireEventsFull(+ new FilterList([+ metadataString("=", under200),+ metadataString("contains", "needle"),+ ]),+ ),+ ).toBe(true);+ });+});diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.tsindex 71d8aded1a..b44752ad1d 100644--- a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts@@ -742,12 +742,105 @@ export class FilterList {}}+// events_core_mv truncates each metadata_values element (and input/output) to+// the first 200 UTF-8 code points via leftUTF8(v, 200). A metadata filter is+// only safe to answer against events_core when 200-char truncation cannot+// change its result.+const EVENTS_CORE_TRUNCATION_LIMIT = 200;++// Metadata operators that can be answered correctly against the truncated+// events_core copy — an allow-list, so any operator not named here defaults to+// events_full. That default is the safe one: routing a decidable filter to+// events_full only costs performance, whereas routing a truncation-sensitive+// filter to events_core silently drops matches past code point 200. When a new+// ClickhouseOperator is added it must be reviewed and added here explicitly+// before it can use events_core.+// - `=` / `starts with`: string ops, subject to the length guard below.+// - `>` `<` `>=` `<=`: numeric comparisons — values are inherently short.+// - `<>`: boolean not-equal — value is inherently short.+// - `is null` / `is not null`: truncation-invariant (metadata_names is not+// truncated; emptiness is unaffected).+// Deliberately excluded (match can live past code point 200): `contains`,+// `does not contain`, `ends with`, and the FTS `matches` operator (which also+// needs the events_full-only index).+const EVENTS_CORE_SAFE_METADATA_OPERATORS = new Set<ClickhouseOperator>([+ "=",+ "starts with",+ ">",+ "<",+ ">=",+ "<=",+ "<>",+ "is null",+ "is not null",+]);++// Count Unicode code points to mirror ClickHouse leftUTF8, which truncates by+// code point rather than UTF-16 unit or byte.+const codePointLength = (value: string): number => Array.from(value).length;++/**+ * Truncation-safety classifier: can a single metadata filter be answered+ * correctly against the truncated events_core copy, or must it read events_full?+ *+ * events_core keeps only the first {@link EVENTS_CORE_TRUNCATION_LIMIT} code+ * points of each metadata value (metadata_names is not truncated). This is an+ * allow-list: an operator is events_core-safe only if it is named in+ * {@link EVENTS_CORE_SAFE_METADATA_OPERATORS} and, for the two length-sensitive+ * string operators, its value fits within the retained prefix:+ * - `starts with`, value length <= 200 — the prefix lives within retained chars.+ * - `=`, value length < 200 — a stored value longer than 200 truncates to 200+ * code points and can never equal a sub-200 value. Strict `<` avoids the+ * exact-200 false positive where a >200 value shares the first 200 chars.+ * - `is null` / `is not null` — truncation invariant (metadata_names is+ * untruncated; emptiness is unaffected).+ * - numeric / boolean metadata comparisons — the value is inherently short,+ * so its truncated copy is complete.+ * Anything else (including any newly added operator) is routed to events_full,+ * because over-routing only costs performance while under-routing silently+ * drops matches past the truncation boundary.+ *+ * Single source of truth for metadata truncation routing. Do not reuse+ * NGRAM_ACCELERATED_METADATA_OPERATORS or FTS_TEXT_OPERATORS: those classify+ * ngram/FTS index eligibility, a different axis (the ngram set includes+ * truncation-unsafe `contains`/`ends with` and excludes truncation-safe `=`).+ */+export const metadataFilterIsEventsCoreSafe = (+ operator: ClickhouseOperator,+ value: unknown,+): boolean => {+ if (!EVENTS_CORE_SAFE_METADATA_OPERATORS.has(operator)) {+ return false;+ }+ if (typeof value === "string") {+ if (operator === "starts with") {+ return codePointLength(value) <= EVENTS_CORE_TRUNCATION_LIMIT;+ }+ if (operator === "=") {+ return codePointLength(value) < EVENTS_CORE_TRUNCATION_LIMIT;+ }+ }+ return true;+};+// events_core stores input/output/metadata_values truncated to 200 chars-// (events_core_mv); filters on these fields must run against events_full or-// matches beyond the truncation point are silently dropped.+// (events_core_mv). A filter must read events_full when truncation could change+// its result: input/output always (truncated, and events_core lacks the I/O FTS+// indices); metadata only when the operator/value is truncation-sensitive.+const filterRequiresEventsFull = (filter: Filter): boolean => {+ if (!isFtsEventsTable(filter.clickhouseTable)) {+ return false;+ }+ if (isFtsTextField(filter.field)) {+ return true;+ }+ if (isFtsMetadataField(filter.field)) {+ const value =+ "value" in filter ? (filter as { value: unknown }).value : undefined;+ return !metadataFilterIsEventsCoreSafe(filter.operator, value);+ }+ return false;+};+export const filtersRequireEventsFull = (filters: FilterList): boolean =>- filters.some(- (f) =>- isFtsEventsTable(f.clickhouseTable) &&- (isFtsTextField(f.field) || isFtsMetadataField(f.field)),- );+ filters.some(filterRequiresEventsFull);diff --git a/packages/shared/src/server/queries/index.ts b/packages/shared/src/server/queries/index.tsindex 5a35767abb..780cab0d25 100644--- a/packages/shared/src/server/queries/index.ts+++ b/packages/shared/src/server/queries/index.ts@@ -20,6 +20,7 @@ export {NullFilter,encodeBooleanScoreEntry,filtersRequireEventsFull,+ metadataFilterIsEventsCoreSafe,type ClickhouseOperator,} from "./clickhouse-sql/clickhouse-filter";export {diff --git a/web/src/__tests__/server/event-query-builder.servertest.ts b/web/src/__tests__/server/event-query-builder.servertest.tsindex adf6894f0d..c9b80eb83c 100644--- a/web/src/__tests__/server/event-query-builder.servertest.ts+++ b/web/src/__tests__/server/event-query-builder.servertest.ts@@ -241,8 +241,11 @@ describe("buildEventsFilterOptionsForColumnsQuery", () => {projectId: "test-project",filter: [{+ // `contains` is truncation-unsafe (a match can sit past char 200),+ // so it must force full-table routing — which is what this test+ // asserts. A short `=`/`starts with` value stays on events_core.column: "metadata",- operator: "=",+ operator: "contains",key: "region",value: "eu",type: "stringObject",