πͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
refactor(query-ast): route a logical events relation to events_core or events_full (#18868)
Niklas Semmler53b19bff82bf
diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.tsindex 7115594f2c..4ce7f0ff28 100644--- a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.ts@@ -5,12 +5,12 @@ import {FilterList,filtersRequireEventsFull,metadataFilterCanPruneEventsFull,- metadataFilterIsEventsCoreSafe,NullFilter,NumberObjectFilter,StringFilter,StringObjectFilter,} from "./clickhouse-filter";+import { metadataFilterIsEventsCoreSafe } from "./events-core-truncation";import { createFilterFromFilterState } from "./factory";import { experimentItemsTableNativeUiColumnDefinitions } from "../../tableMappings/mapExperimentItemsTable";diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.tsindex dd6560e11d..c915b10c68 100644--- a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts@@ -18,6 +18,7 @@ import {isFtsTextTarget,isNgramSubstringTarget,} from "./fts";+import { metadataFilterIsEventsCoreSafe } from "./events-core-truncation";export type ClickhouseOperator =| (typeof filterOperators)[keyof typeof filterOperators][number]@@ -860,91 +861,6 @@ export class FilterList {}}-// events_core_mv truncates each metadata_values element (and input/output) to-// the first 200 UTF-8 code points via leftUTF8(v, 200). A metadata filter is-// only safe to answer against events_core when 200-char truncation cannot-// change its result.-const EVENTS_CORE_TRUNCATION_LIMIT = 200;--// Metadata operators that can be answered correctly against the truncated-// events_core copy β an allow-list, so any operator not named here defaults to-// events_full. That default is the safe one: routing a decidable filter to-// events_full only costs performance, whereas routing a truncation-sensitive-// filter to events_core silently drops matches past code point 200. When a new-// ClickhouseOperator is added it must be reviewed and added here explicitly-// before it can use events_core.-// - `=` / `starts with`: string ops, subject to the length guard below.-// - `>` `<` `>=` `<=`: numeric comparisons β values are inherently short.-// - `<>`: boolean not-equal β value is inherently short.-// - `is null` / `is not null`: truncation-invariant (metadata_names is not-// truncated; emptiness is unaffected).-// - `is set` / `is not set`: key-presence over metadata_names only, which is-// not truncated.-// Deliberately excluded (match can live past code point 200): `contains`,-// `does not contain`, `ends with`, and the FTS `matches` operator (which also-// needs the events_full-only index).-const EVENTS_CORE_SAFE_METADATA_OPERATORS = new Set<ClickhouseOperator>([- "=",- "starts with",- ">",- "<",- ">=",- "<=",- "<>",- "is null",- "is not null",- "is set",- "is not set",-]);--// Count Unicode code points to mirror ClickHouse leftUTF8, which truncates by-// code point rather than UTF-16 unit or byte.-const codePointLength = (value: string): number => Array.from(value).length;--/**- * Truncation-safety classifier: can a single metadata filter be answered- * correctly against the truncated events_core copy, or must it read events_full?- *- * events_core keeps only the first {@link EVENTS_CORE_TRUNCATION_LIMIT} code- * points of each metadata value (metadata_names is not truncated). This is an- * allow-list: an operator is events_core-safe only if it is named in- * {@link EVENTS_CORE_SAFE_METADATA_OPERATORS} and, for the two length-sensitive- * string operators, its value fits within the retained prefix:- * - `starts with`, value length <= 200 β the prefix lives within retained chars.- * - `=`, value length < 200 β a stored value longer than 200 truncates to 200- * code points and can never equal a sub-200 value. Strict `<` avoids the- * exact-200 false positive where a >200 value shares the first 200 chars.- * - `is null` / `is not null` β truncation invariant (metadata_names is- * untruncated; emptiness is unaffected).- * - numeric / boolean metadata comparisons β the value is inherently short,- * so its truncated copy is complete.- * Anything else (including any newly added operator) is routed to events_full,- * because over-routing only costs performance while under-routing silently- * drops matches past the truncation boundary.- *- * Single source of truth for metadata truncation routing. Do not reuse- * NGRAM_ACCELERATED_METADATA_OPERATORS or FTS_TEXT_OPERATORS: those classify- * ngram/FTS index eligibility, a different axis (the ngram set includes- * truncation-unsafe `contains`/`ends with` and excludes truncation-safe `=`).- */-export const metadataFilterIsEventsCoreSafe = (- operator: ClickhouseOperator,- value: unknown,-): boolean => {- if (!EVENTS_CORE_SAFE_METADATA_OPERATORS.has(operator)) {- return false;- }- if (typeof value === "string") {- if (operator === "starts with") {- return codePointLength(value) <= EVENTS_CORE_TRUNCATION_LIMIT;- }- if (operator === "=") {- return codePointLength(value) < EVENTS_CORE_TRUNCATION_LIMIT;- }- }- return true;-};-/*** Whether a metadata string filter on events_full emits a conjunct that an* events_full skip index can prune on: the ngram conjunct for substringdiff --git a/packages/shared/src/server/queries/clickhouse-sql/events-core-truncation.ts b/packages/shared/src/server/queries/clickhouse-sql/events-core-truncation.tsnew file mode 100644index 0000000000..dc113af734--- /dev/null+++ b/packages/shared/src/server/queries/clickhouse-sql/events-core-truncation.ts@@ -0,0 +1,88 @@+import type { ClickhouseOperator } from "./clickhouse-filter";++// events_core_mv truncates each metadata_values element (and input/output) to+// the first 200 UTF-8 code points via leftUTF8(v, 200). A metadata filter is+// only safe to answer against events_core when 200-char truncation cannot+// change its result.+const EVENTS_CORE_TRUNCATION_LIMIT = 200;++// Metadata operators that can be answered correctly against the truncated+// events_core copy β an allow-list, so any operator not named here defaults to+// events_full. That default is the safe one: routing a decidable filter to+// events_full only costs performance, whereas routing a truncation-sensitive+// filter to events_core silently drops matches past code point 200. When a new+// ClickhouseOperator is added it must be reviewed and added here explicitly+// before it can use events_core.+// - `=` / `starts with`: string ops, subject to the length guard below.+// - `>` `<` `>=` `<=`: numeric comparisons β values are inherently short.+// - `<>`: boolean not-equal β value is inherently short.+// - `is null` / `is not null`: truncation-invariant (metadata_names is not+// truncated; emptiness is unaffected).+// - `is set` / `is not set`: key-presence over metadata_names only, which is+// not truncated.+// Deliberately excluded (match can live past code point 200): `contains`,+// `does not contain`, `ends with`, and the FTS `matches` operator (which also+// needs the events_full-only index).+const EVENTS_CORE_SAFE_METADATA_OPERATORS = new Set<ClickhouseOperator>([+ "=",+ "starts with",+ ">",+ "<",+ ">=",+ "<=",+ "<>",+ "is null",+ "is not null",+ "is set",+ "is not set",+]);++/**+ * Truncation-safety classifier: can a single metadata filter be answered+ * correctly against the truncated events_core copy, or must it read events_full?+ *+ * events_core keeps only the first {@link EVENTS_CORE_TRUNCATION_LIMIT} code+ * points of each metadata value (metadata_names is not truncated). This is an+ * allow-list: an operator is events_core-safe only if it is named in+ * {@link EVENTS_CORE_SAFE_METADATA_OPERATORS} and, for the two length-sensitive+ * string operators, its value fits within the retained prefix:+ * - `starts with`, value length <= 200 β the prefix lives within retained chars.+ * - `=`, value length < 200 β a stored value longer than 200 truncates to 200+ * code points and can never equal a sub-200 value. Strict `<` avoids the+ * exact-200 false positive where a >200 value shares the first 200 chars.+ * - `is null` / `is not null` β truncation invariant (metadata_names is+ * untruncated; emptiness is unaffected).+ * - numeric / boolean metadata comparisons β the value is inherently short,+ * so its truncated copy is complete.+ * Anything else (including any newly added operator) is routed to events_full,+ * because over-routing only costs performance while under-routing silently+ * drops matches past the truncation boundary.+ *+ * Single source of truth for metadata truncation routing. Do not reuse+ * NGRAM_ACCELERATED_METADATA_OPERATORS or FTS_TEXT_OPERATORS: those classify+ * ngram/FTS index eligibility, a different axis (the ngram set includes+ * truncation-unsafe `contains`/`ends with` and excludes truncation-safe `=`).+ */+export const metadataFilterIsEventsCoreSafe = (+ operator: ClickhouseOperator,+ value: unknown,+): boolean => {+ if (!EVENTS_CORE_SAFE_METADATA_OPERATORS.has(operator)) {+ return false;+ }+ if (typeof value === "string") {+ if (operator === "starts with") {+ return codePointLength(value) <= EVENTS_CORE_TRUNCATION_LIMIT;+ }+ if (operator === "=") {+ return codePointLength(value) < EVENTS_CORE_TRUNCATION_LIMIT;+ }+ }+ return true;+};++// Count Unicode code points to mirror ClickHouse leftUTF8, which truncates by+// code point rather than UTF-16 unit or byte.+function codePointLength(value: string): number {+ return Array.from(value).length;+}diff --git a/packages/shared/src/server/queries/index.ts b/packages/shared/src/server/queries/index.tsindex 6e452fc7b4..a45527e56c 100644--- a/packages/shared/src/server/queries/index.ts+++ b/packages/shared/src/server/queries/index.ts@@ -21,9 +21,9 @@ export {encodeBooleanScoreEntry,filtersRequireEventsFull,metadataFilterCanPruneEventsFull,- metadataFilterIsEventsCoreSafe,type ClickhouseOperator,} from "./clickhouse-sql/clickhouse-filter";+export { metadataFilterIsEventsCoreSafe } from "./clickhouse-sql/events-core-truncation";export {orderByToClickhouseSql,orderByToEntries,diff --git a/packages/shared/src/server/query-ast/README.md b/packages/shared/src/server/query-ast/README.mdindex 4f1d7b5114..bd6638a0d2 100644--- a/packages/shared/src/server/query-ast/README.md+++ b/packages/shared/src/server/query-ast/README.md@@ -85,6 +85,22 @@ suffix to run there. Promote it to a required check once it has proven stable.- **Never filter `project_id` yourself.** The compile step injects`project_id = {projectId}` into every tenanted relation, so call sites passonly `projectId` (see `../repositories/environments.ts`).+- **Read events through the logical `events` relation.**+ Each `events` relation compiles to `events_core` unless a column reference+ that resolves to it reads a column `events_core` truncates (`input`,+ `output`, `metadata_values`, `input_length`, `output_length`); then that+ relation becomes `events_full`. Metadata `=` / `startsWith` predicates on+ `metadataValue` stay on `events_core` when+ `metadataFilterIsEventsCoreSafe` accepts the value, as in the raw-SQL+ builders. `*` / `e.*` over `events`, raw SQL text in a select that reads+ it, an unqualified truncated column, and a CTE named `events_core` /+ `events_full` are compile errors while `events` is in scope. Name+ `events_core` to read the truncated copy on purpose; `events_core` /+ `events_full` stay selectable by name, which is also the override for a+ shape the pass rejects.+- **Function and aggregate names must be plain identifiers.**+ The compiler writes them verbatim, so anything else fails to compile.+ `collate` is not supported.- **Never filter `is_deleted` on a physical table yourself.** Tables thatdeclare `softDelete` in the registry (`events_core`, `events_full`) get`is_deleted = 0` on every FROM/JOIN. A latest-version read (ordered by@@ -142,7 +158,7 @@ db.selectFrom("observations")**Write a LIMIT BY** (`limitBy`):```ts-db.selectFrom("events_core")+db.selectFrom("events").$call(softDeleteAfterLimit()).select(["span_id", "project_id"]).orderBy("event_ts", "desc")@@ -160,16 +176,19 @@ db.selectFrom("scores as s")// β¦ from scores as s final```-**Select a metadata value** (`metadataValue` β lowers `metadata[key]` to a subscript of `metadataMap`, so the last duplicate key wins; like `mapValue`, a string key is bound and a `literal(...)` key is inlined):+**Filter on a metadata value** (`metadataValue` β lowers `metadata[key]` to a subscript of `metadataMap`, so the last duplicate key wins; like `mapValue`, a string key is bound and a `literal(...)` key is inlined):```tsconst filterKey: string = input.metadataKey; // caller-supplied, so bound-db.selectFrom("events_core as e")- .select((eb) => [metadataValue("e", literal("my_key")).as("my_val")])+db.selectFrom("events as e")+ .select("e.span_id").where((eb) => eb(metadataValue("e", filterKey), "=", "x"));-// select mapfromarrays(arrayreverse(e.metadata_names), arrayreverse(e.metadata_values))['my_key'] as my_val β¦+// select e.span_id from events_core as e where β¦ and mapfromarrays(arrayreverse(e.metadata_names), arrayreverse(e.metadata_values))[{p:String}] = {p:String}```+Selecting `metadataValue(...)` reads the truncated `metadata_values`, so it+routes that relation to `events_full`.+## Developing the dialectSeveral patterns here are deliberately unusual. This is the guide for anyonediff --git a/packages/shared/src/server/query-ast/compile.ts b/packages/shared/src/server/query-ast/compile.tsindex 1e75dc5c7e..2751d23e60 100644--- a/packages/shared/src/server/query-ast/compile.ts+++ b/packages/shared/src/server/query-ast/compile.ts@@ -7,6 +7,7 @@ import {import { ClickHouseQueryCompiler } from "./compiler";import { DedupLoweringPlugin } from "./dedup";+import { EventsTableRoutingPlugin } from "./eventsRouting";import { SoftDeleteFilterPlugin } from "./softDelete";import {TenancyInjectionPlugin,@@ -33,7 +34,8 @@ export type ClickhouseCompilable = {* The choke point. Every query that should produce ClickHouse SQL goes* through here: the `projectId` parameter is required, so a caller cannot* compile without a tenancy scope β omitting it is a compile-time type error.- * The query reads exactly that one project. Soft-deleted rows are filtered+ * The query reads exactly that one project. The logical `events` relation is+ * routed to `events_core` or `events_full`, soft-deleted rows are filtered* out, tenancy is injected from that project, per-table dedup is applied from* the registry, and the dialect compiler refuses to emit SQL unless the tree* was identity-stamped by the tenancy and dedup passes (a copied property is@@ -47,6 +49,7 @@ export function compileClickhouseQuery(): CompiledClickhouseQuery {const scope = requireProjectId(projectId);const stamped = query+ .withPlugin(new EventsTableRoutingPlugin()).withPlugin(new SoftDeleteFilterPlugin()).withPlugin(new TenancyInjectionPlugin(scope)).withPlugin(new DedupLoweringPlugin())diff --git a/packages/shared/src/server/query-ast/compiler.ts b/packages/shared/src/server/query-ast/compiler.tsindex f9a40c7397..def1e47dcd 100644--- a/packages/shared/src/server/query-ast/compiler.ts+++ b/packages/shared/src/server/query-ast/compiler.ts@@ -16,6 +16,7 @@ import {SetOperationNode,TableNode,ValueNode,+ type CollateNode,type CommonTableExpressionNode,type OperationNode,type PrimitiveValueListNode,@@ -43,6 +44,8 @@ const ARRAY_JOIN_SQL: Record<ArrayJoinNode["variant"], string> = {inner: "inner array join",};+const PLAIN_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;+const DATE_WRAPPER_FNS = new Set(["toDate","toDateTime",@@ -274,6 +277,31 @@ export class ClickHouseQueryCompiler extends DefaultQueryCompiler {return "";}+ /**+ * Function and aggregate names are written verbatim, so a name that is not+ * a plain identifier would be raw SQL: `fn("tuple(e.input), f")` emits+ * `tuple(e.input), f(β¦)`.+ */+ protected override visitFunction(node: FunctionNode): void {+ assertPlainName("Function", node.func);+ super.visitFunction(node);+ }++ protected override visitAggregateFunction(node: AggregateFunctionNode): void {+ assertPlainName("Aggregate function", node.func);+ super.visitAggregateFunction(node);+ }++ /**+ * ClickHouse parses only a string literal after `collate`; the name is+ * written verbatim, so no collation compiles to valid SQL.+ */+ protected override visitCollate(node: CollateNode): void {+ throw new QueryCompileError(+ `Collation "${node.collation.name}" is not supported: ClickHouse expects a string literal after collate.`,+ );+ }+protected override appendValue(parameter: unknown): void {this.append(this.bindValue(parameter, this.pendingBindType));}@@ -736,3 +764,11 @@ function withoutCte(scope: SelectScope, name: string): SelectScope {relations: new Map(),};}++function assertPlainName(kind: string, name: string): void {+ if (!PLAIN_IDENTIFIER.test(name)) {+ throw new QueryCompileError(+ `${kind} name "${name}" is not a plain identifier; the compiler writes it verbatim into the SQL.`,+ );+ }+}diff --git a/packages/shared/src/server/query-ast/eventsRouting.test.ts b/packages/shared/src/server/query-ast/eventsRouting.test.tsnew file mode 100644index 0000000000..60cc9c1712--- /dev/null+++ b/packages/shared/src/server/query-ast/eventsRouting.test.ts@@ -0,0 +1,660 @@+import {+ sql,+ type Expression,+ type ExpressionBuilder,+ type SqlBool,+} from "kysely";+import { describe, expect, it } from "vitest";++import {+ FilterList,+ filtersRequireEventsFull,+ StringFilter,+ StringObjectFilter,+} from "../queries/clickhouse-sql/clickhouse-filter";+import { compileClickhouseQuery } from "./compile";+import { getClickhouseKysely } from "./dialect";+import { QueryCompileError } from "./errors";+import {+ includeDeleted,+ limitBy,+ mapValue,+ metadataMap,+ metadataValue,+ startsWith,+ useFinal,+} from "./extensions";+import type { ClickHouseDatabase } from "./schema";++// Raw compiler output, so these run without a `clickhouse` binary.+const projectId = "p";++function compile(builder: Parameters<typeof compileClickhouseQuery>[0]) {+ return compileClickhouseQuery(builder, projectId).sql;+}++/** Physical events tables in FROM/JOIN position, in order. */+function routedTables(query: string): string[] {+ return [+ ...query.matchAll(/\b(?:from|join) (events_core|events_full)\b/g),+ ].map((match) => match[1]);+}++const db = getClickhouseKysely();++type EventsEb = ExpressionBuilder<+ ClickHouseDatabase & { e: ClickHouseDatabase["events"] },+ "e"+>;++describe("logical events routing", () => {+ it("routes a read of untruncated columns to events_core", () => {+ expect(+ compile(+ db+ .selectFrom("events as e")+ .select(["e.span_id", "e.name"])+ .where("e.trace_id", "=", "t"),+ ),+ ).toMatchInlineSnapshot(+ `"select e.span_id, e.name from events_core as e where e.project_id = {p1:String} and e.is_deleted = {p2:UInt8} and e.trace_id = {p3:String}"`,+ );+ });++ it.each([+ "input",+ "output",+ "metadata_values",+ "input_length",+ "output_length",+ ] as const)("routes a projection of %s to events_full", (column) => {+ expect(+ routedTables(compile(db.selectFrom("events as e").select(`e.${column}`))),+ ).toEqual(["events_full"]);+ });++ it("routes a projected metadata value to events_full", () => {+ expect(+ routedTables(+ compile(+ db+ .selectFrom("events as e")+ .select(() => [metadataValue("e", "k").as("v")]),+ ),+ ),+ ).toEqual(["events_full"]);+ });++ it("rejects select * and select e.*, but not count(*)", () => {+ expect(() => compile(db.selectFrom("events").selectAll())).toThrow(+ QueryCompileError,+ );+ expect(() => compile(db.selectFrom("events as e").selectAll("e"))).toThrow(+ QueryCompileError,+ );+ expect(+ routedTables(+ compile(+ db.selectFrom("events as e").select((eb) => eb.fn.countAll().as("n")),+ ),+ ),+ ).toEqual(["events_core"]);+ });++ it("routes count(*) with a clause on a truncated column to events_full", () => {+ expect(+ routedTables(+ compile(+ db+ .selectFrom("events as e")+ .select((eb) =>+ eb.fn.countAll().filterWhere("e.input", "=", "x").as("n"),+ ),+ ),+ ),+ ).toEqual(["events_full"]);+ expect(+ routedTables(+ compile(+ db.selectFrom("events as e").select((eb) =>+ eb.fn+ .countAll()+ .over((ob) => ob.partitionBy("e.input"))+ .as("n"),+ ),+ ),+ ),+ ).toEqual(["events_full"]);+ });++ it("keeps count(*) with a clause on an untruncated column on events_core", () => {+ expect(+ routedTables(+ compile(+ db+ .selectFrom("events as e")+ .select((eb) =>+ eb.fn.countAll().filterWhere("e.name", "=", "x").as("n"),+ ),+ ),+ ),+ ).toEqual(["events_core"]);+ });++ it("ignores an alias named like a truncated column", () => {+ expect(+ routedTables(+ compile(+ db+ .selectFrom("events as e")+ .select((eb) => eb.fn.count("e.span_id").as("input")),+ ),+ ),+ ).toEqual(["events_core"]);+ });++ it.each([+ ["lengthUTF8(e.output)", sql<string>`lengthUTF8(e.output)`],+ ["sql.id e.input", sql<string>`${sql.id("e", "input")}`],+ ["sql.id input", sql<string>`${sql.id("input")}`],+ ["sql.table input", sql<string>`${sql.table("input")}`],+ ["*", sql<string>`*`],+ ["e.*", sql.raw<string>("e.*")],+ ["tuple(*)", sql<string>`tuple(*)`],+ ["COLUMNS(...)", sql<string>`COLUMNS('^inp')`],+ ["sql.ref e.*", sql.ref<string>("e.*")],+ ["sql.ref *", sql.ref<string>("*")],+ ["tuple(sql.ref e.*)", sql<string>`tuple(${sql.ref("e.*")})`],+ ])("rejects raw %s in a select that reads events", (_, expression) => {+ expect(() =>+ compile(db.selectFrom("events as e").select(expression.as("x"))),+ ).toThrow(QueryCompileError);+ });++ it("rejects a raw identifier in the where clause of a select that reads events", () => {+ expect(() =>+ compile(+ db+ .selectFrom("events as e")+ .select("e.span_id")+ .where(sql<string>`${sql.id("input")}`, "=", "x"),+ ),+ ).toThrow(QueryCompileError);+ });++ it("rejects raw SQL in a select nested under one that reads events", () => {+ expect(() =>+ compile(+ db.selectFrom("events as e").select((eb) => [+ "e.span_id",+ eb+ .selectFrom("scores as s")+ .select(sql<number>`1`.as("one"))+ .limit(1)+ .as("x"),+ ]),+ ),+ ).toThrow(QueryCompileError);+ });++ it("rejects COLUMNS() in a select that reads events", () => {+ expect(() =>+ compile(+ db+ .selectFrom("events as e")+ .select((eb) => eb.fn<string>("COLUMNS", [eb.val("^inp")]).as("x")),+ ),+ ).toThrow(QueryCompileError);+ expect(() =>+ compile(+ db+ .selectFrom("events as e")+ .select((eb) =>+ eb.fn.agg<string>("columns", [eb.val("^inp")]).as("x"),+ ),+ ),+ ).toThrow(QueryCompileError);+ });++ it.each([+ "events_core.input",+ "default.events_core.input",+ "events_full.input",+ ])("rejects the physical qualifier %s over a logical relation", (ref) => {+ // ClickHouse resolves a qualifier by table name even when the table is aliased.+ expect(() =>+ compile(+ db.selectFrom("events as e").select(sql.ref<string>(ref).as("x")),+ ),+ ).toThrow(QueryCompileError);+ });++ it("keeps a physical events qualifier when no logical relation is in scope", () => {+ expect(+ routedTables(+ compile(+ db+ .selectFrom("events_core as c")+ .$call(includeDeleted())+ .select(sql.ref<string>("events_core.input").as("x")),+ ),+ ),+ ).toEqual(["events_core"]);+ });++ it("resolves sql.ref like a builder reference", () => {+ const route = (ref: string) =>+ routedTables(+ compile(+ db.selectFrom("events as e").select(sql.ref<string>(ref).as("x")),+ ),+ );+ expect(route("e.input")).toEqual(["events_full"]);+ expect(route("e.span_id")).toEqual(["events_core"]);+ });++ it("allows raw SQL in a select over a subquery that reads events", () => {+ expect(+ routedTables(+ compile(+ db+ .selectFrom(+ db.selectFrom("events as e").select("e.span_id").as("x"),+ )+ .select(sql<number>`1`.as("one")),+ ),+ ),+ ).toEqual(["events_core"]);+ });++ it("rejects an unqualified truncated column of the only relation", () => {+ expect(() => compile(db.selectFrom("events").select("input"))).toThrow(+ QueryCompileError,+ );+ });++ it("routes a table-qualified read of an unaliased logical relation", () => {+ expect(+ routedTables(compile(db.selectFrom("events").select("events.input"))),+ ).toEqual(["events_full"]);+ });++ it("rejects an unqualified limit by column of the only relation", () => {+ expect(() =>+ compile(+ db+ .selectFrom("events")+ .select("span_id")+ .$call(limitBy({ count: 1, columns: ["input"] })),+ ),+ ).toThrow(QueryCompileError);+ expect(() =>+ compile(+ db+ .selectFrom("events")+ .select("span_id")+ .$call(limitBy({ count: 1, columns: ["*"] })),+ ),+ ).toThrow(QueryCompileError);+ });++ it("routes each use of a shared builder by its own reads", () => {+ // Both CTEs hold the same FROM node.+ const base = db.selectFrom("events as e");+ expect(+ routedTables(+ compile(+ db+ .with("a", () => base.select(["e.span_id", "e.input"]))+ .with("b", () => base.select(["e.span_id"]))+ .selectFrom("a")+ .innerJoin("b", "a.span_id", "b.span_id")+ .select("a.span_id"),+ ),+ ),+ ).toEqual(["events_full", "events_core"]);+ });++ it("rejects an unqualified truncated column in a subquery under a select that reads events", () => {+ expect(() =>+ compile(+ db+ .selectFrom("events as e")+ .select((eb) => [+ "e.span_id",+ eb+ .selectFrom("scores as s")+ .select(sql.ref<string>("input").as("i"))+ .limit(1)+ .as("x"),+ ]),+ ),+ ).toThrow(QueryCompileError);+ });++ it("rejects an unqualified truncated column in a subquery with no FROM under a select that reads events", () => {+ // ClickHouse resolves `input` in a no-FROM subquery to the outer relation.+ expect(() =>+ compile(+ db+ .selectFrom("events")+ .select([+ "span_id",+ db.selectNoFrom(sql.ref<string>("input").as("i")).as("x"),+ ]),+ ),+ ).toThrow(QueryCompileError);+ });++ it("resolves a qualifier to an inner FINAL relation before an outer one", () => {+ expect(+ routedTables(+ compile(+ db.selectFrom("events as s").select((eb) => [+ "s.span_id",+ eb+ .selectFrom("scores as s")+ .$call(useFinal(["scores"]))+ .select(sql.ref<string>("s.input").as("i"))+ .limit(1)+ .as("x"),+ ]),+ ),+ ),+ ).toEqual(["events_core"]);+ });++ it("rejects an unqualified truncated column with two relations in scope", () => {+ expect(() =>+ compile(+ db+ .selectFrom("events as a")+ .innerJoin("events as b", "a.span_id", "b.span_id")+ .select(sql.ref<string>("input").as("x")),+ ),+ ).toThrow(QueryCompileError);+ });++ it("keeps an unaliased logical read addressable by its name", () => {+ expect(+ compile(+ db+ .selectFrom("events")+ .select("events.span_id")+ .where("span_id", "=", "s"),+ ),+ ).toMatchInlineSnapshot(+ `"select events.span_id from events_core as events where events.project_id = {p1:String} and events.is_deleted = {p2:UInt8} and span_id = {p3:String}"`,+ );+ });++ it("routes a joined logical relation", () => {+ expect(+ routedTables(+ compile(+ db+ .selectFrom("scores as s")+ .innerJoin("events as e", (join) =>+ join.onRef("e.trace_id", "=", "s.trace_id"),+ )+ .select(["s.id", "e.input"]),+ ),+ ),+ ).toEqual(["events_full"]);+ });++ it("routes each logical relation by its own reads", () => {+ // `e` in the CTE and `e` outside are different relations; `io.input`+ // reads the CTE.+ const query = db+ .with("io", (qb) =>+ qb.selectFrom("events as e").select(["e.span_id", "e.input"]),+ )+ .selectFrom("events as e")+ .innerJoin("io", "io.span_id", "e.span_id")+ .select(["e.span_id", "io.input"]);++ expect(routedTables(compile(query))).toEqual([+ "events_full",+ "events_core",+ ]);+ });++ it("routes joined logical relations separately", () => {+ expect(+ routedTables(+ compile(+ db+ .selectFrom("events as a")+ .innerJoin("events as b", "a.span_id", "b.span_id")+ .select(["a.span_id", "b.input"]),+ ),+ ),+ ).toEqual(["events_core", "events_full"]);+ });++ it("leaves physical events tables unchanged", () => {+ expect(+ routedTables(+ compile(+ db+ .selectFrom("events_core as e")+ .$call(includeDeleted())+ .select("e.input"),+ ),+ ),+ ).toEqual(["events_core"]);+ });++ it("rejects a CTE named events", () => {+ expect(() =>+ compile(+ db+ .with("events", (qb) =>+ qb.selectFrom("events_core").select("span_id"),+ )+ .selectFrom("events")+ .select("span_id"),+ ),+ ).toThrow(QueryCompileError);+ });++ it.each(["events_core", "events_full"] as const)(+ "rejects a CTE named %s alongside a logical relation",+ (name) => {+ // The routed relation would read the CTE instead of the table.+ const withCte = db.with(name, (qb) =>+ qb.selectFrom("scores as s").select("s.id"),+ );+ expect(() =>+ compile(withCte.selectFrom("events as e").select("e.span_id")),+ ).toThrow(QueryCompileError);+ expect(() =>+ compile(withCte.selectFrom(name).select("id")),+ ).not.toThrow();+ },+ );+});++describe("logical events routing of metadata predicates", () => {+ const route = (predicate: (eb: EventsEb) => Expression<SqlBool>) =>+ routedTables(+ compile(+ db.selectFrom("events as e").select("e.span_id").where(predicate),+ ),+ );++ it("keeps a short `=` and `startsWith` on events_core", () => {+ expect(route((eb) => eb(metadataValue("e", "k"), "=", "x"))).toEqual([+ "events_core",+ ]);+ expect(route(() => startsWith(metadataValue("e", "k"), "x"))).toEqual([+ "events_core",+ ]);+ });++ it("keeps a short `=` on events_core under a plugin that rewrites the select", () => {+ expect(+ routedTables(+ compile(+ db+ .selectFrom("events as e")+ .select("e.span_id")+ .where((eb) => eb(metadataValue("e", "k"), "=", "x"))+ .$call(limitBy({ count: 1, columns: ["e.span_id"] })),+ ),+ ),+ ).toEqual(["events_core"]);+ });++ it("keeps a negated short `=` on events_core", () => {+ expect(+ route((eb) => eb.not(eb(metadataValue("e", "k"), "=", "x"))),+ ).toEqual(["events_core"]);+ });++ it("routes other comparison shapes on a metadata value to events_full", () => {+ // Operator outside the allow-list.+ expect(route((eb) => eb(metadataValue("e", "k"), "!=", "x"))).toEqual([+ "events_full",+ ]);+ // Value on the left.+ expect(+ route((eb) => eb(eb.val("x"), "=", metadataValue("e", "k"))),+ ).toEqual(["events_full"]);+ // Compared to a column, not a bound value.+ expect(+ route((eb) => eb(metadataValue("e", "k"), "=", eb.ref("e.name"))),+ ).toEqual(["events_full"]);+ });++ it("routes the relation whose values a hand-built metadata map reads", () => {+ const query = db+ .selectFrom("events as a")+ .innerJoin("events as b", "a.span_id", "b.span_id")+ .select("a.span_id")+ .where((eb) =>+ eb(+ mapValue(+ metadataMap(+ eb.ref("a.metadata_names"),+ eb.ref("b.metadata_values"),+ ),+ "k",+ ),+ "=",+ "x",+ ),+ );++ expect(routedTables(compile(query))).toEqual([+ "events_core",+ "events_full",+ ]);+ });+});++// The raw-SQL builders route through `filtersRequireEventsFull`; the logical+// relation must pick the same table for the same filter.+describe("logical events routing agrees with filtersRequireEventsFull", () => {+ type MetadataOperator =+ | "="+ | "starts with"+ | "contains"+ | "ends with"+ | "does not contain";++ const astPredicate = (operator: MetadataOperator, value: string) => {+ const metadata = metadataValue("e", "k");+ return (eb: EventsEb): Expression<SqlBool> => {+ switch (operator) {+ case "=":+ return eb(metadata, "=", value);+ case "starts with":+ return startsWith(metadata, value);+ case "contains":+ return eb(metadata, "like", `%${value}%`);+ case "ends with":+ return eb(metadata, "like", `%${value}`);+ case "does not contain":+ return eb(metadata, "not like", `%${value}%`);+ }+ };+ };++ // Only `=` and `starts with` route by value length; the others always read+ // events_full, so one value each covers them.+ const lengthSensitive: MetadataOperator[] = ["=", "starts with"];+ const lengthInsensitive: MetadataOperator[] = [+ "contains",+ "ends with",+ "does not contain",+ ];+ // "π" is one code point and two UTF-16 units, so these cases fail if+ // either side counts UTF-16 units instead of code points.+ const values = [199, 200, 201].flatMap((length) => [+ { label: `${length} ascii`, value: "a".repeat(length) },+ { label: `${length} astral`, value: "π".repeat(length) },+ ]);++ const cases = [+ ...lengthSensitive.flatMap((operator) =>+ values.map(({ label, value }) => ({ operator, label, value })),+ ),+ ...lengthInsensitive.map((operator) => ({+ operator,+ label: "short",+ value: "a",+ })),+ ];++ it.each(cases)("metadata $operator, $label", ({ operator, value }) => {+ const legacyNeedsFull = filtersRequireEventsFull(+ new FilterList([+ new StringObjectFilter({+ clickhouseTable: "events_core",+ field: "metadata",+ key: "k",+ operator,+ value,+ tablePrefix: "e",+ }),+ ]),+ );+ const [routed] = routedTables(+ compile(+ db+ .selectFrom("events as e")+ .select("e.span_id")+ .where(astPredicate(operator, value)),+ ),+ );++ expect(routed).toBe(legacyNeedsFull ? "events_full" : "events_core");+ });++ it.each(["input", "output"] as const)("%s filter", (field) => {+ const legacyNeedsFull = filtersRequireEventsFull(+ new FilterList([+ new StringFilter({+ clickhouseTable: "events_core",+ field,+ operator: "=",+ value: "x",+ tablePrefix: "e",+ }),+ ]),+ );+ const [routed] = routedTables(+ compile(+ db+ .selectFrom("events as e")+ .select("e.span_id")+ .where(`e.${field}`, "=", "x"),+ ),+ );++ expect(legacyNeedsFull).toBe(true);+ expect(routed).toBe("events_full");+ });+});diff --git a/packages/shared/src/server/query-ast/eventsRouting.ts b/packages/shared/src/server/query-ast/eventsRouting.tsnew file mode 100644index 0000000000..88b64aac86--- /dev/null+++ b/packages/shared/src/server/query-ast/eventsRouting.ts@@ -0,0 +1,416 @@+/**+ * Logical `events` routing. Runs first in every compile.+ *+ * Call sites read the logical `events` relation; this pass replaces each+ * FROM/JOIN reference to it with `events_core` or `events_full`, one choice+ * per relation. A relation goes to `events_full` only when a column reference+ * resolves to it and names a column in {@link EVENTS_CORE_TRUNCATED_COLUMNS},+ * outside the metadata predicates `metadataFilterIsEventsCoreSafe` accepts:+ *+ * metadataValue(e, k) = 'short' β events_core (< 200 code points)+ * startsWith(metadataValue(e, k), 'p') β events_core (β€ 200 code points)+ * e.input = 'x', select e.input β events_full+ *+ * A qualifier resolves to the innermost FROM/JOIN alias (or unaliased table+ * name) of that name. Shapes that cannot be resolved to one relation are rejected+ * rather than routed to `events_full`, so `events_full` is read only where a+ * query needs it:+ * - `*`, `e.*` or `COLUMNS(β¦)` over a logical relation (`count(*)` reads no+ * column);+ * - an unqualified truncated column while a logical relation is in scope;+ * - a column qualified by `events_core` / `events_full` that names no+ * relation in scope: ClickHouse resolves a qualifier by table name even+ * when the table is aliased, so it would reach the routed relation;+ * - raw SQL text or identifiers (`sql` templates, `sql.raw`, `sql.id`,+ * `sql.table`) in or under a select that reads a logical relation.+ * `sql.ref` and bound values resolve like builder references.+ *+ * Callers that want the truncated copy on purpose name `events_core`;+ * physical `events_core` / `events_full` references pass through unchanged.+ */+import {+ AggregateFunctionNode,+ AliasNode,+ BinaryOperationNode,+ ColumnNode,+ CommonTableExpressionNode,+ FromNode,+ FunctionNode,+ IdentifierNode,+ JoinNode,+ OperatorNode,+ RawNode,+ SelectAllNode,+ SelectionNode,+ TableNode,+ ValueNode,+ type KyselyPlugin,+ type OperationNode,+ type OrderByItemNode,+ type PluginTransformQueryArgs,+ type PluginTransformResultArgs,+ type QueryId,+ type ReferenceNode,+ type RootOperationNode,+ type SelectQueryNode,+} from "kysely";++import { metadataFilterIsEventsCoreSafe } from "../queries/clickhouse-sql/events-core-truncation";+import { QueryCompileError } from "./errors";+import { ArrayIndexNode, FinalTableNode, unwrapFinalTable } from "./nodes";+import { EVENTS_CORE_TRUNCATED_COLUMNS } from "./schema";+import { ClickHouseOperationNodeTransformer } from "./transformer";++const LOGICAL_EVENTS = "events";+const PHYSICAL_EVENTS = new Set(["events_core", "events_full"]);++export class EventsTableRoutingPlugin implements KyselyPlugin {+ transformQuery(args: PluginTransformQueryArgs): RootOperationNode {+ // Builders share unchanged child nodes, so one relation node can sit at+ // two places in the tree. A plain transform rebuilds every node, giving+ // each place its own node to key routing on.+ const tree = new ClickHouseOperationNodeTransformer().transformNode(+ args.node,+ );+ const analysis = new EventsReadAnalysis();+ analysis.transformNode(tree);+ if (!analysis.readsLogicalEvents) return args.node;+ rejectPhysicalEventsCte(analysis);+ return new EventsTableRouter(analysis.fullRelations).transformNode(+ tree,+ ) as RootOperationNode;+ }++ async transformResult(args: PluginTransformResultArgs) {+ return args.result;+ }+}++function rejectPhysicalEventsCte(analysis: EventsReadAnalysis): void {+ if (analysis.physicalEventsCte) {+ throw new QueryCompileError(+ `A CTE named "${analysis.physicalEventsCte}" would capture the routed logical "${LOGICAL_EVENTS}" relation; rename the CTE.`,+ );+ }+}++/**+ * Read-only walk that resolves column references to relations; the+ * transformer is used for its traversal only.+ */+class EventsReadAnalysis extends ClickHouseOperationNodeTransformer {+ readsLogicalEvents = false;+ /** Name of a CTE that shadows `events_core` / `events_full`. */+ physicalEventsCte: string | undefined;+ /** Logical relations that read a truncated column. */+ readonly fullRelations = new Set<OperationNode>();+ /** Relations of each enclosing select, innermost last. */+ private readonly scopes: Relation[][] = [];++ protected override transformNodeImpl<T extends OperationNode>(+ node: T,+ queryId?: QueryId,+ ): T {+ const lookup = coreSafeMetadataLookup(node);+ if (lookup) {+ this.transformNode(lookup.index, queryId);+ return node;+ }+ if (CommonTableExpressionNode.is(node)) this.checkCteName(node);+ return super.transformNodeImpl(node, queryId);+ }++ protected override transformSelectQuery(+ node: SelectQueryNode,+ queryId?: QueryId,+ ): SelectQueryNode {+ const relations = [+ ...(node.from?.froms ?? []),+ ...(node.joins ?? []).map((join) => join.table),+ ].map(describeRelation);+ if (relations.some((relation) => relation.logical)) {+ this.readsLogicalEvents = true;+ }+ this.scopes.push(relations);+ try {+ return super.transformSelectQuery(node, queryId);+ } finally {+ this.scopes.pop();+ }+ }++ protected override transformReference(+ node: ReferenceNode,+ _queryId?: QueryId,+ ): ReferenceNode {+ const qualifier = node.table?.table.identifier.name;+ // `sql.ref("e.*")` parses the star as a column named `*`.+ if (SelectAllNode.is(node.column) || node.column.column.name === "*") {+ if (qualifier === undefined) this.rejectStar("*");+ else if (this.resolveQualified(qualifier)?.logical) {+ throw starError(`${qualifier}.*`);+ }+ } else {+ this.readColumn(node.column.column.name, qualifier);+ }+ return node;+ }++ /** Reached only outside a reference: `selectAll()`, `*` in raw SQL. */+ protected override transformSelectAll(node: SelectAllNode): SelectAllNode {+ this.rejectStar("*");+ return node;+ }++ /** Reached only outside a reference. */+ protected override transformColumn(node: ColumnNode): ColumnNode {+ if (node.column.name === "*") this.rejectStar("*");+ else this.readColumn(node.column.name, undefined);+ return node;+ }++ /**+ * Rejects `columns` like any function ({@link rejectColumnsMatcher}). `count(*)`+ * reads no column; its clauses are walked as usual.+ */+ protected override transformAggregateFunction(+ node: AggregateFunctionNode,+ queryId?: QueryId,+ ): AggregateFunctionNode {+ this.rejectColumnsMatcher(node.func);+ if (isCountStar(node)) {+ super.transformAggregateFunction({ ...node, aggregated: [] }, queryId);+ return node;+ }+ return super.transformAggregateFunction(node, queryId);+ }++ /**+ * The direction is raw `asc` / `desc`, so it is not walked: the+ * `no-deprecated` lint rule keeps `orderBy` from taking an expression.+ */+ protected override transformOrderByItem(+ node: OrderByItemNode,+ queryId?: QueryId,+ ): OrderByItemNode {+ this.transformNode(node.orderBy, queryId);+ return node;+ }++ protected override transformFunction(+ node: FunctionNode,+ queryId?: QueryId,+ ): FunctionNode {+ this.rejectColumnsMatcher(node.func);+ return super.transformFunction(node, queryId);+ }++ protected override transformRaw(node: RawNode, queryId?: QueryId): RawNode {+ if (+ this.readsLogicalInScope() &&+ (node.sqlFragments.some((fragment) => fragment.trim() !== "") ||+ node.parameters.some((p) => IdentifierNode.is(p) || TableNode.is(p)))+ ) {+ throw new QueryCompileError(+ `Raw SQL text or identifiers cannot appear in a select that reads the logical "${LOGICAL_EVENTS}" relation: they may read a truncated column the routing pass cannot see. Use builder expressions, sql.ref() or bound values.`,+ );+ }+ return super.transformRaw(node, queryId);+ }++ private readColumn(column: string, qualifier: string | undefined): void {+ if (!EVENTS_CORE_TRUNCATED_COLUMNS.has(column)) return;+ const relation =+ qualifier === undefined+ ? this.resolveUnqualified(column)+ : this.resolveQualified(qualifier);+ if (relation?.logical) this.fullRelations.add(relation.node);+ }++ private resolveQualified(qualifier: string): Relation | undefined {+ for (const relations of this.scopes.toReversed()) {+ const match = relations.find((relation) => relation.name === qualifier);+ if (match) return match;+ }+ if (PHYSICAL_EVENTS.has(qualifier) && this.readsLogicalInScope()) {+ throw new QueryCompileError(+ `Qualify by the relation's alias, not "${qualifier}": ClickHouse also resolves a table name to an aliased relation, so it may reach the logical "${LOGICAL_EVENTS}" relation after routing.`,+ );+ }+ return undefined;+ }++ private resolveUnqualified(column: string): Relation | undefined {+ if (this.readsLogicalInScope()) {+ throw new QueryCompileError(+ `Qualify "${column}": it is truncated in events_core, so the routing pass needs to know which relation reads it while the logical "${LOGICAL_EVENTS}" relation is in scope.`,+ );+ }+ return undefined;+ }++ private checkCteName(node: CommonTableExpressionNode): void {+ const name = node.name.table.table.identifier.name;+ if (name === LOGICAL_EVENTS) {+ throw new QueryCompileError(+ `A CTE cannot be named "${LOGICAL_EVENTS}": the name is the logical events relation.`,+ );+ }+ if (PHYSICAL_EVENTS.has(name)) this.physicalEventsCte = name;+ }++ /** `columns(β¦)` expands like `*`. */+ private rejectColumnsMatcher(func: string): void {+ if (func.toLowerCase() === "columns") this.rejectStar(`${func}(β¦)`);+ }++ /** Whether the current select or an enclosing one reads a logical relation. */+ private readsLogicalInScope(): boolean {+ return this.scopes.some((relations) => relations.some((r) => r.logical));+ }++ private rejectStar(star: string): void {+ if (this.scopes.at(-1)?.some((relation) => relation.logical)) {+ throw starError(star);+ }+ }+}++function starError(star: string): QueryCompileError {+ return new QueryCompileError(+ `${star} cannot expand the logical "${LOGICAL_EVENTS}" relation: list the columns, so the routing pass can tell whether it reads a truncated one.`,+ );+}++class EventsTableRouter extends ClickHouseOperationNodeTransformer {+ constructor(private readonly fullRelations: ReadonlySet<OperationNode>) {+ super();+ }++ protected override transformFrom(+ node: FromNode,+ queryId?: QueryId,+ ): FromNode {+ return FromNode.create(+ node.froms.map((from) => this.routeRelation(from, queryId)),+ );+ }++ protected override transformJoin(+ node: JoinNode,+ queryId?: QueryId,+ ): JoinNode {+ return {+ ...super.transformJoin(node, queryId),+ table: this.routeRelation(node.table, queryId),+ };+ }++ /**+ * `events as e` β `events_core as e`. An unaliased `events` keeps its name+ * as the alias (`events_core as events`), so `events.col` references and+ * the qualifiers later passes add still resolve.+ */+ private routeRelation(node: OperationNode, queryId?: QueryId): OperationNode {+ const table = TableNode.create(+ this.fullRelations.has(node) ? "events_full" : "events_core",+ );+ if (isLogicalEventsTable(node)) {+ return AliasNode.create(table, IdentifierNode.create(LOGICAL_EVENTS));+ }+ if (AliasNode.is(node) && isLogicalEventsTable(node.node)) {+ return AliasNode.create(table, node.alias);+ }+ return this.transformNode(node, queryId);+ }+}++// FINAL wraps the aliased item: `scores as s FINAL`.+function describeRelation(node: OperationNode): Relation {+ const item = FinalTableNode.is(node) ? unwrapFinalTable(node) : node;+ const table = AliasNode.is(item) ? item.node : item;+ return {+ node,+ name: relationName(item, table),+ logical: isLogicalEventsTable(table),+ };+}++function relationName(+ item: OperationNode,+ table: OperationNode,+): string | undefined {+ if (AliasNode.is(item)) {+ return IdentifierNode.is(item.alias) ? item.alias.name : undefined;+ }+ return TableNode.is(table) && table.table.schema === undefined+ ? table.table.identifier.name+ : undefined;+}++function isLogicalEventsTable(node: OperationNode): boolean {+ return (+ TableNode.is(node) &&+ node.table.schema === undefined &&+ node.table.identifier.name === LOGICAL_EVENTS+ );+}++/** `count(*)` with any `distinct`, `filter`, `over`, β¦ clause. */+function isCountStar(node: AggregateFunctionNode): boolean {+ if (node.func !== "count" || node.aggregated.length !== 1) return false;+ const [arg] = node.aggregated;+ return (+ SelectAllNode.is(arg) ||+ (SelectionNode.is(arg) && SelectAllNode.is(arg.selection))+ );+}++/**+ * The metadata lookup of `metadataValue(e, k) = v` or+ * `startsWith(metadataValue(e, k), v)`, for a bound `v` that+ * `metadataFilterIsEventsCoreSafe` accepts.+ */+function coreSafeMetadataLookup(+ node: OperationNode,+): ArrayIndexNode | undefined {+ if (+ BinaryOperationNode.is(node) &&+ OperatorNode.is(node.operator) &&+ node.operator.operator === "="+ ) {+ return coreSafe(node.leftOperand, "=", node.rightOperand);+ }+ if (+ FunctionNode.is(node) &&+ node.func === "startsWith" &&+ node.arguments.length === 2+ ) {+ return coreSafe(node.arguments[0], "starts with", node.arguments[1]);+ }+ return undefined;+}++// `lookup` is typed `{ kind: string }`: `ArrayIndexNode` is outside Kysely's+// closed `OperationNode` union, so narrowing an `OperationNode` yields `never`.+function coreSafe(+ lookup: { kind: string },+ operator: "=" | "starts with",+ value: OperationNode,+): ArrayIndexNode | undefined {+ return ArrayIndexNode.is(lookup) &&+ lookup.metadataLookup &&+ ValueNode.is(value) &&+ metadataFilterIsEventsCoreSafe(operator, value.value)+ ? lookup+ : undefined;+}++/** A FROM/JOIN source of one select. */+type Relation = {+ /** The FROM item or join table, as the router meets it. */+ node: OperationNode;+ /** Alias, or the table name of an unaliased table. */+ name?: string;+ logical: boolean;+};diff --git a/packages/shared/src/server/query-ast/extensions.ts b/packages/shared/src/server/query-ast/extensions.tsindex 3948c25df2..08b546bf38 100644--- a/packages/shared/src/server/query-ast/extensions.ts+++ b/packages/shared/src/server/query-ast/extensions.ts@@ -228,11 +228,20 @@ export function dateDiff(export function mapValue<V = number>(map: Expression<Record<string, V>>,key: string | Expression<string>,+): ChExpression<V> {+ return subscript(map, key, false);+}++function subscript<V>(+ map: Expression<Record<string, V>>,+ key: string | Expression<string>,+ metadataLookup: boolean,): ChExpression<V> {return new ExpressionWrapper<ClickHouseDatabase, keyof ClickHouseDatabase, V>(ArrayIndexNode.create(map.toOperationNode(),typeof key === "string" ? ValueNode.create(key) : key.toOperationNode(),+ metadataLookup,) as unknown as OperationNode,);}@@ -372,6 +381,14 @@ export function concat(...values: Expression<string>[]): ChExpression<string> {return functionCall("concat", values);}+/** ClickHouse `startsWith(haystack, prefix)`, with `prefix` bound. */+export function startsWith(+ haystack: Expression<string>,+ prefix: string,+): ChExpression<SqlBool> {+ return functionCall("startsWith", [haystack, ValueNode.create(prefix)]);+}+/** ClickHouse `length(array)`. */export function arrayLength(array: Expression<unknown[]>,@@ -648,9 +665,10 @@ export function metadataValue(new ExpressionWrapper<ClickHouseDatabase, "events_core", string[]>(qualifiedColumn(tableAlias, name),);- return mapValue<string>(+ return subscript<string>(metadataMap(column("metadata_names"), column("metadata_values")),key,+ true,);}diff --git a/packages/shared/src/server/query-ast/nodes.test.ts b/packages/shared/src/server/query-ast/nodes.test.tsindex 4f16dffff0..a7428ac39f 100644--- a/packages/shared/src/server/query-ast/nodes.test.ts+++ b/packages/shared/src/server/query-ast/nodes.test.ts@@ -846,3 +846,46 @@ describe("ARRAY JOIN and LIMIT BY nodes", () => {expect(() => useFinal(["events_core"])).toThrow(/immutable/);});});++describe("names the compiler writes verbatim", () => {+ const db = getClickhouseKysely();++ it.each(["columns ", "tuple(s.comment), length", "default.f"])(+ "rejects the function name %j",+ (name) => {+ expect(() =>+ compileClickhouseQuery(+ db+ .selectFrom("scores as s")+ .select((eb) => eb.fn<string>(name, [eb.val("x")]).as("x")),+ projectId,+ ),+ ).toThrow(QueryCompileError);+ expect(() =>+ compileClickhouseQuery(+ db+ .selectFrom("scores as s")+ .select((eb) => eb.fn.agg<string>(name, [eb.val("x")]).as("x")),+ projectId,+ ),+ ).toThrow(QueryCompileError);+ },+ );++ // ClickHouse parses only a string literal after `collate`, which Kysely+ // does not emit.+ it.each(["en", "'en', s.comment"])(+ "rejects the collation %j",+ (collation) => {+ expect(() =>+ compileClickhouseQuery(+ db+ .selectFrom("scores as s")+ .select("s.id")+ .orderBy("s.name", (ob) => ob.collate(collation)),+ projectId,+ ),+ ).toThrow(QueryCompileError);+ },+ );+});diff --git a/packages/shared/src/server/query-ast/nodes.ts b/packages/shared/src/server/query-ast/nodes.tsindex 0356d89f12..1688d6ce8b 100644--- a/packages/shared/src/server/query-ast/nodes.ts+++ b/packages/shared/src/server/query-ast/nodes.ts@@ -80,17 +80,27 @@ export type ArrayIndexNode = {readonly kind: "ArrayIndexNode";readonly array: OperationNode;readonly index: OperationNode;+ /**+ * `metadata[key]` over one relation's `metadata_names`/`metadata_values`,+ * so events routing can keep a short metadata predicate on `events_core`.+ */+ readonly metadataLookup?: true;};export const ArrayIndexNode = {is(node: { kind: string }): node is ArrayIndexNode {return node.kind === "ArrayIndexNode";},- create(array: OperationNode, index: OperationNode): ArrayIndexNode {+ create(+ array: OperationNode,+ index: OperationNode,+ metadataLookup = false,+ ): ArrayIndexNode {return Object.freeze({kind: "ArrayIndexNode",array,index,+ ...(metadataLookup ? { metadataLookup: true as const } : {}),});},};diff --git a/packages/shared/src/server/query-ast/schema.test.ts b/packages/shared/src/server/query-ast/schema.test.tsindex 000e7b9a72..340d388604 100644--- a/packages/shared/src/server/query-ast/schema.test.ts+++ b/packages/shared/src/server/query-ast/schema.test.ts@@ -12,6 +12,7 @@ import { schemaTypeAssertions, extensionTypeAssertions } from "./types.assert";describe("table registry derivation", () => {it("derives the tenanted table set from the registry", () => {expect([...TENANTED_TABLES].sort()).toEqual([+ "events","events_core","events_full","observations",diff --git a/packages/shared/src/server/query-ast/schema.ts b/packages/shared/src/server/query-ast/schema.tsindex 48d466b762..4038bd8b9b 100644--- a/packages/shared/src/server/query-ast/schema.ts+++ b/packages/shared/src/server/query-ast/schema.ts@@ -229,6 +229,23 @@ const EVENTS_COLUMNS = {ingestion_api_key: "String",} as const satisfies Record<string, ChColumnType>;+/**+ * Columns whose `events_core` copy differs from `events_full`:+ * `events_core_mv` keeps the first 200 code points of `input`, `output`, and+ * each `metadata_values` element, and `input_length` / `output_length` are+ * materialized from the truncated copy. Any other read of them needs+ * `events_full`. Must match the truncations in the latest `events_core_mv`+ * definition (`clickhouse/migrations/canonical/0053_add_skill_observation_references.up.sql`);+ * nothing checks this automatically.+ */+export const EVENTS_CORE_TRUNCATED_COLUMNS: ReadonlySet<string> = new Set([+ "input",+ "output",+ "metadata_values",+ "input_length",+ "output_length",+] satisfies ReadonlyArray<keyof typeof EVENTS_COLUMNS>);+const TABLE_REGISTRY = {traces: defineTable({columns: {@@ -262,6 +279,14 @@ const TABLE_REGISTRY = {dedup: { strategy: "none" },softDelete: "is_deleted",}),+ // Logical relation over events_core / events_full. The routing pass+ // (`eventsRouting.ts`) replaces it with one of them before any other pass+ // runs, so ClickHouse never sees this name.+ events: defineTable({+ columns: EVENTS_COLUMNS,+ dedup: { strategy: "none" },+ softDelete: "is_deleted",+ }),scores: defineTable({columns: {id: "String",@@ -303,14 +328,18 @@ export type ClickHouseDatabase = {};/**- * Declared ClickHouse type per column, per registry table. A servertest checks- * it against `system.columns` of the migrated database.+ * Declared ClickHouse type per column, per physical registry table. A+ * servertest checks it against `system.columns` of the migrated database. The+ * logical `events` relation has no table of its own; its columns are+ * `events_core`'s and `events_full`'s.*/export const TABLE_COLUMN_TYPES: Record<string,Record<string, ChColumnType>> = Object.fromEntries(- Object.entries(TABLE_REGISTRY).map(([table, spec]) => [table, spec.columns]),+ Object.entries(TABLE_REGISTRY)+ .filter(([table]) => table !== "events")+ .map(([table, spec]) => [table, spec.columns]),);/**diff --git a/packages/shared/src/server/query-ast/softDelete.test.ts b/packages/shared/src/server/query-ast/softDelete.test.tsindex 6363b538a2..60145fe0d9 100644--- a/packages/shared/src/server/query-ast/softDelete.test.ts+++ b/packages/shared/src/server/query-ast/softDelete.test.ts@@ -24,6 +24,7 @@ function compile(builder: Parameters<typeof compileClickhouseQuery>[0]) {describe("soft-delete filtering", () => {it("declares the flag on the events tables only", () => {expect(SOFT_DELETE_COLUMNS).toEqual({+ events: "is_deleted",events_core: "is_deleted",events_full: "is_deleted",});diff --git a/packages/shared/src/server/query-ast/transformer.ts b/packages/shared/src/server/query-ast/transformer.tsindex 4508812d64..aff947e4d1 100644--- a/packages/shared/src/server/query-ast/transformer.ts+++ b/packages/shared/src/server/query-ast/transformer.ts@@ -50,6 +50,7 @@ export class ClickHouseOperationNodeTransformer extends OperationNodeTransformerreturn ArrayIndexNode.create(this.transformNode(node.array),this.transformNode(node.index),+ node.metadataLookup,);}diff --git a/packages/shared/src/server/repositories/prompt-events.ts b/packages/shared/src/server/repositories/prompt-events.tsindex 728b903ba0..88dca6240d 100644--- a/packages/shared/src/server/repositories/prompt-events.ts+++ b/packages/shared/src/server/repositories/prompt-events.ts@@ -4,7 +4,7 @@ import { getClickhouseKysely } from "../query-ast/dialect";import { limitBy, softDeleteAfterLimit } from "../query-ast/extensions";import type { ClickHouseDatabase } from "../query-ast/schema";-type EventsAsE = ClickHouseDatabase & { e: ClickHouseDatabase["events_core"] };+type EventsAsE = ClickHouseDatabase & { e: ClickHouseDatabase["events"] };/*** Latest version of each generation linked to one of `promptIds`, optionally@@ -19,7 +19,7 @@ export const promptGenerationEvents = <timeWindow: { fromTimestamp?: Date; toTimestamp?: Date } = {},) =>getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").$call(softDeleteAfterLimit()).select(columns).where("e.type", "=", "GENERATION")diff --git a/packages/shared/src/server/repositories/scores.ts b/packages/shared/src/server/repositories/scores.tsindex e548d19b08..a242a6a30f 100644--- a/packages/shared/src/server/repositories/scores.ts+++ b/packages/shared/src/server/repositories/scores.ts@@ -420,7 +420,7 @@ export const getTraceScoresForDatasetRuns = async (const experimentItemEventsQuery = (experimentIds: string[]) =>getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").select(["e.project_id", "e.experiment_id", "e.trace_id"]).where("e.experiment_id", "in", experimentIds).where("e.experiment_id", "!=", "");