Skip to content

πŸͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

refactor(query-ast): route a logical events relation to events_core or events_full (#18868)

Niklas Semmler53b19bff82bf
diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.ts
index 7115594f2c..4ce7f0ff28 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.events-full-routing.test.ts
@@ -5,12 +5,12 @@ import {
FilterList,
filtersRequireEventsFull,
metadataFilterCanPruneEventsFull,
- metadataFilterIsEventsCoreSafe,
NullFilter,
NumberObjectFilter,
StringFilter,
StringObjectFilter,
} from "./clickhouse-filter";
+import { metadataFilterIsEventsCoreSafe } from "./events-core-truncation";
import { createFilterFromFilterState } from "./factory";
import { experimentItemsTableNativeUiColumnDefinitions } from "../../tableMappings/mapExperimentItemsTable";
diff --git a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts
index dd6560e11d..c915b10c68 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts
@@ -18,6 +18,7 @@ import {
isFtsTextTarget,
isNgramSubstringTarget,
} from "./fts";
+import { metadataFilterIsEventsCoreSafe } from "./events-core-truncation";
export type ClickhouseOperator =
| (typeof filterOperators)[keyof typeof filterOperators][number]
@@ -860,91 +861,6 @@ export class FilterList {
}
}
-// events_core_mv truncates each metadata_values element (and input/output) to
-// the first 200 UTF-8 code points via leftUTF8(v, 200). A metadata filter is
-// only safe to answer against events_core when 200-char truncation cannot
-// change its result.
-const EVENTS_CORE_TRUNCATION_LIMIT = 200;
-
-// Metadata operators that can be answered correctly against the truncated
-// events_core copy β€” an allow-list, so any operator not named here defaults to
-// events_full. That default is the safe one: routing a decidable filter to
-// events_full only costs performance, whereas routing a truncation-sensitive
-// filter to events_core silently drops matches past code point 200. When a new
-// ClickhouseOperator is added it must be reviewed and added here explicitly
-// before it can use events_core.
-// - `=` / `starts with`: string ops, subject to the length guard below.
-// - `>` `<` `>=` `<=`: numeric comparisons β€” values are inherently short.
-// - `<>`: boolean not-equal β€” value is inherently short.
-// - `is null` / `is not null`: truncation-invariant (metadata_names is not
-// truncated; emptiness is unaffected).
-// - `is set` / `is not set`: key-presence over metadata_names only, which is
-// not truncated.
-// Deliberately excluded (match can live past code point 200): `contains`,
-// `does not contain`, `ends with`, and the FTS `matches` operator (which also
-// needs the events_full-only index).
-const EVENTS_CORE_SAFE_METADATA_OPERATORS = new Set<ClickhouseOperator>([
- "=",
- "starts with",
- ">",
- "<",
- ">=",
- "<=",
- "<>",
- "is null",
- "is not null",
- "is set",
- "is not set",
-]);
-
-// Count Unicode code points to mirror ClickHouse leftUTF8, which truncates by
-// code point rather than UTF-16 unit or byte.
-const codePointLength = (value: string): number => Array.from(value).length;
-
-/**
- * Truncation-safety classifier: can a single metadata filter be answered
- * correctly against the truncated events_core copy, or must it read events_full?
- *
- * events_core keeps only the first {@link EVENTS_CORE_TRUNCATION_LIMIT} code
- * points of each metadata value (metadata_names is not truncated). This is an
- * allow-list: an operator is events_core-safe only if it is named in
- * {@link EVENTS_CORE_SAFE_METADATA_OPERATORS} and, for the two length-sensitive
- * string operators, its value fits within the retained prefix:
- * - `starts with`, value length <= 200 β€” the prefix lives within retained chars.
- * - `=`, value length < 200 β€” a stored value longer than 200 truncates to 200
- * code points and can never equal a sub-200 value. Strict `<` avoids the
- * exact-200 false positive where a >200 value shares the first 200 chars.
- * - `is null` / `is not null` β€” truncation invariant (metadata_names is
- * untruncated; emptiness is unaffected).
- * - numeric / boolean metadata comparisons β€” the value is inherently short,
- * so its truncated copy is complete.
- * Anything else (including any newly added operator) is routed to events_full,
- * because over-routing only costs performance while under-routing silently
- * drops matches past the truncation boundary.
- *
- * Single source of truth for metadata truncation routing. Do not reuse
- * NGRAM_ACCELERATED_METADATA_OPERATORS or FTS_TEXT_OPERATORS: those classify
- * ngram/FTS index eligibility, a different axis (the ngram set includes
- * truncation-unsafe `contains`/`ends with` and excludes truncation-safe `=`).
- */
-export const metadataFilterIsEventsCoreSafe = (
- operator: ClickhouseOperator,
- value: unknown,
-): boolean => {
- if (!EVENTS_CORE_SAFE_METADATA_OPERATORS.has(operator)) {
- return false;
- }
- if (typeof value === "string") {
- if (operator === "starts with") {
- return codePointLength(value) <= EVENTS_CORE_TRUNCATION_LIMIT;
- }
- if (operator === "=") {
- return codePointLength(value) < EVENTS_CORE_TRUNCATION_LIMIT;
- }
- }
- return true;
-};
-
/**
* Whether a metadata string filter on events_full emits a conjunct that an
* events_full skip index can prune on: the ngram conjunct for substring
diff --git a/packages/shared/src/server/queries/clickhouse-sql/events-core-truncation.ts b/packages/shared/src/server/queries/clickhouse-sql/events-core-truncation.ts
new file mode 100644
index 0000000000..dc113af734
--- /dev/null
+++ b/packages/shared/src/server/queries/clickhouse-sql/events-core-truncation.ts
@@ -0,0 +1,88 @@
+import type { ClickhouseOperator } from "./clickhouse-filter";
+
+// events_core_mv truncates each metadata_values element (and input/output) to
+// the first 200 UTF-8 code points via leftUTF8(v, 200). A metadata filter is
+// only safe to answer against events_core when 200-char truncation cannot
+// change its result.
+const EVENTS_CORE_TRUNCATION_LIMIT = 200;
+
+// Metadata operators that can be answered correctly against the truncated
+// events_core copy β€” an allow-list, so any operator not named here defaults to
+// events_full. That default is the safe one: routing a decidable filter to
+// events_full only costs performance, whereas routing a truncation-sensitive
+// filter to events_core silently drops matches past code point 200. When a new
+// ClickhouseOperator is added it must be reviewed and added here explicitly
+// before it can use events_core.
+// - `=` / `starts with`: string ops, subject to the length guard below.
+// - `>` `<` `>=` `<=`: numeric comparisons β€” values are inherently short.
+// - `<>`: boolean not-equal β€” value is inherently short.
+// - `is null` / `is not null`: truncation-invariant (metadata_names is not
+// truncated; emptiness is unaffected).
+// - `is set` / `is not set`: key-presence over metadata_names only, which is
+// not truncated.
+// Deliberately excluded (match can live past code point 200): `contains`,
+// `does not contain`, `ends with`, and the FTS `matches` operator (which also
+// needs the events_full-only index).
+const EVENTS_CORE_SAFE_METADATA_OPERATORS = new Set<ClickhouseOperator>([
+ "=",
+ "starts with",
+ ">",
+ "<",
+ ">=",
+ "<=",
+ "<>",
+ "is null",
+ "is not null",
+ "is set",
+ "is not set",
+]);
+
+/**
+ * Truncation-safety classifier: can a single metadata filter be answered
+ * correctly against the truncated events_core copy, or must it read events_full?
+ *
+ * events_core keeps only the first {@link EVENTS_CORE_TRUNCATION_LIMIT} code
+ * points of each metadata value (metadata_names is not truncated). This is an
+ * allow-list: an operator is events_core-safe only if it is named in
+ * {@link EVENTS_CORE_SAFE_METADATA_OPERATORS} and, for the two length-sensitive
+ * string operators, its value fits within the retained prefix:
+ * - `starts with`, value length <= 200 β€” the prefix lives within retained chars.
+ * - `=`, value length < 200 β€” a stored value longer than 200 truncates to 200
+ * code points and can never equal a sub-200 value. Strict `<` avoids the
+ * exact-200 false positive where a >200 value shares the first 200 chars.
+ * - `is null` / `is not null` β€” truncation invariant (metadata_names is
+ * untruncated; emptiness is unaffected).
+ * - numeric / boolean metadata comparisons β€” the value is inherently short,
+ * so its truncated copy is complete.
+ * Anything else (including any newly added operator) is routed to events_full,
+ * because over-routing only costs performance while under-routing silently
+ * drops matches past the truncation boundary.
+ *
+ * Single source of truth for metadata truncation routing. Do not reuse
+ * NGRAM_ACCELERATED_METADATA_OPERATORS or FTS_TEXT_OPERATORS: those classify
+ * ngram/FTS index eligibility, a different axis (the ngram set includes
+ * truncation-unsafe `contains`/`ends with` and excludes truncation-safe `=`).
+ */
+export const metadataFilterIsEventsCoreSafe = (
+ operator: ClickhouseOperator,
+ value: unknown,
+): boolean => {
+ if (!EVENTS_CORE_SAFE_METADATA_OPERATORS.has(operator)) {
+ return false;
+ }
+ if (typeof value === "string") {
+ if (operator === "starts with") {
+ return codePointLength(value) <= EVENTS_CORE_TRUNCATION_LIMIT;
+ }
+ if (operator === "=") {
+ return codePointLength(value) < EVENTS_CORE_TRUNCATION_LIMIT;
+ }
+ }
+ return true;
+};
+
+// Count Unicode code points to mirror ClickHouse leftUTF8, which truncates by
+// code point rather than UTF-16 unit or byte.
+function codePointLength(value: string): number {
+ return Array.from(value).length;
+}
diff --git a/packages/shared/src/server/queries/index.ts b/packages/shared/src/server/queries/index.ts
index 6e452fc7b4..a45527e56c 100644
--- a/packages/shared/src/server/queries/index.ts
+++ b/packages/shared/src/server/queries/index.ts
@@ -21,9 +21,9 @@ export {
encodeBooleanScoreEntry,
filtersRequireEventsFull,
metadataFilterCanPruneEventsFull,
- metadataFilterIsEventsCoreSafe,
type ClickhouseOperator,
} from "./clickhouse-sql/clickhouse-filter";
+export { metadataFilterIsEventsCoreSafe } from "./clickhouse-sql/events-core-truncation";
export {
orderByToClickhouseSql,
orderByToEntries,
diff --git a/packages/shared/src/server/query-ast/README.md b/packages/shared/src/server/query-ast/README.md
index 4f1d7b5114..bd6638a0d2 100644
--- a/packages/shared/src/server/query-ast/README.md
+++ b/packages/shared/src/server/query-ast/README.md
@@ -85,6 +85,22 @@ suffix to run there. Promote it to a required check once it has proven stable.
- **Never filter `project_id` yourself.** The compile step injects
`project_id = {projectId}` into every tenanted relation, so call sites pass
only `projectId` (see `../repositories/environments.ts`).
+- **Read events through the logical `events` relation.**
+ Each `events` relation compiles to `events_core` unless a column reference
+ that resolves to it reads a column `events_core` truncates (`input`,
+ `output`, `metadata_values`, `input_length`, `output_length`); then that
+ relation becomes `events_full`. Metadata `=` / `startsWith` predicates on
+ `metadataValue` stay on `events_core` when
+ `metadataFilterIsEventsCoreSafe` accepts the value, as in the raw-SQL
+ builders. `*` / `e.*` over `events`, raw SQL text in a select that reads
+ it, an unqualified truncated column, and a CTE named `events_core` /
+ `events_full` are compile errors while `events` is in scope. Name
+ `events_core` to read the truncated copy on purpose; `events_core` /
+ `events_full` stay selectable by name, which is also the override for a
+ shape the pass rejects.
+- **Function and aggregate names must be plain identifiers.**
+ The compiler writes them verbatim, so anything else fails to compile.
+ `collate` is not supported.
- **Never filter `is_deleted` on a physical table yourself.** Tables that
declare `softDelete` in the registry (`events_core`, `events_full`) get
`is_deleted = 0` on every FROM/JOIN. A latest-version read (ordered by
@@ -142,7 +158,7 @@ db.selectFrom("observations")
**Write a LIMIT BY** (`limitBy`):
```ts
-db.selectFrom("events_core")
+db.selectFrom("events")
.$call(softDeleteAfterLimit())
.select(["span_id", "project_id"])
.orderBy("event_ts", "desc")
@@ -160,16 +176,19 @@ db.selectFrom("scores as s")
// … from scores as s final
```
-**Select a metadata value** (`metadataValue` β€” lowers `metadata[key]` to a subscript of `metadataMap`, so the last duplicate key wins; like `mapValue`, a string key is bound and a `literal(...)` key is inlined):
+**Filter on a metadata value** (`metadataValue` β€” lowers `metadata[key]` to a subscript of `metadataMap`, so the last duplicate key wins; like `mapValue`, a string key is bound and a `literal(...)` key is inlined):
```ts
const filterKey: string = input.metadataKey; // caller-supplied, so bound
-db.selectFrom("events_core as e")
- .select((eb) => [metadataValue("e", literal("my_key")).as("my_val")])
+db.selectFrom("events as e")
+ .select("e.span_id")
.where((eb) => eb(metadataValue("e", filterKey), "=", "x"));
-// select mapfromarrays(arrayreverse(e.metadata_names), arrayreverse(e.metadata_values))['my_key'] as my_val …
+// select e.span_id from events_core as e where … and mapfromarrays(arrayreverse(e.metadata_names), arrayreverse(e.metadata_values))[{p:String}] = {p:String}
```
+Selecting `metadataValue(...)` reads the truncated `metadata_values`, so it
+routes that relation to `events_full`.
+
## Developing the dialect
Several patterns here are deliberately unusual. This is the guide for anyone
diff --git a/packages/shared/src/server/query-ast/compile.ts b/packages/shared/src/server/query-ast/compile.ts
index 1e75dc5c7e..2751d23e60 100644
--- a/packages/shared/src/server/query-ast/compile.ts
+++ b/packages/shared/src/server/query-ast/compile.ts
@@ -7,6 +7,7 @@ import {
import { ClickHouseQueryCompiler } from "./compiler";
import { DedupLoweringPlugin } from "./dedup";
+import { EventsTableRoutingPlugin } from "./eventsRouting";
import { SoftDeleteFilterPlugin } from "./softDelete";
import {
TenancyInjectionPlugin,
@@ -33,7 +34,8 @@ export type ClickhouseCompilable = {
* The choke point. Every query that should produce ClickHouse SQL goes
* through here: the `projectId` parameter is required, so a caller cannot
* compile without a tenancy scope β€” omitting it is a compile-time type error.
- * The query reads exactly that one project. Soft-deleted rows are filtered
+ * The query reads exactly that one project. The logical `events` relation is
+ * routed to `events_core` or `events_full`, soft-deleted rows are filtered
* out, tenancy is injected from that project, per-table dedup is applied from
* the registry, and the dialect compiler refuses to emit SQL unless the tree
* was identity-stamped by the tenancy and dedup passes (a copied property is
@@ -47,6 +49,7 @@ export function compileClickhouseQuery(
): CompiledClickhouseQuery {
const scope = requireProjectId(projectId);
const stamped = query
+ .withPlugin(new EventsTableRoutingPlugin())
.withPlugin(new SoftDeleteFilterPlugin())
.withPlugin(new TenancyInjectionPlugin(scope))
.withPlugin(new DedupLoweringPlugin())
diff --git a/packages/shared/src/server/query-ast/compiler.ts b/packages/shared/src/server/query-ast/compiler.ts
index f9a40c7397..def1e47dcd 100644
--- a/packages/shared/src/server/query-ast/compiler.ts
+++ b/packages/shared/src/server/query-ast/compiler.ts
@@ -16,6 +16,7 @@ import {
SetOperationNode,
TableNode,
ValueNode,
+ type CollateNode,
type CommonTableExpressionNode,
type OperationNode,
type PrimitiveValueListNode,
@@ -43,6 +44,8 @@ const ARRAY_JOIN_SQL: Record<ArrayJoinNode["variant"], string> = {
inner: "inner array join",
};
+const PLAIN_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
const DATE_WRAPPER_FNS = new Set([
"toDate",
"toDateTime",
@@ -274,6 +277,31 @@ export class ClickHouseQueryCompiler extends DefaultQueryCompiler {
return "";
}
+ /**
+ * Function and aggregate names are written verbatim, so a name that is not
+ * a plain identifier would be raw SQL: `fn("tuple(e.input), f")` emits
+ * `tuple(e.input), f(…)`.
+ */
+ protected override visitFunction(node: FunctionNode): void {
+ assertPlainName("Function", node.func);
+ super.visitFunction(node);
+ }
+
+ protected override visitAggregateFunction(node: AggregateFunctionNode): void {
+ assertPlainName("Aggregate function", node.func);
+ super.visitAggregateFunction(node);
+ }
+
+ /**
+ * ClickHouse parses only a string literal after `collate`; the name is
+ * written verbatim, so no collation compiles to valid SQL.
+ */
+ protected override visitCollate(node: CollateNode): void {
+ throw new QueryCompileError(
+ `Collation "${node.collation.name}" is not supported: ClickHouse expects a string literal after collate.`,
+ );
+ }
+
protected override appendValue(parameter: unknown): void {
this.append(this.bindValue(parameter, this.pendingBindType));
}
@@ -736,3 +764,11 @@ function withoutCte(scope: SelectScope, name: string): SelectScope {
relations: new Map(),
};
}
+
+function assertPlainName(kind: string, name: string): void {
+ if (!PLAIN_IDENTIFIER.test(name)) {
+ throw new QueryCompileError(
+ `${kind} name "${name}" is not a plain identifier; the compiler writes it verbatim into the SQL.`,
+ );
+ }
+}
diff --git a/packages/shared/src/server/query-ast/eventsRouting.test.ts b/packages/shared/src/server/query-ast/eventsRouting.test.ts
new file mode 100644
index 0000000000..60cc9c1712
--- /dev/null
+++ b/packages/shared/src/server/query-ast/eventsRouting.test.ts
@@ -0,0 +1,660 @@
+import {
+ sql,
+ type Expression,
+ type ExpressionBuilder,
+ type SqlBool,
+} from "kysely";
+import { describe, expect, it } from "vitest";
+
+import {
+ FilterList,
+ filtersRequireEventsFull,
+ StringFilter,
+ StringObjectFilter,
+} from "../queries/clickhouse-sql/clickhouse-filter";
+import { compileClickhouseQuery } from "./compile";
+import { getClickhouseKysely } from "./dialect";
+import { QueryCompileError } from "./errors";
+import {
+ includeDeleted,
+ limitBy,
+ mapValue,
+ metadataMap,
+ metadataValue,
+ startsWith,
+ useFinal,
+} from "./extensions";
+import type { ClickHouseDatabase } from "./schema";
+
+// Raw compiler output, so these run without a `clickhouse` binary.
+const projectId = "p";
+
+function compile(builder: Parameters<typeof compileClickhouseQuery>[0]) {
+ return compileClickhouseQuery(builder, projectId).sql;
+}
+
+/** Physical events tables in FROM/JOIN position, in order. */
+function routedTables(query: string): string[] {
+ return [
+ ...query.matchAll(/\b(?:from|join) (events_core|events_full)\b/g),
+ ].map((match) => match[1]);
+}
+
+const db = getClickhouseKysely();
+
+type EventsEb = ExpressionBuilder<
+ ClickHouseDatabase & { e: ClickHouseDatabase["events"] },
+ "e"
+>;
+
+describe("logical events routing", () => {
+ it("routes a read of untruncated columns to events_core", () => {
+ expect(
+ compile(
+ db
+ .selectFrom("events as e")
+ .select(["e.span_id", "e.name"])
+ .where("e.trace_id", "=", "t"),
+ ),
+ ).toMatchInlineSnapshot(
+ `"select e.span_id, e.name from events_core as e where e.project_id = {p1:String} and e.is_deleted = {p2:UInt8} and e.trace_id = {p3:String}"`,
+ );
+ });
+
+ it.each([
+ "input",
+ "output",
+ "metadata_values",
+ "input_length",
+ "output_length",
+ ] as const)("routes a projection of %s to events_full", (column) => {
+ expect(
+ routedTables(compile(db.selectFrom("events as e").select(`e.${column}`))),
+ ).toEqual(["events_full"]);
+ });
+
+ it("routes a projected metadata value to events_full", () => {
+ expect(
+ routedTables(
+ compile(
+ db
+ .selectFrom("events as e")
+ .select(() => [metadataValue("e", "k").as("v")]),
+ ),
+ ),
+ ).toEqual(["events_full"]);
+ });
+
+ it("rejects select * and select e.*, but not count(*)", () => {
+ expect(() => compile(db.selectFrom("events").selectAll())).toThrow(
+ QueryCompileError,
+ );
+ expect(() => compile(db.selectFrom("events as e").selectAll("e"))).toThrow(
+ QueryCompileError,
+ );
+ expect(
+ routedTables(
+ compile(
+ db.selectFrom("events as e").select((eb) => eb.fn.countAll().as("n")),
+ ),
+ ),
+ ).toEqual(["events_core"]);
+ });
+
+ it("routes count(*) with a clause on a truncated column to events_full", () => {
+ expect(
+ routedTables(
+ compile(
+ db
+ .selectFrom("events as e")
+ .select((eb) =>
+ eb.fn.countAll().filterWhere("e.input", "=", "x").as("n"),
+ ),
+ ),
+ ),
+ ).toEqual(["events_full"]);
+ expect(
+ routedTables(
+ compile(
+ db.selectFrom("events as e").select((eb) =>
+ eb.fn
+ .countAll()
+ .over((ob) => ob.partitionBy("e.input"))
+ .as("n"),
+ ),
+ ),
+ ),
+ ).toEqual(["events_full"]);
+ });
+
+ it("keeps count(*) with a clause on an untruncated column on events_core", () => {
+ expect(
+ routedTables(
+ compile(
+ db
+ .selectFrom("events as e")
+ .select((eb) =>
+ eb.fn.countAll().filterWhere("e.name", "=", "x").as("n"),
+ ),
+ ),
+ ),
+ ).toEqual(["events_core"]);
+ });
+
+ it("ignores an alias named like a truncated column", () => {
+ expect(
+ routedTables(
+ compile(
+ db
+ .selectFrom("events as e")
+ .select((eb) => eb.fn.count("e.span_id").as("input")),
+ ),
+ ),
+ ).toEqual(["events_core"]);
+ });
+
+ it.each([
+ ["lengthUTF8(e.output)", sql<string>`lengthUTF8(e.output)`],
+ ["sql.id e.input", sql<string>`${sql.id("e", "input")}`],
+ ["sql.id input", sql<string>`${sql.id("input")}`],
+ ["sql.table input", sql<string>`${sql.table("input")}`],
+ ["*", sql<string>`*`],
+ ["e.*", sql.raw<string>("e.*")],
+ ["tuple(*)", sql<string>`tuple(*)`],
+ ["COLUMNS(...)", sql<string>`COLUMNS('^inp')`],
+ ["sql.ref e.*", sql.ref<string>("e.*")],
+ ["sql.ref *", sql.ref<string>("*")],
+ ["tuple(sql.ref e.*)", sql<string>`tuple(${sql.ref("e.*")})`],
+ ])("rejects raw %s in a select that reads events", (_, expression) => {
+ expect(() =>
+ compile(db.selectFrom("events as e").select(expression.as("x"))),
+ ).toThrow(QueryCompileError);
+ });
+
+ it("rejects a raw identifier in the where clause of a select that reads events", () => {
+ expect(() =>
+ compile(
+ db
+ .selectFrom("events as e")
+ .select("e.span_id")
+ .where(sql<string>`${sql.id("input")}`, "=", "x"),
+ ),
+ ).toThrow(QueryCompileError);
+ });
+
+ it("rejects raw SQL in a select nested under one that reads events", () => {
+ expect(() =>
+ compile(
+ db.selectFrom("events as e").select((eb) => [
+ "e.span_id",
+ eb
+ .selectFrom("scores as s")
+ .select(sql<number>`1`.as("one"))
+ .limit(1)
+ .as("x"),
+ ]),
+ ),
+ ).toThrow(QueryCompileError);
+ });
+
+ it("rejects COLUMNS() in a select that reads events", () => {
+ expect(() =>
+ compile(
+ db
+ .selectFrom("events as e")
+ .select((eb) => eb.fn<string>("COLUMNS", [eb.val("^inp")]).as("x")),
+ ),
+ ).toThrow(QueryCompileError);
+ expect(() =>
+ compile(
+ db
+ .selectFrom("events as e")
+ .select((eb) =>
+ eb.fn.agg<string>("columns", [eb.val("^inp")]).as("x"),
+ ),
+ ),
+ ).toThrow(QueryCompileError);
+ });
+
+ it.each([
+ "events_core.input",
+ "default.events_core.input",
+ "events_full.input",
+ ])("rejects the physical qualifier %s over a logical relation", (ref) => {
+ // ClickHouse resolves a qualifier by table name even when the table is aliased.
+ expect(() =>
+ compile(
+ db.selectFrom("events as e").select(sql.ref<string>(ref).as("x")),
+ ),
+ ).toThrow(QueryCompileError);
+ });
+
+ it("keeps a physical events qualifier when no logical relation is in scope", () => {
+ expect(
+ routedTables(
+ compile(
+ db
+ .selectFrom("events_core as c")
+ .$call(includeDeleted())
+ .select(sql.ref<string>("events_core.input").as("x")),
+ ),
+ ),
+ ).toEqual(["events_core"]);
+ });
+
+ it("resolves sql.ref like a builder reference", () => {
+ const route = (ref: string) =>
+ routedTables(
+ compile(
+ db.selectFrom("events as e").select(sql.ref<string>(ref).as("x")),
+ ),
+ );
+ expect(route("e.input")).toEqual(["events_full"]);
+ expect(route("e.span_id")).toEqual(["events_core"]);
+ });
+
+ it("allows raw SQL in a select over a subquery that reads events", () => {
+ expect(
+ routedTables(
+ compile(
+ db
+ .selectFrom(
+ db.selectFrom("events as e").select("e.span_id").as("x"),
+ )
+ .select(sql<number>`1`.as("one")),
+ ),
+ ),
+ ).toEqual(["events_core"]);
+ });
+
+ it("rejects an unqualified truncated column of the only relation", () => {
+ expect(() => compile(db.selectFrom("events").select("input"))).toThrow(
+ QueryCompileError,
+ );
+ });
+
+ it("routes a table-qualified read of an unaliased logical relation", () => {
+ expect(
+ routedTables(compile(db.selectFrom("events").select("events.input"))),
+ ).toEqual(["events_full"]);
+ });
+
+ it("rejects an unqualified limit by column of the only relation", () => {
+ expect(() =>
+ compile(
+ db
+ .selectFrom("events")
+ .select("span_id")
+ .$call(limitBy({ count: 1, columns: ["input"] })),
+ ),
+ ).toThrow(QueryCompileError);
+ expect(() =>
+ compile(
+ db
+ .selectFrom("events")
+ .select("span_id")
+ .$call(limitBy({ count: 1, columns: ["*"] })),
+ ),
+ ).toThrow(QueryCompileError);
+ });
+
+ it("routes each use of a shared builder by its own reads", () => {
+ // Both CTEs hold the same FROM node.
+ const base = db.selectFrom("events as e");
+ expect(
+ routedTables(
+ compile(
+ db
+ .with("a", () => base.select(["e.span_id", "e.input"]))
+ .with("b", () => base.select(["e.span_id"]))
+ .selectFrom("a")
+ .innerJoin("b", "a.span_id", "b.span_id")
+ .select("a.span_id"),
+ ),
+ ),
+ ).toEqual(["events_full", "events_core"]);
+ });
+
+ it("rejects an unqualified truncated column in a subquery under a select that reads events", () => {
+ expect(() =>
+ compile(
+ db
+ .selectFrom("events as e")
+ .select((eb) => [
+ "e.span_id",
+ eb
+ .selectFrom("scores as s")
+ .select(sql.ref<string>("input").as("i"))
+ .limit(1)
+ .as("x"),
+ ]),
+ ),
+ ).toThrow(QueryCompileError);
+ });
+
+ it("rejects an unqualified truncated column in a subquery with no FROM under a select that reads events", () => {
+ // ClickHouse resolves `input` in a no-FROM subquery to the outer relation.
+ expect(() =>
+ compile(
+ db
+ .selectFrom("events")
+ .select([
+ "span_id",
+ db.selectNoFrom(sql.ref<string>("input").as("i")).as("x"),
+ ]),
+ ),
+ ).toThrow(QueryCompileError);
+ });
+
+ it("resolves a qualifier to an inner FINAL relation before an outer one", () => {
+ expect(
+ routedTables(
+ compile(
+ db.selectFrom("events as s").select((eb) => [
+ "s.span_id",
+ eb
+ .selectFrom("scores as s")
+ .$call(useFinal(["scores"]))
+ .select(sql.ref<string>("s.input").as("i"))
+ .limit(1)
+ .as("x"),
+ ]),
+ ),
+ ),
+ ).toEqual(["events_core"]);
+ });
+
+ it("rejects an unqualified truncated column with two relations in scope", () => {
+ expect(() =>
+ compile(
+ db
+ .selectFrom("events as a")
+ .innerJoin("events as b", "a.span_id", "b.span_id")
+ .select(sql.ref<string>("input").as("x")),
+ ),
+ ).toThrow(QueryCompileError);
+ });
+
+ it("keeps an unaliased logical read addressable by its name", () => {
+ expect(
+ compile(
+ db
+ .selectFrom("events")
+ .select("events.span_id")
+ .where("span_id", "=", "s"),
+ ),
+ ).toMatchInlineSnapshot(
+ `"select events.span_id from events_core as events where events.project_id = {p1:String} and events.is_deleted = {p2:UInt8} and span_id = {p3:String}"`,
+ );
+ });
+
+ it("routes a joined logical relation", () => {
+ expect(
+ routedTables(
+ compile(
+ db
+ .selectFrom("scores as s")
+ .innerJoin("events as e", (join) =>
+ join.onRef("e.trace_id", "=", "s.trace_id"),
+ )
+ .select(["s.id", "e.input"]),
+ ),
+ ),
+ ).toEqual(["events_full"]);
+ });
+
+ it("routes each logical relation by its own reads", () => {
+ // `e` in the CTE and `e` outside are different relations; `io.input`
+ // reads the CTE.
+ const query = db
+ .with("io", (qb) =>
+ qb.selectFrom("events as e").select(["e.span_id", "e.input"]),
+ )
+ .selectFrom("events as e")
+ .innerJoin("io", "io.span_id", "e.span_id")
+ .select(["e.span_id", "io.input"]);
+
+ expect(routedTables(compile(query))).toEqual([
+ "events_full",
+ "events_core",
+ ]);
+ });
+
+ it("routes joined logical relations separately", () => {
+ expect(
+ routedTables(
+ compile(
+ db
+ .selectFrom("events as a")
+ .innerJoin("events as b", "a.span_id", "b.span_id")
+ .select(["a.span_id", "b.input"]),
+ ),
+ ),
+ ).toEqual(["events_core", "events_full"]);
+ });
+
+ it("leaves physical events tables unchanged", () => {
+ expect(
+ routedTables(
+ compile(
+ db
+ .selectFrom("events_core as e")
+ .$call(includeDeleted())
+ .select("e.input"),
+ ),
+ ),
+ ).toEqual(["events_core"]);
+ });
+
+ it("rejects a CTE named events", () => {
+ expect(() =>
+ compile(
+ db
+ .with("events", (qb) =>
+ qb.selectFrom("events_core").select("span_id"),
+ )
+ .selectFrom("events")
+ .select("span_id"),
+ ),
+ ).toThrow(QueryCompileError);
+ });
+
+ it.each(["events_core", "events_full"] as const)(
+ "rejects a CTE named %s alongside a logical relation",
+ (name) => {
+ // The routed relation would read the CTE instead of the table.
+ const withCte = db.with(name, (qb) =>
+ qb.selectFrom("scores as s").select("s.id"),
+ );
+ expect(() =>
+ compile(withCte.selectFrom("events as e").select("e.span_id")),
+ ).toThrow(QueryCompileError);
+ expect(() =>
+ compile(withCte.selectFrom(name).select("id")),
+ ).not.toThrow();
+ },
+ );
+});
+
+describe("logical events routing of metadata predicates", () => {
+ const route = (predicate: (eb: EventsEb) => Expression<SqlBool>) =>
+ routedTables(
+ compile(
+ db.selectFrom("events as e").select("e.span_id").where(predicate),
+ ),
+ );
+
+ it("keeps a short `=` and `startsWith` on events_core", () => {
+ expect(route((eb) => eb(metadataValue("e", "k"), "=", "x"))).toEqual([
+ "events_core",
+ ]);
+ expect(route(() => startsWith(metadataValue("e", "k"), "x"))).toEqual([
+ "events_core",
+ ]);
+ });
+
+ it("keeps a short `=` on events_core under a plugin that rewrites the select", () => {
+ expect(
+ routedTables(
+ compile(
+ db
+ .selectFrom("events as e")
+ .select("e.span_id")
+ .where((eb) => eb(metadataValue("e", "k"), "=", "x"))
+ .$call(limitBy({ count: 1, columns: ["e.span_id"] })),
+ ),
+ ),
+ ).toEqual(["events_core"]);
+ });
+
+ it("keeps a negated short `=` on events_core", () => {
+ expect(
+ route((eb) => eb.not(eb(metadataValue("e", "k"), "=", "x"))),
+ ).toEqual(["events_core"]);
+ });
+
+ it("routes other comparison shapes on a metadata value to events_full", () => {
+ // Operator outside the allow-list.
+ expect(route((eb) => eb(metadataValue("e", "k"), "!=", "x"))).toEqual([
+ "events_full",
+ ]);
+ // Value on the left.
+ expect(
+ route((eb) => eb(eb.val("x"), "=", metadataValue("e", "k"))),
+ ).toEqual(["events_full"]);
+ // Compared to a column, not a bound value.
+ expect(
+ route((eb) => eb(metadataValue("e", "k"), "=", eb.ref("e.name"))),
+ ).toEqual(["events_full"]);
+ });
+
+ it("routes the relation whose values a hand-built metadata map reads", () => {
+ const query = db
+ .selectFrom("events as a")
+ .innerJoin("events as b", "a.span_id", "b.span_id")
+ .select("a.span_id")
+ .where((eb) =>
+ eb(
+ mapValue(
+ metadataMap(
+ eb.ref("a.metadata_names"),
+ eb.ref("b.metadata_values"),
+ ),
+ "k",
+ ),
+ "=",
+ "x",
+ ),
+ );
+
+ expect(routedTables(compile(query))).toEqual([
+ "events_core",
+ "events_full",
+ ]);
+ });
+});
+
+// The raw-SQL builders route through `filtersRequireEventsFull`; the logical
+// relation must pick the same table for the same filter.
+describe("logical events routing agrees with filtersRequireEventsFull", () => {
+ type MetadataOperator =
+ | "="
+ | "starts with"
+ | "contains"
+ | "ends with"
+ | "does not contain";
+
+ const astPredicate = (operator: MetadataOperator, value: string) => {
+ const metadata = metadataValue("e", "k");
+ return (eb: EventsEb): Expression<SqlBool> => {
+ switch (operator) {
+ case "=":
+ return eb(metadata, "=", value);
+ case "starts with":
+ return startsWith(metadata, value);
+ case "contains":
+ return eb(metadata, "like", `%${value}%`);
+ case "ends with":
+ return eb(metadata, "like", `%${value}`);
+ case "does not contain":
+ return eb(metadata, "not like", `%${value}%`);
+ }
+ };
+ };
+
+ // Only `=` and `starts with` route by value length; the others always read
+ // events_full, so one value each covers them.
+ const lengthSensitive: MetadataOperator[] = ["=", "starts with"];
+ const lengthInsensitive: MetadataOperator[] = [
+ "contains",
+ "ends with",
+ "does not contain",
+ ];
+ // "πŸ˜€" is one code point and two UTF-16 units, so these cases fail if
+ // either side counts UTF-16 units instead of code points.
+ const values = [199, 200, 201].flatMap((length) => [
+ { label: `${length} ascii`, value: "a".repeat(length) },
+ { label: `${length} astral`, value: "πŸ˜€".repeat(length) },
+ ]);
+
+ const cases = [
+ ...lengthSensitive.flatMap((operator) =>
+ values.map(({ label, value }) => ({ operator, label, value })),
+ ),
+ ...lengthInsensitive.map((operator) => ({
+ operator,
+ label: "short",
+ value: "a",
+ })),
+ ];
+
+ it.each(cases)("metadata $operator, $label", ({ operator, value }) => {
+ const legacyNeedsFull = filtersRequireEventsFull(
+ new FilterList([
+ new StringObjectFilter({
+ clickhouseTable: "events_core",
+ field: "metadata",
+ key: "k",
+ operator,
+ value,
+ tablePrefix: "e",
+ }),
+ ]),
+ );
+ const [routed] = routedTables(
+ compile(
+ db
+ .selectFrom("events as e")
+ .select("e.span_id")
+ .where(astPredicate(operator, value)),
+ ),
+ );
+
+ expect(routed).toBe(legacyNeedsFull ? "events_full" : "events_core");
+ });
+
+ it.each(["input", "output"] as const)("%s filter", (field) => {
+ const legacyNeedsFull = filtersRequireEventsFull(
+ new FilterList([
+ new StringFilter({
+ clickhouseTable: "events_core",
+ field,
+ operator: "=",
+ value: "x",
+ tablePrefix: "e",
+ }),
+ ]),
+ );
+ const [routed] = routedTables(
+ compile(
+ db
+ .selectFrom("events as e")
+ .select("e.span_id")
+ .where(`e.${field}`, "=", "x"),
+ ),
+ );
+
+ expect(legacyNeedsFull).toBe(true);
+ expect(routed).toBe("events_full");
+ });
+});
diff --git a/packages/shared/src/server/query-ast/eventsRouting.ts b/packages/shared/src/server/query-ast/eventsRouting.ts
new file mode 100644
index 0000000000..88b64aac86
--- /dev/null
+++ b/packages/shared/src/server/query-ast/eventsRouting.ts
@@ -0,0 +1,416 @@
+/**
+ * Logical `events` routing. Runs first in every compile.
+ *
+ * Call sites read the logical `events` relation; this pass replaces each
+ * FROM/JOIN reference to it with `events_core` or `events_full`, one choice
+ * per relation. A relation goes to `events_full` only when a column reference
+ * resolves to it and names a column in {@link EVENTS_CORE_TRUNCATED_COLUMNS},
+ * outside the metadata predicates `metadataFilterIsEventsCoreSafe` accepts:
+ *
+ * metadataValue(e, k) = 'short' β†’ events_core (< 200 code points)
+ * startsWith(metadataValue(e, k), 'p') β†’ events_core (≀ 200 code points)
+ * e.input = 'x', select e.input β†’ events_full
+ *
+ * A qualifier resolves to the innermost FROM/JOIN alias (or unaliased table
+ * name) of that name. Shapes that cannot be resolved to one relation are rejected
+ * rather than routed to `events_full`, so `events_full` is read only where a
+ * query needs it:
+ * - `*`, `e.*` or `COLUMNS(…)` over a logical relation (`count(*)` reads no
+ * column);
+ * - an unqualified truncated column while a logical relation is in scope;
+ * - a column qualified by `events_core` / `events_full` that names no
+ * relation in scope: ClickHouse resolves a qualifier by table name even
+ * when the table is aliased, so it would reach the routed relation;
+ * - raw SQL text or identifiers (`sql` templates, `sql.raw`, `sql.id`,
+ * `sql.table`) in or under a select that reads a logical relation.
+ * `sql.ref` and bound values resolve like builder references.
+ *
+ * Callers that want the truncated copy on purpose name `events_core`;
+ * physical `events_core` / `events_full` references pass through unchanged.
+ */
+import {
+ AggregateFunctionNode,
+ AliasNode,
+ BinaryOperationNode,
+ ColumnNode,
+ CommonTableExpressionNode,
+ FromNode,
+ FunctionNode,
+ IdentifierNode,
+ JoinNode,
+ OperatorNode,
+ RawNode,
+ SelectAllNode,
+ SelectionNode,
+ TableNode,
+ ValueNode,
+ type KyselyPlugin,
+ type OperationNode,
+ type OrderByItemNode,
+ type PluginTransformQueryArgs,
+ type PluginTransformResultArgs,
+ type QueryId,
+ type ReferenceNode,
+ type RootOperationNode,
+ type SelectQueryNode,
+} from "kysely";
+
+import { metadataFilterIsEventsCoreSafe } from "../queries/clickhouse-sql/events-core-truncation";
+import { QueryCompileError } from "./errors";
+import { ArrayIndexNode, FinalTableNode, unwrapFinalTable } from "./nodes";
+import { EVENTS_CORE_TRUNCATED_COLUMNS } from "./schema";
+import { ClickHouseOperationNodeTransformer } from "./transformer";
+
+const LOGICAL_EVENTS = "events";
+const PHYSICAL_EVENTS = new Set(["events_core", "events_full"]);
+
+export class EventsTableRoutingPlugin implements KyselyPlugin {
+ transformQuery(args: PluginTransformQueryArgs): RootOperationNode {
+ // Builders share unchanged child nodes, so one relation node can sit at
+ // two places in the tree. A plain transform rebuilds every node, giving
+ // each place its own node to key routing on.
+ const tree = new ClickHouseOperationNodeTransformer().transformNode(
+ args.node,
+ );
+ const analysis = new EventsReadAnalysis();
+ analysis.transformNode(tree);
+ if (!analysis.readsLogicalEvents) return args.node;
+ rejectPhysicalEventsCte(analysis);
+ return new EventsTableRouter(analysis.fullRelations).transformNode(
+ tree,
+ ) as RootOperationNode;
+ }
+
+ async transformResult(args: PluginTransformResultArgs) {
+ return args.result;
+ }
+}
+
+function rejectPhysicalEventsCte(analysis: EventsReadAnalysis): void {
+ if (analysis.physicalEventsCte) {
+ throw new QueryCompileError(
+ `A CTE named "${analysis.physicalEventsCte}" would capture the routed logical "${LOGICAL_EVENTS}" relation; rename the CTE.`,
+ );
+ }
+}
+
+/**
+ * Read-only walk that resolves column references to relations; the
+ * transformer is used for its traversal only.
+ */
+class EventsReadAnalysis extends ClickHouseOperationNodeTransformer {
+ readsLogicalEvents = false;
+ /** Name of a CTE that shadows `events_core` / `events_full`. */
+ physicalEventsCte: string | undefined;
+ /** Logical relations that read a truncated column. */
+ readonly fullRelations = new Set<OperationNode>();
+ /** Relations of each enclosing select, innermost last. */
+ private readonly scopes: Relation[][] = [];
+
+ protected override transformNodeImpl<T extends OperationNode>(
+ node: T,
+ queryId?: QueryId,
+ ): T {
+ const lookup = coreSafeMetadataLookup(node);
+ if (lookup) {
+ this.transformNode(lookup.index, queryId);
+ return node;
+ }
+ if (CommonTableExpressionNode.is(node)) this.checkCteName(node);
+ return super.transformNodeImpl(node, queryId);
+ }
+
+ protected override transformSelectQuery(
+ node: SelectQueryNode,
+ queryId?: QueryId,
+ ): SelectQueryNode {
+ const relations = [
+ ...(node.from?.froms ?? []),
+ ...(node.joins ?? []).map((join) => join.table),
+ ].map(describeRelation);
+ if (relations.some((relation) => relation.logical)) {
+ this.readsLogicalEvents = true;
+ }
+ this.scopes.push(relations);
+ try {
+ return super.transformSelectQuery(node, queryId);
+ } finally {
+ this.scopes.pop();
+ }
+ }
+
+ protected override transformReference(
+ node: ReferenceNode,
+ _queryId?: QueryId,
+ ): ReferenceNode {
+ const qualifier = node.table?.table.identifier.name;
+ // `sql.ref("e.*")` parses the star as a column named `*`.
+ if (SelectAllNode.is(node.column) || node.column.column.name === "*") {
+ if (qualifier === undefined) this.rejectStar("*");
+ else if (this.resolveQualified(qualifier)?.logical) {
+ throw starError(`${qualifier}.*`);
+ }
+ } else {
+ this.readColumn(node.column.column.name, qualifier);
+ }
+ return node;
+ }
+
+ /** Reached only outside a reference: `selectAll()`, `*` in raw SQL. */
+ protected override transformSelectAll(node: SelectAllNode): SelectAllNode {
+ this.rejectStar("*");
+ return node;
+ }
+
+ /** Reached only outside a reference. */
+ protected override transformColumn(node: ColumnNode): ColumnNode {
+ if (node.column.name === "*") this.rejectStar("*");
+ else this.readColumn(node.column.name, undefined);
+ return node;
+ }
+
+ /**
+ * Rejects `columns` like any function ({@link rejectColumnsMatcher}). `count(*)`
+ * reads no column; its clauses are walked as usual.
+ */
+ protected override transformAggregateFunction(
+ node: AggregateFunctionNode,
+ queryId?: QueryId,
+ ): AggregateFunctionNode {
+ this.rejectColumnsMatcher(node.func);
+ if (isCountStar(node)) {
+ super.transformAggregateFunction({ ...node, aggregated: [] }, queryId);
+ return node;
+ }
+ return super.transformAggregateFunction(node, queryId);
+ }
+
+ /**
+ * The direction is raw `asc` / `desc`, so it is not walked: the
+ * `no-deprecated` lint rule keeps `orderBy` from taking an expression.
+ */
+ protected override transformOrderByItem(
+ node: OrderByItemNode,
+ queryId?: QueryId,
+ ): OrderByItemNode {
+ this.transformNode(node.orderBy, queryId);
+ return node;
+ }
+
+ protected override transformFunction(
+ node: FunctionNode,
+ queryId?: QueryId,
+ ): FunctionNode {
+ this.rejectColumnsMatcher(node.func);
+ return super.transformFunction(node, queryId);
+ }
+
+ protected override transformRaw(node: RawNode, queryId?: QueryId): RawNode {
+ if (
+ this.readsLogicalInScope() &&
+ (node.sqlFragments.some((fragment) => fragment.trim() !== "") ||
+ node.parameters.some((p) => IdentifierNode.is(p) || TableNode.is(p)))
+ ) {
+ throw new QueryCompileError(
+ `Raw SQL text or identifiers cannot appear in a select that reads the logical "${LOGICAL_EVENTS}" relation: they may read a truncated column the routing pass cannot see. Use builder expressions, sql.ref() or bound values.`,
+ );
+ }
+ return super.transformRaw(node, queryId);
+ }
+
+ private readColumn(column: string, qualifier: string | undefined): void {
+ if (!EVENTS_CORE_TRUNCATED_COLUMNS.has(column)) return;
+ const relation =
+ qualifier === undefined
+ ? this.resolveUnqualified(column)
+ : this.resolveQualified(qualifier);
+ if (relation?.logical) this.fullRelations.add(relation.node);
+ }
+
+ private resolveQualified(qualifier: string): Relation | undefined {
+ for (const relations of this.scopes.toReversed()) {
+ const match = relations.find((relation) => relation.name === qualifier);
+ if (match) return match;
+ }
+ if (PHYSICAL_EVENTS.has(qualifier) && this.readsLogicalInScope()) {
+ throw new QueryCompileError(
+ `Qualify by the relation's alias, not "${qualifier}": ClickHouse also resolves a table name to an aliased relation, so it may reach the logical "${LOGICAL_EVENTS}" relation after routing.`,
+ );
+ }
+ return undefined;
+ }
+
+ private resolveUnqualified(column: string): Relation | undefined {
+ if (this.readsLogicalInScope()) {
+ throw new QueryCompileError(
+ `Qualify "${column}": it is truncated in events_core, so the routing pass needs to know which relation reads it while the logical "${LOGICAL_EVENTS}" relation is in scope.`,
+ );
+ }
+ return undefined;
+ }
+
+ private checkCteName(node: CommonTableExpressionNode): void {
+ const name = node.name.table.table.identifier.name;
+ if (name === LOGICAL_EVENTS) {
+ throw new QueryCompileError(
+ `A CTE cannot be named "${LOGICAL_EVENTS}": the name is the logical events relation.`,
+ );
+ }
+ if (PHYSICAL_EVENTS.has(name)) this.physicalEventsCte = name;
+ }
+
+ /** `columns(…)` expands like `*`. */
+ private rejectColumnsMatcher(func: string): void {
+ if (func.toLowerCase() === "columns") this.rejectStar(`${func}(…)`);
+ }
+
+ /** Whether the current select or an enclosing one reads a logical relation. */
+ private readsLogicalInScope(): boolean {
+ return this.scopes.some((relations) => relations.some((r) => r.logical));
+ }
+
+ private rejectStar(star: string): void {
+ if (this.scopes.at(-1)?.some((relation) => relation.logical)) {
+ throw starError(star);
+ }
+ }
+}
+
+function starError(star: string): QueryCompileError {
+ return new QueryCompileError(
+ `${star} cannot expand the logical "${LOGICAL_EVENTS}" relation: list the columns, so the routing pass can tell whether it reads a truncated one.`,
+ );
+}
+
+class EventsTableRouter extends ClickHouseOperationNodeTransformer {
+ constructor(private readonly fullRelations: ReadonlySet<OperationNode>) {
+ super();
+ }
+
+ protected override transformFrom(
+ node: FromNode,
+ queryId?: QueryId,
+ ): FromNode {
+ return FromNode.create(
+ node.froms.map((from) => this.routeRelation(from, queryId)),
+ );
+ }
+
+ protected override transformJoin(
+ node: JoinNode,
+ queryId?: QueryId,
+ ): JoinNode {
+ return {
+ ...super.transformJoin(node, queryId),
+ table: this.routeRelation(node.table, queryId),
+ };
+ }
+
+ /**
+ * `events as e` β†’ `events_core as e`. An unaliased `events` keeps its name
+ * as the alias (`events_core as events`), so `events.col` references and
+ * the qualifiers later passes add still resolve.
+ */
+ private routeRelation(node: OperationNode, queryId?: QueryId): OperationNode {
+ const table = TableNode.create(
+ this.fullRelations.has(node) ? "events_full" : "events_core",
+ );
+ if (isLogicalEventsTable(node)) {
+ return AliasNode.create(table, IdentifierNode.create(LOGICAL_EVENTS));
+ }
+ if (AliasNode.is(node) && isLogicalEventsTable(node.node)) {
+ return AliasNode.create(table, node.alias);
+ }
+ return this.transformNode(node, queryId);
+ }
+}
+
+// FINAL wraps the aliased item: `scores as s FINAL`.
+function describeRelation(node: OperationNode): Relation {
+ const item = FinalTableNode.is(node) ? unwrapFinalTable(node) : node;
+ const table = AliasNode.is(item) ? item.node : item;
+ return {
+ node,
+ name: relationName(item, table),
+ logical: isLogicalEventsTable(table),
+ };
+}
+
+function relationName(
+ item: OperationNode,
+ table: OperationNode,
+): string | undefined {
+ if (AliasNode.is(item)) {
+ return IdentifierNode.is(item.alias) ? item.alias.name : undefined;
+ }
+ return TableNode.is(table) && table.table.schema === undefined
+ ? table.table.identifier.name
+ : undefined;
+}
+
+function isLogicalEventsTable(node: OperationNode): boolean {
+ return (
+ TableNode.is(node) &&
+ node.table.schema === undefined &&
+ node.table.identifier.name === LOGICAL_EVENTS
+ );
+}
+
+/** `count(*)` with any `distinct`, `filter`, `over`, … clause. */
+function isCountStar(node: AggregateFunctionNode): boolean {
+ if (node.func !== "count" || node.aggregated.length !== 1) return false;
+ const [arg] = node.aggregated;
+ return (
+ SelectAllNode.is(arg) ||
+ (SelectionNode.is(arg) && SelectAllNode.is(arg.selection))
+ );
+}
+
+/**
+ * The metadata lookup of `metadataValue(e, k) = v` or
+ * `startsWith(metadataValue(e, k), v)`, for a bound `v` that
+ * `metadataFilterIsEventsCoreSafe` accepts.
+ */
+function coreSafeMetadataLookup(
+ node: OperationNode,
+): ArrayIndexNode | undefined {
+ if (
+ BinaryOperationNode.is(node) &&
+ OperatorNode.is(node.operator) &&
+ node.operator.operator === "="
+ ) {
+ return coreSafe(node.leftOperand, "=", node.rightOperand);
+ }
+ if (
+ FunctionNode.is(node) &&
+ node.func === "startsWith" &&
+ node.arguments.length === 2
+ ) {
+ return coreSafe(node.arguments[0], "starts with", node.arguments[1]);
+ }
+ return undefined;
+}
+
+// `lookup` is typed `{ kind: string }`: `ArrayIndexNode` is outside Kysely's
+// closed `OperationNode` union, so narrowing an `OperationNode` yields `never`.
+function coreSafe(
+ lookup: { kind: string },
+ operator: "=" | "starts with",
+ value: OperationNode,
+): ArrayIndexNode | undefined {
+ return ArrayIndexNode.is(lookup) &&
+ lookup.metadataLookup &&
+ ValueNode.is(value) &&
+ metadataFilterIsEventsCoreSafe(operator, value.value)
+ ? lookup
+ : undefined;
+}
+
+/** A FROM/JOIN source of one select. */
+type Relation = {
+ /** The FROM item or join table, as the router meets it. */
+ node: OperationNode;
+ /** Alias, or the table name of an unaliased table. */
+ name?: string;
+ logical: boolean;
+};
diff --git a/packages/shared/src/server/query-ast/extensions.ts b/packages/shared/src/server/query-ast/extensions.ts
index 3948c25df2..08b546bf38 100644
--- a/packages/shared/src/server/query-ast/extensions.ts
+++ b/packages/shared/src/server/query-ast/extensions.ts
@@ -228,11 +228,20 @@ export function dateDiff(
export function mapValue<V = number>(
map: Expression<Record<string, V>>,
key: string | Expression<string>,
+): ChExpression<V> {
+ return subscript(map, key, false);
+}
+
+function subscript<V>(
+ map: Expression<Record<string, V>>,
+ key: string | Expression<string>,
+ metadataLookup: boolean,
): ChExpression<V> {
return new ExpressionWrapper<ClickHouseDatabase, keyof ClickHouseDatabase, V>(
ArrayIndexNode.create(
map.toOperationNode(),
typeof key === "string" ? ValueNode.create(key) : key.toOperationNode(),
+ metadataLookup,
) as unknown as OperationNode,
);
}
@@ -372,6 +381,14 @@ export function concat(...values: Expression<string>[]): ChExpression<string> {
return functionCall("concat", values);
}
+/** ClickHouse `startsWith(haystack, prefix)`, with `prefix` bound. */
+export function startsWith(
+ haystack: Expression<string>,
+ prefix: string,
+): ChExpression<SqlBool> {
+ return functionCall("startsWith", [haystack, ValueNode.create(prefix)]);
+}
+
/** ClickHouse `length(array)`. */
export function arrayLength(
array: Expression<unknown[]>,
@@ -648,9 +665,10 @@ export function metadataValue(
new ExpressionWrapper<ClickHouseDatabase, "events_core", string[]>(
qualifiedColumn(tableAlias, name),
);
- return mapValue<string>(
+ return subscript<string>(
metadataMap(column("metadata_names"), column("metadata_values")),
key,
+ true,
);
}
diff --git a/packages/shared/src/server/query-ast/nodes.test.ts b/packages/shared/src/server/query-ast/nodes.test.ts
index 4f16dffff0..a7428ac39f 100644
--- a/packages/shared/src/server/query-ast/nodes.test.ts
+++ b/packages/shared/src/server/query-ast/nodes.test.ts
@@ -846,3 +846,46 @@ describe("ARRAY JOIN and LIMIT BY nodes", () => {
expect(() => useFinal(["events_core"])).toThrow(/immutable/);
});
});
+
+describe("names the compiler writes verbatim", () => {
+ const db = getClickhouseKysely();
+
+ it.each(["columns ", "tuple(s.comment), length", "default.f"])(
+ "rejects the function name %j",
+ (name) => {
+ expect(() =>
+ compileClickhouseQuery(
+ db
+ .selectFrom("scores as s")
+ .select((eb) => eb.fn<string>(name, [eb.val("x")]).as("x")),
+ projectId,
+ ),
+ ).toThrow(QueryCompileError);
+ expect(() =>
+ compileClickhouseQuery(
+ db
+ .selectFrom("scores as s")
+ .select((eb) => eb.fn.agg<string>(name, [eb.val("x")]).as("x")),
+ projectId,
+ ),
+ ).toThrow(QueryCompileError);
+ },
+ );
+
+ // ClickHouse parses only a string literal after `collate`, which Kysely
+ // does not emit.
+ it.each(["en", "'en', s.comment"])(
+ "rejects the collation %j",
+ (collation) => {
+ expect(() =>
+ compileClickhouseQuery(
+ db
+ .selectFrom("scores as s")
+ .select("s.id")
+ .orderBy("s.name", (ob) => ob.collate(collation)),
+ projectId,
+ ),
+ ).toThrow(QueryCompileError);
+ },
+ );
+});
diff --git a/packages/shared/src/server/query-ast/nodes.ts b/packages/shared/src/server/query-ast/nodes.ts
index 0356d89f12..1688d6ce8b 100644
--- a/packages/shared/src/server/query-ast/nodes.ts
+++ b/packages/shared/src/server/query-ast/nodes.ts
@@ -80,17 +80,27 @@ export type ArrayIndexNode = {
readonly kind: "ArrayIndexNode";
readonly array: OperationNode;
readonly index: OperationNode;
+ /**
+ * `metadata[key]` over one relation's `metadata_names`/`metadata_values`,
+ * so events routing can keep a short metadata predicate on `events_core`.
+ */
+ readonly metadataLookup?: true;
};
export const ArrayIndexNode = {
is(node: { kind: string }): node is ArrayIndexNode {
return node.kind === "ArrayIndexNode";
},
- create(array: OperationNode, index: OperationNode): ArrayIndexNode {
+ create(
+ array: OperationNode,
+ index: OperationNode,
+ metadataLookup = false,
+ ): ArrayIndexNode {
return Object.freeze({
kind: "ArrayIndexNode",
array,
index,
+ ...(metadataLookup ? { metadataLookup: true as const } : {}),
});
},
};
diff --git a/packages/shared/src/server/query-ast/schema.test.ts b/packages/shared/src/server/query-ast/schema.test.ts
index 000e7b9a72..340d388604 100644
--- a/packages/shared/src/server/query-ast/schema.test.ts
+++ b/packages/shared/src/server/query-ast/schema.test.ts
@@ -12,6 +12,7 @@ import { schemaTypeAssertions, extensionTypeAssertions } from "./types.assert";
describe("table registry derivation", () => {
it("derives the tenanted table set from the registry", () => {
expect([...TENANTED_TABLES].sort()).toEqual([
+ "events",
"events_core",
"events_full",
"observations",
diff --git a/packages/shared/src/server/query-ast/schema.ts b/packages/shared/src/server/query-ast/schema.ts
index 48d466b762..4038bd8b9b 100644
--- a/packages/shared/src/server/query-ast/schema.ts
+++ b/packages/shared/src/server/query-ast/schema.ts
@@ -229,6 +229,23 @@ const EVENTS_COLUMNS = {
ingestion_api_key: "String",
} as const satisfies Record<string, ChColumnType>;
+/**
+ * Columns whose `events_core` copy differs from `events_full`:
+ * `events_core_mv` keeps the first 200 code points of `input`, `output`, and
+ * each `metadata_values` element, and `input_length` / `output_length` are
+ * materialized from the truncated copy. Any other read of them needs
+ * `events_full`. Must match the truncations in the latest `events_core_mv`
+ * definition (`clickhouse/migrations/canonical/0053_add_skill_observation_references.up.sql`);
+ * nothing checks this automatically.
+ */
+export const EVENTS_CORE_TRUNCATED_COLUMNS: ReadonlySet<string> = new Set([
+ "input",
+ "output",
+ "metadata_values",
+ "input_length",
+ "output_length",
+] satisfies ReadonlyArray<keyof typeof EVENTS_COLUMNS>);
+
const TABLE_REGISTRY = {
traces: defineTable({
columns: {
@@ -262,6 +279,14 @@ const TABLE_REGISTRY = {
dedup: { strategy: "none" },
softDelete: "is_deleted",
}),
+ // Logical relation over events_core / events_full. The routing pass
+ // (`eventsRouting.ts`) replaces it with one of them before any other pass
+ // runs, so ClickHouse never sees this name.
+ events: defineTable({
+ columns: EVENTS_COLUMNS,
+ dedup: { strategy: "none" },
+ softDelete: "is_deleted",
+ }),
scores: defineTable({
columns: {
id: "String",
@@ -303,14 +328,18 @@ export type ClickHouseDatabase = {
};
/**
- * Declared ClickHouse type per column, per registry table. A servertest checks
- * it against `system.columns` of the migrated database.
+ * Declared ClickHouse type per column, per physical registry table. A
+ * servertest checks it against `system.columns` of the migrated database. The
+ * logical `events` relation has no table of its own; its columns are
+ * `events_core`'s and `events_full`'s.
*/
export const TABLE_COLUMN_TYPES: Record<
string,
Record<string, ChColumnType>
> = Object.fromEntries(
- Object.entries(TABLE_REGISTRY).map(([table, spec]) => [table, spec.columns]),
+ Object.entries(TABLE_REGISTRY)
+ .filter(([table]) => table !== "events")
+ .map(([table, spec]) => [table, spec.columns]),
);
/**
diff --git a/packages/shared/src/server/query-ast/softDelete.test.ts b/packages/shared/src/server/query-ast/softDelete.test.ts
index 6363b538a2..60145fe0d9 100644
--- a/packages/shared/src/server/query-ast/softDelete.test.ts
+++ b/packages/shared/src/server/query-ast/softDelete.test.ts
@@ -24,6 +24,7 @@ function compile(builder: Parameters<typeof compileClickhouseQuery>[0]) {
describe("soft-delete filtering", () => {
it("declares the flag on the events tables only", () => {
expect(SOFT_DELETE_COLUMNS).toEqual({
+ events: "is_deleted",
events_core: "is_deleted",
events_full: "is_deleted",
});
diff --git a/packages/shared/src/server/query-ast/transformer.ts b/packages/shared/src/server/query-ast/transformer.ts
index 4508812d64..aff947e4d1 100644
--- a/packages/shared/src/server/query-ast/transformer.ts
+++ b/packages/shared/src/server/query-ast/transformer.ts
@@ -50,6 +50,7 @@ export class ClickHouseOperationNodeTransformer extends OperationNodeTransformer
return ArrayIndexNode.create(
this.transformNode(node.array),
this.transformNode(node.index),
+ node.metadataLookup,
);
}
diff --git a/packages/shared/src/server/repositories/prompt-events.ts b/packages/shared/src/server/repositories/prompt-events.ts
index 728b903ba0..88dca6240d 100644
--- a/packages/shared/src/server/repositories/prompt-events.ts
+++ b/packages/shared/src/server/repositories/prompt-events.ts
@@ -4,7 +4,7 @@ import { getClickhouseKysely } from "../query-ast/dialect";
import { limitBy, softDeleteAfterLimit } from "../query-ast/extensions";
import type { ClickHouseDatabase } from "../query-ast/schema";
-type EventsAsE = ClickHouseDatabase & { e: ClickHouseDatabase["events_core"] };
+type EventsAsE = ClickHouseDatabase & { e: ClickHouseDatabase["events"] };
/**
* Latest version of each generation linked to one of `promptIds`, optionally
@@ -19,7 +19,7 @@ export const promptGenerationEvents = <
timeWindow: { fromTimestamp?: Date; toTimestamp?: Date } = {},
) =>
getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.$call(softDeleteAfterLimit())
.select(columns)
.where("e.type", "=", "GENERATION")
diff --git a/packages/shared/src/server/repositories/scores.ts b/packages/shared/src/server/repositories/scores.ts
index e548d19b08..a242a6a30f 100644
--- a/packages/shared/src/server/repositories/scores.ts
+++ b/packages/shared/src/server/repositories/scores.ts
@@ -420,7 +420,7 @@ export const getTraceScoresForDatasetRuns = async (
const experimentItemEventsQuery = (experimentIds: string[]) =>
getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.select(["e.project_id", "e.experiment_id", "e.trace_id"])
.where("e.experiment_id", "in", experimentIds)
.where("e.experiment_id", "!=", "");