🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
refactor(query-ast): read events only through logical relations (#19172)
Niklas Semmlerc978cf4a28a2
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.tsindex 703aa2633a..2a51cfcaaa 100644--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts@@ -26,10 +26,7 @@ import {type ClickhouseFilter,} from "./clickhouse-filter";-const eventsRef = expressionBuilder<- { e: ClickHouseDatabase["events_core"] },- "e"->();+const eventsRef = expressionBuilder<{ e: ClickHouseDatabase["events"] }, "e">();const eventsMetadataMap = metadataMap(eventsRef.ref("e.metadata_names"),diff --git a/packages/shared/src/server/query-ast/README.md b/packages/shared/src/server/query-ast/README.mdindex 709710ba0b..5ab469cfba 100644--- a/packages/shared/src/server/query-ast/README.md+++ b/packages/shared/src/server/query-ast/README.md@@ -96,12 +96,15 @@ suffix to run there. Promote it to a required check once it has proven stable.`metadataFilterIsEventsCoreSafe` accepts the value, as in the raw-SQLbuilders. `*` / `e.*` over `events`, raw SQL text in a select that readsit, and an unqualified truncated column are compile errors while `events`- is in scope. Name `events_core` to read the truncated copy on purpose;- `events_core` / `events_full` stay selectable by name, which is also the- override for a shape the pass rejects.+ is in scope. Read `events_truncated` to get the truncated copy on purpose+ (input/output/metadata previews): it always compiles to `events_core`,+ whatever columns it reads, and the pass does not reject `*` or raw SQL over+ it. `events_core` / `events_full` are not part of the Kysely schema, and+ naming them anywhere — FROM/JOIN, alias, column qualifier, raw SQL — is a+ compile error; only the routing pass writes them.- **Give each CTE (and view) its own plain name.** A CTE name is a plainidentifier that is not a registry table (`traces`, `events_core`, the- logical `events`, …) and doesn't reuse the name of a sibling or an+ logical `events` / `events_truncated`, …) and doesn't reuse the name of a sibling or anenclosing CTE (a CTE is not in scope inside its own body). Every FROM/JOINname is an unqualified registry table or a CTE in scope; schema-qualifiednames (`default.traces`) are rejected, since the registry describes tables@@ -219,7 +222,7 @@ db.selectFrom("events")// soft-delete pass so a deleted latest version is dropped```-**Write FINAL** (`useFinal` — ReplacingMergeTree reads; never on `events_core`):+**Write FINAL** (`useFinal` — ReplacingMergeTree reads; never on the events relations):```tsdb.selectFrom("scores as s")diff --git a/packages/shared/src/server/query-ast/binds.test.ts b/packages/shared/src/server/query-ast/binds.test.tsindex e907418e0e..e45a934c90 100644--- a/packages/shared/src/server/query-ast/binds.test.ts+++ b/packages/shared/src/server/query-ast/binds.test.ts@@ -40,7 +40,7 @@ describe("typed params from the column registry", () => {it("binds a number compared to total_cost as Decimal(18, 12)", () => {const { sql } = compile(getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select("span_id").where("total_cost", ">", 1),);@@ -50,7 +50,7 @@ describe("typed params from the column registry", () => {it("wraps an IN list against an Array column as Array(Array(...))", () => {const { sql } = compile(getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select("span_id").where("metadata_names", "in", [["a"], ["b"]]),);@@ -63,7 +63,7 @@ describe("typed params from the column registry", () => {it("inlines an IN list of literals", () => {const { sql, params } = compile(getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select("span_id").where("name", "in", [literal("a"), literal("it's")]),);@@ -74,7 +74,7 @@ describe("typed params from the column registry", () => {it("binds an IN list of value expressions as one array", () => {const { sql } = compile(getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select("span_id").where((eb) => eb("name", "in", [eb.val("a"), eb.val("b")])),);@@ -85,7 +85,7 @@ describe("typed params from the column registry", () => {expect(() =>compile(getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select("span_id").where((eb) => eb("name", "in", [literal("a"), eb.val("b")])),),@@ -95,7 +95,7 @@ describe("typed params from the column registry", () => {it("keeps the column's DateTime precision through toStartOfMinute / toDate", () => {const { sql: minuteSql } = compile(getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select("span_id").where((eb) =>eb(@@ -131,7 +131,7 @@ describe("typed params from the column registry", () => {it("binds a Date as the UTC wall-clock string ingestion writes", () => {const { sql, params } = compile(getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select("span_id").where("start_time", ">=", new Date(-1000)).where("start_time", "<=", START),@@ -148,7 +148,7 @@ describe("typed params from the column registry", () => {it("binds a shared column name with the type of the table it is read from", () => {const { sql } = compile(getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").select("e.span_id").where("e.start_time", ">=", START).where("e.trace_id", "in", (qb) =>@@ -178,7 +178,7 @@ describe("typed params from the column registry", () => {it("resolves a qualifier declared in an enclosing select", () => {const { sql } = compile(getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").select("e.span_id").where((eb) =>eb.exists(@@ -199,7 +199,7 @@ describe("typed params from the column registry", () => {() =>getClickhouseKysely().with("recent", (db) =>- db.selectFrom("events_core as e").select("e.start_time as ts"),+ db.selectFrom("events as e").select("e.start_time as ts"),).selectFrom("recent as r").select("r.ts")@@ -209,9 +209,7 @@ describe("typed params from the column registry", () => {"a CTE reading an earlier CTE",() =>getClickhouseKysely()- .with("first", (db) =>- db.selectFrom("events_core").select("start_time"),- )+ .with("first", (db) => db.selectFrom("events").select("start_time")).with("second", (db) => db.selectFrom("first").selectAll()).selectFrom("second").select("start_time")@@ -222,7 +220,7 @@ describe("typed params from the column registry", () => {() =>getClickhouseKysely().selectFrom((db) =>- db.selectFrom("events_core").select("start_time").as("sub"),+ db.selectFrom("events").select("start_time").as("sub"),).select("sub.start_time").where("sub.start_time", ">=", START),@@ -233,7 +231,7 @@ describe("typed params from the column registry", () => {getClickhouseKysely().with("bounds", (db) =>db- .selectFrom("events_core")+ .selectFrom("events").select((eb) => eb.fn.max("start_time").as("start_time")),).selectFrom("bounds")@@ -246,7 +244,7 @@ describe("typed params from the column registry", () => {getClickhouseKysely().with("bounds", (db) =>db- .selectFrom("events_core")+ .selectFrom("events").select((eb) =>eb.fn<Date>("max", ["start_time"]).as("start_time"),),@@ -261,7 +259,7 @@ describe("typed params from the column registry", () => {getClickhouseKysely().with("latest", (db) =>db- .selectFrom("events_core")+ .selectFrom("events").select((eb) =>eb.fn<Date>("argMax", ["start_time", "event_ts"])@@ -278,7 +276,7 @@ describe("typed params from the column registry", () => {getClickhouseKysely().with("latest", (db) =>db- .selectFrom("events_core")+ .selectFrom("events").select((eb) => eb.fn<Date>("anyLast", ["start_time"]).as("ts")),).selectFrom("latest")@@ -290,7 +288,7 @@ describe("typed params from the column registry", () => {() =>getClickhouseKysely().with("rows", (db) =>- db.selectFrom("events_core as e").selectAll("e"),+ db.selectFrom("events_truncated as e").selectAll("e"),).selectFrom("rows").select("start_time")@@ -307,7 +305,7 @@ describe("typed params from the column registry", () => {describe("CTE name resolution", () => {const db = getClickhouseKysely();const withX = db.with("x", (qb) =>- qb.selectFrom("events_core").select("start_time"),+ qb.selectFrom("events").select("start_time"),);it("types a subquery's read of an enclosing CTE from its body", () => {@@ -333,11 +331,11 @@ describe("typed params from the column registry", () => {db.with("y", (qb) =>qb- .selectFrom("events_core")+ .selectFrom("events").select("span_id").where(db.dynamic.ref("o.start_time"), ">=", START),)- .selectFrom("events_core as o")+ .selectFrom("events as o").select("o.span_id"),),).toThrow(/no relation in scope is named "o"/);@@ -347,7 +345,7 @@ describe("typed params from the column registry", () => {it("propagates the source column type through a JOIN to an aliased subquery", () => {const db = getClickhouseKysely();const sub = db- .selectFrom("events_core")+ .selectFrom("events").select(["trace_id", "start_time"]).as("sub");const { sql } = compile(@@ -364,7 +362,7 @@ describe("typed params from the column registry", () => {it("binds a select alias with the type of the column it names", () => {const { sql } = compile(getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").innerJoin("observations as o", "o.trace_id", "e.trace_id").select((eb) => eb.fn.max("e.start_time").as("start_time")).groupBy("e.trace_id")@@ -389,7 +387,7 @@ describe("typed params from the column registry", () => {it("types a comparison inside an aggregate aliased to the column's name", () => {const { sql } = compile(getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select((eb) => countIf(eb("total_cost", ">", 1)).as("total_cost")).groupBy("trace_id"),);@@ -414,7 +412,7 @@ describe("typed params from the column registry", () => {it("binds an ARRAY JOIN alias by value, not as the column it shadows", () => {const { sql } = compile(getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").select("e.span_id").$call(arrayJoin((eb) => ({ start_time: eb.ref("e.metadata_names") }))).where(sql_.ref("start_time"), "=", "x"),@@ -426,7 +424,7 @@ describe("typed params from the column registry", () => {it("binds a tuple element of an ARRAY JOIN alias by value", () => {const { sql } = compile(getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").select("e.span_id").$call(arrayJoin((eb) => ({ kv: eb.ref("e.metadata_names") }))).where(sql_.ref("kv.1"), "=", "a"),@@ -438,7 +436,7 @@ describe("typed params from the column registry", () => {it("binds a raw expression around a sql.ref column by value", () => {const { sql } = compile(getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events_truncated as e").select("e.span_id").where(sql_<number>`length(${sql_.ref("e.input")})`, ">", 5),);@@ -500,7 +498,7 @@ describe("typed params from the column registry", () => {db.with("t", (qb) =>qb- .selectFrom("events_core")+ .selectFrom("events_truncated").select(sql_`is_deleted`.as("is_deleted")),).selectFrom("t")@@ -514,7 +512,7 @@ describe("typed params from the column registry", () => {it("binds by value a column the CTE does not output", () => {const { sql } = compile(db- .with("t", (qb) => qb.selectFrom("events_core").select("span_id"))+ .with("t", (qb) => qb.selectFrom("events").select("span_id")).selectFrom("t").select("span_id").where(db.dynamic.ref("is_deleted"), "=", 0),@@ -526,7 +524,7 @@ describe("typed params from the column registry", () => {it("compiles an unresolvable column compared to an inlined constant", () => {const { sql } = compile(db- .selectFrom("events_core")+ .selectFrom("events").select("span_id").where((eb) => eb(db.dynamic.ref("x.is_deleted"), "=", eb.lit(0))),);@@ -539,7 +537,7 @@ describe("typed params from the column registry", () => {"a qualifier declared nowhere",() =>db- .selectFrom("events_core")+ .selectFrom("events").select("span_id").where(db.dynamic.ref("x.is_deleted"), "=", 0),/no relation in scope is named "x"/,@@ -549,7 +547,7 @@ describe("typed params from the column registry", () => {"an unqualified column the joined relations type differently",() =>db- .selectFrom("events_core as e")+ .selectFrom("events as e").innerJoin("observations as o", "o.trace_id", "e.trace_id").select("e.span_id").where("start_time", ">=", START),@@ -577,7 +575,7 @@ describe("typed params from the column registry", () => {"a sql.ref column whose qualifier is declared nowhere",() =>db- .selectFrom("events_core")+ .selectFrom("events").select("span_id").where(sql_.ref("x.is_deleted"), "=", 0),/no relation in scope is named "x"/,@@ -587,7 +585,7 @@ describe("typed params from the column registry", () => {() =>db.with("t", (qb) =>- qb.selectFrom("events_core as e").selectAll("x" as never),+ qb.selectFrom("events as e").selectAll("x" as never),).selectFrom("t as t").select(db.dynamic.ref("t.span_id"))@@ -601,7 +599,7 @@ describe("typed params from the column registry", () => {() =>db.with("t", (qb) =>- qb.selectFrom("events_core as e").selectAll("x" as never),+ qb.selectFrom("events as e").selectAll("x" as never),).selectFrom("scores as s").innerJoin("t", (join) => join.onTrue())@@ -614,7 +612,7 @@ describe("typed params from the column registry", () => {"a TupleParam whose tuple has a column declared nowhere",() =>db- .selectFrom("events_core as e")+ .selectFrom("events as e").select("e.span_id").where((eb) =>tupleIn(eb.refTuple(db.dynamic.ref("x.a"), "e.span_id"), [@@ -628,7 +626,7 @@ describe("typed params from the column registry", () => {"a tuple of values against a tuple with a column declared nowhere",() =>db- .selectFrom("events_core as e")+ .selectFrom("events as e").select("e.span_id").where((eb) =>eb(eb.refTuple(db.dynamic.ref("x.a"), "e.span_id"), "in", [@@ -642,7 +640,7 @@ describe("typed params from the column registry", () => {"an empty TupleParam list whose tuple has a column declared nowhere",() =>db- .selectFrom("events_core as e")+ .selectFrom("events as e").select("e.span_id").where((eb) =>tupleIn(eb.refTuple(db.dynamic.ref("x.a"), "e.span_id"), []),@@ -654,7 +652,7 @@ describe("typed params from the column registry", () => {"an IN list against a qualifier declared nowhere",() =>db- .selectFrom("events_core")+ .selectFrom("events").select("span_id").where(db.dynamic.ref("x.trace_id"), "in", ["a", "b"]),/no relation in scope is named "x"/,@@ -670,7 +668,7 @@ describe("typed params from the column registry", () => {getClickhouseKysely().with("totals", (db) =>db- .selectFrom("events_core")+ .selectFrom("events").select((eb) => eb.fn.count("prompt_version").as("prompt_version")),).selectFrom("totals")@@ -683,7 +681,7 @@ describe("typed params from the column registry", () => {it("does not type extra date-wrapper arguments as the column", () => {const { sql } = compile(getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select("span_id").where((eb) =>eb(@@ -699,7 +697,7 @@ describe("typed params from the column registry", () => {it("does not type a subquery LIMIT from an outer column comparison", () => {const { sql } = compile(getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select("trace_id").where("trace_id",@@ -714,7 +712,7 @@ describe("typed params from the column registry", () => {it("binds a TupleParam list as Array(Tuple(...)) for a tuple IN", () => {const { sql, params } = compile(getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").select("e.span_id").where((eb) =>tupleIn(eb.refTuple("e.trace_id", "e.span_id"), [@@ -776,7 +774,9 @@ describe("typed params from the column registry", () => {const db = getClickhouseKysely();const { sql } = compile(db- .with("t", (qb) => qb.selectFrom("events_core as e").selectAll("e"))+ .with("t", (qb) =>+ qb.selectFrom("events_truncated as e").selectAll("e"),+ ).selectFrom("t as t").select(db.dynamic.ref("t.span_id")).where(db.dynamic.ref("t.start_time"), ">=", START),@@ -791,7 +791,7 @@ describe("typed params from the column registry", () => {db.with("t", (qb) =>qb- .selectFrom("events_core as e")+ .selectFrom("events_truncated as e").innerJoin("observations as o", "o.trace_id", "e.trace_id").selectAll(),)@@ -815,7 +815,7 @@ describe("typed params from the column registry", () => {it("binds distinct bare TupleParams as separate Tuple(...) placeholders", () => {const { sql, params } = compile(getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select((eb) => [eb.val(new TupleParam(["a", 1])).as("first"),eb.val(new TupleParam(["b", 2])).as("second"),diff --git a/packages/shared/src/server/query-ast/composition.test.ts b/packages/shared/src/server/query-ast/composition.test.tsindex 92f3ae92e3..fda5d3a9d1 100644--- a/packages/shared/src/server/query-ast/composition.test.ts+++ b/packages/shared/src/server/query-ast/composition.test.ts@@ -20,7 +20,7 @@ const costArrayJoin = () =>const dedup = () =>limitBy({count: 1,- columns: (eb: ExpressionBuilder<ClickHouseDatabase, "events_core">) => [+ columns: (eb: ExpressionBuilder<ClickHouseDatabase, "events">) => [eb.ref("span_id"),eb.ref("project_id"),],@@ -105,7 +105,7 @@ describe("LIMIT BY composition across nesting positions", () => {it("emits LIMIT BY at the top level (baseline)", () => {const db = getClickhouseKysely();const qb = db- .selectFrom("events_core")+ .selectFrom("events").select(["span_id", "project_id"]).orderBy("event_ts", "desc").$call(dedup());@@ -119,7 +119,7 @@ describe("LIMIT BY composition across nesting positions", () => {const qb = db.with("lb", (qb) =>qb- .selectFrom("events_core")+ .selectFrom("events").select(["span_id", "project_id"]).orderBy("event_ts", "desc").$call(dedup()),@@ -172,7 +172,7 @@ describe("FINAL composition across nesting positions", () => {it("keeps FINAL on a JOIN table and not on events_core", () => {const db = getClickhouseKysely();const qb = db- .selectFrom("events_core as e")+ .selectFrom("events as e").innerJoin("scores as s", (join) =>join.onRef("e.trace_id", "=", "s.trace_id")diff --git a/packages/shared/src/server/query-ast/dedup.test.ts b/packages/shared/src/server/query-ast/dedup.test.tsindex 0d1a07a404..1d226d8e01 100644--- a/packages/shared/src/server/query-ast/dedup.test.ts+++ b/packages/shared/src/server/query-ast/dedup.test.ts@@ -22,7 +22,7 @@ describe("per-table dedup lowering", () => {it("does not inject LIMIT BY or FINAL on events_core row reads", () => {const { sql } = compile(getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select(["span_id", "project_id"]),);const lower = sql.toLowerCase();@@ -33,7 +33,7 @@ describe("per-table dedup lowering", () => {it("does not wrap events_core DISTINCT", () => {const { sql } = compile(getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select("environment").distinct(),);@@ -45,7 +45,7 @@ describe("per-table dedup lowering", () => {it("does not wrap events_core aggregations", () => {const { sql } = compile(getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select((eb) => ["environment", eb.fn.countAll().as("n")]).groupBy("environment"),);@@ -58,7 +58,7 @@ describe("per-table dedup lowering", () => {it("still emits an explicit caller LIMIT BY", () => {const { sql } = compile(getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select("span_id").orderBy("event_ts", "desc").$call(diff --git a/packages/shared/src/server/query-ast/eventsRouting.test.ts b/packages/shared/src/server/query-ast/eventsRouting.test.tsindex 36f8b668a1..9ffbc3baa0 100644--- a/packages/shared/src/server/query-ast/eventsRouting.test.ts+++ b/packages/shared/src/server/query-ast/eventsRouting.test.ts@@ -2,6 +2,7 @@ import {AliasNode,FromNode,SelectQueryNode,+ type Kysely,sql,type Expression,type ExpressionBuilder,@@ -20,7 +21,6 @@ import { compileClickhouseQuery } from "./compile";import { getClickhouseKysely } from "./dialect";import { QueryCompileError } from "./errors";import {- includeDeleted,limitBy,mapValue,metadataMap,@@ -47,6 +47,11 @@ function routedTables(query: string): string[] {const db = getClickhouseKysely();+/** The compile DB without its table types, to reach names the types forbid. */+const untypedDb = db as unknown as Kysely<+ Record<string, Record<string, string>>+>;+type EventsEb = ExpressionBuilder<ClickHouseDatabase & { e: ClickHouseDatabase["events"] },"e"@@ -224,19 +229,6 @@ describe("logical events routing", () => {).toThrow(QueryCompileError);});- it.each([- "events_core.input",- "default.events_core.input",- "events_full.input",- ])("rejects the physical qualifier %s over a logical relation", (ref) => {- // ClickHouse resolves a qualifier by table name even when the table is aliased.- expect(() =>- compile(- db.selectFrom("events as e").select(sql.ref<string>(ref).as("x")),- ),- ).toThrow(QueryCompileError);- });-it("rejects FINAL around the logical relation", () => {// useFinal() refuses `events` (dedup: none), so FINAL is wrapped by hand;// routing would otherwise swap the table and drop the FINAL silently.@@ -259,17 +251,64 @@ describe("logical events routing", () => {).toThrow(/FINAL cannot wrap the logical "events" relation/);});- it("keeps a physical events qualifier when no logical relation is in scope", () => {- expect(- routedTables(- compile(- db- .selectFrom("events_core as c")- .$call(includeDeleted())- .select(sql.ref<string>("events_core.input").as("x")),- ),- ),- ).toEqual(["events_core"]);+ it.each<[string, () => Parameters<typeof compile>[0]]>([+ ["FROM", () => untypedDb.selectFrom("events_core").select("span_id")],+ [+ "JOIN",+ () =>+ untypedDb+ .selectFrom("scores as s")+ .innerJoin("events_full as f", "f.span_id", "s.id")+ .select("s.id"),+ ],+ [+ "alias",+ () => untypedDb.selectFrom("events as events_core").select("span_id"),+ ],+ [+ // ClickHouse resolves a qualifier by table name even when the table is aliased.+ "column qualifier",+ () =>+ db+ .selectFrom("scores as s")+ .select(sql.ref<string>("events_core.input").as("x")),+ ],+ [+ "qualifier in a core-safe metadata predicate",+ () =>+ db+ .selectFrom("events as e")+ .select("e.span_id")+ .where((eb) =>+ eb(+ metadataValue(+ eb.ref("events_full.metadata_names" as never),+ eb.ref("events_full.metadata_values" as never),+ "k",+ ),+ "=",+ "x",+ ),+ ),+ ],+ [+ "raw SQL text",+ () =>+ db+ .selectFrom("scores as s")+ .select(sql<number>`(select count() from events_full)`.as("n")),+ ],+ [+ "sql.table",+ () =>+ db+ .selectFrom("scores as s")+ .select(sql<string>`${sql.table("events_core")}.input`.as("x")),+ ],+ ])("rejects a physical events table named as %s", (_, build) => {+ expect(() => compile(build())).toThrow(+ /Read the logical "events" relation instead of "events_(core|full)"/,+ );});it("resolves sql.ref like a builder reference", () => {@@ -463,17 +502,38 @@ describe("logical events routing", () => {).toEqual(["events_core", "events_full"]);});- it("leaves physical events tables unchanged", () => {+ it("routes events_truncated to events_core, also when it reads a truncated column", () => {+ expect(+ compile(+ db+ .selectFrom("events_truncated as e")+ .select(["e.span_id", "e.input"])+ .where((eb) => eb(eventsMetadata(eb, "k"), "!=", "x")),+ ),+ ).toMatchInlineSnapshot(+ `"select e.span_id, e.input from events_core as e where e.project_id = {p1:String} and e.is_deleted = 0 and mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))[{p2:String}] != {p3:String}"`,+ );+ });++ it("keeps an unaliased events_truncated name as the alias", () => {+ expect(+ compile(db.selectFrom("events_truncated").select("input")),+ ).toMatchInlineSnapshot(+ `"select input from events_core as events_truncated where events_truncated.project_id = {p1:String} and events_truncated.is_deleted = 0"`,+ );+ });++ it("routes events and events_truncated in one query separately", () => {expect(routedTables(compile(db- .selectFrom("events_core as e")- .$call(includeDeleted())- .select("e.input"),+ .selectFrom("events as a")+ .innerJoin("events_truncated as b", "a.span_id", "b.span_id")+ .select(["a.input", "b.output"]),),),- ).toEqual(["events_core"]);+ ).toEqual(["events_full", "events_core"]);});});diff --git a/packages/shared/src/server/query-ast/eventsRouting.ts b/packages/shared/src/server/query-ast/eventsRouting.tsindex c0681dd1b9..6354b06c8e 100644--- a/packages/shared/src/server/query-ast/eventsRouting.ts+++ b/packages/shared/src/server/query-ast/eventsRouting.ts@@ -1,5 +1,6 @@/**- * Logical `events` routing. Runs first in every compile.+ * Logical events routing. Runs right after the CTE name check in every+ * compile.** Call sites read the logical `events` relation; this pass replaces each* FROM/JOIN reference to it with `events_core` or `events_full`, one choice@@ -18,15 +19,17 @@* - `*`, `e.*` or `COLUMNS(…)` over a logical relation (`count(*)` reads no* column);* - an unqualified truncated column while a logical relation is in scope;- * - a column qualified by `events_core` / `events_full` that names no- * relation in scope: ClickHouse resolves a qualifier by table name even- * when the table is aliased, so it would reach the routed relation;* - raw SQL text or identifiers (`sql` templates, `sql.raw`, `sql.id`,* `sql.table`) in or under a select that reads a logical relation.* `sql.ref` and bound values resolve like builder references.*- * Callers that want the truncated copy on purpose name `events_core`;- * physical `events_core` / `events_full` references pass through unchanged.+ * Callers that want the truncated copy on purpose read `events_truncated`,+ * which always routes to `events_core`, whatever columns it reads.+ *+ * Only this pass names the physical tables. A call site that does — as a+ * FROM/JOIN table, an alias, a column qualifier, or in raw SQL — is+ * rejected: ClickHouse resolves a qualifier by table name even when the+ * table is aliased, so a physical name could reach a routed relation.*/import {AggregateFunctionNode,@@ -49,20 +52,24 @@ import {type PluginTransformQueryArgs,type PluginTransformResultArgs,type QueryId,- type ReferenceNode,+ ReferenceNode,type RootOperationNode,type SelectQueryNode,} from "kysely";import { metadataFilterIsEventsCoreSafe } from "../queries/clickhouse-sql/events-core-truncation";import { QueryCompileError } from "./errors";-import { ArrayIndexNode, FinalTableNode } from "./nodes";+import { ArrayIndexNode, FinalTableNode, walkNodes } from "./nodes";import { describeRelation, selectRelations, type Relation } from "./relations";-import { EVENTS_CORE_TRUNCATED_COLUMNS } from "./schema";+import {+ EVENTS_CORE_TRUNCATED_COLUMNS,+ LOGICAL_EVENTS_TABLES,+ PHYSICAL_EVENTS_TABLES,+} from "./schema";import { ClickHouseOperationNodeTransformer } from "./transformer";const LOGICAL_EVENTS = "events";-const PHYSICAL_EVENTS = new Set(["events_core", "events_full"]);+const PHYSICAL_EVENT_NAME = /\bevents_(core|full)\b/;export class EventsTableRoutingPlugin implements KyselyPlugin {transformQuery(args: PluginTransformQueryArgs): RootOperationNode {@@ -102,6 +109,9 @@ class EventsReadAnalysis extends ClickHouseOperationNodeTransformer {): T {const lookup = coreSafeMetadataLookup(node);if (lookup) {+ // The lookup's columns read no truncated value, but their qualifiers+ // must still not name a physical table.+ rejectPhysicalQualifiers(lookup.array);this.transformNode(lookup.index, queryId);return node;}@@ -112,8 +122,10 @@ class EventsReadAnalysis extends ClickHouseOperationNodeTransformer {node: SelectQueryNode,queryId?: QueryId,): SelectQueryNode {- const relations = selectRelations(node).map(routingRelation);- if (relations.some((relation) => relation.logical)) {+ const described = selectRelations(node);+ described.forEach(rejectPhysicalRelation);+ const relations = described.map(routingRelation);+ if (relations.some((relation) => relation.routed)) {this.readsLogicalEvents = true;}this.scopes.push(relations);@@ -129,6 +141,7 @@ class EventsReadAnalysis extends ClickHouseOperationNodeTransformer {_queryId?: QueryId,): ReferenceNode {const qualifier = node.table?.table.identifier.name;+ rejectPhysicalName(qualifier);// `sql.ref("e.*")` parses the star as a column named `*`.if (SelectAllNode.is(node.column) || node.column.column.name === "*") {if (qualifier === undefined) this.rejectStar("*");@@ -191,6 +204,7 @@ class EventsReadAnalysis extends ClickHouseOperationNodeTransformer {}protected override transformRaw(node: RawNode, queryId?: QueryId): RawNode {+ rejectPhysicalRawNames(node);if (this.readsLogicalInScope() &&(node.sqlFragments.some((fragment) => fragment.trim() !== "") ||@@ -217,11 +231,6 @@ class EventsReadAnalysis extends ClickHouseOperationNodeTransformer {const match = relations.find((relation) => relation.name === qualifier);if (match) return match;}- if (PHYSICAL_EVENTS.has(qualifier) && this.readsLogicalInScope()) {- throw new QueryCompileError(- `Qualify by the relation's alias, not "${qualifier}": ClickHouse also resolves a table name to an aliased relation, so it may reach the logical "${LOGICAL_EVENTS}" relation after routing.`,- );- }return undefined;}@@ -251,6 +260,45 @@ class EventsReadAnalysis extends ClickHouseOperationNodeTransformer {}}+function rejectPhysicalName(name: string | undefined): void {+ if (name === undefined || !PHYSICAL_EVENTS_TABLES.has(name)) return;+ throw new QueryCompileError(+ `Read the logical "${LOGICAL_EVENTS}" relation instead of "${name}": the routing pass picks events_core or events_full from the columns read. Read "events_truncated" to get the truncated events_core copy on purpose.`,+ );+}++/** Column qualifiers anywhere under `node`. */+function rejectPhysicalQualifiers(node: OperationNode): void {+ walkNodes(node, (child) => {+ if (ReferenceNode.is(child)) {+ rejectPhysicalName(child.table?.table.identifier.name);+ }+ });+}++/** Raw SQL text, and `sql.table` / `sql.id` parameters. */+function rejectPhysicalRawNames(node: RawNode): void {+ for (const fragment of node.sqlFragments) {+ rejectPhysicalName(PHYSICAL_EVENT_NAME.exec(fragment)?.[0]);+ }+ for (const parameter of node.parameters) {+ rejectPhysicalName(rawParameterName(parameter));+ }+}++/** The name a `sql.table` / `sql.id` parameter writes into raw SQL. */+function rawParameterName(node: OperationNode): string | undefined {+ if (TableNode.is(node)) return node.table.identifier.name;+ if (IdentifierNode.is(node)) return node.name;+ return undefined;+}++/** A physical events table as FROM/JOIN table or alias. */+function rejectPhysicalRelation({ table, alias }: Relation): void {+ rejectPhysicalName(table?.table.identifier.name);+ rejectPhysicalName(alias);+}+function starError(star: string): QueryCompileError {return new QueryCompileError(`${star} cannot expand the logical "${LOGICAL_EVENTS}" relation: list the columns, so the routing pass can tell whether it reads a truncated one.`,@@ -282,27 +330,25 @@ class EventsTableRouter extends ClickHouseOperationNodeTransformer {}/**- * `events as e` → `events_core as e`. An unaliased `events` keeps its name- * as the alias (`events_core as events`), so `events.col` references and- * the qualifiers later passes add still resolve.+ * `events as e` → `events_core as e`. An unaliased relation keeps its+ * logical name as the alias (`events_core as events`), so `events.col`+ * references and the qualifiers later passes add still resolve.*/private routeRelation(node: OperationNode, queryId?: QueryId): OperationNode {const table = TableNode.create(this.fullRelations.has(node) ? "events_full" : "events_core",);const { source, alias } = describeRelation(node);- if (!isLogicalEventsTable(source)) {+ const logicalName = logicalEventsTableName(source);+ if (logicalName === undefined) {return this.transformNode(node, queryId);}if (FinalTableNode.is(node)) {throw new QueryCompileError(- `FINAL cannot wrap the logical "${LOGICAL_EVENTS}" relation: events_core and events_full are immutable at read time. Drop the FINAL.`,+ `FINAL cannot wrap the logical "${logicalName}" relation: events_core and events_full are immutable at read time. Drop the FINAL.`,);}- return AliasNode.create(- table,- IdentifierNode.create(alias ?? LOGICAL_EVENTS),- );+ return AliasNode.create(table, IdentifierNode.create(alias ?? logicalName));}}@@ -316,15 +362,20 @@ function routingRelation({table,alias,}: Relation): RoutingRelation {+ const logicalName = logicalEventsTableName(source);return {node,name: alias ?? table?.table.identifier.name,- logical: isLogicalEventsTable(source),+ logical: logicalName === LOGICAL_EVENTS,+ routed: logicalName !== undefined,};}-function isLogicalEventsTable(node: OperationNode): boolean {- return TableNode.is(node) && node.table.identifier.name === LOGICAL_EVENTS;+/** `events` or `events_truncated`, when `node` reads one of them. */+function logicalEventsTableName(node: OperationNode): string | undefined {+ if (!TableNode.is(node)) return undefined;+ const name = node.table.identifier.name;+ return LOGICAL_EVENTS_TABLES.has(name) ? name : undefined;}/** `count(*)` with any `distinct`, `filter`, `over`, … clause. */@@ -383,5 +434,8 @@ type RoutingRelation = {node: OperationNode;/** Alias, or the table name of an unaliased table. */name?: string;+ /** Reads `events`, so its truncated-column reads pick the table. */logical: boolean;+ /** Reads `events` or `events_truncated`, so the router replaces it. */+ routed: boolean;};diff --git a/packages/shared/src/server/query-ast/extensions.test.ts b/packages/shared/src/server/query-ast/extensions.test.tsindex 7755e1800c..43fa3f801b 100644--- a/packages/shared/src/server/query-ast/extensions.test.ts+++ b/packages/shared/src/server/query-ast/extensions.test.ts@@ -37,7 +37,7 @@ import { defineView, fromView } from "./views";const projectId = "proj-1";const eventsMetadata = (- eb: ExpressionBuilder<{ e: ClickHouseDatabase["events_core"] }, "e">,+ eb: ExpressionBuilder<{ e: ClickHouseDatabase["events"] }, "e">,key: string,) =>metadataValue(eb.ref("e.metadata_names"), eb.ref("e.metadata_values"), key);@@ -45,7 +45,7 @@ const eventsMetadata = (describe("schema-typed selection (condition 7)", () => {it("accepts sum() over a numeric column and emits SQL", () => {const qb = getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select((eb) => [eb.fn.sum("total_cost").as("s")]);const { sql: compiled } = compileClickhouseQuery(qb, projectId);@@ -55,7 +55,7 @@ describe("schema-typed selection (condition 7)", () => {it("rejects sum() over a String column at the runtime validation pass", () => {const qb = getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select((eb) => [eb.fn.sum("environment").as("s")]);expect(() => compileClickhouseQuery(qb, projectId)).toThrow(@@ -85,7 +85,7 @@ describe("metadata subscript access (condition 7b)", () => {// parameter-bound) from "the compiler rendered that tree correctly".it("lowers metadata[key] to ArrayIndexNode + mapFromArrays FunctionNode, not RawNode", () => {const qb = getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").select((eb) => [eventsMetadata(eb, "a").as("a")]).where((eb) => eb(eventsMetadata(eb, "a"), "=", "x"));@@ -107,7 +107,7 @@ describe("metadata subscript access (condition 7b)", () => {it("emits subscript + bound key param in SELECT and WHERE", () => {const qb = getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").select((eb) => [eventsMetadata(eb, "a").as("a")]).where((eb) => eb(eventsMetadata(eb, "a"), "=", "x"));@@ -124,7 +124,7 @@ describe("metadata subscript access (condition 7b)", () => {it("emits the same lowering in HAVING", () => {const qb = getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").select("environment").groupBy("environment").having((eb) => eb(eventsMetadata(eb, "a"), "=", "x"));@@ -141,10 +141,7 @@ describe("metadata subscript access (condition 7b)", () => {// by trusting the lookup's marker, so the lookup may only read the metadata// columns of one relation.describe("rejects a lookup over anything but one relation's metadata columns", () => {- const eb = expressionBuilder<- { e: ClickHouseDatabase["events_core"] },- "e"- >();+ const eb = expressionBuilder<{ e: ClickHouseDatabase["events"] }, "e">();it.each([// Any expression, not only `input`: the pass doesn't walk the// arguments, so a column inside `values` would escape routing.@@ -187,8 +184,8 @@ describe("metadata subscript access (condition 7b)", () => {it("rejects names and values from two relations", () => {const joined = expressionBuilder<{- e: ClickHouseDatabase["events_core"];- f: ClickHouseDatabase["events_core"];+ e: ClickHouseDatabase["events"];+ f: ClickHouseDatabase["events"];},"e" | "f">();@@ -207,10 +204,7 @@ describe("virtual view (condition 8)", () => {const environmentsView = defineView("environments_view")<{environment: string;}>(() =>- getClickhouseKysely()- .selectFrom("events_core")- .select("environment")- .distinct(),+ getClickhouseKysely().selectFrom("events").select("environment").distinct(),);it("treats the view as a table and emits a WITH CTE", () => {@@ -259,7 +253,7 @@ describe("ArrayIndexNode identity", () => {describe("leftUTF8", () => {it("inlines a positive integer length", () => {const qb = getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").select((eb) => [leftUTF8(eb.ref("e.input"), 100).as("input")]);const { sql: compiled } = compileClickhouseQuery(qb, projectId);@@ -269,7 +263,7 @@ describe("leftUTF8", () => {it.each([0, -1, 1.5, Number.NaN])("rejects length %s", (length) => {expect(() =>getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").select((eb) => [leftUTF8(eb.ref("e.input"), length).as("input")]),).toThrow(QueryCompileError);});@@ -278,7 +272,7 @@ describe("leftUTF8", () => {describe("arrayMap", () => {it("names nested lambda parameters apart so the inner one does not shadow", () => {const qb = getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").select((eb) => [arrayMap<string, string[]>((name) =>@@ -304,10 +298,7 @@ describe("arrayMap", () => {describe("arrayExists", () => {it("names nested lambda parameters apart so the inner one does not shadow", () => {- const eb = expressionBuilder<- { e: ClickHouseDatabase["events_core"] },- "e"- >();+ const eb = expressionBuilder<{ e: ClickHouseDatabase["events"] }, "e">();const nested = arrayExists<string>((name) =>arrayExists<string>((tag) => eb(tag, "=", name), eb.ref("e.tags")),@@ -321,7 +312,7 @@ describe("arrayExists", () => {});describe("renderClickhouseExpression", () => {- const eb = expressionBuilder<{ e: ClickHouseDatabase["events_core"] }, "e">();+ const eb = expressionBuilder<{ e: ClickHouseDatabase["events"] }, "e">();const metadata = metadataMap(eb.ref("e.metadata_names"),eb.ref("e.metadata_values"),@@ -340,7 +331,7 @@ describe("renderClickhouseExpression", () => {it("rejects a subquery", () => {const subquery = getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select("trace_id");expect(() => renderClickhouseExpression(subquery)).toThrow(UnscopedRelationError,@@ -361,7 +352,7 @@ describe("renderClickhouseExpression", () => {});describe("renderClickhouseSelections", () => {- const events = getClickhouseKysely().selectFrom("events_core as e");+ const events = getClickhouseKysely().selectFrom("events as e");it("keys each select by its output name, in select order", () => {const rendered = renderClickhouseSelections(@@ -410,7 +401,7 @@ describe("renderClickhouseSelections", () => {expect(() =>renderClickhouseSelections(events.select((eb) =>- eb.selectFrom("events_core as x").select("x.trace_id").as("t"),+ eb.selectFrom("events as x").select("x.trace_id").as("t"),),),).toThrow(UnscopedRelationError);@@ -418,7 +409,7 @@ describe("renderClickhouseSelections", () => {});describe("64-bit results", () => {- const eb = expressionBuilder<{ e: ClickHouseDatabase["events_core"] }, "e">();+ const eb = expressionBuilder<{ e: ClickHouseDatabase["events"] }, "e">();const isError = eb("e.level", "=", literal("ERROR"));// ClickHouse 25.8+ (Langfuse v4 needs 25.12) sends (U)Int64 as JSON numbers.@@ -456,7 +447,7 @@ describe("startsWith", () => {it("binds a string prefix and inlines a literal one", () => {const { sql, params } = compileClickhouseQuery(getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").select("e.span_id").where((eb) => startsWith(eb.ref("e.name"), "caller")).where((eb) => startsWith(eb.ref("e.user_id"), literal("const"))),diff --git a/packages/shared/src/server/query-ast/extensions.ts b/packages/shared/src/server/query-ast/extensions.tsindex 7001b760aa..8df187ed4a 100644--- a/packages/shared/src/server/query-ast/extensions.ts+++ b/packages/shared/src/server/query-ast/extensions.ts@@ -845,7 +845,7 @@ class FinalPlugin implements KyselyPlugin {/*** `FROM <table> FINAL` / `JOIN <table> AS alias FINAL` as a `$call` step:* `qb.$call(useFinal(["scores"]))`. Tables declared `dedup: none` (immutable- * at read time, including `events_core`) cannot take FINAL.+ * at read time, including the events relations) cannot take FINAL.*/export function useFinal(tables: ReadonlyArray<keyof ClickHouseDatabase>) {for (const table of tables) {diff --git a/packages/shared/src/server/query-ast/fixtures/catalog.ts b/packages/shared/src/server/query-ast/fixtures/catalog.tsindex 4f150fc040..c19325cc4a 100644--- a/packages/shared/src/server/query-ast/fixtures/catalog.ts+++ b/packages/shared/src/server/query-ast/fixtures/catalog.ts@@ -41,7 +41,7 @@ function environmentsVariant(spec: {}): ClickhouseCompilable {if (spec.writeMode === "events") {return db()- .selectFrom("events_core")+ .selectFrom("events").select("environment").distinct().$if(spec.fromTimestamp !== undefined, (qb) =>@@ -70,11 +70,10 @@ export const CATALOG: CatalogEntry[] = [tier: 0,referenceSql: `SELECT DISTINCT environment- FROM events_core- WHERE (project_id = {p1:String}) AND (events_core.is_deleted = 0)+ FROM events_core AS events+ WHERE (events.project_id = {p1:String}) AND (events.is_deleted = 0)`,- build: () =>- db().selectFrom("events_core").select("environment").distinct(),+ build: () => db().selectFrom("events").select("environment").distinct(),},{id: "environments_legacy_union",@@ -122,12 +121,12 @@ export const CATALOG: CatalogEntry[] = [tier: 1,referenceSql: `SELECT DISTINCT environment- FROM events_core- WHERE (project_id = {p1:String}) AND (events_core.is_deleted = 0) AND (start_time >= {p2:DateTime64(6)})+ FROM events_core AS events+ WHERE (events.project_id = {p1:String}) AND (events.is_deleted = 0) AND (start_time >= {p2:DateTime64(6)})`,build: () =>db()- .selectFrom("events_core")+ .selectFrom("events").select("environment").distinct().where("start_time", ">=", FROM_TS),@@ -137,13 +136,13 @@ export const CATALOG: CatalogEntry[] = [tier: 1,referenceSql: `SELECT environment, count(*) AS n- FROM events_core- WHERE (project_id = {p1:String}) AND (events_core.is_deleted = 0)+ FROM events_core AS events+ WHERE (events.project_id = {p1:String}) AND (events.is_deleted = 0)GROUP BY environment`,build: () =>db()- .selectFrom("events_core")+ .selectFrom("events").select((eb) => ["environment", eb.fn.countAll().as("n")]).groupBy("environment"),},@@ -215,8 +214,8 @@ export const CATALOG: CatalogEntry[] = [referenceSql: `WITH prompt_events AS (SELECT trace_id, span_id- FROM events_core- WHERE (project_id = {p1:String}) AND (events_core.is_deleted = 0)+ FROM events_core AS events+ WHERE (events.project_id = {p1:String}) AND (events.is_deleted = 0))SELECT s.idFROM scores AS s@@ -228,7 +227,7 @@ export const CATALOG: CatalogEntry[] = [build: () =>db().with("prompt_events", (qb) =>- qb.selectFrom("events_core").select(["trace_id", "span_id"]),+ qb.selectFrom("events").select(["trace_id", "span_id"]),).selectFrom("scores as s").select("s.id")@@ -267,14 +266,14 @@ export const CATALOG: CatalogEntry[] = [tier: 3,referenceSql: `SELECT span_id, project_id- FROM events_core- WHERE (project_id = {p1:String}) AND (events_core.is_deleted = 0)+ FROM events_core AS events+ WHERE (events.project_id = {p1:String}) AND (events.is_deleted = 0)ORDER BY event_ts DESCLIMIT 1 BY span_id, project_id`,build: () =>db()- .selectFrom("events_core")+ .selectFrom("events").select(["span_id", "project_id"]).orderBy("event_ts", "desc").$call(@@ -289,12 +288,12 @@ export const CATALOG: CatalogEntry[] = [tier: 4,referenceSql: `SELECT span_id, rank() OVER (PARTITION BY trace_id ORDER BY start_time DESC) AS rk- FROM events_core- WHERE (project_id = {p1:String}) AND (events_core.is_deleted = 0)+ FROM events_core AS events+ WHERE (events.project_id = {p1:String}) AND (events.is_deleted = 0)`,build: () =>db()- .selectFrom("events_core")+ .selectFrom("events").select((eb) => ["span_id",eb.fn@@ -324,14 +323,14 @@ export const CATALOG: CatalogEntry[] = [`,build: () =>db()- .selectFrom("events_core as e")+ .selectFrom("events as e").select("e.span_id").where((eb) =>eb("e.event_ts","=",eb- .selectFrom("events_core as inner")+ .selectFrom("events as inner").select((ieb) => ieb.fn.max("inner.event_ts").as("m")).whereRef("inner.trace_id", "=", "e.trace_id"),),@@ -351,7 +350,7 @@ export const CATALOG: CatalogEntry[] = [`,build: () =>db()- .selectFrom("events_core as e")+ .selectFrom("events_truncated as e").select((eb) => [metadataValue(eb.ref("e.metadata_names"),@@ -390,8 +389,8 @@ export const CATALOG: CatalogEntry[] = [tier: 7,referenceSql: `SELECT DISTINCT environment- FROM events_core- WHERE (project_id = {p1:String}) AND (events_core.is_deleted = 0)+ FROM events_core AS events+ WHERE (events.project_id = {p1:String}) AND (events.is_deleted = 0)`,build: () => environmentsVariant({ writeMode: "events" }),},@@ -401,8 +400,8 @@ export const CATALOG: CatalogEntry[] = [tier: 7,referenceSql: `SELECT DISTINCT environment- FROM events_core- WHERE (project_id = {p1:String}) AND (events_core.is_deleted = 0) AND (start_time >= {p2:DateTime64(6)})+ FROM events_core AS events+ WHERE (events.project_id = {p1:String}) AND (events.is_deleted = 0) AND (start_time >= {p2:DateTime64(6)})`,build: () =>environmentsVariant({ writeMode: "events", fromTimestamp: FROM_TS }),diff --git a/packages/shared/src/server/query-ast/nodes.test.ts b/packages/shared/src/server/query-ast/nodes.test.tsindex 08c67f24f8..0e8abb5e75 100644--- a/packages/shared/src/server/query-ast/nodes.test.ts+++ b/packages/shared/src/server/query-ast/nodes.test.ts@@ -58,9 +58,7 @@ describe("tenancy injection", () => {});it("refuses a scope that is not a single projectId string", () => {- const qb = getClickhouseKysely()- .selectFrom("events_core")- .select("environment");+ const qb = getClickhouseKysely().selectFrom("events").select("environment");expect(() =>// @ts-expect-error - a query is scoped to exactly one project@@ -82,9 +80,7 @@ describe("tenancy injection", () => {});it("injects project_id when the builder omitted it", () => {- const qb = getClickhouseKysely()- .selectFrom("events_core")- .select("environment");+ const qb = getClickhouseKysely().selectFrom("events").select("environment");const { sql: compiled, params } = compileClickhouseQuery(qb, projectId);expect(compiled.toLowerCase()).toContain("project_id");@@ -93,7 +89,7 @@ describe("tenancy injection", () => {it("does not duplicate project_id when the builder already scoped the table", () => {const qb = getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select("environment").where("project_id", "=", "proj-1");@@ -148,7 +144,7 @@ describe("tenancy injection", () => {"rejects the %s table function as a JOIN source",(fn) => {const qb = getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").innerJoin((eb) =>eb@@ -200,13 +196,13 @@ describe("tenancy injection", () => {it("rejects raw SQL fragments in SELECT and WHERE, not only FROM/JOIN", () => {const selectRaw = getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events_truncated").select(sql<string>`(SELECT environment FROM traces LIMIT 1)`.as("stolen"),);const whereRaw = getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events_truncated").select("environment").where(sql<boolean>`id IN (SELECT id FROM traces)`);@@ -282,7 +278,7 @@ describe("tenancy injection", () => {it("injects the context project even when the builder scoped a different project", () => {const qb = getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select("environment").where("project_id", "=", "other-project");@@ -737,7 +733,7 @@ describe("ARRAY JOIN and LIMIT BY nodes", () => {it("attaches a real LimitByNode", () => {const qb = getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select("span_id").where("project_id", "=", "proj-1").$call(@@ -777,7 +773,7 @@ describe("ARRAY JOIN and LIMIT BY nodes", () => {it("emits the LIMIT BY count", () => {const qb = getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select("span_id").$call(limitBy({ count: 2, columns: (eb) => [eb.ref("span_id")] }));@@ -787,7 +783,7 @@ describe("ARRAY JOIN and LIMIT BY nodes", () => {it("emits LIMIT BY before a plain LIMIT", () => {const qb = getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select("span_id").where("project_id", "=", "proj-1").limit(50)@@ -828,8 +824,8 @@ describe("ARRAY JOIN and LIMIT BY nodes", () => {});it("rejects FINAL on events_core", () => {- expect(() => useFinal(["events_core"])).toThrow(QueryCompileError);- expect(() => useFinal(["events_core"])).toThrow(/immutable/);+ expect(() => useFinal(["events"])).toThrow(QueryCompileError);+ expect(() => useFinal(["events"])).toThrow(/immutable/);});});diff --git a/packages/shared/src/server/query-ast/schema.test.ts b/packages/shared/src/server/query-ast/schema.test.tsindex 4f43bc13c9..c25c6c2087 100644--- a/packages/shared/src/server/query-ast/schema.test.ts+++ b/packages/shared/src/server/query-ast/schema.test.ts@@ -13,6 +13,7 @@ describe("table registry derivation", () => {"events","events_core","events_full",+ "events_truncated","observations","scores","traces",diff --git a/packages/shared/src/server/query-ast/schema.ts b/packages/shared/src/server/query-ast/schema.tsindex ac90b7ba0a..ba446f339d 100644--- a/packages/shared/src/server/query-ast/schema.ts+++ b/packages/shared/src/server/query-ast/schema.ts@@ -308,6 +308,14 @@ const TABLE_REGISTRY = {dedup: { strategy: "none" },softDelete: "is_deleted",}),+ // Logical relation for reading the truncated copy on purpose: routing+ // always replaces it with events_core, also when it reads a truncated+ // column (input/output/metadata previews).+ events_truncated: defineTable({+ columns: EVENTS_COLUMNS,+ dedup: { strategy: "none" },+ softDelete: "is_deleted",+ }),scores: defineTable({columns: {id: "String",@@ -341,17 +349,33 @@ type RowOf<Cols extends Record<string, ChColumnType>> = {[C in keyof Cols]: TsTypeOfColumn<Cols[C]>;};-/** Derived Kysely row types. */+/**+ * Relations only the routing pass may name. Call sites read the logical+ * `events` / `events_truncated` instead.+ */+export const PHYSICAL_EVENTS_TABLES: ReadonlySet<string> = new Set([+ "events_core",+ "events_full",+] satisfies ReadonlyArray<keyof typeof TABLE_REGISTRY>);++/** Logical relations the routing pass replaces with a physical events table. */+export const LOGICAL_EVENTS_TABLES: ReadonlySet<string> = new Set([+ "events",+ "events_truncated",+] satisfies ReadonlyArray<keyof typeof TABLE_REGISTRY>);++/** Derived Kysely row types, without the physical events tables. */export type ClickHouseDatabase = {- [T in keyof typeof TABLE_REGISTRY]: RowOf<- (typeof TABLE_REGISTRY)[T]["columns"]- >;+ [T in Exclude<+ keyof typeof TABLE_REGISTRY,+ "events_core" | "events_full"+ >]: RowOf<(typeof TABLE_REGISTRY)[T]["columns"]>;};/*** Declared ClickHouse type per column, per physical registry table. A* servertest checks it against `system.columns` of the migrated database. The- * logical `events` relation has no table of its own; its columns are+ * logical events relations have no table of their own; their columns are* `events_core`'s and `events_full`'s.*/export const TABLE_COLUMN_TYPES: Record<@@ -359,7 +383,7 @@ export const TABLE_COLUMN_TYPES: Record<Record<string, ChColumnType>> = lookupTable(Object.entries(TABLE_REGISTRY)- .filter(([table]) => table !== "events")+ .filter(([table]) => !LOGICAL_EVENTS_TABLES.has(table)).map(([table, spec]) => [table, spec.columns]),);@@ -377,7 +401,7 @@ export const COLUMN_DATA_TYPES: Record<string, ColumnDataType> = lookupTable(),);-/** Every registry relation name, the logical `events` included. */+/** Every registry relation name, the logical events relations included. */export const REGISTRY_TABLES: ReadonlySet<string> = new Set(Object.keys(TABLE_REGISTRY),);diff --git a/packages/shared/src/server/query-ast/softDelete.test.ts b/packages/shared/src/server/query-ast/softDelete.test.tsindex 3eafc32088..9249b2ac9a 100644--- a/packages/shared/src/server/query-ast/softDelete.test.ts+++ b/packages/shared/src/server/query-ast/softDelete.test.ts@@ -22,6 +22,7 @@ describe("soft-delete filtering", () => {events: "is_deleted",events_core: "is_deleted",events_full: "is_deleted",+ events_truncated: "is_deleted",});});@@ -29,11 +30,11 @@ describe("soft-delete filtering", () => {expect(compile(getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").select((eb) => ["span_id", eb.lit(0).as("is_deleted")]),),).toMatchInlineSnapshot(- `"select span_id, 0 as is_deleted from events_core where project_id = {p1:String} and events_core.is_deleted = 0"`,+ `"select span_id, 0 as is_deleted from events_core as events where events.project_id = {p1:String} and events.is_deleted = 0"`,);});@@ -41,12 +42,12 @@ describe("soft-delete filtering", () => {expect(compile(getClickhouseKysely()- .selectFrom("events_full as e")+ .selectFrom("events as e").select("e.span_id").where("e.trace_id", "=", "t"),),).toMatchInlineSnapshot(- `"select e.span_id from events_full as e where e.project_id = {p1:String} and e.is_deleted = 0 and e.trace_id = {p2:String}"`,+ `"select e.span_id from events_core as e where e.project_id = {p1:String} and e.is_deleted = 0 and e.trace_id = {p2:String}"`,);});@@ -55,7 +56,7 @@ describe("soft-delete filtering", () => {compile(getClickhouseKysely().selectFrom("scores as s")- .leftJoin("events_core as e", (join) =>+ .leftJoin("events as e", (join) =>join.onRef("e.trace_id", "=", "s.trace_id"),).select("s.id"),@@ -70,7 +71,7 @@ describe("soft-delete filtering", () => {compile(getClickhouseKysely().selectFrom("scores as s")- .crossJoin("events_core as e")+ .crossJoin("events as e").select("s.id"),),).toMatch(/ where s\.project_id = \{p1:String\} and e\.is_deleted = 0$/);@@ -81,19 +82,19 @@ describe("soft-delete filtering", () => {expect(compile(db- .with("ev", (qb) => qb.selectFrom("events_core").select("trace_id"))+ .with("ev", (qb) => qb.selectFrom("events").select("trace_id")).selectFrom("scores as s").select("s.id").where("s.trace_id", "in", (eb) =>- eb.selectFrom("events_full").select("trace_id"),+ eb.selectFrom("events").select("trace_id"),).where("s.trace_id", "in", (eb) =>eb.selectFrom("ev").select("trace_id"),)- .unionAll(db.selectFrom("events_core").select("span_id as id")),+ .unionAll(db.selectFrom("events").select("span_id as id")),),).toMatchInlineSnapshot(- `"with ev as (select trace_id from events_core where project_id = {p1:String} and events_core.is_deleted = 0) select s.id from scores as s where s.project_id = {p1:String} and s.trace_id in (select trace_id from events_full where project_id = {p1:String} and events_full.is_deleted = 0) and s.trace_id in (select trace_id from ev) union all select span_id as id from events_core where project_id = {p1:String} and events_core.is_deleted = 0"`,+ `"with ev as (select trace_id from events_core as events where events.project_id = {p1:String} and events.is_deleted = 0) select s.id from scores as s where s.project_id = {p1:String} and s.trace_id in (select trace_id from events_core as events where events.project_id = {p1:String} and events.is_deleted = 0) and s.trace_id in (select trace_id from ev) union all select span_id as id from events_core as events where events.project_id = {p1:String} and events.is_deleted = 0"`,);});@@ -102,18 +103,18 @@ describe("soft-delete filtering", () => {expect(compile(db- .selectFrom(db.selectFrom("events_core").select("span_id").as("sub"))+ .selectFrom(db.selectFrom("events").select("span_id").as("sub")).select("sub.span_id"),),).toMatchInlineSnapshot(- `"select sub.span_id from (select span_id from events_core where project_id = {p1:String} and events_core.is_deleted = 0) as sub"`,+ `"select sub.span_id from (select span_id from events_core as events where events.project_id = {p1:String} and events.is_deleted = 0) as sub"`,);});it("filters a soft-delete table wrapped in FINAL", () => {// No soft-delete table allows FINAL yet, so the node is built by hand.const node = getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").select("e.span_id").toOperationNode();if (!SelectQueryNode.is(node) || !node.from)@@ -138,15 +139,15 @@ describe("soft-delete filtering", () => {expect(compile(getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").$call(includeDeleted()).select("e.span_id").where("e.trace_id", "in", (eb) =>- eb.selectFrom("events_core").select("trace_id"),+ eb.selectFrom("events").select("trace_id"),),),).toMatchInlineSnapshot(- `"select e.span_id from events_core as e where e.project_id = {p1:String} and e.trace_id in (select trace_id from events_core where project_id = {p1:String} and events_core.is_deleted = 0)"`,+ `"select e.span_id from events_core as e where e.project_id = {p1:String} and e.trace_id in (select trace_id from events_core as events where events.project_id = {p1:String} and events.is_deleted = 0)"`,);});@@ -155,7 +156,7 @@ describe("soft-delete filtering", () => {compile(getClickhouseKysely().selectFrom("scores as s")- .rightJoin("events_core as e", (join) =>+ .rightJoin("events as e", (join) =>join.onRef("e.trace_id", "=", "s.trace_id"),).select("e.span_id"),@@ -164,7 +165,7 @@ describe("soft-delete filtering", () => {expect(() =>compile(getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").fullJoin("scores as s", (join) =>join.onRef("s.trace_id", "=", "e.trace_id"),)@@ -175,7 +176,7 @@ describe("soft-delete filtering", () => {compile(getClickhouseKysely().selectFrom("scores as s")- .rightJoin("events_core as e", (join) =>+ .rightJoin("events as e", (join) =>join.onRef("e.trace_id", "=", "s.trace_id"),).$call(includeDeleted())@@ -189,7 +190,7 @@ describe("soft-delete filtering", () => {compile(getClickhouseKysely().selectFrom("scores as s")- .innerJoin("events_core as e", (join) =>+ .innerJoin("events as e", (join) =>join.onRef("e.trace_id", "=", "s.trace_id"),).$call(includeDeleted())@@ -202,7 +203,7 @@ describe("soft-delete filtering", () => {expect(compile(getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").$call(includeDeleted()).select("span_id").orderBy("event_ts", "desc")@@ -223,7 +224,7 @@ describe("soft-delete filtering", () => {expect(() =>compile(getClickhouseKysely()- .with("ev", (qb) => qb.selectFrom("events_core").select("trace_id"))+ .with("ev", (qb) => qb.selectFrom("events").select("trace_id")).selectFrom("ev").$call(includeDeleted()).select("trace_id"),@@ -235,12 +236,12 @@ describe("soft-delete filtering", () => {expect(compile(getClickhouseKysely()- .selectFrom("events_full as e")+ .selectFrom("events as e").$call(softDeleteAfterLimit()).select((eb) => ["e.span_id as id","e.event_ts",- "input",+ "e.input",eb.fn<number>("length", ["e.output"]).as("len"),]).where("e.span_id", "=", "o")@@ -248,7 +249,7 @@ describe("soft-delete filtering", () => {.limit(1),),).toMatchInlineSnapshot(- `"select latest.id, latest.event_ts, latest.input, latest.len from (select e.span_id as id, e.event_ts, input, length(e.output) as len, e.is_deleted as is_deleted from events_full as e where e.project_id = {p1:String} and e.span_id = {p2:String} order by e.event_ts desc limit {p3:Int64}) as latest where latest.is_deleted = 0"`,+ `"select latest.id, latest.event_ts, latest.input, latest.len from (select e.span_id as id, e.event_ts, e.input, length(e.output) as len, e.is_deleted as is_deleted from events_full as e where e.project_id = {p1:String} and e.span_id = {p2:String} order by e.event_ts desc limit {p3:Int64}) as latest where latest.is_deleted = 0"`,);});@@ -258,7 +259,7 @@ describe("soft-delete filtering", () => {getClickhouseKysely().with("latest_events", () =>getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").$call(softDeleteAfterLimit()).select(["e.span_id as span_id", "e.trace_id as trace_id"]).orderBy("e.event_ts", "desc")@@ -278,13 +279,13 @@ describe("soft-delete filtering", () => {expect(compile(getClickhouseKysely()- .selectFrom("events_core")+ .selectFrom("events").$call(softDeleteAfterLimit()).select("span_id").limit(1),),).toMatchInlineSnapshot(- `"select latest.span_id from (select span_id, events_core.is_deleted as is_deleted from events_core where project_id = {p1:String} limit {p2:Int64}) as latest where latest.is_deleted = 0"`,+ `"select latest.span_id from (select span_id, events.is_deleted as is_deleted from events_core as events where events.project_id = {p1:String} limit {p2:Int64}) as latest where latest.is_deleted = 0"`,);});@@ -292,21 +293,21 @@ describe("soft-delete filtering", () => {expect(compile(getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events as e").$call(softDeleteAfterLimit()).select("e.span_id").where("e.trace_id", "in", (eb) =>- eb.selectFrom("events_core").select("trace_id"),+ eb.selectFrom("events").select("trace_id"),).limit(1),),- ).toContain("events_core.is_deleted = 0");+ ).toContain("events.is_deleted = 0");});it("rejects softDeleteAfterLimit() on selects it cannot wrap", () => {const marked = () =>getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events_truncated as e").$call(softDeleteAfterLimit()).limit(1);const rejects = (@@ -317,7 +318,7 @@ describe("soft-delete filtering", () => {rejects(marked(), /without outputs/);rejects(getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events_truncated as e").$call(softDeleteAfterLimit()).select("e.span_id"),/needs a LIMIT or LIMIT BY/,@@ -330,7 +331,9 @@ describe("soft-delete filtering", () => {marked().select("e.span_id").unionAll(- getClickhouseKysely().selectFrom("events_core").select("span_id"),+ getClickhouseKysely()+ .selectFrom("events_truncated")+ .select("span_id"),),/cannot wrap a GROUP BY or set operation/,);@@ -350,7 +353,7 @@ describe("soft-delete filtering", () => {);rejects(marked()- .innerJoin("events_full as f", (join) =>+ .innerJoin("events_truncated as f", (join) =>join.onRef("f.span_id", "=", "e.span_id"),).select("e.span_id"),@@ -359,7 +362,7 @@ describe("soft-delete filtering", () => {rejects(getClickhouseKysely().selectFrom("scores as s")- .leftJoin("events_core as e", (join) =>+ .leftJoin("events_truncated as e", (join) =>join.onRef("e.trace_id", "=", "s.trace_id"),).$call(softDeleteAfterLimit())diff --git a/packages/shared/src/server/query-ast/types.assert.ts b/packages/shared/src/server/query-ast/types.assert.tsindex 81832ede8c..e15e1be238 100644--- a/packages/shared/src/server/query-ast/types.assert.ts+++ b/packages/shared/src/server/query-ast/types.assert.ts@@ -20,27 +20,32 @@ import { defineView, fromView } from "./views";export function schemaTypeAssertions(): void {const db = getClickhouseKysely();- db.selectFrom("events_core")+ db.selectFrom("events").select("environment")// @ts-expect-error DateTime column compared to Int.where("start_time", "=", 123);- db.selectFrom("events_core")+ db.selectFrom("events").select("environment").where("start_time", ">=", new Date("2026-01-01T00:00:00.000Z"));- db.selectFrom("events_core").select((eb) => eb.fn.sum("total_cost").as("s"));+ db.selectFrom("events").select((eb) => eb.fn.sum("total_cost").as("s"));// Intentionally valid at the TS layer — Kysely does not constrain sum()// to numeric columns. The runtime pass in typecheck.ts rejects this.- db.selectFrom("events_core").select((eb) => eb.fn.sum("environment").as("s"));+ db.selectFrom("events").select((eb) => eb.fn.sum("environment").as("s"));- db.selectFrom("events_core")+ db.selectFrom("events")// @ts-expect-error unknown column is not in the schema.select("not_a_column");+ // @ts-expect-error physical events tables are reached through `events` / `events_truncated`+ db.selectFrom("events_core");+ // @ts-expect-error physical events tables are reached through `events` / `events_truncated`+ db.selectFrom("events_full as e");+const view = defineView("environments_view")<{ environment: string }>(() =>- db.selectFrom("events_core").select("environment").distinct(),+ db.selectFrom("events").select("environment").distinct(),);fromView(view).select("environment");fromView(view)@@ -50,7 +55,7 @@ export function schemaTypeAssertions(): void {// Named-tuple arrays read as arrays of objects keyed by tuple name; a typo// in the column type would make the row field `never` and reject these.const _skills = db- .selectFrom("events_core")+ .selectFrom("events").select(["skills_available", "skills_resource_loaded"]);type SkillsRow = InferResult<typeof _skills>[number];const _available: SkillsRow["skills_available"] = [@@ -99,7 +104,7 @@ export function extensionTypeAssertions(): void {// limitBy preserves the row type: it declares no new columns.const _limited = db- .selectFrom("events_core")+ .selectFrom("events").select("span_id").$call(limitBy({ count: 1, columns: (eb) => [eb.ref("span_id")] }));type Limited = InferResult<typeof _limited>[number];@@ -108,7 +113,7 @@ export function extensionTypeAssertions(): void {type _NoExtra = Limited["cost_key"];// Helper columns are references checked against the query's tables.- db.selectFrom("events_core as e")+ db.selectFrom("events as e").select("e.span_id")// @ts-expect-error eb.ref resolves against this select's tables.$call(limitBy({ count: 1, columns: (eb) => [eb.ref("e.span_idx")] }));diff --git a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snapindex 73472621e8..d63ab8d6f8 100644--- a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap+++ b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap@@ -742,8 +742,8 @@ exports[`golden: events export streams family > getEventsForBlobStorageExport fiNULL,dateDiff('millisecond', e.start_time, e.completion_start_time)) AS time_to_first_token,- input,- output+ e.input,+ e.outputFROM events_full AS eWHERE (e.project_id = {param1:String})AND (e.is_deleted = 0)@@ -804,8 +804,8 @@ exports[`golden: events export streams family > getEventsForBlobStorageExport fie.parent_span_id AS parent_observation_id,e.start_time AS start_time,e.end_time AS end_time,- input,- output+ e.input,+ e.outputFROM events_full AS eWHERE (e.project_id = {param1:String})AND (e.is_deleted = 0)@@ -1012,8 +1012,8 @@ exports[`golden: events export streams family > getEventsForBlobStorageExport fiNULL,dateDiff('millisecond', e.start_time, e.completion_start_time)) AS time_to_first_token,- input,- output+ e.input,+ e.outputFROM events_full AS eWHERE (e.project_id = {param1:String})AND (e.is_deleted = 0)@@ -1227,8 +1227,8 @@ exports[`golden: events export streams family > getEventsForBlobStorageExportParNULL,dateDiff('millisecond', e.start_time, e.completion_start_time)) AS time_to_first_token,- input,- output+ e.input,+ e.outputFROM events_full AS eWHERE (e.project_id = {param1:String})AND (e.is_deleted = 0)@@ -1297,8 +1297,8 @@ exports[`golden: events export streams family > getEventsForBlobStorageExportPare.tags AS tags,e.release AS release,e.model_id AS model_id,- input,- output,+ e.input,+ e.output,if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time) / 1000) AS latency,if(isNull(e.completion_start_time),@@ -1336,8 +1336,8 @@ exports[`golden: events export streams family > getEventsForBlobStorageExportRawe.parent_span_id AS parent_observation_id,e.start_time AS start_time,e.end_time AS end_time,- input,- output+ e.input,+ e.outputFROM events_full AS eWHERE (e.project_id = {param1:String})AND (e.is_deleted = 0)@@ -1405,8 +1405,8 @@ exports[`golden: events export streams family > getEventsForBlobStorageExportRawe.tags AS tags,e.release AS release,e.model_id AS model_id,- input,- output,+ e.input,+ e.output,if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time) / 1000) AS latency,if(isNull(e.completion_start_time),@@ -1486,8 +1486,8 @@ exports[`golden: events export streams family > getEventsForBlobStorageExportRawNULL,dateDiff('millisecond', e.start_time, e.completion_start_time)) AS time_to_first_token,- input,- output+ e.input,+ e.outputFROM events_full AS eWHERE (e.project_id = {param1:String})AND (e.is_deleted = 0)@@ -2119,8 +2119,8 @@ FROMmapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,e.tags AS tags,e.release AS release,- input,- output,+ e.input,+ e.output,e.is_deleted AS is_deletedFROM events_full AS eWHERE (e.project_id = {param1:String})@@ -2240,8 +2240,8 @@ FROMmapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,e.tags AS tags,e.release AS release,- leftUTF8(input, 1000) AS input,- leftUTF8(output, 1000) AS output,+ leftUTF8(e.input, 1000) AS input,+ leftUTF8(e.output, 1000) AS output,e.is_deleted AS is_deletedFROM events_core AS eWHERE (e.project_id = {param1:String})diff --git a/packages/shared/src/server/repositories/events-selects.types.assert.ts b/packages/shared/src/server/repositories/events-selects.types.assert.tsindex 7450ca9413..77dc5bb00b 100644--- a/packages/shared/src/server/repositories/events-selects.types.assert.ts+++ b/packages/shared/src/server/repositories/events-selects.types.assert.ts@@ -22,15 +22,14 @@ export function eventsSelectsScopeTypeAssertions(): void {const db = getClickhouseKysely();const allTraceAggregates = traceAggregationSelects(TRACE_AGGREGATION_FIELDS);- db.selectFrom("events_core as e").select(allTraceAggregates);- db.selectFrom("events_full as e").select(allTraceAggregates);db.selectFrom("events as e").select(allTraceAggregates);+ db.selectFrom("events_truncated as e").select(allTraceAggregates);db.selectFrom("events as e").innerJoin("traces as t", "t.id", "e.trace_id")// @ts-expect-error a join.select(allTraceAggregates);- db.with("e", (qb) => qb.selectFrom("events_core").selectAll())+ db.with("e", (qb) => qb.selectFrom("events").selectAll()).selectFrom("traces as t")// @ts-expect-error a CTE named `e` is not read in FROM.select(allTraceAggregates);@@ -39,7 +38,7 @@ export function eventsSelectsScopeTypeAssertions(): void {// @ts-expect-error no `e` in scope.select(allTraceAggregates);- db.selectFrom("events_core")+ db.selectFrom("events")// @ts-expect-error the events table is not aliased `e`.select(allTraceAggregates);@@ -59,7 +58,7 @@ export function eventsSelectsScopeTypeAssertions(): void {// @ts-expect-error the subquery `e` lacks `completion_start_time`.select(eventsMetricsSecondsSelects);- db.with("e", (qb) => qb.selectFrom("events_core").selectAll())+ db.with("e", (qb) => qb.selectFrom("events").selectAll())// @ts-expect-error no FROM: `e` is never read.selectNoFrom((eb) => [eventsTraceName(eb).as("name")]);@@ -67,8 +66,8 @@ export function eventsSelectsScopeTypeAssertions(): void {const traceNameSelect = scopedToEvents((eb) =>eventsTraceName(eb).as("name"),);- db.selectFrom("events_core as e").select(traceNameSelect);- db.selectFrom("events_core as e")+ db.selectFrom("events as e").select(traceNameSelect);+ db.selectFrom("events as e").innerJoin("traces as t", "t.id", "e.trace_id")// @ts-expect-error a join, through a composed helper.select(traceNameSelect);@@ -80,7 +79,7 @@ export function eventsSelectsScopeTypeAssertions(): void {// scope of the builder it was made from, and Kysely's `select()` accepts any// aliased expression. Build shared expressions from the query's builder.const detached = expressionBuilder<- { e: ClickHouseDatabase["events_core"] },+ { e: ClickHouseDatabase["events"] },"e">();db.selectFrom("traces as t").select(@@ -90,11 +89,11 @@ export function eventsSelectsScopeTypeAssertions(): void {// Intentionally valid too: upcasting the query's builder to the plain events// builder (here, under a join) drops what the guard checks. Use// scopedToEvents instead of typing a parameter as `ExpressionBuilder`.- db.selectFrom("events_core as e")+ db.selectFrom("events as e").innerJoin("traces as t", "t.id", "e.trace_id").select((eb) => {const events: ExpressionBuilder<- { e: ClickHouseDatabase["events_core"] },+ { e: ClickHouseDatabase["events"] },"e"> = eb;return [eventsTraceName(events).as("name")];diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.tsindex b3dd485687..f2903b70af 100644--- a/packages/shared/src/server/repositories/events.ts+++ b/packages/shared/src/server/repositories/events.ts@@ -1129,8 +1129,8 @@ async function getObservationByIdFromEventsTableInternal({// A soft-deleted latest version reads as not found, not as an older version.const builder = (needsFullTable- ? db.selectFrom("events_full as e")- : db.selectFrom("events_core as e")+ ? db.selectFrom("events as e")+ : db.selectFrom("events_truncated as e")).$call(softDeleteAfterLimit()).select(observationByIdSelects)@@ -1175,11 +1175,11 @@ function observationIoSelects(eb: ExpressionBuilder<{ e: IoColumns }, "e">,truncated: boolean | undefined,) {- if (!truncated) return ["input", "output"] as const;+ if (!truncated) return ["e.input", "e.output"] as const;const charLimit = env.LANGFUSE_SERVER_SIDE_IO_CHAR_LIMIT;return [- leftUTF8(eb.ref("input"), charLimit).as("input"),- leftUTF8(eb.ref("output"), charLimit).as("output"),+ leftUTF8(eb.ref("e.input"), charLimit).as("input"),+ leftUTF8(eb.ref("e.output"), charLimit).as("output"),];}@@ -1216,7 +1216,7 @@ export const getTraceByIdFromEventsTable = async ({const tracesCte = (qb: QueryCreator<ClickHouseDatabase>) =>qb- .selectFrom(truncated ? "events_core as e" : "events_full as e")+ .selectFrom(truncated ? "events_truncated as e" : "events as e").select((eb) => traceAggregationSelects(TRACE_AGGREGATION_FIELDS)(eb)).where("trace_id", "in", [traceId]).$if(fromTimestamp != null, (inner) =>@@ -2658,7 +2658,7 @@ export async function getAgentGraphDataFromEventsTable(params: {const db = getClickhouseKysely();const builder = db- .selectFrom("events_core as e")+ .selectFrom("events_truncated as e").select((eb) => {const metadata = metadataMap(eb.ref("e.metadata_names"),@@ -2830,7 +2830,7 @@ export const getObservationsBatchIOFromEventsTable = async <// Keep the individual filters for primary-key pruning and the tuple filter// for exact trace/observation matching.const builder = getClickhouseKysely()- .selectFrom(truncated ? "events_core as e" : "events_full as e")+ .selectFrom(truncated ? "events_truncated as e" : "events as e").select((eb) => ["e.span_id as id","e.trace_id as trace_id",@@ -3442,9 +3442,9 @@ const buildEventsForBlobStorageExportQuery = (});const builder = getClickhouseKysely()- .selectFrom(includeIO ? "events_full as e" : "events_core as e")+ .selectFrom(includeIO ? "events as e" : "events_truncated as e").select(eventsRowFieldSelects(eventsFieldSetKeys(...fieldSets)))- .$if(includeIO, (qb) => qb.select(["input", "output"]))+ .$if(includeIO, (qb) => qb.select(["e.input", "e.output"])).$if(metricsInSeconds, (qb) => qb.select(eventsMetricsSecondsSelects)).where("e.start_time", ">=", minTimestamp).where("e.start_time", "<", maxTimestamp)@@ -3543,7 +3543,7 @@ export const getEventsForAnalyticsIntegrations = async function* (maxTimestamp: Date,) {const builder = getClickhouseKysely()- .selectFrom("events_core as e")+ .selectFrom("events_truncated as e")// Use export field set for most fields (id, traceId, name, type, level, version,// environment, userId, sessionId, tags, release, traceName, totalCost, latency, etc.).select(eventsRowFieldSelects(eventsFieldSetKeys("export")))diff --git a/packages/shared/src/server/services/sessions-ui-table-events-service.ts b/packages/shared/src/server/services/sessions-ui-table-events-service.tsindex 7d29809bf8..224ea76bf7 100644--- a/packages/shared/src/server/services/sessions-ui-table-events-service.ts+++ b/packages/shared/src/server/services/sessions-ui-table-events-service.ts@@ -53,7 +53,7 @@ export const getSessionTracesFromEvents = async (props: {}) => {const { projectId, sessionId } = props;const builder = getClickhouseKysely()- .selectFrom("events_full as e")+ .selectFrom("events as e").select(traceAggregationSelects(TRACE_AGGREGATION_FIELDS)).where("e.session_id", "=", sessionId).groupBy(["trace_id", "project_id"])