Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

refactor(query-ast): read events only through logical relations (#19172)

Niklas Semmlerc978cf4a28a2
diff --git a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
index 703aa2633a..2a51cfcaaa 100644
--- a/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
+++ b/packages/shared/src/server/queries/clickhouse-sql/event-query-builder.ts
@@ -26,10 +26,7 @@ import {
type ClickhouseFilter,
} from "./clickhouse-filter";
-const eventsRef = expressionBuilder<
- { e: ClickHouseDatabase["events_core"] },
- "e"
->();
+const eventsRef = expressionBuilder<{ e: ClickHouseDatabase["events"] }, "e">();
const eventsMetadataMap = metadataMap(
eventsRef.ref("e.metadata_names"),
diff --git a/packages/shared/src/server/query-ast/README.md b/packages/shared/src/server/query-ast/README.md
index 709710ba0b..5ab469cfba 100644
--- a/packages/shared/src/server/query-ast/README.md
+++ b/packages/shared/src/server/query-ast/README.md
@@ -96,12 +96,15 @@ suffix to run there. Promote it to a required check once it has proven stable.
`metadataFilterIsEventsCoreSafe` accepts the value, as in the raw-SQL
builders. `*` / `e.*` over `events`, raw SQL text in a select that reads
it, and an unqualified truncated column are compile errors while `events`
- is in scope. Name `events_core` to read the truncated copy on purpose;
- `events_core` / `events_full` stay selectable by name, which is also the
- override for a shape the pass rejects.
+ is in scope. Read `events_truncated` to get the truncated copy on purpose
+ (input/output/metadata previews): it always compiles to `events_core`,
+ whatever columns it reads, and the pass does not reject `*` or raw SQL over
+ it. `events_core` / `events_full` are not part of the Kysely schema, and
+ naming them anywhere — FROM/JOIN, alias, column qualifier, raw SQL — is a
+ compile error; only the routing pass writes them.
- **Give each CTE (and view) its own plain name.** A CTE name is a plain
identifier that is not a registry table (`traces`, `events_core`, the
- logical `events`, …) and doesn't reuse the name of a sibling or an
+ logical `events` / `events_truncated`, …) and doesn't reuse the name of a sibling or an
enclosing CTE (a CTE is not in scope inside its own body). Every FROM/JOIN
name is an unqualified registry table or a CTE in scope; schema-qualified
names (`default.traces`) are rejected, since the registry describes tables
@@ -219,7 +222,7 @@ db.selectFrom("events")
// soft-delete pass so a deleted latest version is dropped
```
-**Write FINAL** (`useFinal` — ReplacingMergeTree reads; never on `events_core`):
+**Write FINAL** (`useFinal` — ReplacingMergeTree reads; never on the events relations):
```ts
db.selectFrom("scores as s")
diff --git a/packages/shared/src/server/query-ast/binds.test.ts b/packages/shared/src/server/query-ast/binds.test.ts
index e907418e0e..e45a934c90 100644
--- a/packages/shared/src/server/query-ast/binds.test.ts
+++ b/packages/shared/src/server/query-ast/binds.test.ts
@@ -40,7 +40,7 @@ describe("typed params from the column registry", () => {
it("binds a number compared to total_cost as Decimal(18, 12)", () => {
const { sql } = compile(
getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select("span_id")
.where("total_cost", ">", 1),
);
@@ -50,7 +50,7 @@ describe("typed params from the column registry", () => {
it("wraps an IN list against an Array column as Array(Array(...))", () => {
const { sql } = compile(
getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select("span_id")
.where("metadata_names", "in", [["a"], ["b"]]),
);
@@ -63,7 +63,7 @@ describe("typed params from the column registry", () => {
it("inlines an IN list of literals", () => {
const { sql, params } = compile(
getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select("span_id")
.where("name", "in", [literal("a"), literal("it's")]),
);
@@ -74,7 +74,7 @@ describe("typed params from the column registry", () => {
it("binds an IN list of value expressions as one array", () => {
const { sql } = compile(
getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select("span_id")
.where((eb) => eb("name", "in", [eb.val("a"), eb.val("b")])),
);
@@ -85,7 +85,7 @@ describe("typed params from the column registry", () => {
expect(() =>
compile(
getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select("span_id")
.where((eb) => eb("name", "in", [literal("a"), eb.val("b")])),
),
@@ -95,7 +95,7 @@ describe("typed params from the column registry", () => {
it("keeps the column's DateTime precision through toStartOfMinute / toDate", () => {
const { sql: minuteSql } = compile(
getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select("span_id")
.where((eb) =>
eb(
@@ -131,7 +131,7 @@ describe("typed params from the column registry", () => {
it("binds a Date as the UTC wall-clock string ingestion writes", () => {
const { sql, params } = compile(
getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select("span_id")
.where("start_time", ">=", new Date(-1000))
.where("start_time", "<=", START),
@@ -148,7 +148,7 @@ describe("typed params from the column registry", () => {
it("binds a shared column name with the type of the table it is read from", () => {
const { sql } = compile(
getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.select("e.span_id")
.where("e.start_time", ">=", START)
.where("e.trace_id", "in", (qb) =>
@@ -178,7 +178,7 @@ describe("typed params from the column registry", () => {
it("resolves a qualifier declared in an enclosing select", () => {
const { sql } = compile(
getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.select("e.span_id")
.where((eb) =>
eb.exists(
@@ -199,7 +199,7 @@ describe("typed params from the column registry", () => {
() =>
getClickhouseKysely()
.with("recent", (db) =>
- db.selectFrom("events_core as e").select("e.start_time as ts"),
+ db.selectFrom("events as e").select("e.start_time as ts"),
)
.selectFrom("recent as r")
.select("r.ts")
@@ -209,9 +209,7 @@ describe("typed params from the column registry", () => {
"a CTE reading an earlier CTE",
() =>
getClickhouseKysely()
- .with("first", (db) =>
- db.selectFrom("events_core").select("start_time"),
- )
+ .with("first", (db) => db.selectFrom("events").select("start_time"))
.with("second", (db) => db.selectFrom("first").selectAll())
.selectFrom("second")
.select("start_time")
@@ -222,7 +220,7 @@ describe("typed params from the column registry", () => {
() =>
getClickhouseKysely()
.selectFrom((db) =>
- db.selectFrom("events_core").select("start_time").as("sub"),
+ db.selectFrom("events").select("start_time").as("sub"),
)
.select("sub.start_time")
.where("sub.start_time", ">=", START),
@@ -233,7 +231,7 @@ describe("typed params from the column registry", () => {
getClickhouseKysely()
.with("bounds", (db) =>
db
- .selectFrom("events_core")
+ .selectFrom("events")
.select((eb) => eb.fn.max("start_time").as("start_time")),
)
.selectFrom("bounds")
@@ -246,7 +244,7 @@ describe("typed params from the column registry", () => {
getClickhouseKysely()
.with("bounds", (db) =>
db
- .selectFrom("events_core")
+ .selectFrom("events")
.select((eb) =>
eb.fn<Date>("max", ["start_time"]).as("start_time"),
),
@@ -261,7 +259,7 @@ describe("typed params from the column registry", () => {
getClickhouseKysely()
.with("latest", (db) =>
db
- .selectFrom("events_core")
+ .selectFrom("events")
.select((eb) =>
eb
.fn<Date>("argMax", ["start_time", "event_ts"])
@@ -278,7 +276,7 @@ describe("typed params from the column registry", () => {
getClickhouseKysely()
.with("latest", (db) =>
db
- .selectFrom("events_core")
+ .selectFrom("events")
.select((eb) => eb.fn<Date>("anyLast", ["start_time"]).as("ts")),
)
.selectFrom("latest")
@@ -290,7 +288,7 @@ describe("typed params from the column registry", () => {
() =>
getClickhouseKysely()
.with("rows", (db) =>
- db.selectFrom("events_core as e").selectAll("e"),
+ db.selectFrom("events_truncated as e").selectAll("e"),
)
.selectFrom("rows")
.select("start_time")
@@ -307,7 +305,7 @@ describe("typed params from the column registry", () => {
describe("CTE name resolution", () => {
const db = getClickhouseKysely();
const withX = db.with("x", (qb) =>
- qb.selectFrom("events_core").select("start_time"),
+ qb.selectFrom("events").select("start_time"),
);
it("types a subquery's read of an enclosing CTE from its body", () => {
@@ -333,11 +331,11 @@ describe("typed params from the column registry", () => {
db
.with("y", (qb) =>
qb
- .selectFrom("events_core")
+ .selectFrom("events")
.select("span_id")
.where(db.dynamic.ref("o.start_time"), ">=", START),
)
- .selectFrom("events_core as o")
+ .selectFrom("events as o")
.select("o.span_id"),
),
).toThrow(/no relation in scope is named "o"/);
@@ -347,7 +345,7 @@ describe("typed params from the column registry", () => {
it("propagates the source column type through a JOIN to an aliased subquery", () => {
const db = getClickhouseKysely();
const sub = db
- .selectFrom("events_core")
+ .selectFrom("events")
.select(["trace_id", "start_time"])
.as("sub");
const { sql } = compile(
@@ -364,7 +362,7 @@ describe("typed params from the column registry", () => {
it("binds a select alias with the type of the column it names", () => {
const { sql } = compile(
getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.innerJoin("observations as o", "o.trace_id", "e.trace_id")
.select((eb) => eb.fn.max("e.start_time").as("start_time"))
.groupBy("e.trace_id")
@@ -389,7 +387,7 @@ describe("typed params from the column registry", () => {
it("types a comparison inside an aggregate aliased to the column's name", () => {
const { sql } = compile(
getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select((eb) => countIf(eb("total_cost", ">", 1)).as("total_cost"))
.groupBy("trace_id"),
);
@@ -414,7 +412,7 @@ describe("typed params from the column registry", () => {
it("binds an ARRAY JOIN alias by value, not as the column it shadows", () => {
const { sql } = compile(
getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.select("e.span_id")
.$call(arrayJoin((eb) => ({ start_time: eb.ref("e.metadata_names") })))
.where(sql_.ref("start_time"), "=", "x"),
@@ -426,7 +424,7 @@ describe("typed params from the column registry", () => {
it("binds a tuple element of an ARRAY JOIN alias by value", () => {
const { sql } = compile(
getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.select("e.span_id")
.$call(arrayJoin((eb) => ({ kv: eb.ref("e.metadata_names") })))
.where(sql_.ref("kv.1"), "=", "a"),
@@ -438,7 +436,7 @@ describe("typed params from the column registry", () => {
it("binds a raw expression around a sql.ref column by value", () => {
const { sql } = compile(
getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events_truncated as e")
.select("e.span_id")
.where(sql_<number>`length(${sql_.ref("e.input")})`, ">", 5),
);
@@ -500,7 +498,7 @@ describe("typed params from the column registry", () => {
db
.with("t", (qb) =>
qb
- .selectFrom("events_core")
+ .selectFrom("events_truncated")
.select(sql_`is_deleted`.as("is_deleted")),
)
.selectFrom("t")
@@ -514,7 +512,7 @@ describe("typed params from the column registry", () => {
it("binds by value a column the CTE does not output", () => {
const { sql } = compile(
db
- .with("t", (qb) => qb.selectFrom("events_core").select("span_id"))
+ .with("t", (qb) => qb.selectFrom("events").select("span_id"))
.selectFrom("t")
.select("span_id")
.where(db.dynamic.ref("is_deleted"), "=", 0),
@@ -526,7 +524,7 @@ describe("typed params from the column registry", () => {
it("compiles an unresolvable column compared to an inlined constant", () => {
const { sql } = compile(
db
- .selectFrom("events_core")
+ .selectFrom("events")
.select("span_id")
.where((eb) => eb(db.dynamic.ref("x.is_deleted"), "=", eb.lit(0))),
);
@@ -539,7 +537,7 @@ describe("typed params from the column registry", () => {
"a qualifier declared nowhere",
() =>
db
- .selectFrom("events_core")
+ .selectFrom("events")
.select("span_id")
.where(db.dynamic.ref("x.is_deleted"), "=", 0),
/no relation in scope is named "x"/,
@@ -549,7 +547,7 @@ describe("typed params from the column registry", () => {
"an unqualified column the joined relations type differently",
() =>
db
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.innerJoin("observations as o", "o.trace_id", "e.trace_id")
.select("e.span_id")
.where("start_time", ">=", START),
@@ -577,7 +575,7 @@ describe("typed params from the column registry", () => {
"a sql.ref column whose qualifier is declared nowhere",
() =>
db
- .selectFrom("events_core")
+ .selectFrom("events")
.select("span_id")
.where(sql_.ref("x.is_deleted"), "=", 0),
/no relation in scope is named "x"/,
@@ -587,7 +585,7 @@ describe("typed params from the column registry", () => {
() =>
db
.with("t", (qb) =>
- qb.selectFrom("events_core as e").selectAll("x" as never),
+ qb.selectFrom("events as e").selectAll("x" as never),
)
.selectFrom("t as t")
.select(db.dynamic.ref("t.span_id"))
@@ -601,7 +599,7 @@ describe("typed params from the column registry", () => {
() =>
db
.with("t", (qb) =>
- qb.selectFrom("events_core as e").selectAll("x" as never),
+ qb.selectFrom("events as e").selectAll("x" as never),
)
.selectFrom("scores as s")
.innerJoin("t", (join) => join.onTrue())
@@ -614,7 +612,7 @@ describe("typed params from the column registry", () => {
"a TupleParam whose tuple has a column declared nowhere",
() =>
db
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.select("e.span_id")
.where((eb) =>
tupleIn(eb.refTuple(db.dynamic.ref("x.a"), "e.span_id"), [
@@ -628,7 +626,7 @@ describe("typed params from the column registry", () => {
"a tuple of values against a tuple with a column declared nowhere",
() =>
db
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.select("e.span_id")
.where((eb) =>
eb(eb.refTuple(db.dynamic.ref("x.a"), "e.span_id"), "in", [
@@ -642,7 +640,7 @@ describe("typed params from the column registry", () => {
"an empty TupleParam list whose tuple has a column declared nowhere",
() =>
db
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.select("e.span_id")
.where((eb) =>
tupleIn(eb.refTuple(db.dynamic.ref("x.a"), "e.span_id"), []),
@@ -654,7 +652,7 @@ describe("typed params from the column registry", () => {
"an IN list against a qualifier declared nowhere",
() =>
db
- .selectFrom("events_core")
+ .selectFrom("events")
.select("span_id")
.where(db.dynamic.ref("x.trace_id"), "in", ["a", "b"]),
/no relation in scope is named "x"/,
@@ -670,7 +668,7 @@ describe("typed params from the column registry", () => {
getClickhouseKysely()
.with("totals", (db) =>
db
- .selectFrom("events_core")
+ .selectFrom("events")
.select((eb) => eb.fn.count("prompt_version").as("prompt_version")),
)
.selectFrom("totals")
@@ -683,7 +681,7 @@ describe("typed params from the column registry", () => {
it("does not type extra date-wrapper arguments as the column", () => {
const { sql } = compile(
getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select("span_id")
.where((eb) =>
eb(
@@ -699,7 +697,7 @@ describe("typed params from the column registry", () => {
it("does not type a subquery LIMIT from an outer column comparison", () => {
const { sql } = compile(
getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select("trace_id")
.where(
"trace_id",
@@ -714,7 +712,7 @@ describe("typed params from the column registry", () => {
it("binds a TupleParam list as Array(Tuple(...)) for a tuple IN", () => {
const { sql, params } = compile(
getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.select("e.span_id")
.where((eb) =>
tupleIn(eb.refTuple("e.trace_id", "e.span_id"), [
@@ -776,7 +774,9 @@ describe("typed params from the column registry", () => {
const db = getClickhouseKysely();
const { sql } = compile(
db
- .with("t", (qb) => qb.selectFrom("events_core as e").selectAll("e"))
+ .with("t", (qb) =>
+ qb.selectFrom("events_truncated as e").selectAll("e"),
+ )
.selectFrom("t as t")
.select(db.dynamic.ref("t.span_id"))
.where(db.dynamic.ref("t.start_time"), ">=", START),
@@ -791,7 +791,7 @@ describe("typed params from the column registry", () => {
db
.with("t", (qb) =>
qb
- .selectFrom("events_core as e")
+ .selectFrom("events_truncated as e")
.innerJoin("observations as o", "o.trace_id", "e.trace_id")
.selectAll(),
)
@@ -815,7 +815,7 @@ describe("typed params from the column registry", () => {
it("binds distinct bare TupleParams as separate Tuple(...) placeholders", () => {
const { sql, params } = compile(
getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select((eb) => [
eb.val(new TupleParam(["a", 1])).as("first"),
eb.val(new TupleParam(["b", 2])).as("second"),
diff --git a/packages/shared/src/server/query-ast/composition.test.ts b/packages/shared/src/server/query-ast/composition.test.ts
index 92f3ae92e3..fda5d3a9d1 100644
--- a/packages/shared/src/server/query-ast/composition.test.ts
+++ b/packages/shared/src/server/query-ast/composition.test.ts
@@ -20,7 +20,7 @@ const costArrayJoin = () =>
const dedup = () =>
limitBy({
count: 1,
- columns: (eb: ExpressionBuilder<ClickHouseDatabase, "events_core">) => [
+ columns: (eb: ExpressionBuilder<ClickHouseDatabase, "events">) => [
eb.ref("span_id"),
eb.ref("project_id"),
],
@@ -105,7 +105,7 @@ describe("LIMIT BY composition across nesting positions", () => {
it("emits LIMIT BY at the top level (baseline)", () => {
const db = getClickhouseKysely();
const qb = db
- .selectFrom("events_core")
+ .selectFrom("events")
.select(["span_id", "project_id"])
.orderBy("event_ts", "desc")
.$call(dedup());
@@ -119,7 +119,7 @@ describe("LIMIT BY composition across nesting positions", () => {
const qb = db
.with("lb", (qb) =>
qb
- .selectFrom("events_core")
+ .selectFrom("events")
.select(["span_id", "project_id"])
.orderBy("event_ts", "desc")
.$call(dedup()),
@@ -172,7 +172,7 @@ describe("FINAL composition across nesting positions", () => {
it("keeps FINAL on a JOIN table and not on events_core", () => {
const db = getClickhouseKysely();
const qb = db
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.innerJoin("scores as s", (join) =>
join
.onRef("e.trace_id", "=", "s.trace_id")
diff --git a/packages/shared/src/server/query-ast/dedup.test.ts b/packages/shared/src/server/query-ast/dedup.test.ts
index 0d1a07a404..1d226d8e01 100644
--- a/packages/shared/src/server/query-ast/dedup.test.ts
+++ b/packages/shared/src/server/query-ast/dedup.test.ts
@@ -22,7 +22,7 @@ describe("per-table dedup lowering", () => {
it("does not inject LIMIT BY or FINAL on events_core row reads", () => {
const { sql } = compile(
getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select(["span_id", "project_id"]),
);
const lower = sql.toLowerCase();
@@ -33,7 +33,7 @@ describe("per-table dedup lowering", () => {
it("does not wrap events_core DISTINCT", () => {
const { sql } = compile(
getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select("environment")
.distinct(),
);
@@ -45,7 +45,7 @@ describe("per-table dedup lowering", () => {
it("does not wrap events_core aggregations", () => {
const { sql } = compile(
getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select((eb) => ["environment", eb.fn.countAll().as("n")])
.groupBy("environment"),
);
@@ -58,7 +58,7 @@ describe("per-table dedup lowering", () => {
it("still emits an explicit caller LIMIT BY", () => {
const { sql } = compile(
getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select("span_id")
.orderBy("event_ts", "desc")
.$call(
diff --git a/packages/shared/src/server/query-ast/eventsRouting.test.ts b/packages/shared/src/server/query-ast/eventsRouting.test.ts
index 36f8b668a1..9ffbc3baa0 100644
--- a/packages/shared/src/server/query-ast/eventsRouting.test.ts
+++ b/packages/shared/src/server/query-ast/eventsRouting.test.ts
@@ -2,6 +2,7 @@ import {
AliasNode,
FromNode,
SelectQueryNode,
+ type Kysely,
sql,
type Expression,
type ExpressionBuilder,
@@ -20,7 +21,6 @@ import { compileClickhouseQuery } from "./compile";
import { getClickhouseKysely } from "./dialect";
import { QueryCompileError } from "./errors";
import {
- includeDeleted,
limitBy,
mapValue,
metadataMap,
@@ -47,6 +47,11 @@ function routedTables(query: string): string[] {
const db = getClickhouseKysely();
+/** The compile DB without its table types, to reach names the types forbid. */
+const untypedDb = db as unknown as Kysely<
+ Record<string, Record<string, string>>
+>;
+
type EventsEb = ExpressionBuilder<
ClickHouseDatabase & { e: ClickHouseDatabase["events"] },
"e"
@@ -224,19 +229,6 @@ describe("logical events routing", () => {
).toThrow(QueryCompileError);
});
- it.each([
- "events_core.input",
- "default.events_core.input",
- "events_full.input",
- ])("rejects the physical qualifier %s over a logical relation", (ref) => {
- // ClickHouse resolves a qualifier by table name even when the table is aliased.
- expect(() =>
- compile(
- db.selectFrom("events as e").select(sql.ref<string>(ref).as("x")),
- ),
- ).toThrow(QueryCompileError);
- });
-
it("rejects FINAL around the logical relation", () => {
// useFinal() refuses `events` (dedup: none), so FINAL is wrapped by hand;
// routing would otherwise swap the table and drop the FINAL silently.
@@ -259,17 +251,64 @@ describe("logical events routing", () => {
).toThrow(/FINAL cannot wrap the logical "events" relation/);
});
- it("keeps a physical events qualifier when no logical relation is in scope", () => {
- expect(
- routedTables(
- compile(
- db
- .selectFrom("events_core as c")
- .$call(includeDeleted())
- .select(sql.ref<string>("events_core.input").as("x")),
- ),
- ),
- ).toEqual(["events_core"]);
+ it.each<[string, () => Parameters<typeof compile>[0]]>([
+ ["FROM", () => untypedDb.selectFrom("events_core").select("span_id")],
+ [
+ "JOIN",
+ () =>
+ untypedDb
+ .selectFrom("scores as s")
+ .innerJoin("events_full as f", "f.span_id", "s.id")
+ .select("s.id"),
+ ],
+ [
+ "alias",
+ () => untypedDb.selectFrom("events as events_core").select("span_id"),
+ ],
+ [
+ // ClickHouse resolves a qualifier by table name even when the table is aliased.
+ "column qualifier",
+ () =>
+ db
+ .selectFrom("scores as s")
+ .select(sql.ref<string>("events_core.input").as("x")),
+ ],
+ [
+ "qualifier in a core-safe metadata predicate",
+ () =>
+ db
+ .selectFrom("events as e")
+ .select("e.span_id")
+ .where((eb) =>
+ eb(
+ metadataValue(
+ eb.ref("events_full.metadata_names" as never),
+ eb.ref("events_full.metadata_values" as never),
+ "k",
+ ),
+ "=",
+ "x",
+ ),
+ ),
+ ],
+ [
+ "raw SQL text",
+ () =>
+ db
+ .selectFrom("scores as s")
+ .select(sql<number>`(select count() from events_full)`.as("n")),
+ ],
+ [
+ "sql.table",
+ () =>
+ db
+ .selectFrom("scores as s")
+ .select(sql<string>`${sql.table("events_core")}.input`.as("x")),
+ ],
+ ])("rejects a physical events table named as %s", (_, build) => {
+ expect(() => compile(build())).toThrow(
+ /Read the logical "events" relation instead of "events_(core|full)"/,
+ );
});
it("resolves sql.ref like a builder reference", () => {
@@ -463,17 +502,38 @@ describe("logical events routing", () => {
).toEqual(["events_core", "events_full"]);
});
- it("leaves physical events tables unchanged", () => {
+ it("routes events_truncated to events_core, also when it reads a truncated column", () => {
+ expect(
+ compile(
+ db
+ .selectFrom("events_truncated as e")
+ .select(["e.span_id", "e.input"])
+ .where((eb) => eb(eventsMetadata(eb, "k"), "!=", "x")),
+ ),
+ ).toMatchInlineSnapshot(
+ `"select e.span_id, e.input from events_core as e where e.project_id = {p1:String} and e.is_deleted = 0 and mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values))[{p2:String}] != {p3:String}"`,
+ );
+ });
+
+ it("keeps an unaliased events_truncated name as the alias", () => {
+ expect(
+ compile(db.selectFrom("events_truncated").select("input")),
+ ).toMatchInlineSnapshot(
+ `"select input from events_core as events_truncated where events_truncated.project_id = {p1:String} and events_truncated.is_deleted = 0"`,
+ );
+ });
+
+ it("routes events and events_truncated in one query separately", () => {
expect(
routedTables(
compile(
db
- .selectFrom("events_core as e")
- .$call(includeDeleted())
- .select("e.input"),
+ .selectFrom("events as a")
+ .innerJoin("events_truncated as b", "a.span_id", "b.span_id")
+ .select(["a.input", "b.output"]),
),
),
- ).toEqual(["events_core"]);
+ ).toEqual(["events_full", "events_core"]);
});
});
diff --git a/packages/shared/src/server/query-ast/eventsRouting.ts b/packages/shared/src/server/query-ast/eventsRouting.ts
index c0681dd1b9..6354b06c8e 100644
--- a/packages/shared/src/server/query-ast/eventsRouting.ts
+++ b/packages/shared/src/server/query-ast/eventsRouting.ts
@@ -1,5 +1,6 @@
/**
- * Logical `events` routing. Runs first in every compile.
+ * Logical events routing. Runs right after the CTE name check in every
+ * compile.
*
* Call sites read the logical `events` relation; this pass replaces each
* FROM/JOIN reference to it with `events_core` or `events_full`, one choice
@@ -18,15 +19,17 @@
* - `*`, `e.*` or `COLUMNS(…)` over a logical relation (`count(*)` reads no
* column);
* - an unqualified truncated column while a logical relation is in scope;
- * - a column qualified by `events_core` / `events_full` that names no
- * relation in scope: ClickHouse resolves a qualifier by table name even
- * when the table is aliased, so it would reach the routed relation;
* - raw SQL text or identifiers (`sql` templates, `sql.raw`, `sql.id`,
* `sql.table`) in or under a select that reads a logical relation.
* `sql.ref` and bound values resolve like builder references.
*
- * Callers that want the truncated copy on purpose name `events_core`;
- * physical `events_core` / `events_full` references pass through unchanged.
+ * Callers that want the truncated copy on purpose read `events_truncated`,
+ * which always routes to `events_core`, whatever columns it reads.
+ *
+ * Only this pass names the physical tables. A call site that does — as a
+ * FROM/JOIN table, an alias, a column qualifier, or in raw SQL — is
+ * rejected: ClickHouse resolves a qualifier by table name even when the
+ * table is aliased, so a physical name could reach a routed relation.
*/
import {
AggregateFunctionNode,
@@ -49,20 +52,24 @@ import {
type PluginTransformQueryArgs,
type PluginTransformResultArgs,
type QueryId,
- type ReferenceNode,
+ ReferenceNode,
type RootOperationNode,
type SelectQueryNode,
} from "kysely";
import { metadataFilterIsEventsCoreSafe } from "../queries/clickhouse-sql/events-core-truncation";
import { QueryCompileError } from "./errors";
-import { ArrayIndexNode, FinalTableNode } from "./nodes";
+import { ArrayIndexNode, FinalTableNode, walkNodes } from "./nodes";
import { describeRelation, selectRelations, type Relation } from "./relations";
-import { EVENTS_CORE_TRUNCATED_COLUMNS } from "./schema";
+import {
+ EVENTS_CORE_TRUNCATED_COLUMNS,
+ LOGICAL_EVENTS_TABLES,
+ PHYSICAL_EVENTS_TABLES,
+} from "./schema";
import { ClickHouseOperationNodeTransformer } from "./transformer";
const LOGICAL_EVENTS = "events";
-const PHYSICAL_EVENTS = new Set(["events_core", "events_full"]);
+const PHYSICAL_EVENT_NAME = /\bevents_(core|full)\b/;
export class EventsTableRoutingPlugin implements KyselyPlugin {
transformQuery(args: PluginTransformQueryArgs): RootOperationNode {
@@ -102,6 +109,9 @@ class EventsReadAnalysis extends ClickHouseOperationNodeTransformer {
): T {
const lookup = coreSafeMetadataLookup(node);
if (lookup) {
+ // The lookup's columns read no truncated value, but their qualifiers
+ // must still not name a physical table.
+ rejectPhysicalQualifiers(lookup.array);
this.transformNode(lookup.index, queryId);
return node;
}
@@ -112,8 +122,10 @@ class EventsReadAnalysis extends ClickHouseOperationNodeTransformer {
node: SelectQueryNode,
queryId?: QueryId,
): SelectQueryNode {
- const relations = selectRelations(node).map(routingRelation);
- if (relations.some((relation) => relation.logical)) {
+ const described = selectRelations(node);
+ described.forEach(rejectPhysicalRelation);
+ const relations = described.map(routingRelation);
+ if (relations.some((relation) => relation.routed)) {
this.readsLogicalEvents = true;
}
this.scopes.push(relations);
@@ -129,6 +141,7 @@ class EventsReadAnalysis extends ClickHouseOperationNodeTransformer {
_queryId?: QueryId,
): ReferenceNode {
const qualifier = node.table?.table.identifier.name;
+ rejectPhysicalName(qualifier);
// `sql.ref("e.*")` parses the star as a column named `*`.
if (SelectAllNode.is(node.column) || node.column.column.name === "*") {
if (qualifier === undefined) this.rejectStar("*");
@@ -191,6 +204,7 @@ class EventsReadAnalysis extends ClickHouseOperationNodeTransformer {
}
protected override transformRaw(node: RawNode, queryId?: QueryId): RawNode {
+ rejectPhysicalRawNames(node);
if (
this.readsLogicalInScope() &&
(node.sqlFragments.some((fragment) => fragment.trim() !== "") ||
@@ -217,11 +231,6 @@ class EventsReadAnalysis extends ClickHouseOperationNodeTransformer {
const match = relations.find((relation) => relation.name === qualifier);
if (match) return match;
}
- if (PHYSICAL_EVENTS.has(qualifier) && this.readsLogicalInScope()) {
- throw new QueryCompileError(
- `Qualify by the relation's alias, not "${qualifier}": ClickHouse also resolves a table name to an aliased relation, so it may reach the logical "${LOGICAL_EVENTS}" relation after routing.`,
- );
- }
return undefined;
}
@@ -251,6 +260,45 @@ class EventsReadAnalysis extends ClickHouseOperationNodeTransformer {
}
}
+function rejectPhysicalName(name: string | undefined): void {
+ if (name === undefined || !PHYSICAL_EVENTS_TABLES.has(name)) return;
+ throw new QueryCompileError(
+ `Read the logical "${LOGICAL_EVENTS}" relation instead of "${name}": the routing pass picks events_core or events_full from the columns read. Read "events_truncated" to get the truncated events_core copy on purpose.`,
+ );
+}
+
+/** Column qualifiers anywhere under `node`. */
+function rejectPhysicalQualifiers(node: OperationNode): void {
+ walkNodes(node, (child) => {
+ if (ReferenceNode.is(child)) {
+ rejectPhysicalName(child.table?.table.identifier.name);
+ }
+ });
+}
+
+/** Raw SQL text, and `sql.table` / `sql.id` parameters. */
+function rejectPhysicalRawNames(node: RawNode): void {
+ for (const fragment of node.sqlFragments) {
+ rejectPhysicalName(PHYSICAL_EVENT_NAME.exec(fragment)?.[0]);
+ }
+ for (const parameter of node.parameters) {
+ rejectPhysicalName(rawParameterName(parameter));
+ }
+}
+
+/** The name a `sql.table` / `sql.id` parameter writes into raw SQL. */
+function rawParameterName(node: OperationNode): string | undefined {
+ if (TableNode.is(node)) return node.table.identifier.name;
+ if (IdentifierNode.is(node)) return node.name;
+ return undefined;
+}
+
+/** A physical events table as FROM/JOIN table or alias. */
+function rejectPhysicalRelation({ table, alias }: Relation): void {
+ rejectPhysicalName(table?.table.identifier.name);
+ rejectPhysicalName(alias);
+}
+
function starError(star: string): QueryCompileError {
return new QueryCompileError(
`${star} cannot expand the logical "${LOGICAL_EVENTS}" relation: list the columns, so the routing pass can tell whether it reads a truncated one.`,
@@ -282,27 +330,25 @@ class EventsTableRouter extends ClickHouseOperationNodeTransformer {
}
/**
- * `events as e` → `events_core as e`. An unaliased `events` keeps its name
- * as the alias (`events_core as events`), so `events.col` references and
- * the qualifiers later passes add still resolve.
+ * `events as e` → `events_core as e`. An unaliased relation keeps its
+ * logical name as the alias (`events_core as events`), so `events.col`
+ * references and the qualifiers later passes add still resolve.
*/
private routeRelation(node: OperationNode, queryId?: QueryId): OperationNode {
const table = TableNode.create(
this.fullRelations.has(node) ? "events_full" : "events_core",
);
const { source, alias } = describeRelation(node);
- if (!isLogicalEventsTable(source)) {
+ const logicalName = logicalEventsTableName(source);
+ if (logicalName === undefined) {
return this.transformNode(node, queryId);
}
if (FinalTableNode.is(node)) {
throw new QueryCompileError(
- `FINAL cannot wrap the logical "${LOGICAL_EVENTS}" relation: events_core and events_full are immutable at read time. Drop the FINAL.`,
+ `FINAL cannot wrap the logical "${logicalName}" relation: events_core and events_full are immutable at read time. Drop the FINAL.`,
);
}
- return AliasNode.create(
- table,
- IdentifierNode.create(alias ?? LOGICAL_EVENTS),
- );
+ return AliasNode.create(table, IdentifierNode.create(alias ?? logicalName));
}
}
@@ -316,15 +362,20 @@ function routingRelation({
table,
alias,
}: Relation): RoutingRelation {
+ const logicalName = logicalEventsTableName(source);
return {
node,
name: alias ?? table?.table.identifier.name,
- logical: isLogicalEventsTable(source),
+ logical: logicalName === LOGICAL_EVENTS,
+ routed: logicalName !== undefined,
};
}
-function isLogicalEventsTable(node: OperationNode): boolean {
- return TableNode.is(node) && node.table.identifier.name === LOGICAL_EVENTS;
+/** `events` or `events_truncated`, when `node` reads one of them. */
+function logicalEventsTableName(node: OperationNode): string | undefined {
+ if (!TableNode.is(node)) return undefined;
+ const name = node.table.identifier.name;
+ return LOGICAL_EVENTS_TABLES.has(name) ? name : undefined;
}
/** `count(*)` with any `distinct`, `filter`, `over`, … clause. */
@@ -383,5 +434,8 @@ type RoutingRelation = {
node: OperationNode;
/** Alias, or the table name of an unaliased table. */
name?: string;
+ /** Reads `events`, so its truncated-column reads pick the table. */
logical: boolean;
+ /** Reads `events` or `events_truncated`, so the router replaces it. */
+ routed: boolean;
};
diff --git a/packages/shared/src/server/query-ast/extensions.test.ts b/packages/shared/src/server/query-ast/extensions.test.ts
index 7755e1800c..43fa3f801b 100644
--- a/packages/shared/src/server/query-ast/extensions.test.ts
+++ b/packages/shared/src/server/query-ast/extensions.test.ts
@@ -37,7 +37,7 @@ import { defineView, fromView } from "./views";
const projectId = "proj-1";
const eventsMetadata = (
- eb: ExpressionBuilder<{ e: ClickHouseDatabase["events_core"] }, "e">,
+ eb: ExpressionBuilder<{ e: ClickHouseDatabase["events"] }, "e">,
key: string,
) =>
metadataValue(eb.ref("e.metadata_names"), eb.ref("e.metadata_values"), key);
@@ -45,7 +45,7 @@ const eventsMetadata = (
describe("schema-typed selection (condition 7)", () => {
it("accepts sum() over a numeric column and emits SQL", () => {
const qb = getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select((eb) => [eb.fn.sum("total_cost").as("s")]);
const { sql: compiled } = compileClickhouseQuery(qb, projectId);
@@ -55,7 +55,7 @@ describe("schema-typed selection (condition 7)", () => {
it("rejects sum() over a String column at the runtime validation pass", () => {
const qb = getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select((eb) => [eb.fn.sum("environment").as("s")]);
expect(() => compileClickhouseQuery(qb, projectId)).toThrow(
@@ -85,7 +85,7 @@ describe("metadata subscript access (condition 7b)", () => {
// parameter-bound) from "the compiler rendered that tree correctly".
it("lowers metadata[key] to ArrayIndexNode + mapFromArrays FunctionNode, not RawNode", () => {
const qb = getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.select((eb) => [eventsMetadata(eb, "a").as("a")])
.where((eb) => eb(eventsMetadata(eb, "a"), "=", "x"));
@@ -107,7 +107,7 @@ describe("metadata subscript access (condition 7b)", () => {
it("emits subscript + bound key param in SELECT and WHERE", () => {
const qb = getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.select((eb) => [eventsMetadata(eb, "a").as("a")])
.where((eb) => eb(eventsMetadata(eb, "a"), "=", "x"));
@@ -124,7 +124,7 @@ describe("metadata subscript access (condition 7b)", () => {
it("emits the same lowering in HAVING", () => {
const qb = getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.select("environment")
.groupBy("environment")
.having((eb) => eb(eventsMetadata(eb, "a"), "=", "x"));
@@ -141,10 +141,7 @@ describe("metadata subscript access (condition 7b)", () => {
// by trusting the lookup's marker, so the lookup may only read the metadata
// columns of one relation.
describe("rejects a lookup over anything but one relation's metadata columns", () => {
- const eb = expressionBuilder<
- { e: ClickHouseDatabase["events_core"] },
- "e"
- >();
+ const eb = expressionBuilder<{ e: ClickHouseDatabase["events"] }, "e">();
it.each([
// Any expression, not only `input`: the pass doesn't walk the
// arguments, so a column inside `values` would escape routing.
@@ -187,8 +184,8 @@ describe("metadata subscript access (condition 7b)", () => {
it("rejects names and values from two relations", () => {
const joined = expressionBuilder<
{
- e: ClickHouseDatabase["events_core"];
- f: ClickHouseDatabase["events_core"];
+ e: ClickHouseDatabase["events"];
+ f: ClickHouseDatabase["events"];
},
"e" | "f"
>();
@@ -207,10 +204,7 @@ describe("virtual view (condition 8)", () => {
const environmentsView = defineView("environments_view")<{
environment: string;
}>(() =>
- getClickhouseKysely()
- .selectFrom("events_core")
- .select("environment")
- .distinct(),
+ getClickhouseKysely().selectFrom("events").select("environment").distinct(),
);
it("treats the view as a table and emits a WITH CTE", () => {
@@ -259,7 +253,7 @@ describe("ArrayIndexNode identity", () => {
describe("leftUTF8", () => {
it("inlines a positive integer length", () => {
const qb = getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.select((eb) => [leftUTF8(eb.ref("e.input"), 100).as("input")]);
const { sql: compiled } = compileClickhouseQuery(qb, projectId);
@@ -269,7 +263,7 @@ describe("leftUTF8", () => {
it.each([0, -1, 1.5, Number.NaN])("rejects length %s", (length) => {
expect(() =>
getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.select((eb) => [leftUTF8(eb.ref("e.input"), length).as("input")]),
).toThrow(QueryCompileError);
});
@@ -278,7 +272,7 @@ describe("leftUTF8", () => {
describe("arrayMap", () => {
it("names nested lambda parameters apart so the inner one does not shadow", () => {
const qb = getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.select((eb) => [
arrayMap<string, string[]>(
(name) =>
@@ -304,10 +298,7 @@ describe("arrayMap", () => {
describe("arrayExists", () => {
it("names nested lambda parameters apart so the inner one does not shadow", () => {
- const eb = expressionBuilder<
- { e: ClickHouseDatabase["events_core"] },
- "e"
- >();
+ const eb = expressionBuilder<{ e: ClickHouseDatabase["events"] }, "e">();
const nested = arrayExists<string>(
(name) =>
arrayExists<string>((tag) => eb(tag, "=", name), eb.ref("e.tags")),
@@ -321,7 +312,7 @@ describe("arrayExists", () => {
});
describe("renderClickhouseExpression", () => {
- const eb = expressionBuilder<{ e: ClickHouseDatabase["events_core"] }, "e">();
+ const eb = expressionBuilder<{ e: ClickHouseDatabase["events"] }, "e">();
const metadata = metadataMap(
eb.ref("e.metadata_names"),
eb.ref("e.metadata_values"),
@@ -340,7 +331,7 @@ describe("renderClickhouseExpression", () => {
it("rejects a subquery", () => {
const subquery = getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select("trace_id");
expect(() => renderClickhouseExpression(subquery)).toThrow(
UnscopedRelationError,
@@ -361,7 +352,7 @@ describe("renderClickhouseExpression", () => {
});
describe("renderClickhouseSelections", () => {
- const events = getClickhouseKysely().selectFrom("events_core as e");
+ const events = getClickhouseKysely().selectFrom("events as e");
it("keys each select by its output name, in select order", () => {
const rendered = renderClickhouseSelections(
@@ -410,7 +401,7 @@ describe("renderClickhouseSelections", () => {
expect(() =>
renderClickhouseSelections(
events.select((eb) =>
- eb.selectFrom("events_core as x").select("x.trace_id").as("t"),
+ eb.selectFrom("events as x").select("x.trace_id").as("t"),
),
),
).toThrow(UnscopedRelationError);
@@ -418,7 +409,7 @@ describe("renderClickhouseSelections", () => {
});
describe("64-bit results", () => {
- const eb = expressionBuilder<{ e: ClickHouseDatabase["events_core"] }, "e">();
+ const eb = expressionBuilder<{ e: ClickHouseDatabase["events"] }, "e">();
const isError = eb("e.level", "=", literal("ERROR"));
// ClickHouse 25.8+ (Langfuse v4 needs 25.12) sends (U)Int64 as JSON numbers.
@@ -456,7 +447,7 @@ describe("startsWith", () => {
it("binds a string prefix and inlines a literal one", () => {
const { sql, params } = compileClickhouseQuery(
getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.select("e.span_id")
.where((eb) => startsWith(eb.ref("e.name"), "caller"))
.where((eb) => startsWith(eb.ref("e.user_id"), literal("const"))),
diff --git a/packages/shared/src/server/query-ast/extensions.ts b/packages/shared/src/server/query-ast/extensions.ts
index 7001b760aa..8df187ed4a 100644
--- a/packages/shared/src/server/query-ast/extensions.ts
+++ b/packages/shared/src/server/query-ast/extensions.ts
@@ -845,7 +845,7 @@ class FinalPlugin implements KyselyPlugin {
/**
* `FROM <table> FINAL` / `JOIN <table> AS alias FINAL` as a `$call` step:
* `qb.$call(useFinal(["scores"]))`. Tables declared `dedup: none` (immutable
- * at read time, including `events_core`) cannot take FINAL.
+ * at read time, including the events relations) cannot take FINAL.
*/
export function useFinal(tables: ReadonlyArray<keyof ClickHouseDatabase>) {
for (const table of tables) {
diff --git a/packages/shared/src/server/query-ast/fixtures/catalog.ts b/packages/shared/src/server/query-ast/fixtures/catalog.ts
index 4f150fc040..c19325cc4a 100644
--- a/packages/shared/src/server/query-ast/fixtures/catalog.ts
+++ b/packages/shared/src/server/query-ast/fixtures/catalog.ts
@@ -41,7 +41,7 @@ function environmentsVariant(spec: {
}): ClickhouseCompilable {
if (spec.writeMode === "events") {
return db()
- .selectFrom("events_core")
+ .selectFrom("events")
.select("environment")
.distinct()
.$if(spec.fromTimestamp !== undefined, (qb) =>
@@ -70,11 +70,10 @@ export const CATALOG: CatalogEntry[] = [
tier: 0,
referenceSql: `
SELECT DISTINCT environment
- FROM events_core
- WHERE (project_id = {p1:String}) AND (events_core.is_deleted = 0)
+ FROM events_core AS events
+ WHERE (events.project_id = {p1:String}) AND (events.is_deleted = 0)
`,
- build: () =>
- db().selectFrom("events_core").select("environment").distinct(),
+ build: () => db().selectFrom("events").select("environment").distinct(),
},
{
id: "environments_legacy_union",
@@ -122,12 +121,12 @@ export const CATALOG: CatalogEntry[] = [
tier: 1,
referenceSql: `
SELECT DISTINCT environment
- FROM events_core
- WHERE (project_id = {p1:String}) AND (events_core.is_deleted = 0) AND (start_time >= {p2:DateTime64(6)})
+ FROM events_core AS events
+ WHERE (events.project_id = {p1:String}) AND (events.is_deleted = 0) AND (start_time >= {p2:DateTime64(6)})
`,
build: () =>
db()
- .selectFrom("events_core")
+ .selectFrom("events")
.select("environment")
.distinct()
.where("start_time", ">=", FROM_TS),
@@ -137,13 +136,13 @@ export const CATALOG: CatalogEntry[] = [
tier: 1,
referenceSql: `
SELECT environment, count(*) AS n
- FROM events_core
- WHERE (project_id = {p1:String}) AND (events_core.is_deleted = 0)
+ FROM events_core AS events
+ WHERE (events.project_id = {p1:String}) AND (events.is_deleted = 0)
GROUP BY environment
`,
build: () =>
db()
- .selectFrom("events_core")
+ .selectFrom("events")
.select((eb) => ["environment", eb.fn.countAll().as("n")])
.groupBy("environment"),
},
@@ -215,8 +214,8 @@ export const CATALOG: CatalogEntry[] = [
referenceSql: `
WITH prompt_events AS (
SELECT trace_id, span_id
- FROM events_core
- WHERE (project_id = {p1:String}) AND (events_core.is_deleted = 0)
+ FROM events_core AS events
+ WHERE (events.project_id = {p1:String}) AND (events.is_deleted = 0)
)
SELECT s.id
FROM scores AS s
@@ -228,7 +227,7 @@ export const CATALOG: CatalogEntry[] = [
build: () =>
db()
.with("prompt_events", (qb) =>
- qb.selectFrom("events_core").select(["trace_id", "span_id"]),
+ qb.selectFrom("events").select(["trace_id", "span_id"]),
)
.selectFrom("scores as s")
.select("s.id")
@@ -267,14 +266,14 @@ export const CATALOG: CatalogEntry[] = [
tier: 3,
referenceSql: `
SELECT span_id, project_id
- FROM events_core
- WHERE (project_id = {p1:String}) AND (events_core.is_deleted = 0)
+ FROM events_core AS events
+ WHERE (events.project_id = {p1:String}) AND (events.is_deleted = 0)
ORDER BY event_ts DESC
LIMIT 1 BY span_id, project_id
`,
build: () =>
db()
- .selectFrom("events_core")
+ .selectFrom("events")
.select(["span_id", "project_id"])
.orderBy("event_ts", "desc")
.$call(
@@ -289,12 +288,12 @@ export const CATALOG: CatalogEntry[] = [
tier: 4,
referenceSql: `
SELECT span_id, rank() OVER (PARTITION BY trace_id ORDER BY start_time DESC) AS rk
- FROM events_core
- WHERE (project_id = {p1:String}) AND (events_core.is_deleted = 0)
+ FROM events_core AS events
+ WHERE (events.project_id = {p1:String}) AND (events.is_deleted = 0)
`,
build: () =>
db()
- .selectFrom("events_core")
+ .selectFrom("events")
.select((eb) => [
"span_id",
eb.fn
@@ -324,14 +323,14 @@ export const CATALOG: CatalogEntry[] = [
`,
build: () =>
db()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.select("e.span_id")
.where((eb) =>
eb(
"e.event_ts",
"=",
eb
- .selectFrom("events_core as inner")
+ .selectFrom("events as inner")
.select((ieb) => ieb.fn.max("inner.event_ts").as("m"))
.whereRef("inner.trace_id", "=", "e.trace_id"),
),
@@ -351,7 +350,7 @@ export const CATALOG: CatalogEntry[] = [
`,
build: () =>
db()
- .selectFrom("events_core as e")
+ .selectFrom("events_truncated as e")
.select((eb) => [
metadataValue(
eb.ref("e.metadata_names"),
@@ -390,8 +389,8 @@ export const CATALOG: CatalogEntry[] = [
tier: 7,
referenceSql: `
SELECT DISTINCT environment
- FROM events_core
- WHERE (project_id = {p1:String}) AND (events_core.is_deleted = 0)
+ FROM events_core AS events
+ WHERE (events.project_id = {p1:String}) AND (events.is_deleted = 0)
`,
build: () => environmentsVariant({ writeMode: "events" }),
},
@@ -401,8 +400,8 @@ export const CATALOG: CatalogEntry[] = [
tier: 7,
referenceSql: `
SELECT DISTINCT environment
- FROM events_core
- WHERE (project_id = {p1:String}) AND (events_core.is_deleted = 0) AND (start_time >= {p2:DateTime64(6)})
+ FROM events_core AS events
+ WHERE (events.project_id = {p1:String}) AND (events.is_deleted = 0) AND (start_time >= {p2:DateTime64(6)})
`,
build: () =>
environmentsVariant({ writeMode: "events", fromTimestamp: FROM_TS }),
diff --git a/packages/shared/src/server/query-ast/nodes.test.ts b/packages/shared/src/server/query-ast/nodes.test.ts
index 08c67f24f8..0e8abb5e75 100644
--- a/packages/shared/src/server/query-ast/nodes.test.ts
+++ b/packages/shared/src/server/query-ast/nodes.test.ts
@@ -58,9 +58,7 @@ describe("tenancy injection", () => {
});
it("refuses a scope that is not a single projectId string", () => {
- const qb = getClickhouseKysely()
- .selectFrom("events_core")
- .select("environment");
+ const qb = getClickhouseKysely().selectFrom("events").select("environment");
expect(() =>
// @ts-expect-error - a query is scoped to exactly one project
@@ -82,9 +80,7 @@ describe("tenancy injection", () => {
});
it("injects project_id when the builder omitted it", () => {
- const qb = getClickhouseKysely()
- .selectFrom("events_core")
- .select("environment");
+ const qb = getClickhouseKysely().selectFrom("events").select("environment");
const { sql: compiled, params } = compileClickhouseQuery(qb, projectId);
expect(compiled.toLowerCase()).toContain("project_id");
@@ -93,7 +89,7 @@ describe("tenancy injection", () => {
it("does not duplicate project_id when the builder already scoped the table", () => {
const qb = getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select("environment")
.where("project_id", "=", "proj-1");
@@ -148,7 +144,7 @@ describe("tenancy injection", () => {
"rejects the %s table function as a JOIN source",
(fn) => {
const qb = getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.innerJoin(
(eb) =>
eb
@@ -200,13 +196,13 @@ describe("tenancy injection", () => {
it("rejects raw SQL fragments in SELECT and WHERE, not only FROM/JOIN", () => {
const selectRaw = getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events_truncated")
.select(
sql<string>`(SELECT environment FROM traces LIMIT 1)`.as("stolen"),
);
const whereRaw = getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events_truncated")
.select("environment")
.where(sql<boolean>`id IN (SELECT id FROM traces)`);
@@ -282,7 +278,7 @@ describe("tenancy injection", () => {
it("injects the context project even when the builder scoped a different project", () => {
const qb = getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select("environment")
.where("project_id", "=", "other-project");
@@ -737,7 +733,7 @@ describe("ARRAY JOIN and LIMIT BY nodes", () => {
it("attaches a real LimitByNode", () => {
const qb = getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select("span_id")
.where("project_id", "=", "proj-1")
.$call(
@@ -777,7 +773,7 @@ describe("ARRAY JOIN and LIMIT BY nodes", () => {
it("emits the LIMIT BY count", () => {
const qb = getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select("span_id")
.$call(limitBy({ count: 2, columns: (eb) => [eb.ref("span_id")] }));
@@ -787,7 +783,7 @@ describe("ARRAY JOIN and LIMIT BY nodes", () => {
it("emits LIMIT BY before a plain LIMIT", () => {
const qb = getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select("span_id")
.where("project_id", "=", "proj-1")
.limit(50)
@@ -828,8 +824,8 @@ describe("ARRAY JOIN and LIMIT BY nodes", () => {
});
it("rejects FINAL on events_core", () => {
- expect(() => useFinal(["events_core"])).toThrow(QueryCompileError);
- expect(() => useFinal(["events_core"])).toThrow(/immutable/);
+ expect(() => useFinal(["events"])).toThrow(QueryCompileError);
+ expect(() => useFinal(["events"])).toThrow(/immutable/);
});
});
diff --git a/packages/shared/src/server/query-ast/schema.test.ts b/packages/shared/src/server/query-ast/schema.test.ts
index 4f43bc13c9..c25c6c2087 100644
--- a/packages/shared/src/server/query-ast/schema.test.ts
+++ b/packages/shared/src/server/query-ast/schema.test.ts
@@ -13,6 +13,7 @@ describe("table registry derivation", () => {
"events",
"events_core",
"events_full",
+ "events_truncated",
"observations",
"scores",
"traces",
diff --git a/packages/shared/src/server/query-ast/schema.ts b/packages/shared/src/server/query-ast/schema.ts
index ac90b7ba0a..ba446f339d 100644
--- a/packages/shared/src/server/query-ast/schema.ts
+++ b/packages/shared/src/server/query-ast/schema.ts
@@ -308,6 +308,14 @@ const TABLE_REGISTRY = {
dedup: { strategy: "none" },
softDelete: "is_deleted",
}),
+ // Logical relation for reading the truncated copy on purpose: routing
+ // always replaces it with events_core, also when it reads a truncated
+ // column (input/output/metadata previews).
+ events_truncated: defineTable({
+ columns: EVENTS_COLUMNS,
+ dedup: { strategy: "none" },
+ softDelete: "is_deleted",
+ }),
scores: defineTable({
columns: {
id: "String",
@@ -341,17 +349,33 @@ type RowOf<Cols extends Record<string, ChColumnType>> = {
[C in keyof Cols]: TsTypeOfColumn<Cols[C]>;
};
-/** Derived Kysely row types. */
+/**
+ * Relations only the routing pass may name. Call sites read the logical
+ * `events` / `events_truncated` instead.
+ */
+export const PHYSICAL_EVENTS_TABLES: ReadonlySet<string> = new Set([
+ "events_core",
+ "events_full",
+] satisfies ReadonlyArray<keyof typeof TABLE_REGISTRY>);
+
+/** Logical relations the routing pass replaces with a physical events table. */
+export const LOGICAL_EVENTS_TABLES: ReadonlySet<string> = new Set([
+ "events",
+ "events_truncated",
+] satisfies ReadonlyArray<keyof typeof TABLE_REGISTRY>);
+
+/** Derived Kysely row types, without the physical events tables. */
export type ClickHouseDatabase = {
- [T in keyof typeof TABLE_REGISTRY]: RowOf<
- (typeof TABLE_REGISTRY)[T]["columns"]
- >;
+ [T in Exclude<
+ keyof typeof TABLE_REGISTRY,
+ "events_core" | "events_full"
+ >]: RowOf<(typeof TABLE_REGISTRY)[T]["columns"]>;
};
/**
* Declared ClickHouse type per column, per physical registry table. A
* servertest checks it against `system.columns` of the migrated database. The
- * logical `events` relation has no table of its own; its columns are
+ * logical events relations have no table of their own; their columns are
* `events_core`'s and `events_full`'s.
*/
export const TABLE_COLUMN_TYPES: Record<
@@ -359,7 +383,7 @@ export const TABLE_COLUMN_TYPES: Record<
Record<string, ChColumnType>
> = lookupTable(
Object.entries(TABLE_REGISTRY)
- .filter(([table]) => table !== "events")
+ .filter(([table]) => !LOGICAL_EVENTS_TABLES.has(table))
.map(([table, spec]) => [table, spec.columns]),
);
@@ -377,7 +401,7 @@ export const COLUMN_DATA_TYPES: Record<string, ColumnDataType> = lookupTable(
),
);
-/** Every registry relation name, the logical `events` included. */
+/** Every registry relation name, the logical events relations included. */
export const REGISTRY_TABLES: ReadonlySet<string> = new Set(
Object.keys(TABLE_REGISTRY),
);
diff --git a/packages/shared/src/server/query-ast/softDelete.test.ts b/packages/shared/src/server/query-ast/softDelete.test.ts
index 3eafc32088..9249b2ac9a 100644
--- a/packages/shared/src/server/query-ast/softDelete.test.ts
+++ b/packages/shared/src/server/query-ast/softDelete.test.ts
@@ -22,6 +22,7 @@ describe("soft-delete filtering", () => {
events: "is_deleted",
events_core: "is_deleted",
events_full: "is_deleted",
+ events_truncated: "is_deleted",
});
});
@@ -29,11 +30,11 @@ describe("soft-delete filtering", () => {
expect(
compile(
getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.select((eb) => ["span_id", eb.lit(0).as("is_deleted")]),
),
).toMatchInlineSnapshot(
- `"select span_id, 0 as is_deleted from events_core where project_id = {p1:String} and events_core.is_deleted = 0"`,
+ `"select span_id, 0 as is_deleted from events_core as events where events.project_id = {p1:String} and events.is_deleted = 0"`,
);
});
@@ -41,12 +42,12 @@ describe("soft-delete filtering", () => {
expect(
compile(
getClickhouseKysely()
- .selectFrom("events_full as e")
+ .selectFrom("events as e")
.select("e.span_id")
.where("e.trace_id", "=", "t"),
),
).toMatchInlineSnapshot(
- `"select e.span_id from events_full as e where e.project_id = {p1:String} and e.is_deleted = 0 and e.trace_id = {p2:String}"`,
+ `"select e.span_id from events_core as e where e.project_id = {p1:String} and e.is_deleted = 0 and e.trace_id = {p2:String}"`,
);
});
@@ -55,7 +56,7 @@ describe("soft-delete filtering", () => {
compile(
getClickhouseKysely()
.selectFrom("scores as s")
- .leftJoin("events_core as e", (join) =>
+ .leftJoin("events as e", (join) =>
join.onRef("e.trace_id", "=", "s.trace_id"),
)
.select("s.id"),
@@ -70,7 +71,7 @@ describe("soft-delete filtering", () => {
compile(
getClickhouseKysely()
.selectFrom("scores as s")
- .crossJoin("events_core as e")
+ .crossJoin("events as e")
.select("s.id"),
),
).toMatch(/ where s\.project_id = \{p1:String\} and e\.is_deleted = 0$/);
@@ -81,19 +82,19 @@ describe("soft-delete filtering", () => {
expect(
compile(
db
- .with("ev", (qb) => qb.selectFrom("events_core").select("trace_id"))
+ .with("ev", (qb) => qb.selectFrom("events").select("trace_id"))
.selectFrom("scores as s")
.select("s.id")
.where("s.trace_id", "in", (eb) =>
- eb.selectFrom("events_full").select("trace_id"),
+ eb.selectFrom("events").select("trace_id"),
)
.where("s.trace_id", "in", (eb) =>
eb.selectFrom("ev").select("trace_id"),
)
- .unionAll(db.selectFrom("events_core").select("span_id as id")),
+ .unionAll(db.selectFrom("events").select("span_id as id")),
),
).toMatchInlineSnapshot(
- `"with ev as (select trace_id from events_core where project_id = {p1:String} and events_core.is_deleted = 0) select s.id from scores as s where s.project_id = {p1:String} and s.trace_id in (select trace_id from events_full where project_id = {p1:String} and events_full.is_deleted = 0) and s.trace_id in (select trace_id from ev) union all select span_id as id from events_core where project_id = {p1:String} and events_core.is_deleted = 0"`,
+ `"with ev as (select trace_id from events_core as events where events.project_id = {p1:String} and events.is_deleted = 0) select s.id from scores as s where s.project_id = {p1:String} and s.trace_id in (select trace_id from events_core as events where events.project_id = {p1:String} and events.is_deleted = 0) and s.trace_id in (select trace_id from ev) union all select span_id as id from events_core as events where events.project_id = {p1:String} and events.is_deleted = 0"`,
);
});
@@ -102,18 +103,18 @@ describe("soft-delete filtering", () => {
expect(
compile(
db
- .selectFrom(db.selectFrom("events_core").select("span_id").as("sub"))
+ .selectFrom(db.selectFrom("events").select("span_id").as("sub"))
.select("sub.span_id"),
),
).toMatchInlineSnapshot(
- `"select sub.span_id from (select span_id from events_core where project_id = {p1:String} and events_core.is_deleted = 0) as sub"`,
+ `"select sub.span_id from (select span_id from events_core as events where events.project_id = {p1:String} and events.is_deleted = 0) as sub"`,
);
});
it("filters a soft-delete table wrapped in FINAL", () => {
// No soft-delete table allows FINAL yet, so the node is built by hand.
const node = getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.select("e.span_id")
.toOperationNode();
if (!SelectQueryNode.is(node) || !node.from)
@@ -138,15 +139,15 @@ describe("soft-delete filtering", () => {
expect(
compile(
getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.$call(includeDeleted())
.select("e.span_id")
.where("e.trace_id", "in", (eb) =>
- eb.selectFrom("events_core").select("trace_id"),
+ eb.selectFrom("events").select("trace_id"),
),
),
).toMatchInlineSnapshot(
- `"select e.span_id from events_core as e where e.project_id = {p1:String} and e.trace_id in (select trace_id from events_core where project_id = {p1:String} and events_core.is_deleted = 0)"`,
+ `"select e.span_id from events_core as e where e.project_id = {p1:String} and e.trace_id in (select trace_id from events_core as events where events.project_id = {p1:String} and events.is_deleted = 0)"`,
);
});
@@ -155,7 +156,7 @@ describe("soft-delete filtering", () => {
compile(
getClickhouseKysely()
.selectFrom("scores as s")
- .rightJoin("events_core as e", (join) =>
+ .rightJoin("events as e", (join) =>
join.onRef("e.trace_id", "=", "s.trace_id"),
)
.select("e.span_id"),
@@ -164,7 +165,7 @@ describe("soft-delete filtering", () => {
expect(() =>
compile(
getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.fullJoin("scores as s", (join) =>
join.onRef("s.trace_id", "=", "e.trace_id"),
)
@@ -175,7 +176,7 @@ describe("soft-delete filtering", () => {
compile(
getClickhouseKysely()
.selectFrom("scores as s")
- .rightJoin("events_core as e", (join) =>
+ .rightJoin("events as e", (join) =>
join.onRef("e.trace_id", "=", "s.trace_id"),
)
.$call(includeDeleted())
@@ -189,7 +190,7 @@ describe("soft-delete filtering", () => {
compile(
getClickhouseKysely()
.selectFrom("scores as s")
- .innerJoin("events_core as e", (join) =>
+ .innerJoin("events as e", (join) =>
join.onRef("e.trace_id", "=", "s.trace_id"),
)
.$call(includeDeleted())
@@ -202,7 +203,7 @@ describe("soft-delete filtering", () => {
expect(
compile(
getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.$call(includeDeleted())
.select("span_id")
.orderBy("event_ts", "desc")
@@ -223,7 +224,7 @@ describe("soft-delete filtering", () => {
expect(() =>
compile(
getClickhouseKysely()
- .with("ev", (qb) => qb.selectFrom("events_core").select("trace_id"))
+ .with("ev", (qb) => qb.selectFrom("events").select("trace_id"))
.selectFrom("ev")
.$call(includeDeleted())
.select("trace_id"),
@@ -235,12 +236,12 @@ describe("soft-delete filtering", () => {
expect(
compile(
getClickhouseKysely()
- .selectFrom("events_full as e")
+ .selectFrom("events as e")
.$call(softDeleteAfterLimit())
.select((eb) => [
"e.span_id as id",
"e.event_ts",
- "input",
+ "e.input",
eb.fn<number>("length", ["e.output"]).as("len"),
])
.where("e.span_id", "=", "o")
@@ -248,7 +249,7 @@ describe("soft-delete filtering", () => {
.limit(1),
),
).toMatchInlineSnapshot(
- `"select latest.id, latest.event_ts, latest.input, latest.len from (select e.span_id as id, e.event_ts, input, length(e.output) as len, e.is_deleted as is_deleted from events_full as e where e.project_id = {p1:String} and e.span_id = {p2:String} order by e.event_ts desc limit {p3:Int64}) as latest where latest.is_deleted = 0"`,
+ `"select latest.id, latest.event_ts, latest.input, latest.len from (select e.span_id as id, e.event_ts, e.input, length(e.output) as len, e.is_deleted as is_deleted from events_full as e where e.project_id = {p1:String} and e.span_id = {p2:String} order by e.event_ts desc limit {p3:Int64}) as latest where latest.is_deleted = 0"`,
);
});
@@ -258,7 +259,7 @@ describe("soft-delete filtering", () => {
getClickhouseKysely()
.with("latest_events", () =>
getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.$call(softDeleteAfterLimit())
.select(["e.span_id as span_id", "e.trace_id as trace_id"])
.orderBy("e.event_ts", "desc")
@@ -278,13 +279,13 @@ describe("soft-delete filtering", () => {
expect(
compile(
getClickhouseKysely()
- .selectFrom("events_core")
+ .selectFrom("events")
.$call(softDeleteAfterLimit())
.select("span_id")
.limit(1),
),
).toMatchInlineSnapshot(
- `"select latest.span_id from (select span_id, events_core.is_deleted as is_deleted from events_core where project_id = {p1:String} limit {p2:Int64}) as latest where latest.is_deleted = 0"`,
+ `"select latest.span_id from (select span_id, events.is_deleted as is_deleted from events_core as events where events.project_id = {p1:String} limit {p2:Int64}) as latest where latest.is_deleted = 0"`,
);
});
@@ -292,21 +293,21 @@ describe("soft-delete filtering", () => {
expect(
compile(
getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events as e")
.$call(softDeleteAfterLimit())
.select("e.span_id")
.where("e.trace_id", "in", (eb) =>
- eb.selectFrom("events_core").select("trace_id"),
+ eb.selectFrom("events").select("trace_id"),
)
.limit(1),
),
- ).toContain("events_core.is_deleted = 0");
+ ).toContain("events.is_deleted = 0");
});
it("rejects softDeleteAfterLimit() on selects it cannot wrap", () => {
const marked = () =>
getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events_truncated as e")
.$call(softDeleteAfterLimit())
.limit(1);
const rejects = (
@@ -317,7 +318,7 @@ describe("soft-delete filtering", () => {
rejects(marked(), /without outputs/);
rejects(
getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events_truncated as e")
.$call(softDeleteAfterLimit())
.select("e.span_id"),
/needs a LIMIT or LIMIT BY/,
@@ -330,7 +331,9 @@ describe("soft-delete filtering", () => {
marked()
.select("e.span_id")
.unionAll(
- getClickhouseKysely().selectFrom("events_core").select("span_id"),
+ getClickhouseKysely()
+ .selectFrom("events_truncated")
+ .select("span_id"),
),
/cannot wrap a GROUP BY or set operation/,
);
@@ -350,7 +353,7 @@ describe("soft-delete filtering", () => {
);
rejects(
marked()
- .innerJoin("events_full as f", (join) =>
+ .innerJoin("events_truncated as f", (join) =>
join.onRef("f.span_id", "=", "e.span_id"),
)
.select("e.span_id"),
@@ -359,7 +362,7 @@ describe("soft-delete filtering", () => {
rejects(
getClickhouseKysely()
.selectFrom("scores as s")
- .leftJoin("events_core as e", (join) =>
+ .leftJoin("events_truncated as e", (join) =>
join.onRef("e.trace_id", "=", "s.trace_id"),
)
.$call(softDeleteAfterLimit())
diff --git a/packages/shared/src/server/query-ast/types.assert.ts b/packages/shared/src/server/query-ast/types.assert.ts
index 81832ede8c..e15e1be238 100644
--- a/packages/shared/src/server/query-ast/types.assert.ts
+++ b/packages/shared/src/server/query-ast/types.assert.ts
@@ -20,27 +20,32 @@ import { defineView, fromView } from "./views";
export function schemaTypeAssertions(): void {
const db = getClickhouseKysely();
- db.selectFrom("events_core")
+ db.selectFrom("events")
.select("environment")
// @ts-expect-error DateTime column compared to Int
.where("start_time", "=", 123);
- db.selectFrom("events_core")
+ db.selectFrom("events")
.select("environment")
.where("start_time", ">=", new Date("2026-01-01T00:00:00.000Z"));
- db.selectFrom("events_core").select((eb) => eb.fn.sum("total_cost").as("s"));
+ db.selectFrom("events").select((eb) => eb.fn.sum("total_cost").as("s"));
// Intentionally valid at the TS layer — Kysely does not constrain sum()
// to numeric columns. The runtime pass in typecheck.ts rejects this.
- db.selectFrom("events_core").select((eb) => eb.fn.sum("environment").as("s"));
+ db.selectFrom("events").select((eb) => eb.fn.sum("environment").as("s"));
- db.selectFrom("events_core")
+ db.selectFrom("events")
// @ts-expect-error unknown column is not in the schema
.select("not_a_column");
+ // @ts-expect-error physical events tables are reached through `events` / `events_truncated`
+ db.selectFrom("events_core");
+ // @ts-expect-error physical events tables are reached through `events` / `events_truncated`
+ db.selectFrom("events_full as e");
+
const view = defineView("environments_view")<{ environment: string }>(() =>
- db.selectFrom("events_core").select("environment").distinct(),
+ db.selectFrom("events").select("environment").distinct(),
);
fromView(view).select("environment");
fromView(view)
@@ -50,7 +55,7 @@ export function schemaTypeAssertions(): void {
// Named-tuple arrays read as arrays of objects keyed by tuple name; a typo
// in the column type would make the row field `never` and reject these.
const _skills = db
- .selectFrom("events_core")
+ .selectFrom("events")
.select(["skills_available", "skills_resource_loaded"]);
type SkillsRow = InferResult<typeof _skills>[number];
const _available: SkillsRow["skills_available"] = [
@@ -99,7 +104,7 @@ export function extensionTypeAssertions(): void {
// limitBy preserves the row type: it declares no new columns.
const _limited = db
- .selectFrom("events_core")
+ .selectFrom("events")
.select("span_id")
.$call(limitBy({ count: 1, columns: (eb) => [eb.ref("span_id")] }));
type Limited = InferResult<typeof _limited>[number];
@@ -108,7 +113,7 @@ export function extensionTypeAssertions(): void {
type _NoExtra = Limited["cost_key"];
// Helper columns are references checked against the query's tables.
- db.selectFrom("events_core as e")
+ db.selectFrom("events as e")
.select("e.span_id")
// @ts-expect-error eb.ref resolves against this select's tables
.$call(limitBy({ count: 1, columns: (eb) => [eb.ref("e.span_idx")] }));
diff --git a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap
index 73472621e8..d63ab8d6f8 100644
--- a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap
+++ b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap
@@ -742,8 +742,8 @@ exports[`golden: events export streams family > getEventsForBlobStorageExport fi
NULL,
dateDiff('millisecond', e.start_time, e.completion_start_time)
) AS time_to_first_token,
- input,
- output
+ e.input,
+ e.output
FROM events_full AS e
WHERE (e.project_id = {param1:String})
AND (e.is_deleted = 0)
@@ -804,8 +804,8 @@ exports[`golden: events export streams family > getEventsForBlobStorageExport fi
e.parent_span_id AS parent_observation_id,
e.start_time AS start_time,
e.end_time AS end_time,
- input,
- output
+ e.input,
+ e.output
FROM events_full AS e
WHERE (e.project_id = {param1:String})
AND (e.is_deleted = 0)
@@ -1012,8 +1012,8 @@ exports[`golden: events export streams family > getEventsForBlobStorageExport fi
NULL,
dateDiff('millisecond', e.start_time, e.completion_start_time)
) AS time_to_first_token,
- input,
- output
+ e.input,
+ e.output
FROM events_full AS e
WHERE (e.project_id = {param1:String})
AND (e.is_deleted = 0)
@@ -1227,8 +1227,8 @@ exports[`golden: events export streams family > getEventsForBlobStorageExportPar
NULL,
dateDiff('millisecond', e.start_time, e.completion_start_time)
) AS time_to_first_token,
- input,
- output
+ e.input,
+ e.output
FROM events_full AS e
WHERE (e.project_id = {param1:String})
AND (e.is_deleted = 0)
@@ -1297,8 +1297,8 @@ exports[`golden: events export streams family > getEventsForBlobStorageExportPar
e.tags AS tags,
e.release AS release,
e.model_id AS model_id,
- input,
- output,
+ e.input,
+ e.output,
if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time) / 1000) AS latency,
if(
isNull(e.completion_start_time),
@@ -1336,8 +1336,8 @@ exports[`golden: events export streams family > getEventsForBlobStorageExportRaw
e.parent_span_id AS parent_observation_id,
e.start_time AS start_time,
e.end_time AS end_time,
- input,
- output
+ e.input,
+ e.output
FROM events_full AS e
WHERE (e.project_id = {param1:String})
AND (e.is_deleted = 0)
@@ -1405,8 +1405,8 @@ exports[`golden: events export streams family > getEventsForBlobStorageExportRaw
e.tags AS tags,
e.release AS release,
e.model_id AS model_id,
- input,
- output,
+ e.input,
+ e.output,
if(isNull(e.end_time), NULL, dateDiff('millisecond', e.start_time, e.end_time) / 1000) AS latency,
if(
isNull(e.completion_start_time),
@@ -1486,8 +1486,8 @@ exports[`golden: events export streams family > getEventsForBlobStorageExportRaw
NULL,
dateDiff('millisecond', e.start_time, e.completion_start_time)
) AS time_to_first_token,
- input,
- output
+ e.input,
+ e.output
FROM events_full AS e
WHERE (e.project_id = {param1:String})
AND (e.is_deleted = 0)
@@ -2119,8 +2119,8 @@ FROM
mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,
e.tags AS tags,
e.release AS release,
- input,
- output,
+ e.input,
+ e.output,
e.is_deleted AS is_deleted
FROM events_full AS e
WHERE (e.project_id = {param1:String})
@@ -2240,8 +2240,8 @@ FROM
mapFromArrays(arrayReverse(e.metadata_names), arrayReverse(e.metadata_values)) AS metadata,
e.tags AS tags,
e.release AS release,
- leftUTF8(input, 1000) AS input,
- leftUTF8(output, 1000) AS output,
+ leftUTF8(e.input, 1000) AS input,
+ leftUTF8(e.output, 1000) AS output,
e.is_deleted AS is_deleted
FROM events_core AS e
WHERE (e.project_id = {param1:String})
diff --git a/packages/shared/src/server/repositories/events-selects.types.assert.ts b/packages/shared/src/server/repositories/events-selects.types.assert.ts
index 7450ca9413..77dc5bb00b 100644
--- a/packages/shared/src/server/repositories/events-selects.types.assert.ts
+++ b/packages/shared/src/server/repositories/events-selects.types.assert.ts
@@ -22,15 +22,14 @@ export function eventsSelectsScopeTypeAssertions(): void {
const db = getClickhouseKysely();
const allTraceAggregates = traceAggregationSelects(TRACE_AGGREGATION_FIELDS);
- db.selectFrom("events_core as e").select(allTraceAggregates);
- db.selectFrom("events_full as e").select(allTraceAggregates);
db.selectFrom("events as e").select(allTraceAggregates);
+ db.selectFrom("events_truncated as e").select(allTraceAggregates);
db.selectFrom("events as e")
.innerJoin("traces as t", "t.id", "e.trace_id")
// @ts-expect-error a join
.select(allTraceAggregates);
- db.with("e", (qb) => qb.selectFrom("events_core").selectAll())
+ db.with("e", (qb) => qb.selectFrom("events").selectAll())
.selectFrom("traces as t")
// @ts-expect-error a CTE named `e` is not read in FROM
.select(allTraceAggregates);
@@ -39,7 +38,7 @@ export function eventsSelectsScopeTypeAssertions(): void {
// @ts-expect-error no `e` in scope
.select(allTraceAggregates);
- db.selectFrom("events_core")
+ db.selectFrom("events")
// @ts-expect-error the events table is not aliased `e`
.select(allTraceAggregates);
@@ -59,7 +58,7 @@ export function eventsSelectsScopeTypeAssertions(): void {
// @ts-expect-error the subquery `e` lacks `completion_start_time`
.select(eventsMetricsSecondsSelects);
- db.with("e", (qb) => qb.selectFrom("events_core").selectAll())
+ db.with("e", (qb) => qb.selectFrom("events").selectAll())
// @ts-expect-error no FROM: `e` is never read
.selectNoFrom((eb) => [eventsTraceName(eb).as("name")]);
@@ -67,8 +66,8 @@ export function eventsSelectsScopeTypeAssertions(): void {
const traceNameSelect = scopedToEvents((eb) =>
eventsTraceName(eb).as("name"),
);
- db.selectFrom("events_core as e").select(traceNameSelect);
- db.selectFrom("events_core as e")
+ db.selectFrom("events as e").select(traceNameSelect);
+ db.selectFrom("events as e")
.innerJoin("traces as t", "t.id", "e.trace_id")
// @ts-expect-error a join, through a composed helper
.select(traceNameSelect);
@@ -80,7 +79,7 @@ export function eventsSelectsScopeTypeAssertions(): void {
// scope of the builder it was made from, and Kysely's `select()` accepts any
// aliased expression. Build shared expressions from the query's builder.
const detached = expressionBuilder<
- { e: ClickHouseDatabase["events_core"] },
+ { e: ClickHouseDatabase["events"] },
"e"
>();
db.selectFrom("traces as t").select(
@@ -90,11 +89,11 @@ export function eventsSelectsScopeTypeAssertions(): void {
// Intentionally valid too: upcasting the query's builder to the plain events
// builder (here, under a join) drops what the guard checks. Use
// scopedToEvents instead of typing a parameter as `ExpressionBuilder`.
- db.selectFrom("events_core as e")
+ db.selectFrom("events as e")
.innerJoin("traces as t", "t.id", "e.trace_id")
.select((eb) => {
const events: ExpressionBuilder<
- { e: ClickHouseDatabase["events_core"] },
+ { e: ClickHouseDatabase["events"] },
"e"
> = eb;
return [eventsTraceName(events).as("name")];
diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.ts
index b3dd485687..f2903b70af 100644
--- a/packages/shared/src/server/repositories/events.ts
+++ b/packages/shared/src/server/repositories/events.ts
@@ -1129,8 +1129,8 @@ async function getObservationByIdFromEventsTableInternal({
// A soft-deleted latest version reads as not found, not as an older version.
const builder = (
needsFullTable
- ? db.selectFrom("events_full as e")
- : db.selectFrom("events_core as e")
+ ? db.selectFrom("events as e")
+ : db.selectFrom("events_truncated as e")
)
.$call(softDeleteAfterLimit())
.select(observationByIdSelects)
@@ -1175,11 +1175,11 @@ function observationIoSelects(
eb: ExpressionBuilder<{ e: IoColumns }, "e">,
truncated: boolean | undefined,
) {
- if (!truncated) return ["input", "output"] as const;
+ if (!truncated) return ["e.input", "e.output"] as const;
const charLimit = env.LANGFUSE_SERVER_SIDE_IO_CHAR_LIMIT;
return [
- leftUTF8(eb.ref("input"), charLimit).as("input"),
- leftUTF8(eb.ref("output"), charLimit).as("output"),
+ leftUTF8(eb.ref("e.input"), charLimit).as("input"),
+ leftUTF8(eb.ref("e.output"), charLimit).as("output"),
];
}
@@ -1216,7 +1216,7 @@ export const getTraceByIdFromEventsTable = async ({
const tracesCte = (qb: QueryCreator<ClickHouseDatabase>) =>
qb
- .selectFrom(truncated ? "events_core as e" : "events_full as e")
+ .selectFrom(truncated ? "events_truncated as e" : "events as e")
.select((eb) => traceAggregationSelects(TRACE_AGGREGATION_FIELDS)(eb))
.where("trace_id", "in", [traceId])
.$if(fromTimestamp != null, (inner) =>
@@ -2658,7 +2658,7 @@ export async function getAgentGraphDataFromEventsTable(params: {
const db = getClickhouseKysely();
const builder = db
- .selectFrom("events_core as e")
+ .selectFrom("events_truncated as e")
.select((eb) => {
const metadata = metadataMap(
eb.ref("e.metadata_names"),
@@ -2830,7 +2830,7 @@ export const getObservationsBatchIOFromEventsTable = async <
// Keep the individual filters for primary-key pruning and the tuple filter
// for exact trace/observation matching.
const builder = getClickhouseKysely()
- .selectFrom(truncated ? "events_core as e" : "events_full as e")
+ .selectFrom(truncated ? "events_truncated as e" : "events as e")
.select((eb) => [
"e.span_id as id",
"e.trace_id as trace_id",
@@ -3442,9 +3442,9 @@ const buildEventsForBlobStorageExportQuery = (
});
const builder = getClickhouseKysely()
- .selectFrom(includeIO ? "events_full as e" : "events_core as e")
+ .selectFrom(includeIO ? "events as e" : "events_truncated as e")
.select(eventsRowFieldSelects(eventsFieldSetKeys(...fieldSets)))
- .$if(includeIO, (qb) => qb.select(["input", "output"]))
+ .$if(includeIO, (qb) => qb.select(["e.input", "e.output"]))
.$if(metricsInSeconds, (qb) => qb.select(eventsMetricsSecondsSelects))
.where("e.start_time", ">=", minTimestamp)
.where("e.start_time", "<", maxTimestamp)
@@ -3543,7 +3543,7 @@ export const getEventsForAnalyticsIntegrations = async function* (
maxTimestamp: Date,
) {
const builder = getClickhouseKysely()
- .selectFrom("events_core as e")
+ .selectFrom("events_truncated as e")
// Use export field set for most fields (id, traceId, name, type, level, version,
// environment, userId, sessionId, tags, release, traceName, totalCost, latency, etc.)
.select(eventsRowFieldSelects(eventsFieldSetKeys("export")))
diff --git a/packages/shared/src/server/services/sessions-ui-table-events-service.ts b/packages/shared/src/server/services/sessions-ui-table-events-service.ts
index 7d29809bf8..224ea76bf7 100644
--- a/packages/shared/src/server/services/sessions-ui-table-events-service.ts
+++ b/packages/shared/src/server/services/sessions-ui-table-events-service.ts
@@ -53,7 +53,7 @@ export const getSessionTracesFromEvents = async (props: {
}) => {
const { projectId, sessionId } = props;
const builder = getClickhouseKysely()
- .selectFrom("events_full as e")
+ .selectFrom("events as e")
.select(traceAggregationSelects(TRACE_AGGREGATION_FIELDS))
.where("e.session_id", "=", sessionId)
.groupBy(["trace_id", "project_id"])