πͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
refactor(query-ast): reject CTE names that could mean two relations (#19040)
Niklas Semmler52539f430555
diff --git a/packages/shared/src/server/query-ast/README.md b/packages/shared/src/server/query-ast/README.mdindex 89276009f4..1c98f6f67b 100644--- a/packages/shared/src/server/query-ast/README.md+++ b/packages/shared/src/server/query-ast/README.md@@ -21,6 +21,8 @@ changing how it emits SQL.- `compile.ts` β the one supported entry point,`compileClickhouseQuery(query, projectId)`.+- `ctes.ts` β the first pass: rejects CTE and FROM/JOIN names that could+ mean two relations, so later passes read every registry name as its table.- `tenancy.ts` β the tenancy injection pass.- `schema.ts` β the physical table registry: one `defineTable` declaration perrelation drives the Kysely row types (`ClickHouseDatabase`), the runtime@@ -93,11 +95,20 @@ suffix to run there. Promote it to a required check once it has proven stable.`metadataValue` stay on `events_core` when`metadataFilterIsEventsCoreSafe` accepts the value, as in the raw-SQLbuilders. `*` / `e.*` over `events`, raw SQL text in a select that reads- it, an unqualified truncated column, and a CTE named `events_core` /- `events_full` are compile errors while `events` is in scope. Name- `events_core` to read the truncated copy on purpose; `events_core` /- `events_full` stay selectable by name, which is also the override for a- shape the pass rejects.+ it, and an unqualified truncated column are compile errors while `events`+ is in scope. Name `events_core` to read the truncated copy on purpose;+ `events_core` / `events_full` stay selectable by name, which is also the+ override for a shape the pass rejects.+- **Give each CTE (and view) its own plain name.** A CTE name is a plain+ identifier that is not a registry table (`traces`, `events_core`, the+ logical `events`, β¦) and doesn't reuse the name of a sibling or an+ enclosing CTE (a CTE is not in scope inside its own body). Every FROM/JOIN+ name is an unqualified registry table or a CTE in scope; schema-qualified+ names (`default.traces`) are rejected, since the registry describes tables+ by name only. A CTE body sees only the CTEs declared before it, so+ recursive and cyclic CTEs are out. The passes then read every registry+ name as its table. Rename or reorder the CTEs when the compile rejects+ them; add a physical table to the registry to read it.- **Function and aggregate names must be plain identifiers.**The compiler writes them verbatim, so anything else fails to compile.`collate` is not supported.@@ -262,6 +273,10 @@ and it is where tenancy is enforced:1. A missing/empty `projectId` throws (`QueryCompileError`) β `projectId` is arequired parameter, so omitting it is also a compile-time type error.+ `CteNamePlugin` then rejects the CTE names the rules above forbid; it runs+ before every other pass. A FROM/JOIN name that resolves to no registry+ table or CTE in scope, and a schema-qualified name, throw+ `UnscopedRelationError`; naming errors throw `QueryCompileError`.2. `SoftDeleteFilterPlugin` adds `<flag> = 0` for every FROM/JOIN relationwhose table declares `softDelete`, except in selects marked with`includeDeleted()`. A select marked `softDeleteAfterLimit()` is left@@ -291,8 +306,9 @@ and it is where tenancy is enforced:present, so `qb.compile()` without the plugin also fails. Value binds taketheir ClickHouse type from the compared column's registry entry when one isin scope (`scores.value > 1` β `{p:Float64}`).-6. Raw-SQL table sources (``selectFrom(sql`...`)``) and raw fragments embedding- a `SELECT`/`FROM`/`JOIN` in SELECT/WHERE throw `UnscopedRelationError`, as+6. Raw-SQL table sources (``selectFrom(sql`...`)``), raw fragments embedding+ a `SELECT`/`FROM`/`JOIN` in SELECT/WHERE, and a `sql.table(...)` reference+ inside raw SQL in any position throw `UnscopedRelationError`, asdoes any FROM/JOIN source that is neither a table reference nor a subquery(table functions such as `remote()`, `clusterAllReplicas()`, `merge()`).Kysely's own keyword fragments (`asc`/`desc`) are not relations.diff --git a/packages/shared/src/server/query-ast/binds.test.ts b/packages/shared/src/server/query-ast/binds.test.tsindex 1d95d5aa10..b1a4ffbf5a 100644--- a/packages/shared/src/server/query-ast/binds.test.ts+++ b/packages/shared/src/server/query-ast/binds.test.ts@@ -4,7 +4,7 @@ import { describe, expect, it } from "vitest";import { compileClickhouseQuery } from "./compile";import { getClickhouseKysely } from "./dialect";-import { QueryCompileError, UnscopedRelationError } from "./errors";+import { QueryCompileError } from "./errors";import { arrayJoin, countIf, tupleIn, useFinal } from "./extensions";const projectId = "proj-1";@@ -162,17 +162,6 @@ describe("typed params from the column registry", () => {// A CTE or subquery column keeps the type of the column it selects.it.each([- [- "a CTE named like a registry table",- () =>- getClickhouseKysely()- .with("observations", (db) =>- db.selectFrom("events_core").select("start_time"),- )- .selectFrom("observations")- .select("start_time")- .where("start_time", ">=", START),- ],["an aliased CTE column read through a table alias",() =>@@ -280,119 +269,46 @@ describe("typed params from the column registry", () => {expect(sql).toMatch(/(start_time|ts) >= \{p\d+:DateTime64\(6\)\}/);});- // `observations.start_time` is DateTime64(3); `events_core.start_time` is- // DateTime64(6). Which one a bind gets shows which relation a name resolved to.+ // `events_core.start_time` is DateTime64(6); a Date bound against a column+ // the compiler can't type binds as DateTime64(3). Which one a bind gets+ // shows which relation a name resolved to.describe("CTE name resolution", () => {const db = getClickhouseKysely();- const eventsAsObservations = db.with("observations", (qb) =>+ const withX = db.with("x", (qb) =>qb.selectFrom("events_core").select("start_time"),);- it("resolves a later sibling CTE name to the CTE", () => {- const { sql } = compile(- db- .with("early", (qb) =>- qb.selectFrom("observations").select("start_time"),- )- .with("observations", (qb) =>- qb.selectFrom("events_core").select("start_time"),- )- .selectFrom("early")- .select("start_time")- .where("start_time", ">=", START),- );- expect(sql).toMatch(/start_time >= \{p\d+:DateTime64\(6\)\}$/);- });-- it("resolves a CTE body's read of its own name to the table", () => {+ it("types a subquery's read of an enclosing CTE from its body", () => {const { sql } = compile(- db- .with("observations", (qb) =>+ withX+ .selectFrom((qb) =>qb- .selectFrom("events_core as e")- .innerJoin("observations as o", "o.trace_id", "e.trace_id")- .select("e.start_time")- .where("o.start_time", ">=", START),- )- .selectFrom("observations")- .select("start_time"),- );- expect(sql).toMatch(/o\.start_time >= \{p\d+:DateTime64\(3\)\}/);- });-- it("types a CTE that reads its own name when it is first read in a subquery", () => {- const { sql } = compile(- db- .with("observations", (qb) =>- qb.selectFrom("observations").select(["trace_id", "start_time"]),- )- .selectFrom("traces")- .select("id")- .where(- "id",- "in",- db- .selectFrom("observations")- .select("trace_id")- .where("start_time", ">=", START),- ),- );- expect(sql).toMatch(/start_time >= \{p\d+:DateTime64\(3\)\}/);- });-- it("resolves a nested CTE body's own name to an outer CTE of that name", () => {- const { sql } = compile(- eventsAsObservations- .selectFrom(- db- .with("observations", (qb) =>- qb.selectFrom("observations").select("start_time"),- )- .selectFrom("observations")+ .selectFrom("x").select("start_time")+ .where("start_time", ">=", START).as("sub"),)- .select("sub.start_time")- .where("sub.start_time", ">=", START),- );- expect(sql).toMatch(/sub\.start_time >= \{p\d+:DateTime64\(6\)\}$/);- });-- it("resolves a schema-qualified table to the table, not a CTE of its name", () => {- const { sql } = compile(- eventsAsObservations- .selectFrom("observations")- .select("start_time")- .where(- "start_time",- "in",- db- .withSchema("default")- .selectFrom("observations")- .select("start_time")- .where("start_time", ">=", START),- ),- );- expect(sql).toMatch(- /from default\.observations where project_id = \{p\d+:String\} and start_time >= \{p\d+:DateTime64\(3\)\}/,+ .select("sub.start_time"),);+ expect(sql).toMatch(/start_time >= \{p\d+:DateTime64\(6\)\}\) as sub$/);});- it("rejects CTEs that read each other", () => {+ it("does not let a CTE body see the declaring select's FROM aliases", () => {+ // Inside the body, `o` names nothing, so binding against `o.start_time`+ // is rejected instead of taking the outer `events_core as o` type.expect(() =>compile(db- .with("observations", (qb) =>- qb.selectFrom("traces").select("timestamp as start_time"),- )- .with("traces", (qb) =>- qb.selectFrom("observations").select("start_time as timestamp"),+ .with("y", (qb) =>+ qb+ .selectFrom("events_core")+ .select("span_id")+ .where(db.dynamic.ref("o.start_time"), ">=", START),)- .selectFrom("traces")- .select("timestamp")- .where("timestamp", ">=", START),+ .selectFrom("events_core as o")+ .select("o.span_id"),),- ).toThrow(UnscopedRelationError);+ ).toThrow(/no relation in scope is named "o"/);});});@@ -499,10 +415,6 @@ describe("typed params from the column registry", () => {describe("columns the compiler can't resolve to a relation", () => {const db = getClickhouseKysely();- // A table outside the registry: the compiler can't see its columns.- const unregistered = db.withTables<{- unregistered: { trace_id: string; start_time: Date };- }>();it("binds by value when no relation in scope has the column", () => {const { sql } = compile(@@ -514,6 +426,43 @@ describe("typed params from the column registry", () => {expect(sql).toMatch(/total_cost > \{p\d+:Int64\}/);});+ it("binds by value a column named like an Object prototype key", () => {+ // `scores` has no `constructor` column; the lookup must not find+ // `Object.prototype.constructor` instead.+ const { sql } = compile(+ db+ .selectFrom("scores as s")+ .select("s.id")+ .where(db.dynamic.ref("s.constructor"), ">", 1),+ );+ expect(sql).toMatch(/s\.constructor > \{p\d+:Int64\}/);+ });++ it("binds by value a CTE column named like an Object prototype key", () => {+ // The CTE's output columns are typed from its body, which has no+ // `constructor` column either.+ const { sql } = compile(+ db+ .with("c", (qb) => qb.selectFrom("traces").select("id"))+ .selectFrom("c")+ .select("c.id")+ .where(db.dynamic.ref("c.constructor"), ">", 1),+ );+ expect(sql).toMatch(/c\.constructor > \{p\d+:Int64\}/);+ });++ it("binds by value a column named like an Object prototype key through a CTE *", () => {+ // The `*` expands to the columns of `traces`, which has no `constructor`.+ const { sql } = compile(+ db+ .with("c", (qb) => qb.selectFrom("traces").selectAll())+ .selectFrom("c")+ .select("c.id")+ .where(db.dynamic.ref("c.constructor"), ">", 1),+ );+ expect(sql).toMatch(/c\.constructor > \{p\d+:Int64\}/);+ });+it("binds a named raw-SQL CTE column by value", () => {const { sql } = compile(db@@ -563,16 +512,6 @@ describe("typed params from the column registry", () => {.where(db.dynamic.ref("x.is_deleted"), "=", 0),/no relation in scope is named "x"/,],- [- // `u` is declared, but its table is not in the registry.- "a qualified column of a table outside the registry",- () =>- unregistered- .selectFrom("unregistered as u")- .select("u.trace_id")- .where("u.start_time", ">=", START),- /"unregistered" is not a registered table/,- ],[// start_time is DateTime64(3) on observations, DateTime64(6) on events."an unqualified column the joined relations type differently",@@ -584,68 +523,6 @@ describe("typed params from the column registry", () => {.where("start_time", ">=", START),/joined relations give it different types/,],- [- // span_id is only on events_core in the registry, but `u` may have it.- "an unqualified column next to a table outside the registry",- () =>- unregistered- .selectFrom("events_core as e")- .innerJoin("unregistered as u", "u.trace_id", "e.trace_id")- .select("e.trace_id")- .where("span_id", "=", "s"),- /may come from a relation whose columns are unknown\. "unregistered" is not a registered table/,- ],- [- // The CTE column carries the unknown type of `u.start_time`.- "a CTE column selected from a table outside the registry",- () =>- unregistered- .with("t", (qb) =>- qb.selectFrom("unregistered as u").select("u.start_time"),- )- .selectFrom("t")- .select("start_time")- .where("start_time", ">=", START),- /"unregistered" is not a registered table/,- ],- [- // `t.start_time` is unknown, so the join can't settle the type either.- "an unqualified CTE column of unknown type next to a registry relation",- () =>- unregistered- .with("t", (qb) =>- qb.selectFrom("unregistered as u").select("u.start_time"),- )- .selectFrom("t")- .innerJoin("observations as o", "o.start_time", "t.start_time")- .select("o.trace_id")- .where("start_time", ">=", START),- /"unregistered" is not a registered table/,- ],- [- // `*` over an unregistered table yields unknown columns.- "a CTE * over a table outside the registry",- () =>- unregistered- .with("t", (qb) => qb.selectFrom("unregistered").selectAll())- .selectFrom("t")- .select("trace_id")- .where("start_time", ">=", START),- /may come from a relation whose columns are unknown\. "unregistered" is not a registered table/,- ],- [- // Same, through a qualified `u.*`.- "a CTE qualified * over a table outside the registry",- () =>- unregistered- .with("t", (qb) =>- qb.selectFrom("unregistered as u").selectAll("u"),- )- .selectFrom("t as t")- .select("t.trace_id")- .where("t.start_time", ">=", START),- /"unregistered" is not a registered table/,- ],[// `t` outputs value computed (UInt64); scores types it Float64."an unqualified column one relation outputs computed and another types",@@ -685,6 +562,21 @@ describe("typed params from the column registry", () => {.where(db.dynamic.ref("t.start_time"), ">=", START),/It selects x\.\* but no relation in that select is named "x"/,],+ [+ // `t` selects `x.*` over no relation named `x`, so its columns are+ // unknown and may include trace_id.+ "an unqualified column next to a CTE whose columns are unknown",+ () =>+ db+ .with("t", (qb) =>+ qb.selectFrom("events_core as e").select(db.dynamic.ref("x.*")),+ )+ .selectFrom("scores as s")+ .innerJoin("t", (join) => join.onTrue())+ .select("s.id")+ .where("trace_id", "=", "x"),+ /may come from a relation whose columns are unknown/,+ ],[// Each tuple position takes its column's type; `x.a` has none."a TupleParam whose tuple has a column declared nowhere",diff --git a/packages/shared/src/server/query-ast/compile.ts b/packages/shared/src/server/query-ast/compile.tsindex 2751d23e60..36a1554888 100644--- a/packages/shared/src/server/query-ast/compile.ts+++ b/packages/shared/src/server/query-ast/compile.ts@@ -6,6 +6,7 @@ import {} from "kysely";import { ClickHouseQueryCompiler } from "./compiler";+import { CteNamePlugin } from "./ctes";import { DedupLoweringPlugin } from "./dedup";import { EventsTableRoutingPlugin } from "./eventsRouting";import { SoftDeleteFilterPlugin } from "./softDelete";@@ -34,7 +35,8 @@ export type ClickhouseCompilable = {* The choke point. Every query that should produce ClickHouse SQL goes* through here: the `projectId` parameter is required, so a caller cannot* compile without a tenancy scope β omitting it is a compile-time type error.- * The query reads exactly that one project. The logical `events` relation is+ * The query reads exactly that one project. CTE names that could mean two+ * relations are rejected (`ctes.ts`). The logical `events` relation is* routed to `events_core` or `events_full`, soft-deleted rows are filtered* out, tenancy is injected from that project, per-table dedup is applied from* the registry, and the dialect compiler refuses to emit SQL unless the tree@@ -49,6 +51,7 @@ export function compileClickhouseQuery(): CompiledClickhouseQuery {const scope = requireProjectId(projectId);const stamped = query+ .withPlugin(new CteNamePlugin()).withPlugin(new EventsTableRoutingPlugin()).withPlugin(new SoftDeleteFilterPlugin()).withPlugin(new TenancyInjectionPlugin(scope))diff --git a/packages/shared/src/server/query-ast/compiler.ts b/packages/shared/src/server/query-ast/compiler.tsindex 6f01594d77..5fe0a509e9 100644--- a/packages/shared/src/server/query-ast/compiler.ts+++ b/packages/shared/src/server/query-ast/compiler.ts@@ -19,6 +19,7 @@ import {TupleNode,ValueNode,type CollateNode,+ type CommonTableExpressionNameNode,type CommonTableExpressionNode,type OperationNode,type PrimitiveValueListNode,@@ -36,11 +37,11 @@ import {type ClickHouseSelectQueryNode,type LimitByNode,} from "./nodes";-import { QueryCompileError } from "./errors";+import { QueryCompileError, UnscopedRelationError } from "./errors";import { selectRelations } from "./relations";-import { TABLE_BIND_TYPES } from "./schema";+import { TABLE_BIND_TYPES, lookupTable } from "./schema";import { assertTenancyStamped } from "./tenancy";-import { cteName } from "./transformer";+import { cteName } from "./ctes";const ARRAY_JOIN_SQL: Record<ArrayJoinNode["variant"], string> = {default: "array join",@@ -106,27 +107,26 @@ function inferClickHouseType(value: unknown): string {}/**- * Column bind types of the relation `table` resolves to: the innermost CTE of- * that name, else the registry table, else opaque. A schema-qualified table- * (`default.traces`) is never a CTE.+ * Bind types of the innermost CTE named like `table`, else of its registry+ * table. Any other name throws; `compileClickhouseQuery` rejects it earlier,+ * in `ctes.ts`.*/function relationBindTypes(table: TableNode,scopes: readonly SelectScope[],): RelationBindTypes {const name = table.table.identifier.name;- if (!table.table.schema) {- for (const scope of [...scopes].reverse()) {- const cte = scope.cte(name);- if (cte) return cte;- }+ for (const scope of [...scopes].reverse()) {+ const cte = scope.cte(name);+ if (cte) return cte;}- return (- TABLE_BIND_TYPES[name] ??- new OpaqueRelation(- `"${name}" is not a registered table. Add it to TABLE_REGISTRY in schema.ts, or read the column from a registered table.`,- )- );+ const columns = TABLE_BIND_TYPES[name];+ if (!columns) {+ throw new UnscopedRelationError(+ `"${name}" is not a registry table or a CTE in scope; compile through compileClickhouseQuery(), whose CteNamePlugin (ctes.ts) rejects it.`,+ );+ }+ return columns;}/**@@ -487,13 +487,9 @@ export class ClickHouseQueryCompiler extends DefaultQueryCompiler {/*** Lookup of the CTEs `node`'s WITH declares. A body is typed on its first- * read, against `enclosing` plus {@link withoutCte}, so it can read an- * earlier or a later sibling as ClickHouse does.- *- * Typing a body recurses into the siblings it reads, and each body is typed- * without its own name, so the recursion only follows sibling reads. Those- * never form a cycle: the transformer passes reject- * `WITH a AS (β¦ FROM b), b AS (β¦ FROM a)` before the compiler runs.+ * read, against `enclosing` plus {@link cteBodyScope}. Typing a body+ * recurses into the siblings it reads, which are all declared before it+ * (`ctes.ts` rejects the rest before any pass runs), so the recursion ends.*/private cteLookup(node: SelectQueryNode,@@ -511,7 +507,7 @@ export class ClickHouseQueryCompiler extends DefaultQueryCompiler {if (!cte) return undefined;const known = typed.get(name);if (known) return known;- const bodyScope = withoutCte({ cte: lookup }, name);+ const bodyScope = cteBodyScope({ cte: lookup });const columns = this.outputBindTypes(cte.expression, [...outer,bodyScope,@@ -522,10 +518,15 @@ export class ClickHouseQueryCompiler extends DefaultQueryCompiler {return lookup;}- /**- * The declaring SELECT's scope declares this CTE, so a body reading its own- * name would otherwise bind against the CTE instead of the table.- */+ /** Written verbatim, so `"traces "` can't pass as a name other than `traces`. */+ protected override visitCommonTableExpressionName(+ node: CommonTableExpressionNameNode,+ ): void {+ assertPlainName("CTE", node.table.table.identifier.name);+ super.visitCommonTableExpressionName(node);+ }++ /** A CTE body does not see the declaring SELECT's FROM relations. */protected override visitCommonTableExpression(node: CommonTableExpressionNode,): void {@@ -534,7 +535,7 @@ export class ClickHouseQueryCompiler extends DefaultQueryCompiler {super.visitCommonTableExpression(node);return;}- this.selectScopes.push(withoutCte(declaring, cteName(node)));+ this.selectScopes.push(cteBodyScope(declaring));try {super.visitCommonTableExpression(node);} finally {@@ -561,7 +562,7 @@ export class ClickHouseQueryCompiler extends DefaultQueryCompiler {}const scope = this.selectScopeOf(node, enclosing);const scopes = [...enclosing, scope];- let columns: Record<string, BindType | null> = {};+ let columns = lookupTable<BindType | null>([]);for (const { selection } of node.selections ?? []) {const star = starSelection(selection);if (star) {@@ -826,20 +827,12 @@ export class ClickHouseQueryCompiler extends DefaultQueryCompiler {}/**- * The scope a body of CTE `name` sees in place of its declaring SELECT: every- * sibling CTE but itself, as ClickHouse resolves `WITH traces AS (SELECT β¦- * FROM traces)` to the table, or to an enclosing CTE of that name. The- * declaring SELECT's FROM relations are not visible to the body.+ * The scope a CTE body sees in place of its declaring SELECT: the sibling+ * CTEs, but not the declaring SELECT's FROM relations. A body reads only+ * earlier siblings (`ctes.ts`), so typing a body never reaches itself.*/-function withoutCte(- scope: Pick<SelectScope, "cte">,- name: string,-): SelectScope {- return {- cte: (other) => (other === name ? undefined : scope.cte(other)),- relations: new Map(),- aliases: new Map(),- };+function cteBodyScope(scope: Pick<SelectScope, "cte">): SelectScope {+ return { cte: scope.cte, relations: new Map(), aliases: new Map() };}function assertPlainName(kind: string, name: string): void {@@ -1017,7 +1010,7 @@ function withMissingColumns(columns: ColumnBindTypes,relation: ColumnBindTypes,): Record<string, BindType | null> {- return { ...relation, ...columns };+ return lookupTable([...Object.entries(relation), ...Object.entries(columns)]);}function isTupleBindType(diff --git a/packages/shared/src/server/query-ast/ctes.ts b/packages/shared/src/server/query-ast/ctes.tsnew file mode 100644index 0000000000..89c62c3fb8--- /dev/null+++ b/packages/shared/src/server/query-ast/ctes.ts@@ -0,0 +1,115 @@+import {+ SelectQueryNode,+ type CommonTableExpressionNode,+ type KyselyPlugin,+ type PluginTransformQueryArgs,+ type PluginTransformResultArgs,+ type RootOperationNode,+ type TableNode,+} from "kysely";++import { QueryCompileError, UnscopedRelationError } from "./errors";+import { walkNodes } from "./nodes";+import { selectRelations } from "./relations";+import { REGISTRY_TABLES } from "./schema";++/**+ * Rejects every CTE name and FROM/JOIN name that could mean two relations.+ * Runs first in every compile, so the passes after it can treat a registry+ * name as its table and the compiler can type each CTE read from the one+ * body the name resolves to in scope. Rejected:+ * - a CTE named like a registry table (`with traces as (β¦)`), the logical+ * `events` included;+ * - a CTE that reuses the name of a CTE in scope: a sibling's, or one an+ * enclosing select declares. Inside its own body a CTE is not in scope,+ * so a nested WITH there may reuse its name;+ * - a schema-qualified FROM/JOIN name (`default.traces`): the registry+ * describes tables by name only, not per database;+ * - a FROM/JOIN name that is neither a registry table nor a CTE in scope. A+ * CTE body's scope is where the CTE is declared, without the CTE itself+ * and its later siblings, so recursive and cyclic CTEs are rejected, and+ * so is a body read that ClickHouse would resolve to a CTE declared where+ * the body is read.+ */+export class CteNamePlugin implements KyselyPlugin {+ transformQuery(args: PluginTransformQueryArgs): RootOperationNode {+ assertCteNames(args.node, new Set());+ return args.node;+ }++ async transformResult(args: PluginTransformResultArgs) {+ return args.result;+ }+}++/** `inScope` holds the CTE names visible to the selects under `value`. */+function assertCteNames(value: unknown, inScope: ReadonlySet<string>): void {+ walkNodes(value, (node) => {+ if (!SelectQueryNode.is(node)) return;+ assertSelect(node, inScope);+ return false;+ });+}++function assertSelect(+ node: SelectQueryNode,+ inScope: ReadonlySet<string>,+): void {+ const visible = new Set(inScope);+ for (const cte of node.with?.expressions ?? []) {+ const name = cteName(cte);+ assertNameAllowed(name, visible);+ // The body sees the enclosing CTEs and the earlier siblings only.+ assertCteNames(cte.expression, visible);+ visible.add(name);+ }+ assertReadsResolve(node, visible);+ // Subqueries and UNION branches of this select; the bodies were walked above.+ for (const [key, child] of Object.entries(node)) {+ if (key !== "with") assertCteNames(child, visible);+ }+}++function assertNameAllowed(name: string, visible: ReadonlySet<string>): void {+ if (REGISTRY_TABLES.has(name)) {+ throw new QueryCompileError(+ `A CTE cannot be named "${name}": it is a registry table, and the compile passes read that name as the table. Rename the CTE.`,+ );+ }+ if (visible.has(name)) {+ throw new QueryCompileError(+ `CTE "${name}" is declared twice in one scope: a WITH may not reuse the name of a sibling or of an enclosing CTE. Rename one of them.`,+ );+ }+}++/** Every FROM/JOIN table of `node` is a registry table or a CTE in `visible`. */+function assertReadsResolve(+ node: SelectQueryNode,+ visible: ReadonlySet<string>,+): void {+ for (const { table } of selectRelations(node)) {+ if (table) assertReadResolves(table, visible);+ }+}++function assertReadResolves(+ table: TableNode,+ visible: ReadonlySet<string>,+): void {+ const name = table.table.identifier.name;+ if (table.table.schema) {+ throw new UnscopedRelationError(+ `"${table.table.schema.name}.${name}" is schema-qualified: the registry describes tables by name in the configured database. Read "${name}" unqualified.`,+ );+ }+ if (!REGISTRY_TABLES.has(name) && !visible.has(name)) {+ throw new UnscopedRelationError(+ `"${name}" is neither a registry table nor a CTE in scope. A CTE body reads only CTEs declared before it, so recursive and cyclic CTEs are not supported: declare the CTEs in the order they read each other. To read a physical table, add it to the registry in schema.ts.`,+ );+ }+}++export function cteName(expr: CommonTableExpressionNode): string {+ return expr.name.table.table.identifier.name;+}diff --git a/packages/shared/src/server/query-ast/dedup.test.ts b/packages/shared/src/server/query-ast/dedup.test.tsindex 97d598d137..0d1a07a404 100644--- a/packages/shared/src/server/query-ast/dedup.test.ts+++ b/packages/shared/src/server/query-ast/dedup.test.ts@@ -71,38 +71,6 @@ describe("per-table dedup lowering", () => {expect(sql.toLowerCase().split("limit 1 by").length - 1).toBe(1);});- it("lowers a physical read but not a nested read of an outer CTE of that name", () => {- DEDUP_SPECS.traces = {- strategy: "limitBy",- key: ["id"],- version: "event_ts",- };- try {- const db = getClickhouseKysely();- const { sql } = compile(- db- .with("traces", (qb) =>- qb.selectFrom("scores").select("trace_id as id"),- )- .selectFrom("scores as s")- .select("s.id")- .where("s.trace_id", "in", (eb) =>- eb.selectFrom("traces").select("id"),- )- .where(- "s.trace_id",- "in",- db.withSchema("default").selectFrom("traces").select("id"),- ),- );- expect(sql).toBe(- "with traces as (select trace_id as id from scores where project_id = {p1:String}) select s.id from scores as s where s.project_id = {p1:String} and s.trace_id in (select id from traces) and s.trace_id in (select id from default.traces where project_id = {p1:String} order by event_ts desc limit 1 by id)",- );- } finally {- delete DEDUP_SPECS.traces;- }- });-it("qualifies the version sort and LIMIT BY key with the FROM alias", () => {DEDUP_SPECS.traces = {strategy: "limitBy",@@ -174,6 +142,31 @@ describe("per-table dedup lowering", () => {}});+ it("does not count an aggregate inside a scalar subquery as the outer select's", () => {+ // `count(*)` belongs to the subquery, so the outer read keeps row dedup.+ DEDUP_SPECS.traces = {+ strategy: "limitBy",+ key: ["id"],+ version: "event_ts",+ };+ try {+ const { sql } = compile(+ getClickhouseKysely()+ .selectFrom("traces as t")+ .select((eb) => [+ "t.id",+ eb+ .selectFrom("scores as s")+ .select((inner) => inner.fn.countAll().as("n"))+ .as("n"),+ ]),+ );+ expect(sql).toMatch(/order by t\.event_ts desc limit 1 by t\.id$/);+ } finally {+ delete DEDUP_SPECS.traces;+ }+ });+it("does not rewrite undeclared tables", () => {const { sql } = compile(getClickhouseKysely().selectFrom("traces").select("id"),diff --git a/packages/shared/src/server/query-ast/dedup.ts b/packages/shared/src/server/query-ast/dedup.tsindex 08be4c1d04..644113441d 100644--- a/packages/shared/src/server/query-ast/dedup.ts+++ b/packages/shared/src/server/query-ast/dedup.ts@@ -44,12 +44,13 @@ import { QueryCompileError } from "./errors";import {FinalTableNode,LimitByNode,+ walkNodes,type ClickHouseSelectQueryNode,} from "./nodes";import { describeRelation } from "./relations";import { DEDUP_SPECS, type DedupSpec } from "./schema";import { stampCompiledTree } from "./tenancy";-import { CteScopedTransformer } from "./transformer";+import { SelectRewriteTransformer } from "./transformer";type PhysicalFrom = {tableName: string;@@ -74,19 +75,18 @@ export class DedupLoweringPlugin implements KyselyPlugin {}}-class DedupLoweringTransformer extends CteScopedTransformer {+class DedupLoweringTransformer extends SelectRewriteTransformer {protected rewriteSelect(node: ClickHouseSelectQueryNode,): ClickHouseSelectQueryNode {- return lowerSelect(node, (table) => this.isCte(table));+ return lowerSelect(node);}}function lowerSelect(node: ClickHouseSelectQueryNode,- isCte: (table: TableNode) => boolean,): ClickHouseSelectQueryNode {- const from = singlePhysicalFrom(node, isCte);+ const from = singlePhysicalFrom(node);if (!from) return node;const spec = DEDUP_SPECS[from.tableName];@@ -122,15 +122,12 @@ function assertLowerable(}}-function singlePhysicalFrom(- node: SelectQueryNode,- isCte: (table: TableNode) => boolean,-): PhysicalFrom | undefined {+function singlePhysicalFrom(node: SelectQueryNode): PhysicalFrom | undefined {if (node.joins?.length) return undefined;const froms = node.from?.froms ?? [];if (froms.length !== 1) return undefined;const { node: fromItem, table, alias } = describeRelation(froms[0]);- if (!table || isCte(table)) return undefined;+ if (!table) return undefined;return {tableName: table.table.identifier.name,alias,@@ -182,7 +179,7 @@ function isLimitOne(node: SelectQueryNode): boolean {function hasWrappingAggregate(node: SelectQueryNode): boolean {let found = false;- walk(node.selections, (child) => {+ walkOuterSelect(node.selections, (child) => {if (!AggregateFunctionNode.is(child)) return;if (child.over || DOUBLE_COUNT_AGGREGATES.has(child.func.toLowerCase())) {found = true;@@ -193,7 +190,7 @@ function hasWrappingAggregate(node: SelectQueryNode): boolean {function hasAnyAggregate(node: SelectQueryNode): boolean {let found = false;- walk(node.selections, (child) => {+ walkOuterSelect(node.selections, (child) => {if (AggregateFunctionNode.is(child)) found = true;});return found;@@ -321,23 +318,16 @@ function columnNameOf(node: OperationNode): string | undefined {return undefined;}-function walk(value: unknown, visit: (node: OperationNode) => void): void {- if (!value || typeof value !== "object") return;- if (- "kind" in value &&- typeof (value as { kind: unknown }).kind === "string"- ) {- const node = value as OperationNode;+/**+ * Visits the nodes under `value` but not inside nested selects: those have+ * their own shape, so their aggregates do not belong to the outer query.+ */+function walkOuterSelect(+ value: unknown,+ visit: (node: OperationNode) => void,+): void {+ walkNodes(value, (node) => {visit(node);- // Nested selects have their own shape; do not attribute their- // aggregates to the outer query.- if (SelectQueryNode.is(node)) return;- }- for (const child of Object.values(value)) {- if (Array.isArray(child)) {- for (const item of child) walk(item, visit);- } else {- walk(child, visit);- }- }+ return !SelectQueryNode.is(node);+ });}diff --git a/packages/shared/src/server/query-ast/eventsRouting.test.ts b/packages/shared/src/server/query-ast/eventsRouting.test.tsindex c083088a76..d4568fceea 100644--- a/packages/shared/src/server/query-ast/eventsRouting.test.ts+++ b/packages/shared/src/server/query-ast/eventsRouting.test.ts@@ -237,26 +237,6 @@ describe("logical events routing", () => {).toThrow(QueryCompileError);});- it("gives an unaliased schema-qualified table no qualifier name", () => {- // `default.events_core` is named by neither `events_core` nor- // `default.events_core` here, so the read is rejected, not resolved.- expect(() =>- compile(- db- .selectFrom("events as e")- .select("e.span_id")- .where(- "e.span_id",- "in",- db- .withSchema("default")- .selectFrom("events_core")- .select(sql.ref<string>("events_core.input").as("x")),- ),- ),- ).toThrow(/Qualify by the relation's alias/);- });-it("rejects FINAL around the logical relation", () => {// useFinal() refuses `events` (dedup: none), so FINAL is wrapped by hand;// routing would otherwise swap the table and drop the FINAL silently.@@ -495,35 +475,6 @@ describe("logical events routing", () => {),).toEqual(["events_core"]);});-- it("rejects a CTE named events", () => {- expect(() =>- compile(- db- .with("events", (qb) =>- qb.selectFrom("events_core").select("span_id"),- )- .selectFrom("events")- .select("span_id"),- ),- ).toThrow(QueryCompileError);- });-- it.each(["events_core", "events_full"] as const)(- "rejects a CTE named %s alongside a logical relation",- (name) => {- // The routed relation would read the CTE instead of the table.- const withCte = db.with(name, (qb) =>- qb.selectFrom("scores as s").select("s.id"),- );- expect(() =>- compile(withCte.selectFrom("events as e").select("e.span_id")),- ).toThrow(QueryCompileError);- expect(() =>- compile(withCte.selectFrom(name).select("id")),- ).not.toThrow();- },- );});describe("logical events routing of metadata predicates", () => {diff --git a/packages/shared/src/server/query-ast/eventsRouting.ts b/packages/shared/src/server/query-ast/eventsRouting.tsindex 2365deffc0..c0681dd1b9 100644--- a/packages/shared/src/server/query-ast/eventsRouting.ts+++ b/packages/shared/src/server/query-ast/eventsRouting.ts@@ -33,7 +33,6 @@ import {AliasNode,BinaryOperationNode,ColumnNode,- CommonTableExpressionNode,FromNode,FunctionNode,IdentifierNode,@@ -76,7 +75,6 @@ export class EventsTableRoutingPlugin implements KyselyPlugin {const analysis = new EventsReadAnalysis();analysis.transformNode(tree);if (!analysis.readsLogicalEvents) return args.node;- rejectPhysicalEventsCte(analysis);return new EventsTableRouter(analysis.fullRelations).transformNode(tree,) as RootOperationNode;@@ -87,22 +85,12 @@ export class EventsTableRoutingPlugin implements KyselyPlugin {}}-function rejectPhysicalEventsCte(analysis: EventsReadAnalysis): void {- if (analysis.physicalEventsCte) {- throw new QueryCompileError(- `A CTE named "${analysis.physicalEventsCte}" would capture the routed logical "${LOGICAL_EVENTS}" relation; rename the CTE.`,- );- }-}-/*** Read-only walk that resolves column references to relations; the* transformer is used for its traversal only.*/class EventsReadAnalysis extends ClickHouseOperationNodeTransformer {readsLogicalEvents = false;- /** Name of a CTE that shadows `events_core` / `events_full`. */- physicalEventsCte: string | undefined;/** Logical relations that read a truncated column. */readonly fullRelations = new Set<OperationNode>();/** Relations of each enclosing select, innermost last. */@@ -117,7 +105,6 @@ class EventsReadAnalysis extends ClickHouseOperationNodeTransformer {this.transformNode(lookup.index, queryId);return node;}- if (CommonTableExpressionNode.is(node)) this.checkCteName(node);return super.transformNodeImpl(node, queryId);}@@ -247,16 +234,6 @@ class EventsReadAnalysis extends ClickHouseOperationNodeTransformer {return undefined;}- private checkCteName(node: CommonTableExpressionNode): void {- const name = node.name.table.table.identifier.name;- if (name === LOGICAL_EVENTS) {- throw new QueryCompileError(- `A CTE cannot be named "${LOGICAL_EVENTS}": the name is the logical events relation.`,- );- }- if (PHYSICAL_EVENTS.has(name)) this.physicalEventsCte = name;- }-/** `columns(β¦)` expands like `*`. */private rejectColumnsMatcher(func: string): void {if (func.toLowerCase() === "columns") this.rejectStar(`${func}(β¦)`);@@ -330,8 +307,8 @@ class EventsTableRouter extends ClickHouseOperationNodeTransformer {}/**- * Named by its alias, else by its table name unless schema-qualified:- * `events as e` β `e`, `events` β `events`, `default.events_core` β none.+ * Named by its alias, else by its table name: `events as e` β `e`,+ * `events` β `events`. Schema-qualified names never get here (`ctes.ts`).*/function routingRelation({node,@@ -341,19 +318,13 @@ function routingRelation({}: Relation): RoutingRelation {return {node,- name:- alias ??- (table && !table.table.schema ? table.table.identifier.name : undefined),+ name: alias ?? table?.table.identifier.name,logical: isLogicalEventsTable(source),};}function isLogicalEventsTable(node: OperationNode): boolean {- return (- TableNode.is(node) &&- node.table.schema === undefined &&- node.table.identifier.name === LOGICAL_EVENTS- );+ return TableNode.is(node) && node.table.identifier.name === LOGICAL_EVENTS;}/** `count(*)` with any `distinct`, `filter`, `over`, β¦ clause. */diff --git a/packages/shared/src/server/query-ast/extensions.test.ts b/packages/shared/src/server/query-ast/extensions.test.tsindex 805accf6fd..d5491b795b 100644--- a/packages/shared/src/server/query-ast/extensions.test.ts+++ b/packages/shared/src/server/query-ast/extensions.test.ts@@ -58,6 +58,16 @@ describe("schema-typed selection (condition 7)", () => {);expect(() => compileClickhouseQuery(qb, projectId)).toThrow(/environment/);});++ it("does not type a column named like an Object prototype key", () => {+ // `constructor` is no registry column, so the pass has no type to reject.+ const db = getClickhouseKysely();+ const qb = db+ .selectFrom("scores")+ .select((eb) => [eb.fn.sum(db.dynamic.ref("constructor")).as("s")]);++ expect(() => compileClickhouseQuery(qb, projectId)).not.toThrow();+ });});describe("metadata subscript access (condition 7b)", () => {diff --git a/packages/shared/src/server/query-ast/nodes.test.ts b/packages/shared/src/server/query-ast/nodes.test.tsindex 85fd0fd874..4e09924811 100644--- a/packages/shared/src/server/query-ast/nodes.test.ts+++ b/packages/shared/src/server/query-ast/nodes.test.ts@@ -1,15 +1,19 @@import {AliasNode,+ CommonTableExpressionNode,FromNode,SelectQueryNode,+ WithNode,createQueryId,sql,type KyselyPlugin,+ type OperationNode,} from "kysely";import { describe, expect, it } from "vitest";import { compileClickhouseQuery } from "./compile";import { ClickHouseQueryCompiler } from "./compiler";+import { CteNamePlugin } from "./ctes";import { getClickhouseKysely } from "./dialect";import { QueryCompileError, UnscopedRelationError } from "./errors";import { arrayJoin, limitBy, mapKeys, mapValues, useFinal } from "./extensions";@@ -21,6 +25,7 @@ import {isClickHouseSelectQueryNode,} from "./nodes";import { TenancyInjectionPlugin } from "./tenancy";+import { defineView, fromView } from "./views";const projectId = "proj-1";@@ -213,6 +218,18 @@ describe("tenancy injection", () => {);});+ it("rejects a table reference inside raw SQL outside FROM/JOIN", () => {+ // `s.trace_id IN traces` reads every row of `traces`, unscoped.+ const qb = getClickhouseKysely()+ .selectFrom("scores as s")+ .select("s.id")+ .where(sql<boolean>`s.trace_id IN ${sql.table("traces")}`);++ expect(() => compileClickhouseQuery(qb, projectId)).toThrow(+ /A table reference inside raw SQL/,+ );+ });+it("does not treat a copied langfuseTenancy property as a valid stamp", () => {const qb = getClickhouseKysely().selectFrom("traces").select("environment");const forged = {@@ -309,478 +326,368 @@ describe("tenancy injection", () => {expect(matches.length).toBeGreaterThanOrEqual(2);});- describe("CTE named like a tenanted table", () => {+ describe("CTE names", () => {const db = getClickhouseKysely();- const tracesCte = db.with("traces", (qb) =>- qb.selectFrom("scores").select("trace_id as id"),- );+ // Knows the CTE names `a` and `b` as tables, so a body can read a name+ // before its CTE is declared without a cast.+ const loose = db.withTables<{+ a: { id: string };+ b: { id: string };+ }>();const compile = (query: Parameters<typeof compileClickhouseQuery>[0],): string => compileClickhouseQuery(query, projectId).sql;- it("leaves a scalar subquery's read of an outer CTE unscoped", () => {- expect(- compile(- tracesCte- .selectFrom("scores as s")- .select((eb) => [- eb.selectFrom("traces").select("id").limit(1).as("first_id"),- ]),- ),- ).toBe(- "with traces as (select trace_id as id from scores where project_id = {p1:String}) select (select id from traces limit {p2:Int64}) as first_id from scores as s where s.project_id = {p1:String}",- );- });-- it("leaves IN-subquery, FROM-subquery and UNION reads of an outer CTE unscoped", () => {- expect(- compile(- tracesCte- .selectFrom((eb) => eb.selectFrom("traces").select("id").as("sub"))- .select("sub.id")- .where("sub.id", "in", (eb) => eb.selectFrom("traces").select("id"))- .unionAll(db.selectFrom("traces").select("id")),- ),- ).toBe(- "with traces as (select trace_id as id from scores where project_id = {p1:String}) select sub.id from (select id from traces) as sub where sub.id in (select id from traces) union all select id from traces",- );- });+ // Every pass treats a FROM/JOIN of a registry name as that table, so a+ // CTE of that name is rejected whether or not the query reads it. One+ // physical table, plus the logical `events` that has no table of its own.+ it.each(["traces", "events"] as const)(+ "rejects a CTE named like the registry table %s",+ (name) => {+ expect(() =>+ compile(+ db+ .with(name, (qb) => qb.selectFrom("scores").select("trace_id"))+ .selectFrom("scores")+ .select("id"),+ ),+ ).toThrow(`A CTE cannot be named "${name}"`);+ },+ );- it("leaves a nested CTE body's read of an outer CTE unscoped", () => {- expect(+ it("rejects a nested WITH named like a registry table", () => {+ // The WITH is declared in a FROM subquery, not at the root.+ expect(() =>compile(- tracesCte+ db.selectFrom(db- .with("inner", (qb) => qb.selectFrom("traces").select("id"))- .selectFrom("inner")+ .with("traces", (qb) =>+ qb.selectFrom("scores").select("trace_id as id"),+ )+ .selectFrom("traces").select("id").as("sub"),).select("sub.id"),),- ).toBe(- "with traces as (select trace_id as id from scores where project_id = {p1:String}) select sub.id from (with inner as (select id from traces) select id from inner) as sub",- );+ ).toThrow(`A CTE cannot be named "traces"`);});- it("scopes a CTE body's read of its own name, which is the table", () => {- expect(- compile(- db- .with("traces", (qb) =>- qb- .selectFrom("traces")- .select("id")- .where("id", "in", (eb) =>- eb.selectFrom("traces").select("id"),- ),- )- .selectFrom("traces")- .select("id"),- ),- ).toBe(- "with traces as (select id from traces where project_id = {p1:String} and id in (select id from traces where project_id = {p1:String})) select id from traces",+ it("rejects a view named like a registry table", () => {+ // `fromView` declares the view as a CTE of its name.+ const view = defineView("traces")<{ id: string }>(() =>+ db.selectFrom("scores").select("trace_id as id"),+ );+ expect(() => compile(fromView(view).select("id"))).toThrow(+ `A CTE cannot be named "traces"`,);});- it("resolves a nested CTE body's own name to an outer CTE of that name", () => {- expect(- compile(- tracesCte+ it.each([+ [+ // Two siblings of one WITH.+ "a sibling",+ () =>+ loose+ .with("a", (qb) => qb.selectFrom("scores").select("id"))+ .with("a", (qb) => qb.selectFrom("traces").select("id"))+ .selectFrom("a")+ .select("id"),+ ],+ [+ // A FROM subquery redeclares the root's `a`.+ "an enclosing select's CTE",+ () =>+ loose+ .with("a", (qb) => qb.selectFrom("scores").select("id")).selectFrom(- db- .with("traces", (qb) => qb.selectFrom("traces").select("id"))- .selectFrom("traces")+ loose+ .with("a", (qb) => qb.selectFrom("traces").select("id"))+ .selectFrom("a").select("id").as("sub"),).select("sub.id"),- ),- ).toBe(- "with traces as (select trace_id as id from scores where project_id = {p1:String}) select sub.id from (with traces as (select id from traces) select id from traces) as sub",- );+ ],+ ])("rejects a CTE that reuses the name of %s", (_, build) => {+ expect(() => compile(build())).toThrow(`CTE "a" is declared twice`);});- it("resolves earlier and later sibling CTE names to the CTE", () => {+ it("accepts the same CTE name in two separate subqueries", () => {+ // Neither `x` is in scope of the other.+ const sub = db+ .with("x", (qb) => qb.selectFrom("traces").select("id"))+ .selectFrom("x")+ .select("id");expect(compile(db- .with("before", (qb) => qb.selectFrom("traces").select("id"))- .with("traces", (qb) =>- qb.selectFrom("scores").select("trace_id as id"),- )- .with("after", (qb) => qb.selectFrom("traces").select("id"))- .selectFrom("before")- .innerJoin("after", (join) =>- join.onRef("before.id", "=", "after.id"),- )- .select("before.id"),+ .selectFrom(sub.as("l"))+ .innerJoin(sub.as("r"), "l.id", "r.id")+ .select("l.id"),),).toBe(- "with before as (select id from traces), traces as (select trace_id as id from scores where project_id = {p1:String}), after as (select id from traces) select before.id from before inner join after on before.id = after.id",+ "select l.id from (with x as (select id from traces where project_id = {p1:String}) select id from x) as l inner join (with x as (select id from traces where project_id = {p1:String}) select id from x) as r on l.id = r.id",);});- // Each case is a cycle between sibling CTEs. Reading `traces` reads- // `observations`, which reads `traces` while it is still being resolved,- // so ClickHouse reads the physical `traces` table there. The pass treats- // that read as the CTE and would leave it unscoped, so the cycle is- // rejected whether or not the main select reads it.+ // A CTE body sees only the CTEs declared before it. Reading its own name+ // or a later sibling is the only way to build a cycle, and ClickHouse+ // resolves a read inside a cycle to a table, not the CTE.it.each([[- // traces β observations β traces.- "directly",+ "its own name",() =>- db- .with("traces", (qb) =>- qb.selectFrom("observations").select("trace_id as id"),- )- .with("observations", (qb) =>- qb.selectFrom("traces").select("id as trace_id"),- )- .selectFrom("traces")+ loose+ .with("a", (qb) => qb.selectFrom("a").select("id"))+ .selectFrom("a").select("id"),],[- // traces β observations (in an IN subquery of its body) β traces.- "through a nested subquery",+ "its own name in an IN subquery",() =>- db- .with("traces", (qb) =>+ loose+ .with("a", (qb) =>qb- .selectFrom("scores")- .select("trace_id as id")- .where("trace_id", "in", (eb) =>- eb.selectFrom("observations").select("trace_id"),- ),- )- .with("observations", (qb) =>- qb.selectFrom("traces").select("id as trace_id"),+ .selectFrom("traces")+ .select("id")+ .where("id", "in", (eb) => eb.selectFrom("a").select("id")),)- .selectFrom("scores")+ .selectFrom("a").select("id"),],[- // traces β observations β scores β traces.- "through a third CTE",+ "its own name under an alias",() =>- db- .with("traces", (qb) =>- qb.selectFrom("observations").select("trace_id as id"),- )- .with("observations", (qb) =>- qb.selectFrom("scores").select("trace_id"),- )- .with("scores", (qb) =>- qb.selectFrom("traces").select("id as trace_id"),- )- .selectFrom("traces")+ loose+ .with("a", (qb) => qb.selectFrom("a as x").select("x.id"))+ .selectFrom("a").select("id"),],[- // traces β observations (read as `o`) β traces.- "through an aliased read",+ "its own name on the joined side",() =>- db- .with("traces", (qb) =>- qb.selectFrom("observations as o").select("o.trace_id as id"),- )- .with("observations", (qb) =>- qb.selectFrom("traces").select("id as trace_id"),+ loose+ .with("a", (qb) =>+ qb+ .selectFrom("traces as t")+ .innerJoin("a", "a.id", "t.id")+ .select("t.id"),)- .selectFrom("traces")+ .selectFrom("a").select("id"),],[- // traces β scores (read with FINAL) β traces.- "through a FINAL read",+ "a later sibling",() =>- db- .with("traces", (qb) =>- qb- .selectFrom("scores")- .select("trace_id as id")- .$call(useFinal(["scores"])),- )- .with("scores", (qb) =>- qb.selectFrom("traces").select("id as trace_id"),- )- .selectFrom("traces")+ loose+ .with("a", (qb) => qb.selectFrom("b").select("id"))+ .with("b", (qb) => qb.selectFrom("traces").select("id"))+ .selectFrom("a").select("id"),],[- // traces β observations (the joined side) β traces.- "through a JOIN",+ "itself through a recursive CTE",() =>- db- .with("traces", (qb) =>- qb- .selectFrom("scores as s")- .innerJoin("observations as o", "o.trace_id", "s.trace_id")- .select("s.trace_id as id"),- )- .with("observations", (qb) =>- qb.selectFrom("traces").select("id as trace_id"),- )- .selectFrom("traces")- .select("id"),- ],- ])("rejects sibling CTEs that read each other %s", (_, build) => {- expect(() => compile(build())).toThrow(UnscopedRelationError);- });-- it("scopes a recursive CTE body's read of its own name as the table", () => {- expect(- compile(- db- .withRecursive("traces(id)", (qb) =>- qb.selectFrom("traces").select("id"),- )- .selectFrom("traces")- .select("id"),- ),- ).toBe(- "with recursive traces(id) as (select id from traces where project_id = {p1:String}) select id from traces",- );- });-- it("rejects a CTE body written as raw SQL that reads a relation", () => {- expect(() =>- compile(- db- .with("a", () => sql<{ id: string }>`select id from traces`)+ loose+ .withRecursive("a(id)", (qb) => qb.selectFrom("a").select("id")).selectFrom("a").select("id"),- ),- ).toThrow(UnscopedRelationError);- });-- it("rejects a nested WITH that redirects an enclosing CTE body's read", () => {- // Read inside `sub`, `traces` reads the inner `observations`, which reads- // `a`, which reads `traces` while it is still being resolved: the table.- expect(() =>- compile(- db- .with("traces", (qb) =>- qb.selectFrom("observations").select("trace_id as id"),- )- .with("observations", (qb) =>- qb.selectFrom("scores").select("trace_id"),- )- .with("a", (qb) => qb.selectFrom("traces").select("id"))+ ],+ [+ // `a` is read inside `sub`, where ClickHouse would resolve its `b`+ // to the nested `b`; where `a` is declared, `b` names nothing.+ "a CTE declared only where the body is read",+ () =>+ loose+ .with("a", (qb) => qb.selectFrom("b").select("id")).selectFrom(- db- .withTables<{ a: { id: string } }>()- .with("observations", (qb) =>- qb.selectFrom("a").select("id as trace_id"),- )- .selectFrom("traces")+ loose+ .with("b", (qb) => qb.selectFrom("traces").select("id"))+ .selectFrom("a").select("id").as("sub"),).select("sub.id"),- ),- ).toThrow(UnscopedRelationError);+ ],+ ])("rejects a CTE body that reads %s", (_, build) => {+ expect(() => compile(build())).toThrow(UnscopedRelationError);+ expect(() => compile(build())).toThrow(+ "is neither a registry table nor a CTE in scope",+ );});- it("rejects a nested WITH that redirects a CTE body's own-name read", () => {- // The `traces` CTE inside `a` reads its own name, which escapes to the- // `traces` CTE in scope where `a` is read.- expect(() =>- compile(- db- .with("a", (qb) =>- qb- .selectFrom(- db- .with("traces", (inner) =>- inner.selectFrom("traces").select("id"),- )- .selectFrom("traces")- .select("id")- .as("sub"),- )- .select("sub.id"),- )- .selectFrom(- db- .withTables<{ a: { id: string } }>()- .with("traces", (qb) =>- qb.selectFrom("scores").select("trace_id as id"),- )- .selectFrom("a")- .select("id")- .as("u"),- )- .select("u.id"),+ it("rejects a CTE body that reads itself through FINAL", () => {+ // `useFinal` takes registry tables only, so the FINAL read is built by+ // hand. FinalTableNode is outside Kysely's node union, hence the cast.+ const node = loose+ .with("a", (qb) => qb.selectFrom("a").select("id"))+ .selectFrom("a")+ .select("id")+ .toOperationNode();+ const cte = node.with?.expressions[0];+ const body = cte?.expression;+ if (!cte || !body || !SelectQueryNode.is(body) || !body.from) {+ throw new Error("not a select");+ }+ const final = FinalTableNode.create(body.from.froms[0]) as unknown;+ const finalBody: SelectQueryNode = {+ ...body,+ from: FromNode.create([final as OperationNode]),+ };+ const withFinal = {+ ...node,+ with: WithNode.create(+ CommonTableExpressionNode.create(cte.name, finalBody),),- ).toThrow(UnscopedRelationError);+ };+ const check = () =>+ new CteNamePlugin().transformQuery({+ node: withFinal,+ queryId: createQueryId(),+ });+ expect(check).toThrow(UnscopedRelationError);+ expect(check).toThrow(+ `"a" is neither a registry table nor a CTE in scope`,+ );});- it("accepts a CTE body that reads its own nested CTE", () => {+ it.each(["traces ", "default.recent"])(+ "rejects the CTE name %j, which is not a plain identifier",+ (name) => {+ // The compiler writes a CTE name verbatim, so `traces ` would reach+ // ClickHouse as `traces` while the passes read `traces` as the table.+ expect(() =>+ compile(+ db+ .with(name, (qb) => qb.selectFrom("scores").select("trace_id"))+ .selectFrom("scores")+ .select("id"),+ ),+ ).toThrow("is not a plain identifier");+ },+ );++ it("accepts a nested WITH that reuses the name of the CTE whose body it is in", () => {+ // Inside its own body `a` is not in scope, so the nested `a` is the+ // only `a` the body can read.expect(compile(- db- .with("a", (qb) =>- qb- .selectFrom(- db- .with("x", (inner) =>- inner.selectFrom("scores").select("trace_id"),- )- .selectFrom("x")- .select("trace_id")- .as("sub"),- )- .select("sub.trace_id"),+ loose+ .with("a", () =>+ loose+ .with("a", (qb) => qb.selectFrom("traces").select("id"))+ .selectFrom("a")+ .select("id"),).selectFrom("a")- .select("trace_id"),+ .select("id"),),).toBe(- "with a as (select sub.trace_id from (with x as (select trace_id from scores where project_id = {p1:String}) select trace_id from x) as sub) select trace_id from a",+ "with a as (with a as (select id from traces where project_id = {p1:String}) select id from a) select id from a",);});- it("accepts a CTE body that reads its own nested CTE from a deeper subquery", () => {+ it("accepts reads of an earlier sibling, an enclosing CTE and a CTE of the body's own WITH", () => {expect(compile(db- .with("a", (qb) =>+ .with("x", (qb) => qb.selectFrom("traces").select("id"))+ .with("y", (qb) =>qb- .selectFrom(- db- .with("x", (inner) =>- inner.selectFrom("scores").select("trace_id"),- )- .selectFrom((eb) =>- eb.selectFrom("x").select("trace_id").as("s2"),- )- .select("s2.trace_id")- .as("sub"),- )- .select("sub.trace_id"),+ .selectFrom("x")+ .select("id")+ .where(+ "id",+ "in",+ qb+ .with("z", (inner) => inner.selectFrom("x").select("id"))+ .selectFrom("z")+ .select("id"),+ ),)- .selectFrom("a")- .select("trace_id"),+ .selectFrom("y")+ .select("id"),),).toBe(- "with a as (select sub.trace_id from (with x as (select trace_id from scores where project_id = {p1:String}) select s2.trace_id from (select trace_id from x) as s2) as sub) select trace_id from a",+ "with x as (select id from traces where project_id = {p1:String}), y as (select id from x where id in (with z as (select id from x) select id from z)) select id from y",);});- it("accepts a nested CTE body that reads an enclosing nested CTE", () => {- expect(+ it("rejects a schema-qualified read, even of a registry table", () => {+ // The registry describes `traces`, not `default.traces` or+ // `otherdb.traces`; the read is in a CTE body, so the walk reaches it.+ const build = () =>compile(db.with("a", (qb) =>- qb- .selectFrom(- db- .with("x", (inner) =>- inner.selectFrom("scores").select("trace_id"),- )- .selectFrom(- db- .withTables<{ x: { trace_id: string } }>()- .with("y", (inner) =>- inner.selectFrom("x").select("trace_id"),- )- .selectFrom("y")- .select("trace_id")- .as("s2"),- )- .select("s2.trace_id")- .as("sub"),- )- .select("sub.trace_id"),+ qb.withSchema("otherdb").selectFrom("traces").select("id"),).selectFrom("a")- .select("trace_id"),- ),- ).toBe(- "with a as (select sub.trace_id from (with x as (select trace_id from scores where project_id = {p1:String}) select s2.trace_id from (with y as (select trace_id from x) select trace_id from y) as s2) as sub) select trace_id from a",- );+ .select("id"),+ );+ expect(build).toThrow(UnscopedRelationError);+ expect(build).toThrow(`"otherdb.traces" is schema-qualified`);});- it("does not count a schema-qualified read as a sibling read", () => {- expect(- compile(- db- .with("traces", (qb) =>- qb- .withSchema("default")- .selectFrom("observations")- .select("trace_id as id"),- )- .with("observations", (qb) =>- qb.selectFrom("traces").select("id as trace_id"),- )- .selectFrom("observations")- .select("trace_id"),- ),- ).toBe(- "with traces as (select trace_id as id from default.observations where project_id = {p1:String}), observations as (select id as trace_id from traces) select trace_id from observations",- );+ it("has the compiler reject a table outside the registry that skipped the CTE check", () => {+ // Only tenancy runs here, so the unregistered `a` reaches the compiler,+ // which must not guess its column types.+ const node = loose+ .selectFrom("a")+ .select("id")+ .where("id", "=", "x")+ .toOperationNode();+ const stamped = new TenancyInjectionPlugin(projectId).transformQuery({+ node,+ queryId: createQueryId(),+ });+ const check = () =>+ new ClickHouseQueryCompiler().compileQuery(stamped, createQueryId());+ expect(check).toThrow(UnscopedRelationError);+ expect(check).toThrow(`"a" is not a registry table`);});- it("does not count a column qualifier named like a sibling as a read", () => {+ it("accepts a CTE named like an Object prototype key", () => {+ // The registry lookups of the soft-delete and dedup passes must not+ // find `Object.prototype.constructor` for it.expect(compile(db- .with("a", (qb) =>- qb.selectFrom("scores as traces").select("traces.id"),- )- .with("traces", (qb) => qb.selectFrom("a").select("id"))- .selectFrom("traces")+ .with("constructor", (qb) => qb.selectFrom("traces").select("id"))+ .selectFrom("constructor").select("id"),),).toBe(- "with a as (select traces.id from scores as traces where traces.project_id = {p1:String}), traces as (select id from a) select id from traces",+ "with constructor as (select id from traces where project_id = {p1:String}) select id from constructor",);});- it("scopes a schema-qualified table named like a CTE in scope", () => {+ it("does not count a column qualifier as a read", () => {+ // `b.id` qualifies a column of the alias `b`; the body reads `traces`.expect(compile(- tracesCte- .selectFrom("traces")- .select("id")- .where(- "id",- "in",- db.withSchema("default").selectFrom("traces").select("id"),- ),+ loose+ .with("a", (qb) => qb.selectFrom("traces as b").select("b.id"))+ .with("b", (qb) => qb.selectFrom("traces").select("id"))+ .selectFrom("a")+ .select("id"),),).toBe(- "with traces as (select trace_id as id from scores where project_id = {p1:String}) select id from traces where id in (select id from default.traces where project_id = {p1:String})",+ "with a as (select b.id from traces as b where b.project_id = {p1:String}), b as (select id from traces where project_id = {p1:String}) select id from a",);});- it("scopes the table again once a subquery leaves the CTE's select", () => {- expect(+ it("rejects a CTE body written as raw SQL that reads a relation", () => {+ expect(() =>compile(db- .selectFrom("scores as s")- .select("s.id")- .where(- "s.trace_id",- "in",- db- .with("traces", (qb) =>- qb.selectFrom("scores").select("trace_id as id"),- )- .selectFrom("traces")- .select("id"),- )- .where("s.trace_id", "in", (eb) =>- eb.selectFrom("traces").select("id"),- ),+ .with("a", () => sql<{ id: string }>`select id from traces`)+ .selectFrom("a")+ .select("id"),),- ).toBe(- "select s.id from scores as s where s.project_id = {p1:String} and s.trace_id in (with traces as (select trace_id as id from scores where project_id = {p1:String}) select id from traces) and s.trace_id in (select id from traces where project_id = {p1:String})",- );+ ).toThrow(UnscopedRelationError);});});});diff --git a/packages/shared/src/server/query-ast/nodes.ts b/packages/shared/src/server/query-ast/nodes.tsindex 6eadb3744f..2b96249d99 100644--- a/packages/shared/src/server/query-ast/nodes.ts+++ b/packages/shared/src/server/query-ast/nodes.ts@@ -169,3 +169,19 @@ export function finalTable(table: OperationNode): OperationNode {export function unwrapFinalTable(node: { kind: string }): OperationNode {return (node as unknown as FinalTableNode).table;}++/**+ * Calls `visit` on every node object under `value`, ClickHouse extra fields+ * included, parents before children. `visit` returns `false` to skip a+ * node's children.+ */+export function walkNodes(+ value: unknown,+ visit: (node: OperationNode) => boolean | void,+): void {+ if (!value || typeof value !== "object") return;+ if ("kind" in value && typeof value.kind === "string") {+ if (visit(value as OperationNode) === false) return;+ }+ for (const child of Object.values(value)) walkNodes(child, visit);+}diff --git a/packages/shared/src/server/query-ast/schema.ts b/packages/shared/src/server/query-ast/schema.tsindex 86c2b76d1e..0679474407 100644--- a/packages/shared/src/server/query-ast/schema.ts+++ b/packages/shared/src/server/query-ast/schema.ts@@ -336,7 +336,7 @@ export type ClickHouseDatabase = {export const TABLE_COLUMN_TYPES: Record<string,Record<string, ChColumnType>-> = Object.fromEntries(+> = lookupTable(Object.entries(TABLE_REGISTRY).filter(([table]) => table !== "events").map(([table, spec]) => [table, spec.columns]),@@ -347,15 +347,19 @@ export const TABLE_COLUMN_TYPES: Record<* the aggregate type-check pass. Column names are consistent across tables in* this schema, so a flat map is unambiguous.*/-export const COLUMN_DATA_TYPES: Record<string, ColumnDataType> =- Object.fromEntries(- Object.values(TABLE_REGISTRY).flatMap((table) =>- Object.entries(table.columns).map(([name, chType]) => [- name,- RUNTIME_TYPE[chType],- ]),- ),- );+export const COLUMN_DATA_TYPES: Record<string, ColumnDataType> = lookupTable(+ Object.values(TABLE_REGISTRY).flatMap((table) =>+ Object.entries(table.columns).map(([name, chType]) => [+ name,+ RUNTIME_TYPE[chType],+ ]),+ ),+);++/** Every registry relation name, the logical `events` included. */+export const REGISTRY_TABLES: ReadonlySet<string> = new Set(+ Object.keys(TABLE_REGISTRY),+);/** Derived set of relations the tenancy pass must scope. */export const TENANTED_TABLES = new Set<string>(@@ -365,14 +369,14 @@ export const TENANTED_TABLES = new Set<string>();/** Derived per-table read idioms, keyed by table name. */-export const DEDUP_SPECS: Record<string, DedupSpec> = Object.fromEntries(+export const DEDUP_SPECS: Record<string, DedupSpec> = lookupTable(Object.entries(TABLE_REGISTRY).flatMap(([name, spec]) =>spec.dedup ? [[name, spec.dedup]] : [],),);/** Derived soft-delete flag column, keyed by table name. */-export const SOFT_DELETE_COLUMNS: Record<string, string> = Object.fromEntries(+export const SOFT_DELETE_COLUMNS: Record<string, string> = lookupTable(Object.entries(TABLE_REGISTRY).flatMap(([name, spec]) =>spec.softDelete ? [[name, spec.softDelete]] : [],),@@ -407,10 +411,10 @@ const BIND_TYPE: Record<ChColumnType, string> = {export const TABLE_BIND_TYPES: Record<string,Record<string, string>-> = Object.fromEntries(+> = lookupTable(Object.entries(TABLE_REGISTRY).map(([table, spec]) => [table,- Object.fromEntries(+ lookupTable(Object.entries(spec.columns).map(([name, chType]) => [name,BIND_TYPE[chType],@@ -418,3 +422,14 @@ export const TABLE_BIND_TYPES: Record<),]),);++/**+ * `Object.fromEntries` without `Object.prototype`, so a lookup by a name such+ * as `constructor` or `toString` finds nothing instead of an inherited+ * function: `lookupTable([["traces", 1]]).constructor` β undefined.+ */+export function lookupTable<V>(+ entries: Iterable<readonly [string, V]>,+): Record<string, V> {+ return Object.assign(Object.create(null), Object.fromEntries(entries));+}diff --git a/packages/shared/src/server/query-ast/softDelete.test.ts b/packages/shared/src/server/query-ast/softDelete.test.tsindex 087b1bb6d3..4b2f79e786 100644--- a/packages/shared/src/server/query-ast/softDelete.test.ts+++ b/packages/shared/src/server/query-ast/softDelete.test.ts@@ -112,78 +112,6 @@ describe("soft-delete filtering", () => {);});- it("treats a CTE named like a physical table as the CTE, in nested scopes too", () => {- expect(- compile(- getClickhouseKysely()- .with("events_core", (qb) =>- qb.selectFrom("scores").select("trace_id"),- )- .selectFrom("events_core")- .select("trace_id")- .where("trace_id", "in", (eb) =>- eb.selectFrom("events_core").select("trace_id"),- ),- ),- ).not.toContain("is_deleted");- });-- it("filters a schema-qualified table named like a CTE in scope", () => {- const db = getClickhouseKysely();- expect(- compile(- db- .with("events_core", (qb) =>- qb.selectFrom("scores").select("trace_id"),- )- .selectFrom("events_core")- .select("trace_id")- .where(- "trace_id",- "in",- db- .withSchema("default")- .selectFrom("events_core")- .select("trace_id"),- ),- ),- ).toContain(- "(select trace_id from default.events_core where project_id = {p1:String} and events_core.is_deleted = {p2:UInt8})",- );- });-- it("resolves a CTE body's own name to the table, and sibling CTE names to the CTE", () => {- const sql = compile(- getClickhouseKysely()- .with("before", (qb) => qb.selectFrom("events_core").select("trace_id"))- .with("events_core", (qb) =>- qb.selectFrom("events_core").select("trace_id"),- )- .with("after", (qb) => qb.selectFrom("events_core").select("trace_id"))- .selectFrom("after")- .select("trace_id"),- );- expect(sql).toContain(- "events_core as (select trace_id from events_core where project_id = {p1:String} and events_core.is_deleted = {p2:UInt8})",- );- expect(sql.match(/is_deleted/g)).toHaveLength(1);- });-- it("filters a recursive CTE body that reads its own name, which is the table outside the recursive member", () => {- expect(- compile(- getClickhouseKysely()- .withRecursive("events_core(trace_id)", (qb) =>- qb.selectFrom("events_core").select("trace_id"),- )- .selectFrom("events_core")- .select("trace_id"),- ),- ).toContain(- "(select trace_id from events_core where project_id = {p1:String} and events_core.is_deleted = {p2:UInt8})",- );- });-it("filters a soft-delete table wrapped in FINAL", () => {// No soft-delete table allows FINAL yet, so the node is built by hand.const node = getClickhouseKysely()@@ -297,10 +225,8 @@ describe("soft-delete filtering", () => {expect(() =>compile(getClickhouseKysely()- .with("events_core", (qb) =>- qb.selectFrom("scores").select("trace_id"),- )- .selectFrom("events_core")+ .with("ev", (qb) => qb.selectFrom("events_core").select("trace_id"))+ .selectFrom("ev").$call(includeDeleted()).select("trace_id"),),diff --git a/packages/shared/src/server/query-ast/softDelete.ts b/packages/shared/src/server/query-ast/softDelete.tsindex c837bb3604..02933e6179 100644--- a/packages/shared/src/server/query-ast/softDelete.ts+++ b/packages/shared/src/server/query-ast/softDelete.ts@@ -38,7 +38,7 @@ import { QueryCompileError } from "./errors";import { type ClickHouseSelectQueryNode } from "./nodes";import { describeRelation } from "./relations";import { SOFT_DELETE_COLUMNS } from "./schema";-import { CteScopedTransformer } from "./transformer";+import { SelectRewriteTransformer } from "./transformer";const OUTER_PRESERVING_JOINS = new Set<JoinType>(["RightJoin", "FullJoin"]);const AFTER_LIMIT_ALIAS = "latest";@@ -56,23 +56,21 @@ export class SoftDeleteFilterPlugin implements KyselyPlugin {}}-class SoftDeleteFilterTransformer extends CteScopedTransformer {+class SoftDeleteFilterTransformer extends SelectRewriteTransformer {protected rewriteSelect(node: ClickHouseSelectQueryNode,): ClickHouseSelectQueryNode {- return filterSelect(node, (table) => this.isCte(table));+ return filterSelect(node);}}function filterSelect(node: ClickHouseSelectQueryNode,- isCte: (table: TableNode) => boolean,): ClickHouseSelectQueryNode {- const describe = (item: OperationNode) => softDeleteRelation(item, isCte);- const froms = (node.from?.froms ?? []).map(describe);+ const froms = (node.from?.froms ?? []).map(softDeleteRelation);const joins = (node.joins ?? []).map((join) => ({join,- relation: describe(join.table),+ relation: softDeleteRelation(join.table),}));const readsSoftDelete =@@ -267,13 +265,12 @@ function outputName(node: OperationNode): string | undefined {return undefined;}-/** The soft-delete table `item` reads, unless it reads a CTE. */+/** The soft-delete table `item` reads. */function softDeleteRelation(item: OperationNode,- isCte: (table: TableNode) => boolean,): SoftDeleteRelation | undefined {const { table, alias } = describeRelation(item);- if (!table || isCte(table)) return undefined;+ if (!table) return undefined;const tableName = table.table.identifier.name;const flag = SOFT_DELETE_COLUMNS[tableName];return flag ? { tableName, alias, flag } : undefined;diff --git a/packages/shared/src/server/query-ast/tenancy.ts b/packages/shared/src/server/query-ast/tenancy.tsindex f33cab9bbd..f12f01abfc 100644--- a/packages/shared/src/server/query-ast/tenancy.ts+++ b/packages/shared/src/server/query-ast/tenancy.ts@@ -20,10 +20,10 @@ import {} from "kysely";import { QueryCompileError, UnscopedRelationError } from "./errors";-import { type ClickHouseSelectQueryNode } from "./nodes";+import { walkNodes, type ClickHouseSelectQueryNode } from "./nodes";import { describeRelation } from "./relations";import { TENANTED_TABLES } from "./schema";-import { CteScopedTransformer } from "./transformer";+import { SelectRewriteTransformer } from "./transformer";const PROJECT_ID_COLUMN = "project_id";@@ -35,7 +35,7 @@ const PROJECT_ID_COLUMN = "project_id";const TENANCY_STAMPED = new WeakSet<object>();type ScopedRelation =- | { kind: "table"; tableName: string; alias?: string; node: TableNode }+ | { kind: "table"; tableName: string; alias?: string }| { kind: "subquery" }| { kind: "raw" }| { kind: "other" };@@ -68,7 +68,7 @@ export class TenancyInjectionPlugin implements KyselyPlugin {}}-class TenancyInjectionTransformer extends CteScopedTransformer {+class TenancyInjectionTransformer extends SelectRewriteTransformer {constructor(private readonly projectId: string) {super();}@@ -76,14 +76,13 @@ class TenancyInjectionTransformer extends CteScopedTransformer {protected rewriteSelect(node: ClickHouseSelectQueryNode,): ClickHouseSelectQueryNode {- return injectSelect(node, this.projectId, (table) => this.isCte(table));+ return injectSelect(node, this.projectId);}}function injectSelect(node: ClickHouseSelectQueryNode,projectId: string,- isCte: (table: TableNode) => boolean,): ClickHouseSelectQueryNode {const fromRelations = (node.from?.froms ?? []).map(classifyRelation);const joinRelations = (node.joins ?? []).map((join) => ({@@ -101,7 +100,7 @@ function injectSelect(const isTenantedTable = (r: ScopedRelation,): r is Extract<ScopedRelation, { kind: "table" }> =>- r.kind === "table" && TENANTED_TABLES.has(r.tableName) && !isCte(r.node);+ r.kind === "table" && TENANTED_TABLES.has(r.tableName);const tenantedFrom = fromRelations.filter(isTenantedTable);const tenantedJoinCount = joinRelations.filter(({ relation }) =>@@ -169,7 +168,7 @@ function assertRelationAllowed(relation: ScopedRelation): void {function classifyRelation(node: OperationNode): ScopedRelation {const { source, table, alias } = describeRelation(node);if (table) {- return { kind: "table", tableName: tableNameOf(table), alias, node: table };+ return { kind: "table", tableName: tableNameOf(table), alias };}if (RawNode.is(source)) return { kind: "raw" };if (SelectQueryNode.is(source)) return { kind: "subquery" };@@ -307,23 +306,21 @@ export function assertTenancyStamped(node: RootOperationNode): void {const RAW_RELATION_SQL = /\b(?:from|join|select)\b/i;-function rejectUnscopedRawSql(node: unknown): void {- if (!node || typeof node !== "object") return;- if (Array.isArray(node)) {- for (const item of node) rejectUnscopedRawSql(item);- return;- }- if (RawNode.is(node as OperationNode)) {- const fragments = (node as RawNode).sqlFragments;- if (fragments.some((fragment) => RAW_RELATION_SQL.test(fragment))) {+function rejectUnscopedRawSql(root: unknown): void {+ walkNodes(root, (node) => {+ if (!RawNode.is(node)) return;+ if (node.sqlFragments.some((fragment) => RAW_RELATION_SQL.test(fragment))) {throw new UnscopedRelationError("Raw SQL fragments that introduce a relation (SELECT/FROM/JOIN) are rejected in any position: they can bypass tenancy injection. Use traced table and column references instead.",);}- }- for (const value of Object.values(node)) {- rejectUnscopedRawSql(value);- }+ // `x IN ${sql.table("traces")}` reads a table with no FROM to scope.+ if (node.parameters.some((parameter) => TableNode.is(parameter))) {+ throw new UnscopedRelationError(+ "A table reference inside raw SQL (sql.table) is rejected in any position: it reads a relation the tenancy pass cannot scope. Read the table through selectFrom/innerJoin or a subquery instead.",+ );+ }+ });}/**@@ -336,18 +333,12 @@ export function assertRelationFree(node: OperationNode): void {rejectSubqueries(node);}-function rejectSubqueries(node: unknown): void {- if (!node || typeof node !== "object") return;- if (Array.isArray(node)) {- for (const item of node) rejectSubqueries(item);- return;- }- if ("kind" in node && node.kind === "SelectQueryNode") {- throw new UnscopedRelationError(- "A standalone expression cannot contain a subquery: it has no tenancy scope. Compile the query through compileClickhouseQuery() instead.",- );- }- for (const value of Object.values(node)) {- rejectSubqueries(value);- }+function rejectSubqueries(root: unknown): void {+ walkNodes(root, (node) => {+ if (SelectQueryNode.is(node)) {+ throw new UnscopedRelationError(+ "A standalone expression cannot contain a subquery: it has no tenancy scope. Compile the query through compileClickhouseQuery() instead.",+ );+ }+ });}diff --git a/packages/shared/src/server/query-ast/transformer.ts b/packages/shared/src/server/query-ast/transformer.tsindex f879b3b8f8..094398481f 100644--- a/packages/shared/src/server/query-ast/transformer.ts+++ b/packages/shared/src/server/query-ast/transformer.ts@@ -1,14 +1,10 @@import {OperationNodeTransformer,- SelectQueryNode,- TableNode,- type CommonTableExpressionNode,type OperationNode,type QueryId,- type WithNode,+ type SelectQueryNode,} from "kysely";-import { UnscopedRelationError } from "./errors";import {ArrayJoinNode,ArrayIndexNode,@@ -16,7 +12,6 @@ import {LimitByNode,type ClickHouseSelectQueryNode,} from "./nodes";-import { selectRelations } from "./relations";/*** Transformer that knows the ClickHouse extra fields on SelectQueryNode.@@ -100,218 +95,20 @@ export class ClickHouseOperationNodeTransformer extends OperationNodeTransformer}/**- * Transformer for passes that rewrite physical relations: it tracks which- * names resolve to a CTE, so a relation named like a registry table is left- * alone when it reads a CTE of that name. Subclasses rewrite each select in- * {@link rewriteSelect}, after its children, and ask {@link isCte}. The- * compiler's bind-type scopes (`compiler.ts`) resolve names the same way.+ * Transformer for passes that rewrite each select after its children:+ * subclasses rewrite one select at a time in {@link rewriteSelect}.*/-export abstract class CteScopedTransformer extends ClickHouseOperationNodeTransformer {- /**- * CTE names visible to the select being transformed: one set per enclosing- * select that declares a WITH. Subqueries, nested WITH bodies and UNION- * branches see every enclosing set. A CTE body sees every sibling CTE except- * itself, matching ClickHouse: `WITH events_core AS (SELECT β¦ FROM- * events_core)` reads the table, unless an enclosing select declares- * `events_core` too. Under WITH RECURSIVE only the recursive UNION member- * reads the CTE; it is still treated as the table, so a rewrite errs toward- * ClickHouse rejecting the query. Pushed before a select's children are- * transformed and popped after, so it is empty between compiles.- */- private readonly cteScopes: Set<string>[] = [];-- /**- * Names the CTE bodies of each enclosing WITH read in FROM/JOIN, one set- * per enclosing select that declares a WITH. ClickHouse resolves a CTE body- * where the CTE is read, not where it is declared, so a nested WITH that- * declares one of these names would redirect that read. Pushed and popped- * with {@link cteScopes}.- */- private readonly enclosingBodyReads: Set<string>[] = [];-+export abstract class SelectRewriteTransformer extends ClickHouseOperationNodeTransformer {protected abstract rewriteSelect(node: ClickHouseSelectQueryNode,): ClickHouseSelectQueryNode;- /** A schema-qualified table (`default.traces`) is never a CTE. */- protected isCte(table: TableNode): boolean {- if (table.table.schema) return false;- const name = table.table.identifier.name;- return this.cteScopes.some((scope) => scope.has(name));- }-protected override transformSelectQuery(node: SelectQueryNode,queryId?: QueryId,): SelectQueryNode {- const ctes = cteNames(node.with);- if (ctes.size > 0) {- assertNoRedirectedRead(ctes, this.enclosingBodyReads);- this.cteScopes.push(ctes);- this.enclosingBodyReads.push(- relationNames(node.with?.expressions.map((expr) => expr.expression)),- );- }- try {- return this.rewriteSelect(- super.transformSelectQuery(node, queryId) as ClickHouseSelectQueryNode,- );- } finally {- if (ctes.size > 0) {- this.cteScopes.pop();- this.enclosingBodyReads.pop();- }- }- }-- protected override transformWith(- node: WithNode,- queryId?: QueryId,- ): WithNode {- assertNoSiblingCycle(node);- const full = this.cteScopes.pop() ?? new Set<string>();- try {- const expressions = node.expressions.map((expr) => {- const visible = new Set(full);- visible.delete(cteName(expr));- this.cteScopes.push(visible);- try {- return this.transformNode(expr, queryId);- } finally {- this.cteScopes.pop();- }- });- return { ...node, expressions };- } finally {- if (full.size > 0) this.cteScopes.push(full);- }- }-}--export function cteName(expr: CommonTableExpressionNode): string {- return expr.name.table.table.identifier.name;-}--function cteNames(node: WithNode | undefined): Set<string> {- return new Set((node?.expressions ?? []).map(cteName));-}--/**- * Rejects a WITH that declares a name a CTE body of an enclosing WITH reads:- *- * WITH a AS (SELECT β¦ FROM m)- * SELECT β¦- * FROM (- * WITH m AS (β¦)- * SELECT β¦ FROM a -- `a` is read here, so its `m` is the inner `m`- * )- *- * With this rule every name in a CTE body resolves where the CTE is declared,- * which is what- * {@link CteScopedTransformer} and the compiler's bind-type scopes assume.- * The rule is conservative: it also rejects a nested WITH under which the- * enclosing CTE is never read.- */-function assertNoRedirectedRead(- declared: Set<string>,- enclosingBodyReads: readonly Set<string>[],-): void {- for (const name of declared) {- if (enclosingBodyReads.some((reads) => reads.has(name))) {- throw new UnscopedRelationError(- `A nested WITH may not declare ${name}: an enclosing CTE reads ${name}, and if that CTE is read inside this select, ClickHouse resolves its read to the nested CTE, so the tenancy pass cannot scope it.`,- );- }- }-}--/**- * Rejects sibling CTEs that read each other (`a` β `b` β `a`). ClickHouse- * resolves a read of a CTE that is still being resolved as if that CTE were- * not declared, to an enclosing CTE of that name or else the table, so one- * reference reads the CTE on one path and something else on another, and no- * scope can tell a pass which. A body reading its own name is not a cycle,- * and neither is a body reading a CTE it declares itself in a nested WITH:- * reads are those {@link relationNames} returns.- */-function assertNoSiblingCycle(node: WithNode): void {- const names = cteNames(node);- const reads = new Map(- node.expressions.map((expr) => {- const name = cteName(expr);- const read = relationNames(expr.expression);- return [- name,- [...read].filter((other) => other !== name && names.has(other)),- ];- }),- );- const done = new Set<string>();- const path: string[] = [];- const visit = (name: string) => {- if (path.includes(name)) {- throw new UnscopedRelationError(- `CTEs that read each other (${[...path.slice(path.indexOf(name)), name].join(" β ")}) are rejected: ClickHouse reads the table, not the CTE, on part of the cycle, so the tenancy pass cannot scope it.`,- );- }- if (done.has(name)) return;- path.push(name);- for (const next of reads.get(name) ?? []) visit(next);- path.pop();- done.add(name);- };- for (const name of names) visit(name);-}--/**- * Unqualified table names read in FROM/JOIN anywhere under `value`, except- * reads of a CTE that a select under `value` declares: those always resolve- * to that CTE. A CTE body's read of its own name escapes its WITH, so it- * counts.- *- * relationNames(SELECT β¦ FROM a JOIN (WITH x AS (SELECT β¦ FROM b)- * SELECT β¦ FROM x) AS s)- * β {a, b} -- `x` is declared under `value`, so it is not a read- *- * `shadowed` holds the CTE names declared by the selects between `value`'s- * root and the current node.- */-function relationNames(- value: unknown,- found = new Set<string>(),- shadowed: ReadonlySet<string> = new Set(),-): Set<string> {- if (!value || typeof value !== "object") return found;- // Not a select (an array, a WHERE, a join, β¦): look for selects below it.- if (!isOperationNode(value) || !SelectQueryNode.is(value)) {- for (const child of Object.values(value)) {- relationNames(child, found, shadowed);- }- return found;- }- // A CTE body sees its siblings as CTEs, but not itself: inside its body,- // its own name is the table or an enclosing CTE.- const declared = cteNames(value.with);- for (const expr of value.with?.expressions ?? []) {- const siblings = new Set(declared);- siblings.delete(cteName(expr));- relationNames(expr.expression, found, new Set([...shadowed, ...siblings]));- }- // The rest of this select (FROM, JOIN, WHERE, β¦) sees all of its CTEs.- const inner = new Set([...shadowed, ...declared]);- for (const { table } of selectRelations(value)) {- const name = table?.table.identifier.name;- if (table && name && !table.table.schema && !inner.has(name)) {- found.add(name);- }- }- // Subqueries anywhere in this select; the WITH bodies were walked above.- for (const [key, child] of Object.entries(value)) {- if (key !== "with") relationNames(child, found, inner);+ return this.rewriteSelect(+ super.transformSelectQuery(node, queryId) as ClickHouseSelectQueryNode,+ );}- return found;-}--function isOperationNode(value: object): value is OperationNode {- return "kind" in value && typeof value.kind === "string";}diff --git a/packages/shared/src/server/query-ast/typecheck.ts b/packages/shared/src/server/query-ast/typecheck.tsindex 2a4034ba7b..9e00e16a50 100644--- a/packages/shared/src/server/query-ast/typecheck.ts+++ b/packages/shared/src/server/query-ast/typecheck.ts@@ -6,6 +6,7 @@ import {} from "kysely";import { TypeCompatibilityError } from "./errors";+import { walkNodes } from "./nodes";import { COLUMN_DATA_TYPES } from "./schema";// Kysely already type-checks comparisons against the schema; `sum()` / `avg()`@@ -19,7 +20,7 @@ const NUMERIC_AGGREGATES = new Set(["sum", "avg"]);* Walks every node object (including ClickHouse extra fields).*/export function validateTypeCompatibility(root: OperationNode): void {- walk(root, (node) => {+ walkNodes(root, (node) => {if (!AggregateFunctionNode.is(node)) return;if (!NUMERIC_AGGREGATES.has(node.func.toLowerCase())) return;for (const arg of node.aggregated) {@@ -42,20 +43,3 @@ function columnNameOf(node: OperationNode): string | undefined {}return undefined;}--function walk(value: unknown, visit: (node: OperationNode) => void): void {- if (!value || typeof value !== "object") return;- if (- "kind" in value &&- typeof (value as { kind: unknown }).kind === "string"- ) {- visit(value as OperationNode);- }- for (const child of Object.values(value)) {- if (Array.isArray(child)) {- for (const item of child) walk(item, visit);- } else {- walk(child, visit);- }- }-}diff --git a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snapindex 4578dc3c3c..f98a424f2c 100644--- a/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap+++ b/packages/shared/src/server/repositories/__snapshots__/events.golden.test.ts.snap@@ -2468,7 +2468,7 @@ exports[`golden: events point-reads & existence family > getTraceByIdFromEventsT"param4": 1,},"route": "unknown",- "sql": "WITH traces AS+ "sql": "WITH event_traces AS(SELECTe.trace_id AS id,@@ -2555,7 +2555,7 @@ SELECT0 AS is_deleted,t.input,t.output-FROM traces AS t+FROM event_traces AS tORDER BY t.timestamp DESCLIMIT {param4:Int64}",},@@ -2575,7 +2575,7 @@ exports[`golden: events point-reads & existence family > getTraceByIdFromEventsT"param4": 1,},"route": "unknown",- "sql": "WITH traces AS+ "sql": "WITH event_traces AS(SELECTe.trace_id AS id,@@ -2662,7 +2662,7 @@ SELECT0 AS is_deleted,'' AS input,'' AS output-FROM traces AS t+FROM event_traces AS tORDER BY t.timestamp DESCLIMIT {param4:Int64}",},@@ -2684,7 +2684,7 @@ exports[`golden: events point-reads & existence family > getTraceByIdFromEventsT"param6": 1,},"route": "unknown",- "sql": "WITH traces AS+ "sql": "WITH event_traces AS(SELECTe.trace_id AS id,@@ -2772,7 +2772,7 @@ SELECT0 AS is_deleted,t.input,t.output-FROM traces AS t+FROM event_traces AS tWHERE toDate(t.timestamp) = toDate({param5:DateTime64(6)})ORDER BY t.timestamp DESCLIMIT {param6:Int64}",@@ -2793,7 +2793,7 @@ exports[`golden: events point-reads & existence family > getTraceByIdFromEventsT"param4": 1,},"route": "unknown",- "sql": "WITH traces AS+ "sql": "WITH event_traces AS(SELECTe.trace_id AS id,@@ -2880,7 +2880,7 @@ SELECT0 AS is_deleted,leftUTF8(t.input, 1000) AS input,leftUTF8(t.output, 1000) AS output-FROM traces AS t+FROM event_traces AS tORDER BY t.timestamp DESCLIMIT {param4:Int64}",},diff --git a/packages/shared/src/server/repositories/events.ts b/packages/shared/src/server/repositories/events.tsindex fd3f976398..edf4e72c55 100644--- a/packages/shared/src/server/repositories/events.ts+++ b/packages/shared/src/server/repositories/events.ts@@ -11,6 +11,7 @@ import {type AliasedExpression,type Expression,type ExpressionBuilder,+ type QueryCreator,type SqlBool,} from "kysely";import type {@@ -32,6 +33,7 @@ import { recordDistribution } from "../instrumentation";import { logger } from "../logger";import { compileClickhouseQuery } from "../query-ast/compile";import { getClickhouseKysely } from "../query-ast/dialect";+import type { ClickHouseDatabase } from "../query-ast/schema";import { promptGenerationEvents } from "./prompt-events";import {arrayMap,@@ -1194,12 +1196,10 @@ export const getTraceByIdFromEventsTable = async ({const db = getClickhouseKysely();const truncated = renderingProps.truncated === true;- const tracesCte = (qb: ReturnType<typeof getClickhouseKysely>) => {- const from = truncated- ? qb.selectFrom("events_core as e")- : qb.selectFrom("events_full as e");- return from- .select(traceAggregationSelects)+ const tracesCte = (qb: QueryCreator<ClickHouseDatabase>) =>+ qb+ .selectFrom(truncated ? "events_core as e" : "events_full as e")+ .select((eb) => traceAggregationSelects(eb)).where("trace_id", "in", [traceId]).$if(fromTimestamp != null, (inner) =>inner.where(@@ -1210,17 +1210,14 @@ export const getTraceByIdFromEventsTable = async ({).groupBy(["trace_id", "project_id"]).orderBy("timestamp", "desc");- };const metadataSelect = excludeMetadata? sql`map()`.as("metadata"): "t.metadata";const builder = db- .with("traces", (qb) =>- tracesCte(qb as ReturnType<typeof getClickhouseKysely>),- )- .selectFrom("traces as t")+ .with("event_traces", tracesCte)+ .selectFrom("event_traces as t").select((eb) => ["t.id","t.name",