🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(query-ast): reject identifiers that are not plain names (#19034)
Niklas Semmler68100627486c
diff --git a/packages/shared/src/server/query-ast/README.md b/packages/shared/src/server/query-ast/README.mdindex 8334aba146..a3f7a7a114 100644--- a/packages/shared/src/server/query-ast/README.md+++ b/packages/shared/src/server/query-ast/README.md@@ -109,9 +109,12 @@ suffix to run there. Promote it to a required check once it has proven stable.recursive and cyclic CTEs are out. The passes then read every registryname as its table. Rename or reorder the CTEs when the compile rejectsthem; add a physical table to the registry to read it.-- **Function and aggregate names must be plain identifiers.**- The compiler writes them verbatim, so anything else fails to compile.- `collate` is not supported.+- **Identifiers, function and aggregate names must be plain identifiers**+ (`[A-Za-z_][A-Za-z0-9_]*`). The compiler writes them unquoted, so anything+ else fails to compile. This covers columns, tables, aliases, CTE names,+ `sql.ref` and `sql.id`; `sql.ref("e.*")` is rejected too, list the+ columns instead. A positive integer after a qualifier (`x.1`) is tuple+ element access and compiles as is. `collate` is not supported.- **Never filter `is_deleted` on a physical table yourself.** Tables thatdeclare `softDelete` in the registry (`events_core`, `events_full`) get`is_deleted = 0` on every FROM/JOIN. A latest-version read (ordered bydiff --git a/packages/shared/src/server/query-ast/binds.test.ts b/packages/shared/src/server/query-ast/binds.test.tsindex a6177fabd7..e907418e0e 100644--- a/packages/shared/src/server/query-ast/binds.test.ts+++ b/packages/shared/src/server/query-ast/binds.test.ts@@ -587,7 +587,7 @@ describe("typed params from the column registry", () => {() =>db.with("t", (qb) =>- qb.selectFrom("events_core as e").select(db.dynamic.ref("x.*")),+ qb.selectFrom("events_core as e").selectAll("x" as never),).selectFrom("t as t").select(db.dynamic.ref("t.span_id"))@@ -601,7 +601,7 @@ describe("typed params from the column registry", () => {() =>db.with("t", (qb) =>- qb.selectFrom("events_core as e").select(db.dynamic.ref("x.*")),+ qb.selectFrom("events_core as e").selectAll("x" as never),).selectFrom("scores as s").innerJoin("t", (join) => join.onTrue())@@ -776,9 +776,7 @@ describe("typed params from the column registry", () => {const db = getClickhouseKysely();const { sql } = compile(db- .with("t", (qb) =>- qb.selectFrom("events_core as e").select(db.dynamic.ref("e.*")),- )+ .with("t", (qb) => qb.selectFrom("events_core as e").selectAll("e")).selectFrom("t as t").select(db.dynamic.ref("t.span_id")).where(db.dynamic.ref("t.start_time"), ">=", START),diff --git a/packages/shared/src/server/query-ast/compiler.ts b/packages/shared/src/server/query-ast/compiler.tsindex 48bb2f45ad..0cdffed278 100644--- a/packages/shared/src/server/query-ast/compiler.ts+++ b/packages/shared/src/server/query-ast/compiler.ts@@ -50,6 +50,7 @@ const ARRAY_JOIN_SQL: Record<ArrayJoinNode["variant"], string> = {};const PLAIN_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;+const TUPLE_INDEX = /^[1-9][0-9]*$/;const DATE_WRAPPER_FNS = new Set(["toDate",@@ -292,8 +293,8 @@ export class ClickHouseQueryCompiler extends DefaultQueryCompiler {// Emit identifiers unquoted. `clickhouse format` normalizes to unquoted// identifiers, so returning empty wrappers is what keeps raw compiler output- // byte-comparable with the golden snapshots. (Column/table names in this- // codebase are plain identifiers, so dropping the quotes is safe here.)+ // byte-comparable with the golden snapshots. Dropping the quotes is safe+ // because compileUnwrappedIdentifier accepts plain identifiers only.protected override getLeftIdentifierWrapper(): string {return "";}@@ -302,18 +303,52 @@ export class ClickHouseQueryCompiler extends DefaultQueryCompiler {return "";}+ /**+ * Every identifier (column, table, schema, alias, CTE name, `sql.ref`,+ * `sql.id`) is written unquoted, so a name that is not a plain identifier+ * would be raw SQL: the alias `x from scores --` comments out the tenancy+ * predicate after it.+ */+ protected override compileUnwrappedIdentifier(node: IdentifierNode): void {+ assertPlainName(+ "Identifier",+ node.name,+ "Rename it, or list the columns instead of a star.",+ );+ super.compileUnwrappedIdentifier(node);+ }++ /**+ * A positive integer after a qualifier is ClickHouse tuple element access,+ * not a column: `x.1` in `mapFilter(x -> positionCaseInsensitive(x.1, …), m)`+ * reads the key of map entry `x`. It is written as is; every other column+ * name goes through {@link compileUnwrappedIdentifier}.+ */+ protected override visitReference(node: ReferenceNode): void {+ if (+ node.table &&+ ColumnNode.is(node.column) &&+ TUPLE_INDEX.test(node.column.column.name)+ ) {+ this.visitNode(node.table);+ this.append(`.${node.column.column.name}`);+ return;+ }+ super.visitReference(node);+ }+/*** Function and aggregate names are written verbatim, so a name that is not* a plain identifier would be raw SQL: `fn("tuple(e.input), f")` emits* `tuple(e.input), f(…)`.*/protected override visitFunction(node: FunctionNode): void {- assertPlainName("Function", node.func);+ assertPlainName("Function", node.func, "Rename it.");super.visitFunction(node);}protected override visitAggregateFunction(node: AggregateFunctionNode): void {- assertPlainName("Aggregate function", node.func);+ assertPlainName("Aggregate function", node.func, "Rename it.");super.visitAggregateFunction(node);}@@ -532,7 +567,7 @@ export class ClickHouseQueryCompiler extends DefaultQueryCompiler {protected override visitCommonTableExpressionName(node: CommonTableExpressionNameNode,): void {- assertPlainName("CTE", node.table.table.identifier.name);+ assertPlainName("CTE", node.table.table.identifier.name, "Rename it.");super.visitCommonTableExpressionName(node);}@@ -845,10 +880,10 @@ function cteBodyScope(scope: Pick<SelectScope, "cte">): SelectScope {return { cte: scope.cte, relations: new Map(), aliases: new Map() };}-function assertPlainName(kind: string, name: string): void {+function assertPlainName(kind: string, name: string, fix: string): void {if (!PLAIN_IDENTIFIER.test(name)) {throw new QueryCompileError(- `${kind} name "${name}" is not a plain identifier; the compiler writes it verbatim into the SQL.`,+ `${kind} name "${name}" is not a plain identifier ([A-Za-z_][A-Za-z0-9_]*); the compiler writes it verbatim into the SQL. ${fix}`,);}}diff --git a/packages/shared/src/server/query-ast/nodes.test.ts b/packages/shared/src/server/query-ast/nodes.test.tsindex 4e09924811..08c67f24f8 100644--- a/packages/shared/src/server/query-ast/nodes.test.ts+++ b/packages/shared/src/server/query-ast/nodes.test.ts@@ -858,6 +858,91 @@ describe("names the compiler writes verbatim", () => {},);+ // Identifiers are written unquoted, so a name that is not a plain identifier+ // is raw SQL: the select alias `x from scores --` emits+ // `select s.id as x from scores -- from scores as s where project_id = …`,+ // which comments out the tenancy predicate.+ it.each([+ [+ "select alias",+ db+ .selectFrom("scores as s")+ .select((eb) => eb.ref("s.id").as("x from scores --")),+ ],+ ["table alias", db.selectFrom("scores as s --").select("s --.id")],+ [+ "subquery alias",+ db+ .selectFrom(db.selectFrom("scores as s").select("s.id").as("sub, t"))+ .select("sub, t.id"),+ ],+ [+ "CTE name",+ db+ .with("c --", (qb) => qb.selectFrom("scores as s").select("s.id"))+ .selectFrom("c --")+ .select("id"),+ ],+ [+ "sql.ref column",+ db+ .selectFrom("scores as s")+ .select(sql.ref<string>("s.id from scores --").as("x")),+ ],+ [+ "sql.ref star",+ db.selectFrom("scores as s").select(sql.ref<string>("s.*").as("x")),+ ],+ // An index only reads a tuple element after a qualifier; alone, `1` would+ // compile to the literal 1 instead of a column.+ [+ "sql.ref unqualified tuple index",+ db.selectFrom("scores as s").select(sql.ref<string>("1").as("x")),+ ],+ // Tuple indexes start at 1; ClickHouse rejects `s.0`.+ [+ "sql.ref zero tuple index",+ db.selectFrom("scores as s").select(sql.ref<string>("s.0").as("x")),+ ],+ // Only a whole integer is a tuple index; text after it is raw SQL.+ [+ "sql.ref tuple index followed by text",+ db+ .selectFrom("scores as s")+ .select(sql.ref<string>("s.1 from scores --").as("x")),+ ],+ [+ "sql.id",+ db.selectFrom("scores as s").select(sql.id("s", "id --").as("x")),+ ],+ [+ "array join alias",+ db+ .selectFrom("observations")+ .select("environment")+ .$call(+ arrayJoin((eb) => ({+ "cost_key --": mapKeys(eb.ref("cost_details")),+ })),+ ),+ ],+ ])("rejects a non-plain identifier in a %s", (_kind, qb) => {+ expect(() => compileClickhouseQuery(qb, projectId)).toThrow(+ QueryCompileError,+ );+ expect(() => compileClickhouseQuery(qb, projectId)).toThrow(+ /not a plain identifier/,+ );+ });++ it("writes a positive integer after a qualifier as a tuple element", () => {+ const { sql: compiled } = compileClickhouseQuery(+ db.selectFrom("scores as s").select(sql.ref<string>("s.1").as("x")),+ projectId,+ );+ expect(compiled).toContain("select s.1 as x from scores as s");+ });+// ClickHouse parses only a string literal after `collate`, which Kysely// does not emit.it.each(["en", "'en', s.comment"])(