OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(P2-008): The sql tag: packages/db/src/clickhouse/sql.ts + chQuery params path + the full test battery
ralphdc7516df3f9d
diff --git a/packages/db/package.json b/packages/db/package.jsonindex 3ae01edcb..1e1da401e 100644--- a/packages/db/package.json+++ b/packages/db/package.json@@ -10,6 +10,7 @@"migrate:deploy:db": "pnpm with-env prisma migrate deploy","migrate:deploy": "pnpm migrate:deploy:db && pnpm migrate:deploy:code","duplicate-events": "pnpm with-env jiti ./scripts/find-duplicate-events.ts",+ "test": "vitest run","typecheck": "tsc --noEmit","with-env": "dotenv -e ../../.env -c --"},@@ -44,4 +45,4 @@"prisma": "^6.14.0","typescript": "catalog:"}-}\ No newline at end of file+}diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.tsindex d78a5220c..349352e60 100644--- a/packages/db/src/clickhouse/client.ts+++ b/packages/db/src/clickhouse/client.ts@@ -9,6 +9,7 @@ import { createLogger } from '@openpanel/logger';import type { IInterval } from '@openpanel/validation';import sqlstring from 'sqlstring';import { RoundRobinPicker, withRoundRobinRetry } from './round-robin';+import { type SqlFragment, toStatement } from './sql';export { createClient } from '@clickhouse/client';@@ -324,16 +325,27 @@ export const ch = new Proxy(chTarget as unknown as ClickHouseClient, {},}) as ClickHouseClient;+/**+ * A query is either finished SQL text or a `sql` fragment carrying its own+ * bound params (ADR-013 R1). Both go through the same round-robin/retry path.+ */+export type ChQueryInput = string | SqlFragment;+export async function chQueryWithMeta<T extends Record<string, any>>(- query: string,+ query: ChQueryInput,clickhouseSettings?: ClickHouseSettings): Promise<ResponseJSON<T>> {const start = Date.now();let host: string | undefined;+ const statement = toStatement(query);+ const paramNames = Object.keys(statement.query_params);+ const hasParams = paramNames.length > 0;+ const queryParams = hasParams ? statement.query_params : undefined;const res = await withRetry((client, ctx) => {host = urlHostname(ctx.url);return client.query({- query,+ query: statement.query,+ query_params: queryParams,clickhouse_settings: clickhouseSettings,});});@@ -358,7 +370,9 @@ export async function chQueryWithMeta<T extends Record<string, any>>(logger.info({host,- query: cleanQuery(query),+ query: cleanQuery(statement.query),+ // Names only: bound values are user data and must not reach the logs.+ queryParams: hasParams ? paramNames : undefined,rows: json.rows,stats: response.statistics,elapsed: Date.now() - start,@@ -371,7 +385,7 @@ export async function chQueryWithMeta<T extends Record<string, any>>(}export async function chQuery<T extends Record<string, any>>(- query: string,+ query: ChQueryInput,clickhouseSettings?: ClickHouseSettings): Promise<T[]> {return (await chQueryWithMeta<T>(query, clickhouseSettings)).data;diff --git a/packages/db/src/clickhouse/sql.clickhouse.test.ts b/packages/db/src/clickhouse/sql.clickhouse.test.tsnew file mode 100644index 000000000..5caaf8f75--- /dev/null+++ b/packages/db/src/clickhouse/sql.clickhouse.test.ts@@ -0,0 +1,440 @@+import { describe, expect, it } from 'vitest';+import { sql } from './sql';++/**+ * The ADR-013 P2 round-trip: every param type the codebase uses, bound+ * through `chQuery` (so the `query_params` path is exercised through+ * `withRetry` / round-robin, R1) and read back off a real local ClickHouse.+ *+ * Payloads are hostile on purpose — single quote, double quote, backslash,+ * newline, tab, and a literal `{p1:String}` — because the escaping lives in+ * the driver and the parsing lives in the server, and neither is ours.+ *+ * This suite runs against its own `openpanel_test` database, which it+ * creates. It must never touch the local `openpanel` prod-copy: those tables+ * hold hundreds of millions of rows, and a `FINAL` / `ARRAY JOIN` scan over+ * them times the HTTP client out while the rest of the suite runs in+ * parallel against the same node.+ */++const CLICKHOUSE_TEST_DATABASE = 'openpanel_test';+const DEFAULT_CLICKHOUSE_BASE_URL = 'http://localhost:8123';++/**+ * Well above the app default (30s): this suite shares one local node with+ * every other test file in the run, and a queued request should wait rather+ * than fail.+ */+const CLICKHOUSE_TEST_REQUEST_TIMEOUT_MS = 120_000;++function testDatabaseUrl(): { base: string; withDatabase: string } {+ const configured = (process.env.CLICKHOUSE_URL ?? '').split(',')[0]?.trim();+ const url = new URL(configured || DEFAULT_CLICKHOUSE_BASE_URL);+ url.pathname = '/';+ const base = url.toString();+ url.pathname = `/${CLICKHOUSE_TEST_DATABASE}`;+ return { base, withDatabase: url.toString() };+}++const { base: CLICKHOUSE_BASE_URL, withDatabase: CLICKHOUSE_TEST_URL } =+ testDatabaseUrl();++async function execute(statement: string): Promise<void> {+ const response = await fetch(CLICKHOUSE_BASE_URL, {+ method: 'POST',+ body: statement,+ });+ if (!response.ok) {+ throw new Error(`${response.status}: ${await response.text()}`);+ }+}++/**+ * Minimal stand-ins for the three tables this suite names — same engine and+ * sort key as production, only the columns the assertions touch. They stay+ * empty; every query here asserts on shape, not on data.+ */+const FIXTURE_TABLES = [+ `CREATE TABLE IF NOT EXISTS ${CLICKHOUSE_TEST_DATABASE}.events (+ id UUID DEFAULT generateUUIDv4(),+ project_id String,+ profile_id String,+ groups Array(String) DEFAULT [],+ created_at DateTime64(3)+ ) ENGINE = MergeTree+ PARTITION BY toYYYYMM(created_at)+ ORDER BY (project_id, toDate(created_at), created_at)`,+ `CREATE TABLE IF NOT EXISTS ${CLICKHOUSE_TEST_DATABASE}.profiles (+ id String,+ project_id String,+ first_name String,+ last_name String,+ last_seen_at DateTime64(3)+ ) ENGINE = ReplacingMergeTree(last_seen_at)+ ORDER BY (project_id, id)`,+ `CREATE TABLE IF NOT EXISTS ${CLICKHOUSE_TEST_DATABASE}.sessions (+ id String,+ project_id String,+ created_at DateTime64(3),+ sign Int8,+ version UInt64+ ) ENGINE = VersionedCollapsingMergeTree(sign, version)+ PARTITION BY toYYYYMM(created_at)+ ORDER BY (project_id, toDate(created_at), created_at)`,+];++async function bootstrapTestDatabase(): Promise<boolean> {+ try {+ await execute(`CREATE DATABASE IF NOT EXISTS ${CLICKHOUSE_TEST_DATABASE}`);+ for (const table of FIXTURE_TABLES) {+ await execute(table);+ }+ return true;+ } catch {+ return false;+ }+}++const clickhouseReachable = await bootstrapTestDatabase();++// ./client reads both of these once, at module evaluation, so they have to be+// set before it is imported — hence the dynamic import.+process.env.CLICKHOUSE_URL = CLICKHOUSE_TEST_URL;+process.env.CLICKHOUSE_REQUEST_TIMEOUT_MS = String(+ CLICKHOUSE_TEST_REQUEST_TIMEOUT_MS+);++const { chQuery } = await import('./client');++const HOSTILE = `it's a "quoted" \\ backslash {p1:String} }brace{ \n newline \t tab`;+const BIG_UINT64 = 1_234_567_890_123;++const describeAgainstClickhouse = describe.skipIf(!clickhouseReachable);++/** ClickHouse type names carry `(` and `)`; they are matched literally. */+const escapeRegExp = (literal: string) =>+ literal.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');++const value = async <T>(+ query: Parameters<typeof chQuery>[0],+ settings?: Parameters<typeof chQuery>[1]+) => {+ const rows = await chQuery<{ value: T }>(query, settings);+ return rows[0]?.value;+};++describeAgainstClickhouse('param round-trip against local ClickHouse', () => {+ it('String survives quotes, backslashes, braces and whitespace', async () => {+ expect(await value(sql`SELECT ${sql.string(HOSTILE)} AS value`)).toBe(+ HOSTILE+ );+ });++ it('UInt64', async () => {+ // chQueryWithMeta parseFloats anything whose meta type contains 'Int',+ // so this asserts the value that reaches a caller, not the wire form.+ expect(await value(sql`SELECT ${sql.uint64(BIG_UINT64)} AS value`)).toBe(+ BIG_UINT64+ );+ expect(await value(sql`SELECT ${sql.uint64(42n)} AS value`)).toBe(42);+ });++ it('Float64', async () => {+ expect(await value(sql`SELECT ${sql.float64(-1.5)} AS value`)).toBe(-1.5);+ });++ it('DateTime64(3) keeps millisecond precision', async () => {+ const date = new Date('2026-04-29T13:45:07.123Z');+ expect(+ await value(+ sql`SELECT toUnixTimestamp64Milli(${sql.dateTime64(date)}) AS value`+ )+ ).toBe(date.getTime());+ expect(+ await value(+ sql`SELECT toString(${sql.dateTime64('2026-04-29 13:45:07.123')}) AS value`+ )+ ).toBe('2026-04-29 13:45:07.123');+ });++ it('Date takes the calendar-day text form, not a timestamp', async () => {+ expect(+ await value(+ sql`SELECT toString(${sql.date(new Date('2026-04-29T13:45:00.000Z'))}) AS value`+ )+ ).toBe('2026-04-29');++ // Why sql.date() formats: the driver serializes a JS Date as a unix+ // timestamp, and ClickHouse refuses that for the Date type.+ await expect(+ chQuery(sql`SELECT ${sql.param('Date', new Date())} AS value`)+ ).rejects.toThrow(/cannot be parsed as Date/);+ });++ it('Array(String) with hostile items', async () => {+ const items = [HOSTILE, "', 'injected", '{p1:String}'];+ expect(+ await value(sql`SELECT ${sql.array('String', items)} AS value`)+ ).toEqual(items);+ });++ it('Map(String,String) with hostile keys and values', async () => {+ const map = { [HOSTILE]: HOSTILE, "key'": '}value{' };+ expect(+ await value(sql`SELECT ${sql.map('String', 'String', map)} AS value`)+ ).toEqual(map);+ });++ it('Nullable(String) carries null and a hostile value', async () => {+ expect(+ await value(sql`SELECT isNull(${sql.nullable('String', null)}) AS value`)+ ).toBe(1);+ expect(+ await value(sql`SELECT ${sql.nullable('String', HOSTILE)} AS value`)+ ).toBe(HOSTILE);+ });++ it('UUID', async () => {+ const id = '11111111-2222-3333-4444-555555555555';+ expect(await value(sql`SELECT toString(${sql.uuid(id)}) AS value`)).toBe(+ id+ );+ });++ it('Bool', async () => {+ expect(await value(sql`SELECT ${sql.bool(true)} AS value`)).toBe(true);+ expect(await value(sql`SELECT ${sql.bool(false)} AS value`)).toBe(false);+ });++ it('a payload that looks like SQL stays data', async () => {+ // Interpolated, this returns 1 row. Bound, it returns 0.+ const payload = "nope' OR '1'='1";+ expect(+ await value(+ sql`SELECT count() AS value FROM (SELECT 'safe' AS path) WHERE path = ${sql.string(payload)}`+ )+ ).toBe(0);+ });++ it('rejects a hostile payload on a non-text type instead of inlining it', async () => {+ // The types above cannot carry quotes or braces, so the round-trip for+ // them is the negative one: the server refuses the value as data. The+ // payload never becomes SQL text, which is the property under test.+ const hostileOn: [string, ReturnType<typeof sql.param>][] = [+ ['UInt64', sql.param('UInt64', HOSTILE)],+ ['Float64', sql.param('Float64', HOSTILE)],+ ['DateTime64(3)', sql.dateTime64(HOSTILE)],+ ['Date', sql.date(HOSTILE)],+ ['Bool', sql.param('Bool', HOSTILE)],+ ];++ for (const [type, hostileParam] of hostileOn) {+ await expect(+ chQuery(sql`SELECT ${hostileParam} AS value`),+ type+ ).rejects.toThrow(+ new RegExp(`cannot be parsed as ${escapeRegExp(type)}`)+ );+ }+ });+});++/**+ * Identifier positional matrix — measured against ClickHouse 26.1.3.52 on+ * 2026-09-02. This is the split ADR-013 risk 3 says is unknown until P2 runs:+ * where `{x:Identifier}` works, and where a call site must use `sql.id()`+ * (validated inline text) instead.+ *+ * | Position | `{x:Identifier}` |+ * |--------------------------------------------|------------------|+ * | SELECT column | works |+ * | SELECT alias (`1 AS {c:Identifier}`) | works |+ * | Aggregate/function argument | works |+ * | WHERE column | works |+ * | GROUP BY column | works |+ * | ORDER BY column | works |+ * | FROM table | works |+ * | JOIN table | works |+ * | FROM table inside a CTE | works |+ * | database.table as two separate params | works |+ * | `'db.table'` in ONE param | FAILS — quoted whole |+ * | `'alias.column'` in ONE param | FAILS — quoted whole |+ * | SETTINGS value (`session_timezone = {…}`) | FAILS — syntax error, any type |+ *+ * Consequences for the P5-P8 conversions:+ * - qualified names are two params, or one `sql.id('profiles.name')`;+ * - a per-query `session_timezone` goes through `chQuery`'s+ * `clickhouse_settings` argument, not through the SQL text.+ */+describeAgainstClickhouse('Identifier positional matrix', () => {+ const table = sql.identifier('events');++ it('works in SELECT, alias, function, WHERE, GROUP BY and ORDER BY', async () => {+ const column = sql.identifier('a');+ expect(+ await value(sql`SELECT ${column} AS value FROM (SELECT 1 AS a)`)+ ).toBe(1);+ expect(+ await value(sql`SELECT sum(${column}) AS value FROM (SELECT 1 AS a)`)+ ).toBe(1);+ expect(+ await value(+ sql`SELECT a AS value FROM (SELECT 1 AS a) WHERE ${column} = 1`+ )+ ).toBe(1);+ expect(+ await value(+ sql`SELECT count() AS value FROM (SELECT 1 AS a) GROUP BY ${column} ORDER BY ${column}`+ )+ ).toBe(1);++ const aliased = await chQuery<Record<string, number>>(+ sql`SELECT 1 AS ${sql.identifier('value')}`+ );+ expect(aliased[0]?.value).toBe(1);+ });++ it('works for a table name in FROM, JOIN and a CTE', async () => {+ expect(+ await value(sql`SELECT count() AS value FROM ${table} WHERE 0`)+ ).toBe(0);+ expect(+ await value(+ sql`SELECT count() AS value FROM (SELECT 1 AS a) AS x JOIN ${table} AS y ON 0 WHERE 0`+ )+ ).toBe(0);+ expect(+ await value(+ sql`WITH c AS (SELECT count() AS value FROM ${table} WHERE 0) SELECT value FROM c`+ )+ ).toBe(0);+ });++ it('works for database.table as two params', async () => {+ expect(+ await value(+ sql`SELECT count() AS value FROM ${sql.identifier(CLICKHOUSE_TEST_DATABASE)}.${table} WHERE 0`+ )+ ).toBe(0);+ });++ it('fails for a dotted name in a single param — use sql.id or two params', async () => {+ await expect(+ chQuery(+ sql`SELECT count() AS value FROM ${sql.identifier(`${CLICKHOUSE_TEST_DATABASE}.events`)} WHERE 0`+ )+ ).rejects.toThrow(/Unknown table expression identifier/);+ await expect(+ chQuery(+ sql`SELECT ${sql.identifier('t.a')} AS value FROM (SELECT 1 AS a) AS t`+ )+ ).rejects.toThrow(/Unknown expression identifier/);++ // sql.id() is the supported form for a qualified reference.+ expect(+ await value(+ sql`SELECT ${sql.id('t.a')} AS value FROM (SELECT 1 AS a) AS t`+ )+ ).toBe(1);+ });++ it('fails inside a SETTINGS clause, for any param type', async () => {+ await expect(+ chQuery(sql`SELECT 1 SETTINGS session_timezone = ${sql.string('UTC')}`)+ ).rejects.toThrow(/Syntax error/);+ await expect(+ chQuery(+ sql`SELECT 1 SETTINGS session_timezone = ${sql.identifier('UTC')}`+ )+ ).rejects.toThrow(/Syntax error/);++ // The supported route: settings travel beside the query.+ expect(+ await value(+ sql`SELECT toString(toTimeZone(toDateTime(0), timezone())) AS value`,+ { session_timezone: 'Europe/Stockholm' }+ )+ ).toBe('1970-01-01 01:00:00');+ });++ it('neutralises an injection payload in an Identifier param', async () => {+ await expect(+ chQuery(+ sql`SELECT ${sql.identifier('a) UNION ALL (SELECT 2')} AS value FROM (SELECT 1 AS a)`+ )+ ).rejects.toThrow(/Unknown expression identifier/);+ });+});++describeAgainstClickhouse('local builder fallback executes (R5)', () => {+ const PROFILE_COLUMNS = ['first_name', 'last_name'] as const;++ it('binds every value the imperative assembly produced', async () => {+ // R5 permits local assembly; it does not permit unbound text. Hostile+ // values go in as data and the server matches nothing.+ const filters = [+ { column: 'first_name', value: HOSTILE },+ { column: 'last_name', value: "' OR 1=1 --" },+ ];+ const conditions = filters.map(+ ({ column, value }) =>+ sql`${sql.id(column, PROFILE_COLUMNS)} = ${sql.string(value)}`+ );++ expect(+ await value(+ sql`SELECT count() AS value FROM profiles WHERE ${sql.join(conditions, ' AND ')}`+ )+ ).toBe(0);+ });+});++describeAgainstClickhouse('cluster constructs execute unchanged (R4)', () => {+ it('accepts GLOBAL IN, PREWHERE, ARRAY JOIN, LIMIT BY and a bound LIMIT', async () => {+ const projectId = sql.string('__no_such_project__');+ const rows = await chQuery(sql`+ SELECT e.id AS id, g AS grp+ FROM events AS e+ ARRAY JOIN e.groups AS g+ PREWHERE e.project_id = ${projectId}+ WHERE e.profile_id GLOBAL IN (+ SELECT id FROM profiles WHERE project_id = ${projectId}+ )+ ORDER BY e.created_at+ LIMIT 1 BY e.id+ LIMIT ${sql.param('UInt32', 10)}+ `);+ expect(rows).toEqual([]);+ });++ it('accepts GLOBAL JOIN alongside a trailing literal SETTINGS clause', async () => {+ // The SETTINGS *value* cannot be a param (see the matrix above), but a+ // literal SETTINGS tail must still survive the tag untouched.+ const projectId = sql.string('__no_such_project__');+ expect(+ await value(sql`+ SELECT count() AS value+ FROM events AS e+ GLOBAL JOIN (+ SELECT id FROM profiles WHERE project_id = ${projectId}+ ) AS p ON e.profile_id = p.id+ WHERE e.project_id = ${projectId}+ SETTINGS max_execution_time = 30+ `)+ ).toBe(0);+ });++ it('accepts FINAL and WITH FILL with bound bounds', async () => {+ const from = sql.dateTime64('2026-01-01 00:00:00.000');+ const to = sql.dateTime64('2026-01-04 00:00:00.000');+ const rows = await chQuery<{ d: string; c: number }>(sql`+ SELECT toStartOfDay(created_at) AS d, count() AS c+ FROM sessions FINAL+ WHERE project_id = ${sql.string('__no_such_project__')}+ AND created_at >= ${from} AND created_at < ${to}+ GROUP BY d+ ORDER BY d WITH FILL FROM toStartOfDay(${from}) TO toStartOfDay(${to}) STEP toIntervalDay(1)+ `);+ expect(rows.map((row) => row.c)).toEqual([0, 0, 0]);+ });+});diff --git a/packages/db/src/clickhouse/sql.round-robin.clickhouse.test.ts b/packages/db/src/clickhouse/sql.round-robin.clickhouse.test.tsnew file mode 100644index 000000000..26cd38191--- /dev/null+++ b/packages/db/src/clickhouse/sql.round-robin.clickhouse.test.ts@@ -0,0 +1,112 @@+import { describe, expect, it } from 'vitest';+import { sql } from './sql';++/**+ * ADR-013 R1, the half the round-trip suite cannot reach: bound params must+ * travel through `withRetry` / round-robin, so a parameterised read keeps the+ * failover behaviour `chQueryWithMeta` already had for raw text.+ *+ * The setup is a two-node `CLICKHOUSE_URL` whose first node is dead. The+ * picker starts at index 0, so every query here fails over to node 1 — if the+ * params were dropped anywhere on the retry path, the second attempt would+ * return a different answer or an error instead of the payload.+ */++const CLICKHOUSE_TEST_DATABASE = 'openpanel_test';+const DEFAULT_CLICKHOUSE_BASE_URL = 'http://localhost:8123';++/** Port 1 is privileged and unbindable without root, so it is reliably refused. */+const DEAD_NODE_URL = 'http://127.0.0.1:1';++/** Same reason as the round-trip suite: one local node, many parallel files. */+const CLICKHOUSE_TEST_REQUEST_TIMEOUT_MS = 120_000;++function liveNodeUrl(): { base: string; withDatabase: string } {+ const configured = (process.env.CLICKHOUSE_URL ?? '').split(',')[0]?.trim();+ const url = new URL(configured || DEFAULT_CLICKHOUSE_BASE_URL);+ url.pathname = '/';+ const base = url.toString();+ url.pathname = `/${CLICKHOUSE_TEST_DATABASE}`;+ return { base, withDatabase: url.toString() };+}++const { base: LIVE_BASE_URL, withDatabase: LIVE_NODE_URL } = liveNodeUrl();++async function bootstrapTestDatabase(): Promise<boolean> {+ try {+ const response = await fetch(LIVE_BASE_URL, {+ method: 'POST',+ body: `CREATE DATABASE IF NOT EXISTS ${CLICKHOUSE_TEST_DATABASE}`,+ });+ return response.ok;+ } catch {+ return false;+ }+}++/** If port 1 ever answers, the failover never happens and this suite would+ * pass without testing anything. */+async function deadNodeIsRefused(): Promise<boolean> {+ try {+ await fetch(DEAD_NODE_URL, { method: 'POST', body: 'SELECT 1' });+ return false;+ } catch {+ return true;+ }+}++const clickhouseReachable = await bootstrapTestDatabase();+const failoverIsReal = await deadNodeIsRefused();++// ./client reads these once, at module evaluation.+process.env.CLICKHOUSE_URL = `${DEAD_NODE_URL}/${CLICKHOUSE_TEST_DATABASE},${LIVE_NODE_URL}`;+process.env.CLICKHOUSE_REQUEST_TIMEOUT_MS = String(+ CLICKHOUSE_TEST_REQUEST_TIMEOUT_MS+);+// Without this the dead node is sin-binned after the first failure and the+// later cases never exercise the retry at all.+process.env.CLICKHOUSE_UNHEALTHY_MARK_MS = '0';++const { chQuery, chQueryWithMeta } = await import('./client');++const HOSTILE = `it's a "quoted" \\ backslash {p1:String} }brace{`;++const describeAgainstClickhouse = describe.skipIf(+ !(clickhouseReachable && failoverIsReal)+);++describeAgainstClickhouse(+ 'query_params survive round-robin failover (R1)',+ () => {+ it('carries bound params to the node the retry lands on', async () => {+ const rows = await chQuery<{ value: string }>(+ sql`SELECT ${sql.string(HOSTILE)} AS value`+ );+ expect(rows[0]?.value).toBe(HOSTILE);+ });++ it('carries composite params too', async () => {+ const items = [HOSTILE, "', 'injected"];+ const rows = await chQuery<{ items: string[]; at: string }>(sql`+ SELECT ${sql.array('String', items)} AS items,+ toString(${sql.dateTime64('2026-04-29 13:45:07.123')}) AS at+ `);+ expect(rows[0]?.items).toEqual(items);+ expect(rows[0]?.at).toBe('2026-04-29 13:45:07.123');+ });++ it('keeps the meta path parameterised as well', async () => {+ const result = await chQueryWithMeta<{ value: string }>(+ sql`SELECT ${sql.string(HOSTILE)} AS value`+ );+ expect(result.data[0]?.value).toBe(HOSTILE);+ });++ it('leaves the plain-string path unchanged', async () => {+ // The 122 existing `chQuery(string)` call sites must be unaffected by the+ // params slot: no params in, no params on the wire.+ const rows = await chQuery<{ value: number }>('SELECT 1 AS value');+ expect(rows[0]?.value).toBe(1);+ });+ }+);diff --git a/packages/db/src/clickhouse/sql.test.ts b/packages/db/src/clickhouse/sql.test.tsnew file mode 100644index 000000000..91a65b5b5--- /dev/null+++ b/packages/db/src/clickhouse/sql.test.ts@@ -0,0 +1,273 @@+import { describe, expect, it } from 'vitest';+import { type SqlFragment, SqlIdentifierError, sql, toStatement } from './sql';++describe('sql tag — slot typing (R1)', () => {+ it('is a compile-time error to interpolate a bare value', () => {+ // The whole safety property of ADR-013: a raw string or number can never+ // reach the SQL text. If either of these ever stops erroring, injection+ // has become writable again and `pnpm run typecheck` is the alarm.+ const path = "/blog' OR '1'='1";+ // @ts-expect-error — a bare string is not a SqlFragment | SqlParam+ const withString = sql`SELECT * FROM events WHERE path = ${path}`;+ // @ts-expect-error — a bare number is not a SqlFragment | SqlParam+ const withNumber = sql`SELECT * FROM events LIMIT ${10}`;++ expect(withString).toBeDefined();+ expect(withNumber).toBeDefined();+ });++ it('accepts params and fragments', () => {+ const { query, query_params } = sql`+ SELECT * FROM events WHERE project_id = ${sql.string('p1')}+ `.toStatement();++ expect(query).toContain('{p1:String}');+ expect(query_params).toEqual({ p1: 'p1' });+ });+});++describe('auto-named params (R2)', () => {+ it('numbers params in source order', () => {+ const { query, query_params } = sql`+ SELECT ${sql.string('a')}, ${sql.uint64(2)}, ${sql.float64(3.5)}+ `.toStatement();++ expect(query).toContain('{p1:String}, {p2:UInt64}, {p3:Float64}');+ expect(query_params).toEqual({ p1: 'a', p2: 2, p3: 3.5 });+ });++ it('stays collision-free through three levels of nesting', () => {+ const innermost = sql`name = ${sql.string('inner')}`;+ const middle = sql`(${innermost} AND country = ${sql.string('middle')})`;+ const outer = sql`(${middle} AND project_id = ${sql.string('outer')})`;++ const { query, query_params } = sql`+ SELECT * FROM events WHERE ${outer} AND created_at > ${sql.dateTime64('2026-01-01 00:00:00.000')}+ `.toStatement();++ expect(query).toContain(+ '((name = {p1:String} AND country = {p2:String}) AND project_id = {p3:String})'+ );+ expect(query_params).toEqual({+ p1: 'inner',+ p2: 'middle',+ p3: 'outer',+ p4: '2026-01-01 00:00:00.000',+ });+ });++ it('gives each use of a reused fragment its own param name', () => {+ // Two independently-built fragments — and the same fragment twice — must+ // not share a placeholder, or one value silently binds to both slots.+ const shared = sql`name = ${sql.string('screen_view')}`;+ const { query, query_params } =+ sql`SELECT countIf(${shared}), countIf(${shared})`.toStatement();++ expect(query).toBe(+ 'SELECT countIf(name = {p1:String}), countIf(name = {p2:String})'+ );+ expect(query_params).toEqual({ p1: 'screen_view', p2: 'screen_view' });+ });++ it('composes lists with join without any raw text', () => {+ const names = ['a', 'b', 'c'];+ const clauses = sql.join(+ names.map((name) => sql`name = ${sql.string(name)}`),+ ' OR '+ );+ const { query, query_params } =+ sql`SELECT * FROM events WHERE ${clauses}`.toStatement();++ expect(query).toBe(+ 'SELECT * FROM events WHERE name = {p1:String} OR name = {p2:String} OR name = {p3:String}'+ );+ expect(query_params).toEqual({ p1: 'a', p2: 'b', p3: 'c' });+ });++ it('renders empty fragments and empty joins as nothing', () => {+ const { query, query_params } =+ sql`SELECT 1 ${sql.empty}${sql.join([])}`.toStatement();++ expect(query).toBe('SELECT 1 ');+ expect(query_params).toEqual({});+ });++ it('leaves a plain string query untouched, with no params', () => {+ expect(toStatement('SELECT 1')).toEqual({+ query: 'SELECT 1',+ query_params: {},+ });+ });+});++describe('sql.id (R3)', () => {+ const PROFILE_COLUMNS = ['id', 'first_name', 'last_name'] as const;++ it('inlines a bare identifier', () => {+ expect(sql.id('first_name').toStatement().query).toBe('first_name');+ });++ it('inlines a once-qualified identifier', () => {+ expect(sql.id('profiles.first_name').toStatement().query).toBe(+ 'profiles.first_name'+ );+ });++ it('throws — never falls back to interpolation — for unsafe input', () => {+ const hostile = [+ "name'; DROP TABLE events; --",+ 'name` OR 1=1',+ 'name OR 1=1',+ 'properties["x"]',+ 'a.b.c',+ '',+ '1name',+ 'näme',+ 'a'.repeat(65),+ ];++ for (const identifier of hostile) {+ expect(() => sql.id(identifier)).toThrow(SqlIdentifierError);+ }+ });++ it('throws for an identifier outside the caller whitelist', () => {+ expect(() => sql.id('first_name', PROFILE_COLUMNS)).not.toThrow();+ expect(() => sql.id('password', PROFILE_COLUMNS)).toThrow(+ SqlIdentifierError+ );+ });+});++describe('raw ClickHouse constructs pass through (R4)', () => {+ it('does not obstruct GLOBAL IN/JOIN, PREWHERE, FINAL, LIMIT BY, WITH FILL, ARRAY JOIN or SETTINGS', () => {+ const projectId = sql.string('proj_1');+ const { query } = sql`+ SELECT s.session_id, arrayJoin(s.paths) AS path+ FROM sessions AS s FINAL+ ARRAY JOIN s.paths+ PREWHERE s.project_id = ${projectId}+ WHERE s.profile_id GLOBAL IN (SELECT id FROM profiles WHERE project_id = ${projectId})+ GLOBAL JOIN (SELECT id FROM profiles WHERE project_id = ${projectId}) AS p ON s.profile_id = p.id+ ORDER BY s.created_at+ LIMIT 1 BY s.session_id+ WITH FILL FROM toDateTime(${sql.dateTime64('2026-01-01 00:00:00.000')}) STEP 1+ SETTINGS session_timezone = ${sql.string('Europe/Stockholm')}+ `.toStatement();++ for (const construct of [+ 'FINAL',+ 'ARRAY JOIN',+ 'PREWHERE',+ 'GLOBAL IN',+ 'GLOBAL JOIN',+ 'LIMIT 1 BY',+ 'WITH FILL',+ 'SETTINGS session_timezone',+ ]) {+ expect(query).toContain(construct);+ }+ });+});++describe('local function-scoped builder fallback (R5)', () => {+ const PROFILE_COLUMNS = ['first_name', 'country'] as const;++ /**+ * The shape R5 sanctions: assembly is imperative and local, but the return+ * type is a fragment, so every value still leaves as a bound `{pN:Type}`+ * and every identifier still goes through `sql.id`'s whitelist.+ */+ const buildProfileFilter = (+ filters: readonly { column: string; value: string }[]+ ): SqlFragment => {+ const conditions: SqlFragment[] = [];+ for (const { column, value } of filters) {+ conditions.push(+ sql`${sql.id(column, PROFILE_COLUMNS)} = ${sql.string(value)}`+ );+ }+ return conditions.length === 0+ ? sql.empty+ : sql`WHERE ${sql.join(conditions, ' AND ')}`;+ };++ it('emits bound params from an imperatively assembled query', () => {+ const { query, query_params } =+ sql`SELECT id FROM profiles ${buildProfileFilter([+ { column: 'first_name', value: "O'Brien" },+ { column: 'country', value: 'SE' },+ ])}`.toStatement();++ expect(query).toBe(+ 'SELECT id FROM profiles WHERE first_name = {p1:String} AND country = {p2:String}'+ );+ expect(query_params).toEqual({ p1: "O'Brien", p2: 'SE' });+ });++ it('still refuses an identifier the builder did not whitelist', () => {+ expect(() =>+ buildProfileFilter([{ column: 'email', value: '[email protected]' }])+ ).toThrow(SqlIdentifierError);+ });++ it('degrades to no clause, and no params, when there is nothing to filter', () => {+ expect(+ sql`SELECT id FROM profiles ${buildProfileFilter([])}`.toStatement()+ ).toEqual({ query: 'SELECT id FROM profiles ', query_params: {} });+ });+});++describe('injection (ported from query-builder.test.ts)', () => {+ const bind = (value: string) =>+ sql`SELECT * FROM events WHERE e.path = ${sql.string(value)}`.toStatement();++ it('cannot break out of a string with a date-substring payload', () => {+ const payload = "/blog/2024-01-01' OR '1'='1";+ const { query, query_params } = bind(payload);++ expect(query).toBe('SELECT * FROM events WHERE e.path = {p1:String}');+ expect(query).not.toContain(payload);+ expect(query).not.toContain("'");+ // The value travels beside the query, verbatim — the server escapes it.+ expect(query_params.p1).toBe(payload);+ });++ it('leaves a path containing a date substring intact (the escapeDate bug)', () => {+ const payload = '/blog/2024-03-17-supabase-activity-scheduler';+ const { query_params } = bind(payload);++ // clix re-quoted any date-shaped substring in place; binding cannot.+ expect(query_params.p1).toBe(payload);+ });++ it('does not re-interpret a payload that looks like a placeholder', () => {+ const { query, query_params } = bind('{p1:String}');++ expect(query).toBe('SELECT * FROM events WHERE e.path = {p1:String}');+ expect(query_params.p1).toBe('{p1:String}');+ });+});++describe('param constructors', () => {+ it('formats a Date value as a calendar day for the Date type', () => {+ // ClickHouse rejects a unix timestamp for `Date`; the driver would send+ // one for a JS Date. See sql.clickhouse.test.ts.+ expect(sql.date(new Date('2026-04-29T13:45:00.000Z')).value).toBe(+ '2026-04-29'+ );+ expect(sql.date('2026-04-29').value).toBe('2026-04-29');+ });++ it('declares composite types', () => {+ expect(sql.array('String', ['a']).type).toBe('Array(String)');+ expect(sql.map('String', 'String', { a: 'b' }).type).toBe(+ 'Map(String,String)'+ );+ expect(sql.nullable('String', null).type).toBe('Nullable(String)');+ expect(sql.dateTime64('2026-01-01 00:00:00.000').type).toBe(+ 'DateTime64(3)'+ );+ expect(sql.identifier('created_at').type).toBe('Identifier');+ });+});diff --git a/packages/db/src/clickhouse/sql.ts b/packages/db/src/clickhouse/sql.tsnew file mode 100644index 000000000..b78b31648--- /dev/null+++ b/packages/db/src/clickhouse/sql.ts@@ -0,0 +1,303 @@+/**+ * The ClickHouse `sql` tag (ADR-013).+ *+ * Two tiers, one tag:+ *+ * 1. **Static queries** are plain SQL text — the tag is a no-op wrapper.+ * 2. **Dynamic composition** builds from fragments, and every *value* binds+ * as a ClickHouse-native `{name:Type}` query parameter, escaped by the+ * server rather than by us.+ *+ * The safety property is the type of an interpolation slot: it accepts only+ * `SqlFragment | SqlParam`, so a bare string or number in a slot is a+ * compile-time error. There is deliberately no `sql.raw()` — dynamic+ * structure composes with {@link SqlBuilder.join}, and the only path from an+ * untrusted string to SQL text is {@link SqlBuilder.id}, which validates and+ * throws.+ *+ * Placeholder names are assigned at render time from a single counter, so+ * fragments nest and repeat without colliding (R2).+ *+ * Not re-exported from the package barrel during the clix coexistence window:+ * `query-builder.ts` exports its own `SqlParam` type, and `export *` from both+ * is ambiguous. It joins the barrel when clix is deleted.+ */++/** Auto-generated placeholders are `{p1:Type}`, `{p2:Type}`, … */+const PARAM_NAME_PREFIX = 'p';+const FIRST_PARAM_INDEX = 1;++/** `table.column` is the deepest qualification any call site needs. */+const IDENTIFIER_MAX_PARTS = 2;+const IDENTIFIER_MAX_LENGTH = 64;+const IDENTIFIER_PART_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;++const DEFAULT_DATETIME64_PRECISION = 3;+const ISO_DATE_LENGTH = 'YYYY-MM-DD'.length;++const FRAGMENT_BRAND: unique symbol = Symbol('openpanel.sql.fragment');+const PARAM_BRAND: unique symbol = Symbol('openpanel.sql.param');++/**+ * What `@clickhouse/client` can serialize into `query_params`. Values are+ * handed over untouched — the driver escapes them and the server parses them+ * against the declared type.+ */+export type SqlParamValue =+ | string+ | number+ | bigint+ | boolean+ | Date+ | null+ | readonly SqlParamValue[]+ | ReadonlyMap<string | number, SqlParamValue>+ | { readonly [key: string]: SqlParamValue };++/** A ClickHouse type name as written inside a placeholder, e.g. `Array(String)`. */+export type SqlParamType = string;++/** A single bound value. Only ever produced by the `sql.*` constructors. */+export interface SqlParam {+ readonly [PARAM_BRAND]: true;+ readonly type: SqlParamType;+ readonly value: SqlParamValue;+}++/** The only things an interpolation slot accepts. */+export type SqlSlot = SqlFragment | SqlParam;++/** What goes on the wire: query text plus the params it references. */+export interface SqlStatement {+ readonly query: string;+ readonly query_params: Record<string, SqlParamValue>;+}++interface RenderState {+ readonly text: string[];+ readonly params: Record<string, SqlParamValue>;+ nextIndex: number;+}++/**+ * A piece of SQL and the slots between its literal parts. Immutable and+ * name-free: placeholder names are only assigned by {@link toStatement}, which+ * is what makes nesting and reuse collision-free.+ */+export class SqlFragment {+ readonly [FRAGMENT_BRAND] = true as const;++ constructor(+ readonly strings: readonly string[],+ readonly slots: readonly SqlSlot[]+ ) {}++ toStatement(): SqlStatement {+ const state: RenderState = {+ text: [],+ params: {},+ nextIndex: FIRST_PARAM_INDEX,+ };+ render(this, state);+ return { query: state.text.join(''), query_params: state.params };+ }+}++function isSqlParam(slot: SqlSlot): slot is SqlParam {+ return PARAM_BRAND in slot;+}++export function isSqlFragment(value: unknown): value is SqlFragment {+ return value instanceof SqlFragment;+}++function render(fragment: SqlFragment, state: RenderState): void {+ for (const [index, literal] of fragment.strings.entries()) {+ state.text.push(literal);+ const slot = fragment.slots[index];+ if (!slot) {+ continue;+ }+ if (isSqlParam(slot)) {+ const name = `${PARAM_NAME_PREFIX}${state.nextIndex++}`;+ state.params[name] = slot.value;+ state.text.push(`{${name}:${slot.type}}`);+ continue;+ }+ render(slot, state);+ }+}++/** Accepts the raw-string form too, so `chQuery` keeps one code path. */+export function toStatement(query: string | SqlFragment): SqlStatement {+ return typeof query === 'string'+ ? { query, query_params: {} }+ : query.toStatement();+}++export class SqlIdentifierError extends Error {+ constructor(identifier: string, reason: string) {+ super(+ `Refusing to inline identifier ${JSON.stringify(identifier)}: ${reason}`+ );+ this.name = 'SqlIdentifierError';+ }+}++/**+ * Separators are a closed set on purpose: a free-form string here would be a+ * hole straight through the type-level guarantee.+ */+const SQL_JOIN_SEPARATORS = [+ ', ',+ ' AND ',+ ' OR ',+ ' UNION ALL ',+ ' ',+ '\n',+] as const;+export type SqlJoinSeparator = (typeof SQL_JOIN_SEPARATORS)[number];+const DEFAULT_JOIN_SEPARATOR: SqlJoinSeparator = ', ';++function param(type: SqlParamType, value: SqlParamValue): SqlParam {+ return { [PARAM_BRAND]: true, type, value };+}++function assertSafeIdentifier(+ identifier: string,+ allowed?: readonly string[]+): void {+ if (allowed && !allowed.includes(identifier)) {+ throw new SqlIdentifierError(identifier, 'not in the caller whitelist');+ }+ if (identifier.length === 0) {+ throw new SqlIdentifierError(identifier, 'empty');+ }+ if (identifier.length > IDENTIFIER_MAX_LENGTH) {+ throw new SqlIdentifierError(+ identifier,+ `longer than ${IDENTIFIER_MAX_LENGTH} characters`+ );+ }+ const parts = identifier.split('.');+ if (parts.length > IDENTIFIER_MAX_PARTS) {+ throw new SqlIdentifierError(+ identifier,+ `more than ${IDENTIFIER_MAX_PARTS} dot-separated parts`+ );+ }+ for (const part of parts) {+ if (!IDENTIFIER_PART_PATTERN.test(part)) {+ throw new SqlIdentifierError(+ identifier,+ `${JSON.stringify(part)} is not a bare identifier`+ );+ }+ }+}++/** UTC calendar day — a `Date` param is parsed as text, never as a timestamp. */+function toClickhouseDate(value: Date | string): string {+ return typeof value === 'string'+ ? value+ : value.toISOString().slice(0, ISO_DATE_LENGTH);+}++const EMPTY_FRAGMENT = new SqlFragment([''], []);++interface SqlBuilder {+ (strings: TemplateStringsArray, ...slots: SqlSlot[]): SqlFragment;++ /** A fragment that renders to nothing — the identity for conditionals. */+ readonly empty: SqlFragment;++ /** Escape hatch for a ClickHouse type this module has no shortcut for. */+ param(type: SqlParamType, value: SqlParamValue): SqlParam;++ string(value: string): SqlParam;+ uint64(value: number | bigint | string): SqlParam;+ float64(value: number): SqlParam;+ /** Text form only — ClickHouse rejects a unix timestamp for `Date`. */+ date(value: Date | string): SqlParam;+ dateTime64(value: Date | string | number, precision?: number): SqlParam;+ uuid(value: string): SqlParam;+ bool(value: boolean): SqlParam;+ array(itemType: SqlParamType, values: readonly SqlParamValue[]): SqlParam;+ map(+ keyType: SqlParamType,+ valueType: SqlParamType,+ value:+ | ReadonlyMap<string | number, SqlParamValue>+ | Record<string, SqlParamValue>+ ): SqlParam;+ nullable(innerType: SqlParamType, value: SqlParamValue): SqlParam;++ /**+ * Bind an identifier as `{name:Identifier}`. Server-side and safe, but not+ * accepted in every syntactic position — see the matrix in+ * `sql.clickhouse.test.ts`. Notably a qualified `table.column` must be two+ * separate params, or `sql.id`.+ */+ identifier(name: string): SqlParam;++ /**+ * Inline a validated identifier as SQL text, for the positions where+ * `{name:Identifier}` does not work. Throws on anything that is not a bare+ * (optionally once-qualified) identifier — it never falls back to+ * interpolating the input (R3). Pass `allowed` wherever the call site has a+ * closed column set.+ */+ id(name: string, allowed?: readonly string[]): SqlFragment;++ join(+ fragments: readonly SqlFragment[],+ separator?: SqlJoinSeparator+ ): SqlFragment;+}++const tag = (strings: TemplateStringsArray, ...slots: SqlSlot[]): SqlFragment =>+ new SqlFragment(Array.from(strings), slots);++export const sql: SqlBuilder = Object.assign(tag, {+ empty: EMPTY_FRAGMENT,++ param,+ string: (value: string) => param('String', value),+ uint64: (value: number | bigint | string) => param('UInt64', value),+ float64: (value: number) => param('Float64', value),+ date: (value: Date | string) => param('Date', toClickhouseDate(value)),+ dateTime64: (+ value: Date | string | number,+ precision: number = DEFAULT_DATETIME64_PRECISION+ ) => param(`DateTime64(${precision})`, value),+ uuid: (value: string) => param('UUID', value),+ bool: (value: boolean) => param('Bool', value),+ array: (itemType: SqlParamType, values: readonly SqlParamValue[]) =>+ param(`Array(${itemType})`, values),+ map: (+ keyType: SqlParamType,+ valueType: SqlParamType,+ value:+ | ReadonlyMap<string | number, SqlParamValue>+ | Record<string, SqlParamValue>+ ) => param(`Map(${keyType},${valueType})`, value),+ nullable: (innerType: SqlParamType, value: SqlParamValue) =>+ param(`Nullable(${innerType})`, value),+ identifier: (name: string) => param('Identifier', name),++ id: (name: string, allowed?: readonly string[]): SqlFragment => {+ assertSafeIdentifier(name, allowed);+ return new SqlFragment([name], []);+ },++ join: (+ fragments: readonly SqlFragment[],+ separator: SqlJoinSeparator = DEFAULT_JOIN_SEPARATOR+ ): SqlFragment => {+ if (fragments.length === 0) {+ return EMPTY_FRAGMENT;+ }+ const strings = ['', ...fragments.slice(1).map(() => separator), ''];+ return new SqlFragment(strings, fragments);+ },+});