🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(worker): qualify ClickHouse backfill table names (#15847)
Falk Telzerow1099e2655a87
diff --git a/worker/src/__tests__/v4BackfillDdl.unit.test.ts b/worker/src/__tests__/v4BackfillDdl.unit.test.tsnew file mode 100644index 0000000000..44ad4ff570--- /dev/null+++ b/worker/src/__tests__/v4BackfillDdl.unit.test.ts@@ -0,0 +1,97 @@+import { describe, expect, it } from "vitest";+import {+ PID_TID_SORTING_TABLE,+ buildDisableSharedMergeTreeMergesQuery,+ buildDropScratchTableQuery,+ buildFreezeReplicatedMergesQuery,+ buildStopMergesQuery,+ buildSyncReplicaQuery,+ onClusterClause,+ pidTidSortingTable,+} from "../backgroundMigrations/utils/v4BackfillDdl";++/**+ * Regression coverage for #15845: the V4 historic backfill emitted unqualified+ * `SYSTEM STOP MERGES` / `SYSTEM SYNC REPLICA`. ClickHouse's+ * `AddDefaultDatabaseVisitor` only qualifies `ASTAlterQuery`,+ * `ASTQueryWithTableAndOutput` and `ASTRenameQuery` with the initiator's+ * database — `ASTSystemQuery` is never visited, so each host's DDL worker+ * resolved the scratch table against its own `default` database and the chain+ * failed with `Table default.observations_pid_tid_sorting does not exist`.+ *+ * Every statement below must therefore name the configured database, and the+ * `ON CLUSTER` statements are the ones that actually regress.+ */+const NON_DEFAULT = {+ database: "langfuse",+ clusterEnabled: true,+ clusterName: "default",+};++describe("V4 backfill DDL", () => {+ it("qualifies the ON CLUSTER SYSTEM commands with the configured database", () => {+ expect(buildStopMergesQuery(NON_DEFAULT)).toBe(+ "SYSTEM STOP MERGES ON CLUSTER `default` `langfuse`.`observations_pid_tid_sorting`",+ );+ expect(buildSyncReplicaQuery(NON_DEFAULT)).toBe(+ "SYSTEM SYNC REPLICA ON CLUSTER `default` `langfuse`.`observations_pid_tid_sorting` STRICT",+ );+ });++ it("qualifies the merge-freeze and cleanup statements", () => {+ expect(buildFreezeReplicatedMergesQuery(NON_DEFAULT)).toBe(+ "ALTER TABLE `langfuse`.`observations_pid_tid_sorting` ON CLUSTER `default` " ++ "MODIFY SETTING max_replicated_merges_in_queue = 0, always_fetch_merged_part = 1",+ );+ expect(buildDisableSharedMergeTreeMergesQuery(NON_DEFAULT)).toBe(+ "ALTER TABLE `langfuse`.`observations_pid_tid_sorting` " ++ "MODIFY SETTING shared_merge_tree_disable_merges_and_mutations_assignment = 1",+ );+ expect(buildDropScratchTableQuery(PID_TID_SORTING_TABLE, NON_DEFAULT)).toBe(+ "DROP TABLE IF EXISTS `langfuse`.`observations_pid_tid_sorting` ON CLUSTER `default` SYNC",+ );+ });++ it("omits ON CLUSTER on single-node deployments without leaving double spaces", () => {+ const singleNode = {+ database: "langfuse",+ clusterEnabled: false,+ clusterName: "default",+ };++ expect(buildStopMergesQuery(singleNode)).toBe(+ "SYSTEM STOP MERGES `langfuse`.`observations_pid_tid_sorting`",+ );+ expect(buildDropScratchTableQuery(PID_TID_SORTING_TABLE, singleNode)).toBe(+ "DROP TABLE IF EXISTS `langfuse`.`observations_pid_tid_sorting` SYNC",+ );+ expect(onClusterClause(singleNode)).toBe("");+ });++ it("quotes cluster and database names that are not bare identifiers", () => {+ expect(+ buildStopMergesQuery({+ database: "langfuse-prod",+ clusterEnabled: true,+ clusterName: "prod-cluster",+ }),+ ).toBe(+ "SYSTEM STOP MERGES ON CLUSTER `prod-cluster` `langfuse-prod`.`observations_pid_tid_sorting`",+ );+ });++ it("escapes backticks in configured identifiers", () => {+ expect(pidTidSortingTable({ ...NON_DEFAULT, database: "lang`fuse" })).toBe(+ "`lang\\`fuse`.`observations_pid_tid_sorting`",+ );+ });++ it("rejects empty identifiers rather than emitting an unqualified statement", () => {+ expect(() =>+ buildStopMergesQuery({ ...NON_DEFAULT, database: "" }),+ ).toThrow("Invalid ClickHouse database");+ expect(() =>+ buildStopMergesQuery({ ...NON_DEFAULT, clusterName: "" }),+ ).toThrow("Invalid ClickHouse cluster");+ });+});diff --git a/worker/src/backgroundMigrations/backfillEventsFullFromDatasetRunItems.ts b/worker/src/backgroundMigrations/backfillEventsFullFromDatasetRunItems.tsindex aa5cb3fcb8..1558c2a09c 100644--- a/worker/src/backgroundMigrations/backfillEventsFullFromDatasetRunItems.ts+++ b/worker/src/backgroundMigrations/backfillEventsFullFromDatasetRunItems.ts@@ -24,6 +24,10 @@ import {type TraceProperties,} from "../features/eventPropagation/handleExperimentBackfill";import { checkPredecessorMigrationFinalized } from "./utils/backfillBase";+import {+ PID_TID_SORTING_TABLE,+ pidTidSortingTable,+} from "./utils/v4BackfillDdl";// Hard-coded UUID identifying the row in background_migrations. Must match// the Prisma migration that registers this row.@@ -290,7 +294,7 @@ export default class BackfillEventsFullFromDatasetRunItems implements IBackgroun'' AS trace_name,'' AS user_id,'' AS session_id- FROM observations_pid_tid_sorting o+ FROM ${pidTidSortingTable()} oWHERE (o.project_id, o.trace_id) IN {tracePairs: Array(Tuple(String, String))}AND o.start_time >= {minTime: DateTime64(3)}AND o.start_time <= {maxTime: DateTime64(3)}@@ -583,7 +587,7 @@ export default class BackfillEventsFullFromDatasetRunItems implements IBackgrounconst requiredTables = ["dataset_run_items_rmt",- "observations_pid_tid_sorting",+ PID_TID_SORTING_TABLE,"traces","events_full","events_core",diff --git a/worker/src/backgroundMigrations/backfillEventsFullFromObservations.ts b/worker/src/backgroundMigrations/backfillEventsFullFromObservations.tsindex 808c68d4b4..d25f5030ab 100644--- a/worker/src/backgroundMigrations/backfillEventsFullFromObservations.ts+++ b/worker/src/backgroundMigrations/backfillEventsFullFromObservations.ts@@ -9,6 +9,11 @@ import {detectTableEngine,runBackfillMigrationCli,} from "./utils/backfillBase";+import {+ PID_TID_SORTING_TABLE,+ buildSyncReplicaQuery,+ pidTidSortingTable,+} from "./utils/v4BackfillDdl";// Hard-coded UUID identifying the row in background_migrations. Must match// the Prisma migration that registers this row.@@ -29,7 +34,7 @@ export default class BackfillEventsFullFromObservations extends ChunkedClickhousprotected readonly migrationId = backgroundMigrationId;protected readonly logPrefix = LOG_PREFIX;protected readonly requiredTables = [- "observations_pid_tid_sorting",+ PID_TID_SORTING_TABLE,"traces","events_full","events_core",@@ -72,7 +77,7 @@ export default class BackfillEventsFullFromObservations extends ChunkedClickhousprivate async assertReplicasConverged(attempts: number,): Promise<{ valid: boolean; invalidReason: string | undefined }> {- const engine = await detectTableEngine("observations_pid_tid_sorting");+ const engine = await detectTableEngine(PID_TID_SORTING_TABLE);if (!engine.startsWith("Replicated")) {logger.info(`${this.logPrefix} Skipping cross-replica consistency check (engine=${engine || "unknown"} is not replicated)`,@@ -106,7 +111,7 @@ export default class BackfillEventsFullFromObservations extends ChunkedClickhous`observations_pid_tid_sorting replicas have not converged (${lastSummary}). ` +`M2 freezes merges and syncs all replicas on success, so this indicates a replica joined or ` +`recovered after M2, or M3 resumed after a topology change. Once replication settles (or after ` +- `running SYSTEM SYNC REPLICA ON CLUSTER ${cluster} observations_pid_tid_sorting STRICT), clear ` ++ `running ${buildSyncReplicaQuery()}), clear ` +`failedAt and re-run.`,};}@@ -142,7 +147,7 @@ export default class BackfillEventsFullFromObservations extends ChunkedClickhouscountIf(is_readonly) AS readonly_replicasFROM clusterAllReplicas('${cluster}', 'system.replicas')WHERE database = currentDatabase()- AND table = 'observations_pid_tid_sorting'+ AND table = '${PID_TID_SORTING_TABLE}'`,clickhouseSettings: {skip_unavailable_shards: 1,@@ -210,7 +215,7 @@ export default class BackfillEventsFullFromObservations extends ChunkedClickhousquery: `SELECT partition_id, nameFROM system.parts- WHERE table = 'observations_pid_tid_sorting'+ WHERE table = '${PID_TID_SORTING_TABLE}'AND database = currentDatabase()AND active = 1AND partition_id NOT LIKE 'patch-%'@@ -240,7 +245,7 @@ export default class BackfillEventsFullFromObservations extends ChunkedClickhousquery: `SELECT nameFROM system.parts- WHERE table = 'observations_pid_tid_sorting'+ WHERE table = '${PID_TID_SORTING_TABLE}'AND database = currentDatabase()AND active = 1`,@@ -338,7 +343,7 @@ export default class BackfillEventsFullFromObservations extends ChunkedClickhouso.updated_at,o.event_ts,o.is_deleted- FROM observations_pid_tid_sorting o+ FROM ${pidTidSortingTable()} oLEFT ANY JOIN (SELECT project_id, id, version, release, tags, public, bookmarked, name, user_id, session_idFROM traces t@@ -388,7 +393,7 @@ export default class BackfillEventsFullFromObservations extends ChunkedClickhousquery: `SELECT count() AS countFROM system.parts- WHERE table = 'observations_pid_tid_sorting'+ WHERE table = '${PID_TID_SORTING_TABLE}'AND database = currentDatabase()AND name = {partId: String}AND active = 1diff --git a/worker/src/backgroundMigrations/dropPidTidSortingTables.ts b/worker/src/backgroundMigrations/dropPidTidSortingTables.tsindex 80d776c88d..127cee6c68 100644--- a/worker/src/backgroundMigrations/dropPidTidSortingTables.ts+++ b/worker/src/backgroundMigrations/dropPidTidSortingTables.ts@@ -1,10 +1,11 @@import { IBackgroundMigration } from "./IBackgroundMigration";import { commandClickhouse, logger } from "@langfuse/shared/src/server";import { parseArgs } from "node:util";+import { checkPredecessorMigrationFinalized } from "./utils/backfillBase";import {- checkPredecessorMigrationFinalized,- onClusterClause,-} from "./utils/backfillBase";+ PID_TID_SORTING_TABLE,+ buildDropScratchTableQuery,+} from "./utils/v4BackfillDdl";// Hard-coded UUID identifying the row in background_migrations. Must match the// Prisma migration that registers this row.@@ -50,7 +51,7 @@ export default class DropPidTidSortingTables implements IBackgroundMigration {async run(_args: Record<string, unknown>): Promise<void> {logger.info(`${LOG_PREFIX} Starting cleanup of V4 backfill scratch tables`);- const tables = ["observations_pid_tid_sorting"];+ const tables = [PID_TID_SORTING_TABLE];for (const table of tables) {if (this.isAborted) {@@ -58,11 +59,7 @@ export default class DropPidTidSortingTables implements IBackgroundMigration {return;}- const onCluster = onClusterClause();- // `IF EXISTS` keeps the migration idempotent — re-running the row (or- // running it on a deployment that never created the scratch tables) is- // a no-op rather than an error.- const sql = `DROP TABLE IF EXISTS ${table} ${onCluster} SYNC`.trim();+ const sql = buildDropScratchTableQuery(table);logger.info(`${LOG_PREFIX} Dropping ${table}`);try {diff --git a/worker/src/backgroundMigrations/rewriteObservationsToPidTidSorting.ts b/worker/src/backgroundMigrations/rewriteObservationsToPidTidSorting.tsindex 93497ae909..6dfcccf427 100644--- a/worker/src/backgroundMigrations/rewriteObservationsToPidTidSorting.ts+++ b/worker/src/backgroundMigrations/rewriteObservationsToPidTidSorting.ts@@ -8,9 +8,17 @@ import {assertSafePartition,detectTableEngine,loadPartitionsFromClickhouse,- onClusterClause,runBackfillMigrationCli,} from "./utils/backfillBase";+import {+ PID_TID_SORTING_TABLE,+ buildDisableSharedMergeTreeMergesQuery,+ buildFreezeReplicatedMergesQuery,+ buildStopMergesQuery,+ buildSyncReplicaQuery,+ onClusterClause,+ pidTidSortingTable,+} from "./utils/v4BackfillDdl";// Hard-coded UUID identifying the row in background_migrations.// Must match the Prisma migration that registers this row.@@ -63,7 +71,7 @@ export default class RewriteObservationsToPidTidSorting extends ChunkedClickhous*/private async ensureScratchTable(): Promise<void> {const ddl = `- CREATE TABLE IF NOT EXISTS observations_pid_tid_sorting ${onClusterClause()} (+ CREATE TABLE IF NOT EXISTS ${pidTidSortingTable()} ${onClusterClause()} (\`id\` String,\`trace_id\` String,\`project_id\` String,@@ -124,7 +132,7 @@ export default class RewriteObservationsToPidTidSorting extends ChunkedClickhous// ==========================================================================private async stopMergesOnScratchTable(): Promise<void> {- const engine = await detectTableEngine("observations_pid_tid_sorting");+ const engine = await detectTableEngine(PID_TID_SORTING_TABLE);const isSharedMergeTree = engine.startsWith("Shared");const isReplicatedMergeTree = engine.startsWith("Replicated");@@ -133,11 +141,10 @@ export default class RewriteObservationsToPidTidSorting extends ChunkedClickhous);if (isSharedMergeTree) {- // ClickHouse Cloud (SharedMergeTree) does not support `SYSTEM STOP MERGES`.- // The documented equivalent is the per-table setting below, which is- // assignment-scoped and persists until explicitly reset.+ // ClickHouse Cloud (SharedMergeTree) does not support `SYSTEM STOP MERGES`;+ // the builder emits the documented per-table equivalent instead.await commandClickhouse({- query: `ALTER TABLE observations_pid_tid_sorting MODIFY SETTING shared_merge_tree_disable_merges_and_mutations_assignment = 1`,+ query: buildDisableSharedMergeTreeMergesQuery(),tags: {surface: "worker",route: "background-migration.stopMerges",@@ -149,7 +156,7 @@ export default class RewriteObservationsToPidTidSorting extends ChunkedClickhous// reset on server restart, so on its own it is not a durable freeze// across the M2 -> M3 gap.await commandClickhouse({- query: `SYSTEM STOP MERGES ${onClusterClause()} observations_pid_tid_sorting`,+ query: buildStopMergesQuery(),tags: {surface: "worker",route: "background-migration.stopMerges",@@ -158,13 +165,10 @@ export default class RewriteObservationsToPidTidSorting extends ChunkedClickhous// For the replicated engine, persist the freeze in table metadata on top// of the in-memory lock so it survives restarts and holds on every- // replica: stop enqueuing new merges (`max_replicated_merges_in_queue=0`)- // and stop any replica from executing a merge locally- // (`always_fetch_merged_part=1`).- // See https://github.com/ClickHouse/ClickHouse/issues/22830.+ // replica (see the builder for the settings and the upstream issue).if (isReplicatedMergeTree) {await commandClickhouse({- query: `ALTER TABLE observations_pid_tid_sorting ${onClusterClause()} MODIFY SETTING max_replicated_merges_in_queue = 0, always_fetch_merged_part = 1`,+ query: buildFreezeReplicatedMergesQuery(),tags: {surface: "worker",route: "background-migration.stopMerges",@@ -191,7 +195,7 @@ export default class RewriteObservationsToPidTidSorting extends ChunkedClickhous* a half-synced cluster.*/private async syncReplicasOnScratchTable(): Promise<void> {- const engine = await detectTableEngine("observations_pid_tid_sorting");+ const engine = await detectTableEngine(PID_TID_SORTING_TABLE);if (!engine.startsWith("Replicated")) {logger.info(`${this.logPrefix} Skipping SYSTEM SYNC REPLICA (engine=${engine || "unknown"} is not replicated)`,@@ -203,7 +207,7 @@ export default class RewriteObservationsToPidTidSorting extends ChunkedClickhous`${this.logPrefix} Syncing all replicas of observations_pid_tid_sorting`,);await commandClickhouse({- query: `SYSTEM SYNC REPLICA ${onClusterClause()} observations_pid_tid_sorting STRICT`,+ query: buildSyncReplicaQuery(),tags: {surface: "worker",route: "background-migration.syncReplica",@@ -237,7 +241,7 @@ export default class RewriteObservationsToPidTidSorting extends ChunkedClickhousassertSafePartition(todo.partition);const query = `- INSERT INTO observations_pid_tid_sorting (+ INSERT INTO ${pidTidSortingTable()} (id, trace_id, project_id, environment, type, parent_observation_id,start_time, end_time, name, metadata, level, status_message, version,input, output,diff --git a/worker/src/backgroundMigrations/utils/backfillBase.ts b/worker/src/backgroundMigrations/utils/backfillBase.tsindex 4230fa9830..8862a7c802 100644--- a/worker/src/backgroundMigrations/utils/backfillBase.ts+++ b/worker/src/backgroundMigrations/utils/backfillBase.ts@@ -12,7 +12,6 @@ import {type QueryStatus,} from "@langfuse/shared/src/server";import { prisma } from "@langfuse/shared/src/db";-import { env } from "../../env";import { IBackgroundMigration } from "../IBackgroundMigration";// ============================================================================@@ -80,18 +79,6 @@ export function assertSafePartition(partition: string): void {// Cluster-aware helpers// ============================================================================-/**- * Returns `ON CLUSTER <name>` when running against a clustered ClickHouse- * deployment, an empty string otherwise. Shared by every chain step that- * issues DDL or SYSTEM commands so they fan out to all nodes consistently.- */-export function onClusterClause(): string {- if (env.CLICKHOUSE_CLUSTER_ENABLED === "true") {- return `ON CLUSTER ${env.CLICKHOUSE_CLUSTER_NAME}`;- }- return "";-}-/*** Returns the engine ClickHouse actually picked for a table. On ClickHouse* Cloud the requested engine is silently rewritten to a `Shared*` variantdiff --git a/worker/src/backgroundMigrations/utils/v4BackfillDdl.ts b/worker/src/backgroundMigrations/utils/v4BackfillDdl.tsnew file mode 100644index 0000000000..c3a9c81907--- /dev/null+++ b/worker/src/backgroundMigrations/utils/v4BackfillDdl.ts@@ -0,0 +1,133 @@+import { env } from "../../env";+import {+ qualifiedClickhouseTableName,+ quoteClickhouseIdentifier,+} from "../../utils/clickhouseIdentifiers";++/**+ * Scratch table created by M2, read by M3/M4, dropped by M5. Single source of+ * truth so the DDL, the `requiredTables` guards and the `system.parts` /+ * `system.replicas` predicates cannot drift apart.+ */+export const PID_TID_SORTING_TABLE = "observations_pid_tid_sorting";++/**+ * The ClickHouse topology the V4 chain emits DDL against. Passed explicitly so+ * the query builders below stay pure and assertable in tests, mirroring+ * `buildApplyDeletedMaskQuery`'s `ClickHouseDdlConfig`.+ */+export interface ClickhouseDdlTarget {+ database: string;+ clusterEnabled: boolean;+ clusterName: string;+}++export function clickhouseDdlTarget(): ClickhouseDdlTarget {+ return {+ database: env.CLICKHOUSE_DB,+ clusterEnabled: env.CLICKHOUSE_CLUSTER_ENABLED === "true",+ clusterName: env.CLICKHOUSE_CLUSTER_NAME,+ };+}++/**+ * Returns `ON CLUSTER <name>` when running against a clustered ClickHouse+ * deployment, an empty string otherwise. Shared by every chain step that+ * issues DDL or SYSTEM commands so they fan out to all nodes consistently.+ */+export function onClusterClause(+ target: ClickhouseDdlTarget = clickhouseDdlTarget(),+): string {+ if (!target.clusterEnabled) return "";+ return `ON CLUSTER ${quoteClickhouseIdentifier(target.clusterName, "cluster")}`;+}++/** The scratch table, qualified with the configured database. */+export function pidTidSortingTable(+ target: ClickhouseDdlTarget = clickhouseDdlTarget(),+): string {+ return qualifiedClickhouseTableName(target.database, PID_TID_SORTING_TABLE);+}++function joinSql(...parts: string[]): string {+ return parts.filter((part) => part.length > 0).join(" ");+}++// ============================================================================+// Merge control (M2)+// ============================================================================++/**+ * ClickHouse Cloud (SharedMergeTree) does not support `SYSTEM STOP MERGES`.+ * This per-table setting is the documented equivalent; it is+ * assignment-scoped and persists until explicitly reset.+ */+export function buildDisableSharedMergeTreeMergesQuery(+ target: ClickhouseDdlTarget = clickhouseDdlTarget(),+): string {+ return joinSql(+ "ALTER TABLE",+ pidTidSortingTable(target),+ "MODIFY SETTING shared_merge_tree_disable_merges_and_mutations_assignment = 1",+ );+}++export function buildStopMergesQuery(+ target: ClickhouseDdlTarget = clickhouseDdlTarget(),+): string {+ return joinSql(+ "SYSTEM STOP MERGES",+ onClusterClause(target),+ pidTidSortingTable(target),+ );+}++/**+ * Persists the freeze in table metadata on top of the in-memory action lock so+ * it survives restarts and holds on every replica: stop enqueuing new merges+ * (`max_replicated_merges_in_queue=0`) and stop any replica from executing a+ * merge locally (`always_fetch_merged_part=1`).+ * See https://github.com/ClickHouse/ClickHouse/issues/22830.+ */+export function buildFreezeReplicatedMergesQuery(+ target: ClickhouseDdlTarget = clickhouseDdlTarget(),+): string {+ return joinSql(+ "ALTER TABLE",+ pidTidSortingTable(target),+ onClusterClause(target),+ "MODIFY SETTING max_replicated_merges_in_queue = 0, always_fetch_merged_part = 1",+ );+}++export function buildSyncReplicaQuery(+ target: ClickhouseDdlTarget = clickhouseDdlTarget(),+): string {+ return joinSql(+ "SYSTEM SYNC REPLICA",+ onClusterClause(target),+ pidTidSortingTable(target),+ "STRICT",+ );+}++// ============================================================================+// Cleanup (M5)+// ============================================================================++/**+ * `IF EXISTS` keeps the migration idempotent — re-running the row (or running+ * it on a deployment that never created the scratch table) is a no-op rather+ * than an error.+ */+export function buildDropScratchTableQuery(+ table: string,+ target: ClickhouseDdlTarget = clickhouseDdlTarget(),+): string {+ return joinSql(+ "DROP TABLE IF EXISTS",+ qualifiedClickhouseTableName(target.database, table),+ onClusterClause(target),+ "SYNC",+ );+}diff --git a/worker/src/features/deleted-mask-cleaner/helpers.ts b/worker/src/features/deleted-mask-cleaner/helpers.tsindex 789b19a39b..e8195cab60 100644--- a/worker/src/features/deleted-mask-cleaner/helpers.ts+++ b/worker/src/features/deleted-mask-cleaner/helpers.ts@@ -1,3 +1,10 @@+import {+ assertClickhouseIdentifier,+ qualifiedClickhouseTableName,+ quoteClickhouseIdentifier,+ quoteClickhouseString,+} from "../../utils/clickhouseIdentifiers";+export const DELETED_MASK_CLEANER_TABLES = ["events_core","events_full",@@ -55,12 +62,6 @@ export const DELETED_MASK_CLEANER_WORK_QUERY = `LIMIT 1 BY table`;-function assertClickHouseName(value: string, label: string): void {- if (value.length === 0 || value.includes("\0")) {- throw new Error(`Invalid ClickHouse ${label}: ${value}`);- }-}-export function isDeletedMaskCleanerTable(table: string): boolean {return DELETED_MASK_CLEANER_TABLE_SET.has(table);}@@ -77,21 +78,12 @@ function assertMonthPartition(partition: string): void {}}-function quoteClickhouseString(value: string): string {- return `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;-}--function quoteClickhouseIdentifier(value: string, label: string): string {- assertClickHouseName(value, label);- return `\`${value.replace(/\\/g, "\\\\").replace(/`/g, "\\`")}\``;-}-function buildMutationSource(useClusterAllReplicas: boolean,clusterName: string,): string {if (useClusterAllReplicas) {- assertClickHouseName(clusterName, "cluster");+ assertClickhouseIdentifier(clusterName, "cluster");}return useClusterAllReplicas@@ -161,9 +153,11 @@ export function buildApplyDeletedMaskQuery(candidate: WorkCandidateRow,config: ClickHouseDdlConfig,): string {- const database = quoteClickhouseIdentifier(config.database, "database");assertTargetTable(candidate.table);- const table = quoteClickhouseIdentifier(candidate.table, "table");+ const qualifiedTable = qualifiedClickhouseTableName(+ config.database,+ candidate.table,+ );assertMonthPartition(candidate.partition_to_clean);const clusterClause = config.clusterEnabled@@ -177,7 +171,7 @@ export function buildApplyDeletedMaskQuery(})(): "";- return `ALTER TABLE ${database}.${table}${clusterClause} APPLY DELETED MASK IN PARTITION '${candidate.partition_to_clean}'`;+ return `ALTER TABLE ${qualifiedTable}${clusterClause} APPLY DELETED MASK IN PARTITION '${candidate.partition_to_clean}'`;}export function buildMutationCountQuery(diff --git a/worker/src/utils/clickhouseIdentifiers.ts b/worker/src/utils/clickhouseIdentifiers.tsnew file mode 100644index 0000000000..9d21b9d9f7--- /dev/null+++ b/worker/src/utils/clickhouseIdentifiers.ts@@ -0,0 +1,41 @@+/**+ * Quoting helpers for ClickHouse identifiers that have to be interpolated into+ * SQL text rather than bound as query parameters — table/database/cluster names+ * in DDL and SYSTEM statements cannot be parameterized.+ */++export function assertClickhouseIdentifier(value: string, label: string): void {+ if (value.length === 0 || value.includes("\0")) {+ throw new Error(`Invalid ClickHouse ${label}: ${value}`);+ }+}++export function quoteClickhouseIdentifier(+ value: string,+ label: string,+): string {+ assertClickhouseIdentifier(value, label);+ const escaped = value.replace(/\\/g, "\\\\").replace(/`/g, "\\`");+ return `\`${escaped}\``;+}++export function quoteClickhouseString(value: string): string {+ return `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;+}++/**+ * Qualifies a table with its database.+ *+ * Required for `SYSTEM ... ON CLUSTER` statements: `AddDefaultDatabaseVisitor`+ * only fills in the initiator's current database for `ASTAlterQuery`,+ * `ASTQueryWithTableAndOutput` and `ASTRenameQuery`. `ASTSystemQuery` is not+ * visited, so an unqualified name reaches each host's DDL worker unqualified+ * and resolves against that server's default database (`default`) instead of+ * `CLICKHOUSE_DB`.+ */+export function qualifiedClickhouseTableName(+ database: string,+ table: string,+): string {+ return `${quoteClickhouseIdentifier(database, "database")}.${quoteClickhouseIdentifier(table, "table")}`;+}