Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(worker): qualify ClickHouse backfill table names (#15847)

Falk Telzerow1099e2655a87
diff --git a/worker/src/__tests__/v4BackfillDdl.unit.test.ts b/worker/src/__tests__/v4BackfillDdl.unit.test.ts
new file mode 100644
index 0000000000..44ad4ff570
--- /dev/null
+++ b/worker/src/__tests__/v4BackfillDdl.unit.test.ts
@@ -0,0 +1,97 @@
+import { describe, expect, it } from "vitest";
+import {
+ PID_TID_SORTING_TABLE,
+ buildDisableSharedMergeTreeMergesQuery,
+ buildDropScratchTableQuery,
+ buildFreezeReplicatedMergesQuery,
+ buildStopMergesQuery,
+ buildSyncReplicaQuery,
+ onClusterClause,
+ pidTidSortingTable,
+} from "../backgroundMigrations/utils/v4BackfillDdl";
+
+/**
+ * Regression coverage for #15845: the V4 historic backfill emitted unqualified
+ * `SYSTEM STOP MERGES` / `SYSTEM SYNC REPLICA`. ClickHouse's
+ * `AddDefaultDatabaseVisitor` only qualifies `ASTAlterQuery`,
+ * `ASTQueryWithTableAndOutput` and `ASTRenameQuery` with the initiator's
+ * database — `ASTSystemQuery` is never visited, so each host's DDL worker
+ * resolved the scratch table against its own `default` database and the chain
+ * failed with `Table default.observations_pid_tid_sorting does not exist`.
+ *
+ * Every statement below must therefore name the configured database, and the
+ * `ON CLUSTER` statements are the ones that actually regress.
+ */
+const NON_DEFAULT = {
+ database: "langfuse",
+ clusterEnabled: true,
+ clusterName: "default",
+};
+
+describe("V4 backfill DDL", () => {
+ it("qualifies the ON CLUSTER SYSTEM commands with the configured database", () => {
+ expect(buildStopMergesQuery(NON_DEFAULT)).toBe(
+ "SYSTEM STOP MERGES ON CLUSTER `default` `langfuse`.`observations_pid_tid_sorting`",
+ );
+ expect(buildSyncReplicaQuery(NON_DEFAULT)).toBe(
+ "SYSTEM SYNC REPLICA ON CLUSTER `default` `langfuse`.`observations_pid_tid_sorting` STRICT",
+ );
+ });
+
+ it("qualifies the merge-freeze and cleanup statements", () => {
+ expect(buildFreezeReplicatedMergesQuery(NON_DEFAULT)).toBe(
+ "ALTER TABLE `langfuse`.`observations_pid_tid_sorting` ON CLUSTER `default` " +
+ "MODIFY SETTING max_replicated_merges_in_queue = 0, always_fetch_merged_part = 1",
+ );
+ expect(buildDisableSharedMergeTreeMergesQuery(NON_DEFAULT)).toBe(
+ "ALTER TABLE `langfuse`.`observations_pid_tid_sorting` " +
+ "MODIFY SETTING shared_merge_tree_disable_merges_and_mutations_assignment = 1",
+ );
+ expect(buildDropScratchTableQuery(PID_TID_SORTING_TABLE, NON_DEFAULT)).toBe(
+ "DROP TABLE IF EXISTS `langfuse`.`observations_pid_tid_sorting` ON CLUSTER `default` SYNC",
+ );
+ });
+
+ it("omits ON CLUSTER on single-node deployments without leaving double spaces", () => {
+ const singleNode = {
+ database: "langfuse",
+ clusterEnabled: false,
+ clusterName: "default",
+ };
+
+ expect(buildStopMergesQuery(singleNode)).toBe(
+ "SYSTEM STOP MERGES `langfuse`.`observations_pid_tid_sorting`",
+ );
+ expect(buildDropScratchTableQuery(PID_TID_SORTING_TABLE, singleNode)).toBe(
+ "DROP TABLE IF EXISTS `langfuse`.`observations_pid_tid_sorting` SYNC",
+ );
+ expect(onClusterClause(singleNode)).toBe("");
+ });
+
+ it("quotes cluster and database names that are not bare identifiers", () => {
+ expect(
+ buildStopMergesQuery({
+ database: "langfuse-prod",
+ clusterEnabled: true,
+ clusterName: "prod-cluster",
+ }),
+ ).toBe(
+ "SYSTEM STOP MERGES ON CLUSTER `prod-cluster` `langfuse-prod`.`observations_pid_tid_sorting`",
+ );
+ });
+
+ it("escapes backticks in configured identifiers", () => {
+ expect(pidTidSortingTable({ ...NON_DEFAULT, database: "lang`fuse" })).toBe(
+ "`lang\\`fuse`.`observations_pid_tid_sorting`",
+ );
+ });
+
+ it("rejects empty identifiers rather than emitting an unqualified statement", () => {
+ expect(() =>
+ buildStopMergesQuery({ ...NON_DEFAULT, database: "" }),
+ ).toThrow("Invalid ClickHouse database");
+ expect(() =>
+ buildStopMergesQuery({ ...NON_DEFAULT, clusterName: "" }),
+ ).toThrow("Invalid ClickHouse cluster");
+ });
+});
diff --git a/worker/src/backgroundMigrations/backfillEventsFullFromDatasetRunItems.ts b/worker/src/backgroundMigrations/backfillEventsFullFromDatasetRunItems.ts
index aa5cb3fcb8..1558c2a09c 100644
--- a/worker/src/backgroundMigrations/backfillEventsFullFromDatasetRunItems.ts
+++ b/worker/src/backgroundMigrations/backfillEventsFullFromDatasetRunItems.ts
@@ -24,6 +24,10 @@ import {
type TraceProperties,
} from "../features/eventPropagation/handleExperimentBackfill";
import { checkPredecessorMigrationFinalized } from "./utils/backfillBase";
+import {
+ PID_TID_SORTING_TABLE,
+ pidTidSortingTable,
+} from "./utils/v4BackfillDdl";
// Hard-coded UUID identifying the row in background_migrations. Must match
// the Prisma migration that registers this row.
@@ -290,7 +294,7 @@ export default class BackfillEventsFullFromDatasetRunItems implements IBackgroun
'' AS trace_name,
'' AS user_id,
'' AS session_id
- FROM observations_pid_tid_sorting o
+ FROM ${pidTidSortingTable()} o
WHERE (o.project_id, o.trace_id) IN {tracePairs: Array(Tuple(String, String))}
AND o.start_time >= {minTime: DateTime64(3)}
AND o.start_time <= {maxTime: DateTime64(3)}
@@ -583,7 +587,7 @@ export default class BackfillEventsFullFromDatasetRunItems implements IBackgroun
const requiredTables = [
"dataset_run_items_rmt",
- "observations_pid_tid_sorting",
+ PID_TID_SORTING_TABLE,
"traces",
"events_full",
"events_core",
diff --git a/worker/src/backgroundMigrations/backfillEventsFullFromObservations.ts b/worker/src/backgroundMigrations/backfillEventsFullFromObservations.ts
index 808c68d4b4..d25f5030ab 100644
--- a/worker/src/backgroundMigrations/backfillEventsFullFromObservations.ts
+++ b/worker/src/backgroundMigrations/backfillEventsFullFromObservations.ts
@@ -9,6 +9,11 @@ import {
detectTableEngine,
runBackfillMigrationCli,
} from "./utils/backfillBase";
+import {
+ PID_TID_SORTING_TABLE,
+ buildSyncReplicaQuery,
+ pidTidSortingTable,
+} from "./utils/v4BackfillDdl";
// Hard-coded UUID identifying the row in background_migrations. Must match
// the Prisma migration that registers this row.
@@ -29,7 +34,7 @@ export default class BackfillEventsFullFromObservations extends ChunkedClickhous
protected readonly migrationId = backgroundMigrationId;
protected readonly logPrefix = LOG_PREFIX;
protected readonly requiredTables = [
- "observations_pid_tid_sorting",
+ PID_TID_SORTING_TABLE,
"traces",
"events_full",
"events_core",
@@ -72,7 +77,7 @@ export default class BackfillEventsFullFromObservations extends ChunkedClickhous
private async assertReplicasConverged(
attempts: number,
): Promise<{ valid: boolean; invalidReason: string | undefined }> {
- const engine = await detectTableEngine("observations_pid_tid_sorting");
+ const engine = await detectTableEngine(PID_TID_SORTING_TABLE);
if (!engine.startsWith("Replicated")) {
logger.info(
`${this.logPrefix} Skipping cross-replica consistency check (engine=${engine || "unknown"} is not replicated)`,
@@ -106,7 +111,7 @@ export default class BackfillEventsFullFromObservations extends ChunkedClickhous
`observations_pid_tid_sorting replicas have not converged (${lastSummary}). ` +
`M2 freezes merges and syncs all replicas on success, so this indicates a replica joined or ` +
`recovered after M2, or M3 resumed after a topology change. Once replication settles (or after ` +
- `running SYSTEM SYNC REPLICA ON CLUSTER ${cluster} observations_pid_tid_sorting STRICT), clear ` +
+ `running ${buildSyncReplicaQuery()}), clear ` +
`failedAt and re-run.`,
};
}
@@ -142,7 +147,7 @@ export default class BackfillEventsFullFromObservations extends ChunkedClickhous
countIf(is_readonly) AS readonly_replicas
FROM clusterAllReplicas('${cluster}', 'system.replicas')
WHERE database = currentDatabase()
- AND table = 'observations_pid_tid_sorting'
+ AND table = '${PID_TID_SORTING_TABLE}'
`,
clickhouseSettings: {
skip_unavailable_shards: 1,
@@ -210,7 +215,7 @@ export default class BackfillEventsFullFromObservations extends ChunkedClickhous
query: `
SELECT partition_id, name
FROM system.parts
- WHERE table = 'observations_pid_tid_sorting'
+ WHERE table = '${PID_TID_SORTING_TABLE}'
AND database = currentDatabase()
AND active = 1
AND partition_id NOT LIKE 'patch-%'
@@ -240,7 +245,7 @@ export default class BackfillEventsFullFromObservations extends ChunkedClickhous
query: `
SELECT name
FROM system.parts
- WHERE table = 'observations_pid_tid_sorting'
+ WHERE table = '${PID_TID_SORTING_TABLE}'
AND database = currentDatabase()
AND active = 1
`,
@@ -338,7 +343,7 @@ export default class BackfillEventsFullFromObservations extends ChunkedClickhous
o.updated_at,
o.event_ts,
o.is_deleted
- FROM observations_pid_tid_sorting o
+ FROM ${pidTidSortingTable()} o
LEFT ANY JOIN (
SELECT project_id, id, version, release, tags, public, bookmarked, name, user_id, session_id
FROM traces t
@@ -388,7 +393,7 @@ export default class BackfillEventsFullFromObservations extends ChunkedClickhous
query: `
SELECT count() AS count
FROM system.parts
- WHERE table = 'observations_pid_tid_sorting'
+ WHERE table = '${PID_TID_SORTING_TABLE}'
AND database = currentDatabase()
AND name = {partId: String}
AND active = 1
diff --git a/worker/src/backgroundMigrations/dropPidTidSortingTables.ts b/worker/src/backgroundMigrations/dropPidTidSortingTables.ts
index 80d776c88d..127cee6c68 100644
--- a/worker/src/backgroundMigrations/dropPidTidSortingTables.ts
+++ b/worker/src/backgroundMigrations/dropPidTidSortingTables.ts
@@ -1,10 +1,11 @@
import { IBackgroundMigration } from "./IBackgroundMigration";
import { commandClickhouse, logger } from "@langfuse/shared/src/server";
import { parseArgs } from "node:util";
+import { checkPredecessorMigrationFinalized } from "./utils/backfillBase";
import {
- checkPredecessorMigrationFinalized,
- onClusterClause,
-} from "./utils/backfillBase";
+ PID_TID_SORTING_TABLE,
+ buildDropScratchTableQuery,
+} from "./utils/v4BackfillDdl";
// Hard-coded UUID identifying the row in background_migrations. Must match the
// Prisma migration that registers this row.
@@ -50,7 +51,7 @@ export default class DropPidTidSortingTables implements IBackgroundMigration {
async run(_args: Record<string, unknown>): Promise<void> {
logger.info(`${LOG_PREFIX} Starting cleanup of V4 backfill scratch tables`);
- const tables = ["observations_pid_tid_sorting"];
+ const tables = [PID_TID_SORTING_TABLE];
for (const table of tables) {
if (this.isAborted) {
@@ -58,11 +59,7 @@ export default class DropPidTidSortingTables implements IBackgroundMigration {
return;
}
- const onCluster = onClusterClause();
- // `IF EXISTS` keeps the migration idempotent — re-running the row (or
- // running it on a deployment that never created the scratch tables) is
- // a no-op rather than an error.
- const sql = `DROP TABLE IF EXISTS ${table} ${onCluster} SYNC`.trim();
+ const sql = buildDropScratchTableQuery(table);
logger.info(`${LOG_PREFIX} Dropping ${table}`);
try {
diff --git a/worker/src/backgroundMigrations/rewriteObservationsToPidTidSorting.ts b/worker/src/backgroundMigrations/rewriteObservationsToPidTidSorting.ts
index 93497ae909..6dfcccf427 100644
--- a/worker/src/backgroundMigrations/rewriteObservationsToPidTidSorting.ts
+++ b/worker/src/backgroundMigrations/rewriteObservationsToPidTidSorting.ts
@@ -8,9 +8,17 @@ import {
assertSafePartition,
detectTableEngine,
loadPartitionsFromClickhouse,
- onClusterClause,
runBackfillMigrationCli,
} from "./utils/backfillBase";
+import {
+ PID_TID_SORTING_TABLE,
+ buildDisableSharedMergeTreeMergesQuery,
+ buildFreezeReplicatedMergesQuery,
+ buildStopMergesQuery,
+ buildSyncReplicaQuery,
+ onClusterClause,
+ pidTidSortingTable,
+} from "./utils/v4BackfillDdl";
// Hard-coded UUID identifying the row in background_migrations.
// Must match the Prisma migration that registers this row.
@@ -63,7 +71,7 @@ export default class RewriteObservationsToPidTidSorting extends ChunkedClickhous
*/
private async ensureScratchTable(): Promise<void> {
const ddl = `
- CREATE TABLE IF NOT EXISTS observations_pid_tid_sorting ${onClusterClause()} (
+ CREATE TABLE IF NOT EXISTS ${pidTidSortingTable()} ${onClusterClause()} (
\`id\` String,
\`trace_id\` String,
\`project_id\` String,
@@ -124,7 +132,7 @@ export default class RewriteObservationsToPidTidSorting extends ChunkedClickhous
// ==========================================================================
private async stopMergesOnScratchTable(): Promise<void> {
- const engine = await detectTableEngine("observations_pid_tid_sorting");
+ const engine = await detectTableEngine(PID_TID_SORTING_TABLE);
const isSharedMergeTree = engine.startsWith("Shared");
const isReplicatedMergeTree = engine.startsWith("Replicated");
@@ -133,11 +141,10 @@ export default class RewriteObservationsToPidTidSorting extends ChunkedClickhous
);
if (isSharedMergeTree) {
- // ClickHouse Cloud (SharedMergeTree) does not support `SYSTEM STOP MERGES`.
- // The documented equivalent is the per-table setting below, which is
- // assignment-scoped and persists until explicitly reset.
+ // ClickHouse Cloud (SharedMergeTree) does not support `SYSTEM STOP MERGES`;
+ // the builder emits the documented per-table equivalent instead.
await commandClickhouse({
- query: `ALTER TABLE observations_pid_tid_sorting MODIFY SETTING shared_merge_tree_disable_merges_and_mutations_assignment = 1`,
+ query: buildDisableSharedMergeTreeMergesQuery(),
tags: {
surface: "worker",
route: "background-migration.stopMerges",
@@ -149,7 +156,7 @@ export default class RewriteObservationsToPidTidSorting extends ChunkedClickhous
// reset on server restart, so on its own it is not a durable freeze
// across the M2 -> M3 gap.
await commandClickhouse({
- query: `SYSTEM STOP MERGES ${onClusterClause()} observations_pid_tid_sorting`,
+ query: buildStopMergesQuery(),
tags: {
surface: "worker",
route: "background-migration.stopMerges",
@@ -158,13 +165,10 @@ export default class RewriteObservationsToPidTidSorting extends ChunkedClickhous
// For the replicated engine, persist the freeze in table metadata on top
// of the in-memory lock so it survives restarts and holds on every
- // replica: stop enqueuing new merges (`max_replicated_merges_in_queue=0`)
- // and stop any replica from executing a merge locally
- // (`always_fetch_merged_part=1`).
- // See https://github.com/ClickHouse/ClickHouse/issues/22830.
+ // replica (see the builder for the settings and the upstream issue).
if (isReplicatedMergeTree) {
await commandClickhouse({
- query: `ALTER TABLE observations_pid_tid_sorting ${onClusterClause()} MODIFY SETTING max_replicated_merges_in_queue = 0, always_fetch_merged_part = 1`,
+ query: buildFreezeReplicatedMergesQuery(),
tags: {
surface: "worker",
route: "background-migration.stopMerges",
@@ -191,7 +195,7 @@ export default class RewriteObservationsToPidTidSorting extends ChunkedClickhous
* a half-synced cluster.
*/
private async syncReplicasOnScratchTable(): Promise<void> {
- const engine = await detectTableEngine("observations_pid_tid_sorting");
+ const engine = await detectTableEngine(PID_TID_SORTING_TABLE);
if (!engine.startsWith("Replicated")) {
logger.info(
`${this.logPrefix} Skipping SYSTEM SYNC REPLICA (engine=${engine || "unknown"} is not replicated)`,
@@ -203,7 +207,7 @@ export default class RewriteObservationsToPidTidSorting extends ChunkedClickhous
`${this.logPrefix} Syncing all replicas of observations_pid_tid_sorting`,
);
await commandClickhouse({
- query: `SYSTEM SYNC REPLICA ${onClusterClause()} observations_pid_tid_sorting STRICT`,
+ query: buildSyncReplicaQuery(),
tags: {
surface: "worker",
route: "background-migration.syncReplica",
@@ -237,7 +241,7 @@ export default class RewriteObservationsToPidTidSorting extends ChunkedClickhous
assertSafePartition(todo.partition);
const query = `
- INSERT INTO observations_pid_tid_sorting (
+ INSERT INTO ${pidTidSortingTable()} (
id, trace_id, project_id, environment, type, parent_observation_id,
start_time, end_time, name, metadata, level, status_message, version,
input, output,
diff --git a/worker/src/backgroundMigrations/utils/backfillBase.ts b/worker/src/backgroundMigrations/utils/backfillBase.ts
index 4230fa9830..8862a7c802 100644
--- a/worker/src/backgroundMigrations/utils/backfillBase.ts
+++ b/worker/src/backgroundMigrations/utils/backfillBase.ts
@@ -12,7 +12,6 @@ import {
type QueryStatus,
} from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
-import { env } from "../../env";
import { IBackgroundMigration } from "../IBackgroundMigration";
// ============================================================================
@@ -80,18 +79,6 @@ export function assertSafePartition(partition: string): void {
// Cluster-aware helpers
// ============================================================================
-/**
- * Returns `ON CLUSTER <name>` when running against a clustered ClickHouse
- * deployment, an empty string otherwise. Shared by every chain step that
- * issues DDL or SYSTEM commands so they fan out to all nodes consistently.
- */
-export function onClusterClause(): string {
- if (env.CLICKHOUSE_CLUSTER_ENABLED === "true") {
- return `ON CLUSTER ${env.CLICKHOUSE_CLUSTER_NAME}`;
- }
- return "";
-}
-
/**
* Returns the engine ClickHouse actually picked for a table. On ClickHouse
* Cloud the requested engine is silently rewritten to a `Shared*` variant
diff --git a/worker/src/backgroundMigrations/utils/v4BackfillDdl.ts b/worker/src/backgroundMigrations/utils/v4BackfillDdl.ts
new file mode 100644
index 0000000000..c3a9c81907
--- /dev/null
+++ b/worker/src/backgroundMigrations/utils/v4BackfillDdl.ts
@@ -0,0 +1,133 @@
+import { env } from "../../env";
+import {
+ qualifiedClickhouseTableName,
+ quoteClickhouseIdentifier,
+} from "../../utils/clickhouseIdentifiers";
+
+/**
+ * Scratch table created by M2, read by M3/M4, dropped by M5. Single source of
+ * truth so the DDL, the `requiredTables` guards and the `system.parts` /
+ * `system.replicas` predicates cannot drift apart.
+ */
+export const PID_TID_SORTING_TABLE = "observations_pid_tid_sorting";
+
+/**
+ * The ClickHouse topology the V4 chain emits DDL against. Passed explicitly so
+ * the query builders below stay pure and assertable in tests, mirroring
+ * `buildApplyDeletedMaskQuery`'s `ClickHouseDdlConfig`.
+ */
+export interface ClickhouseDdlTarget {
+ database: string;
+ clusterEnabled: boolean;
+ clusterName: string;
+}
+
+export function clickhouseDdlTarget(): ClickhouseDdlTarget {
+ return {
+ database: env.CLICKHOUSE_DB,
+ clusterEnabled: env.CLICKHOUSE_CLUSTER_ENABLED === "true",
+ clusterName: env.CLICKHOUSE_CLUSTER_NAME,
+ };
+}
+
+/**
+ * Returns `ON CLUSTER <name>` when running against a clustered ClickHouse
+ * deployment, an empty string otherwise. Shared by every chain step that
+ * issues DDL or SYSTEM commands so they fan out to all nodes consistently.
+ */
+export function onClusterClause(
+ target: ClickhouseDdlTarget = clickhouseDdlTarget(),
+): string {
+ if (!target.clusterEnabled) return "";
+ return `ON CLUSTER ${quoteClickhouseIdentifier(target.clusterName, "cluster")}`;
+}
+
+/** The scratch table, qualified with the configured database. */
+export function pidTidSortingTable(
+ target: ClickhouseDdlTarget = clickhouseDdlTarget(),
+): string {
+ return qualifiedClickhouseTableName(target.database, PID_TID_SORTING_TABLE);
+}
+
+function joinSql(...parts: string[]): string {
+ return parts.filter((part) => part.length > 0).join(" ");
+}
+
+// ============================================================================
+// Merge control (M2)
+// ============================================================================
+
+/**
+ * ClickHouse Cloud (SharedMergeTree) does not support `SYSTEM STOP MERGES`.
+ * This per-table setting is the documented equivalent; it is
+ * assignment-scoped and persists until explicitly reset.
+ */
+export function buildDisableSharedMergeTreeMergesQuery(
+ target: ClickhouseDdlTarget = clickhouseDdlTarget(),
+): string {
+ return joinSql(
+ "ALTER TABLE",
+ pidTidSortingTable(target),
+ "MODIFY SETTING shared_merge_tree_disable_merges_and_mutations_assignment = 1",
+ );
+}
+
+export function buildStopMergesQuery(
+ target: ClickhouseDdlTarget = clickhouseDdlTarget(),
+): string {
+ return joinSql(
+ "SYSTEM STOP MERGES",
+ onClusterClause(target),
+ pidTidSortingTable(target),
+ );
+}
+
+/**
+ * Persists the freeze in table metadata on top of the in-memory action lock so
+ * it survives restarts and holds on every replica: stop enqueuing new merges
+ * (`max_replicated_merges_in_queue=0`) and stop any replica from executing a
+ * merge locally (`always_fetch_merged_part=1`).
+ * See https://github.com/ClickHouse/ClickHouse/issues/22830.
+ */
+export function buildFreezeReplicatedMergesQuery(
+ target: ClickhouseDdlTarget = clickhouseDdlTarget(),
+): string {
+ return joinSql(
+ "ALTER TABLE",
+ pidTidSortingTable(target),
+ onClusterClause(target),
+ "MODIFY SETTING max_replicated_merges_in_queue = 0, always_fetch_merged_part = 1",
+ );
+}
+
+export function buildSyncReplicaQuery(
+ target: ClickhouseDdlTarget = clickhouseDdlTarget(),
+): string {
+ return joinSql(
+ "SYSTEM SYNC REPLICA",
+ onClusterClause(target),
+ pidTidSortingTable(target),
+ "STRICT",
+ );
+}
+
+// ============================================================================
+// Cleanup (M5)
+// ============================================================================
+
+/**
+ * `IF EXISTS` keeps the migration idempotent — re-running the row (or running
+ * it on a deployment that never created the scratch table) is a no-op rather
+ * than an error.
+ */
+export function buildDropScratchTableQuery(
+ table: string,
+ target: ClickhouseDdlTarget = clickhouseDdlTarget(),
+): string {
+ return joinSql(
+ "DROP TABLE IF EXISTS",
+ qualifiedClickhouseTableName(target.database, table),
+ onClusterClause(target),
+ "SYNC",
+ );
+}
diff --git a/worker/src/features/deleted-mask-cleaner/helpers.ts b/worker/src/features/deleted-mask-cleaner/helpers.ts
index 789b19a39b..e8195cab60 100644
--- a/worker/src/features/deleted-mask-cleaner/helpers.ts
+++ b/worker/src/features/deleted-mask-cleaner/helpers.ts
@@ -1,3 +1,10 @@
+import {
+ assertClickhouseIdentifier,
+ qualifiedClickhouseTableName,
+ quoteClickhouseIdentifier,
+ quoteClickhouseString,
+} from "../../utils/clickhouseIdentifiers";
+
export const DELETED_MASK_CLEANER_TABLES = [
"events_core",
"events_full",
@@ -55,12 +62,6 @@ export const DELETED_MASK_CLEANER_WORK_QUERY = `
LIMIT 1 BY table
`;
-function assertClickHouseName(value: string, label: string): void {
- if (value.length === 0 || value.includes("\0")) {
- throw new Error(`Invalid ClickHouse ${label}: ${value}`);
- }
-}
-
export function isDeletedMaskCleanerTable(table: string): boolean {
return DELETED_MASK_CLEANER_TABLE_SET.has(table);
}
@@ -77,21 +78,12 @@ function assertMonthPartition(partition: string): void {
}
}
-function quoteClickhouseString(value: string): string {
- return `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
-}
-
-function quoteClickhouseIdentifier(value: string, label: string): string {
- assertClickHouseName(value, label);
- return `\`${value.replace(/\\/g, "\\\\").replace(/`/g, "\\`")}\``;
-}
-
function buildMutationSource(
useClusterAllReplicas: boolean,
clusterName: string,
): string {
if (useClusterAllReplicas) {
- assertClickHouseName(clusterName, "cluster");
+ assertClickhouseIdentifier(clusterName, "cluster");
}
return useClusterAllReplicas
@@ -161,9 +153,11 @@ export function buildApplyDeletedMaskQuery(
candidate: WorkCandidateRow,
config: ClickHouseDdlConfig,
): string {
- const database = quoteClickhouseIdentifier(config.database, "database");
assertTargetTable(candidate.table);
- const table = quoteClickhouseIdentifier(candidate.table, "table");
+ const qualifiedTable = qualifiedClickhouseTableName(
+ config.database,
+ candidate.table,
+ );
assertMonthPartition(candidate.partition_to_clean);
const clusterClause = config.clusterEnabled
@@ -177,7 +171,7 @@ export function buildApplyDeletedMaskQuery(
})()
: "";
- return `ALTER TABLE ${database}.${table}${clusterClause} APPLY DELETED MASK IN PARTITION '${candidate.partition_to_clean}'`;
+ return `ALTER TABLE ${qualifiedTable}${clusterClause} APPLY DELETED MASK IN PARTITION '${candidate.partition_to_clean}'`;
}
export function buildMutationCountQuery(
diff --git a/worker/src/utils/clickhouseIdentifiers.ts b/worker/src/utils/clickhouseIdentifiers.ts
new file mode 100644
index 0000000000..9d21b9d9f7
--- /dev/null
+++ b/worker/src/utils/clickhouseIdentifiers.ts
@@ -0,0 +1,41 @@
+/**
+ * Quoting helpers for ClickHouse identifiers that have to be interpolated into
+ * SQL text rather than bound as query parameters — table/database/cluster names
+ * in DDL and SYSTEM statements cannot be parameterized.
+ */
+
+export function assertClickhouseIdentifier(value: string, label: string): void {
+ if (value.length === 0 || value.includes("\0")) {
+ throw new Error(`Invalid ClickHouse ${label}: ${value}`);
+ }
+}
+
+export function quoteClickhouseIdentifier(
+ value: string,
+ label: string,
+): string {
+ assertClickhouseIdentifier(value, label);
+ const escaped = value.replace(/\\/g, "\\\\").replace(/`/g, "\\`");
+ return `\`${escaped}\``;
+}
+
+export function quoteClickhouseString(value: string): string {
+ return `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
+}
+
+/**
+ * Qualifies a table with its database.
+ *
+ * Required for `SYSTEM ... ON CLUSTER` statements: `AddDefaultDatabaseVisitor`
+ * only fills in the initiator's current database for `ASTAlterQuery`,
+ * `ASTQueryWithTableAndOutput` and `ASTRenameQuery`. `ASTSystemQuery` is not
+ * visited, so an unqualified name reaches each host's DDL worker unqualified
+ * and resolves against that server's default database (`default`) instead of
+ * `CLICKHOUSE_DB`.
+ */
+export function qualifiedClickhouseTableName(
+ database: string,
+ table: string,
+): string {
+ return `${quoteClickhouseIdentifier(database, "database")}.${quoteClickhouseIdentifier(table, "table")}`;
+}