🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { randomUUID } from "crypto";
import { parseArgs } from "node:util";
import {
buildClickHouseLogComment,
clickhouseClient,
commandClickhouse,
getQueryError,
logger,
pollQueryStatus,
queryClickhouse,
sleep,
type QueryStatus,
} from "@langfuse/shared/src/server";
import { prisma } from "@langfuse/shared/src/db";
import { IBackgroundMigration } from "../IBackgroundMigration";
// ============================================================================
// Shared types
// ============================================================================
export interface BaseChunkTodo {
id: string;
partition: string;
status: "pending" | "in_progress" | "completed" | "failed";
queryId?: string;
startedAt?: string;
completedAt?: string;
error?: string;
retryCount?: number;
}
interface ChunkedBackfillArgs {
concurrency?: number;
pollIntervalMs?: number;
maxRetries?: number;
retryFailed?: boolean;
envGate?: string;
/**
* Optional list of yyyymm partitions to restrict scope. Only honored by
* migrations whose `enumerateChunks` supports partition restriction.
*/
partitions?: string[];
}
export interface ResolvedChunkedBackfillConfig {
concurrency: number;
pollIntervalMs: number;
maxRetries: number;
partitions?: string[];
}
export interface ChunkedBackfillState<T extends BaseChunkTodo> {
phase: "init" | "loading_chunks" | "backfill" | "completed";
chunksLoaded: boolean;
todos: T[];
activeQueries: string[];
config: Partial<ResolvedChunkedBackfillConfig>;
}
// ============================================================================
// Helpers
// ============================================================================
function generateQueryId(chunkId: string): string {
return `backfill-${chunkId}-${randomUUID().slice(0, 8)}`;
}
/**
* `partition_id` is either a 6-digit yyyymm string or the literal "REST".
*/
const PARTITION_RE = /^[0-9]{6}$|^REST$/;
export function assertSafePartition(partition: string): void {
if (!PARTITION_RE.test(partition)) {
throw new Error(`Invalid partition_id: ${partition}`);
}
}
// ============================================================================
// Cluster-aware helpers
// ============================================================================
/**
* Returns the engine ClickHouse actually picked for a table. On ClickHouse
* Cloud the requested engine is silently rewritten to a `Shared*` variant
* (e.g. `SharedReplacingMergeTree`), and single-node deployments use the plain
* (non-`Replicated*`) engine — so the prefix is the reliable way to branch on
* deployment topology. Returns "" if the table does not exist.
*/
export async function detectTableEngine(table: string): Promise<string> {
const rows = await queryClickhouse<{ engine: string }>({
query: `
SELECT engine
FROM system.tables
WHERE database = currentDatabase()
AND name = {table: String}
`,
params: { table },
tags: {
surface: "worker",
route: "background-migration.detectTableEngine",
},
});
return rows[0]?.engine ?? "";
}
// ============================================================================
// Migration chain dependency guard
// ============================================================================
/**
* Returns a validation failure unless the prerequisite background-migration row
* has finished cleanly (`finishedAt` set, `failedAt` null).
*
* The `BackgroundMigrationManager` has no dependency model — it runs every eligible
* row in name order and marks each finished/failed independently, so a failed
* upstream step does NOT by itself stop a downstream step from running on
* partial data. Each downstream step calls this from its `validate()`, which
* routes through the manager's validation-failure path and records `failedAt`.
* Because the next step in turn guards on its own predecessor, a single failure
* transitively halts the rest of the chain.
*/
export async function checkPredecessorMigrationFinalized(
predecessorId: string,
predecessorName: string,
): Promise<{ valid: boolean; invalidReason: string | undefined }> {
const predecessor = await prisma.backgroundMigration.findUnique({
where: { id: predecessorId },
select: { finishedAt: true, failedAt: true },
});
if (!predecessor) {
return {
valid: false,
invalidReason: `Prerequisite migration ${predecessorName} (${predecessorId}) is not registered`,
};
}
if (predecessor.failedAt) {
return {
valid: false,
invalidReason: `Prerequisite migration ${predecessorName} failed at ${predecessor.failedAt.toISOString()}; resolve it and clear failedAt before this step can run`,
};
}
if (!predecessor.finishedAt) {
return {
valid: false,
invalidReason: `Prerequisite migration ${predecessorName} has not finished yet`,
};
}
return { valid: true, invalidReason: undefined };
}
// ============================================================================
// Partition discovery
// ============================================================================
/**
* Lists active yyyymm partitions of a ClickHouse table from `system.parts`,
* newest-first. Skips the meta partitions that aren't yyyymm data ranges:
*
* - `all` — appears on tables defined without a PARTITION BY clause.
* - `patch-%` — used by patch tables for ad hoc data corrections.
*
* Optionally restricted to a caller-provided list of yyyymm partitions.
*/
export async function loadPartitionsFromClickhouse(
table: string,
restrictTo?: string[],
logPrefix = "[Backfill]",
): Promise<BaseChunkTodo[]> {
logger.info(`${logPrefix} Discovering ${table} partitions from system.parts`);
const rows = await queryClickhouse<{ partition_id: string }>({
query: `
SELECT DISTINCT partition_id
FROM system.parts
WHERE table = {table: String}
AND database = currentDatabase()
AND active = 1
AND partition_id NOT LIKE 'patch-%'
AND partition_id != 'all'
ORDER BY partition_id DESC
`,
params: { table },
tags: {
surface: "worker",
route: "background-migration.loadPartitionsFromClickhouse",
},
});
const partitions = rows.map((r) => r.partition_id);
const filterSet =
restrictTo && restrictTo.length > 0 ? new Set(restrictTo) : null;
const selected = filterSet
? partitions.filter((p) => filterSet.has(p))
: partitions;
logger.info(
`${logPrefix} Loaded ${selected.length} partitions: ${selected.join(", ")}`,
);
return selected.map((partition) => ({
id: `p-${partition}`,
partition,
status: "pending" as const,
}));
}
// ============================================================================
// Fire query (long-running, abort-and-poll pattern)
// ============================================================================
interface FireQueryRetrySettings {
retry0?: Record<string, string | number>;
retry1?: Record<string, string | number>;
retry2?: Record<string, string | number>;
}
const DEFAULT_RETRY_SETTINGS: FireQueryRetrySettings = {
retry0: {},
retry1: { max_block_size: "4096" },
retry2: { max_threads: 1, max_insert_threads: "1", max_block_size: "2048" },
};
interface FireQueryOptions {
query: string;
queryId: string;
params?: Record<string, unknown>;
/**
* Which attempt this is for the chunk (0-based). Only selects the
* progressively more conservative ClickHouse settings from `retrySettings`;
* retrying itself is the caller's responsibility.
*/
attemptNumber?: number;
retrySettings?: FireQueryRetrySettings;
initialWaitMs?: number;
retryWaitMs?: number;
logPrefix?: string;
}
/**
* Fires a long-running ClickHouse query, confirms it's tracked in
* system.processes, then aborts the HTTP connection so the query continues
* server-side and we poll for completion via pollQueryStatus.
*/
async function fireQuery({
query,
queryId,
params,
attemptNumber = 0,
retrySettings = DEFAULT_RETRY_SETTINGS,
initialWaitMs = 10_000,
retryWaitMs = 30_000,
logPrefix = "[Backfill]",
}: FireQueryOptions): Promise<void> {
logger.info(`${logPrefix} Firing query ${queryId}`);
const abortController = new AbortController();
const retrySetting = (() => {
if (attemptNumber > 1) {
return retrySettings.retry2 ?? {};
}
if (attemptNumber > 0) {
return retrySettings.retry1 ?? {};
}
return retrySettings.retry0 ?? {};
})();
if (attemptNumber > 0) {
logger.info(
`${logPrefix} Applying retry settings for query ${queryId}: ${JSON.stringify(retrySetting)} (attempt ${attemptNumber})`,
);
}
// The rejection handler must attach in the same tick the promise is created:
// the command can reject at any point (fast HTTP/auth/parse errors, the
// expected abort below) and an unhandled rejection kills the worker process
// via the migrations' unhandledRejection handlers.
let earlyError: unknown = null;
const queryPromise = commandClickhouse({
query,
params,
// Set the server-side query_id so pollQueryStatus can track this query in
// system.processes / system.query_log after we abort the HTTP connection.
// (Post-#14493 commandClickhouse only honors the top-level `queryId`; a
// `queryId` inside `tags` is dropped by tag normalization.)
queryId,
tags: {
surface: "worker",
route: "background-migration.fireQuery",
},
clickhouseSettings: {
// The fire-and-poll pattern intentionally outlives the HTTP request, but
// the shared client derives a server-side max_execution_time (~35s) from
// its default request timeout, which kills long chunk queries after the
// abort (#14999). Override per-query so interactive queries stay capped;
// retry settings keep precedence.
max_execution_time: 0,
timeout_before_checking_execution_speed: 0,
...retrySetting,
},
abortSignal: abortController.signal,
}).catch((err) => {
if (err?.name === "AbortError" || err?.message?.includes("aborted")) {
logger.debug(
`${logPrefix} Query ${queryId} HTTP connection aborted as expected`,
);
} else {
earlyError = err;
logger.info(
`${logPrefix} Query ${queryId} promise rejected: ${err?.message}`,
err,
);
}
});
// Surface fast failures (connection refused, auth, SQL parse errors) right
// away instead of polling for a query that never started.
await Promise.race([queryPromise, sleep(initialWaitMs)]);
if (earlyError) {
throw earlyError instanceof Error
? earlyError
: new Error(String(earlyError));
}
try {
const status = await pollQueryStatus(queryId);
if (status === "not_found") {
await sleep(retryWaitMs);
const retryStatus = await pollQueryStatus(queryId);
if (retryStatus === "not_found") {
throw new Error(`Query ${queryId} failed to start on server`);
}
}
} catch (error) {
logger.error(
`${logPrefix} Error verifying query ${queryId} started`,
error,
);
throw error;
}
logger.info(
`${logPrefix} Query ${queryId} confirmed running, aborting HTTP connection`,
);
abortController.abort();
}
// ============================================================================
// Recovery
// ============================================================================
/**
* Reattach to in-flight queries from a previous worker run by polling each
* queryId. Updates each todo in-place with the recovered status.
* Usually, persisted afterwards by a state update, i.e. relies on mutating behaviour
* outside the function.
* Returns the subset that ClickHouse still reports as running so the caller
* can keep tracking them.
*/
async function recoverInProgressTodos<T extends BaseChunkTodo>(
todos: T[],
logPrefix = "[Backfill]",
): Promise<T[]> {
const inProgress = todos.filter(
(t) => t.status === "in_progress" && t.queryId,
);
if (inProgress.length === 0) return [];
logger.info(
`${logPrefix} Recovering ${inProgress.length} in-progress chunks`,
);
const stillRunning: T[] = [];
for (const todo of inProgress) {
try {
const status = await pollQueryStatus(todo.queryId!);
if (status === "completed") {
todo.status = "completed";
todo.completedAt = new Date().toISOString();
logger.info(`${logPrefix} Recovered chunk ${todo.id} as completed`);
} else if (status === "failed") {
todo.status = "pending";
todo.retryCount = (todo.retryCount || 0) + 1;
const error = await getQueryError(todo.queryId!);
logger.warn(
`${logPrefix} Recovered chunk ${todo.id} as failed, will retry: ${error}`,
);
} else if (status === "running") {
logger.info(
`${logPrefix} Recovered chunk ${todo.id} as still running (query ${todo.queryId}), will continue tracking`,
);
stillRunning.push(todo);
} else {
todo.status = "pending";
todo.queryId = undefined;
logger.warn(
`${logPrefix} Recovered chunk ${todo.id} as not_found, resetting to pending`,
);
}
} catch (error) {
logger.error(
`${logPrefix} Error during recovery polling for chunk ${todo.id}, resetting to pending`,
error,
);
todo.status = "pending";
todo.queryId = undefined;
}
}
return stillRunning;
}
// ============================================================================
// Chunked backfill base migration
// ============================================================================
/**
* Base class for backfill migrations that split their work into chunks
* (partitions or parts), run one long-lived server-side ClickHouse query per
* chunk via `fireQuery`, and persist progress in `background_migrations.state`.
*
* `run()` drives a single sequential scheduler loop that polls active queries,
* applies completions/failures, and fills free concurrency slots.
*
* Subclasses provide the chunk enumeration and per-chunk query, plus optional
* hooks for lazy DDL, post-chunk verification, and completion side effects.
*/
export abstract class ChunkedClickhouseBackfillMigration<
T extends BaseChunkTodo = BaseChunkTodo,
> implements IBackgroundMigration {
protected isAborted = false;
/**
* Set when `verifyCompletedChunk` reports a violation. The loop winds down
* (in-flight queries keep their recoverable bookkeeping) and `run()` throws
* this error so the manager records `failedAt`.
*/
private haltError: Error | null = null;
/** Hard-coded UUID of the row in background_migrations. */
protected abstract readonly migrationId: string;
protected abstract readonly logPrefix: string;
/** ClickHouse tables that must exist before the migration may run. */
protected abstract readonly requiredTables: string[];
/** Optional chain guard evaluated first in validate(). */
protected readonly predecessor?: { id: string; name: string };
/** Enumerates the chunk todos. Called exactly once per migration lifetime. */
protected abstract enumerateChunks(
config: ResolvedChunkedBackfillConfig,
): Promise<T[]>;
/** Builds the long-running INSERT for one chunk. */
protected abstract buildChunkQuery(todo: T): {
query: string;
params: Record<string, unknown>;
};
/** Hook: runs after required tables exist (e.g. lazy scratch-table DDL). */
protected async afterTablesValidated(): Promise<void> {}
/**
* Hook: verifies a chunk whose query completed. Returning an error message
* marks the chunk failed and halts the migration (`run()` throws it).
*/
protected async verifyCompletedChunk(_todo: T): Promise<string | null> {
return null;
}
/** Hook: runs once after every chunk completed cleanly; may throw. */
protected async onBackfillSucceeded(
_state: ChunkedBackfillState<T>,
): Promise<void> {}
/** Hook: runs when chunks failed permanently, right before run() throws. */
protected async onBackfillFailed(
_state: ChunkedBackfillState<T>,
): Promise<void> {}
// ==========================================================================
// State management
// ==========================================================================
protected async loadState(): Promise<ChunkedBackfillState<T>> {
const migration = await prisma.backgroundMigration.findUnique({
where: { id: this.migrationId },
select: { state: true },
});
const defaultState: ChunkedBackfillState<T> = {
phase: "init",
chunksLoaded: false,
todos: [],
activeQueries: [],
config: {},
};
if (!migration || !migration.state) {
return defaultState;
}
const state = migration.state as Partial<ChunkedBackfillState<T>>;
return {
phase: state.phase ?? defaultState.phase,
chunksLoaded: state.chunksLoaded ?? defaultState.chunksLoaded,
todos: state.todos ?? defaultState.todos,
activeQueries: state.activeQueries ?? defaultState.activeQueries,
config: state.config ?? defaultState.config,
};
}
protected async updateState(state: ChunkedBackfillState<T>): Promise<void> {
await prisma.backgroundMigration.update({
where: { id: this.migrationId },
data: { state: state as any },
});
}
// ==========================================================================
// Validation
// ==========================================================================
async validate(
_args: Record<string, unknown>,
attempts = 5,
): Promise<{ valid: boolean; invalidReason: string | undefined }> {
if (this.predecessor) {
const predecessor = await checkPredecessorMigrationFinalized(
this.predecessor.id,
this.predecessor.name,
);
if (!predecessor.valid) {
return predecessor;
}
}
for (let attempt = 0; ; attempt++) {
const missing = await this.findFirstMissingTable();
if (!missing) break;
if (attempt >= attempts) {
return {
valid: false,
invalidReason: `ClickHouse ${missing} table does not exist`,
};
}
logger.info(
`${this.logPrefix} ${missing} table does not exist. Retrying in 10s...`,
);
await sleep(10_000);
}
await this.afterTablesValidated();
logger.info(`${this.logPrefix} All prerequisites validated successfully`);
return { valid: true, invalidReason: undefined };
}
private async findFirstMissingTable(): Promise<string | null> {
const tables = await clickhouseClient().query({
query: "SHOW TABLES",
clickhouse_settings: {
log_comment: buildClickHouseLogComment({
surface: "worker",
route: `background-migration.${this.constructor.name}`,
}),
},
});
const tableNames = (await tables.json()).data as { name: string }[];
return (
this.requiredTables.find(
(required) => !tableNames.some((r) => r.name === required),
) ?? null
);
}
// ==========================================================================
// Main run loop
// ==========================================================================
async run(args: Record<string, unknown>): Promise<void> {
const start = Date.now();
const migrationArgs = args as ChunkedBackfillArgs;
const config: ResolvedChunkedBackfillConfig = {
concurrency: migrationArgs.concurrency ?? 1,
pollIntervalMs: migrationArgs.pollIntervalMs ?? 30_000,
maxRetries: migrationArgs.maxRetries ?? 3,
partitions: migrationArgs.partitions,
};
logger.info(
`${this.logPrefix} Starting backfill with config: ${JSON.stringify(config)}`,
);
const state = await this.loadState();
state.config = config;
// Phase 1: enumerate chunks (one-time)
if (!state.chunksLoaded) {
state.phase = "loading_chunks";
await this.updateState(state);
state.todos = await this.enumerateChunks(config);
state.chunksLoaded = true;
state.phase = "backfill";
await this.updateState(state);
}
// Phase 2: re-attach to queries left running by a previous worker
const stillRunning = await recoverInProgressTodos(
state.todos,
this.logPrefix,
);
await this.updateState(state);
// Phase 2.5: reset failed chunks to pending if --retry-failed was passed
if (migrationArgs.retryFailed) {
const failedChunks = state.todos.filter((t) => t.status === "failed");
if (failedChunks.length > 0) {
logger.info(
`${this.logPrefix} Resetting ${failedChunks.length} failed chunks to pending`,
);
for (const todo of failedChunks) {
todo.status = "pending";
todo.error = undefined;
todo.retryCount = 0;
}
await this.updateState(state);
}
}
// Phase 3: single sequential scheduler loop.
const active = new Map<string, T>();
for (const todo of stillRunning) {
active.set(todo.queryId!, todo);
logger.info(
`${this.logPrefix} Tracking recovered running query ${todo.queryId} for chunk ${todo.id}`,
);
}
while (!this.isAborted && !this.haltError) {
await this.pollActiveQueries(active, state, config);
if (this.isAborted || this.haltError) break;
await this.fillFreeSlots(active, state, config);
const outstanding =
active.size > 0 || state.todos.some((t) => t.status === "pending");
if (!outstanding) break;
await sleep(config.pollIntervalMs);
}
if (this.haltError) {
logger.error(this.haltError.message);
throw this.haltError;
}
if (this.isAborted) {
// In-flight queries keep running server-side; the next run re-attaches
// to them via the persisted queryIds.
logger.info(
`${this.logPrefix} Migration aborted. Can be resumed from current state.`,
);
return;
}
const failed = state.todos.filter((t) => t.status === "failed");
if (failed.length > 0) {
await this.onBackfillFailed(state);
const message =
`${this.logPrefix} Migration completed with ${failed.length} failed chunk(s); ` +
`clear failedAt and re-run with --retry-failed before downstream steps can proceed.`;
logger.error(message);
throw new Error(message);
}
logger.info(`${this.logPrefix} All chunks completed!`);
state.phase = "completed";
await this.updateState(state);
await this.onBackfillSucceeded(state);
logger.info(
`${this.logPrefix} Finished backfill in ${((Date.now() - start) / 1000 / 60).toFixed(2)} minutes`,
);
}
/**
* Polls every active query once and applies completions/failures to state.
*/
private async pollActiveQueries(
active: Map<string, T>,
state: ChunkedBackfillState<T>,
config: ResolvedChunkedBackfillConfig,
): Promise<void> {
for (const [queryId, todo] of [...active]) {
let status: QueryStatus;
try {
status = await pollQueryStatus(queryId);
} catch (error) {
logger.warn(
`${this.logPrefix} Error polling query ${queryId} for chunk ${todo.id}, will retry on next poll cycle`,
error,
);
continue;
}
if (status === "running") continue;
active.delete(queryId);
state.activeQueries = state.activeQueries.filter((q) => q !== queryId);
if (status === "completed") {
const verificationError = await this.verifyCompletedChunk(todo);
if (verificationError) {
todo.status = "failed";
todo.error = verificationError;
this.haltError = new Error(`${this.logPrefix} ${verificationError}`);
} else {
todo.status = "completed";
todo.completedAt = new Date().toISOString();
const completed = state.todos.filter(
(t) => t.status === "completed",
).length;
logger.info(
`${this.logPrefix} Completed chunk ${todo.id} (${completed}/${state.todos.length})`,
);
}
} else {
const error =
status === "failed"
? await getQueryError(queryId)
: "Query not found in query_log";
this.applyChunkFailure(todo, error, config);
}
await this.updateState(state);
}
}
/**
* Fires queries for pending chunks until the concurrency limit is reached.
*/
private async fillFreeSlots(
active: Map<string, T>,
state: ChunkedBackfillState<T>,
config: ResolvedChunkedBackfillConfig,
): Promise<void> {
while (active.size < config.concurrency) {
const next = state.todos.find((t) => t.status === "pending");
if (!next) return;
next.status = "in_progress";
next.queryId = generateQueryId(next.id);
next.startedAt = new Date().toISOString();
state.activeQueries.push(next.queryId);
// Persist before firing: a crash in between is recovered as a not_found
// query and the chunk falls back to pending.
await this.updateState(state);
try {
const { query, params } = this.buildChunkQuery(next);
await fireQuery({
query,
queryId: next.queryId,
params,
attemptNumber: next.retryCount || 0,
logPrefix: this.logPrefix,
});
active.set(next.queryId, next);
logger.info(
`${this.logPrefix} Started chunk ${next.id} with query ${next.queryId}`,
);
} catch (err) {
logger.error(
`${this.logPrefix} Failed to start query for chunk ${next.id}`,
err,
);
state.activeQueries = state.activeQueries.filter(
(q) => q !== next.queryId,
);
// Fire-time failures count against maxRetries too — a deterministic
// error (bad SQL, schema drift) must not retry forever.
this.applyChunkFailure(
next,
err instanceof Error ? err.message : String(err),
config,
);
await this.updateState(state);
return; // back off until the next poll cycle
}
}
}
private applyChunkFailure(
todo: T,
error: string | undefined,
config: ResolvedChunkedBackfillConfig,
): void {
todo.retryCount = (todo.retryCount || 0) + 1;
todo.queryId = undefined;
if (todo.retryCount >= config.maxRetries) {
todo.status = "failed";
todo.error = error;
logger.error(
`${this.logPrefix} Chunk ${todo.id} failed permanently: ${error}`,
);
} else {
todo.status = "pending";
logger.warn(
`${this.logPrefix} Chunk ${todo.id} failed, will retry (${todo.retryCount}/${config.maxRetries}): ${error}`,
);
}
}
async abort(): Promise<void> {
logger.info(`${this.logPrefix} Aborting backfill migration`);
this.isAborted = true;
}
}
// ============================================================================
// CLI entry point helper
// ============================================================================
/**
* Shared CLI wrapper for chunked backfill migrations. Installs the fail-fast
* process handlers, parses the standard flags, then validates and runs the
* migration. Call from the migration module's `require.main === module` block.
*/
export function runBackfillMigrationCli(
createMigration: () => IBackgroundMigration,
{
logPrefix,
includePartitions = true,
}: { logPrefix: string; includePartitions?: boolean },
): void {
process.on("unhandledRejection", (reason) => {
logger.error(
`${logPrefix} Unhandled promise rejection - process will exit`,
{
reason: reason instanceof Error ? reason.message : String(reason),
stack: reason instanceof Error ? reason.stack : undefined,
},
);
process.exit(1);
});
process.on("uncaughtException", (error) => {
logger.error(`${logPrefix} Uncaught exception - process will exit`, error);
process.exit(1);
});
async function main(): Promise<void> {
const { values } = parseArgs({
options: {
concurrency: { type: "string", short: "c", default: "1" },
pollIntervalMs: { type: "string", short: "p", default: "30000" },
maxRetries: { type: "string", short: "r", default: "3" },
retryFailed: { type: "boolean", short: "f", default: false },
...(includePartitions
? { partitions: { type: "string", multiple: true } as const }
: {}),
},
}) as {
values: {
concurrency: string;
pollIntervalMs: string;
maxRetries: string;
retryFailed: boolean;
partitions?: string[];
};
};
const migration = createMigration();
const parsedArgs: ChunkedBackfillArgs = {
concurrency: parseInt(values.concurrency, 10),
pollIntervalMs: parseInt(values.pollIntervalMs, 10),
maxRetries: parseInt(values.maxRetries, 10),
retryFailed: values.retryFailed,
partitions: values.partitions,
};
const validation = await migration.validate(
parsedArgs as Record<string, unknown>,
);
if (!validation.valid) {
logger.error(`Validation failed: ${validation.invalidReason}`);
process.exit(1);
}
await migration.run(parsedArgs as Record<string, unknown>);
}
main()
.then(() => {
process.exit(0);
})
.catch((error) => {
logger.error(`Migration execution failed: ${error}`, error);
process.exit(1);
});
}