🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
fix(sites): stop site deletes timing out behind ClickHouse mutations
Bill Yang7bfe1d075ac4
diff --git a/server/src/services/sites/siteConfigurationLifecycle.test.ts b/server/src/services/sites/siteConfigurationLifecycle.test.tsindex 66e4df111..20392ed37 100644--- a/server/src/services/sites/siteConfigurationLifecycle.test.ts+++ b/server/src/services/sites/siteConfigurationLifecycle.test.ts@@ -180,6 +180,14 @@ describe("siteConfigurationLifecycle", () => {expect(mocks.invalidate).toHaveBeenCalledWith(state.site);});+ it("queues the replay deletes instead of waiting behind other mutations", async () => {+ await siteConfigurationLifecycle.delete(1);++ for (const [call] of mocks.clickhouseCommand.mock.calls) {+ expect(call.clickhouse_settings).toEqual({ lightweight_deletes_sync: "0" });+ }+ });+it("keeps replay data when the Postgres delete fails", async () => {state.deleteError = new Error("foreign key violation");diff --git a/server/src/services/sites/siteConfigurationLifecycle.ts b/server/src/services/sites/siteConfigurationLifecycle.tsindex f2edc6a1e..e0fe7690f 100644--- a/server/src/services/sites/siteConfigurationLifecycle.ts+++ b/server/src/services/sites/siteConfigurationLifecycle.ts@@ -556,14 +556,25 @@ class SiteConfigurationLifecycle {}private async deleteReplayData(siteId: number): Promise<void> {+ // A lightweight DELETE is a mutation, and mutations on a table run one at a+ // time. The replay tables also take the identity backfill's UPDATE+ // mutations, so a DELETE that waits for completion (the default,+ // lightweight_deletes_sync=2) can sit behind them for an hour. In prod+ // that ran past the client's 300s timeout, so the Postgres delete rolled+ // back and the site couldn't be deleted. With 0 the server returns once+ // the mutation is queued. ClickHouse keeps queued mutations across+ // restarts, so the replay rows are still removed later.+ const settings = { lightweight_deletes_sync: "0" };await Promise.all([clickhouse.command({query: "DELETE FROM session_replay_events WHERE site_id = {id:UInt32}",query_params: { id: siteId },+ clickhouse_settings: settings,}),clickhouse.command({query: "DELETE FROM session_replay_metadata_v2 WHERE site_id = {id:UInt32}",query_params: { id: siteId },+ clickhouse_settings: settings,}),]);}@@ -572,8 +583,9 @@ class SiteConfigurationLifecycle {const site = await this.findSite(siteId);// Delete the Postgres rows first so a constraint failure can't leave the// Site in place with its replays already wiped. import_status is the one- // table whose foreign key to sites doesn't cascade. Replay cleanup runs- // inside the transaction, so a ClickHouse failure rolls the Site back.+ // table whose foreign key to sites doesn't cascade. Replay cleanup is+ // queued inside the transaction, so a failure to queue it rolls the Site+ // back.await db.transaction(async tx => {await tx.delete(importStatus).where(eq(importStatus.siteId, siteId));await tx.delete(sites).where(eq(sites.siteId, siteId));