TypeScript
fix(server): server updates reclaim partial database backups left by killed updates (#29464)
MarcoDotIOeee8ed4f23b4
diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.tsindex 70550f23792..1da6c2e1951 100644--- a/apps/server/src/serviceLauncher.test.ts+++ b/apps/server/src/serviceLauncher.test.ts@@ -1,5 +1,7 @@// @effect-diagnostics-next-line nodeBuiltinImport:off - observes the children the launcher spawns with Node built-ins.import type * as NodeChildProcess from "node:child_process";+// @effect-diagnostics-next-line nodeBuiltinImport:off - the free-space probe is a plain Promise callback.+import * as NodeFS from "node:fs";import * as NodeServices from "@effect/platform-node/NodeServices";import { assert, it } from "@effect/vitest";@@ -618,6 +620,35 @@ setInterval(() => {}, 1_000);}),);+ it.effect("reclaims partial backups left by killed updates before checking space", () =>+ Effect.gen(function* () {+ const fs = yield* FileSystem.FileSystem;+ const path = yield* Path.Path;+ const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-orphan-" });+ const databasePath = path.join(root, "state.sqlite");+ yield* writeSparseFile(databasePath, 10 * 1024 ** 3);+ const backupRoot = path.join(root, "runtime", "db-backup");+ const orphan = path.join(backupRoot, "earlier-update.staging");+ const completed = path.join(backupRoot, "completed-update");+ yield* fs.makeDirectory(orphan, { recursive: true });+ yield* fs.writeFileString(path.join(orphan, "database"), "partial copy");+ yield* fs.makeDirectory(completed);+ yield* fs.writeFileString(path.join(completed, "database"), "complete copy");++ // The orphan is what keeps this disk too full for a full copy.+ const blocker = yield* Effect.promise(() =>+ databaseBackupBlocker(root, databasePath, {+ readFreeBytes: () => Promise.resolve((NodeFS.existsSync(orphan) ? 8 : 11) * 1024 ** 3),+ canClone: () => Promise.resolve(false),+ }),+ );++ assert.isUndefined(blocker);+ assert.isFalse(yield* fs.exists(orphan));+ assert.isTrue(yield* fs.exists(path.join(completed, "database")));+ }),+ );+it.effect("checks database backup space against clone or full-copy needs", () =>Effect.gen(function* () {const fs = yield* FileSystem.FileSystem;diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.tsindex 9f764b788ef..ee05c632978 100644--- a/apps/server/src/serviceLauncher.ts+++ b/apps/server/src/serviceLauncher.ts@@ -219,6 +219,17 @@ const filesystemBackupSpaceProbe: BackupSpaceProbe = {canClone: backupCanClone,};+// A killed launcher skips the backup's cleanup, and retries only clear their own update's copy.+// Update requests are refused while one is pending, so no staging directory is still in use here.+async function removeAbandonedBackupCopies(backupRoot: string): Promise<void> {+ for (const entry of await NodeFSP.readdir(backupRoot)) {+ if (!entry.endsWith(".staging")) continue;+ await NodeFSP.rm(NodePath.join(backupRoot, entry), { recursive: true, force: true }).catch(+ () => undefined,+ );+ }+}+/** Check before stopping the server so full disks refuse updates without an outage. */export async function databaseBackupBlocker(baseDir: string,@@ -228,6 +239,7 @@ export async function databaseBackupBlocker(const { readFreeBytes, canClone } = { ...filesystemBackupSpaceProbe, ...probe };const backupRoot = NodePath.join(baseDir, "runtime", "db-backup");await NodeFSP.mkdir(backupRoot, { recursive: true, mode: 0o700 });+ await removeAbandonedBackupCopies(backupRoot);const [files, free] = await Promise.all([statDatabaseFiles(dbPath), readFreeBytes(backupRoot)]);const size = files.reduce((total, file) => total + file.size, 0);const fullCopyBytes = size * 1.05;