Skip to content
TypeScript

fix(server): server updates reclaim partial database backups left by killed updates (#29464)

MarcoDotIOeee8ed4f23b4
diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts
index 70550f23792..1da6c2e1951 100644
--- a/apps/server/src/serviceLauncher.test.ts
+++ b/apps/server/src/serviceLauncher.test.ts
@@ -1,5 +1,7 @@
// @effect-diagnostics-next-line nodeBuiltinImport:off - observes the children the launcher spawns with Node built-ins.
import type * as NodeChildProcess from "node:child_process";
+// @effect-diagnostics-next-line nodeBuiltinImport:off - the free-space probe is a plain Promise callback.
+import * as NodeFS from "node:fs";
import * as NodeServices from "@effect/platform-node/NodeServices";
import { assert, it } from "@effect/vitest";
@@ -618,6 +620,35 @@ setInterval(() => {}, 1_000);
}),
);
+ it.effect("reclaims partial backups left by killed updates before checking space", () =>
+ Effect.gen(function* () {
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-orphan-" });
+ const databasePath = path.join(root, "state.sqlite");
+ yield* writeSparseFile(databasePath, 10 * 1024 ** 3);
+ const backupRoot = path.join(root, "runtime", "db-backup");
+ const orphan = path.join(backupRoot, "earlier-update.staging");
+ const completed = path.join(backupRoot, "completed-update");
+ yield* fs.makeDirectory(orphan, { recursive: true });
+ yield* fs.writeFileString(path.join(orphan, "database"), "partial copy");
+ yield* fs.makeDirectory(completed);
+ yield* fs.writeFileString(path.join(completed, "database"), "complete copy");
+
+ // The orphan is what keeps this disk too full for a full copy.
+ const blocker = yield* Effect.promise(() =>
+ databaseBackupBlocker(root, databasePath, {
+ readFreeBytes: () => Promise.resolve((NodeFS.existsSync(orphan) ? 8 : 11) * 1024 ** 3),
+ canClone: () => Promise.resolve(false),
+ }),
+ );
+
+ assert.isUndefined(blocker);
+ assert.isFalse(yield* fs.exists(orphan));
+ assert.isTrue(yield* fs.exists(path.join(completed, "database")));
+ }),
+ );
+
it.effect("checks database backup space against clone or full-copy needs", () =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts
index 9f764b788ef..ee05c632978 100644
--- a/apps/server/src/serviceLauncher.ts
+++ b/apps/server/src/serviceLauncher.ts
@@ -219,6 +219,17 @@ const filesystemBackupSpaceProbe: BackupSpaceProbe = {
canClone: backupCanClone,
};
+// A killed launcher skips the backup's cleanup, and retries only clear their own update's copy.
+// Update requests are refused while one is pending, so no staging directory is still in use here.
+async function removeAbandonedBackupCopies(backupRoot: string): Promise<void> {
+ for (const entry of await NodeFSP.readdir(backupRoot)) {
+ if (!entry.endsWith(".staging")) continue;
+ await NodeFSP.rm(NodePath.join(backupRoot, entry), { recursive: true, force: true }).catch(
+ () => undefined,
+ );
+ }
+}
+
/** Check before stopping the server so full disks refuse updates without an outage. */
export async function databaseBackupBlocker(
baseDir: string,
@@ -228,6 +239,7 @@ export async function databaseBackupBlocker(
const { readFreeBytes, canClone } = { ...filesystemBackupSpaceProbe, ...probe };
const backupRoot = NodePath.join(baseDir, "runtime", "db-backup");
await NodeFSP.mkdir(backupRoot, { recursive: true, mode: 0o700 });
+ await removeAbandonedBackupCopies(backupRoot);
const [files, free] = await Promise.all([statDatabaseFiles(dbPath), readFreeBytes(backupRoot)]);
const size = files.reduce((total, file) => total + file.size, 0);
const fullCopyBytes = size * 1.05;