High performance self-hosted photo and video management solution.
TypeScript backup-toolfluttergoogle-photosgoogle-photos-alternativejavascriptmobile-appnestjsnodejsphoto-galleryphotosphotos-managementself-hostedsveltesveltekittypescriptvideos
feat(server): compress database backups with pigz when it is installed (#32806)
Zack Pollard97af09ca3cbd
diff --git a/server/src/repositories/process.repository.spec.ts b/server/src/repositories/process.repository.spec.tsindex 47ae7a6934..be49d4ccf4 100644--- a/server/src/repositories/process.repository.spec.ts+++ b/server/src/repositories/process.repository.spec.ts@@ -27,6 +27,16 @@ describe(ProcessRepository.name, () => {});});+ describe('isCommandAvailable', () => {+ it('should find an executable on the PATH', async () => {+ await expect(sut.isCommandAvailable('bash')).resolves.toBe(true);+ });++ it('should not find a missing executable', async () => {+ await expect(sut.isCommandAvailable('immich-missing-command')).resolves.toBe(false);+ });+ });+describe('createSpawnDuplexStream', () => {it('should work (drain to stdout)', async () => {const process = sut.spawnDuplexStream('bash', ['-c', 'exit 0']);diff --git a/server/src/repositories/process.repository.ts b/server/src/repositories/process.repository.tsindex 433bbc015b..a5148872c6 100644--- a/server/src/repositories/process.repository.ts+++ b/server/src/repositories/process.repository.ts@@ -1,10 +1,27 @@import { Injectable } from '@nestjs/common';import { SpawnOptionsWithoutStdio, fork, spawn } from 'node:child_process';+import { constants } from 'node:fs';+import { access } from 'node:fs/promises';+import { availableParallelism } from 'node:os';+import path from 'node:path';import { Duplex } from 'node:stream';@Injectable()export class ProcessRepository {spawn = spawn;+ availableParallelism = availableParallelism;++ async isCommandAvailable(command: string): Promise<boolean> {+ const directories = (process.env.PATH ?? '').split(path.delimiter).filter(Boolean);+ const found = await Promise.all(+ directories.map((directory) =>+ access(path.join(directory, command), constants.X_OK)+ .then(() => true)+ .catch(() => false),+ ),+ );+ return found.includes(true);+ }spawnDuplexStream(command: string, args?: readonly string[], options?: SpawnOptionsWithoutStdio): Duplex {let isStdinClosed = false;diff --git a/server/src/services/database-backup.service.spec.ts b/server/src/services/database-backup.service.spec.tsindex 9778f984a0..c6869705cb 100644--- a/server/src/services/database-backup.service.spec.ts+++ b/server/src/services/database-backup.service.spec.ts@@ -16,6 +16,7 @@ describe(DatabaseBackupService.name, () => {beforeEach(() => {mocks = getMocks();+ mocks.process.isCommandAvailable.mockResolvedValue(false);maintenanceHealthRepositoryMock = automock(MaintenanceHealthRepository, {args: [mocks.logger],strict: false,@@ -228,6 +229,19 @@ describe(DatabaseBackupService.name, () => {expect(mocks.storage.rename).toHaveBeenCalled();});+ it('should compress with gzip when pigz is not installed', async () => {+ await sut.handleBackupDatabase();+ expect(mocks.process.isCommandAvailable).toHaveBeenCalledWith('pigz');+ expect(mocks.process.spawnDuplexStream).toHaveBeenLastCalledWith('gzip', ['--rsyncable']);+ });++ it('should compress with pigz on every available core when it is installed', async () => {+ mocks.process.isCommandAvailable.mockResolvedValue(true);+ mocks.process.availableParallelism.mockReturnValue(8);+ await sut.handleBackupDatabase();+ expect(mocks.process.spawnDuplexStream).toHaveBeenLastCalledWith('pigz', ['--rsyncable', '--processes', '8']);+ });+it('should fail if pg_dump fails', async () => {mocks.process.spawnDuplexStream.mockReturnValueOnce(mockDuplex()('pg_dump', 1, '', 'error'));await expect(sut.handleBackupDatabase()).rejects.toThrow('pg_dump non-zero exit code (1)');diff --git a/server/src/services/database-backup.service.ts b/server/src/services/database-backup.service.tsindex 2a62afbe50..a2d1dbed84 100644--- a/server/src/services/database-backup.service.ts+++ b/server/src/services/database-backup.service.ts@@ -236,6 +236,8 @@ export class DatabaseBackupService {const backupFilePath = path.join(StorageCore.getBaseFolder(StorageFolder.Backups), filename);const temporaryFilePath = `${backupFilePath}.tmp`;+ const compression = await this.getCompressionCommand();+let pgdump: Duplex | undefined;let gzip: Duplex | undefined;@@ -247,7 +249,7 @@ export class DatabaseBackupService {},});- gzip = this.processRepository.spawnDuplexStream('gzip', ['--rsyncable']);+ gzip = this.processRepository.spawnDuplexStream(compression.command, compression.args);const fileStream = this.storageRepository.createWriteStream(temporaryFilePath);await pipeline(pgdump, gzip, fileStream);@@ -267,6 +269,15 @@ export class DatabaseBackupService {return backupFilePath;}+ private async getCompressionCommand() {+ if (await this.processRepository.isCommandAvailable('pigz')) {+ const processes = this.processRepository.availableParallelism();+ return { command: 'pigz', args: ['--rsyncable', '--processes', processes.toString()] };+ }++ return { command: 'gzip', args: ['--rsyncable'] };+ }+async uploadBackup(file: Express.Multer.File): Promise<void> {const backupsFolder = StorageCore.getBaseFolder(StorageFolder.Backups);const fn = basename(file.originalname);