Skip to content

High performance self-hosted photo and video management solution.

feat(server): compress database backups with pigz when it is installed (#32806)

Zack Pollard97af09ca3cbd
diff --git a/server/src/repositories/process.repository.spec.ts b/server/src/repositories/process.repository.spec.ts
index 47ae7a6934..be49d4ccf4 100644
--- a/server/src/repositories/process.repository.spec.ts
+++ b/server/src/repositories/process.repository.spec.ts
@@ -27,6 +27,16 @@ describe(ProcessRepository.name, () => {
});
});
+ describe('isCommandAvailable', () => {
+ it('should find an executable on the PATH', async () => {
+ await expect(sut.isCommandAvailable('bash')).resolves.toBe(true);
+ });
+
+ it('should not find a missing executable', async () => {
+ await expect(sut.isCommandAvailable('immich-missing-command')).resolves.toBe(false);
+ });
+ });
+
describe('createSpawnDuplexStream', () => {
it('should work (drain to stdout)', async () => {
const process = sut.spawnDuplexStream('bash', ['-c', 'exit 0']);
diff --git a/server/src/repositories/process.repository.ts b/server/src/repositories/process.repository.ts
index 433bbc015b..a5148872c6 100644
--- a/server/src/repositories/process.repository.ts
+++ b/server/src/repositories/process.repository.ts
@@ -1,10 +1,27 @@
import { Injectable } from '@nestjs/common';
import { SpawnOptionsWithoutStdio, fork, spawn } from 'node:child_process';
+import { constants } from 'node:fs';
+import { access } from 'node:fs/promises';
+import { availableParallelism } from 'node:os';
+import path from 'node:path';
import { Duplex } from 'node:stream';
@Injectable()
export class ProcessRepository {
spawn = spawn;
+ availableParallelism = availableParallelism;
+
+ async isCommandAvailable(command: string): Promise<boolean> {
+ const directories = (process.env.PATH ?? '').split(path.delimiter).filter(Boolean);
+ const found = await Promise.all(
+ directories.map((directory) =>
+ access(path.join(directory, command), constants.X_OK)
+ .then(() => true)
+ .catch(() => false),
+ ),
+ );
+ return found.includes(true);
+ }
spawnDuplexStream(command: string, args?: readonly string[], options?: SpawnOptionsWithoutStdio): Duplex {
let isStdinClosed = false;
diff --git a/server/src/services/database-backup.service.spec.ts b/server/src/services/database-backup.service.spec.ts
index 9778f984a0..c6869705cb 100644
--- a/server/src/services/database-backup.service.spec.ts
+++ b/server/src/services/database-backup.service.spec.ts
@@ -16,6 +16,7 @@ describe(DatabaseBackupService.name, () => {
beforeEach(() => {
mocks = getMocks();
+ mocks.process.isCommandAvailable.mockResolvedValue(false);
maintenanceHealthRepositoryMock = automock(MaintenanceHealthRepository, {
args: [mocks.logger],
strict: false,
@@ -228,6 +229,19 @@ describe(DatabaseBackupService.name, () => {
expect(mocks.storage.rename).toHaveBeenCalled();
});
+ it('should compress with gzip when pigz is not installed', async () => {
+ await sut.handleBackupDatabase();
+ expect(mocks.process.isCommandAvailable).toHaveBeenCalledWith('pigz');
+ expect(mocks.process.spawnDuplexStream).toHaveBeenLastCalledWith('gzip', ['--rsyncable']);
+ });
+
+ it('should compress with pigz on every available core when it is installed', async () => {
+ mocks.process.isCommandAvailable.mockResolvedValue(true);
+ mocks.process.availableParallelism.mockReturnValue(8);
+ await sut.handleBackupDatabase();
+ expect(mocks.process.spawnDuplexStream).toHaveBeenLastCalledWith('pigz', ['--rsyncable', '--processes', '8']);
+ });
+
it('should fail if pg_dump fails', async () => {
mocks.process.spawnDuplexStream.mockReturnValueOnce(mockDuplex()('pg_dump', 1, '', 'error'));
await expect(sut.handleBackupDatabase()).rejects.toThrow('pg_dump non-zero exit code (1)');
diff --git a/server/src/services/database-backup.service.ts b/server/src/services/database-backup.service.ts
index 2a62afbe50..a2d1dbed84 100644
--- a/server/src/services/database-backup.service.ts
+++ b/server/src/services/database-backup.service.ts
@@ -236,6 +236,8 @@ export class DatabaseBackupService {
const backupFilePath = path.join(StorageCore.getBaseFolder(StorageFolder.Backups), filename);
const temporaryFilePath = `${backupFilePath}.tmp`;
+ const compression = await this.getCompressionCommand();
+
let pgdump: Duplex | undefined;
let gzip: Duplex | undefined;
@@ -247,7 +249,7 @@ export class DatabaseBackupService {
},
});
- gzip = this.processRepository.spawnDuplexStream('gzip', ['--rsyncable']);
+ gzip = this.processRepository.spawnDuplexStream(compression.command, compression.args);
const fileStream = this.storageRepository.createWriteStream(temporaryFilePath);
await pipeline(pgdump, gzip, fileStream);
@@ -267,6 +269,15 @@ export class DatabaseBackupService {
return backupFilePath;
}
+ private async getCompressionCommand() {
+ if (await this.processRepository.isCommandAvailable('pigz')) {
+ const processes = this.processRepository.availableParallelism();
+ return { command: 'pigz', args: ['--rsyncable', '--processes', processes.toString()] };
+ }
+
+ return { command: 'gzip', args: ['--rsyncable'] };
+ }
+
async uploadBackup(file: Express.Multer.File): Promise<void> {
const backupsFolder = StorageCore.getBaseFolder(StorageFolder.Backups);
const fn = basename(file.originalname);