An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(backup): use shared compression for database dumps
Andras Bacsai0a6db152167c
diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.phpindex d432a3eeb3..82e35b73ce 100644--- a/app/Jobs/DatabaseBackupJob.php+++ b/app/Jobs/DatabaseBackupJob.php@@ -18,6 +18,7 @@ use App\Notifications\Database\BackupFailed;use App\Notifications\Database\BackupSuccess;use App\Notifications\Database\BackupSuccessWithS3Warning;use App\Rules\SafeWebhookUrl;+use App\Support\BackupCompression;use App\Support\ClickhouseBackupCommand;use Carbon\Carbon;use Illuminate\Bus\Queueable;@@ -609,7 +610,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue}$escapedUsername = escapeshellarg($this->database->postgres_user);if ($this->backup->dump_all) {- $backupCommand .= " $this->container_name pg_dumpall --username $escapedUsername | gzip > $this->backup_location";+ $backupCommand .= " $this->container_name pg_dumpall --username $escapedUsername";+ $backupCommand = $this->buildCompressedDumpCommand($backupCommand).' > '.escapeshellarg($this->backup_location);} else {// Validate and escape database name to prevent command injectionvalidateShellSafePath($database, 'database name');@@ -635,7 +637,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue$commands[] = 'mkdir -p '.$this->backup_dir;$escapedPassword = escapeshellarg($this->database->mysql_root_password);if ($this->backup->dump_all) {- $commands[] = "docker exec $this->container_name mysqldump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false --compress | gzip > $this->backup_location";+ $dumpCommand = "docker exec $this->container_name mysqldump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false";+ $commands[] = $this->buildCompressedDumpCommand($dumpCommand).' > '.escapeshellarg($this->backup_location);} else {// Validate and escape database name to prevent command injectionvalidateShellSafePath($database, 'database name');@@ -659,7 +662,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue$commands[] = 'mkdir -p '.$this->backup_dir;$escapedPassword = escapeshellarg($this->database->mariadb_root_password);if ($this->backup->dump_all) {- $commands[] = "docker exec $this->container_name mariadb-dump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false --compress > $this->backup_location";+ $dumpCommand = "docker exec $this->container_name mariadb-dump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false";+ $commands[] = $this->buildCompressedDumpCommand($dumpCommand).' > '.escapeshellarg($this->backup_location);} else {// Validate and escape database name to prevent command injectionvalidateShellSafePath($database, 'database name');@@ -806,6 +810,15 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueuereturn "{$helperImage}:{$latestVersion}";}+ private function buildCompressedDumpCommand(string $dumpCommand): string+ {+ $cpuPercentage = BackupCompression::cpuPercentage($this->server->settings->backup_compression_cpu_percentage);+ $compressorCommand = BackupCompression::compressorCommand($cpuPercentage);+ $script = "compressor=\$({$compressorCommand}); exec \$compressor";++ return $dumpCommand.' | docker run --rm -i '.escapeshellarg($this->getFullImageName()).' sh -c '.escapeshellarg($script);+ }+private function markStaleExecutionsAsFailed(): void{try {diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.phpindex c1ddaedb85..39998a1f60 100644--- a/app/Jobs/VolumeBackupJob.php+++ b/app/Jobs/VolumeBackupJob.php@@ -8,6 +8,7 @@ use App\Models\ScheduledVolumeBackup;use App\Models\ScheduledVolumeBackupExecution;use App\Models\Server;use App\Rules\SafeWebhookUrl;+use App\Support\BackupCompression;use Carbon\Carbon;use Illuminate\Bus\Queueable;use Illuminate\Contracts\Queue\ShouldBeEncrypted;@@ -77,15 +78,14 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue$source = $this->backup->sourcePath();$containerName = 'volume-backup-'.$this->execution->uuid;$image = coolifyHelperImage().':'.getHelperVersion();- $compressionCpuPercentage = $this->compressionCpuPercentage($server);+ $compressionCpuPercentage = BackupCompression::cpuPercentage($server->settings->backup_compression_cpu_percentage);$this->logCompressorInDevelopment($image, $server, $compressionCpuPercentage);$verifySourceCommand = $target instanceof LocalPersistentVolume && blank($target->host_path)? 'docker volume inspect '.escapeshellarg($source).' >/dev/null': 'test -d '.escapeshellarg($source);- $archiveScript = "compressor='gzip -3'; "- ."if command -v pigz >/dev/null 2>&1; then compressor=\"pigz -3 -p \$(( (\$(nproc) * {$compressionCpuPercentage} + 99) / 100 ))\"; fi; "- .'tar -I "$compressor" -cf - -C /volume .';+ $compressorCommand = BackupCompression::compressorCommand($compressionCpuPercentage);+ $archiveScript = "compressor=\$({$compressorCommand}); tar -I \"\$compressor\" -cf - -C /volume .";$archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName).' -v '.escapeshellarg($source.':/volume:ro').' '.escapeshellarg($image)@@ -344,7 +344,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueuereturn;}- $script = "if command -v pigz >/dev/null 2>&1; then printf 'pigz -3 -p %s' \"\$(( (\$(nproc) * {$compressionCpuPercentage} + 99) / 100 ))\"; else printf 'gzip -3'; fi";+ $script = BackupCompression::compressorCommand($compressionCpuPercentage);$compressor = instant_remote_process(['docker run --rm '.escapeshellarg($image).' sh -c '.escapeshellarg($script)],$server,@@ -361,13 +361,6 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue]);}- private function compressionCpuPercentage(Server $server): int- {- $percentage = (int) ($server->settings->backup_compression_cpu_percentage ?? 25);-- return in_array($percentage, [25, 50, 75, 100], true) ? $percentage : 25;- }-private function removeExpiredBackups(Server $server): void{if ($this->hasRetentionLimits(diff --git a/app/Support/BackupCompression.php b/app/Support/BackupCompression.phpnew file mode 100644index 0000000000..ad70f1fa0b--- /dev/null+++ b/app/Support/BackupCompression.php@@ -0,0 +1,20 @@+<?php++namespace App\Support;++final class BackupCompression+{+ public static function cpuPercentage(int|string|null $configuredPercentage): int+ {+ $percentage = (int) $configuredPercentage;++ return in_array($percentage, [25, 50, 75, 100], true) ? $percentage : 25;+ }++ public static function compressorCommand(int $cpuPercentage): string+ {+ $cpuPercentage = self::cpuPercentage($cpuPercentage);++ return "if command -v pigz >/dev/null 2>&1; then printf 'pigz -3 -p %s' \"\$(( (\$(nproc) * {$cpuPercentage} + 99) / 100 ))\"; else printf 'gzip -3'; fi";+ }+}diff --git a/tests/Feature/Jobs/DatabaseBackupJobTest.php b/tests/Feature/Jobs/DatabaseBackupJobTest.phpindex c21e1be224..7e47126ab1 100644--- a/tests/Feature/Jobs/DatabaseBackupJobTest.php+++ b/tests/Feature/Jobs/DatabaseBackupJobTest.php@@ -1,9 +1,12 @@<?phpuse App\Jobs\DatabaseBackupJob;+use App\Models\InstanceSettings;use App\Models\S3Storage;use App\Models\ScheduledDatabaseBackup;use App\Models\ScheduledDatabaseBackupExecution;+use App\Models\Server;+use App\Models\ServerSetting;use App\Models\Team;use Illuminate\Foundation\Testing\RefreshDatabase;use Illuminate\Support\Facades\Schema;@@ -322,3 +325,40 @@ test('database backup job escapes the S3 copy destination argument', function ()->and($source)->toContain('mc cp {$escapedBackupLocation} {$escapedS3Destination}')->and($source)->not->toContain('mc cp $this->backup_location temporary/$bucket{$this->backup_dir}/');});++test('database dump compression uses the helper image and shared CPU setting', function (int $compressionCpuPercentage) {+ InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));+ $backup = new ScheduledDatabaseBackup(['timeout' => 3600]);+ $job = new DatabaseBackupJob($backup);+ $server = new Server;+ $server->setRelation('settings', new ServerSetting([+ 'backup_compression_cpu_percentage' => $compressionCpuPercentage,+ ]));+ $job->server = $server;++ $command = (new ReflectionClass($job))+ ->getMethod('buildCompressedDumpCommand')+ ->invoke($job, 'docker exec database pg_dumpall');++ expect($command)+ ->toStartWith('docker exec database pg_dumpall | docker run --rm -i')+ ->toContain('coolify-helper')+ ->toContain('command -v pigz')+ ->toContain('pigz -3 -p')+ ->toContain("\$(nproc) * {$compressionCpuPercentage} + 99")+ ->toContain('gzip -3');+})->with([+ 'low' => 25,+ 'high' => 75,+]);++test('all dump all database commands use shared helper compression', function () {+ $source = file_get_contents(app_path('Jobs/DatabaseBackupJob.php'));++ expect($source)+ ->toContain('$this->buildCompressedDumpCommand($backupCommand)')+ ->toContain('mysqldump -u root')+ ->toContain('mariadb-dump -u root')+ ->and(substr_count($source, '$this->buildCompressedDumpCommand($dumpCommand)'))->toBe(2)+ ->and($source)->not->toContain('| gzip >');+});diff --git a/tests/Unit/BackupCompressionTest.php b/tests/Unit/BackupCompressionTest.phpnew file mode 100644index 0000000000..19348248d1--- /dev/null+++ b/tests/Unit/BackupCompressionTest.php@@ -0,0 +1,25 @@+<?php++use App\Support\BackupCompression;++it('normalizes supported backup compression CPU percentages', function (mixed $configuredPercentage, int $expected) {+ expect(BackupCompression::cpuPercentage($configuredPercentage))->toBe($expected);+})->with([+ 'null defaults to low' => [null, 25],+ 'unsupported value defaults to low' => [30, 25],+ 'numeric string is accepted' => ['50', 50],+ 'low' => [25, 25],+ 'medium' => [50, 50],+ 'high' => [75, 75],+ 'maximum' => [100, 100],+]);++it('builds the shared pigz command with gzip fallback', function () {+ $command = BackupCompression::compressorCommand(75);++ expect($command)+ ->toContain('command -v pigz')+ ->toContain('pigz -3 -p')+ ->toContain('$(nproc) * 75 + 99')+ ->toContain('gzip -3');+});