Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

feat(backups): use parallel gzip for volume backups

Andras Bacsaib3aa1fd2327f
diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.php
index 0b85fd2556..232f3aa0de 100644
--- a/app/Jobs/VolumeBackupJob.php
+++ b/app/Jobs/VolumeBackupJob.php
@@ -77,14 +77,19 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
$source = $this->backup->sourcePath();
$containerName = 'volume-backup-'.$this->execution->uuid;
$image = coolifyHelperImage().':'.getHelperVersion();
+ $this->logCompressorInDevelopment($image, $server);
$verifySourceCommand = $target instanceof LocalPersistentVolume && blank($target->host_path)
? 'docker volume inspect '.escapeshellarg($source).' >/dev/null'
: 'test -d '.escapeshellarg($source);
+ $archiveScript = "compressor='gzip -3'; "
+ .'if command -v pigz >/dev/null 2>&1; then compressor="pigz -3 -p $(( ($(nproc) + 1) / 2 ))"; fi; '
+ .'tar -I "$compressor" -cf - -C /volume .';
$archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName)
.' -v '.escapeshellarg($source.':/volume:ro')
.' '.escapeshellarg($image)
- ." tar -I 'gzip -1' -cf - -C /volume . > ".escapeshellarg($backupLocation);
+ .' sh -c '.escapeshellarg($archiveScript)
+ .' > '.escapeshellarg($backupLocation);
if ($this->backup->stop_during_backup) {
$containers = $this->containersUsingVolume($source, $server);
@@ -332,6 +337,28 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
}
}
+ private function logCompressorInDevelopment(string $image, Server $server): void
+ {
+ if (! isDev()) {
+ return;
+ }
+
+ $script = "if command -v pigz >/dev/null 2>&1; then printf 'pigz -3 -p %s' \"$(( ($(nproc) + 1) / 2 ))\"; else printf 'gzip -3'; fi";
+ $compressor = instant_remote_process(
+ ['docker run --rm '.escapeshellarg($image).' sh -c '.escapeshellarg($script)],
+ $server,
+ timeout: 60,
+ disableMultiplexing: true,
+ );
+
+ Log::info('Volume backup compressor selected', [
+ 'backup_id' => $this->backup->id,
+ 'execution_id' => $this->execution?->id,
+ 'compressor' => $compressor,
+ 'helper_image' => $image,
+ ]);
+ }
+
private function removeExpiredBackups(Server $server): void
{
if ($this->hasRetentionLimits(
diff --git a/config/constants.php b/config/constants.php
index 8496f527c6..733a6c183d 100644
--- a/config/constants.php
+++ b/config/constants.php
@@ -3,7 +3,7 @@
return [
'coolify' => [
'version' => env('COOLIFY_VERSION') ?: '4.3.3',
- 'helper_version' => '1.0.14',
+ 'helper_version' => '1.0.15',
'realtime_version' => '1.0.17',
'railpack_version' => '0.23.0',
'self_hosted' => env('SELF_HOSTED', true),
diff --git a/docker/coolify-helper/Dockerfile b/docker/coolify-helper/Dockerfile
index 6bea6ba1bb..567cfbeebe 100644
--- a/docker/coolify-helper/Dockerfile
+++ b/docker/coolify-helper/Dockerfile
@@ -36,7 +36,7 @@ USER root
WORKDIR /artifacts
ENV RAILPACK_VERSION=${RAILPACK_VERSION}
RUN apk upgrade --no-cache && \
- apk add --no-cache bash curl git git-lfs openssh-client tar tini
+ apk add --no-cache bash curl git git-lfs openssh-client pigz tar tini
RUN mkdir -p ~/.docker/cli-plugins
# Install mise (musl build) at the path railpack expects (/tmp/railpack/mise/mise-VERSION).
diff --git a/tests/Feature/VolumeBackupTest.php b/tests/Feature/VolumeBackupTest.php
index 7522538242..bcd0e1ff66 100644
--- a/tests/Feature/VolumeBackupTest.php
+++ b/tests/Feature/VolumeBackupTest.php
@@ -31,6 +31,7 @@ use Illuminate\Routing\Redirector;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\Queue;
use Illuminate\Support\Facades\Route;
@@ -50,6 +51,12 @@ it('provides the volume backup domain classes and relationship', function () {
->and(method_exists(LocalFileVolume::class, 'scheduledBackups'))->toBeTrue();
});
+it('includes parallel gzip support in the Coolify helper image', function () {
+ $dockerfile = file_get_contents(base_path('docker/coolify-helper/Dockerfile'));
+
+ expect($dockerfile)->toContain('pigz');
+});
+
it('keeps the volume backup script inside the Livewire root element', function () {
$view = file_get_contents(resource_path('views/livewire/project/shared/storages/volume-backups.blade.php'));
@@ -1599,12 +1606,51 @@ it('archives a named volume on its server', function () {
Process::assertRan(fn ($process) => str_contains($process->command, 'docker volume inspect')
&& str_contains($process->command, 'docker run --rm --name ')
&& str_contains($process->command, 'app-data:/volume:ro')
- && str_contains($process->command, "tar -I 'gzip -1' -cf -")
+ && str_contains($process->command, 'command -v pigz')
+ && str_contains($process->command, 'pigz -3 -p')
+ && str_contains($process->command, '$(nproc) + 1')
+ && str_contains($process->command, 'gzip -3')
+ && str_contains($process->command, 'tar -I "$compressor" -cf -')
&& str_contains($process->command, '> ')
&& str_contains($process->command, '.tar.gz')
&& ! str_contains($process->command, ':/backup'));
});
+it('logs the selected volume backup compressor in development', function (string $detectedCompressor) {
+ config(['app.env' => 'local', 'broadcasting.default' => 'null']);
+ InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
+ $team = Team::factory()->create();
+ [$application, $volume] = createVolumeBackupApplication($team);
+ $backup = $volume->scheduledBackups()->create([
+ 'team_id' => $team->id,
+ 'frequency' => 'daily',
+ 'retention_amount_locally' => 7,
+ 'retention_days_locally' => 0,
+ 'retention_max_storage_locally' => 0,
+ 'retention_amount_s3' => 7,
+ 'retention_days_s3' => 0,
+ 'retention_max_storage_s3' => 0,
+ ]);
+
+ Process::fake([
+ '*command -v pigz*' => $detectedCompressor,
+ '*du -b*' => '128',
+ '*' => '',
+ ]);
+ Log::spy();
+
+ (new VolumeBackupJob($backup))->handle();
+
+ Log::shouldHaveReceived('info')->once()->with(
+ 'Volume backup compressor selected',
+ Mockery::on(fn (array $context): bool => $context['compressor'] === $detectedCompressor
+ && $context['backup_id'] === $backup->id),
+ );
+})->with([
+ 'pigz' => 'pigz -3 -p 4',
+ 'gzip fallback' => 'gzip -3',
+]);
+
it('keeps the upload destination on the volume backup execution', function () {
config(['broadcasting.default' => 'null']);
InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));