An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
feat(backups): stream S3-only volume archives directly (#11642)
Andras Bacsai6293cd418c48
diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.phpindex b567a71b7f..0b13c66609 100644--- a/app/Jobs/VolumeBackupJob.php+++ b/app/Jobs/VolumeBackupJob.php@@ -73,6 +73,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue$filename = str($this->backup->targetType())->lower().'-'.str($this->backup->targetName())->slug().'-'.Carbon::now()->timestamp.'.tar.gz';$backupLocation = $backupDirectory.'/'.$filename;$this->execution->update(['filename' => $backupLocation]);+ $streamToS3 = $this->backup->save_s3 && $this->backup->disable_local_backup;try {$source = $this->backup->sourcePath();@@ -86,11 +87,17 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue$compressorCommand = BackupCompression::compressorCommand($compressionCpuPercentage);$archiveScript = "compressor=\$({$compressorCommand}); tar -I \"\$compressor\" -cf - -C /volume .";- $archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName)- .' -v '.escapeshellarg($source.':/volume:ro')- .' '.escapeshellarg($image)- .' sh -c '.escapeshellarg($archiveScript)- .' > '.escapeshellarg($backupLocation);+ if ($streamToS3) {+ $this->execution->update(['local_storage_deleted' => true]);+ $archiveCommand = $this->streamToS3Command($archiveScript, $backupLocation, $source, $containerName, $image);+ $this->execution->update(['s3_cleanup_pending' => true]);+ } else {+ $archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName)+ .' -v '.escapeshellarg($source.':/volume:ro')+ .' '.escapeshellarg($image)+ .' sh -c '.escapeshellarg($archiveScript)+ .' > '.escapeshellarg($backupLocation);+ }if ($this->backup->stop_during_backup) {$containers = $this->containersUsingVolume($source, $server);@@ -104,21 +111,23 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue}}- instant_remote_process([+ $archiveOutput = instant_remote_process(array_filter([$verifySourceCommand,- 'mkdir -p '.escapeshellarg($backupDirectory),+ $streamToS3 ? null : 'mkdir -p '.escapeshellarg($backupDirectory),$archiveCommand,- ], $server, timeout: $this->timeout, disableMultiplexing: true);+ ]), $server, timeout: $this->timeout, disableMultiplexing: true);$this->execution->update(['stop_container_ids' => null,'stop_recovery_pending' => false,]);- $size = (int) instant_remote_process(- ['du -b '.escapeshellarg($backupLocation).' | cut -f1'],- $server,- disableMultiplexing: true,- );+ $size = $streamToS3+ ? (int) str($archiveOutput)->trim()->afterLast("\n")->toString()+ : (int) instant_remote_process(+ ['du -b '.escapeshellarg($backupLocation).' | cut -f1'],+ $server,+ disableMultiplexing: true,+ );if ($size <= 0) {throw new \RuntimeException('The storage backup archive is empty or was not created.');@@ -127,9 +136,12 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue$warning = null;$s3Uploaded = null;$s3CleanupPending = false;- $localStorageDeleted = false;+ $localStorageDeleted = $streamToS3;- if ($this->backup->save_s3) {+ if ($streamToS3) {+ $s3Uploaded = true;+ $this->execution->update(['s3_cleanup_pending' => false]);+ } elseif ($this->backup->save_s3) {$s3CleanupPending = true;$this->execution->update(['s3_cleanup_pending' => true]);@@ -181,13 +193,23 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue}} catch (Throwable $exception) {$recoveryError = $this->recoverIncompleteBackup($this->execution);- $archiveDeleted = false;+ $archiveDeleted = $streamToS3;++ if ($streamToS3) {+ $exception = new \RuntimeException(+ 'S3-only streaming backup failed: '.$exception->getMessage()+ .'. The S3 destination may not support streaming uploads. Enable local backups to use the local archive upload method.',+ previous: $exception,+ );+ }- try {- deleteBackupsLocally($backupLocation, $server, throwError: true);- $archiveDeleted = true;- } catch (Throwable $cleanupException) {- $recoveryError .= ' Archive cleanup failed: '.$cleanupException->getMessage();+ if (! $streamToS3) {+ try {+ deleteBackupsLocally($backupLocation, $server, throwError: true);+ $archiveDeleted = true;+ } catch (Throwable $cleanupException) {+ $recoveryError .= ' Archive cleanup failed: '.$cleanupException->getMessage();+ }}$s3CleanupPending = $this->execution->fresh()->s3_cleanup_pending;@@ -195,7 +217,9 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue$this->execution->update(['status' => 'failed','message' => $exception->getMessage().$recoveryError,- 'filename' => $archiveDeleted && ! $s3CleanupPending ? null : $backupLocation,+ 'filename' => $streamToS3+ ? ($s3CleanupPending ? $backupLocation : null)+ : ($archiveDeleted && ! $s3CleanupPending ? null : $backupLocation),'local_storage_deleted' => $archiveDeleted,]);@@ -338,6 +362,34 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue}}+ private function streamToS3Command(string $archiveScript, string $backupLocation, string $source, string $containerName, string $image): string+ {+ $s3 = $this->backup->s3;++ if (! $s3) {+ $this->backup->update(['save_s3' => false, 's3_storage_id' => null]);++ throw new \RuntimeException('The selected S3 storage no longer exists. S3 backup has been disabled.');+ }++ $s3->testConnection(shouldSave: true);+ $resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($s3->endpoint, $s3->trustedInternalHosts()))+ ->map(fn (string $option): string => '--resolve '.escapeshellarg($option))+ ->implode(' ');+ $resolveOptions = $resolveOptions === '' ? '' : ' '.$resolveOptions;+ $destination = 'temporary/'.$s3->bucket.$backupLocation;+ $streamScript = 'set -o pipefail; mc alias set'.$resolveOptions.' temporary '+ .escapeshellarg($s3->endpoint).' '.escapeshellarg($s3->key).' '.escapeshellarg($s3->secret)+ .' >/dev/null && ('.$archiveScript.' | mc pipe --quiet'.$resolveOptions.' '.escapeshellarg($destination).' >/dev/null)'+ .' && mc stat --json'.$resolveOptions.' '.escapeshellarg($destination)+ .' | sed -n '.escapeshellarg('s/.*"size":\([0-9][0-9]*\).*/\1/p');++ return 'docker run --rm --name '.escapeshellarg($containerName)+ .' -v '.escapeshellarg($source.':/volume:ro')+ .' '.escapeshellarg($image)+ .' sh -c '.escapeshellarg($streamScript);+ }+private function logCompressorInDevelopment(string $image, Server $server, int $compressionCpuPercentage): void{if (! isDev()) {diff --git a/tests/Feature/VolumeBackupTest.php b/tests/Feature/VolumeBackupTest.phpindex 9c6897632a..36a04cecaa 100644--- a/tests/Feature/VolumeBackupTest.php+++ b/tests/Feature/VolumeBackupTest.php@@ -28,6 +28,7 @@ use App\Models\StandaloneDocker;use App\Models\Team;use App\Models\User;use Illuminate\Database\Eloquent\Collection;+use Illuminate\Filesystem\FilesystemAdapter;use Illuminate\Foundation\Testing\RefreshDatabase;use Illuminate\Http\RedirectResponse;use Illuminate\Queue\Middleware\WithoutOverlapping;@@ -1902,6 +1903,152 @@ it('keeps the upload destination on the volume backup execution', function () {->and($execution->s3->is($s3Storage))->toBeTrue();});+it('streams S3-only volume backups without creating or copying a local archive', function () {+ config(['broadcasting.default' => 'null']);+ defined('CURLOPT_RESOLVE') || define('CURLOPT_RESOLVE', 10203);+ Carbon::setTestNow('2026-08-15 12:00:00');+ InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));+ $team = Team::factory()->create();+ [$application, $volume] = createVolumeBackupApplication($team);+ $s3Storage = S3Storage::create([+ 'name' => 'Streaming destination',+ 'region' => 'us-east-1',+ 'key' => 'key',+ 'secret' => 'secret',+ 'bucket' => 'bucket',+ 'endpoint' => 'https://s3.amazonaws.com',+ 'team_id' => $team->id,+ 'is_usable' => true,+ ]);+ $backup = $volume->scheduledBackups()->create([+ 'team_id' => $team->id,+ 'frequency' => 'daily',+ 'save_s3' => true,+ 'disable_local_backup' => true,+ 's3_storage_id' => $s3Storage->id,+ ]);+ $sshDisk = Storage::fake('ssh-keys');+ $disk = Mockery::mock(FilesystemAdapter::class);+ $disk->shouldReceive('files')->once()->andReturn([]);+ $disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue();+ Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk);+ Storage::shouldReceive('build')->once()->andReturn($disk);+ Process::fake([+ '*mc pipe*' => "Added `temporary` successfully.\n128 bytes -> `temporary/bucket/archive.tar.gz`\n128",+ '*' => '',+ ]);++ (new VolumeBackupJob($backup))->handle();++ $execution = ScheduledVolumeBackupExecution::query()->sole();+ $expectedFilename = 'volume-app-data-1786795200.tar.gz';+ Process::assertRan(fn ($process) => str_contains($process->command, 'tar -I')+ && str_contains($process->command, 'mc pipe')+ && str_contains($process->command, '>/dev/null && (compressor=')+ && str_contains($process->command, '>/dev/null) && mc stat')+ && str_contains($process->command, 'temporary/bucket/')+ && str_contains($process->command, $expectedFilename)+ && substr_count($process->command, '--resolve') === 3+ && substr_count($process->command, '>/dev/null') >= 2+ && ! str_contains($process->command, ' > ')+ && ! str_contains($process->command, 'mc cp'));+ expect($execution->status)->toBe('success')+ ->and($execution->size)->toBe(128)+ ->and($execution->s3_uploaded)->toBeTrue()+ ->and($execution->local_storage_deleted)->toBeTrue();+});++it('fails an unsupported S3 stream without falling back to a local archive', function () {+ config(['broadcasting.default' => 'null']);+ InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));+ $team = Team::factory()->create();+ [$application, $volume] = createVolumeBackupApplication($team);+ $s3Storage = S3Storage::create([+ 'name' => 'Unsupported streaming destination',+ 'region' => 'us-east-1',+ 'key' => 'key',+ 'secret' => 'secret',+ 'bucket' => 'bucket',+ 'endpoint' => 'https://s3.amazonaws.com',+ 'team_id' => $team->id,+ 'is_usable' => true,+ ]);+ $backup = $volume->scheduledBackups()->create([+ 'team_id' => $team->id,+ 'frequency' => 'daily',+ 'save_s3' => true,+ 'disable_local_backup' => true,+ 's3_storage_id' => $s3Storage->id,+ ]);+ $sshDisk = Storage::fake('ssh-keys');+ $disk = Mockery::mock(FilesystemAdapter::class);+ $disk->shouldReceive('files')->zeroOrMoreTimes()->andReturn([]);+ $disk->shouldReceive('delete')->once()->andReturnTrue();+ Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk);+ Storage::shouldReceive('build')->zeroOrMoreTimes()->andReturn($disk);+ Process::fake([+ '*mc pipe*' => Process::result(errorOutput: 'streaming upload is unsupported', exitCode: 1),+ '*' => '',+ ]);++ expect(fn () => (new VolumeBackupJob($backup))->handle())+ ->toThrow(RuntimeException::class, 'Enable local backups to use the local archive upload method.');++ $execution = ScheduledVolumeBackupExecution::query()->sole();+ expect($execution->status)->toBe('failed')+ ->and($execution->message)->toContain('The S3 destination may not support streaming uploads.')+ ->and($execution->message)->toContain('Enable local backups to use the local archive upload method.')+ ->and($execution->filename)->toBeNull()+ ->and($execution->local_storage_deleted)->toBeTrue();+ Process::assertRan(fn ($process) => str_contains($process->command, 'mc pipe')+ && ! str_contains($process->command, 'mc cp')+ && ! str_contains($process->command, ' > '));+ Process::assertNotRan(fn ($process) => str_contains($process->command, 'mkdir -p')+ || (str_contains($process->command, 'rm -f') && str_contains($process->command, '.tar.gz')));+});++it('keeps local-first archive creation and mc copy when retaining a local volume backup', function () {+ config(['broadcasting.default' => 'null']);+ InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));+ $team = Team::factory()->create();+ [$application, $volume] = createVolumeBackupApplication($team);+ $s3Storage = S3Storage::create([+ 'name' => 'Copied destination',+ 'region' => 'us-east-1',+ 'key' => 'key',+ 'secret' => 'secret',+ 'bucket' => 'bucket',+ 'endpoint' => 'https://s3.amazonaws.com',+ 'team_id' => $team->id,+ 'is_usable' => true,+ ]);+ $backup = $volume->scheduledBackups()->create([+ 'team_id' => $team->id,+ 'frequency' => 'daily',+ 'save_s3' => true,+ 'disable_local_backup' => false,+ 's3_storage_id' => $s3Storage->id,+ ]);+ $sshDisk = Storage::fake('ssh-keys');+ $disk = Mockery::mock(FilesystemAdapter::class);+ $disk->shouldReceive('files')->once()->andReturn([]);+ $disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue();+ Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk);+ Storage::shouldReceive('build')->once()->andReturn($disk);+ Process::fake([+ '*du -b*' => '128',+ '*' => '',+ ]);++ (new VolumeBackupJob($backup))->handle();++ Process::assertRan(fn ($process) => str_contains($process->command, 'tar -I')+ && str_contains($process->command, '>')+ && ! str_contains($process->command, 'mc pipe'));+ Process::assertRan(fn ($process) => str_contains($process->command, 'mc cp')+ && ! str_contains($process->command, 'mc pipe'));+});+it('removes local volume backups older than the configured retention days', function () {config(['broadcasting.default' => 'null']);InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));