Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

feat(backups): stream S3-only volume archives directly (#11642)

Andras Bacsai6293cd418c48
diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.php
index b567a71b7f..0b13c66609 100644
--- a/app/Jobs/VolumeBackupJob.php
+++ b/app/Jobs/VolumeBackupJob.php
@@ -73,6 +73,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
$filename = str($this->backup->targetType())->lower().'-'.str($this->backup->targetName())->slug().'-'.Carbon::now()->timestamp.'.tar.gz';
$backupLocation = $backupDirectory.'/'.$filename;
$this->execution->update(['filename' => $backupLocation]);
+ $streamToS3 = $this->backup->save_s3 && $this->backup->disable_local_backup;
try {
$source = $this->backup->sourcePath();
@@ -86,11 +87,17 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
$compressorCommand = BackupCompression::compressorCommand($compressionCpuPercentage);
$archiveScript = "compressor=\$({$compressorCommand}); tar -I \"\$compressor\" -cf - -C /volume .";
- $archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName)
- .' -v '.escapeshellarg($source.':/volume:ro')
- .' '.escapeshellarg($image)
- .' sh -c '.escapeshellarg($archiveScript)
- .' > '.escapeshellarg($backupLocation);
+ if ($streamToS3) {
+ $this->execution->update(['local_storage_deleted' => true]);
+ $archiveCommand = $this->streamToS3Command($archiveScript, $backupLocation, $source, $containerName, $image);
+ $this->execution->update(['s3_cleanup_pending' => true]);
+ } else {
+ $archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName)
+ .' -v '.escapeshellarg($source.':/volume:ro')
+ .' '.escapeshellarg($image)
+ .' sh -c '.escapeshellarg($archiveScript)
+ .' > '.escapeshellarg($backupLocation);
+ }
if ($this->backup->stop_during_backup) {
$containers = $this->containersUsingVolume($source, $server);
@@ -104,21 +111,23 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
}
}
- instant_remote_process([
+ $archiveOutput = instant_remote_process(array_filter([
$verifySourceCommand,
- 'mkdir -p '.escapeshellarg($backupDirectory),
+ $streamToS3 ? null : 'mkdir -p '.escapeshellarg($backupDirectory),
$archiveCommand,
- ], $server, timeout: $this->timeout, disableMultiplexing: true);
+ ]), $server, timeout: $this->timeout, disableMultiplexing: true);
$this->execution->update([
'stop_container_ids' => null,
'stop_recovery_pending' => false,
]);
- $size = (int) instant_remote_process(
- ['du -b '.escapeshellarg($backupLocation).' | cut -f1'],
- $server,
- disableMultiplexing: true,
- );
+ $size = $streamToS3
+ ? (int) str($archiveOutput)->trim()->afterLast("\n")->toString()
+ : (int) instant_remote_process(
+ ['du -b '.escapeshellarg($backupLocation).' | cut -f1'],
+ $server,
+ disableMultiplexing: true,
+ );
if ($size <= 0) {
throw new \RuntimeException('The storage backup archive is empty or was not created.');
@@ -127,9 +136,12 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
$warning = null;
$s3Uploaded = null;
$s3CleanupPending = false;
- $localStorageDeleted = false;
+ $localStorageDeleted = $streamToS3;
- if ($this->backup->save_s3) {
+ if ($streamToS3) {
+ $s3Uploaded = true;
+ $this->execution->update(['s3_cleanup_pending' => false]);
+ } elseif ($this->backup->save_s3) {
$s3CleanupPending = true;
$this->execution->update(['s3_cleanup_pending' => true]);
@@ -181,13 +193,23 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
}
} catch (Throwable $exception) {
$recoveryError = $this->recoverIncompleteBackup($this->execution);
- $archiveDeleted = false;
+ $archiveDeleted = $streamToS3;
+
+ if ($streamToS3) {
+ $exception = new \RuntimeException(
+ 'S3-only streaming backup failed: '.$exception->getMessage()
+ .'. The S3 destination may not support streaming uploads. Enable local backups to use the local archive upload method.',
+ previous: $exception,
+ );
+ }
- try {
- deleteBackupsLocally($backupLocation, $server, throwError: true);
- $archiveDeleted = true;
- } catch (Throwable $cleanupException) {
- $recoveryError .= ' Archive cleanup failed: '.$cleanupException->getMessage();
+ if (! $streamToS3) {
+ try {
+ deleteBackupsLocally($backupLocation, $server, throwError: true);
+ $archiveDeleted = true;
+ } catch (Throwable $cleanupException) {
+ $recoveryError .= ' Archive cleanup failed: '.$cleanupException->getMessage();
+ }
}
$s3CleanupPending = $this->execution->fresh()->s3_cleanup_pending;
@@ -195,7 +217,9 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
$this->execution->update([
'status' => 'failed',
'message' => $exception->getMessage().$recoveryError,
- 'filename' => $archiveDeleted && ! $s3CleanupPending ? null : $backupLocation,
+ 'filename' => $streamToS3
+ ? ($s3CleanupPending ? $backupLocation : null)
+ : ($archiveDeleted && ! $s3CleanupPending ? null : $backupLocation),
'local_storage_deleted' => $archiveDeleted,
]);
@@ -338,6 +362,34 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
}
}
+ private function streamToS3Command(string $archiveScript, string $backupLocation, string $source, string $containerName, string $image): string
+ {
+ $s3 = $this->backup->s3;
+
+ if (! $s3) {
+ $this->backup->update(['save_s3' => false, 's3_storage_id' => null]);
+
+ throw new \RuntimeException('The selected S3 storage no longer exists. S3 backup has been disabled.');
+ }
+
+ $s3->testConnection(shouldSave: true);
+ $resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($s3->endpoint, $s3->trustedInternalHosts()))
+ ->map(fn (string $option): string => '--resolve '.escapeshellarg($option))
+ ->implode(' ');
+ $resolveOptions = $resolveOptions === '' ? '' : ' '.$resolveOptions;
+ $destination = 'temporary/'.$s3->bucket.$backupLocation;
+ $streamScript = 'set -o pipefail; mc alias set'.$resolveOptions.' temporary '
+ .escapeshellarg($s3->endpoint).' '.escapeshellarg($s3->key).' '.escapeshellarg($s3->secret)
+ .' >/dev/null && ('.$archiveScript.' | mc pipe --quiet'.$resolveOptions.' '.escapeshellarg($destination).' >/dev/null)'
+ .' && mc stat --json'.$resolveOptions.' '.escapeshellarg($destination)
+ .' | sed -n '.escapeshellarg('s/.*"size":\([0-9][0-9]*\).*/\1/p');
+
+ return 'docker run --rm --name '.escapeshellarg($containerName)
+ .' -v '.escapeshellarg($source.':/volume:ro')
+ .' '.escapeshellarg($image)
+ .' sh -c '.escapeshellarg($streamScript);
+ }
+
private function logCompressorInDevelopment(string $image, Server $server, int $compressionCpuPercentage): void
{
if (! isDev()) {
diff --git a/tests/Feature/VolumeBackupTest.php b/tests/Feature/VolumeBackupTest.php
index 9c6897632a..36a04cecaa 100644
--- a/tests/Feature/VolumeBackupTest.php
+++ b/tests/Feature/VolumeBackupTest.php
@@ -28,6 +28,7 @@ use App\Models\StandaloneDocker;
use App\Models\Team;
use App\Models\User;
use Illuminate\Database\Eloquent\Collection;
+use Illuminate\Filesystem\FilesystemAdapter;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Http\RedirectResponse;
use Illuminate\Queue\Middleware\WithoutOverlapping;
@@ -1902,6 +1903,152 @@ it('keeps the upload destination on the volume backup execution', function () {
->and($execution->s3->is($s3Storage))->toBeTrue();
});
+it('streams S3-only volume backups without creating or copying a local archive', function () {
+ config(['broadcasting.default' => 'null']);
+ defined('CURLOPT_RESOLVE') || define('CURLOPT_RESOLVE', 10203);
+ Carbon::setTestNow('2026-08-15 12:00:00');
+ InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
+ $team = Team::factory()->create();
+ [$application, $volume] = createVolumeBackupApplication($team);
+ $s3Storage = S3Storage::create([
+ 'name' => 'Streaming destination',
+ 'region' => 'us-east-1',
+ 'key' => 'key',
+ 'secret' => 'secret',
+ 'bucket' => 'bucket',
+ 'endpoint' => 'https://s3.amazonaws.com',
+ 'team_id' => $team->id,
+ 'is_usable' => true,
+ ]);
+ $backup = $volume->scheduledBackups()->create([
+ 'team_id' => $team->id,
+ 'frequency' => 'daily',
+ 'save_s3' => true,
+ 'disable_local_backup' => true,
+ 's3_storage_id' => $s3Storage->id,
+ ]);
+ $sshDisk = Storage::fake('ssh-keys');
+ $disk = Mockery::mock(FilesystemAdapter::class);
+ $disk->shouldReceive('files')->once()->andReturn([]);
+ $disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue();
+ Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk);
+ Storage::shouldReceive('build')->once()->andReturn($disk);
+ Process::fake([
+ '*mc pipe*' => "Added `temporary` successfully.\n128 bytes -> `temporary/bucket/archive.tar.gz`\n128",
+ '*' => '',
+ ]);
+
+ (new VolumeBackupJob($backup))->handle();
+
+ $execution = ScheduledVolumeBackupExecution::query()->sole();
+ $expectedFilename = 'volume-app-data-1786795200.tar.gz';
+ Process::assertRan(fn ($process) => str_contains($process->command, 'tar -I')
+ && str_contains($process->command, 'mc pipe')
+ && str_contains($process->command, '>/dev/null && (compressor=')
+ && str_contains($process->command, '>/dev/null) && mc stat')
+ && str_contains($process->command, 'temporary/bucket/')
+ && str_contains($process->command, $expectedFilename)
+ && substr_count($process->command, '--resolve') === 3
+ && substr_count($process->command, '>/dev/null') >= 2
+ && ! str_contains($process->command, ' > ')
+ && ! str_contains($process->command, 'mc cp'));
+ expect($execution->status)->toBe('success')
+ ->and($execution->size)->toBe(128)
+ ->and($execution->s3_uploaded)->toBeTrue()
+ ->and($execution->local_storage_deleted)->toBeTrue();
+});
+
+it('fails an unsupported S3 stream without falling back to a local archive', function () {
+ config(['broadcasting.default' => 'null']);
+ InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
+ $team = Team::factory()->create();
+ [$application, $volume] = createVolumeBackupApplication($team);
+ $s3Storage = S3Storage::create([
+ 'name' => 'Unsupported streaming destination',
+ 'region' => 'us-east-1',
+ 'key' => 'key',
+ 'secret' => 'secret',
+ 'bucket' => 'bucket',
+ 'endpoint' => 'https://s3.amazonaws.com',
+ 'team_id' => $team->id,
+ 'is_usable' => true,
+ ]);
+ $backup = $volume->scheduledBackups()->create([
+ 'team_id' => $team->id,
+ 'frequency' => 'daily',
+ 'save_s3' => true,
+ 'disable_local_backup' => true,
+ 's3_storage_id' => $s3Storage->id,
+ ]);
+ $sshDisk = Storage::fake('ssh-keys');
+ $disk = Mockery::mock(FilesystemAdapter::class);
+ $disk->shouldReceive('files')->zeroOrMoreTimes()->andReturn([]);
+ $disk->shouldReceive('delete')->once()->andReturnTrue();
+ Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk);
+ Storage::shouldReceive('build')->zeroOrMoreTimes()->andReturn($disk);
+ Process::fake([
+ '*mc pipe*' => Process::result(errorOutput: 'streaming upload is unsupported', exitCode: 1),
+ '*' => '',
+ ]);
+
+ expect(fn () => (new VolumeBackupJob($backup))->handle())
+ ->toThrow(RuntimeException::class, 'Enable local backups to use the local archive upload method.');
+
+ $execution = ScheduledVolumeBackupExecution::query()->sole();
+ expect($execution->status)->toBe('failed')
+ ->and($execution->message)->toContain('The S3 destination may not support streaming uploads.')
+ ->and($execution->message)->toContain('Enable local backups to use the local archive upload method.')
+ ->and($execution->filename)->toBeNull()
+ ->and($execution->local_storage_deleted)->toBeTrue();
+ Process::assertRan(fn ($process) => str_contains($process->command, 'mc pipe')
+ && ! str_contains($process->command, 'mc cp')
+ && ! str_contains($process->command, ' > '));
+ Process::assertNotRan(fn ($process) => str_contains($process->command, 'mkdir -p')
+ || (str_contains($process->command, 'rm -f') && str_contains($process->command, '.tar.gz')));
+});
+
+it('keeps local-first archive creation and mc copy when retaining a local volume backup', function () {
+ config(['broadcasting.default' => 'null']);
+ InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
+ $team = Team::factory()->create();
+ [$application, $volume] = createVolumeBackupApplication($team);
+ $s3Storage = S3Storage::create([
+ 'name' => 'Copied destination',
+ 'region' => 'us-east-1',
+ 'key' => 'key',
+ 'secret' => 'secret',
+ 'bucket' => 'bucket',
+ 'endpoint' => 'https://s3.amazonaws.com',
+ 'team_id' => $team->id,
+ 'is_usable' => true,
+ ]);
+ $backup = $volume->scheduledBackups()->create([
+ 'team_id' => $team->id,
+ 'frequency' => 'daily',
+ 'save_s3' => true,
+ 'disable_local_backup' => false,
+ 's3_storage_id' => $s3Storage->id,
+ ]);
+ $sshDisk = Storage::fake('ssh-keys');
+ $disk = Mockery::mock(FilesystemAdapter::class);
+ $disk->shouldReceive('files')->once()->andReturn([]);
+ $disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue();
+ Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk);
+ Storage::shouldReceive('build')->once()->andReturn($disk);
+ Process::fake([
+ '*du -b*' => '128',
+ '*' => '',
+ ]);
+
+ (new VolumeBackupJob($backup))->handle();
+
+ Process::assertRan(fn ($process) => str_contains($process->command, 'tar -I')
+ && str_contains($process->command, '>')
+ && ! str_contains($process->command, 'mc pipe'));
+ Process::assertRan(fn ($process) => str_contains($process->command, 'mc cp')
+ && ! str_contains($process->command, 'mc pipe'));
+});
+
it('removes local volume backups older than the configured retention days', function () {
config(['broadcasting.default' => 'null']);
InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));