An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(backups): stop volume backup recovery retry storms
Andras Bacsaif9d10c22fbcf
diff --git a/app/Jobs/ScheduledJobManager.php b/app/Jobs/ScheduledJobManager.phpindex 0ec09c361f..c5d553d850 100644--- a/app/Jobs/ScheduledJobManager.php+++ b/app/Jobs/ScheduledJobManager.php@@ -33,6 +33,11 @@ class ScheduledJobManager implements ShouldQueueprivate const CHUNK_SIZE = 100;+ /**+ * Pending volume backup recovery is retried at most once in this interval for each execution.+ */+ private const VOLUME_BACKUP_RECOVERY_INTERVAL_MINUTES = 5;+/*** The schedule types that run as separate dispatchers, with the delivery job types they own.* Each type has its own overlap lock, so a slow type cannot make another type skip a run.@@ -481,7 +486,9 @@ class ScheduledJobManager implements ShouldQueue->orWhere('s3_cleanup_pending', true))->chunkById(self::CHUNK_SIZE, function ($executions): void {foreach ($executions as $execution) {- VolumeBackupRecoveryJob::dispatch($execution);+ if (Cache::add("volume-backup-recovery-dispatched:{$execution->id}", true, now()->addMinutes(self::VOLUME_BACKUP_RECOVERY_INTERVAL_MINUTES))) {+ VolumeBackupRecoveryJob::dispatch($execution);+ }}});}diff --git a/app/Jobs/VolumeBackupRecoveryJob.php b/app/Jobs/VolumeBackupRecoveryJob.phpindex a3465b192e..c136c45820 100644--- a/app/Jobs/VolumeBackupRecoveryJob.php+++ b/app/Jobs/VolumeBackupRecoveryJob.php@@ -5,27 +5,37 @@ namespace App\Jobs;use App\Models\ScheduledVolumeBackupExecution;use Illuminate\Bus\Queueable;use Illuminate\Contracts\Queue\ShouldBeEncrypted;+use Illuminate\Contracts\Queue\ShouldBeUnique;use Illuminate\Contracts\Queue\ShouldQueue;use Illuminate\Foundation\Bus\Dispatchable;use Illuminate\Queue\InteractsWithQueue;use Illuminate\Queue\Middleware\WithoutOverlapping;use Illuminate\Queue\SerializesModels;+use Illuminate\Support\Facades\Log;-class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldQueue+class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue{use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;- public int $tries = 20;-- public int $backoff = 60;+ /**+ * ScheduledJobManager dispatches pending recovery again, so this job does not retry by itself.+ */+ public int $tries = 1;public int $timeout = 120;+ public int $uniqueFor = 300;+public function __construct(public ScheduledVolumeBackupExecution $execution){$this->onQueue(crons_queue());}+ public function uniqueId(): string+ {+ return (string) $this->execution->id;+ }+public function middleware(): array{return [@@ -62,7 +72,13 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldQueue$server = $execution->scheduledVolumeBackup?->server();if (! $server) {- throw new \RuntimeException('The server is unavailable for container recovery.');+ self::skipRecovery(+ $execution,+ ['stop_container_ids' => null, 'stop_recovery_pending' => false],+ 'Container recovery was skipped because the server or resource no longer exists.',+ );++ return;}$stateFile = self::stateFile($execution);@@ -100,7 +116,13 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldQueue$s3 = $execution->s3;if (! $s3 || blank($execution->filename)) {- throw new \RuntimeException('The S3 storage or backup filename is unavailable for upload cleanup.');+ self::skipRecovery(+ $execution,+ ['s3_cleanup_pending' => false],+ 'S3 upload cleanup was skipped because the S3 storage or backup filename no longer exists.',+ );++ return;}deleteBackupsS3($execution->filename, $s3);@@ -110,6 +132,21 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldQueue]);}+ /**+ * Clears a pending recovery that cannot succeed, so it does not retry forever and block new backups.+ *+ * @param array<string, mixed> $attributes+ */+ private static function skipRecovery(ScheduledVolumeBackupExecution $execution, array $attributes, string $reason): void+ {+ Log::channel('scheduled-errors')->warning($reason, ['execution_id' => $execution->id]);++ $execution->update([+ ...$attributes,+ 'message' => trim(($execution->message ?? '').' '.$reason),+ ]);+ }+public static function stateFile(ScheduledVolumeBackupExecution $execution): string{return '/tmp/coolify-volume-backup-'.$execution->uuid.'.stopped';diff --git a/tests/Feature/VolumeBackupTest.php b/tests/Feature/VolumeBackupTest.phpindex 623e140dfb..1f733f72d4 100644--- a/tests/Feature/VolumeBackupTest.php+++ b/tests/Feature/VolumeBackupTest.php@@ -2614,3 +2614,82 @@ it('dispatches pending recovery without starting another volume backup', functio);Queue::assertNotPushed(VolumeBackupJob::class);});++it('dispatches pending recovery for an execution at most once every five minutes', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow('2026-07-15 12:00:00');+ Queue::fake();+ $team = Team::factory()->create();+ [$application, $volume] = createVolumeBackupApplication($team);+ $backup = $volume->scheduledBackups()->create([+ 'team_id' => $team->id,+ 'frequency' => 'daily',+ 'enabled' => true,+ ]);+ ScheduledVolumeBackupExecution::create([+ 'scheduled_volume_backup_id' => $backup->id,+ 'status' => 'failed',+ 'stop_recovery_pending' => true,+ ]);++ (new ScheduledJobManager)->handle();+ Carbon::setTestNow('2026-07-15 12:04:00');+ (new ScheduledJobManager)->handle();++ Queue::assertPushed(VolumeBackupRecoveryJob::class, 1);++ Carbon::setTestNow('2026-07-15 12:05:01');+ (new ScheduledJobManager)->handle();++ Queue::assertPushed(VolumeBackupRecoveryJob::class, 2);+});++it('stops container recovery when the server no longer exists', function () {+ Process::fake();+ $team = Team::factory()->create();+ [$application, $volume] = createVolumeBackupApplication($team);+ $backup = $volume->scheduledBackups()->create([+ 'team_id' => $team->id,+ 'frequency' => 'daily',+ ]);+ $execution = ScheduledVolumeBackupExecution::create([+ 'scheduled_volume_backup_id' => $backup->id,+ 'status' => 'failed',+ 'message' => 'Worker timed out',+ 'stop_recovery_pending' => true,+ ]);+ $application->update([+ 'destination_id' => null,+ 'destination_type' => null,+ ]);++ (new VolumeBackupRecoveryJob($execution))->handle();++ expect($execution->fresh()->stop_recovery_pending)->toBeFalse()+ ->and($execution->fresh()->message)->toStartWith('Worker timed out')+ ->and($execution->fresh()->message)->toContain('Container recovery was skipped');+ Process::assertNothingRan();+});++it('stops S3 upload cleanup when the S3 storage no longer exists', function () {+ $team = Team::factory()->create();+ [$application, $volume] = createVolumeBackupApplication($team);+ $backup = $volume->scheduledBackups()->create([+ 'team_id' => $team->id,+ 'frequency' => 'daily',+ ]);+ $execution = ScheduledVolumeBackupExecution::create([+ 'scheduled_volume_backup_id' => $backup->id,+ 's3_storage_id' => null,+ 'status' => 'failed',+ 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz',+ 's3_cleanup_pending' => true,+ ]);+ Storage::shouldReceive('build')->never();++ (new VolumeBackupRecoveryJob($execution))->handle();++ expect($execution->fresh()->s3_cleanup_pending)->toBeFalse()+ ->and($execution->fresh()->s3_storage_deleted)->toBeFalsy()+ ->and($execution->fresh()->message)->toContain('S3 upload cleanup was skipped');+});