Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(backups): stop volume backup recovery retry storms

Andras Bacsaif9d10c22fbcf
diff --git a/app/Jobs/ScheduledJobManager.php b/app/Jobs/ScheduledJobManager.php
index 0ec09c361f..c5d553d850 100644
--- a/app/Jobs/ScheduledJobManager.php
+++ b/app/Jobs/ScheduledJobManager.php
@@ -33,6 +33,11 @@ class ScheduledJobManager implements ShouldQueue
private const CHUNK_SIZE = 100;
+ /**
+ * Pending volume backup recovery is retried at most once in this interval for each execution.
+ */
+ private const VOLUME_BACKUP_RECOVERY_INTERVAL_MINUTES = 5;
+
/**
* The schedule types that run as separate dispatchers, with the delivery job types they own.
* Each type has its own overlap lock, so a slow type cannot make another type skip a run.
@@ -481,7 +486,9 @@ class ScheduledJobManager implements ShouldQueue
->orWhere('s3_cleanup_pending', true))
->chunkById(self::CHUNK_SIZE, function ($executions): void {
foreach ($executions as $execution) {
- VolumeBackupRecoveryJob::dispatch($execution);
+ if (Cache::add("volume-backup-recovery-dispatched:{$execution->id}", true, now()->addMinutes(self::VOLUME_BACKUP_RECOVERY_INTERVAL_MINUTES))) {
+ VolumeBackupRecoveryJob::dispatch($execution);
+ }
}
});
}
diff --git a/app/Jobs/VolumeBackupRecoveryJob.php b/app/Jobs/VolumeBackupRecoveryJob.php
index a3465b192e..c136c45820 100644
--- a/app/Jobs/VolumeBackupRecoveryJob.php
+++ b/app/Jobs/VolumeBackupRecoveryJob.php
@@ -5,27 +5,37 @@ namespace App\Jobs;
use App\Models\ScheduledVolumeBackupExecution;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
-class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldQueue
+class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
- public int $tries = 20;
-
- public int $backoff = 60;
+ /**
+ * ScheduledJobManager dispatches pending recovery again, so this job does not retry by itself.
+ */
+ public int $tries = 1;
public int $timeout = 120;
+ public int $uniqueFor = 300;
+
public function __construct(public ScheduledVolumeBackupExecution $execution)
{
$this->onQueue(crons_queue());
}
+ public function uniqueId(): string
+ {
+ return (string) $this->execution->id;
+ }
+
public function middleware(): array
{
return [
@@ -62,7 +72,13 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldQueue
$server = $execution->scheduledVolumeBackup?->server();
if (! $server) {
- throw new \RuntimeException('The server is unavailable for container recovery.');
+ self::skipRecovery(
+ $execution,
+ ['stop_container_ids' => null, 'stop_recovery_pending' => false],
+ 'Container recovery was skipped because the server or resource no longer exists.',
+ );
+
+ return;
}
$stateFile = self::stateFile($execution);
@@ -100,7 +116,13 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldQueue
$s3 = $execution->s3;
if (! $s3 || blank($execution->filename)) {
- throw new \RuntimeException('The S3 storage or backup filename is unavailable for upload cleanup.');
+ self::skipRecovery(
+ $execution,
+ ['s3_cleanup_pending' => false],
+ 'S3 upload cleanup was skipped because the S3 storage or backup filename no longer exists.',
+ );
+
+ return;
}
deleteBackupsS3($execution->filename, $s3);
@@ -110,6 +132,21 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldQueue
]);
}
+ /**
+ * Clears a pending recovery that cannot succeed, so it does not retry forever and block new backups.
+ *
+ * @param array<string, mixed> $attributes
+ */
+ private static function skipRecovery(ScheduledVolumeBackupExecution $execution, array $attributes, string $reason): void
+ {
+ Log::channel('scheduled-errors')->warning($reason, ['execution_id' => $execution->id]);
+
+ $execution->update([
+ ...$attributes,
+ 'message' => trim(($execution->message ?? '').' '.$reason),
+ ]);
+ }
+
public static function stateFile(ScheduledVolumeBackupExecution $execution): string
{
return '/tmp/coolify-volume-backup-'.$execution->uuid.'.stopped';
diff --git a/tests/Feature/VolumeBackupTest.php b/tests/Feature/VolumeBackupTest.php
index 623e140dfb..1f733f72d4 100644
--- a/tests/Feature/VolumeBackupTest.php
+++ b/tests/Feature/VolumeBackupTest.php
@@ -2614,3 +2614,82 @@ it('dispatches pending recovery without starting another volume backup', functio
);
Queue::assertNotPushed(VolumeBackupJob::class);
});
+
+it('dispatches pending recovery for an execution at most once every five minutes', function () {
+ config(['constants.coolify.self_hosted' => true]);
+ Carbon::setTestNow('2026-07-15 12:00:00');
+ Queue::fake();
+ $team = Team::factory()->create();
+ [$application, $volume] = createVolumeBackupApplication($team);
+ $backup = $volume->scheduledBackups()->create([
+ 'team_id' => $team->id,
+ 'frequency' => 'daily',
+ 'enabled' => true,
+ ]);
+ ScheduledVolumeBackupExecution::create([
+ 'scheduled_volume_backup_id' => $backup->id,
+ 'status' => 'failed',
+ 'stop_recovery_pending' => true,
+ ]);
+
+ (new ScheduledJobManager)->handle();
+ Carbon::setTestNow('2026-07-15 12:04:00');
+ (new ScheduledJobManager)->handle();
+
+ Queue::assertPushed(VolumeBackupRecoveryJob::class, 1);
+
+ Carbon::setTestNow('2026-07-15 12:05:01');
+ (new ScheduledJobManager)->handle();
+
+ Queue::assertPushed(VolumeBackupRecoveryJob::class, 2);
+});
+
+it('stops container recovery when the server no longer exists', function () {
+ Process::fake();
+ $team = Team::factory()->create();
+ [$application, $volume] = createVolumeBackupApplication($team);
+ $backup = $volume->scheduledBackups()->create([
+ 'team_id' => $team->id,
+ 'frequency' => 'daily',
+ ]);
+ $execution = ScheduledVolumeBackupExecution::create([
+ 'scheduled_volume_backup_id' => $backup->id,
+ 'status' => 'failed',
+ 'message' => 'Worker timed out',
+ 'stop_recovery_pending' => true,
+ ]);
+ $application->update([
+ 'destination_id' => null,
+ 'destination_type' => null,
+ ]);
+
+ (new VolumeBackupRecoveryJob($execution))->handle();
+
+ expect($execution->fresh()->stop_recovery_pending)->toBeFalse()
+ ->and($execution->fresh()->message)->toStartWith('Worker timed out')
+ ->and($execution->fresh()->message)->toContain('Container recovery was skipped');
+ Process::assertNothingRan();
+});
+
+it('stops S3 upload cleanup when the S3 storage no longer exists', function () {
+ $team = Team::factory()->create();
+ [$application, $volume] = createVolumeBackupApplication($team);
+ $backup = $volume->scheduledBackups()->create([
+ 'team_id' => $team->id,
+ 'frequency' => 'daily',
+ ]);
+ $execution = ScheduledVolumeBackupExecution::create([
+ 'scheduled_volume_backup_id' => $backup->id,
+ 's3_storage_id' => null,
+ 'status' => 'failed',
+ 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz',
+ 's3_cleanup_pending' => true,
+ ]);
+ Storage::shouldReceive('build')->never();
+
+ (new VolumeBackupRecoveryJob($execution))->handle();
+
+ expect($execution->fresh()->s3_cleanup_pending)->toBeFalse()
+ ->and($execution->fresh()->s3_storage_deleted)->toBeFalsy()
+ ->and($execution->fresh()->message)->toContain('S3 upload cleanup was skipped');
+});