An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(volume-backups): back off failed container and S3 recovery
Andras Bacsaic3b32322eb82
diff --git a/app/Jobs/ScheduledJobManager.php b/app/Jobs/ScheduledJobManager.phpindex d3af90a996..e1411412fb 100644--- a/app/Jobs/ScheduledJobManager.php+++ b/app/Jobs/ScheduledJobManager.php@@ -38,6 +38,11 @@ class ScheduledJobManager implements ShouldQueue*/private const VOLUME_BACKUP_RECOVERY_INTERVAL_MINUTES = 5;+ /**+ * Bounds the volume backup recovery jobs dispatched in one scheduler run.+ */+ private const VOLUME_BACKUP_RECOVERY_MAX_DISPATCHES = 100;+/*** The schedule types that run as separate dispatchers, with the delivery job types they own.* Each type has its own overlap lock, so a slow type cannot make another type skip a run.@@ -485,16 +490,29 @@ class ScheduledJobManager implements ShouldQueueprivate function recoverStoppedVolumeBackupContainers(): void{+ $dispatched = 0;+ScheduledVolumeBackupExecution::query()->where(fn (Builder $query) => $query->where('stop_recovery_pending', true)->orWhere('s3_cleanup_pending', true))- ->chunkById(self::CHUNK_SIZE, function ($executions): void {+ ->where('recovery_needs_attention', false)+ ->where(fn (Builder $query) => $query+ ->whereNull('recovery_next_retry_at')+ ->orWhere('recovery_next_retry_at', '<=', now()))+ ->chunkById(self::CHUNK_SIZE, function ($executions) use (&$dispatched): bool {foreach ($executions as $execution) {- if (Cache::add("volume-backup-recovery-dispatched:{$execution->id}", true, now()->addMinutes(self::VOLUME_BACKUP_RECOVERY_INTERVAL_MINUTES))) {+ if ($dispatched >= self::VOLUME_BACKUP_RECOVERY_MAX_DISPATCHES) {+ return false;+ }++ if (Cache::add(VolumeBackupRecoveryJob::dispatchCacheKey($execution->id), true, now()->addMinutes(self::VOLUME_BACKUP_RECOVERY_INTERVAL_MINUTES))) {VolumeBackupRecoveryJob::dispatch($execution);+ $dispatched++;}}++ return true;});}@@ -503,13 +521,9 @@ class ScheduledJobManager implements ShouldQueuetry {$server = $backup->server();- if ($backup->executions()- ->where(fn (Builder $query) => $query- ->where('stop_recovery_pending', true)- ->orWhere('s3_cleanup_pending', true))- ->exists()) {+ if ($backup->executions()->where('stop_recovery_pending', true)->exists()) {$this->skippedCount++;- $this->logSkip('volume_backup', 'container_recovery_pending', [+ $this->logSkip('volume_backup', 'stopped_container_recovery_pending', ['backup_id' => $backup->id,'team_id' => $backup->team_id,]);diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.phpindex 78f091bbe2..f142a65941 100644--- a/app/Jobs/VolumeBackupJob.php+++ b/app/Jobs/VolumeBackupJob.php@@ -488,6 +488,8 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue$this->backup->executions()->where('local_storage_deleted', true)+ ->where('stop_recovery_pending', false)+ ->where('s3_cleanup_pending', false)->where(function (Builder $query): void {$query->where('s3_storage_deleted', true)->orWhereNull('s3_uploaded');})diff --git a/app/Jobs/VolumeBackupRecoveryJob.php b/app/Jobs/VolumeBackupRecoveryJob.phpindex c136c45820..7f475d24ce 100644--- a/app/Jobs/VolumeBackupRecoveryJob.php+++ b/app/Jobs/VolumeBackupRecoveryJob.php@@ -3,6 +3,7 @@namespace App\Jobs;use App\Models\ScheduledVolumeBackupExecution;+use Aws\S3\Exception\S3Exception;use Illuminate\Bus\Queueable;use Illuminate\Contracts\Queue\ShouldBeEncrypted;use Illuminate\Contracts\Queue\ShouldBeUnique;@@ -12,13 +13,15 @@ use Illuminate\Queue\InteractsWithQueue;use Illuminate\Queue\Middleware\WithoutOverlapping;use Illuminate\Queue\SerializesModels;use Illuminate\Support\Facades\Log;+use RuntimeException;+use Throwable;class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue{use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;/**- * ScheduledJobManager dispatches pending recovery again, so this job does not retry by itself.+ * ScheduledJobManager dispatches pending recovery again after a backoff, so this job does not retry by itself.*/public int $tries = 1;@@ -26,6 +29,34 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, Shoupublic int $uniqueFor = 300;+ public const MAX_ATTEMPTS = 8;++ public const BACKOFF_BASE_MINUTES = 5;++ public const BACKOFF_CAP_MINUTES = 360;++ public const CATEGORY_S3_AUTH = 's3_auth';++ public const CATEGORY_S3_BUCKET = 's3_bucket';++ public const CATEGORY_SERVER_UNREACHABLE = 'server_unreachable';++ public const CATEGORY_REMOTE_COMMAND = 'remote_command';++ public const CATEGORY_UNKNOWN = 'unknown';++ private const UNREACHABLE_PATTERNS = [+ 'ssh connection failed',+ 'connection refused',+ 'connection timed out',+ 'operation timed out',+ 'no route to host',+ 'network is unreachable',+ 'could not resolve hostname',+ 'permission denied (publickey',+ 'host key verification failed',+ ];+public function __construct(public ScheduledVolumeBackupExecution $execution){$this->onQueue(crons_queue());@@ -51,20 +82,173 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, Shoupublic function handle(): void{- self::recover($this->execution);+ self::recoverWithBounds($this->execution);}+ /**+ * Runs one recovery attempt and records its outcome instead of throwing, so a persistent failure backs off+ * and eventually needs attention rather than creating a failed job every few minutes.+ */+ public static function recoverWithBounds(ScheduledVolumeBackupExecution $execution): void+ {+ $current = $execution->fresh();+ if (! $current) {+ return;+ }++ $execution->setRawAttributes($current->getAttributes(), true);++ if (! $execution->hasPendingRecovery() || $execution->recovery_needs_attention) {+ return;+ }++ try {+ $failure = self::runRecoverySteps($execution);+ } catch (Throwable $exception) {+ $failure = ['category' => self::CATEGORY_UNKNOWN, 'exception' => $exception];+ }++ if ($failure) {+ self::recordFailure($execution, $failure['category']);++ return;+ }++ self::recordSuccess($execution);+ }++ /**+ * Recovers stopped containers first, then cleans the partial S3 upload. Each step runs even when the other one+ * fails; the first failure is rethrown after both steps ran.+ */public static function recover(ScheduledVolumeBackupExecution $execution): void+ {+ $failure = self::runRecoverySteps($execution);++ if ($failure) {+ throw $failure['exception'];+ }+ }++ /**+ * @return array{category: string, exception: Throwable}|null The first failure, or null when every step succeeded.+ */+ private static function runRecoverySteps(ScheduledVolumeBackupExecution $execution): ?array{$execution->loadMissing('scheduledVolumeBackup.backupable.resource');+ $failure = null;if ($execution->stop_recovery_pending) {- self::recoverContainers($execution);+ try {+ self::recoverContainers($execution);+ } catch (Throwable $exception) {+ $failure = ['category' => self::categorizeContainerFailure($exception), 'exception' => $exception];+ }}if ($execution->s3_cleanup_pending) {- self::cleanupS3Upload($execution);+ try {+ self::cleanupS3Upload($execution);+ } catch (Throwable $exception) {+ $failure ??= ['category' => self::categorizeS3Failure($exception), 'exception' => $exception];+ }+ }++ return $failure;+ }++ /**+ * Returns the minutes to wait before the next automatic attempt after the given number of failed attempts.+ */+ public static function backoffMinutes(int $attempts): int+ {+ $exponent = max(0, min($attempts - 1, 16));++ return (int) min(self::BACKOFF_BASE_MINUTES * (2 ** $exponent), self::BACKOFF_CAP_MINUTES);+ }++ public static function dispatchCacheKey(int $executionId): string+ {+ return "volume-backup-recovery-dispatched:{$executionId}";+ }++ private static function recordSuccess(ScheduledVolumeBackupExecution $execution): void+ {+ $execution->update([+ 'recovery_attempts' => 0,+ 'recovery_last_attempt_at' => null,+ 'recovery_next_retry_at' => null,+ 'recovery_error' => null,+ 'recovery_needs_attention' => false,+ ]);+ }++ private static function recordFailure(ScheduledVolumeBackupExecution $execution, string $category): void+ {+ $recorded = $execution->getConnection()->transaction(function () use ($execution, $category): ?ScheduledVolumeBackupExecution {+ $current = ScheduledVolumeBackupExecution::query()->whereKey($execution->id)->lockForUpdate()->first();+ if (! $current) {+ return null;+ }++ $attempts = $current->recovery_attempts + 1;+ $needsAttention = $attempts >= self::MAX_ATTEMPTS;+ $current->update([+ 'recovery_attempts' => $attempts,+ 'recovery_last_attempt_at' => now(),+ 'recovery_next_retry_at' => $needsAttention ? null : now()->addMinutes(self::backoffMinutes($attempts)),+ 'recovery_error' => $category,+ 'recovery_needs_attention' => $needsAttention,+ ]);++ return $current;+ });++ if (! $recorded) {+ return;+ }++ $execution->setRawAttributes($recorded->getAttributes(), true);++ Log::channel('scheduled-errors')->warning('Volume backup recovery failed', [+ 'execution_id' => $recorded->id,+ 'execution_uuid' => $recorded->uuid,+ 'category' => $category,+ 'attempts' => $recorded->recovery_attempts,+ 'needs_attention' => $recorded->recovery_needs_attention,+ 'next_retry_at' => $recorded->recovery_next_retry_at?->toIso8601String(),+ ]);+ }++ private static function categorizeS3Failure(Throwable $exception): string+ {+ for ($current = $exception; $current; $current = $current->getPrevious()) {+ if ($current instanceof S3Exception) {+ return match ($current->getAwsErrorCode()) {+ 'InvalidAccessKeyId', 'SignatureDoesNotMatch', 'AccessDenied' => self::CATEGORY_S3_AUTH,+ 'NoSuchBucket' => self::CATEGORY_S3_BUCKET,+ default => self::CATEGORY_UNKNOWN,+ };+ }}++ return self::CATEGORY_UNKNOWN;+ }++ private static function categorizeContainerFailure(Throwable $exception): string+ {+ if ($exception->getCode() === 255) {+ return self::CATEGORY_SERVER_UNREACHABLE;+ }++ $message = strtolower($exception->getMessage());+ foreach (self::UNREACHABLE_PATTERNS as $pattern) {+ if (str_contains($message, $pattern)) {+ return self::CATEGORY_SERVER_UNREACHABLE;+ }+ }++ return $exception instanceof RuntimeException ? self::CATEGORY_REMOTE_COMMAND : self::CATEGORY_UNKNOWN;}private static function recoverContainers(ScheduledVolumeBackupExecution $execution): voiddiff --git a/app/Livewire/Project/Shared/Storages/VolumeBackups.php b/app/Livewire/Project/Shared/Storages/VolumeBackups.phpindex b4ae3ead3a..0299c98c8b 100644--- a/app/Livewire/Project/Shared/Storages/VolumeBackups.php+++ b/app/Livewire/Project/Shared/Storages/VolumeBackups.php@@ -4,6 +4,7 @@ namespace App\Livewire\Project\Shared\Storages;use App\Actions\Shared\DeleteScheduledVolumeBackup;use App\Jobs\VolumeBackupJob;+use App\Jobs\VolumeBackupRecoveryJob;use App\Models\LocalFileVolume;use App\Models\LocalPersistentVolume;use App\Models\S3Storage;@@ -13,6 +14,7 @@ use Illuminate\Foundation\Auth\Access\AuthorizesRequests;use Illuminate\Http\RedirectResponse;use Illuminate\Routing\Redirector;use Illuminate\Support\Collection;+use Illuminate\Support\Facades\Cache;use Livewire\Component;use Livewire\WithPagination;use Throwable;@@ -278,6 +280,8 @@ class VolumeBackups extends Component$deletedCount = $this->backup?->executions()->where('local_storage_deleted', true)+ ->where('stop_recovery_pending', false)+ ->where('s3_cleanup_pending', false)->where(fn ($query) => $query->where('s3_storage_deleted', true)->orWhereNull('s3_uploaded')@@ -290,6 +294,33 @@ class VolumeBackups extends Component);}+ public function retryRecovery(int $executionId): void+ {+ $this->authorize('update', $this->resource);++ $execution = $this->backup?->executions()->whereKey($executionId)->first();+ if (! $execution || ! $execution->hasPendingRecovery()) {+ $this->dispatch('error', 'No pending recovery found for this backup execution.');++ return;+ }++ $execution->update([+ 'recovery_attempts' => 0,+ 'recovery_next_retry_at' => null,+ 'recovery_needs_attention' => false,+ ]);+ Cache::forget(VolumeBackupRecoveryJob::dispatchCacheKey($execution->id));+ VolumeBackupRecoveryJob::dispatch($execution);+ auditLog('ui.volume_backup.recovery_retried', [+ 'team_id' => $this->resource->team()?->id,+ 'resource_uuid' => $this->resource->uuid,+ 'backup_uuid' => $this->backup->uuid,+ 'execution_uuid' => $execution->uuid,+ ]);+ $this->dispatch('success', 'Backup recovery queued.');+ }+public function deleteBackup(int $executionId, string $password, array $selectedActions = []): bool|string{$this->authorize('update', $this->resource);diff --git a/app/Models/ScheduledVolumeBackupExecution.php b/app/Models/ScheduledVolumeBackupExecution.phpindex 5287344467..eec799218e 100644--- a/app/Models/ScheduledVolumeBackupExecution.php+++ b/app/Models/ScheduledVolumeBackupExecution.php@@ -21,6 +21,11 @@ class ScheduledVolumeBackupExecution extends BaseModel'local_storage_deleted','s3_storage_deleted','s3_uploaded',+ 'recovery_attempts',+ 'recovery_last_attempt_at',+ 'recovery_next_retry_at',+ 'recovery_error',+ 'recovery_needs_attention',];protected function casts(): array@@ -34,9 +39,18 @@ class ScheduledVolumeBackupExecution extends BaseModel'local_storage_deleted' => 'boolean','s3_storage_deleted' => 'boolean','s3_uploaded' => 'boolean',+ 'recovery_attempts' => 'integer',+ 'recovery_last_attempt_at' => 'datetime',+ 'recovery_next_retry_at' => 'datetime',+ 'recovery_needs_attention' => 'boolean',];}+ public function hasPendingRecovery(): bool+ {+ return $this->stop_recovery_pending || $this->s3_cleanup_pending;+ }+public function scheduledVolumeBackup(): BelongsTo{return $this->belongsTo(ScheduledVolumeBackup::class);diff --git a/bootstrap/helpers/databases.php b/bootstrap/helpers/databases.phpindex a4407c064a..f4a97be5ce 100644--- a/bootstrap/helpers/databases.php+++ b/bootstrap/helpers/databases.php@@ -282,9 +282,16 @@ function deleteBackupsS3(string|array|null $filenames, S3Storage $s3): void'endpoint' => $s3->endpoint,'use_path_style_endpoint' => true,'aws_url' => $s3->awsUrl(),+ 'throw' => true,]);- if (! $disk->delete($filenames)) {+ try {+ $deleted = $disk->delete($filenames);+ } catch (Throwable $exception) {+ throw new RuntimeException('One or more S3 backup files could not be deleted.', previous: $exception);+ }++ if (! $deleted) {throw new RuntimeException('One or more S3 backup files could not be deleted.');}}diff --git a/database/migrations/2026_10_03_172204_add_recovery_bounds_to_scheduled_volume_backup_executions_table.php b/database/migrations/2026_10_03_172204_add_recovery_bounds_to_scheduled_volume_backup_executions_table.phpnew file mode 100644index 0000000000..f10f334912--- /dev/null+++ b/database/migrations/2026_10_03_172204_add_recovery_bounds_to_scheduled_volume_backup_executions_table.php@@ -0,0 +1,44 @@+<?php++use Illuminate\Database\Migrations\Migration;+use Illuminate\Database\Schema\Blueprint;+use Illuminate\Support\Facades\Schema;++return new class extends Migration+{+ /**+ * Run the migrations.+ */+ public function up(): void+ {+ Schema::table('scheduled_volume_backup_executions', function (Blueprint $table) {+ $table->unsignedSmallInteger('recovery_attempts')->default(0);+ $table->timestampTz('recovery_last_attempt_at')->nullable();+ $table->timestampTz('recovery_next_retry_at')->nullable();+ $table->string('recovery_error', 32)->nullable();+ $table->boolean('recovery_needs_attention')->default(false);++ $table->index(+ ['stop_recovery_pending', 's3_cleanup_pending', 'recovery_needs_attention', 'recovery_next_retry_at'],+ 'scheduled_volume_executions_recovery_index',+ );+ });+ }++ /**+ * Reverse the migrations.+ */+ public function down(): void+ {+ Schema::table('scheduled_volume_backup_executions', function (Blueprint $table) {+ $table->dropIndex('scheduled_volume_executions_recovery_index');+ $table->dropColumn([+ 'recovery_attempts',+ 'recovery_last_attempt_at',+ 'recovery_next_retry_at',+ 'recovery_error',+ 'recovery_needs_attention',+ ]);+ });+ }+};diff --git a/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php b/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.phpindex b6dd4ffd4d..b4bf4465ec 100644--- a/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php+++ b/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php@@ -62,6 +62,14 @@if (empty($deleteActions)) {$deleteActions[] = 'This backup execution record will be deleted.';}+ $recoveryPending = $execution->status !== 'running' && $execution->hasPendingRecovery();+ $recoveryErrorLabel = match ($execution->recovery_error) {+ 's3_auth' => 'S3 credentials',+ 's3_bucket' => 'S3 bucket missing',+ 'server_unreachable' => 'server unreachable',+ 'remote_command' => 'remote command failed',+ default => 'unknown error',+ };@endphp<div wire:key="volume-backup-execution-{{ $execution->id }}"@@ -99,6 +107,14 @@? 'neutral': ($execution->s3_uploaded ? 'success' : 'error')" />@endif+ @if ($recoveryPending)+ @if ($execution->recovery_needs_attention)+ <x-status-badge :status="'Needs attention ('.$recoveryErrorLabel.')'" type="error" />+ @else+ <x-status-badge status="Recovery pending" type="warning"+ :title="$execution->recovery_error ? 'Last attempt: '.$recoveryErrorLabel : null" />+ @endif+ @endif</span><span class="flex items-center justify-end gap-1">@@ -109,6 +125,14 @@<x-reicon name="upload" class="size-3.5 rotate-180" /></button>@endif+ @if ($recoveryPending)+ <button type="button" class="icon-button shrink-0"+ wire:click="retryRecovery({{ $execution->id }})"+ wire:loading.attr="disabled" wire:target="retryRecovery({{ $execution->id }})"+ title="Retry recovery" aria-label="Retry recovery">+ <x-reicon name="refresh" class="size-3.5" />+ </button>+ @endif@if ($execution->status !== 'running')<x-modal-confirmation title="Confirm Backup Deletion?" isErrorButtonsubmitAction="deleteBackup({{ $execution->id }})"diff --git a/tests/Feature/VolumeBackupRecoveryBoundsTest.php b/tests/Feature/VolumeBackupRecoveryBoundsTest.phpnew file mode 100644index 0000000000..f65c17ae63--- /dev/null+++ b/tests/Feature/VolumeBackupRecoveryBoundsTest.php@@ -0,0 +1,384 @@+<?php++use App\Jobs\ScheduledJobManager;+use App\Jobs\VolumeBackupJob;+use App\Jobs\VolumeBackupRecoveryJob;+use App\Livewire\Project\Shared\Storages\VolumeBackups;+use App\Models\Application;+use App\Models\Environment;+use App\Models\InstanceSettings;+use App\Models\LocalPersistentVolume;+use App\Models\PrivateKey;+use App\Models\Project;+use App\Models\S3Storage;+use App\Models\ScheduledVolumeBackup;+use App\Models\ScheduledVolumeBackupExecution;+use App\Models\Server;+use App\Models\StandaloneDocker;+use App\Models\Team;+use App\Models\User;+use Aws\Command;+use Aws\S3\Exception\S3Exception;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Carbon;+use Illuminate\Support\Facades\Cache;+use Illuminate\Support\Facades\Log;+use Illuminate\Support\Facades\Process;+use Illuminate\Support\Facades\Queue;+use Illuminate\Support\Facades\Storage;+use League\Flysystem\UnableToDeleteFile;+use Livewire\Livewire;++uses(RefreshDatabase::class);++beforeEach(function () {+ Server::flushIdentityMap();+ config(['broadcasting.default' => 'null']);+ InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));+});++afterEach(function () {+ Carbon::setTestNow();+});++/**+ * @return array{0: Application, 1: LocalPersistentVolume, 2: Server, 3: ScheduledVolumeBackup, 4: S3Storage}+ */+function createRecoveryBoundsBackup(Team $team, array $backupAttributes = []): array+{+ $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);+ $server = Server::factory()->create([+ 'team_id' => $team->id,+ 'private_key_id' => $privateKey->id,+ 'ip' => '203.0.113.20',+ ]);+ $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();+ $project = Project::factory()->create(['team_id' => $team->id]);+ $environment = Environment::factory()->create(['project_id' => $project->id]);+ $application = Application::factory()->create([+ 'environment_id' => $environment->id,+ 'destination_id' => $destination->id,+ 'destination_type' => $destination->getMorphClass(),+ ]);+ $volume = LocalPersistentVolume::create([+ 'name' => 'app-data',+ 'mount_path' => '/data',+ 'resource_id' => $application->id,+ 'resource_type' => $application->getMorphClass(),+ ]);+ $s3 = S3Storage::create([+ 'name' => 'Volume backups',+ 'region' => 'us-east-1',+ 'key' => 'AKIARECOVERYKEY',+ 'secret' => 'recovery-secret',+ 'bucket' => 'bucket',+ 'endpoint' => 'https://s3.example.com',+ 'team_id' => $team->id,+ 'is_usable' => true,+ ]);+ $backup = $volume->scheduledBackups()->create([+ 'team_id' => $team->id,+ 's3_storage_id' => $s3->id,+ 'frequency' => 'daily',+ 'save_s3' => true,+ ...$backupAttributes,+ ]);++ return [$application, $volume, $server, $backup, $s3];+}++function createRecoveryBoundsExecution(ScheduledVolumeBackup $backup, array $attributes = []): ScheduledVolumeBackupExecution+{+ return ScheduledVolumeBackupExecution::create([+ 'scheduled_volume_backup_id' => $backup->id,+ 's3_storage_id' => $backup->s3_storage_id,+ 'status' => 'failed',+ 'filename' => '/data/coolify/backups/volumes/test/partial.tar.gz',+ 's3_cleanup_pending' => true,+ ...$attributes,+ ]);+}++function failRecoveryBoundsS3Delete(string $awsCode): void+{+ $s3Exception = new S3Exception(+ 'The AWS Access Key Id AKIARECOVERYKEY you provided does not exist in our records.',+ new Command('DeleteObject'),+ ['code' => $awsCode],+ );+ $disk = Mockery::mock();+ $disk->shouldReceive('delete')->andThrow(UnableToDeleteFile::atLocation('partial.tar.gz', '', $s3Exception));+ Storage::shouldReceive('build')->andReturn($disk);+}++function signInForRecoveryBounds($testCase, Team $team): User+{+ $user = User::factory()->create();+ $user->teams()->attach($team, ['role' => 'owner']);+ $testCase->actingAs($user);+ session(['currentTeam' => $team]);++ return $user;+}++function makeRecoveryBoundsServerFunctional(Server $server): void+{+ $server->settings()->update([+ 'is_reachable' => true,+ 'is_usable' => true,+ 'force_disabled' => false,+ ]);+}++it('doubles the recovery backoff and caps it at six hours', function () {+ expect(collect(range(1, 9))->map(fn (int $attempts) => VolumeBackupRecoveryJob::backoffMinutes($attempts))->all())+ ->toBe([5, 10, 20, 40, 80, 160, 320, 360, 360]);+});++it('records a categorized S3 cleanup failure without throwing or leaking the S3 message', function () {+ Carbon::setTestNow('2026-10-03 12:00:00');+ [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create());+ $execution = createRecoveryBoundsExecution($backup, ['message' => 'Worker timed out']);+ failRecoveryBoundsS3Delete('InvalidAccessKeyId');+ $logger = Mockery::spy();+ Log::shouldReceive('channel')->with('scheduled-errors')->andReturn($logger);++ (new VolumeBackupRecoveryJob($execution))->handle();++ $execution->refresh();+ expect($execution->s3_cleanup_pending)->toBeTrue()+ ->and($execution->s3_storage_deleted)->toBeFalse()+ ->and($execution->recovery_attempts)->toBe(1)+ ->and($execution->recovery_error)->toBe('s3_auth')+ ->and($execution->recovery_last_attempt_at->equalTo(now()))->toBeTrue()+ ->and($execution->recovery_next_retry_at->equalTo(now()->addMinutes(5)))->toBeTrue()+ ->and($execution->recovery_needs_attention)->toBeFalse()+ ->and($execution->message)->toBe('Worker timed out');+ $logger->shouldHaveReceived('warning')->once()->with(+ 'Volume backup recovery failed',+ Mockery::on(fn (array $context): bool => $context['execution_uuid'] === $execution->uuid+ && $context['category'] === 's3_auth'+ && ! str_contains(json_encode($context), 'AKIA')),+ );+});++it('categorizes a missing S3 bucket and grows the backoff on each failure', function () {+ Carbon::setTestNow('2026-10-03 12:00:00');+ [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create());+ $execution = createRecoveryBoundsExecution($backup);+ failRecoveryBoundsS3Delete('NoSuchBucket');++ VolumeBackupRecoveryJob::recoverWithBounds($execution);+ VolumeBackupRecoveryJob::recoverWithBounds($execution);++ $execution->refresh();+ expect($execution->recovery_error)->toBe('s3_bucket')+ ->and($execution->recovery_attempts)->toBe(2)+ ->and($execution->recovery_next_retry_at->equalTo(now()->addMinutes(10)))->toBeTrue();+});++it('needs attention after the maximum attempts and stops automatic recovery', function () {+ Carbon::setTestNow('2026-10-03 12:00:00');+ config(['constants.coolify.self_hosted' => true]);+ Queue::fake();+ [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create());+ $execution = createRecoveryBoundsExecution($backup, [+ 'recovery_attempts' => VolumeBackupRecoveryJob::MAX_ATTEMPTS - 1,+ ]);+ failRecoveryBoundsS3Delete('AccessDenied');++ VolumeBackupRecoveryJob::recoverWithBounds($execution);+ VolumeBackupRecoveryJob::recoverWithBounds($execution);++ $execution->refresh();+ expect($execution->recovery_needs_attention)->toBeTrue()+ ->and($execution->recovery_attempts)->toBe(VolumeBackupRecoveryJob::MAX_ATTEMPTS)+ ->and($execution->recovery_next_retry_at)->toBeNull();++ Carbon::setTestNow('2026-10-04 12:00:00');+ (new ScheduledJobManager)->handle();++ Queue::assertNotPushed(VolumeBackupRecoveryJob::class);+});++it('waits for the backoff before dispatching recovery again', function () {+ Carbon::setTestNow('2026-10-03 12:00:00');+ config(['constants.coolify.self_hosted' => true]);+ Queue::fake();+ [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create());+ createRecoveryBoundsExecution($backup, [+ 'recovery_attempts' => 3,+ 'recovery_next_retry_at' => now()->addMinutes(20),+ ]);++ (new ScheduledJobManager)->handle();+ Queue::assertNotPushed(VolumeBackupRecoveryJob::class);++ Carbon::setTestNow('2026-10-03 12:20:00');+ (new ScheduledJobManager)->handle();+ Queue::assertPushed(VolumeBackupRecoveryJob::class, 1);+});++it('cleans the S3 upload even when container recovery fails', function () {+ Carbon::setTestNow('2026-10-03 12:00:00');+ config(['constants.ssh.max_retries' => 1, 'constants.ssh.mux_enabled' => false]);+ Storage::fake('ssh-keys');+ Storage::fake('ssh-mux');+ [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create(), ['stop_during_backup' => true]);+ $execution = createRecoveryBoundsExecution($backup, ['stop_recovery_pending' => true]);+ Process::fake([+ '*' => Process::result(errorOutput: 'ssh: connect to host 203.0.113.20 port 22: Connection refused', exitCode: 255),+ ]);+ $disk = Mockery::mock();+ $disk->shouldReceive('delete')->once()->andReturnTrue();+ $filesystem = Mockery::mock(Storage::getFacadeRoot());+ $filesystem->shouldReceive('build')->once()->andReturn($disk);+ Storage::swap($filesystem);++ VolumeBackupRecoveryJob::recoverWithBounds($execution);++ $execution->refresh();+ expect($execution->stop_recovery_pending)->toBeTrue()+ ->and($execution->s3_cleanup_pending)->toBeFalse()+ ->and($execution->s3_storage_deleted)->toBeTrue()+ ->and($execution->recovery_error)->toBe('server_unreachable')+ ->and($execution->recovery_attempts)->toBe(1);+});++it('resets the recovery state after a successful attempt', function () {+ [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create());+ $execution = createRecoveryBoundsExecution($backup, [+ 'recovery_attempts' => 4,+ 'recovery_error' => 's3_auth',+ 'recovery_last_attempt_at' => now()->subHour(),+ 'recovery_next_retry_at' => now()->subMinute(),+ ]);+ $disk = Mockery::mock();+ $disk->shouldReceive('delete')->once()->andReturnTrue();+ Storage::shouldReceive('build')->once()->andReturn($disk);++ (new VolumeBackupRecoveryJob($execution))->handle();++ $execution->refresh();+ expect($execution->s3_cleanup_pending)->toBeFalse()+ ->and($execution->s3_storage_deleted)->toBeTrue()+ ->and($execution->recovery_attempts)->toBe(0)+ ->and($execution->recovery_error)->toBeNull()+ ->and($execution->recovery_next_retry_at)->toBeNull();+});++it('only blocks new scheduled backups while stopped containers await recovery', function (array $pending, bool $dispatched) {+ Carbon::setTestNow('2026-10-03 12:00:00');+ config(['constants.coolify.self_hosted' => true]);+ Queue::fake();+ [, , $server, $backup] = createRecoveryBoundsBackup(Team::factory()->create(), [+ 'frequency' => '* * * * *',+ 'enabled' => true,+ ]);+ makeRecoveryBoundsServerFunctional($server);+ createRecoveryBoundsExecution($backup, $pending);+ Cache::forget("scheduled-volume-backup:{$backup->id}");++ (new ScheduledJobManager)->handle();++ $dispatched+ ? Queue::assertPushed(VolumeBackupJob::class, fn (VolumeBackupJob $job) => $job->backup->is($backup))+ : Queue::assertNotPushed(VolumeBackupJob::class);+})->with([+ 'only S3 cleanup pending' => [['s3_cleanup_pending' => true, 'recovery_needs_attention' => true], true],+ 'stopped containers pending' => [['stop_recovery_pending' => true, 's3_cleanup_pending' => false], false],+ 'stopped containers need attention' => [['stop_recovery_pending' => true, 'recovery_needs_attention' => true], false],+]);++it('keeps executions with pending recovery during retention cleanup', function () {+ Process::fake([+ '*du -b*' => '128',+ '*' => '',+ ]);+ [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create(), ['save_s3' => false]);+ $pending = createRecoveryBoundsExecution($backup, [+ 'local_storage_deleted' => true,+ 's3_uploaded' => null,+ ]);+ $deletable = createRecoveryBoundsExecution($backup, [+ 'filename' => null,+ 'local_storage_deleted' => true,+ 's3_uploaded' => null,+ 's3_cleanup_pending' => false,+ ]);++ (new VolumeBackupJob($backup->fresh()))->handle();++ expect($pending->fresh())->not->toBeNull()+ ->and($deletable->fresh())->toBeNull();+});++it('keeps executions with pending recovery when cleaning up deleted entries', function () {+ $team = Team::factory()->create();+ signInForRecoveryBounds($this, $team);+ [$application, $volume, , $backup] = createRecoveryBoundsBackup($team);+ $pending = createRecoveryBoundsExecution($backup, [+ 'local_storage_deleted' => true,+ 's3_uploaded' => false,+ ]);+ $deletable = createRecoveryBoundsExecution($backup, [+ 'local_storage_deleted' => true,+ 's3_uploaded' => false,+ 's3_cleanup_pending' => false,+ ]);++ Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application])+ ->call('cleanupDeleted');++ expect($pending->fresh())->not->toBeNull()+ ->and($deletable->fresh())->toBeNull();+});++it('shows the recovery state and retries recovery manually', function () {+ Queue::fake();+ $team = Team::factory()->create();+ signInForRecoveryBounds($this, $team);+ [$application, $volume, , $backup] = createRecoveryBoundsBackup($team);+ $execution = createRecoveryBoundsExecution($backup, [+ 'recovery_attempts' => VolumeBackupRecoveryJob::MAX_ATTEMPTS,+ 'recovery_error' => 's3_auth',+ 'recovery_needs_attention' => true,+ ]);+ Cache::put(VolumeBackupRecoveryJob::dispatchCacheKey($execution->id), true, now()->addMinutes(5));++ Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application, 'section' => 'executions'])+ ->assertSee('Needs attention (S3 credentials)')+ ->assertSee('Retry recovery')+ ->call('retryRecovery', $execution->id)+ ->assertDispatched('success')+ ->assertSee('Recovery pending');++ $execution->refresh();+ expect($execution->recovery_attempts)->toBe(0)+ ->and($execution->recovery_needs_attention)->toBeFalse()+ ->and($execution->recovery_next_retry_at)->toBeNull()+ ->and(Cache::has(VolumeBackupRecoveryJob::dispatchCacheKey($execution->id)))->toBeFalse();+ Queue::assertPushed(VolumeBackupRecoveryJob::class, fn (VolumeBackupRecoveryJob $job) => $job->execution->is($execution));+});++it('prevents another team from retrying recovery', function () {+ Queue::fake();+ [$application, $volume, , $backup] = createRecoveryBoundsBackup(Team::factory()->create());+ $execution = createRecoveryBoundsExecution($backup, [+ 'recovery_attempts' => VolumeBackupRecoveryJob::MAX_ATTEMPTS,+ 'recovery_needs_attention' => true,+ ]);+ $otherTeam = Team::factory()->create();+ signInForRecoveryBounds($this, $otherTeam);+ [$otherApplication, $otherVolume] = createRecoveryBoundsBackup($otherTeam);++ Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application])+ ->assertForbidden();+ Livewire::test(VolumeBackups::class, ['storage' => $otherVolume, 'resource' => $otherApplication])+ ->call('retryRecovery', $execution->id)+ ->assertDispatched('error');++ expect($execution->fresh()->recovery_needs_attention)->toBeTrue()+ ->and($execution->fresh()->recovery_attempts)->toBe(VolumeBackupRecoveryJob::MAX_ATTEMPTS);+ Queue::assertNotPushed(VolumeBackupRecoveryJob::class);+});