Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(volume-backups): back off failed container and S3 recovery

Andras Bacsaic3b32322eb82
diff --git a/app/Jobs/ScheduledJobManager.php b/app/Jobs/ScheduledJobManager.php
index d3af90a996..e1411412fb 100644
--- a/app/Jobs/ScheduledJobManager.php
+++ b/app/Jobs/ScheduledJobManager.php
@@ -38,6 +38,11 @@ class ScheduledJobManager implements ShouldQueue
*/
private const VOLUME_BACKUP_RECOVERY_INTERVAL_MINUTES = 5;
+ /**
+ * Bounds the volume backup recovery jobs dispatched in one scheduler run.
+ */
+ private const VOLUME_BACKUP_RECOVERY_MAX_DISPATCHES = 100;
+
/**
* The schedule types that run as separate dispatchers, with the delivery job types they own.
* Each type has its own overlap lock, so a slow type cannot make another type skip a run.
@@ -485,16 +490,29 @@ class ScheduledJobManager implements ShouldQueue
private function recoverStoppedVolumeBackupContainers(): void
{
+ $dispatched = 0;
+
ScheduledVolumeBackupExecution::query()
->where(fn (Builder $query) => $query
->where('stop_recovery_pending', true)
->orWhere('s3_cleanup_pending', true))
- ->chunkById(self::CHUNK_SIZE, function ($executions): void {
+ ->where('recovery_needs_attention', false)
+ ->where(fn (Builder $query) => $query
+ ->whereNull('recovery_next_retry_at')
+ ->orWhere('recovery_next_retry_at', '<=', now()))
+ ->chunkById(self::CHUNK_SIZE, function ($executions) use (&$dispatched): bool {
foreach ($executions as $execution) {
- if (Cache::add("volume-backup-recovery-dispatched:{$execution->id}", true, now()->addMinutes(self::VOLUME_BACKUP_RECOVERY_INTERVAL_MINUTES))) {
+ if ($dispatched >= self::VOLUME_BACKUP_RECOVERY_MAX_DISPATCHES) {
+ return false;
+ }
+
+ if (Cache::add(VolumeBackupRecoveryJob::dispatchCacheKey($execution->id), true, now()->addMinutes(self::VOLUME_BACKUP_RECOVERY_INTERVAL_MINUTES))) {
VolumeBackupRecoveryJob::dispatch($execution);
+ $dispatched++;
}
}
+
+ return true;
});
}
@@ -503,13 +521,9 @@ class ScheduledJobManager implements ShouldQueue
try {
$server = $backup->server();
- if ($backup->executions()
- ->where(fn (Builder $query) => $query
- ->where('stop_recovery_pending', true)
- ->orWhere('s3_cleanup_pending', true))
- ->exists()) {
+ if ($backup->executions()->where('stop_recovery_pending', true)->exists()) {
$this->skippedCount++;
- $this->logSkip('volume_backup', 'container_recovery_pending', [
+ $this->logSkip('volume_backup', 'stopped_container_recovery_pending', [
'backup_id' => $backup->id,
'team_id' => $backup->team_id,
]);
diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.php
index 78f091bbe2..f142a65941 100644
--- a/app/Jobs/VolumeBackupJob.php
+++ b/app/Jobs/VolumeBackupJob.php
@@ -488,6 +488,8 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
$this->backup->executions()
->where('local_storage_deleted', true)
+ ->where('stop_recovery_pending', false)
+ ->where('s3_cleanup_pending', false)
->where(function (Builder $query): void {
$query->where('s3_storage_deleted', true)->orWhereNull('s3_uploaded');
})
diff --git a/app/Jobs/VolumeBackupRecoveryJob.php b/app/Jobs/VolumeBackupRecoveryJob.php
index c136c45820..7f475d24ce 100644
--- a/app/Jobs/VolumeBackupRecoveryJob.php
+++ b/app/Jobs/VolumeBackupRecoveryJob.php
@@ -3,6 +3,7 @@
namespace App\Jobs;
use App\Models\ScheduledVolumeBackupExecution;
+use Aws\S3\Exception\S3Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldBeUnique;
@@ -12,13 +13,15 @@ use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
+use RuntimeException;
+use Throwable;
class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
/**
- * ScheduledJobManager dispatches pending recovery again, so this job does not retry by itself.
+ * ScheduledJobManager dispatches pending recovery again after a backoff, so this job does not retry by itself.
*/
public int $tries = 1;
@@ -26,6 +29,34 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, Shou
public int $uniqueFor = 300;
+ public const MAX_ATTEMPTS = 8;
+
+ public const BACKOFF_BASE_MINUTES = 5;
+
+ public const BACKOFF_CAP_MINUTES = 360;
+
+ public const CATEGORY_S3_AUTH = 's3_auth';
+
+ public const CATEGORY_S3_BUCKET = 's3_bucket';
+
+ public const CATEGORY_SERVER_UNREACHABLE = 'server_unreachable';
+
+ public const CATEGORY_REMOTE_COMMAND = 'remote_command';
+
+ public const CATEGORY_UNKNOWN = 'unknown';
+
+ private const UNREACHABLE_PATTERNS = [
+ 'ssh connection failed',
+ 'connection refused',
+ 'connection timed out',
+ 'operation timed out',
+ 'no route to host',
+ 'network is unreachable',
+ 'could not resolve hostname',
+ 'permission denied (publickey',
+ 'host key verification failed',
+ ];
+
public function __construct(public ScheduledVolumeBackupExecution $execution)
{
$this->onQueue(crons_queue());
@@ -51,20 +82,173 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, Shou
public function handle(): void
{
- self::recover($this->execution);
+ self::recoverWithBounds($this->execution);
}
+ /**
+ * Runs one recovery attempt and records its outcome instead of throwing, so a persistent failure backs off
+ * and eventually needs attention rather than creating a failed job every few minutes.
+ */
+ public static function recoverWithBounds(ScheduledVolumeBackupExecution $execution): void
+ {
+ $current = $execution->fresh();
+ if (! $current) {
+ return;
+ }
+
+ $execution->setRawAttributes($current->getAttributes(), true);
+
+ if (! $execution->hasPendingRecovery() || $execution->recovery_needs_attention) {
+ return;
+ }
+
+ try {
+ $failure = self::runRecoverySteps($execution);
+ } catch (Throwable $exception) {
+ $failure = ['category' => self::CATEGORY_UNKNOWN, 'exception' => $exception];
+ }
+
+ if ($failure) {
+ self::recordFailure($execution, $failure['category']);
+
+ return;
+ }
+
+ self::recordSuccess($execution);
+ }
+
+ /**
+ * Recovers stopped containers first, then cleans the partial S3 upload. Each step runs even when the other one
+ * fails; the first failure is rethrown after both steps ran.
+ */
public static function recover(ScheduledVolumeBackupExecution $execution): void
+ {
+ $failure = self::runRecoverySteps($execution);
+
+ if ($failure) {
+ throw $failure['exception'];
+ }
+ }
+
+ /**
+ * @return array{category: string, exception: Throwable}|null The first failure, or null when every step succeeded.
+ */
+ private static function runRecoverySteps(ScheduledVolumeBackupExecution $execution): ?array
{
$execution->loadMissing('scheduledVolumeBackup.backupable.resource');
+ $failure = null;
if ($execution->stop_recovery_pending) {
- self::recoverContainers($execution);
+ try {
+ self::recoverContainers($execution);
+ } catch (Throwable $exception) {
+ $failure = ['category' => self::categorizeContainerFailure($exception), 'exception' => $exception];
+ }
}
if ($execution->s3_cleanup_pending) {
- self::cleanupS3Upload($execution);
+ try {
+ self::cleanupS3Upload($execution);
+ } catch (Throwable $exception) {
+ $failure ??= ['category' => self::categorizeS3Failure($exception), 'exception' => $exception];
+ }
+ }
+
+ return $failure;
+ }
+
+ /**
+ * Returns the minutes to wait before the next automatic attempt after the given number of failed attempts.
+ */
+ public static function backoffMinutes(int $attempts): int
+ {
+ $exponent = max(0, min($attempts - 1, 16));
+
+ return (int) min(self::BACKOFF_BASE_MINUTES * (2 ** $exponent), self::BACKOFF_CAP_MINUTES);
+ }
+
+ public static function dispatchCacheKey(int $executionId): string
+ {
+ return "volume-backup-recovery-dispatched:{$executionId}";
+ }
+
+ private static function recordSuccess(ScheduledVolumeBackupExecution $execution): void
+ {
+ $execution->update([
+ 'recovery_attempts' => 0,
+ 'recovery_last_attempt_at' => null,
+ 'recovery_next_retry_at' => null,
+ 'recovery_error' => null,
+ 'recovery_needs_attention' => false,
+ ]);
+ }
+
+ private static function recordFailure(ScheduledVolumeBackupExecution $execution, string $category): void
+ {
+ $recorded = $execution->getConnection()->transaction(function () use ($execution, $category): ?ScheduledVolumeBackupExecution {
+ $current = ScheduledVolumeBackupExecution::query()->whereKey($execution->id)->lockForUpdate()->first();
+ if (! $current) {
+ return null;
+ }
+
+ $attempts = $current->recovery_attempts + 1;
+ $needsAttention = $attempts >= self::MAX_ATTEMPTS;
+ $current->update([
+ 'recovery_attempts' => $attempts,
+ 'recovery_last_attempt_at' => now(),
+ 'recovery_next_retry_at' => $needsAttention ? null : now()->addMinutes(self::backoffMinutes($attempts)),
+ 'recovery_error' => $category,
+ 'recovery_needs_attention' => $needsAttention,
+ ]);
+
+ return $current;
+ });
+
+ if (! $recorded) {
+ return;
+ }
+
+ $execution->setRawAttributes($recorded->getAttributes(), true);
+
+ Log::channel('scheduled-errors')->warning('Volume backup recovery failed', [
+ 'execution_id' => $recorded->id,
+ 'execution_uuid' => $recorded->uuid,
+ 'category' => $category,
+ 'attempts' => $recorded->recovery_attempts,
+ 'needs_attention' => $recorded->recovery_needs_attention,
+ 'next_retry_at' => $recorded->recovery_next_retry_at?->toIso8601String(),
+ ]);
+ }
+
+ private static function categorizeS3Failure(Throwable $exception): string
+ {
+ for ($current = $exception; $current; $current = $current->getPrevious()) {
+ if ($current instanceof S3Exception) {
+ return match ($current->getAwsErrorCode()) {
+ 'InvalidAccessKeyId', 'SignatureDoesNotMatch', 'AccessDenied' => self::CATEGORY_S3_AUTH,
+ 'NoSuchBucket' => self::CATEGORY_S3_BUCKET,
+ default => self::CATEGORY_UNKNOWN,
+ };
+ }
}
+
+ return self::CATEGORY_UNKNOWN;
+ }
+
+ private static function categorizeContainerFailure(Throwable $exception): string
+ {
+ if ($exception->getCode() === 255) {
+ return self::CATEGORY_SERVER_UNREACHABLE;
+ }
+
+ $message = strtolower($exception->getMessage());
+ foreach (self::UNREACHABLE_PATTERNS as $pattern) {
+ if (str_contains($message, $pattern)) {
+ return self::CATEGORY_SERVER_UNREACHABLE;
+ }
+ }
+
+ return $exception instanceof RuntimeException ? self::CATEGORY_REMOTE_COMMAND : self::CATEGORY_UNKNOWN;
}
private static function recoverContainers(ScheduledVolumeBackupExecution $execution): void
diff --git a/app/Livewire/Project/Shared/Storages/VolumeBackups.php b/app/Livewire/Project/Shared/Storages/VolumeBackups.php
index b4ae3ead3a..0299c98c8b 100644
--- a/app/Livewire/Project/Shared/Storages/VolumeBackups.php
+++ b/app/Livewire/Project/Shared/Storages/VolumeBackups.php
@@ -4,6 +4,7 @@ namespace App\Livewire\Project\Shared\Storages;
use App\Actions\Shared\DeleteScheduledVolumeBackup;
use App\Jobs\VolumeBackupJob;
+use App\Jobs\VolumeBackupRecoveryJob;
use App\Models\LocalFileVolume;
use App\Models\LocalPersistentVolume;
use App\Models\S3Storage;
@@ -13,6 +14,7 @@ use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Illuminate\Http\RedirectResponse;
use Illuminate\Routing\Redirector;
use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Cache;
use Livewire\Component;
use Livewire\WithPagination;
use Throwable;
@@ -278,6 +280,8 @@ class VolumeBackups extends Component
$deletedCount = $this->backup?->executions()
->where('local_storage_deleted', true)
+ ->where('stop_recovery_pending', false)
+ ->where('s3_cleanup_pending', false)
->where(fn ($query) => $query
->where('s3_storage_deleted', true)
->orWhereNull('s3_uploaded')
@@ -290,6 +294,33 @@ class VolumeBackups extends Component
);
}
+ public function retryRecovery(int $executionId): void
+ {
+ $this->authorize('update', $this->resource);
+
+ $execution = $this->backup?->executions()->whereKey($executionId)->first();
+ if (! $execution || ! $execution->hasPendingRecovery()) {
+ $this->dispatch('error', 'No pending recovery found for this backup execution.');
+
+ return;
+ }
+
+ $execution->update([
+ 'recovery_attempts' => 0,
+ 'recovery_next_retry_at' => null,
+ 'recovery_needs_attention' => false,
+ ]);
+ Cache::forget(VolumeBackupRecoveryJob::dispatchCacheKey($execution->id));
+ VolumeBackupRecoveryJob::dispatch($execution);
+ auditLog('ui.volume_backup.recovery_retried', [
+ 'team_id' => $this->resource->team()?->id,
+ 'resource_uuid' => $this->resource->uuid,
+ 'backup_uuid' => $this->backup->uuid,
+ 'execution_uuid' => $execution->uuid,
+ ]);
+ $this->dispatch('success', 'Backup recovery queued.');
+ }
+
public function deleteBackup(int $executionId, string $password, array $selectedActions = []): bool|string
{
$this->authorize('update', $this->resource);
diff --git a/app/Models/ScheduledVolumeBackupExecution.php b/app/Models/ScheduledVolumeBackupExecution.php
index 5287344467..eec799218e 100644
--- a/app/Models/ScheduledVolumeBackupExecution.php
+++ b/app/Models/ScheduledVolumeBackupExecution.php
@@ -21,6 +21,11 @@ class ScheduledVolumeBackupExecution extends BaseModel
'local_storage_deleted',
's3_storage_deleted',
's3_uploaded',
+ 'recovery_attempts',
+ 'recovery_last_attempt_at',
+ 'recovery_next_retry_at',
+ 'recovery_error',
+ 'recovery_needs_attention',
];
protected function casts(): array
@@ -34,9 +39,18 @@ class ScheduledVolumeBackupExecution extends BaseModel
'local_storage_deleted' => 'boolean',
's3_storage_deleted' => 'boolean',
's3_uploaded' => 'boolean',
+ 'recovery_attempts' => 'integer',
+ 'recovery_last_attempt_at' => 'datetime',
+ 'recovery_next_retry_at' => 'datetime',
+ 'recovery_needs_attention' => 'boolean',
];
}
+ public function hasPendingRecovery(): bool
+ {
+ return $this->stop_recovery_pending || $this->s3_cleanup_pending;
+ }
+
public function scheduledVolumeBackup(): BelongsTo
{
return $this->belongsTo(ScheduledVolumeBackup::class);
diff --git a/bootstrap/helpers/databases.php b/bootstrap/helpers/databases.php
index a4407c064a..f4a97be5ce 100644
--- a/bootstrap/helpers/databases.php
+++ b/bootstrap/helpers/databases.php
@@ -282,9 +282,16 @@ function deleteBackupsS3(string|array|null $filenames, S3Storage $s3): void
'endpoint' => $s3->endpoint,
'use_path_style_endpoint' => true,
'aws_url' => $s3->awsUrl(),
+ 'throw' => true,
]);
- if (! $disk->delete($filenames)) {
+ try {
+ $deleted = $disk->delete($filenames);
+ } catch (Throwable $exception) {
+ throw new RuntimeException('One or more S3 backup files could not be deleted.', previous: $exception);
+ }
+
+ if (! $deleted) {
throw new RuntimeException('One or more S3 backup files could not be deleted.');
}
}
diff --git a/database/migrations/2026_10_03_172204_add_recovery_bounds_to_scheduled_volume_backup_executions_table.php b/database/migrations/2026_10_03_172204_add_recovery_bounds_to_scheduled_volume_backup_executions_table.php
new file mode 100644
index 0000000000..f10f334912
--- /dev/null
+++ b/database/migrations/2026_10_03_172204_add_recovery_bounds_to_scheduled_volume_backup_executions_table.php
@@ -0,0 +1,44 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+ /**
+ * Run the migrations.
+ */
+ public function up(): void
+ {
+ Schema::table('scheduled_volume_backup_executions', function (Blueprint $table) {
+ $table->unsignedSmallInteger('recovery_attempts')->default(0);
+ $table->timestampTz('recovery_last_attempt_at')->nullable();
+ $table->timestampTz('recovery_next_retry_at')->nullable();
+ $table->string('recovery_error', 32)->nullable();
+ $table->boolean('recovery_needs_attention')->default(false);
+
+ $table->index(
+ ['stop_recovery_pending', 's3_cleanup_pending', 'recovery_needs_attention', 'recovery_next_retry_at'],
+ 'scheduled_volume_executions_recovery_index',
+ );
+ });
+ }
+
+ /**
+ * Reverse the migrations.
+ */
+ public function down(): void
+ {
+ Schema::table('scheduled_volume_backup_executions', function (Blueprint $table) {
+ $table->dropIndex('scheduled_volume_executions_recovery_index');
+ $table->dropColumn([
+ 'recovery_attempts',
+ 'recovery_last_attempt_at',
+ 'recovery_next_retry_at',
+ 'recovery_error',
+ 'recovery_needs_attention',
+ ]);
+ });
+ }
+};
diff --git a/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php b/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php
index b6dd4ffd4d..b4bf4465ec 100644
--- a/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php
+++ b/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php
@@ -62,6 +62,14 @@
if (empty($deleteActions)) {
$deleteActions[] = 'This backup execution record will be deleted.';
}
+ $recoveryPending = $execution->status !== 'running' && $execution->hasPendingRecovery();
+ $recoveryErrorLabel = match ($execution->recovery_error) {
+ 's3_auth' => 'S3 credentials',
+ 's3_bucket' => 'S3 bucket missing',
+ 'server_unreachable' => 'server unreachable',
+ 'remote_command' => 'remote command failed',
+ default => 'unknown error',
+ };
@endphp
<div wire:key="volume-backup-execution-{{ $execution->id }}"
@@ -99,6 +107,14 @@
? 'neutral'
: ($execution->s3_uploaded ? 'success' : 'error')" />
@endif
+ @if ($recoveryPending)
+ @if ($execution->recovery_needs_attention)
+ <x-status-badge :status="'Needs attention ('.$recoveryErrorLabel.')'" type="error" />
+ @else
+ <x-status-badge status="Recovery pending" type="warning"
+ :title="$execution->recovery_error ? 'Last attempt: '.$recoveryErrorLabel : null" />
+ @endif
+ @endif
</span>
<span class="flex items-center justify-end gap-1">
@@ -109,6 +125,14 @@
<x-reicon name="upload" class="size-3.5 rotate-180" />
</button>
@endif
+ @if ($recoveryPending)
+ <button type="button" class="icon-button shrink-0"
+ wire:click="retryRecovery({{ $execution->id }})"
+ wire:loading.attr="disabled" wire:target="retryRecovery({{ $execution->id }})"
+ title="Retry recovery" aria-label="Retry recovery">
+ <x-reicon name="refresh" class="size-3.5" />
+ </button>
+ @endif
@if ($execution->status !== 'running')
<x-modal-confirmation title="Confirm Backup Deletion?" isErrorButton
submitAction="deleteBackup({{ $execution->id }})"
diff --git a/tests/Feature/VolumeBackupRecoveryBoundsTest.php b/tests/Feature/VolumeBackupRecoveryBoundsTest.php
new file mode 100644
index 0000000000..f65c17ae63
--- /dev/null
+++ b/tests/Feature/VolumeBackupRecoveryBoundsTest.php
@@ -0,0 +1,384 @@
+<?php
+
+use App\Jobs\ScheduledJobManager;
+use App\Jobs\VolumeBackupJob;
+use App\Jobs\VolumeBackupRecoveryJob;
+use App\Livewire\Project\Shared\Storages\VolumeBackups;
+use App\Models\Application;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\LocalPersistentVolume;
+use App\Models\PrivateKey;
+use App\Models\Project;
+use App\Models\S3Storage;
+use App\Models\ScheduledVolumeBackup;
+use App\Models\ScheduledVolumeBackupExecution;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
+use App\Models\User;
+use Aws\Command;
+use Aws\S3\Exception\S3Exception;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Queue;
+use Illuminate\Support\Facades\Storage;
+use League\Flysystem\UnableToDeleteFile;
+use Livewire\Livewire;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+ Server::flushIdentityMap();
+ config(['broadcasting.default' => 'null']);
+ InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
+});
+
+afterEach(function () {
+ Carbon::setTestNow();
+});
+
+/**
+ * @return array{0: Application, 1: LocalPersistentVolume, 2: Server, 3: ScheduledVolumeBackup, 4: S3Storage}
+ */
+function createRecoveryBoundsBackup(Team $team, array $backupAttributes = []): array
+{
+ $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
+ $server = Server::factory()->create([
+ 'team_id' => $team->id,
+ 'private_key_id' => $privateKey->id,
+ 'ip' => '203.0.113.20',
+ ]);
+ $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
+ $project = Project::factory()->create(['team_id' => $team->id]);
+ $environment = Environment::factory()->create(['project_id' => $project->id]);
+ $application = Application::factory()->create([
+ 'environment_id' => $environment->id,
+ 'destination_id' => $destination->id,
+ 'destination_type' => $destination->getMorphClass(),
+ ]);
+ $volume = LocalPersistentVolume::create([
+ 'name' => 'app-data',
+ 'mount_path' => '/data',
+ 'resource_id' => $application->id,
+ 'resource_type' => $application->getMorphClass(),
+ ]);
+ $s3 = S3Storage::create([
+ 'name' => 'Volume backups',
+ 'region' => 'us-east-1',
+ 'key' => 'AKIARECOVERYKEY',
+ 'secret' => 'recovery-secret',
+ 'bucket' => 'bucket',
+ 'endpoint' => 'https://s3.example.com',
+ 'team_id' => $team->id,
+ 'is_usable' => true,
+ ]);
+ $backup = $volume->scheduledBackups()->create([
+ 'team_id' => $team->id,
+ 's3_storage_id' => $s3->id,
+ 'frequency' => 'daily',
+ 'save_s3' => true,
+ ...$backupAttributes,
+ ]);
+
+ return [$application, $volume, $server, $backup, $s3];
+}
+
+function createRecoveryBoundsExecution(ScheduledVolumeBackup $backup, array $attributes = []): ScheduledVolumeBackupExecution
+{
+ return ScheduledVolumeBackupExecution::create([
+ 'scheduled_volume_backup_id' => $backup->id,
+ 's3_storage_id' => $backup->s3_storage_id,
+ 'status' => 'failed',
+ 'filename' => '/data/coolify/backups/volumes/test/partial.tar.gz',
+ 's3_cleanup_pending' => true,
+ ...$attributes,
+ ]);
+}
+
+function failRecoveryBoundsS3Delete(string $awsCode): void
+{
+ $s3Exception = new S3Exception(
+ 'The AWS Access Key Id AKIARECOVERYKEY you provided does not exist in our records.',
+ new Command('DeleteObject'),
+ ['code' => $awsCode],
+ );
+ $disk = Mockery::mock();
+ $disk->shouldReceive('delete')->andThrow(UnableToDeleteFile::atLocation('partial.tar.gz', '', $s3Exception));
+ Storage::shouldReceive('build')->andReturn($disk);
+}
+
+function signInForRecoveryBounds($testCase, Team $team): User
+{
+ $user = User::factory()->create();
+ $user->teams()->attach($team, ['role' => 'owner']);
+ $testCase->actingAs($user);
+ session(['currentTeam' => $team]);
+
+ return $user;
+}
+
+function makeRecoveryBoundsServerFunctional(Server $server): void
+{
+ $server->settings()->update([
+ 'is_reachable' => true,
+ 'is_usable' => true,
+ 'force_disabled' => false,
+ ]);
+}
+
+it('doubles the recovery backoff and caps it at six hours', function () {
+ expect(collect(range(1, 9))->map(fn (int $attempts) => VolumeBackupRecoveryJob::backoffMinutes($attempts))->all())
+ ->toBe([5, 10, 20, 40, 80, 160, 320, 360, 360]);
+});
+
+it('records a categorized S3 cleanup failure without throwing or leaking the S3 message', function () {
+ Carbon::setTestNow('2026-10-03 12:00:00');
+ [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create());
+ $execution = createRecoveryBoundsExecution($backup, ['message' => 'Worker timed out']);
+ failRecoveryBoundsS3Delete('InvalidAccessKeyId');
+ $logger = Mockery::spy();
+ Log::shouldReceive('channel')->with('scheduled-errors')->andReturn($logger);
+
+ (new VolumeBackupRecoveryJob($execution))->handle();
+
+ $execution->refresh();
+ expect($execution->s3_cleanup_pending)->toBeTrue()
+ ->and($execution->s3_storage_deleted)->toBeFalse()
+ ->and($execution->recovery_attempts)->toBe(1)
+ ->and($execution->recovery_error)->toBe('s3_auth')
+ ->and($execution->recovery_last_attempt_at->equalTo(now()))->toBeTrue()
+ ->and($execution->recovery_next_retry_at->equalTo(now()->addMinutes(5)))->toBeTrue()
+ ->and($execution->recovery_needs_attention)->toBeFalse()
+ ->and($execution->message)->toBe('Worker timed out');
+ $logger->shouldHaveReceived('warning')->once()->with(
+ 'Volume backup recovery failed',
+ Mockery::on(fn (array $context): bool => $context['execution_uuid'] === $execution->uuid
+ && $context['category'] === 's3_auth'
+ && ! str_contains(json_encode($context), 'AKIA')),
+ );
+});
+
+it('categorizes a missing S3 bucket and grows the backoff on each failure', function () {
+ Carbon::setTestNow('2026-10-03 12:00:00');
+ [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create());
+ $execution = createRecoveryBoundsExecution($backup);
+ failRecoveryBoundsS3Delete('NoSuchBucket');
+
+ VolumeBackupRecoveryJob::recoverWithBounds($execution);
+ VolumeBackupRecoveryJob::recoverWithBounds($execution);
+
+ $execution->refresh();
+ expect($execution->recovery_error)->toBe('s3_bucket')
+ ->and($execution->recovery_attempts)->toBe(2)
+ ->and($execution->recovery_next_retry_at->equalTo(now()->addMinutes(10)))->toBeTrue();
+});
+
+it('needs attention after the maximum attempts and stops automatic recovery', function () {
+ Carbon::setTestNow('2026-10-03 12:00:00');
+ config(['constants.coolify.self_hosted' => true]);
+ Queue::fake();
+ [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create());
+ $execution = createRecoveryBoundsExecution($backup, [
+ 'recovery_attempts' => VolumeBackupRecoveryJob::MAX_ATTEMPTS - 1,
+ ]);
+ failRecoveryBoundsS3Delete('AccessDenied');
+
+ VolumeBackupRecoveryJob::recoverWithBounds($execution);
+ VolumeBackupRecoveryJob::recoverWithBounds($execution);
+
+ $execution->refresh();
+ expect($execution->recovery_needs_attention)->toBeTrue()
+ ->and($execution->recovery_attempts)->toBe(VolumeBackupRecoveryJob::MAX_ATTEMPTS)
+ ->and($execution->recovery_next_retry_at)->toBeNull();
+
+ Carbon::setTestNow('2026-10-04 12:00:00');
+ (new ScheduledJobManager)->handle();
+
+ Queue::assertNotPushed(VolumeBackupRecoveryJob::class);
+});
+
+it('waits for the backoff before dispatching recovery again', function () {
+ Carbon::setTestNow('2026-10-03 12:00:00');
+ config(['constants.coolify.self_hosted' => true]);
+ Queue::fake();
+ [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create());
+ createRecoveryBoundsExecution($backup, [
+ 'recovery_attempts' => 3,
+ 'recovery_next_retry_at' => now()->addMinutes(20),
+ ]);
+
+ (new ScheduledJobManager)->handle();
+ Queue::assertNotPushed(VolumeBackupRecoveryJob::class);
+
+ Carbon::setTestNow('2026-10-03 12:20:00');
+ (new ScheduledJobManager)->handle();
+ Queue::assertPushed(VolumeBackupRecoveryJob::class, 1);
+});
+
+it('cleans the S3 upload even when container recovery fails', function () {
+ Carbon::setTestNow('2026-10-03 12:00:00');
+ config(['constants.ssh.max_retries' => 1, 'constants.ssh.mux_enabled' => false]);
+ Storage::fake('ssh-keys');
+ Storage::fake('ssh-mux');
+ [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create(), ['stop_during_backup' => true]);
+ $execution = createRecoveryBoundsExecution($backup, ['stop_recovery_pending' => true]);
+ Process::fake([
+ '*' => Process::result(errorOutput: 'ssh: connect to host 203.0.113.20 port 22: Connection refused', exitCode: 255),
+ ]);
+ $disk = Mockery::mock();
+ $disk->shouldReceive('delete')->once()->andReturnTrue();
+ $filesystem = Mockery::mock(Storage::getFacadeRoot());
+ $filesystem->shouldReceive('build')->once()->andReturn($disk);
+ Storage::swap($filesystem);
+
+ VolumeBackupRecoveryJob::recoverWithBounds($execution);
+
+ $execution->refresh();
+ expect($execution->stop_recovery_pending)->toBeTrue()
+ ->and($execution->s3_cleanup_pending)->toBeFalse()
+ ->and($execution->s3_storage_deleted)->toBeTrue()
+ ->and($execution->recovery_error)->toBe('server_unreachable')
+ ->and($execution->recovery_attempts)->toBe(1);
+});
+
+it('resets the recovery state after a successful attempt', function () {
+ [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create());
+ $execution = createRecoveryBoundsExecution($backup, [
+ 'recovery_attempts' => 4,
+ 'recovery_error' => 's3_auth',
+ 'recovery_last_attempt_at' => now()->subHour(),
+ 'recovery_next_retry_at' => now()->subMinute(),
+ ]);
+ $disk = Mockery::mock();
+ $disk->shouldReceive('delete')->once()->andReturnTrue();
+ Storage::shouldReceive('build')->once()->andReturn($disk);
+
+ (new VolumeBackupRecoveryJob($execution))->handle();
+
+ $execution->refresh();
+ expect($execution->s3_cleanup_pending)->toBeFalse()
+ ->and($execution->s3_storage_deleted)->toBeTrue()
+ ->and($execution->recovery_attempts)->toBe(0)
+ ->and($execution->recovery_error)->toBeNull()
+ ->and($execution->recovery_next_retry_at)->toBeNull();
+});
+
+it('only blocks new scheduled backups while stopped containers await recovery', function (array $pending, bool $dispatched) {
+ Carbon::setTestNow('2026-10-03 12:00:00');
+ config(['constants.coolify.self_hosted' => true]);
+ Queue::fake();
+ [, , $server, $backup] = createRecoveryBoundsBackup(Team::factory()->create(), [
+ 'frequency' => '* * * * *',
+ 'enabled' => true,
+ ]);
+ makeRecoveryBoundsServerFunctional($server);
+ createRecoveryBoundsExecution($backup, $pending);
+ Cache::forget("scheduled-volume-backup:{$backup->id}");
+
+ (new ScheduledJobManager)->handle();
+
+ $dispatched
+ ? Queue::assertPushed(VolumeBackupJob::class, fn (VolumeBackupJob $job) => $job->backup->is($backup))
+ : Queue::assertNotPushed(VolumeBackupJob::class);
+})->with([
+ 'only S3 cleanup pending' => [['s3_cleanup_pending' => true, 'recovery_needs_attention' => true], true],
+ 'stopped containers pending' => [['stop_recovery_pending' => true, 's3_cleanup_pending' => false], false],
+ 'stopped containers need attention' => [['stop_recovery_pending' => true, 'recovery_needs_attention' => true], false],
+]);
+
+it('keeps executions with pending recovery during retention cleanup', function () {
+ Process::fake([
+ '*du -b*' => '128',
+ '*' => '',
+ ]);
+ [, , , $backup] = createRecoveryBoundsBackup(Team::factory()->create(), ['save_s3' => false]);
+ $pending = createRecoveryBoundsExecution($backup, [
+ 'local_storage_deleted' => true,
+ 's3_uploaded' => null,
+ ]);
+ $deletable = createRecoveryBoundsExecution($backup, [
+ 'filename' => null,
+ 'local_storage_deleted' => true,
+ 's3_uploaded' => null,
+ 's3_cleanup_pending' => false,
+ ]);
+
+ (new VolumeBackupJob($backup->fresh()))->handle();
+
+ expect($pending->fresh())->not->toBeNull()
+ ->and($deletable->fresh())->toBeNull();
+});
+
+it('keeps executions with pending recovery when cleaning up deleted entries', function () {
+ $team = Team::factory()->create();
+ signInForRecoveryBounds($this, $team);
+ [$application, $volume, , $backup] = createRecoveryBoundsBackup($team);
+ $pending = createRecoveryBoundsExecution($backup, [
+ 'local_storage_deleted' => true,
+ 's3_uploaded' => false,
+ ]);
+ $deletable = createRecoveryBoundsExecution($backup, [
+ 'local_storage_deleted' => true,
+ 's3_uploaded' => false,
+ 's3_cleanup_pending' => false,
+ ]);
+
+ Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application])
+ ->call('cleanupDeleted');
+
+ expect($pending->fresh())->not->toBeNull()
+ ->and($deletable->fresh())->toBeNull();
+});
+
+it('shows the recovery state and retries recovery manually', function () {
+ Queue::fake();
+ $team = Team::factory()->create();
+ signInForRecoveryBounds($this, $team);
+ [$application, $volume, , $backup] = createRecoveryBoundsBackup($team);
+ $execution = createRecoveryBoundsExecution($backup, [
+ 'recovery_attempts' => VolumeBackupRecoveryJob::MAX_ATTEMPTS,
+ 'recovery_error' => 's3_auth',
+ 'recovery_needs_attention' => true,
+ ]);
+ Cache::put(VolumeBackupRecoveryJob::dispatchCacheKey($execution->id), true, now()->addMinutes(5));
+
+ Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application, 'section' => 'executions'])
+ ->assertSee('Needs attention (S3 credentials)')
+ ->assertSee('Retry recovery')
+ ->call('retryRecovery', $execution->id)
+ ->assertDispatched('success')
+ ->assertSee('Recovery pending');
+
+ $execution->refresh();
+ expect($execution->recovery_attempts)->toBe(0)
+ ->and($execution->recovery_needs_attention)->toBeFalse()
+ ->and($execution->recovery_next_retry_at)->toBeNull()
+ ->and(Cache::has(VolumeBackupRecoveryJob::dispatchCacheKey($execution->id)))->toBeFalse();
+ Queue::assertPushed(VolumeBackupRecoveryJob::class, fn (VolumeBackupRecoveryJob $job) => $job->execution->is($execution));
+});
+
+it('prevents another team from retrying recovery', function () {
+ Queue::fake();
+ [$application, $volume, , $backup] = createRecoveryBoundsBackup(Team::factory()->create());
+ $execution = createRecoveryBoundsExecution($backup, [
+ 'recovery_attempts' => VolumeBackupRecoveryJob::MAX_ATTEMPTS,
+ 'recovery_needs_attention' => true,
+ ]);
+ $otherTeam = Team::factory()->create();
+ signInForRecoveryBounds($this, $otherTeam);
+ [$otherApplication, $otherVolume] = createRecoveryBoundsBackup($otherTeam);
+
+ Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application])
+ ->assertForbidden();
+ Livewire::test(VolumeBackups::class, ['storage' => $otherVolume, 'resource' => $otherApplication])
+ ->call('retryRecovery', $execution->id)
+ ->assertDispatched('error');
+
+ expect($execution->fresh()->recovery_needs_attention)->toBeTrue()
+ ->and($execution->fresh()->recovery_attempts)->toBe(VolumeBackupRecoveryJob::MAX_ATTEMPTS);
+ Queue::assertNotPushed(VolumeBackupRecoveryJob::class);
+});