Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(backups): treat removed containers as recovered after a volume backup

Andras Bacsai4f43c96e53e3
diff --git a/app/Jobs/VolumeBackupRecoveryJob.php b/app/Jobs/VolumeBackupRecoveryJob.php
index bd3310712e..cda0fdd046 100644
--- a/app/Jobs/VolumeBackupRecoveryJob.php
+++ b/app/Jobs/VolumeBackupRecoveryJob.php
@@ -276,10 +276,13 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, Shou
$execution->update(['stop_container_ids' => $containers]);
$remainingFile = $stateFile.'.remaining';
+ // A container that no longer exists (for example after a redeploy) has nothing to restart; only
+ // other inspect or start failures keep the container for another recovery attempt.
$script = 'status=0; : > '.escapeshellarg($remainingFile).'; '
.'if [ -f '.escapeshellarg($stateFile).' ]; then while IFS= read -r container; do '
- .'[ -z "$container" ] && continue; running=$(docker inspect --format \'{{.State.Running}}\' "$container" 2>/dev/null) '
- .'|| { echo "$container" >> '.escapeshellarg($remainingFile).'; status=1; continue; }; '
+ .'[ -z "$container" ] && continue; running=$(docker inspect --format \'{{.State.Running}}\' "$container" 2>&1) '
+ .'|| { case "$running" in *"o such object"*|*"o such container"*) continue ;; esac; '
+ .'echo "$container" >> '.escapeshellarg($remainingFile).'; status=1; continue; }; '
.'if [ "$running" != true ] && ! docker start "$container" >/dev/null; then echo "$container" >> '
.escapeshellarg($remainingFile).'; status=1; fi; done < '.escapeshellarg($stateFile).'; fi; '
.'if [ -s '.escapeshellarg($remainingFile).' ]; then mv '.escapeshellarg($remainingFile).' '.escapeshellarg($stateFile)
diff --git a/tests/Feature/VolumeBackupRecoveryMissingContainerTest.php b/tests/Feature/VolumeBackupRecoveryMissingContainerTest.php
new file mode 100644
index 0000000000..fe5190a7b1
--- /dev/null
+++ b/tests/Feature/VolumeBackupRecoveryMissingContainerTest.php
@@ -0,0 +1,171 @@
+<?php
+
+use App\Jobs\ScheduledJobManager;
+use App\Jobs\VolumeBackupJob;
+use App\Jobs\VolumeBackupRecoveryJob;
+use App\Models\Application;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\LocalPersistentVolume;
+use App\Models\PrivateKey;
+use App\Models\Project;
+use App\Models\ScheduledVolumeBackup;
+use App\Models\ScheduledVolumeBackupExecution;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\File;
+use Illuminate\Support\Facades\Notification;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Queue;
+use Illuminate\Support\Facades\Storage;
+use Symfony\Component\Process\Process as SymfonyProcess;
+
+uses(RefreshDatabase::class);
+
+const MISSING_CONTAINER_GONE = 'aaaaaaaaaaaa';
+const MISSING_CONTAINER_BROKEN = 'bbbbbbbbbbbb';
+const MISSING_CONTAINER_STOPPED = 'cccccccccccc';
+
+beforeEach(function () {
+ Server::flushIdentityMap();
+ Notification::fake();
+ Storage::fake('ssh-keys');
+ Storage::fake('ssh-mux');
+ config(['broadcasting.default' => 'null', 'constants.ssh.mux_enabled' => false, 'constants.ssh.max_retries' => 1]);
+ InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
+
+ $this->fakeDockerDir = sys_get_temp_dir().'/prr-fix-fake-docker-'.uniqid();
+ File::ensureDirectoryExists($this->fakeDockerDir);
+ $this->dockerLog = $this->fakeDockerDir.'/calls.log';
+ File::put($this->fakeDockerDir.'/docker', <<<'SH'
+#!/bin/sh
+echo "$*" >> "$(dirname "$0")/calls.log"
+case "$1" in
+ inspect)
+ case "$4" in
+ aaaaaaaaaaaa) echo "Error: no such object: $4" >&2; exit 1 ;;
+ *) echo false ;;
+ esac ;;
+ start)
+ case "$2" in
+ bbbbbbbbbbbb) echo "Error response from daemon: cannot start" >&2; exit 1 ;;
+ esac ;;
+esac
+SH);
+ chmod($this->fakeDockerDir.'/docker', 0755);
+});
+
+afterEach(function () {
+ File::deleteDirectory($this->fakeDockerDir);
+ foreach (ScheduledVolumeBackupExecution::query()->get() as $execution) {
+ File::delete([VolumeBackupRecoveryJob::stateFile($execution), VolumeBackupRecoveryJob::stateFile($execution).'.remaining']);
+ }
+ Carbon::setTestNow();
+});
+
+/**
+ * @return array{0: Server, 1: ScheduledVolumeBackup}
+ */
+function missingContainerRecoveryBackup(string $user): array
+{
+ $team = Team::factory()->create();
+ $server = Server::factory()->create([
+ 'team_id' => $team->id,
+ 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id,
+ 'user' => $user,
+ ]);
+ $server->settings()->update(['is_reachable' => true, 'is_usable' => true, 'force_disabled' => false]);
+ $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
+ $environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $team->id])->id]);
+ $application = Application::factory()->create([
+ 'environment_id' => $environment->id,
+ 'destination_id' => $destination->id,
+ 'destination_type' => $destination->getMorphClass(),
+ ]);
+ $volume = LocalPersistentVolume::create([
+ 'name' => 'app-data',
+ 'mount_path' => '/data',
+ 'resource_id' => $application->id,
+ 'resource_type' => $application->getMorphClass(),
+ ]);
+ $backup = $volume->scheduledBackups()->create([
+ 'team_id' => $team->id,
+ 'frequency' => '* * * * *',
+ 'enabled' => true,
+ 'save_s3' => false,
+ 'stop_during_backup' => true,
+ ]);
+
+ return [Server::find($server->id), $backup];
+}
+
+/**
+ * Runs the remote container recovery script with a local sh and a fake docker binary, as the server would.
+ */
+function fakeMissingContainerRemote(string $fakeDockerDir): void
+{
+ Process::fake(function ($process) use ($fakeDockerDir) {
+ $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+ $lines = explode(PHP_EOL, $command);
+ $remote = implode(PHP_EOL, array_slice($lines, 1, -1));
+ if (str_starts_with($remote, 'sudo ')) {
+ $remote = substr($remote, strlen('sudo '));
+ }
+ if (! str_starts_with($remote, 'sh -c ') && ! str_starts_with($remote, 'cat ')) {
+ return Process::result(output: '');
+ }
+
+ $result = new SymfonyProcess(['sh', '-c', $remote], null, ['PATH' => $fakeDockerDir.':'.getenv('PATH')]);
+ $result->run();
+
+ return Process::result(output: $result->getOutput(), errorOutput: $result->getErrorOutput(), exitCode: $result->getExitCode());
+ });
+}
+
+it('treats stopped containers that no longer exist as recovered and does not block the schedule', function (string $user) {
+ Carbon::setTestNow('2026-10-03 12:00:00');
+ config(['constants.coolify.self_hosted' => true]);
+ [$server, $backup] = missingContainerRecoveryBackup($user);
+ $execution = ScheduledVolumeBackupExecution::create([
+ 'scheduled_volume_backup_id' => $backup->id,
+ 'status' => 'failed',
+ 'stop_recovery_pending' => true,
+ ]);
+ File::put(VolumeBackupRecoveryJob::stateFile($execution), MISSING_CONTAINER_GONE."\n".MISSING_CONTAINER_STOPPED."\n");
+ fakeMissingContainerRemote($this->fakeDockerDir);
+
+ VolumeBackupRecoveryJob::recoverWithBounds($execution);
+
+ $execution->refresh();
+ expect($execution->stop_recovery_pending)->toBeFalse()
+ ->and($execution->recovery_needs_attention)->toBeFalse()
+ ->and(File::exists(VolumeBackupRecoveryJob::stateFile($execution)))->toBeFalse()
+ ->and(File::get($this->dockerLog))->toContain('start '.MISSING_CONTAINER_STOPPED);
+
+ Queue::fake();
+ Cache::forget("scheduled-volume-backup:{$backup->id}");
+ (new ScheduledJobManager)->handle();
+ Queue::assertPushed(VolumeBackupJob::class, fn (VolumeBackupJob $job) => $job->backup->is($backup));
+})->with(['root user' => ['root'], 'non-root user' => ['ubuntu']]);
+
+it('still needs attention when an existing container fails to start', function (string $user) {
+ [, $backup] = missingContainerRecoveryBackup($user);
+ $execution = ScheduledVolumeBackupExecution::create([
+ 'scheduled_volume_backup_id' => $backup->id,
+ 'status' => 'failed',
+ 'stop_recovery_pending' => true,
+ ]);
+ File::put(VolumeBackupRecoveryJob::stateFile($execution), MISSING_CONTAINER_GONE."\n".MISSING_CONTAINER_BROKEN."\n");
+ fakeMissingContainerRemote($this->fakeDockerDir);
+
+ VolumeBackupRecoveryJob::recoverWithBounds($execution);
+
+ $execution->refresh();
+ expect($execution->stop_recovery_pending)->toBeTrue()
+ ->and($execution->recovery_needs_attention)->toBeTrue()
+ ->and(trim(File::get(VolumeBackupRecoveryJob::stateFile($execution))))->toBe(MISSING_CONTAINER_BROKEN);
+})->with(['root user' => ['root'], 'non-root user' => ['ubuntu']]);