Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(docker): make cleanup commands idempotent (#11463)

Andras Bacsai379abb252621
diff --git a/app/Actions/Application/StopApplicationOneServer.php b/app/Actions/Application/StopApplicationOneServer.php
index 10f5b85f21..b25eb481b6 100644
--- a/app/Actions/Application/StopApplicationOneServer.php
+++ b/app/Actions/Application/StopApplicationOneServer.php
@@ -29,7 +29,7 @@ class StopApplicationOneServer
instant_remote_process(
[
dockerStopCommand($timeout, $containerName, $server),
- "docker rm -f $containerName",
+ dockerRemoveCommand($containerName),
],
$server
);
diff --git a/app/Actions/Database/StopDatabaseProxy.php b/app/Actions/Database/StopDatabaseProxy.php
index 96a1097662..e6789202f3 100644
--- a/app/Actions/Database/StopDatabaseProxy.php
+++ b/app/Actions/Database/StopDatabaseProxy.php
@@ -24,10 +24,10 @@ class StopDatabaseProxy
{
$server = data_get($database, 'destination.server');
$uuid = $database->uuid;
- if ($database->getMorphClass() === \App\Models\ServiceDatabase::class) {
+ if ($database->getMorphClass() === ServiceDatabase::class) {
$server = data_get($database, 'service.server');
}
- instant_remote_process(["docker rm -f {$uuid}-proxy"], $server);
+ instant_remote_process([dockerRemoveCommand("{$uuid}-proxy")], $server);
$database->save();
diff --git a/app/Actions/Destination/RemoveStandaloneDockerNetwork.php b/app/Actions/Destination/RemoveStandaloneDockerNetwork.php
index 21c40a50ad..3e1b5380b6 100644
--- a/app/Actions/Destination/RemoveStandaloneDockerNetwork.php
+++ b/app/Actions/Destination/RemoveStandaloneDockerNetwork.php
@@ -11,6 +11,6 @@ class RemoveStandaloneDockerNetwork
$safeNetwork = escapeshellarg($destination->network);
instant_remote_process(["docker network disconnect {$safeNetwork} coolify-proxy"], $destination->server, throwError: false);
- instant_remote_process(["docker network rm -f {$safeNetwork}"], $destination->server);
+ instant_remote_process([dockerNetworkRemoveCommand($destination->network)], $destination->server);
}
}
diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php
index 1838feb9e7..0b73ed0cf5 100644
--- a/app/Jobs/DatabaseBackupJob.php
+++ b/app/Jobs/DatabaseBackupJob.php
@@ -798,7 +798,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$this->add_to_error_output($e->getMessage());
throw $e;
} finally {
- $command = "docker rm -f backup-of-{$this->backup_log_uuid}";
+ $command = dockerRemoveCommand("backup-of-{$this->backup_log_uuid}");
instant_remote_process([$command], $this->server, true, false, null, disableMultiplexing: true);
}
}
diff --git a/app/Listeners/ProxyStatusChangedNotification.php b/app/Listeners/ProxyStatusChangedNotification.php
index 30ecb2d8d5..9b117d4e13 100644
--- a/app/Listeners/ProxyStatusChangedNotification.php
+++ b/app/Listeners/ProxyStatusChangedNotification.php
@@ -61,7 +61,7 @@ class ProxyStatusChangedNotification implements ShouldQueueAfterCommit
if ($status === 'created') {
instant_remote_process([
- 'docker rm -f coolify-proxy',
+ dockerRemoveCommand('coolify-proxy'),
], $server);
}
}
diff --git a/app/Livewire/Destination/Show.php b/app/Livewire/Destination/Show.php
index 1b344c9056..03fa2b5109 100644
--- a/app/Livewire/Destination/Show.php
+++ b/app/Livewire/Destination/Show.php
@@ -81,7 +81,7 @@ class Show extends Component
}
$safeNetwork = escapeshellarg($this->destination->network);
instant_remote_process(["docker network disconnect {$safeNetwork} coolify-proxy"], $this->destination->server, throwError: false);
- instant_remote_process(["docker network rm -f {$safeNetwork}"], $this->destination->server);
+ instant_remote_process([dockerNetworkRemoveCommand($this->destination->network)], $this->destination->server);
}
$this->destination->delete();
diff --git a/bootstrap/helpers/docker.php b/bootstrap/helpers/docker.php
index 210c84a86a..d152ba58f8 100644
--- a/bootstrap/helpers/docker.php
+++ b/bootstrap/helpers/docker.php
@@ -267,7 +267,28 @@ function dockerStopCommand(int $timeout, string $containers, Server|string|null
function dockerRemoveCommandWithTimeout(string $container, int $timeout = 60, int $killAfter = 10): string
{
$container = escapeShellValue($container);
- $script = "if command -v timeout >/dev/null 2>&1; then timeout -k {$killAfter}s {$timeout}s docker rm -f {$container}; exit_code=\$?; else exit_code=124; fi; if [ \"\$exit_code\" -eq 124 ]; then echo '__COOLIFY_CONTAINER_REMOVE_TIMEOUT__'; fi; exit \$exit_code";
+ $script = "if command -v timeout >/dev/null 2>&1; then output=\$(timeout -k {$killAfter}s {$timeout}s docker rm -f {$container} 2>&1); exit_code=\$?; else output=''; exit_code=124; fi; if [ \"\$exit_code\" -eq 124 ]; then echo '__COOLIFY_CONTAINER_REMOVE_TIMEOUT__'; elif [ \"\$exit_code\" -ne 0 ] && printf '%s' \"\$output\" | grep -q 'No such container:'; then exit 0; elif [ \"\$exit_code\" -ne 0 ]; then printf '%s\\n' \"\$output\" >&2; else printf '%s\\n' \"\$output\"; fi; exit \$exit_code";
+
+ return 'bash -c '.escapeShellValue($script);
+}
+
+function dockerRemoveCommand(string $container): string
+{
+ $command = 'docker rm -f '.escapeShellValue($container);
+
+ return dockerCommandIgnoringError($command, 'No such container:');
+}
+
+function dockerNetworkRemoveCommand(string $network): string
+{
+ $command = 'docker network rm '.escapeShellValue($network);
+
+ return dockerCommandIgnoringError($command, 'network .* not found');
+}
+
+function dockerCommandIgnoringError(string $command, string $ignoredError): string
+{
+ $script = "output=\$({$command} 2>&1); exit_code=\$?; if [ \"\$exit_code\" -ne 0 ] && printf '%s' \"\$output\" | grep -Eq ".escapeShellValue($ignoredError)."; then exit 0; fi; if [ \"\$exit_code\" -ne 0 ]; then printf '%s\\n' \"\$output\" >&2; else printf '%s\\n' \"\$output\"; fi; exit \$exit_code";
return 'bash -c '.escapeShellValue($script);
}
diff --git a/tests/Unit/DockerRemoveWithTimeoutTest.php b/tests/Unit/DockerRemoveWithTimeoutTest.php
index a3b9f15dc5..1f09ae06cb 100644
--- a/tests/Unit/DockerRemoveWithTimeoutTest.php
+++ b/tests/Unit/DockerRemoveWithTimeoutTest.php
@@ -58,6 +58,124 @@ it('escapes container names in bounded removal commands', function () {
rmdir($directory);
});
+it('succeeds when the container was already removed', function () {
+ $directory = sys_get_temp_dir().'/coolify-docker-remove-'.bin2hex(random_bytes(4));
+ mkdir($directory);
+ file_put_contents($directory.'/docker', "#!/bin/sh\necho 'Error response from daemon: No such container: container-name' >&2\nexit 1\n");
+ chmod($directory.'/docker', 0755);
+
+ $process = new Process(['/bin/sh', '-c', dockerRemoveCommandWithTimeout('container-name')], env: [
+ 'PATH' => $directory.':'.getenv('PATH'),
+ ]);
+ $process->run();
+
+ expect($process->isSuccessful())->toBeTrue();
+
+ unlink($directory.'/docker');
+ rmdir($directory);
+});
+
+it('reports a timeout when timeout output also says the container is missing', function () {
+ $directory = sys_get_temp_dir().'/coolify-docker-remove-'.bin2hex(random_bytes(4));
+ mkdir($directory);
+ file_put_contents($directory.'/timeout', "#!/bin/sh\necho 'Error response from daemon: No such container: container-name'\nexit 124\n");
+ chmod($directory.'/timeout', 0755);
+
+ $process = new Process(['/bin/sh', '-c', dockerRemoveCommandWithTimeout('container-name')], env: [
+ 'PATH' => $directory.':'.getenv('PATH'),
+ ]);
+ $process->run();
+
+ expect($process->getExitCode())->toBe(124)
+ ->and($process->getOutput())->toContain('__COOLIFY_CONTAINER_REMOVE_TIMEOUT__');
+
+ unlink($directory.'/timeout');
+ rmdir($directory);
+});
+
+it('fails when Docker cannot remove an existing container', function () {
+ $directory = sys_get_temp_dir().'/coolify-docker-remove-'.bin2hex(random_bytes(4));
+ mkdir($directory);
+ file_put_contents($directory.'/docker', "#!/bin/sh\necho 'Error response from daemon: removal already in progress' >&2\nexit 1\n");
+ chmod($directory.'/docker', 0755);
+
+ $process = new Process(['/bin/sh', '-c', dockerRemoveCommandWithTimeout('container-name')], env: [
+ 'PATH' => $directory.':'.getenv('PATH'),
+ ]);
+ $process->run();
+
+ expect($process->isSuccessful())->toBeFalse()
+ ->and($process->getErrorOutput())->toContain('removal already in progress');
+
+ unlink($directory.'/docker');
+ rmdir($directory);
+});
+
+it('makes regular container removal idempotent without hiding other failures', function () {
+ $directory = sys_get_temp_dir().'/coolify-docker-remove-'.bin2hex(random_bytes(4));
+ mkdir($directory);
+ file_put_contents($directory.'/docker', "#!/bin/sh\necho \"\$DOCKER_ERROR\" >&2\nexit 1\n");
+ chmod($directory.'/docker', 0755);
+
+ $missingContainer = new Process(['/bin/sh', '-c', dockerRemoveCommand('container name')], env: [
+ 'PATH' => $directory.':'.getenv('PATH'),
+ 'DOCKER_ERROR' => 'Error response from daemon: No such container: container name',
+ ]);
+ $missingContainer->run();
+ $realFailure = new Process(['/bin/sh', '-c', dockerRemoveCommand('container name')], env: [
+ 'PATH' => $directory.':'.getenv('PATH'),
+ 'DOCKER_ERROR' => 'Error response from daemon: removal already in progress',
+ ]);
+ $realFailure->run();
+
+ expect($missingContainer->isSuccessful())->toBeTrue()
+ ->and($realFailure->isSuccessful())->toBeFalse();
+
+ unlink($directory.'/docker');
+ rmdir($directory);
+});
+
+it('makes network removal idempotent without hiding other failures', function () {
+ $directory = sys_get_temp_dir().'/coolify-docker-remove-'.bin2hex(random_bytes(4));
+ mkdir($directory);
+ file_put_contents($directory.'/docker', "#!/bin/sh\necho \"\$DOCKER_ERROR\" >&2\nexit 1\n");
+ chmod($directory.'/docker', 0755);
+
+ $missingNetwork = new Process(['/bin/sh', '-c', dockerNetworkRemoveCommand('network name')], env: [
+ 'PATH' => $directory.':'.getenv('PATH'),
+ 'DOCKER_ERROR' => 'Error response from daemon: network network name not found',
+ ]);
+ $missingNetwork->run();
+ $realFailure = new Process(['/bin/sh', '-c', dockerNetworkRemoveCommand('network name')], env: [
+ 'PATH' => $directory.':'.getenv('PATH'),
+ 'DOCKER_ERROR' => 'Error response from daemon: network has active endpoints',
+ ]);
+ $realFailure->run();
+
+ expect($missingNetwork->isSuccessful())->toBeTrue()
+ ->and($realFailure->isSuccessful())->toBeFalse();
+
+ unlink($directory.'/docker');
+ rmdir($directory);
+});
+
+it('uses idempotent commands in strict cleanup paths', function () {
+ $root = dirname(__DIR__, 2);
+
+ expect(file_get_contents($root.'/app/Jobs/DatabaseBackupJob.php'))
+ ->toContain('dockerRemoveCommand("backup-of-{$this->backup_log_uuid}")')
+ ->and(file_get_contents($root.'/app/Actions/Database/StopDatabaseProxy.php'))
+ ->toContain('dockerRemoveCommand("{$uuid}-proxy")')
+ ->and(file_get_contents($root.'/app/Actions/Destination/RemoveStandaloneDockerNetwork.php'))
+ ->toContain('dockerNetworkRemoveCommand($destination->network)')
+ ->and(file_get_contents($root.'/app/Livewire/Destination/Show.php'))
+ ->toContain('dockerNetworkRemoveCommand($this->destination->network)')
+ ->and(file_get_contents($root.'/app/Listeners/ProxyStatusChangedNotification.php'))
+ ->toContain("dockerRemoveCommand('coolify-proxy')")
+ ->and(file_get_contents($root.'/app/Actions/Application/StopApplicationOneServer.php'))
+ ->toContain('dockerRemoveCommand($containerName)');
+});
+
it('configures deferred removal attempts to outlive the shell timeout', function () {
$job = new RemoveContainerJob(123, 'container-name');