An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(upgrade): keep Coolify running when Docker reports IPv6 gateways with a prefix
Andras Bacsai420692c02053
diff --git a/.ai/lessons.md b/.ai/lessons.mdindex c2137bdb38..edfabe8dcf 100644--- a/.ai/lessons.md+++ b/.ai/lessons.md@@ -13,6 +13,7 @@- Redirect browser test output to a file (`> /tmp/x.log 2>&1`); piping it (`| tail`) hangs because the Playwright server keeps the pipe open.- Browser `click('Text')` can match a hidden element first (for example inactive modal steps) and wait forever; use `button:visible:has-text("Text")`.- Never run `php artisan test` inside a dev container (`./scripts/dev exec` or `docker exec coolify-dev-*`). The container sets `DB_CONNECTION=pgsql` as a real environment variable, which wins over `phpunit.xml`, so `RefreshDatabase` runs `migrate:fresh` on the dev database and wipes it. Run tests on the host only.+- The helper image runs `docker compose` for deployments and for the Coolify upgrade itself. Before you bump its Docker CLI or Compose, test them against the oldest supported Docker Engine (for example a `docker:<version>-dind` container); newer clients can reject data from older daemons.- Do not symlink `vendor` into another worktree. Composer resolves the base path from the real `vendor` path, so the tests load the app classes of the other checkout. Run `composer install --ignore-platform-req=ext-zip` in the worktree.- Call `visit()` directly in each `tests/v4/Browser` test body; Pest does not mark a test that only uses helper-wrapped `visit()` as a browser test, so it fails with `sendText() on null`.diff --git a/app/Traits/ExecuteRemoteCommand.php b/app/Traits/ExecuteRemoteCommand.phpindex 449c9c55dd..57b19f91b8 100644--- a/app/Traits/ExecuteRemoteCommand.php+++ b/app/Traits/ExecuteRemoteCommand.php@@ -268,7 +268,22 @@ trait ExecuteRemoteCommand$error = $this->redact_sensitive_info($error);- return "Command execution failed (exit code {$exitCode}): {$commandText}\nError: {$error}";+ return "Command execution failed (exit code {$exitCode}): {$commandText}\nError: {$error}".$this->dockerNetworkGatewayHint($error);+ }++ /**+ * Docker Engine 27.0 to 28.0.0 reports an IPv6 network gateway with its prefix length (fd00::1/64) until the+ * daemon restarts, and newer Docker CLI and Compose versions fail to parse it.+ *+ * @see https://github.com/moby/moby/pull/49520+ */+ private function dockerNetworkGatewayHint(string $error): string+ {+ if (! preg_match('~ParseAddr\("[0-9A-Fa-f:.]+/\d+"\)~', $error)) {+ return '';+ }++ return "\nDocker Engine on this server reports a Docker network gateway with a prefix length, which is a Docker Engine bug fixed in 28.0.1. Run 'systemctl restart docker' on the server (this restarts all containers) or upgrade Docker Engine, then deploy again.";}/**diff --git a/other/nightly/upgrade.sh b/other/nightly/upgrade.shindex 88dd7ea093..4d594cb9e9 100644--- a/other/nightly/upgrade.sh+++ b/other/nightly/upgrade.sh@@ -38,6 +38,47 @@ write_status() {echo "${step}|${message}|$(date -Iseconds)" > "$STATUS_FILE"}+# Print the first gateway of a network that Docker reports with a prefix length (fd00::1/64).+# Docker Engine 27.0 to 28.0.0 does this for IPv6 until the daemon restarts (https://github.com/moby/moby/pull/49520).+# The Docker CLI and Compose in the helper image fail on such a network with a ParseAddr error.+network_gateway_with_prefix() {+ docker network inspect "$1" --format '{{range .IPAM.Config}}{{.Gateway}} {{end}}' 2>&1 | grep -oE '[0-9A-Fa-f:.]+/[0-9]+' | head -n1+}++# Make sure the coolify network exists and the helper image can read it.+# It is created when starting Coolify with docker compose+ensure_coolify_network() {+ log "Checking Docker network 'coolify'..."+ if ! docker network inspect coolify >/dev/null 2>&1; then+ log "Network 'coolify' does not exist, creating..."+ if ! docker network create --attachable --ipv6 coolify 2>/dev/null; then+ log "Failed to create network with IPv6, trying without IPv6..."+ docker network create --attachable coolify 2>/dev/null+ log "Network 'coolify' created without IPv6"+ elif [ -n "$(network_gateway_with_prefix coolify)" ]; then+ # No container uses the new network yet, so it is safe to recreate it.+ log "Docker reports the IPv6 gateway of network 'coolify' with a prefix length, recreating it without IPv6..."+ docker network rm coolify >/dev/null 2>&1+ docker network create --attachable coolify 2>/dev/null+ log "Network 'coolify' created without IPv6"+ else+ log "Network 'coolify' created with IPv6 support"+ fi+ return 0+ fi++ log "Network 'coolify' already exists"+ local gateway+ gateway=$(network_gateway_with_prefix coolify)+ if [ -n "$gateway" ]; then+ local message="Docker Engine $(docker version --format '{{.Server.Version}}' 2>/dev/null) reports the gateway of the 'coolify' network as ${gateway}, which Coolify cannot read (Docker Engine bug, fixed in 28.0.1). Run 'systemctl restart docker' (this restarts all containers) or upgrade Docker Engine, then upgrade Coolify again."+ log "ERROR: ${message}"+ write_status "error" "$message"+ echo " ERROR: ${message}"+ return 1+ fi+}+# Pull an image. Retry with backoff, because the registry can rate limit many instances that pull at the same time.pull_image() {local image="$1"@@ -177,21 +218,7 @@ filog "Environment variables check complete"echo " Done."-# Make sure coolify network exists-# It is created when starting Coolify with docker compose-log "Checking Docker network 'coolify'..."-if ! docker network inspect coolify >/dev/null 2>&1; then- log "Network 'coolify' does not exist, creating..."- if ! docker network create --attachable --ipv6 coolify 2>/dev/null; then- log "Failed to create network with IPv6, trying without IPv6..."- docker network create --attachable coolify 2>/dev/null- log "Network 'coolify' created without IPv6"- else- log "Network 'coolify' created with IPv6 support"- fi-else- log "Network 'coolify' already exists"-fi+ensure_coolify_network || exit 1mkdir -p /data/coolify/images/{avatars,project-icons}chown -R 9999:root /data/coolify/images@@ -316,8 +343,16 @@ nohup bash -c "filog 'Running docker compose up...'- docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-docker.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --env-file /data/coolify/source/.env \${COMPOSE_FILES} up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1- log 'Docker compose up completed'+ if docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-docker.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --env-file /data/coolify/source/.env \${COMPOSE_FILES} up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1; then+ log 'Docker compose up completed'+ elif [ \"\$(docker inspect --format '{{.State.Running}}' coolify 2>/dev/null)\" = 'true' ]; then+ # A slow first start (for example long migrations) can pass the wait timeout. Coolify is still starting.+ log 'Docker compose up did not finish waiting, but the coolify container is running'+ else+ log 'ERROR: Docker compose up failed and the coolify container is not running'+ write_status 'error' \"Coolify did not start. Check the upgrade log: \$LOGFILE\"+ exit 1+ fi# Final log entryecho '' >>\"\$LOGFILE\"diff --git a/scripts/upgrade.sh b/scripts/upgrade.shindex 987ce636a0..1e1e23cadd 100644--- a/scripts/upgrade.sh+++ b/scripts/upgrade.sh@@ -38,6 +38,47 @@ write_status() {echo "${step}|${message}|$(date -Iseconds)" > "$STATUS_FILE"}+# Print the first gateway of a network that Docker reports with a prefix length (fd00::1/64).+# Docker Engine 27.0 to 28.0.0 does this for IPv6 until the daemon restarts (https://github.com/moby/moby/pull/49520).+# The Docker CLI and Compose in the helper image fail on such a network with a ParseAddr error.+network_gateway_with_prefix() {+ docker network inspect "$1" --format '{{range .IPAM.Config}}{{.Gateway}} {{end}}' 2>&1 | grep -oE '[0-9A-Fa-f:.]+/[0-9]+' | head -n1+}++# Make sure the coolify network exists and the helper image can read it.+# It is created when starting Coolify with docker compose+ensure_coolify_network() {+ log "Checking Docker network 'coolify'..."+ if ! docker network inspect coolify >/dev/null 2>&1; then+ log "Network 'coolify' does not exist, creating..."+ if ! docker network create --attachable --ipv6 coolify 2>/dev/null; then+ log "Failed to create network with IPv6, trying without IPv6..."+ docker network create --attachable coolify 2>/dev/null+ log "Network 'coolify' created without IPv6"+ elif [ -n "$(network_gateway_with_prefix coolify)" ]; then+ # No container uses the new network yet, so it is safe to recreate it.+ log "Docker reports the IPv6 gateway of network 'coolify' with a prefix length, recreating it without IPv6..."+ docker network rm coolify >/dev/null 2>&1+ docker network create --attachable coolify 2>/dev/null+ log "Network 'coolify' created without IPv6"+ else+ log "Network 'coolify' created with IPv6 support"+ fi+ return 0+ fi++ log "Network 'coolify' already exists"+ local gateway+ gateway=$(network_gateway_with_prefix coolify)+ if [ -n "$gateway" ]; then+ local message="Docker Engine $(docker version --format '{{.Server.Version}}' 2>/dev/null) reports the gateway of the 'coolify' network as ${gateway}, which Coolify cannot read (Docker Engine bug, fixed in 28.0.1). Run 'systemctl restart docker' (this restarts all containers) or upgrade Docker Engine, then upgrade Coolify again."+ log "ERROR: ${message}"+ write_status "error" "$message"+ echo " ERROR: ${message}"+ return 1+ fi+}+# Pull an image. Retry with backoff, because the registry can rate limit many instances that pull at the same time.pull_image() {local image="$1"@@ -177,21 +218,7 @@ filog "Environment variables check complete"echo " Done."-# Make sure coolify network exists-# It is created when starting Coolify with docker compose-log "Checking Docker network 'coolify'..."-if ! docker network inspect coolify >/dev/null 2>&1; then- log "Network 'coolify' does not exist, creating..."- if ! docker network create --attachable --ipv6 coolify 2>/dev/null; then- log "Failed to create network with IPv6, trying without IPv6..."- docker network create --attachable coolify 2>/dev/null- log "Network 'coolify' created without IPv6"- else- log "Network 'coolify' created with IPv6 support"- fi-else- log "Network 'coolify' already exists"-fi+ensure_coolify_network || exit 1mkdir -p /data/coolify/images/{avatars,project-icons}chown -R 9999:root /data/coolify/images@@ -316,8 +343,16 @@ nohup bash -c "filog 'Running docker compose up...'- docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-docker.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --env-file /data/coolify/source/.env \${COMPOSE_FILES} up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1- log 'Docker compose up completed'+ if docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-docker.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --env-file /data/coolify/source/.env \${COMPOSE_FILES} up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1; then+ log 'Docker compose up completed'+ elif [ \"\$(docker inspect --format '{{.State.Running}}' coolify 2>/dev/null)\" = 'true' ]; then+ # A slow first start (for example long migrations) can pass the wait timeout. Coolify is still starting.+ log 'Docker compose up did not finish waiting, but the coolify container is running'+ else+ log 'ERROR: Docker compose up failed and the coolify container is not running'+ write_status 'error' \"Coolify did not start. Check the upgrade log: \$LOGFILE\"+ exit 1+ fi# Final log entryecho '' >>\"\$LOGFILE\"diff --git a/tests/Feature/DeploymentSensitiveCommandFailureTest.php b/tests/Feature/DeploymentSensitiveCommandFailureTest.phpindex c6d104c9cb..a879ee286e 100644--- a/tests/Feature/DeploymentSensitiveCommandFailureTest.php+++ b/tests/Feature/DeploymentSensitiveCommandFailureTest.php@@ -256,6 +256,33 @@ it('keeps the command in the error message for failed commands that are not sens->not->toContain('[command hidden because it contains sensitive data]');});+it('explains how to fix a Docker network that Docker Engine reports with an IPv6 gateway prefix', function () {+ [$job] = makeSensitiveCommandFailureJob();++ Process::fake(['*' => Process::result(errorOutput: 'ParseAddr("fde4:75c5:b9d4::1/64"): unexpected character, want colon (at "/64")', exitCode: 1)]);++ $exception = captureDeploymentException(fn () => $job->execute_remote_command([+ 'command' => 'docker compose up -d',+ ]));++ expect($exception->getMessage())+ ->toContain('ParseAddr("fde4:75c5:b9d4::1/64")')+ ->toContain('systemctl restart docker')+ ->toContain('28.0.1');+});++it('does not add the Docker network hint to other failures', function () {+ [$job] = makeSensitiveCommandFailureJob();++ Process::fake(['*' => Process::result(errorOutput: 'ParseAddr("fde4:75c5:b9d4::1 "): unexpected character', exitCode: 1)]);++ $exception = captureDeploymentException(fn () => $job->execute_remote_command([+ 'command' => 'docker compose up -d',+ ]));++ expect($exception->getMessage())->not->toContain('systemctl restart docker');+});+it('redacts locked values from the error output of failed commands', function () {[$job] = makeSensitiveCommandFailureJob([['key' => 'LOCKED_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_shown_once' => true],diff --git a/tests/Unit/UpgradeScriptContainerStartTest.php b/tests/Unit/UpgradeScriptContainerStartTest.phpnew file mode 100644index 0000000000..efc16ad510--- /dev/null+++ b/tests/Unit/UpgradeScriptContainerStartTest.php@@ -0,0 +1,63 @@+<?php++/**+ * Runs the detached container restart block of the upgrade script with a fake docker.+ *+ * @return array{exitCode: int, status: string}+ */+function runUpgradeScriptContainerStart(string $script, int $composeExitCode, bool $coolifyRunning): array+{+ $dir = sys_get_temp_dir().'/upgrade-start-'.bin2hex(random_bytes(4));+ mkdir($dir);+ touch("{$dir}/status");+ $source = file_get_contents(getcwd().'/'.$script);+ preg_match('/^nohup bash -c "\n(.*?)^" >>"\$LOGFILE" 2>&1 &$/ms', $source, $match);+ $running = $coolifyRunning ? 'true' : 'false';++ $harness = <<<BASH+ LOGFILE="{$dir}/upgrade.log"+ STATUS_FILE="{$dir}/status"+ DOCKER_CONFIG_MOUNT=''+ REGISTRY_URL='docker.io'+ LATEST_HELPER_VERSION='1.0.17'+ LATEST_IMAGE='4.4.7'+ docker() {+ case "\$1" in+ run) return {$composeExitCode} ;;+ inspect) echo {$running} ;;+ esac+ }+ sleep() { :; }+ export -f docker sleep+ bash -c "+ {$match[1]}"+ BASH;++ exec('bash -c '.escapeshellarg($harness).' >/dev/null 2>&1', result_code: $exitCode);+ $status = is_file("{$dir}/status") ? file_get_contents("{$dir}/status") : '';+ exec('rm -rf '.escapeshellarg($dir));++ return ['exitCode' => $exitCode, 'status' => $status];+}++it('reports an error when the new Coolify containers do not start', function (string $script) {+ $result = runUpgradeScriptContainerStart($script, composeExitCode: 1, coolifyRunning: false);++ expect($result['exitCode'])->toBe(1)+ ->and($result['status'])->toStartWith('error|')+ ->toContain('upgrade log');+})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);++it('completes the upgrade when the coolify container runs but is not healthy within the wait timeout', function (string $script) {+ $result = runUpgradeScriptContainerStart($script, composeExitCode: 1, coolifyRunning: true);++ expect($result['exitCode'])->toBe(0)+ ->and($result['status'])->toBe('');+})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);++it('completes the upgrade when docker compose up succeeds', function (string $script) {+ $result = runUpgradeScriptContainerStart($script, composeExitCode: 0, coolifyRunning: true);++ expect($result['exitCode'])->toBe(0)+ ->and($result['status'])->toBe('');+})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);diff --git a/tests/Unit/UpgradeScriptCoolifyNetworkTest.php b/tests/Unit/UpgradeScriptCoolifyNetworkTest.phpnew file mode 100644index 0000000000..ab3d497730--- /dev/null+++ b/tests/Unit/UpgradeScriptCoolifyNetworkTest.php@@ -0,0 +1,93 @@+<?php++/**+ * Runs ensure_coolify_network() from the upgrade script with a fake docker. Docker Engine 27.x reports an IPv6+ * gateway with its prefix length (fd00:1::1/64), which the Docker CLI and Compose in the helper image cannot parse.+ *+ * @return array{exitCode: int, calls: list<string>, status: string}+ */+function runUpgradeScriptEnsureCoolifyNetwork(string $script, ?string $existingGateways, string $createdIpv6Gateway): array+{+ $dir = sys_get_temp_dir().'/upgrade-network-'.bin2hex(random_bytes(4));+ mkdir($dir);+ touch("{$dir}/calls");+ touch("{$dir}/status");+ if ($existingGateways !== null) {+ file_put_contents("{$dir}/gateways", $existingGateways);+ }+ $source = file_get_contents(getcwd().'/'.$script);+ $functions = collect(['log', 'write_status', 'network_gateway_with_prefix', 'ensure_coolify_network'])+ ->map(function (string $name) use ($source): string {+ preg_match('/^'.$name.'\(\) \{.*?^\}\n/ms', $source, $match);++ return $match[0] ?? '';+ })+ ->implode('');++ $harness = <<<BASH+ LOGFILE="{$dir}/upgrade.log"+ STATUS_FILE="{$dir}/status"+ docker() {+ echo "\$*" >> "{$dir}/calls"+ case "\$1 \$2" in+ "network inspect")+ [ -f "{$dir}/gateways" ] || { echo "Error: network coolify not found" >&2; return 1; }+ cat "{$dir}/gateways"+ ;;+ "network create")+ case "\$*" in+ *--ipv6*) echo "172.19.0.1 {$createdIpv6Gateway} " > "{$dir}/gateways" ;;+ *) echo "172.19.0.1 " > "{$dir}/gateways" ;;+ esac+ ;;+ "network rm") rm -f "{$dir}/gateways" ;;+ version*) echo 27.5.1 ;;+ esac+ }+ {$functions}+ ensure_coolify_network+ BASH;++ exec('bash -c '.escapeshellarg($harness).' >/dev/null 2>&1', result_code: $exitCode);+ $calls = array_values(array_filter(explode("\n", file_get_contents("{$dir}/calls")), fn (string $call): bool => $call !== '' && ! str_starts_with($call, 'network inspect') && ! str_starts_with($call, 'version')));+ $status = file_get_contents("{$dir}/status");+ exec('rm -rf '.escapeshellarg($dir));++ return ['exitCode' => $exitCode, 'calls' => $calls, 'status' => $status];+}++it('stops the upgrade before any container changes when the coolify network has a gateway with a prefix length', function (string $script) {+ $result = runUpgradeScriptEnsureCoolifyNetwork($script, existingGateways: '172.19.0.1 fd00:1::1/64 ', createdIpv6Gateway: 'unused');++ expect($result['exitCode'])->toBe(1)+ ->and($result['calls'])->toBe([])+ ->and($result['status'])->toStartWith('error|')+ ->toContain('fd00:1::1/64')+ ->toContain('systemctl restart docker');+})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);++it('keeps a readable coolify network', function (string $script) {+ $result = runUpgradeScriptEnsureCoolifyNetwork($script, existingGateways: '172.19.0.1 fd00:1::1 ', createdIpv6Gateway: 'unused');++ expect($result)->toBe(['exitCode' => 0, 'calls' => [], 'status' => '']);+})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);++it('creates the coolify network with IPv6', function (string $script) {+ $result = runUpgradeScriptEnsureCoolifyNetwork($script, existingGateways: null, createdIpv6Gateway: 'fd00:1::1');++ expect($result)->toBe(['exitCode' => 0, 'calls' => ['network create --attachable --ipv6 coolify'], 'status' => '']);+})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);++it('recreates a new coolify network without IPv6 when Docker reports its gateway with a prefix length', function (string $script) {+ $result = runUpgradeScriptEnsureCoolifyNetwork($script, existingGateways: null, createdIpv6Gateway: 'fd00:1::1/64');++ expect($result)->toBe([+ 'exitCode' => 0,+ 'calls' => [+ 'network create --attachable --ipv6 coolify',+ 'network rm coolify',+ 'network create --attachable coolify',+ ],+ 'status' => '',+ ]);+})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);