Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(upgrade): keep Coolify running when Docker reports IPv6 gateways with a prefix

Andras Bacsai420692c02053
diff --git a/.ai/lessons.md b/.ai/lessons.md
index c2137bdb38..edfabe8dcf 100644
--- a/.ai/lessons.md
+++ b/.ai/lessons.md
@@ -13,6 +13,7 @@
- Redirect browser test output to a file (`> /tmp/x.log 2>&1`); piping it (`| tail`) hangs because the Playwright server keeps the pipe open.
- Browser `click('Text')` can match a hidden element first (for example inactive modal steps) and wait forever; use `button:visible:has-text("Text")`.
- Never run `php artisan test` inside a dev container (`./scripts/dev exec` or `docker exec coolify-dev-*`). The container sets `DB_CONNECTION=pgsql` as a real environment variable, which wins over `phpunit.xml`, so `RefreshDatabase` runs `migrate:fresh` on the dev database and wipes it. Run tests on the host only.
+- The helper image runs `docker compose` for deployments and for the Coolify upgrade itself. Before you bump its Docker CLI or Compose, test them against the oldest supported Docker Engine (for example a `docker:<version>-dind` container); newer clients can reject data from older daemons.
- Do not symlink `vendor` into another worktree. Composer resolves the base path from the real `vendor` path, so the tests load the app classes of the other checkout. Run `composer install --ignore-platform-req=ext-zip` in the worktree.
- Call `visit()` directly in each `tests/v4/Browser` test body; Pest does not mark a test that only uses helper-wrapped `visit()` as a browser test, so it fails with `sendText() on null`.
diff --git a/app/Traits/ExecuteRemoteCommand.php b/app/Traits/ExecuteRemoteCommand.php
index 449c9c55dd..57b19f91b8 100644
--- a/app/Traits/ExecuteRemoteCommand.php
+++ b/app/Traits/ExecuteRemoteCommand.php
@@ -268,7 +268,22 @@ trait ExecuteRemoteCommand
$error = $this->redact_sensitive_info($error);
- return "Command execution failed (exit code {$exitCode}): {$commandText}\nError: {$error}";
+ return "Command execution failed (exit code {$exitCode}): {$commandText}\nError: {$error}".$this->dockerNetworkGatewayHint($error);
+ }
+
+ /**
+ * Docker Engine 27.0 to 28.0.0 reports an IPv6 network gateway with its prefix length (fd00::1/64) until the
+ * daemon restarts, and newer Docker CLI and Compose versions fail to parse it.
+ *
+ * @see https://github.com/moby/moby/pull/49520
+ */
+ private function dockerNetworkGatewayHint(string $error): string
+ {
+ if (! preg_match('~ParseAddr\("[0-9A-Fa-f:.]+/\d+"\)~', $error)) {
+ return '';
+ }
+
+ return "\nDocker Engine on this server reports a Docker network gateway with a prefix length, which is a Docker Engine bug fixed in 28.0.1. Run 'systemctl restart docker' on the server (this restarts all containers) or upgrade Docker Engine, then deploy again.";
}
/**
diff --git a/other/nightly/upgrade.sh b/other/nightly/upgrade.sh
index 88dd7ea093..4d594cb9e9 100644
--- a/other/nightly/upgrade.sh
+++ b/other/nightly/upgrade.sh
@@ -38,6 +38,47 @@ write_status() {
echo "${step}|${message}|$(date -Iseconds)" > "$STATUS_FILE"
}
+# Print the first gateway of a network that Docker reports with a prefix length (fd00::1/64).
+# Docker Engine 27.0 to 28.0.0 does this for IPv6 until the daemon restarts (https://github.com/moby/moby/pull/49520).
+# The Docker CLI and Compose in the helper image fail on such a network with a ParseAddr error.
+network_gateway_with_prefix() {
+ docker network inspect "$1" --format '{{range .IPAM.Config}}{{.Gateway}} {{end}}' 2>&1 | grep -oE '[0-9A-Fa-f:.]+/[0-9]+' | head -n1
+}
+
+# Make sure the coolify network exists and the helper image can read it.
+# It is created when starting Coolify with docker compose
+ensure_coolify_network() {
+ log "Checking Docker network 'coolify'..."
+ if ! docker network inspect coolify >/dev/null 2>&1; then
+ log "Network 'coolify' does not exist, creating..."
+ if ! docker network create --attachable --ipv6 coolify 2>/dev/null; then
+ log "Failed to create network with IPv6, trying without IPv6..."
+ docker network create --attachable coolify 2>/dev/null
+ log "Network 'coolify' created without IPv6"
+ elif [ -n "$(network_gateway_with_prefix coolify)" ]; then
+ # No container uses the new network yet, so it is safe to recreate it.
+ log "Docker reports the IPv6 gateway of network 'coolify' with a prefix length, recreating it without IPv6..."
+ docker network rm coolify >/dev/null 2>&1
+ docker network create --attachable coolify 2>/dev/null
+ log "Network 'coolify' created without IPv6"
+ else
+ log "Network 'coolify' created with IPv6 support"
+ fi
+ return 0
+ fi
+
+ log "Network 'coolify' already exists"
+ local gateway
+ gateway=$(network_gateway_with_prefix coolify)
+ if [ -n "$gateway" ]; then
+ local message="Docker Engine $(docker version --format '{{.Server.Version}}' 2>/dev/null) reports the gateway of the 'coolify' network as ${gateway}, which Coolify cannot read (Docker Engine bug, fixed in 28.0.1). Run 'systemctl restart docker' (this restarts all containers) or upgrade Docker Engine, then upgrade Coolify again."
+ log "ERROR: ${message}"
+ write_status "error" "$message"
+ echo " ERROR: ${message}"
+ return 1
+ fi
+}
+
# Pull an image. Retry with backoff, because the registry can rate limit many instances that pull at the same time.
pull_image() {
local image="$1"
@@ -177,21 +218,7 @@ fi
log "Environment variables check complete"
echo " Done."
-# Make sure coolify network exists
-# It is created when starting Coolify with docker compose
-log "Checking Docker network 'coolify'..."
-if ! docker network inspect coolify >/dev/null 2>&1; then
- log "Network 'coolify' does not exist, creating..."
- if ! docker network create --attachable --ipv6 coolify 2>/dev/null; then
- log "Failed to create network with IPv6, trying without IPv6..."
- docker network create --attachable coolify 2>/dev/null
- log "Network 'coolify' created without IPv6"
- else
- log "Network 'coolify' created with IPv6 support"
- fi
-else
- log "Network 'coolify' already exists"
-fi
+ensure_coolify_network || exit 1
mkdir -p /data/coolify/images/{avatars,project-icons}
chown -R 9999:root /data/coolify/images
@@ -316,8 +343,16 @@ nohup bash -c "
fi
log 'Running docker compose up...'
- docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-docker.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --env-file /data/coolify/source/.env \${COMPOSE_FILES} up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1
- log 'Docker compose up completed'
+ if docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-docker.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --env-file /data/coolify/source/.env \${COMPOSE_FILES} up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1; then
+ log 'Docker compose up completed'
+ elif [ \"\$(docker inspect --format '{{.State.Running}}' coolify 2>/dev/null)\" = 'true' ]; then
+ # A slow first start (for example long migrations) can pass the wait timeout. Coolify is still starting.
+ log 'Docker compose up did not finish waiting, but the coolify container is running'
+ else
+ log 'ERROR: Docker compose up failed and the coolify container is not running'
+ write_status 'error' \"Coolify did not start. Check the upgrade log: \$LOGFILE\"
+ exit 1
+ fi
# Final log entry
echo '' >>\"\$LOGFILE\"
diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh
index 987ce636a0..1e1e23cadd 100644
--- a/scripts/upgrade.sh
+++ b/scripts/upgrade.sh
@@ -38,6 +38,47 @@ write_status() {
echo "${step}|${message}|$(date -Iseconds)" > "$STATUS_FILE"
}
+# Print the first gateway of a network that Docker reports with a prefix length (fd00::1/64).
+# Docker Engine 27.0 to 28.0.0 does this for IPv6 until the daemon restarts (https://github.com/moby/moby/pull/49520).
+# The Docker CLI and Compose in the helper image fail on such a network with a ParseAddr error.
+network_gateway_with_prefix() {
+ docker network inspect "$1" --format '{{range .IPAM.Config}}{{.Gateway}} {{end}}' 2>&1 | grep -oE '[0-9A-Fa-f:.]+/[0-9]+' | head -n1
+}
+
+# Make sure the coolify network exists and the helper image can read it.
+# It is created when starting Coolify with docker compose
+ensure_coolify_network() {
+ log "Checking Docker network 'coolify'..."
+ if ! docker network inspect coolify >/dev/null 2>&1; then
+ log "Network 'coolify' does not exist, creating..."
+ if ! docker network create --attachable --ipv6 coolify 2>/dev/null; then
+ log "Failed to create network with IPv6, trying without IPv6..."
+ docker network create --attachable coolify 2>/dev/null
+ log "Network 'coolify' created without IPv6"
+ elif [ -n "$(network_gateway_with_prefix coolify)" ]; then
+ # No container uses the new network yet, so it is safe to recreate it.
+ log "Docker reports the IPv6 gateway of network 'coolify' with a prefix length, recreating it without IPv6..."
+ docker network rm coolify >/dev/null 2>&1
+ docker network create --attachable coolify 2>/dev/null
+ log "Network 'coolify' created without IPv6"
+ else
+ log "Network 'coolify' created with IPv6 support"
+ fi
+ return 0
+ fi
+
+ log "Network 'coolify' already exists"
+ local gateway
+ gateway=$(network_gateway_with_prefix coolify)
+ if [ -n "$gateway" ]; then
+ local message="Docker Engine $(docker version --format '{{.Server.Version}}' 2>/dev/null) reports the gateway of the 'coolify' network as ${gateway}, which Coolify cannot read (Docker Engine bug, fixed in 28.0.1). Run 'systemctl restart docker' (this restarts all containers) or upgrade Docker Engine, then upgrade Coolify again."
+ log "ERROR: ${message}"
+ write_status "error" "$message"
+ echo " ERROR: ${message}"
+ return 1
+ fi
+}
+
# Pull an image. Retry with backoff, because the registry can rate limit many instances that pull at the same time.
pull_image() {
local image="$1"
@@ -177,21 +218,7 @@ fi
log "Environment variables check complete"
echo " Done."
-# Make sure coolify network exists
-# It is created when starting Coolify with docker compose
-log "Checking Docker network 'coolify'..."
-if ! docker network inspect coolify >/dev/null 2>&1; then
- log "Network 'coolify' does not exist, creating..."
- if ! docker network create --attachable --ipv6 coolify 2>/dev/null; then
- log "Failed to create network with IPv6, trying without IPv6..."
- docker network create --attachable coolify 2>/dev/null
- log "Network 'coolify' created without IPv6"
- else
- log "Network 'coolify' created with IPv6 support"
- fi
-else
- log "Network 'coolify' already exists"
-fi
+ensure_coolify_network || exit 1
mkdir -p /data/coolify/images/{avatars,project-icons}
chown -R 9999:root /data/coolify/images
@@ -316,8 +343,16 @@ nohup bash -c "
fi
log 'Running docker compose up...'
- docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-docker.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --env-file /data/coolify/source/.env \${COMPOSE_FILES} up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1
- log 'Docker compose up completed'
+ if docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-docker.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --env-file /data/coolify/source/.env \${COMPOSE_FILES} up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1; then
+ log 'Docker compose up completed'
+ elif [ \"\$(docker inspect --format '{{.State.Running}}' coolify 2>/dev/null)\" = 'true' ]; then
+ # A slow first start (for example long migrations) can pass the wait timeout. Coolify is still starting.
+ log 'Docker compose up did not finish waiting, but the coolify container is running'
+ else
+ log 'ERROR: Docker compose up failed and the coolify container is not running'
+ write_status 'error' \"Coolify did not start. Check the upgrade log: \$LOGFILE\"
+ exit 1
+ fi
# Final log entry
echo '' >>\"\$LOGFILE\"
diff --git a/tests/Feature/DeploymentSensitiveCommandFailureTest.php b/tests/Feature/DeploymentSensitiveCommandFailureTest.php
index c6d104c9cb..a879ee286e 100644
--- a/tests/Feature/DeploymentSensitiveCommandFailureTest.php
+++ b/tests/Feature/DeploymentSensitiveCommandFailureTest.php
@@ -256,6 +256,33 @@ it('keeps the command in the error message for failed commands that are not sens
->not->toContain('[command hidden because it contains sensitive data]');
});
+it('explains how to fix a Docker network that Docker Engine reports with an IPv6 gateway prefix', function () {
+ [$job] = makeSensitiveCommandFailureJob();
+
+ Process::fake(['*' => Process::result(errorOutput: 'ParseAddr("fde4:75c5:b9d4::1/64"): unexpected character, want colon (at "/64")', exitCode: 1)]);
+
+ $exception = captureDeploymentException(fn () => $job->execute_remote_command([
+ 'command' => 'docker compose up -d',
+ ]));
+
+ expect($exception->getMessage())
+ ->toContain('ParseAddr("fde4:75c5:b9d4::1/64")')
+ ->toContain('systemctl restart docker')
+ ->toContain('28.0.1');
+});
+
+it('does not add the Docker network hint to other failures', function () {
+ [$job] = makeSensitiveCommandFailureJob();
+
+ Process::fake(['*' => Process::result(errorOutput: 'ParseAddr("fde4:75c5:b9d4::1 "): unexpected character', exitCode: 1)]);
+
+ $exception = captureDeploymentException(fn () => $job->execute_remote_command([
+ 'command' => 'docker compose up -d',
+ ]));
+
+ expect($exception->getMessage())->not->toContain('systemctl restart docker');
+});
+
it('redacts locked values from the error output of failed commands', function () {
[$job] = makeSensitiveCommandFailureJob([
['key' => 'LOCKED_SECRET', 'value' => SENSITIVE_FAILURE_SECRET, 'is_shown_once' => true],
diff --git a/tests/Unit/UpgradeScriptContainerStartTest.php b/tests/Unit/UpgradeScriptContainerStartTest.php
new file mode 100644
index 0000000000..efc16ad510
--- /dev/null
+++ b/tests/Unit/UpgradeScriptContainerStartTest.php
@@ -0,0 +1,63 @@
+<?php
+
+/**
+ * Runs the detached container restart block of the upgrade script with a fake docker.
+ *
+ * @return array{exitCode: int, status: string}
+ */
+function runUpgradeScriptContainerStart(string $script, int $composeExitCode, bool $coolifyRunning): array
+{
+ $dir = sys_get_temp_dir().'/upgrade-start-'.bin2hex(random_bytes(4));
+ mkdir($dir);
+ touch("{$dir}/status");
+ $source = file_get_contents(getcwd().'/'.$script);
+ preg_match('/^nohup bash -c "\n(.*?)^" >>"\$LOGFILE" 2>&1 &$/ms', $source, $match);
+ $running = $coolifyRunning ? 'true' : 'false';
+
+ $harness = <<<BASH
+ LOGFILE="{$dir}/upgrade.log"
+ STATUS_FILE="{$dir}/status"
+ DOCKER_CONFIG_MOUNT=''
+ REGISTRY_URL='docker.io'
+ LATEST_HELPER_VERSION='1.0.17'
+ LATEST_IMAGE='4.4.7'
+ docker() {
+ case "\$1" in
+ run) return {$composeExitCode} ;;
+ inspect) echo {$running} ;;
+ esac
+ }
+ sleep() { :; }
+ export -f docker sleep
+ bash -c "
+ {$match[1]}"
+ BASH;
+
+ exec('bash -c '.escapeshellarg($harness).' >/dev/null 2>&1', result_code: $exitCode);
+ $status = is_file("{$dir}/status") ? file_get_contents("{$dir}/status") : '';
+ exec('rm -rf '.escapeshellarg($dir));
+
+ return ['exitCode' => $exitCode, 'status' => $status];
+}
+
+it('reports an error when the new Coolify containers do not start', function (string $script) {
+ $result = runUpgradeScriptContainerStart($script, composeExitCode: 1, coolifyRunning: false);
+
+ expect($result['exitCode'])->toBe(1)
+ ->and($result['status'])->toStartWith('error|')
+ ->toContain('upgrade log');
+})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);
+
+it('completes the upgrade when the coolify container runs but is not healthy within the wait timeout', function (string $script) {
+ $result = runUpgradeScriptContainerStart($script, composeExitCode: 1, coolifyRunning: true);
+
+ expect($result['exitCode'])->toBe(0)
+ ->and($result['status'])->toBe('');
+})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);
+
+it('completes the upgrade when docker compose up succeeds', function (string $script) {
+ $result = runUpgradeScriptContainerStart($script, composeExitCode: 0, coolifyRunning: true);
+
+ expect($result['exitCode'])->toBe(0)
+ ->and($result['status'])->toBe('');
+})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);
diff --git a/tests/Unit/UpgradeScriptCoolifyNetworkTest.php b/tests/Unit/UpgradeScriptCoolifyNetworkTest.php
new file mode 100644
index 0000000000..ab3d497730
--- /dev/null
+++ b/tests/Unit/UpgradeScriptCoolifyNetworkTest.php
@@ -0,0 +1,93 @@
+<?php
+
+/**
+ * Runs ensure_coolify_network() from the upgrade script with a fake docker. Docker Engine 27.x reports an IPv6
+ * gateway with its prefix length (fd00:1::1/64), which the Docker CLI and Compose in the helper image cannot parse.
+ *
+ * @return array{exitCode: int, calls: list<string>, status: string}
+ */
+function runUpgradeScriptEnsureCoolifyNetwork(string $script, ?string $existingGateways, string $createdIpv6Gateway): array
+{
+ $dir = sys_get_temp_dir().'/upgrade-network-'.bin2hex(random_bytes(4));
+ mkdir($dir);
+ touch("{$dir}/calls");
+ touch("{$dir}/status");
+ if ($existingGateways !== null) {
+ file_put_contents("{$dir}/gateways", $existingGateways);
+ }
+ $source = file_get_contents(getcwd().'/'.$script);
+ $functions = collect(['log', 'write_status', 'network_gateway_with_prefix', 'ensure_coolify_network'])
+ ->map(function (string $name) use ($source): string {
+ preg_match('/^'.$name.'\(\) \{.*?^\}\n/ms', $source, $match);
+
+ return $match[0] ?? '';
+ })
+ ->implode('');
+
+ $harness = <<<BASH
+ LOGFILE="{$dir}/upgrade.log"
+ STATUS_FILE="{$dir}/status"
+ docker() {
+ echo "\$*" >> "{$dir}/calls"
+ case "\$1 \$2" in
+ "network inspect")
+ [ -f "{$dir}/gateways" ] || { echo "Error: network coolify not found" >&2; return 1; }
+ cat "{$dir}/gateways"
+ ;;
+ "network create")
+ case "\$*" in
+ *--ipv6*) echo "172.19.0.1 {$createdIpv6Gateway} " > "{$dir}/gateways" ;;
+ *) echo "172.19.0.1 " > "{$dir}/gateways" ;;
+ esac
+ ;;
+ "network rm") rm -f "{$dir}/gateways" ;;
+ version*) echo 27.5.1 ;;
+ esac
+ }
+ {$functions}
+ ensure_coolify_network
+ BASH;
+
+ exec('bash -c '.escapeshellarg($harness).' >/dev/null 2>&1', result_code: $exitCode);
+ $calls = array_values(array_filter(explode("\n", file_get_contents("{$dir}/calls")), fn (string $call): bool => $call !== '' && ! str_starts_with($call, 'network inspect') && ! str_starts_with($call, 'version')));
+ $status = file_get_contents("{$dir}/status");
+ exec('rm -rf '.escapeshellarg($dir));
+
+ return ['exitCode' => $exitCode, 'calls' => $calls, 'status' => $status];
+}
+
+it('stops the upgrade before any container changes when the coolify network has a gateway with a prefix length', function (string $script) {
+ $result = runUpgradeScriptEnsureCoolifyNetwork($script, existingGateways: '172.19.0.1 fd00:1::1/64 ', createdIpv6Gateway: 'unused');
+
+ expect($result['exitCode'])->toBe(1)
+ ->and($result['calls'])->toBe([])
+ ->and($result['status'])->toStartWith('error|')
+ ->toContain('fd00:1::1/64')
+ ->toContain('systemctl restart docker');
+})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);
+
+it('keeps a readable coolify network', function (string $script) {
+ $result = runUpgradeScriptEnsureCoolifyNetwork($script, existingGateways: '172.19.0.1 fd00:1::1 ', createdIpv6Gateway: 'unused');
+
+ expect($result)->toBe(['exitCode' => 0, 'calls' => [], 'status' => '']);
+})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);
+
+it('creates the coolify network with IPv6', function (string $script) {
+ $result = runUpgradeScriptEnsureCoolifyNetwork($script, existingGateways: null, createdIpv6Gateway: 'fd00:1::1');
+
+ expect($result)->toBe(['exitCode' => 0, 'calls' => ['network create --attachable --ipv6 coolify'], 'status' => '']);
+})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);
+
+it('recreates a new coolify network without IPv6 when Docker reports its gateway with a prefix length', function (string $script) {
+ $result = runUpgradeScriptEnsureCoolifyNetwork($script, existingGateways: null, createdIpv6Gateway: 'fd00:1::1/64');
+
+ expect($result)->toBe([
+ 'exitCode' => 0,
+ 'calls' => [
+ 'network create --attachable --ipv6 coolify',
+ 'network rm coolify',
+ 'network create --attachable coolify',
+ ],
+ 'status' => '',
+ ]);
+})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);