Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(deploy): stop 502 errors during rolling updates

Andras Bacsaia3712f8902f2
diff --git a/.ai/lessons.md b/.ai/lessons.md
index 00dc3cbd89..79f47a68e6 100644
--- a/.ai/lessons.md
+++ b/.ai/lessons.md
@@ -25,6 +25,7 @@
- The helper image runs `docker compose` for deployments and for the Coolify upgrade itself. Before you bump its Docker CLI or Compose, test them against the oldest supported Docker Engine (for example a `docker:<version>-dind` container); newer clients can reject data from older daemons.
- Do not symlink `vendor` into another worktree. Composer resolves the base path from the real `vendor` path, so the tests load the app classes of the other checkout. Run `composer install --ignore-platform-req=ext-zip` in the worktree.
- To assert that the unsaved-changes bar shows, check `el.classList.contains('is-dirty')`. Playwright treats the hidden bar (`opacity-0`) as visible, and its class attribute always contains the text `is-dirty` from Tailwind variants.
+- For load tests against a dev VM, use your own instance (`COOLIFY_DEV_INSTANCE=<name> COOLIFY_DEV_SIDE_BY_SIDE=1 ./scripts/dev start`); other sessions can reseed the localhost VM of a shared instance. The 1 GiB VM drops packets when its conntrack table is full (`dmesg`: `nf_conntrack: table full`), so raise `net.netfilter.nf_conntrack_max` on it and run an idle load first to prove a clean baseline.
- Do not call `sleep()` in a browser test. The app server runs in the test process, so `sleep()` stops it; use `$page->wait()`.
- `Queue::fake()` also captures `dispatchSync()` of a `ShouldQueue` job, so the job never runs. Fake only the jobs to block (`Queue::fake([SomeJob::class])`).
- In tests, turning on `is_metrics_enabled` (and other Sentinel settings) with `save()`/`update()` runs the settings hook, which starts Sentinel over a real SSH connection on the sync queue. Use `saveQuietly()` or `Queue::fake()`.
diff --git a/app/Http/Controllers/Api/ApplicationsController.php b/app/Http/Controllers/Api/ApplicationsController.php
index 4baeefd913..eb3c02685f 100644
--- a/app/Http/Controllers/Api/ApplicationsController.php
+++ b/app/Http/Controllers/Api/ApplicationsController.php
@@ -56,6 +56,7 @@ class ApplicationsController extends Controller
'include_coolify_default_variables',
'is_pr_deployments_public_enabled',
'stop_grace_period',
+ 'rolling_update_drain_seconds',
'docker_images_to_keep',
'is_gzip_enabled',
'is_stripprefix_enabled',
@@ -468,6 +469,7 @@ class ApplicationsController extends Controller
'include_coolify_default_variables' => ['type' => 'boolean', 'default' => true, 'description' => 'Add the predefined COOLIFY_* variables (COOLIFY_URL, COOLIFY_FQDN, ...) to the environment. Requires team admin permissions. Redeploy to apply it.'],
'is_pr_deployments_public_enabled' => ['type' => 'boolean', 'description' => 'Make pull request deployments public.'],
'stop_grace_period' => ['type' => 'integer', 'nullable' => true, 'minimum' => 1, 'maximum' => 3600, 'description' => 'Container stop grace period in seconds.'],
+ 'rolling_update_drain_seconds' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 300, 'description' => 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.'],
'docker_images_to_keep' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 100, 'description' => 'Number of Docker images to retain.'],
'is_gzip_enabled' => ['type' => 'boolean', 'description' => 'Enable gzip compression.'],
'is_stripprefix_enabled' => ['type' => 'boolean', 'description' => 'Enable path prefix stripping.'],
@@ -677,6 +679,7 @@ class ApplicationsController extends Controller
'include_coolify_default_variables' => ['type' => 'boolean', 'default' => true, 'description' => 'Add the predefined COOLIFY_* variables (COOLIFY_URL, COOLIFY_FQDN, ...) to the environment. Requires team admin permissions. Redeploy to apply it.'],
'is_pr_deployments_public_enabled' => ['type' => 'boolean', 'description' => 'Make pull request deployments public.'],
'stop_grace_period' => ['type' => 'integer', 'nullable' => true, 'minimum' => 1, 'maximum' => 3600, 'description' => 'Container stop grace period in seconds.'],
+ 'rolling_update_drain_seconds' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 300, 'description' => 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.'],
'docker_images_to_keep' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 100, 'description' => 'Number of Docker images to retain.'],
'is_gzip_enabled' => ['type' => 'boolean', 'description' => 'Enable gzip compression.'],
'is_stripprefix_enabled' => ['type' => 'boolean', 'description' => 'Enable path prefix stripping.'],
@@ -886,6 +889,7 @@ class ApplicationsController extends Controller
'include_coolify_default_variables' => ['type' => 'boolean', 'default' => true, 'description' => 'Add the predefined COOLIFY_* variables (COOLIFY_URL, COOLIFY_FQDN, ...) to the environment. Requires team admin permissions. Redeploy to apply it.'],
'is_pr_deployments_public_enabled' => ['type' => 'boolean', 'description' => 'Make pull request deployments public.'],
'stop_grace_period' => ['type' => 'integer', 'nullable' => true, 'minimum' => 1, 'maximum' => 3600, 'description' => 'Container stop grace period in seconds.'],
+ 'rolling_update_drain_seconds' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 300, 'description' => 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.'],
'docker_images_to_keep' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 100, 'description' => 'Number of Docker images to retain.'],
'is_gzip_enabled' => ['type' => 'boolean', 'description' => 'Enable gzip compression.'],
'is_stripprefix_enabled' => ['type' => 'boolean', 'description' => 'Enable path prefix stripping.'],
@@ -1065,6 +1069,7 @@ class ApplicationsController extends Controller
'include_coolify_default_variables' => ['type' => 'boolean', 'default' => true, 'description' => 'Add the predefined COOLIFY_* variables (COOLIFY_URL, COOLIFY_FQDN, ...) to the environment. Requires team admin permissions. Redeploy to apply it.'],
'is_pr_deployments_public_enabled' => ['type' => 'boolean', 'description' => 'Make pull request deployments public.'],
'stop_grace_period' => ['type' => 'integer', 'nullable' => true, 'minimum' => 1, 'maximum' => 3600, 'description' => 'Container stop grace period in seconds.'],
+ 'rolling_update_drain_seconds' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 300, 'description' => 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.'],
'docker_images_to_keep' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 100, 'description' => 'Number of Docker images to retain.'],
'is_gzip_enabled' => ['type' => 'boolean', 'description' => 'Enable gzip compression.'],
'is_stripprefix_enabled' => ['type' => 'boolean', 'description' => 'Enable path prefix stripping.'],
@@ -1240,6 +1245,7 @@ class ApplicationsController extends Controller
'include_coolify_default_variables' => ['type' => 'boolean', 'default' => true, 'description' => 'Add the predefined COOLIFY_* variables (COOLIFY_URL, COOLIFY_FQDN, ...) to the environment. Requires team admin permissions. Redeploy to apply it.'],
'is_pr_deployments_public_enabled' => ['type' => 'boolean', 'description' => 'Make pull request deployments public.'],
'stop_grace_period' => ['type' => 'integer', 'nullable' => true, 'minimum' => 1, 'maximum' => 3600, 'description' => 'Container stop grace period in seconds.'],
+ 'rolling_update_drain_seconds' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 300, 'description' => 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.'],
'docker_images_to_keep' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 100, 'description' => 'Number of Docker images to retain.'],
'is_gzip_enabled' => ['type' => 'boolean', 'description' => 'Enable gzip compression.'],
'is_stripprefix_enabled' => ['type' => 'boolean', 'description' => 'Enable path prefix stripping.'],
@@ -3261,6 +3267,7 @@ class ApplicationsController extends Controller
'include_coolify_default_variables' => ['type' => 'boolean', 'default' => true, 'description' => 'Add the predefined COOLIFY_* variables (COOLIFY_URL, COOLIFY_FQDN, ...) to the environment. Requires team admin permissions. Redeploy to apply it.'],
'is_pr_deployments_public_enabled' => ['type' => 'boolean', 'description' => 'Make pull request deployments public.'],
'stop_grace_period' => ['type' => 'integer', 'nullable' => true, 'minimum' => 1, 'maximum' => 3600, 'description' => 'Container stop grace period in seconds.'],
+ 'rolling_update_drain_seconds' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 300, 'description' => 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.'],
'docker_images_to_keep' => ['type' => 'integer', 'minimum' => 0, 'maximum' => 100, 'description' => 'Number of Docker images to retain.'],
'is_gzip_enabled' => ['type' => 'boolean', 'description' => 'Enable gzip compression.'],
'is_stripprefix_enabled' => ['type' => 'boolean', 'description' => 'Enable path prefix stripping.'],
diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php
index 42b2fc4831..92d5917eaf 100644
--- a/app/Jobs/ApplicationDeploymentJob.php
+++ b/app/Jobs/ApplicationDeploymentJob.php
@@ -2709,6 +2709,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
$this->application_deployment_queue->addLogEntry('Rolling update started.');
$this->start_by_compose_file();
$this->health_check();
+ $this->wait_for_proxy_to_route_new_container();
$this->stop_running_container();
$this->application_deployment_queue->addLogEntry('Rolling update completed.');
}
@@ -2718,6 +2719,46 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
}
}
+ /**
+ * The proxy adds the new container some seconds after it is healthy. Stopping the old
+ * container before that sends requests to a closed port (502), see #8627.
+ */
+ private function wait_for_proxy_to_route_new_container(): void
+ {
+ $seconds = $this->application->settings->rollingUpdateDrainSeconds();
+ if ($seconds === 0 || ! $this->newVersionIsHealthy()) {
+ return;
+ }
+ $oldContainerNames = getCurrentApplicationContainerStatus($this->server, $this->application, $this->pull_request_id)
+ ->filter(fn ($container) => data_get($container, 'State') === 'running' && data_get($container, 'Names') !== $this->container_name)
+ ->pluck('Names');
+ if ($oldContainerNames->isEmpty()) {
+ return;
+ }
+ if ($this->proxyLabelsChanged($oldContainerNames)) {
+ // The proxy drops a route that two containers define differently until the old one stops.
+ $this->application_deployment_queue->addLogEntry('The proxy labels changed, so the old containers stop without waiting.');
+
+ return;
+ }
+ $this->application_deployment_queue->addLogEntry("Waiting {$seconds} seconds for the proxy to send traffic to the new container.");
+ Sleep::for($seconds)->seconds();
+ }
+
+ private function proxyLabelsChanged(Collection $oldContainerNames): bool
+ {
+ $containerNames = $oldContainerNames->prepend($this->container_name)->map(fn (string $name) => escapeshellarg($name))->implode(' ');
+ $output = instant_remote_process(["docker inspect --format '{{json .Config.Labels}}' {$containerNames}"], $this->server, false);
+
+ return collect(preg_split('/\R/', trim((string) $output)))
+ ->map(fn (string $labels) => collect(json_decode($labels, true) ?: [])
+ ->filter(fn ($value, string $key) => str_starts_with($key, 'traefik.') || str_starts_with($key, 'caddy'))
+ ->sortKeys()
+ ->all())
+ ->unique()
+ ->count() > 1;
+ }
+
private function health_check()
{
try {
diff --git a/app/Livewire/Project/Application/Advanced.php b/app/Livewire/Project/Application/Advanced.php
index 0075bdddd4..3f3d95573a 100644
--- a/app/Livewire/Project/Application/Advanced.php
+++ b/app/Livewire/Project/Application/Advanced.php
@@ -68,6 +68,8 @@ class Advanced extends Component
#[Validate(['string', 'nullable'])]
public ?string $stopGracePeriod = null;
+ public ?int $rollingUpdateDrainSeconds = DEFAULT_ROLLING_UPDATE_DRAIN_SECONDS;
+
#[Validate(['boolean'])]
public bool $isBuildServerEnabled = false;
@@ -170,6 +172,7 @@ class Advanced extends Component
// Load stop_grace_period separately since it has its own save handler
// Convert null to empty string to prevent dirty detection issues
$this->stopGracePeriod = $this->application->settings->stop_grace_period ?? '';
+ $this->rollingUpdateDrainSeconds = $this->application->settings->rollingUpdateDrainSeconds();
}
private function syncCertificateResolverFromModel(): void
@@ -432,6 +435,30 @@ class Advanced extends Component
}
}
+ public function saveRollingUpdateDrainSeconds()
+ {
+ try {
+ $this->authorize('update', $this->application);
+
+ $validated = $this->validate(
+ ['rollingUpdateDrainSeconds' => ['required', 'integer', 'min:0', 'max:'.MAX_ROLLING_UPDATE_DRAIN_SECONDS]],
+ [],
+ ['rollingUpdateDrainSeconds' => 'rolling update drain time']
+ );
+
+ $this->application->settings->rolling_update_drain_seconds = $validated['rollingUpdateDrainSeconds'];
+ $changedFields = array_keys($this->application->settings->getDirty());
+ $this->application->settings->save();
+ $this->auditSettingsUpdate($changedFields);
+
+ $this->dispatch('success', 'Rolling update drain time updated.');
+ } catch (ValidationException $e) {
+ throw $e;
+ } catch (\Throwable $e) {
+ return handleError($e, $this);
+ }
+ }
+
public function saveMaxRestartCount()
{
try {
diff --git a/app/Models/ApplicationSetting.php b/app/Models/ApplicationSetting.php
index abaf67312e..7d2c5a849a 100644
--- a/app/Models/ApplicationSetting.php
+++ b/app/Models/ApplicationSetting.php
@@ -52,6 +52,7 @@ use Symfony\Component\Yaml\Yaml;
'include_source_commit_in_build' => ['type' => 'boolean'],
'docker_images_to_keep' => ['type' => 'integer'],
'stop_grace_period' => ['type' => 'integer', 'nullable' => true],
+ 'rolling_update_drain_seconds' => ['type' => 'integer', 'description' => 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off.'],
]
)]
class ApplicationSetting extends Model
@@ -84,6 +85,7 @@ class ApplicationSetting extends Model
'is_git_shallow_clone_enabled' => 'boolean',
'docker_images_to_keep' => 'integer',
'stop_grace_period' => 'integer',
+ 'rolling_update_drain_seconds' => 'integer',
'is_log_drain_enabled' => 'boolean',
'is_gpu_enabled' => 'boolean',
'is_include_timestamps' => 'boolean',
@@ -142,6 +144,7 @@ class ApplicationSetting extends Model
'include_source_commit_in_build',
'docker_images_to_keep',
'stop_grace_period',
+ 'rolling_update_drain_seconds',
];
/**
@@ -168,6 +171,11 @@ class ApplicationSetting extends Model
return DEFAULT_STOP_GRACE_PERIOD_SECONDS;
}
+ public function rollingUpdateDrainSeconds(): int
+ {
+ return min(max($this->rolling_update_drain_seconds ?? DEFAULT_ROLLING_UPDATE_DRAIN_SECONDS, 0), MAX_ROLLING_UPDATE_DRAIN_SECONDS);
+ }
+
public function deploymentStopGracePeriodSeconds(): int
{
if (isDev() && $this->stop_grace_period === null) {
diff --git a/bootstrap/helpers/api.php b/bootstrap/helpers/api.php
index d6f05c8095..c4c5eb7a27 100644
--- a/bootstrap/helpers/api.php
+++ b/bootstrap/helpers/api.php
@@ -228,6 +228,7 @@ function sharedDataApplications(?Application $application = null): array
'preview_url_template' => 'string',
'max_restart_count' => 'integer|min:0',
'stop_grace_period' => 'nullable|integer|min:'.MIN_STOP_GRACE_PERIOD_SECONDS.'|max:'.MAX_STOP_GRACE_PERIOD_SECONDS,
+ 'rolling_update_drain_seconds' => 'integer|min:0|max:'.MAX_ROLLING_UPDATE_DRAIN_SECONDS,
'docker_images_to_keep' => 'integer|min:0|max:100',
'static_image' => Rule::enum(StaticImageTypes::class),
'domains' => ValidationPatterns::applicationDomainRules(),
@@ -490,6 +491,7 @@ function removeUnnecessaryFieldsFromRequest(Request $request)
$request->offsetUnset('include_coolify_default_variables');
$request->offsetUnset('is_pr_deployments_public_enabled');
$request->offsetUnset('stop_grace_period');
+ $request->offsetUnset('rolling_update_drain_seconds');
$request->offsetUnset('docker_images_to_keep');
$request->offsetUnset('is_gzip_enabled');
$request->offsetUnset('is_stripprefix_enabled');
diff --git a/bootstrap/helpers/constants.php b/bootstrap/helpers/constants.php
index 6dd1155a20..a235c9b680 100644
--- a/bootstrap/helpers/constants.php
+++ b/bootstrap/helpers/constants.php
@@ -63,6 +63,8 @@ const RESTART_MODE = 'unless-stopped';
const DEFAULT_STOP_GRACE_PERIOD_SECONDS = 30;
const MIN_STOP_GRACE_PERIOD_SECONDS = 1;
const MAX_STOP_GRACE_PERIOD_SECONDS = 3600;
+const DEFAULT_ROLLING_UPDATE_DRAIN_SECONDS = 5;
+const MAX_ROLLING_UPDATE_DRAIN_SECONDS = 300;
const DATABASE_DOCKER_IMAGES = [
'bitnami/mariadb',
diff --git a/bootstrap/helpers/docker.php b/bootstrap/helpers/docker.php
index c27eb4270d..4676b1e754 100644
--- a/bootstrap/helpers/docker.php
+++ b/bootstrap/helpers/docker.php
@@ -787,7 +787,7 @@ function caddySiteAddress(string $scheme, string $host, bool $isForceHttpsEnable
return "{$scheme}://{$host}";
}
-function fqdnLabelsForCaddy(string $network, string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null, ?bool $is_gzip_enabled = true, ?bool $is_stripprefix_enabled = true, ?string $service_name = null, ?string $image = null, string $redirect_direction = 'both', ?string $predefinedPort = null, bool $is_http_basic_auth_enabled = false, ?string $http_basic_auth_username = null, ?string $http_basic_auth_password = null, ?Collection $noindex_domains = null, bool $is_traffic_analytics_enabled = false, array $domainPortOverrides = [], bool $supports_log_append = false, bool $supports_basic_auth_directive = false, ?int $http_basic_auth_bcrypt_cost = null, array $http_basic_auth_argon2id_options = [], bool $is_permanent_redirect_enabled = false)
+function fqdnLabelsForCaddy(string $network, string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null, ?bool $is_gzip_enabled = true, ?bool $is_stripprefix_enabled = true, ?string $service_name = null, ?string $image = null, string $redirect_direction = 'both', ?string $predefinedPort = null, bool $is_http_basic_auth_enabled = false, ?string $http_basic_auth_username = null, ?string $http_basic_auth_password = null, ?Collection $noindex_domains = null, bool $is_traffic_analytics_enabled = false, array $domainPortOverrides = [], bool $supports_log_append = false, bool $supports_basic_auth_directive = false, ?int $http_basic_auth_bcrypt_cost = null, array $http_basic_auth_argon2id_options = [], bool $is_permanent_redirect_enabled = false, bool $is_retry_enabled = false)
{
$labels = collect([]);
if ($serviceLabels) {
@@ -840,6 +840,11 @@ function fqdnLabelsForCaddy(string $network, string $uuid, Collection $domains,
} else {
$labels->push("caddy_{$loop}.{$handle}.{$loop}_reverse_proxy={{upstreams}}");
}
+ if ($is_retry_enabled) {
+ // A failed connection, for example to the old container that stops during a rolling
+ // update, is retried on another upstream. Caddy always retries a connection that failed.
+ $labels->push("caddy_{$loop}.{$handle}.{$loop}_reverse_proxy.lb_try_duration=5s");
+ }
$labels->push("caddy_{$loop}.{$handle}={$path}*");
if ($is_gzip_enabled) {
$labels->push("caddy_{$loop}.encode=zstd gzip");
@@ -944,7 +949,7 @@ function traefikHostRulesWithoutIpv6Brackets(Collection $labels): Collection
) : $label);
}
-function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null, ?bool $is_gzip_enabled = true, ?bool $is_stripprefix_enabled = true, ?string $service_name = null, bool $generate_unique_uuid = false, ?string $image = null, string $redirect_direction = 'both', bool $is_http_basic_auth_enabled = false, ?string $http_basic_auth_username = null, ?string $http_basic_auth_password = null, ?Collection $noindex_domains = null, bool $escape_redirect_replacement_for_compose = true, array $domainPortOverrides = [], ?string $certificate_resolver = 'letsencrypt', ?int $http_basic_auth_bcrypt_cost = null, bool $is_permanent_redirect_enabled = false)
+function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_https_enabled = false, $onlyPort = null, ?Collection $serviceLabels = null, ?bool $is_gzip_enabled = true, ?bool $is_stripprefix_enabled = true, ?string $service_name = null, bool $generate_unique_uuid = false, ?string $image = null, string $redirect_direction = 'both', bool $is_http_basic_auth_enabled = false, ?string $http_basic_auth_username = null, ?string $http_basic_auth_password = null, ?Collection $noindex_domains = null, bool $escape_redirect_replacement_for_compose = true, array $domainPortOverrides = [], ?string $certificate_resolver = 'letsencrypt', ?int $http_basic_auth_bcrypt_cost = null, bool $is_permanent_redirect_enabled = false, bool $is_retry_enabled = false)
{
$labels = collect([]);
$labels->push('traefik.enable=true');
@@ -999,6 +1004,14 @@ function fqdnLabelsForTraefik(string $uuid, Collection $domains, bool $is_force_
->filter()
->unique();
}
+ if ($is_retry_enabled) {
+ // Sends a request to another container when the connection fails, for example to the old
+ // container that stops during a rolling update. Traefik does not retry after a response.
+ // A retry can pick the old container again under load; 3 attempts left about 5% of the
+ // requests failing in a live test, 5 attempts less than 1% (#8627).
+ $labels->push("traefik.http.middlewares.retry-{$uuid}.retry.attempts=5");
+ $middlewares_from_labels->push("retry-{$uuid}");
+ }
foreach ($domains as $loop => $domain) {
try {
if ($generate_unique_uuid) {
@@ -1264,6 +1277,7 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview
http_basic_auth_bcrypt_cost: $application->http_basic_auth_bcrypt_cost,
noindex_domains: $noindexDomains,
domainPortOverrides: $application->domain_port_overrides ?? [],
+ is_retry_enabled: true,
certificate_resolver: $application->destination->server->traefikCertificateResolver($application->settings?->traefik_certificate_resolver),
));
break;
@@ -1288,6 +1302,7 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview
supports_log_append: $application->destination->server->caddySupportsLogAppend(),
supports_basic_auth_directive: $application->destination->server->caddySupportsBasicAuthDirective(),
domainPortOverrides: $application->domain_port_overrides ?? [],
+ is_retry_enabled: true,
));
break;
}
@@ -1308,6 +1323,7 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview
noindex_domains: $noindexDomains,
escape_redirect_replacement_for_compose: false,
domainPortOverrides: $application->domain_port_overrides ?? [],
+ is_retry_enabled: true,
certificate_resolver: $application->destination->server->traefikCertificateResolver($application->settings?->traefik_certificate_resolver),
));
$labels = $labels->merge(fqdnLabelsForCaddy(
@@ -1330,6 +1346,7 @@ function generateLabelsApplication(Application $application, ?ApplicationPreview
supports_log_append: $application->destination->server->caddySupportsLogAppend(),
supports_basic_auth_directive: $application->destination->server->caddySupportsBasicAuthDirective(),
domainPortOverrides: $application->domain_port_overrides ?? [],
+ is_retry_enabled: true,
));
}
}
diff --git a/database/migrations/2026_10_23_224202_add_rolling_update_drain_seconds_to_application_settings.php b/database/migrations/2026_10_23_224202_add_rolling_update_drain_seconds_to_application_settings.php
new file mode 100644
index 0000000000..88edb7081f
--- /dev/null
+++ b/database/migrations/2026_10_23_224202_add_rolling_update_drain_seconds_to_application_settings.php
@@ -0,0 +1,31 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+ /**
+ * Run the migrations.
+ */
+ public function up(): void
+ {
+ Schema::table('application_settings', function (Blueprint $table) {
+ $table->integer('rolling_update_drain_seconds')
+ ->default(5)
+ ->after('stop_grace_period')
+ ->comment('Seconds to wait after the new container is healthy before old containers stop during a rolling update (0-300). 0 turns the wait off.');
+ });
+ }
+
+ /**
+ * Reverse the migrations.
+ */
+ public function down(): void
+ {
+ Schema::table('application_settings', function (Blueprint $table) {
+ $table->dropColumn('rolling_update_drain_seconds');
+ });
+ }
+};
diff --git a/docs/server-transfer-full-coverage.md b/docs/server-transfer-full-coverage.md
index 9aa62f2b6b..3c39e0e15d 100644
--- a/docs/server-transfer-full-coverage.md
+++ b/docs/server-transfer-full-coverage.md
@@ -96,7 +96,7 @@ When you add a model or a setting to the transfer, add it to this file too.
| ✅ | Limits | memory, swap, swappiness, reservation, CPUs, cpuset, CPU shares |
| ➕ | Advanced | custom labels, custom docker run options, pre/post deployment command + container, manual webhook secrets |
| ✅ | Maintenance | maintenance mode on + custom maintenance page |
-| ✅ | All application settings that are not the default | git submodules, LFS, shallow clone, auto deploy off, force HTTPS, debug, preview deployments on, PR deployments public, log drain, GPU (driver, count, device IDs, options), include timestamps, swarm-only worker nodes, build server, consistent container name, gzip off, strip prefix off, connect to Docker network, custom internal name, container label escape off, env sorting, read-only labels, preserve repository, disable build cache, use build secrets, stop grace period 30, inject build args off, include source commit, Docker images to keep 5, custom container name prefix, redact env values in logs, Coolify default variables off, Traefik certificate resolver |
+| ✅ | All application settings that are not the default | git submodules, LFS, shallow clone, auto deploy off, force HTTPS, debug, preview deployments on, PR deployments public, log drain, GPU (driver, count, device IDs, options), include timestamps, swarm-only worker nodes, build server, consistent container name, gzip off, strip prefix off, connect to Docker network, custom internal name, container label escape off, env sorting, read-only labels, preserve repository, disable build cache, use build secrets, stop grace period 30, rolling update drain time 12, inject build args off, include source commit, Docker images to keep 5, custom container name prefix, redact env values in logs, Coolify default variables off, Traefik certificate resolver |
| ✅ | Environment variables | normal, multiline, literal, shown once, build time only, runtime only, required, with comment, preview-only, and references to `{{team.*}}`, `{{project.*}}`, `{{environment.*}}`, `{{server.*}}` |
| ✅ | Persistent volume | the seed has one; add a host-path bind mount and preview suffix on |
| ✅ | File mount + directory mount | content, chown, chmod |
diff --git a/openapi.json b/openapi.json
index 7bd46ddba5..7af9db1b47 100644
--- a/openapi.json
+++ b/openapi.json
@@ -914,6 +914,12 @@
"maximum": 3600,
"description": "Container stop grace period in seconds."
},
+ "rolling_update_drain_seconds": {
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 300,
+ "description": "Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5."
+ },
"docker_images_to_keep": {
"type": "integer",
"minimum": 0,
@@ -1566,6 +1572,12 @@
"maximum": 3600,
"description": "Container stop grace period in seconds."
},
+ "rolling_update_drain_seconds": {
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 300,
+ "description": "Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5."
+ },
"docker_images_to_keep": {
"type": "integer",
"minimum": 0,
@@ -2218,6 +2230,12 @@
"maximum": 3600,
"description": "Container stop grace period in seconds."
},
+ "rolling_update_drain_seconds": {
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 300,
+ "description": "Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5."
+ },
"docker_images_to_keep": {
"type": "integer",
"minimum": 0,
@@ -2760,6 +2778,12 @@
"maximum": 3600,
"description": "Container stop grace period in seconds."
},
+ "rolling_update_drain_seconds": {
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 300,
+ "description": "Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5."
+ },
"docker_images_to_keep": {
"type": "integer",
"minimum": 0,
@@ -3282,6 +3306,12 @@
"maximum": 3600,
"description": "Container stop grace period in seconds."
},
+ "rolling_update_drain_seconds": {
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 300,
+ "description": "Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5."
+ },
"docker_images_to_keep": {
"type": "integer",
"minimum": 0,
@@ -4067,6 +4097,12 @@
"maximum": 3600,
"description": "Container stop grace period in seconds."
},
+ "rolling_update_drain_seconds": {
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 300,
+ "description": "Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5."
+ },
"docker_images_to_keep": {
"type": "integer",
"minimum": 0,
@@ -25104,6 +25140,10 @@
"stop_grace_period": {
"type": "integer",
"nullable": true
+ },
+ "rolling_update_drain_seconds": {
+ "type": "integer",
+ "description": "Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off."
}
},
"type": "object"
diff --git a/openapi.yaml b/openapi.yaml
index 5cfd3c35c8..4018755766 100644
--- a/openapi.yaml
+++ b/openapi.yaml
@@ -594,6 +594,11 @@ paths:
minimum: 1
maximum: 3600
description: 'Container stop grace period in seconds.'
+ rolling_update_drain_seconds:
+ type: integer
+ minimum: 0
+ maximum: 300
+ description: 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.'
docker_images_to_keep:
type: integer
minimum: 0
@@ -1027,6 +1032,11 @@ paths:
minimum: 1
maximum: 3600
description: 'Container stop grace period in seconds.'
+ rolling_update_drain_seconds:
+ type: integer
+ minimum: 0
+ maximum: 300
+ description: 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.'
docker_images_to_keep:
type: integer
minimum: 0
@@ -1460,6 +1470,11 @@ paths:
minimum: 1
maximum: 3600
description: 'Container stop grace period in seconds.'
+ rolling_update_drain_seconds:
+ type: integer
+ minimum: 0
+ maximum: 300
+ description: 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.'
docker_images_to_keep:
type: integer
minimum: 0
@@ -1830,6 +1845,11 @@ paths:
minimum: 1
maximum: 3600
description: 'Container stop grace period in seconds.'
+ rolling_update_drain_seconds:
+ type: integer
+ minimum: 0
+ maximum: 300
+ description: 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.'
docker_images_to_keep:
type: integer
minimum: 0
@@ -2186,6 +2206,11 @@ paths:
minimum: 1
maximum: 3600
description: 'Container stop grace period in seconds.'
+ rolling_update_drain_seconds:
+ type: integer
+ minimum: 0
+ maximum: 300
+ description: 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.'
docker_images_to_keep:
type: integer
minimum: 0
@@ -2709,6 +2734,11 @@ paths:
minimum: 1
maximum: 3600
description: 'Container stop grace period in seconds.'
+ rolling_update_drain_seconds:
+ type: integer
+ minimum: 0
+ maximum: 300
+ description: 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off. Default: 5.'
docker_images_to_keep:
type: integer
minimum: 0
@@ -16039,6 +16069,9 @@ components:
stop_grace_period:
type: integer
nullable: true
+ rolling_update_drain_seconds:
+ type: integer
+ description: 'Seconds to wait after the new container is healthy before old containers stop during a rolling update. 0 turns the wait off.'
type: object
Environment:
description: 'Environment model'
diff --git a/resources/views/livewire/project/application/advanced.blade.php b/resources/views/livewire/project/application/advanced.blade.php
index 8fe5f6eefa..cbf9197f86 100644
--- a/resources/views/livewire/project/application/advanced.blade.php
+++ b/resources/views/livewire/project/application/advanced.blade.php
@@ -126,6 +126,13 @@
helper="How long to wait for graceful shutdown during rolling updates, manual stops, and restarts. Applies to all containers for this application. Saved automatically. Default: {{ DEFAULT_STOP_GRACE_PERIOD_SECONDS }} seconds. Range: {{ MIN_STOP_GRACE_PERIOD_SECONDS }}-{{ MAX_STOP_GRACE_PERIOD_SECONDS }} seconds (1 hour)."
min="{{ MIN_STOP_GRACE_PERIOD_SECONDS }}" max="{{ MAX_STOP_GRACE_PERIOD_SECONDS }}"
canGate="update" :canResource="$application" />
+ @if ($application->build_pack !== 'dockercompose')
+ <x-forms.input type="number" min="0" max="{{ MAX_ROLLING_UPDATE_DRAIN_SECONDS }}"
+ id="rollingUpdateDrainSeconds" label="Rolling update drain time (seconds)"
+ wire:change="saveRollingUpdateDrainSeconds"
+ helper="How long to wait after the new container is healthy before the old containers stop during a rolling update. The proxy needs this time to send traffic to the new container. Saved automatically. Default: {{ DEFAULT_ROLLING_UPDATE_DRAIN_SECONDS }} seconds. Range: 0-{{ MAX_ROLLING_UPDATE_DRAIN_SECONDS }} seconds. Set to 0 to turn off the wait."
+ canGate="update" :canResource="$application" />
+ @endif
<x-forms.input type="number" min="0" id="maxRestartCount" label="Max restart count"
wire:change="saveMaxRestartCount"
helper="Maximum number of restarts before Coolify automatically stops the application and sends a notification. Saved automatically. Set to 0 to disable the limit."
diff --git a/tests/Feature/Api/ApplicationSettingsApiTest.php b/tests/Feature/Api/ApplicationSettingsApiTest.php
index 8698c2bd94..ae75c81e26 100644
--- a/tests/Feature/Api/ApplicationSettingsApiTest.php
+++ b/tests/Feature/Api/ApplicationSettingsApiTest.php
@@ -55,6 +55,7 @@ function recommendedApplicationSettingsPayload(): array
'is_env_sorting_enabled' => true,
'is_pr_deployments_public_enabled' => true,
'stop_grace_period' => 45,
+ 'rolling_update_drain_seconds' => 12,
'docker_images_to_keep' => 7,
'is_gzip_enabled' => false,
'is_stripprefix_enabled' => false,
@@ -345,6 +346,18 @@ test('validates stop grace period bounds', function (int $stopGracePeriod) {
'above maximum' => 3601,
]);
+test('validates rolling update drain time bounds', function (int $seconds) {
+ $this->withHeaders(applicationSettingsApiHeaders($this->bearerToken))
+ ->patchJson("/api/v1/applications/{$this->application->uuid}", [
+ 'rolling_update_drain_seconds' => $seconds,
+ ])
+ ->assertUnprocessable()
+ ->assertJsonValidationErrors('rolling_update_drain_seconds');
+})->with([
+ 'below minimum' => -1,
+ 'above maximum' => 301,
+]);
+
test('validates Docker image retention bounds', function (int $dockerImagesToKeep) {
$this->withHeaders(applicationSettingsApiHeaders($this->bearerToken))
->patchJson("/api/v1/applications/{$this->application->uuid}", [
diff --git a/tests/Feature/RollingUpdateProxyDrainTest.php b/tests/Feature/RollingUpdateProxyDrainTest.php
new file mode 100644
index 0000000000..2c95ddf943
--- /dev/null
+++ b/tests/Feature/RollingUpdateProxyDrainTest.php
@@ -0,0 +1,214 @@
+<?php
+
+use App\Enums\ProxyTypes;
+use App\Jobs\ApplicationDeploymentJob;
+use App\Livewire\Project\Application\Advanced;
+use App\Models\Application;
+use App\Models\ApplicationDeploymentQueue;
+use App\Models\ApplicationPreview;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\PrivateKey;
+use App\Models\Project;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
+use App\Models\User;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Sleep;
+use Livewire\Livewire;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+ Server::flushIdentityMap();
+ InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
+ config(['constants.ssh.mux_enabled' => false]);
+});
+
+class TestableRollingUpdateDrainDeploymentJob extends ApplicationDeploymentJob
+{
+ public function __construct() {}
+}
+
+function createApplicationForRollingUpdateDrainTest(string $proxyType = 'TRAEFIK', bool $exactLabels = false): Application
+{
+ $team = Team::factory()->create();
+ $project = Project::factory()->create(['team_id' => $team->id]);
+ $environment = Environment::factory()->create(['project_id' => $project->id]);
+ $server = Server::factory()->create([
+ 'team_id' => $team->id,
+ 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id,
+ 'proxy' => ['type' => $proxyType],
+ ]);
+ $server->settings->update(['generate_exact_labels' => $exactLabels]);
+ $destination = StandaloneDocker::query()->where('server_id', $server->id)->firstOrFail();
+
+ return Application::factory()->createOne([
+ 'environment_id' => $environment->id,
+ 'destination_id' => $destination->id,
+ 'destination_type' => $destination->getMorphClass(),
+ 'fqdn' => 'https://example.com',
+ 'ports_exposes' => '3000',
+ ])->fresh();
+}
+
+/**
+ * Runs the drain step of a rolling update against a faked server and returns the deployment log lines.
+ *
+ * @param array<string, string>|null $oldProxyLabels labels of the running old container, null when there is none
+ * @param array<string, string> $newProxyLabels
+ * @return list<string>
+ */
+function runRollingUpdateDrainStep(Application $application, bool $newVersionIsHealthy, ?array $oldProxyLabels = ['traefik.enable' => 'true'], array $newProxyLabels = ['traefik.enable' => 'true']): array
+{
+ Process::fake(function ($process) use ($application, $oldProxyLabels, $newProxyLabels) {
+ $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+ if (str_contains($command, 'docker inspect')) {
+ return Process::result(output: json_encode(['coolify.name' => 'new', ...$newProxyLabels])."\n".json_encode(['coolify.name' => 'old', ...$oldProxyLabels]));
+ }
+ if (str_contains($command, 'docker ps') && str_contains($command, "coolify.applicationUuid={$application->uuid}")) {
+ $containers = [['ID' => 'new-id', 'Names' => 'app-new', 'State' => 'running', 'Labels' => '']];
+ if ($oldProxyLabels !== null) {
+ $containers[] = ['ID' => 'old-id', 'Names' => 'app-old', 'State' => 'running', 'Labels' => ''];
+ }
+
+ return Process::result(output: collect($containers)->map(fn (array $container) => json_encode($container))->implode("\n"));
+ }
+
+ return Process::result(output: '');
+ });
+
+ $logs = [];
+ $queue = Mockery::mock(ApplicationDeploymentQueue::class)->makePartial();
+ $queue->shouldReceive('addLogEntry')->andReturnUsing(function (string $message) use (&$logs) {
+ $logs[] = $message;
+ });
+
+ $job = new TestableRollingUpdateDrainDeploymentJob;
+ $reflection = new ReflectionClass(ApplicationDeploymentJob::class);
+ foreach ([
+ 'application' => $application->fresh(),
+ 'application_deployment_queue' => $queue,
+ 'deployment_uuid' => 'rolling-update-drain',
+ 'server' => $application->destination->server,
+ 'container_name' => 'app-new',
+ 'pull_request_id' => 0,
+ ] as $property => $value) {
+ $reflection->getProperty($property)->setValue($job, $value);
+ }
+
+ Cache::forget('deployment-new-version-healthy:rolling-update-drain');
+ if ($newVersionIsHealthy) {
+ Cache::put('deployment-new-version-healthy:rolling-update-drain', true);
+ }
+
+ $reflection->getMethod('wait_for_proxy_to_route_new_container')->invoke($job);
+
+ return $logs;
+}
+
+it('retries failed connections on another container in Traefik application labels', function (bool $exactLabels) {
+ $application = createApplicationForRollingUpdateDrainTest(ProxyTypes::TRAEFIK->value, $exactLabels);
+
+ $labels = collect(generateLabelsApplication($application));
+
+ expect($labels)->toContain("traefik.http.middlewares.retry-{$application->uuid}.retry.attempts=5");
+ $routerMiddlewares = $labels->filter(fn (string $label) => str_contains($label, "routers.https-0-{$application->uuid}.middlewares="));
+ expect($routerMiddlewares)->toHaveCount(1)
+ ->and($routerMiddlewares->first())->toContain("retry-{$application->uuid}");
+})->with([
+ 'generic labels' => false,
+ 'exact labels' => true,
+]);
+
+it('retries failed connections on another upstream in Caddy application labels', function () {
+ $application = createApplicationForRollingUpdateDrainTest(ProxyTypes::CADDY->value, exactLabels: true);
+
+ expect(generateLabelsApplication($application))
+ ->toContain('caddy_0.handle_path.0_reverse_proxy.lb_try_duration=5s');
+});
+
+it('does not add retry labels to preview deployments', function () {
+ $application = createApplicationForRollingUpdateDrainTest();
+ $preview = ApplicationPreview::forceCreate([
+ 'application_id' => $application->id,
+ 'pull_request_id' => 7,
+ 'pull_request_html_url' => 'https://github.com/coollabsio/coolify/pull/7',
+ 'fqdn' => 'https://pr-7.example.com',
+ ]);
+
+ expect(implode("\n", generateLabelsApplication($application, $preview)))
+ ->not->toContain('retry')
+ ->not->toContain('lb_try_duration');
+});
+
+it('waits for the proxy before old containers stop when the new container is healthy', function () {
+ Sleep::fake();
+ $application = createApplicationForRollingUpdateDrainTest();
+
+ $logs = runRollingUpdateDrainStep($application, newVersionIsHealthy: true);
+
+ Sleep::assertSequence([Sleep::for(DEFAULT_ROLLING_UPDATE_DRAIN_SECONDS)->seconds()]);
+ expect($logs)->toBe(['Waiting 5 seconds for the proxy to send traffic to the new container.']);
+});
+
+it('does not wait when the drain time is 0 or the new container is not healthy', function (int $drainSeconds, bool $newVersionIsHealthy) {
+ Sleep::fake();
+ $application = createApplicationForRollingUpdateDrainTest();
+ $application->settings->update(['rolling_update_drain_seconds' => $drainSeconds]);
+
+ $logs = runRollingUpdateDrainStep($application, $newVersionIsHealthy);
+
+ Sleep::assertNeverSlept();
+ expect($logs)->toBe([]);
+})->with([
+ 'drain turned off' => [0, true],
+ 'new container not healthy' => [5, false],
+]);
+
+it('does not wait when no old container runs', function () {
+ Sleep::fake();
+ $application = createApplicationForRollingUpdateDrainTest();
+
+ $logs = runRollingUpdateDrainStep($application, newVersionIsHealthy: true, oldProxyLabels: null);
+
+ Sleep::assertNeverSlept();
+ expect($logs)->toBe([]);
+});
+
+it('stops the old containers without waiting when the proxy labels changed', function () {
+ Sleep::fake();
+ $application = createApplicationForRollingUpdateDrainTest();
+
+ $logs = runRollingUpdateDrainStep(
+ $application,
+ newVersionIsHealthy: true,
+ oldProxyLabels: ['traefik.http.routers.http-0-app.middlewares' => 'gzip-sse'],
+ newProxyLabels: ['traefik.http.routers.http-0-app.middlewares' => 'gzip-sse,retry-app'],
+ );
+
+ Sleep::assertNeverSlept();
+ expect($logs)->toBe(['The proxy labels changed, so the old containers stop without waiting.']);
+});
+
+it('saves the rolling update drain time on the advanced page', function () {
+ $user = User::factory()->create();
+ $application = createApplicationForRollingUpdateDrainTest();
+ $application->team()->members()->attach($user->id, ['role' => 'owner']);
+ $this->actingAs($user);
+ session(['currentTeam' => $application->team()]);
+
+ Livewire::test(Advanced::class, ['application' => $application])
+ ->assertSet('rollingUpdateDrainSeconds', DEFAULT_ROLLING_UPDATE_DRAIN_SECONDS)
+ ->set('rollingUpdateDrainSeconds', 20)
+ ->call('saveRollingUpdateDrainSeconds')
+ ->assertHasNoErrors()
+ ->set('rollingUpdateDrainSeconds', 301)
+ ->call('saveRollingUpdateDrainSeconds')
+ ->assertHasErrors(['rollingUpdateDrainSeconds' => ['max']]);
+
+ expect($application->settings->fresh()->rolling_update_drain_seconds)->toBe(20);
+});