Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(upgrade): prevent duplicate upgrades and stalled progress polling

Andras Bacsaidf953bdcd187
diff --git a/app/Actions/Server/UpdateCoolify.php b/app/Actions/Server/UpdateCoolify.php
index a8c9d21e8f..3c60c34379 100644
--- a/app/Actions/Server/UpdateCoolify.php
+++ b/app/Actions/Server/UpdateCoolify.php
@@ -3,6 +3,7 @@
namespace App\Actions\Server;
use App\Models\Server;
+use App\Services\CoolifyUpgradeStatus;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Sleep;
@@ -116,6 +117,8 @@ class UpdateCoolify
private function update()
{
+ $this->ensureNoUpgradeIsRunning();
+
$latestHelperImageVersion = getHelperVersion();
$upgradeScriptUrl = config('constants.coolify.upgrade_script_url');
$registryUrl = coolifyRegistryUrl();
@@ -128,4 +131,29 @@ class UpdateCoolify
escapeshellarg($registryUrl),
], $this->server);
}
+
+ private function ensureNoUpgradeIsRunning(): void
+ {
+ $status = $this->readUpgradeStatus();
+
+ if (CoolifyUpgradeStatus::isRunning((string) $status)) {
+ Log::warning('Upgrade skipped because another upgrade is running', [
+ 'target_version' => $this->latestVersion,
+ 'status' => $status,
+ ]);
+ throw new \Exception(
+ 'Another Coolify upgrade is already running. Wait for it to finish. '.
+ 'If it has stopped, you can upgrade again '.CoolifyUpgradeStatus::RUNNING_LOCK_EXPIRES_AFTER_MINUTES.' minutes after its last status update.'
+ );
+ }
+ }
+
+ protected function readUpgradeStatus(): ?string
+ {
+ return instant_remote_process(
+ ['cat '.CoolifyUpgradeStatus::FILE.' 2>/dev/null || true'],
+ $this->server,
+ false
+ );
+ }
}
diff --git a/app/Livewire/Upgrade.php b/app/Livewire/Upgrade.php
index e2ab094944..75d9b32b3e 100644
--- a/app/Livewire/Upgrade.php
+++ b/app/Livewire/Upgrade.php
@@ -103,7 +103,7 @@ class Upgrade extends Component
return ['status' => 'none'];
}
- $statusFile = '/data/coolify/source/.upgrade-status';
+ $statusFile = CoolifyUpgradeStatus::FILE;
try {
$content = instant_remote_process(
diff --git a/app/Services/CoolifyUpgradeStatus.php b/app/Services/CoolifyUpgradeStatus.php
index 15f6c22ef2..3f274bb616 100644
--- a/app/Services/CoolifyUpgradeStatus.php
+++ b/app/Services/CoolifyUpgradeStatus.php
@@ -6,8 +6,16 @@ use DateTimeInterface;
class CoolifyUpgradeStatus
{
+ public const FILE = '/data/coolify/source/.upgrade-status';
+
public const STALE_AFTER_MINUTES = 10;
+ /**
+ * An upgrade that has not written a status for this long is treated as stopped,
+ * so a crashed upgrade does not block new ones forever.
+ */
+ public const RUNNING_LOCK_EXPIRES_AFTER_MINUTES = 15;
+
/**
* @return array{status: string, step?: int, message?: string, running_version: string, target_version: string}
*/
@@ -77,6 +85,30 @@ class CoolifyUpgradeStatus
];
}
+ public static function isRunning(string $content, ?DateTimeInterface $now = null): bool
+ {
+ $parts = explode('|', trim($content));
+ if (count($parts) < 3) {
+ return false;
+ }
+
+ [$step, , $timestamp] = $parts;
+ if (! in_array($step, ['1', '2', '3', '4', '5'], true)) {
+ return false;
+ }
+
+ try {
+ $statusTime = new \DateTime($timestamp);
+ } catch (\Throwable) {
+ return false;
+ }
+
+ $now = $now ?? new \DateTime;
+ $diffMinutes = ($now->getTimestamp() - $statusTime->getTimestamp()) / 60;
+
+ return $diffMinutes <= self::RUNNING_LOCK_EXPIRES_AFTER_MINUTES;
+ }
+
public static function hasReachedTargetVersion(string $runningVersion, string $targetVersion): bool
{
if ($runningVersion === '' || $targetVersion === '') {
diff --git a/resources/views/livewire/upgrade.blade.php b/resources/views/livewire/upgrade.blade.php
index b1ad048e1c..2af8873207 100644
--- a/resources/views/livewire/upgrade.blade.php
+++ b/resources/views/livewire/upgrade.blade.php
@@ -186,6 +186,9 @@
checkUpgradeStatusInterval: null,
elapsedInterval: null,
healthCheckAttempts: 0,
+ healthProbePending: false,
+ healthTimeouts: 0,
+ statusRequestPending: false,
livewireFailures: 0,
startTime: null,
elapsedTime: 0,
@@ -261,6 +264,19 @@
return `${minutes}:${seconds.toString().padStart(2, '0')}`;
},
+ // Livewire never settles a $wire call when its request fails, and a request to a removed
+ // container can hang. Time out both so the progress view does not freeze.
+ withTimeout(promise, ms) {
+ return Promise.race([
+ promise,
+ new Promise((_, reject) => setTimeout(() => reject(new Error('Request timed out')), ms)),
+ ]);
+ },
+
+ fetchUpgradeStatus() {
+ return this.withTimeout(this.$wire.getUpgradeStatus(), 15000);
+ },
+
mapStepToUI(apiStep) {
// Map backend steps (1-6) to UI steps (1-4)
// Backend: 1=config, 2=env, 3=pull, 4=stop, 5=start, 6=complete
@@ -281,11 +297,30 @@
},
async probeHealth() {
+ if (this.healthProbePending) {
+ return;
+ }
+ this.healthProbePending = true;
+ try {
+ await this.runHealthProbe();
+ } finally {
+ this.healthProbePending = false;
+ }
+ },
+
+ async runHealthProbe() {
+ if (this.upgradeComplete || this.upgradeError) {
+ return;
+ }
this.healthCheckAttempts++;
const elapsedMinutes = Math.floor((Date.now() - this.startTime) / 60000);
try {
- const response = await fetch('/api/health');
+ const response = await fetch('/api/health', {
+ cache: 'no-store',
+ signal: AbortSignal.timeout(10000),
+ });
+ this.healthTimeouts = 0;
if (!response.ok) {
this.instanceWentDown = true;
this.currentStep = 4;
@@ -298,7 +333,7 @@
let data;
try {
- data = await this.$wire.getUpgradeStatus();
+ data = await this.fetchUpgradeStatus();
} catch (error) {
if (this.instanceWentDown) {
this.showSuccess();
@@ -328,6 +363,10 @@
this.currentStatus = data.message ?? this.getReviveStatusMessage(elapsedMinutes, this.healthCheckAttempts);
}
} catch (error) {
+ // A busy server can answer slowly, so one timeout does not prove that it is down.
+ if (error?.name === 'TimeoutError' && ++this.healthTimeouts < 2) {
+ return;
+ }
console.error('Health check failed:', error);
this.instanceWentDown = true;
this.currentStep = 4;
@@ -356,6 +395,9 @@
},
showSuccess() {
+ if (this.upgradeComplete) {
+ return;
+ }
if (this.checkHealthInterval) {
clearInterval(this.checkHealthInterval);
this.checkHealthInterval = null;
@@ -431,12 +473,17 @@
this.serviceDown = false;
this.instanceWentDown = false;
this.livewireFailures = 0;
+ this.healthTimeouts = 0;
this.startHealthWatch();
// Poll upgrade status via Livewire
this.checkUpgradeStatusInterval = setInterval(async () => {
+ if (this.statusRequestPending) {
+ return;
+ }
+ this.statusRequestPending = true;
try {
- const data = await this.$wire.getUpgradeStatus();
+ const data = await this.fetchUpgradeStatus();
this.livewireFailures = 0;
if (data.status === 'in_progress') {
this.currentStep = this.mapStepToUI(data.step);
@@ -450,6 +497,9 @@
await this.probeHealth();
}
} catch (error) {
+ if (this.upgradeComplete || this.upgradeError) {
+ return;
+ }
this.livewireFailures++;
if (this.livewireFailures < 3) {
this.currentStatus = 'Reconnecting. This is expected during an upgrade...';
@@ -468,6 +518,8 @@
}
this.revive();
}
+ } finally {
+ this.statusRequestPending = false;
}
}, 2000);
}
diff --git a/tests/Unit/CoolifyUpgradeStatusTest.php b/tests/Unit/CoolifyUpgradeStatusTest.php
index f701a8a6d3..1302e97370 100644
--- a/tests/Unit/CoolifyUpgradeStatusTest.php
+++ b/tests/Unit/CoolifyUpgradeStatusTest.php
@@ -128,3 +128,24 @@ it('reports whether the running version has reached the target', function (strin
'missing running version' => ['', '4.3.1', false],
'missing target version' => ['4.3.1', '', false],
]);
+
+it('detects a running upgrade only for recent intermediate steps', function (string $step, int $minutesAgo, bool $expected) {
+ $now = Carbon::parse('2026-08-13T12:00:00+00:00');
+ $content = "{$step}|Some message|".$now->copy()->subMinutes($minutesAgo)->toIso8601String();
+
+ expect(CoolifyUpgradeStatus::isRunning($content, $now))->toBe($expected);
+})->with([
+ 'recent step 1' => ['1', 0, true],
+ 'step 5 at lock expiry' => ['5', 15, true],
+ 'step 3 after lock expiry' => ['3', 16, false],
+ 'complete' => ['6', 1, false],
+ 'error' => ['error', 1, false],
+]);
+
+it('does not detect a running upgrade for empty or malformed status', function (string $content) {
+ expect(CoolifyUpgradeStatus::isRunning($content))->toBeFalse();
+})->with([
+ 'empty' => '',
+ 'missing fields' => '3|Pulling Docker images',
+ 'invalid timestamp' => '3|Pulling Docker images|not-a-date',
+]);
diff --git a/tests/Unit/UpdateCoolifyTest.php b/tests/Unit/UpdateCoolifyTest.php
index d27a634018..ddc8c2546b 100644
--- a/tests/Unit/UpdateCoolifyTest.php
+++ b/tests/Unit/UpdateCoolifyTest.php
@@ -40,6 +40,19 @@ function updateCoolifyTestCreateRootServerAndSettings(array $settings = []): voi
Once::flush();
}
+function updateCoolifyTestActionWithStatus(string $status): UpdateCoolify
+{
+ return new class($status) extends UpdateCoolify
+ {
+ public function __construct(private string $status) {}
+
+ protected function readUpgradeStatus(): ?string
+ {
+ return $this->status;
+ }
+ };
+}
+
afterEach(function () {
Mockery::close();
});
@@ -316,3 +329,37 @@ it('prevents downgrade even with manual update', function () {
expect($e->getMessage())->toContain('4.0.0');
}
});
+
+it('does not start a second upgrade while a recent upgrade status is in progress', function () {
+ Queue::fake();
+ config([
+ 'app.env' => 'testing',
+ 'constants.coolify.version' => '4.0.9',
+ 'constants.ssh.mux_enabled' => false,
+ ]);
+ updateCoolifyTestCreateRootServerAndSettings();
+ Http::fake(['*' => Http::response(['coolify' => ['v4' => ['version' => '4.0.10']]], 200)]);
+ $action = updateCoolifyTestActionWithStatus('3|Pulling Docker images|'.now()->subMinutes(2)->toIso8601String());
+
+ expect(fn () => $action->handle(manual_update: true))
+ ->toThrow(Exception::class, 'Another Coolify upgrade is already running');
+
+ expect(Activity::query()->count())->toBe(0)
+ ->and(InstanceSettings::findOrFail(0)->new_version_available)->toBeFalsy();
+});
+
+it('starts an upgrade when the previous upgrade status is older than the lock expiry', function () {
+ Queue::fake();
+ config([
+ 'app.env' => 'testing',
+ 'constants.coolify.version' => '4.0.9',
+ 'constants.ssh.mux_enabled' => false,
+ ]);
+ updateCoolifyTestCreateRootServerAndSettings();
+ Http::fake(['*' => Http::response(['coolify' => ['v4' => ['version' => '4.0.10']]], 200)]);
+ updateCoolifyTestActionWithStatus('3|Pulling Docker images|'.now()->subMinutes(16)->toIso8601String())
+ ->handle(manual_update: true);
+
+ expect(RemoteProcessCommand::read(Activity::query()->latest('id')->first()))
+ ->toContain("bash /data/coolify/source/upgrade.sh '4.0.10'");
+});