An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(upgrade): retry rate limited image pulls and spread auto updates
Andras Bacsai12fd9e448b93
diff --git a/app/Jobs/UpdateCoolifyJob.php b/app/Jobs/UpdateCoolifyJob.phpindex f0e43cbc0a..e2310f0bee 100644--- a/app/Jobs/UpdateCoolifyJob.php+++ b/app/Jobs/UpdateCoolifyJob.php@@ -44,8 +44,7 @@ class UpdateCoolifyJob implements ShouldBeEncrypted, ShouldQueueLog::info('Starting Coolify update process...');UpdateCoolify::run(false); // false means it's not a manual update- $settings->update(['new_version_available' => false]);- Log::info('Coolify update completed successfully.');+ Log::info('Coolify upgrade script started. Result: /data/coolify/source/.upgrade-status, log: /data/coolify/source/upgrade-*.log.');} catch (\Throwable $e) {Log::error('UpdateCoolifyJob failed: '.$e->getMessage());// Consider implementing a notification to administratorsdiff --git a/app/Livewire/Settings/Updates.php b/app/Livewire/Settings/Updates.phpindex 10b3180640..aba7521dd9 100644--- a/app/Livewire/Settings/Updates.php+++ b/app/Livewire/Settings/Updates.php@@ -130,7 +130,7 @@ class Updates extends Componentif ($this->is_auto_update_enabled && ! validate_cron_expression($this->auto_update_frequency)) {$this->dispatch('error', 'Invalid Cron / Human expression for Auto Update Frequency.');if (empty($this->auto_update_frequency)) {- $this->auto_update_frequency = '0 0 * * *';+ $this->auto_update_frequency = InstanceSettings::randomAutoUpdateFrequency();}return;diff --git a/app/Models/InstanceSettings.php b/app/Models/InstanceSettings.phpindex 16a1d7ea30..79d921fcf8 100644--- a/app/Models/InstanceSettings.php+++ b/app/Models/InstanceSettings.php@@ -104,6 +104,12 @@ class InstanceSettings extends Modelprotected static function booted(): void{+ static::creating(function ($settings) {+ if (blank($settings->getAttributes()['auto_update_frequency'] ?? null)) {+ $settings->auto_update_frequency = self::randomAutoUpdateFrequency();+ }+ });+static::created(function () {Once::flush();});@@ -121,6 +127,14 @@ class InstanceSettings extends Model});}+ /**+ * Daily auto update at a random minute after midnight, so instances do not all pull images from the registry in the same second.+ */+ public static function randomAutoUpdateFrequency(): string+ {+ return random_int(0, 59).' 0 * * *';+ }+public function isPasswordRegistrationAllowed(): bool{if (! $this->is_registration_enabled) {diff --git a/database/migrations/2026_10_15_213357_spread_default_auto_update_frequency.php b/database/migrations/2026_10_15_213357_spread_default_auto_update_frequency.phpnew file mode 100644index 0000000000..9648ab50ec--- /dev/null+++ b/database/migrations/2026_10_15_213357_spread_default_auto_update_frequency.php@@ -0,0 +1,24 @@+<?php++use App\Models\InstanceSettings;+use Illuminate\Database\Migrations\Migration;+use Illuminate\Support\Facades\DB;++return new class extends Migration+{+ /**+ * Move instances that use the old midnight default to a random minute after midnight,+ * so they do not all pull images from the registry at 00:00.+ */+ public function up(): void+ {+ DB::table('instance_settings')+ ->where('auto_update_frequency', '0 0 * * *')+ ->update(['auto_update_frequency' => InstanceSettings::randomAutoUpdateFrequency()]);+ }++ public function down(): void+ {+ //+ }+};diff --git a/other/nightly/upgrade.sh b/other/nightly/upgrade.shindex ce8fdb6589..88dd7ea093 100644--- a/other/nightly/upgrade.sh+++ b/other/nightly/upgrade.sh@@ -38,6 +38,24 @@ write_status() {echo "${step}|${message}|$(date -Iseconds)" > "$STATUS_FILE"}+# Pull an image. Retry with backoff, because the registry can rate limit many instances that pull at the same time.+pull_image() {+ local image="$1"+ local attempt=1+ local max_attempts=5+ local delay=15+ until docker pull "$image" >>"$LOGFILE" 2>&1; do+ if [ "$attempt" -ge "$max_attempts" ]; then+ return 1+ fi+ log "Pull of $image failed (attempt $attempt/$max_attempts). Retrying in ${delay}s..."+ echo " Pull failed (attempt $attempt/$max_attempts). Retrying in ${delay}s..."+ sleep "$((delay + RANDOM % 10))"+ attempt=$((attempt + 1))+ delay=$((delay * 2))+ done+}+echo ""echo "=========================================="echo " Coolify Upgrade - ${DATE}"@@ -205,7 +223,7 @@ echo " This may take a few minutes depending on your connection."HELPER_IMAGE="${REGISTRY_URL:-docker.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION}"echo " - Pulling $HELPER_IMAGE..."log "Pulling image: $HELPER_IMAGE"-if docker pull "$HELPER_IMAGE" >>"$LOGFILE" 2>&1; then+if pull_image "$HELPER_IMAGE"; thenlog "Successfully pulled $HELPER_IMAGE"elselog "ERROR: Failed to pull $HELPER_IMAGE"@@ -220,7 +238,7 @@ for IMAGE in $IMAGES; doif [ -n "$IMAGE" ]; thenecho " - Pulling $IMAGE..."log "Pulling image: $IMAGE"- if docker pull "$IMAGE" >>"$LOGFILE" 2>&1; then+ if pull_image "$IMAGE"; thenlog "Successfully pulled $IMAGE"elselog "ERROR: Failed to pull $IMAGE"diff --git a/scripts/upgrade.sh b/scripts/upgrade.shindex c2bcabc3f7..987ce636a0 100644--- a/scripts/upgrade.sh+++ b/scripts/upgrade.sh@@ -38,6 +38,24 @@ write_status() {echo "${step}|${message}|$(date -Iseconds)" > "$STATUS_FILE"}+# Pull an image. Retry with backoff, because the registry can rate limit many instances that pull at the same time.+pull_image() {+ local image="$1"+ local attempt=1+ local max_attempts=5+ local delay=15+ until docker pull "$image" >>"$LOGFILE" 2>&1; do+ if [ "$attempt" -ge "$max_attempts" ]; then+ return 1+ fi+ log "Pull of $image failed (attempt $attempt/$max_attempts). Retrying in ${delay}s..."+ echo " Pull failed (attempt $attempt/$max_attempts). Retrying in ${delay}s..."+ sleep "$((delay + RANDOM % 10))"+ attempt=$((attempt + 1))+ delay=$((delay * 2))+ done+}+echo ""echo "=========================================="echo " Coolify Upgrade - ${DATE}"@@ -205,7 +223,7 @@ echo " This may take a few minutes depending on your connection."HELPER_IMAGE="${REGISTRY_URL:-docker.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION}"echo " - Pulling $HELPER_IMAGE..."log "Pulling image: $HELPER_IMAGE"-if docker pull "$HELPER_IMAGE" >>"$LOGFILE" 2>&1; then+if pull_image "$HELPER_IMAGE"; thenlog "Successfully pulled $HELPER_IMAGE"elselog "ERROR: Failed to pull $HELPER_IMAGE"@@ -220,7 +238,7 @@ for IMAGE in $IMAGES; doif [ -n "$IMAGE" ]; thenecho " - Pulling $IMAGE..."log "Pulling image: $IMAGE"- if docker pull "$IMAGE" >>"$LOGFILE" 2>&1; then+ if pull_image "$IMAGE"; thenlog "Successfully pulled $IMAGE"elselog "ERROR: Failed to pull $IMAGE"diff --git a/tests/Feature/AutoUpdateFrequencyDefaultTest.php b/tests/Feature/AutoUpdateFrequencyDefaultTest.phpnew file mode 100644index 0000000000..8d64e26b09--- /dev/null+++ b/tests/Feature/AutoUpdateFrequencyDefaultTest.php@@ -0,0 +1,29 @@+<?php++use App\Models\InstanceSettings;+use Illuminate\Foundation\Testing\RefreshDatabase;++uses(RefreshDatabase::class);++it('gives a new instance a random minute after midnight for auto updates', function () {+ $settings = InstanceSettings::forceCreate(['id' => 0]);++ expect($settings->fresh()->auto_update_frequency)->toMatch('/^([0-9]|[1-5][0-9]) 0 \* \* \*$/');+});++it('keeps an explicit auto update frequency on create', function () {+ $settings = InstanceSettings::forceCreate(['id' => 0, 'auto_update_frequency' => '23 3 * * *']);++ expect($settings->fresh()->auto_update_frequency)->toBe('23 3 * * *');+});++it('moves instances off the old midnight default and keeps custom schedules', function (string $frequency, string $expected) {+ InstanceSettings::forceCreate(['id' => 0, 'auto_update_frequency' => $frequency]);++ (require database_path('migrations/2026_10_15_213357_spread_default_auto_update_frequency.php'))->up();++ expect(InstanceSettings::find(0)->auto_update_frequency)->toMatch($expected);+})->with([+ 'old default' => ['0 0 * * *', '/^([0-9]|[1-5][0-9]) 0 \* \* \*$/'],+ 'custom' => ['23 3 * * *', '/^23 3 \* \* \*$/'],+]);diff --git a/tests/Unit/UpgradeScriptPullRetryTest.php b/tests/Unit/UpgradeScriptPullRetryTest.phpnew file mode 100644index 0000000000..d30514a965--- /dev/null+++ b/tests/Unit/UpgradeScriptPullRetryTest.php@@ -0,0 +1,42 @@+<?php++/**+ * Runs pull_image() from the upgrade script with a fake docker that fails the first $failures pulls.+ *+ * @return array{exitCode: int, pulls: int}+ */+function runUpgradeScriptPullImage(string $script, int $failures): array+{+ $dir = sys_get_temp_dir().'/upgrade-pull-'.bin2hex(random_bytes(4));+ mkdir($dir);+ $source = file_get_contents(getcwd().'/'.$script);+ preg_match('/^log\(\) \{.*?^\}\n/ms', $source, $log);+ preg_match('/^pull_image\(\) \{.*?^\}\n/ms', $source, $pullImage);++ $harness = <<<BASH+ LOGFILE="{$dir}/upgrade.log"+ docker() {+ local count=\$(cat "{$dir}/pulls" 2>/dev/null || echo 0)+ echo \$((count + 1)) > "{$dir}/pulls"+ [ "\$count" -ge {$failures} ]+ }+ sleep() { :; }+ {$log[0]}+ {$pullImage[0]}+ pull_image ghcr.io/coollabsio/coolify-helper:1.0.14+ BASH;++ exec('bash -c '.escapeshellarg($harness).' >/dev/null 2>&1', result_code: $exitCode);+ $pulls = (int) @file_get_contents("{$dir}/pulls");+ exec('rm -rf '.escapeshellarg($dir));++ return ['exitCode' => $exitCode, 'pulls' => $pulls];+}++it('retries a rate limited image pull until it succeeds', function (string $script) {+ expect(runUpgradeScriptPullImage($script, failures: 2))->toBe(['exitCode' => 0, 'pulls' => 3]);+})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);++it('fails the image pull after five attempts', function (string $script) {+ expect(runUpgradeScriptPullImage($script, failures: 10))->toBe(['exitCode' => 1, 'pulls' => 5]);+})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);