Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(upgrade): retry rate limited image pulls and spread auto updates

Andras Bacsai12fd9e448b93
diff --git a/app/Jobs/UpdateCoolifyJob.php b/app/Jobs/UpdateCoolifyJob.php
index f0e43cbc0a..e2310f0bee 100644
--- a/app/Jobs/UpdateCoolifyJob.php
+++ b/app/Jobs/UpdateCoolifyJob.php
@@ -44,8 +44,7 @@ class UpdateCoolifyJob implements ShouldBeEncrypted, ShouldQueue
Log::info('Starting Coolify update process...');
UpdateCoolify::run(false); // false means it's not a manual update
- $settings->update(['new_version_available' => false]);
- Log::info('Coolify update completed successfully.');
+ Log::info('Coolify upgrade script started. Result: /data/coolify/source/.upgrade-status, log: /data/coolify/source/upgrade-*.log.');
} catch (\Throwable $e) {
Log::error('UpdateCoolifyJob failed: '.$e->getMessage());
// Consider implementing a notification to administrators
diff --git a/app/Livewire/Settings/Updates.php b/app/Livewire/Settings/Updates.php
index 10b3180640..aba7521dd9 100644
--- a/app/Livewire/Settings/Updates.php
+++ b/app/Livewire/Settings/Updates.php
@@ -130,7 +130,7 @@ class Updates extends Component
if ($this->is_auto_update_enabled && ! validate_cron_expression($this->auto_update_frequency)) {
$this->dispatch('error', 'Invalid Cron / Human expression for Auto Update Frequency.');
if (empty($this->auto_update_frequency)) {
- $this->auto_update_frequency = '0 0 * * *';
+ $this->auto_update_frequency = InstanceSettings::randomAutoUpdateFrequency();
}
return;
diff --git a/app/Models/InstanceSettings.php b/app/Models/InstanceSettings.php
index 16a1d7ea30..79d921fcf8 100644
--- a/app/Models/InstanceSettings.php
+++ b/app/Models/InstanceSettings.php
@@ -104,6 +104,12 @@ class InstanceSettings extends Model
protected static function booted(): void
{
+ static::creating(function ($settings) {
+ if (blank($settings->getAttributes()['auto_update_frequency'] ?? null)) {
+ $settings->auto_update_frequency = self::randomAutoUpdateFrequency();
+ }
+ });
+
static::created(function () {
Once::flush();
});
@@ -121,6 +127,14 @@ class InstanceSettings extends Model
});
}
+ /**
+ * Daily auto update at a random minute after midnight, so instances do not all pull images from the registry in the same second.
+ */
+ public static function randomAutoUpdateFrequency(): string
+ {
+ return random_int(0, 59).' 0 * * *';
+ }
+
public function isPasswordRegistrationAllowed(): bool
{
if (! $this->is_registration_enabled) {
diff --git a/database/migrations/2026_10_15_213357_spread_default_auto_update_frequency.php b/database/migrations/2026_10_15_213357_spread_default_auto_update_frequency.php
new file mode 100644
index 0000000000..9648ab50ec
--- /dev/null
+++ b/database/migrations/2026_10_15_213357_spread_default_auto_update_frequency.php
@@ -0,0 +1,24 @@
+<?php
+
+use App\Models\InstanceSettings;
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Support\Facades\DB;
+
+return new class extends Migration
+{
+ /**
+ * Move instances that use the old midnight default to a random minute after midnight,
+ * so they do not all pull images from the registry at 00:00.
+ */
+ public function up(): void
+ {
+ DB::table('instance_settings')
+ ->where('auto_update_frequency', '0 0 * * *')
+ ->update(['auto_update_frequency' => InstanceSettings::randomAutoUpdateFrequency()]);
+ }
+
+ public function down(): void
+ {
+ //
+ }
+};
diff --git a/other/nightly/upgrade.sh b/other/nightly/upgrade.sh
index ce8fdb6589..88dd7ea093 100644
--- a/other/nightly/upgrade.sh
+++ b/other/nightly/upgrade.sh
@@ -38,6 +38,24 @@ write_status() {
echo "${step}|${message}|$(date -Iseconds)" > "$STATUS_FILE"
}
+# Pull an image. Retry with backoff, because the registry can rate limit many instances that pull at the same time.
+pull_image() {
+ local image="$1"
+ local attempt=1
+ local max_attempts=5
+ local delay=15
+ until docker pull "$image" >>"$LOGFILE" 2>&1; do
+ if [ "$attempt" -ge "$max_attempts" ]; then
+ return 1
+ fi
+ log "Pull of $image failed (attempt $attempt/$max_attempts). Retrying in ${delay}s..."
+ echo " Pull failed (attempt $attempt/$max_attempts). Retrying in ${delay}s..."
+ sleep "$((delay + RANDOM % 10))"
+ attempt=$((attempt + 1))
+ delay=$((delay * 2))
+ done
+}
+
echo ""
echo "=========================================="
echo " Coolify Upgrade - ${DATE}"
@@ -205,7 +223,7 @@ echo " This may take a few minutes depending on your connection."
HELPER_IMAGE="${REGISTRY_URL:-docker.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION}"
echo " - Pulling $HELPER_IMAGE..."
log "Pulling image: $HELPER_IMAGE"
-if docker pull "$HELPER_IMAGE" >>"$LOGFILE" 2>&1; then
+if pull_image "$HELPER_IMAGE"; then
log "Successfully pulled $HELPER_IMAGE"
else
log "ERROR: Failed to pull $HELPER_IMAGE"
@@ -220,7 +238,7 @@ for IMAGE in $IMAGES; do
if [ -n "$IMAGE" ]; then
echo " - Pulling $IMAGE..."
log "Pulling image: $IMAGE"
- if docker pull "$IMAGE" >>"$LOGFILE" 2>&1; then
+ if pull_image "$IMAGE"; then
log "Successfully pulled $IMAGE"
else
log "ERROR: Failed to pull $IMAGE"
diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh
index c2bcabc3f7..987ce636a0 100644
--- a/scripts/upgrade.sh
+++ b/scripts/upgrade.sh
@@ -38,6 +38,24 @@ write_status() {
echo "${step}|${message}|$(date -Iseconds)" > "$STATUS_FILE"
}
+# Pull an image. Retry with backoff, because the registry can rate limit many instances that pull at the same time.
+pull_image() {
+ local image="$1"
+ local attempt=1
+ local max_attempts=5
+ local delay=15
+ until docker pull "$image" >>"$LOGFILE" 2>&1; do
+ if [ "$attempt" -ge "$max_attempts" ]; then
+ return 1
+ fi
+ log "Pull of $image failed (attempt $attempt/$max_attempts). Retrying in ${delay}s..."
+ echo " Pull failed (attempt $attempt/$max_attempts). Retrying in ${delay}s..."
+ sleep "$((delay + RANDOM % 10))"
+ attempt=$((attempt + 1))
+ delay=$((delay * 2))
+ done
+}
+
echo ""
echo "=========================================="
echo " Coolify Upgrade - ${DATE}"
@@ -205,7 +223,7 @@ echo " This may take a few minutes depending on your connection."
HELPER_IMAGE="${REGISTRY_URL:-docker.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION}"
echo " - Pulling $HELPER_IMAGE..."
log "Pulling image: $HELPER_IMAGE"
-if docker pull "$HELPER_IMAGE" >>"$LOGFILE" 2>&1; then
+if pull_image "$HELPER_IMAGE"; then
log "Successfully pulled $HELPER_IMAGE"
else
log "ERROR: Failed to pull $HELPER_IMAGE"
@@ -220,7 +238,7 @@ for IMAGE in $IMAGES; do
if [ -n "$IMAGE" ]; then
echo " - Pulling $IMAGE..."
log "Pulling image: $IMAGE"
- if docker pull "$IMAGE" >>"$LOGFILE" 2>&1; then
+ if pull_image "$IMAGE"; then
log "Successfully pulled $IMAGE"
else
log "ERROR: Failed to pull $IMAGE"
diff --git a/tests/Feature/AutoUpdateFrequencyDefaultTest.php b/tests/Feature/AutoUpdateFrequencyDefaultTest.php
new file mode 100644
index 0000000000..8d64e26b09
--- /dev/null
+++ b/tests/Feature/AutoUpdateFrequencyDefaultTest.php
@@ -0,0 +1,29 @@
+<?php
+
+use App\Models\InstanceSettings;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+
+uses(RefreshDatabase::class);
+
+it('gives a new instance a random minute after midnight for auto updates', function () {
+ $settings = InstanceSettings::forceCreate(['id' => 0]);
+
+ expect($settings->fresh()->auto_update_frequency)->toMatch('/^([0-9]|[1-5][0-9]) 0 \* \* \*$/');
+});
+
+it('keeps an explicit auto update frequency on create', function () {
+ $settings = InstanceSettings::forceCreate(['id' => 0, 'auto_update_frequency' => '23 3 * * *']);
+
+ expect($settings->fresh()->auto_update_frequency)->toBe('23 3 * * *');
+});
+
+it('moves instances off the old midnight default and keeps custom schedules', function (string $frequency, string $expected) {
+ InstanceSettings::forceCreate(['id' => 0, 'auto_update_frequency' => $frequency]);
+
+ (require database_path('migrations/2026_10_15_213357_spread_default_auto_update_frequency.php'))->up();
+
+ expect(InstanceSettings::find(0)->auto_update_frequency)->toMatch($expected);
+})->with([
+ 'old default' => ['0 0 * * *', '/^([0-9]|[1-5][0-9]) 0 \* \* \*$/'],
+ 'custom' => ['23 3 * * *', '/^23 3 \* \* \*$/'],
+]);
diff --git a/tests/Unit/UpgradeScriptPullRetryTest.php b/tests/Unit/UpgradeScriptPullRetryTest.php
new file mode 100644
index 0000000000..d30514a965
--- /dev/null
+++ b/tests/Unit/UpgradeScriptPullRetryTest.php
@@ -0,0 +1,42 @@
+<?php
+
+/**
+ * Runs pull_image() from the upgrade script with a fake docker that fails the first $failures pulls.
+ *
+ * @return array{exitCode: int, pulls: int}
+ */
+function runUpgradeScriptPullImage(string $script, int $failures): array
+{
+ $dir = sys_get_temp_dir().'/upgrade-pull-'.bin2hex(random_bytes(4));
+ mkdir($dir);
+ $source = file_get_contents(getcwd().'/'.$script);
+ preg_match('/^log\(\) \{.*?^\}\n/ms', $source, $log);
+ preg_match('/^pull_image\(\) \{.*?^\}\n/ms', $source, $pullImage);
+
+ $harness = <<<BASH
+ LOGFILE="{$dir}/upgrade.log"
+ docker() {
+ local count=\$(cat "{$dir}/pulls" 2>/dev/null || echo 0)
+ echo \$((count + 1)) > "{$dir}/pulls"
+ [ "\$count" -ge {$failures} ]
+ }
+ sleep() { :; }
+ {$log[0]}
+ {$pullImage[0]}
+ pull_image ghcr.io/coollabsio/coolify-helper:1.0.14
+ BASH;
+
+ exec('bash -c '.escapeshellarg($harness).' >/dev/null 2>&1', result_code: $exitCode);
+ $pulls = (int) @file_get_contents("{$dir}/pulls");
+ exec('rm -rf '.escapeshellarg($dir));
+
+ return ['exitCode' => $exitCode, 'pulls' => $pulls];
+}
+
+it('retries a rate limited image pull until it succeeds', function (string $script) {
+ expect(runUpgradeScriptPullImage($script, failures: 2))->toBe(['exitCode' => 0, 'pulls' => 3]);
+})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);
+
+it('fails the image pull after five attempts', function (string $script) {
+ expect(runUpgradeScriptPullImage($script, failures: 10))->toBe(['exitCode' => 1, 'pulls' => 5]);
+})->with(['scripts/upgrade.sh', 'other/nightly/upgrade.sh']);