An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(server): stagger Sentinel version and patch checks per server
Andras Bacsaia3ed963aed44
diff --git a/app/Console/Commands/ScheduledJobDiagnostics.php b/app/Console/Commands/ScheduledJobDiagnostics.phpindex 0b4f944a1d..a99184cc96 100644--- a/app/Console/Commands/ScheduledJobDiagnostics.php+++ b/app/Console/Commands/ScheduledJobDiagnostics.php@@ -2,6 +2,7 @@namespace App\Console\Commands;+use App\Jobs\ServerManagerJob;use App\Models\DockerCleanupExecution;use App\Models\ScheduledDatabaseBackup;use App\Models\ScheduledJobDelivery;@@ -210,7 +211,8 @@ class ScheduledJobDiagnostics extends Command}$dedupKeys = [- "server-patch-check:{$server->id}" => '0 0 * * 0',+ "server-patch-check:{$server->id}" => ServerManagerJob::patchCheckCron($server),+ "sentinel-version-check:{$server->id}" => ServerManagerJob::sentinelVersionCheckCron($server),"server-check:{$server->id}" => isCloud() ? '*/5 * * * *' : '* * * * *',"server-storage-check:{$server->id}" => data_get($server->settings, 'server_disk_usage_check_frequency', '0 23 * * *'),];diff --git a/app/Jobs/ServerManagerJob.php b/app/Jobs/ServerManagerJob.phpindex fdaa8b54a5..e33e639c61 100644--- a/app/Jobs/ServerManagerJob.php+++ b/app/Jobs/ServerManagerJob.php@@ -173,7 +173,7 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue// Sentinel versions that report their version on push are updated by SentinelController.if ($server->isSentinelEnabled()&& ! Cache::has(Server::sentinelReportedVersionCacheKey($server->id))- && shouldRunCronNow('0 * * * *', $serverTimezone, "sentinel-version-check:{$server->id}", $this->executionTime)+ && shouldRunCronNow(self::sentinelVersionCheckCron($server), $serverTimezone, "sentinel-version-check:{$server->id}", $this->executionTime)) {CheckAndStartSentinelJob::dispatch($server);}@@ -192,16 +192,39 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue}}- // Dispatch ServerPatchCheckJob if due (weekly)- $shouldRunPatchCheck = shouldRunCronNow('0 0 * * 0', $serverTimezone, "server-patch-check:{$server->id}", $this->executionTime);+ // Dispatch ServerPatchCheckJob if due (weekly, staggered per server on Sunday)+ $shouldRunPatchCheck = shouldRunCronNow(self::patchCheckCron($server), $serverTimezone, "server-patch-check:{$server->id}", $this->executionTime);- if ($shouldRunPatchCheck) { // Weekly on Sunday at midnight+ if ($shouldRunPatchCheck) {ServerPatchCheckJob::dispatch($server);}// Crash recovery is handled by sentinelOutOfSync → ServerCheckJob → CheckAndStartSentinelJob.}+ /**+ * Hourly Sentinel version check at a stable per-server minute, so servers do not all SSH at minute 0.+ *+ * ServerManagerJob evaluates this every minute and shouldRunCronNow() catches up a missed minute+ * on the next run, so every minute of the hour is a safe slot.+ */+ public static function sentinelVersionCheckCron(Server $server): string+ {+ return sprintf('%d * * * *', $server->id % 60);+ }++ /**+ * Weekly patch check at a stable per-server time on Sunday between 04:00 and 23:59.+ *+ * The early hours are skipped because most daylight saving transitions happen there.+ */+ public static function patchCheckCron(Server $server): string+ {+ $slot = $server->id % (20 * 60);++ return sprintf('%d %d * * 0', $slot % 60, 4 + intdiv($slot, 60));+ }+/*** Determine the backoff cycle interval based on how many consecutive times a server has been unreachable.* Higher counts → less frequent checks (based on 5-min cloud cycle):diff --git a/tests/Feature/Server/SentinelUpdateCheckIndependenceTest.php b/tests/Feature/Server/SentinelUpdateCheckIndependenceTest.phpindex ca1494840c..65a8157dcd 100644--- a/tests/Feature/Server/SentinelUpdateCheckIndependenceTest.php+++ b/tests/Feature/Server/SentinelUpdateCheckIndependenceTest.php@@ -13,6 +13,7 @@ use Illuminate\Support\Facades\Queue;uses(RefreshDatabase::class);beforeEach(function () {+ Server::flushIdentityMap();Queue::fake();InstanceSettings::forceCreate(['id' => 0]);@@ -32,6 +33,9 @@ beforeEach(function () {]);$this->server->refresh();++ // The hourly Sentinel check runs at a stable per-server minute.+ $this->minute = sprintf('%02d', $this->server->id % 60);});afterEach(function () {@@ -46,8 +50,8 @@ it('dispatches sentinel check hourly regardless of instance update_check_frequen'instance_timezone' => 'UTC',]);- // Set time to top of any hour (sentinel should check every hour)- Carbon::setTestNow('2025-06-15 14:00:00'); // Random hour, not January 1st+ // Set time to the server minute of any hour (sentinel should check every hour)+ Carbon::setTestNow("2025-06-15 14:{$this->minute}:00"); // Random hour, not January 1st// Run ServerManagerJob$job = new ServerManagerJob;@@ -59,7 +63,7 @@ it('dispatches sentinel check hourly regardless of instance update_check_frequen});});-it('does not dispatch sentinel check when not at top of hour', function () {+it('does not dispatch sentinel check when not at the server minute', function () {// Set instance update_check_frequency to hourly (most frequent)$instanceSettings = InstanceSettings::first();$instanceSettings->update([@@ -67,14 +71,15 @@ it('does not dispatch sentinel check when not at top of hour', function () {'instance_timezone' => 'UTC',]);- // Set time to middle of the hour (sentinel check cron won't match)- Carbon::setTestNow('2025-06-15 14:30:00'); // 30 minutes past the hour+ // Set time to another minute of the hour (sentinel check cron won't match)+ $otherMinute = sprintf('%02d', ($this->server->id + 30) % 60);+ Carbon::setTestNow("2025-06-15 14:{$otherMinute}:00");// Run ServerManagerJob$job = new ServerManagerJob;$job->handle();- // Assert that CheckAndStartSentinelJob was NOT dispatched (not top of hour)+ // Assert that CheckAndStartSentinelJob was NOT dispatched (not the server minute)Queue::assertNotPushed(CheckAndStartSentinelJob::class);});@@ -91,7 +96,7 @@ it('dispatches sentinel check at every hour mark throughout the day', function (foreach ($hoursToTest as $hour) {Queue::fake(); // Reset queue for each test- Carbon::setTestNow("2025-06-15 {$hour}:00:00");+ Carbon::setTestNow("2025-06-15 {$hour}:{$this->minute}:00");$job = new ServerManagerJob;$job->handle();@@ -113,13 +118,13 @@ it('respects server timezone when checking sentinel updates', function () {'instance_timezone' => 'UTC',]);- // Set time to 17:00 UTC which is 12:00 PM EST (top of hour in server's timezone)- Carbon::setTestNow('2025-01-15 17:00:00');+ // Set time to 17:xx UTC which is 12:xx PM EST (server minute in server's timezone)+ Carbon::setTestNow("2025-01-15 17:{$this->minute}:00");$job = new ServerManagerJob;$job->handle();- // Should dispatch because it's top of hour in server's timezone (America/New_York)+ // Should dispatch because it's the server minute in server's timezone (America/New_York)Queue::assertPushed(CheckAndStartSentinelJob::class, function ($job) {return $job->server->id === $this->server->id;});@@ -137,7 +142,7 @@ it('does not dispatch sentinel check for build servers', function () {'instance_timezone' => 'UTC',]);- Carbon::setTestNow('2025-06-15 14:00:00');+ Carbon::setTestNow("2025-06-15 14:{$this->minute}:00");$job = new ServerManagerJob;$job->handle();@@ -171,13 +176,17 @@ it('handles multiple servers with different sentinel eligibility', function () {'instance_timezone' => 'UTC',]);- Carbon::setTestNow('2025-06-15 14:00:00');+ // Each server is checked at its own minute+ Carbon::setTestNow("2025-06-15 14:{$this->minute}:00");+ (new ServerManagerJob)->handle();- $job = new ServerManagerJob;- $job->handle();+ Carbon::setTestNow(sprintf('2025-06-15 15:%02d:00', $server3->id % 60));+ (new ServerManagerJob)->handle();- // Should dispatch for server1 (sentinel enabled) and server3 (sentinel enabled)- Queue::assertPushed(CheckAndStartSentinelJob::class, 2);+ // Should dispatch for server1 (sentinel enabled) and server3 (sentinel enabled), never for server2+ Queue::assertNotPushed(CheckAndStartSentinelJob::class, function ($job) use ($server2) {+ return $job->server->id === $server2->id;+ });// Verify it was dispatched for the correct serversQueue::assertPushed(CheckAndStartSentinelJob::class, function ($job) {diff --git a/tests/Feature/Server/ServerManagerJobStaggerTest.php b/tests/Feature/Server/ServerManagerJobStaggerTest.phpnew file mode 100644index 0000000000..9911c2e8fc--- /dev/null+++ b/tests/Feature/Server/ServerManagerJobStaggerTest.php@@ -0,0 +1,115 @@+<?php++use App\Enums\ServerRole;+use App\Jobs\CheckAndStartSentinelJob;+use App\Jobs\ServerManagerJob;+use App\Jobs\ServerPatchCheckJob;+use App\Models\InstanceSettings;+use App\Models\Server;+use App\Models\Team;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Carbon;+use Illuminate\Support\Facades\Queue;++uses(RefreshDatabase::class);++beforeEach(function () {+ Server::flushIdentityMap();+ Queue::fake();+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow('2025-01-15 12:00:00');+ InstanceSettings::forceCreate(['id' => 0, 'instance_timezone' => 'UTC']);+ $this->team = Team::factory()->create();+});++afterEach(function () {+ Carbon::setTestNow();+});++function createStaggerTestServer(Team $team): Server+{+ $server = Server::factory()->create([+ 'team_id' => $team->id,+ 'sentinel_updated_at' => Carbon::now(),+ ]);+ $server->settings->update([+ 'server_timezone' => 'UTC',+ 'server_role' => ServerRole::BOTH,+ 'is_reachable' => true,+ 'is_usable' => true,+ ]);++ return $server->refresh();+}++function runStaggerTestManagerAt(Carbon $time): void+{+ Carbon::setTestNow($time);+ Server::query()->update(['sentinel_updated_at' => $time]);+ Server::flushIdentityMap();+ (new ServerManagerJob)->handle();+}++it('gives each server a stable slot and spreads servers across minutes', function () {+ $servers = collect(range(1, 120))->map(fn (int $id) => (new Server)->forceFill(['id' => $id]));++ $hourlyMinutes = $servers->map(fn (Server $server) => ServerManagerJob::sentinelVersionCheckCron($server));+ $weeklySlots = $servers->map(fn (Server $server) => ServerManagerJob::patchCheckCron($server));++ expect($hourlyMinutes->unique())->toHaveCount(60)+ ->and($weeklySlots->unique())->toHaveCount(120)+ ->and(ServerManagerJob::sentinelVersionCheckCron($servers[6]))->toBe(ServerManagerJob::sentinelVersionCheckCron($servers[6]))+ ->and(ServerManagerJob::patchCheckCron($servers[6]))->toBe(ServerManagerJob::patchCheckCron($servers[6]));++ $weeklySlots->each(function (string $cron) {+ [$minute, $hour, , , $dayOfWeek] = explode(' ', $cron);+ expect((int) $minute)->toBeBetween(0, 59)+ ->and((int) $hour)->toBeBetween(4, 23)+ ->and($dayOfWeek)->toBe('0');+ });+});++it('dispatches the Sentinel version check once per hour at the server minute', function () {+ $server = createStaggerTestServer($this->team);+ $minute = $server->id % 60;+ $slot = Carbon::parse('2025-01-15 12:00:00')->setMinute($minute);++ runStaggerTestManagerAt($slot->copy()->addMinute());+ Queue::assertNotPushed(CheckAndStartSentinelJob::class);++ runStaggerTestManagerAt($slot->copy()->addHour());+ runStaggerTestManagerAt($slot->copy()->addHour()->addSeconds(30));+ runStaggerTestManagerAt($slot->copy()->addHour()->addMinute());+ Queue::assertPushed(CheckAndStartSentinelJob::class, 1);++ runStaggerTestManagerAt($slot->copy()->addHours(2));+ Queue::assertPushed(CheckAndStartSentinelJob::class, 2);+});++it('catches up a Sentinel version check when the server minute was not evaluated', function () {+ $server = createStaggerTestServer($this->team);+ $slot = Carbon::parse('2025-01-15 12:00:00')->setMinute($server->id % 60);++ runStaggerTestManagerAt($slot->copy()->subMinute());+ Queue::assertNotPushed(CheckAndStartSentinelJob::class);++ runStaggerTestManagerAt($slot->copy()->addMinutes(3));+ Queue::assertPushed(CheckAndStartSentinelJob::class, 1);+});++it('dispatches the patch check once per week at the server slot on Sunday', function () {+ $server = createStaggerTestServer($this->team);+ [$minute, $hour] = array_map('intval', explode(' ', ServerManagerJob::patchCheckCron($server)));+ $sunday = Carbon::parse('2025-01-19 00:00:00');+ $slot = $sunday->copy()->setTime($hour, $minute);++ runStaggerTestManagerAt($sunday);+ Queue::assertNotPushed(ServerPatchCheckJob::class);++ runStaggerTestManagerAt($slot);+ runStaggerTestManagerAt($slot->copy()->addMinute());+ Queue::assertPushed(ServerPatchCheckJob::class, 1);++ runStaggerTestManagerAt($slot->copy()->addWeek());+ Queue::assertPushed(ServerPatchCheckJob::class, 2);+});diff --git a/tests/Unit/ServerManagerJobSentinelCheckTest.php b/tests/Unit/ServerManagerJobSentinelCheckTest.phpindex dbbe800ad8..f17b3957be 100644--- a/tests/Unit/ServerManagerJobSentinelCheckTest.php+++ b/tests/Unit/ServerManagerJobSentinelCheckTest.php@@ -16,6 +16,7 @@ use Tests\TestCase;uses(TestCase::class, RefreshDatabase::class);beforeEach(function () {+ Server::flushIdentityMap();Queue::fake();Carbon::setTestNow('2025-01-15 12:00:00');InstanceSettings::forceCreate(['id' => 0, 'instance_timezone' => 'UTC']);@@ -45,6 +46,8 @@ function createSentinelCheckServer(Team $team, Carbon $sentinelUpdatedAt, Serverit('dispatches an hourly Sentinel version check for a healthy Sentinel', function () {$server = createSentinelCheckServer($this->team, Carbon::now());+ Carbon::setTestNow(Carbon::now()->setMinute($server->id % 60));+ $server->update(['sentinel_updated_at' => Carbon::now()]);expect($server->isSentinelEnabled())->toBeTrue()->and($server->isSentinelLive())->toBeTrue();