Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(server): stagger Sentinel version and patch checks per server

Andras Bacsaia3ed963aed44
diff --git a/app/Console/Commands/ScheduledJobDiagnostics.php b/app/Console/Commands/ScheduledJobDiagnostics.php
index 0b4f944a1d..a99184cc96 100644
--- a/app/Console/Commands/ScheduledJobDiagnostics.php
+++ b/app/Console/Commands/ScheduledJobDiagnostics.php
@@ -2,6 +2,7 @@
namespace App\Console\Commands;
+use App\Jobs\ServerManagerJob;
use App\Models\DockerCleanupExecution;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledJobDelivery;
@@ -210,7 +211,8 @@ class ScheduledJobDiagnostics extends Command
}
$dedupKeys = [
- "server-patch-check:{$server->id}" => '0 0 * * 0',
+ "server-patch-check:{$server->id}" => ServerManagerJob::patchCheckCron($server),
+ "sentinel-version-check:{$server->id}" => ServerManagerJob::sentinelVersionCheckCron($server),
"server-check:{$server->id}" => isCloud() ? '*/5 * * * *' : '* * * * *',
"server-storage-check:{$server->id}" => data_get($server->settings, 'server_disk_usage_check_frequency', '0 23 * * *'),
];
diff --git a/app/Jobs/ServerManagerJob.php b/app/Jobs/ServerManagerJob.php
index fdaa8b54a5..e33e639c61 100644
--- a/app/Jobs/ServerManagerJob.php
+++ b/app/Jobs/ServerManagerJob.php
@@ -173,7 +173,7 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
// Sentinel versions that report their version on push are updated by SentinelController.
if ($server->isSentinelEnabled()
&& ! Cache::has(Server::sentinelReportedVersionCacheKey($server->id))
- && shouldRunCronNow('0 * * * *', $serverTimezone, "sentinel-version-check:{$server->id}", $this->executionTime)
+ && shouldRunCronNow(self::sentinelVersionCheckCron($server), $serverTimezone, "sentinel-version-check:{$server->id}", $this->executionTime)
) {
CheckAndStartSentinelJob::dispatch($server);
}
@@ -192,16 +192,39 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
}
}
- // Dispatch ServerPatchCheckJob if due (weekly)
- $shouldRunPatchCheck = shouldRunCronNow('0 0 * * 0', $serverTimezone, "server-patch-check:{$server->id}", $this->executionTime);
+ // Dispatch ServerPatchCheckJob if due (weekly, staggered per server on Sunday)
+ $shouldRunPatchCheck = shouldRunCronNow(self::patchCheckCron($server), $serverTimezone, "server-patch-check:{$server->id}", $this->executionTime);
- if ($shouldRunPatchCheck) { // Weekly on Sunday at midnight
+ if ($shouldRunPatchCheck) {
ServerPatchCheckJob::dispatch($server);
}
// Crash recovery is handled by sentinelOutOfSync → ServerCheckJob → CheckAndStartSentinelJob.
}
+ /**
+ * Hourly Sentinel version check at a stable per-server minute, so servers do not all SSH at minute 0.
+ *
+ * ServerManagerJob evaluates this every minute and shouldRunCronNow() catches up a missed minute
+ * on the next run, so every minute of the hour is a safe slot.
+ */
+ public static function sentinelVersionCheckCron(Server $server): string
+ {
+ return sprintf('%d * * * *', $server->id % 60);
+ }
+
+ /**
+ * Weekly patch check at a stable per-server time on Sunday between 04:00 and 23:59.
+ *
+ * The early hours are skipped because most daylight saving transitions happen there.
+ */
+ public static function patchCheckCron(Server $server): string
+ {
+ $slot = $server->id % (20 * 60);
+
+ return sprintf('%d %d * * 0', $slot % 60, 4 + intdiv($slot, 60));
+ }
+
/**
* Determine the backoff cycle interval based on how many consecutive times a server has been unreachable.
* Higher counts → less frequent checks (based on 5-min cloud cycle):
diff --git a/tests/Feature/Server/SentinelUpdateCheckIndependenceTest.php b/tests/Feature/Server/SentinelUpdateCheckIndependenceTest.php
index ca1494840c..65a8157dcd 100644
--- a/tests/Feature/Server/SentinelUpdateCheckIndependenceTest.php
+++ b/tests/Feature/Server/SentinelUpdateCheckIndependenceTest.php
@@ -13,6 +13,7 @@ use Illuminate\Support\Facades\Queue;
uses(RefreshDatabase::class);
beforeEach(function () {
+ Server::flushIdentityMap();
Queue::fake();
InstanceSettings::forceCreate(['id' => 0]);
@@ -32,6 +33,9 @@ beforeEach(function () {
]);
$this->server->refresh();
+
+ // The hourly Sentinel check runs at a stable per-server minute.
+ $this->minute = sprintf('%02d', $this->server->id % 60);
});
afterEach(function () {
@@ -46,8 +50,8 @@ it('dispatches sentinel check hourly regardless of instance update_check_frequen
'instance_timezone' => 'UTC',
]);
- // Set time to top of any hour (sentinel should check every hour)
- Carbon::setTestNow('2025-06-15 14:00:00'); // Random hour, not January 1st
+ // Set time to the server minute of any hour (sentinel should check every hour)
+ Carbon::setTestNow("2025-06-15 14:{$this->minute}:00"); // Random hour, not January 1st
// Run ServerManagerJob
$job = new ServerManagerJob;
@@ -59,7 +63,7 @@ it('dispatches sentinel check hourly regardless of instance update_check_frequen
});
});
-it('does not dispatch sentinel check when not at top of hour', function () {
+it('does not dispatch sentinel check when not at the server minute', function () {
// Set instance update_check_frequency to hourly (most frequent)
$instanceSettings = InstanceSettings::first();
$instanceSettings->update([
@@ -67,14 +71,15 @@ it('does not dispatch sentinel check when not at top of hour', function () {
'instance_timezone' => 'UTC',
]);
- // Set time to middle of the hour (sentinel check cron won't match)
- Carbon::setTestNow('2025-06-15 14:30:00'); // 30 minutes past the hour
+ // Set time to another minute of the hour (sentinel check cron won't match)
+ $otherMinute = sprintf('%02d', ($this->server->id + 30) % 60);
+ Carbon::setTestNow("2025-06-15 14:{$otherMinute}:00");
// Run ServerManagerJob
$job = new ServerManagerJob;
$job->handle();
- // Assert that CheckAndStartSentinelJob was NOT dispatched (not top of hour)
+ // Assert that CheckAndStartSentinelJob was NOT dispatched (not the server minute)
Queue::assertNotPushed(CheckAndStartSentinelJob::class);
});
@@ -91,7 +96,7 @@ it('dispatches sentinel check at every hour mark throughout the day', function (
foreach ($hoursToTest as $hour) {
Queue::fake(); // Reset queue for each test
- Carbon::setTestNow("2025-06-15 {$hour}:00:00");
+ Carbon::setTestNow("2025-06-15 {$hour}:{$this->minute}:00");
$job = new ServerManagerJob;
$job->handle();
@@ -113,13 +118,13 @@ it('respects server timezone when checking sentinel updates', function () {
'instance_timezone' => 'UTC',
]);
- // Set time to 17:00 UTC which is 12:00 PM EST (top of hour in server's timezone)
- Carbon::setTestNow('2025-01-15 17:00:00');
+ // Set time to 17:xx UTC which is 12:xx PM EST (server minute in server's timezone)
+ Carbon::setTestNow("2025-01-15 17:{$this->minute}:00");
$job = new ServerManagerJob;
$job->handle();
- // Should dispatch because it's top of hour in server's timezone (America/New_York)
+ // Should dispatch because it's the server minute in server's timezone (America/New_York)
Queue::assertPushed(CheckAndStartSentinelJob::class, function ($job) {
return $job->server->id === $this->server->id;
});
@@ -137,7 +142,7 @@ it('does not dispatch sentinel check for build servers', function () {
'instance_timezone' => 'UTC',
]);
- Carbon::setTestNow('2025-06-15 14:00:00');
+ Carbon::setTestNow("2025-06-15 14:{$this->minute}:00");
$job = new ServerManagerJob;
$job->handle();
@@ -171,13 +176,17 @@ it('handles multiple servers with different sentinel eligibility', function () {
'instance_timezone' => 'UTC',
]);
- Carbon::setTestNow('2025-06-15 14:00:00');
+ // Each server is checked at its own minute
+ Carbon::setTestNow("2025-06-15 14:{$this->minute}:00");
+ (new ServerManagerJob)->handle();
- $job = new ServerManagerJob;
- $job->handle();
+ Carbon::setTestNow(sprintf('2025-06-15 15:%02d:00', $server3->id % 60));
+ (new ServerManagerJob)->handle();
- // Should dispatch for server1 (sentinel enabled) and server3 (sentinel enabled)
- Queue::assertPushed(CheckAndStartSentinelJob::class, 2);
+ // Should dispatch for server1 (sentinel enabled) and server3 (sentinel enabled), never for server2
+ Queue::assertNotPushed(CheckAndStartSentinelJob::class, function ($job) use ($server2) {
+ return $job->server->id === $server2->id;
+ });
// Verify it was dispatched for the correct servers
Queue::assertPushed(CheckAndStartSentinelJob::class, function ($job) {
diff --git a/tests/Feature/Server/ServerManagerJobStaggerTest.php b/tests/Feature/Server/ServerManagerJobStaggerTest.php
new file mode 100644
index 0000000000..9911c2e8fc
--- /dev/null
+++ b/tests/Feature/Server/ServerManagerJobStaggerTest.php
@@ -0,0 +1,115 @@
+<?php
+
+use App\Enums\ServerRole;
+use App\Jobs\CheckAndStartSentinelJob;
+use App\Jobs\ServerManagerJob;
+use App\Jobs\ServerPatchCheckJob;
+use App\Models\InstanceSettings;
+use App\Models\Server;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Queue;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+ Server::flushIdentityMap();
+ Queue::fake();
+ config(['constants.coolify.self_hosted' => true]);
+ Carbon::setTestNow('2025-01-15 12:00:00');
+ InstanceSettings::forceCreate(['id' => 0, 'instance_timezone' => 'UTC']);
+ $this->team = Team::factory()->create();
+});
+
+afterEach(function () {
+ Carbon::setTestNow();
+});
+
+function createStaggerTestServer(Team $team): Server
+{
+ $server = Server::factory()->create([
+ 'team_id' => $team->id,
+ 'sentinel_updated_at' => Carbon::now(),
+ ]);
+ $server->settings->update([
+ 'server_timezone' => 'UTC',
+ 'server_role' => ServerRole::BOTH,
+ 'is_reachable' => true,
+ 'is_usable' => true,
+ ]);
+
+ return $server->refresh();
+}
+
+function runStaggerTestManagerAt(Carbon $time): void
+{
+ Carbon::setTestNow($time);
+ Server::query()->update(['sentinel_updated_at' => $time]);
+ Server::flushIdentityMap();
+ (new ServerManagerJob)->handle();
+}
+
+it('gives each server a stable slot and spreads servers across minutes', function () {
+ $servers = collect(range(1, 120))->map(fn (int $id) => (new Server)->forceFill(['id' => $id]));
+
+ $hourlyMinutes = $servers->map(fn (Server $server) => ServerManagerJob::sentinelVersionCheckCron($server));
+ $weeklySlots = $servers->map(fn (Server $server) => ServerManagerJob::patchCheckCron($server));
+
+ expect($hourlyMinutes->unique())->toHaveCount(60)
+ ->and($weeklySlots->unique())->toHaveCount(120)
+ ->and(ServerManagerJob::sentinelVersionCheckCron($servers[6]))->toBe(ServerManagerJob::sentinelVersionCheckCron($servers[6]))
+ ->and(ServerManagerJob::patchCheckCron($servers[6]))->toBe(ServerManagerJob::patchCheckCron($servers[6]));
+
+ $weeklySlots->each(function (string $cron) {
+ [$minute, $hour, , , $dayOfWeek] = explode(' ', $cron);
+ expect((int) $minute)->toBeBetween(0, 59)
+ ->and((int) $hour)->toBeBetween(4, 23)
+ ->and($dayOfWeek)->toBe('0');
+ });
+});
+
+it('dispatches the Sentinel version check once per hour at the server minute', function () {
+ $server = createStaggerTestServer($this->team);
+ $minute = $server->id % 60;
+ $slot = Carbon::parse('2025-01-15 12:00:00')->setMinute($minute);
+
+ runStaggerTestManagerAt($slot->copy()->addMinute());
+ Queue::assertNotPushed(CheckAndStartSentinelJob::class);
+
+ runStaggerTestManagerAt($slot->copy()->addHour());
+ runStaggerTestManagerAt($slot->copy()->addHour()->addSeconds(30));
+ runStaggerTestManagerAt($slot->copy()->addHour()->addMinute());
+ Queue::assertPushed(CheckAndStartSentinelJob::class, 1);
+
+ runStaggerTestManagerAt($slot->copy()->addHours(2));
+ Queue::assertPushed(CheckAndStartSentinelJob::class, 2);
+});
+
+it('catches up a Sentinel version check when the server minute was not evaluated', function () {
+ $server = createStaggerTestServer($this->team);
+ $slot = Carbon::parse('2025-01-15 12:00:00')->setMinute($server->id % 60);
+
+ runStaggerTestManagerAt($slot->copy()->subMinute());
+ Queue::assertNotPushed(CheckAndStartSentinelJob::class);
+
+ runStaggerTestManagerAt($slot->copy()->addMinutes(3));
+ Queue::assertPushed(CheckAndStartSentinelJob::class, 1);
+});
+
+it('dispatches the patch check once per week at the server slot on Sunday', function () {
+ $server = createStaggerTestServer($this->team);
+ [$minute, $hour] = array_map('intval', explode(' ', ServerManagerJob::patchCheckCron($server)));
+ $sunday = Carbon::parse('2025-01-19 00:00:00');
+ $slot = $sunday->copy()->setTime($hour, $minute);
+
+ runStaggerTestManagerAt($sunday);
+ Queue::assertNotPushed(ServerPatchCheckJob::class);
+
+ runStaggerTestManagerAt($slot);
+ runStaggerTestManagerAt($slot->copy()->addMinute());
+ Queue::assertPushed(ServerPatchCheckJob::class, 1);
+
+ runStaggerTestManagerAt($slot->copy()->addWeek());
+ Queue::assertPushed(ServerPatchCheckJob::class, 2);
+});
diff --git a/tests/Unit/ServerManagerJobSentinelCheckTest.php b/tests/Unit/ServerManagerJobSentinelCheckTest.php
index dbbe800ad8..f17b3957be 100644
--- a/tests/Unit/ServerManagerJobSentinelCheckTest.php
+++ b/tests/Unit/ServerManagerJobSentinelCheckTest.php
@@ -16,6 +16,7 @@ use Tests\TestCase;
uses(TestCase::class, RefreshDatabase::class);
beforeEach(function () {
+ Server::flushIdentityMap();
Queue::fake();
Carbon::setTestNow('2025-01-15 12:00:00');
InstanceSettings::forceCreate(['id' => 0, 'instance_timezone' => 'UTC']);
@@ -45,6 +46,8 @@ function createSentinelCheckServer(Team $team, Carbon $sentinelUpdatedAt, Server
it('dispatches an hourly Sentinel version check for a healthy Sentinel', function () {
$server = createSentinelCheckServer($this->team, Carbon::now());
+ Carbon::setTestNow(Carbon::now()->setMinute($server->id % 60));
+ $server->update(['sentinel_updated_at' => Carbon::now()]);
expect($server->isSentinelEnabled())->toBeTrue()
->and($server->isSentinelLive())->toBeTrue();