An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(sentinel): stop hourly check job pile-up and self-heal Sentinel
Andras Bacsai51e6949c3654
diff --git a/app/Actions/Server/StartSentinel.php b/app/Actions/Server/StartSentinel.phpindex 8c9230c954..101f6b2d75 100644--- a/app/Actions/Server/StartSentinel.php+++ b/app/Actions/Server/StartSentinel.php@@ -117,7 +117,7 @@ class StartSentinel? '-v '.escapeshellarg("{$trafficLogDirectory}:{$trafficLogDirectory}:ro").' ': '';$network = $server->isLocalhost() ? ' --network coolify' : '';- $dockerCommand = "docker run -d$network $dockerEnvironments --name coolify-sentinel -v /var/run/docker.sock:/var/run/docker.sock -v $mountDir:/app/db {$trafficMount}--pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-start-period 120s --health-interval 10s --health-retries 3 --add-host=host.docker.internal:host-gateway --label $dockerLabels $image";+ $dockerCommand = "docker run -d$network $dockerEnvironments --name coolify-sentinel --restart unless-stopped -v /var/run/docker.sock:/var/run/docker.sock -v $mountDir:/app/db {$trafficMount}--pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-start-period 120s --health-interval 10s --health-retries 3 --add-host=host.docker.internal:host-gateway --label $dockerLabels $image";$server->sentinelHeartbeat(isReset: true);$server->forceFill(['sentinel_waiting_since' => now()])->save();diff --git a/app/Http/Controllers/Api/SentinelController.php b/app/Http/Controllers/Api/SentinelController.phpindex 3d3978d1c8..23bcf4f13c 100644--- a/app/Http/Controllers/Api/SentinelController.php+++ b/app/Http/Controllers/Api/SentinelController.php@@ -4,6 +4,7 @@ namespace App\Http\Controllers\Api;use App\Events\SentinelSynchronized;use App\Http\Controllers\Controller;+use App\Jobs\CheckAndStartSentinelJob;use App\Jobs\PushServerUpdateJob;use App\Models\Server;use Exception;@@ -103,6 +104,8 @@ class SentinelController extends ControllerSentinelSynchronized::dispatch($server);}+ $this->updateSentinelIfOutdated($server, $request->header('X-Sentinel-Version'));+if ($this->shouldDispatchUpdate($server, $data)) {PushServerUpdateJob::dispatch($server, $data);}@@ -110,6 +113,30 @@ class SentinelController extends Controllerreturn response()->json(['message' => 'ok'], 200);}+ /**+ * Sentinel sends its version on every push, so an outdated Sentinel is found without SSH.+ * The reported version also tells ServerManagerJob to skip its hourly SSH version check.+ * Old Sentinel versions do not send the header and keep the hourly SSH check.+ */+ private function updateSentinelIfOutdated(Server $server, ?string $runningVersion): void+ {+ if (! is_string($runningVersion) || preg_match('/^\d+\.\d+\.\d+$/', $runningVersion) !== 1) {+ return;+ }++ Cache::put(Server::sentinelReportedVersionCacheKey($server->id), $runningVersion, now()->addHours(2));++ $latestVersion = data_get(get_versions_data(), 'coolify.sentinel.version');+ if (blank($latestVersion) || version_compare($runningVersion, $latestVersion, '>=')) {+ return;+ }++ // Retry at most hourly if an update fails.+ if (Cache::add("sentinel:update-dispatched:{$server->id}", true, 3600)) {+ CheckAndStartSentinelJob::dispatch($server);+ }+ }+/*** Decide whether PushServerUpdateJob should be dispatched for this push.*diff --git a/app/Jobs/CheckAndStartSentinelJob.php b/app/Jobs/CheckAndStartSentinelJob.phpindex 5bfe5d504a..72213a9432 100644--- a/app/Jobs/CheckAndStartSentinelJob.php+++ b/app/Jobs/CheckAndStartSentinelJob.php@@ -6,19 +6,30 @@ use App\Actions\Server\StartSentinel;use App\Models\Server;use Illuminate\Bus\Queueable;use Illuminate\Contracts\Queue\ShouldBeEncrypted;+use Illuminate\Contracts\Queue\ShouldBeUnique;use Illuminate\Contracts\Queue\ShouldQueue;use Illuminate\Foundation\Bus\Dispatchable;use Illuminate\Queue\InteractsWithQueue;use Illuminate\Queue\SerializesModels;-class CheckAndStartSentinelJob implements ShouldBeEncrypted, ShouldQueue+class CheckAndStartSentinelJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue{use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;public $timeout = 120;+ /**+ * Only one check per server can wait in the queue, so a slow queue cannot collect copies.+ */+ public $uniqueFor = 600;+public function __construct(public Server $server) {}+ public function uniqueId(): string+ {+ return $this->server->uuid;+ }+public function handle(): void{if (! $this->sentinelIsEnabled()) {diff --git a/app/Jobs/ServerManagerJob.php b/app/Jobs/ServerManagerJob.phpindex a3710532b5..fdaa8b54a5 100644--- a/app/Jobs/ServerManagerJob.php+++ b/app/Jobs/ServerManagerJob.php@@ -13,6 +13,7 @@ use Illuminate\Queue\InteractsWithQueue;use Illuminate\Queue\SerializesModels;use Illuminate\Support\Carbon;use Illuminate\Support\Collection;+use Illuminate\Support\Facades\Cache;use Illuminate\Support\Facades\Log;class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue@@ -169,7 +170,9 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue}}+ // Sentinel versions that report their version on push are updated by SentinelController.if ($server->isSentinelEnabled()+ && ! Cache::has(Server::sentinelReportedVersionCacheKey($server->id))&& shouldRunCronNow('0 * * * *', $serverTimezone, "sentinel-version-check:{$server->id}", $this->executionTime)) {CheckAndStartSentinelJob::dispatch($server);diff --git a/app/Models/Server.php b/app/Models/Server.phpindex 04683c8e2c..d80a924ae6 100644--- a/app/Models/Server.php+++ b/app/Models/Server.php@@ -1079,6 +1079,11 @@ $siteAddress {return max(30, $this->settings->sentinel_push_interval_seconds + 30);}+ public static function sentinelReportedVersionCacheKey(int $serverId): string+ {+ return "sentinel:reported-version:{$serverId}";+ }+public function isSentinelLive(){return Carbon::parse($this->sentinel_updated_at)->isAfter(now()->subSeconds($this->waitBeforeDoingSshCheck()));diff --git a/tests/Feature/SentinelVersionReportTest.php b/tests/Feature/SentinelVersionReportTest.phpnew file mode 100644index 0000000000..9b9227b6e7--- /dev/null+++ b/tests/Feature/SentinelVersionReportTest.php@@ -0,0 +1,61 @@+<?php++use App\Jobs\CheckAndStartSentinelJob;+use App\Models\Server;+use App\Models\User;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Facades\Cache;+use Illuminate\Support\Facades\Queue;++uses(RefreshDatabase::class);++beforeEach(function () {+ config(['app.maintenance.store' => 'array']);++ Queue::fake();+ Cache::flush();+ Cache::put('coolify:versions:all', ['coolify' => ['sentinel' => ['version' => '1.0.2']]], 3600);++ $user = User::factory()->create();+ $this->server = Server::factory()->create(['team_id' => $user->teams()->first()->id]);+ $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]);+});++function pushSentinelWithVersion(Server $server, ?string $version)+{+ $headers = ['Authorization' => 'Bearer '.$server->settings->sentinel_token];+ if ($version !== null) {+ $headers['X-Sentinel-Version'] = $version;+ }++ return test()->postJson('/api/v1/sentinel/push', ['containers' => []], $headers);+}++it('updates an outdated Sentinel once per hour', function () {+ pushSentinelWithVersion($this->server, '1.0.1')->assertOk();+ pushSentinelWithVersion($this->server, '1.0.1')->assertOk();++ Queue::assertPushed(CheckAndStartSentinelJob::class, 1);+ expect(Cache::get(Server::sentinelReportedVersionCacheKey($this->server->id)))->toBe('1.0.1');+});++it('does not update a current Sentinel', function (string $version) {+ pushSentinelWithVersion($this->server, $version)->assertOk();++ Queue::assertNotPushed(CheckAndStartSentinelJob::class);+ expect(Cache::has(Server::sentinelReportedVersionCacheKey($this->server->id)))->toBeTrue();+})->with(['1.0.2', '1.1.0']);++it('keeps the hourly SSH version check for Sentinel without a valid version header', function (?string $version) {+ pushSentinelWithVersion($this->server, $version)->assertOk();++ Queue::assertNotPushed(CheckAndStartSentinelJob::class);+ expect(Cache::has(Server::sentinelReportedVersionCacheKey($this->server->id)))->toBeFalse();+})->with(['no header' => null, 'not a version' => 'next']);++it('queues only one Sentinel check per server while one is waiting', function () {+ CheckAndStartSentinelJob::dispatch($this->server);+ CheckAndStartSentinelJob::dispatch($this->server);++ Queue::assertPushed(CheckAndStartSentinelJob::class, 1);+});diff --git a/tests/Feature/TrafficAnalytics/StartSentinelTrafficTest.php b/tests/Feature/TrafficAnalytics/StartSentinelTrafficTest.phpindex 52ff4ab656..0f23a64cbf 100644--- a/tests/Feature/TrafficAnalytics/StartSentinelTrafficTest.php+++ b/tests/Feature/TrafficAnalytics/StartSentinelTrafficTest.php@@ -164,6 +164,12 @@ it('creates the access log before starting sentinel with traffic analytics', fun'caddy' => ['CADDY', '/data/coolify/proxy/caddy'],]);+it('starts sentinel with a restart policy so Docker restarts it after a crash or reboot', function () {+ $script = runStartSentinelAndCaptureScript(sentinelTrafficServer($this, 'TRAEFIK', analyticsEnabled: false));++ expect($script)->toContain('--name coolify-sentinel --restart unless-stopped ');+});+it('does not touch the access log when traffic analytics cannot run', function (?string $proxyType, bool $analyticsEnabled) {$server = sentinelTrafficServer($this, $proxyType, $analyticsEnabled);diff --git a/tests/Unit/ServerManagerJobSentinelCheckTest.php b/tests/Unit/ServerManagerJobSentinelCheckTest.phpindex df1b1c3ee6..dbbe800ad8 100644--- a/tests/Unit/ServerManagerJobSentinelCheckTest.php+++ b/tests/Unit/ServerManagerJobSentinelCheckTest.php@@ -9,6 +9,7 @@ use App\Models\Server;use App\Models\Team;use Illuminate\Foundation\Testing\RefreshDatabase;use Illuminate\Support\Carbon;+use Illuminate\Support\Facades\Cache;use Illuminate\Support\Facades\Queue;use Tests\TestCase;@@ -56,6 +57,15 @@ it('dispatches an hourly Sentinel version check for a healthy Sentinel', functio});});+it('skips the hourly SSH version check when Sentinel reports its version on push', function () {+ $server = createSentinelCheckServer($this->team, Carbon::now());+ Cache::put(Server::sentinelReportedVersionCacheKey($server->id), '1.0.2', now()->addHours(2));++ (new ServerManagerJob)->handle();++ Queue::assertNotPushed(CheckAndStartSentinelJob::class);+});+it('does not schedule periodic Sentinel restart checks', function () {$root = dirname(__DIR__, 2);$manager = file_get_contents($root.'/app/Jobs/ServerManagerJob.php');