Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(sentinel): stop hourly check job pile-up and self-heal Sentinel

Andras Bacsai51e6949c3654
diff --git a/app/Actions/Server/StartSentinel.php b/app/Actions/Server/StartSentinel.php
index 8c9230c954..101f6b2d75 100644
--- a/app/Actions/Server/StartSentinel.php
+++ b/app/Actions/Server/StartSentinel.php
@@ -117,7 +117,7 @@ class StartSentinel
? '-v '.escapeshellarg("{$trafficLogDirectory}:{$trafficLogDirectory}:ro").' '
: '';
$network = $server->isLocalhost() ? ' --network coolify' : '';
- $dockerCommand = "docker run -d$network $dockerEnvironments --name coolify-sentinel -v /var/run/docker.sock:/var/run/docker.sock -v $mountDir:/app/db {$trafficMount}--pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-start-period 120s --health-interval 10s --health-retries 3 --add-host=host.docker.internal:host-gateway --label $dockerLabels $image";
+ $dockerCommand = "docker run -d$network $dockerEnvironments --name coolify-sentinel --restart unless-stopped -v /var/run/docker.sock:/var/run/docker.sock -v $mountDir:/app/db {$trafficMount}--pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-start-period 120s --health-interval 10s --health-retries 3 --add-host=host.docker.internal:host-gateway --label $dockerLabels $image";
$server->sentinelHeartbeat(isReset: true);
$server->forceFill(['sentinel_waiting_since' => now()])->save();
diff --git a/app/Http/Controllers/Api/SentinelController.php b/app/Http/Controllers/Api/SentinelController.php
index 3d3978d1c8..23bcf4f13c 100644
--- a/app/Http/Controllers/Api/SentinelController.php
+++ b/app/Http/Controllers/Api/SentinelController.php
@@ -4,6 +4,7 @@ namespace App\Http\Controllers\Api;
use App\Events\SentinelSynchronized;
use App\Http\Controllers\Controller;
+use App\Jobs\CheckAndStartSentinelJob;
use App\Jobs\PushServerUpdateJob;
use App\Models\Server;
use Exception;
@@ -103,6 +104,8 @@ class SentinelController extends Controller
SentinelSynchronized::dispatch($server);
}
+ $this->updateSentinelIfOutdated($server, $request->header('X-Sentinel-Version'));
+
if ($this->shouldDispatchUpdate($server, $data)) {
PushServerUpdateJob::dispatch($server, $data);
}
@@ -110,6 +113,30 @@ class SentinelController extends Controller
return response()->json(['message' => 'ok'], 200);
}
+ /**
+ * Sentinel sends its version on every push, so an outdated Sentinel is found without SSH.
+ * The reported version also tells ServerManagerJob to skip its hourly SSH version check.
+ * Old Sentinel versions do not send the header and keep the hourly SSH check.
+ */
+ private function updateSentinelIfOutdated(Server $server, ?string $runningVersion): void
+ {
+ if (! is_string($runningVersion) || preg_match('/^\d+\.\d+\.\d+$/', $runningVersion) !== 1) {
+ return;
+ }
+
+ Cache::put(Server::sentinelReportedVersionCacheKey($server->id), $runningVersion, now()->addHours(2));
+
+ $latestVersion = data_get(get_versions_data(), 'coolify.sentinel.version');
+ if (blank($latestVersion) || version_compare($runningVersion, $latestVersion, '>=')) {
+ return;
+ }
+
+ // Retry at most hourly if an update fails.
+ if (Cache::add("sentinel:update-dispatched:{$server->id}", true, 3600)) {
+ CheckAndStartSentinelJob::dispatch($server);
+ }
+ }
+
/**
* Decide whether PushServerUpdateJob should be dispatched for this push.
*
diff --git a/app/Jobs/CheckAndStartSentinelJob.php b/app/Jobs/CheckAndStartSentinelJob.php
index 5bfe5d504a..72213a9432 100644
--- a/app/Jobs/CheckAndStartSentinelJob.php
+++ b/app/Jobs/CheckAndStartSentinelJob.php
@@ -6,19 +6,30 @@ use App\Actions\Server\StartSentinel;
use App\Models\Server;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
-class CheckAndStartSentinelJob implements ShouldBeEncrypted, ShouldQueue
+class CheckAndStartSentinelJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $timeout = 120;
+ /**
+ * Only one check per server can wait in the queue, so a slow queue cannot collect copies.
+ */
+ public $uniqueFor = 600;
+
public function __construct(public Server $server) {}
+ public function uniqueId(): string
+ {
+ return $this->server->uuid;
+ }
+
public function handle(): void
{
if (! $this->sentinelIsEnabled()) {
diff --git a/app/Jobs/ServerManagerJob.php b/app/Jobs/ServerManagerJob.php
index a3710532b5..fdaa8b54a5 100644
--- a/app/Jobs/ServerManagerJob.php
+++ b/app/Jobs/ServerManagerJob.php
@@ -13,6 +13,7 @@ use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Carbon;
use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Log;
class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
@@ -169,7 +170,9 @@ class ServerManagerJob implements ShouldBeEncrypted, ShouldQueue
}
}
+ // Sentinel versions that report their version on push are updated by SentinelController.
if ($server->isSentinelEnabled()
+ && ! Cache::has(Server::sentinelReportedVersionCacheKey($server->id))
&& shouldRunCronNow('0 * * * *', $serverTimezone, "sentinel-version-check:{$server->id}", $this->executionTime)
) {
CheckAndStartSentinelJob::dispatch($server);
diff --git a/app/Models/Server.php b/app/Models/Server.php
index 04683c8e2c..d80a924ae6 100644
--- a/app/Models/Server.php
+++ b/app/Models/Server.php
@@ -1079,6 +1079,11 @@ $siteAddress {
return max(30, $this->settings->sentinel_push_interval_seconds + 30);
}
+ public static function sentinelReportedVersionCacheKey(int $serverId): string
+ {
+ return "sentinel:reported-version:{$serverId}";
+ }
+
public function isSentinelLive()
{
return Carbon::parse($this->sentinel_updated_at)->isAfter(now()->subSeconds($this->waitBeforeDoingSshCheck()));
diff --git a/tests/Feature/SentinelVersionReportTest.php b/tests/Feature/SentinelVersionReportTest.php
new file mode 100644
index 0000000000..9b9227b6e7
--- /dev/null
+++ b/tests/Feature/SentinelVersionReportTest.php
@@ -0,0 +1,61 @@
+<?php
+
+use App\Jobs\CheckAndStartSentinelJob;
+use App\Models\Server;
+use App\Models\User;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Queue;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+ config(['app.maintenance.store' => 'array']);
+
+ Queue::fake();
+ Cache::flush();
+ Cache::put('coolify:versions:all', ['coolify' => ['sentinel' => ['version' => '1.0.2']]], 3600);
+
+ $user = User::factory()->create();
+ $this->server = Server::factory()->create(['team_id' => $user->teams()->first()->id]);
+ $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]);
+});
+
+function pushSentinelWithVersion(Server $server, ?string $version)
+{
+ $headers = ['Authorization' => 'Bearer '.$server->settings->sentinel_token];
+ if ($version !== null) {
+ $headers['X-Sentinel-Version'] = $version;
+ }
+
+ return test()->postJson('/api/v1/sentinel/push', ['containers' => []], $headers);
+}
+
+it('updates an outdated Sentinel once per hour', function () {
+ pushSentinelWithVersion($this->server, '1.0.1')->assertOk();
+ pushSentinelWithVersion($this->server, '1.0.1')->assertOk();
+
+ Queue::assertPushed(CheckAndStartSentinelJob::class, 1);
+ expect(Cache::get(Server::sentinelReportedVersionCacheKey($this->server->id)))->toBe('1.0.1');
+});
+
+it('does not update a current Sentinel', function (string $version) {
+ pushSentinelWithVersion($this->server, $version)->assertOk();
+
+ Queue::assertNotPushed(CheckAndStartSentinelJob::class);
+ expect(Cache::has(Server::sentinelReportedVersionCacheKey($this->server->id)))->toBeTrue();
+})->with(['1.0.2', '1.1.0']);
+
+it('keeps the hourly SSH version check for Sentinel without a valid version header', function (?string $version) {
+ pushSentinelWithVersion($this->server, $version)->assertOk();
+
+ Queue::assertNotPushed(CheckAndStartSentinelJob::class);
+ expect(Cache::has(Server::sentinelReportedVersionCacheKey($this->server->id)))->toBeFalse();
+})->with(['no header' => null, 'not a version' => 'next']);
+
+it('queues only one Sentinel check per server while one is waiting', function () {
+ CheckAndStartSentinelJob::dispatch($this->server);
+ CheckAndStartSentinelJob::dispatch($this->server);
+
+ Queue::assertPushed(CheckAndStartSentinelJob::class, 1);
+});
diff --git a/tests/Feature/TrafficAnalytics/StartSentinelTrafficTest.php b/tests/Feature/TrafficAnalytics/StartSentinelTrafficTest.php
index 52ff4ab656..0f23a64cbf 100644
--- a/tests/Feature/TrafficAnalytics/StartSentinelTrafficTest.php
+++ b/tests/Feature/TrafficAnalytics/StartSentinelTrafficTest.php
@@ -164,6 +164,12 @@ it('creates the access log before starting sentinel with traffic analytics', fun
'caddy' => ['CADDY', '/data/coolify/proxy/caddy'],
]);
+it('starts sentinel with a restart policy so Docker restarts it after a crash or reboot', function () {
+ $script = runStartSentinelAndCaptureScript(sentinelTrafficServer($this, 'TRAEFIK', analyticsEnabled: false));
+
+ expect($script)->toContain('--name coolify-sentinel --restart unless-stopped ');
+});
+
it('does not touch the access log when traffic analytics cannot run', function (?string $proxyType, bool $analyticsEnabled) {
$server = sentinelTrafficServer($this, $proxyType, $analyticsEnabled);
diff --git a/tests/Unit/ServerManagerJobSentinelCheckTest.php b/tests/Unit/ServerManagerJobSentinelCheckTest.php
index df1b1c3ee6..dbbe800ad8 100644
--- a/tests/Unit/ServerManagerJobSentinelCheckTest.php
+++ b/tests/Unit/ServerManagerJobSentinelCheckTest.php
@@ -9,6 +9,7 @@ use App\Models\Server;
use App\Models\Team;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Queue;
use Tests\TestCase;
@@ -56,6 +57,15 @@ it('dispatches an hourly Sentinel version check for a healthy Sentinel', functio
});
});
+it('skips the hourly SSH version check when Sentinel reports its version on push', function () {
+ $server = createSentinelCheckServer($this->team, Carbon::now());
+ Cache::put(Server::sentinelReportedVersionCacheKey($server->id), '1.0.2', now()->addHours(2));
+
+ (new ServerManagerJob)->handle();
+
+ Queue::assertNotPushed(CheckAndStartSentinelJob::class);
+});
+
it('does not schedule periodic Sentinel restart checks', function () {
$root = dirname(__DIR__, 2);
$manager = file_get_contents($root.'/app/Jobs/ServerManagerJob.php');