An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(sentinel): repair stalled Sentinel and show why pushes fail
Andras Bacsai66cc64e0d914
diff --git a/app/Http/Controllers/Api/SentinelController.php b/app/Http/Controllers/Api/SentinelController.phpindex 23bcf4f13c..109b0db810 100644--- a/app/Http/Controllers/Api/SentinelController.php+++ b/app/Http/Controllers/Api/SentinelController.php@@ -60,6 +60,7 @@ class SentinelController extends Controller'server_uuid' => $server->uuid,'team_id' => $server->team_id,]);+ $server->rememberSentinelPushProblem('Coolify rejected the push: the subscription of this team is not active.');return response()->json(['message' => 'Unauthorized'], 401);}@@ -69,6 +70,7 @@ class SentinelController extends Controller'server_uuid' => $server->uuid,'team_id' => $server->team_id,]);+ $server->rememberSentinelPushProblem('Coolify rejected the push: the server is marked unreachable, not usable, or disabled. Validate the server connection.');return response()->json(['message' => 'Server is not functional'], 401);}@@ -78,6 +80,7 @@ class SentinelController extends Controller'server_uuid' => $server->uuid,'team_id' => $server->team_id,]);+ $server->rememberSentinelPushProblem('Coolify rejected the push: the Sentinel token does not match. Sync Sentinel to apply the current token.');return response()->json(['message' => 'Unauthorized'], 401);}@@ -86,6 +89,8 @@ class SentinelController extends Controller]);if ($validator->fails()) {+ $server->rememberSentinelPushProblem('Coolify rejected the push: the payload is not valid.');+return response()->json(serializeApiResponse(['message' => 'Validation failed.','errors' => $validator->errors(),@@ -101,6 +106,7 @@ class SentinelController extends Controller$server->sentinelHeartbeat();if (! $wasSentinelLive) {+ $server->rememberSentinelPushProblem(null);SentinelSynchronized::dispatch($server);}diff --git a/app/Jobs/CheckAndStartSentinelJob.php b/app/Jobs/CheckAndStartSentinelJob.phpindex 49e08ab1a6..183063f09e 100644--- a/app/Jobs/CheckAndStartSentinelJob.php+++ b/app/Jobs/CheckAndStartSentinelJob.php@@ -11,11 +11,22 @@ use Illuminate\Contracts\Queue\ShouldQueue;use Illuminate\Foundation\Bus\Dispatchable;use Illuminate\Queue\InteractsWithQueue;use Illuminate\Queue\SerializesModels;+use Illuminate\Support\Carbon;+use Illuminate\Support\Facades\Cache;class CheckAndStartSentinelJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue{use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;+ /**+ * A running Sentinel that has not pushed for this long is restarted with the current settings.+ * This repairs a changed Coolify URL or token, and a hung push loop.+ */+ public const STALL_RESTART_AFTER_SECONDS = 600;++ /** A stalled Sentinel is restarted at most once in this period, because a firewall problem stays after a restart. */+ public const STALL_RESTART_INTERVAL_SECONDS = 3600;+public $timeout = 120;/**@@ -40,28 +51,63 @@ class CheckAndStartSentinelJob implements ShouldBeEncrypted, ShouldBeUnique, Sho// An SSH failure throws here, so it is never mistaken for a missing container.// A missing container gives empty output and exit code 0.- $sentinelStatus = instant_remote_process_with_timeout(["docker ps -a --filter 'name=^coolify-sentinel$' --format '{{.State}}'"], $this->server);- if ($sentinelStatus !== 'running') {+ $sentinelStatus = instant_remote_process_with_timeout(["docker ps -a --filter 'name=^coolify-sentinel$' --format '{{.State}} {{.Status}}'"], $this->server);+ if (! str_starts_with((string) $sentinelStatus, 'running')) {$this->startSentinel($latestVersion);return;}- // If sentinel is running, check if it needs an update- $runningVersion = instant_remote_process_with_timeout(['docker exec coolify-sentinel sh -c "curl http://127.0.0.1:8888/api/version"'], $this->server, false);- if (empty($runningVersion)) {- $runningVersion = '0.0.0';+ if (str_contains($sentinelStatus, '(unhealthy)')) {+ $this->server->rememberSentinelPushProblem('The Sentinel container was unhealthy. Coolify restarted it.');+ $this->startSentinel($latestVersion);++ return;}- if ($latestVersion === '0.0.0' && $runningVersion === '0.0.0') {- $this->startSentinel('latest');++ // One SSH call reads the version and the last push result. Sentinel versions without+ // /api/push-status give an empty second line.+ $output = instant_remote_process_with_timeout([+ "docker exec coolify-sentinel sh -c 'curl -s http://127.0.0.1:8888/api/version; echo; curl -sf -H \"Authorization: Bearer \$TOKEN\" http://127.0.0.1:8888/api/push-status'",+ ], $this->server, false);+ [$runningVersion, $pushStatus] = array_pad(explode("\n", trim((string) $output), 2), 2, null);+ $this->rememberPushError($pushStatus);++ // An empty version means that the API does not answer yet, for example while Sentinel starts.+ // A Sentinel whose API stays down fails its health check and is restarted above.+ if (filled($runningVersion) && version_compare(trim($runningVersion), $latestVersion, '<')) {+ $this->startSentinel($latestVersion);return;- } else {- if (version_compare($runningVersion, $latestVersion, '<')) {- $this->startSentinel($latestVersion);+ }++ if ($this->isStalled() && Cache::add("sentinel:stall-restart:{$this->server->id}", true, self::STALL_RESTART_INTERVAL_SECONDS)) {+ $this->startSentinel($latestVersion);+ }+ }++ /**+ * Sentinel is stalled when it has not pushed for STALL_RESTART_AFTER_SECONDS since its last push,+ * or since its last start when it never pushed.+ */+ private function isStalled(): bool+ {+ $lastSignOfLife = $this->server->sentinel_waiting_since ?? $this->server->sentinel_updated_at;+ if ($lastSignOfLife === null) {+ return true;+ }+ $stallAfter = max(self::STALL_RESTART_AFTER_SECONDS, $this->server->waitBeforeDoingSshCheck());++ return Carbon::parse($lastSignOfLife)->isBefore(now()->subSeconds($stallAfter));+ }- return;- }+ private function rememberPushError(?string $pushStatus): void+ {+ $status = json_decode(trim((string) $pushStatus), true);+ if (! is_array($status) || data_get($status, 'consecutive_failures', 0) < 1 || blank(data_get($status, 'last_error'))) {+ return;}++ $this->server->rememberSentinelPushProblem('Sentinel cannot push to Coolify: '.data_get($status, 'last_error'));}private function sentinelIsEnabled(): booldiff --git a/app/Models/InstanceSettings.php b/app/Models/InstanceSettings.phpindex 9c9bc3b4ea..16a1d7ea30 100644--- a/app/Models/InstanceSettings.php+++ b/app/Models/InstanceSettings.php@@ -111,6 +111,13 @@ class InstanceSettings extends Modelstatic::updated(function ($settings) {// Clear once() cache so subsequent calls get fresh dataOnce::flush();++ if ($settings->wasChanged(['fqdn', 'public_ipv4', 'public_ipv6'])) {+ ServerSetting::followInstanceUrlChange(+ ServerSetting::instanceSentinelUrl($settings->getOriginal('fqdn'), $settings->getOriginal('public_ipv4'), $settings->getOriginal('public_ipv6')),+ ServerSetting::instanceSentinelUrl($settings->fqdn, $settings->public_ipv4, $settings->public_ipv6),+ );+ }});}diff --git a/app/Models/Server.php b/app/Models/Server.phpindex cf32ce702a..88f3ecc15a 100644--- a/app/Models/Server.php+++ b/app/Models/Server.php@@ -33,9 +33,11 @@ use Illuminate\Database\Eloquent\Relations\MorphMany;use Illuminate\Database\Eloquent\SoftDeletes;use Illuminate\Support\Carbon;use Illuminate\Support\Collection;+use Illuminate\Support\Facades\Cache;use Illuminate\Support\Facades\DB;use Illuminate\Support\Facades\Log;use Illuminate\Support\Facades\Storage;+use Illuminate\Support\Str;use Illuminate\Support\Stringable;use OpenApi\Attributes as OA;use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;@@ -1082,6 +1084,25 @@ $siteAddress {return "sentinel:reported-version:{$serverId}";}+ /**+ * The last known reason why Sentinel pushes do not arrive. It is shown while Sentinel is out of sync.+ */+ public function sentinelPushProblem(): ?string+ {+ return Cache::get("sentinel:push-problem:{$this->id}");+ }++ public function rememberSentinelPushProblem(?string $problem): void+ {+ if (blank($problem)) {+ Cache::forget("sentinel:push-problem:{$this->id}");++ return;+ }++ Cache::put("sentinel:push-problem:{$this->id}", Str::limit($problem, 500), now()->addDay());+ }+public function isSentinelLive(){return Carbon::parse($this->sentinel_updated_at)->isAfter(now()->subSeconds($this->waitBeforeDoingSshCheck()));diff --git a/app/Models/ServerSetting.php b/app/Models/ServerSetting.phpindex 1fb3af6d6c..a4d51afdae 100644--- a/app/Models/ServerSetting.php+++ b/app/Models/ServerSetting.php@@ -315,6 +315,38 @@ class ServerSetting extends Modelreturn $url;}+ /**+ * The Sentinel push URL that the instance URL or public IP gives to a remote server.+ */+ public static function instanceSentinelUrl(?string $fqdn, ?string $publicIpv4, ?string $publicIpv6): ?string+ {+ return match (true) {+ filled($fqdn) => $fqdn,+ filled($publicIpv4) => 'http://'.$publicIpv4.':8000',+ filled($publicIpv6) => 'http://'.$publicIpv6.':8000',+ default => null,+ };+ }++ /**+ * Move remote servers that use the generated Sentinel URL to the new instance URL. Saving the+ * new URL restarts their Sentinel. Servers with a custom URL keep it.+ */+ public static function followInstanceUrlChange(?string $oldUrl, ?string $newUrl): void+ {+ if (blank($oldUrl) || blank($newUrl) || $oldUrl === $newUrl) {+ return;+ }++ self::query()+ ->where('sentinel_custom_url', $oldUrl)+ ->where('server_id', '!=', 0)+ ->each(function (ServerSetting $setting) use ($newUrl) {+ $setting->sentinel_custom_url = $newUrl;+ $setting->save();+ });+ }+public function restoreDefaultSentinelConfiguration(): void{$this->generateSentinelUrl(save: false, ignoreEvent: true);@@ -327,18 +359,12 @@ class ServerSetting extends Modelpublic function generateSentinelUrl(bool $save = true, bool $ignoreEvent = false): ?string{- $domain = null;$settings = InstanceSettings::get();if ($this->server->isLocalhost()) {$domain = 'http://coolify:8080';- } elseif ($settings->fqdn) {- $domain = $settings->fqdn;- } elseif ($settings->public_ipv4) {- $domain = 'http://'.$settings->public_ipv4.':8000';- } elseif ($settings->public_ipv6) {- $domain = 'http://'.$settings->public_ipv6.':8000';} else {- $domain = $this->sentinelUrlFromCurrentRequest();+ $domain = self::instanceSentinelUrl($settings->fqdn, $settings->public_ipv4, $settings->public_ipv6)+ ?? $this->sentinelUrlFromCurrentRequest();}$this->sentinel_custom_url = $domain;if ($save) {diff --git a/resources/views/livewire/server/sentinel.blade.php b/resources/views/livewire/server/sentinel.blade.phpindex e3a72e8686..02b32b155e 100644--- a/resources/views/livewire/server/sentinel.blade.php+++ b/resources/views/livewire/server/sentinel.blade.php@@ -38,6 +38,9 @@<x-callout type="warning" title="Sentinel is out of sync"><div class="space-y-3"><p>Sentinel has not reported within the expected interval. Check these items before syncing again:</p>+ @if ($sentinelPushProblem = $server->sentinelPushProblem())+ <p><span class="font-medium">Last error:</span> <code class="break-all">{{ $sentinelPushProblem }}</code></p>+ @endif<ul class="list-disc space-y-1 pl-4"><li>Confirm that the <code>coolify-sentinel</code> container is running.</li><li>diff --git a/tests/Feature/SentinelPushProblemTest.php b/tests/Feature/SentinelPushProblemTest.phpnew file mode 100644index 0000000000..18c6abfaf1--- /dev/null+++ b/tests/Feature/SentinelPushProblemTest.php@@ -0,0 +1,76 @@+<?php++use App\Actions\Server\StartSentinel;+use App\Models\InstanceSettings;+use App\Models\Server;+use App\Models\User;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Facades\Cache;+use Illuminate\Support\Facades\Queue;++uses(RefreshDatabase::class);++beforeEach(function () {+ config(['app.maintenance.store' => 'array']);+ Server::flushIdentityMap();+ Queue::fake();+ Cache::flush();+ InstanceSettings::forceCreate(['id' => 0, 'fqdn' => 'https://old.example.com']);++ $user = User::factory()->create();+ $this->server = Server::factory()->create(['team_id' => $user->teams()->first()->id]);+ $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]);+});++function pushSentinelForProblemTest(Server $server, string $token)+{+ return test()->postJson('/api/v1/sentinel/push', ['containers' => []], ['Authorization' => 'Bearer '.$token]);+}++it('remembers when Coolify rejects a push from a server that is not functional', function () {+ $this->server->settings->update(['is_reachable' => false]);++ pushSentinelForProblemTest($this->server, $this->server->settings->sentinel_token)->assertUnauthorized();++ expect($this->server->sentinelPushProblem())->toContain('marked unreachable');+});++it('remembers when Coolify rejects a push with an old token', function () {+ $oldToken = $this->server->settings->sentinel_token;+ $this->server->settings->generateSentinelToken(ignoreEvent: true);++ pushSentinelForProblemTest($this->server, $oldToken)->assertUnauthorized();++ expect($this->server->sentinelPushProblem())->toContain('token does not match');+});++it('forgets the problem after the next accepted push', function () {+ $this->server->rememberSentinelPushProblem('Coolify rejected the push: the Sentinel token does not match.');+ $this->server->forceFill(['sentinel_updated_at' => now()->subHour()])->save();++ pushSentinelForProblemTest($this->server, $this->server->settings->sentinel_token)->assertOk();++ expect($this->server->sentinelPushProblem())->toBeNull();+});++it('moves Sentinel to the new instance URL when the instance URL changes', function () {+ $this->server->settings->update(['sentinel_custom_url' => 'https://old.example.com', 'is_sentinel_enabled' => true]);+ Queue::fake();++ $settings = InstanceSettings::get();+ $settings->fqdn = 'https://new.example.com';+ $settings->save();++ expect($this->server->settings->refresh()->sentinel_custom_url)->toBe('https://new.example.com');+ StartSentinel::assertPushed(fn (StartSentinel $action, array $arguments) => $arguments[0]->is($this->server));+});++it('keeps a custom Sentinel URL when the instance URL changes', function () {+ $this->server->settings->update(['sentinel_custom_url' => 'https://custom.example.com']);++ $settings = InstanceSettings::get();+ $settings->fqdn = 'https://new.example.com';+ $settings->save();++ expect($this->server->settings->refresh()->sentinel_custom_url)->toBe('https://custom.example.com');+});diff --git a/tests/Feature/Server/CheckAndStartSentinelJobTest.php b/tests/Feature/Server/CheckAndStartSentinelJobTest.phpindex b2a33f809d..cb541b3519 100644--- a/tests/Feature/Server/CheckAndStartSentinelJobTest.php+++ b/tests/Feature/Server/CheckAndStartSentinelJobTest.php@@ -7,6 +7,7 @@ use App\Models\PrivateKey;use App\Models\Server;use App\Models\User;use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Facades\Cache;use Illuminate\Support\Facades\Http;use Illuminate\Support\Facades\Process;use Illuminate\Support\Facades\Storage;@@ -19,6 +20,7 @@ beforeEach(function () {Storage::fake('ssh-keys');Storage::fake('ssh-mux');config()->set('constants.ssh.max_retries', 1);+ config()->set('constants.ssh.mux_enabled', false);Http::fake(['*' => Http::response(['coolify' => ['sentinel' => ['version' => '1.0.0']]])]);$user = User::factory()->create();@@ -61,6 +63,7 @@ it('does not treat an SSH failure as a missing container', function () {});it('does not restart a running Sentinel that is up to date', function () {+ $this->server->forceFill(['sentinel_updated_at' => now()])->save();Process::fake(['*docker ps*' => Process::result(output: 'running'),'*api/version*' => Process::result(output: '1.0.0'),@@ -71,7 +74,7 @@ it('does not restart a running Sentinel that is up to date', function () {});it('uses sudo for the status check on a non-root server', function () {- $this->server->update(['user' => 'ubuntu']);+ $this->server->forceFill(['user' => 'ubuntu', 'sentinel_updated_at' => now()])->save();Process::fake(['*docker ps*' => Process::result(output: 'running'),'*api/version*' => Process::result(output: '1.0.0'),@@ -82,3 +85,93 @@ it('uses sudo for the status check on a non-root server', function () {Process::assertRan(fn ($process) => str_contains($process->command, 'sudo docker ps -a'));});++it('restarts an unhealthy Sentinel and remembers why', function () {+ $this->server->forceFill(['sentinel_updated_at' => now()])->save();+ Process::fake(['*docker ps*' => Process::result(output: 'running Up 2 hours (unhealthy)')]);+ StartSentinel::shouldRun()->once();++ (new CheckAndStartSentinelJob($this->server))->handle();++ expect($this->server->sentinelPushProblem())->toContain('unhealthy');+});++it('does not restart a Sentinel whose health check is still starting', function () {+ $this->server->forceFill(['sentinel_updated_at' => now()])->save();+ Process::fake([+ '*docker ps*' => Process::result(output: 'running Up 30 seconds (health: starting)'),+ '*api/version*' => Process::result(output: '1.0.0'),+ ]);+ StartSentinel::shouldNotRun();++ (new CheckAndStartSentinelJob($this->server))->handle();+});++it('restarts a running Sentinel that stopped pushing, at most once per hour', function () {+ Cache::flush();+ $this->server->forceFill(['sentinel_updated_at' => now()->subMinutes(15), 'sentinel_waiting_since' => null])->save();+ Process::fake([+ '*docker ps*' => Process::result(output: 'running Up 2 hours (healthy)'),+ '*api/version*' => Process::result(output: '1.0.0'),+ ]);+ StartSentinel::shouldRun()->once();++ (new CheckAndStartSentinelJob($this->server))->handle();+ (new CheckAndStartSentinelJob($this->server))->handle();+});++it('does not restart a Sentinel that started recently and has not pushed yet', function () {+ Cache::flush();+ $this->server->forceFill(['sentinel_updated_at' => now()->subDays(5), 'sentinel_waiting_since' => now()->subMinutes(3)])->save();+ Process::fake([+ '*docker ps*' => Process::result(output: 'running Up 3 minutes (healthy)'),+ '*api/version*' => Process::result(output: '1.0.0'),+ ]);+ StartSentinel::shouldNotRun();++ (new CheckAndStartSentinelJob($this->server))->handle();+});++it('remembers the push error that Sentinel reports', function () {+ $this->server->forceFill(['sentinel_updated_at' => now()])->save();+ $pushStatus = json_encode([+ 'last_attempt_at' => '2026-10-04T12:00:00Z',+ 'last_success_at' => null,+ 'last_error' => 'http: error sending request: Connection refused',+ 'last_status' => null,+ 'consecutive_failures' => 4,+ ]);+ Process::fake([+ '*docker ps*' => Process::result(output: 'running Up 2 hours (healthy)'),+ '*api/version*' => Process::result(output: "1.0.0\n{$pushStatus}"),+ ]);+ StartSentinel::shouldNotRun();++ (new CheckAndStartSentinelJob($this->server))->handle();++ expect($this->server->sentinelPushProblem())->toBe('Sentinel cannot push to Coolify: http: error sending request: Connection refused');+});++it('reads the push status with the token from inside the Sentinel container on a non-root server', function () {+ $this->server->forceFill(['user' => 'ubuntu', 'sentinel_updated_at' => now()])->save();+ Process::fake([+ '*docker ps*' => Process::result(output: 'running Up 2 hours (healthy)'),+ '*api/version*' => Process::result(output: '1.0.0'),+ ]);++ (new CheckAndStartSentinelJob($this->server->refresh()))->handle();++ Process::assertRan(fn ($process) => str_contains($process->command, "sudo bash -c 'docker exec coolify-sentinel sh -c '\\''curl")+ && str_contains($process->command, 'Bearer $TOKEN'));+});++it('does not restart a starting Sentinel whose API does not answer yet', function () {+ $this->server->forceFill(['sentinel_updated_at' => now()->subDays(5), 'sentinel_waiting_since' => now()])->save();+ Process::fake([+ '*docker ps*' => Process::result(output: 'running Up 3 seconds (health: starting)'),+ '*api/version*' => Process::result(output: ''),+ ]);+ StartSentinel::shouldNotRun();++ (new CheckAndStartSentinelJob($this->server))->handle();+});