Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(sentinel): repair stalled Sentinel and show why pushes fail

Andras Bacsai66cc64e0d914
diff --git a/app/Http/Controllers/Api/SentinelController.php b/app/Http/Controllers/Api/SentinelController.php
index 23bcf4f13c..109b0db810 100644
--- a/app/Http/Controllers/Api/SentinelController.php
+++ b/app/Http/Controllers/Api/SentinelController.php
@@ -60,6 +60,7 @@ class SentinelController extends Controller
'server_uuid' => $server->uuid,
'team_id' => $server->team_id,
]);
+ $server->rememberSentinelPushProblem('Coolify rejected the push: the subscription of this team is not active.');
return response()->json(['message' => 'Unauthorized'], 401);
}
@@ -69,6 +70,7 @@ class SentinelController extends Controller
'server_uuid' => $server->uuid,
'team_id' => $server->team_id,
]);
+ $server->rememberSentinelPushProblem('Coolify rejected the push: the server is marked unreachable, not usable, or disabled. Validate the server connection.');
return response()->json(['message' => 'Server is not functional'], 401);
}
@@ -78,6 +80,7 @@ class SentinelController extends Controller
'server_uuid' => $server->uuid,
'team_id' => $server->team_id,
]);
+ $server->rememberSentinelPushProblem('Coolify rejected the push: the Sentinel token does not match. Sync Sentinel to apply the current token.');
return response()->json(['message' => 'Unauthorized'], 401);
}
@@ -86,6 +89,8 @@ class SentinelController extends Controller
]);
if ($validator->fails()) {
+ $server->rememberSentinelPushProblem('Coolify rejected the push: the payload is not valid.');
+
return response()->json(serializeApiResponse([
'message' => 'Validation failed.',
'errors' => $validator->errors(),
@@ -101,6 +106,7 @@ class SentinelController extends Controller
$server->sentinelHeartbeat();
if (! $wasSentinelLive) {
+ $server->rememberSentinelPushProblem(null);
SentinelSynchronized::dispatch($server);
}
diff --git a/app/Jobs/CheckAndStartSentinelJob.php b/app/Jobs/CheckAndStartSentinelJob.php
index 49e08ab1a6..183063f09e 100644
--- a/app/Jobs/CheckAndStartSentinelJob.php
+++ b/app/Jobs/CheckAndStartSentinelJob.php
@@ -11,11 +11,22 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Cache;
class CheckAndStartSentinelJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+ /**
+ * A running Sentinel that has not pushed for this long is restarted with the current settings.
+ * This repairs a changed Coolify URL or token, and a hung push loop.
+ */
+ public const STALL_RESTART_AFTER_SECONDS = 600;
+
+ /** A stalled Sentinel is restarted at most once in this period, because a firewall problem stays after a restart. */
+ public const STALL_RESTART_INTERVAL_SECONDS = 3600;
+
public $timeout = 120;
/**
@@ -40,28 +51,63 @@ class CheckAndStartSentinelJob implements ShouldBeEncrypted, ShouldBeUnique, Sho
// An SSH failure throws here, so it is never mistaken for a missing container.
// A missing container gives empty output and exit code 0.
- $sentinelStatus = instant_remote_process_with_timeout(["docker ps -a --filter 'name=^coolify-sentinel$' --format '{{.State}}'"], $this->server);
- if ($sentinelStatus !== 'running') {
+ $sentinelStatus = instant_remote_process_with_timeout(["docker ps -a --filter 'name=^coolify-sentinel$' --format '{{.State}} {{.Status}}'"], $this->server);
+ if (! str_starts_with((string) $sentinelStatus, 'running')) {
$this->startSentinel($latestVersion);
return;
}
- // If sentinel is running, check if it needs an update
- $runningVersion = instant_remote_process_with_timeout(['docker exec coolify-sentinel sh -c "curl http://127.0.0.1:8888/api/version"'], $this->server, false);
- if (empty($runningVersion)) {
- $runningVersion = '0.0.0';
+ if (str_contains($sentinelStatus, '(unhealthy)')) {
+ $this->server->rememberSentinelPushProblem('The Sentinel container was unhealthy. Coolify restarted it.');
+ $this->startSentinel($latestVersion);
+
+ return;
}
- if ($latestVersion === '0.0.0' && $runningVersion === '0.0.0') {
- $this->startSentinel('latest');
+
+ // One SSH call reads the version and the last push result. Sentinel versions without
+ // /api/push-status give an empty second line.
+ $output = instant_remote_process_with_timeout([
+ "docker exec coolify-sentinel sh -c 'curl -s http://127.0.0.1:8888/api/version; echo; curl -sf -H \"Authorization: Bearer \$TOKEN\" http://127.0.0.1:8888/api/push-status'",
+ ], $this->server, false);
+ [$runningVersion, $pushStatus] = array_pad(explode("\n", trim((string) $output), 2), 2, null);
+ $this->rememberPushError($pushStatus);
+
+ // An empty version means that the API does not answer yet, for example while Sentinel starts.
+ // A Sentinel whose API stays down fails its health check and is restarted above.
+ if (filled($runningVersion) && version_compare(trim($runningVersion), $latestVersion, '<')) {
+ $this->startSentinel($latestVersion);
return;
- } else {
- if (version_compare($runningVersion, $latestVersion, '<')) {
- $this->startSentinel($latestVersion);
+ }
+
+ if ($this->isStalled() && Cache::add("sentinel:stall-restart:{$this->server->id}", true, self::STALL_RESTART_INTERVAL_SECONDS)) {
+ $this->startSentinel($latestVersion);
+ }
+ }
+
+ /**
+ * Sentinel is stalled when it has not pushed for STALL_RESTART_AFTER_SECONDS since its last push,
+ * or since its last start when it never pushed.
+ */
+ private function isStalled(): bool
+ {
+ $lastSignOfLife = $this->server->sentinel_waiting_since ?? $this->server->sentinel_updated_at;
+ if ($lastSignOfLife === null) {
+ return true;
+ }
+ $stallAfter = max(self::STALL_RESTART_AFTER_SECONDS, $this->server->waitBeforeDoingSshCheck());
+
+ return Carbon::parse($lastSignOfLife)->isBefore(now()->subSeconds($stallAfter));
+ }
- return;
- }
+ private function rememberPushError(?string $pushStatus): void
+ {
+ $status = json_decode(trim((string) $pushStatus), true);
+ if (! is_array($status) || data_get($status, 'consecutive_failures', 0) < 1 || blank(data_get($status, 'last_error'))) {
+ return;
}
+
+ $this->server->rememberSentinelPushProblem('Sentinel cannot push to Coolify: '.data_get($status, 'last_error'));
}
private function sentinelIsEnabled(): bool
diff --git a/app/Models/InstanceSettings.php b/app/Models/InstanceSettings.php
index 9c9bc3b4ea..16a1d7ea30 100644
--- a/app/Models/InstanceSettings.php
+++ b/app/Models/InstanceSettings.php
@@ -111,6 +111,13 @@ class InstanceSettings extends Model
static::updated(function ($settings) {
// Clear once() cache so subsequent calls get fresh data
Once::flush();
+
+ if ($settings->wasChanged(['fqdn', 'public_ipv4', 'public_ipv6'])) {
+ ServerSetting::followInstanceUrlChange(
+ ServerSetting::instanceSentinelUrl($settings->getOriginal('fqdn'), $settings->getOriginal('public_ipv4'), $settings->getOriginal('public_ipv6')),
+ ServerSetting::instanceSentinelUrl($settings->fqdn, $settings->public_ipv4, $settings->public_ipv6),
+ );
+ }
});
}
diff --git a/app/Models/Server.php b/app/Models/Server.php
index cf32ce702a..88f3ecc15a 100644
--- a/app/Models/Server.php
+++ b/app/Models/Server.php
@@ -33,9 +33,11 @@ use Illuminate\Database\Eloquent\Relations\MorphMany;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Carbon;
use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
+use Illuminate\Support\Str;
use Illuminate\Support\Stringable;
use OpenApi\Attributes as OA;
use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
@@ -1082,6 +1084,25 @@ $siteAddress {
return "sentinel:reported-version:{$serverId}";
}
+ /**
+ * The last known reason why Sentinel pushes do not arrive. It is shown while Sentinel is out of sync.
+ */
+ public function sentinelPushProblem(): ?string
+ {
+ return Cache::get("sentinel:push-problem:{$this->id}");
+ }
+
+ public function rememberSentinelPushProblem(?string $problem): void
+ {
+ if (blank($problem)) {
+ Cache::forget("sentinel:push-problem:{$this->id}");
+
+ return;
+ }
+
+ Cache::put("sentinel:push-problem:{$this->id}", Str::limit($problem, 500), now()->addDay());
+ }
+
public function isSentinelLive()
{
return Carbon::parse($this->sentinel_updated_at)->isAfter(now()->subSeconds($this->waitBeforeDoingSshCheck()));
diff --git a/app/Models/ServerSetting.php b/app/Models/ServerSetting.php
index 1fb3af6d6c..a4d51afdae 100644
--- a/app/Models/ServerSetting.php
+++ b/app/Models/ServerSetting.php
@@ -315,6 +315,38 @@ class ServerSetting extends Model
return $url;
}
+ /**
+ * The Sentinel push URL that the instance URL or public IP gives to a remote server.
+ */
+ public static function instanceSentinelUrl(?string $fqdn, ?string $publicIpv4, ?string $publicIpv6): ?string
+ {
+ return match (true) {
+ filled($fqdn) => $fqdn,
+ filled($publicIpv4) => 'http://'.$publicIpv4.':8000',
+ filled($publicIpv6) => 'http://'.$publicIpv6.':8000',
+ default => null,
+ };
+ }
+
+ /**
+ * Move remote servers that use the generated Sentinel URL to the new instance URL. Saving the
+ * new URL restarts their Sentinel. Servers with a custom URL keep it.
+ */
+ public static function followInstanceUrlChange(?string $oldUrl, ?string $newUrl): void
+ {
+ if (blank($oldUrl) || blank($newUrl) || $oldUrl === $newUrl) {
+ return;
+ }
+
+ self::query()
+ ->where('sentinel_custom_url', $oldUrl)
+ ->where('server_id', '!=', 0)
+ ->each(function (ServerSetting $setting) use ($newUrl) {
+ $setting->sentinel_custom_url = $newUrl;
+ $setting->save();
+ });
+ }
+
public function restoreDefaultSentinelConfiguration(): void
{
$this->generateSentinelUrl(save: false, ignoreEvent: true);
@@ -327,18 +359,12 @@ class ServerSetting extends Model
public function generateSentinelUrl(bool $save = true, bool $ignoreEvent = false): ?string
{
- $domain = null;
$settings = InstanceSettings::get();
if ($this->server->isLocalhost()) {
$domain = 'http://coolify:8080';
- } elseif ($settings->fqdn) {
- $domain = $settings->fqdn;
- } elseif ($settings->public_ipv4) {
- $domain = 'http://'.$settings->public_ipv4.':8000';
- } elseif ($settings->public_ipv6) {
- $domain = 'http://'.$settings->public_ipv6.':8000';
} else {
- $domain = $this->sentinelUrlFromCurrentRequest();
+ $domain = self::instanceSentinelUrl($settings->fqdn, $settings->public_ipv4, $settings->public_ipv6)
+ ?? $this->sentinelUrlFromCurrentRequest();
}
$this->sentinel_custom_url = $domain;
if ($save) {
diff --git a/resources/views/livewire/server/sentinel.blade.php b/resources/views/livewire/server/sentinel.blade.php
index e3a72e8686..02b32b155e 100644
--- a/resources/views/livewire/server/sentinel.blade.php
+++ b/resources/views/livewire/server/sentinel.blade.php
@@ -38,6 +38,9 @@
<x-callout type="warning" title="Sentinel is out of sync">
<div class="space-y-3">
<p>Sentinel has not reported within the expected interval. Check these items before syncing again:</p>
+ @if ($sentinelPushProblem = $server->sentinelPushProblem())
+ <p><span class="font-medium">Last error:</span> <code class="break-all">{{ $sentinelPushProblem }}</code></p>
+ @endif
<ul class="list-disc space-y-1 pl-4">
<li>Confirm that the <code>coolify-sentinel</code> container is running.</li>
<li>
diff --git a/tests/Feature/SentinelPushProblemTest.php b/tests/Feature/SentinelPushProblemTest.php
new file mode 100644
index 0000000000..18c6abfaf1
--- /dev/null
+++ b/tests/Feature/SentinelPushProblemTest.php
@@ -0,0 +1,76 @@
+<?php
+
+use App\Actions\Server\StartSentinel;
+use App\Models\InstanceSettings;
+use App\Models\Server;
+use App\Models\User;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Queue;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+ config(['app.maintenance.store' => 'array']);
+ Server::flushIdentityMap();
+ Queue::fake();
+ Cache::flush();
+ InstanceSettings::forceCreate(['id' => 0, 'fqdn' => 'https://old.example.com']);
+
+ $user = User::factory()->create();
+ $this->server = Server::factory()->create(['team_id' => $user->teams()->first()->id]);
+ $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]);
+});
+
+function pushSentinelForProblemTest(Server $server, string $token)
+{
+ return test()->postJson('/api/v1/sentinel/push', ['containers' => []], ['Authorization' => 'Bearer '.$token]);
+}
+
+it('remembers when Coolify rejects a push from a server that is not functional', function () {
+ $this->server->settings->update(['is_reachable' => false]);
+
+ pushSentinelForProblemTest($this->server, $this->server->settings->sentinel_token)->assertUnauthorized();
+
+ expect($this->server->sentinelPushProblem())->toContain('marked unreachable');
+});
+
+it('remembers when Coolify rejects a push with an old token', function () {
+ $oldToken = $this->server->settings->sentinel_token;
+ $this->server->settings->generateSentinelToken(ignoreEvent: true);
+
+ pushSentinelForProblemTest($this->server, $oldToken)->assertUnauthorized();
+
+ expect($this->server->sentinelPushProblem())->toContain('token does not match');
+});
+
+it('forgets the problem after the next accepted push', function () {
+ $this->server->rememberSentinelPushProblem('Coolify rejected the push: the Sentinel token does not match.');
+ $this->server->forceFill(['sentinel_updated_at' => now()->subHour()])->save();
+
+ pushSentinelForProblemTest($this->server, $this->server->settings->sentinel_token)->assertOk();
+
+ expect($this->server->sentinelPushProblem())->toBeNull();
+});
+
+it('moves Sentinel to the new instance URL when the instance URL changes', function () {
+ $this->server->settings->update(['sentinel_custom_url' => 'https://old.example.com', 'is_sentinel_enabled' => true]);
+ Queue::fake();
+
+ $settings = InstanceSettings::get();
+ $settings->fqdn = 'https://new.example.com';
+ $settings->save();
+
+ expect($this->server->settings->refresh()->sentinel_custom_url)->toBe('https://new.example.com');
+ StartSentinel::assertPushed(fn (StartSentinel $action, array $arguments) => $arguments[0]->is($this->server));
+});
+
+it('keeps a custom Sentinel URL when the instance URL changes', function () {
+ $this->server->settings->update(['sentinel_custom_url' => 'https://custom.example.com']);
+
+ $settings = InstanceSettings::get();
+ $settings->fqdn = 'https://new.example.com';
+ $settings->save();
+
+ expect($this->server->settings->refresh()->sentinel_custom_url)->toBe('https://custom.example.com');
+});
diff --git a/tests/Feature/Server/CheckAndStartSentinelJobTest.php b/tests/Feature/Server/CheckAndStartSentinelJobTest.php
index b2a33f809d..cb541b3519 100644
--- a/tests/Feature/Server/CheckAndStartSentinelJobTest.php
+++ b/tests/Feature/Server/CheckAndStartSentinelJobTest.php
@@ -7,6 +7,7 @@ use App\Models\PrivateKey;
use App\Models\Server;
use App\Models\User;
use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\Storage;
@@ -19,6 +20,7 @@ beforeEach(function () {
Storage::fake('ssh-keys');
Storage::fake('ssh-mux');
config()->set('constants.ssh.max_retries', 1);
+ config()->set('constants.ssh.mux_enabled', false);
Http::fake(['*' => Http::response(['coolify' => ['sentinel' => ['version' => '1.0.0']]])]);
$user = User::factory()->create();
@@ -61,6 +63,7 @@ it('does not treat an SSH failure as a missing container', function () {
});
it('does not restart a running Sentinel that is up to date', function () {
+ $this->server->forceFill(['sentinel_updated_at' => now()])->save();
Process::fake([
'*docker ps*' => Process::result(output: 'running'),
'*api/version*' => Process::result(output: '1.0.0'),
@@ -71,7 +74,7 @@ it('does not restart a running Sentinel that is up to date', function () {
});
it('uses sudo for the status check on a non-root server', function () {
- $this->server->update(['user' => 'ubuntu']);
+ $this->server->forceFill(['user' => 'ubuntu', 'sentinel_updated_at' => now()])->save();
Process::fake([
'*docker ps*' => Process::result(output: 'running'),
'*api/version*' => Process::result(output: '1.0.0'),
@@ -82,3 +85,93 @@ it('uses sudo for the status check on a non-root server', function () {
Process::assertRan(fn ($process) => str_contains($process->command, 'sudo docker ps -a'));
});
+
+it('restarts an unhealthy Sentinel and remembers why', function () {
+ $this->server->forceFill(['sentinel_updated_at' => now()])->save();
+ Process::fake(['*docker ps*' => Process::result(output: 'running Up 2 hours (unhealthy)')]);
+ StartSentinel::shouldRun()->once();
+
+ (new CheckAndStartSentinelJob($this->server))->handle();
+
+ expect($this->server->sentinelPushProblem())->toContain('unhealthy');
+});
+
+it('does not restart a Sentinel whose health check is still starting', function () {
+ $this->server->forceFill(['sentinel_updated_at' => now()])->save();
+ Process::fake([
+ '*docker ps*' => Process::result(output: 'running Up 30 seconds (health: starting)'),
+ '*api/version*' => Process::result(output: '1.0.0'),
+ ]);
+ StartSentinel::shouldNotRun();
+
+ (new CheckAndStartSentinelJob($this->server))->handle();
+});
+
+it('restarts a running Sentinel that stopped pushing, at most once per hour', function () {
+ Cache::flush();
+ $this->server->forceFill(['sentinel_updated_at' => now()->subMinutes(15), 'sentinel_waiting_since' => null])->save();
+ Process::fake([
+ '*docker ps*' => Process::result(output: 'running Up 2 hours (healthy)'),
+ '*api/version*' => Process::result(output: '1.0.0'),
+ ]);
+ StartSentinel::shouldRun()->once();
+
+ (new CheckAndStartSentinelJob($this->server))->handle();
+ (new CheckAndStartSentinelJob($this->server))->handle();
+});
+
+it('does not restart a Sentinel that started recently and has not pushed yet', function () {
+ Cache::flush();
+ $this->server->forceFill(['sentinel_updated_at' => now()->subDays(5), 'sentinel_waiting_since' => now()->subMinutes(3)])->save();
+ Process::fake([
+ '*docker ps*' => Process::result(output: 'running Up 3 minutes (healthy)'),
+ '*api/version*' => Process::result(output: '1.0.0'),
+ ]);
+ StartSentinel::shouldNotRun();
+
+ (new CheckAndStartSentinelJob($this->server))->handle();
+});
+
+it('remembers the push error that Sentinel reports', function () {
+ $this->server->forceFill(['sentinel_updated_at' => now()])->save();
+ $pushStatus = json_encode([
+ 'last_attempt_at' => '2026-10-04T12:00:00Z',
+ 'last_success_at' => null,
+ 'last_error' => 'http: error sending request: Connection refused',
+ 'last_status' => null,
+ 'consecutive_failures' => 4,
+ ]);
+ Process::fake([
+ '*docker ps*' => Process::result(output: 'running Up 2 hours (healthy)'),
+ '*api/version*' => Process::result(output: "1.0.0\n{$pushStatus}"),
+ ]);
+ StartSentinel::shouldNotRun();
+
+ (new CheckAndStartSentinelJob($this->server))->handle();
+
+ expect($this->server->sentinelPushProblem())->toBe('Sentinel cannot push to Coolify: http: error sending request: Connection refused');
+});
+
+it('reads the push status with the token from inside the Sentinel container on a non-root server', function () {
+ $this->server->forceFill(['user' => 'ubuntu', 'sentinel_updated_at' => now()])->save();
+ Process::fake([
+ '*docker ps*' => Process::result(output: 'running Up 2 hours (healthy)'),
+ '*api/version*' => Process::result(output: '1.0.0'),
+ ]);
+
+ (new CheckAndStartSentinelJob($this->server->refresh()))->handle();
+
+ Process::assertRan(fn ($process) => str_contains($process->command, "sudo bash -c 'docker exec coolify-sentinel sh -c '\\''curl")
+ && str_contains($process->command, 'Bearer $TOKEN'));
+});
+
+it('does not restart a starting Sentinel whose API does not answer yet', function () {
+ $this->server->forceFill(['sentinel_updated_at' => now()->subDays(5), 'sentinel_waiting_since' => now()])->save();
+ Process::fake([
+ '*docker ps*' => Process::result(output: 'running Up 3 seconds (health: starting)'),
+ '*api/version*' => Process::result(output: ''),
+ ]);
+ StartSentinel::shouldNotRun();
+
+ (new CheckAndStartSentinelJob($this->server))->handle();
+});