Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(metrics): load the Metrics page when a server is slow or unreachable

Andras Bacsai79cf62323a13
diff --git a/app/Livewire/Fleet/Metrics.php b/app/Livewire/Fleet/Metrics.php
index ccce6ae820..a8e11ee0fe 100644
--- a/app/Livewire/Fleet/Metrics.php
+++ b/app/Livewire/Fleet/Metrics.php
@@ -11,6 +11,7 @@ use Illuminate\Contracts\View\View;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Concurrency;
+use Illuminate\Support\Facades\Log;
use Livewire\Attributes\Lazy;
use Livewire\Attributes\Url;
use Livewire\Component;
@@ -228,6 +229,10 @@ class Metrics extends Component
$clients = [];
$summaryTasks = [];
foreach ($targets as $server) {
+ // Coolify does not connect to an unreachable server, and its SSH retries would hold a worker past the process timeout.
+ if (! $server->isFunctional()) {
+ continue;
+ }
$client = $clients[$server->uuid] = $this->metricsClient($server);
$summaryTasks[$server->uuid] = static fn () => $client->summary();
}
@@ -247,7 +252,7 @@ class Metrics extends Component
$results = $this->runMetricTasks($tasks);
foreach ($targets as $server) {
- $summary = $summaries[$server->uuid];
+ $summary = $summaries[$server->uuid] ?? null;
foreach (['cpu', 'memory', 'disk', 'load'] as $metric) {
$series[$metric][] = $results[$server->uuid.':'.$metric] ?? [];
}
@@ -303,7 +308,13 @@ class Metrics extends Component
}
$results = [];
foreach (array_chunk($tasks, 8, true) as $batch) {
- $results += Concurrency::run($batch);
+ try {
+ $results += Concurrency::run($batch);
+ } catch (\Throwable $e) {
+ // A worker that runs past the process timeout fails its batch. Show those metrics as missing, not a server error.
+ Log::warning('Fleet metrics batch failed', ['error' => $e->getMessage()]);
+ $results += array_fill_keys(array_keys($batch), null);
+ }
}
return $results;
diff --git a/app/Services/SentinelMetricsClient.php b/app/Services/SentinelMetricsClient.php
index 615adc5357..6980861e6e 100644
--- a/app/Services/SentinelMetricsClient.php
+++ b/app/Services/SentinelMetricsClient.php
@@ -229,7 +229,9 @@ class SentinelMetricsClient
["docker exec coolify-sentinel sh -c 'curl -s --max-time 10 -H \"Authorization: Bearer {$token}\" \"{$url}\"'"],
$this->server,
false,
- timeout: 15
+ timeout: 15,
+ // Metrics are read again on the next refresh. SSH retries would keep a worker past its timeout.
+ maxAttempts: 1
);
}
diff --git a/tests/Feature/Livewire/Fleet/MetricsChartPayloadTest.php b/tests/Feature/Livewire/Fleet/MetricsChartPayloadTest.php
index 130b4c5ebe..f46da0a532 100644
--- a/tests/Feature/Livewire/Fleet/MetricsChartPayloadTest.php
+++ b/tests/Feature/Livewire/Fleet/MetricsChartPayloadTest.php
@@ -79,7 +79,7 @@ beforeEach(function () {
it('dispatches a chart payload with every trend series', function () {
$s = Server::factory()->create(['team_id' => $this->team->id, 'private_key_id' => $this->privateKey->id]);
- $s->settings->update(['is_metrics_enabled' => true]);
+ $s->settings->forceFill(['is_metrics_enabled' => true, 'is_reachable' => true, 'is_usable' => true])->saveQuietly();
Livewire::test(ChartTestableMetrics::class)
->call('loadData')
@@ -96,7 +96,7 @@ it('dispatches a chart payload with every trend series', function () {
it('includes KPI-tile spark series aligned to one shared time axis', function () {
$s = Server::factory()->create(['team_id' => $this->team->id, 'private_key_id' => $this->privateKey->id]);
- $s->settings->update(['is_metrics_enabled' => true]);
+ $s->settings->forceFill(['is_metrics_enabled' => true, 'is_reachable' => true, 'is_usable' => true])->saveQuietly();
Livewire::test(ChartTestableMetrics::class)
->call('loadData')
@@ -115,7 +115,7 @@ it('includes KPI-tile spark series aligned to one shared time axis', function ()
it('caps KPI-tile sparks to a readable point count on a shared axis', function () {
$s = Server::factory()->create(['team_id' => $this->team->id, 'private_key_id' => $this->privateKey->id]);
- $s->settings->update(['is_metrics_enabled' => true]);
+ $s->settings->forceFill(['is_metrics_enabled' => true, 'is_reachable' => true, 'is_usable' => true])->saveQuietly();
Livewire::test(DenseTestableMetrics::class)
->call('loadData')
diff --git a/tests/Feature/Livewire/Fleet/MetricsDataTest.php b/tests/Feature/Livewire/Fleet/MetricsDataTest.php
index c304fa2610..79154709d8 100644
--- a/tests/Feature/Livewire/Fleet/MetricsDataTest.php
+++ b/tests/Feature/Livewire/Fleet/MetricsDataTest.php
@@ -93,10 +93,11 @@ beforeEach(function () {
$this->team = $team;
});
-function metricsServer($team, $privateKey): Server
+function metricsServer($team, $privateKey, bool $reachable = true): Server
{
$server = Server::factory()->create(['team_id' => $team->id, 'private_key_id' => $privateKey->id]);
$server->settings->update(['is_metrics_enabled' => true]);
+ $server->settings->forceFill(['is_reachable' => $reachable, 'is_usable' => $reachable])->saveQuietly();
return $server;
}
@@ -112,7 +113,7 @@ function onlineSummary(array $overrides = []): array
it('keeps an unreachable server in the table but excludes it from KPIs', function () {
$ok = metricsServer($this->team, $this->privateKey);
- $dead = metricsServer($this->team, $this->privateKey);
+ $dead = metricsServer($this->team, $this->privateKey, reachable: false);
StubMetricsClient::$summaries = [
$ok->uuid => onlineSummary(['cpu' => 30.0]),
@@ -134,7 +135,6 @@ it('keeps an unreachable server in the table but excludes it from KPIs', functio
it('counts a reachable server as online when Sentinel returns no summary', function () {
$server = metricsServer($this->team, $this->privateKey);
$server->update(['name' => 'web-1']);
- $server->settings->forceFill(['is_reachable' => true, 'is_usable' => true])->saveQuietly();
StubMetricsClient::$summaries = [$server->uuid => null];
$component = Livewire::test(TestableMetrics::class)
@@ -165,7 +165,7 @@ it('ranks containers across servers by the selected metric', function () {
it('shows the KPI tiles, a stale badge for offline servers, and the container toggle', function () {
$ok = metricsServer($this->team, $this->privateKey);
$ok->update(['name' => 'web-1']);
- $dead = metricsServer($this->team, $this->privateKey);
+ $dead = metricsServer($this->team, $this->privateKey, reachable: false);
$dead->update(['name' => 'web-2']);
StubMetricsClient::$summaries = [
@@ -299,6 +299,30 @@ it('permits team members to read metrics', function () {
expect(Livewire::test(TestableMetrics::class)->get('kpis')['serversOnline'])->toBe(1);
});
+it('does not query Sentinel on a server that Coolify cannot reach', function () {
+ $server = metricsServer($this->team, $this->privateKey, reachable: false);
+ StubMetricsClient::$summaries = [$server->uuid => onlineSummary()];
+ StubMetricsClient::$histories = [$server->uuid => ['cpu' => [[1000, 25.0]]]];
+
+ $component = Livewire::test(TestableMetrics::class);
+
+ $row = collect($component->get('serverRows'))->firstWhere('uuid', $server->uuid);
+ expect($row['online'])->toBeFalse();
+ expect($row['reporting'])->toBeFalse();
+ expect($component->get('chartData')['cpu'])->toBe([]);
+});
+
+it('shows the page when a metrics worker runs past the process timeout', function () {
+ $server = metricsServer($this->team, $this->privateKey);
+ Concurrency::shouldReceive('run')->andThrow(new RuntimeException('The process exceeded the timeout of 60 seconds.'));
+
+ $component = Livewire::test(TestableMetrics::class)->assertOk();
+
+ $row = collect($component->get('serverRows'))->firstWhere('uuid', $server->uuid);
+ expect($row['reporting'])->toBeFalse();
+ expect($component->get('kpis')['serversReporting'])->toBe(0);
+});
+
it('keeps host trends when an older Sentinel has no summary endpoint', function () {
$server = metricsServer($this->team, $this->privateKey);
StubMetricsClient::$histories = [$server->uuid => ['cpu' => [[1000, 25.0]], 'memory' => [[1000, 100.0]]]];