Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

perf(metrics): fetch all Sentinel metric types in one SSH command

Andras Bacsai9d82a76fc44d
diff --git a/app/Livewire/Dashboard/ServerMetricsChart.php b/app/Livewire/Dashboard/ServerMetricsChart.php
index 83748052ce..f2ba0df17c 100644
--- a/app/Livewire/Dashboard/ServerMetricsChart.php
+++ b/app/Livewire/Dashboard/ServerMetricsChart.php
@@ -22,6 +22,7 @@ class ServerMetricsChart extends Component
}
try {
+ $this->server->prefetchMetrics(['cpu', 'memory'], 10);
$this->dispatch("dashboard-server-metrics-{$this->server->uuid}", [
'cpu' => $this->server->getCpuMetrics(10),
'memory' => $this->server->getMemoryMetrics(10),
diff --git a/app/Livewire/Project/Shared/Metrics.php b/app/Livewire/Project/Shared/Metrics.php
index e5b87b48cd..f4138fdd41 100644
--- a/app/Livewire/Project/Shared/Metrics.php
+++ b/app/Livewire/Project/Shared/Metrics.php
@@ -31,6 +31,7 @@ class Metrics extends Component
public function loadData()
{
try {
+ $this->resource->prefetchMetrics(['cpu', 'memory'], $this->interval);
$cpuMetrics = $this->resource->getCpuMetrics($this->interval);
$memoryMetrics = $this->resource->getMemoryMetrics($this->interval);
$this->dispatch("refreshChartData-{$this->chartId}-cpu", [
diff --git a/app/Livewire/Server/Charts.php b/app/Livewire/Server/Charts.php
index 6ff1f682fa..c63cf47ed3 100644
--- a/app/Livewire/Server/Charts.php
+++ b/app/Livewire/Server/Charts.php
@@ -102,6 +102,7 @@ class Charts extends Component
public function loadData()
{
try {
+ $this->server->prefetchMetrics(['cpu', 'memory', 'disk', 'load', 'network'], $this->interval);
$cpuMetrics = $this->server->getCpuMetrics($this->interval);
$memoryMetrics = $this->server->getMemoryMetrics($this->interval);
// Disk/network/load arrived in a later Sentinel; fetch each optionally so an
diff --git a/app/Services/ServerCanvas.php b/app/Services/ServerCanvas.php
index 5539c481c3..9eaf12c39a 100644
--- a/app/Services/ServerCanvas.php
+++ b/app/Services/ServerCanvas.php
@@ -145,6 +145,8 @@ class ServerCanvas
}
};
+ $server->prefetchMetrics(['cpu', 'memory', 'disk'], 1);
+
return [
'cpu' => $latest(fn () => $server->getCpuMetrics(1)),
'memory' => $latest(fn () => $server->getMemoryMetrics(1)),
diff --git a/app/Traits/HasMetrics.php b/app/Traits/HasMetrics.php
index cdf5a9c2e9..cc096fd1e3 100644
--- a/app/Traits/HasMetrics.php
+++ b/app/Traits/HasMetrics.php
@@ -8,6 +8,57 @@ use Illuminate\Support\Facades\Log;
trait HasMetrics
{
+ /**
+ * Raw Sentinel responses from prefetchMetrics(), keyed by "type:mins"; each is read once.
+ *
+ * @var array<string, string>
+ */
+ protected array $prefetchedMetricResponses = [];
+
+ /**
+ * Fetch several metric types in one SSH round-trip instead of one per type. The
+ * get*Metrics() getters then read these responses. On a failure nothing is stored,
+ * so each getter fetches on its own as before.
+ *
+ * @param array<int, string> $types
+ */
+ public function prefetchMetrics(array $types, int $mins): void
+ {
+ $server = $this->getMetricsServer();
+ if (! $server->isMetricsEnabled()) {
+ return;
+ }
+
+ $from = now()->subMinutes($mins)->toIso8601ZuluString();
+ $token = $this->metricsToken($server);
+ // The token reaches curl on stdin (`-H @-`), so it is not a process argument. The token
+ // only has letters, digits and `._-+/=` (ServerSetting::isValidSentinelToken), so it
+ // cannot end the heredoc. Responses are separated by a 0x1E record separator.
+ $script = implode("\nprintf '\\036'\n", array_map(
+ fn (string $type) => 'curl -s -H @- "'.$this->getMetricsEndpoint($type, $from)."\" <<'COOLIFY_SENTINEL_AUTH'\nAuthorization: Bearer {$token}\nCOOLIFY_SENTINEL_AUTH",
+ $types
+ ));
+
+ try {
+ $output = instant_remote_process(
+ ["docker exec -i coolify-sentinel sh -s <<'COOLIFY_SENTINEL_SCRIPT'\n{$script}\nCOOLIFY_SENTINEL_SCRIPT"],
+ $server,
+ maxAttempts: 1
+ );
+ } catch (\Throwable) {
+ return;
+ }
+
+ $responses = explode("\x1e", (string) $output);
+ if (count($responses) !== count($types)) {
+ return;
+ }
+
+ foreach ($types as $index => $type) {
+ $this->prefetchedMetricResponses["{$type}:{$mins}"] = trim($responses[$index]);
+ }
+ }
+
public function getCpuMetrics(int $mins = 5): ?array
{
return $this->getMetrics('cpu', $mins, 'percent');
@@ -90,26 +141,22 @@ trait HasMetrics
return null;
}
- $from = now()->subMinutes($mins)->toIso8601ZuluString();
- $endpoint = $this->getMetricsEndpoint($type, $from);
-
- $previousToken = null;
- try {
- $previousToken = $server->settings->sentinel_token;
- } catch (DecryptException) {
- // fall through to ensureValidSentinelToken which will regenerate
- }
- $token = $server->settings->ensureValidSentinelToken();
- if ($token !== $previousToken) {
- Log::warning('Regenerated sentinel token during metrics read; sentinel container restart required', ['server_id' => $server->id]);
+ $key = "{$type}:{$mins}";
+ if (array_key_exists($key, $this->prefetchedMetricResponses)) {
+ $response = $this->prefetchedMetricResponses[$key];
+ unset($this->prefetchedMetricResponses[$key]);
+ } else {
+ $from = now()->subMinutes($mins)->toIso8601ZuluString();
+ $endpoint = $this->getMetricsEndpoint($type, $from);
+ $token = $this->metricsToken($server);
+
+ $response = instant_remote_process(
+ ["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$token}\" {$endpoint}'"],
+ $server,
+ false
+ );
}
- $response = instant_remote_process(
- ["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$token}\" {$endpoint}'"],
- $server,
- false
- );
-
if (str($response)->contains('error')) {
$error = json_decode($response, true);
$error = data_get($error, 'error', 'Something is not okay, are you okay?');
@@ -144,6 +191,22 @@ trait HasMetrics
return $metrics;
}
+ private function metricsToken(Server $server): string
+ {
+ $previousToken = null;
+ try {
+ $previousToken = $server->settings->sentinel_token;
+ } catch (DecryptException) {
+ // fall through to ensureValidSentinelToken which will regenerate
+ }
+ $token = $server->settings->ensureValidSentinelToken();
+ if ($token !== $previousToken) {
+ Log::warning('Regenerated sentinel token during metrics read; sentinel container restart required', ['server_id' => $server->id]);
+ }
+
+ return $token;
+ }
+
private function isServerMetrics(): bool
{
return $this instanceof Server;
diff --git a/tests/Feature/ServerCanvasTest.php b/tests/Feature/ServerCanvasTest.php
index 58de97837d..e98e8a0dcb 100644
--- a/tests/Feature/ServerCanvasTest.php
+++ b/tests/Feature/ServerCanvasTest.php
@@ -386,6 +386,7 @@ test('server zones show host facts, the proxy and a usage endpoint only when met
test('the usage endpoint returns the latest Sentinel values and never another team server', function () {
$server = Mockery::mock($this->web)->makePartial();
+ $server->shouldReceive('prefetchMetrics')->once()->with(['cpu', 'memory', 'disk'], 1);
$server->shouldReceive('getCpuMetrics')->andReturn([[1, 10.0], [2, 12.34]]);
$server->shouldReceive('getMemoryMetrics')->andReturn([[2, 40.0]]);
$server->shouldReceive('getDiskMetrics')->andThrow(new RuntimeException('no disk data'));
diff --git a/tests/Feature/ServerMetricsPrefetchTest.php b/tests/Feature/ServerMetricsPrefetchTest.php
new file mode 100644
index 0000000000..ca1d6eeab8
--- /dev/null
+++ b/tests/Feature/ServerMetricsPrefetchTest.php
@@ -0,0 +1,87 @@
+<?php
+
+use App\Models\InstanceSettings;
+use App\Models\PrivateKey;
+use App\Models\Server;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Process\PendingProcess;
+use Illuminate\Support\Facades\Process;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+ InstanceSettings::create(['id' => 0]);
+ config(['constants.ssh.mux_enabled' => false]);
+
+ $team = Team::factory()->create();
+ $key = PrivateKey::factory()->create(['team_id' => $team->id]);
+ $this->server = Server::factory()->create(['team_id' => $team->id, 'private_key_id' => $key->id]);
+ $this->server->settings->forceFill(['is_metrics_enabled' => true])->saveQuietly();
+ $this->server->refresh();
+});
+
+it('fetches several metric types in one SSH command', function () {
+ $commands = [];
+ Process::fake(function (PendingProcess $process) use (&$commands) {
+ $commands[] = $process->command;
+
+ return Process::result(implode("\x1e", [
+ '[{"time":1000,"percent":12.5}]',
+ '[{"time":1000,"usedPercent":40}]',
+ '[{"time":1000,"mount":"/","usedPercent":70}]',
+ ]));
+ });
+
+ $this->server->prefetchMetrics(['cpu', 'memory', 'disk'], 5);
+
+ expect($this->server->getCpuMetrics(5))->toBe([[1000, 12.5]])
+ ->and($this->server->getMemoryMetrics(5))->toBe([[1000, 40.0]])
+ ->and($this->server->getDiskMetrics(5))->toBe([[1000, 70.0]])
+ ->and($commands)->toHaveCount(1)
+ ->and($commands[0])->toContain('/api/cpu/history', '/api/memory/history', '/api/disk/history', '-H @-')
+ ->and($commands[0])->not->toContain('Bearer '.$this->server->settings->sentinel_token.'"');
+});
+
+it('keeps the Sentinel error of one prefetched metric', function () {
+ Process::fake(fn () => Process::result("[{\"time\":1000,\"percent\":1}]\x1e{\"error\":\"Unauthorized\"}"));
+
+ $this->server->prefetchMetrics(['cpu', 'memory'], 5);
+
+ expect($this->server->getCpuMetrics(5))->toBe([[1000, 1.0]])
+ ->and(fn () => $this->server->getMemoryMetrics(5))->toThrow(Exception::class, 'Unauthorized');
+});
+
+it('falls back to one fetch per metric when the batch fails', function () {
+ $commands = [];
+ Process::fake(function (PendingProcess $process) use (&$commands) {
+ $commands[] = $process->command;
+
+ return count($commands) === 1
+ ? Process::result('', 'ssh: connect to host failed', 1)
+ : Process::result('[{"time":2000,"percent":5}]');
+ });
+
+ $this->server->prefetchMetrics(['cpu', 'memory'], 5);
+
+ expect($this->server->getCpuMetrics(5))->toBe([[2000, 5.0]])
+ ->and($commands)->toHaveCount(2);
+});
+
+it('runs the batched metrics script with sudo for a non-root user', function () {
+ $commands = [];
+ Process::fake(function (PendingProcess $process) use (&$commands) {
+ $commands[] = $process->command;
+
+ return Process::result("[]\x1e[]");
+ });
+ $this->server->forceFill(['user' => 'deploy'])->saveQuietly();
+
+ $this->server->prefetchMetrics(['cpu', 'memory'], 5);
+
+ expect($commands)->toHaveCount(1)
+ ->and($commands[0])->toContain('sudo docker exec -i coolify-sentinel sh -s')
+ ->and($commands[0])->toContain("\ncurl -s -H @- ")
+ ->and($commands[0])->not->toContain('sudo curl')
+ ->and($commands[0])->not->toContain('sudo printf');
+});