Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(analytics): fetch Sentinel dashboards of all servers concurrently

Andras Bacsai69bc72a3b514
diff --git a/app/Livewire/Analytics.php b/app/Livewire/Analytics.php
index 62c3aab048..e4cc20741b 100644
--- a/app/Livewire/Analytics.php
+++ b/app/Livewire/Analytics.php
@@ -348,9 +348,14 @@ class Analytics extends Component
$appRows = [];
$hostTotals = [];
+ $clients = $servers->mapWithKeys(fn (Server $server) => [$server->uuid => $this->trafficClient($server)]);
+ if ($resource === null) {
+ SentinelTrafficClient::prefetchServerWideConcurrently($clients, $from, $to, $this->range, appsLimit: self::MAX_LEADERBOARD_APPS);
+ }
+
foreach ($servers as $server) {
try {
- $client = $this->trafficClient($server);
+ $client = $clients[$server->uuid];
if ($resource !== null) {
// One server in Sentinel's resource scope is exact; a merge across servers
diff --git a/app/Services/SentinelTrafficClient.php b/app/Services/SentinelTrafficClient.php
index 1c5c28e2d8..54cb4758c6 100644
--- a/app/Services/SentinelTrafficClient.php
+++ b/app/Services/SentinelTrafficClient.php
@@ -8,6 +8,7 @@ use App\Data\Traffic\TrafficPathData;
use App\Data\Traffic\TrafficSeriesBucketData;
use App\Helpers\SshMultiplexingHelper;
use App\Models\Server;
+use Illuminate\Contracts\Process\InvokedProcess;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Log;
@@ -354,6 +355,56 @@ class SentinelTrafficClient
return $this->safeReportedKeys(array_map(fn ($app) => is_array($app) ? ($app['uuid'] ?? null) : null, $bundle['apps'] ?? []));
}
+ /**
+ * Fetch the server-wide dashboard bundle of several servers at the same time, so a page
+ * waits for the slowest server instead of for every server in turn. Each response warms
+ * the cache that prefetchServerWide() reads; a failed fetch is remembered like in
+ * fetchDashboard(), so prefetchServerWide() does not fetch it again.
+ *
+ * @param iterable<int, self> $clients
+ */
+ public static function prefetchServerWideConcurrently(iterable $clients, string $from, string $to, string $range, int $pathLimit = 50, int $breakdownLimit = 50, int $appsLimit = 200): void
+ {
+ $started = [];
+ foreach ($clients as $client) {
+ $url = $client->dashboardUrl(null, $from, $to, $range, $pathLimit, $breakdownLimit, $appsLimit);
+ if (! $client->fetchesThroughStartRemoteFetch()
+ || Cache::has($client->cacheKey($url))
+ || Cache::get($client->dashboardAbsenceKey()) === true) {
+ continue;
+ }
+
+ try {
+ $started[] = [$client, $url, $client->startRemoteFetch($url)];
+ } catch (\Throwable) {
+ // An unavailable server fails again at once in prefetchServerWide().
+ }
+ }
+
+ foreach ($started as [$client, $url, $process]) {
+ try {
+ Cache::put($client->cacheKey($url), $client->guard($client->awaitRemote($process)), 60);
+ } catch (\Throwable) {
+ Cache::put($client->dashboardAbsenceKey(), true, 60);
+ }
+ }
+ }
+
+ /**
+ * False for a fake that serves canned bodies from raw() or remoteFetch(): the concurrent
+ * prefetch would otherwise reach real SSH.
+ */
+ private function fetchesThroughStartRemoteFetch(): bool
+ {
+ foreach (['raw', 'remoteFetch'] as $method) {
+ if ((new \ReflectionMethod($this, $method))->getDeclaringClass()->getName() !== self::class) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
/**
* The usable keys of a Sentinel-reported key list. A key that would be unsafe in a request
* url is dropped (and logged) on its own, so it doesn't fail the whole server's analytics.
@@ -685,13 +736,21 @@ class SentinelTrafficClient
}
protected function remoteFetch(string $url): string
+ {
+ // Throw the real SSH/docker error (for example, an unreadable SSH key or a missing
+ // container). A literal "null" body comes back empty; guard() rejects the empty string.
+ return $this->awaitRemote($this->startRemoteFetch($url));
+ }
+
+ /**
+ * Start the fetch of one URL without waiting for it, so several servers can be fetched at the same time.
+ */
+ protected function startRemoteFetch(string $url): InvokedProcess
{
$this->ensureRemoteAvailable();
$token = $this->server->settings->ensureValidSentinelToken();
- // Throw the real SSH/docker error (for example, an unreadable SSH key or a missing
- // container). A literal "null" body comes back empty; guard() rejects the empty string.
- return $this->runRemote($this->buildFetchCommand($token, $url));
+ return $this->startRemote($this->buildFetchCommand($token, $url));
}
/**
@@ -702,7 +761,7 @@ class SentinelTrafficClient
$this->ensureRemoteAvailable();
$token = $this->server->settings->ensureValidSentinelToken();
- return $this->runRemote($this->buildBatchCommand($token, $urls));
+ return $this->awaitRemote($this->startRemote($this->buildBatchCommand($token, $urls)));
}
/**
@@ -722,14 +781,25 @@ class SentinelTrafficClient
/**
* Unlike instant_remote_process(), there are no SSH retries; a failure marks the server unavailable.
*/
- private function runRemote(string $command): string
+ private function startRemote(string $command): InvokedProcess
{
$commands = $this->server->isNonRoot() ? parseCommandsByLineForSudo(collect([$command]), $this->server) : [$command];
try {
- $process = Process::timeout(self::REMOTE_TIMEOUT)->run(
+ return Process::timeout(self::REMOTE_TIMEOUT)->start(
SshMultiplexingHelper::generateSshCommand($this->server, implode("\n", $commands), commandTimeout: self::REMOTE_TIMEOUT)
);
+ } catch (\Throwable $e) {
+ Cache::put($this->unavailableKey(), true, self::UNAVAILABLE_TTL);
+
+ throw $e;
+ }
+ }
+
+ private function awaitRemote(InvokedProcess $invoked): string
+ {
+ try {
+ $process = $invoked->wait();
if ($process->exitCode() !== 0) {
excludeCertainErrors($process->errorOutput(), $process->exitCode());
diff --git a/tests/Feature/TrafficAnalytics/GlobalAnalyticsTest.php b/tests/Feature/TrafficAnalytics/GlobalAnalyticsTest.php
index 870befc0f5..1bf1b554e1 100644
--- a/tests/Feature/TrafficAnalytics/GlobalAnalyticsTest.php
+++ b/tests/Feature/TrafficAnalytics/GlobalAnalyticsTest.php
@@ -12,8 +12,11 @@ use App\Models\StandaloneDocker;
use App\Models\Team;
use App\Models\User;
use App\Services\SentinelTrafficClient;
+use Illuminate\Contracts\Process\InvokedProcess;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Process;
use Livewire\Livewire;
uses(RefreshDatabase::class);
@@ -34,6 +37,20 @@ class FakeGlobalAnalyticsTrafficClient extends SentinelTrafficClient
}
}
+/**
+ * Answers every dashboard fetch with a real process that takes one second, so a test can
+ * tell concurrent fetches from serial ones.
+ */
+class SlowDashboardTrafficClient extends SentinelTrafficClient
+{
+ protected function startRemoteFetch(string $url): InvokedProcess
+ {
+ $bundle = json_encode(['overview' => ['requests' => 1000], 'apps' => []]);
+
+ return Process::start('sleep 1 && printf %s '.escapeshellarg($bundle));
+ }
+}
+
function fakeGlobalAnalyticsResponses(array $appUuids = []): array
{
return [
@@ -519,3 +536,17 @@ it('combines the client IP mode of several servers', function (array $modes, str
'anonymized and off' => [['anonymized', 'off'], 'anonymized'],
'full and anonymized' => [['full', 'anonymized'], 'full'],
]);
+
+it('fetches the dashboard of every server at the same time', function () {
+ Cache::flush();
+ collect(range(1, 3))->each(fn () => bootEnabledGlobalServer());
+ app()->bind(SentinelTrafficClient::class, fn ($app, array $parameters) => new SlowDashboardTrafficClient($parameters['server']));
+
+ $started = microtime(true);
+ $component = loadLazy(Livewire::test(Analytics::class));
+ $elapsed = microtime(true) - $started;
+
+ // Three servers fetched one after another take at least 3 seconds.
+ expect($elapsed)->toBeLessThan(2.5)
+ ->and($component->get('overview')['requests'])->toBe(3000);
+});
diff --git a/tests/Feature/TrafficAnalytics/SentinelTrafficAvailabilityTest.php b/tests/Feature/TrafficAnalytics/SentinelTrafficAvailabilityTest.php
index ddaf9c8a58..886da64ed6 100644
--- a/tests/Feature/TrafficAnalytics/SentinelTrafficAvailabilityTest.php
+++ b/tests/Feature/TrafficAnalytics/SentinelTrafficAvailabilityTest.php
@@ -116,5 +116,6 @@ it('shows no data without an error when Sentinel has no traffic routes (0.0.x an
->assertSet('series', [])
->assertNotDispatched('error');
- Process::assertRan(fn (PendingProcess $process) => str_contains(remoteProcessScript($process), '/api/traffic/dashboard'));
+ // The concurrent prefetch remembers the failure, so the per-server pass does not fetch it again.
+ Process::assertRanTimes(fn (PendingProcess $process) => str_contains(remoteProcessScript($process), '/api/traffic/dashboard'), 1);
});