An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(analytics): fetch Sentinel dashboards of all servers concurrently
Andras Bacsai69bc72a3b514
diff --git a/app/Livewire/Analytics.php b/app/Livewire/Analytics.phpindex 62c3aab048..e4cc20741b 100644--- a/app/Livewire/Analytics.php+++ b/app/Livewire/Analytics.php@@ -348,9 +348,14 @@ class Analytics extends Component$appRows = [];$hostTotals = [];+ $clients = $servers->mapWithKeys(fn (Server $server) => [$server->uuid => $this->trafficClient($server)]);+ if ($resource === null) {+ SentinelTrafficClient::prefetchServerWideConcurrently($clients, $from, $to, $this->range, appsLimit: self::MAX_LEADERBOARD_APPS);+ }+foreach ($servers as $server) {try {- $client = $this->trafficClient($server);+ $client = $clients[$server->uuid];if ($resource !== null) {// One server in Sentinel's resource scope is exact; a merge across serversdiff --git a/app/Services/SentinelTrafficClient.php b/app/Services/SentinelTrafficClient.phpindex 1c5c28e2d8..54cb4758c6 100644--- a/app/Services/SentinelTrafficClient.php+++ b/app/Services/SentinelTrafficClient.php@@ -8,6 +8,7 @@ use App\Data\Traffic\TrafficPathData;use App\Data\Traffic\TrafficSeriesBucketData;use App\Helpers\SshMultiplexingHelper;use App\Models\Server;+use Illuminate\Contracts\Process\InvokedProcess;use Illuminate\Support\Collection;use Illuminate\Support\Facades\Cache;use Illuminate\Support\Facades\Log;@@ -354,6 +355,56 @@ class SentinelTrafficClientreturn $this->safeReportedKeys(array_map(fn ($app) => is_array($app) ? ($app['uuid'] ?? null) : null, $bundle['apps'] ?? []));}+ /**+ * Fetch the server-wide dashboard bundle of several servers at the same time, so a page+ * waits for the slowest server instead of for every server in turn. Each response warms+ * the cache that prefetchServerWide() reads; a failed fetch is remembered like in+ * fetchDashboard(), so prefetchServerWide() does not fetch it again.+ *+ * @param iterable<int, self> $clients+ */+ public static function prefetchServerWideConcurrently(iterable $clients, string $from, string $to, string $range, int $pathLimit = 50, int $breakdownLimit = 50, int $appsLimit = 200): void+ {+ $started = [];+ foreach ($clients as $client) {+ $url = $client->dashboardUrl(null, $from, $to, $range, $pathLimit, $breakdownLimit, $appsLimit);+ if (! $client->fetchesThroughStartRemoteFetch()+ || Cache::has($client->cacheKey($url))+ || Cache::get($client->dashboardAbsenceKey()) === true) {+ continue;+ }++ try {+ $started[] = [$client, $url, $client->startRemoteFetch($url)];+ } catch (\Throwable) {+ // An unavailable server fails again at once in prefetchServerWide().+ }+ }++ foreach ($started as [$client, $url, $process]) {+ try {+ Cache::put($client->cacheKey($url), $client->guard($client->awaitRemote($process)), 60);+ } catch (\Throwable) {+ Cache::put($client->dashboardAbsenceKey(), true, 60);+ }+ }+ }++ /**+ * False for a fake that serves canned bodies from raw() or remoteFetch(): the concurrent+ * prefetch would otherwise reach real SSH.+ */+ private function fetchesThroughStartRemoteFetch(): bool+ {+ foreach (['raw', 'remoteFetch'] as $method) {+ if ((new \ReflectionMethod($this, $method))->getDeclaringClass()->getName() !== self::class) {+ return false;+ }+ }++ return true;+ }+/*** The usable keys of a Sentinel-reported key list. A key that would be unsafe in a request* url is dropped (and logged) on its own, so it doesn't fail the whole server's analytics.@@ -685,13 +736,21 @@ class SentinelTrafficClient}protected function remoteFetch(string $url): string+ {+ // Throw the real SSH/docker error (for example, an unreadable SSH key or a missing+ // container). A literal "null" body comes back empty; guard() rejects the empty string.+ return $this->awaitRemote($this->startRemoteFetch($url));+ }++ /**+ * Start the fetch of one URL without waiting for it, so several servers can be fetched at the same time.+ */+ protected function startRemoteFetch(string $url): InvokedProcess{$this->ensureRemoteAvailable();$token = $this->server->settings->ensureValidSentinelToken();- // Throw the real SSH/docker error (for example, an unreadable SSH key or a missing- // container). A literal "null" body comes back empty; guard() rejects the empty string.- return $this->runRemote($this->buildFetchCommand($token, $url));+ return $this->startRemote($this->buildFetchCommand($token, $url));}/**@@ -702,7 +761,7 @@ class SentinelTrafficClient$this->ensureRemoteAvailable();$token = $this->server->settings->ensureValidSentinelToken();- return $this->runRemote($this->buildBatchCommand($token, $urls));+ return $this->awaitRemote($this->startRemote($this->buildBatchCommand($token, $urls)));}/**@@ -722,14 +781,25 @@ class SentinelTrafficClient/*** Unlike instant_remote_process(), there are no SSH retries; a failure marks the server unavailable.*/- private function runRemote(string $command): string+ private function startRemote(string $command): InvokedProcess{$commands = $this->server->isNonRoot() ? parseCommandsByLineForSudo(collect([$command]), $this->server) : [$command];try {- $process = Process::timeout(self::REMOTE_TIMEOUT)->run(+ return Process::timeout(self::REMOTE_TIMEOUT)->start(SshMultiplexingHelper::generateSshCommand($this->server, implode("\n", $commands), commandTimeout: self::REMOTE_TIMEOUT));+ } catch (\Throwable $e) {+ Cache::put($this->unavailableKey(), true, self::UNAVAILABLE_TTL);++ throw $e;+ }+ }++ private function awaitRemote(InvokedProcess $invoked): string+ {+ try {+ $process = $invoked->wait();if ($process->exitCode() !== 0) {excludeCertainErrors($process->errorOutput(), $process->exitCode());diff --git a/tests/Feature/TrafficAnalytics/GlobalAnalyticsTest.php b/tests/Feature/TrafficAnalytics/GlobalAnalyticsTest.phpindex 870befc0f5..1bf1b554e1 100644--- a/tests/Feature/TrafficAnalytics/GlobalAnalyticsTest.php+++ b/tests/Feature/TrafficAnalytics/GlobalAnalyticsTest.php@@ -12,8 +12,11 @@ use App\Models\StandaloneDocker;use App\Models\Team;use App\Models\User;use App\Services\SentinelTrafficClient;+use Illuminate\Contracts\Process\InvokedProcess;use Illuminate\Foundation\Testing\RefreshDatabase;use Illuminate\Support\Carbon;+use Illuminate\Support\Facades\Cache;+use Illuminate\Support\Facades\Process;use Livewire\Livewire;uses(RefreshDatabase::class);@@ -34,6 +37,20 @@ class FakeGlobalAnalyticsTrafficClient extends SentinelTrafficClient}}+/**+ * Answers every dashboard fetch with a real process that takes one second, so a test can+ * tell concurrent fetches from serial ones.+ */+class SlowDashboardTrafficClient extends SentinelTrafficClient+{+ protected function startRemoteFetch(string $url): InvokedProcess+ {+ $bundle = json_encode(['overview' => ['requests' => 1000], 'apps' => []]);++ return Process::start('sleep 1 && printf %s '.escapeshellarg($bundle));+ }+}+function fakeGlobalAnalyticsResponses(array $appUuids = []): array{return [@@ -519,3 +536,17 @@ it('combines the client IP mode of several servers', function (array $modes, str'anonymized and off' => [['anonymized', 'off'], 'anonymized'],'full and anonymized' => [['full', 'anonymized'], 'full'],]);++it('fetches the dashboard of every server at the same time', function () {+ Cache::flush();+ collect(range(1, 3))->each(fn () => bootEnabledGlobalServer());+ app()->bind(SentinelTrafficClient::class, fn ($app, array $parameters) => new SlowDashboardTrafficClient($parameters['server']));++ $started = microtime(true);+ $component = loadLazy(Livewire::test(Analytics::class));+ $elapsed = microtime(true) - $started;++ // Three servers fetched one after another take at least 3 seconds.+ expect($elapsed)->toBeLessThan(2.5)+ ->and($component->get('overview')['requests'])->toBe(3000);+});diff --git a/tests/Feature/TrafficAnalytics/SentinelTrafficAvailabilityTest.php b/tests/Feature/TrafficAnalytics/SentinelTrafficAvailabilityTest.phpindex ddaf9c8a58..886da64ed6 100644--- a/tests/Feature/TrafficAnalytics/SentinelTrafficAvailabilityTest.php+++ b/tests/Feature/TrafficAnalytics/SentinelTrafficAvailabilityTest.php@@ -116,5 +116,6 @@ it('shows no data without an error when Sentinel has no traffic routes (0.0.x an->assertSet('series', [])->assertNotDispatched('error');- Process::assertRan(fn (PendingProcess $process) => str_contains(remoteProcessScript($process), '/api/traffic/dashboard'));+ // The concurrent prefetch remembers the failure, so the per-server pass does not fetch it again.+ Process::assertRanTimes(fn (PendingProcess $process) => str_contains(remoteProcessScript($process), '/api/traffic/dashboard'), 1);});