An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(server): reset disk alerts only after usage recovers
Andras Bacsaied6d6b1e45f7
diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.phpindex b6b358512a..768ebddb22 100644--- a/app/Jobs/PushServerUpdateJob.php+++ b/app/Jobs/PushServerUpdateJob.php@@ -14,6 +14,7 @@ use App\Actions\Shared\ComplexStatusCheck;use App\Events\ServiceChecked;use App\Models\Application;use App\Models\ApplicationPreview;+use App\Models\NotificationThrottle;use App\Models\Server;use App\Models\ServiceApplication;use App\Models\ServiceDatabase;@@ -30,6 +31,7 @@ use App\Models\StandaloneSqlite;use App\Models\SwarmDocker;use App\Notifications\Application\RestartLimitReached as ApplicationRestartLimitReached;use App\Notifications\Container\ContainerRestarted;+use App\Notifications\Server\HighDiskUsage;use App\Services\ContainerStatusAggregator;use App\Services\RestartCountTracker;use App\Traits\CalculatesExcludedStatus;@@ -197,14 +199,22 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced// it is wasted work — and most servers sit well below the threshold.$diskThreshold = data_get($this->server, 'settings.server_disk_usage_notification_threshold', 80);$storageCacheKey = 'storage-check:'.$this->server->id;+ // Set while usage was high, so the throttle is released only once when usage recovers.+ $highUsageCacheKey = 'storage-high:'.$this->server->id;$lastPercentage = Cache::get($storageCacheKey);if ($filesystemUsageRoot !== null&& $filesystemUsageRoot >= $diskThreshold&& (string) $lastPercentage !== (string) $filesystemUsageRoot) {Cache::put($storageCacheKey, $filesystemUsageRoot, 600);+ Cache::forever($highUsageCacheKey, true);ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot);} elseif ($filesystemUsageRoot !== null && $filesystemUsageRoot < $diskThreshold) {Cache::forget($storageCacheKey);+ // The storage check does not run below the threshold, so the next spike alerts again only+ // when the throttle is released here.+ if (HighDiskUsage::hasRecovered($filesystemUsageRoot, $diskThreshold) && Cache::pull($highUsageCacheKey)) {+ NotificationThrottle::release($this->server, HighDiskUsage::class);+ }}if ($this->containers->isEmpty() && ! $this->isCompleteSnapshot()) {diff --git a/app/Jobs/ServerStorageCheckJob.php b/app/Jobs/ServerStorageCheckJob.phpindex 1e1b2c3c2a..f8bac23516 100644--- a/app/Jobs/ServerStorageCheckJob.php+++ b/app/Jobs/ServerStorageCheckJob.php@@ -60,7 +60,7 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silenced}if ($this->percentage > $serverDiskUsageNotificationThreshold) {$team->notify(new HighDiskUsage($this->server, $this->percentage, $serverDiskUsageNotificationThreshold));- } else {+ } elseif (HighDiskUsage::hasRecovered($this->percentage, $serverDiskUsageNotificationThreshold)) {// Usage recovered: the next spike should alert again instead of waiting for the interval.NotificationThrottle::release($this->server, HighDiskUsage::class);}diff --git a/app/Notifications/Server/HighDiskUsage.php b/app/Notifications/Server/HighDiskUsage.phpindex be0517b759..acf938427b 100644--- a/app/Notifications/Server/HighDiskUsage.php+++ b/app/Notifications/Server/HighDiskUsage.php@@ -13,6 +13,17 @@ use Illuminate\Notifications\Messages\MailMessage;class HighDiskUsage extends CustomEmailNotification implements ThrottledNotification{+ /**+ * Percentage points below the threshold that disk usage must reach before a new spike alerts+ * again, so usage that moves around the threshold does not send an alert on every change.+ */+ public const RECOVERY_MARGIN = 5;++ public static function hasRecovered(int|float $diskUsage, int|float $threshold): bool+ {+ return $diskUsage <= $threshold - self::RECOVERY_MARGIN;+ }+public function __construct(public Server $server, public int $disk_usage, public int $server_disk_usage_notification_threshold){$this->onQueue('high');diff --git a/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php b/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.phpindex d4e9563396..58e704c32f 100644--- a/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php+++ b/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php@@ -1,5 +1,6 @@<?php+use App\Jobs\PushServerUpdateJob;use App\Jobs\ServerStorageCheckJob;use App\Livewire\Server\Advanced;use App\Models\InstanceSettings;@@ -139,3 +140,45 @@ it('does not let a member change the notification interval', function () {expect($server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(24);});++function pushDiskUsage(Server $server, int $percentage): void+{+ (new PushServerUpdateJob($server->fresh(), [+ 'containers' => [],+ 'filesystem_usage_root' => ['used_percentage' => $percentage],+ ]))->handle();+}++it('notifies again on a new spike after disk usage recovered on a Sentinel server', function () {+ $server = diskUsageIntervalServer(intervalHours: 24);++ pushDiskUsage($server, 93);+ Carbon::setTestNow(now()->addHour());+ pushDiskUsage($server, 40);+ Carbon::setTestNow(now()->addHour());+ pushDiskUsage($server, 92);++ Notification::assertSentToTimes($server->team, HighDiskUsage::class, 2);+});++it('does not notify again while Sentinel disk usage moves around the threshold', function () {+ $server = diskUsageIntervalServer(intervalHours: 24);++ foreach ([81, 80, 81, 80, 79, 81, 78, 82] as $percentage) {+ pushDiskUsage($server, $percentage);+ Carbon::setTestNow(now()->addMinutes(2));+ }++ Notification::assertSentToTimes($server->team, HighDiskUsage::class, 1);+});++it('does not notify again while checked disk usage moves around the threshold', function () {+ $server = diskUsageIntervalServer(intervalHours: 24);++ foreach ([81, 80, 81, 76, 82] as $percentage) {+ runDiskUsageCheck($server, $percentage);+ Carbon::setTestNow(now()->addMinutes(2));+ }++ Notification::assertSentToTimes($server->team, HighDiskUsage::class, 1);+});