Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(server): reset disk alerts only after usage recovers

Andras Bacsaied6d6b1e45f7
diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php
index b6b358512a..768ebddb22 100644
--- a/app/Jobs/PushServerUpdateJob.php
+++ b/app/Jobs/PushServerUpdateJob.php
@@ -14,6 +14,7 @@ use App\Actions\Shared\ComplexStatusCheck;
use App\Events\ServiceChecked;
use App\Models\Application;
use App\Models\ApplicationPreview;
+use App\Models\NotificationThrottle;
use App\Models\Server;
use App\Models\ServiceApplication;
use App\Models\ServiceDatabase;
@@ -30,6 +31,7 @@ use App\Models\StandaloneSqlite;
use App\Models\SwarmDocker;
use App\Notifications\Application\RestartLimitReached as ApplicationRestartLimitReached;
use App\Notifications\Container\ContainerRestarted;
+use App\Notifications\Server\HighDiskUsage;
use App\Services\ContainerStatusAggregator;
use App\Services\RestartCountTracker;
use App\Traits\CalculatesExcludedStatus;
@@ -197,14 +199,22 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
// it is wasted work — and most servers sit well below the threshold.
$diskThreshold = data_get($this->server, 'settings.server_disk_usage_notification_threshold', 80);
$storageCacheKey = 'storage-check:'.$this->server->id;
+ // Set while usage was high, so the throttle is released only once when usage recovers.
+ $highUsageCacheKey = 'storage-high:'.$this->server->id;
$lastPercentage = Cache::get($storageCacheKey);
if ($filesystemUsageRoot !== null
&& $filesystemUsageRoot >= $diskThreshold
&& (string) $lastPercentage !== (string) $filesystemUsageRoot) {
Cache::put($storageCacheKey, $filesystemUsageRoot, 600);
+ Cache::forever($highUsageCacheKey, true);
ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot);
} elseif ($filesystemUsageRoot !== null && $filesystemUsageRoot < $diskThreshold) {
Cache::forget($storageCacheKey);
+ // The storage check does not run below the threshold, so the next spike alerts again only
+ // when the throttle is released here.
+ if (HighDiskUsage::hasRecovered($filesystemUsageRoot, $diskThreshold) && Cache::pull($highUsageCacheKey)) {
+ NotificationThrottle::release($this->server, HighDiskUsage::class);
+ }
}
if ($this->containers->isEmpty() && ! $this->isCompleteSnapshot()) {
diff --git a/app/Jobs/ServerStorageCheckJob.php b/app/Jobs/ServerStorageCheckJob.php
index 1e1b2c3c2a..f8bac23516 100644
--- a/app/Jobs/ServerStorageCheckJob.php
+++ b/app/Jobs/ServerStorageCheckJob.php
@@ -60,7 +60,7 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silenced
}
if ($this->percentage > $serverDiskUsageNotificationThreshold) {
$team->notify(new HighDiskUsage($this->server, $this->percentage, $serverDiskUsageNotificationThreshold));
- } else {
+ } elseif (HighDiskUsage::hasRecovered($this->percentage, $serverDiskUsageNotificationThreshold)) {
// Usage recovered: the next spike should alert again instead of waiting for the interval.
NotificationThrottle::release($this->server, HighDiskUsage::class);
}
diff --git a/app/Notifications/Server/HighDiskUsage.php b/app/Notifications/Server/HighDiskUsage.php
index be0517b759..acf938427b 100644
--- a/app/Notifications/Server/HighDiskUsage.php
+++ b/app/Notifications/Server/HighDiskUsage.php
@@ -13,6 +13,17 @@ use Illuminate\Notifications\Messages\MailMessage;
class HighDiskUsage extends CustomEmailNotification implements ThrottledNotification
{
+ /**
+ * Percentage points below the threshold that disk usage must reach before a new spike alerts
+ * again, so usage that moves around the threshold does not send an alert on every change.
+ */
+ public const RECOVERY_MARGIN = 5;
+
+ public static function hasRecovered(int|float $diskUsage, int|float $threshold): bool
+ {
+ return $diskUsage <= $threshold - self::RECOVERY_MARGIN;
+ }
+
public function __construct(public Server $server, public int $disk_usage, public int $server_disk_usage_notification_threshold)
{
$this->onQueue('high');
diff --git a/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php b/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php
index d4e9563396..58e704c32f 100644
--- a/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php
+++ b/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php
@@ -1,5 +1,6 @@
<?php
+use App\Jobs\PushServerUpdateJob;
use App\Jobs\ServerStorageCheckJob;
use App\Livewire\Server\Advanced;
use App\Models\InstanceSettings;
@@ -139,3 +140,45 @@ it('does not let a member change the notification interval', function () {
expect($server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(24);
});
+
+function pushDiskUsage(Server $server, int $percentage): void
+{
+ (new PushServerUpdateJob($server->fresh(), [
+ 'containers' => [],
+ 'filesystem_usage_root' => ['used_percentage' => $percentage],
+ ]))->handle();
+}
+
+it('notifies again on a new spike after disk usage recovered on a Sentinel server', function () {
+ $server = diskUsageIntervalServer(intervalHours: 24);
+
+ pushDiskUsage($server, 93);
+ Carbon::setTestNow(now()->addHour());
+ pushDiskUsage($server, 40);
+ Carbon::setTestNow(now()->addHour());
+ pushDiskUsage($server, 92);
+
+ Notification::assertSentToTimes($server->team, HighDiskUsage::class, 2);
+});
+
+it('does not notify again while Sentinel disk usage moves around the threshold', function () {
+ $server = diskUsageIntervalServer(intervalHours: 24);
+
+ foreach ([81, 80, 81, 80, 79, 81, 78, 82] as $percentage) {
+ pushDiskUsage($server, $percentage);
+ Carbon::setTestNow(now()->addMinutes(2));
+ }
+
+ Notification::assertSentToTimes($server->team, HighDiskUsage::class, 1);
+});
+
+it('does not notify again while checked disk usage moves around the threshold', function () {
+ $server = diskUsageIntervalServer(intervalHours: 24);
+
+ foreach ([81, 80, 81, 76, 82] as $percentage) {
+ runDiskUsageCheck($server, $percentage);
+ Carbon::setTestNow(now()->addMinutes(2));
+ }
+
+ Notification::assertSentToTimes($server->team, HighDiskUsage::class, 1);
+});