Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(notifications): throttle repeated notifications per resource

Andras Bacsai99c840922c3d
diff --git a/.ai/lessons.md b/.ai/lessons.md
index d68d184014..eff6d13e5c 100644
--- a/.ai/lessons.md
+++ b/.ai/lessons.md
@@ -5,6 +5,8 @@
- When a symptom matches an earlier fix, inspect that fix and prove why it no longer works before adding another workaround.
- Test old reports against the current branch because later changes can make the report obsolete.
- Use the same regression test before and after the production change so the result shows the behavior difference.
+- Several dev instances can run from one checkout (see `./scripts/dev urls`). After you add a migration, run it on each running instance of that checkout, or those instances fail on the new code.
+- The checkout is shared with other sessions. When the full suite fails, rerun each failing file alone and compare with a clean `git archive HEAD` copy before you connect a failure to your change.
- Redirect browser test output to a file (`> /tmp/x.log 2>&1`); piping it (`| tail`) hangs because the Playwright server keeps the pipe open.
- Call `visit()` directly in each `tests/v4/Browser` test body; Pest does not mark a test that only uses helper-wrapped `visit()` as a browser test, so it fails with `sendText() on null`.
diff --git a/app/Console/Commands/CleanupUnreachableServers.php b/app/Console/Commands/CleanupUnreachableServers.php
index 666e98a182..615dc21d20 100644
--- a/app/Console/Commands/CleanupUnreachableServers.php
+++ b/app/Console/Commands/CleanupUnreachableServers.php
@@ -3,6 +3,7 @@
namespace App\Console\Commands;
use App\Models\Server;
+use App\Notifications\Server\Unreachable;
use Illuminate\Console\Command;
class CleanupUnreachableServers extends Command
@@ -14,7 +15,7 @@ class CleanupUnreachableServers extends Command
public function handle()
{
echo "Running unreachable server cleanup...\n";
- $servers = Server::where('unreachable_count', '>=', 3)->where('unreachable_notification_sent', true)->where('updated_at', '<', now()->subDays(7))->get();
+ $servers = Server::where('unreachable_count', '>=', 3)->whereHas('notificationThrottles', fn ($query) => $query->where('notification', Unreachable::class))->where('updated_at', '<', now()->subDays(7))->get();
if ($servers->count() > 0) {
foreach ($servers as $server) {
echo "Cleanup unreachable server ($server->id) with name $server->name";
diff --git a/app/Contracts/ThrottledNotification.php b/app/Contracts/ThrottledNotification.php
new file mode 100644
index 0000000000..b578f75835
--- /dev/null
+++ b/app/Contracts/ThrottledNotification.php
@@ -0,0 +1,19 @@
+<?php
+
+namespace App\Contracts;
+
+use Illuminate\Database\Eloquent\Model;
+
+/**
+ * A notification that is sent at most once per interval for the same resource.
+ * Team::notify() checks the throttle before it sends the notification.
+ */
+interface ThrottledNotification
+{
+ /**
+ * The resource the throttle belongs to. Return null to send without a throttle.
+ */
+ public function throttleSubject(): ?Model;
+
+ public function throttleIntervalMinutes(): int;
+}
diff --git a/app/Http/Controllers/Api/DatabasesController.php b/app/Http/Controllers/Api/DatabasesController.php
index 81035eca03..3c9c832d31 100644
--- a/app/Http/Controllers/Api/DatabasesController.php
+++ b/app/Http/Controllers/Api/DatabasesController.php
@@ -5119,7 +5119,6 @@ class DatabasesController extends Controller
'created_at',
'updated_at',
'last_execution_at',
- 'missing_backup_notification_sent_at',
])->fill([
'uuid' => new_public_id(),
'database_id' => $newDatabase->id,
diff --git a/app/Http/Controllers/Api/ServersController.php b/app/Http/Controllers/Api/ServersController.php
index 1a5d24b194..ef69f6b424 100644
--- a/app/Http/Controllers/Api/ServersController.php
+++ b/app/Http/Controllers/Api/ServersController.php
@@ -187,6 +187,7 @@ class ServersController extends Controller
$server->load(['settings']);
}
+ $server->append('unreachable_notification_sent');
$settings = $this->removeSensitiveDataFromSettings($server->settings);
$server = $this->removeSensitiveData($server);
data_set($server, 'settings', $settings);
@@ -645,6 +646,7 @@ class ServersController extends Controller
'deployment_queue_limit' => ['type' => 'integer', 'description' => 'Maximum number of queued deployments.'],
'server_disk_usage_notification_threshold' => ['type' => 'integer', 'description' => 'Server disk usage notification threshold (%).'],
'server_disk_usage_check_frequency' => ['type' => 'string', 'description' => 'Cron expression for disk usage check frequency.'],
+ 'server_disk_usage_notification_interval_hours' => ['type' => 'integer', 'description' => 'Minimum hours between high disk usage notifications (1-720). Default: 24.'],
'connection_timeout' => ['type' => 'integer', 'description' => 'SSH connection timeout in seconds (1-300). Default: 10.'],
],
),
@@ -682,7 +684,7 @@ class ServersController extends Controller
)]
public function update_server(Request $request)
{
- $allowedFields = ['name', 'description', 'ip', 'port', 'user', 'private_key_uuid', 'server_role', 'instant_validate', 'proxy_type', 'concurrent_builds', 'dynamic_timeout', 'deployment_queue_limit', 'server_disk_usage_notification_threshold', 'server_disk_usage_check_frequency', 'connection_timeout', 'is_terminal_enabled'];
+ $allowedFields = ['name', 'description', 'ip', 'port', 'user', 'private_key_uuid', 'server_role', 'instant_validate', 'proxy_type', 'concurrent_builds', 'dynamic_timeout', 'deployment_queue_limit', 'server_disk_usage_notification_threshold', 'server_disk_usage_check_frequency', 'server_disk_usage_notification_interval_hours', 'connection_timeout', 'is_terminal_enabled'];
$teamId = getTeamIdFromToken();
if (is_null($teamId)) {
@@ -708,6 +710,7 @@ class ServersController extends Controller
'deployment_queue_limit' => 'integer|min:1',
'server_disk_usage_notification_threshold' => 'integer|min:1|max:100',
'server_disk_usage_check_frequency' => 'string',
+ 'server_disk_usage_notification_interval_hours' => 'integer|min:1|max:720',
'connection_timeout' => 'integer|min:1|max:300',
'is_terminal_enabled' => 'boolean|nullable',
], [
@@ -790,7 +793,7 @@ class ServersController extends Controller
]);
}
- $advancedSettings = $request->only(['concurrent_builds', 'dynamic_timeout', 'deployment_queue_limit', 'server_disk_usage_notification_threshold', 'server_disk_usage_check_frequency', 'connection_timeout']);
+ $advancedSettings = $request->only(['concurrent_builds', 'dynamic_timeout', 'deployment_queue_limit', 'server_disk_usage_notification_threshold', 'server_disk_usage_check_frequency', 'server_disk_usage_notification_interval_hours', 'connection_timeout']);
if (! empty($advancedSettings)) {
$server->settings()->update(array_filter($advancedSettings, fn ($value) => ! is_null($value)));
}
diff --git a/app/Jobs/CheckMissingDatabaseBackupsJob.php b/app/Jobs/CheckMissingDatabaseBackupsJob.php
index 06ba79a33f..957868f44d 100644
--- a/app/Jobs/CheckMissingDatabaseBackupsJob.php
+++ b/app/Jobs/CheckMissingDatabaseBackupsJob.php
@@ -2,6 +2,7 @@
namespace App\Jobs;
+use App\Models\NotificationThrottle;
use App\Models\ScheduledDatabaseBackup;
use App\Notifications\Database\BackupMissing;
use Illuminate\Bus\Queueable;
@@ -39,10 +40,6 @@ class CheckMissingDatabaseBackupsJob implements ShouldBeEncrypted, ShouldBeUniqu
return;
}
- if ($backup->missing_backup_notification_sent_at?->greaterThanOrEqualTo($lastActivityAt)) {
- return;
- }
-
if (! $backup->team) {
Log::warning("Cannot send missing backup notification for backup {$backup->id}: team not found");
@@ -53,7 +50,11 @@ class CheckMissingDatabaseBackupsJob implements ShouldBeEncrypted, ShouldBeUniqu
return;
}
+ // Send once for each period without backup activity.
+ if (! NotificationThrottle::claim($backup, BackupMissing::class, $lastActivityAt)) {
+ return;
+ }
+
$backup->team->notify(new BackupMissing($backup, $lastExecutionAt));
- $backup->forceFill(['missing_backup_notification_sent_at' => now()])->save();
}
}
diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php
index 1324c212ed..3224d3a5e1 100644
--- a/app/Jobs/PushServerUpdateJob.php
+++ b/app/Jobs/PushServerUpdateJob.php
@@ -799,7 +799,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
try {
if (CheckProxy::run($this->server)) {
StartProxy::run($this->server, async: false);
- $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server));
+ $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
}
} catch (\Throwable $e) {
}
@@ -841,7 +841,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
})->first();
if (! $tcpProxyContainerFound) {
StartDatabaseProxy::dispatch($database);
- $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server));
+ $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server, restartedResource: $database));
}
} elseif ($this->isRunning($containerStatus) && ! $tcpProxy) {
// Clean up orphaned proxy containers when is_public=false
diff --git a/app/Jobs/ServerCheckJob.php b/app/Jobs/ServerCheckJob.php
index 10faa7e9b8..4b7203648c 100644
--- a/app/Jobs/ServerCheckJob.php
+++ b/app/Jobs/ServerCheckJob.php
@@ -85,7 +85,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
$shouldStart = CheckProxy::run($this->server);
if ($shouldStart) {
StartProxy::run($this->server, async: false);
- $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server));
+ $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
}
} catch (\Throwable $e) {
}
diff --git a/app/Jobs/ServerStorageCheckJob.php b/app/Jobs/ServerStorageCheckJob.php
index 51426d880a..a457207951 100644
--- a/app/Jobs/ServerStorageCheckJob.php
+++ b/app/Jobs/ServerStorageCheckJob.php
@@ -10,8 +10,8 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\TimeoutExceededException;
use Illuminate\Support\Facades\Log;
-use Illuminate\Support\Facades\RateLimiter;
use Laravel\Horizon\Contracts\Silenced;
class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silenced
@@ -31,7 +31,7 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silenced
public function failed(?\Throwable $exception): void
{
- if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) {
+ if ($exception instanceof TimeoutExceededException) {
Log::warning('ServerStorageCheckJob timed out', [
'server_id' => $this->server->id,
'server_name' => $this->server->name,
@@ -58,20 +58,7 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silenced
return 'No percentage could be retrieved.';
}
if ($this->percentage > $serverDiskUsageNotificationThreshold) {
- $executed = RateLimiter::attempt(
- 'high-disk-usage:'.$this->server->id,
- $maxAttempts = 0,
- function () use ($team, $serverDiskUsageNotificationThreshold) {
- $team->notify(new HighDiskUsage($this->server, $this->percentage, $serverDiskUsageNotificationThreshold));
- },
- $decaySeconds = 3600,
- );
-
- if (! $executed) {
- return 'Too many messages sent!';
- }
- } else {
- RateLimiter::hit('high-disk-usage:'.$this->server->id, 600);
+ $team->notify(new HighDiskUsage($this->server, $this->percentage, $serverDiskUsageNotificationThreshold));
}
} catch (\Throwable $e) {
return handleError($e);
diff --git a/app/Livewire/Server/Advanced.php b/app/Livewire/Server/Advanced.php
index 895ce34e79..34847a12e6 100644
--- a/app/Livewire/Server/Advanced.php
+++ b/app/Livewire/Server/Advanced.php
@@ -18,6 +18,9 @@ class Advanced extends Component
#[Validate(['required', 'integer', 'min:1', 'max:99'])]
public int|string $serverDiskUsageNotificationThreshold = 50;
+ #[Validate(['required', 'integer', 'min:1', 'max:720'])]
+ public int|string $serverDiskUsageNotificationIntervalHours = 24;
+
#[Validate(['required', 'integer', 'min:1'])]
public int|string $concurrentBuilds = 1;
@@ -52,6 +55,7 @@ class Advanced extends Component
$this->server->settings->backup_compression_cpu_percentage = $this->backupCompressionCpuPercentage;
$this->server->settings->server_disk_usage_notification_threshold = $this->serverDiskUsageNotificationThreshold;
$this->server->settings->server_disk_usage_check_frequency = $this->serverDiskUsageCheckFrequency;
+ $this->server->settings->server_disk_usage_notification_interval_hours = $this->serverDiskUsageNotificationIntervalHours;
$this->server->settings->save();
} else {
$this->concurrentBuilds = $this->server->settings->concurrent_builds;
@@ -60,6 +64,7 @@ class Advanced extends Component
$this->backupCompressionCpuPercentage = $this->server->settings->backup_compression_cpu_percentage;
$this->serverDiskUsageNotificationThreshold = $this->server->settings->server_disk_usage_notification_threshold;
$this->serverDiskUsageCheckFrequency = $this->server->settings->server_disk_usage_check_frequency;
+ $this->serverDiskUsageNotificationIntervalHours = $this->server->settings->server_disk_usage_notification_interval_hours;
}
}
diff --git a/app/Models/NotificationThrottle.php b/app/Models/NotificationThrottle.php
new file mode 100644
index 0000000000..46340f9fd3
--- /dev/null
+++ b/app/Models/NotificationThrottle.php
@@ -0,0 +1,111 @@
+<?php
+
+namespace App\Models;
+
+use App\Contracts\ThrottledNotification;
+use Carbon\CarbonInterface;
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\Relations\MorphTo;
+
+/**
+ * Records when a notification was last sent for a resource, so repeated checks
+ * (across queue workers) do not send the same notification again.
+ */
+class NotificationThrottle extends Model
+{
+ public $timestamps = false;
+
+ protected $fillable = [
+ 'notifiable_type',
+ 'notifiable_id',
+ 'notification',
+ 'sent_at',
+ ];
+
+ protected function casts(): array
+ {
+ return [
+ 'sent_at' => 'datetime',
+ ];
+ }
+
+ public function notifiable(): MorphTo
+ {
+ return $this->morphTo();
+ }
+
+ /**
+ * Atomically reserve the right to send a notification.
+ *
+ * Returns true when the notification was never sent, or when it was last sent
+ * before $sentBefore. Without $sentBefore, it is sent only once until released.
+ */
+ public static function claim(Model $notifiable, string $notification, ?CarbonInterface $sentBefore = null): bool
+ {
+ if ($sentBefore !== null) {
+ $renewed = static::query()
+ ->whereMorphedTo('notifiable', $notifiable)
+ ->where('notification', $notification)
+ ->where('sent_at', '<', $sentBefore)
+ ->update(['sent_at' => now()]);
+
+ if ($renewed === 1) {
+ return true;
+ }
+ }
+
+ return static::query()->insertOrIgnore([
+ 'notifiable_type' => $notifiable->getMorphClass(),
+ 'notifiable_id' => $notifiable->getKey(),
+ 'notification' => $notification,
+ 'sent_at' => now(),
+ ]) === 1;
+ }
+
+ /**
+ * Claim the throttle a notification declares. Notifications without a throttle subject are always allowed.
+ */
+ public static function claimFor(ThrottledNotification $notification): bool
+ {
+ $subject = $notification->throttleSubject();
+ if ($subject === null) {
+ return true;
+ }
+
+ return static::claim($subject, $notification::class, now()->subMinutes($notification->throttleIntervalMinutes()));
+ }
+
+ /**
+ * Mark a notification as sent without sending it.
+ */
+ public static function record(Model $notifiable, string $notification): void
+ {
+ static::query()->updateOrInsert(
+ [
+ 'notifiable_type' => $notifiable->getMorphClass(),
+ 'notifiable_id' => $notifiable->getKey(),
+ 'notification' => $notification,
+ ],
+ ['sent_at' => now()],
+ );
+ }
+
+ /**
+ * Forget a sent notification. Returns true when a record was removed.
+ */
+ public static function release(Model $notifiable, string $notification): bool
+ {
+ return static::query()
+ ->whereMorphedTo('notifiable', $notifiable)
+ ->where('notification', $notification)
+ ->delete() > 0;
+ }
+
+ public static function wasSent(Model $notifiable, string $notification): bool
+ {
+ return static::query()
+ ->whereMorphedTo('notifiable', $notifiable)
+ ->where('notification', $notification)
+ ->exists();
+ }
+}
diff --git a/app/Models/ScheduledDatabaseBackup.php b/app/Models/ScheduledDatabaseBackup.php
index 73c162ea81..b44d1b16f4 100644
--- a/app/Models/ScheduledDatabaseBackup.php
+++ b/app/Models/ScheduledDatabaseBackup.php
@@ -18,7 +18,6 @@ class ScheduledDatabaseBackup extends BaseModel
'database_backup_retention_max_storage_locally' => 'float',
'database_backup_retention_max_storage_s3' => 'float',
'missing_backup_notification_days' => 'integer',
- 'missing_backup_notification_sent_at' => 'datetime',
'last_execution_at' => 'datetime',
'next_run_at' => 'datetime',
];
diff --git a/app/Models/Server.php b/app/Models/Server.php
index 00eaad8839..9f4e6667d8 100644
--- a/app/Models/Server.php
+++ b/app/Models/Server.php
@@ -29,6 +29,7 @@ use App\Traits\HasSafeStringAttribute;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Relations\MorphMany;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Carbon;
use Illuminate\Support\Collection;
@@ -235,6 +236,7 @@ class Server extends BaseModel
});
$server->settings()->delete();
$server->sslCertificates()->delete();
+ $server->notificationThrottles()->delete();
});
static::updated(function () {
@@ -280,7 +282,6 @@ class Server extends BaseModel
'logdrain_newrelic_license_key' => 'encrypted',
'delete_unused_volumes' => 'boolean',
'delete_unused_networks' => 'boolean',
- 'unreachable_notification_sent' => 'boolean',
'force_disabled' => 'boolean',
'sentinel_waiting_since' => 'datetime',
];
@@ -1526,6 +1527,19 @@ $siteAddress {
return $this->belongsTo(CloudProviderToken::class);
}
+ public function notificationThrottles(): MorphMany
+ {
+ return $this->morphMany(NotificationThrottle::class, 'notifiable');
+ }
+
+ /**
+ * True while an Unreachable notification was sent and no Reachable notification followed.
+ */
+ protected function unreachableNotificationSent(): Attribute
+ {
+ return Attribute::get(fn (): bool => NotificationThrottle::wasSent($this, Unreachable::class));
+ }
+
public function sslCertificates()
{
return $this->hasMany(SslCertificate::class);
@@ -1777,18 +1791,16 @@ $siteAddress {
public function sendReachableNotification()
{
- $this->unreachable_notification_sent = false;
- $this->save();
- $this->refresh();
- $this->team->notify(new Reachable($this));
+ if (NotificationThrottle::release($this, Unreachable::class)) {
+ $this->team->notify(new Reachable($this));
+ }
}
public function sendUnreachableNotification()
{
- $this->unreachable_notification_sent = true;
- $this->save();
- $this->refresh();
- $this->team->notify(new Unreachable($this));
+ if (NotificationThrottle::claim($this, Unreachable::class)) {
+ $this->team->notify(new Unreachable($this));
+ }
}
public function validateConnection(bool $justCheckingNewKey = false)
diff --git a/app/Models/ServerSetting.php b/app/Models/ServerSetting.php
index 53674dad92..defb842dda 100644
--- a/app/Models/ServerSetting.php
+++ b/app/Models/ServerSetting.php
@@ -117,6 +117,7 @@ class ServerSetting extends Model
'server_disk_usage_notification_threshold',
'is_sentinel_debug_enabled',
'server_disk_usage_check_frequency',
+ 'server_disk_usage_notification_interval_hours',
'is_terminal_enabled',
'deployment_queue_limit',
'backup_compression_cpu_percentage',
@@ -170,6 +171,7 @@ class ServerSetting extends Model
'compose_version_checked_at' => 'datetime',
'backup_compression_cpu_percentage' => 'integer',
'docker_cleanup_next_run_at' => 'datetime',
+ 'server_disk_usage_notification_interval_hours' => 'integer',
];
/**
diff --git a/app/Models/Team.php b/app/Models/Team.php
index 6ce1c1d3c1..da08882f6b 100644
--- a/app/Models/Team.php
+++ b/app/Models/Team.php
@@ -3,11 +3,13 @@
namespace App\Models;
use App\Actions\User\RevokeUserTeamTokens;
+use App\Contracts\ThrottledNotification;
use App\Events\ServerReachabilityChanged;
use App\Notifications\Channels\SendsDiscord;
use App\Notifications\Channels\SendsEmail;
use App\Notifications\Channels\SendsPushover;
use App\Notifications\Channels\SendsSlack;
+use App\Notifications\Server\Unreachable;
use App\Traits\Auditable;
use App\Traits\HasNotificationSettings;
use App\Traits\HasSafeStringAttribute;
@@ -43,7 +45,10 @@ use OpenApi\Attributes as OA;
class Team extends Model implements SendsDiscord, SendsEmail, SendsPushover, SendsSlack
{
- use Auditable, HasFactory, HasNotificationSettings, HasSafeStringAttribute, Notifiable;
+ use Auditable, HasFactory, HasNotificationSettings, HasSafeStringAttribute;
+ use Notifiable {
+ notify as sendNotification;
+ }
protected $fillable = [
'name',
@@ -273,11 +278,23 @@ class Team extends Model implements SendsDiscord, SendsEmail, SendsPushover, Sen
]);
ServerReachabilityChanged::dispatch($server);
$server->unreachable_count = 3;
- $server->unreachable_notification_sent = true;
$server->save();
+ NotificationThrottle::record($server, Unreachable::class);
}
}
+ /**
+ * Send a notification, unless it is throttled and was already sent within its interval.
+ */
+ public function notify($instance): void
+ {
+ if ($instance instanceof ThrottledNotification && ! NotificationThrottle::claimFor($instance)) {
+ return;
+ }
+
+ $this->sendNotification($instance);
+ }
+
public function environment_variables()
{
return $this->hasMany(SharedEnvironmentVariable::class)->where('type', 'team');
diff --git a/app/Notifications/Container/ContainerRestarted.php b/app/Notifications/Container/ContainerRestarted.php
index 2d7eb58b50..012f997754 100644
--- a/app/Notifications/Container/ContainerRestarted.php
+++ b/app/Notifications/Container/ContainerRestarted.php
@@ -2,16 +2,21 @@
namespace App\Notifications\Container;
+use App\Contracts\ThrottledNotification;
use App\Models\Server;
use App\Notifications\CustomEmailNotification;
use App\Notifications\Dto\DiscordMessage;
use App\Notifications\Dto\PushoverMessage;
use App\Notifications\Dto\SlackMessage;
+use Illuminate\Database\Eloquent\Model;
use Illuminate\Notifications\Messages\MailMessage;
-class ContainerRestarted extends CustomEmailNotification
+class ContainerRestarted extends CustomEmailNotification implements ThrottledNotification
{
- public function __construct(public string $name, public Server $server, public ?string $url = null)
+ /**
+ * When $restartedResource is set, a restart loop of that resource sends at most one notification per hour.
+ */
+ public function __construct(public string $name, public Server $server, public ?string $url = null, public ?Model $restartedResource = null)
{
$this->onQueue('high');
}
@@ -21,6 +26,16 @@ class ContainerRestarted extends CustomEmailNotification
return $notifiable->getEnabledChannels('status_change');
}
+ public function throttleSubject(): ?Model
+ {
+ return $this->restartedResource;
+ }
+
+ public function throttleIntervalMinutes(): int
+ {
+ return 60;
+ }
+
public function toMail(): MailMessage
{
$mail = new MailMessage;
diff --git a/app/Notifications/Server/HighDiskUsage.php b/app/Notifications/Server/HighDiskUsage.php
index 149d1bbc86..be0517b759 100644
--- a/app/Notifications/Server/HighDiskUsage.php
+++ b/app/Notifications/Server/HighDiskUsage.php
@@ -2,14 +2,16 @@
namespace App\Notifications\Server;
+use App\Contracts\ThrottledNotification;
use App\Models\Server;
use App\Notifications\CustomEmailNotification;
use App\Notifications\Dto\DiscordMessage;
use App\Notifications\Dto\PushoverMessage;
use App\Notifications\Dto\SlackMessage;
+use Illuminate\Database\Eloquent\Model;
use Illuminate\Notifications\Messages\MailMessage;
-class HighDiskUsage extends CustomEmailNotification
+class HighDiskUsage extends CustomEmailNotification implements ThrottledNotification
{
public function __construct(public Server $server, public int $disk_usage, public int $server_disk_usage_notification_threshold)
{
@@ -21,6 +23,16 @@ class HighDiskUsage extends CustomEmailNotification
return $notifiable->getEnabledChannels('server_disk_usage');
}
+ public function throttleSubject(): ?Model
+ {
+ return $this->server;
+ }
+
+ public function throttleIntervalMinutes(): int
+ {
+ return max(1, (int) data_get($this->server, 'settings.server_disk_usage_notification_interval_hours', 24)) * 60;
+ }
+
public function toMail(): MailMessage
{
$mail = new MailMessage;
diff --git a/database/migrations/2026_09_29_082403_add_disk_usage_notification_interval_to_server_settings_table.php b/database/migrations/2026_09_29_082403_add_disk_usage_notification_interval_to_server_settings_table.php
new file mode 100644
index 0000000000..6ec4a5a312
--- /dev/null
+++ b/database/migrations/2026_09_29_082403_add_disk_usage_notification_interval_to_server_settings_table.php
@@ -0,0 +1,22 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+ public function up(): void
+ {
+ Schema::table('server_settings', function (Blueprint $table) {
+ $table->integer('server_disk_usage_notification_interval_hours')->default(24);
+ });
+ }
+
+ public function down(): void
+ {
+ Schema::table('server_settings', function (Blueprint $table) {
+ $table->dropColumn('server_disk_usage_notification_interval_hours');
+ });
+ }
+};
diff --git a/database/migrations/2026_09_29_083244_create_notification_throttles_table.php b/database/migrations/2026_09_29_083244_create_notification_throttles_table.php
new file mode 100644
index 0000000000..e11a14a567
--- /dev/null
+++ b/database/migrations/2026_09_29_083244_create_notification_throttles_table.php
@@ -0,0 +1,98 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+ private const SERVER = 'App\Models\Server';
+
+ private const BACKUP = 'App\Models\ScheduledDatabaseBackup';
+
+ private const UNREACHABLE = 'App\Notifications\Server\Unreachable';
+
+ private const BACKUP_MISSING = 'App\Notifications\Database\BackupMissing';
+
+ /**
+ * Moves the per-resource "notification sent" markers into one table.
+ */
+ public function up(): void
+ {
+ Schema::create('notification_throttles', function (Blueprint $table) {
+ $table->id();
+ $table->morphs('notifiable');
+ $table->string('notification');
+ $table->timestampTz('sent_at');
+ $table->unique(['notifiable_type', 'notifiable_id', 'notification']);
+ });
+
+ DB::table('servers')
+ ->where('unreachable_notification_sent', true)
+ ->orderBy('id')
+ ->select(['id', 'updated_at'])
+ ->chunk(500, function ($servers) {
+ DB::table('notification_throttles')->insertOrIgnore($servers->map(fn ($server) => [
+ 'notifiable_type' => self::SERVER,
+ 'notifiable_id' => $server->id,
+ 'notification' => self::UNREACHABLE,
+ 'sent_at' => $server->updated_at ?? now(),
+ ])->all());
+ });
+
+ DB::table('scheduled_database_backups')
+ ->whereNotNull('missing_backup_notification_sent_at')
+ ->orderBy('id')
+ ->select(['id', 'missing_backup_notification_sent_at'])
+ ->chunk(500, function ($backups) {
+ DB::table('notification_throttles')->insertOrIgnore($backups->map(fn ($backup) => [
+ 'notifiable_type' => self::BACKUP,
+ 'notifiable_id' => $backup->id,
+ 'notification' => self::BACKUP_MISSING,
+ 'sent_at' => $backup->missing_backup_notification_sent_at,
+ ])->all());
+ });
+
+ Schema::table('servers', function (Blueprint $table) {
+ $table->dropColumn('unreachable_notification_sent');
+ });
+ Schema::table('scheduled_database_backups', function (Blueprint $table) {
+ $table->dropColumn('missing_backup_notification_sent_at');
+ });
+ }
+
+ public function down(): void
+ {
+ Schema::table('servers', function (Blueprint $table) {
+ $table->boolean('unreachable_notification_sent')->default(false);
+ });
+ Schema::table('scheduled_database_backups', function (Blueprint $table) {
+ $table->timestamp('missing_backup_notification_sent_at')->nullable();
+ });
+
+ DB::table('notification_throttles')
+ ->where('notifiable_type', self::SERVER)
+ ->where('notification', self::UNREACHABLE)
+ ->orderBy('id')
+ ->chunk(500, function ($throttles) {
+ DB::table('servers')
+ ->whereIn('id', $throttles->pluck('notifiable_id'))
+ ->update(['unreachable_notification_sent' => true]);
+ });
+
+ DB::table('notification_throttles')
+ ->where('notifiable_type', self::BACKUP)
+ ->where('notification', self::BACKUP_MISSING)
+ ->orderBy('id')
+ ->chunk(500, function ($throttles) {
+ foreach ($throttles as $throttle) {
+ DB::table('scheduled_database_backups')
+ ->where('id', $throttle->notifiable_id)
+ ->update(['missing_backup_notification_sent_at' => $throttle->sent_at]);
+ }
+ });
+
+ Schema::dropIfExists('notification_throttles');
+ }
+};
diff --git a/openapi.json b/openapi.json
index 512f94b9e6..8087d32199 100644
--- a/openapi.json
+++ b/openapi.json
@@ -3531,7 +3531,7 @@
"Applications"
],
"summary": "Get application logs.",
- "description": "Get application logs by UUID.",
+ "description": "Get application logs by UUID. Requires the `read:sensitive` or `root` token ability.",
"operationId": "get-application-logs-by-uuid",
"parameters": [
{
@@ -3616,7 +3616,7 @@
"Applications"
],
"summary": "Get preview application logs.",
- "description": "Get runtime container logs for a preview deployment by application UUID and pull request ID.",
+ "description": "Get runtime container logs for a preview deployment by application UUID and pull request ID. Requires the `read:sensitive` or `root` token ability.",
"operationId": "get-preview-application-logs-by-pull-request-id",
"parameters": [
{
@@ -8293,7 +8293,7 @@
"Databases"
],
"summary": "Get database logs.",
- "description": "Get database logs by UUID.",
+ "description": "Get database logs by UUID. Requires the `read:sensitive` or `root` token ability.",
"operationId": "get-database-logs-by-uuid",
"parameters": [
{
@@ -16766,6 +16766,10 @@
"type": "string",
"description": "Cron expression for disk usage check frequency."
},
+ "server_disk_usage_notification_interval_hours": {
+ "type": "integer",
+ "description": "Minimum hours between high disk usage notifications (1-720). Default: 24."
+ },
"connection_timeout": {
"type": "integer",
"description": "SSH connection timeout in seconds (1-300). Default: 10."
@@ -17265,7 +17269,7 @@
"Service applications"
],
"summary": "Get service application logs",
- "description": "Get Docker logs for a single compose service container.",
+ "description": "Get Docker logs for a single compose service container. Requires the `read:sensitive` or `root` token ability.",
"operationId": "get-service-application-logs-by-service-and-app-uuid",
"parameters": [
{
@@ -17350,7 +17354,7 @@
"Service applications"
],
"summary": "Get service application logs",
- "description": "Get Docker logs for a single compose service container.",
+ "description": "Get Docker logs for a single compose service container. Requires the `read:sensitive` or `root` token ability.",
"operationId": "post-service-application-logs-by-service-and-app-uuid",
"parameters": [
{
@@ -18002,7 +18006,7 @@
"Service databases"
],
"summary": "Get service database logs",
- "description": "Get Docker logs for a compose database container.",
+ "description": "Get Docker logs for a compose database container. Requires the `read:sensitive` or `root` token ability.",
"operationId": "get-service-database-logs-by-service-and-database-uuid",
"parameters": [
{
@@ -18847,7 +18851,7 @@
"Services"
],
"summary": "Get service logs.",
- "description": "Get logs for a specific service sub-resource by service UUID. The `sub_service_name` query parameter must match the `name` field of one of the service applications or databases returned by `GET \/services\/{uuid}`.",
+ "description": "Get logs for a specific service sub-resource by service UUID. The `sub_service_name` query parameter must match the `name` field of one of the service applications or databases returned by `GET \/services\/{uuid}`. Requires the `read:sensitive` or `root` token ability.",
"operationId": "get-service-logs-by-uuid",
"parameters": [
{
diff --git a/openapi.yaml b/openapi.yaml
index e6261818b1..a78f236afc 100644
--- a/openapi.yaml
+++ b/openapi.yaml
@@ -2325,7 +2325,7 @@ paths:
tags:
- Applications
summary: 'Get application logs.'
- description: 'Get application logs by UUID.'
+ description: 'Get application logs by UUID. Requires the `read:sensitive` or `root` token ability.'
operationId: get-application-logs-by-uuid
parameters:
-
@@ -2383,7 +2383,7 @@ paths:
tags:
- Applications
summary: 'Get preview application logs.'
- description: 'Get runtime container logs for a preview deployment by application UUID and pull request ID.'
+ description: 'Get runtime container logs for a preview deployment by application UUID and pull request ID. Requires the `read:sensitive` or `root` token ability.'
operationId: get-preview-application-logs-by-pull-request-id
parameters:
-
@@ -5469,7 +5469,7 @@ paths:
tags:
- Databases
summary: 'Get database logs.'
- description: 'Get database logs by UUID.'
+ description: 'Get database logs by UUID. Requires the `read:sensitive` or `root` token ability.'
operationId: get-database-logs-by-uuid
parameters:
-
@@ -10647,6 +10647,9 @@ paths:
server_disk_usage_check_frequency:
type: string
description: 'Cron expression for disk usage check frequency.'
+ server_disk_usage_notification_interval_hours:
+ type: integer
+ description: 'Minimum hours between high disk usage notifications (1-720). Default: 24.'
connection_timeout:
type: integer
description: 'SSH connection timeout in seconds (1-300). Default: 10.'
@@ -10931,7 +10934,7 @@ paths:
tags:
- 'Service applications'
summary: 'Get service application logs'
- description: 'Get Docker logs for a single compose service container.'
+ description: 'Get Docker logs for a single compose service container. Requires the `read:sensitive` or `root` token ability.'
operationId: get-service-application-logs-by-service-and-app-uuid
parameters:
-
@@ -10989,7 +10992,7 @@ paths:
tags:
- 'Service applications'
summary: 'Get service application logs'
- description: 'Get Docker logs for a single compose service container.'
+ description: 'Get Docker logs for a single compose service container. Requires the `read:sensitive` or `root` token ability.'
operationId: post-service-application-logs-by-service-and-app-uuid
parameters:
-
@@ -11418,7 +11421,7 @@ paths:
tags:
- 'Service databases'
summary: 'Get service database logs'
- description: 'Get Docker logs for a compose database container.'
+ description: 'Get Docker logs for a compose database container. Requires the `read:sensitive` or `root` token ability.'
operationId: get-service-database-logs-by-service-and-database-uuid
parameters:
-
@@ -11912,7 +11915,7 @@ paths:
tags:
- Services
summary: 'Get service logs.'
- description: 'Get logs for a specific service sub-resource by service UUID. The `sub_service_name` query parameter must match the `name` field of one of the service applications or databases returned by `GET /services/{uuid}`.'
+ description: 'Get logs for a specific service sub-resource by service UUID. The `sub_service_name` query parameter must match the `name` field of one of the service applications or databases returned by `GET /services/{uuid}`. Requires the `read:sensitive` or `root` token ability.'
operationId: get-service-logs-by-uuid
parameters:
-
diff --git a/resources/views/livewire/server/advanced.blade.php b/resources/views/livewire/server/advanced.blade.php
index 48f63f02e9..a7244869f3 100644
--- a/resources/views/livewire/server/advanced.blade.php
+++ b/resources/views/livewire/server/advanced.blade.php
@@ -14,7 +14,7 @@
<x-application.settings-section id="server-disk-usage-section" title="Disk usage"
helper="Control when Coolify checks this server and when your team is notified.">
- <div class="grid gap-4 lg:grid-cols-2">
+ <div class="grid gap-4 lg:grid-cols-3">
<x-forms.input canGate="update" :canResource="$server" placeholder="0 23 * * *"
id="serverDiskUsageCheckFrequency" label="Check frequency" required
helper="Cron expression or preset such as hourly, daily, weekly, monthly, or yearly." />
@@ -22,6 +22,10 @@
id="serverDiskUsageNotificationThreshold" type="number" min="1" max="99"
label="Notification threshold" required
helper="Notify the team when root filesystem usage exceeds this percentage." />
+ <x-forms.input canGate="update" :canResource="$server"
+ id="serverDiskUsageNotificationIntervalHours" type="number" min="1" max="720"
+ label="Notification interval (hours)" required
+ helper="Minimum time between high disk usage notifications for this server." />
</div>
</x-application.settings-section>
diff --git a/tests/Feature/Api/LifecycleApisTest.php b/tests/Feature/Api/LifecycleApisTest.php
index 95004550b8..d89dcf414d 100644
--- a/tests/Feature/Api/LifecycleApisTest.php
+++ b/tests/Feature/Api/LifecycleApisTest.php
@@ -7,6 +7,7 @@ use App\Models\CloudInitScript;
use App\Models\Environment;
use App\Models\InstanceSettings;
use App\Models\LocalPersistentVolume;
+use App\Models\NotificationThrottle;
use App\Models\Project;
use App\Models\ScheduledTask;
use App\Models\Server;
@@ -15,6 +16,7 @@ use App\Models\StandaloneDocker;
use App\Models\StandalonePostgresql;
use App\Models\Team;
use App\Models\User;
+use App\Notifications\Database\BackupMissing;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Bus;
use Illuminate\Support\Facades\Queue;
@@ -144,10 +146,8 @@ describe('POST /api/v1/databases/{uuid}/clone', function () {
'frequency' => '0 0 * * *',
'save_s3' => false,
]);
- $backup->forceFill([
- 'last_execution_at' => now()->subDay(),
- 'missing_backup_notification_sent_at' => now(),
- ])->save();
+ $backup->forceFill(['last_execution_at' => now()->subDay()])->save();
+ NotificationThrottle::record($backup, BackupMissing::class);
$response = $this->withHeaders($this->headers)
->postJson("/api/v1/databases/{$database->uuid}/clone", [
@@ -166,7 +166,7 @@ describe('POST /api/v1/databases/{uuid}/clone', function () {
->and($cloned->destination_id)->toBe($this->destination->id)
->and(str($cloned->status)->startsWith('exited'))->toBeTrue()
->and($clonedBackup->last_execution_at)->toBeNull()
- ->and($clonedBackup->missing_backup_notification_sent_at)->toBeNull();
+ ->and(NotificationThrottle::wasSent($clonedBackup, BackupMissing::class))->toBeFalse();
});
test('creates renamed volumes when cloning a database with clone_volumes', function () {
diff --git a/tests/Feature/CleanupUnreachableServersTest.php b/tests/Feature/CleanupUnreachableServersTest.php
index 8849b1ca01..2c58946891 100644
--- a/tests/Feature/CleanupUnreachableServersTest.php
+++ b/tests/Feature/CleanupUnreachableServersTest.php
@@ -1,7 +1,9 @@
<?php
+use App\Models\NotificationThrottle;
use App\Models\Server;
use App\Models\Team;
+use App\Notifications\Server\Unreachable;
use Illuminate\Foundation\Testing\RefreshDatabase;
uses(RefreshDatabase::class);
@@ -13,9 +15,9 @@ it('disables (non-destructively) self-hosted servers with unreachable_count >= 3
$server = Server::factory()->create([
'team_id' => $team->id,
'unreachable_count' => 50,
- 'unreachable_notification_sent' => true,
'updated_at' => now()->subDays(8),
]);
+ NotificationThrottle::record($server, Unreachable::class);
$originalIp = (string) $server->ip;
@@ -34,9 +36,9 @@ it('overwrites the IP with 1.2.3.4 on cloud for servers with unreachable_count >
$server = Server::factory()->create([
'team_id' => $team->id,
'unreachable_count' => 50,
- 'unreachable_notification_sent' => true,
'updated_at' => now()->subDays(8),
]);
+ NotificationThrottle::record($server, Unreachable::class);
$this->artisan('cleanup:unreachable-servers')->assertSuccessful();
@@ -49,9 +51,9 @@ it('does not clean up servers with unreachable_count less than 3', function () {
$server = Server::factory()->create([
'team_id' => $team->id,
'unreachable_count' => 2,
- 'unreachable_notification_sent' => true,
'updated_at' => now()->subDays(8),
]);
+ NotificationThrottle::record($server, Unreachable::class);
$originalIp = (string) $server->ip;
@@ -67,9 +69,9 @@ it('does not clean up servers updated within 7 days', function () {
$server = Server::factory()->create([
'team_id' => $team->id,
'unreachable_count' => 10,
- 'unreachable_notification_sent' => true,
'updated_at' => now()->subDays(3),
]);
+ NotificationThrottle::record($server, Unreachable::class);
$originalIp = (string) $server->ip;
@@ -85,7 +87,6 @@ it('does not clean up servers without notification sent', function () {
$server = Server::factory()->create([
'team_id' => $team->id,
'unreachable_count' => 10,
- 'unreachable_notification_sent' => false,
'updated_at' => now()->subDays(8),
]);
diff --git a/tests/Feature/CleanupUnsubscribedServersTest.php b/tests/Feature/CleanupUnsubscribedServersTest.php
index 53b0160d8a..edf6c63a0b 100644
--- a/tests/Feature/CleanupUnsubscribedServersTest.php
+++ b/tests/Feature/CleanupUnsubscribedServersTest.php
@@ -1,8 +1,10 @@
<?php
+use App\Models\NotificationThrottle;
use App\Models\Server;
use App\Models\Subscription;
use App\Models\Team;
+use App\Notifications\Server\Unreachable;
use Illuminate\Foundation\Testing\RefreshDatabase;
uses(RefreshDatabase::class);
@@ -16,7 +18,6 @@ it('sets unreachable fields on servers when subscription ends', function () {
$server = Server::factory()->create([
'team_id' => $team->id,
'unreachable_count' => 0,
- 'unreachable_notification_sent' => false,
]);
$team->subscriptionEnded();
@@ -34,9 +35,9 @@ it('cleans up unsubscribed server IP after 7 days via cleanup command', function
$server = Server::factory()->create([
'team_id' => $team->id,
'unreachable_count' => 3,
- 'unreachable_notification_sent' => true,
'updated_at' => now()->subDays(8),
]);
+ NotificationThrottle::record($server, Unreachable::class);
$this->artisan('cleanup:unreachable-servers')->assertSuccessful();
@@ -49,9 +50,9 @@ it('does not clean up unsubscribed server IP within 7 day grace period', functio
$server = Server::factory()->create([
'team_id' => $team->id,
'unreachable_count' => 3,
- 'unreachable_notification_sent' => true,
'updated_at' => now()->subDays(3),
]);
+ NotificationThrottle::record($server, Unreachable::class);
$originalIp = (string) $server->ip;
@@ -70,7 +71,6 @@ it('does not affect servers with active subscriptions', function () {
$server = Server::factory()->create([
'team_id' => $team->id,
'unreachable_count' => 0,
- 'unreachable_notification_sent' => false,
]);
$originalCount = $server->unreachable_count;
diff --git a/tests/Feature/MissingDatabaseBackupNotificationTest.php b/tests/Feature/MissingDatabaseBackupNotificationTest.php
index f4f2efd191..9ec50feae3 100644
--- a/tests/Feature/MissingDatabaseBackupNotificationTest.php
+++ b/tests/Feature/MissingDatabaseBackupNotificationTest.php
@@ -1,6 +1,7 @@
<?php
use App\Jobs\CheckMissingDatabaseBackupsJob;
+use App\Models\NotificationThrottle;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledDatabaseBackupExecution;
use App\Models\Team;
@@ -49,7 +50,7 @@ it('notifies the team when an enabled backup has no executions for the configure
(new CheckMissingDatabaseBackupsJob)->handle();
Notification::assertSentTo($team, BackupMissing::class, fn (BackupMissing $notification) => $notification->backup->is($backup));
- expect($backup->fresh()->missing_backup_notification_sent_at)->not->toBeNull();
+ expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeTrue();
});
it('does not notify for recent disabled or unconfigured backup schedules', function () {
@@ -114,7 +115,7 @@ it('waits to mark an incident sent until a notification channel is enabled', fun
$backup = missingBackupSchedule($team);
(new CheckMissingDatabaseBackupsJob)->handle();
- expect($backup->fresh()->missing_backup_notification_sent_at)->toBeNull();
+ expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeFalse();
$team->emailNotificationSettings->update([
'smtp_enabled' => true,
diff --git a/tests/Feature/NotificationThrottleTest.php b/tests/Feature/NotificationThrottleTest.php
new file mode 100644
index 0000000000..b6d3d881f0
--- /dev/null
+++ b/tests/Feature/NotificationThrottleTest.php
@@ -0,0 +1,82 @@
+<?php
+
+use App\Models\InstanceSettings;
+use App\Models\NotificationThrottle;
+use App\Models\Server;
+use App\Models\Team;
+use App\Models\User;
+use App\Notifications\Server\HighDiskUsage;
+use App\Notifications\Server\Unreachable;
+use Carbon\Carbon;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+ Server::flushIdentityMap();
+ $this->team = Team::factory()->create();
+ $this->server = Server::factory()->create(['team_id' => $this->team->id]);
+});
+
+afterEach(function () {
+ Carbon::setTestNow();
+});
+
+it('claims a notification only once until it is released', function () {
+ expect(NotificationThrottle::claim($this->server, Unreachable::class))->toBeTrue()
+ ->and(NotificationThrottle::claim($this->server, Unreachable::class))->toBeFalse()
+ ->and($this->server->unreachable_notification_sent)->toBeTrue()
+ ->and(NotificationThrottle::release($this->server, Unreachable::class))->toBeTrue()
+ ->and(NotificationThrottle::release($this->server, Unreachable::class))->toBeFalse()
+ ->and($this->server->unreachable_notification_sent)->toBeFalse()
+ ->and(NotificationThrottle::claim($this->server, Unreachable::class))->toBeTrue();
+});
+
+it('claims a notification again only when it was sent before the given time', function () {
+ Carbon::setTestNow(Carbon::parse('2026-09-29 12:00:00', 'UTC'));
+
+ expect(NotificationThrottle::claim($this->server, HighDiskUsage::class, now()->subHours(6)))->toBeTrue();
+
+ Carbon::setTestNow(now()->addHours(5));
+ expect(NotificationThrottle::claim($this->server, HighDiskUsage::class, now()->subHours(6)))->toBeFalse();
+
+ Carbon::setTestNow(now()->addHours(2));
+ expect(NotificationThrottle::claim($this->server, HighDiskUsage::class, now()->subHours(6)))->toBeTrue()
+ ->and(NotificationThrottle::query()->count())->toBe(1);
+});
+
+it('keeps throttles separate for each notification and resource', function () {
+ $otherServer = Server::factory()->create(['team_id' => $this->team->id]);
+
+ NotificationThrottle::record($this->server, Unreachable::class);
+
+ expect(NotificationThrottle::claim($this->server, HighDiskUsage::class))->toBeTrue()
+ ->and(NotificationThrottle::claim($otherServer, Unreachable::class))->toBeTrue()
+ ->and(NotificationThrottle::claim($this->server, Unreachable::class))->toBeFalse();
+});
+
+it('removes the throttles of a server when the server is force deleted', function () {
+ NotificationThrottle::record($this->server, Unreachable::class);
+
+ $this->server->forceDelete();
+
+ expect(NotificationThrottle::query()->count())->toBe(0);
+});
+
+it('keeps unreachable_notification_sent in the server API response', function () {
+ config()->set('app.maintenance.driver', 'file');
+ config()->set('cache.default', 'array');
+ InstanceSettings::forceCreate(['id' => 0, 'is_api_enabled' => true]);
+ $user = User::factory()->create();
+ $this->team->members()->attach($user->id, ['role' => 'owner']);
+ session(['currentTeam' => $this->team]);
+ $token = $user->createToken('read-token', ['read']);
+ $token->accessToken->forceFill(['team_id' => $this->team->id])->save();
+
+ NotificationThrottle::record($this->server, Unreachable::class);
+
+ $this->withHeaders(['Authorization' => 'Bearer '.$token->plainTextToken])
+ ->getJson('/api/v1/servers/'.$this->server->uuid)
+ ->assertOk()
+ ->assertJsonPath('unreachable_notification_sent', true);
+});
diff --git a/tests/Feature/NotificationThrottlesMigrationTest.php b/tests/Feature/NotificationThrottlesMigrationTest.php
new file mode 100644
index 0000000000..04a641205d
--- /dev/null
+++ b/tests/Feature/NotificationThrottlesMigrationTest.php
@@ -0,0 +1,68 @@
+<?php
+
+use App\Models\NotificationThrottle;
+use App\Models\ScheduledDatabaseBackup;
+use App\Models\Server;
+use App\Models\Team;
+use App\Notifications\Database\BackupMissing;
+use App\Notifications\Server\Unreachable;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Schema;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+ Server::flushIdentityMap();
+});
+
+it('moves the old notification markers into notification_throttles and back', function () {
+ $team = Team::factory()->create();
+ $localhost = Server::factory()->create(['id' => 0, 'team_id' => $team->id]);
+ $notifiedServer = Server::factory()->create(['team_id' => $team->id]);
+ $quietServer = Server::factory()->create(['team_id' => $team->id]);
+ $notifiedBackup = ScheduledDatabaseBackup::create([
+ 'enabled' => true,
+ 'frequency' => '0 0 * * *',
+ 'save_s3' => false,
+ 'database_type' => 'App\\Models\\StandalonePostgresql',
+ 'database_id' => 999,
+ 'team_id' => $team->id,
+ ]);
+ $quietBackup = $notifiedBackup->replicate(['uuid']);
+ $quietBackup->save();
+
+ $migration = require database_path('migrations/2026_09_29_083244_create_notification_throttles_table.php');
+ $migration->down();
+
+ DB::table('servers')->whereIn('id', [$localhost->id, $notifiedServer->id])->update(['unreachable_notification_sent' => true]);
+ DB::table('scheduled_database_backups')->where('id', $notifiedBackup->id)->update(['missing_backup_notification_sent_at' => '2026-09-01 10:00:00']);
+
+ $migration->up();
+
+ expect(Schema::hasColumn('servers', 'unreachable_notification_sent'))->toBeFalse()
+ ->and(Schema::hasColumn('scheduled_database_backups', 'missing_backup_notification_sent_at'))->toBeFalse()
+ ->and(NotificationThrottle::query()->count())->toBe(3)
+ ->and(NotificationThrottle::wasSent($localhost, Unreachable::class))->toBeTrue()
+ ->and(NotificationThrottle::wasSent($notifiedServer, Unreachable::class))->toBeTrue()
+ ->and(NotificationThrottle::wasSent($quietServer, Unreachable::class))->toBeFalse()
+ ->and(NotificationThrottle::wasSent($quietBackup, BackupMissing::class))->toBeFalse()
+ ->and(NotificationThrottle::query()
+ ->whereMorphedTo('notifiable', $notifiedBackup)
+ ->where('notification', BackupMissing::class)
+ ->sole()
+ ->sent_at
+ ->toDateTimeString())->toBe('2026-09-01 10:00:00');
+
+ $migration->down();
+
+ expect(Schema::hasTable('notification_throttles'))->toBeFalse()
+ ->and(DB::table('servers')->where('unreachable_notification_sent', true)->pluck('id')->sort()->values()->all())
+ ->toBe([$localhost->id, $notifiedServer->id])
+ ->and(DB::table('scheduled_database_backups')->where('id', $notifiedBackup->id)->value('missing_backup_notification_sent_at'))
+ ->toStartWith('2026-09-01 10:00:00')
+ ->and(DB::table('scheduled_database_backups')->where('id', $quietBackup->id)->value('missing_backup_notification_sent_at'))
+ ->toBeNull();
+
+ $migration->up();
+});
diff --git a/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php b/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php
new file mode 100644
index 0000000000..ff0caf1a98
--- /dev/null
+++ b/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php
@@ -0,0 +1,122 @@
+<?php
+
+use App\Jobs\ServerStorageCheckJob;
+use App\Livewire\Server\Advanced;
+use App\Models\InstanceSettings;
+use App\Models\NotificationThrottle;
+use App\Models\Server;
+use App\Models\Team;
+use App\Models\User;
+use App\Notifications\Server\HighDiskUsage;
+use Carbon\Carbon;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Notification;
+use Livewire\Livewire;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+ Server::flushIdentityMap();
+ Notification::fake();
+ InstanceSettings::forceCreate(['id' => 0]);
+ Carbon::setTestNow(Carbon::parse('2026-09-29 12:00:00', 'UTC'));
+});
+
+afterEach(function () {
+ Carbon::setTestNow();
+});
+
+function diskUsageIntervalServer(int $intervalHours = 24): Server
+{
+ $team = Team::factory()->create();
+ $team->emailNotificationSettings()->update([
+ 'use_instance_email_settings' => true,
+ 'server_disk_usage_email_notifications' => true,
+ ]);
+ $server = Server::factory()->create(['team_id' => $team->id, 'ip' => '10.0.0.10']);
+ $server->settings->update([
+ 'is_reachable' => true,
+ 'is_usable' => true,
+ 'force_disabled' => false,
+ 'server_disk_usage_notification_threshold' => 80,
+ 'server_disk_usage_notification_interval_hours' => $intervalHours,
+ ]);
+
+ return $server->fresh();
+}
+
+function runDiskUsageCheck(Server $server, int $percentage): void
+{
+ (new ServerStorageCheckJob($server->fresh(), $percentage))->handle();
+}
+
+it('sends one high disk usage notification per configured interval', function () {
+ $server = diskUsageIntervalServer(intervalHours: 6);
+
+ runDiskUsageCheck($server, 93);
+ Carbon::setTestNow(now()->addMinutes(10));
+ runDiskUsageCheck($server, 93);
+ Carbon::setTestNow(now()->addHours(5));
+ runDiskUsageCheck($server, 94);
+
+ Notification::assertSentToTimes($server->team, HighDiskUsage::class, 1);
+
+ Carbon::setTestNow(now()->addMinutes(51));
+ runDiskUsageCheck($server, 94);
+
+ Notification::assertSentToTimes($server->team, HighDiskUsage::class, 2);
+});
+
+it('defaults to one notification per 24 hours', function () {
+ $team = Team::factory()->create();
+ $server = Server::factory()->create(['team_id' => $team->id]);
+
+ expect($server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(24);
+});
+
+it('does not notify below the threshold', function () {
+ $server = diskUsageIntervalServer();
+
+ runDiskUsageCheck($server, 50);
+
+ Notification::assertNothingSent();
+ expect(NotificationThrottle::wasSent($server, HighDiskUsage::class))->toBeFalse();
+});
+
+it('saves the notification interval from the server advanced settings', function () {
+ $team = Team::factory()->create();
+ $user = User::factory()->create();
+ $team->members()->attach($user, ['role' => 'owner']);
+ $server = Server::factory()->create(['team_id' => $team->id]);
+
+ $this->actingAs($user);
+ session(['currentTeam' => $team]);
+
+ Livewire::test(Advanced::class, ['server_uuid' => $server->uuid])
+ ->assertSet('serverDiskUsageNotificationIntervalHours', 24)
+ ->set('serverDiskUsageNotificationIntervalHours', 0)
+ ->call('submit')
+ ->assertHasErrors(['serverDiskUsageNotificationIntervalHours'])
+ ->set('serverDiskUsageNotificationIntervalHours', 12)
+ ->call('submit')
+ ->assertHasNoErrors()
+ ->assertDispatched('success');
+
+ expect($server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(12);
+});
+
+it('does not let a member change the notification interval', function () {
+ $team = Team::factory()->create();
+ $user = User::factory()->create();
+ $team->members()->attach($user, ['role' => 'member']);
+ $server = Server::factory()->create(['team_id' => $team->id]);
+
+ $this->actingAs($user);
+ session(['currentTeam' => $team]);
+
+ Livewire::test(Advanced::class, ['server_uuid' => $server->uuid])
+ ->set('serverDiskUsageNotificationIntervalHours', 2)
+ ->call('submit');
+
+ expect($server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(24);
+});
diff --git a/tests/Feature/ServerReachabilityNotificationTest.php b/tests/Feature/ServerReachabilityNotificationTest.php
index e996ba0286..4fdd8df86c 100644
--- a/tests/Feature/ServerReachabilityNotificationTest.php
+++ b/tests/Feature/ServerReachabilityNotificationTest.php
@@ -1,6 +1,7 @@
<?php
use App\Events\ServerReachabilityChanged;
+use App\Models\NotificationThrottle;
use App\Models\Server;
use App\Models\Team;
use App\Notifications\Channels\EmailChannel;
@@ -28,7 +29,6 @@ it('sends Unreachable notification when threshold reached and not yet notified',
$this->server->settings()->update(['is_reachable' => false]);
$this->server->forceFill([
'unreachable_count' => 2,
- 'unreachable_notification_sent' => false,
])->save();
ServerReachabilityChanged::dispatch($this->server->fresh());
@@ -41,7 +41,6 @@ it('does not send Unreachable on first transient failure (count=1)', function ()
$this->server->settings()->update(['is_reachable' => false]);
$this->server->forceFill([
'unreachable_count' => 1,
- 'unreachable_notification_sent' => false,
])->save();
ServerReachabilityChanged::dispatch($this->server->fresh());
@@ -53,8 +52,8 @@ it('does not send Unreachable when already notified', function () {
$this->server->settings()->update(['is_reachable' => false]);
$this->server->forceFill([
'unreachable_count' => 5,
- 'unreachable_notification_sent' => true,
])->save();
+ NotificationThrottle::record($this->server, Unreachable::class);
ServerReachabilityChanged::dispatch($this->server->fresh());
@@ -65,8 +64,8 @@ it('sends Reachable notification on recovery when previously notified', function
$this->server->settings()->update(['is_reachable' => true]);
$this->server->forceFill([
'unreachable_count' => 0,
- 'unreachable_notification_sent' => true,
])->save();
+ NotificationThrottle::record($this->server, Unreachable::class);
$fresh = $this->server->fresh();
expect($fresh->unreachable_notification_sent)->toBeTrue();
@@ -82,7 +81,6 @@ it('does not send Reachable when never notified', function () {
$this->server->settings()->update(['is_reachable' => true]);
$this->server->forceFill([
'unreachable_count' => 0,
- 'unreachable_notification_sent' => false,
])->save();
ServerReachabilityChanged::dispatch($this->server->fresh());
@@ -94,7 +92,6 @@ it('routes Unreachable notification through EmailChannel when email toggle is on
$this->server->settings()->update(['is_reachable' => false]);
$this->server->forceFill([
'unreachable_count' => 2,
- 'unreachable_notification_sent' => false,
])->save();
ServerReachabilityChanged::dispatch($this->server->fresh());
diff --git a/tests/Feature/ServerUpdatePrivateKeyApiTest.php b/tests/Feature/ServerUpdatePrivateKeyApiTest.php
index 7940c0d9a1..ae567d2da7 100644
--- a/tests/Feature/ServerUpdatePrivateKeyApiTest.php
+++ b/tests/Feature/ServerUpdatePrivateKeyApiTest.php
@@ -227,3 +227,20 @@ it('rejects an invalid disk usage check frequency without partially updating the
->and($server->settings->server_role->value)->toBe('both')
->and($server->proxy->get('type'))->toBe('TRAEFIK');
});
+
+it('updates the disk usage notification interval', function () {
+ patchServerUpdatePrivateKeyApi($this, $this->server, $this->bearerToken, [
+ 'server_disk_usage_notification_interval_hours' => 48,
+ ])->assertCreated();
+
+ expect($this->server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(48);
+});
+
+it('rejects an out of range disk usage notification interval', function () {
+ patchServerUpdatePrivateKeyApi($this, $this->server, $this->bearerToken, [
+ 'server_disk_usage_notification_interval_hours' => 0,
+ ])->assertUnprocessable()
+ ->assertJsonValidationErrors('server_disk_usage_notification_interval_hours');
+
+ expect($this->server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(24);
+});
diff --git a/tests/Feature/Subscription/SyncStripeSubscriptionsActionTest.php b/tests/Feature/Subscription/SyncStripeSubscriptionsActionTest.php
index 31070053d6..a59ae659de 100644
--- a/tests/Feature/Subscription/SyncStripeSubscriptionsActionTest.php
+++ b/tests/Feature/Subscription/SyncStripeSubscriptionsActionTest.php
@@ -42,7 +42,6 @@ test('fix mode fully ends a locally active subscription with an invalid Stripe s
$server = Server::factory()->create([
'team_id' => $this->team->id,
'unreachable_count' => 0,
- 'unreachable_notification_sent' => false,
]);
$stripe = Mockery::mock(StripeClient::class);
diff --git a/tests/Feature/ThrottledNotificationTest.php b/tests/Feature/ThrottledNotificationTest.php
new file mode 100644
index 0000000000..1528b6ce55
--- /dev/null
+++ b/tests/Feature/ThrottledNotificationTest.php
@@ -0,0 +1,102 @@
+<?php
+
+use App\Jobs\PushServerUpdateJob;
+use App\Models\Environment;
+use App\Models\Project;
+use App\Models\Server;
+use App\Models\StandalonePostgresql;
+use App\Models\Team;
+use App\Notifications\Container\ContainerRestarted;
+use Carbon\Carbon;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Notification;
+use Illuminate\Support\Facades\Queue;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+ Server::flushIdentityMap();
+ Cache::flush();
+ Queue::fake();
+ Notification::fake();
+ Carbon::setTestNow(Carbon::parse('2026-09-29 12:00:00', 'UTC'));
+
+ $this->team = Team::factory()->create();
+ $this->team->emailNotificationSettings()->update([
+ 'use_instance_email_settings' => true,
+ 'status_change_email_notifications' => true,
+ ]);
+ $this->server = Server::factory()->create(['team_id' => $this->team->id]);
+});
+
+afterEach(function () {
+ Carbon::setTestNow();
+});
+
+it('sends a throttled notification once per interval for the same resource', function () {
+ $otherServer = Server::factory()->create(['team_id' => $this->team->id]);
+
+ $this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
+ $this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
+ $this->team->notify(new ContainerRestarted('coolify-proxy', $otherServer, restartedResource: $otherServer));
+
+ Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2);
+
+ Carbon::setTestNow(now()->addMinutes(59));
+ $this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
+ Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2);
+
+ Carbon::setTestNow(now()->addMinutes(2));
+ $this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
+ Notification::assertSentToTimes($this->team, ContainerRestarted::class, 3);
+});
+
+it('does not throttle a notification without a throttle subject', function () {
+ $this->team->notify(new ContainerRestarted('TCP Proxy for db has been disabled', $this->server));
+ $this->team->notify(new ContainerRestarted('TCP Proxy for db has been disabled', $this->server));
+
+ Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2);
+});
+
+it('sends one TCP proxy restart notification per hour while the proxy does not stay up', function () {
+ $project = Project::factory()->create(['team_id' => $this->team->id]);
+ $environment = Environment::factory()->create(['project_id' => $project->id]);
+ $destination = $this->server->standaloneDockers()->firstOrFail();
+ $database = StandalonePostgresql::create([
+ 'name' => 'public-db',
+ 'postgres_user' => 'postgres',
+ 'postgres_password' => encrypt('password'),
+ 'postgres_db' => 'app',
+ 'image' => 'postgres:16-alpine',
+ 'is_public' => true,
+ 'public_port' => 54321,
+ 'environment_id' => $environment->id,
+ 'destination_id' => $destination->id,
+ 'destination_type' => $destination->getMorphClass(),
+ ]);
+
+ // The database runs, but its "-proxy" container is missing on every push.
+ $push = fn () => (new PushServerUpdateJob($this->server, [
+ 'containers' => [[
+ 'name' => $database->uuid,
+ 'state' => 'running',
+ 'health_status' => 'healthy',
+ 'labels' => ['coolify.managed' => true, 'com.docker.compose.service' => $database->uuid],
+ ]],
+ 'filesystem_usage_root' => ['used_percentage' => 10],
+ ]))->handle();
+
+ $push();
+ Carbon::setTestNow(now()->addMinute());
+ $push();
+ Carbon::setTestNow(now()->addMinute());
+ $push();
+
+ Notification::assertSentToTimes($this->team, ContainerRestarted::class, 1);
+
+ Carbon::setTestNow(now()->addHour());
+ $push();
+
+ Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2);
+});