An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(notifications): throttle repeated notifications per resource
Andras Bacsai99c840922c3d
diff --git a/.ai/lessons.md b/.ai/lessons.mdindex d68d184014..eff6d13e5c 100644--- a/.ai/lessons.md+++ b/.ai/lessons.md@@ -5,6 +5,8 @@- When a symptom matches an earlier fix, inspect that fix and prove why it no longer works before adding another workaround.- Test old reports against the current branch because later changes can make the report obsolete.- Use the same regression test before and after the production change so the result shows the behavior difference.+- Several dev instances can run from one checkout (see `./scripts/dev urls`). After you add a migration, run it on each running instance of that checkout, or those instances fail on the new code.+- The checkout is shared with other sessions. When the full suite fails, rerun each failing file alone and compare with a clean `git archive HEAD` copy before you connect a failure to your change.- Redirect browser test output to a file (`> /tmp/x.log 2>&1`); piping it (`| tail`) hangs because the Playwright server keeps the pipe open.- Call `visit()` directly in each `tests/v4/Browser` test body; Pest does not mark a test that only uses helper-wrapped `visit()` as a browser test, so it fails with `sendText() on null`.diff --git a/app/Console/Commands/CleanupUnreachableServers.php b/app/Console/Commands/CleanupUnreachableServers.phpindex 666e98a182..615dc21d20 100644--- a/app/Console/Commands/CleanupUnreachableServers.php+++ b/app/Console/Commands/CleanupUnreachableServers.php@@ -3,6 +3,7 @@namespace App\Console\Commands;use App\Models\Server;+use App\Notifications\Server\Unreachable;use Illuminate\Console\Command;class CleanupUnreachableServers extends Command@@ -14,7 +15,7 @@ class CleanupUnreachableServers extends Commandpublic function handle(){echo "Running unreachable server cleanup...\n";- $servers = Server::where('unreachable_count', '>=', 3)->where('unreachable_notification_sent', true)->where('updated_at', '<', now()->subDays(7))->get();+ $servers = Server::where('unreachable_count', '>=', 3)->whereHas('notificationThrottles', fn ($query) => $query->where('notification', Unreachable::class))->where('updated_at', '<', now()->subDays(7))->get();if ($servers->count() > 0) {foreach ($servers as $server) {echo "Cleanup unreachable server ($server->id) with name $server->name";diff --git a/app/Contracts/ThrottledNotification.php b/app/Contracts/ThrottledNotification.phpnew file mode 100644index 0000000000..b578f75835--- /dev/null+++ b/app/Contracts/ThrottledNotification.php@@ -0,0 +1,19 @@+<?php++namespace App\Contracts;++use Illuminate\Database\Eloquent\Model;++/**+ * A notification that is sent at most once per interval for the same resource.+ * Team::notify() checks the throttle before it sends the notification.+ */+interface ThrottledNotification+{+ /**+ * The resource the throttle belongs to. Return null to send without a throttle.+ */+ public function throttleSubject(): ?Model;++ public function throttleIntervalMinutes(): int;+}diff --git a/app/Http/Controllers/Api/DatabasesController.php b/app/Http/Controllers/Api/DatabasesController.phpindex 81035eca03..3c9c832d31 100644--- a/app/Http/Controllers/Api/DatabasesController.php+++ b/app/Http/Controllers/Api/DatabasesController.php@@ -5119,7 +5119,6 @@ class DatabasesController extends Controller'created_at','updated_at','last_execution_at',- 'missing_backup_notification_sent_at',])->fill(['uuid' => new_public_id(),'database_id' => $newDatabase->id,diff --git a/app/Http/Controllers/Api/ServersController.php b/app/Http/Controllers/Api/ServersController.phpindex 1a5d24b194..ef69f6b424 100644--- a/app/Http/Controllers/Api/ServersController.php+++ b/app/Http/Controllers/Api/ServersController.php@@ -187,6 +187,7 @@ class ServersController extends Controller$server->load(['settings']);}+ $server->append('unreachable_notification_sent');$settings = $this->removeSensitiveDataFromSettings($server->settings);$server = $this->removeSensitiveData($server);data_set($server, 'settings', $settings);@@ -645,6 +646,7 @@ class ServersController extends Controller'deployment_queue_limit' => ['type' => 'integer', 'description' => 'Maximum number of queued deployments.'],'server_disk_usage_notification_threshold' => ['type' => 'integer', 'description' => 'Server disk usage notification threshold (%).'],'server_disk_usage_check_frequency' => ['type' => 'string', 'description' => 'Cron expression for disk usage check frequency.'],+ 'server_disk_usage_notification_interval_hours' => ['type' => 'integer', 'description' => 'Minimum hours between high disk usage notifications (1-720). Default: 24.'],'connection_timeout' => ['type' => 'integer', 'description' => 'SSH connection timeout in seconds (1-300). Default: 10.'],],),@@ -682,7 +684,7 @@ class ServersController extends Controller)]public function update_server(Request $request){- $allowedFields = ['name', 'description', 'ip', 'port', 'user', 'private_key_uuid', 'server_role', 'instant_validate', 'proxy_type', 'concurrent_builds', 'dynamic_timeout', 'deployment_queue_limit', 'server_disk_usage_notification_threshold', 'server_disk_usage_check_frequency', 'connection_timeout', 'is_terminal_enabled'];+ $allowedFields = ['name', 'description', 'ip', 'port', 'user', 'private_key_uuid', 'server_role', 'instant_validate', 'proxy_type', 'concurrent_builds', 'dynamic_timeout', 'deployment_queue_limit', 'server_disk_usage_notification_threshold', 'server_disk_usage_check_frequency', 'server_disk_usage_notification_interval_hours', 'connection_timeout', 'is_terminal_enabled'];$teamId = getTeamIdFromToken();if (is_null($teamId)) {@@ -708,6 +710,7 @@ class ServersController extends Controller'deployment_queue_limit' => 'integer|min:1','server_disk_usage_notification_threshold' => 'integer|min:1|max:100','server_disk_usage_check_frequency' => 'string',+ 'server_disk_usage_notification_interval_hours' => 'integer|min:1|max:720','connection_timeout' => 'integer|min:1|max:300','is_terminal_enabled' => 'boolean|nullable',], [@@ -790,7 +793,7 @@ class ServersController extends Controller]);}- $advancedSettings = $request->only(['concurrent_builds', 'dynamic_timeout', 'deployment_queue_limit', 'server_disk_usage_notification_threshold', 'server_disk_usage_check_frequency', 'connection_timeout']);+ $advancedSettings = $request->only(['concurrent_builds', 'dynamic_timeout', 'deployment_queue_limit', 'server_disk_usage_notification_threshold', 'server_disk_usage_check_frequency', 'server_disk_usage_notification_interval_hours', 'connection_timeout']);if (! empty($advancedSettings)) {$server->settings()->update(array_filter($advancedSettings, fn ($value) => ! is_null($value)));}diff --git a/app/Jobs/CheckMissingDatabaseBackupsJob.php b/app/Jobs/CheckMissingDatabaseBackupsJob.phpindex 06ba79a33f..957868f44d 100644--- a/app/Jobs/CheckMissingDatabaseBackupsJob.php+++ b/app/Jobs/CheckMissingDatabaseBackupsJob.php@@ -2,6 +2,7 @@namespace App\Jobs;+use App\Models\NotificationThrottle;use App\Models\ScheduledDatabaseBackup;use App\Notifications\Database\BackupMissing;use Illuminate\Bus\Queueable;@@ -39,10 +40,6 @@ class CheckMissingDatabaseBackupsJob implements ShouldBeEncrypted, ShouldBeUniqureturn;}- if ($backup->missing_backup_notification_sent_at?->greaterThanOrEqualTo($lastActivityAt)) {- return;- }-if (! $backup->team) {Log::warning("Cannot send missing backup notification for backup {$backup->id}: team not found");@@ -53,7 +50,11 @@ class CheckMissingDatabaseBackupsJob implements ShouldBeEncrypted, ShouldBeUniqureturn;}+ // Send once for each period without backup activity.+ if (! NotificationThrottle::claim($backup, BackupMissing::class, $lastActivityAt)) {+ return;+ }+$backup->team->notify(new BackupMissing($backup, $lastExecutionAt));- $backup->forceFill(['missing_backup_notification_sent_at' => now()])->save();}}diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.phpindex 1324c212ed..3224d3a5e1 100644--- a/app/Jobs/PushServerUpdateJob.php+++ b/app/Jobs/PushServerUpdateJob.php@@ -799,7 +799,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silencedtry {if (CheckProxy::run($this->server)) {StartProxy::run($this->server, async: false);- $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server));+ $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));}} catch (\Throwable $e) {}@@ -841,7 +841,7 @@ class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced})->first();if (! $tcpProxyContainerFound) {StartDatabaseProxy::dispatch($database);- $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server));+ $this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server, restartedResource: $database));}} elseif ($this->isRunning($containerStatus) && ! $tcpProxy) {// Clean up orphaned proxy containers when is_public=falsediff --git a/app/Jobs/ServerCheckJob.php b/app/Jobs/ServerCheckJob.phpindex 10faa7e9b8..4b7203648c 100644--- a/app/Jobs/ServerCheckJob.php+++ b/app/Jobs/ServerCheckJob.php@@ -85,7 +85,7 @@ class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue$shouldStart = CheckProxy::run($this->server);if ($shouldStart) {StartProxy::run($this->server, async: false);- $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server));+ $this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));}} catch (\Throwable $e) {}diff --git a/app/Jobs/ServerStorageCheckJob.php b/app/Jobs/ServerStorageCheckJob.phpindex 51426d880a..a457207951 100644--- a/app/Jobs/ServerStorageCheckJob.php+++ b/app/Jobs/ServerStorageCheckJob.php@@ -10,8 +10,8 @@ use Illuminate\Contracts\Queue\ShouldQueue;use Illuminate\Foundation\Bus\Dispatchable;use Illuminate\Queue\InteractsWithQueue;use Illuminate\Queue\SerializesModels;+use Illuminate\Queue\TimeoutExceededException;use Illuminate\Support\Facades\Log;-use Illuminate\Support\Facades\RateLimiter;use Laravel\Horizon\Contracts\Silenced;class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silenced@@ -31,7 +31,7 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silencedpublic function failed(?\Throwable $exception): void{- if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) {+ if ($exception instanceof TimeoutExceededException) {Log::warning('ServerStorageCheckJob timed out', ['server_id' => $this->server->id,'server_name' => $this->server->name,@@ -58,20 +58,7 @@ class ServerStorageCheckJob implements ShouldBeEncrypted, ShouldQueue, Silencedreturn 'No percentage could be retrieved.';}if ($this->percentage > $serverDiskUsageNotificationThreshold) {- $executed = RateLimiter::attempt(- 'high-disk-usage:'.$this->server->id,- $maxAttempts = 0,- function () use ($team, $serverDiskUsageNotificationThreshold) {- $team->notify(new HighDiskUsage($this->server, $this->percentage, $serverDiskUsageNotificationThreshold));- },- $decaySeconds = 3600,- );-- if (! $executed) {- return 'Too many messages sent!';- }- } else {- RateLimiter::hit('high-disk-usage:'.$this->server->id, 600);+ $team->notify(new HighDiskUsage($this->server, $this->percentage, $serverDiskUsageNotificationThreshold));}} catch (\Throwable $e) {return handleError($e);diff --git a/app/Livewire/Server/Advanced.php b/app/Livewire/Server/Advanced.phpindex 895ce34e79..34847a12e6 100644--- a/app/Livewire/Server/Advanced.php+++ b/app/Livewire/Server/Advanced.php@@ -18,6 +18,9 @@ class Advanced extends Component#[Validate(['required', 'integer', 'min:1', 'max:99'])]public int|string $serverDiskUsageNotificationThreshold = 50;+ #[Validate(['required', 'integer', 'min:1', 'max:720'])]+ public int|string $serverDiskUsageNotificationIntervalHours = 24;+#[Validate(['required', 'integer', 'min:1'])]public int|string $concurrentBuilds = 1;@@ -52,6 +55,7 @@ class Advanced extends Component$this->server->settings->backup_compression_cpu_percentage = $this->backupCompressionCpuPercentage;$this->server->settings->server_disk_usage_notification_threshold = $this->serverDiskUsageNotificationThreshold;$this->server->settings->server_disk_usage_check_frequency = $this->serverDiskUsageCheckFrequency;+ $this->server->settings->server_disk_usage_notification_interval_hours = $this->serverDiskUsageNotificationIntervalHours;$this->server->settings->save();} else {$this->concurrentBuilds = $this->server->settings->concurrent_builds;@@ -60,6 +64,7 @@ class Advanced extends Component$this->backupCompressionCpuPercentage = $this->server->settings->backup_compression_cpu_percentage;$this->serverDiskUsageNotificationThreshold = $this->server->settings->server_disk_usage_notification_threshold;$this->serverDiskUsageCheckFrequency = $this->server->settings->server_disk_usage_check_frequency;+ $this->serverDiskUsageNotificationIntervalHours = $this->server->settings->server_disk_usage_notification_interval_hours;}}diff --git a/app/Models/NotificationThrottle.php b/app/Models/NotificationThrottle.phpnew file mode 100644index 0000000000..46340f9fd3--- /dev/null+++ b/app/Models/NotificationThrottle.php@@ -0,0 +1,111 @@+<?php++namespace App\Models;++use App\Contracts\ThrottledNotification;+use Carbon\CarbonInterface;+use Illuminate\Database\Eloquent\Model;+use Illuminate\Database\Eloquent\Relations\MorphTo;++/**+ * Records when a notification was last sent for a resource, so repeated checks+ * (across queue workers) do not send the same notification again.+ */+class NotificationThrottle extends Model+{+ public $timestamps = false;++ protected $fillable = [+ 'notifiable_type',+ 'notifiable_id',+ 'notification',+ 'sent_at',+ ];++ protected function casts(): array+ {+ return [+ 'sent_at' => 'datetime',+ ];+ }++ public function notifiable(): MorphTo+ {+ return $this->morphTo();+ }++ /**+ * Atomically reserve the right to send a notification.+ *+ * Returns true when the notification was never sent, or when it was last sent+ * before $sentBefore. Without $sentBefore, it is sent only once until released.+ */+ public static function claim(Model $notifiable, string $notification, ?CarbonInterface $sentBefore = null): bool+ {+ if ($sentBefore !== null) {+ $renewed = static::query()+ ->whereMorphedTo('notifiable', $notifiable)+ ->where('notification', $notification)+ ->where('sent_at', '<', $sentBefore)+ ->update(['sent_at' => now()]);++ if ($renewed === 1) {+ return true;+ }+ }++ return static::query()->insertOrIgnore([+ 'notifiable_type' => $notifiable->getMorphClass(),+ 'notifiable_id' => $notifiable->getKey(),+ 'notification' => $notification,+ 'sent_at' => now(),+ ]) === 1;+ }++ /**+ * Claim the throttle a notification declares. Notifications without a throttle subject are always allowed.+ */+ public static function claimFor(ThrottledNotification $notification): bool+ {+ $subject = $notification->throttleSubject();+ if ($subject === null) {+ return true;+ }++ return static::claim($subject, $notification::class, now()->subMinutes($notification->throttleIntervalMinutes()));+ }++ /**+ * Mark a notification as sent without sending it.+ */+ public static function record(Model $notifiable, string $notification): void+ {+ static::query()->updateOrInsert(+ [+ 'notifiable_type' => $notifiable->getMorphClass(),+ 'notifiable_id' => $notifiable->getKey(),+ 'notification' => $notification,+ ],+ ['sent_at' => now()],+ );+ }++ /**+ * Forget a sent notification. Returns true when a record was removed.+ */+ public static function release(Model $notifiable, string $notification): bool+ {+ return static::query()+ ->whereMorphedTo('notifiable', $notifiable)+ ->where('notification', $notification)+ ->delete() > 0;+ }++ public static function wasSent(Model $notifiable, string $notification): bool+ {+ return static::query()+ ->whereMorphedTo('notifiable', $notifiable)+ ->where('notification', $notification)+ ->exists();+ }+}diff --git a/app/Models/ScheduledDatabaseBackup.php b/app/Models/ScheduledDatabaseBackup.phpindex 73c162ea81..b44d1b16f4 100644--- a/app/Models/ScheduledDatabaseBackup.php+++ b/app/Models/ScheduledDatabaseBackup.php@@ -18,7 +18,6 @@ class ScheduledDatabaseBackup extends BaseModel'database_backup_retention_max_storage_locally' => 'float','database_backup_retention_max_storage_s3' => 'float','missing_backup_notification_days' => 'integer',- 'missing_backup_notification_sent_at' => 'datetime','last_execution_at' => 'datetime','next_run_at' => 'datetime',];diff --git a/app/Models/Server.php b/app/Models/Server.phpindex 00eaad8839..9f4e6667d8 100644--- a/app/Models/Server.php+++ b/app/Models/Server.php@@ -29,6 +29,7 @@ use App\Traits\HasSafeStringAttribute;use Illuminate\Database\Eloquent\Builder;use Illuminate\Database\Eloquent\Casts\Attribute;use Illuminate\Database\Eloquent\Factories\HasFactory;+use Illuminate\Database\Eloquent\Relations\MorphMany;use Illuminate\Database\Eloquent\SoftDeletes;use Illuminate\Support\Carbon;use Illuminate\Support\Collection;@@ -235,6 +236,7 @@ class Server extends BaseModel});$server->settings()->delete();$server->sslCertificates()->delete();+ $server->notificationThrottles()->delete();});static::updated(function () {@@ -280,7 +282,6 @@ class Server extends BaseModel'logdrain_newrelic_license_key' => 'encrypted','delete_unused_volumes' => 'boolean','delete_unused_networks' => 'boolean',- 'unreachable_notification_sent' => 'boolean','force_disabled' => 'boolean','sentinel_waiting_since' => 'datetime',];@@ -1526,6 +1527,19 @@ $siteAddress {return $this->belongsTo(CloudProviderToken::class);}+ public function notificationThrottles(): MorphMany+ {+ return $this->morphMany(NotificationThrottle::class, 'notifiable');+ }++ /**+ * True while an Unreachable notification was sent and no Reachable notification followed.+ */+ protected function unreachableNotificationSent(): Attribute+ {+ return Attribute::get(fn (): bool => NotificationThrottle::wasSent($this, Unreachable::class));+ }+public function sslCertificates(){return $this->hasMany(SslCertificate::class);@@ -1777,18 +1791,16 @@ $siteAddress {public function sendReachableNotification(){- $this->unreachable_notification_sent = false;- $this->save();- $this->refresh();- $this->team->notify(new Reachable($this));+ if (NotificationThrottle::release($this, Unreachable::class)) {+ $this->team->notify(new Reachable($this));+ }}public function sendUnreachableNotification(){- $this->unreachable_notification_sent = true;- $this->save();- $this->refresh();- $this->team->notify(new Unreachable($this));+ if (NotificationThrottle::claim($this, Unreachable::class)) {+ $this->team->notify(new Unreachable($this));+ }}public function validateConnection(bool $justCheckingNewKey = false)diff --git a/app/Models/ServerSetting.php b/app/Models/ServerSetting.phpindex 53674dad92..defb842dda 100644--- a/app/Models/ServerSetting.php+++ b/app/Models/ServerSetting.php@@ -117,6 +117,7 @@ class ServerSetting extends Model'server_disk_usage_notification_threshold','is_sentinel_debug_enabled','server_disk_usage_check_frequency',+ 'server_disk_usage_notification_interval_hours','is_terminal_enabled','deployment_queue_limit','backup_compression_cpu_percentage',@@ -170,6 +171,7 @@ class ServerSetting extends Model'compose_version_checked_at' => 'datetime','backup_compression_cpu_percentage' => 'integer','docker_cleanup_next_run_at' => 'datetime',+ 'server_disk_usage_notification_interval_hours' => 'integer',];/**diff --git a/app/Models/Team.php b/app/Models/Team.phpindex 6ce1c1d3c1..da08882f6b 100644--- a/app/Models/Team.php+++ b/app/Models/Team.php@@ -3,11 +3,13 @@namespace App\Models;use App\Actions\User\RevokeUserTeamTokens;+use App\Contracts\ThrottledNotification;use App\Events\ServerReachabilityChanged;use App\Notifications\Channels\SendsDiscord;use App\Notifications\Channels\SendsEmail;use App\Notifications\Channels\SendsPushover;use App\Notifications\Channels\SendsSlack;+use App\Notifications\Server\Unreachable;use App\Traits\Auditable;use App\Traits\HasNotificationSettings;use App\Traits\HasSafeStringAttribute;@@ -43,7 +45,10 @@ use OpenApi\Attributes as OA;class Team extends Model implements SendsDiscord, SendsEmail, SendsPushover, SendsSlack{- use Auditable, HasFactory, HasNotificationSettings, HasSafeStringAttribute, Notifiable;+ use Auditable, HasFactory, HasNotificationSettings, HasSafeStringAttribute;+ use Notifiable {+ notify as sendNotification;+ }protected $fillable = ['name',@@ -273,11 +278,23 @@ class Team extends Model implements SendsDiscord, SendsEmail, SendsPushover, Sen]);ServerReachabilityChanged::dispatch($server);$server->unreachable_count = 3;- $server->unreachable_notification_sent = true;$server->save();+ NotificationThrottle::record($server, Unreachable::class);}}+ /**+ * Send a notification, unless it is throttled and was already sent within its interval.+ */+ public function notify($instance): void+ {+ if ($instance instanceof ThrottledNotification && ! NotificationThrottle::claimFor($instance)) {+ return;+ }++ $this->sendNotification($instance);+ }+public function environment_variables(){return $this->hasMany(SharedEnvironmentVariable::class)->where('type', 'team');diff --git a/app/Notifications/Container/ContainerRestarted.php b/app/Notifications/Container/ContainerRestarted.phpindex 2d7eb58b50..012f997754 100644--- a/app/Notifications/Container/ContainerRestarted.php+++ b/app/Notifications/Container/ContainerRestarted.php@@ -2,16 +2,21 @@namespace App\Notifications\Container;+use App\Contracts\ThrottledNotification;use App\Models\Server;use App\Notifications\CustomEmailNotification;use App\Notifications\Dto\DiscordMessage;use App\Notifications\Dto\PushoverMessage;use App\Notifications\Dto\SlackMessage;+use Illuminate\Database\Eloquent\Model;use Illuminate\Notifications\Messages\MailMessage;-class ContainerRestarted extends CustomEmailNotification+class ContainerRestarted extends CustomEmailNotification implements ThrottledNotification{- public function __construct(public string $name, public Server $server, public ?string $url = null)+ /**+ * When $restartedResource is set, a restart loop of that resource sends at most one notification per hour.+ */+ public function __construct(public string $name, public Server $server, public ?string $url = null, public ?Model $restartedResource = null){$this->onQueue('high');}@@ -21,6 +26,16 @@ class ContainerRestarted extends CustomEmailNotificationreturn $notifiable->getEnabledChannels('status_change');}+ public function throttleSubject(): ?Model+ {+ return $this->restartedResource;+ }++ public function throttleIntervalMinutes(): int+ {+ return 60;+ }+public function toMail(): MailMessage{$mail = new MailMessage;diff --git a/app/Notifications/Server/HighDiskUsage.php b/app/Notifications/Server/HighDiskUsage.phpindex 149d1bbc86..be0517b759 100644--- a/app/Notifications/Server/HighDiskUsage.php+++ b/app/Notifications/Server/HighDiskUsage.php@@ -2,14 +2,16 @@namespace App\Notifications\Server;+use App\Contracts\ThrottledNotification;use App\Models\Server;use App\Notifications\CustomEmailNotification;use App\Notifications\Dto\DiscordMessage;use App\Notifications\Dto\PushoverMessage;use App\Notifications\Dto\SlackMessage;+use Illuminate\Database\Eloquent\Model;use Illuminate\Notifications\Messages\MailMessage;-class HighDiskUsage extends CustomEmailNotification+class HighDiskUsage extends CustomEmailNotification implements ThrottledNotification{public function __construct(public Server $server, public int $disk_usage, public int $server_disk_usage_notification_threshold){@@ -21,6 +23,16 @@ class HighDiskUsage extends CustomEmailNotificationreturn $notifiable->getEnabledChannels('server_disk_usage');}+ public function throttleSubject(): ?Model+ {+ return $this->server;+ }++ public function throttleIntervalMinutes(): int+ {+ return max(1, (int) data_get($this->server, 'settings.server_disk_usage_notification_interval_hours', 24)) * 60;+ }+public function toMail(): MailMessage{$mail = new MailMessage;diff --git a/database/migrations/2026_09_29_082403_add_disk_usage_notification_interval_to_server_settings_table.php b/database/migrations/2026_09_29_082403_add_disk_usage_notification_interval_to_server_settings_table.phpnew file mode 100644index 0000000000..6ec4a5a312--- /dev/null+++ b/database/migrations/2026_09_29_082403_add_disk_usage_notification_interval_to_server_settings_table.php@@ -0,0 +1,22 @@+<?php++use Illuminate\Database\Migrations\Migration;+use Illuminate\Database\Schema\Blueprint;+use Illuminate\Support\Facades\Schema;++return new class extends Migration+{+ public function up(): void+ {+ Schema::table('server_settings', function (Blueprint $table) {+ $table->integer('server_disk_usage_notification_interval_hours')->default(24);+ });+ }++ public function down(): void+ {+ Schema::table('server_settings', function (Blueprint $table) {+ $table->dropColumn('server_disk_usage_notification_interval_hours');+ });+ }+};diff --git a/database/migrations/2026_09_29_083244_create_notification_throttles_table.php b/database/migrations/2026_09_29_083244_create_notification_throttles_table.phpnew file mode 100644index 0000000000..e11a14a567--- /dev/null+++ b/database/migrations/2026_09_29_083244_create_notification_throttles_table.php@@ -0,0 +1,98 @@+<?php++use Illuminate\Database\Migrations\Migration;+use Illuminate\Database\Schema\Blueprint;+use Illuminate\Support\Facades\DB;+use Illuminate\Support\Facades\Schema;++return new class extends Migration+{+ private const SERVER = 'App\Models\Server';++ private const BACKUP = 'App\Models\ScheduledDatabaseBackup';++ private const UNREACHABLE = 'App\Notifications\Server\Unreachable';++ private const BACKUP_MISSING = 'App\Notifications\Database\BackupMissing';++ /**+ * Moves the per-resource "notification sent" markers into one table.+ */+ public function up(): void+ {+ Schema::create('notification_throttles', function (Blueprint $table) {+ $table->id();+ $table->morphs('notifiable');+ $table->string('notification');+ $table->timestampTz('sent_at');+ $table->unique(['notifiable_type', 'notifiable_id', 'notification']);+ });++ DB::table('servers')+ ->where('unreachable_notification_sent', true)+ ->orderBy('id')+ ->select(['id', 'updated_at'])+ ->chunk(500, function ($servers) {+ DB::table('notification_throttles')->insertOrIgnore($servers->map(fn ($server) => [+ 'notifiable_type' => self::SERVER,+ 'notifiable_id' => $server->id,+ 'notification' => self::UNREACHABLE,+ 'sent_at' => $server->updated_at ?? now(),+ ])->all());+ });++ DB::table('scheduled_database_backups')+ ->whereNotNull('missing_backup_notification_sent_at')+ ->orderBy('id')+ ->select(['id', 'missing_backup_notification_sent_at'])+ ->chunk(500, function ($backups) {+ DB::table('notification_throttles')->insertOrIgnore($backups->map(fn ($backup) => [+ 'notifiable_type' => self::BACKUP,+ 'notifiable_id' => $backup->id,+ 'notification' => self::BACKUP_MISSING,+ 'sent_at' => $backup->missing_backup_notification_sent_at,+ ])->all());+ });++ Schema::table('servers', function (Blueprint $table) {+ $table->dropColumn('unreachable_notification_sent');+ });+ Schema::table('scheduled_database_backups', function (Blueprint $table) {+ $table->dropColumn('missing_backup_notification_sent_at');+ });+ }++ public function down(): void+ {+ Schema::table('servers', function (Blueprint $table) {+ $table->boolean('unreachable_notification_sent')->default(false);+ });+ Schema::table('scheduled_database_backups', function (Blueprint $table) {+ $table->timestamp('missing_backup_notification_sent_at')->nullable();+ });++ DB::table('notification_throttles')+ ->where('notifiable_type', self::SERVER)+ ->where('notification', self::UNREACHABLE)+ ->orderBy('id')+ ->chunk(500, function ($throttles) {+ DB::table('servers')+ ->whereIn('id', $throttles->pluck('notifiable_id'))+ ->update(['unreachable_notification_sent' => true]);+ });++ DB::table('notification_throttles')+ ->where('notifiable_type', self::BACKUP)+ ->where('notification', self::BACKUP_MISSING)+ ->orderBy('id')+ ->chunk(500, function ($throttles) {+ foreach ($throttles as $throttle) {+ DB::table('scheduled_database_backups')+ ->where('id', $throttle->notifiable_id)+ ->update(['missing_backup_notification_sent_at' => $throttle->sent_at]);+ }+ });++ Schema::dropIfExists('notification_throttles');+ }+};diff --git a/openapi.json b/openapi.jsonindex 512f94b9e6..8087d32199 100644--- a/openapi.json+++ b/openapi.json@@ -3531,7 +3531,7 @@"Applications"],"summary": "Get application logs.",- "description": "Get application logs by UUID.",+ "description": "Get application logs by UUID. Requires the `read:sensitive` or `root` token ability.","operationId": "get-application-logs-by-uuid","parameters": [{@@ -3616,7 +3616,7 @@"Applications"],"summary": "Get preview application logs.",- "description": "Get runtime container logs for a preview deployment by application UUID and pull request ID.",+ "description": "Get runtime container logs for a preview deployment by application UUID and pull request ID. Requires the `read:sensitive` or `root` token ability.","operationId": "get-preview-application-logs-by-pull-request-id","parameters": [{@@ -8293,7 +8293,7 @@"Databases"],"summary": "Get database logs.",- "description": "Get database logs by UUID.",+ "description": "Get database logs by UUID. Requires the `read:sensitive` or `root` token ability.","operationId": "get-database-logs-by-uuid","parameters": [{@@ -16766,6 +16766,10 @@"type": "string","description": "Cron expression for disk usage check frequency."},+ "server_disk_usage_notification_interval_hours": {+ "type": "integer",+ "description": "Minimum hours between high disk usage notifications (1-720). Default: 24."+ },"connection_timeout": {"type": "integer","description": "SSH connection timeout in seconds (1-300). Default: 10."@@ -17265,7 +17269,7 @@"Service applications"],"summary": "Get service application logs",- "description": "Get Docker logs for a single compose service container.",+ "description": "Get Docker logs for a single compose service container. Requires the `read:sensitive` or `root` token ability.","operationId": "get-service-application-logs-by-service-and-app-uuid","parameters": [{@@ -17350,7 +17354,7 @@"Service applications"],"summary": "Get service application logs",- "description": "Get Docker logs for a single compose service container.",+ "description": "Get Docker logs for a single compose service container. Requires the `read:sensitive` or `root` token ability.","operationId": "post-service-application-logs-by-service-and-app-uuid","parameters": [{@@ -18002,7 +18006,7 @@"Service databases"],"summary": "Get service database logs",- "description": "Get Docker logs for a compose database container.",+ "description": "Get Docker logs for a compose database container. Requires the `read:sensitive` or `root` token ability.","operationId": "get-service-database-logs-by-service-and-database-uuid","parameters": [{@@ -18847,7 +18851,7 @@"Services"],"summary": "Get service logs.",- "description": "Get logs for a specific service sub-resource by service UUID. The `sub_service_name` query parameter must match the `name` field of one of the service applications or databases returned by `GET \/services\/{uuid}`.",+ "description": "Get logs for a specific service sub-resource by service UUID. The `sub_service_name` query parameter must match the `name` field of one of the service applications or databases returned by `GET \/services\/{uuid}`. Requires the `read:sensitive` or `root` token ability.","operationId": "get-service-logs-by-uuid","parameters": [{diff --git a/openapi.yaml b/openapi.yamlindex e6261818b1..a78f236afc 100644--- a/openapi.yaml+++ b/openapi.yaml@@ -2325,7 +2325,7 @@ paths:tags:- Applicationssummary: 'Get application logs.'- description: 'Get application logs by UUID.'+ description: 'Get application logs by UUID. Requires the `read:sensitive` or `root` token ability.'operationId: get-application-logs-by-uuidparameters:-@@ -2383,7 +2383,7 @@ paths:tags:- Applicationssummary: 'Get preview application logs.'- description: 'Get runtime container logs for a preview deployment by application UUID and pull request ID.'+ description: 'Get runtime container logs for a preview deployment by application UUID and pull request ID. Requires the `read:sensitive` or `root` token ability.'operationId: get-preview-application-logs-by-pull-request-idparameters:-@@ -5469,7 +5469,7 @@ paths:tags:- Databasessummary: 'Get database logs.'- description: 'Get database logs by UUID.'+ description: 'Get database logs by UUID. Requires the `read:sensitive` or `root` token ability.'operationId: get-database-logs-by-uuidparameters:-@@ -10647,6 +10647,9 @@ paths:server_disk_usage_check_frequency:type: stringdescription: 'Cron expression for disk usage check frequency.'+ server_disk_usage_notification_interval_hours:+ type: integer+ description: 'Minimum hours between high disk usage notifications (1-720). Default: 24.'connection_timeout:type: integerdescription: 'SSH connection timeout in seconds (1-300). Default: 10.'@@ -10931,7 +10934,7 @@ paths:tags:- 'Service applications'summary: 'Get service application logs'- description: 'Get Docker logs for a single compose service container.'+ description: 'Get Docker logs for a single compose service container. Requires the `read:sensitive` or `root` token ability.'operationId: get-service-application-logs-by-service-and-app-uuidparameters:-@@ -10989,7 +10992,7 @@ paths:tags:- 'Service applications'summary: 'Get service application logs'- description: 'Get Docker logs for a single compose service container.'+ description: 'Get Docker logs for a single compose service container. Requires the `read:sensitive` or `root` token ability.'operationId: post-service-application-logs-by-service-and-app-uuidparameters:-@@ -11418,7 +11421,7 @@ paths:tags:- 'Service databases'summary: 'Get service database logs'- description: 'Get Docker logs for a compose database container.'+ description: 'Get Docker logs for a compose database container. Requires the `read:sensitive` or `root` token ability.'operationId: get-service-database-logs-by-service-and-database-uuidparameters:-@@ -11912,7 +11915,7 @@ paths:tags:- Servicessummary: 'Get service logs.'- description: 'Get logs for a specific service sub-resource by service UUID. The `sub_service_name` query parameter must match the `name` field of one of the service applications or databases returned by `GET /services/{uuid}`.'+ description: 'Get logs for a specific service sub-resource by service UUID. The `sub_service_name` query parameter must match the `name` field of one of the service applications or databases returned by `GET /services/{uuid}`. Requires the `read:sensitive` or `root` token ability.'operationId: get-service-logs-by-uuidparameters:-diff --git a/resources/views/livewire/server/advanced.blade.php b/resources/views/livewire/server/advanced.blade.phpindex 48f63f02e9..a7244869f3 100644--- a/resources/views/livewire/server/advanced.blade.php+++ b/resources/views/livewire/server/advanced.blade.php@@ -14,7 +14,7 @@<x-application.settings-section id="server-disk-usage-section" title="Disk usage"helper="Control when Coolify checks this server and when your team is notified.">- <div class="grid gap-4 lg:grid-cols-2">+ <div class="grid gap-4 lg:grid-cols-3"><x-forms.input canGate="update" :canResource="$server" placeholder="0 23 * * *"id="serverDiskUsageCheckFrequency" label="Check frequency" requiredhelper="Cron expression or preset such as hourly, daily, weekly, monthly, or yearly." />@@ -22,6 +22,10 @@id="serverDiskUsageNotificationThreshold" type="number" min="1" max="99"label="Notification threshold" requiredhelper="Notify the team when root filesystem usage exceeds this percentage." />+ <x-forms.input canGate="update" :canResource="$server"+ id="serverDiskUsageNotificationIntervalHours" type="number" min="1" max="720"+ label="Notification interval (hours)" required+ helper="Minimum time between high disk usage notifications for this server." /></div></x-application.settings-section>diff --git a/tests/Feature/Api/LifecycleApisTest.php b/tests/Feature/Api/LifecycleApisTest.phpindex 95004550b8..d89dcf414d 100644--- a/tests/Feature/Api/LifecycleApisTest.php+++ b/tests/Feature/Api/LifecycleApisTest.php@@ -7,6 +7,7 @@ use App\Models\CloudInitScript;use App\Models\Environment;use App\Models\InstanceSettings;use App\Models\LocalPersistentVolume;+use App\Models\NotificationThrottle;use App\Models\Project;use App\Models\ScheduledTask;use App\Models\Server;@@ -15,6 +16,7 @@ use App\Models\StandaloneDocker;use App\Models\StandalonePostgresql;use App\Models\Team;use App\Models\User;+use App\Notifications\Database\BackupMissing;use Illuminate\Foundation\Testing\RefreshDatabase;use Illuminate\Support\Facades\Bus;use Illuminate\Support\Facades\Queue;@@ -144,10 +146,8 @@ describe('POST /api/v1/databases/{uuid}/clone', function () {'frequency' => '0 0 * * *','save_s3' => false,]);- $backup->forceFill([- 'last_execution_at' => now()->subDay(),- 'missing_backup_notification_sent_at' => now(),- ])->save();+ $backup->forceFill(['last_execution_at' => now()->subDay()])->save();+ NotificationThrottle::record($backup, BackupMissing::class);$response = $this->withHeaders($this->headers)->postJson("/api/v1/databases/{$database->uuid}/clone", [@@ -166,7 +166,7 @@ describe('POST /api/v1/databases/{uuid}/clone', function () {->and($cloned->destination_id)->toBe($this->destination->id)->and(str($cloned->status)->startsWith('exited'))->toBeTrue()->and($clonedBackup->last_execution_at)->toBeNull()- ->and($clonedBackup->missing_backup_notification_sent_at)->toBeNull();+ ->and(NotificationThrottle::wasSent($clonedBackup, BackupMissing::class))->toBeFalse();});test('creates renamed volumes when cloning a database with clone_volumes', function () {diff --git a/tests/Feature/CleanupUnreachableServersTest.php b/tests/Feature/CleanupUnreachableServersTest.phpindex 8849b1ca01..2c58946891 100644--- a/tests/Feature/CleanupUnreachableServersTest.php+++ b/tests/Feature/CleanupUnreachableServersTest.php@@ -1,7 +1,9 @@<?php+use App\Models\NotificationThrottle;use App\Models\Server;use App\Models\Team;+use App\Notifications\Server\Unreachable;use Illuminate\Foundation\Testing\RefreshDatabase;uses(RefreshDatabase::class);@@ -13,9 +15,9 @@ it('disables (non-destructively) self-hosted servers with unreachable_count >= 3$server = Server::factory()->create(['team_id' => $team->id,'unreachable_count' => 50,- 'unreachable_notification_sent' => true,'updated_at' => now()->subDays(8),]);+ NotificationThrottle::record($server, Unreachable::class);$originalIp = (string) $server->ip;@@ -34,9 +36,9 @@ it('overwrites the IP with 1.2.3.4 on cloud for servers with unreachable_count >$server = Server::factory()->create(['team_id' => $team->id,'unreachable_count' => 50,- 'unreachable_notification_sent' => true,'updated_at' => now()->subDays(8),]);+ NotificationThrottle::record($server, Unreachable::class);$this->artisan('cleanup:unreachable-servers')->assertSuccessful();@@ -49,9 +51,9 @@ it('does not clean up servers with unreachable_count less than 3', function () {$server = Server::factory()->create(['team_id' => $team->id,'unreachable_count' => 2,- 'unreachable_notification_sent' => true,'updated_at' => now()->subDays(8),]);+ NotificationThrottle::record($server, Unreachable::class);$originalIp = (string) $server->ip;@@ -67,9 +69,9 @@ it('does not clean up servers updated within 7 days', function () {$server = Server::factory()->create(['team_id' => $team->id,'unreachable_count' => 10,- 'unreachable_notification_sent' => true,'updated_at' => now()->subDays(3),]);+ NotificationThrottle::record($server, Unreachable::class);$originalIp = (string) $server->ip;@@ -85,7 +87,6 @@ it('does not clean up servers without notification sent', function () {$server = Server::factory()->create(['team_id' => $team->id,'unreachable_count' => 10,- 'unreachable_notification_sent' => false,'updated_at' => now()->subDays(8),]);diff --git a/tests/Feature/CleanupUnsubscribedServersTest.php b/tests/Feature/CleanupUnsubscribedServersTest.phpindex 53b0160d8a..edf6c63a0b 100644--- a/tests/Feature/CleanupUnsubscribedServersTest.php+++ b/tests/Feature/CleanupUnsubscribedServersTest.php@@ -1,8 +1,10 @@<?php+use App\Models\NotificationThrottle;use App\Models\Server;use App\Models\Subscription;use App\Models\Team;+use App\Notifications\Server\Unreachable;use Illuminate\Foundation\Testing\RefreshDatabase;uses(RefreshDatabase::class);@@ -16,7 +18,6 @@ it('sets unreachable fields on servers when subscription ends', function () {$server = Server::factory()->create(['team_id' => $team->id,'unreachable_count' => 0,- 'unreachable_notification_sent' => false,]);$team->subscriptionEnded();@@ -34,9 +35,9 @@ it('cleans up unsubscribed server IP after 7 days via cleanup command', function$server = Server::factory()->create(['team_id' => $team->id,'unreachable_count' => 3,- 'unreachable_notification_sent' => true,'updated_at' => now()->subDays(8),]);+ NotificationThrottle::record($server, Unreachable::class);$this->artisan('cleanup:unreachable-servers')->assertSuccessful();@@ -49,9 +50,9 @@ it('does not clean up unsubscribed server IP within 7 day grace period', functio$server = Server::factory()->create(['team_id' => $team->id,'unreachable_count' => 3,- 'unreachable_notification_sent' => true,'updated_at' => now()->subDays(3),]);+ NotificationThrottle::record($server, Unreachable::class);$originalIp = (string) $server->ip;@@ -70,7 +71,6 @@ it('does not affect servers with active subscriptions', function () {$server = Server::factory()->create(['team_id' => $team->id,'unreachable_count' => 0,- 'unreachable_notification_sent' => false,]);$originalCount = $server->unreachable_count;diff --git a/tests/Feature/MissingDatabaseBackupNotificationTest.php b/tests/Feature/MissingDatabaseBackupNotificationTest.phpindex f4f2efd191..9ec50feae3 100644--- a/tests/Feature/MissingDatabaseBackupNotificationTest.php+++ b/tests/Feature/MissingDatabaseBackupNotificationTest.php@@ -1,6 +1,7 @@<?phpuse App\Jobs\CheckMissingDatabaseBackupsJob;+use App\Models\NotificationThrottle;use App\Models\ScheduledDatabaseBackup;use App\Models\ScheduledDatabaseBackupExecution;use App\Models\Team;@@ -49,7 +50,7 @@ it('notifies the team when an enabled backup has no executions for the configure(new CheckMissingDatabaseBackupsJob)->handle();Notification::assertSentTo($team, BackupMissing::class, fn (BackupMissing $notification) => $notification->backup->is($backup));- expect($backup->fresh()->missing_backup_notification_sent_at)->not->toBeNull();+ expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeTrue();});it('does not notify for recent disabled or unconfigured backup schedules', function () {@@ -114,7 +115,7 @@ it('waits to mark an incident sent until a notification channel is enabled', fun$backup = missingBackupSchedule($team);(new CheckMissingDatabaseBackupsJob)->handle();- expect($backup->fresh()->missing_backup_notification_sent_at)->toBeNull();+ expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeFalse();$team->emailNotificationSettings->update(['smtp_enabled' => true,diff --git a/tests/Feature/NotificationThrottleTest.php b/tests/Feature/NotificationThrottleTest.phpnew file mode 100644index 0000000000..b6d3d881f0--- /dev/null+++ b/tests/Feature/NotificationThrottleTest.php@@ -0,0 +1,82 @@+<?php++use App\Models\InstanceSettings;+use App\Models\NotificationThrottle;+use App\Models\Server;+use App\Models\Team;+use App\Models\User;+use App\Notifications\Server\HighDiskUsage;+use App\Notifications\Server\Unreachable;+use Carbon\Carbon;+use Illuminate\Foundation\Testing\RefreshDatabase;++uses(RefreshDatabase::class);++beforeEach(function () {+ Server::flushIdentityMap();+ $this->team = Team::factory()->create();+ $this->server = Server::factory()->create(['team_id' => $this->team->id]);+});++afterEach(function () {+ Carbon::setTestNow();+});++it('claims a notification only once until it is released', function () {+ expect(NotificationThrottle::claim($this->server, Unreachable::class))->toBeTrue()+ ->and(NotificationThrottle::claim($this->server, Unreachable::class))->toBeFalse()+ ->and($this->server->unreachable_notification_sent)->toBeTrue()+ ->and(NotificationThrottle::release($this->server, Unreachable::class))->toBeTrue()+ ->and(NotificationThrottle::release($this->server, Unreachable::class))->toBeFalse()+ ->and($this->server->unreachable_notification_sent)->toBeFalse()+ ->and(NotificationThrottle::claim($this->server, Unreachable::class))->toBeTrue();+});++it('claims a notification again only when it was sent before the given time', function () {+ Carbon::setTestNow(Carbon::parse('2026-09-29 12:00:00', 'UTC'));++ expect(NotificationThrottle::claim($this->server, HighDiskUsage::class, now()->subHours(6)))->toBeTrue();++ Carbon::setTestNow(now()->addHours(5));+ expect(NotificationThrottle::claim($this->server, HighDiskUsage::class, now()->subHours(6)))->toBeFalse();++ Carbon::setTestNow(now()->addHours(2));+ expect(NotificationThrottle::claim($this->server, HighDiskUsage::class, now()->subHours(6)))->toBeTrue()+ ->and(NotificationThrottle::query()->count())->toBe(1);+});++it('keeps throttles separate for each notification and resource', function () {+ $otherServer = Server::factory()->create(['team_id' => $this->team->id]);++ NotificationThrottle::record($this->server, Unreachable::class);++ expect(NotificationThrottle::claim($this->server, HighDiskUsage::class))->toBeTrue()+ ->and(NotificationThrottle::claim($otherServer, Unreachable::class))->toBeTrue()+ ->and(NotificationThrottle::claim($this->server, Unreachable::class))->toBeFalse();+});++it('removes the throttles of a server when the server is force deleted', function () {+ NotificationThrottle::record($this->server, Unreachable::class);++ $this->server->forceDelete();++ expect(NotificationThrottle::query()->count())->toBe(0);+});++it('keeps unreachable_notification_sent in the server API response', function () {+ config()->set('app.maintenance.driver', 'file');+ config()->set('cache.default', 'array');+ InstanceSettings::forceCreate(['id' => 0, 'is_api_enabled' => true]);+ $user = User::factory()->create();+ $this->team->members()->attach($user->id, ['role' => 'owner']);+ session(['currentTeam' => $this->team]);+ $token = $user->createToken('read-token', ['read']);+ $token->accessToken->forceFill(['team_id' => $this->team->id])->save();++ NotificationThrottle::record($this->server, Unreachable::class);++ $this->withHeaders(['Authorization' => 'Bearer '.$token->plainTextToken])+ ->getJson('/api/v1/servers/'.$this->server->uuid)+ ->assertOk()+ ->assertJsonPath('unreachable_notification_sent', true);+});diff --git a/tests/Feature/NotificationThrottlesMigrationTest.php b/tests/Feature/NotificationThrottlesMigrationTest.phpnew file mode 100644index 0000000000..04a641205d--- /dev/null+++ b/tests/Feature/NotificationThrottlesMigrationTest.php@@ -0,0 +1,68 @@+<?php++use App\Models\NotificationThrottle;+use App\Models\ScheduledDatabaseBackup;+use App\Models\Server;+use App\Models\Team;+use App\Notifications\Database\BackupMissing;+use App\Notifications\Server\Unreachable;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Facades\DB;+use Illuminate\Support\Facades\Schema;++uses(RefreshDatabase::class);++beforeEach(function () {+ Server::flushIdentityMap();+});++it('moves the old notification markers into notification_throttles and back', function () {+ $team = Team::factory()->create();+ $localhost = Server::factory()->create(['id' => 0, 'team_id' => $team->id]);+ $notifiedServer = Server::factory()->create(['team_id' => $team->id]);+ $quietServer = Server::factory()->create(['team_id' => $team->id]);+ $notifiedBackup = ScheduledDatabaseBackup::create([+ 'enabled' => true,+ 'frequency' => '0 0 * * *',+ 'save_s3' => false,+ 'database_type' => 'App\\Models\\StandalonePostgresql',+ 'database_id' => 999,+ 'team_id' => $team->id,+ ]);+ $quietBackup = $notifiedBackup->replicate(['uuid']);+ $quietBackup->save();++ $migration = require database_path('migrations/2026_09_29_083244_create_notification_throttles_table.php');+ $migration->down();++ DB::table('servers')->whereIn('id', [$localhost->id, $notifiedServer->id])->update(['unreachable_notification_sent' => true]);+ DB::table('scheduled_database_backups')->where('id', $notifiedBackup->id)->update(['missing_backup_notification_sent_at' => '2026-09-01 10:00:00']);++ $migration->up();++ expect(Schema::hasColumn('servers', 'unreachable_notification_sent'))->toBeFalse()+ ->and(Schema::hasColumn('scheduled_database_backups', 'missing_backup_notification_sent_at'))->toBeFalse()+ ->and(NotificationThrottle::query()->count())->toBe(3)+ ->and(NotificationThrottle::wasSent($localhost, Unreachable::class))->toBeTrue()+ ->and(NotificationThrottle::wasSent($notifiedServer, Unreachable::class))->toBeTrue()+ ->and(NotificationThrottle::wasSent($quietServer, Unreachable::class))->toBeFalse()+ ->and(NotificationThrottle::wasSent($quietBackup, BackupMissing::class))->toBeFalse()+ ->and(NotificationThrottle::query()+ ->whereMorphedTo('notifiable', $notifiedBackup)+ ->where('notification', BackupMissing::class)+ ->sole()+ ->sent_at+ ->toDateTimeString())->toBe('2026-09-01 10:00:00');++ $migration->down();++ expect(Schema::hasTable('notification_throttles'))->toBeFalse()+ ->and(DB::table('servers')->where('unreachable_notification_sent', true)->pluck('id')->sort()->values()->all())+ ->toBe([$localhost->id, $notifiedServer->id])+ ->and(DB::table('scheduled_database_backups')->where('id', $notifiedBackup->id)->value('missing_backup_notification_sent_at'))+ ->toStartWith('2026-09-01 10:00:00')+ ->and(DB::table('scheduled_database_backups')->where('id', $quietBackup->id)->value('missing_backup_notification_sent_at'))+ ->toBeNull();++ $migration->up();+});diff --git a/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php b/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.phpnew file mode 100644index 0000000000..ff0caf1a98--- /dev/null+++ b/tests/Feature/Server/ServerStorageCheckNotificationIntervalTest.php@@ -0,0 +1,122 @@+<?php++use App\Jobs\ServerStorageCheckJob;+use App\Livewire\Server\Advanced;+use App\Models\InstanceSettings;+use App\Models\NotificationThrottle;+use App\Models\Server;+use App\Models\Team;+use App\Models\User;+use App\Notifications\Server\HighDiskUsage;+use Carbon\Carbon;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Facades\Notification;+use Livewire\Livewire;++uses(RefreshDatabase::class);++beforeEach(function () {+ Server::flushIdentityMap();+ Notification::fake();+ InstanceSettings::forceCreate(['id' => 0]);+ Carbon::setTestNow(Carbon::parse('2026-09-29 12:00:00', 'UTC'));+});++afterEach(function () {+ Carbon::setTestNow();+});++function diskUsageIntervalServer(int $intervalHours = 24): Server+{+ $team = Team::factory()->create();+ $team->emailNotificationSettings()->update([+ 'use_instance_email_settings' => true,+ 'server_disk_usage_email_notifications' => true,+ ]);+ $server = Server::factory()->create(['team_id' => $team->id, 'ip' => '10.0.0.10']);+ $server->settings->update([+ 'is_reachable' => true,+ 'is_usable' => true,+ 'force_disabled' => false,+ 'server_disk_usage_notification_threshold' => 80,+ 'server_disk_usage_notification_interval_hours' => $intervalHours,+ ]);++ return $server->fresh();+}++function runDiskUsageCheck(Server $server, int $percentage): void+{+ (new ServerStorageCheckJob($server->fresh(), $percentage))->handle();+}++it('sends one high disk usage notification per configured interval', function () {+ $server = diskUsageIntervalServer(intervalHours: 6);++ runDiskUsageCheck($server, 93);+ Carbon::setTestNow(now()->addMinutes(10));+ runDiskUsageCheck($server, 93);+ Carbon::setTestNow(now()->addHours(5));+ runDiskUsageCheck($server, 94);++ Notification::assertSentToTimes($server->team, HighDiskUsage::class, 1);++ Carbon::setTestNow(now()->addMinutes(51));+ runDiskUsageCheck($server, 94);++ Notification::assertSentToTimes($server->team, HighDiskUsage::class, 2);+});++it('defaults to one notification per 24 hours', function () {+ $team = Team::factory()->create();+ $server = Server::factory()->create(['team_id' => $team->id]);++ expect($server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(24);+});++it('does not notify below the threshold', function () {+ $server = diskUsageIntervalServer();++ runDiskUsageCheck($server, 50);++ Notification::assertNothingSent();+ expect(NotificationThrottle::wasSent($server, HighDiskUsage::class))->toBeFalse();+});++it('saves the notification interval from the server advanced settings', function () {+ $team = Team::factory()->create();+ $user = User::factory()->create();+ $team->members()->attach($user, ['role' => 'owner']);+ $server = Server::factory()->create(['team_id' => $team->id]);++ $this->actingAs($user);+ session(['currentTeam' => $team]);++ Livewire::test(Advanced::class, ['server_uuid' => $server->uuid])+ ->assertSet('serverDiskUsageNotificationIntervalHours', 24)+ ->set('serverDiskUsageNotificationIntervalHours', 0)+ ->call('submit')+ ->assertHasErrors(['serverDiskUsageNotificationIntervalHours'])+ ->set('serverDiskUsageNotificationIntervalHours', 12)+ ->call('submit')+ ->assertHasNoErrors()+ ->assertDispatched('success');++ expect($server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(12);+});++it('does not let a member change the notification interval', function () {+ $team = Team::factory()->create();+ $user = User::factory()->create();+ $team->members()->attach($user, ['role' => 'member']);+ $server = Server::factory()->create(['team_id' => $team->id]);++ $this->actingAs($user);+ session(['currentTeam' => $team]);++ Livewire::test(Advanced::class, ['server_uuid' => $server->uuid])+ ->set('serverDiskUsageNotificationIntervalHours', 2)+ ->call('submit');++ expect($server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(24);+});diff --git a/tests/Feature/ServerReachabilityNotificationTest.php b/tests/Feature/ServerReachabilityNotificationTest.phpindex e996ba0286..4fdd8df86c 100644--- a/tests/Feature/ServerReachabilityNotificationTest.php+++ b/tests/Feature/ServerReachabilityNotificationTest.php@@ -1,6 +1,7 @@<?phpuse App\Events\ServerReachabilityChanged;+use App\Models\NotificationThrottle;use App\Models\Server;use App\Models\Team;use App\Notifications\Channels\EmailChannel;@@ -28,7 +29,6 @@ it('sends Unreachable notification when threshold reached and not yet notified',$this->server->settings()->update(['is_reachable' => false]);$this->server->forceFill(['unreachable_count' => 2,- 'unreachable_notification_sent' => false,])->save();ServerReachabilityChanged::dispatch($this->server->fresh());@@ -41,7 +41,6 @@ it('does not send Unreachable on first transient failure (count=1)', function ()$this->server->settings()->update(['is_reachable' => false]);$this->server->forceFill(['unreachable_count' => 1,- 'unreachable_notification_sent' => false,])->save();ServerReachabilityChanged::dispatch($this->server->fresh());@@ -53,8 +52,8 @@ it('does not send Unreachable when already notified', function () {$this->server->settings()->update(['is_reachable' => false]);$this->server->forceFill(['unreachable_count' => 5,- 'unreachable_notification_sent' => true,])->save();+ NotificationThrottle::record($this->server, Unreachable::class);ServerReachabilityChanged::dispatch($this->server->fresh());@@ -65,8 +64,8 @@ it('sends Reachable notification on recovery when previously notified', function$this->server->settings()->update(['is_reachable' => true]);$this->server->forceFill(['unreachable_count' => 0,- 'unreachable_notification_sent' => true,])->save();+ NotificationThrottle::record($this->server, Unreachable::class);$fresh = $this->server->fresh();expect($fresh->unreachable_notification_sent)->toBeTrue();@@ -82,7 +81,6 @@ it('does not send Reachable when never notified', function () {$this->server->settings()->update(['is_reachable' => true]);$this->server->forceFill(['unreachable_count' => 0,- 'unreachable_notification_sent' => false,])->save();ServerReachabilityChanged::dispatch($this->server->fresh());@@ -94,7 +92,6 @@ it('routes Unreachable notification through EmailChannel when email toggle is on$this->server->settings()->update(['is_reachable' => false]);$this->server->forceFill(['unreachable_count' => 2,- 'unreachable_notification_sent' => false,])->save();ServerReachabilityChanged::dispatch($this->server->fresh());diff --git a/tests/Feature/ServerUpdatePrivateKeyApiTest.php b/tests/Feature/ServerUpdatePrivateKeyApiTest.phpindex 7940c0d9a1..ae567d2da7 100644--- a/tests/Feature/ServerUpdatePrivateKeyApiTest.php+++ b/tests/Feature/ServerUpdatePrivateKeyApiTest.php@@ -227,3 +227,20 @@ it('rejects an invalid disk usage check frequency without partially updating the->and($server->settings->server_role->value)->toBe('both')->and($server->proxy->get('type'))->toBe('TRAEFIK');});++it('updates the disk usage notification interval', function () {+ patchServerUpdatePrivateKeyApi($this, $this->server, $this->bearerToken, [+ 'server_disk_usage_notification_interval_hours' => 48,+ ])->assertCreated();++ expect($this->server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(48);+});++it('rejects an out of range disk usage notification interval', function () {+ patchServerUpdatePrivateKeyApi($this, $this->server, $this->bearerToken, [+ 'server_disk_usage_notification_interval_hours' => 0,+ ])->assertUnprocessable()+ ->assertJsonValidationErrors('server_disk_usage_notification_interval_hours');++ expect($this->server->settings->fresh()->server_disk_usage_notification_interval_hours)->toBe(24);+});diff --git a/tests/Feature/Subscription/SyncStripeSubscriptionsActionTest.php b/tests/Feature/Subscription/SyncStripeSubscriptionsActionTest.phpindex 31070053d6..a59ae659de 100644--- a/tests/Feature/Subscription/SyncStripeSubscriptionsActionTest.php+++ b/tests/Feature/Subscription/SyncStripeSubscriptionsActionTest.php@@ -42,7 +42,6 @@ test('fix mode fully ends a locally active subscription with an invalid Stripe s$server = Server::factory()->create(['team_id' => $this->team->id,'unreachable_count' => 0,- 'unreachable_notification_sent' => false,]);$stripe = Mockery::mock(StripeClient::class);diff --git a/tests/Feature/ThrottledNotificationTest.php b/tests/Feature/ThrottledNotificationTest.phpnew file mode 100644index 0000000000..1528b6ce55--- /dev/null+++ b/tests/Feature/ThrottledNotificationTest.php@@ -0,0 +1,102 @@+<?php++use App\Jobs\PushServerUpdateJob;+use App\Models\Environment;+use App\Models\Project;+use App\Models\Server;+use App\Models\StandalonePostgresql;+use App\Models\Team;+use App\Notifications\Container\ContainerRestarted;+use Carbon\Carbon;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Facades\Cache;+use Illuminate\Support\Facades\Notification;+use Illuminate\Support\Facades\Queue;++uses(RefreshDatabase::class);++beforeEach(function () {+ Server::flushIdentityMap();+ Cache::flush();+ Queue::fake();+ Notification::fake();+ Carbon::setTestNow(Carbon::parse('2026-09-29 12:00:00', 'UTC'));++ $this->team = Team::factory()->create();+ $this->team->emailNotificationSettings()->update([+ 'use_instance_email_settings' => true,+ 'status_change_email_notifications' => true,+ ]);+ $this->server = Server::factory()->create(['team_id' => $this->team->id]);+});++afterEach(function () {+ Carbon::setTestNow();+});++it('sends a throttled notification once per interval for the same resource', function () {+ $otherServer = Server::factory()->create(['team_id' => $this->team->id]);++ $this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));+ $this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));+ $this->team->notify(new ContainerRestarted('coolify-proxy', $otherServer, restartedResource: $otherServer));++ Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2);++ Carbon::setTestNow(now()->addMinutes(59));+ $this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));+ Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2);++ Carbon::setTestNow(now()->addMinutes(2));+ $this->team->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));+ Notification::assertSentToTimes($this->team, ContainerRestarted::class, 3);+});++it('does not throttle a notification without a throttle subject', function () {+ $this->team->notify(new ContainerRestarted('TCP Proxy for db has been disabled', $this->server));+ $this->team->notify(new ContainerRestarted('TCP Proxy for db has been disabled', $this->server));++ Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2);+});++it('sends one TCP proxy restart notification per hour while the proxy does not stay up', function () {+ $project = Project::factory()->create(['team_id' => $this->team->id]);+ $environment = Environment::factory()->create(['project_id' => $project->id]);+ $destination = $this->server->standaloneDockers()->firstOrFail();+ $database = StandalonePostgresql::create([+ 'name' => 'public-db',+ 'postgres_user' => 'postgres',+ 'postgres_password' => encrypt('password'),+ 'postgres_db' => 'app',+ 'image' => 'postgres:16-alpine',+ 'is_public' => true,+ 'public_port' => 54321,+ 'environment_id' => $environment->id,+ 'destination_id' => $destination->id,+ 'destination_type' => $destination->getMorphClass(),+ ]);++ // The database runs, but its "-proxy" container is missing on every push.+ $push = fn () => (new PushServerUpdateJob($this->server, [+ 'containers' => [[+ 'name' => $database->uuid,+ 'state' => 'running',+ 'health_status' => 'healthy',+ 'labels' => ['coolify.managed' => true, 'com.docker.compose.service' => $database->uuid],+ ]],+ 'filesystem_usage_root' => ['used_percentage' => 10],+ ]))->handle();++ $push();+ Carbon::setTestNow(now()->addMinute());+ $push();+ Carbon::setTestNow(now()->addMinute());+ $push();++ Notification::assertSentToTimes($this->team, ContainerRestarted::class, 1);++ Carbon::setTestNow(now()->addHour());+ $push();++ Notification::assertSentToTimes($this->team, ContainerRestarted::class, 2);+});