An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(notifications): disable channels after repeated configuration failures
Andras Bacsai7837a07ba2bf
diff --git a/app/Jobs/SendMessageToDiscordJob.php b/app/Jobs/SendMessageToDiscordJob.phpindex d5c29efb0f..226e9f4cd7 100644--- a/app/Jobs/SendMessageToDiscordJob.php+++ b/app/Jobs/SendMessageToDiscordJob.php@@ -2,8 +2,10 @@namespace App\Jobs;+use App\Models\DiscordNotificationSettings;use App\Notifications\Dto\DiscordMessage;use App\Rules\SafeWebhookUrl;+use App\Traits\HandlesNotificationResponse;use Illuminate\Bus\Queueable;use Illuminate\Contracts\Queue\ShouldBeEncrypted;use Illuminate\Contracts\Queue\ShouldQueue;@@ -16,7 +18,7 @@ use Illuminate\Support\Facades\Validator;class SendMessageToDiscordJob implements ShouldBeEncrypted, ShouldQueue{- use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;+ use Dispatchable, HandlesNotificationResponse, InteractsWithQueue, Queueable, SerializesModels;/*** The number of times the job may be attempted.@@ -34,7 +36,8 @@ class SendMessageToDiscordJob implements ShouldBeEncrypted, ShouldQueuepublic function __construct(public DiscordMessage $message,- public string $webhookUrl+ public string $webhookUrl,+ public ?DiscordNotificationSettings $settings = null,) {$this->onQueue('high');}@@ -69,6 +72,9 @@ class SendMessageToDiscordJob implements ShouldBeEncrypted, ShouldQueuereturn;}- Http::withOptions($httpOptions)->post($this->webhookUrl, $this->message->toPayload());+ $response = Http::withOptions($httpOptions)->post($this->webhookUrl, $this->message->toPayload());++ // 401: invalid webhook token, 403: missing access, 404: unknown (deleted) webhook.+ $this->handleNotificationResponse($response, 'Discord', in_array($response->status(), [401, 403, 404], true));}}diff --git a/app/Jobs/SendMessageToPushoverJob.php b/app/Jobs/SendMessageToPushoverJob.phpindex e2a94cdaa0..406a7e5ea4 100644--- a/app/Jobs/SendMessageToPushoverJob.php+++ b/app/Jobs/SendMessageToPushoverJob.php@@ -2,7 +2,9 @@namespace App\Jobs;+use App\Models\PushoverNotificationSettings;use App\Notifications\Dto\PushoverMessage;+use App\Traits\HandlesNotificationResponse;use Illuminate\Bus\Queueable;use Illuminate\Contracts\Queue\ShouldBeEncrypted;use Illuminate\Contracts\Queue\ShouldQueue;@@ -13,7 +15,7 @@ use Illuminate\Support\Facades\Http;class SendMessageToPushoverJob implements ShouldBeEncrypted, ShouldQueue{- use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;+ use Dispatchable, HandlesNotificationResponse, InteractsWithQueue, Queueable, SerializesModels;/*** The number of times the job may be attempted.@@ -33,6 +35,7 @@ class SendMessageToPushoverJob implements ShouldBeEncrypted, ShouldQueuepublic PushoverMessage $message,public string $token,public string $user,+ public ?PushoverNotificationSettings $settings = null,) {$this->onQueue('high');}@@ -43,8 +46,10 @@ class SendMessageToPushoverJob implements ShouldBeEncrypted, ShouldQueuepublic function handle(): void{$response = Http::post('https://api.pushover.net/1/messages.json', $this->message->toPayload($this->token, $this->user));- if ($response->failed()) {- throw new \RuntimeException('Pushover notification failed with '.$response->status().' status code.'.$response->body());- }++ // Pushover marks the invalid field in the error body, for example {"token": "invalid"}.+ $isConfigurationError = $response->clientError()+ && ($response->json('token') === 'invalid' || $response->json('user') === 'invalid');+ $this->handleNotificationResponse($response, 'Pushover', $isConfigurationError);}}diff --git a/app/Jobs/SendMessageToSlackJob.php b/app/Jobs/SendMessageToSlackJob.phpindex 3a306c23da..0b01f7f658 100644--- a/app/Jobs/SendMessageToSlackJob.php+++ b/app/Jobs/SendMessageToSlackJob.php@@ -2,12 +2,15 @@namespace App\Jobs;+use App\Models\SlackNotificationSettings;use App\Notifications\Dto\SlackMessage;use App\Rules\SafeWebhookUrl;+use App\Traits\HandlesNotificationResponse;use Illuminate\Bus\Queueable;use Illuminate\Contracts\Queue\ShouldBeEncrypted;use Illuminate\Contracts\Queue\ShouldQueue;use Illuminate\Foundation\Bus\Dispatchable;+use Illuminate\Http\Client\Response;use Illuminate\Queue\InteractsWithQueue;use Illuminate\Queue\SerializesModels;use Illuminate\Support\Facades\Http;@@ -16,7 +19,7 @@ use Illuminate\Support\Facades\Validator;class SendMessageToSlackJob implements ShouldBeEncrypted, ShouldQueue{- use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;+ use Dispatchable, HandlesNotificationResponse, InteractsWithQueue, Queueable, SerializesModels;/*** The number of times the job may be attempted.@@ -30,7 +33,8 @@ class SendMessageToSlackJob implements ShouldBeEncrypted, ShouldQueuepublic function __construct(private SlackMessage $message,- private string $webhookUrl+ private string $webhookUrl,+ public ?SlackNotificationSettings $settings = null,) {$this->onQueue('high');}@@ -62,18 +66,17 @@ class SendMessageToSlackJob implements ShouldBeEncrypted, ShouldQueuereturn;}- if ($this->isSlackWebhook()) {- $this->sendToSlack($httpOptions);-- return;- }-/*** This works with Mattermost and as a fallback also with Slack, the notifications just look slightly different and advanced formatting for slack is not supported with Mattermost.** @see https://github.com/coollabsio/coolify/pull/6139#issuecomment-3756777708*/- $this->sendToMattermost($httpOptions);+ $response = $this->isSlackWebhook()+ ? $this->sendToSlack($httpOptions)+ : $this->sendToMattermost($httpOptions);++ // 403: posting is not allowed, 404: the webhook or channel was removed, 410: the channel is archived.+ $this->handleNotificationResponse($response, 'Slack', in_array($response->status(), [403, 404, 410], true));}private function isSlackWebhook(): bool@@ -93,9 +96,9 @@ class SendMessageToSlackJob implements ShouldBeEncrypted, ShouldQueue/*** @param array<string, mixed> $httpOptions*/- private function sendToSlack(array $httpOptions): void+ private function sendToSlack(array $httpOptions): Response{- Http::withOptions($httpOptions)->post($this->webhookUrl, [+ return Http::withOptions($httpOptions)->post($this->webhookUrl, ['text' => $this->message->title,'blocks' => [[@@ -136,11 +139,11 @@ class SendMessageToSlackJob implements ShouldBeEncrypted, ShouldQueue/*** @param array<string, mixed> $httpOptions*/- private function sendToMattermost(array $httpOptions): void+ private function sendToMattermost(array $httpOptions): Response{$username = config('app.name');- Http::withOptions($httpOptions)->post($this->webhookUrl, [+ return Http::withOptions($httpOptions)->post($this->webhookUrl, ['username' => $username,'attachments' => [[diff --git a/app/Jobs/SendMessageToTelegramJob.php b/app/Jobs/SendMessageToTelegramJob.phpindex 6b04d21915..0cb1a7124d 100644--- a/app/Jobs/SendMessageToTelegramJob.php+++ b/app/Jobs/SendMessageToTelegramJob.php@@ -2,6 +2,8 @@namespace App\Jobs;+use App\Models\TelegramNotificationSettings;+use App\Traits\HandlesNotificationResponse;use Illuminate\Bus\Queueable;use Illuminate\Contracts\Queue\ShouldBeEncrypted;use Illuminate\Contracts\Queue\ShouldQueue;@@ -13,7 +15,7 @@ use Illuminate\Support\Str;class SendMessageToTelegramJob implements ShouldBeEncrypted, ShouldQueue{- use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;+ use Dispatchable, HandlesNotificationResponse, InteractsWithQueue, Queueable, SerializesModels;/*** The number of times the job may be attempted.@@ -38,6 +40,7 @@ class SendMessageToTelegramJob implements ShouldBeEncrypted, ShouldQueuepublic string $token,public string $chatId,public ?string $threadId = null,+ public ?TelegramNotificationSettings $settings = null,) {$this->onQueue('high');}@@ -76,8 +79,10 @@ class SendMessageToTelegramJob implements ShouldBeEncrypted, ShouldQueue$payload['message_thread_id'] = $this->threadId;}$response = Http::post($url, $payload);- if ($response->failed()) {- throw new \RuntimeException('Telegram notification failed with '.$response->status().' status code.'.$response->body());- }++ // 401: invalid bot token, 403: the bot was blocked or removed from the chat, 400 "chat not found": wrong chat ID.+ $isConfigurationError = in_array($response->status(), [401, 403], true)+ || ($response->status() === 400 && Str::contains((string) $response->json('description'), 'chat not found', true));+ $this->handleNotificationResponse($response, 'Telegram', $isConfigurationError);}}diff --git a/app/Jobs/SendWebhookJob.php b/app/Jobs/SendWebhookJob.phpindex c14d70a773..29ee58fb2f 100644--- a/app/Jobs/SendWebhookJob.php+++ b/app/Jobs/SendWebhookJob.php@@ -2,7 +2,9 @@namespace App\Jobs;+use App\Models\WebhookNotificationSettings;use App\Rules\SafeWebhookUrl;+use App\Traits\HandlesNotificationResponse;use Illuminate\Bus\Queueable;use Illuminate\Contracts\Queue\ShouldBeEncrypted;use Illuminate\Contracts\Queue\ShouldQueue;@@ -15,7 +17,7 @@ use Illuminate\Support\Facades\Validator;class SendWebhookJob implements ShouldBeEncrypted, ShouldQueue{- use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;+ use Dispatchable, HandlesNotificationResponse, InteractsWithQueue, Queueable, SerializesModels;/*** The number of times the job may be attempted.@@ -33,7 +35,8 @@ class SendWebhookJob implements ShouldBeEncrypted, ShouldQueuepublic function __construct(public array $payload,- public string $webhookUrl+ public string $webhookUrl,+ public ?WebhookNotificationSettings $settings = null,) {$this->onQueue('high');}@@ -68,6 +71,9 @@ class SendWebhookJob implements ShouldBeEncrypted, ShouldQueuereturn;}- Http::withOptions($httpOptions)->post($this->webhookUrl, $this->payload);+ $response = Http::withOptions($httpOptions)->post($this->webhookUrl, $this->payload);++ // 401/403: the endpoint rejects us, 404/410: the endpoint does not exist anymore.+ $this->handleNotificationResponse($response, 'Webhook', in_array($response->status(), [401, 403, 404, 410], true));}}diff --git a/app/Models/DiscordNotificationSettings.php b/app/Models/DiscordNotificationSettings.phpindex 48d5b5d293..4bf3343527 100644--- a/app/Models/DiscordNotificationSettings.php+++ b/app/Models/DiscordNotificationSettings.php@@ -2,12 +2,13 @@namespace App\Models;+use App\Traits\TracksDeliveryFailures;use Illuminate\Database\Eloquent\Model;use Illuminate\Notifications\Notifiable;class DiscordNotificationSettings extends Model{- use Notifiable;+ use Notifiable, TracksDeliveryFailures;public $timestamps = false;@@ -74,4 +75,14 @@ class DiscordNotificationSettings extends Model{return $this->discord_ping_enabled;}++ public function notificationChannelName(): string+ {+ return 'Discord';+ }++ protected function disableNotificationChannel(): void+ {+ $this->discord_enabled = false;+ }}diff --git a/app/Models/EmailNotificationSettings.php b/app/Models/EmailNotificationSettings.phpindex 3b04b482af..94d0802d96 100644--- a/app/Models/EmailNotificationSettings.php+++ b/app/Models/EmailNotificationSettings.php@@ -2,10 +2,13 @@namespace App\Models;+use App\Traits\TracksDeliveryFailures;use Illuminate\Database\Eloquent\Model;class EmailNotificationSettings extends Model{+ use TracksDeliveryFailures;+public $timestamps = false;protected $fillable = [@@ -93,4 +96,15 @@ class EmailNotificationSettings extends Model{return $this->smtp_enabled || $this->resend_enabled || $this->use_instance_email_settings;}++ public function notificationChannelName(): string+ {+ return 'Email';+ }++ protected function disableNotificationChannel(): void+ {+ $this->smtp_enabled = false;+ $this->resend_enabled = false;+ }}diff --git a/app/Models/PushoverNotificationSettings.php b/app/Models/PushoverNotificationSettings.phpindex 2ab6693142..9a8be46626 100644--- a/app/Models/PushoverNotificationSettings.php+++ b/app/Models/PushoverNotificationSettings.php@@ -2,12 +2,13 @@namespace App\Models;+use App\Traits\TracksDeliveryFailures;use Illuminate\Database\Eloquent\Model;use Illuminate\Notifications\Notifiable;class PushoverNotificationSettings extends Model{- use Notifiable;+ use Notifiable, TracksDeliveryFailures;public $timestamps = false;@@ -70,4 +71,14 @@ class PushoverNotificationSettings extends Model{return $this->pushover_enabled;}++ public function notificationChannelName(): string+ {+ return 'Pushover';+ }++ protected function disableNotificationChannel(): void+ {+ $this->pushover_enabled = false;+ }}diff --git a/app/Models/SlackNotificationSettings.php b/app/Models/SlackNotificationSettings.phpindex 648869bafe..53125e1e45 100644--- a/app/Models/SlackNotificationSettings.php+++ b/app/Models/SlackNotificationSettings.php@@ -2,12 +2,13 @@namespace App\Models;+use App\Traits\TracksDeliveryFailures;use Illuminate\Database\Eloquent\Model;use Illuminate\Notifications\Notifiable;class SlackNotificationSettings extends Model{- use Notifiable;+ use Notifiable, TracksDeliveryFailures;public $timestamps = false;@@ -67,4 +68,14 @@ class SlackNotificationSettings extends Model{return $this->slack_enabled;}++ public function notificationChannelName(): string+ {+ return 'Slack';+ }++ protected function disableNotificationChannel(): void+ {+ $this->slack_enabled = false;+ }}diff --git a/app/Models/TelegramNotificationSettings.php b/app/Models/TelegramNotificationSettings.phpindex 3376e239d6..6ab6c7a9e9 100644--- a/app/Models/TelegramNotificationSettings.php+++ b/app/Models/TelegramNotificationSettings.php@@ -2,12 +2,13 @@namespace App\Models;+use App\Traits\TracksDeliveryFailures;use Illuminate\Database\Eloquent\Model;use Illuminate\Notifications\Notifiable;class TelegramNotificationSettings extends Model{- use Notifiable;+ use Notifiable, TracksDeliveryFailures;public $timestamps = false;@@ -117,4 +118,14 @@ class TelegramNotificationSettings extends Model{return $this->telegram_enabled;}++ public function notificationChannelName(): string+ {+ return 'Telegram';+ }++ protected function disableNotificationChannel(): void+ {+ $this->telegram_enabled = false;+ }}diff --git a/app/Models/WebhookNotificationSettings.php b/app/Models/WebhookNotificationSettings.phpindex 7ffd20a8c3..9f2f1244cd 100644--- a/app/Models/WebhookNotificationSettings.php+++ b/app/Models/WebhookNotificationSettings.php@@ -2,12 +2,13 @@namespace App\Models;+use App\Traits\TracksDeliveryFailures;use Illuminate\Database\Eloquent\Model;use Illuminate\Notifications\Notifiable;class WebhookNotificationSettings extends Model{- use Notifiable;+ use Notifiable, TracksDeliveryFailures;public $timestamps = false;@@ -71,4 +72,14 @@ class WebhookNotificationSettings extends Model{return $this->webhook_enabled;}++ public function notificationChannelName(): string+ {+ return 'Webhook';+ }++ protected function disableNotificationChannel(): void+ {+ $this->webhook_enabled = false;+ }}diff --git a/app/Notifications/Channels/DiscordChannel.php b/app/Notifications/Channels/DiscordChannel.phpindex b4ba9bf8cc..4bd16a2d5b 100644--- a/app/Notifications/Channels/DiscordChannel.php+++ b/app/Notifications/Channels/DiscordChannel.php@@ -24,6 +24,6 @@ class DiscordChannel$message->isCritical = false;}- SendMessageToDiscordJob::dispatch($message, $discordSettings->discord_webhook_url);+ SendMessageToDiscordJob::dispatch($message, $discordSettings->discord_webhook_url, data_get($notification, 'isTestNotification', false) ? null : $discordSettings);}}diff --git a/app/Notifications/Channels/EmailChannel.php b/app/Notifications/Channels/EmailChannel.phpindex bcf976f815..9a14da58a1 100644--- a/app/Notifications/Channels/EmailChannel.php+++ b/app/Notifications/Channels/EmailChannel.php@@ -3,13 +3,16 @@namespace App\Notifications\Channels;use App\Exceptions\NonReportableException;+use App\Models\EmailNotificationSettings;use App\Models\Team;use App\Support\SmtpTransportFactory;use Exception;use Illuminate\Notifications\Notification;+use Illuminate\Support\Facades\Log;use Resend;use Resend\Exceptions\ErrorException;use Resend\Exceptions\TransporterException;+use Symfony\Component\Mailer\Exception\TransportExceptionInterface;use Symfony\Component\Mailer\Mailer;use Symfony\Component\Mime\Email;@@ -19,6 +22,10 @@ class EmailChannelpublic function send(SendsEmail $notifiable, Notification $notification): void{+ $settings = null;+ // Test notifications always show their error and never count as delivery failures.+ $isTestNotification = data_get($notification, 'isTestNotification', false);+try {// Get team and validate membership before proceeding$team = data_get($notifiable, 'id');@@ -49,8 +56,6 @@ class EmailChannel}// Skip team membership validation for test notifications- $isTestNotification = data_get($notification, 'isTestNotification', false);-if (! $isTestNotification) {foreach ($recipients as $recipient) {// Check if the recipient is part of the team@@ -103,6 +108,10 @@ class EmailChannel$mailer->send($email);}++ if (! $isTestNotification && $settings instanceof EmailNotificationSettings) {+ $settings->recordDeliverySuccess();+ }} catch (ErrorException $e) {['status' => $statusCode, 'name' => $errorName] = self::resendErrorDetails($e);@@ -116,6 +125,24 @@ class EmailChanneldefault => 'Failed to send email via Resend: '.$e->getErrorMessage(),};+ if (! $isTestNotification && $settings instanceof EmailNotificationSettings) {+ // The team's own settings: an invalid key or domain counts as a delivery failure, and+ // other client errors are only logged. Neither is retried, because a retry fails again.+ if (in_array($statusCode, [401, 403], true)) {+ $settings->recordDeliveryFailure($userMessage);++ return;+ }++ if (in_array($statusCode, [400, 422], true)) {+ Log::warning('Resend rejected the email', ['team_id' => $settings->team_id, 'reason' => $userMessage]);++ return;+ }++ throw new Exception($userMessage, $e->getCode(), $e);+ }+// Log detailed error for admin debugging (redact sensitive data)$emailSettings = $notifiable->emailNotificationSettings ?? instanceSettings();data_set($emailSettings, 'smtp_password', '********');@@ -135,6 +162,15 @@ class EmailChannel}throw new Exception($userMessage, $e->getCode(), $e);+ } catch (TransportExceptionInterface $e) {+ // SMTP 530/535: the SMTP server does not accept the username or password.+ if (! $isTestNotification && $settings instanceof EmailNotificationSettings && in_array($e->getCode(), [530, 535], true)) {+ $settings->recordDeliveryFailure('SMTP authentication failed. Please verify the SMTP username and password.');++ return;+ }++ throw $e;} catch (TransporterException $e) {send_internal_notification("Resend Transport Error: {$e->getMessage()}");throw new Exception('Unable to connect to Resend API. Please check your internet connection and try again.');diff --git a/app/Notifications/Channels/PushoverChannel.php b/app/Notifications/Channels/PushoverChannel.phpindex 3d3728d01f..1323d874e3 100644--- a/app/Notifications/Channels/PushoverChannel.php+++ b/app/Notifications/Channels/PushoverChannel.php@@ -16,6 +16,6 @@ class PushoverChannelreturn;}- SendMessageToPushoverJob::dispatch($message, $pushoverSettings->pushover_api_token, $pushoverSettings->pushover_user_key);+ SendMessageToPushoverJob::dispatch($message, $pushoverSettings->pushover_api_token, $pushoverSettings->pushover_user_key, data_get($notification, 'isTestNotification', false) ? null : $pushoverSettings);}}diff --git a/app/Notifications/Channels/SlackChannel.php b/app/Notifications/Channels/SlackChannel.phpindex cddb7a5611..82d235a269 100644--- a/app/Notifications/Channels/SlackChannel.php+++ b/app/Notifications/Channels/SlackChannel.php@@ -19,6 +19,6 @@ class SlackChannelreturn;}- SendMessageToSlackJob::dispatch($message, $slackSettings->slack_webhook_url);+ SendMessageToSlackJob::dispatch($message, $slackSettings->slack_webhook_url, data_get($notification, 'isTestNotification', false) ? null : $slackSettings);}}diff --git a/app/Notifications/Channels/TelegramChannel.php b/app/Notifications/Channels/TelegramChannel.phpindex d0e1c95ed9..5b23f54413 100644--- a/app/Notifications/Channels/TelegramChannel.php+++ b/app/Notifications/Channels/TelegramChannel.php@@ -73,6 +73,6 @@ class TelegramChannelreturn;}- SendMessageToTelegramJob::dispatch($message, $buttons, $telegramToken, $chatId, $threadId);+ SendMessageToTelegramJob::dispatch($message, $buttons, $telegramToken, $chatId, $threadId, data_get($notification, 'isTestNotification', false) ? null : $settings);}}diff --git a/app/Notifications/Channels/WebhookChannel.php b/app/Notifications/Channels/WebhookChannel.phpindex e735b88bb8..12b59dd600 100644--- a/app/Notifications/Channels/WebhookChannel.php+++ b/app/Notifications/Channels/WebhookChannel.php@@ -20,6 +20,6 @@ class WebhookChannel$payload = $notification->toWebhook();- SendWebhookJob::dispatch($payload, $webhookSettings->webhook_url);+ SendWebhookJob::dispatch($payload, $webhookSettings->webhook_url, data_get($notification, 'isTestNotification', false) ? null : $webhookSettings);}}diff --git a/app/Traits/HandlesNotificationResponse.php b/app/Traits/HandlesNotificationResponse.phpnew file mode 100644index 0000000000..77d8b8477d--- /dev/null+++ b/app/Traits/HandlesNotificationResponse.php@@ -0,0 +1,45 @@+<?php++namespace App\Traits;++use Illuminate\Http\Client\Response;+use Illuminate\Support\Facades\Log;+use Illuminate\Support\Str;+use RuntimeException;++/**+ * Shared response handling for the notification jobs. The job must have a+ * nullable `$settings` property with a model that uses TracksDeliveryFailures+ * (null for test notifications, so that they never count).+ */+trait HandlesNotificationResponse+{+ /**+ * Rate limits and server errors are retried by the queue. A configuration+ * error (invalid token, deleted webhook) counts as a delivery failure and+ * is not retried. Other client errors come from our payload: they are only+ * logged, because they must never disable the user's channel.+ */+ protected function handleNotificationResponse(Response $response, string $channel, bool $isConfigurationError): void+ {+ if ($response->successful()) {+ $this->settings?->recordDeliverySuccess();++ return;+ }++ if ($response->status() === 429 || $response->serverError()) {+ throw new RuntimeException("{$channel} notification failed with {$response->status()} status code.");+ }++ $reason = "{$channel} returned HTTP {$response->status()}: ".Str::limit($response->body(), 200);++ if ($isConfigurationError) {+ $this->settings?->recordDeliveryFailure($reason);++ return;+ }++ Log::warning("{$channel} rejected the notification", ['reason' => $reason]);+ }+}diff --git a/app/Traits/TracksDeliveryFailures.php b/app/Traits/TracksDeliveryFailures.phpnew file mode 100644index 0000000000..5b8c785437--- /dev/null+++ b/app/Traits/TracksDeliveryFailures.php@@ -0,0 +1,136 @@+<?php++namespace App\Traits;++use App\Notifications\Internal\GeneralNotification;+use Illuminate\Notifications\Messages\MailMessage;+use Illuminate\Support\Facades\Log;+use Illuminate\Support\Str;+use Throwable;++/**+ * Counts permanent delivery failures of a notification channel, the same way+ * servers count failed connection checks in `unreachable_count`.+ *+ * Only configuration errors (an invalid key, token, or URL) count. After+ * DELIVERY_FAILURE_THRESHOLD failures in a row the channel is disabled, and the+ * team admins and the instance admins are told one time. A successful delivery+ * or a change to the settings sets the counter back to zero.+ */+trait TracksDeliveryFailures+{+ public const DELIVERY_FAILURE_THRESHOLD = 3;++ /**+ * The channel name for messages, for example "Discord".+ */+ abstract public function notificationChannelName(): string;++ /**+ * Turn off the channel on the model, without saving.+ */+ abstract protected function disableNotificationChannel(): void;++ public static function bootTracksDeliveryFailures(): void+ {+ static::saving(function (self $settings) {+ if ($settings->exists && array_diff(array_keys($settings->getDirty()), ['failure_count']) !== []) {+ $settings->failure_count = 0;+ }+ });+ }++ public function recordDeliveryFailure(string $reason): void+ {+ $this->increment('failure_count');++ Log::warning('Notification delivery failed', [+ 'channel' => $this->notificationChannelName(),+ 'team_id' => $this->team_id,+ 'failure_count' => $this->failure_count,+ 'reason' => $reason,+ ]);++ if ($this->failure_count < self::DELIVERY_FAILURE_THRESHOLD || ! $this->isEnabled()) {+ return;+ }++ $this->disableNotificationChannel();+ $this->save();++ $channel = $this->notificationChannelName();++ auditLog('notifications.'.Str::lower($channel).'.auto_disabled', [+ 'team_id' => $this->team_id,+ 'reason' => $reason,+ ], 'warning');++ send_internal_notification(sprintf(+ "%s notifications disabled for team %s after %d failed deliveries.\nReason: %s",+ $channel,+ $this->team_id,+ self::DELIVERY_FAILURE_THRESHOLD,+ $reason+ ));++ $this->sendChannelDisabledEmail($reason);+ $this->notifyOtherChannels($reason);+ }++ public function recordDeliverySuccess(): void+ {+ if ($this->failure_count > 0) {+ $this->failure_count = 0;+ $this->save();+ }+ }++ /**+ * Tell the team through its other enabled channels. The disabled channel is+ * not in the list anymore, and the "general" event never uses email. The+ * root team already gets the internal notification on the same channels.+ */+ private function notifyOtherChannels(string $reason): void+ {+ if ($this->team_id === 0 || ! $this->team) {+ return;+ }++ $channel = $this->notificationChannelName();++ $this->team->notify(new GeneralNotification(+ "{$channel} notifications were disabled after ".self::DELIVERY_FAILURE_THRESHOLD." failed deliveries in a row.\nReason: {$reason}\nPlease fix the settings and enable {$channel} notifications again: ".base_url().'/notifications/'.Str::lower($channel),+ success: false,+ ));+ }++ /**+ * Tell the team admins through the instance email settings, because the+ * team's own channel does not work.+ */+ private function sendChannelDisabledEmail(string $reason): void+ {+ if (! is_transactional_emails_enabled() || ! $this->team) {+ return;+ }++ try {+ $channel = $this->notificationChannelName();+ $mail = new MailMessage;+ $mail->subject("Coolify: {$channel} notifications were disabled");+ $mail->view('emails.notification-channel-disabled', [+ 'channel' => $channel,+ 'failures' => self::DELIVERY_FAILURE_THRESHOLD,+ 'reason' => $reason,+ 'url' => base_url().'/notifications/'.Str::lower($channel),+ ]);++ $admins = $this->team->members()->wherePivotIn('role', ['admin', 'owner'])->get(['users.id', 'users.email']);+ foreach ($admins as $admin) {+ send_user_an_email($mail, $admin->email);+ }+ } catch (Throwable $e) {+ Log::warning('Failed to send notification channel disabled email: '.$e->getMessage());+ }+ }+}diff --git a/database/migrations/2026_10_12_121520_add_failure_count_to_notification_settings_tables.php b/database/migrations/2026_10_12_121520_add_failure_count_to_notification_settings_tables.phpnew file mode 100644index 0000000000..4e7e1c54f9--- /dev/null+++ b/database/migrations/2026_10_12_121520_add_failure_count_to_notification_settings_tables.php@@ -0,0 +1,45 @@+<?php++use Illuminate\Database\Migrations\Migration;+use Illuminate\Database\Schema\Blueprint;+use Illuminate\Support\Facades\Schema;++return new class extends Migration+{+ private array $tables = [+ 'email_notification_settings',+ 'discord_notification_settings',+ 'slack_notification_settings',+ 'telegram_notification_settings',+ 'pushover_notification_settings',+ 'webhook_notification_settings',+ ];++ /**+ * Run the migrations.+ */+ public function up(): void+ {+ foreach ($this->tables as $table) {+ if (! Schema::hasColumn($table, 'failure_count')) {+ Schema::table($table, function (Blueprint $table) {+ $table->integer('failure_count')->default(0);+ });+ }+ }+ }++ /**+ * Reverse the migrations.+ */+ public function down(): void+ {+ foreach ($this->tables as $table) {+ if (Schema::hasColumn($table, 'failure_count')) {+ Schema::table($table, function (Blueprint $table) {+ $table->dropColumn('failure_count');+ });+ }+ }+ }+};diff --git a/resources/views/emails/notification-channel-disabled.blade.php b/resources/views/emails/notification-channel-disabled.blade.phpnew file mode 100644index 0000000000..8faf5121c4--- /dev/null+++ b/resources/views/emails/notification-channel-disabled.blade.php@@ -0,0 +1,8 @@+<x-emails.layout>++Coolify could not send {{ $channel }} notifications for your team {{ $failures }} times in a row, so we disabled {{ $channel }} notifications.++Reason: {{ $reason }}++Please fix the settings and enable {{ $channel }} notifications again [here]({{ $url }}).+</x-emails.layout>diff --git a/tests/Feature/NotificationDeliveryFailureTest.php b/tests/Feature/NotificationDeliveryFailureTest.phpnew file mode 100644index 0000000000..64e31385c9--- /dev/null+++ b/tests/Feature/NotificationDeliveryFailureTest.php@@ -0,0 +1,166 @@+<?php++use App\Jobs\SendMessageToDiscordJob;+use App\Jobs\SendMessageToPushoverJob;+use App\Jobs\SendMessageToSlackJob;+use App\Jobs\SendMessageToTelegramJob;+use App\Jobs\SendWebhookJob;+use App\Models\InstanceSettings;+use App\Models\Team;+use App\Notifications\Channels\DiscordChannel;+use App\Notifications\Channels\SlackChannel;+use App\Notifications\Dto\DiscordMessage;+use App\Notifications\Dto\PushoverMessage;+use App\Notifications\Dto\SlackMessage;+use App\Notifications\Internal\GeneralNotification;+use App\Notifications\Test;+use Illuminate\Database\Eloquent\Model;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Notifications\Notification;+use Illuminate\Support\Facades\Http;+use Illuminate\Support\Facades\Notification as NotificationFacade;+use Illuminate\Support\Facades\Queue;++uses(RefreshDatabase::class);++beforeEach(function () {+ InstanceSettings::unguarded(fn () => InstanceSettings::query()->firstOrCreate(['id' => 0]));+ $this->team = Team::factory()->create();+});++/**+ * @return array{0: Model, 1: Closure(Model|null): object, 2: string}+ */+function deliveryFailureChannel(Team $team, string $channel): array+{+ return match ($channel) {+ 'discord' => [+ tap($team->discordNotificationSettings)->update(['discord_enabled' => true, 'discord_webhook_url' => 'https://example.com/discord']),+ fn ($settings) => new SendMessageToDiscordJob(new DiscordMessage('Title', 'Description', DiscordMessage::infoColor()), 'https://example.com/discord', $settings),+ 'discord_enabled',+ ],+ 'slack' => [+ tap($team->slackNotificationSettings)->update(['slack_enabled' => true, 'slack_webhook_url' => 'https://example.com/slack']),+ fn ($settings) => new SendMessageToSlackJob(new SlackMessage('Title', 'Description'), 'https://example.com/slack', $settings),+ 'slack_enabled',+ ],+ 'telegram' => [+ tap($team->telegramNotificationSettings)->update(['telegram_enabled' => true, 'telegram_token' => 'token', 'telegram_chat_id' => '123']),+ fn ($settings) => new SendMessageToTelegramJob('Message', [], 'token', '123', null, $settings),+ 'telegram_enabled',+ ],+ 'pushover' => [+ tap($team->pushoverNotificationSettings)->update(['pushover_enabled' => true, 'pushover_api_token' => 'token', 'pushover_user_key' => 'user']),+ fn ($settings) => new SendMessageToPushoverJob(new PushoverMessage('Title', 'Message'), 'token', 'user', $settings),+ 'pushover_enabled',+ ],+ 'webhook' => [+ tap($team->webhookNotificationSettings)->update(['webhook_enabled' => true, 'webhook_url' => 'https://example.com/webhook']),+ fn ($settings) => new SendWebhookJob(['event' => 'test'], 'https://example.com/webhook', $settings),+ 'webhook_enabled',+ ],+ };+}++dataset('configuration errors', [+ 'discord deleted webhook' => ['discord', 404, ['message' => 'Unknown Webhook', 'code' => 10015]],+ 'slack archived channel' => ['slack', 410, 'channel_is_archived'],+ 'telegram invalid token' => ['telegram', 401, ['ok' => false, 'description' => 'Unauthorized']],+ 'telegram wrong chat' => ['telegram', 400, ['ok' => false, 'description' => 'Bad Request: chat not found']],+ 'pushover invalid token' => ['pushover', 400, ['token' => 'invalid', 'errors' => ['application token is invalid'], 'status' => 0]],+ 'webhook not found' => ['webhook', 404, 'Not Found'],+]);++it('disables the channel after three configuration errors in a row', function (string $channel, int $status, array|string $body) {+ [$settings, $makeJob, $enabledColumn] = deliveryFailureChannel($this->team, $channel);+ Http::fake(['*' => Http::response($body, $status)]);++ $makeJob($settings->fresh())->handle();+ $makeJob($settings->fresh())->handle();+ expect($settings->fresh()->failure_count)->toBe(2)+ ->and($settings->fresh()->{$enabledColumn})->toBeTrue();++ $makeJob($settings->fresh())->handle();+ expect($settings->fresh()->{$enabledColumn})->toBeFalse();+})->with('configuration errors');++it('retries rate limits and server errors without counting them', function (string $channel, int $status) {+ [$settings, $makeJob] = deliveryFailureChannel($this->team, $channel);+ Http::fake(['*' => Http::response('error', $status)]);++ expect(fn () => $makeJob($settings->fresh())->handle())->toThrow(RuntimeException::class);+ expect($settings->fresh()->failure_count)->toBe(0);+})->with(['discord', 'slack', 'telegram', 'pushover', 'webhook'])->with([429, 500]);++it('does not count errors caused by the message payload', function (string $channel, array|string $body) {+ [$settings, $makeJob] = deliveryFailureChannel($this->team, $channel);+ Http::fake(['*' => Http::response($body, 400)]);++ $makeJob($settings->fresh())->handle();++ expect($settings->fresh()->failure_count)->toBe(0);+})->with([+ 'discord' => ['discord', ['message' => 'Invalid Form Body', 'code' => 50035]],+ 'slack' => ['slack', 'invalid_payload'],+ 'telegram' => ['telegram', ['ok' => false, 'description' => "Bad Request: can't parse entities"]],+ 'pushover' => ['pushover', ['message' => 'cannot be longer than 1024 characters', 'status' => 0]],+ 'webhook' => ['webhook', 'Bad Request'],+]);++it('resets the failure count after a successful delivery', function () {+ [$settings, $makeJob] = deliveryFailureChannel($this->team, 'discord');+ $settings->increment('failure_count', 2);+ Http::fake(['*' => Http::response('', 204)]);++ $makeJob($settings->fresh())->handle();++ expect($settings->fresh()->failure_count)->toBe(0);+});++it('resets the failure count when the user changes the settings', function () {+ [$settings] = deliveryFailureChannel($this->team, 'discord');+ $settings->increment('failure_count', 2);++ $settings->fresh()->update(['discord_webhook_url' => 'https://example.com/new-discord']);++ expect($settings->fresh()->failure_count)->toBe(0);+});++it('does not track test notifications', function () {+ deliveryFailureChannel($this->team, 'discord');+ Queue::fake();++ (new DiscordChannel)->send($this->team, new Test(channel: 'discord'));++ Queue::assertPushed(SendMessageToDiscordJob::class, fn (SendMessageToDiscordJob $job) => $job->settings === null);+});++it('tracks normal notifications', function () {+ [$settings] = deliveryFailureChannel($this->team, 'discord');+ Queue::fake();++ (new DiscordChannel)->send($this->team, new class extends Notification+ {+ public function toDiscord(): DiscordMessage+ {+ return new DiscordMessage('Title', 'Description', DiscordMessage::infoColor());+ }+ });++ Queue::assertPushed(SendMessageToDiscordJob::class, fn (SendMessageToDiscordJob $job) => $job->settings?->is($settings));+});++it('tells the team through its other enabled channels when a channel is disabled', function () {+ [$settings, $makeJob] = deliveryFailureChannel($this->team, 'discord');+ deliveryFailureChannel($this->team, 'slack');+ $settings->increment('failure_count', 2);+ Http::fake(['*' => Http::response(['message' => 'Unknown Webhook', 'code' => 10015], 404)]);+ NotificationFacade::fake();++ $makeJob($settings->fresh())->handle();++ NotificationFacade::assertSentTo($this->team, GeneralNotification::class, function (GeneralNotification $notification, array $channels) {+ return $channels === [SlackChannel::class]+ && str_contains($notification->message, 'Discord notifications were disabled');+ });+});diff --git a/tests/Unit/Notifications/Channels/EmailChannelTest.php b/tests/Unit/Notifications/Channels/EmailChannelTest.phpindex 57cc6a7ea1..f5918d6b00 100644--- a/tests/Unit/Notifications/Channels/EmailChannelTest.php+++ b/tests/Unit/Notifications/Channels/EmailChannelTest.php@@ -1,6 +1,7 @@<?phpuse App\Exceptions\NonReportableException;+use App\Models\InstanceSettings;use App\Models\Team;use App\Models\User;use App\Notifications\Channels\EmailChannel;@@ -13,6 +14,7 @@ use Tests\TestCase;uses(TestCase::class, RefreshDatabase::class);beforeEach(function () {+ InstanceSettings::forceCreate(['id' => 0]);$this->resendStub = null;$this->team = Team::factory()->create();$user = User::factory()->create(['email' => '[email protected]']);@@ -28,13 +30,24 @@ beforeEach(function () {]);$this->team->refresh();+ // A test notification shows every error to the user and never counts as a delivery failure.$this->notification = new class extends Notification{+ public bool $isTestNotification = true;+public function toMail(object $notifiable): MailMessage{return (new MailMessage)->subject('Test Email')->line('Test');}};++ $this->deliveryNotification = new class extends Notification+ {+ public function toMail(object $notifiable): MailMessage+ {+ return (new MailMessage)->subject('Deployment')->line('Deployed');+ }+ };});afterEach(function () {@@ -165,3 +178,52 @@ it('sends successfully when Resend returns an email id', function () {expect(fn () => (new EmailChannel)->send($this->team, $this->notification))->not->toThrow(Throwable::class);});++it('counts an invalid team Resend key as a delivery failure without a retry', function () {+ $this->resendStub = fakeResendResponse(401, ['statusCode' => 401, 'name' => 'validation_error', 'message' => 'API key is invalid']);++ (new EmailChannel)->send($this->team, $this->deliveryNotification);++ $settings = $this->team->emailNotificationSettings->fresh();+ expect($settings->failure_count)->toBe(1)+ ->and($settings->resend_enabled)->toBeTrue();+});++it('disables team Resend after three invalid key failures in a row', function () {+ $this->resendStub = fakeResendResponse(401, ['statusCode' => 401, 'name' => 'validation_error', 'message' => 'API key is invalid']);++ foreach (range(1, 3) as $attempt) {+ (new EmailChannel)->send($this->team, $this->deliveryNotification);+ }++ $settings = $this->team->emailNotificationSettings->fresh();+ expect($settings->resend_enabled)->toBeFalse()+ ->and($settings->failure_count)->toBe(0)+ ->and($settings->resend_api_key)->toBe('test_api_key');+});++it('retries temporary Resend errors without counting them', function () {+ $this->resendStub = fakeResendResponse(500, ['statusCode' => 500, 'name' => 'application_error', 'message' => 'Internal server error.']);++ expect(fn () => (new EmailChannel)->send($this->team, $this->deliveryNotification))+ ->toThrow(Exception::class, 'Failed to send email via Resend: Internal server error.');++ expect($this->team->emailNotificationSettings->fresh()->failure_count)->toBe(0);+});++it('does not count Resend errors for test emails', function () {+ $this->resendStub = fakeResendResponse(401, ['statusCode' => 401, 'name' => 'validation_error', 'message' => 'API key is invalid']);++ expect(fn () => (new EmailChannel)->send($this->team, $this->notification))->toThrow(NonReportableException::class);++ expect($this->team->emailNotificationSettings->fresh()->failure_count)->toBe(0);+});++it('resets the failure count after a successful delivery', function () {+ $this->team->emailNotificationSettings->increment('failure_count', 2);+ $this->resendStub = fakeResendResponse(200, ['id' => '49a3999c-0ce1-4ea6-ab68-afcd6dc2e794']);++ (new EmailChannel)->send($this->team, $this->deliveryNotification);++ expect($this->team->emailNotificationSettings->fresh()->failure_count)->toBe(0);+});