An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
feat(backups): add configurable missing volume backup alerts
Andras Bacsai407c384b09ae
diff --git a/app/Console/Kernel.php b/app/Console/Kernel.phpindex b13d8c99c4..a88ba64a81 100644--- a/app/Console/Kernel.php+++ b/app/Console/Kernel.php@@ -6,6 +6,7 @@ use App\Jobs\ApiTokenExpirationWarningJob;use App\Jobs\CheckForUpdatesJob;use App\Jobs\CheckHelperImageJob;use App\Jobs\CheckMissingDatabaseBackupsJob;+use App\Jobs\CheckMissingVolumeBackupsJob;use App\Jobs\CleanupInstanceStuffsJob;use App\Jobs\CleanupOrphanedPreviewContainersJob;use App\Jobs\CleanupStaleMultiplexedConnections;@@ -67,6 +68,7 @@ class Kernel extends ConsoleKernel->runInBackground();$this->scheduleInstance->job(new ApiTokenExpirationWarningJob)->hourly()->onOneServer();$this->scheduleInstance->job(new CheckMissingDatabaseBackupsJob)->hourly()->onOneServer();+ $this->scheduleInstance->job(new CheckMissingVolumeBackupsJob)->hourly()->onOneServer();$this->scheduleInstance->job(new RevalidateUnusableS3StoragesJob)->hourly()->onOneServer();$this->scheduleInstance->job(new ReconcileGithubRunnersJob)->everyMinute()->onOneServer();diff --git a/app/Http/Controllers/Api/VolumeBackupsController.php b/app/Http/Controllers/Api/VolumeBackupsController.phpindex 26ff938a10..e3b48a367a 100644--- a/app/Http/Controllers/Api/VolumeBackupsController.php+++ b/app/Http/Controllers/Api/VolumeBackupsController.php@@ -35,13 +35,14 @@ use RuntimeException;new OA\Property(property: 'retention_days_s3', type: 'integer', default: 0, maximum: 2147483647, minimum: 0),new OA\Property(property: 'retention_max_storage_s3', type: 'number', format: 'float', default: 0, maximum: 9999999999, minimum: 0),new OA\Property(property: 'timeout', type: 'integer', default: ScheduledVolumeBackup::DEFAULT_TIMEOUT, minimum: 60, maximum: 36000),+ new OA\Property(property: 'missing_backup_notification_days', type: 'integer', description: 'Alert after this many days without an execution; 0 disables alerts', default: 0, minimum: 0, maximum: 365),],type: 'object',additionalProperties: false,)]#[OA\Schema(schema: 'VolumeBackupScheduleResponse',- required: ['uuid', 'message', 'storage_uuid', 'storage_type', 'frequency', 'enabled', 'save_s3', 'disable_local_backup', 'stop_during_backup', 'retention_amount_locally', 'retention_days_locally', 'retention_max_storage_locally', 'retention_amount_s3', 'retention_days_s3', 'retention_max_storage_s3', 'timeout'],+ required: ['uuid', 'message', 'storage_uuid', 'storage_type', 'frequency', 'enabled', 'save_s3', 'disable_local_backup', 'stop_during_backup', 'retention_amount_locally', 'retention_days_locally', 'retention_max_storage_locally', 'retention_amount_s3', 'retention_days_s3', 'retention_max_storage_s3', 'timeout', 'missing_backup_notification_days'],properties: [new OA\Property(property: 'uuid', type: 'string'),new OA\Property(property: 'message', type: 'string'),@@ -60,6 +61,7 @@ use RuntimeException;new OA\Property(property: 'retention_days_s3', type: 'integer'),new OA\Property(property: 'retention_max_storage_s3', type: 'number', format: 'float'),new OA\Property(property: 'timeout', type: 'integer'),+ new OA\Property(property: 'missing_backup_notification_days', type: 'integer'),],type: 'object',)]@@ -197,6 +199,7 @@ class VolumeBackupsController extends Controller'retention_days_s3' => 'integer|min:0|max:2147483647','retention_max_storage_s3' => 'numeric|min:0|max:9999999999','timeout' => 'integer|min:60|max:36000',+ 'missing_backup_notification_days' => 'integer|min:0|max:365',]);$errors = $validator->errors();$allowedFields = [@@ -213,6 +216,7 @@ class VolumeBackupsController extends Controller'retention_days_s3','retention_max_storage_s3','timeout',+ 'missing_backup_notification_days',];foreach (array_diff(array_keys($request->all()), $allowedFields) as $field) {@@ -279,6 +283,9 @@ class VolumeBackupsController extends Controllerif ($request->has('timeout')) {$attributes['timeout'] = $request->integer('timeout');}+ if ($request->has('missing_backup_notification_days')) {+ $attributes['missing_backup_notification_days'] = $request->integer('missing_backup_notification_days');+ }$backup = $storage->scheduledBackups()->updateOrCreate([], $attributes);$created = $backup->wasRecentlyCreated;@@ -445,6 +452,7 @@ class VolumeBackupsController extends Controller'retention_days_s3' => $backup->retention_days_s3,'retention_max_storage_s3' => $backup->retention_max_storage_s3,'timeout' => $backup->timeout,+ 'missing_backup_notification_days' => $backup->missing_backup_notification_days ?? 0,];}diff --git a/app/Jobs/CheckMissingVolumeBackupsJob.php b/app/Jobs/CheckMissingVolumeBackupsJob.phpnew file mode 100644index 0000000000..1f8ab30b4f--- /dev/null+++ b/app/Jobs/CheckMissingVolumeBackupsJob.php@@ -0,0 +1,69 @@+<?php++namespace App\Jobs;++use App\Models\NotificationThrottle;+use App\Models\ScheduledVolumeBackup;+use App\Notifications\VolumeBackup\BackupMissing;+use Illuminate\Bus\Queueable;+use Illuminate\Contracts\Queue\ShouldBeEncrypted;+use Illuminate\Contracts\Queue\ShouldBeUnique;+use Illuminate\Contracts\Queue\ShouldQueue;+use Illuminate\Foundation\Bus\Dispatchable;+use Illuminate\Queue\InteractsWithQueue;+use Illuminate\Queue\SerializesModels;+use Illuminate\Support\Facades\Log;++class CheckMissingVolumeBackupsJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue+{+ use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;++ public $timeout = 1800;++ /**+ * Releases the unique lock of a killed worker after one run (the timeout), so the next hourly run is not+ * blocked until the queue retry_after.+ */+ public int $uniqueFor = 1800;++ public function handle(): void+ {+ ScheduledVolumeBackup::query()+ ->with(['team', 'latestExecution'])+ ->where('enabled', true)+ ->where('missing_backup_notification_days', '>', 0)+ ->chunkById(100, function ($backups): void {+ foreach ($backups as $backup) {+ $this->notifyIfMissing($backup);+ }+ });+ }++ private function notifyIfMissing(ScheduledVolumeBackup $backup): void+ {+ $lastExecutionAt = $backup->latestExecution?->created_at;+ $lastActivityAt = $lastExecutionAt ?? $backup->created_at;++ if (! $lastActivityAt || $lastActivityAt->isAfter(now()->subDays($backup->missing_backup_notification_days))) {+ return;+ }++ if (! $backup->team) {+ Log::warning("Cannot send missing volume backup notification for backup {$backup->id}: team not found");++ return;+ }++ if ($backup->team->getEnabledChannels('backup_failure') === []) {+ return;+ }++ // Send once for each period without backup activity.+ NotificationThrottle::sendOnce(+ $backup,+ BackupMissing::class,+ $lastActivityAt,+ fn () => $backup->team->notify(new BackupMissing($backup, $lastExecutionAt)),+ );+ }+}diff --git a/app/Livewire/Project/Shared/Storages/VolumeBackups.php b/app/Livewire/Project/Shared/Storages/VolumeBackups.phpindex 215c8a68f9..d741bcc0e4 100644--- a/app/Livewire/Project/Shared/Storages/VolumeBackups.php+++ b/app/Livewire/Project/Shared/Storages/VolumeBackups.php@@ -61,6 +61,8 @@ class VolumeBackups extends Componentpublic int $timeout = ScheduledVolumeBackup::DEFAULT_TIMEOUT;+ public int $missingBackupNotificationDays = 0;+public int $perPage = 10;public function updatedPerPage(): void@@ -94,6 +96,7 @@ class VolumeBackups extends Component'retentionDaysS3' => ['required', 'integer', 'min:0'],'retentionMaxStorageS3' => ['required', 'numeric', 'min:0'],'timeout' => ['required', 'integer', 'min:60', 'max:36000'],+ 'missingBackupNotificationDays' => ['required', 'integer', 'min:0', 'max:365'],];}@@ -123,6 +126,7 @@ class VolumeBackups extends Component$this->retentionDaysS3 = $this->backup->retention_days_s3;$this->retentionMaxStorageS3 = $this->backup->retention_max_storage_s3;$this->timeout = $this->backup->timeout;+ $this->missingBackupNotificationDays = $this->backup->missing_backup_notification_days;} else {$this->s3StorageId = $this->availableS3Storages->first()?->id;}@@ -435,6 +439,7 @@ class VolumeBackups extends Component'retention_days_s3' => $this->retentionDaysS3,'retention_max_storage_s3' => $this->retentionMaxStorageS3,'timeout' => $this->timeout,+ 'missing_backup_notification_days' => $this->missingBackupNotificationDays,]);$changedFields = auditChangedFields($backup);$backup->save();diff --git a/app/Models/ScheduledVolumeBackup.php b/app/Models/ScheduledVolumeBackup.phpindex 7f33fd92b9..9ad676d953 100644--- a/app/Models/ScheduledVolumeBackup.php+++ b/app/Models/ScheduledVolumeBackup.php@@ -31,6 +31,7 @@ class ScheduledVolumeBackup extends BaseModel'retention_days_s3','retention_max_storage_s3','timeout',+ 'missing_backup_notification_days',];protected function casts(): array@@ -47,6 +48,7 @@ class ScheduledVolumeBackup extends BaseModel'retention_days_s3' => 'integer','retention_max_storage_s3' => 'float','timeout' => 'integer',+ 'missing_backup_notification_days' => 'integer',];}diff --git a/app/Notifications/Channels/TelegramChannel.php b/app/Notifications/Channels/TelegramChannel.phpindex 118ad4269c..d0e1c95ed9 100644--- a/app/Notifications/Channels/TelegramChannel.php+++ b/app/Notifications/Channels/TelegramChannel.php@@ -11,6 +11,7 @@ use App\Notifications\Container\ContainerRestarted;use App\Notifications\Database\BackupFailed;use App\Notifications\Database\BackupMissing;use App\Notifications\Database\BackupSuccess;+use App\Notifications\Database\BackupSuccessWithS3Warning;use App\Notifications\ScheduledTask\TaskFailed;use App\Notifications\ScheduledTask\TaskSuccess;use App\Notifications\Server\DockerCleanupFailed;@@ -20,6 +21,10 @@ use App\Notifications\Server\Reachable;use App\Notifications\Server\ServerPatchCheck;use App\Notifications\Server\TraefikVersionOutdated;use App\Notifications\Server\Unreachable;+use App\Notifications\VolumeBackup\BackupFailed as VolumeBackupFailed;+use App\Notifications\VolumeBackup\BackupMissing as VolumeBackupMissing;+use App\Notifications\VolumeBackup\BackupSuccess as VolumeBackupSuccess;+use App\Notifications\VolumeBackup\RecoveryFailed as VolumeBackupRecoveryFailed;class TelegramChannel{@@ -41,8 +46,15 @@ class TelegramChannelRestartLimitReached::class => $settings->telegram_notifications_restart_limit_reached_thread_id,BackupSuccess::class => $settings->telegram_notifications_backup_success_thread_id,+ VolumeBackupSuccess::class => $notification->warning+ ? $settings->telegram_notifications_backup_failure_thread_id+ : $settings->telegram_notifications_backup_success_thread_id,BackupFailed::class,- BackupMissing::class => $settings->telegram_notifications_backup_failure_thread_id,+ BackupMissing::class,+ BackupSuccessWithS3Warning::class,+ VolumeBackupFailed::class,+ VolumeBackupMissing::class,+ VolumeBackupRecoveryFailed::class => $settings->telegram_notifications_backup_failure_thread_id,TaskSuccess::class => $settings->telegram_notifications_scheduled_task_success_thread_id,TaskFailed::class => $settings->telegram_notifications_scheduled_task_failure_thread_id,diff --git a/app/Notifications/VolumeBackup/BackupMissing.php b/app/Notifications/VolumeBackup/BackupMissing.phpnew file mode 100644index 0000000000..63282e3c13--- /dev/null+++ b/app/Notifications/VolumeBackup/BackupMissing.php@@ -0,0 +1,117 @@+<?php++namespace App\Notifications\VolumeBackup;++use App\Models\ScheduledVolumeBackup;+use App\Models\ServiceApplication;+use App\Models\ServiceDatabase;+use App\Notifications\CustomEmailNotification;+use App\Notifications\Dto\DiscordMessage;+use App\Notifications\Dto\PushoverMessage;+use App\Notifications\Dto\SlackMessage;+use Illuminate\Notifications\Messages\MailMessage;+use Illuminate\Support\Carbon;++class BackupMissing extends CustomEmailNotification+{+ public string $resourceName;++ public string $target;++ public ?string $url = null;++ public function __construct(public ScheduledVolumeBackup $backup, public ?Carbon $lastExecutionAt)+ {+ $this->onQueue('high');+ $resource = $backup->targetResource();+ $this->resourceName = $resource?->name ?? 'Unknown resource';+ $this->target = $backup->targetType().' '.$backup->targetName();++ $linkable = $resource instanceof ServiceApplication || $resource instanceof ServiceDatabase ? $resource->service : $resource;+ if ($linkable && method_exists($linkable, 'link')) {+ $this->url = $linkable->link();+ }+ }++ public function via(object $notifiable): array+ {+ return $notifiable->getEnabledChannels('backup_failure');+ }++ public function summary(): string+ {+ return "The enabled storage backup schedule for {$this->target} ({$this->resourceName}) has produced no executions in the last {$this->backup->missing_backup_notification_days} day(s).";+ }++ public function toMail(): MailMessage+ {+ return (new MailMessage)+ ->subject("Coolify: [ACTION REQUIRED] No recent storage backup for {$this->resourceName}")+ ->view('emails.volume-backup-missing', [+ 'summary' => $this->summary(),+ 'last_execution_at' => $this->lastExecutionAt?->toDateTimeString(),+ 'url' => $this->url,+ ]);+ }++ public function toDiscord(): DiscordMessage+ {+ $message = new DiscordMessage(+ title: ':warning: Scheduled storage backup missing',+ description: $this->summary(),+ color: DiscordMessage::errorColor(),+ isCritical: true,+ );++ if ($this->url) {+ $message->addField('Resource', '[Link]('.$this->url.')');+ }++ return $message;+ }++ public function toTelegram(): array+ {+ $data = ['message' => "Coolify: {$this->summary()}"];++ if ($this->url) {+ $data['buttons'] = [['text' => 'Open resource in Coolify', 'url' => $this->url]];+ }++ return $data;+ }++ public function toPushover(): PushoverMessage+ {+ return new PushoverMessage(+ title: 'Scheduled storage backup missing',+ level: 'error',+ message: $this->summary(),+ buttons: $this->url ? [['text' => 'Open resource in Coolify', 'url' => $this->url]] : [],+ );+ }++ public function toSlack(): SlackMessage+ {+ return new SlackMessage(+ title: 'Scheduled storage backup missing',+ description: $this->summary().($this->url ? "\n\n<{$this->url}|Open resource in Coolify>" : ''),+ color: SlackMessage::errorColor(),+ );+ }++ public function toWebhook(): array+ {+ return [+ 'success' => false,+ 'message' => 'Scheduled storage backup missing',+ 'event' => 'volume_backup_missing',+ 'resource_name' => $this->resourceName,+ 'storage' => $this->target,+ 'backup_uuid' => $this->backup->uuid,+ 'days' => $this->backup->missing_backup_notification_days,+ 'last_execution_at' => $this->lastExecutionAt?->toDateTimeString(),+ 'url' => $this->url,+ ];+ }+}diff --git a/app/Services/ScheduledJobDeliveryService.php b/app/Services/ScheduledJobDeliveryService.phpindex 8609b78b80..0d5ddd633a 100644--- a/app/Services/ScheduledJobDeliveryService.php+++ b/app/Services/ScheduledJobDeliveryService.php@@ -2,6 +2,7 @@namespace App\Services;+use App\Events\BackupCreated;use App\Events\DockerCleanupDone;use App\Events\ScheduledTaskDone;use App\Jobs\DatabaseBackupJob;@@ -11,15 +12,19 @@ use App\Jobs\VolumeBackupJob;use App\Models\DockerCleanupExecution;use App\Models\NotificationThrottle;use App\Models\ScheduledDatabaseBackup;+use App\Models\ScheduledDatabaseBackupExecution;use App\Models\ScheduledJobDelivery;use App\Models\ScheduledJobState;use App\Models\ScheduledTask;use App\Models\ScheduledTaskExecution;use App\Models\ScheduledVolumeBackup;+use App\Models\ScheduledVolumeBackupExecution;use App\Models\Server;use App\Models\Team;+use App\Notifications\Database\BackupFailed;use App\Notifications\ScheduledTask\TaskFailed;use App\Notifications\Server\DockerCleanupFailed;+use App\Notifications\VolumeBackup\BackupFailed as VolumeBackupFailed;use Closure;use Cron\CronExpression;use Illuminate\Database\Eloquent\Model;@@ -353,7 +358,7 @@ class ScheduledJobDeliveryService}/**- * Show a missed task or Docker cleanup as a failed execution and send the failure notification.+ * Show a missed task, Docker cleanup, or backup as a failed execution and send the failure notification.* The caller moved the occurrence out of `enqueued` atomically, so this runs once per occurrence,* and claim() rejects the late job if it starts afterwards.*/@@ -391,6 +396,38 @@ class ScheduledJobDeliveryService]);event(new DockerCleanupDone($execution));$this->notifyMissedOccurrence($server, fn () => $server->team?->notify(new DockerCleanupFailed($server, "Docker cleanup job failed with the following error: {$message}")));+ } elseif ($occurrence->job_type === 'database-backup') {+ $backup = ScheduledDatabaseBackup::with(['team', 'database'])->find($occurrence->resource_id);+ if (! $backup?->database) {+ return;+ }++ $databaseName = $backup->dump_all ? 'all' : $backup->databases_to_backup;+ ScheduledDatabaseBackupExecution::create([+ 'uuid' => new_public_id(),+ 'database_name' => filled($databaseName) ? str($databaseName)->limit(250)->toString() : null,+ 'scheduled_database_backup_id' => $backup->id,+ 'status' => 'failed',+ 'message' => $message,+ 'local_storage_deleted' => false,+ 'finished_at' => now(),+ ]);+ BackupCreated::dispatch($backup->team_id);+ $this->notifyMissedOccurrence($backup, fn () => $backup->team?->notify(new BackupFailed($backup, $backup->database, $message, $databaseName ?? 'unknown')));+ } elseif ($occurrence->job_type === 'volume-backup') {+ $backup = ScheduledVolumeBackup::with(['team', 'backupable.resource'])->find($occurrence->resource_id);+ if (! $backup) {+ return;+ }++ ScheduledVolumeBackupExecution::create([+ 'scheduled_volume_backup_id' => $backup->id,+ 'status' => 'failed',+ 'message' => $message,+ 'finished_at' => now(),+ ]);+ BackupCreated::dispatch($backup->team_id);+ $this->notifyMissedOccurrence($backup, fn () => $backup->team?->notify(new VolumeBackupFailed($backup, $message)));}} catch (\Throwable $e) {Log::channel('scheduled-errors')->error('Failed to report missed scheduled occurrence', [diff --git a/database/migrations/2026_10_07_072758_add_missing_backup_notification_days_to_scheduled_volume_backups_table.php b/database/migrations/2026_10_07_072758_add_missing_backup_notification_days_to_scheduled_volume_backups_table.phpnew file mode 100644index 0000000000..b3aae9916b--- /dev/null+++ b/database/migrations/2026_10_07_072758_add_missing_backup_notification_days_to_scheduled_volume_backups_table.php@@ -0,0 +1,22 @@+<?php++use Illuminate\Database\Migrations\Migration;+use Illuminate\Database\Schema\Blueprint;+use Illuminate\Support\Facades\Schema;++return new class extends Migration+{+ public function up(): void+ {+ Schema::table('scheduled_volume_backups', function (Blueprint $table) {+ $table->unsignedInteger('missing_backup_notification_days')->default(0);+ });+ }++ public function down(): void+ {+ Schema::table('scheduled_volume_backups', function (Blueprint $table) {+ $table->dropColumn('missing_backup_notification_days');+ });+ }+};diff --git a/resources/views/emails/volume-backup-missing.blade.php b/resources/views/emails/volume-backup-missing.blade.phpnew file mode 100644index 0000000000..e9952ff24a--- /dev/null+++ b/resources/views/emails/volume-backup-missing.blade.php@@ -0,0 +1,13 @@+<x-emails.layout>+{{ $summary }}++@if ($last_execution_at)+The last execution was at {{ $last_execution_at }}.+@else+This schedule has never produced an execution.+@endif++@if ($url)+Click [here]({{ $url }}) to open the resource.+@endif+</x-emails.layout>diff --git a/resources/views/livewire/project/shared/storages/volume-backups/general.blade.php b/resources/views/livewire/project/shared/storages/volume-backups/general.blade.phpindex 5b4ece52f1..a48b6cb303 100644--- a/resources/views/livewire/project/shared/storages/volume-backups/general.blade.php+++ b/resources/views/livewire/project/shared/storages/volume-backups/general.blade.php@@ -44,6 +44,9 @@helper="Uses the backup server timezone, or the instance timezone when none is configured." required /><x-forms.input id="timeout" type="number" min="60" max="36000" label="Timeout"helper="Maximum backup runtime in seconds." required />+ <x-forms.input label="Missing backup alert after" id="missingBackupNotificationDays" type="number"+ min="0" max="365" suffix="days"+ helper="Notify through backup failure channels after this many days without an execution. Use 0 to disable." required /></div></x-application.settings-section></form>diff --git a/tests/Feature/Api/VolumeBackupScheduleApiTest.php b/tests/Feature/Api/VolumeBackupScheduleApiTest.phpindex e60f2919da..e494261d51 100644--- a/tests/Feature/Api/VolumeBackupScheduleApiTest.php+++ b/tests/Feature/Api/VolumeBackupScheduleApiTest.php@@ -101,9 +101,12 @@ it('sets an application volume backup schedule through the API', function () {'retention_days_s3' => 7,'retention_max_storage_s3' => 8.5,'timeout' => 600,+ 'missing_backup_notification_days' => 5,]);- $response->assertCreated()->assertJsonStructure(['uuid', 'message']);+ $response->assertCreated()+ ->assertJsonStructure(['uuid', 'message'])+ ->assertJsonPath('missing_backup_notification_days', 5);$backup = ScheduledVolumeBackup::query()->sole();expect($backup->backupable->is($this->volume))->toBeTrue()@@ -120,7 +123,20 @@ it('sets an application volume backup schedule through the API', function () {->and($backup->retention_amount_s3)->toBe(6)->and($backup->retention_days_s3)->toBe(7)->and($backup->retention_max_storage_s3)->toBe(8.5)- ->and($backup->timeout)->toBe(600);+ ->and($backup->timeout)->toBe(600)+ ->and($backup->missing_backup_notification_days)->toBe(5);+});++it('rejects a missing backup alert period above one year through the API', function () {+ $this->withHeaders($this->headers)+ ->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups", [+ 'frequency' => 'daily',+ 'missing_backup_notification_days' => 366,+ ])+ ->assertUnprocessable()+ ->assertJsonValidationErrors('missing_backup_notification_days');++ expect(ScheduledVolumeBackup::query()->exists())->toBeFalse();});it('updates the existing backup schedule instead of creating another one', function () {diff --git a/tests/Feature/MissingVolumeBackupNotificationTest.php b/tests/Feature/MissingVolumeBackupNotificationTest.phpnew file mode 100644index 0000000000..18f5c05c5f--- /dev/null+++ b/tests/Feature/MissingVolumeBackupNotificationTest.php@@ -0,0 +1,109 @@+<?php++use App\Jobs\CheckMissingVolumeBackupsJob;+use App\Models\LocalPersistentVolume;+use App\Models\NotificationThrottle;+use App\Models\ScheduledVolumeBackup;+use App\Models\ScheduledVolumeBackupExecution;+use App\Models\Team;+use App\Notifications\VolumeBackup\BackupMissing;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Carbon;+use Illuminate\Support\Facades\Notification;++uses(RefreshDatabase::class);++afterEach(fn () => Carbon::setTestNow());++function missingVolumeBackupSchedule(Team $team, array $attributes = []): ScheduledVolumeBackup+{+ $backup = ScheduledVolumeBackup::create(array_merge([+ 'backupable_type' => (new LocalPersistentVolume)->getMorphClass(),+ 'backupable_id' => ScheduledVolumeBackup::query()->max('backupable_id') + 1000,+ 'team_id' => $team->id,+ 'frequency' => '0 0 * * *',+ 'enabled' => true,+ 'missing_backup_notification_days' => 2,+ ], $attributes));++ ScheduledVolumeBackup::whereKey($backup->id)->update(['created_at' => now()->subDays(3)]);++ return $backup->refresh();+}++function teamWithVolumeBackupFailureNotifications(): Team+{+ $team = Team::create(['name' => 'Test team']);+ $team->emailNotificationSettings->update([+ 'smtp_enabled' => true,+ 'backup_failure_email_notifications' => true,+ ]);++ return $team;+}++it('notifies the team when an enabled volume backup has no executions for the configured days', function () {+ Notification::fake();+ $team = teamWithVolumeBackupFailureNotifications();+ $backup = missingVolumeBackupSchedule($team);++ (new CheckMissingVolumeBackupsJob)->handle();++ Notification::assertSentTo($team, BackupMissing::class, fn (BackupMissing $notification) => $notification->backup->is($backup)+ && $notification->lastExecutionAt === null+ && $notification->toWebhook()['event'] === 'volume_backup_missing');+ expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeTrue()+ ->and((string) (new BackupMissing($backup, null))->toMail()->render())->toContain('This schedule has never produced an execution.');+});++it('does not notify for recent, disabled or unconfigured volume backup schedules', function () {+ Notification::fake();+ $team = teamWithVolumeBackupFailureNotifications();+ missingVolumeBackupSchedule($team, ['enabled' => false]);+ missingVolumeBackupSchedule($team, ['missing_backup_notification_days' => 0]);+ $recentlyCreated = missingVolumeBackupSchedule($team);+ ScheduledVolumeBackup::whereKey($recentlyCreated->id)->update(['created_at' => now()->subDay()]);+ $recentlyExecuted = missingVolumeBackupSchedule($team);+ ScheduledVolumeBackupExecution::create([+ 'scheduled_volume_backup_id' => $recentlyExecuted->id,+ 'status' => 'success',+ ]);++ (new CheckMissingVolumeBackupsJob)->handle();++ Notification::assertNothingSent();+});++it('does not notify when the team has no backup failure channels', function () {+ Notification::fake();+ $team = Team::create(['name' => 'Test team']);+ $backup = missingVolumeBackupSchedule($team);++ (new CheckMissingVolumeBackupsJob)->handle();++ Notification::assertNothingSent();+ expect(NotificationThrottle::wasSent($backup, BackupMissing::class))->toBeFalse();+});++it('notifies once per period without volume backup executions and rearms after another execution', function () {+ Notification::fake();+ $team = teamWithVolumeBackupFailureNotifications();+ $backup = missingVolumeBackupSchedule($team);++ (new CheckMissingVolumeBackupsJob)->handle();+ (new CheckMissingVolumeBackupsJob)->handle();++ Notification::assertSentToTimes($team, BackupMissing::class, 1);++ Carbon::setTestNow(now()->addMinute());+ $execution = ScheduledVolumeBackupExecution::create([+ 'scheduled_volume_backup_id' => $backup->id,+ 'status' => 'success',+ ]);+ Carbon::setTestNow(now()->addDays(3));++ (new CheckMissingVolumeBackupsJob)->handle();++ Notification::assertSentToTimes($team, BackupMissing::class, 2);+ Notification::assertSentTo($team, BackupMissing::class, fn (BackupMissing $notification) => $notification->lastExecutionAt?->equalTo($execution->created_at) === true);+});diff --git a/tests/Feature/ScheduledInstanceJobsUniqueLockTest.php b/tests/Feature/ScheduledInstanceJobsUniqueLockTest.phpindex 4ee8d66780..4da77c49d4 100644--- a/tests/Feature/ScheduledInstanceJobsUniqueLockTest.php+++ b/tests/Feature/ScheduledInstanceJobsUniqueLockTest.php@@ -1,6 +1,7 @@<?phpuse App\Jobs\CheckMissingDatabaseBackupsJob;+use App\Jobs\CheckMissingVolumeBackupsJob;use App\Jobs\CleanupInstanceStuffsJob;use App\Jobs\CleanupOrphanedPreviewContainersJob;use Illuminate\Support\Facades\Queue;@@ -14,6 +15,7 @@ dataset('scheduled instance jobs', ['orphaned preview containers cleanup' => [CleanupOrphanedPreviewContainersJob::class, 600],'instance cleanup' => [CleanupInstanceStuffsJob::class, 60],'missing database backups check' => [CheckMissingDatabaseBackupsJob::class, 1800],+ 'missing volume backups check' => [CheckMissingVolumeBackupsJob::class, 1800],]);it('does not queue the job again while its unique lock is held', function (string $jobClass, int $lockSeconds) {diff --git a/tests/Feature/ScheduledJobManagerDispatchTest.php b/tests/Feature/ScheduledJobManagerDispatchTest.phpindex 82f024c848..d09bb865a1 100644--- a/tests/Feature/ScheduledJobManagerDispatchTest.php+++ b/tests/Feature/ScheduledJobManagerDispatchTest.php@@ -13,6 +13,7 @@ use App\Models\InstanceSettings;use App\Models\PrivateKey;use App\Models\Project;use App\Models\ScheduledDatabaseBackup;+use App\Models\ScheduledDatabaseBackupExecution;use App\Models\ScheduledJobDelivery;use App\Models\ScheduledJobState;use App\Models\ScheduledTask;@@ -24,6 +25,7 @@ use App\Models\ServiceDatabase;use App\Models\StandaloneDocker;use App\Models\StandalonePostgresql;use App\Models\Team;+use App\Notifications\Database\BackupFailed;use App\Notifications\ScheduledTask\TaskFailed;use App\Notifications\Server\DockerCleanupFailed;use App\Services\ScheduledJobDeliveryService;@@ -752,6 +754,31 @@ it('records a missed Docker cleanup as a failed execution and notifies the teamEvent::assertDispatched(DockerCleanupDone::class, 1);});+it('records a missed database backup that is too old to run late as a failed execution and notifies the team once', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));+ Queue::fake();+ Notification::fake();+ Event::fake([BackupCreated::class]);+ $backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);+ $backup->team->emailNotificationSettings->update(['smtp_enabled' => true, 'backup_failure_email_notifications' => true]);+ $occurrence = createStaleEnqueuedOccurrence("scheduled-backup:{$backup->id}", 'database-backup', $backup->id, Carbon::create(2026, 9, 16, 0, 0, 0, 'UTC'));++ app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();+ app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();++ $executions = ScheduledDatabaseBackupExecution::query()->where('scheduled_database_backup_id', $backup->id)->get();+ expect($executions)->toHaveCount(1)+ ->and($executions->first()->status)->toBe('failed')+ ->and($executions->first()->message)->toBe('Skipped: the queued job did not start within 60 minutes.')+ ->and($executions->first()->finished_at)->not->toBeNull()+ ->and($occurrence->fresh()->status)->toBe('failed');+ Queue::assertNotPushed(DatabaseBackupJob::class);+ Notification::assertSentToTimes($backup->team, BackupFailed::class, 1);+ Notification::assertSentTo($backup->team, BackupFailed::class, fn (BackupFailed $notification) => $notification->output === 'Skipped: the queued job did not start within 60 minutes.');+ Event::assertDispatched(BackupCreated::class);+});+it('sends at most one missed-run notification per hour for the same schedule', function (string $jobType) {config(['constants.coolify.self_hosted' => true]);Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));diff --git a/tests/Feature/TelegramNotificationThreadRoutingTest.php b/tests/Feature/TelegramNotificationThreadRoutingTest.phpindex 474d2b157d..00a8bc327a 100644--- a/tests/Feature/TelegramNotificationThreadRoutingTest.php+++ b/tests/Feature/TelegramNotificationThreadRoutingTest.php@@ -2,11 +2,14 @@use App\Jobs\SendMessageToTelegramJob;use App\Models\InstanceSettings;+use App\Models\ScheduledVolumeBackup;use App\Models\Server;use App\Models\Team;use App\Notifications\Channels\TelegramChannel;use App\Notifications\Server\ServerPatchCheck;use App\Notifications\Server\TraefikVersionOutdated;+use App\Notifications\VolumeBackup\BackupFailed;+use App\Notifications\VolumeBackup\BackupSuccess;use Illuminate\Foundation\Testing\RefreshDatabase;use Illuminate\Support\Facades\Queue;@@ -87,3 +90,22 @@ it('falls back to the main chat when no topic is configured', function () {fn (SendMessageToTelegramJob $job) => $job->threadId === null);});++it('sends volume backup notifications to the backup topics', function (Closure $makeNotification, string $threadId) {+ $this->team->telegramNotificationSettings->update([+ 'telegram_notifications_backup_success_thread_id' => '11',+ 'telegram_notifications_backup_failure_thread_id' => '12',+ ]);+ $backup = new ScheduledVolumeBackup(['frequency' => 'daily']);++ (new TelegramChannel)->send($this->team->fresh(), $makeNotification($backup));++ Queue::assertPushed(+ SendMessageToTelegramJob::class,+ fn (SendMessageToTelegramJob $job) => $job->threadId === $threadId+ );+})->with([+ 'success' => [fn (ScheduledVolumeBackup $backup) => new BackupSuccess($backup), '11'],+ 'success with warning' => [fn (ScheduledVolumeBackup $backup) => new BackupSuccess($backup, 'S3 upload failed'), '12'],+ 'failed' => [fn (ScheduledVolumeBackup $backup) => new BackupFailed($backup, 'Archive command failed'), '12'],+]);diff --git a/tests/Feature/VolumeBackupTest.php b/tests/Feature/VolumeBackupTest.phpindex 75fbe6b9c6..d247a7a4e6 100644--- a/tests/Feature/VolumeBackupTest.php+++ b/tests/Feature/VolumeBackupTest.php@@ -18,6 +18,7 @@ use App\Models\LocalPersistentVolume;use App\Models\PrivateKey;use App\Models\Project;use App\Models\S3Storage;+use App\Models\ScheduledJobDelivery;use App\Models\ScheduledVolumeBackup;use App\Models\ScheduledVolumeBackupExecution;use App\Models\Server;@@ -29,6 +30,7 @@ use App\Models\Team;use App\Models\User;use App\Notifications\VolumeBackup\BackupFailed;use App\Notifications\VolumeBackup\BackupSuccess;+use App\Services\ScheduledJobDeliveryService;use Illuminate\Database\Eloquent\Collection;use Illuminate\Filesystem\FilesystemAdapter;use Illuminate\Foundation\Testing\RefreshDatabase;@@ -1142,6 +1144,19 @@ it('does not change S3 storage when another volume backup setting is invalid', f->and($backup->frequency)->toBe('daily');});+it('rejects a missing volume backup alert period above one year', function () {+ $team = Team::factory()->create();+ signInForVolumeBackups($this, $team);+ [$application, $volume] = createVolumeBackupApplication($team);++ Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application])+ ->set('missingBackupNotificationDays', 366)+ ->call('save')+ ->assertHasErrors(['missingBackupNotificationDays' => 'max']);++ expect(ScheduledVolumeBackup::query()->exists())->toBeFalse();+});+function createVolumeBackupApplication(Team $team): array{InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));@@ -1208,12 +1223,14 @@ it('creates a local scheduled backup for a persistent volume', function () {->set('retentionDaysLocally', 14)->set('retentionMaxStorageLocally', 1.5)->set('stopDuringBackup', true)+ ->set('missingBackupNotificationDays', 3)->call('save')->assertDispatched('success');$backup = ScheduledVolumeBackup::query()->sole();expect($backup->backupable->is($volume))->toBeTrue()+ ->and($backup->missing_backup_notification_days)->toBe(3)->and($backup->team_id)->toBe($team->id)->and($backup->frequency)->toBe('daily')->and($backup->retention_amount_locally)->toBe(5)@@ -2850,3 +2867,37 @@ it('sends volume backup success with a warning through the backup failure channe'without warning' => [null, 'backup_success'],'with warning' => ['S3 upload failed: denied', 'backup_failure'],]);++it('records a missed volume backup that is too old to run late as a failed execution and notifies the team once', function () {+ config(['broadcasting.default' => 'null']);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));+ Queue::fake();+ Notification::fake();+ $team = Team::factory()->create();+ $team->discordNotificationSettings()->update(['discord_enabled' => true, 'backup_failure_discord_notifications' => true]);+ [$application, $volume] = createVolumeBackupApplication($team);+ $backup = $volume->scheduledBackups()->create([+ 'team_id' => $team->id,+ 'frequency' => 'daily',+ ]);+ $occurrence = ScheduledJobDelivery::create([+ 'schedule_key' => "scheduled-volume-backup:{$backup->id}",+ 'scheduled_for' => Carbon::create(2026, 9, 16, 0, 0, 0, 'UTC'),+ 'job_type' => 'volume-backup',+ 'resource_id' => $backup->id,+ 'status' => 'enqueued',+ 'enqueued_at' => now()->subMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1),+ ]);++ app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();+ app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();++ $execution = ScheduledVolumeBackupExecution::query()->sole();+ expect($execution->status)->toBe('failed')+ ->and($execution->message)->toBe('Skipped: the queued job did not start within 60 minutes.')+ ->and($execution->finished_at)->not->toBeNull()+ ->and($occurrence->fresh()->status)->toBe('failed');+ Queue::assertNotPushed(VolumeBackupJob::class);+ Notification::assertSentToTimes($team, BackupFailed::class, 1);+ Notification::assertSentTo($team, BackupFailed::class, fn (BackupFailed $notification) => $notification->output === 'Skipped: the queued job did not start within 60 minutes.');+});