An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(backups): make overlapping and concurrent backup runs safe
Andras Bacsai1c68eafa8499
diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.phpindex 890e14462e..630e1153fc 100644--- a/app/Jobs/DatabaseBackupJob.php+++ b/app/Jobs/DatabaseBackupJob.php@@ -22,6 +22,8 @@ use App\Rules\SafeWebhookUrl;use App\Services\ScheduledJobDeliveryService;use App\Support\BackupCompression;use App\Support\ClickhouseBackupCommand;+use App\Support\DatabaseOperationReservation;+use App\Support\ResourceStartActivity;use App\Support\ValidationPatterns;use Carbon\Carbon;use Illuminate\Bus\Queueable;@@ -29,10 +31,10 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;use Illuminate\Contracts\Queue\ShouldQueue;use Illuminate\Foundation\Bus\Dispatchable;use Illuminate\Queue\InteractsWithQueue;-use Illuminate\Queue\Middleware\WithoutOverlapping;use Illuminate\Queue\SerializesModels;use Illuminate\Support\Facades\Log;use Illuminate\Support\Str;+use Spatie\Activitylog\Models\Activity;use Throwable;class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue@@ -91,7 +93,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue{$expireAfter = ($this->backup->timeout ?? 3600) + 300;- return [(new WithoutOverlapping('database-backup-'.$this->backup->id))->expireAfter($expireAfter)->dontRelease()];+ return [ScheduledJobDeliveryService::withoutOverlapping('database-backup-'.$this->backup->id, $this->occurrenceUuid)->expireAfter($expireAfter)->dontRelease()];}public function handle(): void@@ -138,6 +140,12 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueuereturn;}+ if ($this->databaseOperationInProgress()) {+ $this->recordSkippedBackup('Skipped: a start, restart or import of this database was in progress. A backup taken now could contain a partly restored database.');+ $this->logSkippedRun('database_operation_in_progress');++ return;+ }if (data_get($this->backup, 'database_type') === ServiceDatabase::class) {$databaseType = $this->database->databaseType();$serviceUuid = $this->database->service->uuid;@@ -146,7 +154,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueueif (! ValidationPatterns::isValidContainerName($this->container_name)) {throw new \Exception('Invalid database container name.');}- $this->directory_name = $serviceName.'-'.str($this->database->name)->slug().'-'.$serviceUuid;+ // The container name is validated above, so it is a safe folder name. Keep the folder of earlier releases.+ $this->directory_name = $serviceName.'-'.$this->container_name;$escapedContainerName = escapeshellarg($this->container_name);if (str($databaseType)->contains('postgres')) {$commands[] = "docker exec {$escapedContainerName} env | grep POSTGRES_";@@ -321,9 +330,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueuetry {if (str($databaseType)->contains('postgres')) {$fileDatabaseName = $this->backupFilenamePart($database);- $this->backup_file = "/pg-dump-$fileDatabaseName-".Carbon::now()->timestamp.'.dmp';+ $this->backup_file = "/pg-dump-$fileDatabaseName-".Carbon::now()->timestamp."-{$this->backup_log_uuid}.dmp";if ($this->backup->dump_all) {- $this->backup_file = '/pg-dump-all-'.Carbon::now()->timestamp.'.gz';+ $this->backup_file = '/pg-dump-all-'.Carbon::now()->timestamp."-{$this->backup_log_uuid}.gz";}$this->backup_location = $this->backup_dir.$this->backup_file;$this->backup_log = ScheduledDatabaseBackupExecution::create([@@ -347,7 +356,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue}}$fileDatabaseName = $this->backupFilenamePart($databaseName);- $this->backup_file = "/mongo-dump-$fileDatabaseName-".Carbon::now()->timestamp.'.tar.gz';+ $this->backup_file = "/mongo-dump-$fileDatabaseName-".Carbon::now()->timestamp."-{$this->backup_log_uuid}.tar.gz";$this->backup_location = $this->backup_dir.$this->backup_file;$this->backup_log = ScheduledDatabaseBackupExecution::create(['uuid' => $this->backup_log_uuid,@@ -360,9 +369,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue$this->backup_standalone_mongodb($database);} elseif (str($databaseType)->contains('mysql')) {$fileDatabaseName = $this->backupFilenamePart($database);- $this->backup_file = "/mysql-dump-$fileDatabaseName-".Carbon::now()->timestamp.'.dmp';+ $this->backup_file = "/mysql-dump-$fileDatabaseName-".Carbon::now()->timestamp."-{$this->backup_log_uuid}.dmp";if ($this->backup->dump_all) {- $this->backup_file = '/mysql-dump-all-'.Carbon::now()->timestamp.'.gz';+ $this->backup_file = '/mysql-dump-all-'.Carbon::now()->timestamp."-{$this->backup_log_uuid}.gz";}$this->backup_location = $this->backup_dir.$this->backup_file;$this->backup_log = ScheduledDatabaseBackupExecution::create([@@ -376,9 +385,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue$this->backup_standalone_mysql($database);} elseif (str($databaseType)->contains('mariadb')) {$fileDatabaseName = $this->backupFilenamePart($database);- $this->backup_file = "/mariadb-dump-$fileDatabaseName-".Carbon::now()->timestamp.'.dmp';+ $this->backup_file = "/mariadb-dump-$fileDatabaseName-".Carbon::now()->timestamp."-{$this->backup_log_uuid}.dmp";if ($this->backup->dump_all) {- $this->backup_file = '/mariadb-dump-all-'.Carbon::now()->timestamp.'.gz';+ $this->backup_file = '/mariadb-dump-all-'.Carbon::now()->timestamp."-{$this->backup_log_uuid}.gz";}$this->backup_location = $this->backup_dir.$this->backup_file;$this->backup_log = ScheduledDatabaseBackupExecution::create([@@ -403,7 +412,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueueBackupCreated::dispatch($this->team->id);$this->backup_standalone_clickhouse($database);} elseif ($this->database instanceof StandaloneSqlite) {- $this->backup_file = "/sqlite-backup-$database-".Carbon::now()->timestamp.'.gz';+ $this->backup_file = "/sqlite-backup-$database-".Carbon::now()->timestamp."-{$this->backup_log_uuid}.gz";$this->backup_location = $this->backup_dir.$this->backup_file;$this->backup_log = ScheduledDatabaseBackupExecution::create(['uuid' => $this->backup_log_uuid,@@ -925,6 +934,49 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue]);}+ /**+ * A start, restart or import of the database is reserved, queued or running. Stale activities do+ * not count; they are failed by the start and import paths.+ */+ private function databaseOperationInProgress(): bool+ {+ $uuid = $this->database->uuid;+ if (blank($uuid)) {+ return false;+ }++ if (DatabaseOperationReservation::isHeldByOther($uuid)) {+ return true;+ }++ $liveOperation = collect([ResourceStartActivity::DATABASE_START_OPERATION, ResourceStartActivity::DATABASE_IMPORT_OPERATION])+ ->flatMap(fn (string $operation) => ResourceStartActivity::active($uuid, $operation))+ ->contains(fn (Activity $activity): bool => ! ResourceStartActivity::isStale($activity));++ return $liveOperation+ || ResourceStartActivity::latestRunning($uuid) !== null+ || ($this->database instanceof ServiceDatabase && ResourceStartActivity::latestRunning($this->database->service->uuid) !== null);+ }++ /**+ * Show a backup run that did not take a backup in the execution list. The run is expected, so it+ * sends no failure notification.+ */+ private function recordSkippedBackup(string $message): void+ {+ $databaseNames = $this->backup->dump_all ? 'all' : data_get($this->backup, 'databases_to_backup');+ $this->backup_log_uuid = new_public_id();+ $this->backup_log = ScheduledDatabaseBackupExecution::create([+ 'uuid' => $this->backup_log_uuid,+ 'database_name' => filled($databaseNames) ? Str::limit($databaseNames, 250) : null,+ 'scheduled_database_backup_id' => $this->backup->id,+ 'status' => 'failed',+ 'message' => $message,+ 'filename' => null,+ 'local_storage_deleted' => false,+ ]);+ }+private function markStaleExecutionsAsFailed(): void{try {diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.phpindex fb7e3376b5..78f091bbe2 100644--- a/app/Jobs/VolumeBackupJob.php+++ b/app/Jobs/VolumeBackupJob.php@@ -17,7 +17,6 @@ use Illuminate\Contracts\Queue\ShouldQueue;use Illuminate\Database\Eloquent\Builder;use Illuminate\Foundation\Bus\Dispatchable;use Illuminate\Queue\InteractsWithQueue;-use Illuminate\Queue\Middleware\WithoutOverlapping;use Illuminate\Queue\SerializesModels;use Illuminate\Support\Collection;use Illuminate\Support\Facades\Log;@@ -42,7 +41,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueuepublic function middleware(): array{return [- (new WithoutOverlapping('volume-backup-'.$this->backup->id))+ ScheduledJobDeliveryService::withoutOverlapping('volume-backup-'.$this->backup->id, $this->occurrenceUuid)->shared()->expireAfter($this->timeout + 60)->dontRelease(),diff --git a/app/Services/ScheduledJobDeliveryService.php b/app/Services/ScheduledJobDeliveryService.phpindex 84e8d4210f..330889c151 100644--- a/app/Services/ScheduledJobDeliveryService.php+++ b/app/Services/ScheduledJobDeliveryService.php@@ -2,17 +2,25 @@namespace App\Services;+use App\Events\DockerCleanupDone;+use App\Events\ScheduledTaskDone;use App\Jobs\DatabaseBackupJob;use App\Jobs\DockerCleanupJob;use App\Jobs\ScheduledTaskJob;use App\Jobs\VolumeBackupJob;+use App\Models\DockerCleanupExecution;use App\Models\ScheduledDatabaseBackup;use App\Models\ScheduledJobDelivery;use App\Models\ScheduledJobState;use App\Models\ScheduledTask;+use App\Models\ScheduledTaskExecution;use App\Models\ScheduledVolumeBackup;use App\Models\Server;+use App\Models\Team;+use App\Notifications\ScheduledTask\TaskFailed;+use App\Notifications\Server\DockerCleanupFailed;use Cron\CronExpression;+use Illuminate\Queue\Middleware\WithoutOverlapping;use Illuminate\Support\Carbon;use Illuminate\Support\Facades\DB;use Illuminate\Support\Facades\Log;@@ -29,7 +37,8 @@ class ScheduledJobDeliveryServicepublic const ENQUEUED_STALE_AFTER_MINUTES = 60;/**- * Job types that run once when they are late. A lost task or Docker cleanup is logged as missed.+ * Job types that run once when they are late. A lost task or Docker cleanup is not run; it is+ * recorded as a failed execution and the team is notified.*/private const LATE_RUN_JOB_TYPES = ['database-backup', 'volume-backup'];@@ -167,7 +176,7 @@ class ScheduledJobDeliveryService}/**- * Publish backups again whose queued job was not started, and log lost tasks and Docker+ * Publish backups again whose queued job was not started, and report lost tasks and Docker* cleanups as missed. A lost backup is not run when a newer occurrence of the same schedule* exists, so a backup that runs longer than its interval does not build up a backlog.*@@ -196,10 +205,15 @@ class ScheduledJobDeliveryServiceprivate function recoverStaleEnqueuedOccurrence(ScheduledJobDelivery $occurrence): void{- $hasNewerOccurrence = ScheduledJobDelivery::query()+ // complete() deletes the row of a finished occurrence, so the schedule state also counts.+ $hasNewerOccurrence = ScheduledJobState::query()->where('schedule_key', $occurrence->schedule_key)- ->where('scheduled_for', '>', $occurrence->scheduled_for)- ->exists();+ ->where('last_scheduled_for', '>', $occurrence->scheduled_for)+ ->exists()+ || ScheduledJobDelivery::query()+ ->where('schedule_key', $occurrence->schedule_key)+ ->where('scheduled_for', '>', $occurrence->scheduled_for)+ ->exists();$runLate = in_array($occurrence->job_type, self::LATE_RUN_JOB_TYPES, true) && ! $hasNewerOccurrence;// Only one scheduler node can move each occurrence.@@ -217,6 +231,7 @@ class ScheduledJobDeliveryService$this->logOccurrence($occurrence->uuid, $hasNewerOccurrence? 'Scheduled occurrence skipped: its queued job was not started and a newer occurrence exists': 'Scheduled occurrence missed: its queued job was not started', 'warning');+ $this->reportMissedOccurrence($occurrence);return;}@@ -226,6 +241,55 @@ class ScheduledJobDeliveryService$this->publish($occurrence);}+ /**+ * Show a missed task or Docker cleanup as a failed execution and send the failure notification.+ * The caller moved the occurrence out of `enqueued` atomically, so this runs once per occurrence,+ * and claim() rejects the late job if it starts afterwards.+ */+ private function reportMissedOccurrence(ScheduledJobDelivery $occurrence): void+ {+ $message = 'Skipped: the queued job did not start within '.self::ENQUEUED_STALE_AFTER_MINUTES.' minutes.';++ try {+ if ($occurrence->job_type === 'scheduled-task') {+ $task = ScheduledTask::find($occurrence->resource_id);+ if (! $task) {+ return;+ }++ ScheduledTaskExecution::create([+ 'scheduled_task_id' => $task->id,+ 'status' => 'failed',+ 'message' => $message,+ 'retry_count' => 0,+ 'finished_at' => now(),+ ]);+ ScheduledTaskDone::dispatch($task->team_id);+ Team::find($task->team_id)?->notify(new TaskFailed($task, $message));+ } elseif ($occurrence->job_type === 'docker-cleanup') {+ $server = Server::find($occurrence->resource_id);+ if (! $server) {+ return;+ }++ $execution = DockerCleanupExecution::create([+ 'server_id' => $server->id,+ 'status' => 'failed',+ 'message' => $message,+ 'finished_at' => now(),+ ]);+ event(new DockerCleanupDone($execution));+ $server->team?->notify(new DockerCleanupFailed($server, "Docker cleanup job failed with the following error: {$message}"));+ }+ } catch (\Throwable $e) {+ Log::channel('scheduled-errors')->error('Failed to report missed scheduled occurrence', [+ 'occurrence_uuid' => $occurrence->uuid,+ 'schedule_key' => $occurrence->schedule_key,+ 'error' => $e->getMessage(),+ ]);+ }+ }+public function deleteOldOccurrences(): void{ScheduledJobDelivery::query()@@ -276,6 +340,53 @@ class ScheduledJobDeliveryService->delete();}+ /**+ * Skip an occurrence that no job has claimed, so stale recovery does not publish it again.+ */+ public function skip(string $uuid, string $reason): void+ {+ $skipped = ScheduledJobDelivery::query()+ ->where('uuid', $uuid)+ ->whereIn('status', ['pending', 'enqueued'])+ ->update([+ 'status' => 'skipped',+ 'updated_at' => now(),+ ]) === 1;++ if ($skipped) {+ $this->logOccurrence($uuid, "Scheduled occurrence skipped: {$reason}", 'warning');+ }+ }++ /**+ * WithoutOverlapping for the job of a scheduled occurrence. When another run holds the lock,+ * the job does not run and its occurrence is skipped, so stale recovery does not run it later.+ */+ public static function withoutOverlapping(string $key, ?string $occurrenceUuid): WithoutOverlapping+ {+ return new class($key, $occurrenceUuid) extends WithoutOverlapping+ {+ public function __construct(string $key, private ?string $occurrenceUuid)+ {+ parent::__construct($key);+ }++ public function handle($job, $next)+ {+ $started = false;+ parent::handle($job, function ($job) use ($next, &$started) {+ $started = true;++ return $next($job);+ });++ if (! $started && $this->occurrenceUuid !== null) {+ app(ScheduledJobDeliveryService::class)->skip($this->occurrenceUuid, 'the previous run of this schedule is still running');+ }+ }+ };+ }+public function fail(string $uuid, string $claimToken): void{ScheduledJobDelivery::query()@@ -359,8 +470,8 @@ class ScheduledJobDeliveryServicereturn false;}- // An open previous occurrence usually means the previous run is still running. Jobs with- // WithoutOverlapping(...)->dontRelease() then drop this one without running it.+ // An open previous occurrence usually means the previous run is still running. Backup jobs+ // then skip this one without running it (see withoutOverlapping()).$previousOpen = ScheduledJobDelivery::query()->where('schedule_key', $occurrence->schedule_key)->where('scheduled_for', '<', $occurrence->scheduled_for)diff --git a/tests/Feature/BackupRunSafetyTest.php b/tests/Feature/BackupRunSafetyTest.phpnew file mode 100644index 0000000000..d1fb7480af--- /dev/null+++ b/tests/Feature/BackupRunSafetyTest.php@@ -0,0 +1,292 @@+<?php++use App\Enums\ProcessStatus;+use App\Events\BackupCreated;+use App\Jobs\DatabaseBackupJob;+use App\Jobs\VolumeBackupJob;+use App\Models\Application;+use App\Models\Environment;+use App\Models\InstanceSettings;+use App\Models\LocalPersistentVolume;+use App\Models\PrivateKey;+use App\Models\Project;+use App\Models\ScheduledDatabaseBackup;+use App\Models\ScheduledDatabaseBackupExecution;+use App\Models\ScheduledJobDelivery;+use App\Models\ScheduledJobState;+use App\Models\ScheduledVolumeBackupExecution;+use App\Models\Server;+use App\Models\Service;+use App\Models\ServiceDatabase;+use App\Models\StandaloneDocker;+use App\Models\Team;+use App\Services\ScheduledJobDeliveryService;+use App\Support\DatabaseOperationReservation;+use App\Support\ResourceStartActivity;+use Illuminate\Database\Eloquent\Model;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Carbon;+use Illuminate\Support\Collection;+use Illuminate\Support\Facades\Cache;+use Illuminate\Support\Facades\Event;+use Illuminate\Support\Facades\Notification;+use Illuminate\Support\Facades\Process;+use Illuminate\Support\Facades\Queue;+use Illuminate\Support\Facades\Storage;+use Spatie\Activitylog\Models\Activity;++uses(RefreshDatabase::class);++beforeEach(function () {+ Server::flushIdentityMap();+ Storage::fake('ssh-keys');+ Storage::fake('ssh-mux');+ Event::fake([BackupCreated::class]);+ Notification::fake();+ InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));++ $this->team = Team::factory()->create();+ $this->server = Server::factory()->create([+ 'team_id' => $this->team->id,+ 'private_key_id' => PrivateKey::factory()->create(['team_id' => $this->team->id])->id,+ ]);+ $this->server->settings()->update(['is_reachable' => true, 'is_usable' => true]);+ $this->destination = StandaloneDocker::where('server_id', $this->server->id)->firstOrFail();+ $project = Project::factory()->create(['team_id' => $this->team->id]);+ $this->environment = Environment::factory()->create(['project_id' => $project->id]);+});++/**+ * @return Collection<int, string> The remote commands the job ran.+ */+function backupRunSafetyFakeRemoteCommands(array $outputs = []): Collection+{+ $commands = collect();+ Process::fake(function ($process) use ($commands, $outputs) {+ $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;+ $commands->push($command);+ foreach ($outputs as $needle => $output) {+ if (str_contains($command, $needle)) {+ return Process::result(output: $output);+ }+ }++ return Process::result(output: str_contains($command, 'du -b') ? '128' : '');+ });++ return $commands;+}++function backupRunSafetySchedule(Model $database, Team $team, array $overrides = []): ScheduledDatabaseBackup+{+ $backup = new ScheduledDatabaseBackup;+ $backup->forceFill([+ 'enabled' => true,+ 'save_s3' => false,+ 'frequency' => 'daily',+ 'database_id' => $database->id,+ 'database_type' => $database->getMorphClass(),+ 'team_id' => $team->id,+ ...$overrides,+ ]);+ $backup->save();++ return $backup->fresh();+}++function backupRunSafetyOccurrence(string $scheduleKey, string $jobType, int $resourceId): ScheduledJobDelivery+{+ return ScheduledJobDelivery::create([+ 'schedule_key' => $scheduleKey,+ 'scheduled_for' => now()->startOfMinute(),+ 'job_type' => $jobType,+ 'resource_id' => $resourceId,+ 'status' => 'enqueued',+ 'enqueued_at' => now(),+ ]);+}++function backupRunSafetyActivity(string $typeUuid, string $operation, ProcessStatus $status): Activity+{+ return Activity::create([+ 'log_name' => 'default',+ 'description' => '[]',+ 'properties' => [+ 'type_uuid' => $typeUuid,+ 'operation' => $operation,+ 'status' => $status->value,+ ],+ ]);+}++// --- Overlapped occurrences (M5) ---++it('skips a database backup occurrence when the previous run still holds the overlap lock, so recovery does not run it later', function () {+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'));+ $database = create_standalone_postgresql($this->environment->id, $this->destination, ['status' => 'running:healthy']);+ $backup = backupRunSafetySchedule($database, $this->team);+ $occurrence = backupRunSafetyOccurrence("scheduled-backup:{$backup->id}", 'database-backup', $backup->id);+ $job = new DatabaseBackupJob($backup, $occurrence->uuid);+ $runningBackupLock = Cache::lock($job->middleware()[0]->getLockKey($job), 3600);+ $runningBackupLock->get();+ $commands = backupRunSafetyFakeRemoteCommands();++ dispatch_sync($job);+ $runningBackupLock->release();++ expect($occurrence->fresh()->status)->toBe('skipped')+ ->and(ScheduledDatabaseBackupExecution::query()->exists())->toBeFalse()+ ->and($commands)->toBeEmpty();++ Carbon::setTestNow(now()->addMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1));+ Queue::fake();+ app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();++ Queue::assertNothingPushed();+});++it('skips a volume backup occurrence when the previous run still holds the overlap lock', function () {+ $volume = backupRunSafetyVolume($this->environment, $this->destination);+ $backup = $volume->scheduledBackups()->create(['team_id' => $this->team->id, 'frequency' => 'daily', 'enabled' => true]);+ $occurrence = backupRunSafetyOccurrence("scheduled-volume-backup:{$backup->id}", 'volume-backup', $backup->id);+ $runningBackupLock = Cache::lock(VolumeBackupJob::lockKey($backup->id), 3600);+ $runningBackupLock->get();+ $commands = backupRunSafetyFakeRemoteCommands();++ dispatch_sync(new VolumeBackupJob($backup, $occurrence->uuid));+ $runningBackupLock->release();++ expect($occurrence->fresh()->status)->toBe('skipped')+ ->and(ScheduledVolumeBackupExecution::query()->exists())->toBeFalse()+ ->and($commands)->toBeEmpty();+});++it('does not run a lost backup occurrence again when a newer occurrence already completed', function () {+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));+ Queue::fake();+ $database = create_standalone_postgresql($this->environment->id, $this->destination, ['status' => 'running:healthy']);+ $backup = backupRunSafetySchedule($database, $this->team, ['frequency' => 'hourly']);+ $scheduleKey = "scheduled-backup:{$backup->id}";+ $lost = ScheduledJobDelivery::create([+ 'schedule_key' => $scheduleKey,+ 'scheduled_for' => Carbon::create(2026, 9, 17, 10, 0, 0, 'UTC'),+ 'job_type' => 'database-backup',+ 'resource_id' => $backup->id,+ 'status' => 'enqueued',+ 'enqueued_at' => now()->subMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1),+ ]);+ // The 11:00 occurrence ran and completed; complete() deleted its row.+ ScheduledJobState::create(['schedule_key' => $scheduleKey, 'last_scheduled_for' => Carbon::create(2026, 9, 17, 11, 0, 0, 'UTC')]);++ app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();++ Queue::assertNotPushed(DatabaseBackupJob::class);+ expect($lost->fresh()->status)->toBe('failed');+});++// --- Imports and starts (M10) ---++it('does not back up a database while a start or import of it is in progress', function (Closure $startOperation) {+ $database = create_standalone_postgresql($this->environment->id, $this->destination, ['status' => 'running:healthy']);+ $backup = backupRunSafetySchedule($database, $this->team);+ $occurrence = backupRunSafetyOccurrence("scheduled-backup:{$backup->id}", 'database-backup', $backup->id);+ $startOperation($database);+ $commands = backupRunSafetyFakeRemoteCommands();++ (new DatabaseBackupJob($backup, $occurrence->uuid))->handle();++ $execution = ScheduledDatabaseBackupExecution::query()->sole();+ expect($commands->filter(fn (string $command) => str_contains($command, 'pg_dump')))->toBeEmpty()+ ->and($execution->status)->toBe('failed')+ ->and($execution->message)->toContain('Skipped')+ ->and($execution->filename)->toBeNull()+ ->and($execution->finished_at)->not->toBeNull()+ ->and(ScheduledJobDelivery::query()->whereKey($occurrence->id)->exists())->toBeFalse();+ Notification::assertNothingSent();+})->with([+ 'reserved import, start or restart' => [fn (Model $database) => DatabaseOperationReservation::acquire($database->uuid)],+ 'running import' => [fn (Model $database) => backupRunSafetyActivity($database->uuid, ResourceStartActivity::DATABASE_IMPORT_OPERATION, ProcessStatus::IN_PROGRESS)],+ 'queued start' => [fn (Model $database) => backupRunSafetyActivity($database->uuid, ResourceStartActivity::DATABASE_START_OPERATION, ProcessStatus::QUEUED)],+]);++it('backs up a database when its last import is stale', function () {+ $database = create_standalone_postgresql($this->environment->id, $this->destination, ['status' => 'running:healthy']);+ $backup = backupRunSafetySchedule($database, $this->team);+ $import = backupRunSafetyActivity($database->uuid, ResourceStartActivity::DATABASE_IMPORT_OPERATION, ProcessStatus::IN_PROGRESS);+ Activity::query()->whereKey($import->id)->update(['updated_at' => now()->subSeconds(ResourceStartActivity::importStaleAfterSeconds() + 60)]);+ $commands = backupRunSafetyFakeRemoteCommands();++ (new DatabaseBackupJob($backup))->handle();++ expect($commands->filter(fn (string $command) => str_contains($command, 'pg_dump')))->not->toBeEmpty()+ ->and(ScheduledDatabaseBackupExecution::query()->sole()->status)->toBe('success');+});++// --- Unique backup file names (M11) ---++it('gives each backup run of the same database its own file in the same second', function (Closure $createDatabase, array $backupSettings, string $extension) {+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'));+ $database = $createDatabase($this->environment->id, $this->destination, ['status' => 'running:healthy']);+ $first = backupRunSafetySchedule($database, $this->team, $backupSettings);+ $second = backupRunSafetySchedule($database, $this->team, $backupSettings);+ backupRunSafetyFakeRemoteCommands();++ (new DatabaseBackupJob($first))->handle();+ (new DatabaseBackupJob($second))->handle();++ $executions = ScheduledDatabaseBackupExecution::query()->orderBy('id')->get();+ expect($executions)->toHaveCount(2)+ ->and($executions->pluck('status')->unique()->all())->toBe(['success'])+ ->and($executions->pluck('filename')->unique())->toHaveCount(2);+ foreach ($executions as $execution) {+ expect($execution->filename)->toEndWith("-{$execution->uuid}{$extension}");+ }+})->with([+ 'postgresql' => [fn (...$arguments) => create_standalone_postgresql(...$arguments), [], '.dmp'],+ 'postgresql full dump' => [fn (...$arguments) => create_standalone_postgresql(...$arguments), ['dump_all' => true], '.gz'],+ 'mysql' => [fn (...$arguments) => create_standalone_mysql(...$arguments), [], '.dmp'],+ 'mariadb' => [fn (...$arguments) => create_standalone_mariadb(...$arguments), [], '.dmp'],+ 'mongodb' => [fn (...$arguments) => create_standalone_mongodb(...$arguments), [], '.tar.gz'],+ 'sqlite' => [fn (...$arguments) => create_standalone_sqlite(...$arguments), [], '.gz'],+]);++// --- Service database backup folder (M12) ---++it('stores service database backups in the folder of earlier releases', function () {+ $service = Service::factory()->create([+ 'name' => 'Rallly',+ 'server_id' => $this->server->id,+ 'destination_id' => $this->destination->id,+ 'destination_type' => $this->destination->getMorphClass(),+ 'environment_id' => $this->environment->id,+ ]);+ $database = ServiceDatabase::create([+ 'service_id' => $service->id,+ 'name' => 'rallly_db',+ 'image' => 'postgres:16-alpine',+ 'status' => 'running:healthy',+ ]);+ $backup = backupRunSafetySchedule($database, $this->team);+ backupRunSafetyFakeRemoteCommands(['env | grep POSTGRES_' => "POSTGRES_USER=rallly\nPOSTGRES_DB=rallly"]);++ (new DatabaseBackupJob($backup))->handle();++ expect(ScheduledDatabaseBackupExecution::query()->sole()->filename)+ ->toContain("/rallly-rallly_db-{$service->uuid}/pg-dump-rallly-");+});++function backupRunSafetyVolume(Environment $environment, StandaloneDocker $destination): LocalPersistentVolume+{+ $application = Application::factory()->create([+ 'environment_id' => $environment->id,+ 'destination_id' => $destination->id,+ 'destination_type' => $destination->getMorphClass(),+ ]);++ return LocalPersistentVolume::create([+ 'name' => 'app-data',+ 'mount_path' => '/data',+ 'resource_id' => $application->id,+ 'resource_type' => $application->getMorphClass(),+ ]);+}diff --git a/tests/Feature/DatabaseDumpExitStatusTest.php b/tests/Feature/DatabaseDumpExitStatusTest.phpindex e614db4250..7a9382a74b 100644--- a/tests/Feature/DatabaseDumpExitStatusTest.php+++ b/tests/Feature/DatabaseDumpExitStatusTest.php@@ -107,5 +107,5 @@ it('records a failed SQLite dump as failed and removes the partial backup file',$execution = $backup->executions()->latest('id')->firstOrFail();expect($execution->status)->toBe('failed')->and($execution->filename)->toBeNull()- ->and($commands->implode("\n"))->toMatch("/rm -f '[^']*sqlite-backup-app\\.db-\\d+\\.gz'/");+ ->and($commands->implode("\n"))->toMatch("/rm -f '[^']*sqlite-backup-app\\.db-\\d+-[a-z0-9]+\\.gz'/");});diff --git a/tests/Feature/ScheduledJobManagerDispatchTest.php b/tests/Feature/ScheduledJobManagerDispatchTest.phpindex 95f241b046..9c84560399 100644--- a/tests/Feature/ScheduledJobManagerDispatchTest.php+++ b/tests/Feature/ScheduledJobManagerDispatchTest.php@@ -1,18 +1,22 @@<?phpuse App\Events\BackupCreated;+use App\Events\DockerCleanupDone;use App\Jobs\DatabaseBackupJob;use App\Jobs\DockerCleanupJob;use App\Jobs\ScheduledJobManager;use App\Jobs\ScheduledTaskJob;use App\Models\Application;+use App\Models\DockerCleanupExecution;use App\Models\Environment;+use App\Models\InstanceSettings;use App\Models\PrivateKey;use App\Models\Project;use App\Models\ScheduledDatabaseBackup;use App\Models\ScheduledJobDelivery;use App\Models\ScheduledJobState;use App\Models\ScheduledTask;+use App\Models\ScheduledTaskExecution;use App\Models\Server;use App\Models\ServerSetting;use App\Models\Service;@@ -20,6 +24,8 @@ use App\Models\ServiceDatabase;use App\Models\StandaloneDocker;use App\Models\StandalonePostgresql;use App\Models\Team;+use App\Notifications\ScheduledTask\TaskFailed;+use App\Notifications\Server\DockerCleanupFailed;use App\Services\ScheduledJobDeliveryService;use Illuminate\Foundation\Testing\RefreshDatabase;use Illuminate\Queue\Middleware\WithoutOverlapping;@@ -29,6 +35,7 @@ use Illuminate\Support\Facades\Cache;use Illuminate\Support\Facades\DB;use Illuminate\Support\Facades\Event;use Illuminate\Support\Facades\Log;+use Illuminate\Support\Facades\Notification;use Illuminate\Support\Facades\Queue;uses(RefreshDatabase::class);@@ -691,6 +698,58 @@ it('logs a lost scheduled task occurrence as missed instead of running it late',->and($log)->toContain('Scheduled occurrence missed: its queued job was not started');});+it('records a missed scheduled task as a failed execution and notifies the team once', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));+ Queue::fake();+ Notification::fake();+ $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);+ InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));+ Team::find($task->team_id)->emailNotificationSettings->update(['smtp_enabled' => true, 'scheduled_task_failure_email_notifications' => true]);+ $occurrence = createStaleEnqueuedOccurrence("scheduled-task:{$task->id}", 'scheduled-task', $task->id, Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'));++ (new ScheduledJobManager)->handle();+ (new ScheduledJobManager)->handle();++ $executions = ScheduledTaskExecution::query()->where('scheduled_task_id', $task->id)->get();+ expect($executions)->toHaveCount(1)+ ->and($executions->first()->status)->toBe('failed')+ ->and($executions->first()->message)->toBe('Skipped: the queued job did not start within 60 minutes.')+ ->and($executions->first()->finished_at)->not->toBeNull();+ Notification::assertSentToTimes(Team::find($task->team_id), TaskFailed::class, 1);+ Notification::assertSentTo(Team::find($task->team_id), TaskFailed::class, fn (TaskFailed $notification) => $notification->task->is($task)+ && $notification->output === 'Skipped: the queued job did not start within 60 minutes.');++ $lateJob = new ScheduledTaskJob($task, $occurrence->uuid);+ $lateJob->handle();++ expect(ScheduledTaskExecution::query()->where('scheduled_task_id', $task->id)->count())->toBe(1);+});++it('records a missed Docker cleanup as a failed execution and notifies the team once', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));+ Queue::fake();+ Notification::fake();+ Event::fake([DockerCleanupDone::class]);+ $server = createScheduledTaskApplication()->destination->server;+ $server->team->emailNotificationSettings->update(['smtp_enabled' => true, 'docker_cleanup_failure_email_notifications' => true]);+ $occurrence = createStaleEnqueuedOccurrence("docker-cleanup:{$server->id}", 'docker-cleanup', $server->id, Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'));++ (new ScheduledJobManager)->handle();+ (new ScheduledJobManager)->handle();++ $executions = DockerCleanupExecution::query()->where('server_id', $server->id)->get();+ expect($executions)->toHaveCount(1)+ ->and($executions->first()->status)->toBe('failed')+ ->and($executions->first()->message)->toBe('Skipped: the queued job did not start within 60 minutes.')+ ->and($executions->first()->finished_at)->not->toBeNull()+ ->and($occurrence->fresh()->status)->toBe('failed')+ ->and(app(ScheduledJobDeliveryService::class)->claim($occurrence->uuid, 'original-job'))->toBeFalse();+ Notification::assertSentToTimes($server->team, DockerCleanupFailed::class, 1);+ Event::assertDispatched(DockerCleanupDone::class, 1);+});+it('leaves recently enqueued occurrences alone', function () {config(['constants.coolify.self_hosted' => true]);Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));