An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(backups): enforce retention and clean up stale executions
Andras Bacsaiffb5045c6ab7
diff --git a/app/Jobs/CleanupInstanceStuffsJob.php b/app/Jobs/CleanupInstanceStuffsJob.phpindex 011c586396..e37a39c3dc 100644--- a/app/Jobs/CleanupInstanceStuffsJob.php+++ b/app/Jobs/CleanupInstanceStuffsJob.php@@ -2,6 +2,7 @@namespace App\Jobs;+use App\Models\ScheduledDatabaseBackup;use App\Models\TeamInvitation;use App\Models\User;use Illuminate\Bus\Queueable;@@ -12,6 +13,7 @@ use Illuminate\Foundation\Bus\Dispatchable;use Illuminate\Queue\InteractsWithQueue;use Illuminate\Queue\Middleware\WithoutOverlapping;use Illuminate\Queue\SerializesModels;+use Illuminate\Support\Facades\Cache;use Illuminate\Support\Facades\Log;class CleanupInstanceStuffsJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue@@ -32,6 +34,7 @@ class CleanupInstanceStuffsJob implements ShouldBeEncrypted, ShouldBeUnique, Shotry {$this->cleanupInvitationLink();$this->cleanupExpiredEmailChangeRequests();+ $this->enforceBackupRetention();} catch (\Throwable $e) {Log::error('CleanupInstanceStuffsJob failed with error: '.$e->getMessage());}@@ -55,4 +58,25 @@ class CleanupInstanceStuffsJob implements ShouldBeEncrypted, ShouldBeUnique, Sho'email_change_code_expires_at' => null,]);}++ private function enforceBackupRetention(): void+ {+ if (! Cache::add('backup-retention-enforcement', true, 1800)) {+ return;+ }++ try {+ $backups = ScheduledDatabaseBackup::where('enabled', true)->get();+ foreach ($backups as $backup) {+ try {+ removeOldBackups($backup);+ } catch (\Throwable $e) {+ Log::warning('Failed to enforce retention for backup '.$backup->id.': '.$e->getMessage());+ }+ }+ } catch (\Throwable $e) {+ Log::error('Failed to enforce backup retention: '.$e->getMessage());+ Cache::forget('backup-retention-enforcement');+ }+ }}diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.phpindex a2d08e1e85..207191cbd2 100644--- a/app/Jobs/DatabaseBackupJob.php+++ b/app/Jobs/DatabaseBackupJob.php@@ -22,6 +22,7 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;use Illuminate\Contracts\Queue\ShouldQueue;use Illuminate\Foundation\Bus\Dispatchable;use Illuminate\Queue\InteractsWithQueue;+use Illuminate\Queue\Middleware\WithoutOverlapping;use Illuminate\Queue\SerializesModels;use Illuminate\Support\Facades\Log;use Illuminate\Support\Str;@@ -80,6 +81,13 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue$this->timeout = $backup->timeout ?? 3600;}+ public function middleware(): array+ {+ $expireAfter = ($this->backup->timeout ?? 3600) + 300;++ return [(new WithoutOverlapping('database-backup-'.$this->backup->id))->expireAfter($expireAfter)->dontRelease()];+ }+public function handle(): void{try {@@ -107,6 +115,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueuethrow new \Exception('Database not found?!');}+ $this->markStaleExecutionsAsFailed();+BackupCreated::dispatch($this->team->id);$status = str(data_get($this->database, 'status'));@@ -727,6 +737,31 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueuereturn "{$helperImage}:{$latestVersion}";}+ private function markStaleExecutionsAsFailed(): void+ {+ try {+ $timeoutSeconds = ($this->backup->timeout ?? 3600) * 2;++ $staleExecutions = $this->backup->executions()+ ->where('status', 'running')+ ->where('created_at', '<', now()->subSeconds($timeoutSeconds))+ ->get();++ foreach ($staleExecutions as $execution) {+ $execution->update([+ 'status' => 'failed',+ 'message' => 'Marked as failed - backup execution exceeded maximum allowed time',+ 'finished_at' => now(),+ ]);+ }+ } catch (Throwable $e) {+ Log::channel('scheduled-errors')->warning('Failed to clean up stale backup executions', [+ 'backup_id' => $this->backup->uuid,+ 'error' => $e->getMessage(),+ ]);+ }+ }+public function failed(?Throwable $exception): void{Log::channel('scheduled-errors')->error('DatabaseBackup permanently failed', [diff --git a/app/Models/ScheduledDatabaseBackup.php b/app/Models/ScheduledDatabaseBackup.phpindex 6308bae8b7..4038c6288c 100644--- a/app/Models/ScheduledDatabaseBackup.php+++ b/app/Models/ScheduledDatabaseBackup.php@@ -8,6 +8,14 @@ use Illuminate\Database\Eloquent\Relations\MorphTo;class ScheduledDatabaseBackup extends BaseModel{+ protected function casts(): array+ {+ return [+ 'database_backup_retention_max_storage_locally' => 'float',+ 'database_backup_retention_max_storage_s3' => 'float',+ ];+ }+protected $fillable = ['uuid','team_id',diff --git a/tests/Feature/BackupRetentionAndStaleDetectionTest.php b/tests/Feature/BackupRetentionAndStaleDetectionTest.phpnew file mode 100644index 0000000000..cd4a76feb1--- /dev/null+++ b/tests/Feature/BackupRetentionAndStaleDetectionTest.php@@ -0,0 +1,252 @@+<?php++use App\Jobs\CleanupInstanceStuffsJob;+use App\Jobs\DatabaseBackupJob;+use App\Models\ScheduledDatabaseBackup;+use App\Models\ScheduledDatabaseBackupExecution;+use App\Models\Team;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Queue\Middleware\WithoutOverlapping;+use Illuminate\Support\Facades\Cache;++uses(RefreshDatabase::class);++// --- WithoutOverlapping middleware on DatabaseBackupJob ---++test('database backup job has WithoutOverlapping middleware keyed by backup id', function () {+ $team = Team::factory()->create();++ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => '0 0 * * *',+ 'save_s3' => false,+ 'database_type' => 'App\Models\StandalonePostgresql',+ 'database_id' => 1,+ 'team_id' => $team->id,+ ]);++ $job = new DatabaseBackupJob($backup);+ $middleware = $job->middleware();++ expect($middleware)->toHaveCount(1);+ expect($middleware[0])->toBeInstanceOf(WithoutOverlapping::class);+});++test('database backup job middleware uses custom timeout plus buffer', function () {+ $team = Team::factory()->create();++ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => '0 0 * * *',+ 'save_s3' => false,+ 'database_type' => 'App\Models\StandalonePostgresql',+ 'database_id' => 1,+ 'team_id' => $team->id,+ 'timeout' => 7200,+ ]);++ $job = new DatabaseBackupJob($backup);+ $middleware = $job->middleware();++ $reflection = new ReflectionClass($middleware[0]);+ $expiresAfterProp = $reflection->getProperty('expiresAfter');+ $releaseAfterProp = $reflection->getProperty('releaseAfter');++ expect($expiresAfterProp->getValue($middleware[0]))->toBe(7500); // 7200 + 300+ expect($releaseAfterProp->getValue($middleware[0]))->toBeNull(); // dontRelease sets to null+});++// --- Stale backup detection in DatabaseBackupJob ---++test('markStaleExecutionsAsFailed marks stale running executions as failed', function () {+ $team = Team::factory()->create();++ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => '0 0 * * *',+ 'save_s3' => false,+ 'database_type' => 'App\Models\StandalonePostgresql',+ 'database_id' => 1,+ 'team_id' => $team->id,+ 'timeout' => 3600,+ ]);++ // Create a stale execution (3 hours old, timeout is 1h, threshold is 2h)+ $staleExecution = ScheduledDatabaseBackupExecution::create([+ 'uuid' => 'stale-exec-uuid',+ 'database_name' => 'test_db',+ 'scheduled_database_backup_id' => $backup->id,+ 'status' => 'running',+ ]);+ ScheduledDatabaseBackupExecution::where('id', $staleExecution->id)+ ->update(['created_at' => now()->subHours(3)]);++ // Call the private method via reflection+ $job = new DatabaseBackupJob($backup);+ $reflection = new ReflectionClass($job);+ $method = $reflection->getMethod('markStaleExecutionsAsFailed');+ $method->invoke($job);++ $staleExecution->refresh();+ expect($staleExecution->status)->toBe('failed');+ expect($staleExecution->message)->toContain('exceeded maximum allowed time');+ expect($staleExecution->finished_at)->not->toBeNull();+});++test('markStaleExecutionsAsFailed does not mark recent running executions as failed', function () {+ $team = Team::factory()->create();++ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => '0 0 * * *',+ 'save_s3' => false,+ 'database_type' => 'App\Models\StandalonePostgresql',+ 'database_id' => 1,+ 'team_id' => $team->id,+ 'timeout' => 3600,+ ]);++ // Create a recent execution (30 minutes old, threshold is 2 hours)+ $recentExecution = ScheduledDatabaseBackupExecution::create([+ 'uuid' => 'recent-exec-uuid',+ 'database_name' => 'test_db',+ 'scheduled_database_backup_id' => $backup->id,+ 'status' => 'running',+ ]);+ ScheduledDatabaseBackupExecution::where('id', $recentExecution->id)+ ->update(['created_at' => now()->subMinutes(30)]);++ $job = new DatabaseBackupJob($backup);+ $reflection = new ReflectionClass($job);+ $method = $reflection->getMethod('markStaleExecutionsAsFailed');+ $method->invoke($job);++ $recentExecution->refresh();+ expect($recentExecution->status)->toBe('running');+ expect($recentExecution->finished_at)->toBeNull();+});++// --- Decimal cast on ScheduledDatabaseBackup model ---++test('scheduled database backup model casts max storage fields to float', function () {+ $model = new ScheduledDatabaseBackup;+ $casts = $model->getCasts();++ expect($casts)->toHaveKey('database_backup_retention_max_storage_locally');+ expect($casts['database_backup_retention_max_storage_locally'])->toBe('float');+ expect($casts)->toHaveKey('database_backup_retention_max_storage_s3');+ expect($casts['database_backup_retention_max_storage_s3'])->toBe('float');+});++test('max storage retention field returns float zero that passes strict comparison', function () {+ $team = Team::factory()->create();++ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => '0 0 * * *',+ 'save_s3' => false,+ 'database_type' => 'App\Models\StandalonePostgresql',+ 'database_id' => 1,+ 'team_id' => $team->id,+ 'database_backup_retention_max_storage_locally' => 0,+ ]);++ $backup->refresh();++ // With the float cast, strict comparison to 0.0 works+ expect($backup->database_backup_retention_max_storage_locally)->toBe(0.0);+ // And loose comparison to int 0 also works+ expect($backup->database_backup_retention_max_storage_locally == 0)->toBeTrue();+});++// --- Periodic retention enforcement in CleanupInstanceStuffsJob ---++test('cleanup instance stuffs job calls retention enforcement without errors', function () {+ $team = Team::factory()->create();++ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => '0 0 * * *',+ 'save_s3' => false,+ 'enabled' => true,+ 'database_type' => 'App\Models\StandalonePostgresql',+ 'database_id' => 1,+ 'team_id' => $team->id,+ 'database_backup_retention_amount_locally' => 1,+ ]);++ // Create 3 successful executions+ foreach (range(1, 3) as $i) {+ ScheduledDatabaseBackupExecution::create([+ 'uuid' => "exec-uuid-{$i}",+ 'database_name' => 'test_db',+ 'filename' => "/backup/test_{$i}.dmp",+ 'scheduled_database_backup_id' => $backup->id,+ 'status' => 'success',+ 'size' => 1024,+ 'local_storage_deleted' => false,+ 'created_at' => now()->subDays($i),+ ]);+ }++ Cache::forget('backup-retention-enforcement');++ // Should not throw — the job gracefully handles missing servers+ $job = new CleanupInstanceStuffsJob;+ $job->handle();++ // All 3 executions should still exist (no server to delete files from)+ expect($backup->executions()->count())->toBe(3);+});++test('deleteOldBackupsLocally identifies correct backups for deletion by retention count', function () {+ $team = Team::factory()->create();++ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => '0 0 * * *',+ 'save_s3' => false,+ 'enabled' => true,+ 'database_type' => 'App\Models\StandalonePostgresql',+ 'database_id' => 1,+ 'team_id' => $team->id,+ 'database_backup_retention_amount_locally' => 1,+ ]);++ foreach (range(1, 3) as $i) {+ $exec = ScheduledDatabaseBackupExecution::create([+ 'uuid' => "exec-uuid-{$i}",+ 'database_name' => 'test_db',+ 'filename' => "/backup/test_{$i}.dmp",+ 'scheduled_database_backup_id' => $backup->id,+ 'status' => 'success',+ 'size' => 1024,+ 'local_storage_deleted' => false,+ ]);+ ScheduledDatabaseBackupExecution::where('id', $exec->id)+ ->update(['created_at' => now()->subDays($i)]);+ }++ // Test the selection logic directly+ $successfulBackups = $backup->executions()+ ->where('status', 'success')+ ->where('local_storage_deleted', false)+ ->orderBy('created_at', 'desc')+ ->get();++ $retentionAmount = $backup->database_backup_retention_amount_locally;+ $backupsToDelete = $successfulBackups->skip($retentionAmount);++ // exec-uuid-1 is newest (1 day ago), exec-uuid-2 and exec-uuid-3 should be deleted+ expect($backupsToDelete)->toHaveCount(2);+ expect($successfulBackups->first()->uuid)->toBe('exec-uuid-1');+});++test('cleanup instance stuffs job throttles retention enforcement via cache', function () {+ Cache::forget('backup-retention-enforcement');++ $job = new CleanupInstanceStuffsJob;+ $job->handle();++ // The cache key should now exist+ expect(Cache::has('backup-retention-enforcement'))->toBeTrue();++ // Running again should skip retention enforcement (cache hit)+ // We verify by checking the lock is still held — no error thrown+ $job->handle();++ expect(Cache::has('backup-retention-enforcement'))->toBeTrue();+});