An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(jobs): resume S3 revalidation and remove orphaned notification throttles
Andras Bacsai62d861d23a3e
diff --git a/app/Jobs/CleanupInstanceStuffsJob.php b/app/Jobs/CleanupInstanceStuffsJob.phpindex e37a39c3dc..85c39a8e08 100644--- a/app/Jobs/CleanupInstanceStuffsJob.php+++ b/app/Jobs/CleanupInstanceStuffsJob.php@@ -2,6 +2,7 @@namespace App\Jobs;+use App\Models\NotificationThrottle;use App\Models\ScheduledDatabaseBackup;use App\Models\TeamInvitation;use App\Models\User;@@ -35,6 +36,7 @@ class CleanupInstanceStuffsJob implements ShouldBeEncrypted, ShouldBeUnique, Sho$this->cleanupInvitationLink();$this->cleanupExpiredEmailChangeRequests();$this->enforceBackupRetention();+ $this->deleteOrphanedNotificationThrottles();} catch (\Throwable $e) {Log::error('CleanupInstanceStuffsJob failed with error: '.$e->getMessage());}@@ -59,6 +61,20 @@ class CleanupInstanceStuffsJob implements ShouldBeEncrypted, ShouldBeUnique, Sho]);}+ private function deleteOrphanedNotificationThrottles(): void+ {+ if (! Cache::add('notification-throttle-orphan-cleanup', true, 3600)) {+ return;+ }++ try {+ NotificationThrottle::deleteOrphans();+ } catch (\Throwable $e) {+ Log::error('Failed to delete orphaned notification throttles: '.$e->getMessage());+ Cache::forget('notification-throttle-orphan-cleanup');+ }+ }+private function enforceBackupRetention(): void{if (! Cache::add('backup-retention-enforcement', true, 1800)) {diff --git a/app/Jobs/RevalidateUnusableS3StoragesJob.php b/app/Jobs/RevalidateUnusableS3StoragesJob.phpindex fe9593b6fc..2240b457fb 100644--- a/app/Jobs/RevalidateUnusableS3StoragesJob.php+++ b/app/Jobs/RevalidateUnusableS3StoragesJob.php@@ -10,9 +10,13 @@ use Illuminate\Contracts\Queue\ShouldQueue;use Illuminate\Foundation\Bus\Dispatchable;use Illuminate\Queue\InteractsWithQueue;use Illuminate\Queue\SerializesModels;+use Illuminate\Support\Facades\Cache;/*** Re-tests S3 storages that failed validation, so a storage that becomes reachable again is usable without manual action.+ *+ * Each run stops after a time budget and saves the last tested id, so the next run continues from there+ * instead of re-testing the same slow storages from the start.*/class RevalidateUnusableS3StoragesJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue{@@ -22,18 +26,41 @@ class RevalidateUnusableS3StoragesJob implements ShouldBeEncrypted, ShouldBeUniqpublic $timeout = 1800;+ /**+ * Stay well below $timeout, because a single unreachable endpoint can take a while to fail.+ */+ public const TIME_BUDGET_SECONDS = 1200;++ public const CURSOR_CACHE_KEY = 'revalidate-unusable-s3-storages:last-id';+public function handle(): void{- S3Storage::query()+ $deadline = now()->addSeconds(self::TIME_BUDGET_SECONDS);+ $lastId = Cache::get(self::CURSOR_CACHE_KEY);++ $completed = S3Storage::query()->where('is_usable', false)- ->chunkById(100, function ($storages): void {+ ->when($lastId !== null, fn ($query) => $query->where('id', '>', $lastId))+ ->chunkById(100, function ($storages) use ($deadline): bool {foreach ($storages as $storage) {+ if (now()->greaterThanOrEqualTo($deadline)) {+ return false;+ }+try {$storage->testConnection(shouldSave: true);} catch (\Throwable) {// testConnection() keeps the storage unusable and notifies the team once.}++ Cache::forever(self::CURSOR_CACHE_KEY, $storage->id);}++ return true;});++ if ($completed) {+ Cache::forget(self::CURSOR_CACHE_KEY);+ }}}diff --git a/app/Models/NotificationThrottle.php b/app/Models/NotificationThrottle.phpindex b33a6ac5f7..a43e24c433 100644--- a/app/Models/NotificationThrottle.php+++ b/app/Models/NotificationThrottle.php@@ -7,6 +7,7 @@ use Carbon\CarbonInterface;use Closure;use Illuminate\Database\Eloquent\Model;use Illuminate\Database\Eloquent\Relations\MorphTo;+use Illuminate\Database\Eloquent\Relations\Relation;use Throwable;/**@@ -133,4 +134,36 @@ class NotificationThrottle extends Model->where('notification', $notification)->exists();}++ /**+ * Delete rows whose subject no longer exists. Subjects are often removed with bulk or cascading+ * deletes that skip model events, so this sweep is the reliable cleanup. Soft-deleted subjects are kept.+ *+ * Returns the number of deleted rows.+ */+ public static function deleteOrphans(): int+ {+ $deleted = 0;++ foreach (static::query()->distinct()->pluck('notifiable_type') as $type) {+ $class = Relation::getMorphedModel($type) ?? $type;+ $throttles = static::query()->where('notifiable_type', $type);++ if (! is_a($class, Model::class, true)) {+ $deleted += $throttles->delete();++ continue;+ }++ $subject = new $class;+ $deleted += $throttles->whereNotExists(+ $class::query()+ ->withoutGlobalScopes()+ ->selectRaw('1')+ ->whereColumn($subject->getQualifiedKeyName(), (new static)->qualifyColumn('notifiable_id'))+ )->delete();+ }++ return $deleted;+ }}diff --git a/tests/Feature/NotificationThrottleTest.php b/tests/Feature/NotificationThrottleTest.phpindex 77ad73cc80..2f1a2d2be6 100644--- a/tests/Feature/NotificationThrottleTest.php+++ b/tests/Feature/NotificationThrottleTest.php@@ -1,7 +1,9 @@<?php+use App\Jobs\CleanupInstanceStuffsJob;use App\Models\InstanceSettings;use App\Models\NotificationThrottle;+use App\Models\ScheduledDatabaseBackup;use App\Models\Server;use App\Models\Team;use App\Models\User;@@ -137,3 +139,36 @@ it('releases the unreachable claim when sending the unreachable notification threxpect(fn () => $this->server->sendUnreachableNotification())->toThrow(RuntimeException::class)->and(NotificationThrottle::wasSent($this->server, Unreachable::class))->toBeFalse();});++it('deletes throttle rows whose subject no longer exists', function () {+ $trashedServer = Server::factory()->create(['team_id' => $this->team->id]);+ $trashedServer->delete();+ $rows = [+ 'kept' => ['notifiable_type' => $this->server->getMorphClass(), 'notifiable_id' => $this->server->id],+ 'kept-soft-deleted' => ['notifiable_type' => $trashedServer->getMorphClass(), 'notifiable_id' => $trashedServer->id],+ 'missing-server' => ['notifiable_type' => $this->server->getMorphClass(), 'notifiable_id' => 999999],+ 'missing-backup' => ['notifiable_type' => ScheduledDatabaseBackup::class, 'notifiable_id' => 999999],+ 'missing-class' => ['notifiable_type' => 'App\Models\RemovedModel', 'notifiable_id' => 1],+ ];+ foreach ($rows as $notification => $row) {+ NotificationThrottle::query()->insert($row + ['notification' => $notification, 'sent_at' => now()]);+ }++ expect(NotificationThrottle::deleteOrphans())->toBe(3)+ ->and(NotificationThrottle::query()->orderBy('notification')->pluck('notification')->all())->toBe(['kept', 'kept-soft-deleted']);+});++it('deletes orphaned throttle rows during the instance cleanup', function () {+ InstanceSettings::query()->firstOrCreate(['id' => 0]);+ NotificationThrottle::record($this->server, Unreachable::class);+ NotificationThrottle::query()->insert([+ 'notifiable_type' => ScheduledDatabaseBackup::class,+ 'notifiable_id' => 999999,+ 'notification' => 'App\Notifications\Database\BackupMissing',+ 'sent_at' => now(),+ ]);++ (new CleanupInstanceStuffsJob)->handle();++ expect(NotificationThrottle::query()->pluck('notifiable_type')->all())->toBe([$this->server->getMorphClass()]);+});diff --git a/tests/Feature/RevalidateUnusableS3StoragesJobTest.php b/tests/Feature/RevalidateUnusableS3StoragesJobTest.phpindex 94b3667841..7fec8808a5 100644--- a/tests/Feature/RevalidateUnusableS3StoragesJobTest.php+++ b/tests/Feature/RevalidateUnusableS3StoragesJobTest.php@@ -3,6 +3,7 @@use App\Jobs\RevalidateUnusableS3StoragesJob;use App\Models\S3Storage;use App\Models\Team;+use Carbon\Carbon;use Illuminate\Filesystem\FilesystemAdapter;use Illuminate\Foundation\Testing\RefreshDatabase;use Illuminate\Support\Facades\Storage;@@ -61,3 +62,48 @@ it('does not test storages that are already usable', function () {(new RevalidateUnusableS3StoragesJob)->handle();});++it('includes the storage with id 0 on the first run', function () {+ $storage = revalidationStorage(Team::factory()->create());+ S3Storage::query()->whereKey($storage->id)->update(['id' => 0]);+ $disk = Mockery::mock(FilesystemAdapter::class);+ $disk->shouldReceive('files')->once()->andReturn([]);+ Storage::shouldReceive('build')->once()->andReturn($disk);++ (new RevalidateUnusableS3StoragesJob)->handle();++ expect(S3Storage::find(0)->is_usable)->toBeTrue();+});++it('stops when the time budget is used and continues with the next storage on the next run', function () {+ $team = Team::factory()->create();+ $first = revalidationStorage($team, ['bucket' => 'first-bucket']);+ $second = revalidationStorage($team, ['bucket' => 'second-bucket']);+ $tested = [];+ Storage::shouldReceive('build')->andReturnUsing(function (array $config) use (&$tested) {+ $tested[] = $config['bucket'];+ $disk = Mockery::mock(FilesystemAdapter::class);+ $disk->shouldReceive('files')->andReturnUsing(function () {+ // A slow endpoint uses the whole time budget of the run.+ Carbon::setTestNow(now()->addSeconds(RevalidateUnusableS3StoragesJob::TIME_BUDGET_SECONDS + 1));++ throw new RuntimeException('Connection timed out');+ });++ return $disk;+ });++ (new RevalidateUnusableS3StoragesJob)->handle();+ expect($tested)->toBe(['first-bucket']);++ (new RevalidateUnusableS3StoragesJob)->handle();+ expect($tested)->toBe(['first-bucket', 'second-bucket']);++ // After the last storage, the next run starts from the beginning again.+ (new RevalidateUnusableS3StoragesJob)->handle();+ expect($tested)->toBe(['first-bucket', 'second-bucket', 'first-bucket'])+ ->and($first->fresh()->is_usable)->toBeFalse()+ ->and($second->fresh()->is_usable)->toBeFalse();++ Carbon::setTestNow();+});