Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

feat(s3): revalidate unusable S3 storages hourly

Andras Bacsai639a07e4cb57
diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php
index 150d956d5f..c827586b49 100644
--- a/app/Console/Kernel.php
+++ b/app/Console/Kernel.php
@@ -12,6 +12,7 @@ use App\Jobs\CleanupStaleMultiplexedConnections;
use App\Jobs\PullChangelog;
use App\Jobs\PullTemplatesFromCDN;
use App\Jobs\RegenerateSslCertJob;
+use App\Jobs\RevalidateUnusableS3StoragesJob;
use App\Jobs\ServerManagerJob;
use App\Jobs\UpdateCoolifyJob;
use App\Models\InstanceSettings;
@@ -63,6 +64,7 @@ class Kernel extends ConsoleKernel
->runInBackground();
$this->scheduleInstance->job(new ApiTokenExpirationWarningJob)->hourly()->onOneServer();
$this->scheduleInstance->job(new CheckMissingDatabaseBackupsJob)->hourly()->onOneServer();
+ $this->scheduleInstance->job(new RevalidateUnusableS3StoragesJob)->hourly()->onOneServer();
if (isDev()) {
// Instance Jobs
diff --git a/app/Jobs/RevalidateUnusableS3StoragesJob.php b/app/Jobs/RevalidateUnusableS3StoragesJob.php
new file mode 100644
index 0000000000..fe9593b6fc
--- /dev/null
+++ b/app/Jobs/RevalidateUnusableS3StoragesJob.php
@@ -0,0 +1,39 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\S3Storage;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+
+/**
+ * Re-tests S3 storages that failed validation, so a storage that becomes reachable again is usable without manual action.
+ */
+class RevalidateUnusableS3StoragesJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue
+{
+ use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+ public $tries = 1;
+
+ public $timeout = 1800;
+
+ public function handle(): void
+ {
+ S3Storage::query()
+ ->where('is_usable', false)
+ ->chunkById(100, function ($storages): void {
+ foreach ($storages as $storage) {
+ try {
+ $storage->testConnection(shouldSave: true);
+ } catch (\Throwable) {
+ // testConnection() keeps the storage unusable and notifies the team once.
+ }
+ }
+ });
+ }
+}
diff --git a/tests/Feature/RevalidateUnusableS3StoragesJobTest.php b/tests/Feature/RevalidateUnusableS3StoragesJobTest.php
new file mode 100644
index 0000000000..94b3667841
--- /dev/null
+++ b/tests/Feature/RevalidateUnusableS3StoragesJobTest.php
@@ -0,0 +1,63 @@
+<?php
+
+use App\Jobs\RevalidateUnusableS3StoragesJob;
+use App\Models\S3Storage;
+use App\Models\Team;
+use Illuminate\Filesystem\FilesystemAdapter;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Storage;
+
+uses(RefreshDatabase::class);
+
+function revalidationStorage(Team $team, array $attributes = []): S3Storage
+{
+ return S3Storage::create(array_merge([
+ 'name' => 'Storage',
+ 'region' => 'us-east-1',
+ 'key' => 'key',
+ 'secret' => 'secret',
+ 'bucket' => 'bucket',
+ 'endpoint' => 'https://93.184.216.34',
+ 'team_id' => $team->id,
+ 'is_usable' => false,
+ 'unusable_email_sent' => true,
+ ], $attributes));
+}
+
+it('marks an unusable storage as usable when the connection works again', function () {
+ $storage = revalidationStorage(Team::factory()->create());
+ $disk = Mockery::mock(FilesystemAdapter::class);
+ $disk->shouldReceive('files')->once()->andReturn([]);
+ Storage::shouldReceive('build')->once()->andReturn($disk);
+
+ (new RevalidateUnusableS3StoragesJob)->handle();
+
+ $storage->refresh();
+ expect($storage->is_usable)->toBeTrue()
+ ->and($storage->unusable_email_sent)->toBeFalsy();
+});
+
+it('keeps a storage unusable when the connection still fails and continues with the next one', function () {
+ $team = Team::factory()->create();
+ $failing = revalidationStorage($team, ['bucket' => 'failing-bucket']);
+ $working = revalidationStorage($team, ['bucket' => 'working-bucket']);
+ $failingDisk = Mockery::mock(FilesystemAdapter::class);
+ $failingDisk->shouldReceive('files')->once()->andThrow(new RuntimeException('Access denied'));
+ $workingDisk = Mockery::mock(FilesystemAdapter::class);
+ $workingDisk->shouldReceive('files')->once()->andReturn([]);
+ Storage::shouldReceive('build')->twice()->andReturnUsing(
+ fn (array $config) => $config['bucket'] === 'failing-bucket' ? $failingDisk : $workingDisk
+ );
+
+ (new RevalidateUnusableS3StoragesJob)->handle();
+
+ expect($failing->fresh()->is_usable)->toBeFalse()
+ ->and($working->fresh()->is_usable)->toBeTrue();
+});
+
+it('does not test storages that are already usable', function () {
+ revalidationStorage(Team::factory()->create(), ['is_usable' => true]);
+ Storage::shouldReceive('build')->never();
+
+ (new RevalidateUnusableS3StoragesJob)->handle();
+});