Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(database-import): delete staged API uploads that were never imported

Andras Bacsai476ba3c6ab37
diff --git a/app/Console/Commands/CleanupDatabaseImportUploads.php b/app/Console/Commands/CleanupDatabaseImportUploads.php
new file mode 100644
index 0000000000..d70fb4eb36
--- /dev/null
+++ b/app/Console/Commands/CleanupDatabaseImportUploads.php
@@ -0,0 +1,72 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Actions\Database\StartDatabaseImport;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Storage;
+
+class CleanupDatabaseImportUploads extends Command
+{
+ public const MAX_AGE_HOURS = 24;
+
+ protected $signature = 'cleanup:database-import-uploads';
+
+ protected $description = 'Delete staged API database import uploads that were not imported within a day';
+
+ /**
+ * Uploads are staged at upload/imports/{team}/{database uuid}/{upload id}/restore. Starting an import
+ * copies the file to the server and deletes it while it holds the import lock of the database, so an
+ * upload is only deleted while this command holds that lock too.
+ */
+ public function handle(): int
+ {
+ $deleted = 0;
+ $cutoff = now()->subHours(self::MAX_AGE_HOURS)->getTimestamp();
+
+ foreach (Storage::directories('upload/imports') as $teamDirectory) {
+ foreach (Storage::directories($teamDirectory) as $databaseDirectory) {
+ foreach (Storage::directories($databaseDirectory) as $uploadDirectory) {
+ if ($this->lastModified($uploadDirectory) < $cutoff && $this->deleteUnlessImporting($uploadDirectory, basename($databaseDirectory))) {
+ $deleted++;
+ }
+ }
+ $this->deleteIfEmpty($databaseDirectory);
+ }
+ $this->deleteIfEmpty($teamDirectory);
+ }
+
+ $this->info("Deleted {$deleted} staged database import upload(s).");
+
+ return self::SUCCESS;
+ }
+
+ private function lastModified(string $directory): int
+ {
+ $timestamps = array_map(fn (string $file): int => Storage::lastModified($file), Storage::allFiles($directory));
+
+ return $timestamps === [] ? (int) filemtime(Storage::path($directory)) : max($timestamps);
+ }
+
+ private function deleteUnlessImporting(string $directory, string $databaseUuid): bool
+ {
+ $lock = Cache::lock(StartDatabaseImport::lockKey($databaseUuid), 60);
+ if (! $lock->get()) {
+ return false;
+ }
+
+ try {
+ return Storage::deleteDirectory($directory);
+ } finally {
+ $lock->release();
+ }
+ }
+
+ private function deleteIfEmpty(string $directory): void
+ {
+ if (Storage::allFiles($directory) === [] && Storage::allDirectories($directory) === []) {
+ Storage::deleteDirectory($directory);
+ }
+ }
+}
diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php
index 4e4d921b08..a1fb63b40b 100644
--- a/app/Console/Kernel.php
+++ b/app/Console/Kernel.php
@@ -59,6 +59,7 @@ class Kernel extends ConsoleKernel
->withoutOverlapping(60)
->runInBackground();
$this->scheduleInstance->command('sanctum:prune-expired --hours=1')->hourly()->onOneServer();
+ $this->scheduleInstance->command('cleanup:database-import-uploads')->hourly()->onOneServer();
$this->scheduleInstance->command('dns:release-orphaned-records')
->hourly()
->onOneServer()
diff --git a/tests/Feature/CleanupDatabaseImportUploadsTest.php b/tests/Feature/CleanupDatabaseImportUploadsTest.php
new file mode 100644
index 0000000000..1564c1e9ce
--- /dev/null
+++ b/tests/Feature/CleanupDatabaseImportUploadsTest.php
@@ -0,0 +1,46 @@
+<?php
+
+use App\Actions\Database\StartDatabaseImport;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Storage;
+
+beforeEach(function () {
+ Storage::fake('local');
+ Cache::flush();
+});
+
+function stageDatabaseImportUploadForCleanupTest(string $databaseUuid, string $uploadId, Carbon $modifiedAt): string
+{
+ $directory = "upload/imports/1/{$databaseUuid}/{$uploadId}";
+ Storage::put("{$directory}/restore", 'dump');
+ touch(Storage::path("{$directory}/restore"), $modifiedAt->getTimestamp());
+
+ return $directory;
+}
+
+it('deletes staged API import uploads older than a day and keeps newer ones', function () {
+ $old = stageDatabaseImportUploadForCleanupTest('db-old', 'f4c1d3a2-0000-4000-8000-000000000001', now()->subHours(25));
+ $fresh = stageDatabaseImportUploadForCleanupTest('db-fresh', 'f4c1d3a2-0000-4000-8000-000000000002', now()->subHours(23));
+
+ $this->artisan('cleanup:database-import-uploads')->assertSuccessful();
+
+ expect(Storage::exists($old))->toBeFalse()
+ ->and(Storage::exists('upload/imports/1/db-old'))->toBeFalse()
+ ->and(Storage::exists("{$fresh}/restore"))->toBeTrue();
+});
+
+it('keeps an old staged upload while an import of its database holds the import lock', function () {
+ $directory = stageDatabaseImportUploadForCleanupTest('db-busy', 'f4c1d3a2-0000-4000-8000-000000000003', now()->subDays(2));
+ $lock = Cache::lock(StartDatabaseImport::lockKey('db-busy'), 60);
+ $lock->get();
+
+ $this->artisan('cleanup:database-import-uploads')->assertSuccessful();
+
+ expect(Storage::exists("{$directory}/restore"))->toBeTrue();
+
+ $lock->release();
+ $this->artisan('cleanup:database-import-uploads')->assertSuccessful();
+
+ expect(Storage::exists($directory))->toBeFalse();
+});