An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
docs(backups): add multi-destination S3 planning notes (#11789)
Andras Bacsaid4d8bba51467
diff --git a/.ai/lessons.md b/.ai/lessons.mdindex 5cde3ef34e..7b94e29b3b 100644--- a/.ai/lessons.md+++ b/.ai/lessons.md@@ -12,6 +12,7 @@- SSH retries replay the full command batch. Only exit code 255 is an SSH transport failure; other exit codes come from the remote command, so do not retry them. Keep batches safe to run twice (for example `docker rm -f <name> || true` before `docker run --name <name>`).- Redirect browser test output to a file (`> /tmp/x.log 2>&1`); piping it (`| tail`) hangs because the Playwright server keeps the pipe open.- Browser `click('Text')` can match a hidden element first (for example inactive modal steps) and wait forever; use `button:visible:has-text("Text")`.+- A new worktree has no `vendor/` or `public/build/`. Run `composer install --ignore-platform-req=ext-zip` and copy `public/build` from the main checkout before page-rendering tests; otherwise they fail with a Vite manifest error.- Never run `php artisan test` inside a dev container (`./scripts/dev exec` or `docker exec coolify-dev-*`). The container sets `DB_CONNECTION=pgsql` as a real environment variable, which wins over `phpunit.xml`, so `RefreshDatabase` runs `migrate:fresh` on the dev database and wipes it. Run tests on the host only.- Call `visit()` directly in each `tests/v4/Browser` test body; Pest does not mark a test that only uses helper-wrapped `visit()` as a browser test, so it fails with `sendText() on null`.diff --git a/app/Actions/Development/StartDevelopmentInstanceDatabase.php b/app/Actions/Development/StartDevelopmentInstanceDatabase.phpnew file mode 100644index 0000000000..41b8428bd4--- /dev/null+++ b/app/Actions/Development/StartDevelopmentInstanceDatabase.php@@ -0,0 +1,33 @@+<?php++namespace App\Actions\Development;++use App\Models\Server;+use Lorisleiva\Actions\Concerns\AsAction;++/**+ * Starts a `coolify-db` container on a development VM that acts as the localhost server.+ *+ * The instance backup reads the credentials from this container and dumps it, like on a self-hosted install.+ * The real development database runs on the host, so the VM cannot reach it.+ */+class StartDevelopmentInstanceDatabase+{+ use AsAction;++ public function handle(Server $server): void+ {+ $environment = collect([+ 'POSTGRES_USER' => config('database.connections.pgsql.username'),+ 'POSTGRES_PASSWORD' => config('database.connections.pgsql.password'),+ 'POSTGRES_DB' => config('database.connections.pgsql.database'),+ ])->map(fn ($value, string $key): string => '-e '.escapeshellarg("{$key}={$value}"))->implode(' ');++ instant_remote_process([+ 'if ! docker inspect coolify-db >/dev/null 2>&1; then',+ "docker run -d --name coolify-db --restart unless-stopped {$environment} postgres:15-alpine",+ 'fi',+ 'docker start coolify-db',+ ], $server, disableMultiplexing: true);+ }+}diff --git a/app/Actions/Shared/DeleteScheduledVolumeBackup.php b/app/Actions/Shared/DeleteScheduledVolumeBackup.phpindex d56cce9384..afeb44fce6 100644--- a/app/Actions/Shared/DeleteScheduledVolumeBackup.php+++ b/app/Actions/Shared/DeleteScheduledVolumeBackup.php@@ -4,6 +4,7 @@ namespace App\Actions\Shared;use App\Jobs\VolumeBackupJob;use App\Models\ScheduledVolumeBackup;+use App\Models\ScheduledVolumeBackupExecution;use App\Models\Server;use Illuminate\Support\Facades\Cache;use Lorisleiva\Actions\Concerns\AsAction;@@ -52,23 +53,10 @@ class DeleteScheduledVolumeBackup}if ($deleteS3Archives) {- $s3Executions = $backup->executions()- ->with('s3')- ->where('s3_uploaded', true)- ->where('s3_storage_deleted', false)- ->get();-- foreach ($s3Executions->groupBy('s3_storage_id') as $executions) {- $s3 = $executions->first()->s3;- if (! $s3) {- throw new \RuntimeException('The S3 storage used by an existing backup is unavailable.');- }-- $filenames = $executions->pluck('filename')->filter()->all();- if ($filenames !== []) {- deleteBackupsS3($filenames, $s3);- }- }+ $backup->executions()+ ->whereHas('s3Replicas', fn ($query) => $query->where('s3_uploaded', true)->where('s3_storage_deleted', false))+ ->get()+ ->each(fn (ScheduledVolumeBackupExecution $execution) => $execution->deleteS3Copies());}$backup->delete();diff --git a/app/Console/Commands/SeedDevelopmentQemuServerCommand.php b/app/Console/Commands/SeedDevelopmentQemuServerCommand.phpindex 082ad83ae2..26fc463ff7 100644--- a/app/Console/Commands/SeedDevelopmentQemuServerCommand.php+++ b/app/Console/Commands/SeedDevelopmentQemuServerCommand.php@@ -3,7 +3,9 @@namespace App\Console\Commands;use App\Actions\Development\SeedDevelopmentQemuServer;+use App\Actions\Development\StartDevelopmentInstanceDatabase;use Illuminate\Console\Command;+use Throwable;class SeedDevelopmentQemuServerCommand extends Command{@@ -22,6 +24,15 @@ class SeedDevelopmentQemuServerCommand extends Command$server = SeedDevelopmentQemuServer::run($this->argument('profile'), ! $this->option('keep-others'), (bool) $this->option('as-localhost'), $this->option('ip') ?: null, (int) $this->option('port'));$this->info("Seeded {$server->name} at {$server->ip}.");+ if ($this->option('as-localhost')) {+ try {+ StartDevelopmentInstanceDatabase::run($server);+ $this->info('Started the coolify-db container for instance backups.');+ } catch (Throwable $e) {+ $this->warn("Could not start the coolify-db container for instance backups: {$e->getMessage()}");+ }+ }+return self::SUCCESS;}}diff --git a/app/Http/Controllers/Api/DatabasesController.php b/app/Http/Controllers/Api/DatabasesController.phpindex d71b64563b..b6fe02d8bc 100644--- a/app/Http/Controllers/Api/DatabasesController.php+++ b/app/Http/Controllers/Api/DatabasesController.php@@ -32,6 +32,8 @@ use Illuminate\Database\Eloquent\Collection as EloquentCollection;use Illuminate\Database\Eloquent\Model;use Illuminate\Http\JsonResponse;use Illuminate\Http\Request;+use Illuminate\Support\Arr;+use Illuminate\Support\Collection;use Illuminate\Support\Facades\Bus;use Illuminate\Support\Facades\DB;use OpenApi\Attributes as OA;@@ -152,6 +154,58 @@ class DatabasesController extends Controller}}+ /**+ * Resolves the S3 destinations of a backup request, primary first. `s3_storage_uuids` lists every destination;+ * `s3_storage_uuid` alone selects a single destination, and sent with `s3_storage_uuids` it must be in the list and+ * becomes the primary destination.+ *+ * @return Collection<int, S3Storage>|JsonResponse|null Null when the request does not select destinations.+ */+ private function requestedBackupS3Storages(Request $request, int|string $teamId): Collection|JsonResponse|null+ {+ $hasList = $request->has('s3_storage_uuids');+ if (! $hasList && ! $request->filled('s3_storage_uuid')) {+ return null;+ }++ $uuids = collect($hasList ? $request->input('s3_storage_uuids') : [$request->input('s3_storage_uuid')])->unique()->values();+ $primaryUuid = $request->input('s3_storage_uuid') ?? $uuids->first();+ $errorField = $hasList ? 's3_storage_uuids' : 's3_storage_uuid';++ if (filled($primaryUuid) && ! $uuids->contains($primaryUuid)) {+ return response()->json([+ 'message' => 'Validation failed.',+ 'errors' => ['s3_storage_uuid' => ['The s3_storage_uuid must be one of the s3_storage_uuids.']],+ ], 422);+ }++ $storages = S3Storage::ownedByCurrentTeamAPI($teamId)->whereIn('uuid', $uuids->all())->get();+ if ($storages->count() !== $uuids->count()) {+ return response()->json([+ 'message' => 'Validation failed.',+ 'errors' => [$errorField => ['The selected S3 storage is invalid for this team.']],+ ], 422);+ }++ return $storages->sortBy(fn (S3Storage $storage): int => $storage->uuid === $primaryUuid ? 0 : 1)->values();+ }++ /**+ * Adds the S3 destination UUIDs (primary first) to a serialized backup without exposing the storages themselves.+ */+ private function withBackupS3StorageUuids(ScheduledDatabaseBackup $backup): ScheduledDatabaseBackup+ {+ $storages = $backup->s3Storages->sortBy(fn (S3Storage $storage): int => $storage->id === $backup->s3_storage_id ? 0 : 1);+ if ($storages->isEmpty() && $backup->s3) {+ $storages = collect([$backup->s3]);+ }++ $backup->setAttribute('s3_storage_uuid', $backup->s3?->uuid);+ $backup->setAttribute('s3_storage_uuids', $storages->pluck('uuid')->values()->all());++ return $backup->unsetRelation('s3Storages')->unsetRelation('s3');+ }+private function removeSensitiveData($database, bool $loadNestedServerSecrets = false){$database->makeHidden([@@ -302,9 +356,10 @@ class DatabasesController extends Controller$databaseIds = $databases->pluck('id')->toArray();- $backupConfigs = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId)->with('latest_log')+ $backupConfigs = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId)->with(['latest_log', 's3', 's3Storages'])->whereIn('database_id', $databaseIds)->get()+ ->map(fn (ScheduledDatabaseBackup $backup) => $this->withBackupS3StorageUuids($backup))->groupBy(fn (ScheduledDatabaseBackup $backup) => $backup->database_type.':'.$backup->database_id);$databases = $databases->map(function ($database) use ($backupConfigs) {@@ -340,7 +395,7 @@ class DatabasesController extends Controllerresponses: [new OA\Response(response: 200,- description: 'Get all backups for a database',+ description: 'Get all backups for a database. Each backup includes s3_storage_uuid (primary S3 destination) and s3_storage_uuids (every S3 destination, primary first).',content: new OA\JsonContent(type: 'string',example: 'Content is very complex. Will be implemented later.',@@ -376,7 +431,8 @@ class DatabasesController extends Controller$this->authorize('view', $database);- $backupConfig = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId)->with('executions')->where('database_id', $database->id)->get();+ $backupConfig = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId)->with(['executions', 's3', 's3Storages'])->where('database_id', $database->id)->get()+ ->map(fn (ScheduledDatabaseBackup $backup) => $this->withBackupS3StorageUuids($backup));return response()->json($backupConfig);}@@ -873,7 +929,8 @@ class DatabasesController extends Controller'frequency' => ['type' => 'string', 'description' => 'Backup frequency (cron expression or: every_minute, hourly, daily, weekly, monthly, yearly)'],'enabled' => ['type' => 'boolean', 'description' => 'Whether the backup is enabled', 'default' => true],'save_s3' => ['type' => 'boolean', 'description' => 'Whether to save backups to S3', 'default' => false],- 's3_storage_uuid' => ['type' => 'string', 'description' => 'S3 storage UUID (required if save_s3 is true)'],+ 's3_storage_uuid' => ['type' => 'string', 'description' => 'Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination. Either field is required if save_s3 is true.'],+ 's3_storage_uuids' => ['type' => 'array', 'items' => ['type' => 'string'], 'description' => 'UUIDs of every S3 storage the backup is uploaded to. Replaces the current destinations. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination.'],'databases_to_backup' => ['type' => 'string', 'description' => 'Comma separated list of databases to backup'],'dump_all' => ['type' => 'boolean', 'description' => 'Whether to dump all databases', 'default' => false],'backup_now' => ['type' => 'boolean', 'description' => 'Whether to trigger backup immediately after creation'],@@ -921,7 +978,7 @@ class DatabasesController extends Controller)]public function create_backup(Request $request){- $backupConfigFields = ['save_s3', 'enabled', 'dump_all', 'frequency', 'databases_to_backup', 'database_backup_retention_amount_locally', 'database_backup_retention_days_locally', 'database_backup_retention_max_storage_locally', 'database_backup_retention_amount_s3', 'database_backup_retention_days_s3', 'database_backup_retention_max_storage_s3', 's3_storage_uuid', 'timeout', 'missing_backup_notification_days'];+ $backupConfigFields = ['save_s3', 'enabled', 'dump_all', 'frequency', 'databases_to_backup', 'database_backup_retention_amount_locally', 'database_backup_retention_days_locally', 'database_backup_retention_max_storage_locally', 'database_backup_retention_amount_s3', 'database_backup_retention_days_s3', 'database_backup_retention_max_storage_s3', 's3_storage_uuid', 's3_storage_uuids', 'timeout', 'missing_backup_notification_days'];$teamId = getTeamIdFromToken();if (is_null($teamId)) {@@ -941,6 +998,8 @@ class DatabasesController extends Controller'dump_all' => 'boolean','backup_now' => 'boolean|nullable','s3_storage_uuid' => 'string|exists:s3_storages,uuid|nullable',+ 's3_storage_uuids' => 'array',+ 's3_storage_uuids.*' => 'string|distinct','databases_to_backup' => 'string|nullable','database_backup_retention_amount_locally' => 'integer|min:0','database_backup_retention_days_locally' => 'integer|min:0',@@ -986,24 +1045,17 @@ class DatabasesController extends Controller], 422);}- // Validate S3 storage if save_s3 is true- if ($request->boolean('save_s3') && ! $request->filled('s3_storage_uuid')) {+ $s3Storages = $this->requestedBackupS3Storages($request, $teamId);+ if ($s3Storages instanceof JsonResponse) {+ return $s3Storages;+ }+ if ($request->boolean('save_s3') && blank($s3Storages)) {return response()->json(['message' => 'Validation failed.',- 'errors' => ['s3_storage_uuid' => ['The s3_storage_uuid field is required when save_s3 is true.']],+ 'errors' => ['s3_storage_uuid' => ['The s3_storage_uuid or s3_storage_uuids field is required when save_s3 is true.']],], 422);}- if ($request->filled('s3_storage_uuid')) {- $existsInTeam = S3Storage::ownedByCurrentTeamAPI($teamId)->where('uuid', $request->s3_storage_uuid)->exists();- if (! $existsInTeam) {- return response()->json([- 'message' => 'Validation failed.',- 'errors' => ['s3_storage_uuid' => ['The selected S3 storage is invalid for this team.']],- ], 422);- }- }-// Check for extra fields$extraFields = array_diff(array_keys($request->all()), $backupConfigFields, ['backup_now']);if (! empty($extraFields)) {@@ -1018,20 +1070,9 @@ class DatabasesController extends Controller], 422);}- $backupData = $request->only($backupConfigFields);-- // Convert s3_storage_uuid to s3_storage_id- if (isset($backupData['s3_storage_uuid'])) {- $s3Storage = S3Storage::ownedByCurrentTeamAPI($teamId)->where('uuid', $backupData['s3_storage_uuid'])->first();- if ($s3Storage) {- $backupData['s3_storage_id'] = $s3Storage->id;- } elseif ($request->boolean('save_s3')) {- return response()->json([- 'message' => 'Validation failed.',- 'errors' => ['s3_storage_uuid' => ['The selected S3 storage is invalid for this team.']],- ], 422);- }- unset($backupData['s3_storage_uuid']);+ $backupData = Arr::except($request->only($backupConfigFields), ['s3_storage_uuid', 's3_storage_uuids']);+ if ($s3Storages !== null) {+ $backupData['s3_storage_id'] = $s3Storages->first()?->id;}// Set default databases_to_backup based on database type if not provided@@ -1070,6 +1111,9 @@ class DatabasesController extends Controller}$backupConfig = ScheduledDatabaseBackup::create($backupData);+ if ($s3Storages !== null) {+ $backupConfig->syncS3Storages($s3Storages->pluck('id')->all());+ }// Trigger immediate backup if requestedif ($request->backup_now) {@@ -1129,7 +1173,8 @@ class DatabasesController extends Controllertype: 'object',properties: ['save_s3' => ['type' => 'boolean', 'description' => 'Whether data is saved in s3 or not'],- 's3_storage_uuid' => ['type' => 'string', 'description' => 'S3 storage UUID'],+ 's3_storage_uuid' => ['type' => 'string', 'description' => 'Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination; when it is already a destination, it becomes the primary destination and the other destinations stay.'],+ 's3_storage_uuids' => ['type' => 'array', 'items' => ['type' => 'string'], 'description' => 'UUIDs of every S3 storage the backup is uploaded to. Replaces the current destinations. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination.'],'backup_now' => ['type' => 'boolean', 'description' => 'Whether to take a backup now or not'],'enabled' => ['type' => 'boolean', 'description' => 'Whether the backup is enabled or not'],'databases_to_backup' => ['type' => 'string', 'description' => 'Comma separated list of databases to backup'],@@ -1172,7 +1217,7 @@ class DatabasesController extends Controller)]public function update_backup(Request $request){- $backupConfigFields = ['save_s3', 'enabled', 'dump_all', 'frequency', 'databases_to_backup', 'database_backup_retention_amount_locally', 'database_backup_retention_days_locally', 'database_backup_retention_max_storage_locally', 'database_backup_retention_amount_s3', 'database_backup_retention_days_s3', 'database_backup_retention_max_storage_s3', 's3_storage_uuid', 'timeout', 'missing_backup_notification_days'];+ $backupConfigFields = ['save_s3', 'enabled', 'dump_all', 'frequency', 'databases_to_backup', 'database_backup_retention_amount_locally', 'database_backup_retention_days_locally', 'database_backup_retention_max_storage_locally', 'database_backup_retention_amount_s3', 'database_backup_retention_days_s3', 'database_backup_retention_max_storage_s3', 's3_storage_uuid', 's3_storage_uuids', 'timeout', 'missing_backup_notification_days'];$teamId = getTeamIdFromToken();if (is_null($teamId)) {@@ -1189,6 +1234,8 @@ class DatabasesController extends Controller'enabled' => 'boolean','dump_all' => 'boolean','s3_storage_uuid' => 'string|exists:s3_storages,uuid|nullable',+ 's3_storage_uuids' => 'array',+ 's3_storage_uuids.*' => 'string|distinct','databases_to_backup' => 'string|nullable','frequency' => 'string','database_backup_retention_amount_locally' => 'integer|min:0',@@ -1236,20 +1283,9 @@ class DatabasesController extends Controller}}- if ($request->boolean('save_s3') && ! $request->filled('s3_storage_uuid')) {- return response()->json([- 'message' => 'Validation failed.',- 'errors' => ['s3_storage_uuid' => ['The s3_storage_uuid field is required when save_s3 is true.']],- ], 422);- }- if ($request->filled('s3_storage_uuid')) {- $existsInTeam = S3Storage::ownedByCurrentTeamAPI($teamId)->where('uuid', $request->s3_storage_uuid)->exists();- if (! $existsInTeam) {- return response()->json([- 'message' => 'Validation failed.',- 'errors' => ['s3_storage_uuid' => ['The selected S3 storage is invalid for this team.']],- ], 422);- }+ $s3Storages = $this->requestedBackupS3Storages($request, $teamId);+ if ($s3Storages instanceof JsonResponse) {+ return $s3Storages;}$backupConfig = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId)->where('database_id', $database->id)@@ -1259,6 +1295,14 @@ class DatabasesController extends Controllerreturn response()->json(['message' => 'Backup config not found.'], 404);}+ $saveToS3 = $request->has('save_s3') ? $request->boolean('save_s3') : (bool) $backupConfig->save_s3;+ if (($request->boolean('save_s3') && $s3Storages === null) || ($saveToS3 && $s3Storages?->isEmpty())) {+ return response()->json([+ 'message' => 'Validation failed.',+ 'errors' => ['s3_storage_uuid' => ['The s3_storage_uuid or s3_storage_uuids field is required when save_s3 is true.']],+ ], 422);+ }+$extraFields = array_diff(array_keys($request->all()), $backupConfigFields, ['backup_now']);if (! empty($extraFields)) {$errors = $validator->errors();@@ -1272,20 +1316,9 @@ class DatabasesController extends Controller], 422);}- $backupData = $request->only($backupConfigFields);-- // Convert s3_storage_uuid to s3_storage_id- if (isset($backupData['s3_storage_uuid'])) {- $s3Storage = S3Storage::ownedByCurrentTeamAPI($teamId)->where('uuid', $backupData['s3_storage_uuid'])->first();- if ($s3Storage) {- $backupData['s3_storage_id'] = $s3Storage->id;- } elseif ($request->boolean('save_s3')) {- return response()->json([- 'message' => 'Validation failed.',- 'errors' => ['s3_storage_uuid' => ['The selected S3 storage is invalid for this team.']],- ], 422);- }- unset($backupData['s3_storage_uuid']);+ $backupData = Arr::except($request->only($backupConfigFields), ['s3_storage_uuid', 's3_storage_uuids']);+ if ($s3Storages !== null) {+ $backupData['s3_storage_id'] = $s3Storages->first()?->id;}// Validate databases_to_backup input@@ -1300,7 +1333,19 @@ class DatabasesController extends Controller}}+ $storageIds = $s3Storages?->pluck('id');+ // An older client sends only s3_storage_uuid. When it is already a destination, the other destinations stay.+ if ($storageIds !== null && ! $request->has('s3_storage_uuids')) {+ $currentStorageIds = $backupConfig->selectedS3Storages()->pluck('id');+ if ($currentStorageIds->contains($storageIds->first())) {+ $storageIds = $currentStorageIds;+ }+ }+$backupConfig->update($backupData);+ if ($storageIds !== null) {+ $backupConfig->syncS3Storages($storageIds->all());+ }if ($request->backup_now) {dispatch(new DatabaseBackupJob($backupConfig));@@ -2938,9 +2983,10 @@ class DatabasesController extends Controllerif ($execution->filename) {deleteBackupsLocally($execution->filename, $database->destination->server);- if ($deleteS3 && $backup->s3) {- deleteBackupsS3($execution->filename, $backup->s3);- }+ }++ if ($deleteS3) {+ $execution->deleteS3Copies();}$execution->delete();@@ -3073,9 +3119,10 @@ class DatabasesController extends Controllerif ($execution->filename) {deleteBackupsLocally($execution->filename, $database->destination->server);- if ($deleteS3 && $backup->s3) {- deleteBackupsS3($execution->filename, $backup->s3);- }+ }++ if ($deleteS3) {+ $execution->deleteS3Copies();}$execution->delete();@@ -5195,7 +5242,7 @@ class DatabasesController extends Controller}foreach ($database->scheduledBackups()->get() as $backup) {- $backup->replicate([+ $newBackup = $backup->replicate(['id','created_at','updated_at',@@ -5205,7 +5252,9 @@ class DatabasesController extends Controller'database_id' => $newDatabase->id,'database_type' => $newDatabase->getMorphClass(),'team_id' => $teamId,- ])->save();+ ]);+ $newBackup->save();+ $backup->copyS3StoragesTo($newBackup);}foreach ($database->environment_variables()->get() as $environmentVariable) {diff --git a/app/Http/Controllers/Api/ServicesController.php b/app/Http/Controllers/Api/ServicesController.phpindex 7411f8a472..01f3657434 100644--- a/app/Http/Controllers/Api/ServicesController.php+++ b/app/Http/Controllers/Api/ServicesController.php@@ -3399,7 +3399,7 @@ class ServicesController extends Controller}foreach ($sourceDatabase->scheduledBackups()->get() as $backup) {- $backup->replicate([+ $newBackup = $backup->replicate(['id','created_at','updated_at',@@ -3408,7 +3408,9 @@ class ServicesController extends Controller'database_id' => $database->id,'database_type' => $database->getMorphClass(),'team_id' => $teamId,- ])->save();+ ]);+ $newBackup->save();+ $backup->copyS3StoragesTo($newBackup);}}diff --git a/app/Http/Controllers/Api/VolumeBackupsController.php b/app/Http/Controllers/Api/VolumeBackupsController.phpindex e3b48a367a..a199f530db 100644--- a/app/Http/Controllers/Api/VolumeBackupsController.php+++ b/app/Http/Controllers/Api/VolumeBackupsController.php@@ -14,6 +14,7 @@ use App\Models\Service;use Illuminate\Database\Eloquent\Model;use Illuminate\Http\JsonResponse;use Illuminate\Http\Request;+use Illuminate\Support\Collection;use Illuminate\Support\MessageBag;use OpenApi\Attributes as OA;use RuntimeException;@@ -27,7 +28,8 @@ use RuntimeException;new OA\Property(property: 'save_s3', type: 'boolean', default: false),new OA\Property(property: 'disable_local_backup', type: 'boolean', default: false),new OA\Property(property: 'stop_during_backup', type: 'boolean', default: false),- new OA\Property(property: 's3_storage_uuid', type: 'string', nullable: true),+ new OA\Property(property: 's3_storage_uuid', type: 'string', nullable: true, description: 'Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination.'),+ new OA\Property(property: 's3_storage_uuids', type: 'array', items: new OA\Items(type: 'string'), description: 'UUIDs of every S3 storage the backup is uploaded to. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination. save_s3 requires at least one destination.'),new OA\Property(property: 'retention_amount_locally', type: 'integer', default: 7, minimum: 0, maximum: 10000),new OA\Property(property: 'retention_days_locally', type: 'integer', default: 0, maximum: 2147483647, minimum: 0),new OA\Property(property: 'retention_max_storage_locally', type: 'number', format: 'float', default: 0, maximum: 9999999999, minimum: 0),@@ -53,7 +55,8 @@ use RuntimeException;new OA\Property(property: 'save_s3', type: 'boolean'),new OA\Property(property: 'disable_local_backup', type: 'boolean'),new OA\Property(property: 'stop_during_backup', type: 'boolean'),- new OA\Property(property: 's3_storage_uuid', type: 'string', nullable: true),+ new OA\Property(property: 's3_storage_uuid', type: 'string', nullable: true, description: 'Primary S3 destination.'),+ new OA\Property(property: 's3_storage_uuids', type: 'array', items: new OA\Items(type: 'string'), description: 'Every S3 destination, primary first.'),new OA\Property(property: 'retention_amount_locally', type: 'integer'),new OA\Property(property: 'retention_days_locally', type: 'integer'),new OA\Property(property: 'retention_max_storage_locally', type: 'number', format: 'float'),@@ -165,7 +168,7 @@ class VolumeBackupsController extends Controllerreturn response()->json(['message' => 'Storage not found.'], 404);}- ['errors' => $errors, 's3Storage' => $s3Storage, 'saveToS3' => $saveToS3] = $this->validateUpsertRequest($request, $storage, $teamId);+ ['errors' => $errors, 's3Storages' => $s3Storages, 'saveToS3' => $saveToS3] = $this->validateUpsertRequest($request, $storage, $teamId);if ($errors->isNotEmpty()) {return response()->json([@@ -174,11 +177,11 @@ class VolumeBackupsController extends Controller], 422);}- return $this->persistSchedule($request, $storage, $teamId, $s3Storage, $saveToS3, $resourceType, $resource);+ return $this->persistSchedule($request, $storage, $teamId, $s3Storages, $saveToS3, $resourceType, $resource);}/**- * @return array{errors: MessageBag, s3Storage: S3Storage|null, saveToS3: bool}+ * @return array{errors: MessageBag, s3Storages: Collection<int, S3Storage>, saveToS3: bool}*/private function validateUpsertRequest(Request $request,@@ -192,6 +195,8 @@ class VolumeBackupsController extends Controller'disable_local_backup' => 'boolean','stop_during_backup' => 'boolean','s3_storage_uuid' => 'nullable|string',+ 's3_storage_uuids' => 'array',+ 's3_storage_uuids.*' => 'string|distinct','retention_amount_locally' => 'integer|min:0|max:10000','retention_days_locally' => 'integer|min:0|max:2147483647','retention_max_storage_locally' => 'numeric|min:0|max:9999999999',@@ -209,6 +214,7 @@ class VolumeBackupsController extends Controller'disable_local_backup','stop_during_backup','s3_storage_uuid',+ 's3_storage_uuids','retention_amount_locally','retention_days_locally','retention_max_storage_locally',@@ -232,16 +238,24 @@ class VolumeBackupsController extends Controller$errors->add('disable_local_backup', 'Local backups can only be disabled when S3 backups are enabled.');}- $s3Storage = null;- if ($saveToS3) {- $s3Storage = S3Storage::query()+ $s3Storages = collect();+ if ($saveToS3 && ! $errors->hasAny(['s3_storage_uuid', 's3_storage_uuids', 's3_storage_uuids.*'])) {+ $hasList = $request->has('s3_storage_uuids');+ $uuids = collect($hasList ? $request->input('s3_storage_uuids') : [$request->input('s3_storage_uuid')])->filter()->unique()->values();+ $primaryUuid = $request->input('s3_storage_uuid') ?? $uuids->first();++ $s3Storages = S3Storage::query()->where('team_id', $teamId)->where('is_usable', true)- ->where('uuid', $request->input('s3_storage_uuid'))- ->first();-- if (! $s3Storage) {- $errors->add('s3_storage_uuid', 'Select a usable S3 storage owned by your team.');+ ->whereIn('uuid', $uuids->all())+ ->get()+ ->sortBy(fn (S3Storage $s3Storage): int => $s3Storage->uuid === $primaryUuid ? 0 : 1)+ ->values();++ if ($hasList && filled($primaryUuid) && ! $uuids->contains($primaryUuid)) {+ $errors->add('s3_storage_uuid', 'The s3_storage_uuid must be one of the s3_storage_uuids.');+ } elseif ($uuids->isEmpty() || $s3Storages->count() !== $uuids->count()) {+ $errors->add($hasList ? 's3_storage_uuids' : 's3_storage_uuid', 'Select a usable S3 storage owned by your team.');}}@@ -251,7 +265,7 @@ class VolumeBackupsController extends Controllerreturn ['errors' => $errors,- 's3Storage' => $s3Storage,+ 's3Storages' => $s3Storages,'saveToS3' => $saveToS3,];}@@ -260,7 +274,7 @@ class VolumeBackupsController extends ControllerRequest $request,LocalPersistentVolume|LocalFileVolume $storage,int|string $teamId,- ?S3Storage $s3Storage,+ Collection $s3Storages,bool $saveToS3,string $resourceType,Model $resource,@@ -272,7 +286,7 @@ class VolumeBackupsController extends Controller'save_s3' => $saveToS3,'disable_local_backup' => $saveToS3 && $request->boolean('disable_local_backup'),'stop_during_backup' => $request->boolean('stop_during_backup'),- 's3_storage_id' => $s3Storage?->id,+ 's3_storage_id' => $s3Storages->first()?->id,'retention_amount_locally' => $request->integer('retention_amount_locally', 7),'retention_days_locally' => $request->integer('retention_days_locally'),'retention_max_storage_locally' => $request->float('retention_max_storage_locally'),@@ -288,6 +302,7 @@ class VolumeBackupsController extends Controller}$backup = $storage->scheduledBackups()->updateOrCreate([], $attributes);+ $backup->syncS3Storages($s3Storages->pluck('id')->all());$created = $backup->wasRecentlyCreated;auditLog('api.volume_backup.schedule_set', [@@ -298,7 +313,7 @@ class VolumeBackupsController extends Controller'backup_uuid' => $backup->uuid,]);- return response()->json($this->responseData($backup, $storage, $s3Storage, $created), $created ? 201 : 200);+ return response()->json($this->responseData($backup, $storage, $s3Storages, $created), $created ? 201 : 200);}#[OA\Delete(@@ -431,7 +446,7 @@ class VolumeBackupsController extends Controllerprivate function responseData(ScheduledVolumeBackup $backup,LocalPersistentVolume|LocalFileVolume $storage,- ?S3Storage $s3Storage,+ Collection $s3Storages,bool $created,): array {return [@@ -444,7 +459,8 @@ class VolumeBackupsController extends Controller'save_s3' => $backup->save_s3,'disable_local_backup' => $backup->disable_local_backup,'stop_during_backup' => $backup->stop_during_backup,- 's3_storage_uuid' => $s3Storage?->uuid,+ 's3_storage_uuid' => $s3Storages->first()?->uuid,+ 's3_storage_uuids' => $s3Storages->pluck('uuid')->all(),'retention_amount_locally' => $backup->retention_amount_locally,'retention_days_locally' => $backup->retention_days_locally,'retention_max_storage_locally' => $backup->retention_max_storage_locally,diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.phpindex 92d24bab3d..eb0959865b 100644--- a/app/Jobs/DatabaseBackupJob.php+++ b/app/Jobs/DatabaseBackupJob.php@@ -83,8 +83,6 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueuepublic ?string $mongo_root_password = null;- public ?S3Storage $s3 = null;-public $timeout = 3600 + self::WORKER_TIMEOUT_MARGIN_SECONDS;public ?string $backup_log_uuid = null;@@ -129,11 +127,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueueif (data_get($this->backup, 'database_type') === ServiceDatabase::class) {$this->database = data_get($this->backup, 'database');$this->server = $this->database->service->server;- $this->s3 = $this->backup->s3;} else {$this->database = data_get($this->backup, 'database');$this->server = $this->database->destination->server;- $this->s3 = $this->backup->s3;}if (is_null($this->server)) {throw new \Exception('Server not found?!');@@ -329,6 +325,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue$this->backup_location = null;$this->backup_output = null;$this->error_output = null;+ $this->s3_uploaded = false;// Generate unique UUID for each database backup execution$attempts = 0;@@ -484,10 +481,10 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue$localStorageDeleted = false;if ($this->backup->save_s3 && $localBackupSucceeded) {try {- $this->upload_to_s3();+ $s3UploadError = $this->uploadToS3Destinations();- // If local backup is disabled, delete the local file immediately after S3 upload- if ($this->backup->disable_local_backup) {+ // If local backup is disabled, delete the local file only when every destination has a copy+ if ($s3UploadError === null && $this->backup->disable_local_backup) {deleteBackupsLocally($this->backup_location, $this->server);$localStorageDeleted = true;}@@ -843,9 +840,15 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueuereturn devHostDockerPath($this->server, $this->backup_location);}- private function upload_to_s3(): void+ /**+ * Uploads the backup file to every selected S3 destination and records one replica per destination.+ *+ * @return string|null The failed destinations and their errors, or null when every upload succeeded.+ */+ private function uploadToS3Destinations(): ?string{- if (is_null($this->s3)) {+ $storages = $this->backup->selectedS3Storages();+ if ($storages->isEmpty()) {$previousS3StorageId = $this->backup->s3_storage_id;$this->backup->update([@@ -856,13 +859,43 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueuethrow new \Exception('S3 storage configuration is missing or has been deleted (S3 storage ID: '.($previousS3StorageId ?? 'null').'). S3 backup has been disabled for this schedule.');}+ $failures = [];+ foreach ($storages as $storage) {+ $uploaded = false;+ $uploadError = null;+ try {+ $this->upload_to_s3($storage);+ $uploaded = true;+ } catch (Throwable $e) {+ $uploadError = $e->getMessage();+ $failures[] = "{$storage->name}: {$uploadError}";+ }++ try {+ $this->backup_log->s3Replicas()->create([+ 's3_storage_id' => $storage->id,+ 's3_uploaded' => $uploaded,+ 'message' => $uploadError,+ ]);+ } catch (Throwable $e) {+ $failures[] = "{$storage->name}: {$e->getMessage()}";+ }+ }++ $this->backup_log->refreshS3Summary();+ $this->s3_uploaded = $failures === [];++ return $failures === [] ? null : implode("\n", $failures);+ }++ protected function upload_to_s3(S3Storage $s3): void+ {try {- $key = $this->s3->key;- $secret = $this->s3->secret;- // $region = $this->s3->region;- $bucket = $this->s3->bucket;- $endpoint = $this->s3->endpoint;- $this->s3->testConnection(shouldSave: true);+ $key = $s3->key;+ $secret = $s3->secret;+ $bucket = $s3->bucket;+ $endpoint = $s3->endpoint;+ $s3->testConnection(shouldSave: true);if (data_get($this->backup, 'database_type') === ServiceDatabase::class) {$network = $this->database->service->destination->network;} else {@@ -883,7 +916,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue$escapedSecret = escapeshellarg($secret);$escapedBackupLocation = escapeshellarg($this->backup_location);$escapedS3Destination = escapeshellarg("temporary/{$bucket}{$this->backup_dir}/");- $resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($endpoint, $this->s3->trustedInternalHosts()))+ $resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($endpoint, $s3->trustedInternalHosts()))->map(fn (string $resolveOption): string => '--resolve '.escapeshellarg($resolveOption))->implode(' ');$resolveOptions = $resolveOptions === '' ? '' : ' '.$resolveOptions;@@ -892,9 +925,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue$commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc cp {$escapedBackupLocation} {$escapedS3Destination}";instant_remote_process($commands, $this->server, true, false, $this->commandTimeout(), disableMultiplexing: true);- $this->s3_uploaded = true;} catch (Throwable $e) {- $this->s3_uploaded = false;$this->add_to_error_output($e->getMessage());throw $e;} finally {diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.phpindex d635894b09..db226e5675 100644--- a/app/Jobs/VolumeBackupJob.php+++ b/app/Jobs/VolumeBackupJob.php@@ -4,9 +4,11 @@ namespace App\Jobs;use App\Events\BackupCreated;use App\Models\LocalPersistentVolume;+use App\Models\S3Storage;use App\Models\ScheduledVolumeBackup;use App\Models\ScheduledVolumeBackupExecution;use App\Models\Server;+use App\Models\VolumeBackupS3Replica;use App\Notifications\VolumeBackup\BackupFailed;use App\Notifications\VolumeBackup\BackupSuccess;use App\Rules\SafeWebhookUrl;@@ -16,7 +18,6 @@ use Carbon\Carbon;use Illuminate\Bus\Queueable;use Illuminate\Contracts\Queue\ShouldBeEncrypted;use Illuminate\Contracts\Queue\ShouldQueue;-use Illuminate\Database\Eloquent\Builder;use Illuminate\Foundation\Bus\Dispatchable;use Illuminate\Queue\InteractsWithQueue;use Illuminate\Queue\SerializesModels;@@ -71,16 +72,21 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueuethrow new \RuntimeException('The storage backup resource, team, or server no longer exists.');}+ $destinations = $this->backup->save_s3 ? $this->backup->selectedS3Storages() : collect();$this->execution = $this->backup->executions()->create([- 's3_storage_id' => $this->backup->save_s3 ? $this->backup->s3_storage_id : null,+ 's3_storage_id' => $destinations->first()?->id,]);+ foreach ($destinations as $s3) {+ $this->execution->s3Replicas()->create(['s3_storage_id' => $s3->id]);+ }BackupCreated::dispatch($team->id);$backupDirectory = backup_dir().'/volumes/'.str($team->name)->slug().'-'.$team->id.'/'.$target->uuid;$filename = str($this->backup->targetType())->lower().'-'.str($this->backup->targetName())->slug().'-'.Carbon::now()->timestamp.'.tar.gz';$backupLocation = $backupDirectory.'/'.$filename;$this->execution->update(['filename' => $backupLocation]);- $streamToS3 = $this->backup->save_s3 && $this->backup->disable_local_backup;+ // Streaming writes no local archive, so it can upload to one destination only.+ $streamToS3 = $this->backup->disable_local_backup && $destinations->count() === 1;try {$source = $this->backup->sourcePath();@@ -96,7 +102,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue$archiveScript = "compressor=\$({$compressorCommand}); tar -I \"\$compressor\" -cf - -C /volume .";if ($streamToS3) {$this->execution->update(['local_storage_deleted' => true]);- $archiveCommand = $this->streamToS3Command($archiveScript, $backupLocation, $source, $containerName, $image);+ $archiveCommand = $this->streamToS3Command($destinations->first(), $archiveScript, $backupLocation, $source, $containerName, $image);$this->execution->update(['s3_cleanup_pending' => true]);} else {$archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName)@@ -141,24 +147,27 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue}$warning = null;- $s3Uploaded = null;+ $s3Uploaded = false;$s3CleanupPending = false;$localStorageDeleted = $streamToS3;if ($streamToS3) {$s3Uploaded = true;+ $this->execution->s3Replicas()->update(['s3_uploaded' => true]);+ $this->execution->refreshS3Summary();$this->execution->update(['s3_cleanup_pending' => false]);+ } elseif ($this->backup->save_s3 && $destinations->isEmpty()) {+ $warning = 'S3 upload failed: The selected S3 storage no longer exists.';} elseif ($this->backup->save_s3) {$s3CleanupPending = true;$this->execution->update(['s3_cleanup_pending' => true]);+ $uploadErrors = $this->uploadToDestinations($backupLocation, $backupDirectory, $server);+ $s3Uploaded = $uploadErrors === [];- try {- $this->uploadToS3($backupLocation, $backupDirectory, $server);- $s3Uploaded = true;+ if ($s3Uploaded) {$s3CleanupPending = false;- } catch (Throwable $exception) {- $s3Uploaded = false;- $warning = 'S3 upload failed: '.$exception->getMessage();+ } else {+ $warning = 'S3 upload failed: '.implode(' ', $uploadErrors);try {VolumeBackupRecoveryJob::cleanupS3Upload($this->execution);@@ -184,7 +193,6 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue'message' => $warning,'size' => $size,'filename' => $backupLocation,- 's3_uploaded' => $s3Uploaded,'s3_cleanup_pending' => $s3CleanupPending,'local_storage_deleted' => $localStorageDeleted,]);@@ -220,6 +228,8 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue}}+ $this->execution->s3Replicas()->whereNull('s3_uploaded')->update(['s3_uploaded' => false]);+ $this->execution->refreshS3Summary();$s3CleanupPending = $this->execution->fresh()->s3_cleanup_pending;$this->execution->update([@@ -347,16 +357,37 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue}}- private function uploadToS3(string $backupLocation, string $backupDirectory, Server $server): void+ /**+ * Uploads the local archive to every destination of the execution and records the result on each copy.+ *+ * @return array<int, string> One error message per failed destination.+ */+ private function uploadToDestinations(string $backupLocation, string $backupDirectory, Server $server): array{- $s3 = $this->backup->s3;+ $errors = [];- if (! $s3) {- $this->backup->update(['save_s3' => false, 's3_storage_id' => null]);+ foreach ($this->execution->s3Replicas()->with('s3')->get() as $replica) {+ /** @var VolumeBackupS3Replica $replica */+ try {+ if (! $replica->s3) {+ throw new \RuntimeException('The selected S3 storage no longer exists.');+ }- throw new \RuntimeException('The selected S3 storage no longer exists. S3 backup has been disabled.');+ $this->uploadToS3($replica->s3, $backupLocation, $backupDirectory, $server);+ $replica->update(['s3_uploaded' => true, 'message' => null]);+ } catch (Throwable $exception) {+ $replica->update(['s3_uploaded' => false, 'message' => $exception->getMessage()]);+ $errors[] = ($replica->s3?->name ?? 'Deleted storage').': '.$exception->getMessage();+ }}+ $this->execution->refreshS3Summary();++ return $errors;+ }++ private function uploadToS3(S3Storage $s3, string $backupLocation, string $backupDirectory, Server $server): void+ {$s3->testConnection(shouldSave: true);$containerName = 'volume-upload-'.$this->execution->uuid;$image = coolifyHelperImage().':'.getHelperVersion();@@ -385,16 +416,8 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue}}- private function streamToS3Command(string $archiveScript, string $backupLocation, string $source, string $containerName, string $image): string+ private function streamToS3Command(S3Storage $s3, string $archiveScript, string $backupLocation, string $source, string $containerName, string $image): string{- $s3 = $this->backup->s3;-- if (! $s3) {- $this->backup->update(['save_s3' => false, 's3_storage_id' => null]);-- throw new \RuntimeException('The selected S3 storage no longer exists. S3 backup has been disabled.');- }-$s3->testConnection(shouldSave: true);$resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($s3->endpoint, $s3->trustedInternalHosts()))->map(fn (string $option): string => '--resolve '.escapeshellarg($option))@@ -462,36 +485,45 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue}}- if ($this->backup->save_s3 && $this->backup->s3 && $this->hasRetentionLimits(+ if ($this->backup->save_s3 && $this->hasRetentionLimits($this->backup->retention_amount_s3,$this->backup->retention_days_s3,$this->backup->retention_max_storage_s3,)) {+ $liveCopies = fn ($query) => $query->where('s3_uploaded', true)->where('s3_storage_deleted', false);$s3Executions = $this->backup->executions()- ->with('s3')+ ->with(['s3Replicas' => fn ($query) => $liveCopies($query)->with('s3')])->where('status', 'success')- ->where('s3_uploaded', true)- ->where('s3_storage_deleted', false)+ ->whereHas('s3Replicas', $liveCopies)->get();- $s3Executions = $this->executionsOutsideRetention(- $s3Executions,- $this->backup->retention_amount_s3,- $this->backup->retention_days_s3,- $this->backup->retention_max_storage_s3,- );- foreach ($s3Executions->groupBy('s3_storage_id') as $executions) {- $s3 = $executions->first()->s3;+ // Each destination keeps its own copies within the same retention settings.+ foreach ($s3Executions->flatMap->s3Replicas->pluck('s3_storage_id')->unique() as $storageId) {+ $executions = $this->executionsOutsideRetention(+ $s3Executions->filter(fn (ScheduledVolumeBackupExecution $execution): bool => $execution->s3Replicas->contains('s3_storage_id', $storageId))->values(),+ $this->backup->retention_amount_s3,+ $this->backup->retention_days_s3,+ $this->backup->retention_max_storage_s3,+ );+ if ($executions->isEmpty()) {+ continue;+ }++ $replicas = $executions->map(fn (ScheduledVolumeBackupExecution $execution): VolumeBackupS3Replica => $execution->s3Replicas->firstWhere('s3_storage_id', $storageId));+ $s3 = $replicas->first()->s3;if (! $s3) {- throw new \RuntimeException('The S3 storage used by an existing backup is unavailable.');+ VolumeBackupS3Replica::query()->whereKey($replicas->pluck('id')->all())->update(['s3_storage_deleted' => true]);+ $executions->each(fn (ScheduledVolumeBackupExecution $execution) => $execution->refreshS3Summary());++ continue;}$filenames = $executions->pluck('filename')->filter()->all();if ($filenames !== []) {deleteBackupsS3($filenames, $s3);- $this->backup->executions()->whereKey($executions->pluck('id')->all())- ->update(['s3_storage_deleted' => true]);}+ VolumeBackupS3Replica::query()->whereKey($replicas->pluck('id')->all())->update(['s3_storage_deleted' => true]);+ $executions->each(fn (ScheduledVolumeBackupExecution $execution) => $execution->refreshS3Summary());}}@@ -499,9 +531,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue->where('local_storage_deleted', true)->where('stop_recovery_pending', false)->where('s3_cleanup_pending', false)- ->where(function (Builder $query): void {- $query->where('s3_storage_deleted', true)->orWhereNull('s3_uploaded');- })+ ->withoutLiveS3Copies()->delete();}diff --git a/app/Jobs/VolumeBackupRecoveryJob.php b/app/Jobs/VolumeBackupRecoveryJob.phpindex cda0fdd046..4965df1379 100644--- a/app/Jobs/VolumeBackupRecoveryJob.php+++ b/app/Jobs/VolumeBackupRecoveryJob.php@@ -295,12 +295,19 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, Shou]);}+ /**+ * Deletes the partial upload from every destination whose copy did not finish uploading. Every destination is+ * tried; the first failure is rethrown afterwards and keeps the cleanup pending.+ */public static function cleanupS3Upload(ScheduledVolumeBackupExecution $execution): void{- $execution->loadMissing('s3');- $s3 = $execution->s3;+ $replicas = $execution->s3Replicas()+ ->with('s3')+ ->where('s3_storage_deleted', false)+ ->where(fn ($query) => $query->whereNull('s3_uploaded')->orWhere('s3_uploaded', false))+ ->get();- if (! $s3 || blank($execution->filename)) {+ if ($replicas->isEmpty() || blank($execution->filename)) {self::skipRecovery($execution,['s3_cleanup_pending' => false],@@ -310,11 +317,25 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, Shoureturn;}- deleteBackupsS3($execution->filename, $s3);- $execution->update([- 's3_cleanup_pending' => false,- 's3_storage_deleted' => true,- ]);+ $failure = null;+ foreach ($replicas as $replica) {+ try {+ if ($replica->s3) {+ deleteBackupsS3($execution->filename, $replica->s3);+ }+ $replica->update(['s3_storage_deleted' => true]);+ } catch (Throwable $exception) {+ $failure ??= $exception;+ }+ }++ $execution->refreshS3Summary();++ if ($failure) {+ throw $failure;+ }++ $execution->update(['s3_cleanup_pending' => false]);}/**diff --git a/app/Livewire/Project/Application/Backup/Index.php b/app/Livewire/Project/Application/Backup/Index.phpindex fe7e2eac5d..7ca097db96 100644--- a/app/Livewire/Project/Application/Backup/Index.php+++ b/app/Livewire/Project/Application/Backup/Index.php@@ -31,7 +31,7 @@ class Index extends Componentpublic function render(): View{$backups = ScheduledVolumeBackup::query()- ->with(['backupable', 'latestExecution', 's3'])+ ->with(['backupable', 'latestExecution', 's3', 's3Storages'])->withCount('executions')->forApplication($this->application)->latest()diff --git a/app/Livewire/Project/CloneMe.php b/app/Livewire/Project/CloneMe.phpindex 8f2ff9af40..b310c248ee 100644--- a/app/Livewire/Project/CloneMe.php+++ b/app/Livewire/Project/CloneMe.php@@ -279,6 +279,7 @@ class CloneMe extends Component'team_id' => $teamId,]);$newBackup->save();+ $backup->copyS3StoragesTo($newBackup);}$environmentVaribles = $database->environment_variables()->get();@@ -469,6 +470,7 @@ class CloneMe extends Component'team_id' => $teamId,]);$newBackup->save();+ $backup->copyS3StoragesTo($newBackup);}}diff --git a/app/Livewire/Project/Database/BackupEdit.php b/app/Livewire/Project/Database/BackupEdit.phpindex 59dd1a29c0..a7c49eeb58 100644--- a/app/Livewire/Project/Database/BackupEdit.php+++ b/app/Livewire/Project/Database/BackupEdit.php@@ -9,6 +9,7 @@ use App\Models\ServiceDatabase;use App\Models\StandalonePostgresql;use App\Traits\ListensToTeamChannel;use Exception;+use Illuminate\Database\Eloquent\Builder;use Illuminate\Foundation\Auth\Access\AuthorizesRequests;use Illuminate\Support\Collection;use Illuminate\Support\Facades\Auth;@@ -76,8 +77,9 @@ class BackupEdit extends Component#[Validate(['required', 'boolean'])]public bool $disableLocalBackup = false;- #[Validate(['nullable', 'integer'])]- public ?int $s3StorageId = null;+ /** @var array<int, int|string> */+ #[Validate(['array'])]+ public array $s3StorageIds = [];#[Validate(['nullable', 'string'])]public ?string $databasesToBackup = null;@@ -113,6 +115,10 @@ class BackupEdit extends Component{try {$this->authorize('view', $this->backup->database);+ $this->availableS3Storages = S3Storage::query()+ ->whereKey($this->availableS3StorageIds()->all())+ ->orderBy('name')+ ->get();$this->parameters = get_route_parameters();$this->syncData();$this->refreshStatus();@@ -148,7 +154,6 @@ class BackupEdit extends Component$this->backup->database_backup_retention_max_storage_s3 = $this->databaseBackupRetentionMaxStorageS3;$this->backup->save_s3 = $this->saveS3;$this->backup->disable_local_backup = $this->disableLocalBackup;- $this->backup->s3_storage_id = $this->s3StorageId;// Validate databases_to_backup to prevent command injection// Handles all formats including MongoDB's "db:col1,col2|db2:col3"@@ -160,10 +165,16 @@ class BackupEdit extends Component$this->backup->dump_all = $this->dumpAll;$this->backup->timeout = $this->timeout;$this->backup->missing_backup_notification_days = $this->missingBackupNotificationDays;- $this->customValidate();+ $storageIds = $this->customValidate();$changedFields = auditChangedFields($this->backup);$this->backup->save();+ $previousStorageIds = $this->backup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all();+ $this->backup->syncS3Storages($storageIds->all());+ if ($previousStorageIds !== $storageIds->sort()->values()->all()) {+ $changedFields[] = 's3_storages';+ }+return $changedFields;} else {$this->backupEnabled = $this->backup->enabled;@@ -177,7 +188,8 @@ class BackupEdit extends Component$this->databaseBackupRetentionMaxStorageS3 = $this->backup->database_backup_retention_max_storage_s3;$this->saveS3 = $this->backup->save_s3;$this->disableLocalBackup = $this->backup->disable_local_backup ?? false;- $this->s3StorageId = $this->backup->s3_storage_id ?? $this->availableS3StorageIds()->first();+ $selectedIds = $this->backup->selectedS3Storages()->pluck('id');+ $this->s3StorageIds = ($selectedIds->isNotEmpty() ? $selectedIds : $this->availableS3StorageIds()->take(1))->all();$this->databasesToBackup = $this->backup->databases_to_backup;$this->dumpAll = $this->backup->dump_all;$this->timeout = $this->backup->timeout;@@ -216,10 +228,10 @@ class BackupEdit extends Componentif ($this->delete_associated_backups_locally && $server) {deleteBackupsLocally($filenames, $server);}+ }- if ($this->delete_associated_backups_s3 && $this->backup->s3) {- deleteBackupsS3($filenames, $this->backup->s3);- }+ if ($this->delete_associated_backups_s3) {+ $this->backup->executions()->each(fn ($execution) => $execution->deleteS3Copies());}$backupUuid = $this->backup->uuid;@@ -333,14 +345,14 @@ class BackupEdit extends Component}}- public function updatedS3StorageId(): void+ public function updatedS3StorageIds(): void{$this->instantSave();}public function toggleS3(): void{- if (! $this->saveS3 && $this->availableS3StorageIds()->isEmpty()) {+ if (! $this->saveS3 && $this->selectedAvailableS3StorageIds()->isEmpty()) {$this->dispatch('error', 'Select a usable S3 storage before enabling S3 backups.');return;@@ -351,21 +363,22 @@ class BackupEdit extends Component$this->instantSave();}- private function customValidate()+ /**+ * @return Collection<int, int> The usable S3 storage IDs to keep as destinations.+ */+ private function customValidate(): Collection{- if (! is_numeric($this->backup->s3_storage_id)) {- $this->backup->s3_storage_id = null;- }+ // Only usable S3 storages of the database's team can be destinations; others are dropped.+ $storageIds = $this->selectedAvailableS3StorageIds();+ $this->s3StorageIds = $storageIds->all();- // S3 backup cannot be enabled without a valid S3 storage owned by the team- $availableS3Ids = $this->availableS3StorageIds();- if ($availableS3Ids->isEmpty()) {- $this->backup->s3_storage_id = $this->s3StorageId = null;- if ($this->backup->save_s3) {- $this->backup->save_s3 = $this->saveS3 = false;- }- } elseif (! $availableS3Ids->contains($this->backup->s3_storage_id)) {- $this->backup->s3_storage_id = $this->s3StorageId = $availableS3Ids->first();+ if ($this->availableS3StorageIds()->isEmpty()) {+ $this->backup->save_s3 = $this->saveS3 = false;+ } elseif ($this->backup->save_s3 && $storageIds->isEmpty()) {+ $message = 'Select at least one usable S3 storage.';+ $this->addError('s3StorageIds', $message);++ throw new Exception($message);}// Validate that disable_local_backup can only be true when S3 backup is enabled@@ -378,6 +391,8 @@ class BackupEdit extends Componentthrow new Exception('Invalid Cron / Human expression');}$this->validate();++ return $storageIds;}/**@@ -399,6 +414,20 @@ class BackupEdit extends Component]);}+ /**+ * @return Collection<int, int>+ */+ private function selectedAvailableS3StorageIds(): Collection+ {+ $availableIds = $this->availableS3StorageIds();++ return collect($this->s3StorageIds)+ ->map(fn ($id): int => (int) $id)+ ->filter(fn (int $id): bool => $availableIds->contains($id))+ ->unique()+ ->values();+ }+private function availableS3StorageIds(): Collection{$database = $this->backup->database;@@ -409,9 +438,12 @@ class BackupEdit extends Componentreturn collect();}+ // Current destinations stay selectable while unusable, so saving other settings does not remove them.+ $selectedIds = $this->backup->selectedS3Storages()->pluck('id')->all();+return S3Storage::query()->where('team_id', $teamId)- ->where('is_usable', true)+ ->where(fn (Builder $query) => $query->where('is_usable', true)->orWhereIn('id', $selectedIds))->pluck('id');}@@ -432,7 +464,7 @@ class BackupEdit extends Componentreturn view('livewire.project.database.backup-edit', ['checkboxes' => [['id' => 'delete_associated_backups_locally', 'label' => __('database.delete_backups_locally')],- ['id' => 'delete_associated_backups_s3', 'label' => 'All backups will be permanently deleted (associated with this backup job) from the selected S3 Storage.'],+ ['id' => 'delete_associated_backups_s3', 'label' => 'All backups will be permanently deleted (associated with this backup job) from every S3 storage of this schedule.'],// ['id' => 'delete_associated_backups_sftp', 'label' => 'All backups associated with this backup job from this database will be permanently deleted from the selected SFTP Storage.']],]);diff --git a/app/Livewire/Project/Database/BackupExecutions.php b/app/Livewire/Project/Database/BackupExecutions.phpindex 6f17e2fcf5..f9d4658688 100644--- a/app/Livewire/Project/Database/BackupExecutions.php+++ b/app/Livewire/Project/Database/BackupExecutions.php@@ -109,12 +109,8 @@ class BackupExecutions extends ComponentdeleteBackupsLocally($execution->filename, $server, throwError: true);}- if ($deleteFromS3 && $execution->s3_uploaded && ! $execution->s3_storage_deleted) {- if (! $execution->scheduledDatabaseBackup->s3) {- throw new \RuntimeException('The S3 storage is unavailable.');- }-- deleteBackupsS3($execution->filename, $execution->scheduledDatabaseBackup->s3);+ if ($deleteFromS3) {+ $execution->deleteS3Copies();}$execution->delete();diff --git a/app/Livewire/Project/Database/ScheduledBackups.php b/app/Livewire/Project/Database/ScheduledBackups.phpindex ece99b45f4..2071337b4b 100644--- a/app/Livewire/Project/Database/ScheduledBackups.php+++ b/app/Livewire/Project/Database/ScheduledBackups.php@@ -73,7 +73,7 @@ class ScheduledBackups extends Componentpublic function render(): View{- $this->database->loadMissing('scheduledBackups.s3');+ $this->database->loadMissing(['scheduledBackups.s3', 'scheduledBackups.s3Storages']);return view('livewire.project.database.scheduled-backups');}diff --git a/app/Livewire/Project/Service/BackupExecutions.php b/app/Livewire/Project/Service/BackupExecutions.phpindex 81be0ff8ad..b319f66412 100644--- a/app/Livewire/Project/Service/BackupExecutions.php+++ b/app/Livewire/Project/Service/BackupExecutions.php@@ -114,7 +114,7 @@ class BackupExecutions extends Componentprivate function formatExecutions(Collection $rows): Collection{$databaseExecutions = ScheduledDatabaseBackupExecution::query()- ->with(['scheduledDatabaseBackup.database', 'scheduledDatabaseBackup.s3'])+ ->with(['scheduledDatabaseBackup.database', 'scheduledDatabaseBackup.s3', 'scheduledDatabaseBackup.s3Storages'])->whereIn('id', $rows->where('type', 'database')->pluck('id'))->get()->keyBy('id');$volumeExecutions = ScheduledVolumeBackupExecution::query()@@ -128,11 +128,16 @@ class BackupExecutions extends Component$isDatabase = $row->type === 'database';$execution = $isDatabase ? $databaseExecutions->get($row->id) : $volumeExecutions->get($row->id);$schedule = $isDatabase ? $execution->scheduledDatabaseBackup : $execution->scheduledVolumeBackup;- $storage = $isDatabase ? ($schedule->save_s3 ? $schedule->s3 : null) : $execution->s3;- if ($storage?->team_id !== $serviceTeamId) {- $storage = null;- }- $storageLabel = $storage ? $storage->name.' (bucket: '.$storage->bucket.')' : 'Unavailable';+ $storages = match (true) {+ ! $isDatabase => collect([$execution->s3]),+ ! $schedule->save_s3 => collect(),+ $schedule->s3Storages->isNotEmpty() => $schedule->s3Storages,+ default => collect([$schedule->s3]),+ };+ $storageLabel = $storages+ ->filter(fn ($storage) => $storage?->team_id === $serviceTeamId)+ ->map(fn ($storage) => $storage->name.' (bucket: '.$storage->bucket.')')+ ->implode(', ') ?: 'Unavailable';if ($isDatabase && ! $schedule->save_s3) {$storageLabel = 'Not configured';} elseif (! $isDatabase && ! $execution->s3_storage_id && ! $execution->s3_uploaded && ! $execution->s3_storage_deleted) {diff --git a/app/Livewire/Project/Service/VolumeBackup/Index.php b/app/Livewire/Project/Service/VolumeBackup/Index.phpindex 60a78e5c15..64dbbb8df2 100644--- a/app/Livewire/Project/Service/VolumeBackup/Index.php+++ b/app/Livewire/Project/Service/VolumeBackup/Index.php@@ -118,7 +118,7 @@ class Index extends Component);$databaseBackups = ScheduledDatabaseBackup::query()- ->with(['database', 'latest_log', 's3'])+ ->with(['database', 'latest_log', 's3', 's3Storages'])->withCount('executions')->where('database_type', (new ServiceDatabase)->getMorphClass())->whereHasMorph('database', [ServiceDatabase::class], fn ($query) => $query->where('service_id', $this->service->id))@@ -126,7 +126,7 @@ class Index extends Component->get();$backups = ScheduledVolumeBackup::query()- ->with(['backupable.resource', 'latestExecution', 's3'])+ ->with(['backupable.resource', 'latestExecution', 's3', 's3Storages'])->withCount('executions')->forService($this->service)->latest()diff --git a/app/Livewire/Project/Shared/ResourceOperations.php b/app/Livewire/Project/Shared/ResourceOperations.phpindex 2b6f5b4587..a459b988e0 100644--- a/app/Livewire/Project/Shared/ResourceOperations.php+++ b/app/Livewire/Project/Shared/ResourceOperations.php@@ -232,6 +232,7 @@ class ResourceOperations extends Component'team_id' => $this->resourceTeamId(),]);$newBackup->save();+ $backup->copyS3StoragesTo($newBackup);}$environmentVaribles = $this->resource->environment_variables()->get();diff --git a/app/Livewire/Project/Shared/Storages/VolumeBackups.php b/app/Livewire/Project/Shared/Storages/VolumeBackups.phpindex d741bcc0e4..083585fee6 100644--- a/app/Livewire/Project/Shared/Storages/VolumeBackups.php+++ b/app/Livewire/Project/Shared/Storages/VolumeBackups.php@@ -11,6 +11,7 @@ use App\Models\LocalPersistentVolume;use App\Models\S3Storage;use App\Models\ScheduledVolumeBackup;use App\Models\Service;+use Illuminate\Database\Eloquent\Builder;use Illuminate\Foundation\Auth\Access\AuthorizesRequests;use Illuminate\Http\RedirectResponse;use Illuminate\Routing\Redirector;@@ -43,7 +44,8 @@ class VolumeBackups extends Componentpublic bool $stopDuringBackup = false;- public ?int $s3StorageId = null;+ /** @var array<int, int|string> */+ public array $s3StorageIds = [];public int $retentionAmountLocally = 7;@@ -88,7 +90,7 @@ class VolumeBackups extends Component'saveToS3' => ['required', 'boolean'],'disableLocalBackup' => ['required', 'boolean'],'stopDuringBackup' => ['required', 'boolean'],- 's3StorageId' => ['nullable', 'integer'],+ 's3StorageIds' => ['array'],'retentionAmountLocally' => ['required', 'integer', 'min:0', 'max:10000'],'retentionDaysLocally' => ['required', 'integer', 'min:0'],'retentionMaxStorageLocally' => ['required', 'numeric', 'min:0'],@@ -103,12 +105,8 @@ class VolumeBackups extends Componentpublic function mount(): void{$this->authorize('view', $this->resource);- $this->availableS3Storages = S3Storage::query()- ->where('team_id', $this->resourceTeamId())- ->where('is_usable', true)- ->orderBy('name')- ->get();$this->backup = $this->storage->scheduledBackups()->first();+ $this->availableS3Storages = $this->selectableS3Storages()->orderBy('name')->get();$server = $this->backup?->server() ?? data_get($this->resource, 'destination.server');$this->timezone = data_get($server, 'settings.server_timezone', 'Instance timezone');@@ -118,7 +116,8 @@ class VolumeBackups extends Component$this->saveToS3 = $this->backup->save_s3;$this->disableLocalBackup = $this->backup->disable_local_backup;$this->stopDuringBackup = $this->backup->stop_during_backup;- $this->s3StorageId = $this->backup->s3_storage_id ?? $this->availableS3Storages->first()?->id;+ $selectedIds = $this->backup->selectedS3Storages()->pluck('id');+ $this->s3StorageIds = ($selectedIds->isNotEmpty() ? $selectedIds : $this->availableS3Storages->take(1)->pluck('id'))->all();$this->retentionAmountLocally = $this->backup->retention_amount_locally;$this->retentionDaysLocally = $this->backup->retention_days_locally;$this->retentionMaxStorageLocally = $this->backup->retention_max_storage_locally;@@ -128,7 +127,7 @@ class VolumeBackups extends Component$this->timeout = $this->backup->timeout;$this->missingBackupNotificationDays = $this->backup->missing_backup_notification_days;} else {- $this->s3StorageId = $this->availableS3Storages->first()?->id;+ $this->s3StorageIds = $this->availableS3Storages->take(1)->pluck('id')->all();}}@@ -149,30 +148,24 @@ class VolumeBackups extends Component$this->save();}- public function updatedS3StorageId(): void+ public function updatedS3StorageIds(): void{$this->authorize('update', $this->resource);- if (! $this->hasValidS3Storage()) {- $this->addError('s3StorageId', 'Select a usable S3 storage owned by your team.');-- return;- }-- $this->resetErrorBag('s3StorageId');if (! $this->validateSettings()) {return;}+ $this->resetErrorBag('s3StorageIds');$this->backup = $this->persistBackup($this->enabled);- $this->dispatch('success', 'S3 storage updated.');+ $this->dispatch('success', 'S3 storages updated.');}public function toggleS3(): void{$this->authorize('update', $this->resource);- if (! $this->saveToS3 && ! $this->hasValidS3Storage()) {+ if (! $this->saveToS3 && $this->selectedS3StorageIds()->isEmpty()) {$this->dispatch('error', 'Select a usable S3 storage before enabling S3 backups.');return;@@ -302,6 +295,7 @@ class VolumeBackups extends Component->where('s3_storage_deleted', true)->orWhereNull('s3_uploaded')->orWhere('s3_uploaded', false))+ ->withoutLiveS3Copies()->delete() ?? 0;$this->dispatch(@@ -366,12 +360,8 @@ class VolumeBackups extends ComponentdeleteBackupsLocally($execution->filename, $server, throwError: true);}- if ($this->delete_backup_s3 && $execution->s3_uploaded && ! $execution->s3_storage_deleted) {- if (! $execution->s3) {- throw new \RuntimeException('The S3 storage is unavailable.');- }-- deleteBackupsS3($execution->filename, $execution->s3);+ if ($this->delete_backup_s3) {+ $execution->deleteS3Copies();}$execution->delete();@@ -388,7 +378,7 @@ class VolumeBackups extends Componentpublic function render(){- $executions = $this->backup?->executions()->paginate($this->perPage);+ $executions = $this->backup?->executions()->with('s3Replicas.s3')->paginate($this->perPage);return view('livewire.project.shared.storages.volume-backups', ['executions' => $executions ?? collect(),@@ -410,8 +400,8 @@ class VolumeBackups extends Componentreturn false;}- if ($this->saveToS3 && ! $this->hasValidS3Storage()) {- $this->addError('s3StorageId', 'Select a usable S3 storage owned by your team.');+ if ($this->saveToS3 && $this->selectedS3StorageIds()->isEmpty()) {+ $this->addError('s3StorageIds', 'Select at least one usable S3 storage owned by your team.');return false;}@@ -431,7 +421,6 @@ class VolumeBackups extends Component'save_s3' => $this->saveToS3,'disable_local_backup' => $this->disableLocalBackup,'stop_during_backup' => $this->stopDuringBackup,- 's3_storage_id' => $this->hasValidS3Storage() ? $this->s3StorageId : $this->backup?->s3_storage_id,'retention_amount_locally' => $this->retentionAmountLocally,'retention_days_locally' => $this->retentionDaysLocally,'retention_max_storage_locally' => $this->retentionMaxStorageLocally,@@ -443,6 +432,15 @@ class VolumeBackups extends Component]);$changedFields = auditChangedFields($backup);$backup->save();++ $storageIds = $this->selectedS3StorageIds();+ $previousStorageIds = $backup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all();+ $backup->syncS3Storages($storageIds->all());+ $this->s3StorageIds = $storageIds->all();+ if (! $backup->wasRecentlyCreated && $previousStorageIds !== $storageIds->sort()->values()->all()) {+ $changedFields[] = 's3_storages';+ }+$this->auditScheduleSet($backup, $changedFields);return $backup;@@ -485,14 +483,33 @@ class VolumeBackups extends Component];}- private function hasValidS3Storage(): bool+ /**+ * The submitted destinations that are selectable S3 storages of the resource's team. Others are dropped.+ *+ * @return Collection<int, int>+ */+ private function selectedS3StorageIds(): Collection+ {+ $submittedIds = collect($this->s3StorageIds)->map(fn ($id): int => (int) $id)->filter()->unique();++ return $this->selectableS3Storages()+ ->whereKey($submittedIds->all())+ ->pluck('id')+ ->sortBy(fn (int $id): int|false => $submittedIds->search($id))+ ->values();+ }++ /**+ * Usable storages of the resource's team, plus the current destinations. A destination that is unusable for a+ * while stays selected, so saving other settings does not remove it.+ */+ private function selectableS3Storages(): Builder{- return $this->s3StorageId !== null- && S3Storage::query()- ->whereKey($this->s3StorageId)- ->where('team_id', $this->resourceTeamId())- ->where('is_usable', true)- ->exists();+ $selectedIds = $this->backup?->selectedS3Storages()->pluck('id')->all() ?? [];++ return S3Storage::query()+ ->where('team_id', $this->resourceTeamId())+ ->where(fn (Builder $query) => $query->where('is_usable', true)->orWhereIn('id', $selectedIds));}private function resourceTeamId(): intdiff --git a/app/Livewire/Storage/Resources.php b/app/Livewire/Storage/Resources.phpindex 95e7917515..306a6a4a5f 100644--- a/app/Livewire/Storage/Resources.php+++ b/app/Livewire/Storage/Resources.php@@ -9,6 +9,7 @@ use App\Models\ScheduledVolumeBackup;use App\Models\Service;use App\Models\ServiceApplication;use App\Models\ServiceDatabase;+use Illuminate\Database\Eloquent\Builder;use Illuminate\Foundation\Auth\Access\AuthorizesRequests;use Livewire\Component;@@ -26,7 +27,7 @@ class Resources extends Component{$this->authorize('view', $this->storage);- $backups = ScheduledDatabaseBackup::where('s3_storage_id', $this->storage->id)+ $backups = $this->backupsUsingStorage(ScheduledDatabaseBackup::query())->where('save_s3', true)->get();@@ -34,8 +35,7 @@ class Resources extends Component$this->selectedStorages[$backup->id] = $this->storage->id;}- ScheduledVolumeBackup::query()- ->where('s3_storage_id', $this->storage->id)+ $this->backupsUsingStorage(ScheduledVolumeBackup::query())->where('save_s3', true)->each(function (ScheduledVolumeBackup $backup): void {$this->selectedVolumeStorages[$backup->id] = $this->storage->id;@@ -46,29 +46,26 @@ class Resources extends Component{$this->authorize('update', $this->storage);- $backup = ScheduledDatabaseBackup::where('id', $backupId)- ->where('s3_storage_id', $this->storage->id)+ $backup = $this->backupsUsingStorage(ScheduledDatabaseBackup::query())+ ->whereKey($backupId)->firstOrFail();- $backup->fill([- 'save_s3' => false,- 's3_storage_id' => null,- ]);- $changedFields = auditChangedFields($backup);- $backup->save();+ $changedFields = $this->replaceStorage($backup, null);$this->auditDatabaseBackupUpdated($backup, $changedFields);unset($this->selectedStorages[$backupId]);- $this->dispatch('success', 'S3 disabled.', 'S3 backup has been disabled for this schedule.');+ $this->dispatch('success', 'Storage removed.', $backup->save_s3+ ? 'This storage was removed from the schedule destinations.'+ : 'S3 backup has been disabled for this schedule.');}public function moveBackup(int $backupId): void{$this->authorize('update', $this->storage);- $backup = ScheduledDatabaseBackup::where('id', $backupId)- ->where('s3_storage_id', $this->storage->id)+ $backup = $this->backupsUsingStorage(ScheduledDatabaseBackup::query())+ ->whereKey($backupId)->firstOrFail();$newStorageId = $this->selectedStorages[$backupId] ?? null;@@ -90,9 +87,7 @@ class Resources extends Component$this->authorize('update', $newStorage);- $backup->s3_storage_id = $newStorage->id;- $changedFields = auditChangedFields($backup);- $backup->save();+ $changedFields = $this->replaceStorage($backup, $newStorage->id);$this->auditDatabaseBackupUpdated($backup, $changedFields);unset($this->selectedStorages[$backupId]);@@ -104,31 +99,26 @@ class Resources extends Component{$this->authorize('update', $this->storage);- $backup = ScheduledVolumeBackup::query()- ->where('id', $backupId)- ->where('s3_storage_id', $this->storage->id)+ $backup = $this->backupsUsingStorage(ScheduledVolumeBackup::query())+ ->whereKey($backupId)->firstOrFail();- $backup->fill([- 'save_s3' => false,- 's3_storage_id' => null,- ]);- $changedFields = auditChangedFields($backup);- $backup->save();+ $changedFields = $this->replaceStorage($backup, null);$this->auditVolumeBackupUpdated($backup, $changedFields);unset($this->selectedVolumeStorages[$backupId]);- $this->dispatch('success', 'S3 disabled.', 'S3 backup has been disabled for this schedule.');+ $this->dispatch('success', 'Storage removed.', $backup->save_s3+ ? 'This storage was removed from the schedule destinations.'+ : 'S3 backup has been disabled for this schedule.');}public function moveVolumeBackup(int $backupId): void{$this->authorize('update', $this->storage);- $backup = ScheduledVolumeBackup::query()- ->where('id', $backupId)- ->where('s3_storage_id', $this->storage->id)+ $backup = $this->backupsUsingStorage(ScheduledVolumeBackup::query())+ ->whereKey($backupId)->firstOrFail();$newStorageId = $this->selectedVolumeStorages[$backupId] ?? null;@@ -151,9 +141,7 @@ class Resources extends Component$this->authorize('update', $newStorage);- $backup->s3_storage_id = $newStorage->id;- $changedFields = auditChangedFields($backup);- $backup->save();+ $changedFields = $this->replaceStorage($backup, $newStorage->id);$this->auditVolumeBackupUpdated($backup, $changedFields);unset($this->selectedVolumeStorages[$backupId]);@@ -161,6 +149,49 @@ class Resources extends Component$this->dispatch('success', 'Backup moved.', "Moved to {$newStorage->name}.");}+ /**+ * Schedules that have this storage among their destinations.+ *+ * @template TModel of ScheduledDatabaseBackup|ScheduledVolumeBackup+ *+ * @param Builder<TModel> $query+ * @return Builder<TModel>+ */+ private function backupsUsingStorage(Builder $query): Builder+ {+ return $query->usingS3Storage($this->storage->id);+ }++ /**+ * Replaces this storage in the destinations of a schedule, or removes it when no replacement is given. S3 backups+ * turn off when no destination remains.+ *+ * @return array<int, string> Names of the changed backup fields.+ */+ private function replaceStorage(ScheduledDatabaseBackup|ScheduledVolumeBackup $backup, ?int $newStorageId): array+ {+ $storageIds = $backup->selectedS3Storages()+ ->pluck('id')+ ->map(fn (int $id): ?int => $id === $this->storage->id ? $newStorageId : $id)+ ->filter()+ ->unique()+ ->values();+ $previousPrimaryId = $backup->s3_storage_id;++ $backup->syncS3Storages($storageIds->all());++ $changedFields = ['s3_storages'];+ if ($backup->s3_storage_id !== $previousPrimaryId) {+ $changedFields[] = 's3_storage_id';+ }+ if ($storageIds->isEmpty() && $backup->save_s3) {+ $backup->update(['save_s3' => false]);+ $changedFields[] = 'save_s3';+ }++ return $changedFields;+ }+/*** @param array<int, string> $changedFields*/@@ -211,7 +242,7 @@ class Resources extends Componentpublic function render(){- $backups = ScheduledDatabaseBackup::where('s3_storage_id', $this->storage->id)+ $backups = $this->backupsUsingStorage(ScheduledDatabaseBackup::query())->where('save_s3', true)->with('database')->get()@@ -221,8 +252,7 @@ class Resources extends Component->orderBy('name')->get(['id', 'name', 'is_usable']);- $volumeBackups = ScheduledVolumeBackup::query()- ->where('s3_storage_id', $this->storage->id)+ $volumeBackups = $this->backupsUsingStorage(ScheduledVolumeBackup::query())->where('save_s3', true)->with('backupable.resource')->get();diff --git a/app/Livewire/Storage/Show.php b/app/Livewire/Storage/Show.phpindex 17abd19e51..b7ddfcf5e2 100644--- a/app/Livewire/Storage/Show.php+++ b/app/Livewire/Storage/Show.php@@ -32,8 +32,10 @@ class Show extends Componentreturn $this->redirectRoute('storage.index', navigate: true);}$this->currentRoute = request()->route()->getName();- $this->backupCount = ScheduledDatabaseBackup::where('s3_storage_id', $this->storage->id)->count()- + ScheduledVolumeBackup::where('s3_storage_id', $this->storage->id)->count();+ $this->backupCount = collect([ScheduledDatabaseBackup::class, ScheduledVolumeBackup::class])+ ->sum(fn (string $backupModel): int => $backupModel::query()+ ->usingS3Storage($this->storage->id)+ ->count());}public function delete()diff --git a/app/Mcp/Tools/ListDatabaseBackups.php b/app/Mcp/Tools/ListDatabaseBackups.phpindex 7c65130935..92bee0da01 100644--- a/app/Mcp/Tools/ListDatabaseBackups.php+++ b/app/Mcp/Tools/ListDatabaseBackups.php@@ -43,6 +43,7 @@ class ListDatabaseBackups extends Tool$backups = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId)->where('database_id', $database->id)->where('database_type', $database->getMorphClass())+ ->with(['s3', 's3Storages'])->get()->map(fn ($backup) => $this->scrubSensitive(['uuid' => $backup->uuid,@@ -51,6 +52,7 @@ class ListDatabaseBackups extends Tool'database_backup_retention_amount_locally' => $backup->database_backup_retention_amount_locally ?? null,'save_s3' => $backup->save_s3 ?? null,'s3_storage_uuid' => $backup->s3?->uuid ?? null,+ 's3_storage_uuids' => $backup->selectedS3Storages()->pluck('uuid')->all(),'created_at' => $backup->created_at,'updated_at' => $backup->updated_at,]))diff --git a/app/Models/DatabaseBackupS3Replica.php b/app/Models/DatabaseBackupS3Replica.phpnew file mode 100644index 0000000000..1dee319c2b--- /dev/null+++ b/app/Models/DatabaseBackupS3Replica.php@@ -0,0 +1,38 @@+<?php++namespace App\Models;++use Illuminate\Database\Eloquent\Model;+use Illuminate\Database\Eloquent\Relations\BelongsTo;++/**+ * The copy of one database backup execution in one S3 destination.+ */+class DatabaseBackupS3Replica extends Model+{+ protected $fillable = [+ 'execution_id',+ 's3_storage_id',+ 's3_uploaded',+ 's3_storage_deleted',+ 'message',+ ];++ protected function casts(): array+ {+ return [+ 's3_uploaded' => 'boolean',+ 's3_storage_deleted' => 'boolean',+ ];+ }++ public function execution(): BelongsTo+ {+ return $this->belongsTo(ScheduledDatabaseBackupExecution::class, 'execution_id');+ }++ public function s3(): BelongsTo+ {+ return $this->belongsTo(S3Storage::class, 's3_storage_id');+ }+}diff --git a/app/Models/S3Storage.php b/app/Models/S3Storage.phpindex 3c0d9e7e95..3ee7d5f269 100644--- a/app/Models/S3Storage.php+++ b/app/Models/S3Storage.php@@ -6,10 +6,12 @@ use App\Rules\SafeWebhookUrl;use App\Rules\ValidS3BucketName;use App\Traits\Auditable;use App\Traits\HasSafeStringAttribute;+use Illuminate\Database\Eloquent\Builder;use Illuminate\Database\Eloquent\Casts\Attribute;use Illuminate\Database\Eloquent\Factories\HasFactory;use Illuminate\Filesystem\FilesystemAdapter;use Illuminate\Notifications\Messages\MailMessage;+use Illuminate\Support\Facades\DB;use Illuminate\Support\Facades\Storage;use Illuminate\Support\Facades\Validator;@@ -67,19 +69,48 @@ class S3Storage extends BaseModel});static::deleting(function (S3Storage $storage) {- ScheduledDatabaseBackup::where('s3_storage_id', $storage->id)->update([- 'save_s3' => false,- 's3_storage_id' => null,- ]);- ScheduledVolumeBackupExecution::where('s3_storage_id', $storage->id)- ->update([- 's3_storage_deleted' => true,- 's3_cleanup_pending' => false,- ]);- ScheduledVolumeBackup::where('s3_storage_id', $storage->id)->update([- 'save_s3' => false,- 's3_storage_id' => null,- ]);+ $storage->detachFromBackups();+ });+ }++ /**+ * Removes this storage from every backup schedule and marks the copies in it as deleted. A schedule moves its+ * primary destination to a remaining storage, or stops S3 backups when none remains.+ */+ public function detachFromBackups(): void+ {+ DB::transaction(function (): void {+ foreach ([ScheduledDatabaseBackup::class, ScheduledVolumeBackup::class] as $backupModel) {+ $backupModel::query()+ ->usingS3Storage($this->id)+ ->lazyById()+ ->each(function (ScheduledDatabaseBackup|ScheduledVolumeBackup $backup): void {+ $backup->syncS3Storages($backup->s3Storages()->whereKeyNot($this->id)->pluck('s3_storages.id')->all());+ if ($backup->s3_storage_id === null && $backup->save_s3) {+ $backup->update(['save_s3' => false]);+ }+ });+ }++ foreach ([DatabaseBackupS3Replica::class, VolumeBackupS3Replica::class] as $replicaModel) {+ $replicaModel::query()+ ->with('execution')+ ->where('s3_storage_id', $this->id)+ ->where('s3_storage_deleted', false)+ ->lazyById()+ ->each(function (DatabaseBackupS3Replica|VolumeBackupS3Replica $replica): void {+ $replica->update(['s3_storage_deleted' => true]);+ $replica->execution?->refreshS3Summary();+ });+ }++ ScheduledVolumeBackupExecution::query()+ ->where('s3_cleanup_pending', true)+ ->whereHas('s3Replicas', fn (Builder $query) => $query->where('s3_storage_id', $this->id))+ ->whereDoesntHave('s3Replicas', fn (Builder $query) => $query+ ->where('s3_storage_deleted', false)+ ->where(fn (Builder $query) => $query->whereNull('s3_uploaded')->orWhere('s3_uploaded', false)))+ ->update(['s3_cleanup_pending' => false]);});}diff --git a/app/Models/ScheduledDatabaseBackup.php b/app/Models/ScheduledDatabaseBackup.phpindex 53ae084349..3c31c0fbc7 100644--- a/app/Models/ScheduledDatabaseBackup.php+++ b/app/Models/ScheduledDatabaseBackup.php@@ -2,12 +2,16 @@namespace App\Models;+use App\Traits\HasS3Destinations;+use Illuminate\Database\Eloquent\Relations\BelongsToMany;use Illuminate\Database\Eloquent\Relations\HasMany;use Illuminate\Database\Eloquent\Relations\HasOne;use Illuminate\Database\Eloquent\Relations\MorphTo;class ScheduledDatabaseBackup extends BaseModel{+ use HasS3Destinations;+protected function casts(): array{return [@@ -78,6 +82,11 @@ class ScheduledDatabaseBackup extends BaseModelreturn $this->belongsTo(S3Storage::class, 's3_storage_id');}+ public function s3Storages(): BelongsToMany+ {+ return $this->belongsToMany(S3Storage::class, 'scheduled_database_backup_s3_storage')->withTimestamps();+ }+public function get_last_days_backup_status($days = 7){return $this->hasMany(ScheduledDatabaseBackupExecution::class)->where('created_at', '>=', now()->subDays($days))->get();@@ -85,7 +94,7 @@ class ScheduledDatabaseBackup extends BaseModelpublic function executionsPaginated(int $skip = 0, int $take = 10){- $executions = $this->hasMany(ScheduledDatabaseBackupExecution::class)->orderBy('created_at', 'desc');+ $executions = $this->hasMany(ScheduledDatabaseBackupExecution::class)->with('s3Replicas.s3')->orderBy('created_at', 'desc');$count = $executions->count();$executions = $executions->skip($skip)->take($take)->get();diff --git a/app/Models/ScheduledDatabaseBackupExecution.php b/app/Models/ScheduledDatabaseBackupExecution.phpindex 1a479772cf..eb43a1a2e3 100644--- a/app/Models/ScheduledDatabaseBackupExecution.php+++ b/app/Models/ScheduledDatabaseBackupExecution.php@@ -2,10 +2,14 @@namespace App\Models;+use App\Traits\HasS3Replicas;use Illuminate\Database\Eloquent\Relations\BelongsTo;+use Illuminate\Database\Eloquent\Relations\HasMany;class ScheduledDatabaseBackupExecution extends BaseModel{+ use HasS3Replicas;+protected static function booted(): void{static::created(function (ScheduledDatabaseBackupExecution $execution): void {@@ -42,4 +46,9 @@ class ScheduledDatabaseBackupExecution extends BaseModel{return $this->belongsTo(ScheduledDatabaseBackup::class);}++ public function s3Replicas(): HasMany+ {+ return $this->hasMany(DatabaseBackupS3Replica::class, 'execution_id');+ }}diff --git a/app/Models/ScheduledVolumeBackup.php b/app/Models/ScheduledVolumeBackup.phpindex 9ad676d953..0c4443b2e3 100644--- a/app/Models/ScheduledVolumeBackup.php+++ b/app/Models/ScheduledVolumeBackup.php@@ -2,15 +2,19 @@namespace App\Models;+use App\Traits\HasS3Destinations;use Illuminate\Database\Eloquent\Builder;use Illuminate\Database\Eloquent\Model;use Illuminate\Database\Eloquent\Relations\BelongsTo;+use Illuminate\Database\Eloquent\Relations\BelongsToMany;use Illuminate\Database\Eloquent\Relations\HasMany;use Illuminate\Database\Eloquent\Relations\HasOne;use Illuminate\Database\Eloquent\Relations\MorphTo;class ScheduledVolumeBackup extends BaseModel{+ use HasS3Destinations;+public const int DEFAULT_TIMEOUT = 36000;protected $fillable = [@@ -123,6 +127,11 @@ class ScheduledVolumeBackup extends BaseModelreturn $this->belongsTo(S3Storage::class, 's3_storage_id');}+ public function s3Storages(): BelongsToMany+ {+ return $this->belongsToMany(S3Storage::class, 'scheduled_volume_backup_s3_storage')->withTimestamps();+ }+public function executions(): HasMany{return $this->hasMany(ScheduledVolumeBackupExecution::class)->latest();diff --git a/app/Models/ScheduledVolumeBackupExecution.php b/app/Models/ScheduledVolumeBackupExecution.phpindex 664efa11c1..82dfe0ee3b 100644--- a/app/Models/ScheduledVolumeBackupExecution.php+++ b/app/Models/ScheduledVolumeBackupExecution.php@@ -2,10 +2,14 @@namespace App\Models;+use App\Traits\HasS3Replicas;use Illuminate\Database\Eloquent\Relations\BelongsTo;+use Illuminate\Database\Eloquent\Relations\HasMany;class ScheduledVolumeBackupExecution extends BaseModel{+ use HasS3Replicas;+protected $fillable = ['uuid','scheduled_volume_backup_id',@@ -56,4 +60,9 @@ class ScheduledVolumeBackupExecution extends BaseModel{return $this->belongsTo(S3Storage::class, 's3_storage_id');}++ public function s3Replicas(): HasMany+ {+ return $this->hasMany(VolumeBackupS3Replica::class, 'execution_id');+ }}diff --git a/app/Models/VolumeBackupS3Replica.php b/app/Models/VolumeBackupS3Replica.phpnew file mode 100644index 0000000000..779ee7571c--- /dev/null+++ b/app/Models/VolumeBackupS3Replica.php@@ -0,0 +1,38 @@+<?php++namespace App\Models;++use Illuminate\Database\Eloquent\Model;+use Illuminate\Database\Eloquent\Relations\BelongsTo;++/**+ * The copy of one volume or directory backup execution in one S3 destination.+ */+class VolumeBackupS3Replica extends Model+{+ protected $fillable = [+ 'execution_id',+ 's3_storage_id',+ 's3_uploaded',+ 's3_storage_deleted',+ 'message',+ ];++ protected function casts(): array+ {+ return [+ 's3_uploaded' => 'boolean',+ 's3_storage_deleted' => 'boolean',+ ];+ }++ public function execution(): BelongsTo+ {+ return $this->belongsTo(ScheduledVolumeBackupExecution::class, 'execution_id');+ }++ public function s3(): BelongsTo+ {+ return $this->belongsTo(S3Storage::class, 's3_storage_id');+ }+}diff --git a/app/Services/ServerTransfer/ServerTransferExporter.php b/app/Services/ServerTransfer/ServerTransferExporter.phpindex 8db6e5afd1..db7ddf556e 100644--- a/app/Services/ServerTransfer/ServerTransferExporter.php+++ b/app/Services/ServerTransfer/ServerTransferExporter.php@@ -16,6 +16,7 @@ use App\Models\Project;use App\Models\S3Storage;use App\Models\ScheduledDatabaseBackup;use App\Models\ScheduledTask;+use App\Models\ScheduledVolumeBackup;use App\Models\Server;use App\Models\Service;use App\Models\SharedEnvironmentVariable;@@ -202,22 +203,22 @@ class ServerTransferExporter}}- $collectVolumeS3 = function ($resource) use (&$s3Ids): void {+ $collectBackupS3 = function (ScheduledDatabaseBackup|ScheduledVolumeBackup $backup) use (&$s3Ids): void {+ $s3Ids->push($backup->s3_storage_id, ...$backup->s3Storages()->pluck('s3_storages.id')->all());+ };++ $collectVolumeS3 = function ($resource) use ($collectBackupS3): void {if (method_exists($resource, 'persistentStorages')) {foreach ($resource->persistentStorages as $volume) {foreach ($volume->scheduledBackups as $vb) {- if ($vb->s3_storage_id) {- $s3Ids->push($vb->s3_storage_id);- }+ $collectBackupS3($vb);}}}if (method_exists($resource, 'fileStorages')) {foreach ($resource->fileStorages as $file) {foreach ($file->scheduledBackups as $vb) {- if ($vb->s3_storage_id) {- $s3Ids->push($vb->s3_storage_id);- }+ $collectBackupS3($vb);}}}@@ -226,9 +227,7 @@ class ServerTransferExporterforeach ($databases as $database) {if (method_exists($database, 'scheduledBackups')) {foreach ($database->scheduledBackups as $backup) {- if ($backup->s3_storage_id) {- $s3Ids->push($backup->s3_storage_id);- }+ $collectBackupS3($backup);}}$database->loadMissing(['persistentStorages.scheduledBackups', 'fileStorages.scheduledBackups']);@@ -257,9 +256,7 @@ class ServerTransferExporter}foreach ($service->databases as $serviceDb) {foreach ($serviceDb->scheduledBackups as $backup) {- if ($backup->s3_storage_id) {- $s3Ids->push($backup->s3_storage_id);- }+ $collectBackupS3($backup);}$collectVolumeS3($serviceDb);}@@ -981,6 +978,7 @@ class ServerTransferExporter'timeout' => $backup->timeout,'disable_local_backup' => (bool) ($backup->disable_local_backup ?? false),'s3_storage_uuid' => $backup->s3?->uuid,+ 's3_storage_uuids' => $backup->selectedS3Storages()->pluck('uuid')->values()->all(),'had_s3_storage' => filled($backup->s3_storage_id),];}@@ -1193,6 +1191,7 @@ class ServerTransferExporter'retention_max_storage_s3' => $backup->retention_max_storage_s3,'timeout' => $backup->timeout,'s3_storage_uuid' => $backup->s3?->uuid,+ 's3_storage_uuids' => $backup->selectedS3Storages()->pluck('uuid')->values()->all(),'had_s3_storage' => filled($backup->s3_storage_id),];}diff --git a/app/Services/ServerTransfer/ServerTransferImporter.php b/app/Services/ServerTransfer/ServerTransferImporter.phpindex 90fe5954d0..d1714d4e70 100644--- a/app/Services/ServerTransfer/ServerTransferImporter.php+++ b/app/Services/ServerTransfer/ServerTransferImporter.php@@ -1237,13 +1237,10 @@ class ServerTransferImporter$uuid = new_public_id();}- $s3Id = null;- $s3Uuid = data_get($backup, 's3_storage_uuid');- if ($s3Uuid && isset($this->s3StorageMap[$s3Uuid])) {- $s3Id = $this->s3StorageMap[$s3Uuid]->id;- }+ $s3Ids = $this->importedBackupS3StorageIds($backup);+ $s3Id = $s3Ids[0] ?? null;- ScheduledDatabaseBackup::create([+ $importedBackup = ScheduledDatabaseBackup::create(['uuid' => $uuid,'team_id' => $teamId,'description' => data_get($backup, 'description'),@@ -1264,9 +1261,26 @@ class ServerTransferImporter'database_type' => $database->getMorphClass(),'database_id' => $database->id,]);+ $importedBackup->syncS3Storages($s3Ids);}}+ /**+ * Maps the exported S3 destinations of a backup to imported storages, primary first. Older bundles only have+ * `s3_storage_uuid`.+ *+ * @param array<string, mixed> $backup+ * @return list<int>+ */+ private function importedBackupS3StorageIds(array $backup): array+ {+ $uuids = collect(data_get($backup, 's3_storage_uuids', []))+ ->prepend(data_get($backup, 's3_storage_uuid'))+ ->filter(fn ($uuid): bool => is_string($uuid) && isset($this->s3StorageMap[$uuid]));++ return $uuids->map(fn (string $uuid): int => $this->s3StorageMap[$uuid]->id)->unique()->values()->all();+ }+/*** @param array<string, mixed> $payload*/@@ -1612,13 +1626,10 @@ class ServerTransferImporter$uuid = new_public_id();}- $s3Id = null;- $s3Uuid = data_get($backupPayload, 's3_storage_uuid');- if ($s3Uuid && isset($this->s3StorageMap[$s3Uuid])) {- $s3Id = $this->s3StorageMap[$s3Uuid]->id;- }+ $s3Ids = $this->importedBackupS3StorageIds($backupPayload);+ $s3Id = $s3Ids[0] ?? null;- ScheduledVolumeBackup::create([+ $importedBackup = ScheduledVolumeBackup::create(['uuid' => $uuid,'backupable_type' => $volume->getMorphClass(),'backupable_id' => $volume->id,@@ -1637,6 +1648,7 @@ class ServerTransferImporter'retention_max_storage_s3' => data_get($backupPayload, 'retention_max_storage_s3'),'timeout' => data_get($backupPayload, 'timeout'),]);+ $importedBackup->syncS3Storages($s3Ids);$count++;}diff --git a/app/Traits/HasS3Destinations.php b/app/Traits/HasS3Destinations.phpnew file mode 100644index 0000000000..59668f3961--- /dev/null+++ b/app/Traits/HasS3Destinations.php@@ -0,0 +1,68 @@+<?php++namespace App\Traits;++use App\Models\S3Storage;+use Illuminate\Database\Eloquent\Builder;+use Illuminate\Database\Eloquent\Relations\BelongsToMany;+use Illuminate\Support\Collection;++/**+ * S3 destinations of a backup schedule. The `s3_storage_id` column keeps the primary destination for older clients.+ */+trait HasS3Destinations+{+ abstract public function s3Storages(): BelongsToMany;++ /**+ * @return Collection<int, S3Storage> The primary destination comes first.+ */+ public function selectedS3Storages(): Collection+ {+ $storages = $this->relationLoaded('s3Storages')+ ? $this->s3Storages->sortBy('name')+ : $this->s3Storages()->orderBy('s3_storages.name')->get();++ if ($storages->isEmpty()) {+ return $this->s3 ? collect([$this->s3]) : collect();+ }++ return $storages+ ->sortBy(fn (S3Storage $storage): int => $storage->id === $this->s3_storage_id ? 0 : 1)+ ->values();+ }++ /**+ * Schedules that send backups to the given storage.+ */+ public function scopeUsingS3Storage(Builder $query, int $storageId): Builder+ {+ return $query->where(fn (Builder $query) => $query+ ->where('s3_storage_id', $storageId)+ ->orWhereHas('s3Storages', fn (Builder $query) => $query->whereKey($storageId)));+ }++ /**+ * Gives a cloned schedule the same destinations, limited to storages of the clone's team.+ */+ public function copyS3StoragesTo(self $copy): void+ {+ $copy->s3Storages()->sync(+ $this->s3Storages()->where('s3_storages.team_id', $copy->team_id)->pluck('s3_storages.id')->all(),+ );+ }++ /**+ * Replaces the selected destinations. The current primary destination stays primary when it is still selected.+ *+ * @param array<int, int|string> $storageIds+ */+ public function syncS3Storages(array $storageIds): void+ {+ $storageIds = collect($storageIds)->map(fn ($id): int => (int) $id)->filter()->unique()->values();++ $this->s3Storages()->sync($storageIds->all());+ $this->s3_storage_id = $storageIds->contains($this->s3_storage_id) ? $this->s3_storage_id : $storageIds->first();+ $this->save();+ }+}diff --git a/app/Traits/HasS3Replicas.php b/app/Traits/HasS3Replicas.phpnew file mode 100644index 0000000000..81e935e174--- /dev/null+++ b/app/Traits/HasS3Replicas.php@@ -0,0 +1,75 @@+<?php++namespace App\Traits;++use Illuminate\Database\Eloquent\Builder;+use Illuminate\Database\Eloquent\Relations\HasMany;++/**+ * S3 copies of a backup execution, one per destination. The `s3_uploaded` and `s3_storage_deleted` columns of the+ * execution summarize them.+ */+trait HasS3Replicas+{+ abstract public function s3Replicas(): HasMany;++ /**+ * Deletes the backup file from every destination that still has a copy.+ */+ public function deleteS3Copies(): void+ {+ $replicas = $this->s3Replicas()+ ->with('s3')+ ->where('s3_uploaded', true)+ ->where('s3_storage_deleted', false)+ ->get();++ try {+ foreach ($replicas as $replica) {+ if (! $replica->s3) {+ throw new \RuntimeException('The S3 storage used by an existing backup is unavailable.');+ }+ if (filled($this->filename)) {+ deleteBackupsS3($this->filename, $replica->s3);+ }+ $replica->update(['s3_storage_deleted' => true]);+ }+ } finally {+ $this->refreshS3Summary();+ }+ }++ public function hasLiveS3Copies(): bool+ {+ return $this->s3Replicas()->where('s3_uploaded', true)->where('s3_storage_deleted', false)->exists();+ }++ /**+ * Writes the replica states to the summary columns: uploaded when every copy uploaded, deleted when every uploaded+ * copy is deleted.+ */+ public function refreshS3Summary(): void+ {+ $replicas = $this->s3Replicas()->get();+ if ($replicas->isEmpty()) {+ return;+ }++ $uploaded = $replicas->where('s3_uploaded', true);++ $this->update([+ 's3_uploaded' => $replicas->every(fn ($replica): bool => $replica->s3_uploaded === true),+ 's3_storage_deleted' => $uploaded->isNotEmpty() && $uploaded->every(fn ($replica): bool => $replica->s3_storage_deleted),+ ]);+ }++ /**+ * Executions that have no S3 copy left to keep.+ */+ public function scopeWithoutLiveS3Copies(Builder $query): Builder+ {+ return $query->whereDoesntHave('s3Replicas', fn (Builder $query) => $query+ ->where('s3_uploaded', true)+ ->where('s3_storage_deleted', false));+ }+}diff --git a/bootstrap/helpers/databases.php b/bootstrap/helpers/databases.phpindex f4a97be5ce..966b2de3e6 100644--- a/bootstrap/helpers/databases.php+++ b/bootstrap/helpers/databases.php@@ -1,7 +1,9 @@<?php+use App\Models\DatabaseBackupS3Replica;use App\Models\EnvironmentVariable;use App\Models\S3Storage;+use App\Models\ScheduledDatabaseBackupExecution;use App\Models\Server;use App\Models\ServiceDatabase;use App\Models\StandaloneClickhouse;@@ -314,110 +316,96 @@ function deleteEmptyBackupFolder($folderPath, Server $server): voidfunction removeOldBackups($backup): void{- try {- if ($backup->executions) {- // Delete old local backups (only if local backup is NOT disabled)- // Note: When disable_local_backup is enabled, each execution already marks its own- // local_storage_deleted status at the time of backup, so we don't need to retroactively- // update old executions- if (! $backup->disable_local_backup) {- $localBackupsToDelete = deleteOldBackupsLocally($backup);- if ($localBackupsToDelete->isNotEmpty()) {- $backup->executions()- ->whereIn('id', $localBackupsToDelete->pluck('id'))- ->update(['local_storage_deleted' => true]);- }- }- }-- if ($backup->save_s3 && $backup->executions) {- $s3BackupsToDelete = deleteOldBackupsFromS3($backup);- if ($s3BackupsToDelete->isNotEmpty()) {- $backup->executions()- ->whereIn('id', $s3BackupsToDelete->pluck('id'))- ->update(['s3_storage_deleted' => true]);- }- }-- // Delete execution records where all backup copies are gone- // Case 1: Both local and S3 backups are deleted- $backup->executions()- ->where('local_storage_deleted', true)- ->where('s3_storage_deleted', true)- ->delete();-- // Case 2: Local backup is deleted and S3 was never used (s3_uploaded is null)+ // When local backups are disabled, retain the local file until an S3 copy is available.+ $localBackupsToDelete = deleteOldBackupsLocally($backup);+ if ($localBackupsToDelete->isNotEmpty()) {$backup->executions()- ->where('local_storage_deleted', true)- ->whereNull('s3_uploaded')- ->delete();-- } catch (Exception $e) {- throw $e;- }-}--function deleteOldBackupsLocally($backup): Collection-{- if (! $backup || ! $backup->executions) {- return collect();+ ->whereIn('id', $localBackupsToDelete->pluck('id'))+ ->update(['local_storage_deleted' => true]);}- $successfulBackups = $backup->executions()- ->where('status', 'success')- ->where('local_storage_deleted', false)- ->orderBy('created_at', 'desc')- ->get();-- if ($successfulBackups->isEmpty()) {- return collect();+ if ($backup->save_s3) {+ deleteOldBackupsFromS3($backup);}- $retentionAmount = $backup->database_backup_retention_amount_locally;- $retentionDays = $backup->database_backup_retention_days_locally;- $maxStorageGB = $backup->database_backup_retention_max_storage_locally;+ // Delete execution records when no local file and no S3 copy is left+ $backup->executions()+ ->where('local_storage_deleted', true)+ ->withoutLiveS3Copies()+ ->delete();+}- if ($retentionAmount === 0 && $retentionDays === 0 && $maxStorageGB === 0) {+/**+ * Successful executions that fall outside the retention settings, newest execution first.+ *+ * @param Collection<int, ScheduledDatabaseBackupExecution> $executions Sorted newest first.+ * @return Collection<int, ScheduledDatabaseBackupExecution>+ */+function backupExecutionsOutsideRetention(Collection $executions, ?int $retentionAmount, ?int $retentionDays, int|float|null $maxStorageGB): Collection+{+ if ($executions->isEmpty()) {return collect();}$backupsToDelete = collect();if ($retentionAmount > 0) {- $byAmount = $successfulBackups->skip($retentionAmount);- $backupsToDelete = $backupsToDelete->merge($byAmount);+ $backupsToDelete = $backupsToDelete->merge($executions->skip($retentionAmount));}if ($retentionDays > 0) {- $oldestAllowedDate = $successfulBackups->first()->created_at->clone()->utc()->subDays($retentionDays);- $oldBackups = $successfulBackups->filter(fn ($execution) => $execution->created_at->utc() < $oldestAllowedDate);- $backupsToDelete = $backupsToDelete->merge($oldBackups);+ $oldestAllowedDate = $executions->first()->created_at->clone()->utc()->subDays($retentionDays);+ $backupsToDelete = $backupsToDelete->merge(+ $executions->filter(fn ($execution) => $execution->created_at->utc() < $oldestAllowedDate)+ );}if ($maxStorageGB > 0) {$maxStorageBytes = $maxStorageGB * pow(1024, 3);$totalSize = 0;- $backupsOverLimit = collect();-- $backupsToCheck = $successfulBackups->skip(1);- foreach ($backupsToCheck as $backupExecution) {+ foreach ($executions->skip(1) as $backupExecution) {$totalSize += (int) $backupExecution->size;if ($totalSize > $maxStorageBytes) {- $backupsOverLimit = $successfulBackups->filter(+ $backupsToDelete = $backupsToDelete->merge($executions->filter(fn ($b) => $b->created_at->utc() <= $backupExecution->created_at->utc()- )->skip(1);+ )->skip(1));break;}}+ }++ return $backupsToDelete->unique('id')->values();+}- $backupsToDelete = $backupsToDelete->merge($backupsOverLimit);+function deleteOldBackupsLocally($backup): Collection+{+ if (! $backup) {+ return collect();}- $backupsToDelete = $backupsToDelete->unique('id');- $processedBackups = collect();+ $successfulBackups = $backup->executions()+ ->where('status', 'success')+ ->where('local_storage_deleted', false)+ ->when($backup->disable_local_backup, fn ($query) => $query+ ->whereHas('s3Replicas', fn ($query) => $query+ ->where('s3_uploaded', true)+ ->where('s3_storage_deleted', false)))+ ->orderBy('created_at', 'desc')+ ->orderBy('id', 'desc')+ ->get();++ $backupsToDelete = backupExecutionsOutsideRetention(+ $successfulBackups,+ $backup->database_backup_retention_amount_locally,+ $backup->database_backup_retention_days_locally,+ $backup->database_backup_retention_max_storage_locally,+ );++ if ($backupsToDelete->isEmpty()) {+ return collect();+ }- $server = null;if ($backup->database_type === ServiceDatabase::class) {$server = $backup->database->service->server;} else {@@ -433,89 +421,81 @@ function deleteOldBackupsLocally($backup): Collection->pluck('filename')->all();- if (! empty($filesToDelete)) {- deleteBackupsLocally($filesToDelete, $server);- $processedBackups = $backupsToDelete;+ if (empty($filesToDelete)) {+ return collect();}- return $processedBackups;+ deleteBackupsLocally($filesToDelete, $server);++ return $backupsToDelete;}+/**+ * Applies the S3 retention settings to each destination separately. Copies in a storage that no longer exists are+ * marked as deleted.+ *+ * @return Collection<int, ScheduledDatabaseBackupExecution> The executions that lost at least one S3 copy.+ */function deleteOldBackupsFromS3($backup): Collection{- if (! $backup || ! $backup->executions || ! $backup->s3) {+ if (! $backup) {return collect();}- $successfulBackups = $backup->executions()- ->where('status', 'success')+ $liveReplicas = DatabaseBackupS3Replica::query()+ ->with('s3')+ ->whereHas('execution', fn ($query) => $query+ ->where('scheduled_database_backup_id', $backup->id)+ ->where('status', 'success'))+ ->where('s3_uploaded', true)->where('s3_storage_deleted', false)- ->orderBy('created_at', 'desc')->get();- if ($successfulBackups->isEmpty()) {- return collect();- }-- $retentionAmount = $backup->database_backup_retention_amount_s3;- $retentionDays = $backup->database_backup_retention_days_s3;- $maxStorageGB = $backup->database_backup_retention_max_storage_s3;-- if ($retentionAmount === 0 && $retentionDays === 0 && $maxStorageGB === 0) {- return collect();- }-- $backupsToDelete = collect();-- if ($retentionAmount > 0) {- $byAmount = $successfulBackups->skip($retentionAmount);- $backupsToDelete = $backupsToDelete->merge($byAmount);- }-- if ($retentionDays > 0) {- $oldestAllowedDate = $successfulBackups->first()->created_at->clone()->utc()->subDays($retentionDays);- $oldBackups = $successfulBackups->filter(fn ($execution) => $execution->created_at->utc() < $oldestAllowedDate);- $backupsToDelete = $backupsToDelete->merge($oldBackups);- }-- if ($maxStorageGB > 0) {- $maxStorageBytes = $maxStorageGB * pow(1024, 3);- $totalSize = 0;- $backupsOverLimit = collect();-- $backupsToCheck = $successfulBackups->skip(1);-- foreach ($backupsToCheck as $backupExecution) {- $totalSize += (int) $backupExecution->size;- if ($totalSize > $maxStorageBytes) {- $backupsOverLimit = $successfulBackups->filter(- fn ($b) => $b->created_at->utc() <= $backupExecution->created_at->utc()- )->skip(1);- break;- }+ $deletedReplicas = $liveReplicas->filter(fn (DatabaseBackupS3Replica $replica) => ! $replica->s3);++ foreach ($liveReplicas->filter(fn (DatabaseBackupS3Replica $replica) => $replica->s3)->groupBy('s3_storage_id') as $replicas) {+ $successfulBackups = $backup->executions()+ ->whereIn('id', $replicas->pluck('execution_id'))+ ->orderBy('created_at', 'desc')+ ->orderBy('id', 'desc')+ ->get();++ $backupsToDelete = backupExecutionsOutsideRetention(+ $successfulBackups,+ $backup->database_backup_retention_amount_s3,+ $backup->database_backup_retention_days_s3,+ $backup->database_backup_retention_max_storage_s3,+ );++ $filesToDelete = $backupsToDelete+ ->filter(fn ($execution) => ! empty($execution->filename))+ ->pluck('filename')+ ->all();++ if (empty($filesToDelete)) {+ continue;}- $backupsToDelete = $backupsToDelete->merge($backupsOverLimit);- }-- $backupsToDelete = $backupsToDelete->unique('id');- $processedBackups = collect();-- $filesToDelete = $backupsToDelete- ->filter(fn ($execution) => ! empty($execution->filename))- ->pluck('filename')- ->all();-- if (! empty($filesToDelete)) {try {- deleteBackupsS3($filesToDelete, $backup->s3);- $processedBackups = $backupsToDelete;+ deleteBackupsS3($filesToDelete, $replicas->first()->s3);+ $deletedReplicas = $deletedReplicas->merge($replicas->whereIn('execution_id', $backupsToDelete->pluck('id')));} catch (Throwable $e) {report($e);}}- return $processedBackups;+ if ($deletedReplicas->isEmpty()) {+ return collect();+ }++ DatabaseBackupS3Replica::query()+ ->whereKey($deletedReplicas->pluck('id'))+ ->update(['s3_storage_deleted' => true]);++ $executions = $backup->executions()->whereIn('id', $deletedReplicas->pluck('execution_id')->unique())->get();+ $executions->each(fn (ScheduledDatabaseBackupExecution $execution) => $execution->refreshS3Summary());++ return $executions;}function isPublicPortAlreadyUsed(Server $server, int $port, ?string $id = null): booldiff --git a/database/migrations/2026_10_07_150000_create_backup_s3_destination_tables.php b/database/migrations/2026_10_07_150000_create_backup_s3_destination_tables.phpnew file mode 100644index 0000000000..39aff70ad2--- /dev/null+++ b/database/migrations/2026_10_07_150000_create_backup_s3_destination_tables.php@@ -0,0 +1,126 @@+<?php++use Illuminate\Database\Migrations\Migration;+use Illuminate\Database\Schema\Blueprint;+use Illuminate\Support\Facades\DB;+use Illuminate\Support\Facades\Schema;++return new class extends Migration+{+ public function up(): void+ {+ Schema::create('scheduled_database_backup_s3_storage', function (Blueprint $table) {+ $table->id();+ $table->foreignId('scheduled_database_backup_id')+ ->constrained(indexName: 'db_backup_s3_storage_backup_fk')+ ->cascadeOnDelete();+ $table->foreignId('s3_storage_id')+ ->constrained(indexName: 'db_backup_s3_storage_storage_fk')+ ->cascadeOnDelete();+ $table->timestamps();+ $table->unique(['scheduled_database_backup_id', 's3_storage_id'], 'db_backup_s3_storage_unique');+ });++ Schema::create('scheduled_volume_backup_s3_storage', function (Blueprint $table) {+ $table->id();+ $table->foreignId('scheduled_volume_backup_id')+ ->constrained(indexName: 'volume_backup_s3_storage_backup_fk')+ ->cascadeOnDelete();+ $table->foreignId('s3_storage_id')+ ->constrained(indexName: 'volume_backup_s3_storage_storage_fk')+ ->cascadeOnDelete();+ $table->timestamps();+ $table->unique(['scheduled_volume_backup_id', 's3_storage_id'], 'volume_backup_s3_storage_unique');+ });++ Schema::create('database_backup_s3_replicas', function (Blueprint $table) {+ $table->id();+ $table->foreignId('execution_id')+ ->index()+ ->constrained('scheduled_database_backup_executions', indexName: 'db_backup_s3_replicas_execution_fk')+ ->cascadeOnDelete();+ $table->foreignId('s3_storage_id')+ ->nullable()+ ->constrained(indexName: 'db_backup_s3_replicas_storage_fk')+ ->nullOnDelete();+ $table->boolean('s3_uploaded')->nullable();+ $table->boolean('s3_storage_deleted')->default(false);+ $table->text('message')->nullable();+ $table->timestamps();+ });++ Schema::create('volume_backup_s3_replicas', function (Blueprint $table) {+ $table->id();+ $table->foreignId('execution_id')+ ->index()+ ->constrained('scheduled_volume_backup_executions', indexName: 'volume_backup_s3_replicas_execution_fk')+ ->cascadeOnDelete();+ $table->foreignId('s3_storage_id')+ ->nullable()+ ->constrained(indexName: 'volume_backup_s3_replicas_storage_fk')+ ->nullOnDelete();+ $table->boolean('s3_uploaded')->nullable();+ $table->boolean('s3_storage_deleted')->default(false);+ $table->text('message')->nullable();+ $table->timestamps();+ });++ $this->backfill();+ }++ public function down(): void+ {+ Schema::dropIfExists('volume_backup_s3_replicas');+ Schema::dropIfExists('database_backup_s3_replicas');+ Schema::dropIfExists('scheduled_volume_backup_s3_storage');+ Schema::dropIfExists('scheduled_database_backup_s3_storage');+ }++ /**+ * Copies the single S3 destination of existing schedules and executions into the new tables.+ */+ private function backfill(): void+ {+ $now = now();++ DB::table('scheduled_database_backup_s3_storage')->insertUsing(+ ['scheduled_database_backup_id', 's3_storage_id', 'created_at', 'updated_at'],+ DB::table('scheduled_database_backups')+ ->join('s3_storages', 's3_storages.id', '=', 'scheduled_database_backups.s3_storage_id')+ ->select('scheduled_database_backups.id', 'scheduled_database_backups.s3_storage_id')+ ->selectRaw('?, ?', [$now, $now]),+ );++ DB::table('scheduled_volume_backup_s3_storage')->insertUsing(+ ['scheduled_volume_backup_id', 's3_storage_id', 'created_at', 'updated_at'],+ DB::table('scheduled_volume_backups')+ ->whereNotNull('s3_storage_id')+ ->select('id', 's3_storage_id')+ ->selectRaw('?, ?', [$now, $now]),+ );++ // Database executions have no storage column: the copy went to the storage of the schedule.+ DB::table('database_backup_s3_replicas')->insertUsing(+ ['execution_id', 's3_storage_id', 's3_uploaded', 's3_storage_deleted', 'created_at', 'updated_at'],+ DB::table('scheduled_database_backup_executions')+ ->join('scheduled_database_backups', 'scheduled_database_backups.id', '=', 'scheduled_database_backup_executions.scheduled_database_backup_id')+ ->leftJoin('s3_storages', 's3_storages.id', '=', 'scheduled_database_backups.s3_storage_id')+ ->whereNotNull('scheduled_database_backup_executions.s3_uploaded')+ ->select(+ 'scheduled_database_backup_executions.id',+ 's3_storages.id',+ 'scheduled_database_backup_executions.s3_uploaded',+ 'scheduled_database_backup_executions.s3_storage_deleted',+ )+ ->selectRaw('?, ?', [$now, $now]),+ );++ DB::table('volume_backup_s3_replicas')->insertUsing(+ ['execution_id', 's3_storage_id', 's3_uploaded', 's3_storage_deleted', 'created_at', 'updated_at'],+ DB::table('scheduled_volume_backup_executions')+ ->whereNotNull('s3_storage_id')+ ->select('id', 's3_storage_id', 's3_uploaded', 's3_storage_deleted')+ ->selectRaw('?, ?', [$now, $now]),+ );+ }+};diff --git a/openapi.json b/openapi.jsonindex 192d6f49b9..e418f93489 100644--- a/openapi.json+++ b/openapi.json@@ -86,9 +86,9 @@"properties": {"pull_request_id": {"type": "integer",+ "example": 42,"maximum": 2147483647,- "minimum": 1,- "example": 42+ "minimum": 1},"pull_request_html_url": {"type": [@@ -6857,7 +6857,7 @@],"responses": {"200": {- "description": "Get all backups for a database",+ "description": "Get all backups for a database. Each backup includes s3_storage_uuid (primary S3 destination) and s3_storage_uuids (every S3 destination, primary first).","content": {"application\/json": {"schema": {@@ -6927,7 +6927,14 @@},"s3_storage_uuid": {"type": "string",- "description": "S3 storage UUID (required if save_s3 is true)"+ "description": "Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination. Either field is required if save_s3 is true."+ },+ "s3_storage_uuids": {+ "type": "array",+ "items": {+ "type": "string"+ },+ "description": "UUIDs of every S3 storage the backup is uploaded to. Replaces the current destinations. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination."},"databases_to_backup": {"type": "string",@@ -7530,7 +7537,14 @@},"s3_storage_uuid": {"type": "string",- "description": "S3 storage UUID"+ "description": "Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination; when it is already a destination, it becomes the primary destination and the other destinations stay."+ },+ "s3_storage_uuids": {+ "type": "array",+ "items": {+ "type": "string"+ },+ "description": "UUIDs of every S3 storage the backup is uploaded to. Replaces the current destinations. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination."},"backup_now": {"type": "boolean",@@ -10878,8 +10892,8 @@},"network": {"type": "string",- "maxLength": 255,- "pattern": "^[a-zA-Z0-9][a-zA-Z0-9._-]*$"+ "pattern": "^[a-zA-Z0-9][a-zA-Z0-9._-]*$",+ "maxLength": 255},"type": {"type": "string",@@ -23541,8 +23555,8 @@"properties": {"frequency": {"type": "string",- "maxLength": 255,- "example": "0 2 * * *"+ "example": "0 2 * * *",+ "maxLength": 255},"enabled": {"type": "boolean",@@ -23561,11 +23575,19 @@"default": false},"s3_storage_uuid": {+ "description": "Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination.","type": ["string","null"]},+ "s3_storage_uuids": {+ "description": "UUIDs of every S3 storage the backup is uploaded to. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination. save_s3 requires at least one destination.",+ "type": "array",+ "items": {+ "type": "string"+ }+ },"retention_amount_locally": {"type": "integer","default": 7,@@ -23674,11 +23696,19 @@"type": "boolean"},"s3_storage_uuid": {+ "description": "Primary S3 destination.","type": ["string","null"]},+ "s3_storage_uuids": {+ "description": "Every S3 destination, primary first.",+ "type": "array",+ "items": {+ "type": "string"+ }+ },"retention_amount_locally": {"type": "integer"},diff --git a/openapi.yaml b/openapi.yamlindex 5983cbe38d..a7e4087848 100644--- a/openapi.yaml+++ b/openapi.yaml@@ -62,9 +62,9 @@ paths:properties:pull_request_id:type: integer+ example: 42maximum: 2147483647minimum: 1- example: 42pull_request_html_url:type: [string, 'null']example: 'https://github.com/org/repo/pull/42'@@ -4401,7 +4401,7 @@ paths:type: stringresponses:'200':- description: 'Get all backups for a database'+ description: 'Get all backups for a database. Each backup includes s3_storage_uuid (primary S3 destination) and s3_storage_uuids (every S3 destination, primary first).'content:application/json:schema:@@ -4452,7 +4452,11 @@ paths:default: falses3_storage_uuid:type: string- description: 'S3 storage UUID (required if save_s3 is true)'+ description: 'Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination. Either field is required if save_s3 is true.'+ s3_storage_uuids:+ type: array+ items: { type: string }+ description: 'UUIDs of every S3 storage the backup is uploaded to. Replaces the current destinations. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination.'databases_to_backup:type: stringdescription: 'Comma separated list of databases to backup'@@ -4875,7 +4879,11 @@ paths:description: 'Whether data is saved in s3 or not's3_storage_uuid:type: string- description: 'S3 storage UUID'+ description: 'Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination; when it is already a destination, it becomes the primary destination and the other destinations stay.'+ s3_storage_uuids:+ type: array+ items: { type: string }+ description: 'UUIDs of every S3 storage the backup is uploaded to. Replaces the current destinations. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination.'backup_now:type: booleandescription: 'Whether to take a backup now or not'@@ -7099,8 +7107,8 @@ paths:maxLength: 255network:type: string- maxLength: 255pattern: '^[a-zA-Z0-9][a-zA-Z0-9._-]*$'+ maxLength: 255type:type: stringenum: [standalone, swarm]@@ -14947,8 +14955,8 @@ components:properties:frequency:type: string- maxLength: 255example: '0 2 * * *'+ maxLength: 255enabled:type: booleandefault: true@@ -14962,9 +14970,15 @@ components:type: booleandefault: falses3_storage_uuid:+ description: 'Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination.'type:- string- 'null'+ s3_storage_uuids:+ description: 'UUIDs of every S3 storage the backup is uploaded to. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination. save_s3 requires at least one destination.'+ type: array+ items:+ type: stringretention_amount_locally:type: integerdefault: 7@@ -15052,9 +15066,15 @@ components:stop_during_backup:type: booleans3_storage_uuid:+ description: 'Primary S3 destination.'type:- string- 'null'+ s3_storage_uuids:+ description: 'Every S3 destination, primary first.'+ type: array+ items:+ type: stringretention_amount_locally:type: integerretention_days_locally:diff --git a/resources/views/components/status-badge.blade.php b/resources/views/components/status-badge.blade.phpindex 5aa02454bd..a8f49e8bf5 100644--- a/resources/views/components/status-badge.blade.php+++ b/resources/views/components/status-badge.blade.php@@ -4,6 +4,7 @@'type' => 'neutral','as' => 'span','dynamic' => false,+ 'pulse' => false, // ping the dot for a state that is still in progress])@php@@ -22,7 +23,12 @@@if ($dynamic){{ $slot }}@else- <span class="size-1.5 shrink-0 rounded-full {{ $dotClasses[$type] ?? $dotClasses['neutral'] }}"></span>+ <span class="relative flex size-1.5 shrink-0">+ @if ($pulse)+ <span class="absolute inline-flex size-full rounded-full opacity-75 motion-safe:animate-ping {{ $dotClasses[$type] ?? $dotClasses['neutral'] }}"></span>+ @endif+ <span class="relative inline-flex size-1.5 rounded-full {{ $dotClasses[$type] ?? $dotClasses['neutral'] }}"></span>+ </span><span class="truncate">{{ collect([$label, $status])->filter()->join(' ') }}</span>@endif</button>@@ -31,7 +37,12 @@@if ($dynamic){{ $slot }}@else- <span class="size-1.5 shrink-0 rounded-full {{ $dotClasses[$type] ?? $dotClasses['neutral'] }}"></span>+ <span class="relative flex size-1.5 shrink-0">+ @if ($pulse)+ <span class="absolute inline-flex size-full rounded-full opacity-75 motion-safe:animate-ping {{ $dotClasses[$type] ?? $dotClasses['neutral'] }}"></span>+ @endif+ <span class="relative inline-flex size-1.5 rounded-full {{ $dotClasses[$type] ?? $dotClasses['neutral'] }}"></span>+ </span><span class="truncate">{{ collect([$label, $status])->filter()->join(' ') }}</span>@endif</a>@@ -40,7 +51,12 @@@if ($dynamic){{ $slot }}@else- <span class="size-1.5 shrink-0 rounded-full {{ $dotClasses[$type] ?? $dotClasses['neutral'] }}"></span>+ <span class="relative flex size-1.5 shrink-0">+ @if ($pulse)+ <span class="absolute inline-flex size-full rounded-full opacity-75 motion-safe:animate-ping {{ $dotClasses[$type] ?? $dotClasses['neutral'] }}"></span>+ @endif+ <span class="relative inline-flex size-1.5 rounded-full {{ $dotClasses[$type] ?? $dotClasses['neutral'] }}"></span>+ </span><span class="truncate">{{ collect([$label, $status])->filter()->join(' ') }}</span>@endif</span>diff --git a/resources/views/livewire/project/application/backup/index.blade.php b/resources/views/livewire/project/application/backup/index.blade.phpindex 7e89bbcd07..785b855cae 100644--- a/resources/views/livewire/project/application/backup/index.blade.php+++ b/resources/views/livewire/project/application/backup/index.blade.php@@ -181,6 +181,7 @@@foreach ($backups as $backup)@php$latestExecution = $backup->latestExecution;+ $s3StorageNames = ($backup->s3Storages->isNotEmpty() ? $backup->s3Storages : collect([$backup->s3])->filter())->pluck('name');$status = $latestExecution?->status;$statusLabel = match ($status) {'running' => 'In progress',@@ -208,9 +209,9 @@<span>{{ $backup->targetType() }}</span><span>{{ $backup->frequency }}</span><span><x-status-badge :status="$statusLabel" :type="$statusType" /></span>- <span title="{{ $backup->save_s3 ? ($backup->s3?->name ?? 'S3 storage unavailable') : 'S3 storage is not configured' }}">- <x-status-badge :status="$backup->save_s3 ? ($backup->s3 ? 'Configured' : 'Unavailable') : 'Not set'"- :type="$backup->save_s3 ? ($backup->s3 ? 'success' : 'error') : 'neutral'" />+ <span title="{{ $backup->save_s3 ? ($s3StorageNames->isNotEmpty() ? $s3StorageNames->join(', ') : 'S3 storage unavailable') : 'S3 storage is not configured' }}">+ <x-status-badge :status="$backup->save_s3 ? match ($s3StorageNames->count()) { 0 => 'Unavailable', 1 => 'Configured', default => $s3StorageNames->count().' storages' } : 'Not set'"+ :type="$backup->save_s3 ? ($s3StorageNames->isNotEmpty() ? 'success' : 'error') : 'neutral'" /></span><span>{{ $latestExecution?->finished_at?->diffForHumans() ?? ($status === 'running' ? 'Running now' : 'Never') }}diff --git a/resources/views/livewire/project/database/backup-edit/s3.blade.php b/resources/views/livewire/project/database/backup-edit/s3.blade.phpindex dcfb7c3815..4f0f7b43fd 100644--- a/resources/views/livewire/project/database/backup-edit/s3.blade.php+++ b/resources/views/livewire/project/database/backup-edit/s3.blade.php@@ -25,11 +25,12 @@@endif</x-slot:actions><div class="grid gap-4 sm:grid-cols-2">- <x-forms.listbox id="s3StorageId" label="S3 storage" portal :required="$saveS3"- :options="$availableS3Storages->map(fn ($s3) => [- 'value' => $s3->id,- 'label' => $s3->name,- ])->values()->all()" />+ <x-forms.listbox id="s3StorageIds" label="S3 storages" multiple live portal+ helper="Each backup is uploaded to every selected storage." placeholder="Select S3 storages"+ :options="$availableS3Storages->map(fn ($storage) => [+ 'value' => $storage->id,+ 'label' => $storage->name.($storage->is_usable ? '' : ' (unavailable)'),+ ])->all()" /><x-forms.listbox id="disableLocalBackup" label="Local copy" portal :disabled="! $saveS3":options="[['value' => false, 'label' => 'Keep local backup'],diff --git a/resources/views/livewire/project/database/backup-executions.blade.php b/resources/views/livewire/project/database/backup-executions.blade.phpindex 6bd2851b73..f267908207 100644--- a/resources/views/livewire/project/database/backup-executions.blade.php+++ b/resources/views/livewire/project/database/backup-executions.blade.php@@ -55,6 +55,10 @@'failed' => ['Failed', 'error'],default => [str($executionStatus)->headline(), 'neutral'],};+ $s3Replicas = $execution->s3Replicas;+ $hasLiveS3Copies = $s3Replicas->isNotEmpty()+ ? $s3Replicas->contains(fn ($replica) => $replica->s3_uploaded === true && ! $replica->s3_storage_deleted)+ : data_get($execution, 's3_uploaded') === true && ! data_get($execution, 's3_storage_deleted', false);$executionCheckboxes = [];$deleteActions = [];@@ -62,11 +66,10 @@$deleteActions[] = 'This backup will be permanently deleted from local storage.';}- if (data_get($execution, 's3_uploaded') === true- && ! data_get($execution, 's3_storage_deleted', false)) {+ if ($hasLiveS3Copies) {$executionCheckboxes[] = ['id' => 'delete_backup_s3',- 'label' => 'Delete the selected backup permanently from S3 Storage',+ 'label' => 'Delete the selected backup permanently from every S3 storage',];}@@ -79,10 +82,7 @@<div class="data-table-row backup-executions-table-grid min-h-14 border-b border-neutral-200 px-4 py-2.5 dark:border-white/[0.06]"><div class="flex items-center gap-2"><x-status-badge :status="$executionStatusLabel"- :type="$executionStatusType" />- @if ($executionStatus === 'running')- <x-loading />- @endif+ :type="$executionStatusType" :pulse="$executionStatus === 'running'" /></div><div class="truncate text-[12px] font-medium text-black dark:text-fg">{{ data_get($execution, 'database_name', 'N/A') }}@@ -112,7 +112,22 @@<x-status-badge label="Local":status="data_get($execution, 'local_storage_deleted', false) ? 'Deleted' : 'Available'":type="data_get($execution, 'local_storage_deleted', false) ? 'neutral' : 'success'" />- @if (data_get($execution, 's3_uploaded') !== null)+ @foreach ($s3Replicas as $replica)+ <x-status-badge wire:key="database-backup-replica-{{ $replica->id }}"+ :label="$replica->s3?->name ?? 'Removed S3 storage'"+ :status="$replica->s3_storage_deleted+ ? 'Deleted'+ : match ($replica->s3_uploaded) {+ true => 'Available',+ false => 'Failed',+ default => $executionStatus === 'running' ? 'Pending' : 'Not uploaded',+ }"+ :type="$replica->s3_storage_deleted || $replica->s3_uploaded === null+ ? 'neutral'+ : ($replica->s3_uploaded ? 'success' : 'error')"+ :title="$replica->message" />+ @endforeach+ @if ($s3Replicas->isEmpty() && data_get($execution, 's3_uploaded') !== null)<x-status-badge label="S3":status="data_get($execution, 's3_storage_deleted', false)? 'Deleted'diff --git a/resources/views/livewire/project/database/scheduled-backups.blade.php b/resources/views/livewire/project/database/scheduled-backups.blade.phpindex 793cd2ff47..aba61ec48c 100644--- a/resources/views/livewire/project/database/scheduled-backups.blade.php+++ b/resources/views/livewire/project/database/scheduled-backups.blade.php@@ -3,7 +3,7 @@backups: @js($database->scheduledBackups->map(fn ($backup) => ['name' => strtolower($database->name),'frequency' => strtolower($backup->frequency),- 's3_storage' => strtolower($backup->s3?->name ?? ''),+ 's3_storage' => strtolower(($backup->s3Storages->isNotEmpty() ? $backup->s3Storages : collect([$backup->s3])->filter())->pluck('name')->join(', ')),])->values()),hasMatches() {const query = this.search.toLowerCase();@@ -67,6 +67,7 @@@foreach ($database->scheduledBackups as $backup)@php$latestStatus = data_get($backup->latest_log, 'status');+ $s3StorageNames = ($backup->s3Storages->isNotEmpty() ? $backup->s3Storages : collect([$backup->s3])->filter())->pluck('name')->join(', ');[$statusLabel, $statusType] = match ($latestStatus) {'success' => ['Success', 'success'],'running' => ['In progress', 'warning'],@@ -83,7 +84,7 @@<div x-show="search === ''|| @js(strtolower($database->name)).includes(search.toLowerCase())|| @js(strtolower($backup->frequency)).includes(search.toLowerCase())- || @js(strtolower($backup->s3?->name ?? '')).includes(search.toLowerCase())"+ || @js(strtolower($s3StorageNames)).includes(search.toLowerCase())"class="data-table-row scheduled-backups-table-grid border-b border-neutral-200 last:border-b-0 dark:border-white/[0.06]"><div class="min-w-0"><span class="block truncate text-[12px] font-semibold text-black dark:text-fg">@@ -96,8 +97,8 @@<x-loading />@endif</div>- <div class="truncate text-[11px] text-neutral-600 dark:text-fg-dim">- {{ $backup->save_s3 ? ($backup->s3?->name ?? 'Unavailable') : 'Local only' }}+ <div class="truncate text-[11px] text-neutral-600 dark:text-fg-dim" title="{{ $backup->save_s3 ? $s3StorageNames : '' }}">+ {{ $backup->save_s3 ? ($s3StorageNames ?: 'Unavailable') : 'Local only' }}</div><div class="text-[11px] text-neutral-600 dark:text-fg-dim"><a {{ wireNavigate() }} href="{{ $backupExecutionsRoute }}"diff --git a/resources/views/livewire/project/service/volume-backup/index.blade.php b/resources/views/livewire/project/service/volume-backup/index.blade.phpindex 26b835cdc8..acde772a26 100644--- a/resources/views/livewire/project/service/volume-backup/index.blade.php+++ b/resources/views/livewire/project/service/volume-backup/index.blade.php@@ -268,8 +268,8 @@default => 'neutral',};$databaseBackupId = 'database:'.$databaseBackup->id;- $databaseS3 = $databaseBackup->s3?->team_id === $serviceTeamId ? $databaseBackup->s3 : null;- $databaseS3Tooltip = ! $databaseBackup->save_s3 ? 'S3 storage: Not configured' : ($databaseS3 ? 'S3 storage: '.$databaseS3->name.' (bucket: '.$databaseS3->bucket.')' : 'S3 storage: Unavailable');+ $databaseS3s = ($databaseBackup->s3Storages->isNotEmpty() ? $databaseBackup->s3Storages : collect([$databaseBackup->s3])->filter())->where('team_id', $serviceTeamId);+ $databaseS3Tooltip = ! $databaseBackup->save_s3 ? 'S3 storage: Not configured' : ($databaseS3s->isNotEmpty() ? 'S3 storage: '.$databaseS3s->map(fn ($s3) => $s3->name.' (bucket: '.$s3->bucket.')')->join(', ') : 'S3 storage: Unavailable');@endphp<div wire:key="database-backup-{{ $databaseBackup->uuid }}"x-show="isVisible(@js($databaseBackupId))"@@ -284,8 +284,8 @@<span>{{ $databaseBackup->frequency }}</span><span><x-status-badge :status="$statusLabel" :type="$statusType" /></span><span>- <x-status-badge :status="$databaseBackup->save_s3 ? ($databaseS3 ? 'Configured' : 'Unavailable') : 'Not set'"- :type="$databaseBackup->save_s3 ? ($databaseS3 ? 'success' : 'error') : 'neutral'"+ <x-status-badge :status="$databaseBackup->save_s3 ? match ($databaseS3s->count()) { 0 => 'Unavailable', 1 => 'Configured', default => $databaseS3s->count().' storages' } : 'Not set'"+ :type="$databaseBackup->save_s3 ? ($databaseS3s->isNotEmpty() ? 'success' : 'error') : 'neutral'":data-tooltip="$databaseS3Tooltip" :aria-label="$databaseS3Tooltip" tabindex="0" /></span><span>{{ $latestExecution?->finished_at?->diffForHumans() ?? ($status === 'running' ? 'Running now' : 'Never') }}</span>@@ -308,8 +308,8 @@@foreach ($backups as $backup)@php$latestExecution = $backup->latestExecution;- $volumeS3 = $backup->s3?->team_id === $serviceTeamId ? $backup->s3 : null;- $volumeS3Tooltip = ! $backup->save_s3 ? 'S3 storage: Not configured' : ($volumeS3 ? 'S3 storage: '.$volumeS3->name.' (bucket: '.$volumeS3->bucket.')' : 'S3 storage: Unavailable');+ $volumeS3s = ($backup->s3Storages->isNotEmpty() ? $backup->s3Storages : collect([$backup->s3])->filter())->where('team_id', $serviceTeamId);+ $volumeS3Tooltip = ! $backup->save_s3 ? 'S3 storage: Not configured' : ($volumeS3s->isNotEmpty() ? 'S3 storage: '.$volumeS3s->map(fn ($s3) => $s3->name.' (bucket: '.$s3->bucket.')')->join(', ') : 'S3 storage: Unavailable');$status = $latestExecution?->status;$statusLabel = match ($status) {'running' => 'In progress',@@ -338,8 +338,8 @@<span>{{ $backup->frequency }}</span><span><x-status-badge :status="$statusLabel" :type="$statusType" /></span><span>- <x-status-badge :status="$backup->save_s3 ? ($volumeS3 ? 'Configured' : 'Unavailable') : 'Not set'"- :type="$backup->save_s3 ? ($volumeS3 ? 'success' : 'error') : 'neutral'"+ <x-status-badge :status="$backup->save_s3 ? match ($volumeS3s->count()) { 0 => 'Unavailable', 1 => 'Configured', default => $volumeS3s->count().' storages' } : 'Not set'"+ :type="$backup->save_s3 ? ($volumeS3s->isNotEmpty() ? 'success' : 'error') : 'neutral'":data-tooltip="$volumeS3Tooltip" :aria-label="$volumeS3Tooltip" tabindex="0" /></span><span>diff --git a/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php b/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.phpindex b4bf4465ec..041fb4470d 100644--- a/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php+++ b/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php@@ -48,15 +48,19 @@'failed' => 'error',default => 'neutral',};+ $s3Replicas = $execution->s3Replicas;+ $hasLiveS3Copies = $s3Replicas->isNotEmpty()+ ? $s3Replicas->contains(fn ($replica) => $replica->s3_uploaded === true && ! $replica->s3_storage_deleted)+ : $execution->s3_uploaded === true && ! $execution->s3_storage_deleted;$executionCheckboxes = [];$deleteActions = [];if (! $execution->local_storage_deleted) {$deleteActions[] = 'This backup will be permanently deleted from local storage.';}- if ($execution->s3_uploaded === true && ! $execution->s3_storage_deleted) {+ if ($hasLiveS3Copies) {$executionCheckboxes[] = ['id' => 'delete_backup_s3',- 'label' => 'Delete the selected backup permanently from S3 Storage',+ 'label' => 'Delete the selected backup permanently from every S3 storage',];}if (empty($deleteActions)) {@@ -75,7 +79,7 @@<div wire:key="volume-backup-execution-{{ $execution->id }}"class="data-table-row volume-backup-executions-grid min-h-16 items-center gap-x-3 border-b border-neutral-200 text-[12px] last:border-b-0 dark:border-white/[0.07]"><span>- <x-status-badge :status="$statusLabel" :type="$statusType" />+ <x-status-badge :status="$statusLabel" :type="$statusType" :pulse="$execution->status === 'running'" /></span><span class="min-w-0">@@ -98,7 +102,22 @@<span class="flex flex-wrap gap-1"><x-status-badge :status="$execution->local_storage_deleted ? 'Local deleted' : 'Local'":type="$execution->local_storage_deleted ? 'neutral' : 'success'" />- @if ($execution->s3_uploaded !== null)+ @foreach ($s3Replicas as $replica)+ <x-status-badge wire:key="volume-backup-replica-{{ $replica->id }}"+ :label="$replica->s3?->name ?? 'Removed S3 storage'"+ :status="$replica->s3_storage_deleted+ ? 'deleted'+ : match ($replica->s3_uploaded) {+ true => null,+ false => 'failed',+ default => $execution->status === 'running' ? 'pending' : 'not uploaded',+ }"+ :type="$replica->s3_storage_deleted || $replica->s3_uploaded === null+ ? 'neutral'+ : ($replica->s3_uploaded ? 'success' : 'error')"+ :title="$replica->message" />+ @endforeach+ @if ($s3Replicas->isEmpty() && $execution->s3_uploaded !== null)<x-status-badge:status="$execution->s3_storage_deleted? 'S3 deleted'diff --git a/resources/views/livewire/project/shared/storages/volume-backups/s3.blade.php b/resources/views/livewire/project/shared/storages/volume-backups/s3.blade.phpindex 7c3f471bfa..3655a94753 100644--- a/resources/views/livewire/project/shared/storages/volume-backups/s3.blade.php+++ b/resources/views/livewire/project/shared/storages/volume-backups/s3.blade.php@@ -29,12 +29,12 @@@endif</x-slot:actions><div class="grid gap-4 sm:grid-cols-2">- <x-forms.listbox canGate="update" :canResource="$resource" id="s3StorageId" label="S3 storage" :required="$saveToS3"- :disabled="! auth()->user()?->can('update', $resource)"- :options="$availableS3Storages->map(fn ($s3Storage) => [- 'value' => $s3Storage->id,- 'label' => $s3Storage->name,- ])->values()->all()" />+ <x-forms.listbox canGate="update" :canResource="$resource" id="s3StorageIds" label="S3 storages" multiple live portal+ helper="Each backup is uploaded to every selected storage." placeholder="Select S3 storages"+ :options="$availableS3Storages->map(fn ($storage) => [+ 'value' => $storage->id,+ 'label' => $storage->name.($storage->is_usable ? '' : ' (unavailable)'),+ ])->all()" /><x-forms.listbox canGate="update" :canResource="$resource" id="disableLocalBackup" label="Local copy":disabled="! $saveToS3 || ! auth()->user()?->can('update', $resource)" :options="[['value' => false, 'label' => 'Keep local backup'],diff --git a/resources/views/livewire/storage/resources.blade.php b/resources/views/livewire/storage/resources.blade.phpindex dd8b004e4c..3caa1f6c94 100644--- a/resources/views/livewire/storage/resources.blade.php+++ b/resources/views/livewire/storage/resources.blade.php@@ -43,6 +43,8 @@]);}}+ } elseif ($database instanceof \App\Models\StandalonePostgresql && $database->id === 0) {+ $resourceLink = route('settings.backup');} elseif ($database) {$environment = $database->environment;$project = $environment?->project;@@ -103,10 +105,10 @@<button type="button" class="button shrink-0"wire:click="moveBackup({{ $backup->id }})">Move</button><div wire:key="disableS3-{{ $backup->id }}">- <x-modal-confirmation title="Disable S3 for this backup schedule?"+ <x-modal-confirmation title="Remove this storage from the backup schedule?"submitAction="disableS3({{ $backup->id }})" :confirmWithText="false"- :confirmWithPassword="false" step2ButtonText="Disable S3"- :actions="['Backups from this schedule will no longer be uploaded to S3.']">+ :confirmWithPassword="false" step2ButtonText="Remove storage"+ :actions="['Backups from this schedule will no longer be uploaded to this storage. S3 backup turns off when no other storage remains.']"><x-slot:trigger><button type="button" class="button shrink-0 text-error">Disable</button></x-slot:trigger>@@ -148,10 +150,10 @@<button type="button" class="button shrink-0"wire:click="moveVolumeBackup({{ $backup->id }})">Move</button><div wire:key="disableVolumeS3-{{ $backup->id }}">- <x-modal-confirmation title="Disable S3 for this backup schedule?"+ <x-modal-confirmation title="Remove this storage from the backup schedule?"submitAction="disableVolumeS3({{ $backup->id }})" :confirmWithText="false"- :confirmWithPassword="false" step2ButtonText="Disable S3"- :actions="['Backups from this schedule will no longer be uploaded to S3.']">+ :confirmWithPassword="false" step2ButtonText="Remove storage"+ :actions="['Backups from this schedule will no longer be uploaded to this storage. S3 backup turns off when no other storage remains.']"><x-slot:trigger><button type="button" class="button shrink-0 text-error">Disable</button></x-slot:trigger>diff --git a/tests/Feature/Api/BackupS3DestinationsApiTest.php b/tests/Feature/Api/BackupS3DestinationsApiTest.phpnew file mode 100644index 0000000000..5428dd51b7--- /dev/null+++ b/tests/Feature/Api/BackupS3DestinationsApiTest.php@@ -0,0 +1,292 @@+<?php++use App\Models\Application;+use App\Models\Environment;+use App\Models\InstanceSettings;+use App\Models\LocalPersistentVolume;+use App\Models\Project;+use App\Models\S3Storage;+use App\Models\ScheduledDatabaseBackup;+use App\Models\ScheduledVolumeBackup;+use App\Models\Server;+use App\Models\StandaloneDocker;+use App\Models\StandalonePostgresql;+use App\Models\Team;+use App\Models\User;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Str;++uses(RefreshDatabase::class);++beforeEach(function () {+ Server::flushIdentityMap();+ config()->set('app.maintenance.store', 'array');+ InstanceSettings::forceCreate(['id' => 0, 'is_api_enabled' => true]);++ $this->team = Team::factory()->create();+ $user = User::factory()->create();+ $this->team->members()->attach($user->id, ['role' => 'owner']);++ $plainTextToken = Str::random(40);+ $token = $user->tokens()->create([+ 'name' => 's3-destinations-test',+ 'token' => hash('sha256', $plainTextToken),+ 'abilities' => ['*'],+ 'team_id' => $this->team->id,+ ]);+ $this->headers = ['Authorization' => 'Bearer '.$token->getKey().'|'.$plainTextToken];++ $server = Server::factory()->create(['team_id' => $this->team->id]);+ $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();+ $project = Project::factory()->create(['team_id' => $this->team->id]);+ $environment = Environment::factory()->create(['project_id' => $project->id]);++ $this->database = StandalonePostgresql::create([+ 'name' => 'db',+ 'postgres_user' => 'postgres',+ 'postgres_password' => 'password',+ 'postgres_db' => 'app',+ 'image' => 'postgres:16',+ 'environment_id' => $environment->id,+ 'destination_id' => $destination->id,+ 'destination_type' => $destination->getMorphClass(),+ ]);++ $this->application = Application::factory()->create([+ 'environment_id' => $environment->id,+ 'destination_id' => $destination->id,+ 'destination_type' => $destination->getMorphClass(),+ ]);+ $this->volume = LocalPersistentVolume::create([+ 'name' => 'data',+ 'mount_path' => '/data',+ 'resource_id' => $this->application->id,+ 'resource_type' => $this->application->getMorphClass(),+ ]);++ $this->firstS3 = createBackupDestinationS3($this->team, 'first');+ $this->secondS3 = createBackupDestinationS3($this->team, 'second');+ $this->otherTeamS3 = createBackupDestinationS3(Team::factory()->create(), 'foreign');+});++function createBackupDestinationS3(Team $team, string $name): S3Storage+{+ return S3Storage::create([+ 'name' => $name,+ 'region' => 'us-east-1',+ 'key' => 'key',+ 'secret' => 'secret',+ 'bucket' => $name.'-bucket',+ 'endpoint' => 'https://s3.example.com',+ 'team_id' => $team->id,+ 'is_usable' => true,+ ]);+}++/**+ * @return list<string>+ */+function backupDestinationUuids(ScheduledDatabaseBackup|ScheduledVolumeBackup $backup): array+{+ return $backup->refresh()->selectedS3Storages()->pluck('uuid')->all();+}++describe('database backups', function () {+ test('creates a backup with several destinations and lists them primary first', function () {+ $response = $this->withHeaders($this->headers)->postJson("/api/v1/databases/{$this->database->uuid}/backups", [+ 'frequency' => 'daily',+ 'save_s3' => true,+ 's3_storage_uuids' => [$this->firstS3->uuid, $this->secondS3->uuid],+ 's3_storage_uuid' => $this->secondS3->uuid,+ ])->assertCreated();++ $backup = ScheduledDatabaseBackup::where('uuid', $response->json('uuid'))->sole();+ expect($backup->s3_storage_id)->toBe($this->secondS3->id)+ ->and(backupDestinationUuids($backup))->toBe([$this->secondS3->uuid, $this->firstS3->uuid]);++ $this->withHeaders($this->headers)->getJson("/api/v1/databases/{$this->database->uuid}/backups")+ ->assertOk()+ ->assertJsonPath('0.s3_storage_uuid', $this->secondS3->uuid)+ ->assertJsonPath('0.s3_storage_uuids', [$this->secondS3->uuid, $this->firstS3->uuid])+ ->assertJsonMissingPath('0.s3_storages')+ ->assertJsonMissingPath('0.s3');+ });++ test('uses the first listed destination as primary when s3_storage_uuid is not sent', function () {+ $response = $this->withHeaders($this->headers)->postJson("/api/v1/databases/{$this->database->uuid}/backups", [+ 'frequency' => 'daily',+ 'save_s3' => true,+ 's3_storage_uuids' => [$this->secondS3->uuid, $this->firstS3->uuid],+ ])->assertCreated();++ expect(ScheduledDatabaseBackup::where('uuid', $response->json('uuid'))->sole()->s3_storage_id)->toBe($this->secondS3->id);+ });++ test('a single s3_storage_uuid that is already a destination becomes primary and keeps the others', function () {+ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => 'daily',+ 'save_s3' => true,+ 'database_id' => $this->database->id,+ 'database_type' => $this->database->getMorphClass(),+ 'team_id' => $this->team->id,+ ]);+ $backup->syncS3Storages([$this->firstS3->id, $this->secondS3->id]);++ $this->withHeaders($this->headers)->patchJson("/api/v1/databases/{$this->database->uuid}/backups/{$backup->uuid}", [+ 'save_s3' => true,+ 's3_storage_uuid' => $this->secondS3->uuid,+ ])->assertOk();++ expect(backupDestinationUuids($backup))->toBe([$this->secondS3->uuid, $this->firstS3->uuid])+ ->and($backup->s3_storage_id)->toBe($this->secondS3->id);+ });++ test('a single new s3_storage_uuid replaces every destination', function () {+ $thirdS3 = createBackupDestinationS3($this->team, 'third');+ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => 'daily',+ 'save_s3' => true,+ 'database_id' => $this->database->id,+ 'database_type' => $this->database->getMorphClass(),+ 'team_id' => $this->team->id,+ ]);+ $backup->syncS3Storages([$this->firstS3->id, $this->secondS3->id]);++ $this->withHeaders($this->headers)->patchJson("/api/v1/databases/{$this->database->uuid}/backups/{$backup->uuid}", [+ 's3_storage_uuid' => $thirdS3->uuid,+ ])->assertOk();++ expect(backupDestinationUuids($backup))->toBe([$thirdS3->uuid])+ ->and($backup->s3_storage_id)->toBe($thirdS3->id);+ });++ test('updates the destination list', function () {+ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => 'daily',+ 'database_id' => $this->database->id,+ 'database_type' => $this->database->getMorphClass(),+ 'team_id' => $this->team->id,+ ]);++ $this->withHeaders($this->headers)->patchJson("/api/v1/databases/{$this->database->uuid}/backups/{$backup->uuid}", [+ 'save_s3' => true,+ 's3_storage_uuids' => [$this->firstS3->uuid, $this->secondS3->uuid],+ ])->assertOk();++ expect(backupDestinationUuids($backup))->toBe([$this->firstS3->uuid, $this->secondS3->uuid])+ ->and((bool) $backup->save_s3)->toBeTrue();+ });++ test('rejects an S3 storage of another team without changing the backup', function () {+ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => 'daily',+ 'save_s3' => true,+ 's3_storage_id' => $this->firstS3->id,+ 'database_id' => $this->database->id,+ 'database_type' => $this->database->getMorphClass(),+ 'team_id' => $this->team->id,+ ]);+ $backup->syncS3Storages([$this->firstS3->id]);++ $this->withHeaders($this->headers)->patchJson("/api/v1/databases/{$this->database->uuid}/backups/{$backup->uuid}", [+ 's3_storage_uuids' => [$this->firstS3->uuid, $this->otherTeamS3->uuid],+ ])->assertUnprocessable()+ ->assertJsonValidationErrors(['s3_storage_uuids'])+ ->assertDontSee($this->otherTeamS3->name);++ expect(backupDestinationUuids($backup))->toBe([$this->firstS3->uuid]);+ });++ test('rejects invalid destination combinations', function (array $payload, string $errorField) {+ $payload = array_map(fn ($value) => is_array($value)+ ? array_map(fn (string $name) => $this->{$name}->uuid, $value)+ : (is_string($value) && property_exists($this, $value) ? $this->{$value}->uuid : $value), $payload);++ $this->withHeaders($this->headers)->postJson("/api/v1/databases/{$this->database->uuid}/backups", ['frequency' => 'daily', ...$payload])+ ->assertUnprocessable()+ ->assertJsonValidationErrors([$errorField]);++ expect(ScheduledDatabaseBackup::count())->toBe(0);+ })->with([+ 'primary outside the list' => [['s3_storage_uuids' => ['firstS3'], 's3_storage_uuid' => 'secondS3'], 's3_storage_uuid'],+ 'save_s3 without destinations' => [['save_s3' => true, 's3_storage_uuids' => []], 's3_storage_uuid'],+ 'foreign storage in the list' => [['save_s3' => true, 's3_storage_uuids' => ['firstS3', 'otherTeamS3']], 's3_storage_uuids'],+ ]);++ test('rejects clearing the destinations of a backup that saves to S3', function () {+ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => 'daily',+ 'save_s3' => true,+ 's3_storage_id' => $this->firstS3->id,+ 'database_id' => $this->database->id,+ 'database_type' => $this->database->getMorphClass(),+ 'team_id' => $this->team->id,+ ]);++ $this->withHeaders($this->headers)->patchJson("/api/v1/databases/{$this->database->uuid}/backups/{$backup->uuid}", [+ 's3_storage_uuids' => [],+ ])->assertUnprocessable();++ expect($backup->refresh()->s3_storage_id)->toBe($this->firstS3->id);+ });+});++describe('volume backups', function () {+ test('sets several destinations and returns them primary first', function () {+ $this->withHeaders($this->headers)+ ->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups", [+ 'frequency' => 'daily',+ 'save_s3' => true,+ 's3_storage_uuids' => [$this->firstS3->uuid, $this->secondS3->uuid],+ 's3_storage_uuid' => $this->secondS3->uuid,+ ])+ ->assertCreated()+ ->assertJsonPath('s3_storage_uuid', $this->secondS3->uuid)+ ->assertJsonPath('s3_storage_uuids', [$this->secondS3->uuid, $this->firstS3->uuid]);++ expect(backupDestinationUuids(ScheduledVolumeBackup::sole()))->toBe([$this->secondS3->uuid, $this->firstS3->uuid]);+ });++ test('a single s3_storage_uuid replaces every destination', function () {+ $url = "/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups";+ $this->withHeaders($this->headers)->putJson($url, [+ 'frequency' => 'daily',+ 'save_s3' => true,+ 's3_storage_uuids' => [$this->firstS3->uuid, $this->secondS3->uuid],+ ])->assertCreated();++ $this->withHeaders($this->headers)->putJson($url, [+ 'frequency' => 'daily',+ 'save_s3' => true,+ 's3_storage_uuid' => $this->secondS3->uuid,+ ])->assertOk()->assertJsonPath('s3_storage_uuids', [$this->secondS3->uuid]);++ expect(backupDestinationUuids(ScheduledVolumeBackup::sole()))->toBe([$this->secondS3->uuid]);+ });++ test('rejects an S3 storage of another team', function () {+ $this->withHeaders($this->headers)+ ->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups", [+ 'frequency' => 'daily',+ 'save_s3' => true,+ 's3_storage_uuids' => [$this->firstS3->uuid, $this->otherTeamS3->uuid],+ ])+ ->assertUnprocessable()+ ->assertJsonValidationErrors(['s3_storage_uuids'])+ ->assertDontSee($this->otherTeamS3->name);++ expect(ScheduledVolumeBackup::count())->toBe(0);+ });++ test('rejects a primary destination outside the list', function () {+ $this->withHeaders($this->headers)+ ->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups", [+ 'frequency' => 'daily',+ 'save_s3' => true,+ 's3_storage_uuids' => [$this->firstS3->uuid],+ 's3_storage_uuid' => $this->secondS3->uuid,+ ])+ ->assertUnprocessable()+ ->assertJsonValidationErrors(['s3_storage_uuid']);+ });+});diff --git a/tests/Feature/BackupEditValidationTest.php b/tests/Feature/BackupEditValidationTest.phpindex 66643ffc46..3479e8db11 100644--- a/tests/Feature/BackupEditValidationTest.php+++ b/tests/Feature/BackupEditValidationTest.php@@ -424,7 +424,7 @@ it('defaults to the first available storage when multiple storages are available]);Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s])- ->assertSet('s3StorageId', $firstS3->id)+ ->assertSet('s3StorageIds', [$firstS3->id])->set('saveS3', true)->call('instantSave')->assertDispatched('success');@@ -461,7 +461,7 @@ it('accepts a root team S3 storage for the instance database backup', function (Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => collect([$s3])])->set('saveS3', true)- ->set('s3StorageId', $s3->id)+ ->set('s3StorageIds', [$s3->id])->call('instantSave')->assertDispatched('success');@@ -548,7 +548,7 @@ it('saves selected S3 storage immediately when it changes', function () {]);Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s])- ->set('s3StorageId', $secondS3->id)+ ->set('s3StorageIds', [$secondS3->id])->assertDispatched('success');$backup->refresh();diff --git a/tests/Feature/BackupRetentionAndStaleDetectionTest.php b/tests/Feature/BackupRetentionAndStaleDetectionTest.phpindex 97e45599ff..8e235a6501 100644--- a/tests/Feature/BackupRetentionAndStaleDetectionTest.php+++ b/tests/Feature/BackupRetentionAndStaleDetectionTest.php@@ -276,6 +276,8 @@ test('database S3 retention remains best effort when deletion fails', function (]);ScheduledDatabaseBackupExecution::whereKey($newestExecution->id)->update(['created_at' => now()]);ScheduledDatabaseBackupExecution::whereKey($oldExecution->id)->update(['created_at' => now()->subDay()]);+ $newestExecution->s3Replicas()->create(['s3_storage_id' => $s3->id, 's3_uploaded' => true]);+ $oldExecution->s3Replicas()->create(['s3_storage_id' => $s3->id, 's3_uploaded' => true]);$disk = Mockery::mock();$disk->shouldReceive('delete')->once()->with(['/backup/old.dmp'])->andReturnFalse();Storage::shouldReceive('build')->once()->andReturn($disk);diff --git a/tests/Feature/BackupS3DestinationsFormTest.php b/tests/Feature/BackupS3DestinationsFormTest.phpnew file mode 100644index 0000000000..211360408a--- /dev/null+++ b/tests/Feature/BackupS3DestinationsFormTest.php@@ -0,0 +1,322 @@+<?php++use App\Livewire\Project\Database\BackupEdit;+use App\Livewire\Project\Database\BackupExecutions;+use App\Livewire\Project\Shared\Storages\VolumeBackups;+use App\Livewire\Storage\Resources as StorageResources;+use App\Models\Application;+use App\Models\Environment;+use App\Models\InstanceSettings;+use App\Models\LocalPersistentVolume;+use App\Models\PrivateKey;+use App\Models\Project;+use App\Models\S3Storage;+use App\Models\ScheduledDatabaseBackup;+use App\Models\ScheduledDatabaseBackupExecution;+use App\Models\ScheduledVolumeBackup;+use App\Models\ScheduledVolumeBackupExecution;+use App\Models\Server;+use App\Models\StandaloneDocker;+use App\Models\StandalonePostgresql;+use App\Models\Team;+use App\Models\User;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Livewire\Livewire;++uses(RefreshDatabase::class);++function createS3StorageForDestinationsFormTest(Team $team, string $name): S3Storage+{+ return S3Storage::create([+ 'name' => $name,+ 'region' => 'us-east-1',+ 'key' => 'key',+ 'secret' => 'secret',+ 'bucket' => str($name)->slug()->toString(),+ 'endpoint' => 'https://s3.example.com',+ 'is_usable' => true,+ 'team_id' => $team->id,+ ]);+}++/**+ * @return array{0: Application, 1: LocalPersistentVolume, 2: StandalonePostgresql}+ */+function createResourcesForDestinationsFormTest(Team $team): array+{+ $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);+ $server = Server::factory()->create(['team_id' => $team->id, 'private_key_id' => $privateKey->id]);+ $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();+ $project = Project::factory()->create(['team_id' => $team->id]);+ $environment = Environment::factory()->create(['project_id' => $project->id]);+ $application = Application::factory()->create([+ 'environment_id' => $environment->id,+ 'destination_id' => $destination->id,+ 'destination_type' => $destination->getMorphClass(),+ ]);+ $volume = LocalPersistentVolume::create([+ 'name' => 'app-data',+ 'mount_path' => '/data',+ 'resource_id' => $application->id,+ 'resource_type' => $application->getMorphClass(),+ ]);+ $database = StandalonePostgresql::create([+ 'name' => 'pg-destinations',+ 'image' => 'postgres:16-alpine',+ 'postgres_user' => 'postgres',+ 'postgres_password' => 'password',+ 'postgres_db' => 'postgres',+ 'environment_id' => $environment->id,+ 'destination_id' => $destination->id,+ 'destination_type' => $destination->getMorphClass(),+ ]);++ return [$application, $volume, $database];+}++function createDatabaseBackupForDestinationsFormTest(StandalonePostgresql $database, array $overrides = []): ScheduledDatabaseBackup+{+ return ScheduledDatabaseBackup::create([+ 'frequency' => '0 0 * * *',+ 'save_s3' => false,+ 'database_type' => $database->getMorphClass(),+ 'database_id' => $database->id,+ 'team_id' => $database->team()->id,+ ...$overrides,+ ]);+}++beforeEach(function () {+ Server::flushIdentityMap();+ InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));++ $this->team = Team::factory()->create();+ $this->otherTeam = Team::factory()->create();+ $this->user = User::factory()->create();+ $this->user->teams()->attach($this->team, ['role' => 'admin']);+ $this->user->teams()->attach($this->otherTeam, ['role' => 'owner']);+ $this->actingAs($this->user);+ session(['currentTeam' => $this->team]);++ [$this->application, $this->volume, $this->database] = createResourcesForDestinationsFormTest($this->team);+ $this->firstStorage = createS3StorageForDestinationsFormTest($this->team, 'First S3');+ $this->secondStorage = createS3StorageForDestinationsFormTest($this->team, 'Second S3');+ $this->foreignStorage = createS3StorageForDestinationsFormTest($this->otherTeam, 'Foreign S3');+});++it('saves two S3 destinations for a database backup schedule', function () {+ $backup = createDatabaseBackupForDestinationsFormTest($this->database);++ Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s, 'section' => 's3'])+ ->assertSee('First S3')+ ->assertSee('Second S3')+ ->assertSet('s3StorageIds', [$this->firstStorage->id])+ ->set('saveS3', true)+ ->set('s3StorageIds', [(string) $this->firstStorage->id, (string) $this->secondStorage->id])+ ->call('submit')+ ->assertHasNoErrors()+ ->assertDispatched('success');++ $backup->refresh();+ expect($backup->save_s3)->toBeTruthy()+ ->and($backup->s3_storage_id)->toBe($this->firstStorage->id)+ ->and($backup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all())+ ->toBe([$this->firstStorage->id, $this->secondStorage->id]);+});++it('saves two S3 destinations for a volume backup schedule', function () {+ Livewire::test(VolumeBackups::class, ['storage' => $this->volume, 'resource' => $this->application])+ ->set('frequency', 'daily')+ ->set('saveToS3', true)+ ->set('s3StorageIds', [(string) $this->secondStorage->id, (string) $this->firstStorage->id])+ ->assertHasNoErrors()+ ->assertDispatched('success');++ $backup = ScheduledVolumeBackup::query()->sole();+ expect($backup->save_s3)->toBeTrue()+ ->and($backup->s3_storage_id)->toBe($this->secondStorage->id)+ ->and($backup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all())+ ->toBe([$this->firstStorage->id, $this->secondStorage->id]);+});++it('drops S3 storages of another team after a team switch', function () {+ $backup = createDatabaseBackupForDestinationsFormTest($this->database, [+ 'save_s3' => true,+ 's3_storage_id' => $this->firstStorage->id,+ ]);+ $backup->syncS3Storages([$this->firstStorage->id]);++ $databaseComponent = Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s]);+ $volumeComponent = Livewire::test(VolumeBackups::class, ['storage' => $this->volume, 'resource' => $this->application]);+ session(['currentTeam' => $this->otherTeam]);++ $databaseComponent->set('s3StorageIds', [$this->foreignStorage->id])+ ->assertHasErrors('s3StorageIds');+ expect($backup->fresh()->s3Storages()->pluck('s3_storages.id')->all())->toBe([$this->firstStorage->id]);++ $databaseComponent->set('s3StorageIds', [$this->secondStorage->id, $this->foreignStorage->id])+ ->assertHasNoErrors();+ expect($backup->fresh()->s3_storage_id)->toBe($this->secondStorage->id)+ ->and($backup->fresh()->s3Storages()->pluck('s3_storages.id')->all())->toBe([$this->secondStorage->id]);++ $volumeComponent->set('frequency', 'daily')+ ->set('saveToS3', true)+ ->set('s3StorageIds', [$this->foreignStorage->id])+ ->assertHasErrors('s3StorageIds');+ expect(ScheduledVolumeBackup::query()->count())->toBe(0);++ $volumeComponent->set('s3StorageIds', [$this->firstStorage->id, $this->foreignStorage->id])+ ->assertHasNoErrors();+ expect(ScheduledVolumeBackup::query()->sole()->s3Storages()->pluck('s3_storages.id')->all())+ ->toBe([$this->firstStorage->id]);+});++it('rejects S3 backups without a destination', function () {+ $backup = createDatabaseBackupForDestinationsFormTest($this->database, [+ 'save_s3' => true,+ 's3_storage_id' => $this->firstStorage->id,+ ]);+ $backup->syncS3Storages([$this->firstStorage->id]);++ Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s])+ ->set('s3StorageIds', [])+ ->call('submit')+ ->assertHasErrors('s3StorageIds')+ ->assertDispatched('error');++ expect($backup->fresh()->save_s3)->toBeTruthy()+ ->and($backup->fresh()->s3Storages()->pluck('s3_storages.id')->all())->toBe([$this->firstStorage->id]);++ Livewire::test(VolumeBackups::class, ['storage' => $this->volume, 'resource' => $this->application])+ ->set('frequency', 'daily')+ ->set('saveToS3', true)+ ->set('s3StorageIds', [])+ ->call('save')+ ->assertHasErrors('s3StorageIds');++ expect(ScheduledVolumeBackup::query()->count())->toBe(0);+});++it('moves and removes one destination of a multi-destination schedule from the storage page', function () {+ $thirdStorage = createS3StorageForDestinationsFormTest($this->team, 'Third S3');+ $databaseBackup = createDatabaseBackupForDestinationsFormTest($this->database, ['save_s3' => true]);+ $databaseBackup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]);+ $volumeBackup = ScheduledVolumeBackup::create([+ 'backupable_type' => $this->volume->getMorphClass(),+ 'backupable_id' => $this->volume->id,+ 'team_id' => $this->team->id,+ 'frequency' => 'daily',+ 'enabled' => true,+ 'save_s3' => true,+ ]);+ $volumeBackup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]);++ Livewire::test(StorageResources::class, ['storage' => $this->secondStorage])+ ->assertSet("selectedStorages.{$databaseBackup->id}", $this->secondStorage->id)+ ->assertSet("selectedVolumeStorages.{$volumeBackup->id}", $this->secondStorage->id)+ ->assertSee('pg-destinations')+ ->assertSee('app-data');++ Livewire::test(StorageResources::class, ['storage' => $this->firstStorage])+ ->set("selectedStorages.{$databaseBackup->id}", $thirdStorage->id)+ ->call('moveBackup', $databaseBackup->id)+ ->set("selectedVolumeStorages.{$volumeBackup->id}", $thirdStorage->id)+ ->call('moveVolumeBackup', $volumeBackup->id);++ foreach ([$databaseBackup, $volumeBackup] as $backup) {+ $backup->refresh();+ expect($backup->s3_storage_id)->toBe($thirdStorage->id)+ ->and($backup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all())+ ->toBe([$this->secondStorage->id, $thirdStorage->id]);+ }++ Livewire::test(StorageResources::class, ['storage' => $this->secondStorage])+ ->call('disableS3', $databaseBackup->id)+ ->call('disableVolumeS3', $volumeBackup->id);++ foreach ([$databaseBackup, $volumeBackup] as $backup) {+ $backup->refresh();+ expect($backup->save_s3)->toBeTruthy()+ ->and($backup->s3_storage_id)->toBe($thirdStorage->id)+ ->and($backup->s3Storages()->pluck('s3_storages.id')->all())->toBe([$thirdStorage->id]);+ }++ Livewire::test(StorageResources::class, ['storage' => $thirdStorage])+ ->call('disableS3', $databaseBackup->id)+ ->call('disableVolumeS3', $volumeBackup->id);++ foreach ([$databaseBackup, $volumeBackup] as $backup) {+ $backup->refresh();+ expect($backup->save_s3)->toBeFalsy()+ ->and($backup->s3_storage_id)->toBeNull()+ ->and($backup->s3Storages()->count())->toBe(0);+ }+});++it('shows the copy status of each destination in the execution lists', function () {+ $databaseBackup = createDatabaseBackupForDestinationsFormTest($this->database, ['save_s3' => true]);+ $databaseBackup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]);+ $databaseExecution = ScheduledDatabaseBackupExecution::create([+ 'scheduled_database_backup_id' => $databaseBackup->id,+ 'status' => 'success',+ 'filename' => '/backups/dump.dmp',+ 's3_uploaded' => false,+ ]);+ $volumeBackup = ScheduledVolumeBackup::create([+ 'backupable_type' => $this->volume->getMorphClass(),+ 'backupable_id' => $this->volume->id,+ 'team_id' => $this->team->id,+ 'frequency' => 'daily',+ 'save_s3' => true,+ ]);+ $volumeBackup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]);+ $volumeExecution = ScheduledVolumeBackupExecution::create([+ 'scheduled_volume_backup_id' => $volumeBackup->id,+ 'status' => 'success',+ 'filename' => '/backups/volume.tar.gz',+ 's3_uploaded' => false,+ ]);+ foreach ([$databaseExecution, $volumeExecution] as $execution) {+ $execution->s3Replicas()->create(['s3_storage_id' => $this->firstStorage->id, 's3_uploaded' => true]);+ $execution->s3Replicas()->create(['s3_storage_id' => $this->secondStorage->id, 's3_uploaded' => false, 'message' => 'Access denied']);+ }++ Livewire::test(BackupExecutions::class, ['backup' => $databaseBackup])+ ->assertSeeInOrder(['First S3 Available', 'Second S3 Failed'])+ ->assertSee('Access denied')+ ->assertSee('Delete the selected backup permanently from every S3 storage');++ Livewire::test(VolumeBackups::class, ['storage' => $this->volume, 'resource' => $this->application, 'section' => 'executions'])+ ->assertSeeInOrder(['First S3', 'Second S3 failed'])+ ->assertSee('Delete the selected backup permanently from every S3 storage');+});++it('keeps a selected destination that is unusable for a while when other settings are saved', function () {+ $databaseBackup = createDatabaseBackupForDestinationsFormTest($this->database, ['save_s3' => true]);+ $databaseBackup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]);+ $volumeBackup = ScheduledVolumeBackup::create([+ 'backupable_type' => $this->volume->getMorphClass(),+ 'backupable_id' => $this->volume->id,+ 'team_id' => $this->team->id,+ 'frequency' => '0 0 * * *',+ 'save_s3' => true,+ ]);+ $volumeBackup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]);+ $this->secondStorage->update(['is_usable' => false]);++ Livewire::test(BackupEdit::class, ['backup' => $databaseBackup->fresh(), 'availableS3Storages' => $this->team->s3s, 'section' => 's3'])+ ->assertSee('Second S3 (unavailable)')+ ->set('timeout', 7200)+ ->call('submit')+ ->assertHasNoErrors();+ Livewire::test(VolumeBackups::class, ['storage' => $this->volume, 'resource' => $this->application, 'section' => 's3'])+ ->assertSee('Second S3 (unavailable)')+ ->set('stopDuringBackup', true)+ ->call('save')+ ->assertHasNoErrors();++ expect($databaseBackup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all())+ ->toBe([$this->firstStorage->id, $this->secondStorage->id])+ ->and($volumeBackup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all())+ ->toBe([$this->firstStorage->id, $this->secondStorage->id]);+});diff --git a/tests/Feature/DatabaseBackupMultipleS3DestinationsTest.php b/tests/Feature/DatabaseBackupMultipleS3DestinationsTest.phpnew file mode 100644index 0000000000..0191aa09d7--- /dev/null+++ b/tests/Feature/DatabaseBackupMultipleS3DestinationsTest.php@@ -0,0 +1,434 @@+<?php++use App\Events\BackupCreated;+use App\Jobs\DatabaseBackupJob;+use App\Models\DatabaseBackupS3Replica;+use App\Models\Environment;+use App\Models\InstanceSettings;+use App\Models\PrivateKey;+use App\Models\Project;+use App\Models\S3Storage;+use App\Models\ScheduledDatabaseBackup;+use App\Models\ScheduledDatabaseBackupExecution;+use App\Models\Server;+use App\Models\StandaloneDocker;+use App\Models\Team;+use App\Models\User;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Collection;+use Illuminate\Support\Facades\Event;+use Illuminate\Support\Facades\Notification;+use Illuminate\Support\Facades\Process;+use Illuminate\Support\Facades\Storage;++uses(RefreshDatabase::class);++beforeEach(function () {+ Server::flushIdentityMap();+});++function multiS3Storage(Team $team, string $name): S3Storage+{+ return S3Storage::create([+ 'name' => $name,+ 'region' => 'us-east-1',+ 'key' => 'key',+ 'secret' => 'secret',+ 'bucket' => strtolower($name).'-bucket',+ 'endpoint' => 'https://s3.example.com',+ 'team_id' => $team->id,+ ]);+}++function multiS3Execution(ScheduledDatabaseBackup $backup, string $name, array $attributes = []): ScheduledDatabaseBackupExecution+{+ $execution = ScheduledDatabaseBackupExecution::create([+ 'uuid' => $name,+ 'database_name' => 'app',+ 'filename' => "/backup/{$name}.dmp",+ 'scheduled_database_backup_id' => $backup->id,+ 'status' => 'success',+ 's3_uploaded' => true,+ 'local_storage_deleted' => true,+ ...$attributes,+ ]);++ return $execution;+}++/**+ * Records the files that deleteBackupsS3() removes, keyed by bucket.+ */+function fakeMultiS3Deletes(): Collection+{+ $deleted = collect();+ Storage::shouldReceive('build')->andReturnUsing(function (array $config) use ($deleted) {+ $disk = Mockery::mock();+ $disk->shouldReceive('delete')->andReturnUsing(function (array $files) use ($config, $deleted) {+ $deleted->put($config['bucket'], [...$deleted->get($config['bucket'], []), ...$files]);++ return true;+ });++ return $disk;+ });++ return $deleted;+}++/**+ * Runs a backup with two S3 destinations, failing the upload to the storages named in $failingStorages.+ *+ * @return array{0: ScheduledDatabaseBackupExecution, 1: Collection<int, string>}+ */+function runMultiS3BackupJob(array $failingStorages): array+{+ $team = Team::factory()->create();+ $server = Server::factory()->create([+ 'team_id' => $team->id,+ 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id,+ ]);+ $server->settings()->update(['is_reachable' => true, 'is_usable' => true]);+ $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();+ $environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $team->id])->id]);+ $database = create_standalone_postgresql($environment->id, $destination, ['status' => 'running:healthy']);+ $first = multiS3Storage($team, 'First');+ $second = multiS3Storage($team, 'Second');+ $backup = new ScheduledDatabaseBackup;+ $backup->forceFill([+ 'enabled' => true,+ 'save_s3' => true,+ 'disable_local_backup' => true,+ 's3_storage_id' => $first->id,+ 'frequency' => 'daily',+ 'databases_to_backup' => 'app',+ 'database_id' => $database->id,+ 'database_type' => $database->getMorphClass(),+ 'team_id' => $team->id,+ ])->save();+ $backup->syncS3Storages([$first->id, $second->id]);+ $commands = collect();+ Process::fake(function ($process) use ($commands) {+ $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;+ $commands->push($command);++ return Process::result(output: str_contains($command, 'du -b') ? '128' : '');+ });++ $job = Mockery::mock(DatabaseBackupJob::class.'[upload_to_s3]', [$backup->fresh()])->shouldAllowMockingProtectedMethods();+ $job->shouldReceive('upload_to_s3')->andReturnUsing(function (S3Storage $storage) use ($failingStorages) {+ if (in_array($storage->name, $failingStorages, true)) {+ throw new RuntimeException('access denied');+ }+ });+ $job->handle();++ return [ScheduledDatabaseBackupExecution::query()->sole(), $commands];+}++it('applies S3 retention to each destination separately', function () {+ $team = Team::factory()->create();+ $first = multiS3Storage($team, 'First');+ $second = multiS3Storage($team, 'Second');+ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => '0 0 * * *',+ 'save_s3' => true,+ 'disable_local_backup' => true,+ 'database_type' => 'App\Models\StandalonePostgresql',+ 'database_id' => 1,+ 'team_id' => $team->id,+ 'database_backup_retention_amount_s3' => 1,+ ]);+ $backup->syncS3Storages([$first->id, $second->id]);+ $old = multiS3Execution($backup, 'old');+ $new = multiS3Execution($backup, 'new', ['s3_uploaded' => false]);+ ScheduledDatabaseBackupExecution::whereKey($old->id)->update(['created_at' => now()->subDay()]);+ $old->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]);+ $old->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => true]);+ $new->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]);+ $new->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => false, 'message' => 'denied']);+ $deleted = fakeMultiS3Deletes();++ removeOldBackups($backup);++ expect($deleted->all())->toBe(['first-bucket' => ['/backup/old.dmp']])+ ->and($old->s3Replicas()->where('s3_storage_id', $first->id)->sole()->s3_storage_deleted)->toBeTrue()+ ->and($old->s3Replicas()->where('s3_storage_id', $second->id)->sole()->s3_storage_deleted)->toBeFalse()+ ->and($old->fresh()->s3_storage_deleted)->toBeFalse()+ ->and($backup->executions()->count())->toBe(2);+});++it('deletes the execution record when its last S3 copy expires', function () {+ $team = Team::factory()->create();+ $first = multiS3Storage($team, 'First');+ $second = multiS3Storage($team, 'Second');+ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => '0 0 * * *',+ 'save_s3' => true,+ 'disable_local_backup' => true,+ 'database_type' => 'App\Models\StandalonePostgresql',+ 'database_id' => 1,+ 'team_id' => $team->id,+ 'database_backup_retention_amount_s3' => 1,+ ]);+ $backup->syncS3Storages([$first->id, $second->id]);+ $old = multiS3Execution($backup, 'old');+ $new = multiS3Execution($backup, 'new');+ ScheduledDatabaseBackupExecution::whereKey($old->id)->update(['created_at' => now()->subDay()]);+ foreach ([$old, $new] as $execution) {+ $execution->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]);+ $execution->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => true]);+ }+ $deleted = fakeMultiS3Deletes();++ removeOldBackups($backup);++ expect($deleted->all())->toBe(['first-bucket' => ['/backup/old.dmp'], 'second-bucket' => ['/backup/old.dmp']])+ ->and($old->fresh())->toBeNull()+ ->and(DatabaseBackupS3Replica::where('execution_id', $old->id)->exists())->toBeFalse()+ ->and($new->fresh())->not->toBeNull();+});++it('marks copies in a deleted storage as gone without calling S3', function () {+ $team = Team::factory()->create();+ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => '0 0 * * *',+ 'save_s3' => true,+ 'disable_local_backup' => true,+ 'database_type' => 'App\Models\StandalonePostgresql',+ 'database_id' => 1,+ 'team_id' => $team->id,+ ]);+ $execution = multiS3Execution($backup, 'orphaned');+ $execution->s3Replicas()->create(['s3_storage_id' => null, 's3_uploaded' => true]);+ Storage::shouldReceive('build')->never();++ removeOldBackups($backup);++ expect($execution->fresh())->toBeNull();+});++it('keeps an execution whose local file still exists after its S3 copies expire', function () {+ $team = Team::factory()->create();+ $storage = multiS3Storage($team, 'First');+ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => '0 0 * * *',+ 'save_s3' => true,+ 'database_type' => 'App\Models\StandalonePostgresql',+ 'database_id' => 1,+ 'team_id' => $team->id,+ ]);+ $execution = multiS3Execution($backup, 'local-copy', ['local_storage_deleted' => false]);+ $execution->s3Replicas()->create(['s3_storage_id' => $storage->id, 's3_uploaded' => true, 's3_storage_deleted' => true]);++ removeOldBackups($backup);++ expect($execution->fresh())->not->toBeNull();+});++it('deletes the S3 copies of an execution from every destination that has one', function () {+ $team = Team::factory()->create();+ $first = multiS3Storage($team, 'First');+ $second = multiS3Storage($team, 'Second');+ $failed = multiS3Storage($team, 'Failed');+ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => '0 0 * * *',+ 'save_s3' => true,+ 'database_type' => 'App\Models\StandalonePostgresql',+ 'database_id' => 1,+ 'team_id' => $team->id,+ ]);+ $execution = multiS3Execution($backup, 'manual', ['s3_uploaded' => false]);+ $execution->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]);+ $execution->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => true]);+ $execution->s3Replicas()->create(['s3_storage_id' => $failed->id, 's3_uploaded' => false]);+ $deleted = fakeMultiS3Deletes();++ $execution->deleteS3Copies();++ expect($deleted->keys()->sort()->values()->all())->toBe(['first-bucket', 'second-bucket'])+ ->and($execution->fresh()->s3_storage_deleted)->toBeTrue()+ ->and($execution->hasLiveS3Copies())->toBeFalse();+});++describe('backup job', function () {+ beforeEach(function () {+ Storage::fake('ssh-keys');+ Storage::fake('ssh-mux');+ Event::fake([BackupCreated::class]);+ Notification::fake();+ InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));+ });++ it('uploads to every destination and deletes the local file when all uploads succeed', function () {+ [$execution, $commands] = runMultiS3BackupJob([]);++ expect($execution->status)->toBe('success')+ ->and($execution->s3_uploaded)->toBeTrue()+ ->and($execution->local_storage_deleted)->toBeTrue()+ ->and($execution->s3Replicas()->pluck('s3_uploaded')->all())->toBe([true, true])+ ->and($commands->contains(fn (string $command) => str_contains($command, 'rm -f') && str_contains($command, $execution->filename)))->toBeTrue();+ });++ it('continues to later destinations and keeps the local file when a replica write fails', function (array $failingStorages) {+ $event = 'eloquent.creating: '.DatabaseBackupS3Replica::class;+ Event::listen($event, function (DatabaseBackupS3Replica $replica): void {+ if (S3Storage::find($replica->s3_storage_id)?->name === 'First') {+ throw new RuntimeException('replica persistence failed');+ }+ });++ try {+ [$execution, $commands] = runMultiS3BackupJob($failingStorages);+ } finally {+ Event::forget($event);+ }++ $replica = $execution->s3Replicas()->with('s3')->sole();+ expect($replica->s3->name)->toBe('Second')+ ->and($replica->s3_uploaded)->toBeTrue()+ ->and($execution->s3_storage_deleted)->toBeFalse()+ ->and($execution->s3_uploaded)->toBeFalse()+ ->and($execution->local_storage_deleted)->toBeFalse()+ ->and($execution->message)->toContain('First: replica persistence failed')+ ->and($commands->contains(fn (string $command) => str_contains($command, 'rm -f') && str_contains($command, $execution->filename)))->toBeFalse();++ if ($failingStorages !== []) {+ expect($execution->message)->toContain('First: access denied');+ }+ })->with([+ 'successful upload' => [[]],+ 'failed upload' => [['First']],+ ]);++ it('keeps the local file and names the failed destination when one upload fails', function () {+ [$execution, $commands] = runMultiS3BackupJob(['Second']);++ $replicas = $execution->s3Replicas()->with('s3')->get()->keyBy(fn ($replica) => $replica->s3->name);+ expect($execution->status)->toBe('success')+ ->and($execution->s3_uploaded)->toBeFalse()+ ->and($execution->local_storage_deleted)->toBeFalse()+ ->and($execution->message)->toContain('Second: access denied')+ ->and($replicas['First']->s3_uploaded)->toBeTrue()+ ->and($replicas['Second']->s3_uploaded)->toBeFalse()+ ->and($replicas['Second']->message)->toBe('access denied')+ ->and($commands->contains(fn (string $command) => str_contains($command, 'rm -f') && str_contains($command, $execution->filename)))->toBeFalse();+ });+});++it('deletes an execution from every S3 destination through the API', function () {+ InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));+ $sshKeys = Storage::fake('ssh-keys');+ $sshMux = Storage::fake('ssh-mux');+ Process::fake();+ $team = Team::factory()->create();+ $user = User::factory()->create();+ $team->members()->attach($user, ['role' => 'owner']);+ session(['currentTeam' => $team]);+ $token = $user->createToken('test-token', ['*'])->plainTextToken;+ $server = Server::factory()->create([+ 'team_id' => $team->id,+ 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id,+ ]);+ $server->settings()->update(['is_reachable' => true, 'is_usable' => true]);+ $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();+ $environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $team->id])->id]);+ $database = create_standalone_postgresql($environment->id, $destination);+ $first = multiS3Storage($team, 'First');+ $second = multiS3Storage($team, 'Second');+ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => '0 0 * * *',+ 'save_s3' => true,+ 'database_type' => $database->getMorphClass(),+ 'database_id' => $database->id,+ 'team_id' => $team->id,+ ]);+ $backup->syncS3Storages([$first->id, $second->id]);+ $execution = multiS3Execution($backup, 'api', ['local_storage_deleted' => false]);+ $execution->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]);+ $execution->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => true]);+ $deleted = fakeMultiS3Deletes();+ Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshKeys);+ Storage::shouldReceive('disk')->with('ssh-mux')->andReturn($sshMux);++ $this->withHeaders(['Authorization' => 'Bearer '.$token])+ ->deleteJson("/api/v1/databases/{$database->uuid}/backups/{$backup->uuid}/executions/{$execution->uuid}?delete_s3=true")+ ->assertOk();++ expect($deleted->all())->toBe(['first-bucket' => ['/backup/api.dmp'], 'second-bucket' => ['/backup/api.dmp']])+ ->and($execution->fresh())->toBeNull();+});++it('copies the destinations of the same team to a cloned schedule', function () {+ $team = Team::factory()->create();+ $otherTeam = Team::factory()->create();+ $first = multiS3Storage($team, 'First');+ $second = multiS3Storage($team, 'Second');+ $foreign = multiS3Storage($otherTeam, 'Foreign');+ $attributes = [+ 'frequency' => '0 0 * * *',+ 'save_s3' => true,+ 'database_type' => 'App\Models\StandalonePostgresql',+ 'database_id' => 1,+ 'team_id' => $team->id,+ ];+ $backup = ScheduledDatabaseBackup::create($attributes);+ $backup->syncS3Storages([$first->id, $second->id]);+ $backup->s3Storages()->attach($foreign->id);+ $copy = ScheduledDatabaseBackup::create($attributes);++ $backup->copyS3StoragesTo($copy);++ expect($copy->s3Storages()->pluck('s3_storages.id')->sort()->values()->all())->toBe([$first->id, $second->id]);+});++it('requires a live S3 copy for local retention only when local backups are disabled', function (bool $disabled, ?array $replica, bool $deleted) {+ $team = Team::factory()->create();+ $server = Server::factory()->create([+ 'team_id' => $team->id,+ 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id,+ ]);+ $server->settings()->update(['is_reachable' => true, 'is_usable' => true]);+ $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();+ $environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $team->id])->id]);+ $database = create_standalone_postgresql($environment->id, $destination);+ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => '0 0 * * *',+ 'save_s3' => true,+ 'disable_local_backup' => $disabled,+ 'database_backup_retention_amount_locally' => 1,+ 'database_type' => $database->getMorphClass(),+ 'database_id' => $database->id,+ 'team_id' => $team->id,+ ]);+ $older = multiS3Execution($backup, 'older', ['local_storage_deleted' => false, 'created_at' => now()->subDay()]);+ $newer = multiS3Execution($backup, 'newer', ['local_storage_deleted' => false]);+ if ($replica !== null) {+ $storage = multiS3Storage($team, 'Retention');+ foreach ([$older, $newer] as $execution) {+ $execution->s3Replicas()->create(['s3_storage_id' => $storage->id, ...$replica]);+ }+ }+ $commands = collect();+ Process::fake(function ($process) use ($commands) {+ $commands->push(is_array($process->command) ? implode(' ', $process->command) : $process->command);++ return Process::result();+ });++ removeOldBackups($backup);++ expect($commands->contains(fn (string $command): bool => str_contains($command, '/backup/older.dmp')))->toBe($deleted);+ if ($deleted && $replica === null) {+ $this->assertModelMissing($older);+ } else {+ expect($older->fresh()->local_storage_deleted)->toBe($deleted);+ }+ expect($newer->fresh()->local_storage_deleted)->toBeFalse();+})->with([+ 'no replica' => [true, null, false],+ 'failed upload' => [true, ['s3_uploaded' => false], false],+ 'deleted replica' => [true, ['s3_uploaded' => true, 's3_storage_deleted' => true], false],+ 'live replica' => [true, ['s3_uploaded' => true, 's3_storage_deleted' => false], true],+ 'normal local retention' => [false, null, true],+]);diff --git a/tests/Feature/DevelopmentQemuVmTest.php b/tests/Feature/DevelopmentQemuVmTest.phpindex ad6d1d76d0..864536cfa7 100644--- a/tests/Feature/DevelopmentQemuVmTest.php+++ b/tests/Feature/DevelopmentQemuVmTest.php@@ -173,16 +173,41 @@ it('can change the localhost qemu profile without adding a server', function ()it('seeds localhost through the container command', function () {$this->seed(ServerSeeder::class);+ Process::fake();expect(Artisan::call('dev:qemu:seed', ['profile' => 'ubuntu-root','--as-localhost' => true,]))->toBe(Command::SUCCESS)->and(Server::query()->findOrFail(0)->ip)->toBe('192.168.122.10');++ Process::assertRan(fn ($process) => str_contains($process->command, 'docker run -d --name coolify-db')+ && str_contains($process->command, 'POSTGRES_DB='));+});++it('starts the instance database container with sudo on a non-root localhost vm', function () {+ $this->seed(ServerSeeder::class);+ Process::fake();++ Artisan::call('dev:qemu:seed', ['profile' => 'ubuntu-non-root', '--as-localhost' => true]);++ Process::assertRan(fn ($process) => str_contains($process->command, 'if ! sudo docker inspect coolify-db')+ && str_contains($process->command, 'sudo docker run -d --name coolify-db')+ && str_contains($process->command, 'sudo docker start coolify-db'));+});++it('still seeds localhost when the instance database container cannot start', function () {+ $this->seed(ServerSeeder::class);+ Process::fake(fn () => Process::result(errorOutput: 'Cannot connect to the Docker daemon', exitCode: 1));++ expect(Artisan::call('dev:qemu:seed', ['profile' => 'ubuntu-root', '--as-localhost' => true]))->toBe(Command::SUCCESS)+ ->and(Artisan::output())->toContain('Could not start the coolify-db container')+ ->and(Server::query()->findOrFail(0)->ip)->toBe('192.168.122.10');});it('seeds localhost with the forwarded ssh address of a lima vm', function () {$this->seed(ServerSeeder::class);+ Process::fake();expect(Artisan::call('dev:qemu:seed', ['profile' => 'debian-non-root',diff --git a/tests/Feature/InstanceBackupMultipleS3DestinationsTest.php b/tests/Feature/InstanceBackupMultipleS3DestinationsTest.phpnew file mode 100644index 0000000000..2cbac04a38--- /dev/null+++ b/tests/Feature/InstanceBackupMultipleS3DestinationsTest.php@@ -0,0 +1,137 @@+<?php++use App\Jobs\DatabaseBackupJob;+use App\Livewire\Project\Database\BackupEdit;+use App\Livewire\SettingsBackup;+use App\Livewire\Storage\Resources as StorageResources;+use App\Models\InstanceSettings;+use App\Models\PrivateKey;+use App\Models\S3Storage;+use App\Models\ScheduledDatabaseBackup;+use App\Models\ScheduledDatabaseBackupExecution;+use App\Models\Server;+use App\Models\StandaloneDocker;+use App\Models\StandalonePostgresql;+use App\Models\Team;+use App\Models\User;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Facades\Process;+use Illuminate\Support\Once;+use Livewire\Livewire;++uses(RefreshDatabase::class);++function instanceBackupS3Storage(Team $team, string $name): S3Storage+{+ return S3Storage::create([+ 'name' => $name,+ 'region' => 'us-east-1',+ 'key' => 'key',+ 'secret' => 'secret',+ 'bucket' => str($name)->slug()->toString(),+ 'endpoint' => 'https://s3.example.com',+ 'is_usable' => true,+ 'team_id' => $team->id,+ ]);+}++beforeEach(function () {+ Server::flushIdentityMap();+ $this->rootTeam = Team::find(0) ?? Team::factory()->create(['id' => 0]);+ $server = Server::factory()->create([+ 'id' => 0,+ 'team_id' => 0,+ 'ip' => '127.0.0.1',+ 'private_key_id' => PrivateKey::factory()->create(['team_id' => 0])->id,+ ]);+ $server->settings()->update(['is_reachable' => true, 'is_usable' => true, 'force_disabled' => false]);+ StandaloneDocker::query()->where('server_id', $server->id)->firstOrFail()->forceFill(['id' => 0])->save();+ InstanceSettings::forceCreate(['id' => 0]);+ Once::flush();++ $this->database = new StandalonePostgresql;+ $this->database->forceFill([+ 'id' => 0,+ 'name' => 'coolify-db',+ 'postgres_user' => 'coolify',+ 'postgres_password' => 'password',+ 'postgres_db' => 'coolify',+ 'status' => 'running',+ 'destination_type' => StandaloneDocker::class,+ 'destination_id' => 0,+ ])->save();+ $this->backup = ScheduledDatabaseBackup::create([+ 'enabled' => true,+ 'save_s3' => false,+ 'frequency' => '0 0 * * *',+ 'database_id' => 0,+ 'database_type' => StandalonePostgresql::class,+ 'team_id' => 0,+ ]);++ $this->firstStorage = instanceBackupS3Storage($this->rootTeam, 'First S3');+ $this->secondStorage = instanceBackupS3Storage($this->rootTeam, 'Second S3');+ $this->otherTeam = Team::factory()->create();+ $this->foreignStorage = instanceBackupS3Storage($this->otherTeam, 'Foreign S3');++ $user = User::factory()->create();+ $this->rootTeam->members()->attach($user->id, ['role' => 'admin']);+ $user->teams()->attach($this->otherTeam->id, ['role' => 'owner']);+ $this->actingAs($user);+ session(['currentTeam' => ['id' => 0]]);+});++it('offers every root team S3 storage on the instance backup page', function () {+ Livewire::test(SettingsBackup::class)+ ->assertOk()+ ->assertSee('First S3')+ ->assertSee('Second S3')+ ->assertDontSee('Foreign S3');+});++it('saves several S3 destinations for the instance backup and drops storages of other teams', function () {+ // The session team can change in another tab after the page loads.+ session(['currentTeam' => ['id' => $this->otherTeam->id]]);++ Livewire::test(BackupEdit::class, ['backup' => $this->backup->fresh(), 'availableS3Storages' => collect(), 'section' => 's3'])+ ->set('saveS3', true)+ ->set('s3StorageIds', [$this->firstStorage->id, $this->secondStorage->id, $this->foreignStorage->id])+ ->call('submit')+ ->assertHasNoErrors()+ ->assertDispatched('success');++ $backup = $this->backup->fresh();++ expect($backup->save_s3)->toBeTruthy()+ ->and($backup->s3_storage_id)->toBe($this->firstStorage->id)+ ->and($backup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all())+ ->toBe([$this->firstStorage->id, $this->secondStorage->id]);+});++it('uploads the instance backup to every selected destination', function () {+ $this->backup->forceFill(['save_s3' => true, 'databases_to_backup' => 'coolify'])->save();+ $this->backup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]);+ Process::fake(fn ($process) => Process::result(output: str_contains(implode(' ', (array) $process->command), 'du -b') ? '128' : ''));++ $uploadedTo = collect();+ $job = Mockery::mock(DatabaseBackupJob::class.'[upload_to_s3]', [$this->backup->fresh()])->shouldAllowMockingProtectedMethods();+ $job->shouldReceive('upload_to_s3')->andReturnUsing(fn (S3Storage $storage) => $uploadedTo->push($storage->id));+ $job->handle();++ $execution = ScheduledDatabaseBackupExecution::query()->sole();++ expect($uploadedTo->sort()->values()->all())->toBe([$this->firstStorage->id, $this->secondStorage->id])+ ->and($execution->status)->toBe('success')+ ->and($execution->s3_uploaded)->toBeTrue()+ ->and($execution->s3Replicas()->where('s3_uploaded', true)->pluck('s3_storage_id')->sort()->values()->all())+ ->toBe([$this->firstStorage->id, $this->secondStorage->id]);+});++it('links the instance backup from the storage resources page', function () {+ $this->backup->forceFill(['save_s3' => true])->save();+ $this->backup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]);++ Livewire::test(StorageResources::class, ['storage' => $this->secondStorage])+ ->assertSee('coolify-db')+ ->assertSeeHtml(route('settings.backup'));+});diff --git a/tests/Feature/Jobs/DatabaseBackupJobTest.php b/tests/Feature/Jobs/DatabaseBackupJobTest.phpindex a19958608f..d0ba825511 100644--- a/tests/Feature/Jobs/DatabaseBackupJobTest.php+++ b/tests/Feature/Jobs/DatabaseBackupJobTest.php@@ -52,7 +52,7 @@ test('scheduled database backup casts full dump selection to boolean', function->and((new ScheduledDatabaseBackup(['dump_all' => '1']))->dump_all)->toBeTrue();});-test('upload_to_s3 throws exception and disables s3 when storage is null', function () {+test('s3 upload throws exception and disables s3 when no destination exists', function () {$backup = ScheduledDatabaseBackup::create(['frequency' => '0 0 * * *','save_s3' => true,@@ -64,11 +64,7 @@ test('upload_to_s3 throws exception and disables s3 when storage is null', funct$job = new DatabaseBackupJob($backup);- $reflection = new ReflectionClass($job);- $s3Property = $reflection->getProperty('s3');- $s3Property->setValue($job, null);-- $method = $reflection->getMethod('upload_to_s3');+ $method = (new ReflectionClass($job))->getMethod('uploadToS3Destinations');expect(fn () => $method->invoke($job))->toThrow(Exception::class, 'S3 storage configuration is missing or has been deleted');@@ -78,7 +74,7 @@ test('upload_to_s3 throws exception and disables s3 when storage is null', functexpect($backup->s3_storage_id)->toBeNull();});-test('upload_to_s3 exception message reports the previous s3 storage id', function () {+test('s3 upload exception message reports the previous s3 storage id', function () {$backup = ScheduledDatabaseBackup::create(['frequency' => '0 0 * * *','save_s3' => true,@@ -90,10 +86,7 @@ test('upload_to_s3 exception message reports the previous s3 storage id', functi$job = new DatabaseBackupJob($backup);- $reflection = new ReflectionClass($job);- $reflection->getProperty('s3')->setValue($job, null);-- expect(fn () => $reflection->getMethod('upload_to_s3')->invoke($job))+ expect(fn () => (new ReflectionClass($job))->getMethod('uploadToS3Destinations')->invoke($job))->toThrow(Exception::class, 'S3 storage ID: 12345');$backup->refresh();@@ -101,7 +94,7 @@ test('upload_to_s3 exception message reports the previous s3 storage id', functiexpect($backup->s3_storage_id)->toBeNull();});-test('upload_to_s3 exception message reports null when no previous s3 storage id exists', function () {+test('s3 upload exception message reports null when no previous s3 storage id exists', function () {$backup = ScheduledDatabaseBackup::create(['frequency' => '0 0 * * *','save_s3' => true,@@ -113,10 +106,7 @@ test('upload_to_s3 exception message reports null when no previous s3 storage id$job = new DatabaseBackupJob($backup);- $reflection = new ReflectionClass($job);- $reflection->getProperty('s3')->setValue($job, null);-- expect(fn () => $reflection->getMethod('upload_to_s3')->invoke($job))+ expect(fn () => (new ReflectionClass($job))->getMethod('uploadToS3Destinations')->invoke($job))->toThrow(Exception::class, 'S3 storage ID: null');});@@ -281,7 +271,7 @@ test('retention cleanup failure does not fail a successful database backup', fun'local_storage_deleted' => true,'created_at' => now()->subDay(),]);- ScheduledDatabaseBackupExecution::create([+ $newExecution = ScheduledDatabaseBackupExecution::create(['uuid' => 'new-backup','database_name' => 'database','filename' => '/backup/new.dmp',@@ -290,6 +280,8 @@ test('retention cleanup failure does not fail a successful database backup', fun's3_uploaded' => true,'local_storage_deleted' => true,]);+ $oldExecution->s3Replicas()->create(['s3_storage_id' => $s3->id, 's3_uploaded' => true]);+ $newExecution->s3Replicas()->create(['s3_storage_id' => $s3->id, 's3_uploaded' => true]);$disk = Mockery::mock();$disk->shouldReceive('delete')->once()->with(['/backup/old.dmp'])->andReturnFalse();Storage::shouldReceive('build')->once()->andReturn($disk);diff --git a/tests/Feature/Mcp/McpReadToolsTest.php b/tests/Feature/Mcp/McpReadToolsTest.phpindex 21df936c94..f45757b8d1 100644--- a/tests/Feature/Mcp/McpReadToolsTest.php+++ b/tests/Feature/Mcp/McpReadToolsTest.php@@ -10,6 +10,7 @@ use App\Models\EnvironmentVariable;use App\Models\GithubApp;use App\Models\InstanceSettings;use App\Models\Project;+use App\Models\S3Storage;use App\Models\ScheduledDatabaseBackup;use App\Models\ScheduledDatabaseBackupExecution;use App\Models\ScheduledTask;@@ -217,6 +218,36 @@ test('database backup tools scope schedules by database type and id', function (expect($response->json('result.isError'))->toBeTrue();});+test('list_database_backups returns every S3 destination primary first', function () {+ $postgres = StandalonePostgresql::create([+ 'name' => 'multi-s3-db',+ 'postgres_password' => 'password',+ 'environment_id' => $this->environment->id,+ 'destination_id' => $this->destination->id,+ 'destination_type' => $this->destination->getMorphClass(),+ ]);+ [$first, $second] = collect(['first', 'second'])->map(fn (string $name) => S3Storage::create([+ 'name' => $name, 'region' => 'us-east-1', 'key' => 'key', 'secret' => 'secret',+ 'bucket' => $name, 'endpoint' => 'https://s3.example.com', 'team_id' => $this->team->id,+ ]))->all();+ $backup = ScheduledDatabaseBackup::create([+ 'team_id' => $this->team->id,+ 'frequency' => '0 0 * * *',+ 'save_s3' => true,+ 's3_storage_id' => $second->id,+ 'database_id' => $postgres->id,+ 'database_type' => $postgres->getMorphClass(),+ ]);+ $backup->syncS3Storages([$first->id, $second->id]);++ $response = mcpReadCall('list_database_backups', ['uuid' => $postgres->uuid]);+ $response->assertOk();++ $listed = mcpReadJson($response)['data']['backups'][0];+ expect($listed['s3_storage_uuid'])->toBe($second->uuid)+ ->and($listed['s3_storage_uuids'])->toBe([$second->uuid, $first->uuid]);+});+test('list_backup_executions omits messages without sensitive read and redacts when included', function () {$postgres = StandalonePostgresql::create(['name' => 'backup-msg-db',diff --git a/tests/Feature/MutableLivewireComponentsAuthorizationTest.php b/tests/Feature/MutableLivewireComponentsAuthorizationTest.phpindex 5563e87769..47b215d5f0 100644--- a/tests/Feature/MutableLivewireComponentsAuthorizationTest.php+++ b/tests/Feature/MutableLivewireComponentsAuthorizationTest.php@@ -182,7 +182,7 @@ it('authorizes every volume backup form control', function (string $path, array['/<x-forms\.button(?=[^>]*wire:click="toggleS3")(?=[^>]*canGate="update")(?=[^>]*:canResource="\$resource")[^>]*>\s*Enable S3\s*<\/x-forms\.button>/s','/<x-forms\.button(?=[^>]*wire:click="toggleS3")(?=[^>]*canGate="update")(?=[^>]*:canResource="\$resource")[^>]*>\s*Disable S3\s*<\/x-forms\.button>/s',- '/<x-forms\.listbox canGate="update" :canResource="\$resource" id="s3StorageId"[\s\S]*?:disabled="! auth\(\)->user\(\)\?->can\(\'update\', \$resource\)"/',+ '/<x-forms\.listbox canGate="update" :canResource="\$resource" id="s3StorageIds"/','/<x-forms\.listbox canGate="update" :canResource="\$resource" id="disableLocalBackup"[\s\S]*?:disabled="! \$saveToS3 \|\| ! auth\(\)->user\(\)\?->can\(\'update\', \$resource\)"/',],],diff --git a/tests/Feature/ResourceTeamAfterTeamSwitchTest.php b/tests/Feature/ResourceTeamAfterTeamSwitchTest.phpindex 6a434a035a..dfa4f32181 100644--- a/tests/Feature/ResourceTeamAfterTeamSwitchTest.php+++ b/tests/Feature/ResourceTeamAfterTeamSwitchTest.php@@ -175,7 +175,7 @@ test('backup edit keeps the S3 storage of the backup team when given a foreign lexpect($backup->fresh()->s3_storage_id)->toBe($ownS3->id)->and($backup->fresh()->save_s3)->toBeTruthy();- $component->set('s3StorageId', $foreignS3->id)->call('instantSave');+ $component->set('s3StorageIds', [$foreignS3->id])->call('instantSave');expect($backup->fresh()->s3_storage_id)->toBe($ownS3->id);});diff --git a/tests/Feature/ScheduleAuditTest.php b/tests/Feature/ScheduleAuditTest.phpindex 281fdd42b4..6c700eae69 100644--- a/tests/Feature/ScheduleAuditTest.php+++ b/tests/Feature/ScheduleAuditTest.php@@ -300,12 +300,12 @@ test('disabling or moving S3 storage for backups records audit events', function$volumeEvent = scheduleAuditEvent('ui.volume_backup.schedule_set');expect($databaseEvent->resource_uuid)->toBe($database->uuid)- ->and($databaseEvent->metadata['changed_fields'])->toEqualCanonicalizing(['save_s3', 's3_storage_id'])+ ->and($databaseEvent->metadata['changed_fields'])->toEqualCanonicalizing(['save_s3', 's3_storage_id', 's3_storages'])->and($volumeEvent->metadata['resource_type'])->toBe('application')->and($volumeEvent->metadata['resource_uuid'])->toBe($this->application->uuid)->and($volumeEvent->metadata['storage_uuid'])->toBe($volume->uuid)->and($volumeEvent->metadata['backup_uuid'])->toBe($volumeBackup->uuid)- ->and($volumeEvent->metadata['changed_fields'])->toBe(['s3_storage_id']);+ ->and($volumeEvent->metadata['changed_fields'])->toEqualCanonicalizing(['s3_storage_id', 's3_storages']);});test('configuring the instance database backup records an audit event', function () {diff --git a/tests/Feature/VolumeBackupMultipleS3DestinationsTest.php b/tests/Feature/VolumeBackupMultipleS3DestinationsTest.phpnew file mode 100644index 0000000000..3eadab443a--- /dev/null+++ b/tests/Feature/VolumeBackupMultipleS3DestinationsTest.php@@ -0,0 +1,386 @@+<?php++use App\Actions\Shared\DeleteScheduledVolumeBackup;+use App\Jobs\VolumeBackupJob;+use App\Jobs\VolumeBackupRecoveryJob;+use App\Models\Application;+use App\Models\Environment;+use App\Models\InstanceSettings;+use App\Models\LocalPersistentVolume;+use App\Models\PrivateKey;+use App\Models\Project;+use App\Models\S3Storage;+use App\Models\ScheduledDatabaseBackup;+use App\Models\ScheduledVolumeBackup;+use App\Models\ScheduledVolumeBackupExecution;+use App\Models\Server;+use App\Models\StandaloneDocker;+use App\Models\Team;+use Illuminate\Filesystem\FilesystemAdapter;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Facades\Event;+use Illuminate\Support\Facades\Notification;+use Illuminate\Support\Facades\Process;+use Illuminate\Support\Facades\Storage;++uses(RefreshDatabase::class);++beforeEach(function () {+ Server::flushIdentityMap();+ Notification::fake();+ config(['broadcasting.default' => 'null']);+ defined('CURLOPT_RESOLVE') || define('CURLOPT_RESOLVE', 10203);+});++/**+ * @return array{0: ScheduledVolumeBackup, 1: S3Storage, 2: S3Storage, 3: Server}+ */+function createMultiDestinationVolumeBackup(array $backupAttributes = []): array+{+ InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));+ $team = Team::factory()->create();+ $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);+ $server = Server::factory()->create([+ 'team_id' => $team->id,+ 'private_key_id' => $privateKey->id,+ 'ip' => '203.0.113.10',+ ]);+ $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();+ $project = Project::factory()->create(['team_id' => $team->id]);+ $environment = Environment::factory()->create(['project_id' => $project->id]);+ $application = Application::factory()->create([+ 'environment_id' => $environment->id,+ 'destination_id' => $destination->id,+ 'destination_type' => $destination->getMorphClass(),+ ]);+ $volume = LocalPersistentVolume::create([+ 'name' => 'app-data',+ 'mount_path' => '/data',+ 'resource_id' => $application->id,+ 'resource_type' => $application->getMorphClass(),+ ]);+ $storages = collect(['first', 'second'])->map(fn (string $name): S3Storage => S3Storage::create([+ 'name' => "{$name} storage",+ 'region' => 'us-east-1',+ 'key' => "{$name}-key",+ 'secret' => 'secret',+ 'bucket' => "{$name}-bucket",+ 'endpoint' => 'https://s3.amazonaws.com',+ 'team_id' => $team->id,+ 'is_usable' => true,+ ]));+ $backup = $volume->scheduledBackups()->create([+ 'team_id' => $team->id,+ 'frequency' => 'daily',+ 'save_s3' => true,+ ...$backupAttributes,+ ]);+ $backup->syncS3Storages($storages->pluck('id')->all());++ return [$backup->fresh(), $storages[0], $storages[1], $server];+}++function fakeMultiDestinationS3Disks(): void+{+ $sshDisk = Storage::fake('ssh-keys');+ $disk = Mockery::mock(FilesystemAdapter::class);+ $disk->shouldReceive('files')->zeroOrMoreTimes()->andReturn([]);+ $disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue();+ Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk);+ Storage::shouldReceive('build')->zeroOrMoreTimes()->andReturn($disk);+}++it('archives locally and uploads to every destination instead of streaming', function () {+ [$backup, $first, $second] = createMultiDestinationVolumeBackup(['disable_local_backup' => true]);+ fakeMultiDestinationS3Disks();+ Process::fake([+ '*du -b*' => '128',+ '*' => '',+ ]);++ (new VolumeBackupJob($backup))->handle();++ $execution = ScheduledVolumeBackupExecution::query()->sole();+ Process::assertNotRan(fn ($process) => str_contains($process->command, 'mc pipe'));+ Process::assertRan(fn ($process) => str_contains($process->command, 'tar -I') && str_contains($process->command, ' > '));+ Process::assertRan(fn ($process) => str_contains($process->command, 'mc cp') && str_contains($process->command, 'temporary/first-bucket/'));+ Process::assertRan(fn ($process) => str_contains($process->command, 'mc cp') && str_contains($process->command, 'temporary/second-bucket/'));+ Process::assertRan(fn ($process) => preg_match("/rm -f '[^']+\\.tar\\.gz'/", $process->command) === 1);+ expect($execution->status)->toBe('success')+ ->and($execution->s3_storage_id)->toBe($first->id)+ ->and($execution->s3_uploaded)->toBeTrue()+ ->and($execution->local_storage_deleted)->toBeTrue()+ ->and($execution->s3Replicas()->orderBy('s3_storage_id')->get()->map->only(['s3_storage_id', 's3_uploaded'])->all())->toBe([+ ['s3_storage_id' => $first->id, 's3_uploaded' => true],+ ['s3_storage_id' => $second->id, 's3_uploaded' => true],+ ]);+});++it('keeps the local archive and cleans the partial upload when one destination fails', function () {+ [$backup, $first, $second] = createMultiDestinationVolumeBackup(['disable_local_backup' => true]);+ fakeMultiDestinationS3Disks();+ Process::fake([+ '*du -b*' => '128',+ '*temporary/second-bucket/*' => Process::result(errorOutput: 'access denied', exitCode: 1),+ '*' => '',+ ]);++ (new VolumeBackupJob($backup))->handle();++ $execution = ScheduledVolumeBackupExecution::query()->sole();+ $firstReplica = $execution->s3Replicas()->where('s3_storage_id', $first->id)->sole();+ $secondReplica = $execution->s3Replicas()->where('s3_storage_id', $second->id)->sole();+ Process::assertNotRan(fn ($process) => preg_match("/rm -f '[^']+\\.tar\\.gz'/", $process->command) === 1);+ expect($execution->status)->toBe('success')+ ->and($execution->message)->toContain('S3 upload failed: second storage:')+ ->and($execution->s3_uploaded)->toBeFalse()+ ->and($execution->s3_cleanup_pending)->toBeFalse()+ ->and($execution->local_storage_deleted)->toBeFalse()+ ->and($firstReplica->s3_uploaded)->toBeTrue()+ ->and($firstReplica->s3_storage_deleted)->toBeFalse()+ ->and($secondReplica->s3_uploaded)->toBeFalse()+ ->and($secondReplica->s3_storage_deleted)->toBeTrue()+ ->and($secondReplica->message)->not->toBeEmpty();+});++it('applies S3 retention to each destination separately', function () {+ [$backup, $first, $second, $server] = createMultiDestinationVolumeBackup([+ 'retention_amount_locally' => 0,+ 'retention_days_locally' => 0,+ 'retention_max_storage_locally' => 0,+ 'retention_amount_s3' => 1,+ 'retention_days_s3' => 0,+ 'retention_max_storage_s3' => 0,+ ]);+ $createExecution = function (string $name, int $daysAgo, array $uploadedTo) use ($backup, $first, $second): ScheduledVolumeBackupExecution {+ $execution = ScheduledVolumeBackupExecution::create([+ 'scheduled_volume_backup_id' => $backup->id,+ 's3_storage_id' => $first->id,+ 'status' => 'success',+ 'filename' => "/data/coolify/backups/volumes/test/{$name}.tar.gz",+ 'local_storage_deleted' => true,+ 'created_at' => now()->subDays($daysAgo),+ ]);+ foreach ([$first, $second] as $storage) {+ $execution->s3Replicas()->create([+ 's3_storage_id' => $storage->id,+ 's3_uploaded' => in_array($storage->id, $uploadedTo, true),+ ]);+ }+ $execution->refreshS3Summary();++ return $execution;+ };+ $oldest = $createExecution('oldest', 3, [$first->id]);+ $older = $createExecution('older', 2, [$first->id, $second->id]);+ $latest = $createExecution('latest', 1, [$first->id]);+ $disk = Mockery::mock();+ $disk->shouldReceive('delete')+ ->once()+ ->with(['/data/coolify/backups/volumes/test/older.tar.gz', '/data/coolify/backups/volumes/test/oldest.tar.gz'])+ ->andReturnTrue();+ Storage::shouldReceive('build')+ ->once()+ ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'first-key'))+ ->andReturn($disk);+ $job = new VolumeBackupJob($backup);++ (new ReflectionClass($job))->getMethod('removeExpiredBackups')->invoke($job, $server);++ expect($oldest->fresh())->toBeNull()+ ->and($older->fresh())->not->toBeNull()+ ->and($older->s3Replicas()->where('s3_storage_id', $first->id)->sole()->s3_storage_deleted)->toBeTrue()+ ->and($older->s3Replicas()->where('s3_storage_id', $second->id)->sole()->s3_storage_deleted)->toBeFalse()+ ->and($older->fresh()->s3_storage_deleted)->toBeFalse()+ ->and($latest->s3Replicas()->where('s3_storage_id', $first->id)->sole()->s3_storage_deleted)->toBeFalse();+});++it('continues retention and execution cleanup when a replica storage is missing', function () {+ $this->freezeTime();+ [$backup, , $second, $server] = createMultiDestinationVolumeBackup([+ 'retention_amount_locally' => 0,+ 'retention_days_locally' => 0,+ 'retention_max_storage_locally' => 0,+ 'retention_amount_s3' => 0,+ 'retention_days_s3' => 7,+ 'retention_max_storage_s3' => 0,+ ]);+ $executions = collect([false, true])->map(function (bool $localDeleted) use ($backup, $second): ScheduledVolumeBackupExecution {+ $execution = $backup->executions()->create([+ 'status' => 'success',+ 'filename' => '/data/coolify/backups/volumes/test/'.($localDeleted ? 'deleted' : 'local').'.tar.gz',+ 'local_storage_deleted' => $localDeleted,+ ]);+ $execution->forceFill(['created_at' => now()->subDays(10)])->save();+ $execution->s3Replicas()->create(['s3_storage_id' => null, 's3_uploaded' => true]);+ $execution->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => true]);+ $execution->refreshS3Summary();++ return $execution;+ });+ $disk = Mockery::mock();+ $disk->shouldReceive('delete')->once()->with(Mockery::on(+ fn (array $filenames): bool => count($filenames) === 2+ && in_array($executions[0]->filename, $filenames, true)+ && in_array($executions[1]->filename, $filenames, true),+ ))->andReturnTrue();+ Storage::shouldReceive('build')->once()+ ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'second-key'))+ ->andReturn($disk);+ $job = new VolumeBackupJob($backup);++ (new ReflectionClass($job))->getMethod('removeExpiredBackups')->invoke($job, $server);++ expect($executions[0]->fresh()->s3_storage_deleted)->toBeTrue()+ ->and($executions[0]->s3Replicas()->where('s3_storage_deleted', false)->count())->toBe(0);+ $this->assertModelMissing($executions[1]);+});++it('cleans an interrupted upload from every destination whose copy did not finish', function () {+ [$backup, $first, $second] = createMultiDestinationVolumeBackup();+ $execution = ScheduledVolumeBackupExecution::create([+ 'scheduled_volume_backup_id' => $backup->id,+ 's3_storage_id' => $first->id,+ 'status' => 'failed',+ 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz',+ 's3_cleanup_pending' => true,+ ]);+ $uploaded = $execution->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]);+ $interrupted = $execution->s3Replicas()->create(['s3_storage_id' => $second->id]);+ $withoutStorage = $execution->s3Replicas()->create(['s3_storage_id' => null, 's3_uploaded' => false]);+ $disk = Mockery::mock();+ $disk->shouldReceive('delete')->once()->with(['/data/coolify/backups/volumes/test/interrupted.tar.gz'])->andReturnTrue();+ Storage::shouldReceive('build')+ ->once()+ ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'second-key'))+ ->andReturn($disk);++ VolumeBackupRecoveryJob::cleanupS3Upload($execution);++ expect($execution->fresh()->s3_cleanup_pending)->toBeFalse()+ ->and($uploaded->fresh()->s3_storage_deleted)->toBeFalse()+ ->and($interrupted->fresh()->s3_storage_deleted)->toBeTrue()+ ->and($withoutStorage->fresh()->s3_storage_deleted)->toBeTrue();+});++it('keeps the cleanup pending when one destination cannot delete the partial upload', function () {+ [$backup, $first, $second] = createMultiDestinationVolumeBackup();+ $execution = ScheduledVolumeBackupExecution::create([+ 'scheduled_volume_backup_id' => $backup->id,+ 's3_storage_id' => $first->id,+ 'status' => 'failed',+ 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz',+ 's3_cleanup_pending' => true,+ ]);+ $failing = $execution->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => false]);+ $cleaned = $execution->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => false]);+ $failingDisk = Mockery::mock();+ $failingDisk->shouldReceive('delete')->once()->andReturnFalse();+ $cleanedDisk = Mockery::mock();+ $cleanedDisk->shouldReceive('delete')->once()->andReturnTrue();+ Storage::shouldReceive('build')->with(Mockery::on(fn (array $config): bool => $config['key'] === 'first-key'))->andReturn($failingDisk);+ Storage::shouldReceive('build')->with(Mockery::on(fn (array $config): bool => $config['key'] === 'second-key'))->andReturn($cleanedDisk);++ expect(fn () => VolumeBackupRecoveryJob::cleanupS3Upload($execution))->toThrow(RuntimeException::class);++ expect($execution->fresh()->s3_cleanup_pending)->toBeTrue()+ ->and($failing->fresh()->s3_storage_deleted)->toBeFalse()+ ->and($cleaned->fresh()->s3_storage_deleted)->toBeTrue();+});++it('moves the primary destination to a remaining storage when the primary storage is deleted', function () {+ [$backup, $first, $second] = createMultiDestinationVolumeBackup();+ $execution = ScheduledVolumeBackupExecution::create([+ 'scheduled_volume_backup_id' => $backup->id,+ 's3_storage_id' => $first->id,+ 'status' => 'success',+ 'filename' => '/data/coolify/backups/volumes/test/archive.tar.gz',+ ]);+ $execution->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]);+ $execution->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => true]);+ $execution->refreshS3Summary();++ expect($backup->s3_storage_id)->toBe($first->id);++ $first->delete();++ $backup->refresh();+ expect($backup->save_s3)->toBeTrue()+ ->and($backup->s3_storage_id)->toBe($second->id)+ ->and($backup->s3Storages()->pluck('s3_storages.id')->all())->toBe([$second->id])+ ->and($execution->fresh()->s3_storage_deleted)->toBeFalse()+ ->and($execution->hasLiveS3Copies())->toBeTrue();+});++it('preserves the schedule and uploaded replica when its S3 storage is unavailable', function () {+ [$backup] = createMultiDestinationVolumeBackup();+ $execution = ScheduledVolumeBackupExecution::create([+ 'scheduled_volume_backup_id' => $backup->id,+ 'status' => 'success',+ 'filename' => '/data/coolify/backups/volumes/test/archive.tar.gz',+ 'local_storage_deleted' => true,+ ]);+ $replica = $execution->s3Replicas()->create([+ 's3_storage_id' => null,+ 's3_uploaded' => true,+ 's3_storage_deleted' => false,+ ]);+ $execution->refreshS3Summary();++ expect(fn () => (new DeleteScheduledVolumeBackup)->handle($backup, deleteLocalArchives: false))+ ->toThrow(RuntimeException::class, 'The S3 storage used by an existing backup is unavailable.');++ expect($backup->fresh())->not->toBeNull();+ expect($replica->fresh()->s3_storage_deleted)->toBeFalse();+ expect($execution->fresh()->s3_storage_deleted)->toBeFalse();+ expect($execution->hasLiveS3Copies())->toBeTrue();+});++it('rolls back all backup detachment changes when an execution update fails', function () {+ [$backup, $storage] = createMultiDestinationVolumeBackup();+ $backup->syncS3Storages([$storage->id]);+ $databaseBackup = ScheduledDatabaseBackup::create([+ 'frequency' => 'daily',+ 'save_s3' => true,+ 'database_type' => 'App\\Models\\StandalonePostgresql',+ 'database_id' => 1,+ 'team_id' => $storage->team_id,+ ]);+ $databaseBackup->syncS3Storages([$storage->id]);+ $databaseExecution = $databaseBackup->executions()->create([+ 'uuid' => 'rollback-database',+ 'database_name' => 'app',+ 'status' => 'success',+ 's3_uploaded' => true,+ ]);+ $databaseReplica = $databaseExecution->s3Replicas()->create([+ 's3_storage_id' => $storage->id,+ 's3_uploaded' => true,+ ]);+ $execution = ScheduledVolumeBackupExecution::create([+ 'scheduled_volume_backup_id' => $backup->id,+ 's3_storage_id' => $storage->id,+ 'status' => 'failed',+ 's3_cleanup_pending' => true,+ ]);+ $replica = $execution->s3Replicas()->create([+ 's3_storage_id' => $storage->id,+ 's3_uploaded' => false,+ ]);+ Event::listen('eloquent.updating: '.ScheduledVolumeBackupExecution::class, function (): void {+ throw new RuntimeException('Execution update failed');+ });++ expect(fn () => $storage->delete())->toThrow(RuntimeException::class, 'Execution update failed');++ $this->assertModelExists($storage);+ foreach ([$databaseBackup, $backup] as $schedule) {+ expect($schedule->fresh()->s3_storage_id)->toBe($storage->id)+ ->and((bool) $schedule->fresh()->save_s3)->toBeTrue()+ ->and($schedule->s3Storages()->pluck('s3_storages.id')->all())->toBe([$storage->id]);+ }+ expect($databaseReplica->fresh()->s3_storage_deleted)->toBeFalse()+ ->and($databaseExecution->fresh()->s3_storage_deleted)->toBeFalse()+ ->and($replica->fresh()->s3_storage_deleted)->toBeFalse()+ ->and($execution->fresh()->s3_cleanup_pending)->toBeTrue();+});diff --git a/tests/Feature/VolumeBackupRecoveryBoundsTest.php b/tests/Feature/VolumeBackupRecoveryBoundsTest.phpindex ff3ed2d031..08d90ab774 100644--- a/tests/Feature/VolumeBackupRecoveryBoundsTest.php+++ b/tests/Feature/VolumeBackupRecoveryBoundsTest.php@@ -93,7 +93,7 @@ function createRecoveryBoundsBackup(Team $team, array $backupAttributes = []): afunction createRecoveryBoundsExecution(ScheduledVolumeBackup $backup, array $attributes = []): ScheduledVolumeBackupExecution{- return ScheduledVolumeBackupExecution::create([+ $execution = ScheduledVolumeBackupExecution::create(['scheduled_volume_backup_id' => $backup->id,'s3_storage_id' => $backup->s3_storage_id,'status' => 'failed',@@ -101,6 +101,9 @@ function createRecoveryBoundsExecution(ScheduledVolumeBackup $backup, array $att's3_cleanup_pending' => true,...$attributes,]);+ $execution->s3Replicas()->create(['s3_storage_id' => $backup->s3_storage_id]);++ return $execution;}function failRecoveryBoundsS3Delete(string $awsCode): void@@ -164,7 +167,7 @@ it('records a categorized S3 cleanup failure without throwing or leaking the S3$execution->refresh();expect($execution->s3_cleanup_pending)->toBeTrue()- ->and($execution->s3_storage_deleted)->toBeFalse()+ ->and($execution->s3Replicas()->sole()->s3_storage_deleted)->toBeFalse()->and($execution->recovery_error)->toBe('s3_auth')->and($execution->recovery_last_attempt_at->equalTo(now()))->toBeTrue()->and($execution->recovery_needs_attention)->toBeTrue()@@ -261,7 +264,7 @@ it('cleans the S3 upload even when container recovery fails', function () {$execution->refresh();expect($execution->stop_recovery_pending)->toBeTrue()->and($execution->s3_cleanup_pending)->toBeFalse()- ->and($execution->s3_storage_deleted)->toBeTrue()+ ->and($execution->s3Replicas()->sole()->s3_storage_deleted)->toBeTrue()->and($execution->recovery_error)->toBe('server_unreachable')->and($execution->recovery_needs_attention)->toBeFalse();Notification::assertNothingSent();@@ -281,7 +284,7 @@ it('resets the recovery state after a successful attempt', function () {$execution->refresh();expect($execution->s3_cleanup_pending)->toBeFalse()- ->and($execution->s3_storage_deleted)->toBeTrue()+ ->and($execution->s3Replicas()->sole()->s3_storage_deleted)->toBeTrue()->and($execution->recovery_error)->toBeNull()->and($execution->recovery_last_attempt_at)->toBeNull()->and($execution->recovery_needs_attention)->toBeFalse();diff --git a/tests/Feature/VolumeBackupTest.php b/tests/Feature/VolumeBackupTest.phpindex d247a7a4e6..66eadf98ca 100644--- a/tests/Feature/VolumeBackupTest.php+++ b/tests/Feature/VolumeBackupTest.php@@ -867,6 +867,7 @@ it('deletes an individual S3 archive from the storage recorded on its execution''local_storage_deleted' => true,'s3_uploaded' => true,]);+ $execution->s3Replicas()->create(['s3_storage_id' => $originalStorage->id, 's3_uploaded' => true]);$disk = Mockery::mock();$disk->shouldReceive('delete')->once()@@ -981,7 +982,7 @@ it('persists S3 settings the first time when a volume backup schedule does not e'resource' => $application,'section' => 's3',])- ->assertSet('s3StorageId', $s3Storage->id)+ ->assertSet('s3StorageIds', [$s3Storage->id])->call('toggleS3')->assertSet('saveToS3', true)->assertDispatched('success');@@ -1013,7 +1014,7 @@ it('persists the selected S3 storage when a volume backup schedule does not exis'resource' => $application,'section' => 's3',])- ->set('s3StorageId', $s3Storage->id)+ ->set('s3StorageIds', [$s3Storage->id])->assertDispatched('success');$backup = ScheduledVolumeBackup::query()->sole();@@ -1136,7 +1137,7 @@ it('does not change S3 storage when another volume backup setting is invalid', f'section' => 's3',])->set('frequency', 'not a valid schedule')- ->set('s3StorageId', $secondS3Storage->id)+ ->set('s3StorageIds', [$secondS3Storage->id])->assertHasErrors('frequency')->assertNotDispatched('success');@@ -1254,18 +1255,18 @@ it('lists and accepts only S3 storages of the resource team when the session tea$component = Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]);expect($component->get('availableS3Storages')->pluck('id')->all())->toBe([$resourceTeamStorage->id])- ->and($component->get('s3StorageId'))->toBe($resourceTeamStorage->id);+ ->and($component->get('s3StorageIds'))->toBe([$resourceTeamStorage->id]);$component->set('frequency', 'daily')->set('saveToS3', true)- ->set('s3StorageId', $otherTeamStorage->id)- ->assertHasErrors('s3StorageId')+ ->set('s3StorageIds', [$otherTeamStorage->id])+ ->assertHasErrors('s3StorageIds')->call('save')- ->assertHasErrors('s3StorageId');+ ->assertHasErrors('s3StorageIds');expect(ScheduledVolumeBackup::query()->count())->toBe(0);- $component->set('s3StorageId', $resourceTeamStorage->id)+ $component->set('s3StorageIds', [$resourceTeamStorage->id])->call('save')->assertHasNoErrors();@@ -1295,9 +1296,9 @@ it('only accepts a usable S3 storage owned by the current team', function () {Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application])->set('frequency', 'daily')->set('saveToS3', true)- ->set('s3StorageId', $foreignStorage->id)+ ->set('s3StorageIds', [$foreignStorage->id])->call('save')- ->assertHasErrors('s3StorageId');+ ->assertHasErrors('s3StorageIds');expect(ScheduledVolumeBackup::query()->count())->toBe(0);});@@ -1322,7 +1323,7 @@ it('saves the database-style S3 backup controls immediately', function () {]);$component = Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application])- ->assertSet('s3StorageId', $s3Storage->id)+ ->assertSet('s3StorageIds', [$s3Storage->id])->set('saveToS3', true)->call('instantSave')->set('disableLocalBackup', true)@@ -1378,7 +1379,7 @@ it('saves the selected volume backup S3 storage immediately while S3 is disabled]);Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application, 'section' => 's3'])- ->set('s3StorageId', $secondS3Storage->id)+ ->set('s3StorageIds', [$secondS3Storage->id])->assertDispatched('success');$backup = ScheduledVolumeBackup::query()->sole();@@ -1486,6 +1487,7 @@ it('disables S3 volume backups when the storage is deleted', function () {'s3_uploaded' => true,'s3_cleanup_pending' => true,]);+ $execution->s3Replicas()->create(['s3_storage_id' => $s3Storage->id, 's3_uploaded' => true]);$s3Storage->delete();@@ -1530,6 +1532,7 @@ it('marks historical executions deleted when their recorded S3 storage is remove's3_uploaded' => true,'s3_cleanup_pending' => true,]);+ $execution->s3Replicas()->create(['s3_storage_id' => $originalStorage->id, 's3_uploaded' => true]);$originalStorage->delete();@@ -1704,7 +1707,7 @@ it('deletes S3 archives from the storage recorded on each execution', function ('frequency' => 'daily','save_s3' => true,]);- ScheduledVolumeBackupExecution::create([+ $execution = ScheduledVolumeBackupExecution::create(['scheduled_volume_backup_id' => $backup->id,'s3_storage_id' => $originalStorage->id,'status' => 'success',@@ -1712,6 +1715,7 @@ it('deletes S3 archives from the storage recorded on each execution', function ('local_storage_deleted' => true,'s3_uploaded' => true,]);+ $execution->s3Replicas()->create(['s3_storage_id' => $originalStorage->id, 's3_uploaded' => true]);$disk = Mockery::mock();$disk->shouldReceive('delete')->once()@@ -1977,7 +1981,8 @@ it('keeps the upload destination on the volume backup execution', function () {$execution = ScheduledVolumeBackupExecution::query()->sole();expect($execution->s3_storage_id)->toBe($s3Storage->id)- ->and($execution->s3->is($s3Storage))->toBeTrue();+ ->and($execution->s3->is($s3Storage))->toBeTrue()+ ->and($execution->s3Replicas()->sole()->s3_storage_id)->toBe($s3Storage->id);});it('streams S3-only volume backups without creating or copying a local archive', function () {@@ -2032,6 +2037,7 @@ it('streams S3-only volume backups without creating or copying a local archive',expect($execution->status)->toBe('success')->and($execution->size)->toBe(128)->and($execution->s3_uploaded)->toBeTrue()+ ->and($execution->s3Replicas()->sole()->s3_uploaded)->toBeTrue()->and($execution->local_storage_deleted)->toBeTrue();});@@ -2240,15 +2246,26 @@ it('removes retained S3 archives from the storage recorded on each execution', f'status' => 'success','filename' => '/data/coolify/backups/volumes/test/old.tar.gz','s3_uploaded' => true,+ 'created_at' => now()->subDays(2),+ ]);+ $oldExecution->s3Replicas()->create(['s3_storage_id' => $originalStorage->id, 's3_uploaded' => true]);+ $retainedOriginalExecution = ScheduledVolumeBackupExecution::create([+ 'scheduled_volume_backup_id' => $backup->id,+ 's3_storage_id' => $originalStorage->id,+ 'status' => 'success',+ 'filename' => '/data/coolify/backups/volumes/test/retained-original.tar.gz',+ 's3_uploaded' => true,'created_at' => now()->subDay(),]);- ScheduledVolumeBackupExecution::create([+ $retainedOriginalExecution->s3Replicas()->create(['s3_storage_id' => $originalStorage->id, 's3_uploaded' => true]);+ $newExecution = ScheduledVolumeBackupExecution::create(['scheduled_volume_backup_id' => $backup->id,'s3_storage_id' => $newStorage->id,'status' => 'success','filename' => '/data/coolify/backups/volumes/test/new.tar.gz','s3_uploaded' => true,]);+ $newExecution->s3Replicas()->create(['s3_storage_id' => $newStorage->id, 's3_uploaded' => true]);$disk = Mockery::mock();$disk->shouldReceive('delete')->once()->with(['/data/coolify/backups/volumes/test/old.tar.gz'])->andReturnTrue();Storage::shouldReceive('build')@@ -2260,7 +2277,10 @@ it('removes retained S3 archives from the storage recorded on each execution', f$method->invoke($job, $server);- expect($oldExecution->fresh()->s3_storage_deleted)->toBeTrue();+ expect($oldExecution->fresh()->s3_storage_deleted)->toBeTrue()+ ->and($oldExecution->s3Replicas()->sole()->s3_storage_deleted)->toBeTrue()+ ->and($retainedOriginalExecution->fresh()->s3_storage_deleted)->toBeFalse()+ ->and($newExecution->fresh()->s3_storage_deleted)->toBeFalse();});it('does not query execution history when local retention is unlimited', function () {@@ -2514,6 +2534,7 @@ it('cleans an interrupted S3 upload and coordinates recovery with the backup loc'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz','s3_cleanup_pending' => true,]);+ $execution->s3Replicas()->create(['s3_storage_id' => $s3Storage->id]);$disk = Mockery::mock();$disk->shouldReceive('delete')->once()->andReturnTrue();Storage::shouldReceive('build')->once()->andReturn($disk);@@ -2523,7 +2544,7 @@ it('cleans an interrupted S3 upload and coordinates recovery with the backup loc$job->handle();expect($execution->fresh()->s3_cleanup_pending)->toBeFalse()- ->and($execution->fresh()->s3_storage_deleted)->toBeTrue();+ ->and($execution->s3Replicas()->sole()->s3_storage_deleted)->toBeTrue();});it('cleans an interrupted upload from the execution S3 storage after a schedule switch', function () {@@ -2562,6 +2583,7 @@ it('cleans an interrupted upload from the execution S3 storage after a schedule'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz','s3_cleanup_pending' => true,]);+ $execution->s3Replicas()->create(['s3_storage_id' => $originalStorage->id]);$disk = Mockery::mock();$disk->shouldReceive('delete')->once()->andReturnTrue();Storage::shouldReceive('build')@@ -2572,7 +2594,7 @@ it('cleans an interrupted upload from the execution S3 storage after a scheduleVolumeBackupRecoveryJob::cleanupS3Upload($execution);expect($execution->fresh()->s3_cleanup_pending)->toBeFalse()- ->and($execution->fresh()->s3_storage_deleted)->toBeTrue();+ ->and($execution->s3Replicas()->sole()->s3_storage_deleted)->toBeTrue();});it('keeps the S3 key tracked when interrupted upload cleanup must be retried', function () {@@ -2603,6 +2625,7 @@ it('keeps the S3 key tracked when interrupted upload cleanup must be retried', f'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz','s3_cleanup_pending' => true,]);+ $execution->s3Replicas()->create(['s3_storage_id' => $s3Storage->id]);$disk = Mockery::mock();$disk->shouldReceive('delete')->once()->andReturnFalse();Storage::shouldReceive('build')->once()->andReturn($disk);diff --git a/tests/Unit/ServerTransfer/ServerTransferS3DestinationsTest.php b/tests/Unit/ServerTransfer/ServerTransferS3DestinationsTest.phpnew file mode 100644index 0000000000..479367ee70--- /dev/null+++ b/tests/Unit/ServerTransfer/ServerTransferS3DestinationsTest.php@@ -0,0 +1,144 @@+<?php++use App\Models\Application;+use App\Models\Environment;+use App\Models\InstanceSettings;+use App\Models\LocalPersistentVolume;+use App\Models\PrivateKey;+use App\Models\Project;+use App\Models\S3Storage;+use App\Models\ScheduledDatabaseBackup;+use App\Models\ScheduledVolumeBackup;+use App\Models\Server;+use App\Models\StandaloneDocker;+use App\Models\StandalonePostgresql;+use App\Models\Team;+use App\Services\ServerTransfer\ServerTransferExporter;+use App\Services\ServerTransfer\ServerTransferImporter;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Tests\TestCase;++uses(TestCase::class, RefreshDatabase::class);++beforeEach(function () {+ Server::flushIdentityMap();+ InstanceSettings::forceCreate(['id' => 0, 'is_api_enabled' => true, 'fqdn' => 'https://coolify-a.test']);++ $this->team = Team::factory()->create();+ $privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]);+ $this->server = Server::factory()->create([+ 'team_id' => $this->team->id,+ 'private_key_id' => $privateKey->id,+ 'ip' => '10.56.0.10',+ ]);+ $destination = StandaloneDocker::where('server_id', $this->server->id)->firstOrFail();+ $project = Project::factory()->create(['team_id' => $this->team->id]);+ $environment = $project->environments()->first()+ ?? Environment::factory()->create(['project_id' => $project->id]);++ $this->application = Application::factory()->create([+ 'environment_id' => $environment->id,+ 'destination_id' => $destination->id,+ 'destination_type' => $destination->getMorphClass(),+ ]);+ $volume = LocalPersistentVolume::create([+ 'name' => $this->application->uuid.'-data',+ 'mount_path' => '/app/data',+ 'resource_type' => $this->application->getMorphClass(),+ 'resource_id' => $this->application->id,+ ]);++ StandalonePostgresql::withoutEvents(function () use ($environment, $destination) {+ $database = new StandalonePostgresql;+ $database->forceFill([+ 'name' => 'app-db',+ 'postgres_user' => 'postgres',+ 'postgres_password' => 'secret',+ 'postgres_db' => 'app',+ 'environment_id' => $environment->id,+ 'destination_id' => $destination->id,+ 'destination_type' => $destination->getMorphClass(),+ ]);+ $database->uuid = new_public_id();+ $database->save();+ $this->database = $database;+ });++ $this->firstS3 = S3Storage::create([+ 'name' => 'first', 'region' => 'us-east-1', 'key' => 'k1', 'secret' => 's1',+ 'bucket' => 'first-bucket', 'endpoint' => 'https://first.example.com', 'team_id' => $this->team->id,+ ]);+ $this->secondS3 = S3Storage::create([+ 'name' => 'second', 'region' => 'us-east-1', 'key' => 'k2', 'secret' => 's2',+ 'bucket' => 'second-bucket', 'endpoint' => 'https://second.example.com', 'team_id' => $this->team->id,+ ]);++ $this->databaseBackup = ScheduledDatabaseBackup::create([+ 'team_id' => $this->team->id,+ 'save_s3' => true,+ 'frequency' => '0 2 * * *',+ 's3_storage_id' => $this->secondS3->id,+ 'database_type' => $this->database->getMorphClass(),+ 'database_id' => $this->database->id,+ ]);+ $this->databaseBackup->syncS3Storages([$this->firstS3->id, $this->secondS3->id]);++ $this->volumeBackup = $volume->scheduledBackups()->create([+ 'team_id' => $this->team->id,+ 'save_s3' => true,+ 'frequency' => '0 3 * * *',+ 's3_storage_id' => $this->secondS3->id,+ ]);+ $this->volumeBackup->syncS3Storages([$this->firstS3->id, $this->secondS3->id]);+});++/**+ * @return list<string> Buckets of the backup's destinations, primary first.+ */+function transferredBackupBuckets(ScheduledDatabaseBackup|ScheduledVolumeBackup $backup): array+{+ return $backup->selectedS3Storages()->pluck('bucket')->all();+}++function importTransferBundleIntoNewTeam(array $bundle, $context): Team+{+ $targetTeam = Team::factory()->create();+ $context->application->forceDelete();+ $context->database->forceDelete();+ $context->server->forceDelete();++ app(ServerTransferImporter::class)->import($bundle, teamId: $targetTeam->id);++ return $targetTeam;+}++test('transfer keeps every S3 destination of database and volume backups', function () {+ $bundle = app(ServerTransferExporter::class)->export($this->server);++ expect(collect($bundle['s3_storages'])->pluck('uuid')->sort()->values()->all())+ ->toEqual(collect([$this->firstS3->uuid, $this->secondS3->uuid])->sort()->values()->all());++ $targetTeam = importTransferBundleIntoNewTeam($bundle, $this);++ $databaseBackup = ScheduledDatabaseBackup::where('team_id', $targetTeam->id)->sole();+ $volumeBackup = ScheduledVolumeBackup::where('team_id', $targetTeam->id)->sole();++ expect(transferredBackupBuckets($databaseBackup))->toBe(['second-bucket', 'first-bucket'])+ ->and($databaseBackup->save_s3)->toBeTruthy()+ ->and($databaseBackup->s3->team_id)->toBe($targetTeam->id)+ ->and(transferredBackupBuckets($volumeBackup))->toBe(['second-bucket', 'first-bucket'])+ ->and($volumeBackup->save_s3)->toBeTrue()+ ->and($volumeBackup->s3->team_id)->toBe($targetTeam->id);+});++test('transfer imports bundles that only have the primary S3 destination', function () {+ $bundle = app(ServerTransferExporter::class)->export($this->server);+ data_forget($bundle, 'projects.*.environments.*.databases.*.scheduled_backups.*.s3_storage_uuids');+ data_forget($bundle, 'volume_backups.*.s3_storage_uuids');+ expect(data_get($bundle, 'volume_backups.0'))->not->toHaveKey('s3_storage_uuids');++ $targetTeam = importTransferBundleIntoNewTeam($bundle, $this);++ expect(transferredBackupBuckets(ScheduledDatabaseBackup::where('team_id', $targetTeam->id)->sole()))->toBe(['second-bucket'])+ ->and(transferredBackupBuckets(ScheduledVolumeBackup::where('team_id', $targetTeam->id)->sole()))->toBe(['second-bucket']);+});