Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

docs(backups): add multi-destination S3 planning notes (#11789)

Andras Bacsaid4d8bba51467
diff --git a/.ai/lessons.md b/.ai/lessons.md
index 5cde3ef34e..7b94e29b3b 100644
--- a/.ai/lessons.md
+++ b/.ai/lessons.md
@@ -12,6 +12,7 @@
- SSH retries replay the full command batch. Only exit code 255 is an SSH transport failure; other exit codes come from the remote command, so do not retry them. Keep batches safe to run twice (for example `docker rm -f <name> || true` before `docker run --name <name>`).
- Redirect browser test output to a file (`> /tmp/x.log 2>&1`); piping it (`| tail`) hangs because the Playwright server keeps the pipe open.
- Browser `click('Text')` can match a hidden element first (for example inactive modal steps) and wait forever; use `button:visible:has-text("Text")`.
+- A new worktree has no `vendor/` or `public/build/`. Run `composer install --ignore-platform-req=ext-zip` and copy `public/build` from the main checkout before page-rendering tests; otherwise they fail with a Vite manifest error.
- Never run `php artisan test` inside a dev container (`./scripts/dev exec` or `docker exec coolify-dev-*`). The container sets `DB_CONNECTION=pgsql` as a real environment variable, which wins over `phpunit.xml`, so `RefreshDatabase` runs `migrate:fresh` on the dev database and wipes it. Run tests on the host only.
- Call `visit()` directly in each `tests/v4/Browser` test body; Pest does not mark a test that only uses helper-wrapped `visit()` as a browser test, so it fails with `sendText() on null`.
diff --git a/app/Actions/Development/StartDevelopmentInstanceDatabase.php b/app/Actions/Development/StartDevelopmentInstanceDatabase.php
new file mode 100644
index 0000000000..41b8428bd4
--- /dev/null
+++ b/app/Actions/Development/StartDevelopmentInstanceDatabase.php
@@ -0,0 +1,33 @@
+<?php
+
+namespace App\Actions\Development;
+
+use App\Models\Server;
+use Lorisleiva\Actions\Concerns\AsAction;
+
+/**
+ * Starts a `coolify-db` container on a development VM that acts as the localhost server.
+ *
+ * The instance backup reads the credentials from this container and dumps it, like on a self-hosted install.
+ * The real development database runs on the host, so the VM cannot reach it.
+ */
+class StartDevelopmentInstanceDatabase
+{
+ use AsAction;
+
+ public function handle(Server $server): void
+ {
+ $environment = collect([
+ 'POSTGRES_USER' => config('database.connections.pgsql.username'),
+ 'POSTGRES_PASSWORD' => config('database.connections.pgsql.password'),
+ 'POSTGRES_DB' => config('database.connections.pgsql.database'),
+ ])->map(fn ($value, string $key): string => '-e '.escapeshellarg("{$key}={$value}"))->implode(' ');
+
+ instant_remote_process([
+ 'if ! docker inspect coolify-db >/dev/null 2>&1; then',
+ "docker run -d --name coolify-db --restart unless-stopped {$environment} postgres:15-alpine",
+ 'fi',
+ 'docker start coolify-db',
+ ], $server, disableMultiplexing: true);
+ }
+}
diff --git a/app/Actions/Shared/DeleteScheduledVolumeBackup.php b/app/Actions/Shared/DeleteScheduledVolumeBackup.php
index d56cce9384..afeb44fce6 100644
--- a/app/Actions/Shared/DeleteScheduledVolumeBackup.php
+++ b/app/Actions/Shared/DeleteScheduledVolumeBackup.php
@@ -4,6 +4,7 @@ namespace App\Actions\Shared;
use App\Jobs\VolumeBackupJob;
use App\Models\ScheduledVolumeBackup;
+use App\Models\ScheduledVolumeBackupExecution;
use App\Models\Server;
use Illuminate\Support\Facades\Cache;
use Lorisleiva\Actions\Concerns\AsAction;
@@ -52,23 +53,10 @@ class DeleteScheduledVolumeBackup
}
if ($deleteS3Archives) {
- $s3Executions = $backup->executions()
- ->with('s3')
- ->where('s3_uploaded', true)
- ->where('s3_storage_deleted', false)
- ->get();
-
- foreach ($s3Executions->groupBy('s3_storage_id') as $executions) {
- $s3 = $executions->first()->s3;
- if (! $s3) {
- throw new \RuntimeException('The S3 storage used by an existing backup is unavailable.');
- }
-
- $filenames = $executions->pluck('filename')->filter()->all();
- if ($filenames !== []) {
- deleteBackupsS3($filenames, $s3);
- }
- }
+ $backup->executions()
+ ->whereHas('s3Replicas', fn ($query) => $query->where('s3_uploaded', true)->where('s3_storage_deleted', false))
+ ->get()
+ ->each(fn (ScheduledVolumeBackupExecution $execution) => $execution->deleteS3Copies());
}
$backup->delete();
diff --git a/app/Console/Commands/SeedDevelopmentQemuServerCommand.php b/app/Console/Commands/SeedDevelopmentQemuServerCommand.php
index 082ad83ae2..26fc463ff7 100644
--- a/app/Console/Commands/SeedDevelopmentQemuServerCommand.php
+++ b/app/Console/Commands/SeedDevelopmentQemuServerCommand.php
@@ -3,7 +3,9 @@
namespace App\Console\Commands;
use App\Actions\Development\SeedDevelopmentQemuServer;
+use App\Actions\Development\StartDevelopmentInstanceDatabase;
use Illuminate\Console\Command;
+use Throwable;
class SeedDevelopmentQemuServerCommand extends Command
{
@@ -22,6 +24,15 @@ class SeedDevelopmentQemuServerCommand extends Command
$server = SeedDevelopmentQemuServer::run($this->argument('profile'), ! $this->option('keep-others'), (bool) $this->option('as-localhost'), $this->option('ip') ?: null, (int) $this->option('port'));
$this->info("Seeded {$server->name} at {$server->ip}.");
+ if ($this->option('as-localhost')) {
+ try {
+ StartDevelopmentInstanceDatabase::run($server);
+ $this->info('Started the coolify-db container for instance backups.');
+ } catch (Throwable $e) {
+ $this->warn("Could not start the coolify-db container for instance backups: {$e->getMessage()}");
+ }
+ }
+
return self::SUCCESS;
}
}
diff --git a/app/Http/Controllers/Api/DatabasesController.php b/app/Http/Controllers/Api/DatabasesController.php
index d71b64563b..b6fe02d8bc 100644
--- a/app/Http/Controllers/Api/DatabasesController.php
+++ b/app/Http/Controllers/Api/DatabasesController.php
@@ -32,6 +32,8 @@ use Illuminate\Database\Eloquent\Collection as EloquentCollection;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Http\JsonResponse;
use Illuminate\Http\Request;
+use Illuminate\Support\Arr;
+use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Bus;
use Illuminate\Support\Facades\DB;
use OpenApi\Attributes as OA;
@@ -152,6 +154,58 @@ class DatabasesController extends Controller
}
}
+ /**
+ * Resolves the S3 destinations of a backup request, primary first. `s3_storage_uuids` lists every destination;
+ * `s3_storage_uuid` alone selects a single destination, and sent with `s3_storage_uuids` it must be in the list and
+ * becomes the primary destination.
+ *
+ * @return Collection<int, S3Storage>|JsonResponse|null Null when the request does not select destinations.
+ */
+ private function requestedBackupS3Storages(Request $request, int|string $teamId): Collection|JsonResponse|null
+ {
+ $hasList = $request->has('s3_storage_uuids');
+ if (! $hasList && ! $request->filled('s3_storage_uuid')) {
+ return null;
+ }
+
+ $uuids = collect($hasList ? $request->input('s3_storage_uuids') : [$request->input('s3_storage_uuid')])->unique()->values();
+ $primaryUuid = $request->input('s3_storage_uuid') ?? $uuids->first();
+ $errorField = $hasList ? 's3_storage_uuids' : 's3_storage_uuid';
+
+ if (filled($primaryUuid) && ! $uuids->contains($primaryUuid)) {
+ return response()->json([
+ 'message' => 'Validation failed.',
+ 'errors' => ['s3_storage_uuid' => ['The s3_storage_uuid must be one of the s3_storage_uuids.']],
+ ], 422);
+ }
+
+ $storages = S3Storage::ownedByCurrentTeamAPI($teamId)->whereIn('uuid', $uuids->all())->get();
+ if ($storages->count() !== $uuids->count()) {
+ return response()->json([
+ 'message' => 'Validation failed.',
+ 'errors' => [$errorField => ['The selected S3 storage is invalid for this team.']],
+ ], 422);
+ }
+
+ return $storages->sortBy(fn (S3Storage $storage): int => $storage->uuid === $primaryUuid ? 0 : 1)->values();
+ }
+
+ /**
+ * Adds the S3 destination UUIDs (primary first) to a serialized backup without exposing the storages themselves.
+ */
+ private function withBackupS3StorageUuids(ScheduledDatabaseBackup $backup): ScheduledDatabaseBackup
+ {
+ $storages = $backup->s3Storages->sortBy(fn (S3Storage $storage): int => $storage->id === $backup->s3_storage_id ? 0 : 1);
+ if ($storages->isEmpty() && $backup->s3) {
+ $storages = collect([$backup->s3]);
+ }
+
+ $backup->setAttribute('s3_storage_uuid', $backup->s3?->uuid);
+ $backup->setAttribute('s3_storage_uuids', $storages->pluck('uuid')->values()->all());
+
+ return $backup->unsetRelation('s3Storages')->unsetRelation('s3');
+ }
+
private function removeSensitiveData($database, bool $loadNestedServerSecrets = false)
{
$database->makeHidden([
@@ -302,9 +356,10 @@ class DatabasesController extends Controller
$databaseIds = $databases->pluck('id')->toArray();
- $backupConfigs = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId)->with('latest_log')
+ $backupConfigs = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId)->with(['latest_log', 's3', 's3Storages'])
->whereIn('database_id', $databaseIds)
->get()
+ ->map(fn (ScheduledDatabaseBackup $backup) => $this->withBackupS3StorageUuids($backup))
->groupBy(fn (ScheduledDatabaseBackup $backup) => $backup->database_type.':'.$backup->database_id);
$databases = $databases->map(function ($database) use ($backupConfigs) {
@@ -340,7 +395,7 @@ class DatabasesController extends Controller
responses: [
new OA\Response(
response: 200,
- description: 'Get all backups for a database',
+ description: 'Get all backups for a database. Each backup includes s3_storage_uuid (primary S3 destination) and s3_storage_uuids (every S3 destination, primary first).',
content: new OA\JsonContent(
type: 'string',
example: 'Content is very complex. Will be implemented later.',
@@ -376,7 +431,8 @@ class DatabasesController extends Controller
$this->authorize('view', $database);
- $backupConfig = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId)->with('executions')->where('database_id', $database->id)->get();
+ $backupConfig = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId)->with(['executions', 's3', 's3Storages'])->where('database_id', $database->id)->get()
+ ->map(fn (ScheduledDatabaseBackup $backup) => $this->withBackupS3StorageUuids($backup));
return response()->json($backupConfig);
}
@@ -873,7 +929,8 @@ class DatabasesController extends Controller
'frequency' => ['type' => 'string', 'description' => 'Backup frequency (cron expression or: every_minute, hourly, daily, weekly, monthly, yearly)'],
'enabled' => ['type' => 'boolean', 'description' => 'Whether the backup is enabled', 'default' => true],
'save_s3' => ['type' => 'boolean', 'description' => 'Whether to save backups to S3', 'default' => false],
- 's3_storage_uuid' => ['type' => 'string', 'description' => 'S3 storage UUID (required if save_s3 is true)'],
+ 's3_storage_uuid' => ['type' => 'string', 'description' => 'Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination. Either field is required if save_s3 is true.'],
+ 's3_storage_uuids' => ['type' => 'array', 'items' => ['type' => 'string'], 'description' => 'UUIDs of every S3 storage the backup is uploaded to. Replaces the current destinations. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination.'],
'databases_to_backup' => ['type' => 'string', 'description' => 'Comma separated list of databases to backup'],
'dump_all' => ['type' => 'boolean', 'description' => 'Whether to dump all databases', 'default' => false],
'backup_now' => ['type' => 'boolean', 'description' => 'Whether to trigger backup immediately after creation'],
@@ -921,7 +978,7 @@ class DatabasesController extends Controller
)]
public function create_backup(Request $request)
{
- $backupConfigFields = ['save_s3', 'enabled', 'dump_all', 'frequency', 'databases_to_backup', 'database_backup_retention_amount_locally', 'database_backup_retention_days_locally', 'database_backup_retention_max_storage_locally', 'database_backup_retention_amount_s3', 'database_backup_retention_days_s3', 'database_backup_retention_max_storage_s3', 's3_storage_uuid', 'timeout', 'missing_backup_notification_days'];
+ $backupConfigFields = ['save_s3', 'enabled', 'dump_all', 'frequency', 'databases_to_backup', 'database_backup_retention_amount_locally', 'database_backup_retention_days_locally', 'database_backup_retention_max_storage_locally', 'database_backup_retention_amount_s3', 'database_backup_retention_days_s3', 'database_backup_retention_max_storage_s3', 's3_storage_uuid', 's3_storage_uuids', 'timeout', 'missing_backup_notification_days'];
$teamId = getTeamIdFromToken();
if (is_null($teamId)) {
@@ -941,6 +998,8 @@ class DatabasesController extends Controller
'dump_all' => 'boolean',
'backup_now' => 'boolean|nullable',
's3_storage_uuid' => 'string|exists:s3_storages,uuid|nullable',
+ 's3_storage_uuids' => 'array',
+ 's3_storage_uuids.*' => 'string|distinct',
'databases_to_backup' => 'string|nullable',
'database_backup_retention_amount_locally' => 'integer|min:0',
'database_backup_retention_days_locally' => 'integer|min:0',
@@ -986,24 +1045,17 @@ class DatabasesController extends Controller
], 422);
}
- // Validate S3 storage if save_s3 is true
- if ($request->boolean('save_s3') && ! $request->filled('s3_storage_uuid')) {
+ $s3Storages = $this->requestedBackupS3Storages($request, $teamId);
+ if ($s3Storages instanceof JsonResponse) {
+ return $s3Storages;
+ }
+ if ($request->boolean('save_s3') && blank($s3Storages)) {
return response()->json([
'message' => 'Validation failed.',
- 'errors' => ['s3_storage_uuid' => ['The s3_storage_uuid field is required when save_s3 is true.']],
+ 'errors' => ['s3_storage_uuid' => ['The s3_storage_uuid or s3_storage_uuids field is required when save_s3 is true.']],
], 422);
}
- if ($request->filled('s3_storage_uuid')) {
- $existsInTeam = S3Storage::ownedByCurrentTeamAPI($teamId)->where('uuid', $request->s3_storage_uuid)->exists();
- if (! $existsInTeam) {
- return response()->json([
- 'message' => 'Validation failed.',
- 'errors' => ['s3_storage_uuid' => ['The selected S3 storage is invalid for this team.']],
- ], 422);
- }
- }
-
// Check for extra fields
$extraFields = array_diff(array_keys($request->all()), $backupConfigFields, ['backup_now']);
if (! empty($extraFields)) {
@@ -1018,20 +1070,9 @@ class DatabasesController extends Controller
], 422);
}
- $backupData = $request->only($backupConfigFields);
-
- // Convert s3_storage_uuid to s3_storage_id
- if (isset($backupData['s3_storage_uuid'])) {
- $s3Storage = S3Storage::ownedByCurrentTeamAPI($teamId)->where('uuid', $backupData['s3_storage_uuid'])->first();
- if ($s3Storage) {
- $backupData['s3_storage_id'] = $s3Storage->id;
- } elseif ($request->boolean('save_s3')) {
- return response()->json([
- 'message' => 'Validation failed.',
- 'errors' => ['s3_storage_uuid' => ['The selected S3 storage is invalid for this team.']],
- ], 422);
- }
- unset($backupData['s3_storage_uuid']);
+ $backupData = Arr::except($request->only($backupConfigFields), ['s3_storage_uuid', 's3_storage_uuids']);
+ if ($s3Storages !== null) {
+ $backupData['s3_storage_id'] = $s3Storages->first()?->id;
}
// Set default databases_to_backup based on database type if not provided
@@ -1070,6 +1111,9 @@ class DatabasesController extends Controller
}
$backupConfig = ScheduledDatabaseBackup::create($backupData);
+ if ($s3Storages !== null) {
+ $backupConfig->syncS3Storages($s3Storages->pluck('id')->all());
+ }
// Trigger immediate backup if requested
if ($request->backup_now) {
@@ -1129,7 +1173,8 @@ class DatabasesController extends Controller
type: 'object',
properties: [
'save_s3' => ['type' => 'boolean', 'description' => 'Whether data is saved in s3 or not'],
- 's3_storage_uuid' => ['type' => 'string', 'description' => 'S3 storage UUID'],
+ 's3_storage_uuid' => ['type' => 'string', 'description' => 'Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination; when it is already a destination, it becomes the primary destination and the other destinations stay.'],
+ 's3_storage_uuids' => ['type' => 'array', 'items' => ['type' => 'string'], 'description' => 'UUIDs of every S3 storage the backup is uploaded to. Replaces the current destinations. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination.'],
'backup_now' => ['type' => 'boolean', 'description' => 'Whether to take a backup now or not'],
'enabled' => ['type' => 'boolean', 'description' => 'Whether the backup is enabled or not'],
'databases_to_backup' => ['type' => 'string', 'description' => 'Comma separated list of databases to backup'],
@@ -1172,7 +1217,7 @@ class DatabasesController extends Controller
)]
public function update_backup(Request $request)
{
- $backupConfigFields = ['save_s3', 'enabled', 'dump_all', 'frequency', 'databases_to_backup', 'database_backup_retention_amount_locally', 'database_backup_retention_days_locally', 'database_backup_retention_max_storage_locally', 'database_backup_retention_amount_s3', 'database_backup_retention_days_s3', 'database_backup_retention_max_storage_s3', 's3_storage_uuid', 'timeout', 'missing_backup_notification_days'];
+ $backupConfigFields = ['save_s3', 'enabled', 'dump_all', 'frequency', 'databases_to_backup', 'database_backup_retention_amount_locally', 'database_backup_retention_days_locally', 'database_backup_retention_max_storage_locally', 'database_backup_retention_amount_s3', 'database_backup_retention_days_s3', 'database_backup_retention_max_storage_s3', 's3_storage_uuid', 's3_storage_uuids', 'timeout', 'missing_backup_notification_days'];
$teamId = getTeamIdFromToken();
if (is_null($teamId)) {
@@ -1189,6 +1234,8 @@ class DatabasesController extends Controller
'enabled' => 'boolean',
'dump_all' => 'boolean',
's3_storage_uuid' => 'string|exists:s3_storages,uuid|nullable',
+ 's3_storage_uuids' => 'array',
+ 's3_storage_uuids.*' => 'string|distinct',
'databases_to_backup' => 'string|nullable',
'frequency' => 'string',
'database_backup_retention_amount_locally' => 'integer|min:0',
@@ -1236,20 +1283,9 @@ class DatabasesController extends Controller
}
}
- if ($request->boolean('save_s3') && ! $request->filled('s3_storage_uuid')) {
- return response()->json([
- 'message' => 'Validation failed.',
- 'errors' => ['s3_storage_uuid' => ['The s3_storage_uuid field is required when save_s3 is true.']],
- ], 422);
- }
- if ($request->filled('s3_storage_uuid')) {
- $existsInTeam = S3Storage::ownedByCurrentTeamAPI($teamId)->where('uuid', $request->s3_storage_uuid)->exists();
- if (! $existsInTeam) {
- return response()->json([
- 'message' => 'Validation failed.',
- 'errors' => ['s3_storage_uuid' => ['The selected S3 storage is invalid for this team.']],
- ], 422);
- }
+ $s3Storages = $this->requestedBackupS3Storages($request, $teamId);
+ if ($s3Storages instanceof JsonResponse) {
+ return $s3Storages;
}
$backupConfig = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId)->where('database_id', $database->id)
@@ -1259,6 +1295,14 @@ class DatabasesController extends Controller
return response()->json(['message' => 'Backup config not found.'], 404);
}
+ $saveToS3 = $request->has('save_s3') ? $request->boolean('save_s3') : (bool) $backupConfig->save_s3;
+ if (($request->boolean('save_s3') && $s3Storages === null) || ($saveToS3 && $s3Storages?->isEmpty())) {
+ return response()->json([
+ 'message' => 'Validation failed.',
+ 'errors' => ['s3_storage_uuid' => ['The s3_storage_uuid or s3_storage_uuids field is required when save_s3 is true.']],
+ ], 422);
+ }
+
$extraFields = array_diff(array_keys($request->all()), $backupConfigFields, ['backup_now']);
if (! empty($extraFields)) {
$errors = $validator->errors();
@@ -1272,20 +1316,9 @@ class DatabasesController extends Controller
], 422);
}
- $backupData = $request->only($backupConfigFields);
-
- // Convert s3_storage_uuid to s3_storage_id
- if (isset($backupData['s3_storage_uuid'])) {
- $s3Storage = S3Storage::ownedByCurrentTeamAPI($teamId)->where('uuid', $backupData['s3_storage_uuid'])->first();
- if ($s3Storage) {
- $backupData['s3_storage_id'] = $s3Storage->id;
- } elseif ($request->boolean('save_s3')) {
- return response()->json([
- 'message' => 'Validation failed.',
- 'errors' => ['s3_storage_uuid' => ['The selected S3 storage is invalid for this team.']],
- ], 422);
- }
- unset($backupData['s3_storage_uuid']);
+ $backupData = Arr::except($request->only($backupConfigFields), ['s3_storage_uuid', 's3_storage_uuids']);
+ if ($s3Storages !== null) {
+ $backupData['s3_storage_id'] = $s3Storages->first()?->id;
}
// Validate databases_to_backup input
@@ -1300,7 +1333,19 @@ class DatabasesController extends Controller
}
}
+ $storageIds = $s3Storages?->pluck('id');
+ // An older client sends only s3_storage_uuid. When it is already a destination, the other destinations stay.
+ if ($storageIds !== null && ! $request->has('s3_storage_uuids')) {
+ $currentStorageIds = $backupConfig->selectedS3Storages()->pluck('id');
+ if ($currentStorageIds->contains($storageIds->first())) {
+ $storageIds = $currentStorageIds;
+ }
+ }
+
$backupConfig->update($backupData);
+ if ($storageIds !== null) {
+ $backupConfig->syncS3Storages($storageIds->all());
+ }
if ($request->backup_now) {
dispatch(new DatabaseBackupJob($backupConfig));
@@ -2938,9 +2983,10 @@ class DatabasesController extends Controller
if ($execution->filename) {
deleteBackupsLocally($execution->filename, $database->destination->server);
- if ($deleteS3 && $backup->s3) {
- deleteBackupsS3($execution->filename, $backup->s3);
- }
+ }
+
+ if ($deleteS3) {
+ $execution->deleteS3Copies();
}
$execution->delete();
@@ -3073,9 +3119,10 @@ class DatabasesController extends Controller
if ($execution->filename) {
deleteBackupsLocally($execution->filename, $database->destination->server);
- if ($deleteS3 && $backup->s3) {
- deleteBackupsS3($execution->filename, $backup->s3);
- }
+ }
+
+ if ($deleteS3) {
+ $execution->deleteS3Copies();
}
$execution->delete();
@@ -5195,7 +5242,7 @@ class DatabasesController extends Controller
}
foreach ($database->scheduledBackups()->get() as $backup) {
- $backup->replicate([
+ $newBackup = $backup->replicate([
'id',
'created_at',
'updated_at',
@@ -5205,7 +5252,9 @@ class DatabasesController extends Controller
'database_id' => $newDatabase->id,
'database_type' => $newDatabase->getMorphClass(),
'team_id' => $teamId,
- ])->save();
+ ]);
+ $newBackup->save();
+ $backup->copyS3StoragesTo($newBackup);
}
foreach ($database->environment_variables()->get() as $environmentVariable) {
diff --git a/app/Http/Controllers/Api/ServicesController.php b/app/Http/Controllers/Api/ServicesController.php
index 7411f8a472..01f3657434 100644
--- a/app/Http/Controllers/Api/ServicesController.php
+++ b/app/Http/Controllers/Api/ServicesController.php
@@ -3399,7 +3399,7 @@ class ServicesController extends Controller
}
foreach ($sourceDatabase->scheduledBackups()->get() as $backup) {
- $backup->replicate([
+ $newBackup = $backup->replicate([
'id',
'created_at',
'updated_at',
@@ -3408,7 +3408,9 @@ class ServicesController extends Controller
'database_id' => $database->id,
'database_type' => $database->getMorphClass(),
'team_id' => $teamId,
- ])->save();
+ ]);
+ $newBackup->save();
+ $backup->copyS3StoragesTo($newBackup);
}
}
diff --git a/app/Http/Controllers/Api/VolumeBackupsController.php b/app/Http/Controllers/Api/VolumeBackupsController.php
index e3b48a367a..a199f530db 100644
--- a/app/Http/Controllers/Api/VolumeBackupsController.php
+++ b/app/Http/Controllers/Api/VolumeBackupsController.php
@@ -14,6 +14,7 @@ use App\Models\Service;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Http\JsonResponse;
use Illuminate\Http\Request;
+use Illuminate\Support\Collection;
use Illuminate\Support\MessageBag;
use OpenApi\Attributes as OA;
use RuntimeException;
@@ -27,7 +28,8 @@ use RuntimeException;
new OA\Property(property: 'save_s3', type: 'boolean', default: false),
new OA\Property(property: 'disable_local_backup', type: 'boolean', default: false),
new OA\Property(property: 'stop_during_backup', type: 'boolean', default: false),
- new OA\Property(property: 's3_storage_uuid', type: 'string', nullable: true),
+ new OA\Property(property: 's3_storage_uuid', type: 'string', nullable: true, description: 'Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination.'),
+ new OA\Property(property: 's3_storage_uuids', type: 'array', items: new OA\Items(type: 'string'), description: 'UUIDs of every S3 storage the backup is uploaded to. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination. save_s3 requires at least one destination.'),
new OA\Property(property: 'retention_amount_locally', type: 'integer', default: 7, minimum: 0, maximum: 10000),
new OA\Property(property: 'retention_days_locally', type: 'integer', default: 0, maximum: 2147483647, minimum: 0),
new OA\Property(property: 'retention_max_storage_locally', type: 'number', format: 'float', default: 0, maximum: 9999999999, minimum: 0),
@@ -53,7 +55,8 @@ use RuntimeException;
new OA\Property(property: 'save_s3', type: 'boolean'),
new OA\Property(property: 'disable_local_backup', type: 'boolean'),
new OA\Property(property: 'stop_during_backup', type: 'boolean'),
- new OA\Property(property: 's3_storage_uuid', type: 'string', nullable: true),
+ new OA\Property(property: 's3_storage_uuid', type: 'string', nullable: true, description: 'Primary S3 destination.'),
+ new OA\Property(property: 's3_storage_uuids', type: 'array', items: new OA\Items(type: 'string'), description: 'Every S3 destination, primary first.'),
new OA\Property(property: 'retention_amount_locally', type: 'integer'),
new OA\Property(property: 'retention_days_locally', type: 'integer'),
new OA\Property(property: 'retention_max_storage_locally', type: 'number', format: 'float'),
@@ -165,7 +168,7 @@ class VolumeBackupsController extends Controller
return response()->json(['message' => 'Storage not found.'], 404);
}
- ['errors' => $errors, 's3Storage' => $s3Storage, 'saveToS3' => $saveToS3] = $this->validateUpsertRequest($request, $storage, $teamId);
+ ['errors' => $errors, 's3Storages' => $s3Storages, 'saveToS3' => $saveToS3] = $this->validateUpsertRequest($request, $storage, $teamId);
if ($errors->isNotEmpty()) {
return response()->json([
@@ -174,11 +177,11 @@ class VolumeBackupsController extends Controller
], 422);
}
- return $this->persistSchedule($request, $storage, $teamId, $s3Storage, $saveToS3, $resourceType, $resource);
+ return $this->persistSchedule($request, $storage, $teamId, $s3Storages, $saveToS3, $resourceType, $resource);
}
/**
- * @return array{errors: MessageBag, s3Storage: S3Storage|null, saveToS3: bool}
+ * @return array{errors: MessageBag, s3Storages: Collection<int, S3Storage>, saveToS3: bool}
*/
private function validateUpsertRequest(
Request $request,
@@ -192,6 +195,8 @@ class VolumeBackupsController extends Controller
'disable_local_backup' => 'boolean',
'stop_during_backup' => 'boolean',
's3_storage_uuid' => 'nullable|string',
+ 's3_storage_uuids' => 'array',
+ 's3_storage_uuids.*' => 'string|distinct',
'retention_amount_locally' => 'integer|min:0|max:10000',
'retention_days_locally' => 'integer|min:0|max:2147483647',
'retention_max_storage_locally' => 'numeric|min:0|max:9999999999',
@@ -209,6 +214,7 @@ class VolumeBackupsController extends Controller
'disable_local_backup',
'stop_during_backup',
's3_storage_uuid',
+ 's3_storage_uuids',
'retention_amount_locally',
'retention_days_locally',
'retention_max_storage_locally',
@@ -232,16 +238,24 @@ class VolumeBackupsController extends Controller
$errors->add('disable_local_backup', 'Local backups can only be disabled when S3 backups are enabled.');
}
- $s3Storage = null;
- if ($saveToS3) {
- $s3Storage = S3Storage::query()
+ $s3Storages = collect();
+ if ($saveToS3 && ! $errors->hasAny(['s3_storage_uuid', 's3_storage_uuids', 's3_storage_uuids.*'])) {
+ $hasList = $request->has('s3_storage_uuids');
+ $uuids = collect($hasList ? $request->input('s3_storage_uuids') : [$request->input('s3_storage_uuid')])->filter()->unique()->values();
+ $primaryUuid = $request->input('s3_storage_uuid') ?? $uuids->first();
+
+ $s3Storages = S3Storage::query()
->where('team_id', $teamId)
->where('is_usable', true)
- ->where('uuid', $request->input('s3_storage_uuid'))
- ->first();
-
- if (! $s3Storage) {
- $errors->add('s3_storage_uuid', 'Select a usable S3 storage owned by your team.');
+ ->whereIn('uuid', $uuids->all())
+ ->get()
+ ->sortBy(fn (S3Storage $s3Storage): int => $s3Storage->uuid === $primaryUuid ? 0 : 1)
+ ->values();
+
+ if ($hasList && filled($primaryUuid) && ! $uuids->contains($primaryUuid)) {
+ $errors->add('s3_storage_uuid', 'The s3_storage_uuid must be one of the s3_storage_uuids.');
+ } elseif ($uuids->isEmpty() || $s3Storages->count() !== $uuids->count()) {
+ $errors->add($hasList ? 's3_storage_uuids' : 's3_storage_uuid', 'Select a usable S3 storage owned by your team.');
}
}
@@ -251,7 +265,7 @@ class VolumeBackupsController extends Controller
return [
'errors' => $errors,
- 's3Storage' => $s3Storage,
+ 's3Storages' => $s3Storages,
'saveToS3' => $saveToS3,
];
}
@@ -260,7 +274,7 @@ class VolumeBackupsController extends Controller
Request $request,
LocalPersistentVolume|LocalFileVolume $storage,
int|string $teamId,
- ?S3Storage $s3Storage,
+ Collection $s3Storages,
bool $saveToS3,
string $resourceType,
Model $resource,
@@ -272,7 +286,7 @@ class VolumeBackupsController extends Controller
'save_s3' => $saveToS3,
'disable_local_backup' => $saveToS3 && $request->boolean('disable_local_backup'),
'stop_during_backup' => $request->boolean('stop_during_backup'),
- 's3_storage_id' => $s3Storage?->id,
+ 's3_storage_id' => $s3Storages->first()?->id,
'retention_amount_locally' => $request->integer('retention_amount_locally', 7),
'retention_days_locally' => $request->integer('retention_days_locally'),
'retention_max_storage_locally' => $request->float('retention_max_storage_locally'),
@@ -288,6 +302,7 @@ class VolumeBackupsController extends Controller
}
$backup = $storage->scheduledBackups()->updateOrCreate([], $attributes);
+ $backup->syncS3Storages($s3Storages->pluck('id')->all());
$created = $backup->wasRecentlyCreated;
auditLog('api.volume_backup.schedule_set', [
@@ -298,7 +313,7 @@ class VolumeBackupsController extends Controller
'backup_uuid' => $backup->uuid,
]);
- return response()->json($this->responseData($backup, $storage, $s3Storage, $created), $created ? 201 : 200);
+ return response()->json($this->responseData($backup, $storage, $s3Storages, $created), $created ? 201 : 200);
}
#[OA\Delete(
@@ -431,7 +446,7 @@ class VolumeBackupsController extends Controller
private function responseData(
ScheduledVolumeBackup $backup,
LocalPersistentVolume|LocalFileVolume $storage,
- ?S3Storage $s3Storage,
+ Collection $s3Storages,
bool $created,
): array {
return [
@@ -444,7 +459,8 @@ class VolumeBackupsController extends Controller
'save_s3' => $backup->save_s3,
'disable_local_backup' => $backup->disable_local_backup,
'stop_during_backup' => $backup->stop_during_backup,
- 's3_storage_uuid' => $s3Storage?->uuid,
+ 's3_storage_uuid' => $s3Storages->first()?->uuid,
+ 's3_storage_uuids' => $s3Storages->pluck('uuid')->all(),
'retention_amount_locally' => $backup->retention_amount_locally,
'retention_days_locally' => $backup->retention_days_locally,
'retention_max_storage_locally' => $backup->retention_max_storage_locally,
diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php
index 92d24bab3d..eb0959865b 100644
--- a/app/Jobs/DatabaseBackupJob.php
+++ b/app/Jobs/DatabaseBackupJob.php
@@ -83,8 +83,6 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
public ?string $mongo_root_password = null;
- public ?S3Storage $s3 = null;
-
public $timeout = 3600 + self::WORKER_TIMEOUT_MARGIN_SECONDS;
public ?string $backup_log_uuid = null;
@@ -129,11 +127,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
if (data_get($this->backup, 'database_type') === ServiceDatabase::class) {
$this->database = data_get($this->backup, 'database');
$this->server = $this->database->service->server;
- $this->s3 = $this->backup->s3;
} else {
$this->database = data_get($this->backup, 'database');
$this->server = $this->database->destination->server;
- $this->s3 = $this->backup->s3;
}
if (is_null($this->server)) {
throw new \Exception('Server not found?!');
@@ -329,6 +325,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$this->backup_location = null;
$this->backup_output = null;
$this->error_output = null;
+ $this->s3_uploaded = false;
// Generate unique UUID for each database backup execution
$attempts = 0;
@@ -484,10 +481,10 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$localStorageDeleted = false;
if ($this->backup->save_s3 && $localBackupSucceeded) {
try {
- $this->upload_to_s3();
+ $s3UploadError = $this->uploadToS3Destinations();
- // If local backup is disabled, delete the local file immediately after S3 upload
- if ($this->backup->disable_local_backup) {
+ // If local backup is disabled, delete the local file only when every destination has a copy
+ if ($s3UploadError === null && $this->backup->disable_local_backup) {
deleteBackupsLocally($this->backup_location, $this->server);
$localStorageDeleted = true;
}
@@ -843,9 +840,15 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
return devHostDockerPath($this->server, $this->backup_location);
}
- private function upload_to_s3(): void
+ /**
+ * Uploads the backup file to every selected S3 destination and records one replica per destination.
+ *
+ * @return string|null The failed destinations and their errors, or null when every upload succeeded.
+ */
+ private function uploadToS3Destinations(): ?string
{
- if (is_null($this->s3)) {
+ $storages = $this->backup->selectedS3Storages();
+ if ($storages->isEmpty()) {
$previousS3StorageId = $this->backup->s3_storage_id;
$this->backup->update([
@@ -856,13 +859,43 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
throw new \Exception('S3 storage configuration is missing or has been deleted (S3 storage ID: '.($previousS3StorageId ?? 'null').'). S3 backup has been disabled for this schedule.');
}
+ $failures = [];
+ foreach ($storages as $storage) {
+ $uploaded = false;
+ $uploadError = null;
+ try {
+ $this->upload_to_s3($storage);
+ $uploaded = true;
+ } catch (Throwable $e) {
+ $uploadError = $e->getMessage();
+ $failures[] = "{$storage->name}: {$uploadError}";
+ }
+
+ try {
+ $this->backup_log->s3Replicas()->create([
+ 's3_storage_id' => $storage->id,
+ 's3_uploaded' => $uploaded,
+ 'message' => $uploadError,
+ ]);
+ } catch (Throwable $e) {
+ $failures[] = "{$storage->name}: {$e->getMessage()}";
+ }
+ }
+
+ $this->backup_log->refreshS3Summary();
+ $this->s3_uploaded = $failures === [];
+
+ return $failures === [] ? null : implode("\n", $failures);
+ }
+
+ protected function upload_to_s3(S3Storage $s3): void
+ {
try {
- $key = $this->s3->key;
- $secret = $this->s3->secret;
- // $region = $this->s3->region;
- $bucket = $this->s3->bucket;
- $endpoint = $this->s3->endpoint;
- $this->s3->testConnection(shouldSave: true);
+ $key = $s3->key;
+ $secret = $s3->secret;
+ $bucket = $s3->bucket;
+ $endpoint = $s3->endpoint;
+ $s3->testConnection(shouldSave: true);
if (data_get($this->backup, 'database_type') === ServiceDatabase::class) {
$network = $this->database->service->destination->network;
} else {
@@ -883,7 +916,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$escapedSecret = escapeshellarg($secret);
$escapedBackupLocation = escapeshellarg($this->backup_location);
$escapedS3Destination = escapeshellarg("temporary/{$bucket}{$this->backup_dir}/");
- $resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($endpoint, $this->s3->trustedInternalHosts()))
+ $resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($endpoint, $s3->trustedInternalHosts()))
->map(fn (string $resolveOption): string => '--resolve '.escapeshellarg($resolveOption))
->implode(' ');
$resolveOptions = $resolveOptions === '' ? '' : ' '.$resolveOptions;
@@ -892,9 +925,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc cp {$escapedBackupLocation} {$escapedS3Destination}";
instant_remote_process($commands, $this->server, true, false, $this->commandTimeout(), disableMultiplexing: true);
- $this->s3_uploaded = true;
} catch (Throwable $e) {
- $this->s3_uploaded = false;
$this->add_to_error_output($e->getMessage());
throw $e;
} finally {
diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.php
index d635894b09..db226e5675 100644
--- a/app/Jobs/VolumeBackupJob.php
+++ b/app/Jobs/VolumeBackupJob.php
@@ -4,9 +4,11 @@ namespace App\Jobs;
use App\Events\BackupCreated;
use App\Models\LocalPersistentVolume;
+use App\Models\S3Storage;
use App\Models\ScheduledVolumeBackup;
use App\Models\ScheduledVolumeBackupExecution;
use App\Models\Server;
+use App\Models\VolumeBackupS3Replica;
use App\Notifications\VolumeBackup\BackupFailed;
use App\Notifications\VolumeBackup\BackupSuccess;
use App\Rules\SafeWebhookUrl;
@@ -16,7 +18,6 @@ use Carbon\Carbon;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
-use Illuminate\Database\Eloquent\Builder;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
@@ -71,16 +72,21 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
throw new \RuntimeException('The storage backup resource, team, or server no longer exists.');
}
+ $destinations = $this->backup->save_s3 ? $this->backup->selectedS3Storages() : collect();
$this->execution = $this->backup->executions()->create([
- 's3_storage_id' => $this->backup->save_s3 ? $this->backup->s3_storage_id : null,
+ 's3_storage_id' => $destinations->first()?->id,
]);
+ foreach ($destinations as $s3) {
+ $this->execution->s3Replicas()->create(['s3_storage_id' => $s3->id]);
+ }
BackupCreated::dispatch($team->id);
$backupDirectory = backup_dir().'/volumes/'.str($team->name)->slug().'-'.$team->id.'/'.$target->uuid;
$filename = str($this->backup->targetType())->lower().'-'.str($this->backup->targetName())->slug().'-'.Carbon::now()->timestamp.'.tar.gz';
$backupLocation = $backupDirectory.'/'.$filename;
$this->execution->update(['filename' => $backupLocation]);
- $streamToS3 = $this->backup->save_s3 && $this->backup->disable_local_backup;
+ // Streaming writes no local archive, so it can upload to one destination only.
+ $streamToS3 = $this->backup->disable_local_backup && $destinations->count() === 1;
try {
$source = $this->backup->sourcePath();
@@ -96,7 +102,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
$archiveScript = "compressor=\$({$compressorCommand}); tar -I \"\$compressor\" -cf - -C /volume .";
if ($streamToS3) {
$this->execution->update(['local_storage_deleted' => true]);
- $archiveCommand = $this->streamToS3Command($archiveScript, $backupLocation, $source, $containerName, $image);
+ $archiveCommand = $this->streamToS3Command($destinations->first(), $archiveScript, $backupLocation, $source, $containerName, $image);
$this->execution->update(['s3_cleanup_pending' => true]);
} else {
$archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName)
@@ -141,24 +147,27 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
}
$warning = null;
- $s3Uploaded = null;
+ $s3Uploaded = false;
$s3CleanupPending = false;
$localStorageDeleted = $streamToS3;
if ($streamToS3) {
$s3Uploaded = true;
+ $this->execution->s3Replicas()->update(['s3_uploaded' => true]);
+ $this->execution->refreshS3Summary();
$this->execution->update(['s3_cleanup_pending' => false]);
+ } elseif ($this->backup->save_s3 && $destinations->isEmpty()) {
+ $warning = 'S3 upload failed: The selected S3 storage no longer exists.';
} elseif ($this->backup->save_s3) {
$s3CleanupPending = true;
$this->execution->update(['s3_cleanup_pending' => true]);
+ $uploadErrors = $this->uploadToDestinations($backupLocation, $backupDirectory, $server);
+ $s3Uploaded = $uploadErrors === [];
- try {
- $this->uploadToS3($backupLocation, $backupDirectory, $server);
- $s3Uploaded = true;
+ if ($s3Uploaded) {
$s3CleanupPending = false;
- } catch (Throwable $exception) {
- $s3Uploaded = false;
- $warning = 'S3 upload failed: '.$exception->getMessage();
+ } else {
+ $warning = 'S3 upload failed: '.implode(' ', $uploadErrors);
try {
VolumeBackupRecoveryJob::cleanupS3Upload($this->execution);
@@ -184,7 +193,6 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
'message' => $warning,
'size' => $size,
'filename' => $backupLocation,
- 's3_uploaded' => $s3Uploaded,
's3_cleanup_pending' => $s3CleanupPending,
'local_storage_deleted' => $localStorageDeleted,
]);
@@ -220,6 +228,8 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
}
}
+ $this->execution->s3Replicas()->whereNull('s3_uploaded')->update(['s3_uploaded' => false]);
+ $this->execution->refreshS3Summary();
$s3CleanupPending = $this->execution->fresh()->s3_cleanup_pending;
$this->execution->update([
@@ -347,16 +357,37 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
}
}
- private function uploadToS3(string $backupLocation, string $backupDirectory, Server $server): void
+ /**
+ * Uploads the local archive to every destination of the execution and records the result on each copy.
+ *
+ * @return array<int, string> One error message per failed destination.
+ */
+ private function uploadToDestinations(string $backupLocation, string $backupDirectory, Server $server): array
{
- $s3 = $this->backup->s3;
+ $errors = [];
- if (! $s3) {
- $this->backup->update(['save_s3' => false, 's3_storage_id' => null]);
+ foreach ($this->execution->s3Replicas()->with('s3')->get() as $replica) {
+ /** @var VolumeBackupS3Replica $replica */
+ try {
+ if (! $replica->s3) {
+ throw new \RuntimeException('The selected S3 storage no longer exists.');
+ }
- throw new \RuntimeException('The selected S3 storage no longer exists. S3 backup has been disabled.');
+ $this->uploadToS3($replica->s3, $backupLocation, $backupDirectory, $server);
+ $replica->update(['s3_uploaded' => true, 'message' => null]);
+ } catch (Throwable $exception) {
+ $replica->update(['s3_uploaded' => false, 'message' => $exception->getMessage()]);
+ $errors[] = ($replica->s3?->name ?? 'Deleted storage').': '.$exception->getMessage();
+ }
}
+ $this->execution->refreshS3Summary();
+
+ return $errors;
+ }
+
+ private function uploadToS3(S3Storage $s3, string $backupLocation, string $backupDirectory, Server $server): void
+ {
$s3->testConnection(shouldSave: true);
$containerName = 'volume-upload-'.$this->execution->uuid;
$image = coolifyHelperImage().':'.getHelperVersion();
@@ -385,16 +416,8 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
}
}
- private function streamToS3Command(string $archiveScript, string $backupLocation, string $source, string $containerName, string $image): string
+ private function streamToS3Command(S3Storage $s3, string $archiveScript, string $backupLocation, string $source, string $containerName, string $image): string
{
- $s3 = $this->backup->s3;
-
- if (! $s3) {
- $this->backup->update(['save_s3' => false, 's3_storage_id' => null]);
-
- throw new \RuntimeException('The selected S3 storage no longer exists. S3 backup has been disabled.');
- }
-
$s3->testConnection(shouldSave: true);
$resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($s3->endpoint, $s3->trustedInternalHosts()))
->map(fn (string $option): string => '--resolve '.escapeshellarg($option))
@@ -462,36 +485,45 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
}
}
- if ($this->backup->save_s3 && $this->backup->s3 && $this->hasRetentionLimits(
+ if ($this->backup->save_s3 && $this->hasRetentionLimits(
$this->backup->retention_amount_s3,
$this->backup->retention_days_s3,
$this->backup->retention_max_storage_s3,
)) {
+ $liveCopies = fn ($query) => $query->where('s3_uploaded', true)->where('s3_storage_deleted', false);
$s3Executions = $this->backup->executions()
- ->with('s3')
+ ->with(['s3Replicas' => fn ($query) => $liveCopies($query)->with('s3')])
->where('status', 'success')
- ->where('s3_uploaded', true)
- ->where('s3_storage_deleted', false)
+ ->whereHas('s3Replicas', $liveCopies)
->get();
- $s3Executions = $this->executionsOutsideRetention(
- $s3Executions,
- $this->backup->retention_amount_s3,
- $this->backup->retention_days_s3,
- $this->backup->retention_max_storage_s3,
- );
- foreach ($s3Executions->groupBy('s3_storage_id') as $executions) {
- $s3 = $executions->first()->s3;
+ // Each destination keeps its own copies within the same retention settings.
+ foreach ($s3Executions->flatMap->s3Replicas->pluck('s3_storage_id')->unique() as $storageId) {
+ $executions = $this->executionsOutsideRetention(
+ $s3Executions->filter(fn (ScheduledVolumeBackupExecution $execution): bool => $execution->s3Replicas->contains('s3_storage_id', $storageId))->values(),
+ $this->backup->retention_amount_s3,
+ $this->backup->retention_days_s3,
+ $this->backup->retention_max_storage_s3,
+ );
+ if ($executions->isEmpty()) {
+ continue;
+ }
+
+ $replicas = $executions->map(fn (ScheduledVolumeBackupExecution $execution): VolumeBackupS3Replica => $execution->s3Replicas->firstWhere('s3_storage_id', $storageId));
+ $s3 = $replicas->first()->s3;
if (! $s3) {
- throw new \RuntimeException('The S3 storage used by an existing backup is unavailable.');
+ VolumeBackupS3Replica::query()->whereKey($replicas->pluck('id')->all())->update(['s3_storage_deleted' => true]);
+ $executions->each(fn (ScheduledVolumeBackupExecution $execution) => $execution->refreshS3Summary());
+
+ continue;
}
$filenames = $executions->pluck('filename')->filter()->all();
if ($filenames !== []) {
deleteBackupsS3($filenames, $s3);
- $this->backup->executions()->whereKey($executions->pluck('id')->all())
- ->update(['s3_storage_deleted' => true]);
}
+ VolumeBackupS3Replica::query()->whereKey($replicas->pluck('id')->all())->update(['s3_storage_deleted' => true]);
+ $executions->each(fn (ScheduledVolumeBackupExecution $execution) => $execution->refreshS3Summary());
}
}
@@ -499,9 +531,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
->where('local_storage_deleted', true)
->where('stop_recovery_pending', false)
->where('s3_cleanup_pending', false)
- ->where(function (Builder $query): void {
- $query->where('s3_storage_deleted', true)->orWhereNull('s3_uploaded');
- })
+ ->withoutLiveS3Copies()
->delete();
}
diff --git a/app/Jobs/VolumeBackupRecoveryJob.php b/app/Jobs/VolumeBackupRecoveryJob.php
index cda0fdd046..4965df1379 100644
--- a/app/Jobs/VolumeBackupRecoveryJob.php
+++ b/app/Jobs/VolumeBackupRecoveryJob.php
@@ -295,12 +295,19 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, Shou
]);
}
+ /**
+ * Deletes the partial upload from every destination whose copy did not finish uploading. Every destination is
+ * tried; the first failure is rethrown afterwards and keeps the cleanup pending.
+ */
public static function cleanupS3Upload(ScheduledVolumeBackupExecution $execution): void
{
- $execution->loadMissing('s3');
- $s3 = $execution->s3;
+ $replicas = $execution->s3Replicas()
+ ->with('s3')
+ ->where('s3_storage_deleted', false)
+ ->where(fn ($query) => $query->whereNull('s3_uploaded')->orWhere('s3_uploaded', false))
+ ->get();
- if (! $s3 || blank($execution->filename)) {
+ if ($replicas->isEmpty() || blank($execution->filename)) {
self::skipRecovery(
$execution,
['s3_cleanup_pending' => false],
@@ -310,11 +317,25 @@ class VolumeBackupRecoveryJob implements ShouldBeEncrypted, ShouldBeUnique, Shou
return;
}
- deleteBackupsS3($execution->filename, $s3);
- $execution->update([
- 's3_cleanup_pending' => false,
- 's3_storage_deleted' => true,
- ]);
+ $failure = null;
+ foreach ($replicas as $replica) {
+ try {
+ if ($replica->s3) {
+ deleteBackupsS3($execution->filename, $replica->s3);
+ }
+ $replica->update(['s3_storage_deleted' => true]);
+ } catch (Throwable $exception) {
+ $failure ??= $exception;
+ }
+ }
+
+ $execution->refreshS3Summary();
+
+ if ($failure) {
+ throw $failure;
+ }
+
+ $execution->update(['s3_cleanup_pending' => false]);
}
/**
diff --git a/app/Livewire/Project/Application/Backup/Index.php b/app/Livewire/Project/Application/Backup/Index.php
index fe7e2eac5d..7ca097db96 100644
--- a/app/Livewire/Project/Application/Backup/Index.php
+++ b/app/Livewire/Project/Application/Backup/Index.php
@@ -31,7 +31,7 @@ class Index extends Component
public function render(): View
{
$backups = ScheduledVolumeBackup::query()
- ->with(['backupable', 'latestExecution', 's3'])
+ ->with(['backupable', 'latestExecution', 's3', 's3Storages'])
->withCount('executions')
->forApplication($this->application)
->latest()
diff --git a/app/Livewire/Project/CloneMe.php b/app/Livewire/Project/CloneMe.php
index 8f2ff9af40..b310c248ee 100644
--- a/app/Livewire/Project/CloneMe.php
+++ b/app/Livewire/Project/CloneMe.php
@@ -279,6 +279,7 @@ class CloneMe extends Component
'team_id' => $teamId,
]);
$newBackup->save();
+ $backup->copyS3StoragesTo($newBackup);
}
$environmentVaribles = $database->environment_variables()->get();
@@ -469,6 +470,7 @@ class CloneMe extends Component
'team_id' => $teamId,
]);
$newBackup->save();
+ $backup->copyS3StoragesTo($newBackup);
}
}
diff --git a/app/Livewire/Project/Database/BackupEdit.php b/app/Livewire/Project/Database/BackupEdit.php
index 59dd1a29c0..a7c49eeb58 100644
--- a/app/Livewire/Project/Database/BackupEdit.php
+++ b/app/Livewire/Project/Database/BackupEdit.php
@@ -9,6 +9,7 @@ use App\Models\ServiceDatabase;
use App\Models\StandalonePostgresql;
use App\Traits\ListensToTeamChannel;
use Exception;
+use Illuminate\Database\Eloquent\Builder;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Auth;
@@ -76,8 +77,9 @@ class BackupEdit extends Component
#[Validate(['required', 'boolean'])]
public bool $disableLocalBackup = false;
- #[Validate(['nullable', 'integer'])]
- public ?int $s3StorageId = null;
+ /** @var array<int, int|string> */
+ #[Validate(['array'])]
+ public array $s3StorageIds = [];
#[Validate(['nullable', 'string'])]
public ?string $databasesToBackup = null;
@@ -113,6 +115,10 @@ class BackupEdit extends Component
{
try {
$this->authorize('view', $this->backup->database);
+ $this->availableS3Storages = S3Storage::query()
+ ->whereKey($this->availableS3StorageIds()->all())
+ ->orderBy('name')
+ ->get();
$this->parameters = get_route_parameters();
$this->syncData();
$this->refreshStatus();
@@ -148,7 +154,6 @@ class BackupEdit extends Component
$this->backup->database_backup_retention_max_storage_s3 = $this->databaseBackupRetentionMaxStorageS3;
$this->backup->save_s3 = $this->saveS3;
$this->backup->disable_local_backup = $this->disableLocalBackup;
- $this->backup->s3_storage_id = $this->s3StorageId;
// Validate databases_to_backup to prevent command injection
// Handles all formats including MongoDB's "db:col1,col2|db2:col3"
@@ -160,10 +165,16 @@ class BackupEdit extends Component
$this->backup->dump_all = $this->dumpAll;
$this->backup->timeout = $this->timeout;
$this->backup->missing_backup_notification_days = $this->missingBackupNotificationDays;
- $this->customValidate();
+ $storageIds = $this->customValidate();
$changedFields = auditChangedFields($this->backup);
$this->backup->save();
+ $previousStorageIds = $this->backup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all();
+ $this->backup->syncS3Storages($storageIds->all());
+ if ($previousStorageIds !== $storageIds->sort()->values()->all()) {
+ $changedFields[] = 's3_storages';
+ }
+
return $changedFields;
} else {
$this->backupEnabled = $this->backup->enabled;
@@ -177,7 +188,8 @@ class BackupEdit extends Component
$this->databaseBackupRetentionMaxStorageS3 = $this->backup->database_backup_retention_max_storage_s3;
$this->saveS3 = $this->backup->save_s3;
$this->disableLocalBackup = $this->backup->disable_local_backup ?? false;
- $this->s3StorageId = $this->backup->s3_storage_id ?? $this->availableS3StorageIds()->first();
+ $selectedIds = $this->backup->selectedS3Storages()->pluck('id');
+ $this->s3StorageIds = ($selectedIds->isNotEmpty() ? $selectedIds : $this->availableS3StorageIds()->take(1))->all();
$this->databasesToBackup = $this->backup->databases_to_backup;
$this->dumpAll = $this->backup->dump_all;
$this->timeout = $this->backup->timeout;
@@ -216,10 +228,10 @@ class BackupEdit extends Component
if ($this->delete_associated_backups_locally && $server) {
deleteBackupsLocally($filenames, $server);
}
+ }
- if ($this->delete_associated_backups_s3 && $this->backup->s3) {
- deleteBackupsS3($filenames, $this->backup->s3);
- }
+ if ($this->delete_associated_backups_s3) {
+ $this->backup->executions()->each(fn ($execution) => $execution->deleteS3Copies());
}
$backupUuid = $this->backup->uuid;
@@ -333,14 +345,14 @@ class BackupEdit extends Component
}
}
- public function updatedS3StorageId(): void
+ public function updatedS3StorageIds(): void
{
$this->instantSave();
}
public function toggleS3(): void
{
- if (! $this->saveS3 && $this->availableS3StorageIds()->isEmpty()) {
+ if (! $this->saveS3 && $this->selectedAvailableS3StorageIds()->isEmpty()) {
$this->dispatch('error', 'Select a usable S3 storage before enabling S3 backups.');
return;
@@ -351,21 +363,22 @@ class BackupEdit extends Component
$this->instantSave();
}
- private function customValidate()
+ /**
+ * @return Collection<int, int> The usable S3 storage IDs to keep as destinations.
+ */
+ private function customValidate(): Collection
{
- if (! is_numeric($this->backup->s3_storage_id)) {
- $this->backup->s3_storage_id = null;
- }
+ // Only usable S3 storages of the database's team can be destinations; others are dropped.
+ $storageIds = $this->selectedAvailableS3StorageIds();
+ $this->s3StorageIds = $storageIds->all();
- // S3 backup cannot be enabled without a valid S3 storage owned by the team
- $availableS3Ids = $this->availableS3StorageIds();
- if ($availableS3Ids->isEmpty()) {
- $this->backup->s3_storage_id = $this->s3StorageId = null;
- if ($this->backup->save_s3) {
- $this->backup->save_s3 = $this->saveS3 = false;
- }
- } elseif (! $availableS3Ids->contains($this->backup->s3_storage_id)) {
- $this->backup->s3_storage_id = $this->s3StorageId = $availableS3Ids->first();
+ if ($this->availableS3StorageIds()->isEmpty()) {
+ $this->backup->save_s3 = $this->saveS3 = false;
+ } elseif ($this->backup->save_s3 && $storageIds->isEmpty()) {
+ $message = 'Select at least one usable S3 storage.';
+ $this->addError('s3StorageIds', $message);
+
+ throw new Exception($message);
}
// Validate that disable_local_backup can only be true when S3 backup is enabled
@@ -378,6 +391,8 @@ class BackupEdit extends Component
throw new Exception('Invalid Cron / Human expression');
}
$this->validate();
+
+ return $storageIds;
}
/**
@@ -399,6 +414,20 @@ class BackupEdit extends Component
]);
}
+ /**
+ * @return Collection<int, int>
+ */
+ private function selectedAvailableS3StorageIds(): Collection
+ {
+ $availableIds = $this->availableS3StorageIds();
+
+ return collect($this->s3StorageIds)
+ ->map(fn ($id): int => (int) $id)
+ ->filter(fn (int $id): bool => $availableIds->contains($id))
+ ->unique()
+ ->values();
+ }
+
private function availableS3StorageIds(): Collection
{
$database = $this->backup->database;
@@ -409,9 +438,12 @@ class BackupEdit extends Component
return collect();
}
+ // Current destinations stay selectable while unusable, so saving other settings does not remove them.
+ $selectedIds = $this->backup->selectedS3Storages()->pluck('id')->all();
+
return S3Storage::query()
->where('team_id', $teamId)
- ->where('is_usable', true)
+ ->where(fn (Builder $query) => $query->where('is_usable', true)->orWhereIn('id', $selectedIds))
->pluck('id');
}
@@ -432,7 +464,7 @@ class BackupEdit extends Component
return view('livewire.project.database.backup-edit', [
'checkboxes' => [
['id' => 'delete_associated_backups_locally', 'label' => __('database.delete_backups_locally')],
- ['id' => 'delete_associated_backups_s3', 'label' => 'All backups will be permanently deleted (associated with this backup job) from the selected S3 Storage.'],
+ ['id' => 'delete_associated_backups_s3', 'label' => 'All backups will be permanently deleted (associated with this backup job) from every S3 storage of this schedule.'],
// ['id' => 'delete_associated_backups_sftp', 'label' => 'All backups associated with this backup job from this database will be permanently deleted from the selected SFTP Storage.']
],
]);
diff --git a/app/Livewire/Project/Database/BackupExecutions.php b/app/Livewire/Project/Database/BackupExecutions.php
index 6f17e2fcf5..f9d4658688 100644
--- a/app/Livewire/Project/Database/BackupExecutions.php
+++ b/app/Livewire/Project/Database/BackupExecutions.php
@@ -109,12 +109,8 @@ class BackupExecutions extends Component
deleteBackupsLocally($execution->filename, $server, throwError: true);
}
- if ($deleteFromS3 && $execution->s3_uploaded && ! $execution->s3_storage_deleted) {
- if (! $execution->scheduledDatabaseBackup->s3) {
- throw new \RuntimeException('The S3 storage is unavailable.');
- }
-
- deleteBackupsS3($execution->filename, $execution->scheduledDatabaseBackup->s3);
+ if ($deleteFromS3) {
+ $execution->deleteS3Copies();
}
$execution->delete();
diff --git a/app/Livewire/Project/Database/ScheduledBackups.php b/app/Livewire/Project/Database/ScheduledBackups.php
index ece99b45f4..2071337b4b 100644
--- a/app/Livewire/Project/Database/ScheduledBackups.php
+++ b/app/Livewire/Project/Database/ScheduledBackups.php
@@ -73,7 +73,7 @@ class ScheduledBackups extends Component
public function render(): View
{
- $this->database->loadMissing('scheduledBackups.s3');
+ $this->database->loadMissing(['scheduledBackups.s3', 'scheduledBackups.s3Storages']);
return view('livewire.project.database.scheduled-backups');
}
diff --git a/app/Livewire/Project/Service/BackupExecutions.php b/app/Livewire/Project/Service/BackupExecutions.php
index 81be0ff8ad..b319f66412 100644
--- a/app/Livewire/Project/Service/BackupExecutions.php
+++ b/app/Livewire/Project/Service/BackupExecutions.php
@@ -114,7 +114,7 @@ class BackupExecutions extends Component
private function formatExecutions(Collection $rows): Collection
{
$databaseExecutions = ScheduledDatabaseBackupExecution::query()
- ->with(['scheduledDatabaseBackup.database', 'scheduledDatabaseBackup.s3'])
+ ->with(['scheduledDatabaseBackup.database', 'scheduledDatabaseBackup.s3', 'scheduledDatabaseBackup.s3Storages'])
->whereIn('id', $rows->where('type', 'database')->pluck('id'))
->get()->keyBy('id');
$volumeExecutions = ScheduledVolumeBackupExecution::query()
@@ -128,11 +128,16 @@ class BackupExecutions extends Component
$isDatabase = $row->type === 'database';
$execution = $isDatabase ? $databaseExecutions->get($row->id) : $volumeExecutions->get($row->id);
$schedule = $isDatabase ? $execution->scheduledDatabaseBackup : $execution->scheduledVolumeBackup;
- $storage = $isDatabase ? ($schedule->save_s3 ? $schedule->s3 : null) : $execution->s3;
- if ($storage?->team_id !== $serviceTeamId) {
- $storage = null;
- }
- $storageLabel = $storage ? $storage->name.' (bucket: '.$storage->bucket.')' : 'Unavailable';
+ $storages = match (true) {
+ ! $isDatabase => collect([$execution->s3]),
+ ! $schedule->save_s3 => collect(),
+ $schedule->s3Storages->isNotEmpty() => $schedule->s3Storages,
+ default => collect([$schedule->s3]),
+ };
+ $storageLabel = $storages
+ ->filter(fn ($storage) => $storage?->team_id === $serviceTeamId)
+ ->map(fn ($storage) => $storage->name.' (bucket: '.$storage->bucket.')')
+ ->implode(', ') ?: 'Unavailable';
if ($isDatabase && ! $schedule->save_s3) {
$storageLabel = 'Not configured';
} elseif (! $isDatabase && ! $execution->s3_storage_id && ! $execution->s3_uploaded && ! $execution->s3_storage_deleted) {
diff --git a/app/Livewire/Project/Service/VolumeBackup/Index.php b/app/Livewire/Project/Service/VolumeBackup/Index.php
index 60a78e5c15..64dbbb8df2 100644
--- a/app/Livewire/Project/Service/VolumeBackup/Index.php
+++ b/app/Livewire/Project/Service/VolumeBackup/Index.php
@@ -118,7 +118,7 @@ class Index extends Component
);
$databaseBackups = ScheduledDatabaseBackup::query()
- ->with(['database', 'latest_log', 's3'])
+ ->with(['database', 'latest_log', 's3', 's3Storages'])
->withCount('executions')
->where('database_type', (new ServiceDatabase)->getMorphClass())
->whereHasMorph('database', [ServiceDatabase::class], fn ($query) => $query->where('service_id', $this->service->id))
@@ -126,7 +126,7 @@ class Index extends Component
->get();
$backups = ScheduledVolumeBackup::query()
- ->with(['backupable.resource', 'latestExecution', 's3'])
+ ->with(['backupable.resource', 'latestExecution', 's3', 's3Storages'])
->withCount('executions')
->forService($this->service)
->latest()
diff --git a/app/Livewire/Project/Shared/ResourceOperations.php b/app/Livewire/Project/Shared/ResourceOperations.php
index 2b6f5b4587..a459b988e0 100644
--- a/app/Livewire/Project/Shared/ResourceOperations.php
+++ b/app/Livewire/Project/Shared/ResourceOperations.php
@@ -232,6 +232,7 @@ class ResourceOperations extends Component
'team_id' => $this->resourceTeamId(),
]);
$newBackup->save();
+ $backup->copyS3StoragesTo($newBackup);
}
$environmentVaribles = $this->resource->environment_variables()->get();
diff --git a/app/Livewire/Project/Shared/Storages/VolumeBackups.php b/app/Livewire/Project/Shared/Storages/VolumeBackups.php
index d741bcc0e4..083585fee6 100644
--- a/app/Livewire/Project/Shared/Storages/VolumeBackups.php
+++ b/app/Livewire/Project/Shared/Storages/VolumeBackups.php
@@ -11,6 +11,7 @@ use App\Models\LocalPersistentVolume;
use App\Models\S3Storage;
use App\Models\ScheduledVolumeBackup;
use App\Models\Service;
+use Illuminate\Database\Eloquent\Builder;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Illuminate\Http\RedirectResponse;
use Illuminate\Routing\Redirector;
@@ -43,7 +44,8 @@ class VolumeBackups extends Component
public bool $stopDuringBackup = false;
- public ?int $s3StorageId = null;
+ /** @var array<int, int|string> */
+ public array $s3StorageIds = [];
public int $retentionAmountLocally = 7;
@@ -88,7 +90,7 @@ class VolumeBackups extends Component
'saveToS3' => ['required', 'boolean'],
'disableLocalBackup' => ['required', 'boolean'],
'stopDuringBackup' => ['required', 'boolean'],
- 's3StorageId' => ['nullable', 'integer'],
+ 's3StorageIds' => ['array'],
'retentionAmountLocally' => ['required', 'integer', 'min:0', 'max:10000'],
'retentionDaysLocally' => ['required', 'integer', 'min:0'],
'retentionMaxStorageLocally' => ['required', 'numeric', 'min:0'],
@@ -103,12 +105,8 @@ class VolumeBackups extends Component
public function mount(): void
{
$this->authorize('view', $this->resource);
- $this->availableS3Storages = S3Storage::query()
- ->where('team_id', $this->resourceTeamId())
- ->where('is_usable', true)
- ->orderBy('name')
- ->get();
$this->backup = $this->storage->scheduledBackups()->first();
+ $this->availableS3Storages = $this->selectableS3Storages()->orderBy('name')->get();
$server = $this->backup?->server() ?? data_get($this->resource, 'destination.server');
$this->timezone = data_get($server, 'settings.server_timezone', 'Instance timezone');
@@ -118,7 +116,8 @@ class VolumeBackups extends Component
$this->saveToS3 = $this->backup->save_s3;
$this->disableLocalBackup = $this->backup->disable_local_backup;
$this->stopDuringBackup = $this->backup->stop_during_backup;
- $this->s3StorageId = $this->backup->s3_storage_id ?? $this->availableS3Storages->first()?->id;
+ $selectedIds = $this->backup->selectedS3Storages()->pluck('id');
+ $this->s3StorageIds = ($selectedIds->isNotEmpty() ? $selectedIds : $this->availableS3Storages->take(1)->pluck('id'))->all();
$this->retentionAmountLocally = $this->backup->retention_amount_locally;
$this->retentionDaysLocally = $this->backup->retention_days_locally;
$this->retentionMaxStorageLocally = $this->backup->retention_max_storage_locally;
@@ -128,7 +127,7 @@ class VolumeBackups extends Component
$this->timeout = $this->backup->timeout;
$this->missingBackupNotificationDays = $this->backup->missing_backup_notification_days;
} else {
- $this->s3StorageId = $this->availableS3Storages->first()?->id;
+ $this->s3StorageIds = $this->availableS3Storages->take(1)->pluck('id')->all();
}
}
@@ -149,30 +148,24 @@ class VolumeBackups extends Component
$this->save();
}
- public function updatedS3StorageId(): void
+ public function updatedS3StorageIds(): void
{
$this->authorize('update', $this->resource);
- if (! $this->hasValidS3Storage()) {
- $this->addError('s3StorageId', 'Select a usable S3 storage owned by your team.');
-
- return;
- }
-
- $this->resetErrorBag('s3StorageId');
if (! $this->validateSettings()) {
return;
}
+ $this->resetErrorBag('s3StorageIds');
$this->backup = $this->persistBackup($this->enabled);
- $this->dispatch('success', 'S3 storage updated.');
+ $this->dispatch('success', 'S3 storages updated.');
}
public function toggleS3(): void
{
$this->authorize('update', $this->resource);
- if (! $this->saveToS3 && ! $this->hasValidS3Storage()) {
+ if (! $this->saveToS3 && $this->selectedS3StorageIds()->isEmpty()) {
$this->dispatch('error', 'Select a usable S3 storage before enabling S3 backups.');
return;
@@ -302,6 +295,7 @@ class VolumeBackups extends Component
->where('s3_storage_deleted', true)
->orWhereNull('s3_uploaded')
->orWhere('s3_uploaded', false))
+ ->withoutLiveS3Copies()
->delete() ?? 0;
$this->dispatch(
@@ -366,12 +360,8 @@ class VolumeBackups extends Component
deleteBackupsLocally($execution->filename, $server, throwError: true);
}
- if ($this->delete_backup_s3 && $execution->s3_uploaded && ! $execution->s3_storage_deleted) {
- if (! $execution->s3) {
- throw new \RuntimeException('The S3 storage is unavailable.');
- }
-
- deleteBackupsS3($execution->filename, $execution->s3);
+ if ($this->delete_backup_s3) {
+ $execution->deleteS3Copies();
}
$execution->delete();
@@ -388,7 +378,7 @@ class VolumeBackups extends Component
public function render()
{
- $executions = $this->backup?->executions()->paginate($this->perPage);
+ $executions = $this->backup?->executions()->with('s3Replicas.s3')->paginate($this->perPage);
return view('livewire.project.shared.storages.volume-backups', [
'executions' => $executions ?? collect(),
@@ -410,8 +400,8 @@ class VolumeBackups extends Component
return false;
}
- if ($this->saveToS3 && ! $this->hasValidS3Storage()) {
- $this->addError('s3StorageId', 'Select a usable S3 storage owned by your team.');
+ if ($this->saveToS3 && $this->selectedS3StorageIds()->isEmpty()) {
+ $this->addError('s3StorageIds', 'Select at least one usable S3 storage owned by your team.');
return false;
}
@@ -431,7 +421,6 @@ class VolumeBackups extends Component
'save_s3' => $this->saveToS3,
'disable_local_backup' => $this->disableLocalBackup,
'stop_during_backup' => $this->stopDuringBackup,
- 's3_storage_id' => $this->hasValidS3Storage() ? $this->s3StorageId : $this->backup?->s3_storage_id,
'retention_amount_locally' => $this->retentionAmountLocally,
'retention_days_locally' => $this->retentionDaysLocally,
'retention_max_storage_locally' => $this->retentionMaxStorageLocally,
@@ -443,6 +432,15 @@ class VolumeBackups extends Component
]);
$changedFields = auditChangedFields($backup);
$backup->save();
+
+ $storageIds = $this->selectedS3StorageIds();
+ $previousStorageIds = $backup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all();
+ $backup->syncS3Storages($storageIds->all());
+ $this->s3StorageIds = $storageIds->all();
+ if (! $backup->wasRecentlyCreated && $previousStorageIds !== $storageIds->sort()->values()->all()) {
+ $changedFields[] = 's3_storages';
+ }
+
$this->auditScheduleSet($backup, $changedFields);
return $backup;
@@ -485,14 +483,33 @@ class VolumeBackups extends Component
];
}
- private function hasValidS3Storage(): bool
+ /**
+ * The submitted destinations that are selectable S3 storages of the resource's team. Others are dropped.
+ *
+ * @return Collection<int, int>
+ */
+ private function selectedS3StorageIds(): Collection
+ {
+ $submittedIds = collect($this->s3StorageIds)->map(fn ($id): int => (int) $id)->filter()->unique();
+
+ return $this->selectableS3Storages()
+ ->whereKey($submittedIds->all())
+ ->pluck('id')
+ ->sortBy(fn (int $id): int|false => $submittedIds->search($id))
+ ->values();
+ }
+
+ /**
+ * Usable storages of the resource's team, plus the current destinations. A destination that is unusable for a
+ * while stays selected, so saving other settings does not remove it.
+ */
+ private function selectableS3Storages(): Builder
{
- return $this->s3StorageId !== null
- && S3Storage::query()
- ->whereKey($this->s3StorageId)
- ->where('team_id', $this->resourceTeamId())
- ->where('is_usable', true)
- ->exists();
+ $selectedIds = $this->backup?->selectedS3Storages()->pluck('id')->all() ?? [];
+
+ return S3Storage::query()
+ ->where('team_id', $this->resourceTeamId())
+ ->where(fn (Builder $query) => $query->where('is_usable', true)->orWhereIn('id', $selectedIds));
}
private function resourceTeamId(): int
diff --git a/app/Livewire/Storage/Resources.php b/app/Livewire/Storage/Resources.php
index 95e7917515..306a6a4a5f 100644
--- a/app/Livewire/Storage/Resources.php
+++ b/app/Livewire/Storage/Resources.php
@@ -9,6 +9,7 @@ use App\Models\ScheduledVolumeBackup;
use App\Models\Service;
use App\Models\ServiceApplication;
use App\Models\ServiceDatabase;
+use Illuminate\Database\Eloquent\Builder;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Livewire\Component;
@@ -26,7 +27,7 @@ class Resources extends Component
{
$this->authorize('view', $this->storage);
- $backups = ScheduledDatabaseBackup::where('s3_storage_id', $this->storage->id)
+ $backups = $this->backupsUsingStorage(ScheduledDatabaseBackup::query())
->where('save_s3', true)
->get();
@@ -34,8 +35,7 @@ class Resources extends Component
$this->selectedStorages[$backup->id] = $this->storage->id;
}
- ScheduledVolumeBackup::query()
- ->where('s3_storage_id', $this->storage->id)
+ $this->backupsUsingStorage(ScheduledVolumeBackup::query())
->where('save_s3', true)
->each(function (ScheduledVolumeBackup $backup): void {
$this->selectedVolumeStorages[$backup->id] = $this->storage->id;
@@ -46,29 +46,26 @@ class Resources extends Component
{
$this->authorize('update', $this->storage);
- $backup = ScheduledDatabaseBackup::where('id', $backupId)
- ->where('s3_storage_id', $this->storage->id)
+ $backup = $this->backupsUsingStorage(ScheduledDatabaseBackup::query())
+ ->whereKey($backupId)
->firstOrFail();
- $backup->fill([
- 'save_s3' => false,
- 's3_storage_id' => null,
- ]);
- $changedFields = auditChangedFields($backup);
- $backup->save();
+ $changedFields = $this->replaceStorage($backup, null);
$this->auditDatabaseBackupUpdated($backup, $changedFields);
unset($this->selectedStorages[$backupId]);
- $this->dispatch('success', 'S3 disabled.', 'S3 backup has been disabled for this schedule.');
+ $this->dispatch('success', 'Storage removed.', $backup->save_s3
+ ? 'This storage was removed from the schedule destinations.'
+ : 'S3 backup has been disabled for this schedule.');
}
public function moveBackup(int $backupId): void
{
$this->authorize('update', $this->storage);
- $backup = ScheduledDatabaseBackup::where('id', $backupId)
- ->where('s3_storage_id', $this->storage->id)
+ $backup = $this->backupsUsingStorage(ScheduledDatabaseBackup::query())
+ ->whereKey($backupId)
->firstOrFail();
$newStorageId = $this->selectedStorages[$backupId] ?? null;
@@ -90,9 +87,7 @@ class Resources extends Component
$this->authorize('update', $newStorage);
- $backup->s3_storage_id = $newStorage->id;
- $changedFields = auditChangedFields($backup);
- $backup->save();
+ $changedFields = $this->replaceStorage($backup, $newStorage->id);
$this->auditDatabaseBackupUpdated($backup, $changedFields);
unset($this->selectedStorages[$backupId]);
@@ -104,31 +99,26 @@ class Resources extends Component
{
$this->authorize('update', $this->storage);
- $backup = ScheduledVolumeBackup::query()
- ->where('id', $backupId)
- ->where('s3_storage_id', $this->storage->id)
+ $backup = $this->backupsUsingStorage(ScheduledVolumeBackup::query())
+ ->whereKey($backupId)
->firstOrFail();
- $backup->fill([
- 'save_s3' => false,
- 's3_storage_id' => null,
- ]);
- $changedFields = auditChangedFields($backup);
- $backup->save();
+ $changedFields = $this->replaceStorage($backup, null);
$this->auditVolumeBackupUpdated($backup, $changedFields);
unset($this->selectedVolumeStorages[$backupId]);
- $this->dispatch('success', 'S3 disabled.', 'S3 backup has been disabled for this schedule.');
+ $this->dispatch('success', 'Storage removed.', $backup->save_s3
+ ? 'This storage was removed from the schedule destinations.'
+ : 'S3 backup has been disabled for this schedule.');
}
public function moveVolumeBackup(int $backupId): void
{
$this->authorize('update', $this->storage);
- $backup = ScheduledVolumeBackup::query()
- ->where('id', $backupId)
- ->where('s3_storage_id', $this->storage->id)
+ $backup = $this->backupsUsingStorage(ScheduledVolumeBackup::query())
+ ->whereKey($backupId)
->firstOrFail();
$newStorageId = $this->selectedVolumeStorages[$backupId] ?? null;
@@ -151,9 +141,7 @@ class Resources extends Component
$this->authorize('update', $newStorage);
- $backup->s3_storage_id = $newStorage->id;
- $changedFields = auditChangedFields($backup);
- $backup->save();
+ $changedFields = $this->replaceStorage($backup, $newStorage->id);
$this->auditVolumeBackupUpdated($backup, $changedFields);
unset($this->selectedVolumeStorages[$backupId]);
@@ -161,6 +149,49 @@ class Resources extends Component
$this->dispatch('success', 'Backup moved.', "Moved to {$newStorage->name}.");
}
+ /**
+ * Schedules that have this storage among their destinations.
+ *
+ * @template TModel of ScheduledDatabaseBackup|ScheduledVolumeBackup
+ *
+ * @param Builder<TModel> $query
+ * @return Builder<TModel>
+ */
+ private function backupsUsingStorage(Builder $query): Builder
+ {
+ return $query->usingS3Storage($this->storage->id);
+ }
+
+ /**
+ * Replaces this storage in the destinations of a schedule, or removes it when no replacement is given. S3 backups
+ * turn off when no destination remains.
+ *
+ * @return array<int, string> Names of the changed backup fields.
+ */
+ private function replaceStorage(ScheduledDatabaseBackup|ScheduledVolumeBackup $backup, ?int $newStorageId): array
+ {
+ $storageIds = $backup->selectedS3Storages()
+ ->pluck('id')
+ ->map(fn (int $id): ?int => $id === $this->storage->id ? $newStorageId : $id)
+ ->filter()
+ ->unique()
+ ->values();
+ $previousPrimaryId = $backup->s3_storage_id;
+
+ $backup->syncS3Storages($storageIds->all());
+
+ $changedFields = ['s3_storages'];
+ if ($backup->s3_storage_id !== $previousPrimaryId) {
+ $changedFields[] = 's3_storage_id';
+ }
+ if ($storageIds->isEmpty() && $backup->save_s3) {
+ $backup->update(['save_s3' => false]);
+ $changedFields[] = 'save_s3';
+ }
+
+ return $changedFields;
+ }
+
/**
* @param array<int, string> $changedFields
*/
@@ -211,7 +242,7 @@ class Resources extends Component
public function render()
{
- $backups = ScheduledDatabaseBackup::where('s3_storage_id', $this->storage->id)
+ $backups = $this->backupsUsingStorage(ScheduledDatabaseBackup::query())
->where('save_s3', true)
->with('database')
->get()
@@ -221,8 +252,7 @@ class Resources extends Component
->orderBy('name')
->get(['id', 'name', 'is_usable']);
- $volumeBackups = ScheduledVolumeBackup::query()
- ->where('s3_storage_id', $this->storage->id)
+ $volumeBackups = $this->backupsUsingStorage(ScheduledVolumeBackup::query())
->where('save_s3', true)
->with('backupable.resource')
->get();
diff --git a/app/Livewire/Storage/Show.php b/app/Livewire/Storage/Show.php
index 17abd19e51..b7ddfcf5e2 100644
--- a/app/Livewire/Storage/Show.php
+++ b/app/Livewire/Storage/Show.php
@@ -32,8 +32,10 @@ class Show extends Component
return $this->redirectRoute('storage.index', navigate: true);
}
$this->currentRoute = request()->route()->getName();
- $this->backupCount = ScheduledDatabaseBackup::where('s3_storage_id', $this->storage->id)->count()
- + ScheduledVolumeBackup::where('s3_storage_id', $this->storage->id)->count();
+ $this->backupCount = collect([ScheduledDatabaseBackup::class, ScheduledVolumeBackup::class])
+ ->sum(fn (string $backupModel): int => $backupModel::query()
+ ->usingS3Storage($this->storage->id)
+ ->count());
}
public function delete()
diff --git a/app/Mcp/Tools/ListDatabaseBackups.php b/app/Mcp/Tools/ListDatabaseBackups.php
index 7c65130935..92bee0da01 100644
--- a/app/Mcp/Tools/ListDatabaseBackups.php
+++ b/app/Mcp/Tools/ListDatabaseBackups.php
@@ -43,6 +43,7 @@ class ListDatabaseBackups extends Tool
$backups = ScheduledDatabaseBackup::ownedByCurrentTeamAPI($teamId)
->where('database_id', $database->id)
->where('database_type', $database->getMorphClass())
+ ->with(['s3', 's3Storages'])
->get()
->map(fn ($backup) => $this->scrubSensitive([
'uuid' => $backup->uuid,
@@ -51,6 +52,7 @@ class ListDatabaseBackups extends Tool
'database_backup_retention_amount_locally' => $backup->database_backup_retention_amount_locally ?? null,
'save_s3' => $backup->save_s3 ?? null,
's3_storage_uuid' => $backup->s3?->uuid ?? null,
+ 's3_storage_uuids' => $backup->selectedS3Storages()->pluck('uuid')->all(),
'created_at' => $backup->created_at,
'updated_at' => $backup->updated_at,
]))
diff --git a/app/Models/DatabaseBackupS3Replica.php b/app/Models/DatabaseBackupS3Replica.php
new file mode 100644
index 0000000000..1dee319c2b
--- /dev/null
+++ b/app/Models/DatabaseBackupS3Replica.php
@@ -0,0 +1,38 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\Relations\BelongsTo;
+
+/**
+ * The copy of one database backup execution in one S3 destination.
+ */
+class DatabaseBackupS3Replica extends Model
+{
+ protected $fillable = [
+ 'execution_id',
+ 's3_storage_id',
+ 's3_uploaded',
+ 's3_storage_deleted',
+ 'message',
+ ];
+
+ protected function casts(): array
+ {
+ return [
+ 's3_uploaded' => 'boolean',
+ 's3_storage_deleted' => 'boolean',
+ ];
+ }
+
+ public function execution(): BelongsTo
+ {
+ return $this->belongsTo(ScheduledDatabaseBackupExecution::class, 'execution_id');
+ }
+
+ public function s3(): BelongsTo
+ {
+ return $this->belongsTo(S3Storage::class, 's3_storage_id');
+ }
+}
diff --git a/app/Models/S3Storage.php b/app/Models/S3Storage.php
index 3c0d9e7e95..3ee7d5f269 100644
--- a/app/Models/S3Storage.php
+++ b/app/Models/S3Storage.php
@@ -6,10 +6,12 @@ use App\Rules\SafeWebhookUrl;
use App\Rules\ValidS3BucketName;
use App\Traits\Auditable;
use App\Traits\HasSafeStringAttribute;
+use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Filesystem\FilesystemAdapter;
use Illuminate\Notifications\Messages\MailMessage;
+use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Facades\Validator;
@@ -67,19 +69,48 @@ class S3Storage extends BaseModel
});
static::deleting(function (S3Storage $storage) {
- ScheduledDatabaseBackup::where('s3_storage_id', $storage->id)->update([
- 'save_s3' => false,
- 's3_storage_id' => null,
- ]);
- ScheduledVolumeBackupExecution::where('s3_storage_id', $storage->id)
- ->update([
- 's3_storage_deleted' => true,
- 's3_cleanup_pending' => false,
- ]);
- ScheduledVolumeBackup::where('s3_storage_id', $storage->id)->update([
- 'save_s3' => false,
- 's3_storage_id' => null,
- ]);
+ $storage->detachFromBackups();
+ });
+ }
+
+ /**
+ * Removes this storage from every backup schedule and marks the copies in it as deleted. A schedule moves its
+ * primary destination to a remaining storage, or stops S3 backups when none remains.
+ */
+ public function detachFromBackups(): void
+ {
+ DB::transaction(function (): void {
+ foreach ([ScheduledDatabaseBackup::class, ScheduledVolumeBackup::class] as $backupModel) {
+ $backupModel::query()
+ ->usingS3Storage($this->id)
+ ->lazyById()
+ ->each(function (ScheduledDatabaseBackup|ScheduledVolumeBackup $backup): void {
+ $backup->syncS3Storages($backup->s3Storages()->whereKeyNot($this->id)->pluck('s3_storages.id')->all());
+ if ($backup->s3_storage_id === null && $backup->save_s3) {
+ $backup->update(['save_s3' => false]);
+ }
+ });
+ }
+
+ foreach ([DatabaseBackupS3Replica::class, VolumeBackupS3Replica::class] as $replicaModel) {
+ $replicaModel::query()
+ ->with('execution')
+ ->where('s3_storage_id', $this->id)
+ ->where('s3_storage_deleted', false)
+ ->lazyById()
+ ->each(function (DatabaseBackupS3Replica|VolumeBackupS3Replica $replica): void {
+ $replica->update(['s3_storage_deleted' => true]);
+ $replica->execution?->refreshS3Summary();
+ });
+ }
+
+ ScheduledVolumeBackupExecution::query()
+ ->where('s3_cleanup_pending', true)
+ ->whereHas('s3Replicas', fn (Builder $query) => $query->where('s3_storage_id', $this->id))
+ ->whereDoesntHave('s3Replicas', fn (Builder $query) => $query
+ ->where('s3_storage_deleted', false)
+ ->where(fn (Builder $query) => $query->whereNull('s3_uploaded')->orWhere('s3_uploaded', false)))
+ ->update(['s3_cleanup_pending' => false]);
});
}
diff --git a/app/Models/ScheduledDatabaseBackup.php b/app/Models/ScheduledDatabaseBackup.php
index 53ae084349..3c31c0fbc7 100644
--- a/app/Models/ScheduledDatabaseBackup.php
+++ b/app/Models/ScheduledDatabaseBackup.php
@@ -2,12 +2,16 @@
namespace App\Models;
+use App\Traits\HasS3Destinations;
+use Illuminate\Database\Eloquent\Relations\BelongsToMany;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Database\Eloquent\Relations\HasOne;
use Illuminate\Database\Eloquent\Relations\MorphTo;
class ScheduledDatabaseBackup extends BaseModel
{
+ use HasS3Destinations;
+
protected function casts(): array
{
return [
@@ -78,6 +82,11 @@ class ScheduledDatabaseBackup extends BaseModel
return $this->belongsTo(S3Storage::class, 's3_storage_id');
}
+ public function s3Storages(): BelongsToMany
+ {
+ return $this->belongsToMany(S3Storage::class, 'scheduled_database_backup_s3_storage')->withTimestamps();
+ }
+
public function get_last_days_backup_status($days = 7)
{
return $this->hasMany(ScheduledDatabaseBackupExecution::class)->where('created_at', '>=', now()->subDays($days))->get();
@@ -85,7 +94,7 @@ class ScheduledDatabaseBackup extends BaseModel
public function executionsPaginated(int $skip = 0, int $take = 10)
{
- $executions = $this->hasMany(ScheduledDatabaseBackupExecution::class)->orderBy('created_at', 'desc');
+ $executions = $this->hasMany(ScheduledDatabaseBackupExecution::class)->with('s3Replicas.s3')->orderBy('created_at', 'desc');
$count = $executions->count();
$executions = $executions->skip($skip)->take($take)->get();
diff --git a/app/Models/ScheduledDatabaseBackupExecution.php b/app/Models/ScheduledDatabaseBackupExecution.php
index 1a479772cf..eb43a1a2e3 100644
--- a/app/Models/ScheduledDatabaseBackupExecution.php
+++ b/app/Models/ScheduledDatabaseBackupExecution.php
@@ -2,10 +2,14 @@
namespace App\Models;
+use App\Traits\HasS3Replicas;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
+use Illuminate\Database\Eloquent\Relations\HasMany;
class ScheduledDatabaseBackupExecution extends BaseModel
{
+ use HasS3Replicas;
+
protected static function booted(): void
{
static::created(function (ScheduledDatabaseBackupExecution $execution): void {
@@ -42,4 +46,9 @@ class ScheduledDatabaseBackupExecution extends BaseModel
{
return $this->belongsTo(ScheduledDatabaseBackup::class);
}
+
+ public function s3Replicas(): HasMany
+ {
+ return $this->hasMany(DatabaseBackupS3Replica::class, 'execution_id');
+ }
}
diff --git a/app/Models/ScheduledVolumeBackup.php b/app/Models/ScheduledVolumeBackup.php
index 9ad676d953..0c4443b2e3 100644
--- a/app/Models/ScheduledVolumeBackup.php
+++ b/app/Models/ScheduledVolumeBackup.php
@@ -2,15 +2,19 @@
namespace App\Models;
+use App\Traits\HasS3Destinations;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
+use Illuminate\Database\Eloquent\Relations\BelongsToMany;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Database\Eloquent\Relations\HasOne;
use Illuminate\Database\Eloquent\Relations\MorphTo;
class ScheduledVolumeBackup extends BaseModel
{
+ use HasS3Destinations;
+
public const int DEFAULT_TIMEOUT = 36000;
protected $fillable = [
@@ -123,6 +127,11 @@ class ScheduledVolumeBackup extends BaseModel
return $this->belongsTo(S3Storage::class, 's3_storage_id');
}
+ public function s3Storages(): BelongsToMany
+ {
+ return $this->belongsToMany(S3Storage::class, 'scheduled_volume_backup_s3_storage')->withTimestamps();
+ }
+
public function executions(): HasMany
{
return $this->hasMany(ScheduledVolumeBackupExecution::class)->latest();
diff --git a/app/Models/ScheduledVolumeBackupExecution.php b/app/Models/ScheduledVolumeBackupExecution.php
index 664efa11c1..82dfe0ee3b 100644
--- a/app/Models/ScheduledVolumeBackupExecution.php
+++ b/app/Models/ScheduledVolumeBackupExecution.php
@@ -2,10 +2,14 @@
namespace App\Models;
+use App\Traits\HasS3Replicas;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
+use Illuminate\Database\Eloquent\Relations\HasMany;
class ScheduledVolumeBackupExecution extends BaseModel
{
+ use HasS3Replicas;
+
protected $fillable = [
'uuid',
'scheduled_volume_backup_id',
@@ -56,4 +60,9 @@ class ScheduledVolumeBackupExecution extends BaseModel
{
return $this->belongsTo(S3Storage::class, 's3_storage_id');
}
+
+ public function s3Replicas(): HasMany
+ {
+ return $this->hasMany(VolumeBackupS3Replica::class, 'execution_id');
+ }
}
diff --git a/app/Models/VolumeBackupS3Replica.php b/app/Models/VolumeBackupS3Replica.php
new file mode 100644
index 0000000000..779ee7571c
--- /dev/null
+++ b/app/Models/VolumeBackupS3Replica.php
@@ -0,0 +1,38 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\Relations\BelongsTo;
+
+/**
+ * The copy of one volume or directory backup execution in one S3 destination.
+ */
+class VolumeBackupS3Replica extends Model
+{
+ protected $fillable = [
+ 'execution_id',
+ 's3_storage_id',
+ 's3_uploaded',
+ 's3_storage_deleted',
+ 'message',
+ ];
+
+ protected function casts(): array
+ {
+ return [
+ 's3_uploaded' => 'boolean',
+ 's3_storage_deleted' => 'boolean',
+ ];
+ }
+
+ public function execution(): BelongsTo
+ {
+ return $this->belongsTo(ScheduledVolumeBackupExecution::class, 'execution_id');
+ }
+
+ public function s3(): BelongsTo
+ {
+ return $this->belongsTo(S3Storage::class, 's3_storage_id');
+ }
+}
diff --git a/app/Services/ServerTransfer/ServerTransferExporter.php b/app/Services/ServerTransfer/ServerTransferExporter.php
index 8db6e5afd1..db7ddf556e 100644
--- a/app/Services/ServerTransfer/ServerTransferExporter.php
+++ b/app/Services/ServerTransfer/ServerTransferExporter.php
@@ -16,6 +16,7 @@ use App\Models\Project;
use App\Models\S3Storage;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledTask;
+use App\Models\ScheduledVolumeBackup;
use App\Models\Server;
use App\Models\Service;
use App\Models\SharedEnvironmentVariable;
@@ -202,22 +203,22 @@ class ServerTransferExporter
}
}
- $collectVolumeS3 = function ($resource) use (&$s3Ids): void {
+ $collectBackupS3 = function (ScheduledDatabaseBackup|ScheduledVolumeBackup $backup) use (&$s3Ids): void {
+ $s3Ids->push($backup->s3_storage_id, ...$backup->s3Storages()->pluck('s3_storages.id')->all());
+ };
+
+ $collectVolumeS3 = function ($resource) use ($collectBackupS3): void {
if (method_exists($resource, 'persistentStorages')) {
foreach ($resource->persistentStorages as $volume) {
foreach ($volume->scheduledBackups as $vb) {
- if ($vb->s3_storage_id) {
- $s3Ids->push($vb->s3_storage_id);
- }
+ $collectBackupS3($vb);
}
}
}
if (method_exists($resource, 'fileStorages')) {
foreach ($resource->fileStorages as $file) {
foreach ($file->scheduledBackups as $vb) {
- if ($vb->s3_storage_id) {
- $s3Ids->push($vb->s3_storage_id);
- }
+ $collectBackupS3($vb);
}
}
}
@@ -226,9 +227,7 @@ class ServerTransferExporter
foreach ($databases as $database) {
if (method_exists($database, 'scheduledBackups')) {
foreach ($database->scheduledBackups as $backup) {
- if ($backup->s3_storage_id) {
- $s3Ids->push($backup->s3_storage_id);
- }
+ $collectBackupS3($backup);
}
}
$database->loadMissing(['persistentStorages.scheduledBackups', 'fileStorages.scheduledBackups']);
@@ -257,9 +256,7 @@ class ServerTransferExporter
}
foreach ($service->databases as $serviceDb) {
foreach ($serviceDb->scheduledBackups as $backup) {
- if ($backup->s3_storage_id) {
- $s3Ids->push($backup->s3_storage_id);
- }
+ $collectBackupS3($backup);
}
$collectVolumeS3($serviceDb);
}
@@ -981,6 +978,7 @@ class ServerTransferExporter
'timeout' => $backup->timeout,
'disable_local_backup' => (bool) ($backup->disable_local_backup ?? false),
's3_storage_uuid' => $backup->s3?->uuid,
+ 's3_storage_uuids' => $backup->selectedS3Storages()->pluck('uuid')->values()->all(),
'had_s3_storage' => filled($backup->s3_storage_id),
];
}
@@ -1193,6 +1191,7 @@ class ServerTransferExporter
'retention_max_storage_s3' => $backup->retention_max_storage_s3,
'timeout' => $backup->timeout,
's3_storage_uuid' => $backup->s3?->uuid,
+ 's3_storage_uuids' => $backup->selectedS3Storages()->pluck('uuid')->values()->all(),
'had_s3_storage' => filled($backup->s3_storage_id),
];
}
diff --git a/app/Services/ServerTransfer/ServerTransferImporter.php b/app/Services/ServerTransfer/ServerTransferImporter.php
index 90fe5954d0..d1714d4e70 100644
--- a/app/Services/ServerTransfer/ServerTransferImporter.php
+++ b/app/Services/ServerTransfer/ServerTransferImporter.php
@@ -1237,13 +1237,10 @@ class ServerTransferImporter
$uuid = new_public_id();
}
- $s3Id = null;
- $s3Uuid = data_get($backup, 's3_storage_uuid');
- if ($s3Uuid && isset($this->s3StorageMap[$s3Uuid])) {
- $s3Id = $this->s3StorageMap[$s3Uuid]->id;
- }
+ $s3Ids = $this->importedBackupS3StorageIds($backup);
+ $s3Id = $s3Ids[0] ?? null;
- ScheduledDatabaseBackup::create([
+ $importedBackup = ScheduledDatabaseBackup::create([
'uuid' => $uuid,
'team_id' => $teamId,
'description' => data_get($backup, 'description'),
@@ -1264,9 +1261,26 @@ class ServerTransferImporter
'database_type' => $database->getMorphClass(),
'database_id' => $database->id,
]);
+ $importedBackup->syncS3Storages($s3Ids);
}
}
+ /**
+ * Maps the exported S3 destinations of a backup to imported storages, primary first. Older bundles only have
+ * `s3_storage_uuid`.
+ *
+ * @param array<string, mixed> $backup
+ * @return list<int>
+ */
+ private function importedBackupS3StorageIds(array $backup): array
+ {
+ $uuids = collect(data_get($backup, 's3_storage_uuids', []))
+ ->prepend(data_get($backup, 's3_storage_uuid'))
+ ->filter(fn ($uuid): bool => is_string($uuid) && isset($this->s3StorageMap[$uuid]));
+
+ return $uuids->map(fn (string $uuid): int => $this->s3StorageMap[$uuid]->id)->unique()->values()->all();
+ }
+
/**
* @param array<string, mixed> $payload
*/
@@ -1612,13 +1626,10 @@ class ServerTransferImporter
$uuid = new_public_id();
}
- $s3Id = null;
- $s3Uuid = data_get($backupPayload, 's3_storage_uuid');
- if ($s3Uuid && isset($this->s3StorageMap[$s3Uuid])) {
- $s3Id = $this->s3StorageMap[$s3Uuid]->id;
- }
+ $s3Ids = $this->importedBackupS3StorageIds($backupPayload);
+ $s3Id = $s3Ids[0] ?? null;
- ScheduledVolumeBackup::create([
+ $importedBackup = ScheduledVolumeBackup::create([
'uuid' => $uuid,
'backupable_type' => $volume->getMorphClass(),
'backupable_id' => $volume->id,
@@ -1637,6 +1648,7 @@ class ServerTransferImporter
'retention_max_storage_s3' => data_get($backupPayload, 'retention_max_storage_s3'),
'timeout' => data_get($backupPayload, 'timeout'),
]);
+ $importedBackup->syncS3Storages($s3Ids);
$count++;
}
diff --git a/app/Traits/HasS3Destinations.php b/app/Traits/HasS3Destinations.php
new file mode 100644
index 0000000000..59668f3961
--- /dev/null
+++ b/app/Traits/HasS3Destinations.php
@@ -0,0 +1,68 @@
+<?php
+
+namespace App\Traits;
+
+use App\Models\S3Storage;
+use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Database\Eloquent\Relations\BelongsToMany;
+use Illuminate\Support\Collection;
+
+/**
+ * S3 destinations of a backup schedule. The `s3_storage_id` column keeps the primary destination for older clients.
+ */
+trait HasS3Destinations
+{
+ abstract public function s3Storages(): BelongsToMany;
+
+ /**
+ * @return Collection<int, S3Storage> The primary destination comes first.
+ */
+ public function selectedS3Storages(): Collection
+ {
+ $storages = $this->relationLoaded('s3Storages')
+ ? $this->s3Storages->sortBy('name')
+ : $this->s3Storages()->orderBy('s3_storages.name')->get();
+
+ if ($storages->isEmpty()) {
+ return $this->s3 ? collect([$this->s3]) : collect();
+ }
+
+ return $storages
+ ->sortBy(fn (S3Storage $storage): int => $storage->id === $this->s3_storage_id ? 0 : 1)
+ ->values();
+ }
+
+ /**
+ * Schedules that send backups to the given storage.
+ */
+ public function scopeUsingS3Storage(Builder $query, int $storageId): Builder
+ {
+ return $query->where(fn (Builder $query) => $query
+ ->where('s3_storage_id', $storageId)
+ ->orWhereHas('s3Storages', fn (Builder $query) => $query->whereKey($storageId)));
+ }
+
+ /**
+ * Gives a cloned schedule the same destinations, limited to storages of the clone's team.
+ */
+ public function copyS3StoragesTo(self $copy): void
+ {
+ $copy->s3Storages()->sync(
+ $this->s3Storages()->where('s3_storages.team_id', $copy->team_id)->pluck('s3_storages.id')->all(),
+ );
+ }
+
+ /**
+ * Replaces the selected destinations. The current primary destination stays primary when it is still selected.
+ *
+ * @param array<int, int|string> $storageIds
+ */
+ public function syncS3Storages(array $storageIds): void
+ {
+ $storageIds = collect($storageIds)->map(fn ($id): int => (int) $id)->filter()->unique()->values();
+
+ $this->s3Storages()->sync($storageIds->all());
+ $this->s3_storage_id = $storageIds->contains($this->s3_storage_id) ? $this->s3_storage_id : $storageIds->first();
+ $this->save();
+ }
+}
diff --git a/app/Traits/HasS3Replicas.php b/app/Traits/HasS3Replicas.php
new file mode 100644
index 0000000000..81e935e174
--- /dev/null
+++ b/app/Traits/HasS3Replicas.php
@@ -0,0 +1,75 @@
+<?php
+
+namespace App\Traits;
+
+use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Database\Eloquent\Relations\HasMany;
+
+/**
+ * S3 copies of a backup execution, one per destination. The `s3_uploaded` and `s3_storage_deleted` columns of the
+ * execution summarize them.
+ */
+trait HasS3Replicas
+{
+ abstract public function s3Replicas(): HasMany;
+
+ /**
+ * Deletes the backup file from every destination that still has a copy.
+ */
+ public function deleteS3Copies(): void
+ {
+ $replicas = $this->s3Replicas()
+ ->with('s3')
+ ->where('s3_uploaded', true)
+ ->where('s3_storage_deleted', false)
+ ->get();
+
+ try {
+ foreach ($replicas as $replica) {
+ if (! $replica->s3) {
+ throw new \RuntimeException('The S3 storage used by an existing backup is unavailable.');
+ }
+ if (filled($this->filename)) {
+ deleteBackupsS3($this->filename, $replica->s3);
+ }
+ $replica->update(['s3_storage_deleted' => true]);
+ }
+ } finally {
+ $this->refreshS3Summary();
+ }
+ }
+
+ public function hasLiveS3Copies(): bool
+ {
+ return $this->s3Replicas()->where('s3_uploaded', true)->where('s3_storage_deleted', false)->exists();
+ }
+
+ /**
+ * Writes the replica states to the summary columns: uploaded when every copy uploaded, deleted when every uploaded
+ * copy is deleted.
+ */
+ public function refreshS3Summary(): void
+ {
+ $replicas = $this->s3Replicas()->get();
+ if ($replicas->isEmpty()) {
+ return;
+ }
+
+ $uploaded = $replicas->where('s3_uploaded', true);
+
+ $this->update([
+ 's3_uploaded' => $replicas->every(fn ($replica): bool => $replica->s3_uploaded === true),
+ 's3_storage_deleted' => $uploaded->isNotEmpty() && $uploaded->every(fn ($replica): bool => $replica->s3_storage_deleted),
+ ]);
+ }
+
+ /**
+ * Executions that have no S3 copy left to keep.
+ */
+ public function scopeWithoutLiveS3Copies(Builder $query): Builder
+ {
+ return $query->whereDoesntHave('s3Replicas', fn (Builder $query) => $query
+ ->where('s3_uploaded', true)
+ ->where('s3_storage_deleted', false));
+ }
+}
diff --git a/bootstrap/helpers/databases.php b/bootstrap/helpers/databases.php
index f4a97be5ce..966b2de3e6 100644
--- a/bootstrap/helpers/databases.php
+++ b/bootstrap/helpers/databases.php
@@ -1,7 +1,9 @@
<?php
+use App\Models\DatabaseBackupS3Replica;
use App\Models\EnvironmentVariable;
use App\Models\S3Storage;
+use App\Models\ScheduledDatabaseBackupExecution;
use App\Models\Server;
use App\Models\ServiceDatabase;
use App\Models\StandaloneClickhouse;
@@ -314,110 +316,96 @@ function deleteEmptyBackupFolder($folderPath, Server $server): void
function removeOldBackups($backup): void
{
- try {
- if ($backup->executions) {
- // Delete old local backups (only if local backup is NOT disabled)
- // Note: When disable_local_backup is enabled, each execution already marks its own
- // local_storage_deleted status at the time of backup, so we don't need to retroactively
- // update old executions
- if (! $backup->disable_local_backup) {
- $localBackupsToDelete = deleteOldBackupsLocally($backup);
- if ($localBackupsToDelete->isNotEmpty()) {
- $backup->executions()
- ->whereIn('id', $localBackupsToDelete->pluck('id'))
- ->update(['local_storage_deleted' => true]);
- }
- }
- }
-
- if ($backup->save_s3 && $backup->executions) {
- $s3BackupsToDelete = deleteOldBackupsFromS3($backup);
- if ($s3BackupsToDelete->isNotEmpty()) {
- $backup->executions()
- ->whereIn('id', $s3BackupsToDelete->pluck('id'))
- ->update(['s3_storage_deleted' => true]);
- }
- }
-
- // Delete execution records where all backup copies are gone
- // Case 1: Both local and S3 backups are deleted
- $backup->executions()
- ->where('local_storage_deleted', true)
- ->where('s3_storage_deleted', true)
- ->delete();
-
- // Case 2: Local backup is deleted and S3 was never used (s3_uploaded is null)
+ // When local backups are disabled, retain the local file until an S3 copy is available.
+ $localBackupsToDelete = deleteOldBackupsLocally($backup);
+ if ($localBackupsToDelete->isNotEmpty()) {
$backup->executions()
- ->where('local_storage_deleted', true)
- ->whereNull('s3_uploaded')
- ->delete();
-
- } catch (Exception $e) {
- throw $e;
- }
-}
-
-function deleteOldBackupsLocally($backup): Collection
-{
- if (! $backup || ! $backup->executions) {
- return collect();
+ ->whereIn('id', $localBackupsToDelete->pluck('id'))
+ ->update(['local_storage_deleted' => true]);
}
- $successfulBackups = $backup->executions()
- ->where('status', 'success')
- ->where('local_storage_deleted', false)
- ->orderBy('created_at', 'desc')
- ->get();
-
- if ($successfulBackups->isEmpty()) {
- return collect();
+ if ($backup->save_s3) {
+ deleteOldBackupsFromS3($backup);
}
- $retentionAmount = $backup->database_backup_retention_amount_locally;
- $retentionDays = $backup->database_backup_retention_days_locally;
- $maxStorageGB = $backup->database_backup_retention_max_storage_locally;
+ // Delete execution records when no local file and no S3 copy is left
+ $backup->executions()
+ ->where('local_storage_deleted', true)
+ ->withoutLiveS3Copies()
+ ->delete();
+}
- if ($retentionAmount === 0 && $retentionDays === 0 && $maxStorageGB === 0) {
+/**
+ * Successful executions that fall outside the retention settings, newest execution first.
+ *
+ * @param Collection<int, ScheduledDatabaseBackupExecution> $executions Sorted newest first.
+ * @return Collection<int, ScheduledDatabaseBackupExecution>
+ */
+function backupExecutionsOutsideRetention(Collection $executions, ?int $retentionAmount, ?int $retentionDays, int|float|null $maxStorageGB): Collection
+{
+ if ($executions->isEmpty()) {
return collect();
}
$backupsToDelete = collect();
if ($retentionAmount > 0) {
- $byAmount = $successfulBackups->skip($retentionAmount);
- $backupsToDelete = $backupsToDelete->merge($byAmount);
+ $backupsToDelete = $backupsToDelete->merge($executions->skip($retentionAmount));
}
if ($retentionDays > 0) {
- $oldestAllowedDate = $successfulBackups->first()->created_at->clone()->utc()->subDays($retentionDays);
- $oldBackups = $successfulBackups->filter(fn ($execution) => $execution->created_at->utc() < $oldestAllowedDate);
- $backupsToDelete = $backupsToDelete->merge($oldBackups);
+ $oldestAllowedDate = $executions->first()->created_at->clone()->utc()->subDays($retentionDays);
+ $backupsToDelete = $backupsToDelete->merge(
+ $executions->filter(fn ($execution) => $execution->created_at->utc() < $oldestAllowedDate)
+ );
}
if ($maxStorageGB > 0) {
$maxStorageBytes = $maxStorageGB * pow(1024, 3);
$totalSize = 0;
- $backupsOverLimit = collect();
-
- $backupsToCheck = $successfulBackups->skip(1);
- foreach ($backupsToCheck as $backupExecution) {
+ foreach ($executions->skip(1) as $backupExecution) {
$totalSize += (int) $backupExecution->size;
if ($totalSize > $maxStorageBytes) {
- $backupsOverLimit = $successfulBackups->filter(
+ $backupsToDelete = $backupsToDelete->merge($executions->filter(
fn ($b) => $b->created_at->utc() <= $backupExecution->created_at->utc()
- )->skip(1);
+ )->skip(1));
break;
}
}
+ }
+
+ return $backupsToDelete->unique('id')->values();
+}
- $backupsToDelete = $backupsToDelete->merge($backupsOverLimit);
+function deleteOldBackupsLocally($backup): Collection
+{
+ if (! $backup) {
+ return collect();
}
- $backupsToDelete = $backupsToDelete->unique('id');
- $processedBackups = collect();
+ $successfulBackups = $backup->executions()
+ ->where('status', 'success')
+ ->where('local_storage_deleted', false)
+ ->when($backup->disable_local_backup, fn ($query) => $query
+ ->whereHas('s3Replicas', fn ($query) => $query
+ ->where('s3_uploaded', true)
+ ->where('s3_storage_deleted', false)))
+ ->orderBy('created_at', 'desc')
+ ->orderBy('id', 'desc')
+ ->get();
+
+ $backupsToDelete = backupExecutionsOutsideRetention(
+ $successfulBackups,
+ $backup->database_backup_retention_amount_locally,
+ $backup->database_backup_retention_days_locally,
+ $backup->database_backup_retention_max_storage_locally,
+ );
+
+ if ($backupsToDelete->isEmpty()) {
+ return collect();
+ }
- $server = null;
if ($backup->database_type === ServiceDatabase::class) {
$server = $backup->database->service->server;
} else {
@@ -433,89 +421,81 @@ function deleteOldBackupsLocally($backup): Collection
->pluck('filename')
->all();
- if (! empty($filesToDelete)) {
- deleteBackupsLocally($filesToDelete, $server);
- $processedBackups = $backupsToDelete;
+ if (empty($filesToDelete)) {
+ return collect();
}
- return $processedBackups;
+ deleteBackupsLocally($filesToDelete, $server);
+
+ return $backupsToDelete;
}
+/**
+ * Applies the S3 retention settings to each destination separately. Copies in a storage that no longer exists are
+ * marked as deleted.
+ *
+ * @return Collection<int, ScheduledDatabaseBackupExecution> The executions that lost at least one S3 copy.
+ */
function deleteOldBackupsFromS3($backup): Collection
{
- if (! $backup || ! $backup->executions || ! $backup->s3) {
+ if (! $backup) {
return collect();
}
- $successfulBackups = $backup->executions()
- ->where('status', 'success')
+ $liveReplicas = DatabaseBackupS3Replica::query()
+ ->with('s3')
+ ->whereHas('execution', fn ($query) => $query
+ ->where('scheduled_database_backup_id', $backup->id)
+ ->where('status', 'success'))
+ ->where('s3_uploaded', true)
->where('s3_storage_deleted', false)
- ->orderBy('created_at', 'desc')
->get();
- if ($successfulBackups->isEmpty()) {
- return collect();
- }
-
- $retentionAmount = $backup->database_backup_retention_amount_s3;
- $retentionDays = $backup->database_backup_retention_days_s3;
- $maxStorageGB = $backup->database_backup_retention_max_storage_s3;
-
- if ($retentionAmount === 0 && $retentionDays === 0 && $maxStorageGB === 0) {
- return collect();
- }
-
- $backupsToDelete = collect();
-
- if ($retentionAmount > 0) {
- $byAmount = $successfulBackups->skip($retentionAmount);
- $backupsToDelete = $backupsToDelete->merge($byAmount);
- }
-
- if ($retentionDays > 0) {
- $oldestAllowedDate = $successfulBackups->first()->created_at->clone()->utc()->subDays($retentionDays);
- $oldBackups = $successfulBackups->filter(fn ($execution) => $execution->created_at->utc() < $oldestAllowedDate);
- $backupsToDelete = $backupsToDelete->merge($oldBackups);
- }
-
- if ($maxStorageGB > 0) {
- $maxStorageBytes = $maxStorageGB * pow(1024, 3);
- $totalSize = 0;
- $backupsOverLimit = collect();
-
- $backupsToCheck = $successfulBackups->skip(1);
-
- foreach ($backupsToCheck as $backupExecution) {
- $totalSize += (int) $backupExecution->size;
- if ($totalSize > $maxStorageBytes) {
- $backupsOverLimit = $successfulBackups->filter(
- fn ($b) => $b->created_at->utc() <= $backupExecution->created_at->utc()
- )->skip(1);
- break;
- }
+ $deletedReplicas = $liveReplicas->filter(fn (DatabaseBackupS3Replica $replica) => ! $replica->s3);
+
+ foreach ($liveReplicas->filter(fn (DatabaseBackupS3Replica $replica) => $replica->s3)->groupBy('s3_storage_id') as $replicas) {
+ $successfulBackups = $backup->executions()
+ ->whereIn('id', $replicas->pluck('execution_id'))
+ ->orderBy('created_at', 'desc')
+ ->orderBy('id', 'desc')
+ ->get();
+
+ $backupsToDelete = backupExecutionsOutsideRetention(
+ $successfulBackups,
+ $backup->database_backup_retention_amount_s3,
+ $backup->database_backup_retention_days_s3,
+ $backup->database_backup_retention_max_storage_s3,
+ );
+
+ $filesToDelete = $backupsToDelete
+ ->filter(fn ($execution) => ! empty($execution->filename))
+ ->pluck('filename')
+ ->all();
+
+ if (empty($filesToDelete)) {
+ continue;
}
- $backupsToDelete = $backupsToDelete->merge($backupsOverLimit);
- }
-
- $backupsToDelete = $backupsToDelete->unique('id');
- $processedBackups = collect();
-
- $filesToDelete = $backupsToDelete
- ->filter(fn ($execution) => ! empty($execution->filename))
- ->pluck('filename')
- ->all();
-
- if (! empty($filesToDelete)) {
try {
- deleteBackupsS3($filesToDelete, $backup->s3);
- $processedBackups = $backupsToDelete;
+ deleteBackupsS3($filesToDelete, $replicas->first()->s3);
+ $deletedReplicas = $deletedReplicas->merge($replicas->whereIn('execution_id', $backupsToDelete->pluck('id')));
} catch (Throwable $e) {
report($e);
}
}
- return $processedBackups;
+ if ($deletedReplicas->isEmpty()) {
+ return collect();
+ }
+
+ DatabaseBackupS3Replica::query()
+ ->whereKey($deletedReplicas->pluck('id'))
+ ->update(['s3_storage_deleted' => true]);
+
+ $executions = $backup->executions()->whereIn('id', $deletedReplicas->pluck('execution_id')->unique())->get();
+ $executions->each(fn (ScheduledDatabaseBackupExecution $execution) => $execution->refreshS3Summary());
+
+ return $executions;
}
function isPublicPortAlreadyUsed(Server $server, int $port, ?string $id = null): bool
diff --git a/database/migrations/2026_10_07_150000_create_backup_s3_destination_tables.php b/database/migrations/2026_10_07_150000_create_backup_s3_destination_tables.php
new file mode 100644
index 0000000000..39aff70ad2
--- /dev/null
+++ b/database/migrations/2026_10_07_150000_create_backup_s3_destination_tables.php
@@ -0,0 +1,126 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+ public function up(): void
+ {
+ Schema::create('scheduled_database_backup_s3_storage', function (Blueprint $table) {
+ $table->id();
+ $table->foreignId('scheduled_database_backup_id')
+ ->constrained(indexName: 'db_backup_s3_storage_backup_fk')
+ ->cascadeOnDelete();
+ $table->foreignId('s3_storage_id')
+ ->constrained(indexName: 'db_backup_s3_storage_storage_fk')
+ ->cascadeOnDelete();
+ $table->timestamps();
+ $table->unique(['scheduled_database_backup_id', 's3_storage_id'], 'db_backup_s3_storage_unique');
+ });
+
+ Schema::create('scheduled_volume_backup_s3_storage', function (Blueprint $table) {
+ $table->id();
+ $table->foreignId('scheduled_volume_backup_id')
+ ->constrained(indexName: 'volume_backup_s3_storage_backup_fk')
+ ->cascadeOnDelete();
+ $table->foreignId('s3_storage_id')
+ ->constrained(indexName: 'volume_backup_s3_storage_storage_fk')
+ ->cascadeOnDelete();
+ $table->timestamps();
+ $table->unique(['scheduled_volume_backup_id', 's3_storage_id'], 'volume_backup_s3_storage_unique');
+ });
+
+ Schema::create('database_backup_s3_replicas', function (Blueprint $table) {
+ $table->id();
+ $table->foreignId('execution_id')
+ ->index()
+ ->constrained('scheduled_database_backup_executions', indexName: 'db_backup_s3_replicas_execution_fk')
+ ->cascadeOnDelete();
+ $table->foreignId('s3_storage_id')
+ ->nullable()
+ ->constrained(indexName: 'db_backup_s3_replicas_storage_fk')
+ ->nullOnDelete();
+ $table->boolean('s3_uploaded')->nullable();
+ $table->boolean('s3_storage_deleted')->default(false);
+ $table->text('message')->nullable();
+ $table->timestamps();
+ });
+
+ Schema::create('volume_backup_s3_replicas', function (Blueprint $table) {
+ $table->id();
+ $table->foreignId('execution_id')
+ ->index()
+ ->constrained('scheduled_volume_backup_executions', indexName: 'volume_backup_s3_replicas_execution_fk')
+ ->cascadeOnDelete();
+ $table->foreignId('s3_storage_id')
+ ->nullable()
+ ->constrained(indexName: 'volume_backup_s3_replicas_storage_fk')
+ ->nullOnDelete();
+ $table->boolean('s3_uploaded')->nullable();
+ $table->boolean('s3_storage_deleted')->default(false);
+ $table->text('message')->nullable();
+ $table->timestamps();
+ });
+
+ $this->backfill();
+ }
+
+ public function down(): void
+ {
+ Schema::dropIfExists('volume_backup_s3_replicas');
+ Schema::dropIfExists('database_backup_s3_replicas');
+ Schema::dropIfExists('scheduled_volume_backup_s3_storage');
+ Schema::dropIfExists('scheduled_database_backup_s3_storage');
+ }
+
+ /**
+ * Copies the single S3 destination of existing schedules and executions into the new tables.
+ */
+ private function backfill(): void
+ {
+ $now = now();
+
+ DB::table('scheduled_database_backup_s3_storage')->insertUsing(
+ ['scheduled_database_backup_id', 's3_storage_id', 'created_at', 'updated_at'],
+ DB::table('scheduled_database_backups')
+ ->join('s3_storages', 's3_storages.id', '=', 'scheduled_database_backups.s3_storage_id')
+ ->select('scheduled_database_backups.id', 'scheduled_database_backups.s3_storage_id')
+ ->selectRaw('?, ?', [$now, $now]),
+ );
+
+ DB::table('scheduled_volume_backup_s3_storage')->insertUsing(
+ ['scheduled_volume_backup_id', 's3_storage_id', 'created_at', 'updated_at'],
+ DB::table('scheduled_volume_backups')
+ ->whereNotNull('s3_storage_id')
+ ->select('id', 's3_storage_id')
+ ->selectRaw('?, ?', [$now, $now]),
+ );
+
+ // Database executions have no storage column: the copy went to the storage of the schedule.
+ DB::table('database_backup_s3_replicas')->insertUsing(
+ ['execution_id', 's3_storage_id', 's3_uploaded', 's3_storage_deleted', 'created_at', 'updated_at'],
+ DB::table('scheduled_database_backup_executions')
+ ->join('scheduled_database_backups', 'scheduled_database_backups.id', '=', 'scheduled_database_backup_executions.scheduled_database_backup_id')
+ ->leftJoin('s3_storages', 's3_storages.id', '=', 'scheduled_database_backups.s3_storage_id')
+ ->whereNotNull('scheduled_database_backup_executions.s3_uploaded')
+ ->select(
+ 'scheduled_database_backup_executions.id',
+ 's3_storages.id',
+ 'scheduled_database_backup_executions.s3_uploaded',
+ 'scheduled_database_backup_executions.s3_storage_deleted',
+ )
+ ->selectRaw('?, ?', [$now, $now]),
+ );
+
+ DB::table('volume_backup_s3_replicas')->insertUsing(
+ ['execution_id', 's3_storage_id', 's3_uploaded', 's3_storage_deleted', 'created_at', 'updated_at'],
+ DB::table('scheduled_volume_backup_executions')
+ ->whereNotNull('s3_storage_id')
+ ->select('id', 's3_storage_id', 's3_uploaded', 's3_storage_deleted')
+ ->selectRaw('?, ?', [$now, $now]),
+ );
+ }
+};
diff --git a/openapi.json b/openapi.json
index 192d6f49b9..e418f93489 100644
--- a/openapi.json
+++ b/openapi.json
@@ -86,9 +86,9 @@
"properties": {
"pull_request_id": {
"type": "integer",
+ "example": 42,
"maximum": 2147483647,
- "minimum": 1,
- "example": 42
+ "minimum": 1
},
"pull_request_html_url": {
"type": [
@@ -6857,7 +6857,7 @@
],
"responses": {
"200": {
- "description": "Get all backups for a database",
+ "description": "Get all backups for a database. Each backup includes s3_storage_uuid (primary S3 destination) and s3_storage_uuids (every S3 destination, primary first).",
"content": {
"application\/json": {
"schema": {
@@ -6927,7 +6927,14 @@
},
"s3_storage_uuid": {
"type": "string",
- "description": "S3 storage UUID (required if save_s3 is true)"
+ "description": "Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination. Either field is required if save_s3 is true."
+ },
+ "s3_storage_uuids": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "description": "UUIDs of every S3 storage the backup is uploaded to. Replaces the current destinations. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination."
},
"databases_to_backup": {
"type": "string",
@@ -7530,7 +7537,14 @@
},
"s3_storage_uuid": {
"type": "string",
- "description": "S3 storage UUID"
+ "description": "Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination; when it is already a destination, it becomes the primary destination and the other destinations stay."
+ },
+ "s3_storage_uuids": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "description": "UUIDs of every S3 storage the backup is uploaded to. Replaces the current destinations. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination."
},
"backup_now": {
"type": "boolean",
@@ -10878,8 +10892,8 @@
},
"network": {
"type": "string",
- "maxLength": 255,
- "pattern": "^[a-zA-Z0-9][a-zA-Z0-9._-]*$"
+ "pattern": "^[a-zA-Z0-9][a-zA-Z0-9._-]*$",
+ "maxLength": 255
},
"type": {
"type": "string",
@@ -23541,8 +23555,8 @@
"properties": {
"frequency": {
"type": "string",
- "maxLength": 255,
- "example": "0 2 * * *"
+ "example": "0 2 * * *",
+ "maxLength": 255
},
"enabled": {
"type": "boolean",
@@ -23561,11 +23575,19 @@
"default": false
},
"s3_storage_uuid": {
+ "description": "Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination.",
"type": [
"string",
"null"
]
},
+ "s3_storage_uuids": {
+ "description": "UUIDs of every S3 storage the backup is uploaded to. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination. save_s3 requires at least one destination.",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
"retention_amount_locally": {
"type": "integer",
"default": 7,
@@ -23674,11 +23696,19 @@
"type": "boolean"
},
"s3_storage_uuid": {
+ "description": "Primary S3 destination.",
"type": [
"string",
"null"
]
},
+ "s3_storage_uuids": {
+ "description": "Every S3 destination, primary first.",
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
"retention_amount_locally": {
"type": "integer"
},
diff --git a/openapi.yaml b/openapi.yaml
index 5983cbe38d..a7e4087848 100644
--- a/openapi.yaml
+++ b/openapi.yaml
@@ -62,9 +62,9 @@ paths:
properties:
pull_request_id:
type: integer
+ example: 42
maximum: 2147483647
minimum: 1
- example: 42
pull_request_html_url:
type: [string, 'null']
example: 'https://github.com/org/repo/pull/42'
@@ -4401,7 +4401,7 @@ paths:
type: string
responses:
'200':
- description: 'Get all backups for a database'
+ description: 'Get all backups for a database. Each backup includes s3_storage_uuid (primary S3 destination) and s3_storage_uuids (every S3 destination, primary first).'
content:
application/json:
schema:
@@ -4452,7 +4452,11 @@ paths:
default: false
s3_storage_uuid:
type: string
- description: 'S3 storage UUID (required if save_s3 is true)'
+ description: 'Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination. Either field is required if save_s3 is true.'
+ s3_storage_uuids:
+ type: array
+ items: { type: string }
+ description: 'UUIDs of every S3 storage the backup is uploaded to. Replaces the current destinations. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination.'
databases_to_backup:
type: string
description: 'Comma separated list of databases to backup'
@@ -4875,7 +4879,11 @@ paths:
description: 'Whether data is saved in s3 or not'
s3_storage_uuid:
type: string
- description: 'S3 storage UUID'
+ description: 'Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination; when it is already a destination, it becomes the primary destination and the other destinations stay.'
+ s3_storage_uuids:
+ type: array
+ items: { type: string }
+ description: 'UUIDs of every S3 storage the backup is uploaded to. Replaces the current destinations. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination.'
backup_now:
type: boolean
description: 'Whether to take a backup now or not'
@@ -7099,8 +7107,8 @@ paths:
maxLength: 255
network:
type: string
- maxLength: 255
pattern: '^[a-zA-Z0-9][a-zA-Z0-9._-]*$'
+ maxLength: 255
type:
type: string
enum: [standalone, swarm]
@@ -14947,8 +14955,8 @@ components:
properties:
frequency:
type: string
- maxLength: 255
example: '0 2 * * *'
+ maxLength: 255
enabled:
type: boolean
default: true
@@ -14962,9 +14970,15 @@ components:
type: boolean
default: false
s3_storage_uuid:
+ description: 'Primary S3 storage UUID. Sent without s3_storage_uuids, it selects this storage as the only destination.'
type:
- string
- 'null'
+ s3_storage_uuids:
+ description: 'UUIDs of every S3 storage the backup is uploaded to. When s3_storage_uuid is also sent, it must be in this list and becomes the primary destination; otherwise the first entry is the primary destination. save_s3 requires at least one destination.'
+ type: array
+ items:
+ type: string
retention_amount_locally:
type: integer
default: 7
@@ -15052,9 +15066,15 @@ components:
stop_during_backup:
type: boolean
s3_storage_uuid:
+ description: 'Primary S3 destination.'
type:
- string
- 'null'
+ s3_storage_uuids:
+ description: 'Every S3 destination, primary first.'
+ type: array
+ items:
+ type: string
retention_amount_locally:
type: integer
retention_days_locally:
diff --git a/resources/views/components/status-badge.blade.php b/resources/views/components/status-badge.blade.php
index 5aa02454bd..a8f49e8bf5 100644
--- a/resources/views/components/status-badge.blade.php
+++ b/resources/views/components/status-badge.blade.php
@@ -4,6 +4,7 @@
'type' => 'neutral',
'as' => 'span',
'dynamic' => false,
+ 'pulse' => false, // ping the dot for a state that is still in progress
])
@php
@@ -22,7 +23,12 @@
@if ($dynamic)
{{ $slot }}
@else
- <span class="size-1.5 shrink-0 rounded-full {{ $dotClasses[$type] ?? $dotClasses['neutral'] }}"></span>
+ <span class="relative flex size-1.5 shrink-0">
+ @if ($pulse)
+ <span class="absolute inline-flex size-full rounded-full opacity-75 motion-safe:animate-ping {{ $dotClasses[$type] ?? $dotClasses['neutral'] }}"></span>
+ @endif
+ <span class="relative inline-flex size-1.5 rounded-full {{ $dotClasses[$type] ?? $dotClasses['neutral'] }}"></span>
+ </span>
<span class="truncate">{{ collect([$label, $status])->filter()->join(' ') }}</span>
@endif
</button>
@@ -31,7 +37,12 @@
@if ($dynamic)
{{ $slot }}
@else
- <span class="size-1.5 shrink-0 rounded-full {{ $dotClasses[$type] ?? $dotClasses['neutral'] }}"></span>
+ <span class="relative flex size-1.5 shrink-0">
+ @if ($pulse)
+ <span class="absolute inline-flex size-full rounded-full opacity-75 motion-safe:animate-ping {{ $dotClasses[$type] ?? $dotClasses['neutral'] }}"></span>
+ @endif
+ <span class="relative inline-flex size-1.5 rounded-full {{ $dotClasses[$type] ?? $dotClasses['neutral'] }}"></span>
+ </span>
<span class="truncate">{{ collect([$label, $status])->filter()->join(' ') }}</span>
@endif
</a>
@@ -40,7 +51,12 @@
@if ($dynamic)
{{ $slot }}
@else
- <span class="size-1.5 shrink-0 rounded-full {{ $dotClasses[$type] ?? $dotClasses['neutral'] }}"></span>
+ <span class="relative flex size-1.5 shrink-0">
+ @if ($pulse)
+ <span class="absolute inline-flex size-full rounded-full opacity-75 motion-safe:animate-ping {{ $dotClasses[$type] ?? $dotClasses['neutral'] }}"></span>
+ @endif
+ <span class="relative inline-flex size-1.5 rounded-full {{ $dotClasses[$type] ?? $dotClasses['neutral'] }}"></span>
+ </span>
<span class="truncate">{{ collect([$label, $status])->filter()->join(' ') }}</span>
@endif
</span>
diff --git a/resources/views/livewire/project/application/backup/index.blade.php b/resources/views/livewire/project/application/backup/index.blade.php
index 7e89bbcd07..785b855cae 100644
--- a/resources/views/livewire/project/application/backup/index.blade.php
+++ b/resources/views/livewire/project/application/backup/index.blade.php
@@ -181,6 +181,7 @@
@foreach ($backups as $backup)
@php
$latestExecution = $backup->latestExecution;
+ $s3StorageNames = ($backup->s3Storages->isNotEmpty() ? $backup->s3Storages : collect([$backup->s3])->filter())->pluck('name');
$status = $latestExecution?->status;
$statusLabel = match ($status) {
'running' => 'In progress',
@@ -208,9 +209,9 @@
<span>{{ $backup->targetType() }}</span>
<span>{{ $backup->frequency }}</span>
<span><x-status-badge :status="$statusLabel" :type="$statusType" /></span>
- <span title="{{ $backup->save_s3 ? ($backup->s3?->name ?? 'S3 storage unavailable') : 'S3 storage is not configured' }}">
- <x-status-badge :status="$backup->save_s3 ? ($backup->s3 ? 'Configured' : 'Unavailable') : 'Not set'"
- :type="$backup->save_s3 ? ($backup->s3 ? 'success' : 'error') : 'neutral'" />
+ <span title="{{ $backup->save_s3 ? ($s3StorageNames->isNotEmpty() ? $s3StorageNames->join(', ') : 'S3 storage unavailable') : 'S3 storage is not configured' }}">
+ <x-status-badge :status="$backup->save_s3 ? match ($s3StorageNames->count()) { 0 => 'Unavailable', 1 => 'Configured', default => $s3StorageNames->count().' storages' } : 'Not set'"
+ :type="$backup->save_s3 ? ($s3StorageNames->isNotEmpty() ? 'success' : 'error') : 'neutral'" />
</span>
<span>
{{ $latestExecution?->finished_at?->diffForHumans() ?? ($status === 'running' ? 'Running now' : 'Never') }}
diff --git a/resources/views/livewire/project/database/backup-edit/s3.blade.php b/resources/views/livewire/project/database/backup-edit/s3.blade.php
index dcfb7c3815..4f0f7b43fd 100644
--- a/resources/views/livewire/project/database/backup-edit/s3.blade.php
+++ b/resources/views/livewire/project/database/backup-edit/s3.blade.php
@@ -25,11 +25,12 @@
@endif
</x-slot:actions>
<div class="grid gap-4 sm:grid-cols-2">
- <x-forms.listbox id="s3StorageId" label="S3 storage" portal :required="$saveS3"
- :options="$availableS3Storages->map(fn ($s3) => [
- 'value' => $s3->id,
- 'label' => $s3->name,
- ])->values()->all()" />
+ <x-forms.listbox id="s3StorageIds" label="S3 storages" multiple live portal
+ helper="Each backup is uploaded to every selected storage." placeholder="Select S3 storages"
+ :options="$availableS3Storages->map(fn ($storage) => [
+ 'value' => $storage->id,
+ 'label' => $storage->name.($storage->is_usable ? '' : ' (unavailable)'),
+ ])->all()" />
<x-forms.listbox id="disableLocalBackup" label="Local copy" portal :disabled="! $saveS3"
:options="[
['value' => false, 'label' => 'Keep local backup'],
diff --git a/resources/views/livewire/project/database/backup-executions.blade.php b/resources/views/livewire/project/database/backup-executions.blade.php
index 6bd2851b73..f267908207 100644
--- a/resources/views/livewire/project/database/backup-executions.blade.php
+++ b/resources/views/livewire/project/database/backup-executions.blade.php
@@ -55,6 +55,10 @@
'failed' => ['Failed', 'error'],
default => [str($executionStatus)->headline(), 'neutral'],
};
+ $s3Replicas = $execution->s3Replicas;
+ $hasLiveS3Copies = $s3Replicas->isNotEmpty()
+ ? $s3Replicas->contains(fn ($replica) => $replica->s3_uploaded === true && ! $replica->s3_storage_deleted)
+ : data_get($execution, 's3_uploaded') === true && ! data_get($execution, 's3_storage_deleted', false);
$executionCheckboxes = [];
$deleteActions = [];
@@ -62,11 +66,10 @@
$deleteActions[] = 'This backup will be permanently deleted from local storage.';
}
- if (data_get($execution, 's3_uploaded') === true
- && ! data_get($execution, 's3_storage_deleted', false)) {
+ if ($hasLiveS3Copies) {
$executionCheckboxes[] = [
'id' => 'delete_backup_s3',
- 'label' => 'Delete the selected backup permanently from S3 Storage',
+ 'label' => 'Delete the selected backup permanently from every S3 storage',
];
}
@@ -79,10 +82,7 @@
<div class="data-table-row backup-executions-table-grid min-h-14 border-b border-neutral-200 px-4 py-2.5 dark:border-white/[0.06]">
<div class="flex items-center gap-2">
<x-status-badge :status="$executionStatusLabel"
- :type="$executionStatusType" />
- @if ($executionStatus === 'running')
- <x-loading />
- @endif
+ :type="$executionStatusType" :pulse="$executionStatus === 'running'" />
</div>
<div class="truncate text-[12px] font-medium text-black dark:text-fg">
{{ data_get($execution, 'database_name', 'N/A') }}
@@ -112,7 +112,22 @@
<x-status-badge label="Local"
:status="data_get($execution, 'local_storage_deleted', false) ? 'Deleted' : 'Available'"
:type="data_get($execution, 'local_storage_deleted', false) ? 'neutral' : 'success'" />
- @if (data_get($execution, 's3_uploaded') !== null)
+ @foreach ($s3Replicas as $replica)
+ <x-status-badge wire:key="database-backup-replica-{{ $replica->id }}"
+ :label="$replica->s3?->name ?? 'Removed S3 storage'"
+ :status="$replica->s3_storage_deleted
+ ? 'Deleted'
+ : match ($replica->s3_uploaded) {
+ true => 'Available',
+ false => 'Failed',
+ default => $executionStatus === 'running' ? 'Pending' : 'Not uploaded',
+ }"
+ :type="$replica->s3_storage_deleted || $replica->s3_uploaded === null
+ ? 'neutral'
+ : ($replica->s3_uploaded ? 'success' : 'error')"
+ :title="$replica->message" />
+ @endforeach
+ @if ($s3Replicas->isEmpty() && data_get($execution, 's3_uploaded') !== null)
<x-status-badge label="S3"
:status="data_get($execution, 's3_storage_deleted', false)
? 'Deleted'
diff --git a/resources/views/livewire/project/database/scheduled-backups.blade.php b/resources/views/livewire/project/database/scheduled-backups.blade.php
index 793cd2ff47..aba61ec48c 100644
--- a/resources/views/livewire/project/database/scheduled-backups.blade.php
+++ b/resources/views/livewire/project/database/scheduled-backups.blade.php
@@ -3,7 +3,7 @@
backups: @js($database->scheduledBackups->map(fn ($backup) => [
'name' => strtolower($database->name),
'frequency' => strtolower($backup->frequency),
- 's3_storage' => strtolower($backup->s3?->name ?? ''),
+ 's3_storage' => strtolower(($backup->s3Storages->isNotEmpty() ? $backup->s3Storages : collect([$backup->s3])->filter())->pluck('name')->join(', ')),
])->values()),
hasMatches() {
const query = this.search.toLowerCase();
@@ -67,6 +67,7 @@
@foreach ($database->scheduledBackups as $backup)
@php
$latestStatus = data_get($backup->latest_log, 'status');
+ $s3StorageNames = ($backup->s3Storages->isNotEmpty() ? $backup->s3Storages : collect([$backup->s3])->filter())->pluck('name')->join(', ');
[$statusLabel, $statusType] = match ($latestStatus) {
'success' => ['Success', 'success'],
'running' => ['In progress', 'warning'],
@@ -83,7 +84,7 @@
<div x-show="search === ''
|| @js(strtolower($database->name)).includes(search.toLowerCase())
|| @js(strtolower($backup->frequency)).includes(search.toLowerCase())
- || @js(strtolower($backup->s3?->name ?? '')).includes(search.toLowerCase())"
+ || @js(strtolower($s3StorageNames)).includes(search.toLowerCase())"
class="data-table-row scheduled-backups-table-grid border-b border-neutral-200 last:border-b-0 dark:border-white/[0.06]">
<div class="min-w-0">
<span class="block truncate text-[12px] font-semibold text-black dark:text-fg">
@@ -96,8 +97,8 @@
<x-loading />
@endif
</div>
- <div class="truncate text-[11px] text-neutral-600 dark:text-fg-dim">
- {{ $backup->save_s3 ? ($backup->s3?->name ?? 'Unavailable') : 'Local only' }}
+ <div class="truncate text-[11px] text-neutral-600 dark:text-fg-dim" title="{{ $backup->save_s3 ? $s3StorageNames : '' }}">
+ {{ $backup->save_s3 ? ($s3StorageNames ?: 'Unavailable') : 'Local only' }}
</div>
<div class="text-[11px] text-neutral-600 dark:text-fg-dim">
<a {{ wireNavigate() }} href="{{ $backupExecutionsRoute }}"
diff --git a/resources/views/livewire/project/service/volume-backup/index.blade.php b/resources/views/livewire/project/service/volume-backup/index.blade.php
index 26b835cdc8..acde772a26 100644
--- a/resources/views/livewire/project/service/volume-backup/index.blade.php
+++ b/resources/views/livewire/project/service/volume-backup/index.blade.php
@@ -268,8 +268,8 @@
default => 'neutral',
};
$databaseBackupId = 'database:'.$databaseBackup->id;
- $databaseS3 = $databaseBackup->s3?->team_id === $serviceTeamId ? $databaseBackup->s3 : null;
- $databaseS3Tooltip = ! $databaseBackup->save_s3 ? 'S3 storage: Not configured' : ($databaseS3 ? 'S3 storage: '.$databaseS3->name.' (bucket: '.$databaseS3->bucket.')' : 'S3 storage: Unavailable');
+ $databaseS3s = ($databaseBackup->s3Storages->isNotEmpty() ? $databaseBackup->s3Storages : collect([$databaseBackup->s3])->filter())->where('team_id', $serviceTeamId);
+ $databaseS3Tooltip = ! $databaseBackup->save_s3 ? 'S3 storage: Not configured' : ($databaseS3s->isNotEmpty() ? 'S3 storage: '.$databaseS3s->map(fn ($s3) => $s3->name.' (bucket: '.$s3->bucket.')')->join(', ') : 'S3 storage: Unavailable');
@endphp
<div wire:key="database-backup-{{ $databaseBackup->uuid }}"
x-show="isVisible(@js($databaseBackupId))"
@@ -284,8 +284,8 @@
<span>{{ $databaseBackup->frequency }}</span>
<span><x-status-badge :status="$statusLabel" :type="$statusType" /></span>
<span>
- <x-status-badge :status="$databaseBackup->save_s3 ? ($databaseS3 ? 'Configured' : 'Unavailable') : 'Not set'"
- :type="$databaseBackup->save_s3 ? ($databaseS3 ? 'success' : 'error') : 'neutral'"
+ <x-status-badge :status="$databaseBackup->save_s3 ? match ($databaseS3s->count()) { 0 => 'Unavailable', 1 => 'Configured', default => $databaseS3s->count().' storages' } : 'Not set'"
+ :type="$databaseBackup->save_s3 ? ($databaseS3s->isNotEmpty() ? 'success' : 'error') : 'neutral'"
:data-tooltip="$databaseS3Tooltip" :aria-label="$databaseS3Tooltip" tabindex="0" />
</span>
<span>{{ $latestExecution?->finished_at?->diffForHumans() ?? ($status === 'running' ? 'Running now' : 'Never') }}</span>
@@ -308,8 +308,8 @@
@foreach ($backups as $backup)
@php
$latestExecution = $backup->latestExecution;
- $volumeS3 = $backup->s3?->team_id === $serviceTeamId ? $backup->s3 : null;
- $volumeS3Tooltip = ! $backup->save_s3 ? 'S3 storage: Not configured' : ($volumeS3 ? 'S3 storage: '.$volumeS3->name.' (bucket: '.$volumeS3->bucket.')' : 'S3 storage: Unavailable');
+ $volumeS3s = ($backup->s3Storages->isNotEmpty() ? $backup->s3Storages : collect([$backup->s3])->filter())->where('team_id', $serviceTeamId);
+ $volumeS3Tooltip = ! $backup->save_s3 ? 'S3 storage: Not configured' : ($volumeS3s->isNotEmpty() ? 'S3 storage: '.$volumeS3s->map(fn ($s3) => $s3->name.' (bucket: '.$s3->bucket.')')->join(', ') : 'S3 storage: Unavailable');
$status = $latestExecution?->status;
$statusLabel = match ($status) {
'running' => 'In progress',
@@ -338,8 +338,8 @@
<span>{{ $backup->frequency }}</span>
<span><x-status-badge :status="$statusLabel" :type="$statusType" /></span>
<span>
- <x-status-badge :status="$backup->save_s3 ? ($volumeS3 ? 'Configured' : 'Unavailable') : 'Not set'"
- :type="$backup->save_s3 ? ($volumeS3 ? 'success' : 'error') : 'neutral'"
+ <x-status-badge :status="$backup->save_s3 ? match ($volumeS3s->count()) { 0 => 'Unavailable', 1 => 'Configured', default => $volumeS3s->count().' storages' } : 'Not set'"
+ :type="$backup->save_s3 ? ($volumeS3s->isNotEmpty() ? 'success' : 'error') : 'neutral'"
:data-tooltip="$volumeS3Tooltip" :aria-label="$volumeS3Tooltip" tabindex="0" />
</span>
<span>
diff --git a/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php b/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php
index b4bf4465ec..041fb4470d 100644
--- a/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php
+++ b/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php
@@ -48,15 +48,19 @@
'failed' => 'error',
default => 'neutral',
};
+ $s3Replicas = $execution->s3Replicas;
+ $hasLiveS3Copies = $s3Replicas->isNotEmpty()
+ ? $s3Replicas->contains(fn ($replica) => $replica->s3_uploaded === true && ! $replica->s3_storage_deleted)
+ : $execution->s3_uploaded === true && ! $execution->s3_storage_deleted;
$executionCheckboxes = [];
$deleteActions = [];
if (! $execution->local_storage_deleted) {
$deleteActions[] = 'This backup will be permanently deleted from local storage.';
}
- if ($execution->s3_uploaded === true && ! $execution->s3_storage_deleted) {
+ if ($hasLiveS3Copies) {
$executionCheckboxes[] = [
'id' => 'delete_backup_s3',
- 'label' => 'Delete the selected backup permanently from S3 Storage',
+ 'label' => 'Delete the selected backup permanently from every S3 storage',
];
}
if (empty($deleteActions)) {
@@ -75,7 +79,7 @@
<div wire:key="volume-backup-execution-{{ $execution->id }}"
class="data-table-row volume-backup-executions-grid min-h-16 items-center gap-x-3 border-b border-neutral-200 text-[12px] last:border-b-0 dark:border-white/[0.07]">
<span>
- <x-status-badge :status="$statusLabel" :type="$statusType" />
+ <x-status-badge :status="$statusLabel" :type="$statusType" :pulse="$execution->status === 'running'" />
</span>
<span class="min-w-0">
@@ -98,7 +102,22 @@
<span class="flex flex-wrap gap-1">
<x-status-badge :status="$execution->local_storage_deleted ? 'Local deleted' : 'Local'"
:type="$execution->local_storage_deleted ? 'neutral' : 'success'" />
- @if ($execution->s3_uploaded !== null)
+ @foreach ($s3Replicas as $replica)
+ <x-status-badge wire:key="volume-backup-replica-{{ $replica->id }}"
+ :label="$replica->s3?->name ?? 'Removed S3 storage'"
+ :status="$replica->s3_storage_deleted
+ ? 'deleted'
+ : match ($replica->s3_uploaded) {
+ true => null,
+ false => 'failed',
+ default => $execution->status === 'running' ? 'pending' : 'not uploaded',
+ }"
+ :type="$replica->s3_storage_deleted || $replica->s3_uploaded === null
+ ? 'neutral'
+ : ($replica->s3_uploaded ? 'success' : 'error')"
+ :title="$replica->message" />
+ @endforeach
+ @if ($s3Replicas->isEmpty() && $execution->s3_uploaded !== null)
<x-status-badge
:status="$execution->s3_storage_deleted
? 'S3 deleted'
diff --git a/resources/views/livewire/project/shared/storages/volume-backups/s3.blade.php b/resources/views/livewire/project/shared/storages/volume-backups/s3.blade.php
index 7c3f471bfa..3655a94753 100644
--- a/resources/views/livewire/project/shared/storages/volume-backups/s3.blade.php
+++ b/resources/views/livewire/project/shared/storages/volume-backups/s3.blade.php
@@ -29,12 +29,12 @@
@endif
</x-slot:actions>
<div class="grid gap-4 sm:grid-cols-2">
- <x-forms.listbox canGate="update" :canResource="$resource" id="s3StorageId" label="S3 storage" :required="$saveToS3"
- :disabled="! auth()->user()?->can('update', $resource)"
- :options="$availableS3Storages->map(fn ($s3Storage) => [
- 'value' => $s3Storage->id,
- 'label' => $s3Storage->name,
- ])->values()->all()" />
+ <x-forms.listbox canGate="update" :canResource="$resource" id="s3StorageIds" label="S3 storages" multiple live portal
+ helper="Each backup is uploaded to every selected storage." placeholder="Select S3 storages"
+ :options="$availableS3Storages->map(fn ($storage) => [
+ 'value' => $storage->id,
+ 'label' => $storage->name.($storage->is_usable ? '' : ' (unavailable)'),
+ ])->all()" />
<x-forms.listbox canGate="update" :canResource="$resource" id="disableLocalBackup" label="Local copy"
:disabled="! $saveToS3 || ! auth()->user()?->can('update', $resource)" :options="[
['value' => false, 'label' => 'Keep local backup'],
diff --git a/resources/views/livewire/storage/resources.blade.php b/resources/views/livewire/storage/resources.blade.php
index dd8b004e4c..3caa1f6c94 100644
--- a/resources/views/livewire/storage/resources.blade.php
+++ b/resources/views/livewire/storage/resources.blade.php
@@ -43,6 +43,8 @@
]);
}
}
+ } elseif ($database instanceof \App\Models\StandalonePostgresql && $database->id === 0) {
+ $resourceLink = route('settings.backup');
} elseif ($database) {
$environment = $database->environment;
$project = $environment?->project;
@@ -103,10 +105,10 @@
<button type="button" class="button shrink-0"
wire:click="moveBackup({{ $backup->id }})">Move</button>
<div wire:key="disableS3-{{ $backup->id }}">
- <x-modal-confirmation title="Disable S3 for this backup schedule?"
+ <x-modal-confirmation title="Remove this storage from the backup schedule?"
submitAction="disableS3({{ $backup->id }})" :confirmWithText="false"
- :confirmWithPassword="false" step2ButtonText="Disable S3"
- :actions="['Backups from this schedule will no longer be uploaded to S3.']">
+ :confirmWithPassword="false" step2ButtonText="Remove storage"
+ :actions="['Backups from this schedule will no longer be uploaded to this storage. S3 backup turns off when no other storage remains.']">
<x-slot:trigger>
<button type="button" class="button shrink-0 text-error">Disable</button>
</x-slot:trigger>
@@ -148,10 +150,10 @@
<button type="button" class="button shrink-0"
wire:click="moveVolumeBackup({{ $backup->id }})">Move</button>
<div wire:key="disableVolumeS3-{{ $backup->id }}">
- <x-modal-confirmation title="Disable S3 for this backup schedule?"
+ <x-modal-confirmation title="Remove this storage from the backup schedule?"
submitAction="disableVolumeS3({{ $backup->id }})" :confirmWithText="false"
- :confirmWithPassword="false" step2ButtonText="Disable S3"
- :actions="['Backups from this schedule will no longer be uploaded to S3.']">
+ :confirmWithPassword="false" step2ButtonText="Remove storage"
+ :actions="['Backups from this schedule will no longer be uploaded to this storage. S3 backup turns off when no other storage remains.']">
<x-slot:trigger>
<button type="button" class="button shrink-0 text-error">Disable</button>
</x-slot:trigger>
diff --git a/tests/Feature/Api/BackupS3DestinationsApiTest.php b/tests/Feature/Api/BackupS3DestinationsApiTest.php
new file mode 100644
index 0000000000..5428dd51b7
--- /dev/null
+++ b/tests/Feature/Api/BackupS3DestinationsApiTest.php
@@ -0,0 +1,292 @@
+<?php
+
+use App\Models\Application;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\LocalPersistentVolume;
+use App\Models\Project;
+use App\Models\S3Storage;
+use App\Models\ScheduledDatabaseBackup;
+use App\Models\ScheduledVolumeBackup;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\StandalonePostgresql;
+use App\Models\Team;
+use App\Models\User;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Str;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+ Server::flushIdentityMap();
+ config()->set('app.maintenance.store', 'array');
+ InstanceSettings::forceCreate(['id' => 0, 'is_api_enabled' => true]);
+
+ $this->team = Team::factory()->create();
+ $user = User::factory()->create();
+ $this->team->members()->attach($user->id, ['role' => 'owner']);
+
+ $plainTextToken = Str::random(40);
+ $token = $user->tokens()->create([
+ 'name' => 's3-destinations-test',
+ 'token' => hash('sha256', $plainTextToken),
+ 'abilities' => ['*'],
+ 'team_id' => $this->team->id,
+ ]);
+ $this->headers = ['Authorization' => 'Bearer '.$token->getKey().'|'.$plainTextToken];
+
+ $server = Server::factory()->create(['team_id' => $this->team->id]);
+ $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
+ $project = Project::factory()->create(['team_id' => $this->team->id]);
+ $environment = Environment::factory()->create(['project_id' => $project->id]);
+
+ $this->database = StandalonePostgresql::create([
+ 'name' => 'db',
+ 'postgres_user' => 'postgres',
+ 'postgres_password' => 'password',
+ 'postgres_db' => 'app',
+ 'image' => 'postgres:16',
+ 'environment_id' => $environment->id,
+ 'destination_id' => $destination->id,
+ 'destination_type' => $destination->getMorphClass(),
+ ]);
+
+ $this->application = Application::factory()->create([
+ 'environment_id' => $environment->id,
+ 'destination_id' => $destination->id,
+ 'destination_type' => $destination->getMorphClass(),
+ ]);
+ $this->volume = LocalPersistentVolume::create([
+ 'name' => 'data',
+ 'mount_path' => '/data',
+ 'resource_id' => $this->application->id,
+ 'resource_type' => $this->application->getMorphClass(),
+ ]);
+
+ $this->firstS3 = createBackupDestinationS3($this->team, 'first');
+ $this->secondS3 = createBackupDestinationS3($this->team, 'second');
+ $this->otherTeamS3 = createBackupDestinationS3(Team::factory()->create(), 'foreign');
+});
+
+function createBackupDestinationS3(Team $team, string $name): S3Storage
+{
+ return S3Storage::create([
+ 'name' => $name,
+ 'region' => 'us-east-1',
+ 'key' => 'key',
+ 'secret' => 'secret',
+ 'bucket' => $name.'-bucket',
+ 'endpoint' => 'https://s3.example.com',
+ 'team_id' => $team->id,
+ 'is_usable' => true,
+ ]);
+}
+
+/**
+ * @return list<string>
+ */
+function backupDestinationUuids(ScheduledDatabaseBackup|ScheduledVolumeBackup $backup): array
+{
+ return $backup->refresh()->selectedS3Storages()->pluck('uuid')->all();
+}
+
+describe('database backups', function () {
+ test('creates a backup with several destinations and lists them primary first', function () {
+ $response = $this->withHeaders($this->headers)->postJson("/api/v1/databases/{$this->database->uuid}/backups", [
+ 'frequency' => 'daily',
+ 'save_s3' => true,
+ 's3_storage_uuids' => [$this->firstS3->uuid, $this->secondS3->uuid],
+ 's3_storage_uuid' => $this->secondS3->uuid,
+ ])->assertCreated();
+
+ $backup = ScheduledDatabaseBackup::where('uuid', $response->json('uuid'))->sole();
+ expect($backup->s3_storage_id)->toBe($this->secondS3->id)
+ ->and(backupDestinationUuids($backup))->toBe([$this->secondS3->uuid, $this->firstS3->uuid]);
+
+ $this->withHeaders($this->headers)->getJson("/api/v1/databases/{$this->database->uuid}/backups")
+ ->assertOk()
+ ->assertJsonPath('0.s3_storage_uuid', $this->secondS3->uuid)
+ ->assertJsonPath('0.s3_storage_uuids', [$this->secondS3->uuid, $this->firstS3->uuid])
+ ->assertJsonMissingPath('0.s3_storages')
+ ->assertJsonMissingPath('0.s3');
+ });
+
+ test('uses the first listed destination as primary when s3_storage_uuid is not sent', function () {
+ $response = $this->withHeaders($this->headers)->postJson("/api/v1/databases/{$this->database->uuid}/backups", [
+ 'frequency' => 'daily',
+ 'save_s3' => true,
+ 's3_storage_uuids' => [$this->secondS3->uuid, $this->firstS3->uuid],
+ ])->assertCreated();
+
+ expect(ScheduledDatabaseBackup::where('uuid', $response->json('uuid'))->sole()->s3_storage_id)->toBe($this->secondS3->id);
+ });
+
+ test('a single s3_storage_uuid that is already a destination becomes primary and keeps the others', function () {
+ $backup = ScheduledDatabaseBackup::create([
+ 'frequency' => 'daily',
+ 'save_s3' => true,
+ 'database_id' => $this->database->id,
+ 'database_type' => $this->database->getMorphClass(),
+ 'team_id' => $this->team->id,
+ ]);
+ $backup->syncS3Storages([$this->firstS3->id, $this->secondS3->id]);
+
+ $this->withHeaders($this->headers)->patchJson("/api/v1/databases/{$this->database->uuid}/backups/{$backup->uuid}", [
+ 'save_s3' => true,
+ 's3_storage_uuid' => $this->secondS3->uuid,
+ ])->assertOk();
+
+ expect(backupDestinationUuids($backup))->toBe([$this->secondS3->uuid, $this->firstS3->uuid])
+ ->and($backup->s3_storage_id)->toBe($this->secondS3->id);
+ });
+
+ test('a single new s3_storage_uuid replaces every destination', function () {
+ $thirdS3 = createBackupDestinationS3($this->team, 'third');
+ $backup = ScheduledDatabaseBackup::create([
+ 'frequency' => 'daily',
+ 'save_s3' => true,
+ 'database_id' => $this->database->id,
+ 'database_type' => $this->database->getMorphClass(),
+ 'team_id' => $this->team->id,
+ ]);
+ $backup->syncS3Storages([$this->firstS3->id, $this->secondS3->id]);
+
+ $this->withHeaders($this->headers)->patchJson("/api/v1/databases/{$this->database->uuid}/backups/{$backup->uuid}", [
+ 's3_storage_uuid' => $thirdS3->uuid,
+ ])->assertOk();
+
+ expect(backupDestinationUuids($backup))->toBe([$thirdS3->uuid])
+ ->and($backup->s3_storage_id)->toBe($thirdS3->id);
+ });
+
+ test('updates the destination list', function () {
+ $backup = ScheduledDatabaseBackup::create([
+ 'frequency' => 'daily',
+ 'database_id' => $this->database->id,
+ 'database_type' => $this->database->getMorphClass(),
+ 'team_id' => $this->team->id,
+ ]);
+
+ $this->withHeaders($this->headers)->patchJson("/api/v1/databases/{$this->database->uuid}/backups/{$backup->uuid}", [
+ 'save_s3' => true,
+ 's3_storage_uuids' => [$this->firstS3->uuid, $this->secondS3->uuid],
+ ])->assertOk();
+
+ expect(backupDestinationUuids($backup))->toBe([$this->firstS3->uuid, $this->secondS3->uuid])
+ ->and((bool) $backup->save_s3)->toBeTrue();
+ });
+
+ test('rejects an S3 storage of another team without changing the backup', function () {
+ $backup = ScheduledDatabaseBackup::create([
+ 'frequency' => 'daily',
+ 'save_s3' => true,
+ 's3_storage_id' => $this->firstS3->id,
+ 'database_id' => $this->database->id,
+ 'database_type' => $this->database->getMorphClass(),
+ 'team_id' => $this->team->id,
+ ]);
+ $backup->syncS3Storages([$this->firstS3->id]);
+
+ $this->withHeaders($this->headers)->patchJson("/api/v1/databases/{$this->database->uuid}/backups/{$backup->uuid}", [
+ 's3_storage_uuids' => [$this->firstS3->uuid, $this->otherTeamS3->uuid],
+ ])->assertUnprocessable()
+ ->assertJsonValidationErrors(['s3_storage_uuids'])
+ ->assertDontSee($this->otherTeamS3->name);
+
+ expect(backupDestinationUuids($backup))->toBe([$this->firstS3->uuid]);
+ });
+
+ test('rejects invalid destination combinations', function (array $payload, string $errorField) {
+ $payload = array_map(fn ($value) => is_array($value)
+ ? array_map(fn (string $name) => $this->{$name}->uuid, $value)
+ : (is_string($value) && property_exists($this, $value) ? $this->{$value}->uuid : $value), $payload);
+
+ $this->withHeaders($this->headers)->postJson("/api/v1/databases/{$this->database->uuid}/backups", ['frequency' => 'daily', ...$payload])
+ ->assertUnprocessable()
+ ->assertJsonValidationErrors([$errorField]);
+
+ expect(ScheduledDatabaseBackup::count())->toBe(0);
+ })->with([
+ 'primary outside the list' => [['s3_storage_uuids' => ['firstS3'], 's3_storage_uuid' => 'secondS3'], 's3_storage_uuid'],
+ 'save_s3 without destinations' => [['save_s3' => true, 's3_storage_uuids' => []], 's3_storage_uuid'],
+ 'foreign storage in the list' => [['save_s3' => true, 's3_storage_uuids' => ['firstS3', 'otherTeamS3']], 's3_storage_uuids'],
+ ]);
+
+ test('rejects clearing the destinations of a backup that saves to S3', function () {
+ $backup = ScheduledDatabaseBackup::create([
+ 'frequency' => 'daily',
+ 'save_s3' => true,
+ 's3_storage_id' => $this->firstS3->id,
+ 'database_id' => $this->database->id,
+ 'database_type' => $this->database->getMorphClass(),
+ 'team_id' => $this->team->id,
+ ]);
+
+ $this->withHeaders($this->headers)->patchJson("/api/v1/databases/{$this->database->uuid}/backups/{$backup->uuid}", [
+ 's3_storage_uuids' => [],
+ ])->assertUnprocessable();
+
+ expect($backup->refresh()->s3_storage_id)->toBe($this->firstS3->id);
+ });
+});
+
+describe('volume backups', function () {
+ test('sets several destinations and returns them primary first', function () {
+ $this->withHeaders($this->headers)
+ ->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups", [
+ 'frequency' => 'daily',
+ 'save_s3' => true,
+ 's3_storage_uuids' => [$this->firstS3->uuid, $this->secondS3->uuid],
+ 's3_storage_uuid' => $this->secondS3->uuid,
+ ])
+ ->assertCreated()
+ ->assertJsonPath('s3_storage_uuid', $this->secondS3->uuid)
+ ->assertJsonPath('s3_storage_uuids', [$this->secondS3->uuid, $this->firstS3->uuid]);
+
+ expect(backupDestinationUuids(ScheduledVolumeBackup::sole()))->toBe([$this->secondS3->uuid, $this->firstS3->uuid]);
+ });
+
+ test('a single s3_storage_uuid replaces every destination', function () {
+ $url = "/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups";
+ $this->withHeaders($this->headers)->putJson($url, [
+ 'frequency' => 'daily',
+ 'save_s3' => true,
+ 's3_storage_uuids' => [$this->firstS3->uuid, $this->secondS3->uuid],
+ ])->assertCreated();
+
+ $this->withHeaders($this->headers)->putJson($url, [
+ 'frequency' => 'daily',
+ 'save_s3' => true,
+ 's3_storage_uuid' => $this->secondS3->uuid,
+ ])->assertOk()->assertJsonPath('s3_storage_uuids', [$this->secondS3->uuid]);
+
+ expect(backupDestinationUuids(ScheduledVolumeBackup::sole()))->toBe([$this->secondS3->uuid]);
+ });
+
+ test('rejects an S3 storage of another team', function () {
+ $this->withHeaders($this->headers)
+ ->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups", [
+ 'frequency' => 'daily',
+ 'save_s3' => true,
+ 's3_storage_uuids' => [$this->firstS3->uuid, $this->otherTeamS3->uuid],
+ ])
+ ->assertUnprocessable()
+ ->assertJsonValidationErrors(['s3_storage_uuids'])
+ ->assertDontSee($this->otherTeamS3->name);
+
+ expect(ScheduledVolumeBackup::count())->toBe(0);
+ });
+
+ test('rejects a primary destination outside the list', function () {
+ $this->withHeaders($this->headers)
+ ->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups", [
+ 'frequency' => 'daily',
+ 'save_s3' => true,
+ 's3_storage_uuids' => [$this->firstS3->uuid],
+ 's3_storage_uuid' => $this->secondS3->uuid,
+ ])
+ ->assertUnprocessable()
+ ->assertJsonValidationErrors(['s3_storage_uuid']);
+ });
+});
diff --git a/tests/Feature/BackupEditValidationTest.php b/tests/Feature/BackupEditValidationTest.php
index 66643ffc46..3479e8db11 100644
--- a/tests/Feature/BackupEditValidationTest.php
+++ b/tests/Feature/BackupEditValidationTest.php
@@ -424,7 +424,7 @@ it('defaults to the first available storage when multiple storages are available
]);
Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s])
- ->assertSet('s3StorageId', $firstS3->id)
+ ->assertSet('s3StorageIds', [$firstS3->id])
->set('saveS3', true)
->call('instantSave')
->assertDispatched('success');
@@ -461,7 +461,7 @@ it('accepts a root team S3 storage for the instance database backup', function (
Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => collect([$s3])])
->set('saveS3', true)
- ->set('s3StorageId', $s3->id)
+ ->set('s3StorageIds', [$s3->id])
->call('instantSave')
->assertDispatched('success');
@@ -548,7 +548,7 @@ it('saves selected S3 storage immediately when it changes', function () {
]);
Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s])
- ->set('s3StorageId', $secondS3->id)
+ ->set('s3StorageIds', [$secondS3->id])
->assertDispatched('success');
$backup->refresh();
diff --git a/tests/Feature/BackupRetentionAndStaleDetectionTest.php b/tests/Feature/BackupRetentionAndStaleDetectionTest.php
index 97e45599ff..8e235a6501 100644
--- a/tests/Feature/BackupRetentionAndStaleDetectionTest.php
+++ b/tests/Feature/BackupRetentionAndStaleDetectionTest.php
@@ -276,6 +276,8 @@ test('database S3 retention remains best effort when deletion fails', function (
]);
ScheduledDatabaseBackupExecution::whereKey($newestExecution->id)->update(['created_at' => now()]);
ScheduledDatabaseBackupExecution::whereKey($oldExecution->id)->update(['created_at' => now()->subDay()]);
+ $newestExecution->s3Replicas()->create(['s3_storage_id' => $s3->id, 's3_uploaded' => true]);
+ $oldExecution->s3Replicas()->create(['s3_storage_id' => $s3->id, 's3_uploaded' => true]);
$disk = Mockery::mock();
$disk->shouldReceive('delete')->once()->with(['/backup/old.dmp'])->andReturnFalse();
Storage::shouldReceive('build')->once()->andReturn($disk);
diff --git a/tests/Feature/BackupS3DestinationsFormTest.php b/tests/Feature/BackupS3DestinationsFormTest.php
new file mode 100644
index 0000000000..211360408a
--- /dev/null
+++ b/tests/Feature/BackupS3DestinationsFormTest.php
@@ -0,0 +1,322 @@
+<?php
+
+use App\Livewire\Project\Database\BackupEdit;
+use App\Livewire\Project\Database\BackupExecutions;
+use App\Livewire\Project\Shared\Storages\VolumeBackups;
+use App\Livewire\Storage\Resources as StorageResources;
+use App\Models\Application;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\LocalPersistentVolume;
+use App\Models\PrivateKey;
+use App\Models\Project;
+use App\Models\S3Storage;
+use App\Models\ScheduledDatabaseBackup;
+use App\Models\ScheduledDatabaseBackupExecution;
+use App\Models\ScheduledVolumeBackup;
+use App\Models\ScheduledVolumeBackupExecution;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\StandalonePostgresql;
+use App\Models\Team;
+use App\Models\User;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Livewire\Livewire;
+
+uses(RefreshDatabase::class);
+
+function createS3StorageForDestinationsFormTest(Team $team, string $name): S3Storage
+{
+ return S3Storage::create([
+ 'name' => $name,
+ 'region' => 'us-east-1',
+ 'key' => 'key',
+ 'secret' => 'secret',
+ 'bucket' => str($name)->slug()->toString(),
+ 'endpoint' => 'https://s3.example.com',
+ 'is_usable' => true,
+ 'team_id' => $team->id,
+ ]);
+}
+
+/**
+ * @return array{0: Application, 1: LocalPersistentVolume, 2: StandalonePostgresql}
+ */
+function createResourcesForDestinationsFormTest(Team $team): array
+{
+ $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
+ $server = Server::factory()->create(['team_id' => $team->id, 'private_key_id' => $privateKey->id]);
+ $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
+ $project = Project::factory()->create(['team_id' => $team->id]);
+ $environment = Environment::factory()->create(['project_id' => $project->id]);
+ $application = Application::factory()->create([
+ 'environment_id' => $environment->id,
+ 'destination_id' => $destination->id,
+ 'destination_type' => $destination->getMorphClass(),
+ ]);
+ $volume = LocalPersistentVolume::create([
+ 'name' => 'app-data',
+ 'mount_path' => '/data',
+ 'resource_id' => $application->id,
+ 'resource_type' => $application->getMorphClass(),
+ ]);
+ $database = StandalonePostgresql::create([
+ 'name' => 'pg-destinations',
+ 'image' => 'postgres:16-alpine',
+ 'postgres_user' => 'postgres',
+ 'postgres_password' => 'password',
+ 'postgres_db' => 'postgres',
+ 'environment_id' => $environment->id,
+ 'destination_id' => $destination->id,
+ 'destination_type' => $destination->getMorphClass(),
+ ]);
+
+ return [$application, $volume, $database];
+}
+
+function createDatabaseBackupForDestinationsFormTest(StandalonePostgresql $database, array $overrides = []): ScheduledDatabaseBackup
+{
+ return ScheduledDatabaseBackup::create([
+ 'frequency' => '0 0 * * *',
+ 'save_s3' => false,
+ 'database_type' => $database->getMorphClass(),
+ 'database_id' => $database->id,
+ 'team_id' => $database->team()->id,
+ ...$overrides,
+ ]);
+}
+
+beforeEach(function () {
+ Server::flushIdentityMap();
+ InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
+
+ $this->team = Team::factory()->create();
+ $this->otherTeam = Team::factory()->create();
+ $this->user = User::factory()->create();
+ $this->user->teams()->attach($this->team, ['role' => 'admin']);
+ $this->user->teams()->attach($this->otherTeam, ['role' => 'owner']);
+ $this->actingAs($this->user);
+ session(['currentTeam' => $this->team]);
+
+ [$this->application, $this->volume, $this->database] = createResourcesForDestinationsFormTest($this->team);
+ $this->firstStorage = createS3StorageForDestinationsFormTest($this->team, 'First S3');
+ $this->secondStorage = createS3StorageForDestinationsFormTest($this->team, 'Second S3');
+ $this->foreignStorage = createS3StorageForDestinationsFormTest($this->otherTeam, 'Foreign S3');
+});
+
+it('saves two S3 destinations for a database backup schedule', function () {
+ $backup = createDatabaseBackupForDestinationsFormTest($this->database);
+
+ Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s, 'section' => 's3'])
+ ->assertSee('First S3')
+ ->assertSee('Second S3')
+ ->assertSet('s3StorageIds', [$this->firstStorage->id])
+ ->set('saveS3', true)
+ ->set('s3StorageIds', [(string) $this->firstStorage->id, (string) $this->secondStorage->id])
+ ->call('submit')
+ ->assertHasNoErrors()
+ ->assertDispatched('success');
+
+ $backup->refresh();
+ expect($backup->save_s3)->toBeTruthy()
+ ->and($backup->s3_storage_id)->toBe($this->firstStorage->id)
+ ->and($backup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all())
+ ->toBe([$this->firstStorage->id, $this->secondStorage->id]);
+});
+
+it('saves two S3 destinations for a volume backup schedule', function () {
+ Livewire::test(VolumeBackups::class, ['storage' => $this->volume, 'resource' => $this->application])
+ ->set('frequency', 'daily')
+ ->set('saveToS3', true)
+ ->set('s3StorageIds', [(string) $this->secondStorage->id, (string) $this->firstStorage->id])
+ ->assertHasNoErrors()
+ ->assertDispatched('success');
+
+ $backup = ScheduledVolumeBackup::query()->sole();
+ expect($backup->save_s3)->toBeTrue()
+ ->and($backup->s3_storage_id)->toBe($this->secondStorage->id)
+ ->and($backup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all())
+ ->toBe([$this->firstStorage->id, $this->secondStorage->id]);
+});
+
+it('drops S3 storages of another team after a team switch', function () {
+ $backup = createDatabaseBackupForDestinationsFormTest($this->database, [
+ 'save_s3' => true,
+ 's3_storage_id' => $this->firstStorage->id,
+ ]);
+ $backup->syncS3Storages([$this->firstStorage->id]);
+
+ $databaseComponent = Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s]);
+ $volumeComponent = Livewire::test(VolumeBackups::class, ['storage' => $this->volume, 'resource' => $this->application]);
+ session(['currentTeam' => $this->otherTeam]);
+
+ $databaseComponent->set('s3StorageIds', [$this->foreignStorage->id])
+ ->assertHasErrors('s3StorageIds');
+ expect($backup->fresh()->s3Storages()->pluck('s3_storages.id')->all())->toBe([$this->firstStorage->id]);
+
+ $databaseComponent->set('s3StorageIds', [$this->secondStorage->id, $this->foreignStorage->id])
+ ->assertHasNoErrors();
+ expect($backup->fresh()->s3_storage_id)->toBe($this->secondStorage->id)
+ ->and($backup->fresh()->s3Storages()->pluck('s3_storages.id')->all())->toBe([$this->secondStorage->id]);
+
+ $volumeComponent->set('frequency', 'daily')
+ ->set('saveToS3', true)
+ ->set('s3StorageIds', [$this->foreignStorage->id])
+ ->assertHasErrors('s3StorageIds');
+ expect(ScheduledVolumeBackup::query()->count())->toBe(0);
+
+ $volumeComponent->set('s3StorageIds', [$this->firstStorage->id, $this->foreignStorage->id])
+ ->assertHasNoErrors();
+ expect(ScheduledVolumeBackup::query()->sole()->s3Storages()->pluck('s3_storages.id')->all())
+ ->toBe([$this->firstStorage->id]);
+});
+
+it('rejects S3 backups without a destination', function () {
+ $backup = createDatabaseBackupForDestinationsFormTest($this->database, [
+ 'save_s3' => true,
+ 's3_storage_id' => $this->firstStorage->id,
+ ]);
+ $backup->syncS3Storages([$this->firstStorage->id]);
+
+ Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s])
+ ->set('s3StorageIds', [])
+ ->call('submit')
+ ->assertHasErrors('s3StorageIds')
+ ->assertDispatched('error');
+
+ expect($backup->fresh()->save_s3)->toBeTruthy()
+ ->and($backup->fresh()->s3Storages()->pluck('s3_storages.id')->all())->toBe([$this->firstStorage->id]);
+
+ Livewire::test(VolumeBackups::class, ['storage' => $this->volume, 'resource' => $this->application])
+ ->set('frequency', 'daily')
+ ->set('saveToS3', true)
+ ->set('s3StorageIds', [])
+ ->call('save')
+ ->assertHasErrors('s3StorageIds');
+
+ expect(ScheduledVolumeBackup::query()->count())->toBe(0);
+});
+
+it('moves and removes one destination of a multi-destination schedule from the storage page', function () {
+ $thirdStorage = createS3StorageForDestinationsFormTest($this->team, 'Third S3');
+ $databaseBackup = createDatabaseBackupForDestinationsFormTest($this->database, ['save_s3' => true]);
+ $databaseBackup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]);
+ $volumeBackup = ScheduledVolumeBackup::create([
+ 'backupable_type' => $this->volume->getMorphClass(),
+ 'backupable_id' => $this->volume->id,
+ 'team_id' => $this->team->id,
+ 'frequency' => 'daily',
+ 'enabled' => true,
+ 'save_s3' => true,
+ ]);
+ $volumeBackup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]);
+
+ Livewire::test(StorageResources::class, ['storage' => $this->secondStorage])
+ ->assertSet("selectedStorages.{$databaseBackup->id}", $this->secondStorage->id)
+ ->assertSet("selectedVolumeStorages.{$volumeBackup->id}", $this->secondStorage->id)
+ ->assertSee('pg-destinations')
+ ->assertSee('app-data');
+
+ Livewire::test(StorageResources::class, ['storage' => $this->firstStorage])
+ ->set("selectedStorages.{$databaseBackup->id}", $thirdStorage->id)
+ ->call('moveBackup', $databaseBackup->id)
+ ->set("selectedVolumeStorages.{$volumeBackup->id}", $thirdStorage->id)
+ ->call('moveVolumeBackup', $volumeBackup->id);
+
+ foreach ([$databaseBackup, $volumeBackup] as $backup) {
+ $backup->refresh();
+ expect($backup->s3_storage_id)->toBe($thirdStorage->id)
+ ->and($backup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all())
+ ->toBe([$this->secondStorage->id, $thirdStorage->id]);
+ }
+
+ Livewire::test(StorageResources::class, ['storage' => $this->secondStorage])
+ ->call('disableS3', $databaseBackup->id)
+ ->call('disableVolumeS3', $volumeBackup->id);
+
+ foreach ([$databaseBackup, $volumeBackup] as $backup) {
+ $backup->refresh();
+ expect($backup->save_s3)->toBeTruthy()
+ ->and($backup->s3_storage_id)->toBe($thirdStorage->id)
+ ->and($backup->s3Storages()->pluck('s3_storages.id')->all())->toBe([$thirdStorage->id]);
+ }
+
+ Livewire::test(StorageResources::class, ['storage' => $thirdStorage])
+ ->call('disableS3', $databaseBackup->id)
+ ->call('disableVolumeS3', $volumeBackup->id);
+
+ foreach ([$databaseBackup, $volumeBackup] as $backup) {
+ $backup->refresh();
+ expect($backup->save_s3)->toBeFalsy()
+ ->and($backup->s3_storage_id)->toBeNull()
+ ->and($backup->s3Storages()->count())->toBe(0);
+ }
+});
+
+it('shows the copy status of each destination in the execution lists', function () {
+ $databaseBackup = createDatabaseBackupForDestinationsFormTest($this->database, ['save_s3' => true]);
+ $databaseBackup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]);
+ $databaseExecution = ScheduledDatabaseBackupExecution::create([
+ 'scheduled_database_backup_id' => $databaseBackup->id,
+ 'status' => 'success',
+ 'filename' => '/backups/dump.dmp',
+ 's3_uploaded' => false,
+ ]);
+ $volumeBackup = ScheduledVolumeBackup::create([
+ 'backupable_type' => $this->volume->getMorphClass(),
+ 'backupable_id' => $this->volume->id,
+ 'team_id' => $this->team->id,
+ 'frequency' => 'daily',
+ 'save_s3' => true,
+ ]);
+ $volumeBackup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]);
+ $volumeExecution = ScheduledVolumeBackupExecution::create([
+ 'scheduled_volume_backup_id' => $volumeBackup->id,
+ 'status' => 'success',
+ 'filename' => '/backups/volume.tar.gz',
+ 's3_uploaded' => false,
+ ]);
+ foreach ([$databaseExecution, $volumeExecution] as $execution) {
+ $execution->s3Replicas()->create(['s3_storage_id' => $this->firstStorage->id, 's3_uploaded' => true]);
+ $execution->s3Replicas()->create(['s3_storage_id' => $this->secondStorage->id, 's3_uploaded' => false, 'message' => 'Access denied']);
+ }
+
+ Livewire::test(BackupExecutions::class, ['backup' => $databaseBackup])
+ ->assertSeeInOrder(['First S3 Available', 'Second S3 Failed'])
+ ->assertSee('Access denied')
+ ->assertSee('Delete the selected backup permanently from every S3 storage');
+
+ Livewire::test(VolumeBackups::class, ['storage' => $this->volume, 'resource' => $this->application, 'section' => 'executions'])
+ ->assertSeeInOrder(['First S3', 'Second S3 failed'])
+ ->assertSee('Delete the selected backup permanently from every S3 storage');
+});
+
+it('keeps a selected destination that is unusable for a while when other settings are saved', function () {
+ $databaseBackup = createDatabaseBackupForDestinationsFormTest($this->database, ['save_s3' => true]);
+ $databaseBackup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]);
+ $volumeBackup = ScheduledVolumeBackup::create([
+ 'backupable_type' => $this->volume->getMorphClass(),
+ 'backupable_id' => $this->volume->id,
+ 'team_id' => $this->team->id,
+ 'frequency' => '0 0 * * *',
+ 'save_s3' => true,
+ ]);
+ $volumeBackup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]);
+ $this->secondStorage->update(['is_usable' => false]);
+
+ Livewire::test(BackupEdit::class, ['backup' => $databaseBackup->fresh(), 'availableS3Storages' => $this->team->s3s, 'section' => 's3'])
+ ->assertSee('Second S3 (unavailable)')
+ ->set('timeout', 7200)
+ ->call('submit')
+ ->assertHasNoErrors();
+ Livewire::test(VolumeBackups::class, ['storage' => $this->volume, 'resource' => $this->application, 'section' => 's3'])
+ ->assertSee('Second S3 (unavailable)')
+ ->set('stopDuringBackup', true)
+ ->call('save')
+ ->assertHasNoErrors();
+
+ expect($databaseBackup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all())
+ ->toBe([$this->firstStorage->id, $this->secondStorage->id])
+ ->and($volumeBackup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all())
+ ->toBe([$this->firstStorage->id, $this->secondStorage->id]);
+});
diff --git a/tests/Feature/DatabaseBackupMultipleS3DestinationsTest.php b/tests/Feature/DatabaseBackupMultipleS3DestinationsTest.php
new file mode 100644
index 0000000000..0191aa09d7
--- /dev/null
+++ b/tests/Feature/DatabaseBackupMultipleS3DestinationsTest.php
@@ -0,0 +1,434 @@
+<?php
+
+use App\Events\BackupCreated;
+use App\Jobs\DatabaseBackupJob;
+use App\Models\DatabaseBackupS3Replica;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\PrivateKey;
+use App\Models\Project;
+use App\Models\S3Storage;
+use App\Models\ScheduledDatabaseBackup;
+use App\Models\ScheduledDatabaseBackupExecution;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
+use App\Models\User;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Event;
+use Illuminate\Support\Facades\Notification;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Storage;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+ Server::flushIdentityMap();
+});
+
+function multiS3Storage(Team $team, string $name): S3Storage
+{
+ return S3Storage::create([
+ 'name' => $name,
+ 'region' => 'us-east-1',
+ 'key' => 'key',
+ 'secret' => 'secret',
+ 'bucket' => strtolower($name).'-bucket',
+ 'endpoint' => 'https://s3.example.com',
+ 'team_id' => $team->id,
+ ]);
+}
+
+function multiS3Execution(ScheduledDatabaseBackup $backup, string $name, array $attributes = []): ScheduledDatabaseBackupExecution
+{
+ $execution = ScheduledDatabaseBackupExecution::create([
+ 'uuid' => $name,
+ 'database_name' => 'app',
+ 'filename' => "/backup/{$name}.dmp",
+ 'scheduled_database_backup_id' => $backup->id,
+ 'status' => 'success',
+ 's3_uploaded' => true,
+ 'local_storage_deleted' => true,
+ ...$attributes,
+ ]);
+
+ return $execution;
+}
+
+/**
+ * Records the files that deleteBackupsS3() removes, keyed by bucket.
+ */
+function fakeMultiS3Deletes(): Collection
+{
+ $deleted = collect();
+ Storage::shouldReceive('build')->andReturnUsing(function (array $config) use ($deleted) {
+ $disk = Mockery::mock();
+ $disk->shouldReceive('delete')->andReturnUsing(function (array $files) use ($config, $deleted) {
+ $deleted->put($config['bucket'], [...$deleted->get($config['bucket'], []), ...$files]);
+
+ return true;
+ });
+
+ return $disk;
+ });
+
+ return $deleted;
+}
+
+/**
+ * Runs a backup with two S3 destinations, failing the upload to the storages named in $failingStorages.
+ *
+ * @return array{0: ScheduledDatabaseBackupExecution, 1: Collection<int, string>}
+ */
+function runMultiS3BackupJob(array $failingStorages): array
+{
+ $team = Team::factory()->create();
+ $server = Server::factory()->create([
+ 'team_id' => $team->id,
+ 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id,
+ ]);
+ $server->settings()->update(['is_reachable' => true, 'is_usable' => true]);
+ $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
+ $environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $team->id])->id]);
+ $database = create_standalone_postgresql($environment->id, $destination, ['status' => 'running:healthy']);
+ $first = multiS3Storage($team, 'First');
+ $second = multiS3Storage($team, 'Second');
+ $backup = new ScheduledDatabaseBackup;
+ $backup->forceFill([
+ 'enabled' => true,
+ 'save_s3' => true,
+ 'disable_local_backup' => true,
+ 's3_storage_id' => $first->id,
+ 'frequency' => 'daily',
+ 'databases_to_backup' => 'app',
+ 'database_id' => $database->id,
+ 'database_type' => $database->getMorphClass(),
+ 'team_id' => $team->id,
+ ])->save();
+ $backup->syncS3Storages([$first->id, $second->id]);
+ $commands = collect();
+ Process::fake(function ($process) use ($commands) {
+ $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+ $commands->push($command);
+
+ return Process::result(output: str_contains($command, 'du -b') ? '128' : '');
+ });
+
+ $job = Mockery::mock(DatabaseBackupJob::class.'[upload_to_s3]', [$backup->fresh()])->shouldAllowMockingProtectedMethods();
+ $job->shouldReceive('upload_to_s3')->andReturnUsing(function (S3Storage $storage) use ($failingStorages) {
+ if (in_array($storage->name, $failingStorages, true)) {
+ throw new RuntimeException('access denied');
+ }
+ });
+ $job->handle();
+
+ return [ScheduledDatabaseBackupExecution::query()->sole(), $commands];
+}
+
+it('applies S3 retention to each destination separately', function () {
+ $team = Team::factory()->create();
+ $first = multiS3Storage($team, 'First');
+ $second = multiS3Storage($team, 'Second');
+ $backup = ScheduledDatabaseBackup::create([
+ 'frequency' => '0 0 * * *',
+ 'save_s3' => true,
+ 'disable_local_backup' => true,
+ 'database_type' => 'App\Models\StandalonePostgresql',
+ 'database_id' => 1,
+ 'team_id' => $team->id,
+ 'database_backup_retention_amount_s3' => 1,
+ ]);
+ $backup->syncS3Storages([$first->id, $second->id]);
+ $old = multiS3Execution($backup, 'old');
+ $new = multiS3Execution($backup, 'new', ['s3_uploaded' => false]);
+ ScheduledDatabaseBackupExecution::whereKey($old->id)->update(['created_at' => now()->subDay()]);
+ $old->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]);
+ $old->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => true]);
+ $new->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]);
+ $new->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => false, 'message' => 'denied']);
+ $deleted = fakeMultiS3Deletes();
+
+ removeOldBackups($backup);
+
+ expect($deleted->all())->toBe(['first-bucket' => ['/backup/old.dmp']])
+ ->and($old->s3Replicas()->where('s3_storage_id', $first->id)->sole()->s3_storage_deleted)->toBeTrue()
+ ->and($old->s3Replicas()->where('s3_storage_id', $second->id)->sole()->s3_storage_deleted)->toBeFalse()
+ ->and($old->fresh()->s3_storage_deleted)->toBeFalse()
+ ->and($backup->executions()->count())->toBe(2);
+});
+
+it('deletes the execution record when its last S3 copy expires', function () {
+ $team = Team::factory()->create();
+ $first = multiS3Storage($team, 'First');
+ $second = multiS3Storage($team, 'Second');
+ $backup = ScheduledDatabaseBackup::create([
+ 'frequency' => '0 0 * * *',
+ 'save_s3' => true,
+ 'disable_local_backup' => true,
+ 'database_type' => 'App\Models\StandalonePostgresql',
+ 'database_id' => 1,
+ 'team_id' => $team->id,
+ 'database_backup_retention_amount_s3' => 1,
+ ]);
+ $backup->syncS3Storages([$first->id, $second->id]);
+ $old = multiS3Execution($backup, 'old');
+ $new = multiS3Execution($backup, 'new');
+ ScheduledDatabaseBackupExecution::whereKey($old->id)->update(['created_at' => now()->subDay()]);
+ foreach ([$old, $new] as $execution) {
+ $execution->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]);
+ $execution->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => true]);
+ }
+ $deleted = fakeMultiS3Deletes();
+
+ removeOldBackups($backup);
+
+ expect($deleted->all())->toBe(['first-bucket' => ['/backup/old.dmp'], 'second-bucket' => ['/backup/old.dmp']])
+ ->and($old->fresh())->toBeNull()
+ ->and(DatabaseBackupS3Replica::where('execution_id', $old->id)->exists())->toBeFalse()
+ ->and($new->fresh())->not->toBeNull();
+});
+
+it('marks copies in a deleted storage as gone without calling S3', function () {
+ $team = Team::factory()->create();
+ $backup = ScheduledDatabaseBackup::create([
+ 'frequency' => '0 0 * * *',
+ 'save_s3' => true,
+ 'disable_local_backup' => true,
+ 'database_type' => 'App\Models\StandalonePostgresql',
+ 'database_id' => 1,
+ 'team_id' => $team->id,
+ ]);
+ $execution = multiS3Execution($backup, 'orphaned');
+ $execution->s3Replicas()->create(['s3_storage_id' => null, 's3_uploaded' => true]);
+ Storage::shouldReceive('build')->never();
+
+ removeOldBackups($backup);
+
+ expect($execution->fresh())->toBeNull();
+});
+
+it('keeps an execution whose local file still exists after its S3 copies expire', function () {
+ $team = Team::factory()->create();
+ $storage = multiS3Storage($team, 'First');
+ $backup = ScheduledDatabaseBackup::create([
+ 'frequency' => '0 0 * * *',
+ 'save_s3' => true,
+ 'database_type' => 'App\Models\StandalonePostgresql',
+ 'database_id' => 1,
+ 'team_id' => $team->id,
+ ]);
+ $execution = multiS3Execution($backup, 'local-copy', ['local_storage_deleted' => false]);
+ $execution->s3Replicas()->create(['s3_storage_id' => $storage->id, 's3_uploaded' => true, 's3_storage_deleted' => true]);
+
+ removeOldBackups($backup);
+
+ expect($execution->fresh())->not->toBeNull();
+});
+
+it('deletes the S3 copies of an execution from every destination that has one', function () {
+ $team = Team::factory()->create();
+ $first = multiS3Storage($team, 'First');
+ $second = multiS3Storage($team, 'Second');
+ $failed = multiS3Storage($team, 'Failed');
+ $backup = ScheduledDatabaseBackup::create([
+ 'frequency' => '0 0 * * *',
+ 'save_s3' => true,
+ 'database_type' => 'App\Models\StandalonePostgresql',
+ 'database_id' => 1,
+ 'team_id' => $team->id,
+ ]);
+ $execution = multiS3Execution($backup, 'manual', ['s3_uploaded' => false]);
+ $execution->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]);
+ $execution->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => true]);
+ $execution->s3Replicas()->create(['s3_storage_id' => $failed->id, 's3_uploaded' => false]);
+ $deleted = fakeMultiS3Deletes();
+
+ $execution->deleteS3Copies();
+
+ expect($deleted->keys()->sort()->values()->all())->toBe(['first-bucket', 'second-bucket'])
+ ->and($execution->fresh()->s3_storage_deleted)->toBeTrue()
+ ->and($execution->hasLiveS3Copies())->toBeFalse();
+});
+
+describe('backup job', function () {
+ beforeEach(function () {
+ Storage::fake('ssh-keys');
+ Storage::fake('ssh-mux');
+ Event::fake([BackupCreated::class]);
+ Notification::fake();
+ InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
+ });
+
+ it('uploads to every destination and deletes the local file when all uploads succeed', function () {
+ [$execution, $commands] = runMultiS3BackupJob([]);
+
+ expect($execution->status)->toBe('success')
+ ->and($execution->s3_uploaded)->toBeTrue()
+ ->and($execution->local_storage_deleted)->toBeTrue()
+ ->and($execution->s3Replicas()->pluck('s3_uploaded')->all())->toBe([true, true])
+ ->and($commands->contains(fn (string $command) => str_contains($command, 'rm -f') && str_contains($command, $execution->filename)))->toBeTrue();
+ });
+
+ it('continues to later destinations and keeps the local file when a replica write fails', function (array $failingStorages) {
+ $event = 'eloquent.creating: '.DatabaseBackupS3Replica::class;
+ Event::listen($event, function (DatabaseBackupS3Replica $replica): void {
+ if (S3Storage::find($replica->s3_storage_id)?->name === 'First') {
+ throw new RuntimeException('replica persistence failed');
+ }
+ });
+
+ try {
+ [$execution, $commands] = runMultiS3BackupJob($failingStorages);
+ } finally {
+ Event::forget($event);
+ }
+
+ $replica = $execution->s3Replicas()->with('s3')->sole();
+ expect($replica->s3->name)->toBe('Second')
+ ->and($replica->s3_uploaded)->toBeTrue()
+ ->and($execution->s3_storage_deleted)->toBeFalse()
+ ->and($execution->s3_uploaded)->toBeFalse()
+ ->and($execution->local_storage_deleted)->toBeFalse()
+ ->and($execution->message)->toContain('First: replica persistence failed')
+ ->and($commands->contains(fn (string $command) => str_contains($command, 'rm -f') && str_contains($command, $execution->filename)))->toBeFalse();
+
+ if ($failingStorages !== []) {
+ expect($execution->message)->toContain('First: access denied');
+ }
+ })->with([
+ 'successful upload' => [[]],
+ 'failed upload' => [['First']],
+ ]);
+
+ it('keeps the local file and names the failed destination when one upload fails', function () {
+ [$execution, $commands] = runMultiS3BackupJob(['Second']);
+
+ $replicas = $execution->s3Replicas()->with('s3')->get()->keyBy(fn ($replica) => $replica->s3->name);
+ expect($execution->status)->toBe('success')
+ ->and($execution->s3_uploaded)->toBeFalse()
+ ->and($execution->local_storage_deleted)->toBeFalse()
+ ->and($execution->message)->toContain('Second: access denied')
+ ->and($replicas['First']->s3_uploaded)->toBeTrue()
+ ->and($replicas['Second']->s3_uploaded)->toBeFalse()
+ ->and($replicas['Second']->message)->toBe('access denied')
+ ->and($commands->contains(fn (string $command) => str_contains($command, 'rm -f') && str_contains($command, $execution->filename)))->toBeFalse();
+ });
+});
+
+it('deletes an execution from every S3 destination through the API', function () {
+ InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
+ $sshKeys = Storage::fake('ssh-keys');
+ $sshMux = Storage::fake('ssh-mux');
+ Process::fake();
+ $team = Team::factory()->create();
+ $user = User::factory()->create();
+ $team->members()->attach($user, ['role' => 'owner']);
+ session(['currentTeam' => $team]);
+ $token = $user->createToken('test-token', ['*'])->plainTextToken;
+ $server = Server::factory()->create([
+ 'team_id' => $team->id,
+ 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id,
+ ]);
+ $server->settings()->update(['is_reachable' => true, 'is_usable' => true]);
+ $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
+ $environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $team->id])->id]);
+ $database = create_standalone_postgresql($environment->id, $destination);
+ $first = multiS3Storage($team, 'First');
+ $second = multiS3Storage($team, 'Second');
+ $backup = ScheduledDatabaseBackup::create([
+ 'frequency' => '0 0 * * *',
+ 'save_s3' => true,
+ 'database_type' => $database->getMorphClass(),
+ 'database_id' => $database->id,
+ 'team_id' => $team->id,
+ ]);
+ $backup->syncS3Storages([$first->id, $second->id]);
+ $execution = multiS3Execution($backup, 'api', ['local_storage_deleted' => false]);
+ $execution->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]);
+ $execution->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => true]);
+ $deleted = fakeMultiS3Deletes();
+ Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshKeys);
+ Storage::shouldReceive('disk')->with('ssh-mux')->andReturn($sshMux);
+
+ $this->withHeaders(['Authorization' => 'Bearer '.$token])
+ ->deleteJson("/api/v1/databases/{$database->uuid}/backups/{$backup->uuid}/executions/{$execution->uuid}?delete_s3=true")
+ ->assertOk();
+
+ expect($deleted->all())->toBe(['first-bucket' => ['/backup/api.dmp'], 'second-bucket' => ['/backup/api.dmp']])
+ ->and($execution->fresh())->toBeNull();
+});
+
+it('copies the destinations of the same team to a cloned schedule', function () {
+ $team = Team::factory()->create();
+ $otherTeam = Team::factory()->create();
+ $first = multiS3Storage($team, 'First');
+ $second = multiS3Storage($team, 'Second');
+ $foreign = multiS3Storage($otherTeam, 'Foreign');
+ $attributes = [
+ 'frequency' => '0 0 * * *',
+ 'save_s3' => true,
+ 'database_type' => 'App\Models\StandalonePostgresql',
+ 'database_id' => 1,
+ 'team_id' => $team->id,
+ ];
+ $backup = ScheduledDatabaseBackup::create($attributes);
+ $backup->syncS3Storages([$first->id, $second->id]);
+ $backup->s3Storages()->attach($foreign->id);
+ $copy = ScheduledDatabaseBackup::create($attributes);
+
+ $backup->copyS3StoragesTo($copy);
+
+ expect($copy->s3Storages()->pluck('s3_storages.id')->sort()->values()->all())->toBe([$first->id, $second->id]);
+});
+
+it('requires a live S3 copy for local retention only when local backups are disabled', function (bool $disabled, ?array $replica, bool $deleted) {
+ $team = Team::factory()->create();
+ $server = Server::factory()->create([
+ 'team_id' => $team->id,
+ 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id,
+ ]);
+ $server->settings()->update(['is_reachable' => true, 'is_usable' => true]);
+ $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
+ $environment = Environment::factory()->create(['project_id' => Project::factory()->create(['team_id' => $team->id])->id]);
+ $database = create_standalone_postgresql($environment->id, $destination);
+ $backup = ScheduledDatabaseBackup::create([
+ 'frequency' => '0 0 * * *',
+ 'save_s3' => true,
+ 'disable_local_backup' => $disabled,
+ 'database_backup_retention_amount_locally' => 1,
+ 'database_type' => $database->getMorphClass(),
+ 'database_id' => $database->id,
+ 'team_id' => $team->id,
+ ]);
+ $older = multiS3Execution($backup, 'older', ['local_storage_deleted' => false, 'created_at' => now()->subDay()]);
+ $newer = multiS3Execution($backup, 'newer', ['local_storage_deleted' => false]);
+ if ($replica !== null) {
+ $storage = multiS3Storage($team, 'Retention');
+ foreach ([$older, $newer] as $execution) {
+ $execution->s3Replicas()->create(['s3_storage_id' => $storage->id, ...$replica]);
+ }
+ }
+ $commands = collect();
+ Process::fake(function ($process) use ($commands) {
+ $commands->push(is_array($process->command) ? implode(' ', $process->command) : $process->command);
+
+ return Process::result();
+ });
+
+ removeOldBackups($backup);
+
+ expect($commands->contains(fn (string $command): bool => str_contains($command, '/backup/older.dmp')))->toBe($deleted);
+ if ($deleted && $replica === null) {
+ $this->assertModelMissing($older);
+ } else {
+ expect($older->fresh()->local_storage_deleted)->toBe($deleted);
+ }
+ expect($newer->fresh()->local_storage_deleted)->toBeFalse();
+})->with([
+ 'no replica' => [true, null, false],
+ 'failed upload' => [true, ['s3_uploaded' => false], false],
+ 'deleted replica' => [true, ['s3_uploaded' => true, 's3_storage_deleted' => true], false],
+ 'live replica' => [true, ['s3_uploaded' => true, 's3_storage_deleted' => false], true],
+ 'normal local retention' => [false, null, true],
+]);
diff --git a/tests/Feature/DevelopmentQemuVmTest.php b/tests/Feature/DevelopmentQemuVmTest.php
index ad6d1d76d0..864536cfa7 100644
--- a/tests/Feature/DevelopmentQemuVmTest.php
+++ b/tests/Feature/DevelopmentQemuVmTest.php
@@ -173,16 +173,41 @@ it('can change the localhost qemu profile without adding a server', function ()
it('seeds localhost through the container command', function () {
$this->seed(ServerSeeder::class);
+ Process::fake();
expect(Artisan::call('dev:qemu:seed', [
'profile' => 'ubuntu-root',
'--as-localhost' => true,
]))->toBe(Command::SUCCESS)
->and(Server::query()->findOrFail(0)->ip)->toBe('192.168.122.10');
+
+ Process::assertRan(fn ($process) => str_contains($process->command, 'docker run -d --name coolify-db')
+ && str_contains($process->command, 'POSTGRES_DB='));
+});
+
+it('starts the instance database container with sudo on a non-root localhost vm', function () {
+ $this->seed(ServerSeeder::class);
+ Process::fake();
+
+ Artisan::call('dev:qemu:seed', ['profile' => 'ubuntu-non-root', '--as-localhost' => true]);
+
+ Process::assertRan(fn ($process) => str_contains($process->command, 'if ! sudo docker inspect coolify-db')
+ && str_contains($process->command, 'sudo docker run -d --name coolify-db')
+ && str_contains($process->command, 'sudo docker start coolify-db'));
+});
+
+it('still seeds localhost when the instance database container cannot start', function () {
+ $this->seed(ServerSeeder::class);
+ Process::fake(fn () => Process::result(errorOutput: 'Cannot connect to the Docker daemon', exitCode: 1));
+
+ expect(Artisan::call('dev:qemu:seed', ['profile' => 'ubuntu-root', '--as-localhost' => true]))->toBe(Command::SUCCESS)
+ ->and(Artisan::output())->toContain('Could not start the coolify-db container')
+ ->and(Server::query()->findOrFail(0)->ip)->toBe('192.168.122.10');
});
it('seeds localhost with the forwarded ssh address of a lima vm', function () {
$this->seed(ServerSeeder::class);
+ Process::fake();
expect(Artisan::call('dev:qemu:seed', [
'profile' => 'debian-non-root',
diff --git a/tests/Feature/InstanceBackupMultipleS3DestinationsTest.php b/tests/Feature/InstanceBackupMultipleS3DestinationsTest.php
new file mode 100644
index 0000000000..2cbac04a38
--- /dev/null
+++ b/tests/Feature/InstanceBackupMultipleS3DestinationsTest.php
@@ -0,0 +1,137 @@
+<?php
+
+use App\Jobs\DatabaseBackupJob;
+use App\Livewire\Project\Database\BackupEdit;
+use App\Livewire\SettingsBackup;
+use App\Livewire\Storage\Resources as StorageResources;
+use App\Models\InstanceSettings;
+use App\Models\PrivateKey;
+use App\Models\S3Storage;
+use App\Models\ScheduledDatabaseBackup;
+use App\Models\ScheduledDatabaseBackupExecution;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\StandalonePostgresql;
+use App\Models\Team;
+use App\Models\User;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Once;
+use Livewire\Livewire;
+
+uses(RefreshDatabase::class);
+
+function instanceBackupS3Storage(Team $team, string $name): S3Storage
+{
+ return S3Storage::create([
+ 'name' => $name,
+ 'region' => 'us-east-1',
+ 'key' => 'key',
+ 'secret' => 'secret',
+ 'bucket' => str($name)->slug()->toString(),
+ 'endpoint' => 'https://s3.example.com',
+ 'is_usable' => true,
+ 'team_id' => $team->id,
+ ]);
+}
+
+beforeEach(function () {
+ Server::flushIdentityMap();
+ $this->rootTeam = Team::find(0) ?? Team::factory()->create(['id' => 0]);
+ $server = Server::factory()->create([
+ 'id' => 0,
+ 'team_id' => 0,
+ 'ip' => '127.0.0.1',
+ 'private_key_id' => PrivateKey::factory()->create(['team_id' => 0])->id,
+ ]);
+ $server->settings()->update(['is_reachable' => true, 'is_usable' => true, 'force_disabled' => false]);
+ StandaloneDocker::query()->where('server_id', $server->id)->firstOrFail()->forceFill(['id' => 0])->save();
+ InstanceSettings::forceCreate(['id' => 0]);
+ Once::flush();
+
+ $this->database = new StandalonePostgresql;
+ $this->database->forceFill([
+ 'id' => 0,
+ 'name' => 'coolify-db',
+ 'postgres_user' => 'coolify',
+ 'postgres_password' => 'password',
+ 'postgres_db' => 'coolify',
+ 'status' => 'running',
+ 'destination_type' => StandaloneDocker::class,
+ 'destination_id' => 0,
+ ])->save();
+ $this->backup = ScheduledDatabaseBackup::create([
+ 'enabled' => true,
+ 'save_s3' => false,
+ 'frequency' => '0 0 * * *',
+ 'database_id' => 0,
+ 'database_type' => StandalonePostgresql::class,
+ 'team_id' => 0,
+ ]);
+
+ $this->firstStorage = instanceBackupS3Storage($this->rootTeam, 'First S3');
+ $this->secondStorage = instanceBackupS3Storage($this->rootTeam, 'Second S3');
+ $this->otherTeam = Team::factory()->create();
+ $this->foreignStorage = instanceBackupS3Storage($this->otherTeam, 'Foreign S3');
+
+ $user = User::factory()->create();
+ $this->rootTeam->members()->attach($user->id, ['role' => 'admin']);
+ $user->teams()->attach($this->otherTeam->id, ['role' => 'owner']);
+ $this->actingAs($user);
+ session(['currentTeam' => ['id' => 0]]);
+});
+
+it('offers every root team S3 storage on the instance backup page', function () {
+ Livewire::test(SettingsBackup::class)
+ ->assertOk()
+ ->assertSee('First S3')
+ ->assertSee('Second S3')
+ ->assertDontSee('Foreign S3');
+});
+
+it('saves several S3 destinations for the instance backup and drops storages of other teams', function () {
+ // The session team can change in another tab after the page loads.
+ session(['currentTeam' => ['id' => $this->otherTeam->id]]);
+
+ Livewire::test(BackupEdit::class, ['backup' => $this->backup->fresh(), 'availableS3Storages' => collect(), 'section' => 's3'])
+ ->set('saveS3', true)
+ ->set('s3StorageIds', [$this->firstStorage->id, $this->secondStorage->id, $this->foreignStorage->id])
+ ->call('submit')
+ ->assertHasNoErrors()
+ ->assertDispatched('success');
+
+ $backup = $this->backup->fresh();
+
+ expect($backup->save_s3)->toBeTruthy()
+ ->and($backup->s3_storage_id)->toBe($this->firstStorage->id)
+ ->and($backup->s3Storages()->pluck('s3_storages.id')->sort()->values()->all())
+ ->toBe([$this->firstStorage->id, $this->secondStorage->id]);
+});
+
+it('uploads the instance backup to every selected destination', function () {
+ $this->backup->forceFill(['save_s3' => true, 'databases_to_backup' => 'coolify'])->save();
+ $this->backup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]);
+ Process::fake(fn ($process) => Process::result(output: str_contains(implode(' ', (array) $process->command), 'du -b') ? '128' : ''));
+
+ $uploadedTo = collect();
+ $job = Mockery::mock(DatabaseBackupJob::class.'[upload_to_s3]', [$this->backup->fresh()])->shouldAllowMockingProtectedMethods();
+ $job->shouldReceive('upload_to_s3')->andReturnUsing(fn (S3Storage $storage) => $uploadedTo->push($storage->id));
+ $job->handle();
+
+ $execution = ScheduledDatabaseBackupExecution::query()->sole();
+
+ expect($uploadedTo->sort()->values()->all())->toBe([$this->firstStorage->id, $this->secondStorage->id])
+ ->and($execution->status)->toBe('success')
+ ->and($execution->s3_uploaded)->toBeTrue()
+ ->and($execution->s3Replicas()->where('s3_uploaded', true)->pluck('s3_storage_id')->sort()->values()->all())
+ ->toBe([$this->firstStorage->id, $this->secondStorage->id]);
+});
+
+it('links the instance backup from the storage resources page', function () {
+ $this->backup->forceFill(['save_s3' => true])->save();
+ $this->backup->syncS3Storages([$this->firstStorage->id, $this->secondStorage->id]);
+
+ Livewire::test(StorageResources::class, ['storage' => $this->secondStorage])
+ ->assertSee('coolify-db')
+ ->assertSeeHtml(route('settings.backup'));
+});
diff --git a/tests/Feature/Jobs/DatabaseBackupJobTest.php b/tests/Feature/Jobs/DatabaseBackupJobTest.php
index a19958608f..d0ba825511 100644
--- a/tests/Feature/Jobs/DatabaseBackupJobTest.php
+++ b/tests/Feature/Jobs/DatabaseBackupJobTest.php
@@ -52,7 +52,7 @@ test('scheduled database backup casts full dump selection to boolean', function
->and((new ScheduledDatabaseBackup(['dump_all' => '1']))->dump_all)->toBeTrue();
});
-test('upload_to_s3 throws exception and disables s3 when storage is null', function () {
+test('s3 upload throws exception and disables s3 when no destination exists', function () {
$backup = ScheduledDatabaseBackup::create([
'frequency' => '0 0 * * *',
'save_s3' => true,
@@ -64,11 +64,7 @@ test('upload_to_s3 throws exception and disables s3 when storage is null', funct
$job = new DatabaseBackupJob($backup);
- $reflection = new ReflectionClass($job);
- $s3Property = $reflection->getProperty('s3');
- $s3Property->setValue($job, null);
-
- $method = $reflection->getMethod('upload_to_s3');
+ $method = (new ReflectionClass($job))->getMethod('uploadToS3Destinations');
expect(fn () => $method->invoke($job))
->toThrow(Exception::class, 'S3 storage configuration is missing or has been deleted');
@@ -78,7 +74,7 @@ test('upload_to_s3 throws exception and disables s3 when storage is null', funct
expect($backup->s3_storage_id)->toBeNull();
});
-test('upload_to_s3 exception message reports the previous s3 storage id', function () {
+test('s3 upload exception message reports the previous s3 storage id', function () {
$backup = ScheduledDatabaseBackup::create([
'frequency' => '0 0 * * *',
'save_s3' => true,
@@ -90,10 +86,7 @@ test('upload_to_s3 exception message reports the previous s3 storage id', functi
$job = new DatabaseBackupJob($backup);
- $reflection = new ReflectionClass($job);
- $reflection->getProperty('s3')->setValue($job, null);
-
- expect(fn () => $reflection->getMethod('upload_to_s3')->invoke($job))
+ expect(fn () => (new ReflectionClass($job))->getMethod('uploadToS3Destinations')->invoke($job))
->toThrow(Exception::class, 'S3 storage ID: 12345');
$backup->refresh();
@@ -101,7 +94,7 @@ test('upload_to_s3 exception message reports the previous s3 storage id', functi
expect($backup->s3_storage_id)->toBeNull();
});
-test('upload_to_s3 exception message reports null when no previous s3 storage id exists', function () {
+test('s3 upload exception message reports null when no previous s3 storage id exists', function () {
$backup = ScheduledDatabaseBackup::create([
'frequency' => '0 0 * * *',
'save_s3' => true,
@@ -113,10 +106,7 @@ test('upload_to_s3 exception message reports null when no previous s3 storage id
$job = new DatabaseBackupJob($backup);
- $reflection = new ReflectionClass($job);
- $reflection->getProperty('s3')->setValue($job, null);
-
- expect(fn () => $reflection->getMethod('upload_to_s3')->invoke($job))
+ expect(fn () => (new ReflectionClass($job))->getMethod('uploadToS3Destinations')->invoke($job))
->toThrow(Exception::class, 'S3 storage ID: null');
});
@@ -281,7 +271,7 @@ test('retention cleanup failure does not fail a successful database backup', fun
'local_storage_deleted' => true,
'created_at' => now()->subDay(),
]);
- ScheduledDatabaseBackupExecution::create([
+ $newExecution = ScheduledDatabaseBackupExecution::create([
'uuid' => 'new-backup',
'database_name' => 'database',
'filename' => '/backup/new.dmp',
@@ -290,6 +280,8 @@ test('retention cleanup failure does not fail a successful database backup', fun
's3_uploaded' => true,
'local_storage_deleted' => true,
]);
+ $oldExecution->s3Replicas()->create(['s3_storage_id' => $s3->id, 's3_uploaded' => true]);
+ $newExecution->s3Replicas()->create(['s3_storage_id' => $s3->id, 's3_uploaded' => true]);
$disk = Mockery::mock();
$disk->shouldReceive('delete')->once()->with(['/backup/old.dmp'])->andReturnFalse();
Storage::shouldReceive('build')->once()->andReturn($disk);
diff --git a/tests/Feature/Mcp/McpReadToolsTest.php b/tests/Feature/Mcp/McpReadToolsTest.php
index 21df936c94..f45757b8d1 100644
--- a/tests/Feature/Mcp/McpReadToolsTest.php
+++ b/tests/Feature/Mcp/McpReadToolsTest.php
@@ -10,6 +10,7 @@ use App\Models\EnvironmentVariable;
use App\Models\GithubApp;
use App\Models\InstanceSettings;
use App\Models\Project;
+use App\Models\S3Storage;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledDatabaseBackupExecution;
use App\Models\ScheduledTask;
@@ -217,6 +218,36 @@ test('database backup tools scope schedules by database type and id', function (
expect($response->json('result.isError'))->toBeTrue();
});
+test('list_database_backups returns every S3 destination primary first', function () {
+ $postgres = StandalonePostgresql::create([
+ 'name' => 'multi-s3-db',
+ 'postgres_password' => 'password',
+ 'environment_id' => $this->environment->id,
+ 'destination_id' => $this->destination->id,
+ 'destination_type' => $this->destination->getMorphClass(),
+ ]);
+ [$first, $second] = collect(['first', 'second'])->map(fn (string $name) => S3Storage::create([
+ 'name' => $name, 'region' => 'us-east-1', 'key' => 'key', 'secret' => 'secret',
+ 'bucket' => $name, 'endpoint' => 'https://s3.example.com', 'team_id' => $this->team->id,
+ ]))->all();
+ $backup = ScheduledDatabaseBackup::create([
+ 'team_id' => $this->team->id,
+ 'frequency' => '0 0 * * *',
+ 'save_s3' => true,
+ 's3_storage_id' => $second->id,
+ 'database_id' => $postgres->id,
+ 'database_type' => $postgres->getMorphClass(),
+ ]);
+ $backup->syncS3Storages([$first->id, $second->id]);
+
+ $response = mcpReadCall('list_database_backups', ['uuid' => $postgres->uuid]);
+ $response->assertOk();
+
+ $listed = mcpReadJson($response)['data']['backups'][0];
+ expect($listed['s3_storage_uuid'])->toBe($second->uuid)
+ ->and($listed['s3_storage_uuids'])->toBe([$second->uuid, $first->uuid]);
+});
+
test('list_backup_executions omits messages without sensitive read and redacts when included', function () {
$postgres = StandalonePostgresql::create([
'name' => 'backup-msg-db',
diff --git a/tests/Feature/MutableLivewireComponentsAuthorizationTest.php b/tests/Feature/MutableLivewireComponentsAuthorizationTest.php
index 5563e87769..47b215d5f0 100644
--- a/tests/Feature/MutableLivewireComponentsAuthorizationTest.php
+++ b/tests/Feature/MutableLivewireComponentsAuthorizationTest.php
@@ -182,7 +182,7 @@ it('authorizes every volume backup form control', function (string $path, array
[
'/<x-forms\.button(?=[^>]*wire:click="toggleS3")(?=[^>]*canGate="update")(?=[^>]*:canResource="\$resource")[^>]*>\s*Enable S3\s*<\/x-forms\.button>/s',
'/<x-forms\.button(?=[^>]*wire:click="toggleS3")(?=[^>]*canGate="update")(?=[^>]*:canResource="\$resource")[^>]*>\s*Disable S3\s*<\/x-forms\.button>/s',
- '/<x-forms\.listbox canGate="update" :canResource="\$resource" id="s3StorageId"[\s\S]*?:disabled="! auth\(\)->user\(\)\?->can\(\'update\', \$resource\)"/',
+ '/<x-forms\.listbox canGate="update" :canResource="\$resource" id="s3StorageIds"/',
'/<x-forms\.listbox canGate="update" :canResource="\$resource" id="disableLocalBackup"[\s\S]*?:disabled="! \$saveToS3 \|\| ! auth\(\)->user\(\)\?->can\(\'update\', \$resource\)"/',
],
],
diff --git a/tests/Feature/ResourceTeamAfterTeamSwitchTest.php b/tests/Feature/ResourceTeamAfterTeamSwitchTest.php
index 6a434a035a..dfa4f32181 100644
--- a/tests/Feature/ResourceTeamAfterTeamSwitchTest.php
+++ b/tests/Feature/ResourceTeamAfterTeamSwitchTest.php
@@ -175,7 +175,7 @@ test('backup edit keeps the S3 storage of the backup team when given a foreign l
expect($backup->fresh()->s3_storage_id)->toBe($ownS3->id)
->and($backup->fresh()->save_s3)->toBeTruthy();
- $component->set('s3StorageId', $foreignS3->id)->call('instantSave');
+ $component->set('s3StorageIds', [$foreignS3->id])->call('instantSave');
expect($backup->fresh()->s3_storage_id)->toBe($ownS3->id);
});
diff --git a/tests/Feature/ScheduleAuditTest.php b/tests/Feature/ScheduleAuditTest.php
index 281fdd42b4..6c700eae69 100644
--- a/tests/Feature/ScheduleAuditTest.php
+++ b/tests/Feature/ScheduleAuditTest.php
@@ -300,12 +300,12 @@ test('disabling or moving S3 storage for backups records audit events', function
$volumeEvent = scheduleAuditEvent('ui.volume_backup.schedule_set');
expect($databaseEvent->resource_uuid)->toBe($database->uuid)
- ->and($databaseEvent->metadata['changed_fields'])->toEqualCanonicalizing(['save_s3', 's3_storage_id'])
+ ->and($databaseEvent->metadata['changed_fields'])->toEqualCanonicalizing(['save_s3', 's3_storage_id', 's3_storages'])
->and($volumeEvent->metadata['resource_type'])->toBe('application')
->and($volumeEvent->metadata['resource_uuid'])->toBe($this->application->uuid)
->and($volumeEvent->metadata['storage_uuid'])->toBe($volume->uuid)
->and($volumeEvent->metadata['backup_uuid'])->toBe($volumeBackup->uuid)
- ->and($volumeEvent->metadata['changed_fields'])->toBe(['s3_storage_id']);
+ ->and($volumeEvent->metadata['changed_fields'])->toEqualCanonicalizing(['s3_storage_id', 's3_storages']);
});
test('configuring the instance database backup records an audit event', function () {
diff --git a/tests/Feature/VolumeBackupMultipleS3DestinationsTest.php b/tests/Feature/VolumeBackupMultipleS3DestinationsTest.php
new file mode 100644
index 0000000000..3eadab443a
--- /dev/null
+++ b/tests/Feature/VolumeBackupMultipleS3DestinationsTest.php
@@ -0,0 +1,386 @@
+<?php
+
+use App\Actions\Shared\DeleteScheduledVolumeBackup;
+use App\Jobs\VolumeBackupJob;
+use App\Jobs\VolumeBackupRecoveryJob;
+use App\Models\Application;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\LocalPersistentVolume;
+use App\Models\PrivateKey;
+use App\Models\Project;
+use App\Models\S3Storage;
+use App\Models\ScheduledDatabaseBackup;
+use App\Models\ScheduledVolumeBackup;
+use App\Models\ScheduledVolumeBackupExecution;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
+use Illuminate\Filesystem\FilesystemAdapter;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Event;
+use Illuminate\Support\Facades\Notification;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Storage;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+ Server::flushIdentityMap();
+ Notification::fake();
+ config(['broadcasting.default' => 'null']);
+ defined('CURLOPT_RESOLVE') || define('CURLOPT_RESOLVE', 10203);
+});
+
+/**
+ * @return array{0: ScheduledVolumeBackup, 1: S3Storage, 2: S3Storage, 3: Server}
+ */
+function createMultiDestinationVolumeBackup(array $backupAttributes = []): array
+{
+ InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
+ $team = Team::factory()->create();
+ $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
+ $server = Server::factory()->create([
+ 'team_id' => $team->id,
+ 'private_key_id' => $privateKey->id,
+ 'ip' => '203.0.113.10',
+ ]);
+ $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
+ $project = Project::factory()->create(['team_id' => $team->id]);
+ $environment = Environment::factory()->create(['project_id' => $project->id]);
+ $application = Application::factory()->create([
+ 'environment_id' => $environment->id,
+ 'destination_id' => $destination->id,
+ 'destination_type' => $destination->getMorphClass(),
+ ]);
+ $volume = LocalPersistentVolume::create([
+ 'name' => 'app-data',
+ 'mount_path' => '/data',
+ 'resource_id' => $application->id,
+ 'resource_type' => $application->getMorphClass(),
+ ]);
+ $storages = collect(['first', 'second'])->map(fn (string $name): S3Storage => S3Storage::create([
+ 'name' => "{$name} storage",
+ 'region' => 'us-east-1',
+ 'key' => "{$name}-key",
+ 'secret' => 'secret',
+ 'bucket' => "{$name}-bucket",
+ 'endpoint' => 'https://s3.amazonaws.com',
+ 'team_id' => $team->id,
+ 'is_usable' => true,
+ ]));
+ $backup = $volume->scheduledBackups()->create([
+ 'team_id' => $team->id,
+ 'frequency' => 'daily',
+ 'save_s3' => true,
+ ...$backupAttributes,
+ ]);
+ $backup->syncS3Storages($storages->pluck('id')->all());
+
+ return [$backup->fresh(), $storages[0], $storages[1], $server];
+}
+
+function fakeMultiDestinationS3Disks(): void
+{
+ $sshDisk = Storage::fake('ssh-keys');
+ $disk = Mockery::mock(FilesystemAdapter::class);
+ $disk->shouldReceive('files')->zeroOrMoreTimes()->andReturn([]);
+ $disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue();
+ Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk);
+ Storage::shouldReceive('build')->zeroOrMoreTimes()->andReturn($disk);
+}
+
+it('archives locally and uploads to every destination instead of streaming', function () {
+ [$backup, $first, $second] = createMultiDestinationVolumeBackup(['disable_local_backup' => true]);
+ fakeMultiDestinationS3Disks();
+ Process::fake([
+ '*du -b*' => '128',
+ '*' => '',
+ ]);
+
+ (new VolumeBackupJob($backup))->handle();
+
+ $execution = ScheduledVolumeBackupExecution::query()->sole();
+ Process::assertNotRan(fn ($process) => str_contains($process->command, 'mc pipe'));
+ Process::assertRan(fn ($process) => str_contains($process->command, 'tar -I') && str_contains($process->command, ' > '));
+ Process::assertRan(fn ($process) => str_contains($process->command, 'mc cp') && str_contains($process->command, 'temporary/first-bucket/'));
+ Process::assertRan(fn ($process) => str_contains($process->command, 'mc cp') && str_contains($process->command, 'temporary/second-bucket/'));
+ Process::assertRan(fn ($process) => preg_match("/rm -f '[^']+\\.tar\\.gz'/", $process->command) === 1);
+ expect($execution->status)->toBe('success')
+ ->and($execution->s3_storage_id)->toBe($first->id)
+ ->and($execution->s3_uploaded)->toBeTrue()
+ ->and($execution->local_storage_deleted)->toBeTrue()
+ ->and($execution->s3Replicas()->orderBy('s3_storage_id')->get()->map->only(['s3_storage_id', 's3_uploaded'])->all())->toBe([
+ ['s3_storage_id' => $first->id, 's3_uploaded' => true],
+ ['s3_storage_id' => $second->id, 's3_uploaded' => true],
+ ]);
+});
+
+it('keeps the local archive and cleans the partial upload when one destination fails', function () {
+ [$backup, $first, $second] = createMultiDestinationVolumeBackup(['disable_local_backup' => true]);
+ fakeMultiDestinationS3Disks();
+ Process::fake([
+ '*du -b*' => '128',
+ '*temporary/second-bucket/*' => Process::result(errorOutput: 'access denied', exitCode: 1),
+ '*' => '',
+ ]);
+
+ (new VolumeBackupJob($backup))->handle();
+
+ $execution = ScheduledVolumeBackupExecution::query()->sole();
+ $firstReplica = $execution->s3Replicas()->where('s3_storage_id', $first->id)->sole();
+ $secondReplica = $execution->s3Replicas()->where('s3_storage_id', $second->id)->sole();
+ Process::assertNotRan(fn ($process) => preg_match("/rm -f '[^']+\\.tar\\.gz'/", $process->command) === 1);
+ expect($execution->status)->toBe('success')
+ ->and($execution->message)->toContain('S3 upload failed: second storage:')
+ ->and($execution->s3_uploaded)->toBeFalse()
+ ->and($execution->s3_cleanup_pending)->toBeFalse()
+ ->and($execution->local_storage_deleted)->toBeFalse()
+ ->and($firstReplica->s3_uploaded)->toBeTrue()
+ ->and($firstReplica->s3_storage_deleted)->toBeFalse()
+ ->and($secondReplica->s3_uploaded)->toBeFalse()
+ ->and($secondReplica->s3_storage_deleted)->toBeTrue()
+ ->and($secondReplica->message)->not->toBeEmpty();
+});
+
+it('applies S3 retention to each destination separately', function () {
+ [$backup, $first, $second, $server] = createMultiDestinationVolumeBackup([
+ 'retention_amount_locally' => 0,
+ 'retention_days_locally' => 0,
+ 'retention_max_storage_locally' => 0,
+ 'retention_amount_s3' => 1,
+ 'retention_days_s3' => 0,
+ 'retention_max_storage_s3' => 0,
+ ]);
+ $createExecution = function (string $name, int $daysAgo, array $uploadedTo) use ($backup, $first, $second): ScheduledVolumeBackupExecution {
+ $execution = ScheduledVolumeBackupExecution::create([
+ 'scheduled_volume_backup_id' => $backup->id,
+ 's3_storage_id' => $first->id,
+ 'status' => 'success',
+ 'filename' => "/data/coolify/backups/volumes/test/{$name}.tar.gz",
+ 'local_storage_deleted' => true,
+ 'created_at' => now()->subDays($daysAgo),
+ ]);
+ foreach ([$first, $second] as $storage) {
+ $execution->s3Replicas()->create([
+ 's3_storage_id' => $storage->id,
+ 's3_uploaded' => in_array($storage->id, $uploadedTo, true),
+ ]);
+ }
+ $execution->refreshS3Summary();
+
+ return $execution;
+ };
+ $oldest = $createExecution('oldest', 3, [$first->id]);
+ $older = $createExecution('older', 2, [$first->id, $second->id]);
+ $latest = $createExecution('latest', 1, [$first->id]);
+ $disk = Mockery::mock();
+ $disk->shouldReceive('delete')
+ ->once()
+ ->with(['/data/coolify/backups/volumes/test/older.tar.gz', '/data/coolify/backups/volumes/test/oldest.tar.gz'])
+ ->andReturnTrue();
+ Storage::shouldReceive('build')
+ ->once()
+ ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'first-key'))
+ ->andReturn($disk);
+ $job = new VolumeBackupJob($backup);
+
+ (new ReflectionClass($job))->getMethod('removeExpiredBackups')->invoke($job, $server);
+
+ expect($oldest->fresh())->toBeNull()
+ ->and($older->fresh())->not->toBeNull()
+ ->and($older->s3Replicas()->where('s3_storage_id', $first->id)->sole()->s3_storage_deleted)->toBeTrue()
+ ->and($older->s3Replicas()->where('s3_storage_id', $second->id)->sole()->s3_storage_deleted)->toBeFalse()
+ ->and($older->fresh()->s3_storage_deleted)->toBeFalse()
+ ->and($latest->s3Replicas()->where('s3_storage_id', $first->id)->sole()->s3_storage_deleted)->toBeFalse();
+});
+
+it('continues retention and execution cleanup when a replica storage is missing', function () {
+ $this->freezeTime();
+ [$backup, , $second, $server] = createMultiDestinationVolumeBackup([
+ 'retention_amount_locally' => 0,
+ 'retention_days_locally' => 0,
+ 'retention_max_storage_locally' => 0,
+ 'retention_amount_s3' => 0,
+ 'retention_days_s3' => 7,
+ 'retention_max_storage_s3' => 0,
+ ]);
+ $executions = collect([false, true])->map(function (bool $localDeleted) use ($backup, $second): ScheduledVolumeBackupExecution {
+ $execution = $backup->executions()->create([
+ 'status' => 'success',
+ 'filename' => '/data/coolify/backups/volumes/test/'.($localDeleted ? 'deleted' : 'local').'.tar.gz',
+ 'local_storage_deleted' => $localDeleted,
+ ]);
+ $execution->forceFill(['created_at' => now()->subDays(10)])->save();
+ $execution->s3Replicas()->create(['s3_storage_id' => null, 's3_uploaded' => true]);
+ $execution->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => true]);
+ $execution->refreshS3Summary();
+
+ return $execution;
+ });
+ $disk = Mockery::mock();
+ $disk->shouldReceive('delete')->once()->with(Mockery::on(
+ fn (array $filenames): bool => count($filenames) === 2
+ && in_array($executions[0]->filename, $filenames, true)
+ && in_array($executions[1]->filename, $filenames, true),
+ ))->andReturnTrue();
+ Storage::shouldReceive('build')->once()
+ ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'second-key'))
+ ->andReturn($disk);
+ $job = new VolumeBackupJob($backup);
+
+ (new ReflectionClass($job))->getMethod('removeExpiredBackups')->invoke($job, $server);
+
+ expect($executions[0]->fresh()->s3_storage_deleted)->toBeTrue()
+ ->and($executions[0]->s3Replicas()->where('s3_storage_deleted', false)->count())->toBe(0);
+ $this->assertModelMissing($executions[1]);
+});
+
+it('cleans an interrupted upload from every destination whose copy did not finish', function () {
+ [$backup, $first, $second] = createMultiDestinationVolumeBackup();
+ $execution = ScheduledVolumeBackupExecution::create([
+ 'scheduled_volume_backup_id' => $backup->id,
+ 's3_storage_id' => $first->id,
+ 'status' => 'failed',
+ 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz',
+ 's3_cleanup_pending' => true,
+ ]);
+ $uploaded = $execution->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]);
+ $interrupted = $execution->s3Replicas()->create(['s3_storage_id' => $second->id]);
+ $withoutStorage = $execution->s3Replicas()->create(['s3_storage_id' => null, 's3_uploaded' => false]);
+ $disk = Mockery::mock();
+ $disk->shouldReceive('delete')->once()->with(['/data/coolify/backups/volumes/test/interrupted.tar.gz'])->andReturnTrue();
+ Storage::shouldReceive('build')
+ ->once()
+ ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'second-key'))
+ ->andReturn($disk);
+
+ VolumeBackupRecoveryJob::cleanupS3Upload($execution);
+
+ expect($execution->fresh()->s3_cleanup_pending)->toBeFalse()
+ ->and($uploaded->fresh()->s3_storage_deleted)->toBeFalse()
+ ->and($interrupted->fresh()->s3_storage_deleted)->toBeTrue()
+ ->and($withoutStorage->fresh()->s3_storage_deleted)->toBeTrue();
+});
+
+it('keeps the cleanup pending when one destination cannot delete the partial upload', function () {
+ [$backup, $first, $second] = createMultiDestinationVolumeBackup();
+ $execution = ScheduledVolumeBackupExecution::create([
+ 'scheduled_volume_backup_id' => $backup->id,
+ 's3_storage_id' => $first->id,
+ 'status' => 'failed',
+ 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz',
+ 's3_cleanup_pending' => true,
+ ]);
+ $failing = $execution->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => false]);
+ $cleaned = $execution->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => false]);
+ $failingDisk = Mockery::mock();
+ $failingDisk->shouldReceive('delete')->once()->andReturnFalse();
+ $cleanedDisk = Mockery::mock();
+ $cleanedDisk->shouldReceive('delete')->once()->andReturnTrue();
+ Storage::shouldReceive('build')->with(Mockery::on(fn (array $config): bool => $config['key'] === 'first-key'))->andReturn($failingDisk);
+ Storage::shouldReceive('build')->with(Mockery::on(fn (array $config): bool => $config['key'] === 'second-key'))->andReturn($cleanedDisk);
+
+ expect(fn () => VolumeBackupRecoveryJob::cleanupS3Upload($execution))->toThrow(RuntimeException::class);
+
+ expect($execution->fresh()->s3_cleanup_pending)->toBeTrue()
+ ->and($failing->fresh()->s3_storage_deleted)->toBeFalse()
+ ->and($cleaned->fresh()->s3_storage_deleted)->toBeTrue();
+});
+
+it('moves the primary destination to a remaining storage when the primary storage is deleted', function () {
+ [$backup, $first, $second] = createMultiDestinationVolumeBackup();
+ $execution = ScheduledVolumeBackupExecution::create([
+ 'scheduled_volume_backup_id' => $backup->id,
+ 's3_storage_id' => $first->id,
+ 'status' => 'success',
+ 'filename' => '/data/coolify/backups/volumes/test/archive.tar.gz',
+ ]);
+ $execution->s3Replicas()->create(['s3_storage_id' => $first->id, 's3_uploaded' => true]);
+ $execution->s3Replicas()->create(['s3_storage_id' => $second->id, 's3_uploaded' => true]);
+ $execution->refreshS3Summary();
+
+ expect($backup->s3_storage_id)->toBe($first->id);
+
+ $first->delete();
+
+ $backup->refresh();
+ expect($backup->save_s3)->toBeTrue()
+ ->and($backup->s3_storage_id)->toBe($second->id)
+ ->and($backup->s3Storages()->pluck('s3_storages.id')->all())->toBe([$second->id])
+ ->and($execution->fresh()->s3_storage_deleted)->toBeFalse()
+ ->and($execution->hasLiveS3Copies())->toBeTrue();
+});
+
+it('preserves the schedule and uploaded replica when its S3 storage is unavailable', function () {
+ [$backup] = createMultiDestinationVolumeBackup();
+ $execution = ScheduledVolumeBackupExecution::create([
+ 'scheduled_volume_backup_id' => $backup->id,
+ 'status' => 'success',
+ 'filename' => '/data/coolify/backups/volumes/test/archive.tar.gz',
+ 'local_storage_deleted' => true,
+ ]);
+ $replica = $execution->s3Replicas()->create([
+ 's3_storage_id' => null,
+ 's3_uploaded' => true,
+ 's3_storage_deleted' => false,
+ ]);
+ $execution->refreshS3Summary();
+
+ expect(fn () => (new DeleteScheduledVolumeBackup)->handle($backup, deleteLocalArchives: false))
+ ->toThrow(RuntimeException::class, 'The S3 storage used by an existing backup is unavailable.');
+
+ expect($backup->fresh())->not->toBeNull();
+ expect($replica->fresh()->s3_storage_deleted)->toBeFalse();
+ expect($execution->fresh()->s3_storage_deleted)->toBeFalse();
+ expect($execution->hasLiveS3Copies())->toBeTrue();
+});
+
+it('rolls back all backup detachment changes when an execution update fails', function () {
+ [$backup, $storage] = createMultiDestinationVolumeBackup();
+ $backup->syncS3Storages([$storage->id]);
+ $databaseBackup = ScheduledDatabaseBackup::create([
+ 'frequency' => 'daily',
+ 'save_s3' => true,
+ 'database_type' => 'App\\Models\\StandalonePostgresql',
+ 'database_id' => 1,
+ 'team_id' => $storage->team_id,
+ ]);
+ $databaseBackup->syncS3Storages([$storage->id]);
+ $databaseExecution = $databaseBackup->executions()->create([
+ 'uuid' => 'rollback-database',
+ 'database_name' => 'app',
+ 'status' => 'success',
+ 's3_uploaded' => true,
+ ]);
+ $databaseReplica = $databaseExecution->s3Replicas()->create([
+ 's3_storage_id' => $storage->id,
+ 's3_uploaded' => true,
+ ]);
+ $execution = ScheduledVolumeBackupExecution::create([
+ 'scheduled_volume_backup_id' => $backup->id,
+ 's3_storage_id' => $storage->id,
+ 'status' => 'failed',
+ 's3_cleanup_pending' => true,
+ ]);
+ $replica = $execution->s3Replicas()->create([
+ 's3_storage_id' => $storage->id,
+ 's3_uploaded' => false,
+ ]);
+ Event::listen('eloquent.updating: '.ScheduledVolumeBackupExecution::class, function (): void {
+ throw new RuntimeException('Execution update failed');
+ });
+
+ expect(fn () => $storage->delete())->toThrow(RuntimeException::class, 'Execution update failed');
+
+ $this->assertModelExists($storage);
+ foreach ([$databaseBackup, $backup] as $schedule) {
+ expect($schedule->fresh()->s3_storage_id)->toBe($storage->id)
+ ->and((bool) $schedule->fresh()->save_s3)->toBeTrue()
+ ->and($schedule->s3Storages()->pluck('s3_storages.id')->all())->toBe([$storage->id]);
+ }
+ expect($databaseReplica->fresh()->s3_storage_deleted)->toBeFalse()
+ ->and($databaseExecution->fresh()->s3_storage_deleted)->toBeFalse()
+ ->and($replica->fresh()->s3_storage_deleted)->toBeFalse()
+ ->and($execution->fresh()->s3_cleanup_pending)->toBeTrue();
+});
diff --git a/tests/Feature/VolumeBackupRecoveryBoundsTest.php b/tests/Feature/VolumeBackupRecoveryBoundsTest.php
index ff3ed2d031..08d90ab774 100644
--- a/tests/Feature/VolumeBackupRecoveryBoundsTest.php
+++ b/tests/Feature/VolumeBackupRecoveryBoundsTest.php
@@ -93,7 +93,7 @@ function createRecoveryBoundsBackup(Team $team, array $backupAttributes = []): a
function createRecoveryBoundsExecution(ScheduledVolumeBackup $backup, array $attributes = []): ScheduledVolumeBackupExecution
{
- return ScheduledVolumeBackupExecution::create([
+ $execution = ScheduledVolumeBackupExecution::create([
'scheduled_volume_backup_id' => $backup->id,
's3_storage_id' => $backup->s3_storage_id,
'status' => 'failed',
@@ -101,6 +101,9 @@ function createRecoveryBoundsExecution(ScheduledVolumeBackup $backup, array $att
's3_cleanup_pending' => true,
...$attributes,
]);
+ $execution->s3Replicas()->create(['s3_storage_id' => $backup->s3_storage_id]);
+
+ return $execution;
}
function failRecoveryBoundsS3Delete(string $awsCode): void
@@ -164,7 +167,7 @@ it('records a categorized S3 cleanup failure without throwing or leaking the S3
$execution->refresh();
expect($execution->s3_cleanup_pending)->toBeTrue()
- ->and($execution->s3_storage_deleted)->toBeFalse()
+ ->and($execution->s3Replicas()->sole()->s3_storage_deleted)->toBeFalse()
->and($execution->recovery_error)->toBe('s3_auth')
->and($execution->recovery_last_attempt_at->equalTo(now()))->toBeTrue()
->and($execution->recovery_needs_attention)->toBeTrue()
@@ -261,7 +264,7 @@ it('cleans the S3 upload even when container recovery fails', function () {
$execution->refresh();
expect($execution->stop_recovery_pending)->toBeTrue()
->and($execution->s3_cleanup_pending)->toBeFalse()
- ->and($execution->s3_storage_deleted)->toBeTrue()
+ ->and($execution->s3Replicas()->sole()->s3_storage_deleted)->toBeTrue()
->and($execution->recovery_error)->toBe('server_unreachable')
->and($execution->recovery_needs_attention)->toBeFalse();
Notification::assertNothingSent();
@@ -281,7 +284,7 @@ it('resets the recovery state after a successful attempt', function () {
$execution->refresh();
expect($execution->s3_cleanup_pending)->toBeFalse()
- ->and($execution->s3_storage_deleted)->toBeTrue()
+ ->and($execution->s3Replicas()->sole()->s3_storage_deleted)->toBeTrue()
->and($execution->recovery_error)->toBeNull()
->and($execution->recovery_last_attempt_at)->toBeNull()
->and($execution->recovery_needs_attention)->toBeFalse();
diff --git a/tests/Feature/VolumeBackupTest.php b/tests/Feature/VolumeBackupTest.php
index d247a7a4e6..66eadf98ca 100644
--- a/tests/Feature/VolumeBackupTest.php
+++ b/tests/Feature/VolumeBackupTest.php
@@ -867,6 +867,7 @@ it('deletes an individual S3 archive from the storage recorded on its execution'
'local_storage_deleted' => true,
's3_uploaded' => true,
]);
+ $execution->s3Replicas()->create(['s3_storage_id' => $originalStorage->id, 's3_uploaded' => true]);
$disk = Mockery::mock();
$disk->shouldReceive('delete')
->once()
@@ -981,7 +982,7 @@ it('persists S3 settings the first time when a volume backup schedule does not e
'resource' => $application,
'section' => 's3',
])
- ->assertSet('s3StorageId', $s3Storage->id)
+ ->assertSet('s3StorageIds', [$s3Storage->id])
->call('toggleS3')
->assertSet('saveToS3', true)
->assertDispatched('success');
@@ -1013,7 +1014,7 @@ it('persists the selected S3 storage when a volume backup schedule does not exis
'resource' => $application,
'section' => 's3',
])
- ->set('s3StorageId', $s3Storage->id)
+ ->set('s3StorageIds', [$s3Storage->id])
->assertDispatched('success');
$backup = ScheduledVolumeBackup::query()->sole();
@@ -1136,7 +1137,7 @@ it('does not change S3 storage when another volume backup setting is invalid', f
'section' => 's3',
])
->set('frequency', 'not a valid schedule')
- ->set('s3StorageId', $secondS3Storage->id)
+ ->set('s3StorageIds', [$secondS3Storage->id])
->assertHasErrors('frequency')
->assertNotDispatched('success');
@@ -1254,18 +1255,18 @@ it('lists and accepts only S3 storages of the resource team when the session tea
$component = Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]);
expect($component->get('availableS3Storages')->pluck('id')->all())->toBe([$resourceTeamStorage->id])
- ->and($component->get('s3StorageId'))->toBe($resourceTeamStorage->id);
+ ->and($component->get('s3StorageIds'))->toBe([$resourceTeamStorage->id]);
$component->set('frequency', 'daily')
->set('saveToS3', true)
- ->set('s3StorageId', $otherTeamStorage->id)
- ->assertHasErrors('s3StorageId')
+ ->set('s3StorageIds', [$otherTeamStorage->id])
+ ->assertHasErrors('s3StorageIds')
->call('save')
- ->assertHasErrors('s3StorageId');
+ ->assertHasErrors('s3StorageIds');
expect(ScheduledVolumeBackup::query()->count())->toBe(0);
- $component->set('s3StorageId', $resourceTeamStorage->id)
+ $component->set('s3StorageIds', [$resourceTeamStorage->id])
->call('save')
->assertHasNoErrors();
@@ -1295,9 +1296,9 @@ it('only accepts a usable S3 storage owned by the current team', function () {
Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application])
->set('frequency', 'daily')
->set('saveToS3', true)
- ->set('s3StorageId', $foreignStorage->id)
+ ->set('s3StorageIds', [$foreignStorage->id])
->call('save')
- ->assertHasErrors('s3StorageId');
+ ->assertHasErrors('s3StorageIds');
expect(ScheduledVolumeBackup::query()->count())->toBe(0);
});
@@ -1322,7 +1323,7 @@ it('saves the database-style S3 backup controls immediately', function () {
]);
$component = Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application])
- ->assertSet('s3StorageId', $s3Storage->id)
+ ->assertSet('s3StorageIds', [$s3Storage->id])
->set('saveToS3', true)
->call('instantSave')
->set('disableLocalBackup', true)
@@ -1378,7 +1379,7 @@ it('saves the selected volume backup S3 storage immediately while S3 is disabled
]);
Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application, 'section' => 's3'])
- ->set('s3StorageId', $secondS3Storage->id)
+ ->set('s3StorageIds', [$secondS3Storage->id])
->assertDispatched('success');
$backup = ScheduledVolumeBackup::query()->sole();
@@ -1486,6 +1487,7 @@ it('disables S3 volume backups when the storage is deleted', function () {
's3_uploaded' => true,
's3_cleanup_pending' => true,
]);
+ $execution->s3Replicas()->create(['s3_storage_id' => $s3Storage->id, 's3_uploaded' => true]);
$s3Storage->delete();
@@ -1530,6 +1532,7 @@ it('marks historical executions deleted when their recorded S3 storage is remove
's3_uploaded' => true,
's3_cleanup_pending' => true,
]);
+ $execution->s3Replicas()->create(['s3_storage_id' => $originalStorage->id, 's3_uploaded' => true]);
$originalStorage->delete();
@@ -1704,7 +1707,7 @@ it('deletes S3 archives from the storage recorded on each execution', function (
'frequency' => 'daily',
'save_s3' => true,
]);
- ScheduledVolumeBackupExecution::create([
+ $execution = ScheduledVolumeBackupExecution::create([
'scheduled_volume_backup_id' => $backup->id,
's3_storage_id' => $originalStorage->id,
'status' => 'success',
@@ -1712,6 +1715,7 @@ it('deletes S3 archives from the storage recorded on each execution', function (
'local_storage_deleted' => true,
's3_uploaded' => true,
]);
+ $execution->s3Replicas()->create(['s3_storage_id' => $originalStorage->id, 's3_uploaded' => true]);
$disk = Mockery::mock();
$disk->shouldReceive('delete')
->once()
@@ -1977,7 +1981,8 @@ it('keeps the upload destination on the volume backup execution', function () {
$execution = ScheduledVolumeBackupExecution::query()->sole();
expect($execution->s3_storage_id)->toBe($s3Storage->id)
- ->and($execution->s3->is($s3Storage))->toBeTrue();
+ ->and($execution->s3->is($s3Storage))->toBeTrue()
+ ->and($execution->s3Replicas()->sole()->s3_storage_id)->toBe($s3Storage->id);
});
it('streams S3-only volume backups without creating or copying a local archive', function () {
@@ -2032,6 +2037,7 @@ it('streams S3-only volume backups without creating or copying a local archive',
expect($execution->status)->toBe('success')
->and($execution->size)->toBe(128)
->and($execution->s3_uploaded)->toBeTrue()
+ ->and($execution->s3Replicas()->sole()->s3_uploaded)->toBeTrue()
->and($execution->local_storage_deleted)->toBeTrue();
});
@@ -2240,15 +2246,26 @@ it('removes retained S3 archives from the storage recorded on each execution', f
'status' => 'success',
'filename' => '/data/coolify/backups/volumes/test/old.tar.gz',
's3_uploaded' => true,
+ 'created_at' => now()->subDays(2),
+ ]);
+ $oldExecution->s3Replicas()->create(['s3_storage_id' => $originalStorage->id, 's3_uploaded' => true]);
+ $retainedOriginalExecution = ScheduledVolumeBackupExecution::create([
+ 'scheduled_volume_backup_id' => $backup->id,
+ 's3_storage_id' => $originalStorage->id,
+ 'status' => 'success',
+ 'filename' => '/data/coolify/backups/volumes/test/retained-original.tar.gz',
+ 's3_uploaded' => true,
'created_at' => now()->subDay(),
]);
- ScheduledVolumeBackupExecution::create([
+ $retainedOriginalExecution->s3Replicas()->create(['s3_storage_id' => $originalStorage->id, 's3_uploaded' => true]);
+ $newExecution = ScheduledVolumeBackupExecution::create([
'scheduled_volume_backup_id' => $backup->id,
's3_storage_id' => $newStorage->id,
'status' => 'success',
'filename' => '/data/coolify/backups/volumes/test/new.tar.gz',
's3_uploaded' => true,
]);
+ $newExecution->s3Replicas()->create(['s3_storage_id' => $newStorage->id, 's3_uploaded' => true]);
$disk = Mockery::mock();
$disk->shouldReceive('delete')->once()->with(['/data/coolify/backups/volumes/test/old.tar.gz'])->andReturnTrue();
Storage::shouldReceive('build')
@@ -2260,7 +2277,10 @@ it('removes retained S3 archives from the storage recorded on each execution', f
$method->invoke($job, $server);
- expect($oldExecution->fresh()->s3_storage_deleted)->toBeTrue();
+ expect($oldExecution->fresh()->s3_storage_deleted)->toBeTrue()
+ ->and($oldExecution->s3Replicas()->sole()->s3_storage_deleted)->toBeTrue()
+ ->and($retainedOriginalExecution->fresh()->s3_storage_deleted)->toBeFalse()
+ ->and($newExecution->fresh()->s3_storage_deleted)->toBeFalse();
});
it('does not query execution history when local retention is unlimited', function () {
@@ -2514,6 +2534,7 @@ it('cleans an interrupted S3 upload and coordinates recovery with the backup loc
'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz',
's3_cleanup_pending' => true,
]);
+ $execution->s3Replicas()->create(['s3_storage_id' => $s3Storage->id]);
$disk = Mockery::mock();
$disk->shouldReceive('delete')->once()->andReturnTrue();
Storage::shouldReceive('build')->once()->andReturn($disk);
@@ -2523,7 +2544,7 @@ it('cleans an interrupted S3 upload and coordinates recovery with the backup loc
$job->handle();
expect($execution->fresh()->s3_cleanup_pending)->toBeFalse()
- ->and($execution->fresh()->s3_storage_deleted)->toBeTrue();
+ ->and($execution->s3Replicas()->sole()->s3_storage_deleted)->toBeTrue();
});
it('cleans an interrupted upload from the execution S3 storage after a schedule switch', function () {
@@ -2562,6 +2583,7 @@ it('cleans an interrupted upload from the execution S3 storage after a schedule
'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz',
's3_cleanup_pending' => true,
]);
+ $execution->s3Replicas()->create(['s3_storage_id' => $originalStorage->id]);
$disk = Mockery::mock();
$disk->shouldReceive('delete')->once()->andReturnTrue();
Storage::shouldReceive('build')
@@ -2572,7 +2594,7 @@ it('cleans an interrupted upload from the execution S3 storage after a schedule
VolumeBackupRecoveryJob::cleanupS3Upload($execution);
expect($execution->fresh()->s3_cleanup_pending)->toBeFalse()
- ->and($execution->fresh()->s3_storage_deleted)->toBeTrue();
+ ->and($execution->s3Replicas()->sole()->s3_storage_deleted)->toBeTrue();
});
it('keeps the S3 key tracked when interrupted upload cleanup must be retried', function () {
@@ -2603,6 +2625,7 @@ it('keeps the S3 key tracked when interrupted upload cleanup must be retried', f
'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz',
's3_cleanup_pending' => true,
]);
+ $execution->s3Replicas()->create(['s3_storage_id' => $s3Storage->id]);
$disk = Mockery::mock();
$disk->shouldReceive('delete')->once()->andReturnFalse();
Storage::shouldReceive('build')->once()->andReturn($disk);
diff --git a/tests/Unit/ServerTransfer/ServerTransferS3DestinationsTest.php b/tests/Unit/ServerTransfer/ServerTransferS3DestinationsTest.php
new file mode 100644
index 0000000000..479367ee70
--- /dev/null
+++ b/tests/Unit/ServerTransfer/ServerTransferS3DestinationsTest.php
@@ -0,0 +1,144 @@
+<?php
+
+use App\Models\Application;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\LocalPersistentVolume;
+use App\Models\PrivateKey;
+use App\Models\Project;
+use App\Models\S3Storage;
+use App\Models\ScheduledDatabaseBackup;
+use App\Models\ScheduledVolumeBackup;
+use App\Models\Server;
+use App\Models\StandaloneDocker;
+use App\Models\StandalonePostgresql;
+use App\Models\Team;
+use App\Services\ServerTransfer\ServerTransferExporter;
+use App\Services\ServerTransfer\ServerTransferImporter;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Tests\TestCase;
+
+uses(TestCase::class, RefreshDatabase::class);
+
+beforeEach(function () {
+ Server::flushIdentityMap();
+ InstanceSettings::forceCreate(['id' => 0, 'is_api_enabled' => true, 'fqdn' => 'https://coolify-a.test']);
+
+ $this->team = Team::factory()->create();
+ $privateKey = PrivateKey::factory()->create(['team_id' => $this->team->id]);
+ $this->server = Server::factory()->create([
+ 'team_id' => $this->team->id,
+ 'private_key_id' => $privateKey->id,
+ 'ip' => '10.56.0.10',
+ ]);
+ $destination = StandaloneDocker::where('server_id', $this->server->id)->firstOrFail();
+ $project = Project::factory()->create(['team_id' => $this->team->id]);
+ $environment = $project->environments()->first()
+ ?? Environment::factory()->create(['project_id' => $project->id]);
+
+ $this->application = Application::factory()->create([
+ 'environment_id' => $environment->id,
+ 'destination_id' => $destination->id,
+ 'destination_type' => $destination->getMorphClass(),
+ ]);
+ $volume = LocalPersistentVolume::create([
+ 'name' => $this->application->uuid.'-data',
+ 'mount_path' => '/app/data',
+ 'resource_type' => $this->application->getMorphClass(),
+ 'resource_id' => $this->application->id,
+ ]);
+
+ StandalonePostgresql::withoutEvents(function () use ($environment, $destination) {
+ $database = new StandalonePostgresql;
+ $database->forceFill([
+ 'name' => 'app-db',
+ 'postgres_user' => 'postgres',
+ 'postgres_password' => 'secret',
+ 'postgres_db' => 'app',
+ 'environment_id' => $environment->id,
+ 'destination_id' => $destination->id,
+ 'destination_type' => $destination->getMorphClass(),
+ ]);
+ $database->uuid = new_public_id();
+ $database->save();
+ $this->database = $database;
+ });
+
+ $this->firstS3 = S3Storage::create([
+ 'name' => 'first', 'region' => 'us-east-1', 'key' => 'k1', 'secret' => 's1',
+ 'bucket' => 'first-bucket', 'endpoint' => 'https://first.example.com', 'team_id' => $this->team->id,
+ ]);
+ $this->secondS3 = S3Storage::create([
+ 'name' => 'second', 'region' => 'us-east-1', 'key' => 'k2', 'secret' => 's2',
+ 'bucket' => 'second-bucket', 'endpoint' => 'https://second.example.com', 'team_id' => $this->team->id,
+ ]);
+
+ $this->databaseBackup = ScheduledDatabaseBackup::create([
+ 'team_id' => $this->team->id,
+ 'save_s3' => true,
+ 'frequency' => '0 2 * * *',
+ 's3_storage_id' => $this->secondS3->id,
+ 'database_type' => $this->database->getMorphClass(),
+ 'database_id' => $this->database->id,
+ ]);
+ $this->databaseBackup->syncS3Storages([$this->firstS3->id, $this->secondS3->id]);
+
+ $this->volumeBackup = $volume->scheduledBackups()->create([
+ 'team_id' => $this->team->id,
+ 'save_s3' => true,
+ 'frequency' => '0 3 * * *',
+ 's3_storage_id' => $this->secondS3->id,
+ ]);
+ $this->volumeBackup->syncS3Storages([$this->firstS3->id, $this->secondS3->id]);
+});
+
+/**
+ * @return list<string> Buckets of the backup's destinations, primary first.
+ */
+function transferredBackupBuckets(ScheduledDatabaseBackup|ScheduledVolumeBackup $backup): array
+{
+ return $backup->selectedS3Storages()->pluck('bucket')->all();
+}
+
+function importTransferBundleIntoNewTeam(array $bundle, $context): Team
+{
+ $targetTeam = Team::factory()->create();
+ $context->application->forceDelete();
+ $context->database->forceDelete();
+ $context->server->forceDelete();
+
+ app(ServerTransferImporter::class)->import($bundle, teamId: $targetTeam->id);
+
+ return $targetTeam;
+}
+
+test('transfer keeps every S3 destination of database and volume backups', function () {
+ $bundle = app(ServerTransferExporter::class)->export($this->server);
+
+ expect(collect($bundle['s3_storages'])->pluck('uuid')->sort()->values()->all())
+ ->toEqual(collect([$this->firstS3->uuid, $this->secondS3->uuid])->sort()->values()->all());
+
+ $targetTeam = importTransferBundleIntoNewTeam($bundle, $this);
+
+ $databaseBackup = ScheduledDatabaseBackup::where('team_id', $targetTeam->id)->sole();
+ $volumeBackup = ScheduledVolumeBackup::where('team_id', $targetTeam->id)->sole();
+
+ expect(transferredBackupBuckets($databaseBackup))->toBe(['second-bucket', 'first-bucket'])
+ ->and($databaseBackup->save_s3)->toBeTruthy()
+ ->and($databaseBackup->s3->team_id)->toBe($targetTeam->id)
+ ->and(transferredBackupBuckets($volumeBackup))->toBe(['second-bucket', 'first-bucket'])
+ ->and($volumeBackup->save_s3)->toBeTrue()
+ ->and($volumeBackup->s3->team_id)->toBe($targetTeam->id);
+});
+
+test('transfer imports bundles that only have the primary S3 destination', function () {
+ $bundle = app(ServerTransferExporter::class)->export($this->server);
+ data_forget($bundle, 'projects.*.environments.*.databases.*.scheduled_backups.*.s3_storage_uuids');
+ data_forget($bundle, 'volume_backups.*.s3_storage_uuids');
+ expect(data_get($bundle, 'volume_backups.0'))->not->toHaveKey('s3_storage_uuids');
+
+ $targetTeam = importTransferBundleIntoNewTeam($bundle, $this);
+
+ expect(transferredBackupBuckets(ScheduledDatabaseBackup::where('team_id', $targetTeam->id)->sole()))->toBe(['second-bucket'])
+ ->and(transferredBackupBuckets(ScheduledVolumeBackup::where('team_id', $targetTeam->id)->sole()))->toBe(['second-bucket']);
+});