Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(backups): make overlapping and concurrent backup runs safe

Andras Bacsai1c68eafa8499
diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php
index 890e14462e..630e1153fc 100644
--- a/app/Jobs/DatabaseBackupJob.php
+++ b/app/Jobs/DatabaseBackupJob.php
@@ -22,6 +22,8 @@ use App\Rules\SafeWebhookUrl;
use App\Services\ScheduledJobDeliveryService;
use App\Support\BackupCompression;
use App\Support\ClickhouseBackupCommand;
+use App\Support\DatabaseOperationReservation;
+use App\Support\ResourceStartActivity;
use App\Support\ValidationPatterns;
use Carbon\Carbon;
use Illuminate\Bus\Queueable;
@@ -29,10 +31,10 @@ use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
-use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Str;
+use Spatie\Activitylog\Models\Activity;
use Throwable;
class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
@@ -91,7 +93,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
{
$expireAfter = ($this->backup->timeout ?? 3600) + 300;
- return [(new WithoutOverlapping('database-backup-'.$this->backup->id))->expireAfter($expireAfter)->dontRelease()];
+ return [ScheduledJobDeliveryService::withoutOverlapping('database-backup-'.$this->backup->id, $this->occurrenceUuid)->expireAfter($expireAfter)->dontRelease()];
}
public function handle(): void
@@ -138,6 +140,12 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
return;
}
+ if ($this->databaseOperationInProgress()) {
+ $this->recordSkippedBackup('Skipped: a start, restart or import of this database was in progress. A backup taken now could contain a partly restored database.');
+ $this->logSkippedRun('database_operation_in_progress');
+
+ return;
+ }
if (data_get($this->backup, 'database_type') === ServiceDatabase::class) {
$databaseType = $this->database->databaseType();
$serviceUuid = $this->database->service->uuid;
@@ -146,7 +154,8 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
if (! ValidationPatterns::isValidContainerName($this->container_name)) {
throw new \Exception('Invalid database container name.');
}
- $this->directory_name = $serviceName.'-'.str($this->database->name)->slug().'-'.$serviceUuid;
+ // The container name is validated above, so it is a safe folder name. Keep the folder of earlier releases.
+ $this->directory_name = $serviceName.'-'.$this->container_name;
$escapedContainerName = escapeshellarg($this->container_name);
if (str($databaseType)->contains('postgres')) {
$commands[] = "docker exec {$escapedContainerName} env | grep POSTGRES_";
@@ -321,9 +330,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
try {
if (str($databaseType)->contains('postgres')) {
$fileDatabaseName = $this->backupFilenamePart($database);
- $this->backup_file = "/pg-dump-$fileDatabaseName-".Carbon::now()->timestamp.'.dmp';
+ $this->backup_file = "/pg-dump-$fileDatabaseName-".Carbon::now()->timestamp."-{$this->backup_log_uuid}.dmp";
if ($this->backup->dump_all) {
- $this->backup_file = '/pg-dump-all-'.Carbon::now()->timestamp.'.gz';
+ $this->backup_file = '/pg-dump-all-'.Carbon::now()->timestamp."-{$this->backup_log_uuid}.gz";
}
$this->backup_location = $this->backup_dir.$this->backup_file;
$this->backup_log = ScheduledDatabaseBackupExecution::create([
@@ -347,7 +356,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
}
}
$fileDatabaseName = $this->backupFilenamePart($databaseName);
- $this->backup_file = "/mongo-dump-$fileDatabaseName-".Carbon::now()->timestamp.'.tar.gz';
+ $this->backup_file = "/mongo-dump-$fileDatabaseName-".Carbon::now()->timestamp."-{$this->backup_log_uuid}.tar.gz";
$this->backup_location = $this->backup_dir.$this->backup_file;
$this->backup_log = ScheduledDatabaseBackupExecution::create([
'uuid' => $this->backup_log_uuid,
@@ -360,9 +369,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$this->backup_standalone_mongodb($database);
} elseif (str($databaseType)->contains('mysql')) {
$fileDatabaseName = $this->backupFilenamePart($database);
- $this->backup_file = "/mysql-dump-$fileDatabaseName-".Carbon::now()->timestamp.'.dmp';
+ $this->backup_file = "/mysql-dump-$fileDatabaseName-".Carbon::now()->timestamp."-{$this->backup_log_uuid}.dmp";
if ($this->backup->dump_all) {
- $this->backup_file = '/mysql-dump-all-'.Carbon::now()->timestamp.'.gz';
+ $this->backup_file = '/mysql-dump-all-'.Carbon::now()->timestamp."-{$this->backup_log_uuid}.gz";
}
$this->backup_location = $this->backup_dir.$this->backup_file;
$this->backup_log = ScheduledDatabaseBackupExecution::create([
@@ -376,9 +385,9 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
$this->backup_standalone_mysql($database);
} elseif (str($databaseType)->contains('mariadb')) {
$fileDatabaseName = $this->backupFilenamePart($database);
- $this->backup_file = "/mariadb-dump-$fileDatabaseName-".Carbon::now()->timestamp.'.dmp';
+ $this->backup_file = "/mariadb-dump-$fileDatabaseName-".Carbon::now()->timestamp."-{$this->backup_log_uuid}.dmp";
if ($this->backup->dump_all) {
- $this->backup_file = '/mariadb-dump-all-'.Carbon::now()->timestamp.'.gz';
+ $this->backup_file = '/mariadb-dump-all-'.Carbon::now()->timestamp."-{$this->backup_log_uuid}.gz";
}
$this->backup_location = $this->backup_dir.$this->backup_file;
$this->backup_log = ScheduledDatabaseBackupExecution::create([
@@ -403,7 +412,7 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
BackupCreated::dispatch($this->team->id);
$this->backup_standalone_clickhouse($database);
} elseif ($this->database instanceof StandaloneSqlite) {
- $this->backup_file = "/sqlite-backup-$database-".Carbon::now()->timestamp.'.gz';
+ $this->backup_file = "/sqlite-backup-$database-".Carbon::now()->timestamp."-{$this->backup_log_uuid}.gz";
$this->backup_location = $this->backup_dir.$this->backup_file;
$this->backup_log = ScheduledDatabaseBackupExecution::create([
'uuid' => $this->backup_log_uuid,
@@ -925,6 +934,49 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
]);
}
+ /**
+ * A start, restart or import of the database is reserved, queued or running. Stale activities do
+ * not count; they are failed by the start and import paths.
+ */
+ private function databaseOperationInProgress(): bool
+ {
+ $uuid = $this->database->uuid;
+ if (blank($uuid)) {
+ return false;
+ }
+
+ if (DatabaseOperationReservation::isHeldByOther($uuid)) {
+ return true;
+ }
+
+ $liveOperation = collect([ResourceStartActivity::DATABASE_START_OPERATION, ResourceStartActivity::DATABASE_IMPORT_OPERATION])
+ ->flatMap(fn (string $operation) => ResourceStartActivity::active($uuid, $operation))
+ ->contains(fn (Activity $activity): bool => ! ResourceStartActivity::isStale($activity));
+
+ return $liveOperation
+ || ResourceStartActivity::latestRunning($uuid) !== null
+ || ($this->database instanceof ServiceDatabase && ResourceStartActivity::latestRunning($this->database->service->uuid) !== null);
+ }
+
+ /**
+ * Show a backup run that did not take a backup in the execution list. The run is expected, so it
+ * sends no failure notification.
+ */
+ private function recordSkippedBackup(string $message): void
+ {
+ $databaseNames = $this->backup->dump_all ? 'all' : data_get($this->backup, 'databases_to_backup');
+ $this->backup_log_uuid = new_public_id();
+ $this->backup_log = ScheduledDatabaseBackupExecution::create([
+ 'uuid' => $this->backup_log_uuid,
+ 'database_name' => filled($databaseNames) ? Str::limit($databaseNames, 250) : null,
+ 'scheduled_database_backup_id' => $this->backup->id,
+ 'status' => 'failed',
+ 'message' => $message,
+ 'filename' => null,
+ 'local_storage_deleted' => false,
+ ]);
+ }
+
private function markStaleExecutionsAsFailed(): void
{
try {
diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.php
index fb7e3376b5..78f091bbe2 100644
--- a/app/Jobs/VolumeBackupJob.php
+++ b/app/Jobs/VolumeBackupJob.php
@@ -17,7 +17,6 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
-use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
@@ -42,7 +41,7 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
public function middleware(): array
{
return [
- (new WithoutOverlapping('volume-backup-'.$this->backup->id))
+ ScheduledJobDeliveryService::withoutOverlapping('volume-backup-'.$this->backup->id, $this->occurrenceUuid)
->shared()
->expireAfter($this->timeout + 60)
->dontRelease(),
diff --git a/app/Services/ScheduledJobDeliveryService.php b/app/Services/ScheduledJobDeliveryService.php
index 84e8d4210f..330889c151 100644
--- a/app/Services/ScheduledJobDeliveryService.php
+++ b/app/Services/ScheduledJobDeliveryService.php
@@ -2,17 +2,25 @@
namespace App\Services;
+use App\Events\DockerCleanupDone;
+use App\Events\ScheduledTaskDone;
use App\Jobs\DatabaseBackupJob;
use App\Jobs\DockerCleanupJob;
use App\Jobs\ScheduledTaskJob;
use App\Jobs\VolumeBackupJob;
+use App\Models\DockerCleanupExecution;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledJobDelivery;
use App\Models\ScheduledJobState;
use App\Models\ScheduledTask;
+use App\Models\ScheduledTaskExecution;
use App\Models\ScheduledVolumeBackup;
use App\Models\Server;
+use App\Models\Team;
+use App\Notifications\ScheduledTask\TaskFailed;
+use App\Notifications\Server\DockerCleanupFailed;
use Cron\CronExpression;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
@@ -29,7 +37,8 @@ class ScheduledJobDeliveryService
public const ENQUEUED_STALE_AFTER_MINUTES = 60;
/**
- * Job types that run once when they are late. A lost task or Docker cleanup is logged as missed.
+ * Job types that run once when they are late. A lost task or Docker cleanup is not run; it is
+ * recorded as a failed execution and the team is notified.
*/
private const LATE_RUN_JOB_TYPES = ['database-backup', 'volume-backup'];
@@ -167,7 +176,7 @@ class ScheduledJobDeliveryService
}
/**
- * Publish backups again whose queued job was not started, and log lost tasks and Docker
+ * Publish backups again whose queued job was not started, and report lost tasks and Docker
* cleanups as missed. A lost backup is not run when a newer occurrence of the same schedule
* exists, so a backup that runs longer than its interval does not build up a backlog.
*
@@ -196,10 +205,15 @@ class ScheduledJobDeliveryService
private function recoverStaleEnqueuedOccurrence(ScheduledJobDelivery $occurrence): void
{
- $hasNewerOccurrence = ScheduledJobDelivery::query()
+ // complete() deletes the row of a finished occurrence, so the schedule state also counts.
+ $hasNewerOccurrence = ScheduledJobState::query()
->where('schedule_key', $occurrence->schedule_key)
- ->where('scheduled_for', '>', $occurrence->scheduled_for)
- ->exists();
+ ->where('last_scheduled_for', '>', $occurrence->scheduled_for)
+ ->exists()
+ || ScheduledJobDelivery::query()
+ ->where('schedule_key', $occurrence->schedule_key)
+ ->where('scheduled_for', '>', $occurrence->scheduled_for)
+ ->exists();
$runLate = in_array($occurrence->job_type, self::LATE_RUN_JOB_TYPES, true) && ! $hasNewerOccurrence;
// Only one scheduler node can move each occurrence.
@@ -217,6 +231,7 @@ class ScheduledJobDeliveryService
$this->logOccurrence($occurrence->uuid, $hasNewerOccurrence
? 'Scheduled occurrence skipped: its queued job was not started and a newer occurrence exists'
: 'Scheduled occurrence missed: its queued job was not started', 'warning');
+ $this->reportMissedOccurrence($occurrence);
return;
}
@@ -226,6 +241,55 @@ class ScheduledJobDeliveryService
$this->publish($occurrence);
}
+ /**
+ * Show a missed task or Docker cleanup as a failed execution and send the failure notification.
+ * The caller moved the occurrence out of `enqueued` atomically, so this runs once per occurrence,
+ * and claim() rejects the late job if it starts afterwards.
+ */
+ private function reportMissedOccurrence(ScheduledJobDelivery $occurrence): void
+ {
+ $message = 'Skipped: the queued job did not start within '.self::ENQUEUED_STALE_AFTER_MINUTES.' minutes.';
+
+ try {
+ if ($occurrence->job_type === 'scheduled-task') {
+ $task = ScheduledTask::find($occurrence->resource_id);
+ if (! $task) {
+ return;
+ }
+
+ ScheduledTaskExecution::create([
+ 'scheduled_task_id' => $task->id,
+ 'status' => 'failed',
+ 'message' => $message,
+ 'retry_count' => 0,
+ 'finished_at' => now(),
+ ]);
+ ScheduledTaskDone::dispatch($task->team_id);
+ Team::find($task->team_id)?->notify(new TaskFailed($task, $message));
+ } elseif ($occurrence->job_type === 'docker-cleanup') {
+ $server = Server::find($occurrence->resource_id);
+ if (! $server) {
+ return;
+ }
+
+ $execution = DockerCleanupExecution::create([
+ 'server_id' => $server->id,
+ 'status' => 'failed',
+ 'message' => $message,
+ 'finished_at' => now(),
+ ]);
+ event(new DockerCleanupDone($execution));
+ $server->team?->notify(new DockerCleanupFailed($server, "Docker cleanup job failed with the following error: {$message}"));
+ }
+ } catch (\Throwable $e) {
+ Log::channel('scheduled-errors')->error('Failed to report missed scheduled occurrence', [
+ 'occurrence_uuid' => $occurrence->uuid,
+ 'schedule_key' => $occurrence->schedule_key,
+ 'error' => $e->getMessage(),
+ ]);
+ }
+ }
+
public function deleteOldOccurrences(): void
{
ScheduledJobDelivery::query()
@@ -276,6 +340,53 @@ class ScheduledJobDeliveryService
->delete();
}
+ /**
+ * Skip an occurrence that no job has claimed, so stale recovery does not publish it again.
+ */
+ public function skip(string $uuid, string $reason): void
+ {
+ $skipped = ScheduledJobDelivery::query()
+ ->where('uuid', $uuid)
+ ->whereIn('status', ['pending', 'enqueued'])
+ ->update([
+ 'status' => 'skipped',
+ 'updated_at' => now(),
+ ]) === 1;
+
+ if ($skipped) {
+ $this->logOccurrence($uuid, "Scheduled occurrence skipped: {$reason}", 'warning');
+ }
+ }
+
+ /**
+ * WithoutOverlapping for the job of a scheduled occurrence. When another run holds the lock,
+ * the job does not run and its occurrence is skipped, so stale recovery does not run it later.
+ */
+ public static function withoutOverlapping(string $key, ?string $occurrenceUuid): WithoutOverlapping
+ {
+ return new class($key, $occurrenceUuid) extends WithoutOverlapping
+ {
+ public function __construct(string $key, private ?string $occurrenceUuid)
+ {
+ parent::__construct($key);
+ }
+
+ public function handle($job, $next)
+ {
+ $started = false;
+ parent::handle($job, function ($job) use ($next, &$started) {
+ $started = true;
+
+ return $next($job);
+ });
+
+ if (! $started && $this->occurrenceUuid !== null) {
+ app(ScheduledJobDeliveryService::class)->skip($this->occurrenceUuid, 'the previous run of this schedule is still running');
+ }
+ }
+ };
+ }
+
public function fail(string $uuid, string $claimToken): void
{
ScheduledJobDelivery::query()
@@ -359,8 +470,8 @@ class ScheduledJobDeliveryService
return false;
}
- // An open previous occurrence usually means the previous run is still running. Jobs with
- // WithoutOverlapping(...)->dontRelease() then drop this one without running it.
+ // An open previous occurrence usually means the previous run is still running. Backup jobs
+ // then skip this one without running it (see withoutOverlapping()).
$previousOpen = ScheduledJobDelivery::query()
->where('schedule_key', $occurrence->schedule_key)
->where('scheduled_for', '<', $occurrence->scheduled_for)
diff --git a/tests/Feature/BackupRunSafetyTest.php b/tests/Feature/BackupRunSafetyTest.php
new file mode 100644
index 0000000000..d1fb7480af
--- /dev/null
+++ b/tests/Feature/BackupRunSafetyTest.php
@@ -0,0 +1,292 @@
+<?php
+
+use App\Enums\ProcessStatus;
+use App\Events\BackupCreated;
+use App\Jobs\DatabaseBackupJob;
+use App\Jobs\VolumeBackupJob;
+use App\Models\Application;
+use App\Models\Environment;
+use App\Models\InstanceSettings;
+use App\Models\LocalPersistentVolume;
+use App\Models\PrivateKey;
+use App\Models\Project;
+use App\Models\ScheduledDatabaseBackup;
+use App\Models\ScheduledDatabaseBackupExecution;
+use App\Models\ScheduledJobDelivery;
+use App\Models\ScheduledJobState;
+use App\Models\ScheduledVolumeBackupExecution;
+use App\Models\Server;
+use App\Models\Service;
+use App\Models\ServiceDatabase;
+use App\Models\StandaloneDocker;
+use App\Models\Team;
+use App\Services\ScheduledJobDeliveryService;
+use App\Support\DatabaseOperationReservation;
+use App\Support\ResourceStartActivity;
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Event;
+use Illuminate\Support\Facades\Notification;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Queue;
+use Illuminate\Support\Facades\Storage;
+use Spatie\Activitylog\Models\Activity;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+ Server::flushIdentityMap();
+ Storage::fake('ssh-keys');
+ Storage::fake('ssh-mux');
+ Event::fake([BackupCreated::class]);
+ Notification::fake();
+ InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
+
+ $this->team = Team::factory()->create();
+ $this->server = Server::factory()->create([
+ 'team_id' => $this->team->id,
+ 'private_key_id' => PrivateKey::factory()->create(['team_id' => $this->team->id])->id,
+ ]);
+ $this->server->settings()->update(['is_reachable' => true, 'is_usable' => true]);
+ $this->destination = StandaloneDocker::where('server_id', $this->server->id)->firstOrFail();
+ $project = Project::factory()->create(['team_id' => $this->team->id]);
+ $this->environment = Environment::factory()->create(['project_id' => $project->id]);
+});
+
+/**
+ * @return Collection<int, string> The remote commands the job ran.
+ */
+function backupRunSafetyFakeRemoteCommands(array $outputs = []): Collection
+{
+ $commands = collect();
+ Process::fake(function ($process) use ($commands, $outputs) {
+ $command = is_array($process->command) ? implode(' ', $process->command) : $process->command;
+ $commands->push($command);
+ foreach ($outputs as $needle => $output) {
+ if (str_contains($command, $needle)) {
+ return Process::result(output: $output);
+ }
+ }
+
+ return Process::result(output: str_contains($command, 'du -b') ? '128' : '');
+ });
+
+ return $commands;
+}
+
+function backupRunSafetySchedule(Model $database, Team $team, array $overrides = []): ScheduledDatabaseBackup
+{
+ $backup = new ScheduledDatabaseBackup;
+ $backup->forceFill([
+ 'enabled' => true,
+ 'save_s3' => false,
+ 'frequency' => 'daily',
+ 'database_id' => $database->id,
+ 'database_type' => $database->getMorphClass(),
+ 'team_id' => $team->id,
+ ...$overrides,
+ ]);
+ $backup->save();
+
+ return $backup->fresh();
+}
+
+function backupRunSafetyOccurrence(string $scheduleKey, string $jobType, int $resourceId): ScheduledJobDelivery
+{
+ return ScheduledJobDelivery::create([
+ 'schedule_key' => $scheduleKey,
+ 'scheduled_for' => now()->startOfMinute(),
+ 'job_type' => $jobType,
+ 'resource_id' => $resourceId,
+ 'status' => 'enqueued',
+ 'enqueued_at' => now(),
+ ]);
+}
+
+function backupRunSafetyActivity(string $typeUuid, string $operation, ProcessStatus $status): Activity
+{
+ return Activity::create([
+ 'log_name' => 'default',
+ 'description' => '[]',
+ 'properties' => [
+ 'type_uuid' => $typeUuid,
+ 'operation' => $operation,
+ 'status' => $status->value,
+ ],
+ ]);
+}
+
+// --- Overlapped occurrences (M5) ---
+
+it('skips a database backup occurrence when the previous run still holds the overlap lock, so recovery does not run it later', function () {
+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'));
+ $database = create_standalone_postgresql($this->environment->id, $this->destination, ['status' => 'running:healthy']);
+ $backup = backupRunSafetySchedule($database, $this->team);
+ $occurrence = backupRunSafetyOccurrence("scheduled-backup:{$backup->id}", 'database-backup', $backup->id);
+ $job = new DatabaseBackupJob($backup, $occurrence->uuid);
+ $runningBackupLock = Cache::lock($job->middleware()[0]->getLockKey($job), 3600);
+ $runningBackupLock->get();
+ $commands = backupRunSafetyFakeRemoteCommands();
+
+ dispatch_sync($job);
+ $runningBackupLock->release();
+
+ expect($occurrence->fresh()->status)->toBe('skipped')
+ ->and(ScheduledDatabaseBackupExecution::query()->exists())->toBeFalse()
+ ->and($commands)->toBeEmpty();
+
+ Carbon::setTestNow(now()->addMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1));
+ Queue::fake();
+ app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
+
+ Queue::assertNothingPushed();
+});
+
+it('skips a volume backup occurrence when the previous run still holds the overlap lock', function () {
+ $volume = backupRunSafetyVolume($this->environment, $this->destination);
+ $backup = $volume->scheduledBackups()->create(['team_id' => $this->team->id, 'frequency' => 'daily', 'enabled' => true]);
+ $occurrence = backupRunSafetyOccurrence("scheduled-volume-backup:{$backup->id}", 'volume-backup', $backup->id);
+ $runningBackupLock = Cache::lock(VolumeBackupJob::lockKey($backup->id), 3600);
+ $runningBackupLock->get();
+ $commands = backupRunSafetyFakeRemoteCommands();
+
+ dispatch_sync(new VolumeBackupJob($backup, $occurrence->uuid));
+ $runningBackupLock->release();
+
+ expect($occurrence->fresh()->status)->toBe('skipped')
+ ->and(ScheduledVolumeBackupExecution::query()->exists())->toBeFalse()
+ ->and($commands)->toBeEmpty();
+});
+
+it('does not run a lost backup occurrence again when a newer occurrence already completed', function () {
+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));
+ Queue::fake();
+ $database = create_standalone_postgresql($this->environment->id, $this->destination, ['status' => 'running:healthy']);
+ $backup = backupRunSafetySchedule($database, $this->team, ['frequency' => 'hourly']);
+ $scheduleKey = "scheduled-backup:{$backup->id}";
+ $lost = ScheduledJobDelivery::create([
+ 'schedule_key' => $scheduleKey,
+ 'scheduled_for' => Carbon::create(2026, 9, 17, 10, 0, 0, 'UTC'),
+ 'job_type' => 'database-backup',
+ 'resource_id' => $backup->id,
+ 'status' => 'enqueued',
+ 'enqueued_at' => now()->subMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1),
+ ]);
+ // The 11:00 occurrence ran and completed; complete() deleted its row.
+ ScheduledJobState::create(['schedule_key' => $scheduleKey, 'last_scheduled_for' => Carbon::create(2026, 9, 17, 11, 0, 0, 'UTC')]);
+
+ app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();
+
+ Queue::assertNotPushed(DatabaseBackupJob::class);
+ expect($lost->fresh()->status)->toBe('failed');
+});
+
+// --- Imports and starts (M10) ---
+
+it('does not back up a database while a start or import of it is in progress', function (Closure $startOperation) {
+ $database = create_standalone_postgresql($this->environment->id, $this->destination, ['status' => 'running:healthy']);
+ $backup = backupRunSafetySchedule($database, $this->team);
+ $occurrence = backupRunSafetyOccurrence("scheduled-backup:{$backup->id}", 'database-backup', $backup->id);
+ $startOperation($database);
+ $commands = backupRunSafetyFakeRemoteCommands();
+
+ (new DatabaseBackupJob($backup, $occurrence->uuid))->handle();
+
+ $execution = ScheduledDatabaseBackupExecution::query()->sole();
+ expect($commands->filter(fn (string $command) => str_contains($command, 'pg_dump')))->toBeEmpty()
+ ->and($execution->status)->toBe('failed')
+ ->and($execution->message)->toContain('Skipped')
+ ->and($execution->filename)->toBeNull()
+ ->and($execution->finished_at)->not->toBeNull()
+ ->and(ScheduledJobDelivery::query()->whereKey($occurrence->id)->exists())->toBeFalse();
+ Notification::assertNothingSent();
+})->with([
+ 'reserved import, start or restart' => [fn (Model $database) => DatabaseOperationReservation::acquire($database->uuid)],
+ 'running import' => [fn (Model $database) => backupRunSafetyActivity($database->uuid, ResourceStartActivity::DATABASE_IMPORT_OPERATION, ProcessStatus::IN_PROGRESS)],
+ 'queued start' => [fn (Model $database) => backupRunSafetyActivity($database->uuid, ResourceStartActivity::DATABASE_START_OPERATION, ProcessStatus::QUEUED)],
+]);
+
+it('backs up a database when its last import is stale', function () {
+ $database = create_standalone_postgresql($this->environment->id, $this->destination, ['status' => 'running:healthy']);
+ $backup = backupRunSafetySchedule($database, $this->team);
+ $import = backupRunSafetyActivity($database->uuid, ResourceStartActivity::DATABASE_IMPORT_OPERATION, ProcessStatus::IN_PROGRESS);
+ Activity::query()->whereKey($import->id)->update(['updated_at' => now()->subSeconds(ResourceStartActivity::importStaleAfterSeconds() + 60)]);
+ $commands = backupRunSafetyFakeRemoteCommands();
+
+ (new DatabaseBackupJob($backup))->handle();
+
+ expect($commands->filter(fn (string $command) => str_contains($command, 'pg_dump')))->not->toBeEmpty()
+ ->and(ScheduledDatabaseBackupExecution::query()->sole()->status)->toBe('success');
+});
+
+// --- Unique backup file names (M11) ---
+
+it('gives each backup run of the same database its own file in the same second', function (Closure $createDatabase, array $backupSettings, string $extension) {
+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'));
+ $database = $createDatabase($this->environment->id, $this->destination, ['status' => 'running:healthy']);
+ $first = backupRunSafetySchedule($database, $this->team, $backupSettings);
+ $second = backupRunSafetySchedule($database, $this->team, $backupSettings);
+ backupRunSafetyFakeRemoteCommands();
+
+ (new DatabaseBackupJob($first))->handle();
+ (new DatabaseBackupJob($second))->handle();
+
+ $executions = ScheduledDatabaseBackupExecution::query()->orderBy('id')->get();
+ expect($executions)->toHaveCount(2)
+ ->and($executions->pluck('status')->unique()->all())->toBe(['success'])
+ ->and($executions->pluck('filename')->unique())->toHaveCount(2);
+ foreach ($executions as $execution) {
+ expect($execution->filename)->toEndWith("-{$execution->uuid}{$extension}");
+ }
+})->with([
+ 'postgresql' => [fn (...$arguments) => create_standalone_postgresql(...$arguments), [], '.dmp'],
+ 'postgresql full dump' => [fn (...$arguments) => create_standalone_postgresql(...$arguments), ['dump_all' => true], '.gz'],
+ 'mysql' => [fn (...$arguments) => create_standalone_mysql(...$arguments), [], '.dmp'],
+ 'mariadb' => [fn (...$arguments) => create_standalone_mariadb(...$arguments), [], '.dmp'],
+ 'mongodb' => [fn (...$arguments) => create_standalone_mongodb(...$arguments), [], '.tar.gz'],
+ 'sqlite' => [fn (...$arguments) => create_standalone_sqlite(...$arguments), [], '.gz'],
+]);
+
+// --- Service database backup folder (M12) ---
+
+it('stores service database backups in the folder of earlier releases', function () {
+ $service = Service::factory()->create([
+ 'name' => 'Rallly',
+ 'server_id' => $this->server->id,
+ 'destination_id' => $this->destination->id,
+ 'destination_type' => $this->destination->getMorphClass(),
+ 'environment_id' => $this->environment->id,
+ ]);
+ $database = ServiceDatabase::create([
+ 'service_id' => $service->id,
+ 'name' => 'rallly_db',
+ 'image' => 'postgres:16-alpine',
+ 'status' => 'running:healthy',
+ ]);
+ $backup = backupRunSafetySchedule($database, $this->team);
+ backupRunSafetyFakeRemoteCommands(['env | grep POSTGRES_' => "POSTGRES_USER=rallly\nPOSTGRES_DB=rallly"]);
+
+ (new DatabaseBackupJob($backup))->handle();
+
+ expect(ScheduledDatabaseBackupExecution::query()->sole()->filename)
+ ->toContain("/rallly-rallly_db-{$service->uuid}/pg-dump-rallly-");
+});
+
+function backupRunSafetyVolume(Environment $environment, StandaloneDocker $destination): LocalPersistentVolume
+{
+ $application = Application::factory()->create([
+ 'environment_id' => $environment->id,
+ 'destination_id' => $destination->id,
+ 'destination_type' => $destination->getMorphClass(),
+ ]);
+
+ return LocalPersistentVolume::create([
+ 'name' => 'app-data',
+ 'mount_path' => '/data',
+ 'resource_id' => $application->id,
+ 'resource_type' => $application->getMorphClass(),
+ ]);
+}
diff --git a/tests/Feature/DatabaseDumpExitStatusTest.php b/tests/Feature/DatabaseDumpExitStatusTest.php
index e614db4250..7a9382a74b 100644
--- a/tests/Feature/DatabaseDumpExitStatusTest.php
+++ b/tests/Feature/DatabaseDumpExitStatusTest.php
@@ -107,5 +107,5 @@ it('records a failed SQLite dump as failed and removes the partial backup file',
$execution = $backup->executions()->latest('id')->firstOrFail();
expect($execution->status)->toBe('failed')
->and($execution->filename)->toBeNull()
- ->and($commands->implode("\n"))->toMatch("/rm -f '[^']*sqlite-backup-app\\.db-\\d+\\.gz'/");
+ ->and($commands->implode("\n"))->toMatch("/rm -f '[^']*sqlite-backup-app\\.db-\\d+-[a-z0-9]+\\.gz'/");
});
diff --git a/tests/Feature/ScheduledJobManagerDispatchTest.php b/tests/Feature/ScheduledJobManagerDispatchTest.php
index 95f241b046..9c84560399 100644
--- a/tests/Feature/ScheduledJobManagerDispatchTest.php
+++ b/tests/Feature/ScheduledJobManagerDispatchTest.php
@@ -1,18 +1,22 @@
<?php
use App\Events\BackupCreated;
+use App\Events\DockerCleanupDone;
use App\Jobs\DatabaseBackupJob;
use App\Jobs\DockerCleanupJob;
use App\Jobs\ScheduledJobManager;
use App\Jobs\ScheduledTaskJob;
use App\Models\Application;
+use App\Models\DockerCleanupExecution;
use App\Models\Environment;
+use App\Models\InstanceSettings;
use App\Models\PrivateKey;
use App\Models\Project;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledJobDelivery;
use App\Models\ScheduledJobState;
use App\Models\ScheduledTask;
+use App\Models\ScheduledTaskExecution;
use App\Models\Server;
use App\Models\ServerSetting;
use App\Models\Service;
@@ -20,6 +24,8 @@ use App\Models\ServiceDatabase;
use App\Models\StandaloneDocker;
use App\Models\StandalonePostgresql;
use App\Models\Team;
+use App\Notifications\ScheduledTask\TaskFailed;
+use App\Notifications\Server\DockerCleanupFailed;
use App\Services\ScheduledJobDeliveryService;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Queue\Middleware\WithoutOverlapping;
@@ -29,6 +35,7 @@ use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Event;
use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Notification;
use Illuminate\Support\Facades\Queue;
uses(RefreshDatabase::class);
@@ -691,6 +698,58 @@ it('logs a lost scheduled task occurrence as missed instead of running it late',
->and($log)->toContain('Scheduled occurrence missed: its queued job was not started');
});
+it('records a missed scheduled task as a failed execution and notifies the team once', function () {
+ config(['constants.coolify.self_hosted' => true]);
+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));
+ Queue::fake();
+ Notification::fake();
+ $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);
+ InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));
+ Team::find($task->team_id)->emailNotificationSettings->update(['smtp_enabled' => true, 'scheduled_task_failure_email_notifications' => true]);
+ $occurrence = createStaleEnqueuedOccurrence("scheduled-task:{$task->id}", 'scheduled-task', $task->id, Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'));
+
+ (new ScheduledJobManager)->handle();
+ (new ScheduledJobManager)->handle();
+
+ $executions = ScheduledTaskExecution::query()->where('scheduled_task_id', $task->id)->get();
+ expect($executions)->toHaveCount(1)
+ ->and($executions->first()->status)->toBe('failed')
+ ->and($executions->first()->message)->toBe('Skipped: the queued job did not start within 60 minutes.')
+ ->and($executions->first()->finished_at)->not->toBeNull();
+ Notification::assertSentToTimes(Team::find($task->team_id), TaskFailed::class, 1);
+ Notification::assertSentTo(Team::find($task->team_id), TaskFailed::class, fn (TaskFailed $notification) => $notification->task->is($task)
+ && $notification->output === 'Skipped: the queued job did not start within 60 minutes.');
+
+ $lateJob = new ScheduledTaskJob($task, $occurrence->uuid);
+ $lateJob->handle();
+
+ expect(ScheduledTaskExecution::query()->where('scheduled_task_id', $task->id)->count())->toBe(1);
+});
+
+it('records a missed Docker cleanup as a failed execution and notifies the team once', function () {
+ config(['constants.coolify.self_hosted' => true]);
+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));
+ Queue::fake();
+ Notification::fake();
+ Event::fake([DockerCleanupDone::class]);
+ $server = createScheduledTaskApplication()->destination->server;
+ $server->team->emailNotificationSettings->update(['smtp_enabled' => true, 'docker_cleanup_failure_email_notifications' => true]);
+ $occurrence = createStaleEnqueuedOccurrence("docker-cleanup:{$server->id}", 'docker-cleanup', $server->id, Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'));
+
+ (new ScheduledJobManager)->handle();
+ (new ScheduledJobManager)->handle();
+
+ $executions = DockerCleanupExecution::query()->where('server_id', $server->id)->get();
+ expect($executions)->toHaveCount(1)
+ ->and($executions->first()->status)->toBe('failed')
+ ->and($executions->first()->message)->toBe('Skipped: the queued job did not start within 60 minutes.')
+ ->and($executions->first()->finished_at)->not->toBeNull()
+ ->and($occurrence->fresh()->status)->toBe('failed')
+ ->and(app(ScheduledJobDeliveryService::class)->claim($occurrence->uuid, 'original-job'))->toBeFalse();
+ Notification::assertSentToTimes($server->team, DockerCleanupFailed::class, 1);
+ Event::assertDispatched(DockerCleanupDone::class, 1);
+});
+
it('leaves recently enqueued occurrences alone', function () {
config(['constants.coolify.self_hosted' => true]);
Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));