An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
feat(scheduler): dispatch scheduled jobs by their stored next run time
Andras Bacsai452259aacc98
diff --git a/.ai/lessons.md b/.ai/lessons.mdindex 834b839331..d68d184014 100644--- a/.ai/lessons.md+++ b/.ai/lessons.md@@ -56,6 +56,8 @@## Make distributed schedules durable- Use the database as the correctness source for dynamic cron occurrences shared by multiple scheduler and Horizon nodes; Redis locks are load controls, not a durable execution ledger.- Give each schedule occurrence a unique database identity and make queue consumers claim it atomically before external work.+- Store the next due time on each schedule and select only due rows; claim a run with `UPDATE ... WHERE next_run_at = :old`. Do not evaluate every cron expression each minute: on a remote database the per-row queries alone exceed the one-minute budget.+- Run a per-minute dispatcher from the scheduler process, not as a queued job, so queue backlog cannot delay it.- Keep pending occurrences recoverable across publisher interruptions, and define an explicit bounded policy for late or offline schedules.- Horizon workers are long-lived: flush every static or `once()` cache (for example `Server::flushIdentityMap()`) in `Queue::before`, or later jobs decide with stale state.diff --git a/app/Actions/Shared/MigrateResourceToDestination.php b/app/Actions/Shared/MigrateResourceToDestination.phpindex b0979e1677..ad69093412 100644--- a/app/Actions/Shared/MigrateResourceToDestination.php+++ b/app/Actions/Shared/MigrateResourceToDestination.php@@ -23,6 +23,7 @@ use App\Models\StandalonePostgresql;use App\Models\StandaloneRedis;use App\Models\StandaloneSqlite;use App\Models\SwarmDocker;+use App\Services\ScheduleNextRunRecalculator;use Illuminate\Support\Collection;use Illuminate\Support\Facades\Bus;use Illuminate\Validation\ValidationException;@@ -151,6 +152,7 @@ class MigrateResourceToDestination}$resource->fill($payload)->save();+ app(ScheduleNextRunRecalculator::class)->forResource($resource);if ($resource instanceof Application) {$resource->additional_networks()->detach();diff --git a/app/Console/Commands/DispatchScheduledJobs.php b/app/Console/Commands/DispatchScheduledJobs.phpnew file mode 100644index 0000000000..3d4b53c2ae--- /dev/null+++ b/app/Console/Commands/DispatchScheduledJobs.php@@ -0,0 +1,20 @@+<?php++namespace App\Console\Commands;++use App\Jobs\ScheduledJobManager;+use Illuminate\Console\Command;++class DispatchScheduledJobs extends Command+{+ protected $signature = 'scheduled:dispatch';++ protected $description = 'Dispatch the due scheduled backups, tasks, volume backups, and Docker cleanups';++ public function handle(ScheduledJobManager $manager): int+ {+ $manager->handle();++ return self::SUCCESS;+ }+}diff --git a/app/Console/Commands/ScheduledJobDiagnostics.php b/app/Console/Commands/ScheduledJobDiagnostics.phpindex 61f26265a9..f66efd57b2 100644--- a/app/Console/Commands/ScheduledJobDiagnostics.php+++ b/app/Console/Commands/ScheduledJobDiagnostics.php@@ -2,254 +2,67 @@namespace App\Console\Commands;-use App\Models\DockerCleanupExecution;+use App\Jobs\ScheduledJobManager;use App\Models\ScheduledDatabaseBackup;+use App\Models\ScheduledJobDelivery;use App\Models\ScheduledTask;-use App\Models\Server;-use App\Models\Team;+use App\Models\ScheduledVolumeBackup;+use App\Models\ServerSetting;use Illuminate\Console\Command;+use Illuminate\Database\Eloquent\Builder;use Illuminate\Support\Carbon;-use Illuminate\Support\Collection;use Illuminate\Support\Facades\Cache;class ScheduledJobDiagnostics extends Command{- protected $signature = 'scheduled:diagnostics- {--type=all : Type to inspect: docker-cleanup, backups, tasks, server-jobs, all}- {--server= : Filter by server ID}';+ protected $signature = 'scheduled:diagnostics {--all : List every schedule, not only the overdue ones and the ones without a next run}';- protected $description = 'Inspect dedup cache state and scheduling decisions for all scheduled jobs';+ protected $description = 'Show the scheduler heartbeat, overdue schedules, schedules without a next run, and open deliveries';public function handle(): int- {- $type = $this->option('type');- $serverFilter = $this->option('server');-- $this->outputHeartbeat();-- if (in_array($type, ['all', 'docker-cleanup'])) {- $this->inspectDockerCleanups($serverFilter);- }-- if (in_array($type, ['all', 'backups'])) {- $this->inspectBackups();- }-- if (in_array($type, ['all', 'tasks'])) {- $this->inspectTasks();- }-- if (in_array($type, ['all', 'server-jobs'])) {- $this->inspectServerJobs($serverFilter);- }-- return self::SUCCESS;- }-- private function outputHeartbeat(): void{$heartbeat = Cache::get('scheduled-job-manager:heartbeat');- if ($heartbeat) {- $age = Carbon::parse($heartbeat)->diffForHumans();- $this->info("Scheduler heartbeat: {$heartbeat} ({$age})");- } else {- $this->error('Scheduler heartbeat: MISSING — ScheduledJobManager may not be running');- }+ $heartbeat+ ? $this->info("Scheduler heartbeat: {$heartbeat} (".Carbon::parse($heartbeat)->diffForHumans().')')+ : $this->error('Scheduler heartbeat: missing. The scheduled:dispatch command does not run.');$this->newLine();- }-- private function inspectDockerCleanups(?string $serverFilter): void- {- $this->info('=== Docker Cleanup Jobs ===');-- $servers = $this->getServers($serverFilter);-- $rows = [];- foreach ($servers as $server) {- $frequency = data_get($server->settings, 'docker_cleanup_frequency', '0 * * * *');- if (isset(VALID_CRON_STRINGS[$frequency])) {- $frequency = VALID_CRON_STRINGS[$frequency];- }- $dedupKey = "docker-cleanup:{$server->id}";- $cacheValue = Cache::get($dedupKey);- $timezone = data_get($server->settings, 'server_timezone', config('app.timezone'));-- if (validate_timezone($timezone) === false) {- $timezone = config('app.timezone');+ $overdueBefore = now()->subMinutes(ScheduledJobManager::LATE_RUN_WINDOW_MINUTES);+ $schedules = [+ 'Database backups' => [ScheduledDatabaseBackup::query()->where('enabled', true), 'next_run_at', 'frequency'],+ 'Scheduled tasks' => [ScheduledTask::query()->where('enabled', true), 'next_run_at', 'frequency'],+ 'Volume backups' => [ScheduledVolumeBackup::query()->where('enabled', true), 'next_run_at', 'frequency'],+ 'Docker cleanups' => [ServerSetting::query(), 'docker_cleanup_next_run_at', 'docker_cleanup_frequency'],+ ];++ foreach ($schedules as $label => [$query, $column, $frequencyColumn]) {+ /** @var Builder $query */+ $problems = (clone $query)->where(fn (Builder $query) => $query->whereNull($column)->orWhere($column, '<', $overdueBefore));+ $this->info(sprintf('=== %s: %d enabled, %d overdue or without a next run ===', $label, (clone $query)->count(), (clone $problems)->count()));++ $rows = ($this->option('all') ? $query : $problems)+ ->orderBy($column)+ ->limit(100)+ ->get()+ ->map(fn ($schedule) => [$schedule->getKey(), $schedule->getRawOriginal($frequencyColumn), $schedule->{$column}?->toIso8601String() ?? (next_cron_run_at((string) $schedule->getRawOriginal($frequencyColumn), null, now()) ? 'not calculated' : 'invalid frequency')]);++ if ($rows->isNotEmpty()) {+ $this->table(['ID', 'Frequency', 'Next run (UTC)'], $rows);}-- $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);-- $lastExecution = DockerCleanupExecution::where('server_id', $server->id)- ->latest()- ->first();-- $rows[] = [- $server->id,- $server->name,- $timezone,- $frequency,- $dedupKey,- $cacheValue ?? '<missing>',- $wouldFire ? 'YES' : 'no',- $lastExecution ? $lastExecution->status.' @ '.$lastExecution->created_at : 'never',- ];+ $this->newLine();}+ $this->info('=== Open and failed deliveries (failed ones are kept for 30 days) ===');$this->table(- ['ID', 'Server', 'TZ', 'Frequency', 'Dedup Key', 'Cache Value', 'Would Fire', 'Last Execution'],- $rows+ ['Job type', 'Status', 'Count', 'Oldest due (UTC)'],+ ScheduledJobDelivery::query()+ ->whereIn('status', ['pending', 'enqueued', 'claimed', 'failed'])+ ->selectRaw('job_type, status, count(*) as total, min(scheduled_for) as oldest')+ ->groupBy('job_type', 'status')+ ->get()+ ->map(fn ($row) => [$row->job_type, $row->status, $row->total, $row->oldest]),);- $this->newLine();- }-- private function inspectBackups(): void- {- $this->info('=== Scheduled Backups ===');-- $backups = ScheduledDatabaseBackup::with(['database'])- ->where('enabled', true)- ->get();-- $rows = [];- foreach ($backups as $backup) {- $server = $backup->server();- $frequency = $backup->frequency;- if (isset(VALID_CRON_STRINGS[$frequency])) {- $frequency = VALID_CRON_STRINGS[$frequency];- }- $dedupKey = "scheduled-backup:{$backup->id}";- $cacheValue = Cache::get($dedupKey);- $timezone = $server ? data_get($server->settings, 'server_timezone', config('app.timezone')) : config('app.timezone');-- if (validate_timezone($timezone) === false) {- $timezone = config('app.timezone');- }-- $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);-- $rows[] = [- $backup->id,- $backup->database_type ?? 'unknown',- $server?->name ?? 'N/A',- $frequency,- $cacheValue ?? '<missing>',- $wouldFire ? 'YES' : 'no',- ];- }-- $this->table(- ['Backup ID', 'DB Type', 'Server', 'Frequency', 'Cache Value', 'Would Fire'],- $rows- );- $this->newLine();- }-- private function inspectTasks(): void- {- $this->info('=== Scheduled Tasks ===');-- $tasks = ScheduledTask::with(['service', 'application'])- ->where('enabled', true)- ->get();-- $rows = [];- foreach ($tasks as $task) {- $server = $task->server();- $frequency = $task->frequency;- if (isset(VALID_CRON_STRINGS[$frequency])) {- $frequency = VALID_CRON_STRINGS[$frequency];- }-- $dedupKey = "scheduled-task:{$task->id}";- $cacheValue = Cache::get($dedupKey);- $timezone = $server ? data_get($server->settings, 'server_timezone', config('app.timezone')) : config('app.timezone');-- if (validate_timezone($timezone) === false) {- $timezone = config('app.timezone');- }-- $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);-- $rows[] = [- $task->id,- $task->name,- $server?->name ?? 'N/A',- $frequency,- $cacheValue ?? '<missing>',- $wouldFire ? 'YES' : 'no',- ];- }-- $this->table(- ['Task ID', 'Name', 'Server', 'Frequency', 'Cache Value', 'Would Fire'],- $rows- );- $this->newLine();- }-- private function inspectServerJobs(?string $serverFilter): void- {- $this->info('=== Server Manager Jobs ===');-- $servers = $this->getServers($serverFilter);-- $rows = [];- foreach ($servers as $server) {- $timezone = data_get($server->settings, 'server_timezone', config('app.timezone'));- if (validate_timezone($timezone) === false) {- $timezone = config('app.timezone');- }-- $dedupKeys = [- "server-patch-check:{$server->id}" => '0 0 * * 0',- "server-check:{$server->id}" => isCloud() ? '*/5 * * * *' : '* * * * *',- "server-storage-check:{$server->id}" => data_get($server->settings, 'server_disk_usage_check_frequency', '0 23 * * *'),- ];-- foreach ($dedupKeys as $dedupKey => $frequency) {- if (isset(VALID_CRON_STRINGS[$frequency])) {- $frequency = VALID_CRON_STRINGS[$frequency];- }-- $cacheValue = Cache::get($dedupKey);- $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);-- $rows[] = [- $server->id,- $server->name,- $dedupKey,- $frequency,- $cacheValue ?? '<missing>',- $wouldFire ? 'YES' : 'no',- ];- }- }-- $this->table(- ['Server ID', 'Server', 'Dedup Key', 'Frequency', 'Cache Value', 'Would Fire'],- $rows- );- $this->newLine();- }-- private function getServers(?string $serverFilter): Collection- {- $query = Server::with('settings')->where('ip', '!=', '1.2.3.4');-- if ($serverFilter) {- $query->where('id', $serverFilter);- }-- if (isCloud()) {- $servers = $query->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get();- $own = Team::find(0)?->servers()->with('settings')->get() ?? collect();-- return $servers->merge($own);- }-- return $query->get();+ return self::SUCCESS;}}diff --git a/app/Console/Kernel.php b/app/Console/Kernel.phpindex 32c6a2ee78..150d956d5f 100644--- a/app/Console/Kernel.php+++ b/app/Console/Kernel.php@@ -12,7 +12,6 @@ use App\Jobs\CleanupStaleMultiplexedConnections;use App\Jobs\PullChangelog;use App\Jobs\PullTemplatesFromCDN;use App\Jobs\RegenerateSslCertJob;-use App\Jobs\ScheduledJobManager;use App\Jobs\ServerManagerJob;use App\Jobs\UpdateCoolifyJob;use App\Models\InstanceSettings;@@ -75,7 +74,7 @@ class Kernel extends ConsoleKernel$this->scheduleInstance->job(new ServerManagerJob)->everyMinute()->onOneServer();// Scheduled Jobs (Backups & Tasks)- $this->scheduleInstance->job(new ScheduledJobManager)->everyMinute()->onOneServer();+ $this->scheduleScheduledJobs();$this->scheduleInstance->command('uploads:clear')->everyTwoMinutes();@@ -96,7 +95,7 @@ class Kernel extends ConsoleKernel$this->pullImages();// Scheduled Jobs (Backups & Tasks)- $this->scheduleInstance->job(new ScheduledJobManager)->everyMinute()->onOneServer();+ $this->scheduleScheduledJobs();$this->scheduleInstance->job(new RegenerateSslCertJob)->twiceDaily()->onOneServer();@@ -116,6 +115,18 @@ class Kernel extends ConsoleKernel->onOneServer();}+ /**+ * Run the dispatcher in the scheduler process, so a busy queue cannot delay it. Parallel runs+ * are safe because each occurrence is claimed with an atomic update of next_run_at.+ */+ private function scheduleScheduledJobs(): void+ {+ $this->scheduleInstance->command('scheduled:dispatch')+ ->everyMinute()+ ->onOneServer()+ ->runInBackground();+ }+private function scheduleUpdates(): void{$this->scheduleInstance->job(new CheckForUpdatesJob)diff --git a/app/Jobs/ScheduledJobManager.php b/app/Jobs/ScheduledJobManager.phpindex 7a051345e3..43c35b2b91 100644--- a/app/Jobs/ScheduledJobManager.php+++ b/app/Jobs/ScheduledJobManager.php@@ -4,418 +4,198 @@ namespace App\Jobs;use App\Actions\Shared\DeleteScheduledVolumeBackup;use App\Models\ScheduledDatabaseBackup;+use App\Models\ScheduledJobDelivery;use App\Models\ScheduledTask;use App\Models\ScheduledVolumeBackup;use App\Models\ScheduledVolumeBackupExecution;use App\Models\Server;-use App\Models\ServerSetting;-use App\Models\Subscription;-use App\Models\Team;+use App\Models\ServiceDatabase;use App\Services\ScheduledJobDeliveryService;-use Cron\CronExpression;-use Illuminate\Bus\Queueable;-use Illuminate\Contracts\Queue\ShouldQueue;+use Carbon\CarbonImmutable;use Illuminate\Database\Eloquent\Builder;-use Illuminate\Foundation\Bus\Dispatchable;-use Illuminate\Queue\InteractsWithQueue;-use Illuminate\Queue\Middleware\WithoutOverlapping;-use Illuminate\Queue\SerializesModels;-use Illuminate\Support\Carbon;+use Illuminate\Database\Eloquent\Model;+use Illuminate\Database\Eloquent\Relations\MorphTo;use Illuminate\Support\Facades\Cache;+use Illuminate\Support\Facades\DB;use Illuminate\Support\Facades\Log;-use Illuminate\Support\Facades\Redis;-class ScheduledJobManager implements ShouldQueue+/**+ * Dispatches the due scheduled backups, tasks, volume backups, and Docker cleanups.+ *+ * Each schedule stores its next due time in next_run_at. A run selects only due rows and claims+ * each occurrence with an atomic update of next_run_at, so parallel runs on several nodes never+ * dispatch the same occurrence twice. The `scheduled:dispatch` command runs it every minute.+ */+class ScheduledJobManager{- use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;-private const CHUNK_SIZE = 100;/**- * The time when this job execution started.- * Used to ensure all scheduled items are evaluated against the same point in time.- */- private ?Carbon $executionTime = null;-- private int $dispatchedCount = 0;-- private int $skippedCount = 0;-- /**- * Create a new job instance.- */- public function __construct()- {- $this->onQueue(crons_queue());- }-- /**- * Get the middleware the job should pass through.+ * A task or Docker cleanup that is due longer than this is logged as missed and not run.+ * Database and volume backups always run once, also when they are late.*/- public function middleware(): array- {- // Self-healing: clear any stale lock before WithoutOverlapping tries to acquire it.- // Stale locks (TTL = -1) can occur during upgrades, Redis restarts, or edge cases.- // @see https://github.com/coollabsio/coolify/issues/8327- self::clearStaleLockIfPresent();-- return [- (new WithoutOverlapping('scheduled-job-manager'))- ->expireAfter(90) // Lock expires after 90s to handle high-load environments with many tasks- ->dontRelease(), // Don't re-queue on lock conflict- ];- }+ public const LATE_RUN_WINDOW_MINUTES = 10;- /**- * Clear a stale WithoutOverlapping lock if it has no TTL (TTL = -1).- *- * This provides continuous self-healing since it runs every time the job is dispatched.- * Stale locks permanently block all scheduled job executions with no user-visible error.- */- private static function clearStaleLockIfPresent(): void- {- try {- $cachePrefix = config('cache.prefix', '');- $lockKey = $cachePrefix.'laravel-queue-overlap:'.self::class.':scheduled-job-manager';+ private CarbonImmutable $now;- $ttl = Redis::connection('default')->ttl($lockKey);+ private ScheduledJobDeliveryService $deliveries;- if ($ttl === -1) {- Redis::connection('default')->del($lockKey);- Log::channel('scheduled')->warning('Cleared stale ScheduledJobManager lock', [- 'lock_key' => $lockKey,- ]);- }- } catch (\Throwable $e) {- // Never let lock cleanup failure prevent the job from running- Log::channel('scheduled-errors')->error('Failed to check/clear stale lock', [- 'error' => $e->getMessage(),- ]);- }- }+ /** @var array{dispatched: int, skipped: int, missed: int} */+ private array $counts = ['dispatched' => 0, 'skipped' => 0, 'missed' => 0];public function handle(): void{- // Freeze the execution time at the start of the job- $this->executionTime = Carbon::now();- $this->dispatchedCount = 0;- $this->skippedCount = 0;+ $this->now = CarbonImmutable::now();+ $this->deliveries = app(ScheduledJobDeliveryService::class);+ $this->counts = ['dispatched' => 0, 'skipped' => 0, 'missed' => 0];- $this->logStart();+ // Read the current server state, not servers cached by an earlier run in this process.+ Server::flushIdentityMap();- app(ScheduledJobDeliveryService::class)->publishPending();+ Log::channel('scheduled')->info('ScheduledJobManager started', ['execution_time' => $this->now->toIso8601String()]);- // Process scheduled backups and tasks together so neither type starves the other.- try {- $this->processScheduledBackupsAndTasks();- } catch (\Exception $e) {- Log::channel('scheduled-errors')->error('Failed to process scheduled backups and tasks', [- 'error' => $e->getMessage(),- 'trace' => $e->getTraceAsString(),- ]);- }-- try {+ $this->deliveries->publishPending();+ $this->runStep('database backups', fn () => $this->processDatabaseBackups());+ $this->runStep('scheduled tasks', fn () => $this->processScheduledTasks());+ $this->runStep('volume backups', function () {$this->recoverStoppedVolumeBackupContainers();- $this->processScheduledVolumeBackups();- } catch (\Exception $e) {- Log::channel('scheduled-errors')->error('Failed to process scheduled volume backups', [- 'error' => $e->getMessage(),- 'trace' => $e->getTraceAsString(),- ]);- }-- // Process Docker cleanups - don't let failures stop the job manager- try {- $this->processDockerCleanups();- } catch (\Exception $e) {- Log::channel('scheduled-errors')->error('Failed to process docker cleanups', [- 'error' => $e->getMessage(),- 'trace' => $e->getTraceAsString(),- ]);- }+ $this->processVolumeBackups();+ });+ $this->runStep('docker cleanups', fn () => $this->processDockerCleanups());Log::channel('scheduled')->info('ScheduledJobManager completed', [- 'execution_time' => $this->executionTime->toIso8601String(),- 'duration_ms' => $this->executionTime->diffInMilliseconds(Carbon::now()),- 'dispatched' => $this->dispatchedCount,- 'skipped' => $this->skippedCount,- 'host' => gethostname(),- 'pid' => getmypid(),- 'memory_mb' => round(memory_get_usage(true) / 1048576, 1),- 'memory_peak_mb' => round(memory_get_peak_usage(true) / 1048576, 1),- 'open_occurrences_over_15_minutes' => rescue(fn () => app(ScheduledJobDeliveryService::class)->staleOpenOccurrenceCounts(), [], report: false),+ 'execution_time' => $this->now->toIso8601String(),+ 'duration_ms' => $this->now->diffInMilliseconds(CarbonImmutable::now()),+ ...$this->counts,]);- // Write heartbeat so the UI can detect when the scheduler has stopped- try {- Cache::put('scheduled-job-manager:heartbeat', now()->toIso8601String(), 300);- } catch (\Throwable) {- // Non-critical; don't let heartbeat failure affect the job- }+ // The UI uses the heartbeat to show when the scheduler has stopped.+ rescue(fn () => Cache::put('scheduled-job-manager:heartbeat', now()->toIso8601String(), 300), report: false);}- /**- * Log which worker runs the manager and how late it starts. The manager only catches up- * occurrences from the last CATCH_UP_WINDOW_MINUTES, so a larger delay skips them silently.- */- private function logStart(): void+ private function runStep(string $name, callable $step): void{- $previousStartedAt = rescue(fn () => Cache::get('scheduled-job-manager:last-started-at'), report: false);- rescue(fn () => Cache::put('scheduled-job-manager:last-started-at', $this->executionTime->toIso8601String(), 86400), report: false);- $queuedAt = data_get($this->job?->payload(), 'createdAt');-- $context = [- 'execution_time' => $this->executionTime->toIso8601String(),- 'host' => gethostname(),- 'pid' => getmypid(),- 'queue_wait_seconds' => $queuedAt ? $this->executionTime->timestamp - (int) $queuedAt : null,- 'seconds_since_previous_start' => $previousStartedAt ? (int) Carbon::parse($previousStartedAt)->diffInSeconds($this->executionTime) : null,- 'memory_mb' => round(memory_get_usage(true) / 1048576, 1),- ];-- Log::channel('scheduled')->info('ScheduledJobManager started', $context);-- $catchUpWindowSeconds = ScheduledJobDeliveryService::CATCH_UP_WINDOW_MINUTES * 60;- if ($context['queue_wait_seconds'] > $catchUpWindowSeconds || $context['seconds_since_previous_start'] > $catchUpWindowSeconds) {- Log::channel('scheduled-errors')->warning('ScheduledJobManager started late; occurrences due before the catch-up window were not run', $context);+ try {+ $step();+ } catch (\Throwable $e) {+ Log::channel('scheduled-errors')->error("Failed to process {$name}", [+ 'error' => $e->getMessage(),+ 'trace' => $e->getTraceAsString(),+ ]);}}- private function processScheduledBackupsAndTasks(): void+ private function processDatabaseBackups(): void{- $lastBackupId = null;- $lastTaskId = null;-- do {- $backups = $this->scheduledBackupQuery($lastBackupId)->get();- $tasks = $this->scheduledTaskQuery($lastTaskId)->get();-- if ($backups->isNotEmpty()) {- $lastBackupId = $backups->last()->id;- }-- if ($tasks->isNotEmpty()) {- $lastTaskId = $tasks->last()->id;- }-- $this->processInterleavedDueSchedules(- $this->dueScheduledBackups($backups),- $this->dueScheduledTasks($tasks),- );- } while ($backups->isNotEmpty() || $tasks->isNotEmpty());- }-- /**- * @param array<int, array{backup: ScheduledDatabaseBackup, server: Server}> $dueBackups- * @param array<int, array{task: ScheduledTask, server: Server}> $dueTasks- */- private function processInterleavedDueSchedules(array $dueBackups, array $dueTasks): void+ $this->due(ScheduledDatabaseBackup::query())+ ->with([+ 'team.subscription',+ 'database' => fn (MorphTo $morphTo) => $morphTo->morphWith([+ ServiceDatabase::class => ['service.destination.server.settings', 'service.destination.server.team.subscription'],+ ...array_fill_keys(STANDALONE_DATABASE_MODELS, ['destination.server.settings', 'destination.server.team.subscription']),+ ]),+ ])+ ->chunkById(self::CHUNK_SIZE, fn ($backups) => $backups->each(fn (ScheduledDatabaseBackup $backup) => $this->processItem(+ 'backup',+ ['backup_id' => $backup->id, 'database_id' => $backup->database_id, 'database_type' => $backup->database_type, 'team_id' => $backup->team_id],+ fn () => $this->processDatabaseBackup($backup),+ )));+ }++ private function processDatabaseBackup(ScheduledDatabaseBackup $backup): array{- $maxCount = max(count($dueBackups), count($dueTasks));+ $server = $backup->server();- for ($index = 0; $index < $maxCount; $index++) {- if (isset($dueBackups[$index])) {- $this->processScheduledBackup($dueBackups[$index]['backup'], $dueBackups[$index]['server']);- }+ if (blank($backup->database) || blank($server)) {+ $backup->delete();- if (isset($dueTasks[$index])) {- $this->processScheduledTask($dueTasks[$index]['task'], $dueTasks[$index]['server']);- }+ return ['deleted' => blank($backup->database) ? 'database_deleted' : 'server_deleted'];}- }- private function scheduledBackupQuery(?int $lastBackupId): Builder- {- return ScheduledDatabaseBackup::with(['database', 'team.subscription'])- ->where('enabled', true)- ->when($lastBackupId !== null, fn (Builder $query) => $query->where('id', '>', $lastBackupId))- ->orderBy('id')- ->limit(self::CHUNK_SIZE);+ return $this->runOccurrence(+ schedule: $backup,+ frequency: $backup->frequency,+ server: $server,+ runLate: true,+ skipReason: fn () => $this->serverSkipReason($server),+ delivery: ['schedule_key' => "scheduled-backup:{$backup->id}", 'job_type' => 'database-backup', 'resource_id' => $backup->id],+ );}- private function scheduledTaskQuery(?int $lastTaskId): Builder+ private function processScheduledTasks(): void{- return ScheduledTask::with([- 'service.destination.server.settings',- 'service.destination.server.team.subscription',- 'application.destination.server.settings',- 'application.destination.server.team.subscription',- ])- ->where('enabled', true)- ->when($lastTaskId !== null, fn (Builder $query) => $query->where('id', '>', $lastTaskId))- ->orderBy('id')- ->limit(self::CHUNK_SIZE);- }-- /**- * @param iterable<ScheduledDatabaseBackup> $backups- * @return array<int, array{backup: ScheduledDatabaseBackup, server: Server}>- */- private function dueScheduledBackups(iterable $backups): array+ $this->due(ScheduledTask::query())+ ->with([+ 'service.destination.server.settings',+ 'service.destination.server.team.subscription',+ 'application.destination.server.settings',+ 'application.destination.server.team.subscription',+ ])+ ->chunkById(self::CHUNK_SIZE, fn ($tasks) => $tasks->each(fn (ScheduledTask $task) => $this->processItem(+ 'task',+ ['task_id' => $task->id, 'task_name' => $task->name, 'team_id' => $task->team_id],+ fn () => $this->processScheduledTask($task),+ )));+ }++ private function processScheduledTask(ScheduledTask $task): array{- $dueBackups = [];-- foreach ($backups as $backup) {- try {- $server = $backup->server();+ $server = $task->server();- if (blank(data_get($backup, 'database')) || blank($server)) {- $this->processScheduledBackup($backup, $server);-- continue;- }+ if (blank($server) || (! $task->service && ! $task->application)) {+ $task->delete();- if ($this->isDueCandidateBeforeExpensiveChecks($backup->frequency, $server)) {- $dueBackups[] = [- 'backup' => $backup,- 'server' => $server,- ];- }- } catch (\Exception $e) {- Log::channel('scheduled-errors')->error('Error prechecking backup', [- 'backup_id' => $backup->id,- 'error' => $e->getMessage(),- ]);- }+ return ['deleted' => blank($server) ? 'server_deleted' : 'resource_deleted'];}- return $dueBackups;+ return $this->runOccurrence(+ schedule: $task,+ frequency: $task->frequency,+ server: $server,+ runLate: false,+ skipReason: fn () => $this->serverSkipReason($server) ?? $this->taskResourceSkipReason($task),+ delivery: ['schedule_key' => "scheduled-task:{$task->id}", 'job_type' => 'scheduled-task', 'resource_id' => $task->id],+ );}- /**- * @param iterable<ScheduledTask> $tasks- * @return array<int, array{task: ScheduledTask, server: Server}>- */- private function dueScheduledTasks(iterable $tasks): array+ private function processVolumeBackups(): void{- $dueTasks = [];-- foreach ($tasks as $task) {- try {- $server = $task->server();-- if (blank($server) || (! $task->service && ! $task->application)) {- $this->processScheduledTask($task, $server);-- continue;- }-- if ($this->isDueCandidateBeforeExpensiveChecks($task->frequency, $server)) {- $dueTasks[] = [- 'task' => $task,- 'server' => $server,- ];- }- } catch (\Exception $e) {- Log::channel('scheduled-errors')->error('Error prechecking task', [- 'task_id' => $task->id,- 'error' => $e->getMessage(),- ]);- }- }-- return $dueTasks;+ $this->due(ScheduledVolumeBackup::query())+ ->with(['backupable', 'team.subscription'])+ ->chunkById(self::CHUNK_SIZE, fn ($backups) => $backups->each(fn (ScheduledVolumeBackup $backup) => $this->processItem(+ 'volume_backup',+ ['backup_id' => $backup->id, 'backupable_type' => $backup->backupable_type, 'backupable_id' => $backup->backupable_id, 'team_id' => $backup->team_id],+ fn () => $this->processVolumeBackup($backup),+ )));}- private function processScheduledBackup(ScheduledDatabaseBackup $backup, ?Server $precheckedServer = null): void+ private function processVolumeBackup(ScheduledVolumeBackup $backup): array{- try {- $server = $precheckedServer ?? $backup->server();- $skipReason = $this->getBackupSkipReason($backup, $server);- if ($skipReason !== null) {- if ($server === null || $this->recordSkippedOccurrence($backup->frequency, $server, "scheduled-backup:{$backup->id}")) {- $this->skippedCount++;- $this->logBackupSkip($backup, $skipReason, $server);- }-- return;- }+ $server = $backup->server();- if ($this->dispatchOccurrence(- $backup->frequency,- $server,- "scheduled-backup:{$backup->id}",- 'database-backup',- $backup->id,- )) {- $this->dispatchedCount++;- Log::channel('scheduled')->info('Backup dispatched', [- 'backup_id' => $backup->id,- 'database_id' => $backup->database_id,- 'database_type' => $backup->database_type,- 'team_id' => $backup->team_id ?? null,- 'server_id' => $server->id,- ]);- }- } catch (\Exception $e) {- Log::channel('scheduled-errors')->error('Error processing backup', [- 'backup_id' => $backup->id,- 'error' => $e->getMessage(),- ]);+ // Wait without claiming: the backup runs late when the recovery is done.+ if ($backup->executions()->where(fn (Builder $query) => $query->where('stop_recovery_pending', true)->orWhere('s3_cleanup_pending', true))->exists()) {+ return ['waiting' => 'container_recovery_pending'];}- }-- private function processScheduledTask(ScheduledTask $task, ?Server $precheckedServer = null): void- {- try {- $server = $precheckedServer ?? $task->server();- $criticalSkip = $this->getTaskCriticalSkipReason($task, $server);- if ($criticalSkip !== null) {- if ($server === null || $this->recordSkippedOccurrence($task->frequency, $server, "scheduled-task:{$task->id}")) {- $this->skippedCount++;- $this->logTaskSkip($task, $criticalSkip, $server);- }-- return;- }-- $runtimeSkip = $this->getTaskRuntimeSkipReason($task);- if ($runtimeSkip !== null) {- if ($this->recordSkippedOccurrence($task->frequency, $server, "scheduled-task:{$task->id}")) {- $this->skippedCount++;- $this->logTaskSkip($task, $runtimeSkip, $server);- }-- return;- }- if (! $this->dispatchOccurrence(- $task->frequency,- $server,- "scheduled-task:{$task->id}",- 'scheduled-task',- $task->id,- )) {+ if (! $backup->backupable) {+ DeleteScheduledVolumeBackup::run($backup, $server);- return;- }+ return ['deleted' => 'resource_deleted'];+ }- $this->dispatchedCount++;- Log::channel('scheduled')->info('Task dispatched', [- 'task_id' => $task->id,- 'task_name' => $task->name,- 'team_id' => $server->team_id,- 'server_id' => $server->id,- ]);- } catch (\Exception $e) {- Log::channel('scheduled-errors')->error('Error processing task', [- 'task_id' => $task->id,- 'error' => $e->getMessage(),- ]);+ if (! $server) {+ return ['waiting' => 'server_missing'];}- }- private function processScheduledVolumeBackups(): void- {- ScheduledVolumeBackup::query()- ->with(['backupable', 'team.subscription'])- ->where('enabled', true)- ->chunkById(self::CHUNK_SIZE, function ($backups): void {- foreach ($backups as $backup) {- $this->processScheduledVolumeBackup($backup);- }- });+ return $this->runOccurrence(+ schedule: $backup,+ frequency: $backup->frequency,+ server: $server,+ runLate: true,+ skipReason: fn () => $this->serverSkipReason($server),+ delivery: ['schedule_key' => "scheduled-volume-backup:{$backup->id}", 'job_type' => 'volume-backup', 'resource_id' => $backup->id],+ );}private function recoverStoppedVolumeBackupContainers(): void@@ -431,367 +211,216 @@ class ScheduledJobManager implements ShouldQueue});}- private function processScheduledVolumeBackup(ScheduledVolumeBackup $backup): void+ private function processDockerCleanups(): void{- try {- $server = $backup->server();-- if ($backup->executions()- ->where(fn (Builder $query) => $query- ->where('stop_recovery_pending', true)- ->orWhere('s3_cleanup_pending', true))- ->exists()) {- $this->skippedCount++;- $this->logSkip('volume_backup', 'container_recovery_pending', [- 'backup_id' => $backup->id,- 'team_id' => $backup->team_id,- ]);-- return;- }+ $query = Server::query()+ ->with('settings')+ ->whereNotNull('ip')+ ->where('ip', '!=', '')+ ->whereNotIn('ip', Server::PLACEHOLDER_IPS)+ ->whereHas('settings', fn (Builder $query) => $query+ ->whereNull('docker_cleanup_next_run_at')+ ->orWhere('docker_cleanup_next_run_at', '<=', $this->now));- if (! $backup->backupable) {- DeleteScheduledVolumeBackup::run($backup, $server);- $this->skippedCount++;- $this->logSkip('volume_backup', 'resource_deleted', [- 'backup_id' => $backup->id,- 'team_id' => $backup->team_id,- ]);+ if (isCloud()) {+ $query->with('team.subscription')->where(fn (Builder $query) => $query+ ->where('team_id', 0)+ ->orWhereRelation('team.subscription', 'stripe_invoice_paid', true));+ }- return;- }+ $query->chunkById(self::CHUNK_SIZE, fn ($servers) => $servers->each(fn (Server $server) => $this->processItem(+ 'docker_cleanup',+ ['server_id' => $server->id, 'server_name' => $server->name, 'team_id' => $server->team_id],+ fn () => $this->runOccurrence(+ schedule: $server->settings,+ frequency: $server->settings->docker_cleanup_frequency ?? '0 0 * * *',+ server: $server,+ runLate: false,+ skipReason: fn () => $this->serverSkipReason($server),+ delivery: [+ 'schedule_key' => "docker-cleanup:{$server->id}",+ 'job_type' => 'docker-cleanup',+ 'resource_id' => $server->id,+ 'payload' => [+ 'delete_unused_volumes' => $server->settings->delete_unused_volumes,+ 'delete_unused_networks' => $server->settings->delete_unused_networks,+ ],+ ],+ column: 'docker_cleanup_next_run_at',+ ),+ )));+ }- if (! $server) {- $this->skippedCount++;- $this->logSkip('volume_backup', 'server_missing', [- 'backup_id' => $backup->id,- 'team_id' => $backup->team_id,- ]);+ /**+ * Enabled schedules that are due, or that have no next run yet.+ */+ private function due(Builder $query): Builder+ {+ return $query+ ->where('enabled', true)+ ->where(fn (Builder $query) => $query->whereNull('next_run_at')->orWhere('next_run_at', '<=', $this->now));+ }- return;+ /**+ * Claim the due occurrence of a schedule and dispatch it, skip it, or log it as missed.+ *+ * @param array{schedule_key: string, job_type: string, resource_id: int, payload?: array<string, mixed>} $delivery+ * @return array<string, mixed> The outcome, for the log.+ */+ private function runOccurrence(+ Model $schedule,+ string $frequency,+ Server $server,+ bool $runLate,+ callable $skipReason,+ array $delivery,+ string $column = 'next_run_at',+ ): array {+ $timezone = data_get($server, 'settings.server_timezone');+ $stored = $schedule->getRawOriginal($column);++ if ($stored === null) {+ // First run after the upgrade, a timezone change, or a move to another server.+ $first = next_cron_run_at($frequency, $timezone, $this->now, includeCurrentMinute: true);+ if ($first === null || ! $this->advance($schedule, $column, null, $first)) {+ return $first === null ? ['invalid_frequency' => $frequency] : [];}-- if (! $this->isDueCandidateBeforeExpensiveChecks($backup->frequency, $server)) {- return;+ if ($first->gt($this->now)) {+ return [];}+ $stored = $first;+ }- if (! $server->isFunctional()) {- if ($this->recordSkippedOccurrence($backup->frequency, $server, "scheduled-volume-backup:{$backup->id}")) {- $this->skippedCount++;- $this->logSkip('volume_backup', 'server_not_functional', [- 'backup_id' => $backup->id,- 'team_id' => $backup->team_id,- ...$this->serverStateContext($server),- ]);- }-- return;- }+ $dueAt = CarbonImmutable::instance($schedule->getAttribute($column))->utc();+ $next = next_cron_run_at($frequency, $timezone, $this->now);+ if ($next === null) {+ return ['invalid_frequency' => $frequency];+ }+ // When the clocks go back, a local time occurs two times. Run it only once.+ if ($this->sameLocalTime($next, $dueAt, $timezone)) {+ $next = next_cron_run_at($frequency, $timezone, $next) ?? $next;+ }- if (isCloud() && $backup->team_id !== 0 && ! data_get($backup, 'team.subscription.stripe_invoice_paid', false)) {- if ($this->recordSkippedOccurrence($backup->frequency, $server, "scheduled-volume-backup:{$backup->id}")) {- $this->skippedCount++;- $this->logSkip('volume_backup', 'subscription_unpaid', [- 'backup_id' => $backup->id,- 'team_id' => $backup->team_id,- ...$this->serverStateContext($server),- ]);- }+ $minutesLate = (int) $dueAt->diffInMinutes($this->now);+ if (! $runLate && $minutesLate > self::LATE_RUN_WINDOW_MINUTES) {+ $outcome = ['missed' => $dueAt->toIso8601String(), 'minutes_late' => $minutesLate];+ } elseif (($reason = $skipReason()) !== null) {+ $outcome = ['skipped' => $reason];+ } else {+ $outcome = ['dispatched' => $dueAt->toIso8601String()];+ }- return;+ // Claim and record the occurrence together, so a crash cannot claim it without a delivery.+ $occurrence = DB::transaction(function () use ($schedule, $column, $stored, $next, $outcome, $delivery, $dueAt): ScheduledJobDelivery|bool {+ if (! $this->advance($schedule, $column, $stored, $next)) {+ return false;}- if ($this->dispatchOccurrence(- $backup->frequency,- $server,- "scheduled-volume-backup:{$backup->id}",- 'volume-backup',- $backup->id,- )) {- $this->dispatchedCount++;- Log::channel('scheduled')->info('Volume backup dispatched', [- 'backup_id' => $backup->id,- 'backupable_type' => $backup->backupable_type,- 'backupable_id' => $backup->backupable_id,- 'team_id' => $backup->team_id,- 'server_id' => $server->id,- ]);+ if (! isset($outcome['dispatched'])) {+ return true;}- } catch (\Exception $e) {- Log::channel('scheduled-errors')->error('Error processing volume backup', [- 'backup_id' => $backup->id,- 'error' => $e->getMessage(),- ]);- }- }- private function getBackupSkipReason(ScheduledDatabaseBackup $backup, ?Server $server): ?string- {- if (blank(data_get($backup, 'database'))) {- $backup->delete();+ return $this->deliveries->create($delivery['schedule_key'], $dueAt, $delivery['job_type'], $delivery['resource_id'], $delivery['payload'] ?? []) ?? true;+ });- return 'database_deleted';+ if ($occurrence === false) {+ return [];}- if (blank($server)) {- $backup->delete();-- return 'server_deleted';+ if ($occurrence instanceof ScheduledJobDelivery) {+ $this->deliveries->publish($occurrence);}- if ($server->isFunctional() === false) {- return 'server_not_functional';- }+ return [...$outcome, 'server_id' => $server->id, 'next_run_at' => $next->toIso8601String()];+ }- if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) {- return 'subscription_unpaid';+ /**+ * Move next_run_at from the stored value to $to. Only one process can win this update.+ */+ private function advance(Model $schedule, string $column, mixed $from, CarbonImmutable $to): bool+ {+ $updated = $schedule->newQuery()+ ->whereKey($schedule->getKey())+ ->when($from === null, fn (Builder $query) => $query->whereNull($column), fn (Builder $query) => $query->where($column, $from))+ ->toBase()+ ->update([$column => $to]);++ if ($updated === 1) {+ $schedule->setAttribute($column, $to)->syncOriginalAttribute($column);}- return null;+ return $updated === 1;}- private function getTaskCriticalSkipReason(ScheduledTask $task, ?Server $server): ?string+ private function sameLocalTime(CarbonImmutable $first, CarbonImmutable $second, ?string $timezone): bool{- if (blank($server)) {- $task->delete();+ $timezone = filled($timezone) && validate_timezone($timezone) ? $timezone : config('app.timezone');- return 'server_deleted';- }+ return $first->setTimezone($timezone)->format('Y-m-d H:i') === $second->setTimezone($timezone)->format('Y-m-d H:i');+ }- if ($server->isFunctional() === false) {+ private function serverSkipReason(Server $server): ?string+ {+ if (! $server->isFunctional()) {return 'server_not_functional';}- if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) {+ if (isCloud() && $server->team_id !== 0 && data_get($server->team?->subscription, 'stripe_invoice_paid', false) === false) {return 'subscription_unpaid';}- if (! $task->service && ! $task->application) {- $task->delete();-- return 'resource_deleted';- }-return null;}- private function getTaskRuntimeSkipReason(ScheduledTask $task): ?string+ private function taskResourceSkipReason(ScheduledTask $task): ?string{- if ($task->application && str($task->application->status)->contains('running') === false) {+ if ($task->application && ! str($task->application->status)->contains('running')) {return 'application_not_running';}- if ($task->service && str($task->service->status)->contains('running') === false) {+ if ($task->service && ! str($task->service->status)->contains('running')) {return 'service_not_running';}return null;}- private function processDockerCleanups(): void- {- $this->getServersForCleanupQuery()- ->chunkById(self::CHUNK_SIZE, function ($servers): void {- foreach ($servers as $server) {- $this->processDockerCleanup($server);- }- });- }-- private function processDockerCleanup(Server $server): void+ /**+ * Process one schedule and log its outcome. An error in one schedule never stops the others.+ *+ * @param array<string, mixed> $context+ */+ private function processItem(string $type, array $context, callable $process): void{try {- $frequency = data_get($server->settings, 'docker_cleanup_frequency', '0 * * * *');- if (! $this->isDueCandidateBeforeExpensiveChecks($frequency, $server)) {- return;- }-- $skipReason = $this->getDockerCleanupSkipReason($server);- if ($skipReason !== null) {- if ($this->recordSkippedOccurrence($frequency, $server, "docker-cleanup:{$server->id}")) {- $this->skippedCount++;- $this->logSkip('docker_cleanup', $skipReason, [- 'server_id' => $server->id,- 'server_name' => $server->name,- 'team_id' => $server->team_id,- ]);- }-- return;- }-- if ($this->dispatchOccurrence(- $frequency,- $server,- "docker-cleanup:{$server->id}",- 'docker-cleanup',- $server->id,- [- 'delete_unused_volumes' => $server->settings->delete_unused_volumes,- 'delete_unused_networks' => $server->settings->delete_unused_networks,- ],- )) {- $this->dispatchedCount++;- Log::channel('scheduled')->info('Docker cleanup dispatched', [- 'server_id' => $server->id,- 'server_name' => $server->name,- 'team_id' => $server->team_id,- ]);- }- } catch (\Exception $e) {- Log::channel('scheduled-errors')->error('Error processing docker cleanup', [- 'server_id' => $server->id,- 'server_name' => $server->name,- 'error' => $e->getMessage(),- ]);- }- }-- private function getServersForCleanupQuery(): Builder- {- $query = Server::with('settings')- ->whereNotNull('ip')- ->where('ip', '!=', '')- ->whereNotIn('ip', Server::PLACEHOLDER_IPS);-- if (isCloud()) {- $query- ->with('team.subscription')- ->where(function (Builder $query): void {- $query- ->where('team_id', 0)- ->orWhereRelation('team.subscription', 'stripe_invoice_paid', true);- });- }-- return $query;- }-- private function getDockerCleanupSkipReason(Server $server): ?string- {- if (! $server->isFunctional()) {- return 'server_not_functional';- }+ $outcome = $process();+ } catch (\Throwable $e) {+ Log::channel('scheduled-errors')->error("Error processing {$type}", [...$context, 'error' => $e->getMessage()]);- // In cloud, check subscription status (except team 0)- if (isCloud() && $server->team_id !== 0) {- if (data_get($server->team->subscription, 'stripe_invoice_paid', false) === false) {- return 'subscription_unpaid';- }+ return;}- return null;- }-- private function logSkip(string $type, string $reason, array $context = []): void- {- Log::channel('scheduled')->info(ucfirst(str_replace('_', ' ', $type)).' skipped', array_merge([- 'type' => $type,- 'skip_reason' => $reason,- 'execution_time' => $this->executionTime?->toIso8601String(),- ], $context));- }-- private function dispatchOccurrence(- string $frequency,- Server $server,- string $scheduleKey,- string $jobType,- int $resourceId,- array $payload = [],- ): bool {- return app(ScheduledJobDeliveryService::class)->recordAndPublish(- $scheduleKey,- $frequency,- $this->serverTimezone($server),- $jobType,- $resourceId,- $payload,- $this->executionTime,- );- }-- private function recordSkippedOccurrence(string $frequency, Server $server, string $scheduleKey): bool- {- return app(ScheduledJobDeliveryService::class)->recordSkipped(- $scheduleKey,- $frequency,- $this->serverTimezone($server),- $this->executionTime,- );- }-- private function isDueCandidateBeforeExpensiveChecks(string $frequency, Server $server): bool- {- $cron = new CronExpression($this->normalizeFrequency($frequency));- $executionTime = ($this->executionTime ?? Carbon::now())->copy()->setTimezone($this->serverTimezone($server));- $previousDue = Carbon::instance($cron->getPreviousRunDate($executionTime, allowCurrentDate: true));-- return $previousDue->gte(- $executionTime->copy()->subMinutes(ScheduledJobDeliveryService::CATCH_UP_WINDOW_MINUTES)- );- }+ $label = ucfirst(str_replace('_', ' ', $type));+ $context = [...$context, ...$outcome, 'execution_time' => $this->now->toIso8601String()];- private function normalizeFrequency(string $frequency): string- {- return VALID_CRON_STRINGS[$frequency] ?? $frequency;- }-- private function serverTimezone(Server $server): string- {- $timezone = data_get($server->settings, 'server_timezone', config('app.timezone'));-- return validate_timezone($timezone) ? $timezone : config('app.timezone');- }-- private function logBackupSkip(ScheduledDatabaseBackup $backup, string $reason, ?Server $server): void- {- $this->logSkip('backup', $reason, [- 'backup_id' => $backup->id,- 'database_id' => $backup->database_id,- 'database_type' => $backup->database_type,- 'team_id' => $backup->team_id ?? null,- ...$this->serverStateContext($server),- ]);- }-- private function logTaskSkip(ScheduledTask $task, string $reason, ?Server $server): void- {- $this->logSkip('task', $reason, [- 'task_id' => $task->id,- 'task_name' => $task->name,- 'team_id' => $server?->team_id,- ...$this->serverStateContext($server),- ]);+ match (true) {+ isset($outcome['dispatched']) => $this->log('dispatched', 'info', "{$label} dispatched", $context),+ isset($outcome['skipped']) => $this->log('skipped', 'info', "{$label} skipped", ['skip_reason' => $outcome['skipped'], ...$context]),+ isset($outcome['deleted']) => $this->log('skipped', 'info', "{$label} skipped", ['skip_reason' => $outcome['deleted'], ...$context]),+ isset($outcome['waiting']) => $this->log('skipped', 'info', "{$label} skipped", ['skip_reason' => $outcome['waiting'], ...$context]),+ isset($outcome['missed']) => $this->log('missed', 'warning', "{$label} missed", $context),+ isset($outcome['invalid_frequency']) => Log::channel('scheduled-errors')->warning("{$label} has an invalid frequency", $context),+ default => null,+ };}/**- * Compare the server state that decided the skip with the current database state.- * A difference shows that the worker used stale server data.- *- * @return array<string, mixed>+ * @param 'dispatched'|'skipped'|'missed' $counter+ * @param array<string, mixed> $context*/- private function serverStateContext(?Server $server): array+ private function log(string $counter, string $level, string $message, array $context): void{- if ($server === null) {- return [];- }-- $settings = ServerSetting::query()- ->where('server_id', $server->id)- ->first(['is_reachable', 'is_usable', 'force_disabled']);-- return [- 'server_id' => $server->id,- 'server_ip_is_placeholder' => $server->hasPlaceholderIp(),- 'used_is_reachable' => data_get($server->settings, 'is_reachable'),- 'used_is_usable' => data_get($server->settings, 'is_usable'),- 'used_force_disabled' => data_get($server->settings, 'force_disabled'),- 'used_invoice_paid' => data_get($server->team?->subscription, 'stripe_invoice_paid'),- 'db_is_reachable' => $settings?->is_reachable,- 'db_is_usable' => $settings?->is_usable,- 'db_force_disabled' => $settings?->force_disabled,- 'db_invoice_paid' => Subscription::query()->where('team_id', $server->team_id)->value('stripe_invoice_paid'),- ];+ $this->counts[$counter]++;+ Log::channel('scheduled')->log($level, $message, $context);}}diff --git a/app/Livewire/Project/Shared/Destination.php b/app/Livewire/Project/Shared/Destination.phpindex 65af4b5ab7..1bb46b2aa8 100644--- a/app/Livewire/Project/Shared/Destination.php+++ b/app/Livewire/Project/Shared/Destination.php@@ -8,6 +8,7 @@ use App\Events\ApplicationStatusChanged;use App\Models\Application;use App\Models\Server;use App\Models\StandaloneDocker;+use App\Services\ScheduleNextRunRecalculator;use Illuminate\Foundation\Auth\Access\AuthorizesRequests;use Illuminate\Support\Collection;use Livewire\Component;@@ -136,6 +137,7 @@ class Destination extends Component->wherePivot('server_id', $server->id)->detach($network->id);$this->resource->additional_networks()->attach($mainDestination->id, ['server_id' => $mainDestination->server->id]);+ app(ScheduleNextRunRecalculator::class)->forResource($this->resource);});$this->resource->refresh();$this->refreshServers();diff --git a/app/Livewire/Project/Shared/ScheduledTask/Show.php b/app/Livewire/Project/Shared/ScheduledTask/Show.phpindex 3bfe0700d9..2428ff4e69 100644--- a/app/Livewire/Project/Shared/ScheduledTask/Show.php+++ b/app/Livewire/Project/Shared/ScheduledTask/Show.php@@ -7,6 +7,7 @@ use App\Models\Application;use App\Models\ScheduledTask;use App\Models\Service;use Illuminate\Foundation\Auth\Access\AuthorizesRequests;+use Livewire\Attributes\Computed;use Livewire\Attributes\Locked;use Livewire\Attributes\Validate;use Livewire\Component;@@ -87,6 +88,15 @@ class Show extends Component}}+ /**+ * Timezone of the server the task runs on, used to display its next run.+ */+ #[Computed]+ public function serverTimezone(): ?string+ {+ return $this->task->scheduleTimezone();+ }+private function syncData(bool $toModel = false): void{if ($toModel) {diff --git a/app/Livewire/Server/DockerCleanup.php b/app/Livewire/Server/DockerCleanup.phpindex d0a8d8ca9d..6e6d1480b2 100644--- a/app/Livewire/Server/DockerCleanup.php+++ b/app/Livewire/Server/DockerCleanup.php@@ -5,9 +5,7 @@ namespace App\Livewire\Server;use App\Jobs\DockerCleanupJob;use App\Models\DockerCleanupExecution;use App\Models\Server;-use Cron\CronExpression;use Illuminate\Foundation\Auth\Access\AuthorizesRequests;-use Illuminate\Support\Carbon;use Illuminate\Support\Facades\Cache;use Livewire\Attributes\Computed;use Livewire\Attributes\Validate;@@ -39,35 +37,21 @@ class DockerCleanup extends Component#[Validate('boolean')]public bool $disableApplicationImageRetention = false;+ /**+ * The scheduled cleanup is stale when its next run is more than 10 minutes overdue,+ * meaning the scheduled job dispatcher did not run it.+ */#[Computed]public function isCleanupStale(): bool{- try {- $lastExecution = DockerCleanupExecution::where('server_id', $this->server->id)- ->orderBy('created_at', 'desc')- ->first();-- if (! $lastExecution) {- return false;- }-- $frequency = $this->server->settings->docker_cleanup_frequency ?? '0 0 * * *';- if (isset(VALID_CRON_STRINGS[$frequency])) {- $frequency = VALID_CRON_STRINGS[$frequency];- }-- $cron = new CronExpression($frequency);- $now = Carbon::now();- $nextRun = Carbon::parse($cron->getNextRunDate($now));- $afterThat = Carbon::parse($cron->getNextRunDate($nextRun));- $intervalMinutes = $nextRun->diffInMinutes($afterThat);-- $threshold = max($intervalMinutes * 2, 10);-- return Carbon::parse($lastExecution->created_at)->diffInMinutes($now) > $threshold;- } catch (\Throwable) {+ // The dispatcher does not run cleanups on servers without a real IP address.+ if (blank($this->server->ip) || $this->server->hasPlaceholderIp()) {return false;}++ $nextRunAt = $this->server->settings->docker_cleanup_next_run_at;++ return $nextRunAt !== null && $nextRunAt->lt(now()->subMinutes(10));}#[Computed]diff --git a/app/Models/ScheduledDatabaseBackup.php b/app/Models/ScheduledDatabaseBackup.phpindex 7a26658e4f..73c162ea81 100644--- a/app/Models/ScheduledDatabaseBackup.php+++ b/app/Models/ScheduledDatabaseBackup.php@@ -2,12 +2,15 @@namespace App\Models;+use App\Traits\HasNextRunAt;use Illuminate\Database\Eloquent\Relations\HasMany;use Illuminate\Database\Eloquent\Relations\HasOne;use Illuminate\Database\Eloquent\Relations\MorphTo;class ScheduledDatabaseBackup extends BaseModel{+ use HasNextRunAt;+protected function casts(): array{return [@@ -17,6 +20,7 @@ class ScheduledDatabaseBackup extends BaseModel'missing_backup_notification_days' => 'integer','missing_backup_notification_sent_at' => 'datetime','last_execution_at' => 'datetime',+ 'next_run_at' => 'datetime',];}diff --git a/app/Models/ScheduledJobState.php b/app/Models/ScheduledJobState.phpdeleted file mode 100644index e267cd6a1d..0000000000--- a/app/Models/ScheduledJobState.php+++ /dev/null@@ -1,18 +0,0 @@-<?php--namespace App\Models;--class ScheduledJobState extends BaseModel-{- protected $fillable = [- 'schedule_key',- 'last_scheduled_for',- ];-- protected function casts(): array- {- return [- 'last_scheduled_for' => 'immutable_datetime',- ];- }-}diff --git a/app/Models/ScheduledTask.php b/app/Models/ScheduledTask.phpindex 0a53395d31..f69ae77b82 100644--- a/app/Models/ScheduledTask.php+++ b/app/Models/ScheduledTask.php@@ -2,6 +2,7 @@namespace App\Models;+use App\Traits\HasNextRunAt;use App\Traits\HasSafeStringAttribute;use Illuminate\Database\Eloquent\Factories\HasFactory;use Illuminate\Database\Eloquent\Relations\HasMany;@@ -20,6 +21,7 @@ use OpenApi\Attributes as OA;'frequency' => ['type' => 'string', 'description' => 'The frequency of the scheduled task.'],'container' => ['type' => 'string', 'nullable' => true, 'description' => 'The container where the command should be executed.'],'timeout' => ['type' => 'integer', 'description' => 'The timeout of the scheduled task in seconds.'],+ 'next_run_at' => ['type' => 'string', 'format' => 'date-time', 'nullable' => true, 'description' => 'The next time the scheduled task runs, in UTC. Null when it is disabled or not calculated yet.'],'created_at' => ['type' => 'string', 'format' => 'date-time', 'description' => 'The date and time when the scheduled task was created.'],'updated_at' => ['type' => 'string', 'format' => 'date-time', 'description' => 'The date and time when the scheduled task was last updated.'],],@@ -27,6 +29,7 @@ use OpenApi\Attributes as OA;class ScheduledTask extends BaseModel{use HasFactory;+ use HasNextRunAt;use HasSafeStringAttribute;protected $fillable = [@@ -52,6 +55,7 @@ class ScheduledTask extends BaseModelreturn ['enabled' => 'boolean','timeout' => 'integer',+ 'next_run_at' => 'datetime',];}diff --git a/app/Models/ScheduledVolumeBackup.php b/app/Models/ScheduledVolumeBackup.phpindex 7f33fd92b9..4732d60adc 100644--- a/app/Models/ScheduledVolumeBackup.php+++ b/app/Models/ScheduledVolumeBackup.php@@ -2,6 +2,7 @@namespace App\Models;+use App\Traits\HasNextRunAt;use Illuminate\Database\Eloquent\Builder;use Illuminate\Database\Eloquent\Model;use Illuminate\Database\Eloquent\Relations\BelongsTo;@@ -11,6 +12,8 @@ use Illuminate\Database\Eloquent\Relations\MorphTo;class ScheduledVolumeBackup extends BaseModel{+ use HasNextRunAt;+public const int DEFAULT_TIMEOUT = 36000;protected $fillable = [@@ -47,6 +50,7 @@ class ScheduledVolumeBackup extends BaseModel'retention_days_s3' => 'integer','retention_max_storage_s3' => 'float','timeout' => 'integer',+ 'next_run_at' => 'datetime',];}diff --git a/app/Models/ServerSetting.php b/app/Models/ServerSetting.phpindex 32afb2a2a7..53674dad92 100644--- a/app/Models/ServerSetting.php+++ b/app/Models/ServerSetting.php@@ -3,6 +3,7 @@namespace App\Models;use App\Enums\ServerRole;+use App\Services\ScheduleNextRunRecalculator;use Illuminate\Contracts\Encryption\DecryptException;use Illuminate\Database\Eloquent\Casts\Attribute;use Illuminate\Database\Eloquent\Model;@@ -168,6 +169,7 @@ class ServerSetting extends Model'docker_version_checked_at' => 'datetime','compose_version_checked_at' => 'datetime','backup_compression_cpu_percentage' => 'integer',+ 'docker_cleanup_next_run_at' => 'datetime',];/**@@ -188,6 +190,15 @@ class ServerSetting extends Modelprotected static function booted(){+ static::saving(function ($setting) {+ if (! $setting->exists || $setting->isDirty(['docker_cleanup_frequency', 'server_timezone'])) {+ // Without a frequency (a new row that uses the column default), the dispatcher calculates it.+ $frequency = $setting->getAttributes()['docker_cleanup_frequency'] ?? null;+ $setting->docker_cleanup_next_run_at = $frequency === null+ ? null+ : next_cron_run_at($frequency, $setting->server_timezone, now());+ }+ });static::creating(function ($setting) {try {if (str($setting->sentinel_token)->isEmpty()) {@@ -201,6 +212,11 @@ class ServerSetting extends Model}});static::updated(function ($settings) {+ if ($settings->wasChanged('server_timezone') && $settings->server) {+ // A failure must not fail the settings save: the schedules correct themselves after their next run.+ rescue(fn () => app(ScheduleNextRunRecalculator::class)->forServer($settings->server));+ }+if ($settings->wasChanged('sentinel_token') ||$settings->wasChanged('sentinel_custom_url') ||diff --git a/app/Services/ScheduleNextRunRecalculator.php b/app/Services/ScheduleNextRunRecalculator.phpnew file mode 100644index 0000000000..87127c27c2--- /dev/null+++ b/app/Services/ScheduleNextRunRecalculator.php@@ -0,0 +1,85 @@+<?php++namespace App\Services;++use App\Models\ScheduledVolumeBackup;+use App\Models\Server;+use App\Models\Service;+use App\Models\ServiceApplication;+use App\Models\ServiceDatabase;+use App\Models\StandalonePostgresql;+use Illuminate\Database\Eloquent\Builder;+use Illuminate\Database\Eloquent\Model;+use Illuminate\Database\Eloquent\Relations\Relation;++/**+ * Recalculates next_run_at of schedules after their server timezone can have changed:+ * a server timezone change, or a resource that moved to another server.+ */+class ScheduleNextRunRecalculator+{+ /**+ * @param string|null $timezone The timezone of the resource's server. Read from the database when null.+ */+ public function forResource(Model $resource, ?string $timezone = null): void+ {+ $timezone ??= $this->serverTimezone($resource);++ foreach (['scheduled_tasks', 'scheduledBackups'] as $relation) {+ if (method_exists($resource, $relation)) {+ $this->recalculate($resource->{$relation}(), $timezone);+ }+ }++ foreach (['persistentStorages', 'fileStorages'] as $relation) {+ if (method_exists($resource, $relation)) {+ $storages = $resource->{$relation}();+ $this->recalculate(ScheduledVolumeBackup::query()+ ->where('backupable_type', $storages->getRelated()->getMorphClass())+ ->whereIn('backupable_id', $storages->select('id')), $timezone);+ }+ }++ if ($resource instanceof Service) {+ $resource->applications()->get()->each(fn (Model $application) => $this->forResource($application, $timezone));+ $resource->databases()->get()->each(fn (Model $database) => $this->forResource($database, $timezone));+ }+ }++ /**+ * Each resource uses the timezone of its own main server: Server::applications() also returns+ * applications for which this server is only an additional server.+ */+ public function forServer(Server $server): void+ {+ $server->applications()->each(fn (Model $application) => $this->forResource($application));+ $server->databases()->each(fn (Model $database) => $this->forResource($database));+ $server->services()->get()->each(fn (Service $service) => $this->forResource($service));++ // Server::databases() leaves out the instance database, which has the instance backup.+ if ($server->id === 0 && ($instanceDatabase = StandalonePostgresql::find(0))) {+ $this->forResource($instanceDatabase);+ }+ }++ /**+ * Move enabled schedules to their next due time after now. The current minute is excluded,+ * so an occurrence that already ran in this minute does not run again.+ */+ private function recalculate(Relation|Builder $schedules, ?string $timezone): void+ {+ $schedules->where('enabled', true)->get()->each(fn (Model $schedule) => $schedule->forceFill([+ 'next_run_at' => next_cron_run_at((string) $schedule->frequency, $timezone, now()),+ ])->saveQuietly());+ }++ private function serverTimezone(Model $resource): ?string+ {+ $owner = $resource instanceof ServiceApplication || $resource instanceof ServiceDatabase+ ? $resource->service()->first()+ : $resource;+ $server = $owner?->destination()->first()?->server()->first();++ return $server?->settings()->value('server_timezone');+ }+}diff --git a/app/Services/ScheduledJobDeliveryService.php b/app/Services/ScheduledJobDeliveryService.phpindex 3a8c09e5d4..4aab2231cb 100644--- a/app/Services/ScheduledJobDeliveryService.php+++ b/app/Services/ScheduledJobDeliveryService.php@@ -8,107 +8,28 @@ use App\Jobs\ScheduledTaskJob;use App\Jobs\VolumeBackupJob;use App\Models\ScheduledDatabaseBackup;use App\Models\ScheduledJobDelivery;-use App\Models\ScheduledJobState;use App\Models\ScheduledTask;use App\Models\ScheduledVolumeBackup;use App\Models\Server;-use Cron\CronExpression;-use Illuminate\Support\Carbon;-use Illuminate\Support\Facades\DB;+use Carbon\CarbonInterface;use Illuminate\Support\Facades\Log;class ScheduledJobDeliveryService{- public const CATCH_UP_WINDOW_MINUTES = 10;-- public function recordAndPublish(- string $scheduleKey,- string $frequency,- string $timezone,- string $jobType,- int $resourceId,- array $payload = [],- ?Carbon $executionTime = null,- ): bool {- $executionTime = ($executionTime ?? Carbon::now())->copy()->setTimezone($timezone);- $cron = new CronExpression(VALID_CRON_STRINGS[$frequency] ?? $frequency);- $scheduledFor = Carbon::instance($cron->getPreviousRunDate($executionTime, allowCurrentDate: true));-- if (! $scheduledFor->gte($executionTime->copy()->subMinutes(self::CATCH_UP_WINDOW_MINUTES))) {- return false;- }-- $delivery = DB::transaction(function () use ($scheduleKey, $scheduledFor, $jobType, $resourceId, $payload): ?ScheduledJobDelivery {- ScheduledJobState::query()->insertOrIgnore([- 'uuid' => new_public_id(),- 'schedule_key' => $scheduleKey,- 'created_at' => now(),- 'updated_at' => now(),- ]);-- $state = ScheduledJobState::query()- ->where('schedule_key', $scheduleKey)- ->lockForUpdate()- ->firstOrFail();-- if ($state->last_scheduled_for?->gte($scheduledFor)) {- return null;- }-- $state->update(['last_scheduled_for' => $scheduledFor->utc()]);-- return ScheduledJobDelivery::create([- 'schedule_key' => $scheduleKey,- 'scheduled_for' => $scheduledFor->utc(),- 'job_type' => $jobType,- 'resource_id' => $resourceId,- 'payload' => $payload,- 'status' => 'pending',- ]);- });-- if ($delivery === null) {- return false;- }-- return $this->publish($delivery);- }-- public function recordSkipped(- string $scheduleKey,- string $frequency,- string $timezone,- ?Carbon $executionTime = null,- ): bool {- $executionTime = ($executionTime ?? Carbon::now())->copy()->setTimezone($timezone);- $cron = new CronExpression(VALID_CRON_STRINGS[$frequency] ?? $frequency);- $scheduledFor = Carbon::instance($cron->getPreviousRunDate($executionTime, allowCurrentDate: true));-- if (! $scheduledFor->gte($executionTime->copy()->subMinutes(self::CATCH_UP_WINDOW_MINUTES))) {- return false;- }-- return DB::transaction(function () use ($scheduleKey, $scheduledFor): bool {- ScheduledJobState::query()->insertOrIgnore([- 'uuid' => new_public_id(),- 'schedule_key' => $scheduleKey,- 'created_at' => now(),- 'updated_at' => now(),- ]);-- $state = ScheduledJobState::query()- ->where('schedule_key', $scheduleKey)- ->lockForUpdate()- ->firstOrFail();-- if ($state->last_scheduled_for?->gte($scheduledFor)) {- return false;- }-- $state->update(['last_scheduled_for' => $scheduledFor->utc()]);+ /**+ * Record a pending delivery for one occurrence of a schedule. Returns null when the+ * occurrence already has a delivery.+ *+ * @param array<string, mixed> $payload+ */+ public function create(string $scheduleKey, CarbonInterface $scheduledFor, string $jobType, int $resourceId, array $payload = []): ?ScheduledJobDelivery+ {+ $delivery = ScheduledJobDelivery::createOrFirst(+ ['schedule_key' => $scheduleKey, 'scheduled_for' => $scheduledFor->utc()],+ ['job_type' => $jobType, 'resource_id' => $resourceId, 'payload' => $payload, 'status' => 'pending'],+ );- return true;- });+ return $delivery->wasRecentlyCreated ? $delivery : null;}public function publishPending(): void@@ -132,24 +53,6 @@ class ScheduledJobDeliveryService});}- /**- * Count occurrences that are still open 15 minutes after they were created. Pending or- * enqueued rows here never started; claimed rows are still running or their worker died.- *- * @return array<string, int>- */- public function staleOpenOccurrenceCounts(): array- {- return ScheduledJobDelivery::query()- ->whereIn('status', ['pending', 'enqueued', 'claimed'])- ->where('created_at', '<', now()->subMinutes(15))- ->selectRaw('status, count(*) as aggregate')- ->groupBy('status')- ->pluck('aggregate', 'status')- ->map(fn ($count) => (int) $count)- ->all();- }-public function deleteOldOccurrences(): void{ScheduledJobDelivery::query()@@ -244,7 +147,7 @@ class ScheduledJobDeliveryService}}- private function publish(ScheduledJobDelivery $occurrence): bool+ public function publish(ScheduledJobDelivery $occurrence): bool{if ($occurrence->status !== 'pending') {return false;@@ -283,15 +186,6 @@ class ScheduledJobDeliveryServicereturn false;}- // An open previous occurrence usually means the previous run is still running. Jobs with- // WithoutOverlapping(...)->dontRelease() then drop this one without running it.- $previousOpen = ScheduledJobDelivery::query()- ->where('schedule_key', $occurrence->schedule_key)- ->where('scheduled_for', '<', $occurrence->scheduled_for)- ->whereIn('status', ['pending', 'enqueued', 'claimed'])- ->orderByDesc('scheduled_for')- ->first(['uuid', 'status', 'scheduled_for']);-dispatch($job);ScheduledJobDelivery::query()@@ -304,17 +198,7 @@ class ScheduledJobDeliveryService]);// Log from the loaded row: a fast worker can complete and delete the occurrence already.- $enqueued = ['status' => 'enqueued', 'enqueued_at' => now()->toIso8601String()];- if ($previousOpen) {- $this->logOccurrence($occurrence, 'Scheduled occurrence enqueued while the previous occurrence is still open', 'warning', [- ...$enqueued,- 'previous_occurrence_uuid' => $previousOpen->uuid,- 'previous_status' => $previousOpen->status,- 'previous_scheduled_for' => $previousOpen->scheduled_for->toIso8601String(),- ]);- } else {- $this->logOccurrence($occurrence, 'Scheduled occurrence enqueued', context: $enqueued);- }+ $this->logOccurrence($occurrence, 'Scheduled occurrence enqueued', context: ['status' => 'enqueued', 'enqueued_at' => now()->toIso8601String()]);return true;}diff --git a/app/Traits/HasNextRunAt.php b/app/Traits/HasNextRunAt.phpnew file mode 100644index 0000000000..856ebe3391--- /dev/null+++ b/app/Traits/HasNextRunAt.php@@ -0,0 +1,38 @@+<?php++namespace App\Traits;++use Carbon\CarbonImmutable;+use DateTimeInterface;++/**+ * Keeps next_run_at of a schedule model in sync with its frequency and enabled flag.+ * The model must have `frequency`, `enabled`, `next_run_at`, and a `server()` method.+ */+trait HasNextRunAt+{+ protected static function bootHasNextRunAt(): void+ {+ static::saving(function ($schedule) {+ if ($schedule->exists && ! $schedule->isDirty(['frequency', 'enabled'])) {+ return;+ }++ $schedule->next_run_at = $schedule->calculateNextRunAt(now());+ });+ }++ public function calculateNextRunAt(DateTimeInterface $after, bool $includeCurrentMinute = false): ?CarbonImmutable+ {+ if ($this->getAttribute('enabled') !== null && ! $this->enabled) {+ return null;+ }++ return next_cron_run_at((string) $this->frequency, $this->scheduleTimezone(), $after, $includeCurrentMinute);+ }++ public function scheduleTimezone(): ?string+ {+ return data_get($this->server(), 'settings.server_timezone');+ }+}diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.phpindex d50d1f28fc..336ba1f0be 100644--- a/bootstrap/helpers/shared.php+++ b/bootstrap/helpers/shared.php@@ -1111,6 +1111,28 @@ function validate_timezone(string $timezone): boolreturn in_array($timezone, timezone_identifiers_list());}+/**+ * The next due time of a cron frequency after $after, in UTC. An invalid frequency returns null.+ * With $includeCurrentMinute, a frequency that is due in the minute of $after returns that minute.+ */+function next_cron_run_at(string $frequency, ?string $timezone, DateTimeInterface $after, bool $includeCurrentMinute = false): ?CarbonImmutable+{+ $frequency = trim($frequency);+ $timezone = filled($timezone) && validate_timezone($timezone) ? $timezone : config('app.timezone');++ try {+ $cron = new Cron\CronExpression(VALID_CRON_STRINGS[$frequency] ?? $frequency);+ $time = CarbonImmutable::instance($after)->setTimezone($timezone);+ if ($includeCurrentMinute) {+ $time = $time->startOfMinute();+ }++ return CarbonImmutable::instance($cron->getNextRunDate($time, 0, $includeCurrentMinute))->utc();+ } catch (Throwable) {+ return null;+ }+}+function parseEnvFormatToArray($env_file_contents){$env_array = [];diff --git a/database/migrations/2026_09_28_194652_add_next_run_at_to_scheduled_jobs.php b/database/migrations/2026_09_28_194652_add_next_run_at_to_scheduled_jobs.phpnew file mode 100644index 0000000000..dd413fb653--- /dev/null+++ b/database/migrations/2026_09_28_194652_add_next_run_at_to_scheduled_jobs.php@@ -0,0 +1,94 @@+<?php++use App\Models\ScheduledDatabaseBackup;+use App\Models\ScheduledTask;+use App\Models\ScheduledVolumeBackup;+use App\Models\ServerSetting;+use Illuminate\Database\Migrations\Migration;+use Illuminate\Database\Schema\Blueprint;+use Illuminate\Support\Carbon;+use Illuminate\Support\Facades\DB;+use Illuminate\Support\Facades\Schema;++return new class extends Migration+{+ /**+ * Schedules without a recorded occurrence keep next_run_at = null. The dispatcher calculates it on its first run.+ */+ public function up(): void+ {+ foreach (['scheduled_database_backups', 'scheduled_tasks', 'scheduled_volume_backups'] as $table) {+ Schema::table($table, function (Blueprint $table) {+ $table->timestampTz('next_run_at')->nullable();+ $table->index(['enabled', 'next_run_at']);+ });+ }++ Schema::table('server_settings', function (Blueprint $table) {+ $table->timestampTz('docker_cleanup_next_run_at')->nullable()->index();+ });++ $this->continueAfterLastScheduledOccurrences();++ Schema::dropIfExists('scheduled_job_states');+ }++ public function down(): void+ {+ Schema::create('scheduled_job_states', function (Blueprint $table) {+ $table->id();+ $table->string('uuid')->unique();+ $table->string('schedule_key')->unique();+ $table->timestampTz('last_scheduled_for')->nullable();+ $table->timestamps();+ });++ Schema::table('server_settings', function (Blueprint $table) {+ $table->dropIndex(['docker_cleanup_next_run_at']);+ $table->dropColumn('docker_cleanup_next_run_at');+ });++ foreach (['scheduled_database_backups', 'scheduled_tasks', 'scheduled_volume_backups'] as $table) {+ Schema::table($table, function (Blueprint $table) {+ $table->dropIndex(['enabled', 'next_run_at']);+ $table->dropColumn('next_run_at');+ });+ }+ }++ /**+ * Continue each schedule after the last occurrence that the previous dispatcher recorded, so+ * an occurrence that was due during the upgrade still runs late instead of being skipped.+ */+ public function continueAfterLastScheduledOccurrences(): void+ {+ if (! Schema::hasTable('scheduled_job_states')) {+ return;+ }++ DB::table('scheduled_job_states')->whereNotNull('last_scheduled_for')->orderBy('id')->chunkById(500, function ($states) {+ foreach ($states as $state) {+ try {+ [$type, $id] = explode(':', $state->schedule_key, 2) + [null, null];+ $schedule = match ($type) {+ 'scheduled-backup' => ScheduledDatabaseBackup::find($id),+ 'scheduled-task' => ScheduledTask::find($id),+ 'scheduled-volume-backup' => ScheduledVolumeBackup::find($id),+ 'docker-cleanup' => ServerSetting::query()->where('server_id', $id)->first(),+ default => null,+ };++ if ($schedule instanceof ServerSetting) {+ $schedule->forceFill(['docker_cleanup_next_run_at' => next_cron_run_at(+ (string) $schedule->docker_cleanup_frequency, $schedule->server_timezone, Carbon::parse($state->last_scheduled_for),+ )])->saveQuietly();+ } elseif ($schedule?->enabled) {+ $schedule->forceFill(['next_run_at' => $schedule->calculateNextRunAt(Carbon::parse($state->last_scheduled_for))])->saveQuietly();+ }+ } catch (Throwable) {+ // The dispatcher calculates the next run of this schedule on its first run.+ }+ }+ });+ }+};diff --git a/openapi.json b/openapi.jsonindex f0d628691b..ec644582c4 100644--- a/openapi.json+++ b/openapi.json@@ -23557,6 +23557,12 @@"type": "integer","description": "The timeout of the scheduled task in seconds."},+ "next_run_at": {+ "type": "string",+ "format": "date-time",+ "nullable": true,+ "description": "The next time the scheduled task runs, in UTC. Null when it is disabled or not calculated yet."+ },"created_at": {"type": "string","format": "date-time",diff --git a/openapi.yaml b/openapi.yamlindex 3801868b4d..dd507d853f 100644--- a/openapi.yaml+++ b/openapi.yaml@@ -15091,6 +15091,11 @@ components:timeout:type: integerdescription: 'The timeout of the scheduled task in seconds.'+ next_run_at:+ type: string+ format: date-time+ nullable: true+ description: 'The next time the scheduled task runs, in UTC. Null when it is disabled or not calculated yet.'created_at:type: stringformat: date-timediff --git a/resources/views/components/next-run.blade.php b/resources/views/components/next-run.blade.phpnew file mode 100644index 0000000000..f0db83ffda--- /dev/null+++ b/resources/views/components/next-run.blade.php@@ -0,0 +1,30 @@+@props([+ 'at' => null,+ 'timezone' => null,+ 'enabled' => true,+ 'frequency' => null,+])++@php+ // Same fallback as next_cron_run_at(): schedules without a valid server timezone run in the instance timezone.+ $displayTimezone = filled($timezone) && validate_timezone((string) $timezone) ? $timezone : config('app.timezone');+ $nextRunAt = $at ? \Illuminate\Support\Carbon::parse($at) : null;+ // Without a next run, the frequency is either not calculated yet or cannot be calculated at all.+ $isFrequencyInvalid = ! $nextRunAt && filled($frequency) && next_cron_run_at((string) $frequency, $displayTimezone, now()) === null;+@endphp++<p {{ $attributes->merge(['class' => 'text-xs leading-5 text-neutral-500 dark:text-fg-dim']) }}>+ <span class="font-medium text-neutral-700 dark:text-fg">Next run</span>+ @if (! $enabled)+ <span>Disabled</span>+ @elseif ($isFrequencyInvalid)+ <span class="text-red-500">Invalid frequency. Change it to a valid cron expression, or this schedule never runs.</span>+ @elseif (! $nextRunAt)+ <span>Calculating…</span>+ @else+ <time datetime="{{ $nextRunAt->toIso8601String() }}"+ title="{{ $nextRunAt->copy()->utc()->format('Y-m-d H:i') }} UTC">{{ $nextRunAt->copy()->setTimezone($displayTimezone)->format('Y-m-d H:i') }} ({{ $displayTimezone }})</time>+ <span>·+ {{ $nextRunAt->isFuture() ? 'in '.$nextRunAt->diffForHumans(syntax: \Carbon\CarbonInterface::DIFF_ABSOLUTE) : $nextRunAt->diffForHumans() }}</span>+ @endif+</p>diff --git a/resources/views/livewire/project/database/backup-edit/general.blade.php b/resources/views/livewire/project/database/backup-edit/general.blade.phpindex 49385b0258..1dd6f28eb8 100644--- a/resources/views/livewire/project/database/backup-edit/general.blade.php+++ b/resources/views/livewire/project/database/backup-edit/general.blade.php@@ -96,6 +96,7 @@min="0" max="365" suffix="days" canGate="manageBackups" :canResource="$backup->database"helper="Notify through backup failure channels after this many days without an execution. Use 0 to disable." required /></div>+ <x-next-run :at="$backup->next_run_at" :timezone="$timezone" :enabled="$backup->enabled" :frequency="$backup->frequency" /></div></x-application.settings-section></form>diff --git a/resources/views/livewire/project/shared/scheduled-task/show.blade.php b/resources/views/livewire/project/shared/scheduled-task/show.blade.phpindex c5be9974ba..7849f35dcf 100644--- a/resources/views/livewire/project/shared/scheduled-task/show.blade.php+++ b/resources/views/livewire/project/shared/scheduled-task/show.blade.php@@ -44,6 +44,7 @@<x-forms.input canGate="update" :canResource="$resource" :disabled="!auth()->user()->can('update', $resource)"placeholder="php artisan schedule:run" id="command" label="Command" required /></div>+ <x-next-run class="mt-4" :at="$task->next_run_at" :timezone="$this->serverTimezone" :enabled="$task->enabled" :frequency="$task->frequency" /></x-application.settings-section></form>diff --git a/resources/views/livewire/project/shared/storages/volume-backups/general.blade.php b/resources/views/livewire/project/shared/storages/volume-backups/general.blade.phpindex 5b4ece52f1..48764b9c36 100644--- a/resources/views/livewire/project/shared/storages/volume-backups/general.blade.php+++ b/resources/views/livewire/project/shared/storages/volume-backups/general.blade.php@@ -45,5 +45,6 @@<x-forms.input id="timeout" type="number" min="60" max="36000" label="Timeout"helper="Maximum backup runtime in seconds." required /></div>+ <x-next-run class="mt-4" :at="$backup?->next_run_at" :timezone="$timezone" :enabled="(bool) $backup?->enabled" :frequency="$backup?->frequency" /></x-application.settings-section></form>diff --git a/resources/views/livewire/server/docker-cleanup.blade.php b/resources/views/livewire/server/docker-cleanup.blade.phpindex e12f4cd3da..a530f7c746 100644--- a/resources/views/livewire/server/docker-cleanup.blade.php+++ b/resources/views/livewire/server/docker-cleanup.blade.php@@ -33,6 +33,8 @@@if (!isCloud() && $this->isCleanupStale)<x-callout type="warning" title="Docker cleanup may be stalled">+ A scheduled cleanup was due+ {{ $server->settings->docker_cleanup_next_run_at?->diffForHumans() }} but has not started.The last cleanup ran {{ $this->lastExecutionTime ?? 'at an unknown time' }}.@if (!$this->isSchedulerHealthy)The scheduled job manager also appears inactive.@@ -52,6 +54,8 @@<p class="mt-1 text-xs leading-5 text-neutral-500 dark:text-fg-dim">Cleanup runs automatically using the schedule and threshold configured below.</p>+ <x-next-run class="mt-1" :at="$server->settings->docker_cleanup_next_run_at"+ :timezone="$server->settings->server_timezone" :frequency="$server->settings->docker_cleanup_frequency" /></div></div>@endifdiff --git a/tests/Feature/Jobs/ScheduledJobManagerStaleLockTest.php b/tests/Feature/Jobs/ScheduledJobManagerStaleLockTest.phpdeleted file mode 100644index dba5b6041a..0000000000--- a/tests/Feature/Jobs/ScheduledJobManagerStaleLockTest.php+++ /dev/null@@ -1,50 +0,0 @@-<?php--use App\Jobs\ScheduledJobManager;-use Illuminate\Support\Facades\Redis;--function scheduledJobManagerLockKey(): string-{- return config('cache.prefix').'laravel-queue-overlap:'.ScheduledJobManager::class.':scheduled-job-manager';-}--it('clears stale lock when TTL is -1', function () {- $lockKey = scheduledJobManagerLockKey();-- $redis = Mockery::mock();- $redis->shouldReceive('ttl')->once()->with($lockKey)->andReturn(-1);- $redis->shouldReceive('del')->once()->with($lockKey)->andReturn(1);-- Redis::shouldReceive('connection')->with('default')->andReturn($redis);-- $job = new ScheduledJobManager;- $job->middleware();-});--it('preserves valid lock with positive TTL', function () {- $lockKey = scheduledJobManagerLockKey();-- $redis = Mockery::mock();- $redis->shouldReceive('ttl')->once()->with($lockKey)->andReturn(60);- $redis->shouldNotReceive('del');-- Redis::shouldReceive('connection')->with('default')->andReturn($redis);-- $job = new ScheduledJobManager;- $job->middleware();-});--it('does not fail when no lock exists', function () {- $lockKey = scheduledJobManagerLockKey();-- $redis = Mockery::mock();- $redis->shouldReceive('ttl')->once()->with($lockKey)->andReturn(-2);- $redis->shouldNotReceive('del');-- Redis::shouldReceive('connection')->with('default')->andReturn($redis);-- $job = new ScheduledJobManager;- $middleware = $job->middleware();-- expect($middleware)->toBeArray()->toHaveCount(1);-});diff --git a/tests/Feature/QueueRoutingTest.php b/tests/Feature/QueueRoutingTest.phpindex 1bb837be89..a35d2d8555 100644--- a/tests/Feature/QueueRoutingTest.php+++ b/tests/Feature/QueueRoutingTest.php@@ -4,7 +4,6 @@ use App\Actions\Database\StartDatabase;use App\Actions\Database\StartDatabaseProxy;use App\Actions\Service\StartService;use App\Jobs\DatabaseBackupJob;-use App\Jobs\ScheduledJobManager;use App\Models\ScheduledDatabaseBackup;describe('deployment_queue helper', function () {@@ -61,14 +60,12 @@ describe('scheduled job routing', function () {test('scheduled jobs use the crons queue on cloud', function () {config(['constants.coolify.self_hosted' => false]);- expect((new ScheduledJobManager)->queue)->toBe('crons');expect((new DatabaseBackupJob(new ScheduledDatabaseBackup))->queue)->toBe('crons');});test('scheduled jobs use the high queue on self-hosted', function () {config(['constants.coolify.self_hosted' => true]);- expect((new ScheduledJobManager)->queue)->toBe('high');expect((new DatabaseBackupJob(new ScheduledDatabaseBackup))->queue)->toBe('high');});});diff --git a/tests/Feature/ScheduleNextRunAtTest.php b/tests/Feature/ScheduleNextRunAtTest.phpnew file mode 100644index 0000000000..e05acc2fb3--- /dev/null+++ b/tests/Feature/ScheduleNextRunAtTest.php@@ -0,0 +1,179 @@+<?php++use App\Models\Application;+use App\Models\Environment;+use App\Models\PrivateKey;+use App\Models\Project;+use App\Models\ScheduledTask;+use App\Models\Server;+use App\Models\StandaloneDocker;+use App\Models\Team;+use App\Services\ScheduleNextRunRecalculator;+use Carbon\CarbonImmutable;+use Illuminate\Database\Schema\Blueprint;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Carbon;+use Illuminate\Support\Facades\DB;+use Illuminate\Support\Facades\Schema;++uses(RefreshDatabase::class);++beforeEach(function () {+ Server::flushIdentityMap();+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 10, 20, 30, 'UTC'));+});++it('calculates the next due time of a frequency in the server timezone', function (string $frequency, string $timezone, string $after, bool $includeCurrentMinute, ?string $expected) {+ $next = next_cron_run_at($frequency, $timezone, CarbonImmutable::parse($after, 'UTC'), $includeCurrentMinute);++ expect($next?->toDateTimeString())->toBe($expected);+})->with([+ 'alias' => ['daily', 'UTC', '2026-09-17 10:20:30', false, '2026-09-18 00:00:00'],+ 'server timezone' => ['0 2 * * *', 'Europe/Budapest', '2026-09-17 10:20:30', false, '2026-09-18 00:00:00'],+ 'strictly after' => ['20 10 * * *', 'UTC', '2026-09-17 10:20:00', false, '2026-09-18 10:20:00'],+ 'current minute included' => ['20 10 * * *', 'UTC', '2026-09-17 10:20:30', true, '2026-09-17 10:20:00'],+ 'a time that does not exist when daylight saving time starts' => ['30 2 * * *', 'Europe/Budapest', '2026-03-28 12:00:00', false, '2026-03-29 01:30:00'],+ 'invalid timezone uses the app timezone' => ['0 2 * * *', 'Mars/Olympus', '2026-09-17 10:20:30', false, '2026-09-18 02:00:00'],+ 'invalid frequency' => ['not a cron', 'UTC', '2026-09-17 10:20:30', false, null],+]);++it('calculates the next run of a schedule when it is created or its frequency or enabled flag changes', function () {+ $task = nextRunAtTestTask(nextRunAtTestApplication('Europe/Budapest'), '0 2 * * *');++ expect($task->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00');++ $task->update(['frequency' => 'hourly']);+ expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-17 11:00:00');++ $task->update(['enabled' => false]);+ expect($task->fresh()->next_run_at)->toBeNull();++ Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 30, 0, 'UTC'));+ $task->update(['enabled' => true]);+ expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-17 13:00:00');++ $task->update(['name' => 'renamed']);+ expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-17 13:00:00');+});++it('calculates the next Docker cleanup when its frequency or the server timezone changes', function () {+ $settings = nextRunAtTestApplication()->destination->server->settings;++ $settings->update(['docker_cleanup_frequency' => '0 2 * * *']);+ expect($settings->fresh()->docker_cleanup_next_run_at->toDateTimeString())->toBe('2026-09-18 02:00:00');++ $settings->update(['server_timezone' => 'Europe/Budapest']);+ expect($settings->fresh()->docker_cleanup_next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00');+});++it('recalculates the next run of the server schedules when the server timezone changes', function () {+ $application = nextRunAtTestApplication();+ $task = nextRunAtTestTask($application, '0 2 * * *');+ $otherTask = nextRunAtTestTask(nextRunAtTestApplication(), '0 2 * * *');++ $application->destination->server->settings->update(['server_timezone' => 'Europe/Budapest']);++ expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00')+ ->and($otherTask->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 02:00:00');+});++it('keeps the timezone of the main server for applications that also run on the changed server', function () {+ $application = nextRunAtTestApplication();+ $task = nextRunAtTestTask($application, '0 2 * * *');+ $additionalServer = nextRunAtTestApplication()->destination->server;+ $application->additional_networks()->attach(+ StandaloneDocker::where('server_id', $additionalServer->id)->value('id'),+ ['server_id' => $additionalServer->id],+ );++ $additionalServer->settings->update(['server_timezone' => 'Europe/Budapest']);++ expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 02:00:00');+});++it('does not run an occurrence again when the timezone changes in the minute it ran', function () {+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 11, 59, 0, 'UTC'));+ $application = nextRunAtTestApplication();+ $task = nextRunAtTestTask($application, '0 * * * *');++ Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 30, 'UTC'));+ $application->destination->server->settings->update(['server_timezone' => 'Europe/Budapest']);++ expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-17 13:00:00');+});++it('recalculates the next run of the resource schedules with the timezone of its current server', function () {+ $application = nextRunAtTestApplication();+ $task = nextRunAtTestTask($application, '0 2 * * *');+ $application->destination->server->settings()->update(['server_timezone' => 'Europe/Budapest']);++ app(ScheduleNextRunRecalculator::class)->forResource($application);++ expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00');+});++it('lists overdue schedules and schedules without a next run in the diagnostics', function () {+ $overdue = nextRunAtTestTask(nextRunAtTestApplication(), '0 2 * * *');+ ScheduledTask::query()->whereKey($overdue->id)->update(['next_run_at' => now()->subHour()]);+ $invalid = nextRunAtTestTask(nextRunAtTestApplication(), '0 2 * * *');+ ScheduledTask::query()->whereKey($invalid->id)->update(['frequency' => '60 * * * *', 'next_run_at' => null]);+ nextRunAtTestTask(nextRunAtTestApplication(), '0 3 * * *');++ $this->artisan('scheduled:diagnostics')+ ->expectsOutputToContain('Scheduled tasks: 3 enabled, 2 overdue or without a next run')+ ->expectsOutputToContain('invalid frequency')+ ->assertSuccessful();+});++it('continues each schedule after the occurrence that the previous dispatcher recorded last', function () {+ $task = nextRunAtTestTask(nextRunAtTestApplication(), '0 2 * * *');+ Schema::create('scheduled_job_states', function (Blueprint $table) {+ $table->id();+ $table->string('uuid')->unique();+ $table->string('schedule_key')->unique();+ $table->timestampTz('last_scheduled_for')->nullable();+ $table->timestamps();+ });+ DB::table('scheduled_job_states')->insert([+ 'uuid' => 'state-1',+ 'schedule_key' => "scheduled-task:{$task->id}",+ 'last_scheduled_for' => '2026-09-16 02:00:00',+ ]);++ $migration = require database_path('migrations/2026_09_28_194652_add_next_run_at_to_scheduled_jobs.php');+ $migration->continueAfterLastScheduledOccurrences();++ // The occurrence of 2026-09-17 02:00 was not recorded, so the dispatcher runs it on its next run.+ expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-17 02:00:00');+});++function nextRunAtTestApplication(string $timezone = 'UTC'): Application+{+ $team = Team::factory()->create();+ $server = Server::factory()->create([+ 'team_id' => $team->id,+ 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id,+ ]);+ $server->settings()->update(['server_timezone' => $timezone]);+ $destination = StandaloneDocker::where('server_id', $server->id)->first()+ ?? StandaloneDocker::factory()->create(['server_id' => $server->id]);+ $environment = Environment::factory()->create([+ 'project_id' => Project::factory()->create(['team_id' => $team->id])->id,+ ]);++ return Application::factory()->create([+ 'environment_id' => $environment->id,+ 'destination_id' => $destination->id,+ 'destination_type' => StandaloneDocker::class,+ ]);+}++function nextRunAtTestTask(Application $application, string $frequency): ScheduledTask+{+ return ScheduledTask::factory()->create([+ 'team_id' => $application->environment->project->team_id,+ 'application_id' => $application->id,+ 'frequency' => $frequency,+ 'enabled' => true,+ ])->fresh();+}diff --git a/tests/Feature/ScheduleNextRunDisplayTest.php b/tests/Feature/ScheduleNextRunDisplayTest.phpnew file mode 100644index 0000000000..d8910b1d6c--- /dev/null+++ b/tests/Feature/ScheduleNextRunDisplayTest.php@@ -0,0 +1,222 @@+<?php++use App\Livewire\Project\Database\BackupEdit;+use App\Livewire\Project\Shared\ScheduledTask\Show;+use App\Livewire\Project\Shared\Storages\VolumeBackups;+use App\Livewire\Server\DockerCleanup;+use App\Models\Application;+use App\Models\Environment;+use App\Models\InstanceSettings;+use App\Models\LocalPersistentVolume;+use App\Models\Project;+use App\Models\ScheduledDatabaseBackup;+use App\Models\ScheduledTask;+use App\Models\Server;+use App\Models\ServerSetting;+use App\Models\StandaloneDocker;+use App\Models\StandalonePostgresql;+use App\Models\Team;+use App\Models\User;+use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Routing\Events\RouteMatched;+use Illuminate\Support\Carbon;+use Illuminate\Support\Facades\Event;+use Livewire\Livewire;++uses(RefreshDatabase::class);++beforeEach(function () {+ Server::flushIdentityMap();+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 10, 20, 30, 'UTC'));+ InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));++ $this->team = Team::factory()->create();+ $this->user = User::factory()->create();+ $this->user->teams()->attach($this->team, ['role' => 'owner']);+ $this->actingAs($this->user);+ session(['currentTeam' => $this->team]);++ $this->server = Server::factory()->create(['team_id' => $this->team->id]);+ $this->server->settings->update(['server_timezone' => 'Europe/Budapest']);+ $this->destination = StandaloneDocker::where('server_id', $this->server->id)->firstOrFail();+ $project = Project::factory()->create(['team_id' => $this->team->id]);+ $this->environment = Environment::factory()->create(['project_id' => $project->id]);+ $this->application = Application::factory()->create([+ 'environment_id' => $this->environment->id,+ 'destination_id' => $this->destination->id,+ 'destination_type' => $this->destination->getMorphClass(),+ ]);+});++afterEach(function () {+ Event::forget(RouteMatched::class);+ Carbon::setTestNow();+});++function nextRunDisplayBindTaskRoute(Application $application, ScheduledTask $task): void+{+ $application->loadMissing('environment.project');++ Event::listen(RouteMatched::class, function (RouteMatched $event) use ($application, $task): void {+ $event->route->setParameter('task_uuid', $task->uuid);+ $event->route->setParameter('project_uuid', $application->environment->project->uuid);+ $event->route->setParameter('environment_uuid', $application->environment->uuid);+ $event->route->setParameter('application_uuid', $application->uuid);+ });+}++function nextRunDisplaySetCleanupNextRun(Server $server, ?Carbon $nextRunAt): void+{+ ServerSetting::query()->where('server_id', $server->id)->update(['docker_cleanup_next_run_at' => $nextRunAt]);+}++it('shows the next Docker cleanup in the server timezone and refreshes it after saving a new frequency', function () {+ $this->server->settings->update(['docker_cleanup_frequency' => '0 2 * * *']);++ Livewire::test(DockerCleanup::class, ['server_uuid' => $this->server->uuid])+ ->assertSee('Next run')+ ->assertSee('2026-09-18 02:00 (Europe/Budapest)', false)+ ->set('dockerCleanupFrequency', 'hourly')+ ->call('submit')+ ->assertSee('2026-09-17 13:00 (Europe/Budapest)', false)+ ->assertSee('in 39 minutes');+});++it('shows Calculating when the Docker cleanup next run is not calculated yet', function () {+ nextRunDisplaySetCleanupNextRun($this->server, null);++ Livewire::test(DockerCleanup::class, ['server_uuid' => $this->server->uuid])+ ->assertSee('Next run')+ ->assertSee('Calculating…');+});++it('marks the Docker cleanup as stale only when its next run is more than 10 minutes overdue', function (?int $minutesFromNow, bool $stale) {+ nextRunDisplaySetCleanupNextRun($this->server, $minutesFromNow === null ? null : now()->addMinutes($minutesFromNow));++ $component = Livewire::test(DockerCleanup::class, ['server_uuid' => $this->server->uuid]);++ expect($component->instance()->isCleanupStale())->toBe($stale);+ if ($stale) {+ $component->assertSee('Docker cleanup may be stalled');+ } else {+ $component->assertDontSee('Docker cleanup may be stalled');+ }+})->with([+ '30 minutes overdue' => [-30, true],+ '5 minutes overdue' => [-5, false],+ 'in the future' => [30, false],+ 'not calculated yet' => [null, false],+]);++it('does not mark the Docker cleanup as stale on a server without a real IP address', function () {+ $this->server->update(['ip' => Server::PLACEHOLDER_IP]);+ nextRunDisplaySetCleanupNextRun($this->server, now()->subMinutes(30));++ $component = Livewire::test(DockerCleanup::class, ['server_uuid' => $this->server->uuid]);++ expect($component->instance()->isCleanupStale())->toBeFalse();+});++it('shows the next scheduled task run in the server timezone and refreshes it on changes', function () {+ $task = ScheduledTask::factory()->create([+ 'team_id' => $this->team->id,+ 'application_id' => $this->application->id,+ 'frequency' => '0 2 * * *',+ 'enabled' => true,+ ]);+ nextRunDisplayBindTaskRoute($this->application, $task);++ $component = Livewire::test(Show::class)+ ->assertSee('Next run')+ ->assertSee('2026-09-18 02:00 (Europe/Budapest)', false)+ ->assertSee('in 13 hours');++ $component->call('toggleEnabled')+ ->assertDontSee('2026-09-18 02:00 (Europe/Budapest)', false)+ ->assertSeeInOrder(['Next run', 'Disabled']);++ $component->call('toggleEnabled')+ ->set('frequency', 'hourly')+ ->call('submit')+ ->assertSee('2026-09-17 13:00 (Europe/Budapest)', false);++ expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-17 11:00:00');+});++it('shows Calculating for an enabled scheduled task without a next run', function () {+ $task = ScheduledTask::factory()->create([+ 'team_id' => $this->team->id,+ 'application_id' => $this->application->id,+ 'frequency' => '0 2 * * *',+ 'enabled' => true,+ ]);+ ScheduledTask::query()->whereKey($task->id)->update(['next_run_at' => null]);+ nextRunDisplayBindTaskRoute($this->application, $task);++ Livewire::test(Show::class)->assertSee('Calculating…');+});++it('shows an invalid frequency instead of Calculating when the next run cannot be calculated', function () {+ $task = ScheduledTask::factory()->create([+ 'team_id' => $this->team->id,+ 'application_id' => $this->application->id,+ 'frequency' => '0 2 * * *',+ 'enabled' => true,+ ]);+ // The forms reject such a value, so only old data or a direct database change can have it.+ ScheduledTask::query()->whereKey($task->id)->update(['frequency' => '60 * * * *', 'next_run_at' => null]);+ nextRunDisplayBindTaskRoute($this->application, $task);++ Livewire::test(Show::class)+ ->assertSee('Invalid frequency')+ ->assertDontSee('Calculating…');+});++it('shows an invalid Docker cleanup frequency', function () {+ ServerSetting::query()->where('server_id', $this->server->id)->update(['docker_cleanup_frequency' => '0 0 31 2 *', 'docker_cleanup_next_run_at' => null]);++ Livewire::test(DockerCleanup::class, ['server_uuid' => $this->server->uuid])+ ->assertSee('Invalid frequency')+ ->assertDontSee('Calculating…');+});++it('shows the next database backup run in the server timezone', function () {+ $database = StandalonePostgresql::create([+ 'name' => 'pg-next-run',+ 'image' => 'postgres:16-alpine',+ 'postgres_user' => 'postgres',+ 'postgres_password' => 'password',+ 'postgres_db' => 'postgres',+ 'environment_id' => $this->environment->id,+ 'destination_id' => $this->destination->id,+ 'destination_type' => $this->destination->getMorphClass(),+ ]);+ $backup = ScheduledDatabaseBackup::create([+ 'frequency' => '0 2 * * *',+ 'enabled' => true,+ 'database_type' => $database->getMorphClass(),+ 'database_id' => $database->id,+ 'team_id' => $this->team->id,+ ]);++ Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => collect()])+ ->assertSee('2026-09-18 02:00 (Europe/Budapest)', false)+ ->call('toggleEnabled')+ ->assertDontSee('2026-09-18 02:00 (Europe/Budapest)', false)+ ->assertSeeInOrder(['Next run', 'Disabled']);+});++it('shows the next volume backup run in the server timezone', function () {+ $volume = LocalPersistentVolume::create([+ 'name' => 'app-data',+ 'mount_path' => '/data',+ 'resource_id' => $this->application->id,+ 'resource_type' => $this->application->getMorphClass(),+ ]);++ Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $this->application])+ ->assertSeeInOrder(['Next run', 'Disabled'])+ ->set('frequency', '0 2 * * *')+ ->call('toggleEnabled')+ ->assertSee('2026-09-18 02:00 (Europe/Budapest)', false);+});diff --git a/tests/Feature/ScheduleOnOneServerTest.php b/tests/Feature/ScheduleOnOneServerTest.phpindex 3ad3808631..9a38bff4ed 100644--- a/tests/Feature/ScheduleOnOneServerTest.php+++ b/tests/Feature/ScheduleOnOneServerTest.php@@ -72,3 +72,15 @@ it('schedules stuck resource cleanup in the background once per day', function (->and($event->withoutOverlapping)->toBeTrue()->and($event->runInBackground)->toBeTrue();});++it('runs the scheduled job dispatcher from the scheduler instead of a queue', function () {+ $events = collect(app(Schedule::class)->events());++ $event = $events->first(fn ($event) => str_contains((string) $event->command, 'scheduled:dispatch'));++ expect($event)->not->toBeNull()+ ->and($event->expression)->toBe('* * * * *')+ ->and($event->onOneServer)->toBeTrue()+ ->and($event->runInBackground)->toBeTrue()+ ->and($events->contains(fn ($event) => str_contains((string) $event->description, 'ScheduledJobManager')))->toBeFalse();+});diff --git a/tests/Feature/ScheduledJobManagerDispatchTest.php b/tests/Feature/ScheduledJobManagerDispatchTest.phpindex ebb34bb8ff..a869d8e6a0 100644--- a/tests/Feature/ScheduledJobManagerDispatchTest.php+++ b/tests/Feature/ScheduledJobManagerDispatchTest.php@@ -2,6 +2,7 @@use App\Events\BackupCreated;use App\Jobs\DatabaseBackupJob;+use App\Jobs\DockerCleanupJob;use App\Jobs\ScheduledJobManager;use App\Jobs\ScheduledTaskJob;use App\Models\Application;@@ -10,10 +11,11 @@ use App\Models\PrivateKey;use App\Models\Project;use App\Models\ScheduledDatabaseBackup;use App\Models\ScheduledJobDelivery;-use App\Models\ScheduledJobState;use App\Models\ScheduledTask;use App\Models\Server;use App\Models\ServerSetting;+use App\Models\Service;+use App\Models\ServiceDatabase;use App\Models\StandaloneDocker;use App\Models\StandalonePostgresql;use App\Models\Team;@@ -21,7 +23,6 @@ use App\Services\ScheduledJobDeliveryService;use Illuminate\Foundation\Testing\RefreshDatabase;use Illuminate\Support\Carbon;use Illuminate\Support\Facades\Bus;-use Illuminate\Support\Facades\Cache;use Illuminate\Support\Facades\DB;use Illuminate\Support\Facades\Event;use Illuminate\Support\Facades\Log;@@ -31,173 +32,219 @@ uses(RefreshDatabase::class);beforeEach(function () {Server::flushIdentityMap();-});--it('dispatches scheduled tasks across chunks', function () {config(['constants.coolify.self_hosted' => true]);- Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));Queue::fake();+});- $team = Team::factory()->create();- $privateKey = PrivateKey::create([- 'name' => 'Test Key',- 'private_key' => '-----BEGIN OPENSSH PRIVATE KEY------b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW-QyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevAAAAJi/QySHv0Mk-hwAAAAtzc2gtZWQyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevA-AAAECBQw4jg1WRT2IGHMncCiZhURCts2s24HoDS0thHnnRKVuGmoeGq/pojrsyP1pszcNV-uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA==------END OPENSSH PRIVATE KEY-----',- 'team_id' => $team->id,- ]);- $server = Server::factory()->create([- 'team_id' => $team->id,- 'private_key_id' => $privateKey->id,- ]);- $server->settings()->update([- 'is_reachable' => true,- 'is_usable' => true,- 'force_disabled' => false,- 'docker_cleanup_frequency' => '0 * * * *',- ]);-- $destination = StandaloneDocker::where('server_id', $server->id)->first()- ?? StandaloneDocker::factory()->create(['server_id' => $server->id]);- $project = Project::factory()->create(['team_id' => $team->id]);- $environment = Environment::factory()->create(['project_id' => $project->id]);- $application = Application::factory()->create([- 'environment_id' => $environment->id,- 'destination_id' => $destination->id,- 'destination_type' => StandaloneDocker::class,- 'status' => 'running',- ]);-- ScheduledTask::factory()- ->count(101)- ->create([- 'team_id' => $team->id,- 'application_id' => $application->id,- 'frequency' => '* * * * *',- 'enabled' => true,- ]);+it('dispatches due scheduled tasks across chunks and moves their next run forward', function () {+ Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 0, 30, 'UTC'));+ $application = createScheduledTaskApplication();+ foreach (range(1, 101) as $ignored) {+ createScheduledApplicationTask($application, ['frequency' => '* * * * *']);+ }+ Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));(new ScheduledJobManager)->handle();Queue::assertPushed(ScheduledTaskJob::class, 101);+ expect(ScheduledTask::query()->pluck('next_run_at')->map->toDateTimeString()->unique()->all())+ ->toBe(['2026-05-27 00:02:00']);});-it('skips expensive dispatch for schedules outside the catch-up window', function () {- config(['constants.coolify.self_hosted' => true]);- Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));- Queue::fake();-- $application = createScheduledTaskApplication();-- $task = ScheduledTask::factory()->create([- 'team_id' => $application->environment->project->team_id,- 'application_id' => $application->id,- 'frequency' => '0 2 * * *',- 'enabled' => true,- ]);+it('runs a new schedule first at its next due time', function () {+ Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 20, 'UTC'));+ $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => '* * * * *']);(new ScheduledJobManager)->handle();Queue::assertNotPushed(ScheduledTaskJob::class);- expect(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-task:{$task->id}")->exists())->toBeFalse();+ expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-05-27 00:02:00');});-it('dispatches a recently missed daily task when deduplication cache is empty', function () {- config(['constants.coolify.self_hosted' => true]);- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 10, 0, 'UTC'));- Queue::fake();-+it('does not query schedules that are not due', function () {+ Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 0, 0, 'UTC'));$application = createScheduledTaskApplication();+ $database = createScheduledBackupDatabase();+ $countQueries = function (): int {+ DB::flushQueryLog();+ DB::enableQueryLog();+ (new ScheduledJobManager)->handle();+ $count = count(DB::getQueryLog());+ DB::disableQueryLog();++ return $count;+ };+ // Initialize the Docker cleanup schedules of both servers, so only the schedules below change.+ Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));+ $countQueries();+ $withoutSchedules = $countQueries();- ScheduledTask::factory()->create([- 'team_id' => $application->environment->project->team_id,- 'application_id' => $application->id,- 'frequency' => 'daily',- 'enabled' => true,- ]);-- (new ScheduledJobManager)->handle();+ foreach (range(1, 5) as $ignored) {+ createScheduledApplicationTask($application, ['frequency' => '0 2 * * *']);+ createScheduledDatabaseBackup($database, ['frequency' => '0 2 * * *']);+ }- Queue::assertPushed(ScheduledTaskJob::class, 1);+ expect($countQueries())->toBe($withoutSchedules);+ Queue::assertNothingPushed();});-it('dispatches one job when multiple managers evaluate the same occurrence', function () {- config(['constants.coolify.self_hosted' => true]);- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));- Queue::fake();-- $application = createScheduledTaskApplication();- $task = ScheduledTask::factory()->create([- 'team_id' => $application->environment->project->team_id,- 'application_id' => $application->id,- 'frequency' => 'daily',- 'enabled' => true,- ]);+it('dispatches one job when several managers run for the same occurrence', function () {+ Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));+ $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 5, 'UTC'));(new ScheduledJobManager)->handle();(new ScheduledJobManager)->handle();Queue::assertPushed(ScheduledTaskJob::class, 1);expect(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-task:{$task->id}")->count())->toBe(1)- ->and(ScheduledJobState::query()->where('schedule_key', "scheduled-task:{$task->id}")->count())->toBe(1);+ ->and($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00');});-it('does not retry an occurrence skipped while its server is not functional', function () {- config(['constants.coolify.self_hosted' => true]);- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));- Queue::fake();+it('does not dispatch an occurrence that another manager claims after this one read it', function () {+ Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));+ $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 5, 'UTC'));++ // Another node claims the occurrence between this manager's select and its update.+ $claimedByOtherNode = false;+ ScheduledTask::retrieved(function (ScheduledTask $retrieved) use (&$claimedByOtherNode) {+ if (! $claimedByOtherNode) {+ $claimedByOtherNode = true;+ ScheduledTask::query()->whereKey($retrieved->id)->toBase()->update(['next_run_at' => '2026-09-18 00:00:00']);+ }+ });- $application = createScheduledTaskApplication();- $task = createScheduledApplicationTask($application, ['frequency' => 'daily']);+ (new ScheduledJobManager)->handle();++ Queue::assertNotPushed(ScheduledTaskJob::class);+ expect($claimedByOtherNode)->toBeTrue()+ ->and(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-task:{$task->id}")->exists())->toBeFalse();+});++it('calculates the next run of a schedule without one and runs it when it is due now', function (string $frequency, bool $runsNow, string $nextRunAt) {+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 20, 'UTC'));+ $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => $frequency]);+ ScheduledTask::query()->whereKey($task->id)->update(['next_run_at' => null]);++ (new ScheduledJobManager)->handle();++ Queue::assertPushed(ScheduledTaskJob::class, $runsNow ? 1 : 0);+ expect($task->fresh()->next_run_at->toDateTimeString())->toBe($nextRunAt);+})->with([+ 'due in the current minute' => ['0 0 * * *', true, '2026-09-18 00:00:00'],+ 'due later' => ['30 * * * *', false, '2026-09-17 00:30:00'],+]);++it('moves the next run forward when an occurrence is skipped, and does not retry it', function () {+ Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));+ $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);$server = $task->server();$server->settings()->update(['is_reachable' => false]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 5, 'UTC'));(new ScheduledJobManager)->handle();-$server->settings()->update(['is_reachable' => true]);- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 6, 0, 'UTC'));+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 1, 0, 'UTC'));(new ScheduledJobManager)->handle();Queue::assertNotPushed(ScheduledTaskJob::class);- expect(ScheduledJobState::query()->where('schedule_key', "scheduled-task:{$task->id}")->value('last_scheduled_for'))- ->not->toBeNull()- ->and(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-task:{$task->id}")->exists())- ->toBeFalse();+ expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00')+ ->and(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-task:{$task->id}")->exists())->toBeFalse();});-it('does not publish a task occurrence when its application is not running', function () {- config(['constants.coolify.self_hosted' => true]);- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));- Queue::fake();-+it('does not dispatch a task when its application is not running', function () {+ Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));$application = createScheduledTaskApplication();$application->update(['status' => 'stopped']);$task = createScheduledApplicationTask($application, ['frequency' => 'daily']);+ $logPath = captureScheduledJobManagerTestLog('scheduled');+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 5, 'UTC'));(new ScheduledJobManager)->handle();Queue::assertNotPushed(ScheduledTaskJob::class);- expect(ScheduledJobState::query()->where('schedule_key', "scheduled-task:{$task->id}")->exists())->toBeTrue()- ->and(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-task:{$task->id}")->exists())- ->toBeFalse();+ expect(file_get_contents($logPath))->toContain('Task skipped')->toContain('"skip_reason":"application_not_running"');+ @unlink($logPath);});-it('publishes a pending occurrence after a previous publisher interruption', function () {- config(['constants.coolify.self_hosted' => true]);- Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));- Queue::fake();+it('runs late backups once, but logs late tasks outside the late-run window as missed', function (int $minutesLate, bool $taskRuns) {+ Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));+ $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);+ $backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);+ $logPath = captureScheduledJobManagerTestLog('scheduled');++ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC')->addMinutes($minutesLate));+ (new ScheduledJobManager)->handle();+ Queue::assertPushed(ScheduledTaskJob::class, $taskRuns ? 1 : 0);+ Queue::assertPushed(DatabaseBackupJob::class, 1);+ expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00')+ ->and($backup->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00')+ ->and(str_contains(file_get_contents($logPath), 'Task missed'))->toBe(! $taskRuns);+ @unlink($logPath);+})->with([+ 'five minutes late' => [5, true],+ 'three hours late' => [180, false],+]);++it('runs a daily schedule once on the night the clocks go back', function () {+ Carbon::setTestNow(Carbon::create(2026, 10, 24, 12, 0, 0, 'UTC'));$application = createScheduledTaskApplication();- $task = ScheduledTask::factory()->create([- 'team_id' => $application->environment->project->team_id,- 'application_id' => $application->id,- 'frequency' => 'daily',- 'enabled' => true,+ $application->destination->server->settings->update(['server_timezone' => 'Europe/Budapest']);+ $task = createScheduledApplicationTask($application, ['frequency' => '30 2 * * *']);++ // 02:30 CEST. At 03:00 CEST the clocks go back to 02:00, so 02:30 CET follows one hour later.+ Carbon::setTestNow(Carbon::create(2026, 10, 25, 0, 30, 5, 'UTC'));+ (new ScheduledJobManager)->handle();++ Queue::assertPushed(ScheduledTaskJob::class, 1);+ expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-10-26 01:30:00');+});++it('dispatches due Docker cleanups with their settings', function () {+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 30, 0, 'UTC'));+ $server = createScheduledTaskApplication()->destination->server;+ $server->settings->update(['docker_cleanup_frequency' => '0 * * * *', 'delete_unused_volumes' => true]);++ Carbon::setTestNow(Carbon::create(2026, 9, 17, 1, 0, 10, 'UTC'));+ (new ScheduledJobManager)->handle();++ Queue::assertPushed(DockerCleanupJob::class, fn (DockerCleanupJob $job) => $job->server->is($server) && $job->deleteUnusedVolumes);+ expect($server->settings->fresh()->docker_cleanup_next_run_at->toDateTimeString())->toBe('2026-09-17 02:00:00');+});++it('does not dispatch an occurrence that already has a delivery', function () {+ Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));+ $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);+ ScheduledJobDelivery::create([+ 'schedule_key' => "scheduled-task:{$task->id}",+ 'scheduled_for' => Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'),+ 'job_type' => 'scheduled-task',+ 'resource_id' => $task->id,+ 'status' => 'claimed',]);++ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 5, 'UTC'));+ (new ScheduledJobManager)->handle();++ Queue::assertNotPushed(ScheduledTaskJob::class);+ expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00');+});++it('dispatches a due backup of a service database', function () {+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 4, 30, 'UTC'));+ $backup = createScheduledServiceDatabaseBackup(['frequency' => '* * * * *']);++ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));+ (new ScheduledJobManager)->handle();++ Queue::assertPushed(DatabaseBackupJob::class, fn (DatabaseBackupJob $job) => $job->backup->is($backup));+});++it('publishes a pending occurrence after a previous publisher interruption', function () {+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));+ $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);$occurrence = ScheduledJobDelivery::create(['schedule_key' => "scheduled-task:{$task->id}",'scheduled_for' => Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'),@@ -211,6 +258,37 @@ it('publishes a pending occurrence after a previous publisher interruption', funexpect($occurrence->fresh()->status)->toBe('enqueued');});+it('keeps publishing pending occurrences after one of them fails', function () {+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));+ $logPath = captureScheduledJobManagerTestLog('scheduled-errors');+ $database = createScheduledBackupDatabase();+ $occurrences = collect([createScheduledDatabaseBackup($database), createScheduledDatabaseBackup($database)])+ ->map(fn (ScheduledDatabaseBackup $backup) => ScheduledJobDelivery::create([+ 'schedule_key' => "scheduled-backup:{$backup->id}",+ 'scheduled_for' => now()->startOfMinute(),+ 'job_type' => 'database-backup',+ 'resource_id' => $backup->id,+ 'status' => 'pending',+ ]));++ $dispatchAttempts = 0;+ Bus::shouldReceive('dispatch')->andReturnUsing(function () use (&$dispatchAttempts) {+ if (++$dispatchAttempts === 1) {+ throw new RuntimeException('queue connection lost');+ }+ });++ app(ScheduledJobDeliveryService::class)->publishPending();++ $log = file_get_contents($logPath);+ @unlink($logPath);++ expect($occurrences->first()->fresh()->status)->toBe('pending')+ ->and($occurrences->last()->fresh()->status)->toBe('enqueued')+ ->and($log)->toContain('Failed to publish pending scheduled occurrence')+ ->toContain('queue connection lost');+});+it('allows only one worker to claim an occurrence', function () {$occurrence = ScheduledJobDelivery::create(['schedule_key' => 'scheduled-task:claim-test',@@ -274,30 +352,8 @@ it('marks stale claimed deliveries as failed', function () {expect($delivery->fresh()->status)->toBe('failed');});-it('dispatches the instance coolify-db backup even when its id is zero', function () {- config(['constants.coolify.self_hosted' => true]);- Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));- Queue::fake();-- $database = createScheduledBackupDatabase();- $backup = createScheduledDatabaseBackup($database, [- 'id' => 0,- 'frequency' => '* * * * *',- ]);-- expect($backup->id)->toBe(0);-- (new ScheduledJobManager)->handle();-- Queue::assertPushed(DatabaseBackupJob::class, 1);- Queue::assertPushed(DatabaseBackupJob::class, fn (DatabaseBackupJob $job) => $job->backup->id === 0);-});-it('dispatches zero-id schedules and continues with positive ids', function () {- config(['constants.coolify.self_hosted' => true]);- Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));- Queue::fake();-+ Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 0, 30, 'UTC'));$application = createScheduledTaskApplication();$database = StandalonePostgresql::create(['name' => 'coolify-db',@@ -310,17 +366,14 @@ it('dispatches zero-id schedules and continues with positive ids', function () {'destination_id' => $application->destination_id,'destination_type' => $application->destination_type,]);-$zeroIdBackup = createScheduledDatabaseBackup($database, ['id' => 0]);$positiveIdBackup = createScheduledDatabaseBackup($database);$zeroIdTask = createScheduledApplicationTask($application, ['id' => 0]);$positiveIdTask = createScheduledApplicationTask($application);- expect($zeroIdBackup->id)->toBe(0)- ->and($positiveIdBackup->id)->toBeGreaterThan(0)- ->and($zeroIdTask->id)->toBe(0)- ->and($positiveIdTask->id)->toBeGreaterThan(0);+ expect($zeroIdBackup->id)->toBe(0)->and($zeroIdTask->id)->toBe(0);+ Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));(new ScheduledJobManager)->handle();Queue::assertPushed(DatabaseBackupJob::class, 2);@@ -332,14 +385,7 @@ it('dispatches zero-id schedules and continues with positive ids', function () {});it('does not query relationships when constructing scheduled task jobs', function () {- $application = createScheduledTaskApplication();-- $task = ScheduledTask::factory()->create([- 'team_id' => $application->environment->project->team_id,- 'application_id' => $application->id,- 'frequency' => '* * * * *',- 'enabled' => true,- ])->fresh();+ $task = createScheduledApplicationTask(createScheduledTaskApplication());DB::flushQueryLog();DB::enableQueryLog();@@ -351,56 +397,32 @@ it('does not query relationships when constructing scheduled task jobs', functio->and($job->timeout)->toBe(300);});-it('reads current server settings in each queued manager run of the same worker', function () {- config(['constants.coolify.self_hosted' => true]);- Queue::fakeExcept(ScheduledJobManager::class);-+it('reads current server settings in each manager run of the same process', function () {+ Carbon::setTestNow(Carbon::create(2026, 9, 15, 12, 0, 0, 'UTC'));$database = createScheduledBackupDatabase();createScheduledDatabaseBackup($database, ['frequency' => 'daily']);$serverId = $database->destination->server_id;ServerSetting::query()->where('server_id', $serverId)->update(['is_reachable' => false]);- Carbon::setTestNow(Carbon::create(2026, 9, 16, 0, 5, 0, 'UTC'));- dispatch(new ScheduledJobManager);+ Carbon::setTestNow(Carbon::create(2026, 9, 16, 0, 0, 5, 'UTC'));+ $this->artisan('scheduled:dispatch')->assertSuccessful();Queue::assertNotPushed(DatabaseBackupJob::class);// Another worker marks the server reachable again before the next daily run.ServerSetting::query()->where('server_id', $serverId)->update(['is_reachable' => true]);- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));- dispatch(new ScheduledJobManager);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 5, 'UTC'));+ $this->artisan('scheduled:dispatch')->assertSuccessful();Queue::assertPushed(DatabaseBackupJob::class, 1);});-it('logs the current database server state when a backup is skipped', function () {- config(['constants.coolify.self_hosted' => true]);- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));- Queue::fake();- $logPath = captureScheduledJobManagerTestLog('scheduled');-- $database = createScheduledBackupDatabase();- $backup = createScheduledDatabaseBackup($database, ['frequency' => 'daily']);- ServerSetting::query()->where('server_id', $database->destination->server_id)->update(['is_reachable' => false]);-- (new ScheduledJobManager)->handle();-- $log = file_get_contents($logPath);- @unlink($logPath);-- expect($log)->toContain('Backup skipped')- ->toContain('"skip_reason":"server_not_functional"')- ->toContain("\"backup_id\":{$backup->id}")- ->toContain('"db_is_reachable":false');-});-it('logs each step of a scheduled occurrence', function () {- config(['constants.coolify.self_hosted' => true]);- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));- Queue::fake();+ Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));+ $backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);$logPath = captureScheduledJobManagerTestLog('scheduled');- $backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));(new ScheduledJobManager)->handle();$occurrenceUuid = ScheduledJobDelivery::query()->where('schedule_key', "scheduled-backup:{$backup->id}")->value('uuid');$service = app(ScheduledJobDeliveryService::class);@@ -415,7 +437,6 @@ it('logs each step of a scheduled occurrence', function () {expect($log)->toContain('Backup dispatched')->toContain('Scheduled occurrence enqueued')->toContain('"occurrence_uuid":"'.$occurrenceUuid.'"')- ->toContain('"host":"'.gethostname().'"')->toContain('Scheduled occurrence claimed')->toContain('Scheduled occurrence claim rejected')->toContain('Scheduled occurrence completed')@@ -423,128 +444,23 @@ it('logs each step of a scheduled occurrence', function () {->toContain('"seconds_since_due":300');});-it('warns when the manager starts after the catch-up window', function (int $minutesSincePreviousStart, bool $expectWarning) {- config(['constants.coolify.self_hosted' => true]);- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 30, 0, 'UTC'));- Queue::fake();- $logPath = captureScheduledJobManagerTestLog('scheduled-errors');- Cache::put('scheduled-job-manager:last-started-at', now()->subMinutes($minutesSincePreviousStart)->toIso8601String());-- (new ScheduledJobManager)->handle();-- $log = file_get_contents($logPath);- @unlink($logPath);-- expect(str_contains($log, 'ScheduledJobManager started late'))->toBe($expectWarning);-})->with([- 'normal one-minute interval' => [1, false],- 'twenty-minute gap' => [20, true],-]);--it('warns when an occurrence is enqueued while the previous one is still running', function () {- config(['constants.coolify.self_hosted' => true]);- Queue::fake();- $logPath = captureScheduledJobManagerTestLog('scheduled');- $backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);-- Carbon::setTestNow(Carbon::create(2026, 9, 16, 0, 5, 0, 'UTC'));- (new ScheduledJobManager)->handle();- $previousUuid = ScheduledJobDelivery::query()->where('schedule_key', "scheduled-backup:{$backup->id}")->value('uuid');- app(ScheduledJobDeliveryService::class)->claim($previousUuid, 'worker-a');-- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));- (new ScheduledJobManager)->handle();-- $log = file_get_contents($logPath);- @unlink($logPath);-- expect($log)->toContain('Scheduled occurrence enqueued while the previous occurrence is still open')- ->toContain('"previous_occurrence_uuid":"'.$previousUuid.'"')- ->toContain('"previous_status":"claimed"');-});--it('reports occurrences that stay open after fifteen minutes', function () {- config(['constants.coolify.self_hosted' => true]);- Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 30, 0, 'UTC'));- Queue::fake();- $logPath = captureScheduledJobManagerTestLog('scheduled');-- foreach (['enqueued' => 20, 'claimed' => 20, 'failed' => 20] as $status => $minutesAgo) {- ScheduledJobDelivery::create([- 'schedule_key' => "scheduled-backup:{$status}",- 'scheduled_for' => now()->subMinutes($minutesAgo),- 'job_type' => 'database-backup',- 'resource_id' => 1,- 'status' => $status,- ])->forceFill(['created_at' => now()->subMinutes($minutesAgo)])->save();- }- ScheduledJobDelivery::create([- 'schedule_key' => 'scheduled-backup:recent',- 'scheduled_for' => now()->subMinutes(2),- 'job_type' => 'database-backup',- 'resource_id' => 1,- 'status' => 'enqueued',- ]);-- (new ScheduledJobManager)->handle();-- $log = file_get_contents($logPath);- @unlink($logPath);-- expect($log)->toContain('"open_occurrences_over_15_minutes":{"claimed":1,"enqueued":1}');-});--it('keeps publishing pending occurrences after one of them fails', function () {- config(['constants.coolify.self_hosted' => true]);- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));- $logPath = captureScheduledJobManagerTestLog('scheduled-errors');- $database = createScheduledBackupDatabase();- $occurrences = collect([createScheduledDatabaseBackup($database), createScheduledDatabaseBackup($database)])- ->map(fn (ScheduledDatabaseBackup $backup) => ScheduledJobDelivery::create([- 'schedule_key' => "scheduled-backup:{$backup->id}",- 'scheduled_for' => now()->startOfMinute(),- 'job_type' => 'database-backup',- 'resource_id' => $backup->id,- 'status' => 'pending',- ]));-- $dispatchAttempts = 0;- Bus::shouldReceive('dispatch')->andReturnUsing(function () use (&$dispatchAttempts) {- if (++$dispatchAttempts === 1) {- throw new RuntimeException('queue connection lost');- }- });-- app(ScheduledJobDeliveryService::class)->publishPending();-- $log = file_get_contents($logPath);- @unlink($logPath);-- expect($occurrences->first()->fresh()->status)->toBe('pending')- ->and($occurrences->last()->fresh()->status)->toBe('enqueued')- ->and($log)->toContain('Failed to publish pending scheduled occurrence')- ->toContain('queue connection lost');-});-it('logs a backup run that ends early, even when it completes before the enqueue log', function () {- config(['constants.coolify.self_hosted' => true]);- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));+ Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));Event::fake([BackupCreated::class]);- Queue::fakeExcept([ScheduledJobManager::class, DatabaseBackupJob::class]);- $logPath = captureScheduledJobManagerTestLog('scheduled');-+ Queue::fakeExcept([DatabaseBackupJob::class]);$database = createScheduledBackupDatabase();$database->update(['status' => 'exited']);$backup = createScheduledDatabaseBackup($database, ['frequency' => 'daily']);+ $logPath = captureScheduledJobManagerTestLog('scheduled');- dispatch(new ScheduledJobManager);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));+ $this->artisan('scheduled:dispatch')->assertSuccessful();$log = file_get_contents($logPath);@unlink($logPath);expect($log)->toContain('Database backup job ended without a backup')->toContain('"skip_reason":"database_not_running"')- ->toContain('"database_status":"exited:unhealthy"')->toContain('Scheduled occurrence completed')->toMatch('/Scheduled occurrence enqueued \{[^\n]*"schedule_key":"scheduled-backup:'.$backup->id.'"[^\n]*"status":"enqueued"/')->and(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-backup:{$backup->id}")->exists())->toBeFalse();@@ -630,6 +546,34 @@ function createScheduledDatabaseBackup(StandalonePostgresql $database, array $ovreturn $backup->fresh();}+function createScheduledServiceDatabaseBackup(array $overrides = []): ScheduledDatabaseBackup+{+ $application = createScheduledTaskApplication();+ $service = Service::factory()->create([+ 'environment_id' => $application->environment_id,+ 'destination_id' => $application->destination_id,+ 'destination_type' => $application->destination_type,+ ]);+ $database = ServiceDatabase::create([+ 'name' => 'postgres',+ 'image' => 'postgres:16-alpine',+ 'service_id' => $service->id,+ ]);++ $backup = new ScheduledDatabaseBackup;+ $backup->forceFill(array_merge([+ 'enabled' => true,+ 'save_s3' => false,+ 'frequency' => '* * * * *',+ 'database_id' => $database->id,+ 'database_type' => $database->getMorphClass(),+ 'team_id' => $application->environment->project->team_id,+ ], $overrides));+ $backup->save();++ return $backup->fresh();+}+function createScheduledApplicationTask(Application $application, array $overrides = []): ScheduledTask{$task = new ScheduledTask;diff --git a/tests/Feature/VolumeBackupTest.php b/tests/Feature/VolumeBackupTest.phpindex 623e140dfb..a161864408 100644--- a/tests/Feature/VolumeBackupTest.php+++ b/tests/Feature/VolumeBackupTest.php@@ -2519,9 +2519,9 @@ it('dispatches due scheduled volume backups', function () {'frequency' => '* * * * *','enabled' => true,]);- Cache::forget("scheduled-volume-backup:{$backup->id}");expect($backup->server()?->isFunctional())->toBeTrue();+ $this->travel(1)->minutes();(new ScheduledJobManager)->handle();Queue::assertPushed(@@ -2547,8 +2547,8 @@ it('dispatches due scheduled directory backups', function () {'frequency' => '* * * * *','enabled' => true,]);- Cache::forget("scheduled-volume-backup:{$backup->id}");+ $this->travel(1)->minutes();(new ScheduledJobManager)->handle();Queue::assertPushed(@@ -2577,6 +2577,7 @@ it('retains scheduled volume backups and archive metadata when the server is mis'destination_type' => null,]);+ $this->travel(1)->minutes();(new ScheduledJobManager)->handle();expect($backup->fresh())->not->toBeNull()@@ -2606,6 +2607,7 @@ it('dispatches pending recovery without starting another volume backup', functio'stop_recovery_pending' => true,]);+ $this->travel(1)->minutes();(new ScheduledJobManager)->handle();Queue::assertPushed(diff --git a/tests/Unit/ScheduledJobManagerLockTest.php b/tests/Unit/ScheduledJobManagerLockTest.phpdeleted file mode 100644index 924f6f43cf..0000000000--- a/tests/Unit/ScheduledJobManagerLockTest.php+++ /dev/null@@ -1,63 +0,0 @@-<?php--use App\Jobs\ScheduledJobManager;-use Illuminate\Queue\Middleware\WithoutOverlapping;-use Tests\TestCase;--uses(TestCase::class);--it('uses WithoutOverlapping middleware with expireAfter to prevent stale locks', function () {- $job = new ScheduledJobManager;- $middleware = $job->middleware();-- // Assert middleware exists- expect($middleware)->toBeArray()- ->and($middleware)->toHaveCount(1);-- $overlappingMiddleware = $middleware[0];-- // Assert it's a WithoutOverlapping instance- expect($overlappingMiddleware)->toBeInstanceOf(WithoutOverlapping::class);-- // Use reflection to check private properties- $reflection = new ReflectionClass($overlappingMiddleware);-- // Check expireAfter is set (should be 60 seconds - matches job frequency)- $expiresAfterProperty = $reflection->getProperty('expiresAfter');- $expiresAfterProperty->setAccessible(true);- $expiresAfter = $expiresAfterProperty->getValue($overlappingMiddleware);-- expect($expiresAfter)->toBe(90)- ->and($expiresAfter)->toBeGreaterThan(0, 'expireAfter must be set to prevent stale locks');-- // Check releaseAfter is NOT set (we use dontRelease)- $releaseAfterProperty = $reflection->getProperty('releaseAfter');- $releaseAfterProperty->setAccessible(true);- $releaseAfter = $releaseAfterProperty->getValue($overlappingMiddleware);-- expect($releaseAfter)->toBeNull('releaseAfter should be null when using dontRelease()');-- // Check the lock key- $keyProperty = $reflection->getProperty('key');- $keyProperty->setAccessible(true);- $key = $keyProperty->getValue($overlappingMiddleware);-- expect($key)->toBe('scheduled-job-manager');-});--it('prevents stale locks by ensuring expireAfter is always set', function () {- $job = new ScheduledJobManager;- $middleware = $job->middleware();-- $overlappingMiddleware = $middleware[0];- $reflection = new ReflectionClass($overlappingMiddleware);-- $expiresAfterProperty = $reflection->getProperty('expiresAfter');- $expiresAfterProperty->setAccessible(true);- $expiresAfter = $expiresAfterProperty->getValue($overlappingMiddleware);-- // Critical check: expireAfter MUST be set to prevent GitHub issue #4539- expect($expiresAfter)->not->toBeNull(- 'expireAfter() is required to prevent stale locks (see GitHub #4539)'- );-});diff --git a/tests/Unit/ServerPlaceholderIpTest.php b/tests/Unit/ServerPlaceholderIpTest.phpindex 94787b4d47..e7c0f25069 100644--- a/tests/Unit/ServerPlaceholderIpTest.php+++ b/tests/Unit/ServerPlaceholderIpTest.php@@ -4,8 +4,10 @@ use App\Jobs\CleanupOrphanedPreviewContainersJob;use App\Jobs\ScheduledJobManager;use App\Models\PrivateKey;use App\Models\Server;+use App\Models\ServerSetting;use App\Models\Team;use Illuminate\Foundation\Testing\RefreshDatabase;+use Illuminate\Support\Facades\Queue;use Tests\TestCase;uses(TestCase::class, RefreshDatabase::class);@@ -99,10 +101,13 @@ it('excludes every placeholder address from scheduled Docker cleanup', function]);}- $method = new ReflectionMethod(ScheduledJobManager::class, 'getServersForCleanupQuery');- $servers = $method->invoke(new ScheduledJobManager)->get();+ ServerSetting::query()->update(['docker_cleanup_next_run_at' => null]);+ Queue::fake();- expect($servers->modelKeys())->toBe([$realServer->id]);+ (new ScheduledJobManager)->handle();++ // The dispatcher calculates the next cleanup only for servers that it selects.+ expect(ServerSetting::query()->whereNotNull('docker_cleanup_next_run_at')->pluck('server_id')->all())->toBe([$realServer->id]);});it('excludes every placeholder address from orphaned preview cleanup', function () {