An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
revert(scheduler): go back to the cron scan and keep the reliability fixes
Andras Bacsai50be16be8fe6
diff --git a/.ai/lessons.md b/.ai/lessons.mdindex 040431f38c..af4080da3b 100644--- a/.ai/lessons.md+++ b/.ai/lessons.md@@ -60,8 +60,8 @@## Make distributed schedules durable- Use the database as the correctness source for dynamic cron occurrences shared by multiple scheduler and Horizon nodes; Redis locks are load controls, not a durable execution ledger.- Give each schedule occurrence a unique database identity and make queue consumers claim it atomically before external work.-- Store the next due time on each schedule and select only due rows; claim a run with `UPDATE ... WHERE next_run_at = :old`. Do not evaluate every cron expression each minute: on a remote database the per-row queries alone exceed the one-minute budget.- Run a per-minute dispatcher from the scheduler process, not as a queued job, so queue backlog cannot delay it.+- A Redis instance that holds queues must use `maxmemory-policy noeviction`; an evicting policy deletes queued jobs without an error. Before changing scheduler code for missed runs, check `INFO stats` `evicted_keys`.- Keep pending occurrences recoverable across publisher interruptions, and define an explicit bounded policy for late or offline schedules.- Horizon workers are long-lived: flush every static or `once()` cache (for example `Server::flushIdentityMap()`) in `Queue::before`, or later jobs decide with stale state.diff --git a/app/Actions/Shared/MigrateResourceToDestination.php b/app/Actions/Shared/MigrateResourceToDestination.phpindex ad69093412..b0979e1677 100644--- a/app/Actions/Shared/MigrateResourceToDestination.php+++ b/app/Actions/Shared/MigrateResourceToDestination.php@@ -23,7 +23,6 @@ use App\Models\StandalonePostgresql;use App\Models\StandaloneRedis;use App\Models\StandaloneSqlite;use App\Models\SwarmDocker;-use App\Services\ScheduleNextRunRecalculator;use Illuminate\Support\Collection;use Illuminate\Support\Facades\Bus;use Illuminate\Validation\ValidationException;@@ -152,7 +151,6 @@ class MigrateResourceToDestination}$resource->fill($payload)->save();- app(ScheduleNextRunRecalculator::class)->forResource($resource);if ($resource instanceof Application) {$resource->additional_networks()->detach();diff --git a/app/Console/Commands/DispatchScheduledJobs.php b/app/Console/Commands/DispatchScheduledJobs.phpindex 3d4b53c2ae..2c1cc8df04 100644--- a/app/Console/Commands/DispatchScheduledJobs.php+++ b/app/Console/Commands/DispatchScheduledJobs.php@@ -11,9 +11,9 @@ class DispatchScheduledJobs extends Commandprotected $description = 'Dispatch the due scheduled backups, tasks, volume backups, and Docker cleanups';- public function handle(ScheduledJobManager $manager): int+ public function handle(): int{- $manager->handle();+ ScheduledJobManager::dispatchSync();return self::SUCCESS;}diff --git a/app/Console/Commands/ScheduledJobDiagnostics.php b/app/Console/Commands/ScheduledJobDiagnostics.phpindex f66efd57b2..61f26265a9 100644--- a/app/Console/Commands/ScheduledJobDiagnostics.php+++ b/app/Console/Commands/ScheduledJobDiagnostics.php@@ -2,67 +2,254 @@namespace App\Console\Commands;-use App\Jobs\ScheduledJobManager;+use App\Models\DockerCleanupExecution;use App\Models\ScheduledDatabaseBackup;-use App\Models\ScheduledJobDelivery;use App\Models\ScheduledTask;-use App\Models\ScheduledVolumeBackup;-use App\Models\ServerSetting;+use App\Models\Server;+use App\Models\Team;use Illuminate\Console\Command;-use Illuminate\Database\Eloquent\Builder;use Illuminate\Support\Carbon;+use Illuminate\Support\Collection;use Illuminate\Support\Facades\Cache;class ScheduledJobDiagnostics extends Command{- protected $signature = 'scheduled:diagnostics {--all : List every schedule, not only the overdue ones and the ones without a next run}';+ protected $signature = 'scheduled:diagnostics+ {--type=all : Type to inspect: docker-cleanup, backups, tasks, server-jobs, all}+ {--server= : Filter by server ID}';- protected $description = 'Show the scheduler heartbeat, overdue schedules, schedules without a next run, and open deliveries';+ protected $description = 'Inspect dedup cache state and scheduling decisions for all scheduled jobs';public function handle(): int+ {+ $type = $this->option('type');+ $serverFilter = $this->option('server');++ $this->outputHeartbeat();++ if (in_array($type, ['all', 'docker-cleanup'])) {+ $this->inspectDockerCleanups($serverFilter);+ }++ if (in_array($type, ['all', 'backups'])) {+ $this->inspectBackups();+ }++ if (in_array($type, ['all', 'tasks'])) {+ $this->inspectTasks();+ }++ if (in_array($type, ['all', 'server-jobs'])) {+ $this->inspectServerJobs($serverFilter);+ }++ return self::SUCCESS;+ }++ private function outputHeartbeat(): void{$heartbeat = Cache::get('scheduled-job-manager:heartbeat');- $heartbeat- ? $this->info("Scheduler heartbeat: {$heartbeat} (".Carbon::parse($heartbeat)->diffForHumans().')')- : $this->error('Scheduler heartbeat: missing. The scheduled:dispatch command does not run.');+ if ($heartbeat) {+ $age = Carbon::parse($heartbeat)->diffForHumans();+ $this->info("Scheduler heartbeat: {$heartbeat} ({$age})");+ } else {+ $this->error('Scheduler heartbeat: MISSING — ScheduledJobManager may not be running');+ }$this->newLine();+ }++ private function inspectDockerCleanups(?string $serverFilter): void+ {+ $this->info('=== Docker Cleanup Jobs ===');++ $servers = $this->getServers($serverFilter);++ $rows = [];+ foreach ($servers as $server) {+ $frequency = data_get($server->settings, 'docker_cleanup_frequency', '0 * * * *');+ if (isset(VALID_CRON_STRINGS[$frequency])) {+ $frequency = VALID_CRON_STRINGS[$frequency];+ }- $overdueBefore = now()->subMinutes(ScheduledJobManager::LATE_RUN_WINDOW_MINUTES);- $schedules = [- 'Database backups' => [ScheduledDatabaseBackup::query()->where('enabled', true), 'next_run_at', 'frequency'],- 'Scheduled tasks' => [ScheduledTask::query()->where('enabled', true), 'next_run_at', 'frequency'],- 'Volume backups' => [ScheduledVolumeBackup::query()->where('enabled', true), 'next_run_at', 'frequency'],- 'Docker cleanups' => [ServerSetting::query(), 'docker_cleanup_next_run_at', 'docker_cleanup_frequency'],- ];-- foreach ($schedules as $label => [$query, $column, $frequencyColumn]) {- /** @var Builder $query */- $problems = (clone $query)->where(fn (Builder $query) => $query->whereNull($column)->orWhere($column, '<', $overdueBefore));- $this->info(sprintf('=== %s: %d enabled, %d overdue or without a next run ===', $label, (clone $query)->count(), (clone $problems)->count()));-- $rows = ($this->option('all') ? $query : $problems)- ->orderBy($column)- ->limit(100)- ->get()- ->map(fn ($schedule) => [$schedule->getKey(), $schedule->getRawOriginal($frequencyColumn), $schedule->{$column}?->toIso8601String() ?? (next_cron_run_at((string) $schedule->getRawOriginal($frequencyColumn), null, now()) ? 'not calculated' : 'invalid frequency')]);-- if ($rows->isNotEmpty()) {- $this->table(['ID', 'Frequency', 'Next run (UTC)'], $rows);+ $dedupKey = "docker-cleanup:{$server->id}";+ $cacheValue = Cache::get($dedupKey);+ $timezone = data_get($server->settings, 'server_timezone', config('app.timezone'));++ if (validate_timezone($timezone) === false) {+ $timezone = config('app.timezone');}- $this->newLine();++ $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);++ $lastExecution = DockerCleanupExecution::where('server_id', $server->id)+ ->latest()+ ->first();++ $rows[] = [+ $server->id,+ $server->name,+ $timezone,+ $frequency,+ $dedupKey,+ $cacheValue ?? '<missing>',+ $wouldFire ? 'YES' : 'no',+ $lastExecution ? $lastExecution->status.' @ '.$lastExecution->created_at : 'never',+ ];}- $this->info('=== Open and failed deliveries (failed ones are kept for 30 days) ===');$this->table(- ['Job type', 'Status', 'Count', 'Oldest due (UTC)'],- ScheduledJobDelivery::query()- ->whereIn('status', ['pending', 'enqueued', 'claimed', 'failed'])- ->selectRaw('job_type, status, count(*) as total, min(scheduled_for) as oldest')- ->groupBy('job_type', 'status')- ->get()- ->map(fn ($row) => [$row->job_type, $row->status, $row->total, $row->oldest]),+ ['ID', 'Server', 'TZ', 'Frequency', 'Dedup Key', 'Cache Value', 'Would Fire', 'Last Execution'],+ $rows);+ $this->newLine();+ }- return self::SUCCESS;+ private function inspectBackups(): void+ {+ $this->info('=== Scheduled Backups ===');++ $backups = ScheduledDatabaseBackup::with(['database'])+ ->where('enabled', true)+ ->get();++ $rows = [];+ foreach ($backups as $backup) {+ $server = $backup->server();+ $frequency = $backup->frequency;+ if (isset(VALID_CRON_STRINGS[$frequency])) {+ $frequency = VALID_CRON_STRINGS[$frequency];+ }++ $dedupKey = "scheduled-backup:{$backup->id}";+ $cacheValue = Cache::get($dedupKey);+ $timezone = $server ? data_get($server->settings, 'server_timezone', config('app.timezone')) : config('app.timezone');++ if (validate_timezone($timezone) === false) {+ $timezone = config('app.timezone');+ }++ $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);++ $rows[] = [+ $backup->id,+ $backup->database_type ?? 'unknown',+ $server?->name ?? 'N/A',+ $frequency,+ $cacheValue ?? '<missing>',+ $wouldFire ? 'YES' : 'no',+ ];+ }++ $this->table(+ ['Backup ID', 'DB Type', 'Server', 'Frequency', 'Cache Value', 'Would Fire'],+ $rows+ );+ $this->newLine();+ }++ private function inspectTasks(): void+ {+ $this->info('=== Scheduled Tasks ===');++ $tasks = ScheduledTask::with(['service', 'application'])+ ->where('enabled', true)+ ->get();++ $rows = [];+ foreach ($tasks as $task) {+ $server = $task->server();+ $frequency = $task->frequency;+ if (isset(VALID_CRON_STRINGS[$frequency])) {+ $frequency = VALID_CRON_STRINGS[$frequency];+ }++ $dedupKey = "scheduled-task:{$task->id}";+ $cacheValue = Cache::get($dedupKey);+ $timezone = $server ? data_get($server->settings, 'server_timezone', config('app.timezone')) : config('app.timezone');++ if (validate_timezone($timezone) === false) {+ $timezone = config('app.timezone');+ }++ $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);++ $rows[] = [+ $task->id,+ $task->name,+ $server?->name ?? 'N/A',+ $frequency,+ $cacheValue ?? '<missing>',+ $wouldFire ? 'YES' : 'no',+ ];+ }++ $this->table(+ ['Task ID', 'Name', 'Server', 'Frequency', 'Cache Value', 'Would Fire'],+ $rows+ );+ $this->newLine();+ }++ private function inspectServerJobs(?string $serverFilter): void+ {+ $this->info('=== Server Manager Jobs ===');++ $servers = $this->getServers($serverFilter);++ $rows = [];+ foreach ($servers as $server) {+ $timezone = data_get($server->settings, 'server_timezone', config('app.timezone'));+ if (validate_timezone($timezone) === false) {+ $timezone = config('app.timezone');+ }++ $dedupKeys = [+ "server-patch-check:{$server->id}" => '0 0 * * 0',+ "server-check:{$server->id}" => isCloud() ? '*/5 * * * *' : '* * * * *',+ "server-storage-check:{$server->id}" => data_get($server->settings, 'server_disk_usage_check_frequency', '0 23 * * *'),+ ];++ foreach ($dedupKeys as $dedupKey => $frequency) {+ if (isset(VALID_CRON_STRINGS[$frequency])) {+ $frequency = VALID_CRON_STRINGS[$frequency];+ }++ $cacheValue = Cache::get($dedupKey);+ $wouldFire = shouldRunCronNow($frequency, $timezone, $dedupKey);++ $rows[] = [+ $server->id,+ $server->name,+ $dedupKey,+ $frequency,+ $cacheValue ?? '<missing>',+ $wouldFire ? 'YES' : 'no',+ ];+ }+ }++ $this->table(+ ['Server ID', 'Server', 'Dedup Key', 'Frequency', 'Cache Value', 'Would Fire'],+ $rows+ );+ $this->newLine();+ }++ private function getServers(?string $serverFilter): Collection+ {+ $query = Server::with('settings')->where('ip', '!=', '1.2.3.4');++ if ($serverFilter) {+ $query->where('id', $serverFilter);+ }++ if (isCloud()) {+ $servers = $query->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get();+ $own = Team::find(0)?->servers()->with('settings')->get() ?? collect();++ return $servers->merge($own);+ }++ return $query->get();}}diff --git a/app/Console/Kernel.php b/app/Console/Kernel.phpindex 4bf442d3bf..7d45ff4b53 100644--- a/app/Console/Kernel.php+++ b/app/Console/Kernel.php@@ -78,7 +78,7 @@ class Kernel extends ConsoleKernel$this->scheduleInstance->job(new ServerManagerJob)->everyMinute()->onOneServer();// Scheduled Jobs (Backups & Tasks)- $this->scheduleScheduledJobs();+ $this->scheduleScheduledJobManager();$this->scheduleInstance->command('uploads:clear')->everyTwoMinutes();@@ -99,7 +99,7 @@ class Kernel extends ConsoleKernel$this->pullImages();// Scheduled Jobs (Backups & Tasks)- $this->scheduleScheduledJobs();+ $this->scheduleScheduledJobManager();$this->scheduleInstance->job(new RegenerateSslCertJob)->twiceDaily()->onOneServer();@@ -120,14 +120,15 @@ class Kernel extends ConsoleKernel}/**- * Run the dispatcher in the scheduler process, so a busy queue cannot delay it. Parallel runs- * are safe because each occurrence is claimed with an atomic update of next_run_at.+ * Run the manager from the scheduler, not from a queue worker. A busy queue could delay it+ * past the catch-up window, and then due backups and tasks would be skipped.*/- private function scheduleScheduledJobs(): void+ private function scheduleScheduledJobManager(): void{$this->scheduleInstance->command('scheduled:dispatch')->everyMinute()->onOneServer()+ ->withoutOverlapping(5)->runInBackground();}diff --git a/app/Jobs/ScheduledJobManager.php b/app/Jobs/ScheduledJobManager.phpindex 4815a853c9..8e4a5b26a8 100644--- a/app/Jobs/ScheduledJobManager.php+++ b/app/Jobs/ScheduledJobManager.php@@ -4,199 +4,438 @@ namespace App\Jobs;use App\Actions\Shared\DeleteScheduledVolumeBackup;use App\Models\ScheduledDatabaseBackup;-use App\Models\ScheduledJobDelivery;use App\Models\ScheduledTask;use App\Models\ScheduledVolumeBackup;use App\Models\ScheduledVolumeBackupExecution;use App\Models\Server;+use App\Models\ServerSetting;use App\Models\ServiceDatabase;+use App\Models\Subscription;+use App\Models\Team;use App\Services\ScheduledJobDeliveryService;-use Carbon\CarbonImmutable;+use Cron\CronExpression;+use Illuminate\Bus\Queueable;+use Illuminate\Contracts\Queue\ShouldQueue;use Illuminate\Database\Eloquent\Builder;-use Illuminate\Database\Eloquent\Model;use Illuminate\Database\Eloquent\Relations\MorphTo;+use Illuminate\Foundation\Bus\Dispatchable;+use Illuminate\Queue\InteractsWithQueue;+use Illuminate\Queue\Middleware\WithoutOverlapping;+use Illuminate\Queue\SerializesModels;+use Illuminate\Support\Carbon;use Illuminate\Support\Facades\Cache;-use Illuminate\Support\Facades\DB;use Illuminate\Support\Facades\Log;+use Illuminate\Support\Facades\Redis;-/**- * Dispatches the due scheduled backups, tasks, volume backups, and Docker cleanups.- *- * Each schedule stores its next due time in next_run_at. A run selects only due rows and claims- * each occurrence with an atomic update of next_run_at, so parallel runs on several nodes never- * dispatch the same occurrence twice. The `scheduled:dispatch` command runs it every minute.- */-class ScheduledJobManager+class ScheduledJobManager implements ShouldQueue{+ use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;+private const CHUNK_SIZE = 100;/**- * A task or Docker cleanup that is due longer than this is logged as missed and not run.- * Database and volume backups always run once, also when they are late.+ * The time when this job execution started.+ * Used to ensure all scheduled items are evaluated against the same point in time.*/- public const LATE_RUN_WINDOW_MINUTES = 10;+ private ?Carbon $executionTime = null;++ private int $dispatchedCount = 0;++ private int $skippedCount = 0;- private CarbonImmutable $now;+ /**+ * Create a new job instance.+ */+ public function __construct()+ {+ $this->onQueue(crons_queue());+ }++ /**+ * Get the middleware the job should pass through.+ */+ public function middleware(): array+ {+ // Self-healing: clear any stale lock before WithoutOverlapping tries to acquire it.+ // Stale locks (TTL = -1) can occur during upgrades, Redis restarts, or edge cases.+ // @see https://github.com/coollabsio/coolify/issues/8327+ self::clearStaleLockIfPresent();++ return [+ (new WithoutOverlapping('scheduled-job-manager'))+ ->expireAfter(90) // Lock expires after 90s to handle high-load environments with many tasks+ ->dontRelease(), // Don't re-queue on lock conflict+ ];+ }++ /**+ * Clear a stale WithoutOverlapping lock if it has no TTL (TTL = -1).+ *+ * This provides continuous self-healing since it runs every time the job is dispatched.+ * Stale locks permanently block all scheduled job executions with no user-visible error.+ */+ private static function clearStaleLockIfPresent(): void+ {+ try {+ $cachePrefix = config('cache.prefix', '');+ $lockKey = $cachePrefix.'laravel-queue-overlap:'.self::class.':scheduled-job-manager';- private ScheduledJobDeliveryService $deliveries;+ $ttl = Redis::connection('default')->ttl($lockKey);- /** @var array{dispatched: int, skipped: int, missed: int} */- private array $counts = ['dispatched' => 0, 'skipped' => 0, 'missed' => 0];+ if ($ttl === -1) {+ Redis::connection('default')->del($lockKey);+ Log::channel('scheduled')->warning('Cleared stale ScheduledJobManager lock', [+ 'lock_key' => $lockKey,+ ]);+ }+ } catch (\Throwable $e) {+ // Never let lock cleanup failure prevent the job from running+ Log::channel('scheduled-errors')->error('Failed to check/clear stale lock', [+ 'error' => $e->getMessage(),+ ]);+ }+ }public function handle(): void{- $this->now = CarbonImmutable::now();- $this->deliveries = app(ScheduledJobDeliveryService::class);- $this->counts = ['dispatched' => 0, 'skipped' => 0, 'missed' => 0];+ // Freeze the execution time at the start of the job+ $this->executionTime = Carbon::now();+ $this->dispatchedCount = 0;+ $this->skippedCount = 0;- // Read the current server state, not servers cached by an earlier run in this process.- Server::flushIdentityMap();+ $this->logStart();- Log::channel('scheduled')->info('ScheduledJobManager started', ['execution_time' => $this->now->toIso8601String()]);+ try {+ app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();+ } catch (\Throwable $e) {+ Log::channel('scheduled-errors')->error('Failed to recover stale enqueued occurrences', [+ 'error' => $e->getMessage(),+ ]);+ }- $this->runStep('stale enqueued occurrences', fn () => $this->deliveries->recoverStaleEnqueued());- $this->deliveries->publishPending();- $this->runStep('database backups', fn () => $this->processDatabaseBackups());- $this->runStep('scheduled tasks', fn () => $this->processScheduledTasks());- $this->runStep('volume backups', function () {- $this->recoverStoppedVolumeBackupContainers();- $this->processVolumeBackups();- });- $this->runStep('docker cleanups', fn () => $this->processDockerCleanups());+ app(ScheduledJobDeliveryService::class)->publishPending();- Log::channel('scheduled')->info('ScheduledJobManager completed', [- 'execution_time' => $this->now->toIso8601String(),- 'duration_ms' => $this->now->diffInMilliseconds(CarbonImmutable::now()),- ...$this->counts,- ]);+ // Process scheduled backups and tasks together so neither type starves the other.+ try {+ $this->processScheduledBackupsAndTasks();+ } catch (\Throwable $e) {+ Log::channel('scheduled-errors')->error('Failed to process scheduled backups and tasks', [+ 'error' => $e->getMessage(),+ 'trace' => $e->getTraceAsString(),+ ]);+ }- // The UI uses the heartbeat to show when the scheduler has stopped.- rescue(fn () => Cache::put('scheduled-job-manager:heartbeat', now()->toIso8601String(), 300), report: false);- }+ try {+ $this->recoverStoppedVolumeBackupContainers();+ $this->processScheduledVolumeBackups();+ } catch (\Throwable $e) {+ Log::channel('scheduled-errors')->error('Failed to process scheduled volume backups', [+ 'error' => $e->getMessage(),+ 'trace' => $e->getTraceAsString(),+ ]);+ }- private function runStep(string $name, callable $step): void- {+ // Process Docker cleanups - don't let failures stop the job managertry {- $step();+ $this->processDockerCleanups();} catch (\Throwable $e) {- Log::channel('scheduled-errors')->error("Failed to process {$name}", [+ Log::channel('scheduled-errors')->error('Failed to process docker cleanups', ['error' => $e->getMessage(),'trace' => $e->getTraceAsString(),]);}++ Log::channel('scheduled')->info('ScheduledJobManager completed', [+ 'execution_time' => $this->executionTime->toIso8601String(),+ 'duration_ms' => $this->executionTime->diffInMilliseconds(Carbon::now()),+ 'dispatched' => $this->dispatchedCount,+ 'skipped' => $this->skippedCount,+ 'host' => gethostname(),+ 'pid' => getmypid(),+ 'memory_mb' => round(memory_get_usage(true) / 1048576, 1),+ 'memory_peak_mb' => round(memory_get_peak_usage(true) / 1048576, 1),+ 'open_occurrences_over_15_minutes' => rescue(fn () => app(ScheduledJobDeliveryService::class)->staleOpenOccurrenceCounts(), [], report: false),+ ]);++ // Write heartbeat so the UI can detect when the scheduler has stopped+ try {+ Cache::put('scheduled-job-manager:heartbeat', now()->toIso8601String(), 300);+ } catch (\Throwable) {+ // Non-critical; don't let heartbeat failure affect the job+ }}- private function processDatabaseBackups(): void+ /**+ * Log which worker runs the manager and how late it starts. The manager only catches up+ * occurrences from the last CATCH_UP_WINDOW_MINUTES, so a larger delay skips them silently.+ */+ private function logStart(): void{- $this->due(ScheduledDatabaseBackup::query())- ->with([- 'team.subscription',- 'database' => fn (MorphTo $morphTo) => $morphTo->morphWith([- ServiceDatabase::class => ['service.destination.server.settings', 'service.destination.server.team.subscription'],- ...array_fill_keys(STANDALONE_DATABASE_MODELS, ['destination.server.settings', 'destination.server.team.subscription']),- ]),- ])- ->chunkById(self::CHUNK_SIZE, fn ($backups) => $backups->each(fn (ScheduledDatabaseBackup $backup) => $this->processItem(- 'backup',- ['backup_id' => $backup->id, 'database_id' => $backup->database_id, 'database_type' => $backup->database_type, 'team_id' => $backup->team_id],- fn () => $this->processDatabaseBackup($backup),- )));- }-- private function processDatabaseBackup(ScheduledDatabaseBackup $backup): array+ $previousStartedAt = rescue(fn () => Cache::get('scheduled-job-manager:last-started-at'), report: false);+ rescue(fn () => Cache::put('scheduled-job-manager:last-started-at', $this->executionTime->toIso8601String(), 86400), report: false);+ $queuedAt = data_get($this->job?->payload(), 'createdAt');++ $context = [+ 'execution_time' => $this->executionTime->toIso8601String(),+ 'host' => gethostname(),+ 'pid' => getmypid(),+ 'queue_wait_seconds' => $queuedAt ? $this->executionTime->timestamp - (int) $queuedAt : null,+ 'seconds_since_previous_start' => $previousStartedAt ? (int) Carbon::parse($previousStartedAt)->diffInSeconds($this->executionTime) : null,+ 'memory_mb' => round(memory_get_usage(true) / 1048576, 1),+ ];++ Log::channel('scheduled')->info('ScheduledJobManager started', $context);++ $catchUpWindowSeconds = ScheduledJobDeliveryService::CATCH_UP_WINDOW_MINUTES * 60;+ if ($context['queue_wait_seconds'] > $catchUpWindowSeconds || $context['seconds_since_previous_start'] > $catchUpWindowSeconds) {+ Log::channel('scheduled-errors')->warning('ScheduledJobManager started late; occurrences due before the catch-up window were not run', $context);+ }+ }++ private function processScheduledBackupsAndTasks(): void{- $server = $backup->server();+ $lastBackupId = null;+ $lastTaskId = null;- if (blank($backup->database) || blank($server)) {- $backup->delete();+ do {+ $backups = $this->scheduledBackupQuery($lastBackupId)->get();+ $tasks = $this->scheduledTaskQuery($lastTaskId)->get();++ if ($backups->isNotEmpty()) {+ $lastBackupId = $backups->last()->id;+ }++ if ($tasks->isNotEmpty()) {+ $lastTaskId = $tasks->last()->id;+ }++ $this->processInterleavedDueSchedules(+ $this->dueScheduledBackups($backups),+ $this->dueScheduledTasks($tasks),+ );+ } while ($backups->isNotEmpty() || $tasks->isNotEmpty());+ }++ /**+ * @param array<int, array{backup: ScheduledDatabaseBackup, server: Server}> $dueBackups+ * @param array<int, array{task: ScheduledTask, server: Server}> $dueTasks+ */+ private function processInterleavedDueSchedules(array $dueBackups, array $dueTasks): void+ {+ $maxCount = max(count($dueBackups), count($dueTasks));- return ['deleted' => blank($backup->database) ? 'database_deleted' : 'server_deleted'];+ for ($index = 0; $index < $maxCount; $index++) {+ if (isset($dueBackups[$index])) {+ $this->processScheduledBackup($dueBackups[$index]['backup'], $dueBackups[$index]['server']);+ }++ if (isset($dueTasks[$index])) {+ $this->processScheduledTask($dueTasks[$index]['task'], $dueTasks[$index]['server']);+ }}+ }- return $this->runOccurrence(- schedule: $backup,- frequency: $backup->frequency,- server: $server,- runLate: true,- skipReason: fn () => $this->serverSkipReason($server),- delivery: ['schedule_key' => "scheduled-backup:{$backup->id}", 'job_type' => 'database-backup', 'resource_id' => $backup->id],- );+ /**+ * Load the server chain of each backup with the chunk. Otherwise $backup->server()+ * queries the destination, service, server, and settings once for each backup.+ */+ private function scheduledBackupQuery(?int $lastBackupId): Builder+ {+ return ScheduledDatabaseBackup::with([+ 'team.subscription',+ 'database' => fn (MorphTo $morphTo) => $morphTo->morphWith([+ ServiceDatabase::class => ['service.destination.server.settings', 'service.destination.server.team.subscription'],+ ...array_fill_keys(STANDALONE_DATABASE_MODELS, ['destination.server.settings', 'destination.server.team.subscription']),+ ]),+ ])+ ->where('enabled', true)+ ->when($lastBackupId !== null, fn (Builder $query) => $query->where('id', '>', $lastBackupId))+ ->orderBy('id')+ ->limit(self::CHUNK_SIZE);}- private function processScheduledTasks(): void+ private function scheduledTaskQuery(?int $lastTaskId): Builder{- $this->due(ScheduledTask::query())- ->with([- 'service.destination.server.settings',- 'service.destination.server.team.subscription',- 'application.destination.server.settings',- 'application.destination.server.team.subscription',- ])- ->chunkById(self::CHUNK_SIZE, fn ($tasks) => $tasks->each(fn (ScheduledTask $task) => $this->processItem(- 'task',- ['task_id' => $task->id, 'task_name' => $task->name, 'team_id' => $task->team_id],- fn () => $this->processScheduledTask($task),- )));- }-- private function processScheduledTask(ScheduledTask $task): array+ return ScheduledTask::with([+ 'service.destination.server.settings',+ 'service.destination.server.team.subscription',+ 'application.destination.server.settings',+ 'application.destination.server.team.subscription',+ ])+ ->where('enabled', true)+ ->when($lastTaskId !== null, fn (Builder $query) => $query->where('id', '>', $lastTaskId))+ ->orderBy('id')+ ->limit(self::CHUNK_SIZE);+ }++ /**+ * @param iterable<ScheduledDatabaseBackup> $backups+ * @return array<int, array{backup: ScheduledDatabaseBackup, server: Server}>+ */+ private function dueScheduledBackups(iterable $backups): array{- $server = $task->server();+ $dueBackups = [];- if (blank($server) || (! $task->service && ! $task->application)) {- $task->delete();+ foreach ($backups as $backup) {+ try {+ $server = $backup->server();- return ['deleted' => blank($server) ? 'server_deleted' : 'resource_deleted'];+ if (blank(data_get($backup, 'database')) || blank($server)) {+ $this->processScheduledBackup($backup, $server);++ continue;+ }++ if ($this->isDueCandidateBeforeExpensiveChecks($backup->frequency, $server)) {+ $dueBackups[] = [+ 'backup' => $backup,+ 'server' => $server,+ ];+ }+ } catch (\Throwable $e) {+ Log::channel('scheduled-errors')->error('Error prechecking backup', [+ 'backup_id' => $backup->id,+ 'error' => $e->getMessage(),+ ]);+ }}- return $this->runOccurrence(- schedule: $task,- frequency: $task->frequency,- server: $server,- runLate: false,- skipReason: fn () => $this->serverSkipReason($server) ?? $this->taskResourceSkipReason($task),- delivery: ['schedule_key' => "scheduled-task:{$task->id}", 'job_type' => 'scheduled-task', 'resource_id' => $task->id],- );+ return $dueBackups;}- private function processVolumeBackups(): void+ /**+ * @param iterable<ScheduledTask> $tasks+ * @return array<int, array{task: ScheduledTask, server: Server}>+ */+ private function dueScheduledTasks(iterable $tasks): array{- $this->due(ScheduledVolumeBackup::query())- ->with(['backupable', 'team.subscription'])- ->chunkById(self::CHUNK_SIZE, fn ($backups) => $backups->each(fn (ScheduledVolumeBackup $backup) => $this->processItem(- 'volume_backup',- ['backup_id' => $backup->id, 'backupable_type' => $backup->backupable_type, 'backupable_id' => $backup->backupable_id, 'team_id' => $backup->team_id],- fn () => $this->processVolumeBackup($backup),- )));+ $dueTasks = [];++ foreach ($tasks as $task) {+ try {+ $server = $task->server();++ if (blank($server) || (! $task->service && ! $task->application)) {+ $this->processScheduledTask($task, $server);++ continue;+ }++ if ($this->isDueCandidateBeforeExpensiveChecks($task->frequency, $server)) {+ $dueTasks[] = [+ 'task' => $task,+ 'server' => $server,+ ];+ }+ } catch (\Throwable $e) {+ Log::channel('scheduled-errors')->error('Error prechecking task', [+ 'task_id' => $task->id,+ 'error' => $e->getMessage(),+ ]);+ }+ }++ return $dueTasks;}- private function processVolumeBackup(ScheduledVolumeBackup $backup): array+ private function processScheduledBackup(ScheduledDatabaseBackup $backup, ?Server $precheckedServer = null): void{- $server = $backup->server();+ try {+ $server = $precheckedServer ?? $backup->server();+ $skipReason = $this->getBackupSkipReason($backup, $server);+ if ($skipReason !== null) {+ if ($server === null || $this->recordSkippedOccurrence($backup->frequency, $server, "scheduled-backup:{$backup->id}")) {+ $this->skippedCount++;+ $this->logBackupSkip($backup, $skipReason, $server);+ }++ return;+ }- // Wait without claiming: the backup runs late when the recovery is done.- if ($backup->executions()->where(fn (Builder $query) => $query->where('stop_recovery_pending', true)->orWhere('s3_cleanup_pending', true))->exists()) {- return ['waiting' => 'container_recovery_pending'];+ if ($this->dispatchOccurrence(+ $backup->frequency,+ $server,+ "scheduled-backup:{$backup->id}",+ 'database-backup',+ $backup->id,+ )) {+ $this->dispatchedCount++;+ Log::channel('scheduled')->info('Backup dispatched', [+ 'backup_id' => $backup->id,+ 'database_id' => $backup->database_id,+ 'database_type' => $backup->database_type,+ 'team_id' => $backup->team_id ?? null,+ 'server_id' => $server->id,+ ]);+ }+ } catch (\Throwable $e) {+ Log::channel('scheduled-errors')->error('Error processing backup', [+ 'backup_id' => $backup->id,+ 'error' => $e->getMessage(),+ ]);}+ }++ private function processScheduledTask(ScheduledTask $task, ?Server $precheckedServer = null): void+ {+ try {+ $server = $precheckedServer ?? $task->server();+ $criticalSkip = $this->getTaskCriticalSkipReason($task, $server);+ if ($criticalSkip !== null) {+ if ($server === null || $this->recordSkippedOccurrence($task->frequency, $server, "scheduled-task:{$task->id}")) {+ $this->skippedCount++;+ $this->logTaskSkip($task, $criticalSkip, $server);+ }- if (! $backup->backupable) {- DeleteScheduledVolumeBackup::run($backup, $server);+ return;+ }- return ['deleted' => 'resource_deleted'];- }+ $runtimeSkip = $this->getTaskRuntimeSkipReason($task);+ if ($runtimeSkip !== null) {+ if ($this->recordSkippedOccurrence($task->frequency, $server, "scheduled-task:{$task->id}")) {+ $this->skippedCount++;+ $this->logTaskSkip($task, $runtimeSkip, $server);+ }- if (! $server) {- return ['waiting' => 'server_missing'];+ return;+ }++ if (! $this->dispatchOccurrence(+ $task->frequency,+ $server,+ "scheduled-task:{$task->id}",+ 'scheduled-task',+ $task->id,+ )) {++ return;+ }++ $this->dispatchedCount++;+ Log::channel('scheduled')->info('Task dispatched', [+ 'task_id' => $task->id,+ 'task_name' => $task->name,+ 'team_id' => $server->team_id,+ 'server_id' => $server->id,+ ]);+ } catch (\Throwable $e) {+ Log::channel('scheduled-errors')->error('Error processing task', [+ 'task_id' => $task->id,+ 'error' => $e->getMessage(),+ ]);}+ }- return $this->runOccurrence(- schedule: $backup,- frequency: $backup->frequency,- server: $server,- runLate: true,- skipReason: fn () => $this->serverSkipReason($server),- delivery: ['schedule_key' => "scheduled-volume-backup:{$backup->id}", 'job_type' => 'volume-backup', 'resource_id' => $backup->id],- );+ private function processScheduledVolumeBackups(): void+ {+ ScheduledVolumeBackup::query()+ ->with(['backupable', 'team.subscription'])+ ->where('enabled', true)+ ->chunkById(self::CHUNK_SIZE, function ($backups): void {+ foreach ($backups as $backup) {+ $this->processScheduledVolumeBackup($backup);+ }+ });}private function recoverStoppedVolumeBackupContainers(): void@@ -212,241 +451,367 @@ class ScheduledJobManager});}- private function processDockerCleanups(): void+ private function processScheduledVolumeBackup(ScheduledVolumeBackup $backup): void{- $query = Server::query()- ->with('settings')- ->whereNotNull('ip')- ->where('ip', '!=', '')- ->whereNotIn('ip', Server::PLACEHOLDER_IPS)- ->whereHas('settings', fn (Builder $query) => $query- ->whereNull('docker_cleanup_next_run_at')- ->orWhere('docker_cleanup_next_run_at', '<=', $this->now));+ try {+ $server = $backup->server();++ if ($backup->executions()+ ->where(fn (Builder $query) => $query+ ->where('stop_recovery_pending', true)+ ->orWhere('s3_cleanup_pending', true))+ ->exists()) {+ $this->skippedCount++;+ $this->logSkip('volume_backup', 'container_recovery_pending', [+ 'backup_id' => $backup->id,+ 'team_id' => $backup->team_id,+ ]);++ return;+ }- if (isCloud()) {- $query->with('team.subscription')->where(fn (Builder $query) => $query- ->where('team_id', 0)- ->orWhereRelation('team.subscription', 'stripe_invoice_paid', true));- }+ if (! $backup->backupable) {+ DeleteScheduledVolumeBackup::run($backup, $server);+ $this->skippedCount++;+ $this->logSkip('volume_backup', 'resource_deleted', [+ 'backup_id' => $backup->id,+ 'team_id' => $backup->team_id,+ ]);- $query->chunkById(self::CHUNK_SIZE, fn ($servers) => $servers->each(fn (Server $server) => $this->processItem(- 'docker_cleanup',- ['server_id' => $server->id, 'server_name' => $server->name, 'team_id' => $server->team_id],- fn () => $this->runOccurrence(- schedule: $server->settings,- frequency: $server->settings->docker_cleanup_frequency ?? '0 0 * * *',- server: $server,- runLate: false,- skipReason: fn () => $this->serverSkipReason($server),- delivery: [- 'schedule_key' => "docker-cleanup:{$server->id}",- 'job_type' => 'docker-cleanup',- 'resource_id' => $server->id,- 'payload' => [- 'delete_unused_volumes' => $server->settings->delete_unused_volumes,- 'delete_unused_networks' => $server->settings->delete_unused_networks,- ],- ],- column: 'docker_cleanup_next_run_at',- ),- )));- }+ return;+ }- /**- * Enabled schedules that are due, or that have no next run yet.- */- private function due(Builder $query): Builder- {- return $query- ->where('enabled', true)- ->where(fn (Builder $query) => $query->whereNull('next_run_at')->orWhere('next_run_at', '<=', $this->now));- }+ if (! $server) {+ $this->skippedCount++;+ $this->logSkip('volume_backup', 'server_missing', [+ 'backup_id' => $backup->id,+ 'team_id' => $backup->team_id,+ ]);- /**- * Claim the due occurrence of a schedule and dispatch it, skip it, or log it as missed.- *- * @param array{schedule_key: string, job_type: string, resource_id: int, payload?: array<string, mixed>} $delivery- * @return array<string, mixed> The outcome, for the log.- */- private function runOccurrence(- Model $schedule,- string $frequency,- Server $server,- bool $runLate,- callable $skipReason,- array $delivery,- string $column = 'next_run_at',- ): array {- $timezone = data_get($server, 'settings.server_timezone');- $stored = $schedule->getRawOriginal($column);-- if ($stored === null) {- // First run after the upgrade, a timezone change, or a move to another server.- $first = next_cron_run_at($frequency, $timezone, $this->now, includeCurrentMinute: true);- if ($first === null) {- return $this->invalidFrequency($schedule, $column, null, $frequency, $delivery['schedule_key']);+ return;}- if (! $this->advance($schedule, $column, null, $first)) {- return [];++ if (! $this->isDueCandidateBeforeExpensiveChecks($backup->frequency, $server)) {+ return;}- if ($first->gt($this->now)) {- return [];++ if (! $server->isFunctional()) {+ if ($this->recordSkippedOccurrence($backup->frequency, $server, "scheduled-volume-backup:{$backup->id}")) {+ $this->skippedCount++;+ $this->logSkip('volume_backup', 'server_not_functional', [+ 'backup_id' => $backup->id,+ 'team_id' => $backup->team_id,+ ...$this->serverStateContext($server),+ ]);+ }++ return;}- $stored = $first;++ if (isCloud() && $backup->team_id !== 0 && ! data_get($backup, 'team.subscription.stripe_invoice_paid', false)) {+ if ($this->recordSkippedOccurrence($backup->frequency, $server, "scheduled-volume-backup:{$backup->id}")) {+ $this->skippedCount++;+ $this->logSkip('volume_backup', 'subscription_unpaid', [+ 'backup_id' => $backup->id,+ 'team_id' => $backup->team_id,+ ...$this->serverStateContext($server),+ ]);+ }++ return;+ }++ if ($this->dispatchOccurrence(+ $backup->frequency,+ $server,+ "scheduled-volume-backup:{$backup->id}",+ 'volume-backup',+ $backup->id,+ )) {+ $this->dispatchedCount++;+ Log::channel('scheduled')->info('Volume backup dispatched', [+ 'backup_id' => $backup->id,+ 'backupable_type' => $backup->backupable_type,+ 'backupable_id' => $backup->backupable_id,+ 'team_id' => $backup->team_id,+ 'server_id' => $server->id,+ ]);+ }+ } catch (\Throwable $e) {+ Log::channel('scheduled-errors')->error('Error processing volume backup', [+ 'backup_id' => $backup->id,+ 'error' => $e->getMessage(),+ ]);}+ }- $dueAt = CarbonImmutable::instance($schedule->getAttribute($column))->utc();- $next = next_cron_run_at($frequency, $timezone, $this->now);- if ($next === null) {- return $this->invalidFrequency($schedule, $column, $stored, $frequency, $delivery['schedule_key']);+ private function getBackupSkipReason(ScheduledDatabaseBackup $backup, ?Server $server): ?string+ {+ if (blank(data_get($backup, 'database'))) {+ $backup->delete();++ return 'database_deleted';}- // When the clocks go back, a local time occurs two times. Run it only once.- if ($this->sameLocalTime($next, $dueAt, $timezone)) {- $next = next_cron_run_at($frequency, $timezone, $next) ?? $next;++ if (blank($server)) {+ $backup->delete();++ return 'server_deleted';}- $minutesLate = (int) $dueAt->diffInMinutes($this->now);- if (! $runLate && $minutesLate > self::LATE_RUN_WINDOW_MINUTES) {- $outcome = ['missed' => $dueAt->toIso8601String(), 'minutes_late' => $minutesLate];- } elseif (($reason = $skipReason()) !== null) {- $outcome = ['skipped' => $reason];- } else {- $outcome = ['dispatched' => $dueAt->toIso8601String()];+ if ($server->isFunctional() === false) {+ return 'server_not_functional';}- // Claim and record the occurrence together, so a crash cannot claim it without a delivery.- $occurrence = DB::transaction(function () use ($schedule, $column, $stored, $next, $outcome, $delivery, $dueAt): ScheduledJobDelivery|bool {- if (! $this->advance($schedule, $column, $stored, $next)) {- return false;- }+ if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) {+ return 'subscription_unpaid';+ }- if (! isset($outcome['dispatched'])) {- return true;- }+ return null;+ }- return $this->deliveries->create($delivery['schedule_key'], $dueAt, $delivery['job_type'], $delivery['resource_id'], $delivery['payload'] ?? []) ?? true;- });+ private function getTaskCriticalSkipReason(ScheduledTask $task, ?Server $server): ?string+ {+ if (blank($server)) {+ $task->delete();- if ($occurrence === false) {- return [];+ return 'server_deleted';}- if ($occurrence instanceof ScheduledJobDelivery) {- $this->deliveries->publish($occurrence);+ if ($server->isFunctional() === false) {+ return 'server_not_functional';}- return [...$outcome, 'server_id' => $server->id, 'next_run_at' => $next->toIso8601String()];+ if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) {+ return 'subscription_unpaid';+ }++ if (! $task->service && ! $task->application) {+ $task->delete();++ return 'resource_deleted';+ }++ return null;}- /**- * Park a schedule whose frequency does not parse: set next_run_at to null, which the UI, the API,- * and `scheduled:diagnostics` show as an invalid frequency, and log it once a day instead of- * every minute. Saving a valid frequency calculates next_run_at again (HasNextRunAt, ServerSetting).- *- * @return array<string, mixed>- */- private function invalidFrequency(Model $schedule, string $column, mixed $stored, string $frequency, string $scheduleKey): array+ private function getTaskRuntimeSkipReason(ScheduledTask $task): ?string{- if ($stored !== null) {- $this->advance($schedule, $column, $stored, null);+ if ($task->application && str($task->application->status)->contains('running') === false) {+ return 'application_not_running';}- $firstReportToday = rescue(- fn () => Cache::add("scheduled-job-manager:invalid-frequency:{$scheduleKey}:".md5($frequency), true, now()->addDay()),- true,- report: false,- );+ if ($task->service && str($task->service->status)->contains('running') === false) {+ return 'service_not_running';+ }- return $firstReportToday ? ['invalid_frequency' => $frequency] : [];+ return null;}- /**- * Move next_run_at from the stored value to $to. Only one process can win this update.- */- private function advance(Model $schedule, string $column, mixed $from, ?CarbonImmutable $to): bool+ private function processDockerCleanups(): void{- $updated = $schedule->newQuery()- ->whereKey($schedule->getKey())- ->when($from === null, fn (Builder $query) => $query->whereNull($column), fn (Builder $query) => $query->where($column, $from))- ->toBase()- ->update([$column => $to]);-- if ($updated === 1) {- $schedule->setAttribute($column, $to)->syncOriginalAttribute($column);- }+ $this->getServersForCleanupQuery()+ ->chunkById(self::CHUNK_SIZE, function ($servers): void {+ foreach ($servers as $server) {+ $this->processDockerCleanup($server);+ }+ });+ }- return $updated === 1;+ private function processDockerCleanup(Server $server): void+ {+ try {+ $frequency = data_get($server->settings, 'docker_cleanup_frequency', '0 * * * *');+ if (! $this->isDueCandidateBeforeExpensiveChecks($frequency, $server)) {+ return;+ }++ $skipReason = $this->getDockerCleanupSkipReason($server);+ if ($skipReason !== null) {+ if ($this->recordSkippedOccurrence($frequency, $server, "docker-cleanup:{$server->id}")) {+ $this->skippedCount++;+ $this->logSkip('docker_cleanup', $skipReason, [+ 'server_id' => $server->id,+ 'server_name' => $server->name,+ 'team_id' => $server->team_id,+ ]);+ }++ return;+ }++ if ($this->dispatchOccurrence(+ $frequency,+ $server,+ "docker-cleanup:{$server->id}",+ 'docker-cleanup',+ $server->id,+ [+ 'delete_unused_volumes' => $server->settings->delete_unused_volumes,+ 'delete_unused_networks' => $server->settings->delete_unused_networks,+ ],+ )) {+ $this->dispatchedCount++;+ Log::channel('scheduled')->info('Docker cleanup dispatched', [+ 'server_id' => $server->id,+ 'server_name' => $server->name,+ 'team_id' => $server->team_id,+ ]);+ }+ } catch (\Throwable $e) {+ Log::channel('scheduled-errors')->error('Error processing docker cleanup', [+ 'server_id' => $server->id,+ 'server_name' => $server->name,+ 'error' => $e->getMessage(),+ ]);+ }}- private function sameLocalTime(CarbonImmutable $first, CarbonImmutable $second, ?string $timezone): bool+ private function getServersForCleanupQuery(): Builder{- $timezone = filled($timezone) && validate_timezone($timezone) ? $timezone : config('app.timezone');+ $query = Server::with('settings')+ ->whereNotNull('ip')+ ->where('ip', '!=', '')+ ->whereNotIn('ip', Server::PLACEHOLDER_IPS);++ if (isCloud()) {+ $query+ ->with('team.subscription')+ ->where(function (Builder $query): void {+ $query+ ->where('team_id', 0)+ ->orWhereRelation('team.subscription', 'stripe_invoice_paid', true);+ });+ }- return $first->setTimezone($timezone)->format('Y-m-d H:i') === $second->setTimezone($timezone)->format('Y-m-d H:i');+ return $query;}- private function serverSkipReason(Server $server): ?string+ private function getDockerCleanupSkipReason(Server $server): ?string{if (! $server->isFunctional()) {return 'server_not_functional';}- if (isCloud() && $server->team_id !== 0 && data_get($server->team?->subscription, 'stripe_invoice_paid', false) === false) {- return 'subscription_unpaid';+ // In cloud, check subscription status (except team 0)+ if (isCloud() && $server->team_id !== 0) {+ if (data_get($server->team->subscription, 'stripe_invoice_paid', false) === false) {+ return 'subscription_unpaid';+ }}return null;}- private function taskResourceSkipReason(ScheduledTask $task): ?string+ private function logSkip(string $type, string $reason, array $context = []): void{- if ($task->application && ! str($task->application->status)->contains('running')) {- return 'application_not_running';- }+ Log::channel('scheduled')->info(ucfirst(str_replace('_', ' ', $type)).' skipped', array_merge([+ 'type' => $type,+ 'skip_reason' => $reason,+ 'execution_time' => $this->executionTime?->toIso8601String(),+ ], $context));+ }- if ($task->service && ! str($task->service->status)->contains('running')) {- return 'service_not_running';- }+ private function dispatchOccurrence(+ string $frequency,+ Server $server,+ string $scheduleKey,+ string $jobType,+ int $resourceId,+ array $payload = [],+ ): bool {+ return app(ScheduledJobDeliveryService::class)->recordAndPublish(+ $scheduleKey,+ $frequency,+ $this->serverTimezone($server),+ $jobType,+ $resourceId,+ $payload,+ $this->executionTime,+ );+ }- return null;+ private function recordSkippedOccurrence(string $frequency, Server $server, string $scheduleKey): bool+ {+ return app(ScheduledJobDeliveryService::class)->recordSkipped(+ $scheduleKey,+ $frequency,+ $this->serverTimezone($server),+ $this->executionTime,+ );}- /**- * Process one schedule and log its outcome. An error in one schedule never stops the others.- *- * @param array<string, mixed> $context- */- private function processItem(string $type, array $context, callable $process): void+ private function isDueCandidateBeforeExpensiveChecks(string $frequency, Server $server): bool{- try {- $outcome = $process();- } catch (\Throwable $e) {- Log::channel('scheduled-errors')->error("Error processing {$type}", [...$context, 'error' => $e->getMessage()]);+ $cron = new CronExpression($this->normalizeFrequency($frequency));+ $executionTime = ($this->executionTime ?? Carbon::now())->copy()->setTimezone($this->serverTimezone($server));+ $previousDue = Carbon::instance($cron->getPreviousRunDate($executionTime, allowCurrentDate: true));- return;- }+ return $previousDue->gte(+ $executionTime->copy()->subMinutes(ScheduledJobDeliveryService::CATCH_UP_WINDOW_MINUTES)+ );+ }- $label = ucfirst(str_replace('_', ' ', $type));- $context = [...$context, ...$outcome, 'execution_time' => $this->now->toIso8601String()];+ private function normalizeFrequency(string $frequency): string+ {+ return VALID_CRON_STRINGS[$frequency] ?? $frequency;+ }++ private function serverTimezone(Server $server): string+ {+ $timezone = data_get($server->settings, 'server_timezone', config('app.timezone'));++ return validate_timezone($timezone) ? $timezone : config('app.timezone');+ }++ private function logBackupSkip(ScheduledDatabaseBackup $backup, string $reason, ?Server $server): void+ {+ $this->logSkip('backup', $reason, [+ 'backup_id' => $backup->id,+ 'database_id' => $backup->database_id,+ 'database_type' => $backup->database_type,+ 'team_id' => $backup->team_id ?? null,+ ...$this->serverStateContext($server),+ ]);+ }- match (true) {- isset($outcome['dispatched']) => $this->log('dispatched', 'info', "{$label} dispatched", $context),- isset($outcome['skipped']) => $this->log('skipped', 'info', "{$label} skipped", ['skip_reason' => $outcome['skipped'], ...$context]),- isset($outcome['deleted']) => $this->log('skipped', 'info', "{$label} skipped", ['skip_reason' => $outcome['deleted'], ...$context]),- isset($outcome['waiting']) => $this->log('skipped', 'info', "{$label} skipped", ['skip_reason' => $outcome['waiting'], ...$context]),- isset($outcome['missed']) => $this->log('missed', 'warning', "{$label} missed", $context),- isset($outcome['invalid_frequency']) => Log::channel('scheduled-errors')->warning("{$label} has an invalid frequency", $context),- default => null,- };+ private function logTaskSkip(ScheduledTask $task, string $reason, ?Server $server): void+ {+ $this->logSkip('task', $reason, [+ 'task_id' => $task->id,+ 'task_name' => $task->name,+ 'team_id' => $server?->team_id,+ ...$this->serverStateContext($server),+ ]);}/**- * @param 'dispatched'|'skipped'|'missed' $counter- * @param array<string, mixed> $context+ * Compare the server state that decided the skip with the current database state.+ * A difference shows that the worker used stale server data.+ *+ * @return array<string, mixed>*/- private function log(string $counter, string $level, string $message, array $context): void+ private function serverStateContext(?Server $server): array{- $this->counts[$counter]++;- Log::channel('scheduled')->log($level, $message, $context);+ if ($server === null) {+ return [];+ }++ $settings = ServerSetting::query()+ ->where('server_id', $server->id)+ ->first(['is_reachable', 'is_usable', 'force_disabled']);++ return [+ 'server_id' => $server->id,+ 'server_ip_is_placeholder' => $server->hasPlaceholderIp(),+ 'used_is_reachable' => data_get($server->settings, 'is_reachable'),+ 'used_is_usable' => data_get($server->settings, 'is_usable'),+ 'used_force_disabled' => data_get($server->settings, 'force_disabled'),+ 'used_invoice_paid' => data_get($server->team?->subscription, 'stripe_invoice_paid'),+ 'db_is_reachable' => $settings?->is_reachable,+ 'db_is_usable' => $settings?->is_usable,+ 'db_force_disabled' => $settings?->force_disabled,+ 'db_invoice_paid' => Subscription::query()->where('team_id', $server->team_id)->value('stripe_invoice_paid'),+ ];}}diff --git a/app/Livewire/Project/Shared/Destination.php b/app/Livewire/Project/Shared/Destination.phpindex 24c3b51090..1e5a6fb2d4 100644--- a/app/Livewire/Project/Shared/Destination.php+++ b/app/Livewire/Project/Shared/Destination.php@@ -8,7 +8,6 @@ use App\Events\ApplicationStatusChanged;use App\Models\Application;use App\Models\Server;use App\Models\StandaloneDocker;-use App\Services\ScheduleNextRunRecalculator;use App\Traits\ListensToTeamChannel;use Illuminate\Foundation\Auth\Access\AuthorizesRequests;use Illuminate\Support\Collection;@@ -139,7 +138,6 @@ class Destination extends Component->wherePivot('server_id', $server->id)->detach($network->id);$this->resource->additional_networks()->attach($mainDestination->id, ['server_id' => $mainDestination->server->id]);- app(ScheduleNextRunRecalculator::class)->forResource($this->resource);});$this->resource->refresh();$this->refreshServers();diff --git a/app/Livewire/Project/Shared/ScheduledTask/Show.php b/app/Livewire/Project/Shared/ScheduledTask/Show.phpindex 2428ff4e69..3bfe0700d9 100644--- a/app/Livewire/Project/Shared/ScheduledTask/Show.php+++ b/app/Livewire/Project/Shared/ScheduledTask/Show.php@@ -7,7 +7,6 @@ use App\Models\Application;use App\Models\ScheduledTask;use App\Models\Service;use Illuminate\Foundation\Auth\Access\AuthorizesRequests;-use Livewire\Attributes\Computed;use Livewire\Attributes\Locked;use Livewire\Attributes\Validate;use Livewire\Component;@@ -88,15 +87,6 @@ class Show extends Component}}- /**- * Timezone of the server the task runs on, used to display its next run.- */- #[Computed]- public function serverTimezone(): ?string- {- return $this->task->scheduleTimezone();- }-private function syncData(bool $toModel = false): void{if ($toModel) {diff --git a/app/Livewire/Server/DockerCleanup.php b/app/Livewire/Server/DockerCleanup.phpindex 6e6d1480b2..d0a8d8ca9d 100644--- a/app/Livewire/Server/DockerCleanup.php+++ b/app/Livewire/Server/DockerCleanup.php@@ -5,7 +5,9 @@ namespace App\Livewire\Server;use App\Jobs\DockerCleanupJob;use App\Models\DockerCleanupExecution;use App\Models\Server;+use Cron\CronExpression;use Illuminate\Foundation\Auth\Access\AuthorizesRequests;+use Illuminate\Support\Carbon;use Illuminate\Support\Facades\Cache;use Livewire\Attributes\Computed;use Livewire\Attributes\Validate;@@ -37,21 +39,35 @@ class DockerCleanup extends Component#[Validate('boolean')]public bool $disableApplicationImageRetention = false;- /**- * The scheduled cleanup is stale when its next run is more than 10 minutes overdue,- * meaning the scheduled job dispatcher did not run it.- */#[Computed]public function isCleanupStale(): bool{- // The dispatcher does not run cleanups on servers without a real IP address.- if (blank($this->server->ip) || $this->server->hasPlaceholderIp()) {- return false;- }+ try {+ $lastExecution = DockerCleanupExecution::where('server_id', $this->server->id)+ ->orderBy('created_at', 'desc')+ ->first();- $nextRunAt = $this->server->settings->docker_cleanup_next_run_at;+ if (! $lastExecution) {+ return false;+ }++ $frequency = $this->server->settings->docker_cleanup_frequency ?? '0 0 * * *';+ if (isset(VALID_CRON_STRINGS[$frequency])) {+ $frequency = VALID_CRON_STRINGS[$frequency];+ }- return $nextRunAt !== null && $nextRunAt->lt(now()->subMinutes(10));+ $cron = new CronExpression($frequency);+ $now = Carbon::now();+ $nextRun = Carbon::parse($cron->getNextRunDate($now));+ $afterThat = Carbon::parse($cron->getNextRunDate($nextRun));+ $intervalMinutes = $nextRun->diffInMinutes($afterThat);++ $threshold = max($intervalMinutes * 2, 10);++ return Carbon::parse($lastExecution->created_at)->diffInMinutes($now) > $threshold;+ } catch (\Throwable) {+ return false;+ }}#[Computed]diff --git a/app/Models/ScheduledDatabaseBackup.php b/app/Models/ScheduledDatabaseBackup.phpindex b44d1b16f4..53ae084349 100644--- a/app/Models/ScheduledDatabaseBackup.php+++ b/app/Models/ScheduledDatabaseBackup.php@@ -2,15 +2,12 @@namespace App\Models;-use App\Traits\HasNextRunAt;use Illuminate\Database\Eloquent\Relations\HasMany;use Illuminate\Database\Eloquent\Relations\HasOne;use Illuminate\Database\Eloquent\Relations\MorphTo;class ScheduledDatabaseBackup extends BaseModel{- use HasNextRunAt;-protected function casts(): array{return [@@ -19,7 +16,6 @@ class ScheduledDatabaseBackup extends BaseModel'database_backup_retention_max_storage_s3' => 'float','missing_backup_notification_days' => 'integer','last_execution_at' => 'datetime',- 'next_run_at' => 'datetime',];}diff --git a/app/Models/ScheduledJobState.php b/app/Models/ScheduledJobState.phpnew file mode 100644index 0000000000..e267cd6a1d--- /dev/null+++ b/app/Models/ScheduledJobState.php@@ -0,0 +1,18 @@+<?php++namespace App\Models;++class ScheduledJobState extends BaseModel+{+ protected $fillable = [+ 'schedule_key',+ 'last_scheduled_for',+ ];++ protected function casts(): array+ {+ return [+ 'last_scheduled_for' => 'immutable_datetime',+ ];+ }+}diff --git a/app/Models/ScheduledTask.php b/app/Models/ScheduledTask.phpindex f69ae77b82..0a53395d31 100644--- a/app/Models/ScheduledTask.php+++ b/app/Models/ScheduledTask.php@@ -2,7 +2,6 @@namespace App\Models;-use App\Traits\HasNextRunAt;use App\Traits\HasSafeStringAttribute;use Illuminate\Database\Eloquent\Factories\HasFactory;use Illuminate\Database\Eloquent\Relations\HasMany;@@ -21,7 +20,6 @@ use OpenApi\Attributes as OA;'frequency' => ['type' => 'string', 'description' => 'The frequency of the scheduled task.'],'container' => ['type' => 'string', 'nullable' => true, 'description' => 'The container where the command should be executed.'],'timeout' => ['type' => 'integer', 'description' => 'The timeout of the scheduled task in seconds.'],- 'next_run_at' => ['type' => 'string', 'format' => 'date-time', 'nullable' => true, 'description' => 'The next time the scheduled task runs, in UTC. Null when it is disabled or not calculated yet.'],'created_at' => ['type' => 'string', 'format' => 'date-time', 'description' => 'The date and time when the scheduled task was created.'],'updated_at' => ['type' => 'string', 'format' => 'date-time', 'description' => 'The date and time when the scheduled task was last updated.'],],@@ -29,7 +27,6 @@ use OpenApi\Attributes as OA;class ScheduledTask extends BaseModel{use HasFactory;- use HasNextRunAt;use HasSafeStringAttribute;protected $fillable = [@@ -55,7 +52,6 @@ class ScheduledTask extends BaseModelreturn ['enabled' => 'boolean','timeout' => 'integer',- 'next_run_at' => 'datetime',];}diff --git a/app/Models/ScheduledVolumeBackup.php b/app/Models/ScheduledVolumeBackup.phpindex 4732d60adc..7f33fd92b9 100644--- a/app/Models/ScheduledVolumeBackup.php+++ b/app/Models/ScheduledVolumeBackup.php@@ -2,7 +2,6 @@namespace App\Models;-use App\Traits\HasNextRunAt;use Illuminate\Database\Eloquent\Builder;use Illuminate\Database\Eloquent\Model;use Illuminate\Database\Eloquent\Relations\BelongsTo;@@ -12,8 +11,6 @@ use Illuminate\Database\Eloquent\Relations\MorphTo;class ScheduledVolumeBackup extends BaseModel{- use HasNextRunAt;-public const int DEFAULT_TIMEOUT = 36000;protected $fillable = [@@ -50,7 +47,6 @@ class ScheduledVolumeBackup extends BaseModel'retention_days_s3' => 'integer','retention_max_storage_s3' => 'float','timeout' => 'integer',- 'next_run_at' => 'datetime',];}diff --git a/app/Models/ServerSetting.php b/app/Models/ServerSetting.phpindex 4f7e76bbd6..1fb3af6d6c 100644--- a/app/Models/ServerSetting.php+++ b/app/Models/ServerSetting.php@@ -3,7 +3,6 @@namespace App\Models;use App\Enums\ServerRole;-use App\Services\ScheduleNextRunRecalculator;use Illuminate\Contracts\Encryption\DecryptException;use Illuminate\Database\Eloquent\Casts\Attribute;use Illuminate\Database\Eloquent\Model;@@ -171,7 +170,6 @@ class ServerSetting extends Model'docker_version_checked_at' => 'datetime','compose_version_checked_at' => 'datetime','backup_compression_cpu_percentage' => 'integer',- 'docker_cleanup_next_run_at' => 'datetime','server_disk_usage_notification_interval_hours' => 'integer',];@@ -193,15 +191,6 @@ class ServerSetting extends Modelprotected static function booted(){- static::saving(function ($setting) {- if (! $setting->exists || $setting->isDirty(['docker_cleanup_frequency', 'server_timezone'])) {- // Without a frequency (a new row that uses the column default), the dispatcher calculates it.- $frequency = $setting->getAttributes()['docker_cleanup_frequency'] ?? null;- $setting->docker_cleanup_next_run_at = $frequency === null- ? null- : next_cron_run_at($frequency, $setting->server_timezone, now());- }- });static::creating(function ($setting) {try {if (str($setting->sentinel_token)->isEmpty()) {@@ -215,11 +204,6 @@ class ServerSetting extends Model}});static::updated(function ($settings) {- if ($settings->wasChanged('server_timezone') && $settings->server) {- // A failure must not fail the settings save: the schedules correct themselves after their next run.- rescue(fn () => app(ScheduleNextRunRecalculator::class)->forServer($settings->server));- }-if ($settings->wasChanged('sentinel_token') ||$settings->wasChanged('sentinel_custom_url') ||diff --git a/app/Services/ScheduleNextRunRecalculator.php b/app/Services/ScheduleNextRunRecalculator.phpdeleted file mode 100644index 87127c27c2..0000000000--- a/app/Services/ScheduleNextRunRecalculator.php+++ /dev/null@@ -1,85 +0,0 @@-<?php--namespace App\Services;--use App\Models\ScheduledVolumeBackup;-use App\Models\Server;-use App\Models\Service;-use App\Models\ServiceApplication;-use App\Models\ServiceDatabase;-use App\Models\StandalonePostgresql;-use Illuminate\Database\Eloquent\Builder;-use Illuminate\Database\Eloquent\Model;-use Illuminate\Database\Eloquent\Relations\Relation;--/**- * Recalculates next_run_at of schedules after their server timezone can have changed:- * a server timezone change, or a resource that moved to another server.- */-class ScheduleNextRunRecalculator-{- /**- * @param string|null $timezone The timezone of the resource's server. Read from the database when null.- */- public function forResource(Model $resource, ?string $timezone = null): void- {- $timezone ??= $this->serverTimezone($resource);-- foreach (['scheduled_tasks', 'scheduledBackups'] as $relation) {- if (method_exists($resource, $relation)) {- $this->recalculate($resource->{$relation}(), $timezone);- }- }-- foreach (['persistentStorages', 'fileStorages'] as $relation) {- if (method_exists($resource, $relation)) {- $storages = $resource->{$relation}();- $this->recalculate(ScheduledVolumeBackup::query()- ->where('backupable_type', $storages->getRelated()->getMorphClass())- ->whereIn('backupable_id', $storages->select('id')), $timezone);- }- }-- if ($resource instanceof Service) {- $resource->applications()->get()->each(fn (Model $application) => $this->forResource($application, $timezone));- $resource->databases()->get()->each(fn (Model $database) => $this->forResource($database, $timezone));- }- }-- /**- * Each resource uses the timezone of its own main server: Server::applications() also returns- * applications for which this server is only an additional server.- */- public function forServer(Server $server): void- {- $server->applications()->each(fn (Model $application) => $this->forResource($application));- $server->databases()->each(fn (Model $database) => $this->forResource($database));- $server->services()->get()->each(fn (Service $service) => $this->forResource($service));-- // Server::databases() leaves out the instance database, which has the instance backup.- if ($server->id === 0 && ($instanceDatabase = StandalonePostgresql::find(0))) {- $this->forResource($instanceDatabase);- }- }-- /**- * Move enabled schedules to their next due time after now. The current minute is excluded,- * so an occurrence that already ran in this minute does not run again.- */- private function recalculate(Relation|Builder $schedules, ?string $timezone): void- {- $schedules->where('enabled', true)->get()->each(fn (Model $schedule) => $schedule->forceFill([- 'next_run_at' => next_cron_run_at((string) $schedule->frequency, $timezone, now()),- ])->saveQuietly());- }-- private function serverTimezone(Model $resource): ?string- {- $owner = $resource instanceof ServiceApplication || $resource instanceof ServiceDatabase- ? $resource->service()->first()- : $resource;- $server = $owner?->destination()->first()?->server()->first();-- return $server?->settings()->value('server_timezone');- }-}diff --git a/app/Services/ScheduledJobDeliveryService.php b/app/Services/ScheduledJobDeliveryService.phpindex 9067391e6d..5dce4691dc 100644--- a/app/Services/ScheduledJobDeliveryService.php+++ b/app/Services/ScheduledJobDeliveryService.php@@ -8,41 +8,119 @@ use App\Jobs\ScheduledTaskJob;use App\Jobs\VolumeBackupJob;use App\Models\ScheduledDatabaseBackup;use App\Models\ScheduledJobDelivery;+use App\Models\ScheduledJobState;use App\Models\ScheduledTask;use App\Models\ScheduledVolumeBackup;use App\Models\Server;-use Carbon\CarbonInterface;+use Cron\CronExpression;+use Illuminate\Support\Carbon;+use Illuminate\Support\Facades\DB;use Illuminate\Support\Facades\Log;class ScheduledJobDeliveryService{+ public const CATCH_UP_WINDOW_MINUTES = 10;+/*** An occurrence that is enqueued longer than this has most likely lost its queued job, for- * example after a Redis restart. A job that is only waiting in a long queue is safe as well:- * the first job that claims the occurrence runs it, and the other job exits.+ * example when Redis evicted or lost the queue. A job that is only waiting in a long queue is+ * safe as well: the first job that claims the occurrence runs it, and the other job exits.*/public const ENQUEUED_STALE_AFTER_MINUTES = 60;/**- * Job types that run once when they are late, like the dispatcher does. A lost task or Docker- * cleanup is logged as missed instead.+ * Job types that run once when they are late. A lost task or Docker cleanup is logged as missed.*/private const LATE_RUN_JOB_TYPES = ['database-backup', 'volume-backup'];- /**- * Record a pending delivery for one occurrence of a schedule. Returns null when the- * occurrence already has a delivery.- *- * @param array<string, mixed> $payload- */- public function create(string $scheduleKey, CarbonInterface $scheduledFor, string $jobType, int $resourceId, array $payload = []): ?ScheduledJobDelivery- {- $delivery = ScheduledJobDelivery::createOrFirst(- ['schedule_key' => $scheduleKey, 'scheduled_for' => $scheduledFor->utc()],- ['job_type' => $jobType, 'resource_id' => $resourceId, 'payload' => $payload, 'status' => 'pending'],- );+ public function recordAndPublish(+ string $scheduleKey,+ string $frequency,+ string $timezone,+ string $jobType,+ int $resourceId,+ array $payload = [],+ ?Carbon $executionTime = null,+ ): bool {+ $executionTime = ($executionTime ?? Carbon::now())->copy()->setTimezone($timezone);+ $cron = new CronExpression(VALID_CRON_STRINGS[$frequency] ?? $frequency);+ $scheduledFor = Carbon::instance($cron->getPreviousRunDate($executionTime, allowCurrentDate: true));++ if (! $scheduledFor->gte($executionTime->copy()->subMinutes(self::CATCH_UP_WINDOW_MINUTES))) {+ return false;+ }++ $delivery = DB::transaction(function () use ($scheduleKey, $scheduledFor, $jobType, $resourceId, $payload): ?ScheduledJobDelivery {+ ScheduledJobState::query()->insertOrIgnore([+ 'uuid' => new_public_id(),+ 'schedule_key' => $scheduleKey,+ 'created_at' => now(),+ 'updated_at' => now(),+ ]);++ $state = ScheduledJobState::query()+ ->where('schedule_key', $scheduleKey)+ ->lockForUpdate()+ ->firstOrFail();++ if ($state->last_scheduled_for?->gte($scheduledFor)) {+ return null;+ }++ $state->update(['last_scheduled_for' => $scheduledFor->utc()]);++ return ScheduledJobDelivery::create([+ 'schedule_key' => $scheduleKey,+ 'scheduled_for' => $scheduledFor->utc(),+ 'job_type' => $jobType,+ 'resource_id' => $resourceId,+ 'payload' => $payload,+ 'status' => 'pending',+ ]);+ });++ if ($delivery === null) {+ return false;+ }++ return $this->publish($delivery);+ }++ public function recordSkipped(+ string $scheduleKey,+ string $frequency,+ string $timezone,+ ?Carbon $executionTime = null,+ ): bool {+ $executionTime = ($executionTime ?? Carbon::now())->copy()->setTimezone($timezone);+ $cron = new CronExpression(VALID_CRON_STRINGS[$frequency] ?? $frequency);+ $scheduledFor = Carbon::instance($cron->getPreviousRunDate($executionTime, allowCurrentDate: true));++ if (! $scheduledFor->gte($executionTime->copy()->subMinutes(self::CATCH_UP_WINDOW_MINUTES))) {+ return false;+ }++ return DB::transaction(function () use ($scheduleKey, $scheduledFor): bool {+ ScheduledJobState::query()->insertOrIgnore([+ 'uuid' => new_public_id(),+ 'schedule_key' => $scheduleKey,+ 'created_at' => now(),+ 'updated_at' => now(),+ ]);++ $state = ScheduledJobState::query()+ ->where('schedule_key', $scheduleKey)+ ->lockForUpdate()+ ->firstOrFail();- return $delivery->wasRecentlyCreated ? $delivery : null;+ if ($state->last_scheduled_for?->gte($scheduledFor)) {+ return false;+ }++ $state->update(['last_scheduled_for' => $scheduledFor->utc()]);++ return true;+ });}public function publishPending(): void@@ -66,9 +144,28 @@ class ScheduledJobDeliveryService});}+ /**+ * Count occurrences that are still open 15 minutes after they were created. Pending or+ * enqueued rows here never started; claimed rows are still running or their worker died.+ *+ * @return array<string, int>+ */+ public function staleOpenOccurrenceCounts(): array+ {+ return ScheduledJobDelivery::query()+ ->whereIn('status', ['pending', 'enqueued', 'claimed'])+ ->where('created_at', '<', now()->subMinutes(15))+ ->selectRaw('status, count(*) as aggregate')+ ->groupBy('status')+ ->pluck('aggregate', 'status')+ ->map(fn ($count) => (int) $count)+ ->all();+ }+/*** Publish backups again whose queued job was not started, and log lost tasks and Docker- * cleanups as missed. Only one scheduler node can move each occurrence.+ * cleanups as missed. A lost backup is not run when a newer occurrence of the same schedule+ * exists, so a backup that runs longer than its interval does not build up a backlog.*/public function recoverStaleEnqueued(): void{@@ -78,26 +175,7 @@ class ScheduledJobDeliveryService->chunkById(100, function ($occurrences): void {foreach ($occurrences as $occurrence) {try {- $runLate = in_array($occurrence->job_type, self::LATE_RUN_JOB_TYPES, true);- $moved = ScheduledJobDelivery::query()- ->whereKey($occurrence->id)- ->where('status', 'enqueued')- ->where('enqueued_at', '<', now()->subMinutes(self::ENQUEUED_STALE_AFTER_MINUTES))- ->update(['status' => $runLate ? 'pending' : 'failed', 'updated_at' => now()]) === 1;-- if (! $moved) {- continue;- }-- if (! $runLate) {- $this->logOccurrence($occurrence->uuid, 'Scheduled occurrence missed: its queued job was not started', 'warning');-- continue;- }-- $occurrence->status = 'pending';- $this->logOccurrence($occurrence->uuid, 'Scheduled occurrence publishing again: its queued job was not started', 'warning');- $this->publish($occurrence);+ $this->recoverStaleEnqueuedOccurrence($occurrence);} catch (\Throwable $e) {Log::channel('scheduled-errors')->error('Failed to recover enqueued scheduled occurrence', ['occurrence_uuid' => $occurrence->uuid,@@ -109,6 +187,38 @@ class ScheduledJobDeliveryService});}+ private function recoverStaleEnqueuedOccurrence(ScheduledJobDelivery $occurrence): void+ {+ $hasNewerOccurrence = ScheduledJobDelivery::query()+ ->where('schedule_key', $occurrence->schedule_key)+ ->where('scheduled_for', '>', $occurrence->scheduled_for)+ ->exists();+ $runLate = in_array($occurrence->job_type, self::LATE_RUN_JOB_TYPES, true) && ! $hasNewerOccurrence;++ // Only one scheduler node can move each occurrence.+ $moved = ScheduledJobDelivery::query()+ ->whereKey($occurrence->id)+ ->where('status', 'enqueued')+ ->where('enqueued_at', '<', now()->subMinutes(self::ENQUEUED_STALE_AFTER_MINUTES))+ ->update(['status' => $runLate ? 'pending' : 'failed', 'updated_at' => now()]) === 1;++ if (! $moved) {+ return;+ }++ if (! $runLate) {+ $this->logOccurrence($occurrence->uuid, $hasNewerOccurrence+ ? 'Scheduled occurrence skipped: its queued job was not started and a newer occurrence exists'+ : 'Scheduled occurrence missed: its queued job was not started', 'warning');++ return;+ }++ $occurrence->status = 'pending';+ $this->logOccurrence($occurrence->uuid, 'Scheduled occurrence publishing again: its queued job was not started', 'warning');+ $this->publish($occurrence);+ }+public function deleteOldOccurrences(): void{ScheduledJobDelivery::query()@@ -203,7 +313,7 @@ class ScheduledJobDeliveryService}}- public function publish(ScheduledJobDelivery $occurrence): bool+ private function publish(ScheduledJobDelivery $occurrence): bool{if ($occurrence->status !== 'pending') {return false;@@ -242,6 +352,15 @@ class ScheduledJobDeliveryServicereturn false;}+ // An open previous occurrence usually means the previous run is still running. Jobs with+ // WithoutOverlapping(...)->dontRelease() then drop this one without running it.+ $previousOpen = ScheduledJobDelivery::query()+ ->where('schedule_key', $occurrence->schedule_key)+ ->where('scheduled_for', '<', $occurrence->scheduled_for)+ ->whereIn('status', ['pending', 'enqueued', 'claimed'])+ ->orderByDesc('scheduled_for')+ ->first(['uuid', 'status', 'scheduled_for']);+dispatch($job);ScheduledJobDelivery::query()@@ -254,7 +373,17 @@ class ScheduledJobDeliveryService]);// Log from the loaded row: a fast worker can complete and delete the occurrence already.- $this->logOccurrence($occurrence, 'Scheduled occurrence enqueued', context: ['status' => 'enqueued', 'enqueued_at' => now()->toIso8601String()]);+ $enqueued = ['status' => 'enqueued', 'enqueued_at' => now()->toIso8601String()];+ if ($previousOpen) {+ $this->logOccurrence($occurrence, 'Scheduled occurrence enqueued while the previous occurrence is still open', 'warning', [+ ...$enqueued,+ 'previous_occurrence_uuid' => $previousOpen->uuid,+ 'previous_status' => $previousOpen->status,+ 'previous_scheduled_for' => $previousOpen->scheduled_for->toIso8601String(),+ ]);+ } else {+ $this->logOccurrence($occurrence, 'Scheduled occurrence enqueued', context: $enqueued);+ }return true;}diff --git a/app/Traits/HasNextRunAt.php b/app/Traits/HasNextRunAt.phpdeleted file mode 100644index 856ebe3391..0000000000--- a/app/Traits/HasNextRunAt.php+++ /dev/null@@ -1,38 +0,0 @@-<?php--namespace App\Traits;--use Carbon\CarbonImmutable;-use DateTimeInterface;--/**- * Keeps next_run_at of a schedule model in sync with its frequency and enabled flag.- * The model must have `frequency`, `enabled`, `next_run_at`, and a `server()` method.- */-trait HasNextRunAt-{- protected static function bootHasNextRunAt(): void- {- static::saving(function ($schedule) {- if ($schedule->exists && ! $schedule->isDirty(['frequency', 'enabled'])) {- return;- }-- $schedule->next_run_at = $schedule->calculateNextRunAt(now());- });- }-- public function calculateNextRunAt(DateTimeInterface $after, bool $includeCurrentMinute = false): ?CarbonImmutable- {- if ($this->getAttribute('enabled') !== null && ! $this->enabled) {- return null;- }-- return next_cron_run_at((string) $this->frequency, $this->scheduleTimezone(), $after, $includeCurrentMinute);- }-- public function scheduleTimezone(): ?string- {- return data_get($this->server(), 'settings.server_timezone');- }-}diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.phpindex 94a35140c0..9b97c551c5 100644--- a/bootstrap/helpers/shared.php+++ b/bootstrap/helpers/shared.php@@ -1112,28 +1112,6 @@ function validate_timezone(string $timezone): boolreturn in_array($timezone, timezone_identifiers_list());}-/**- * The next due time of a cron frequency after $after, in UTC. An invalid frequency returns null.- * With $includeCurrentMinute, a frequency that is due in the minute of $after returns that minute.- */-function next_cron_run_at(string $frequency, ?string $timezone, DateTimeInterface $after, bool $includeCurrentMinute = false): ?CarbonImmutable-{- $frequency = trim($frequency);- $timezone = filled($timezone) && validate_timezone($timezone) ? $timezone : config('app.timezone');-- try {- $cron = new Cron\CronExpression(VALID_CRON_STRINGS[$frequency] ?? $frequency);- $time = CarbonImmutable::instance($after)->setTimezone($timezone);- if ($includeCurrentMinute) {- $time = $time->startOfMinute();- }-- return CarbonImmutable::instance($cron->getNextRunDate($time, 0, $includeCurrentMinute))->utc();- } catch (Throwable) {- return null;- }-}-function parseEnvFormatToArray($env_file_contents){$env_array = [];diff --git a/database/migrations/2026_09_28_194652_add_next_run_at_to_scheduled_jobs.php b/database/migrations/2026_09_28_194652_add_next_run_at_to_scheduled_jobs.phpdeleted file mode 100644index dd413fb653..0000000000--- a/database/migrations/2026_09_28_194652_add_next_run_at_to_scheduled_jobs.php+++ /dev/null@@ -1,94 +0,0 @@-<?php--use App\Models\ScheduledDatabaseBackup;-use App\Models\ScheduledTask;-use App\Models\ScheduledVolumeBackup;-use App\Models\ServerSetting;-use Illuminate\Database\Migrations\Migration;-use Illuminate\Database\Schema\Blueprint;-use Illuminate\Support\Carbon;-use Illuminate\Support\Facades\DB;-use Illuminate\Support\Facades\Schema;--return new class extends Migration-{- /**- * Schedules without a recorded occurrence keep next_run_at = null. The dispatcher calculates it on its first run.- */- public function up(): void- {- foreach (['scheduled_database_backups', 'scheduled_tasks', 'scheduled_volume_backups'] as $table) {- Schema::table($table, function (Blueprint $table) {- $table->timestampTz('next_run_at')->nullable();- $table->index(['enabled', 'next_run_at']);- });- }-- Schema::table('server_settings', function (Blueprint $table) {- $table->timestampTz('docker_cleanup_next_run_at')->nullable()->index();- });-- $this->continueAfterLastScheduledOccurrences();-- Schema::dropIfExists('scheduled_job_states');- }-- public function down(): void- {- Schema::create('scheduled_job_states', function (Blueprint $table) {- $table->id();- $table->string('uuid')->unique();- $table->string('schedule_key')->unique();- $table->timestampTz('last_scheduled_for')->nullable();- $table->timestamps();- });-- Schema::table('server_settings', function (Blueprint $table) {- $table->dropIndex(['docker_cleanup_next_run_at']);- $table->dropColumn('docker_cleanup_next_run_at');- });-- foreach (['scheduled_database_backups', 'scheduled_tasks', 'scheduled_volume_backups'] as $table) {- Schema::table($table, function (Blueprint $table) {- $table->dropIndex(['enabled', 'next_run_at']);- $table->dropColumn('next_run_at');- });- }- }-- /**- * Continue each schedule after the last occurrence that the previous dispatcher recorded, so- * an occurrence that was due during the upgrade still runs late instead of being skipped.- */- public function continueAfterLastScheduledOccurrences(): void- {- if (! Schema::hasTable('scheduled_job_states')) {- return;- }-- DB::table('scheduled_job_states')->whereNotNull('last_scheduled_for')->orderBy('id')->chunkById(500, function ($states) {- foreach ($states as $state) {- try {- [$type, $id] = explode(':', $state->schedule_key, 2) + [null, null];- $schedule = match ($type) {- 'scheduled-backup' => ScheduledDatabaseBackup::find($id),- 'scheduled-task' => ScheduledTask::find($id),- 'scheduled-volume-backup' => ScheduledVolumeBackup::find($id),- 'docker-cleanup' => ServerSetting::query()->where('server_id', $id)->first(),- default => null,- };-- if ($schedule instanceof ServerSetting) {- $schedule->forceFill(['docker_cleanup_next_run_at' => next_cron_run_at(- (string) $schedule->docker_cleanup_frequency, $schedule->server_timezone, Carbon::parse($state->last_scheduled_for),- )])->saveQuietly();- } elseif ($schedule?->enabled) {- $schedule->forceFill(['next_run_at' => $schedule->calculateNextRunAt(Carbon::parse($state->last_scheduled_for))])->saveQuietly();- }- } catch (Throwable) {- // The dispatcher calculates the next run of this schedule on its first run.- }- }- });- }-};diff --git a/database/migrations/2026_10_02_055938_remove_next_run_at_from_scheduled_jobs.php b/database/migrations/2026_10_02_055938_remove_next_run_at_from_scheduled_jobs.phpnew file mode 100644index 0000000000..e3d205a4ce--- /dev/null+++ b/database/migrations/2026_10_02_055938_remove_next_run_at_from_scheduled_jobs.php@@ -0,0 +1,46 @@+<?php++use Illuminate\Database\Migrations\Migration;+use Illuminate\Database\Schema\Blueprint;+use Illuminate\Support\Facades\Schema;++return new class extends Migration+{+ /**+ * Undo the removed migration 2026_09_28_194652_add_next_run_at_to_scheduled_jobs where it ran.+ * Environments that never ran it are not changed.+ */+ public function up(): void+ {+ foreach (['scheduled_database_backups', 'scheduled_tasks', 'scheduled_volume_backups'] as $tableName) {+ if (Schema::hasColumn($tableName, 'next_run_at')) {+ Schema::table($tableName, function (Blueprint $table) {+ $table->dropIndex(['enabled', 'next_run_at']);+ $table->dropColumn('next_run_at');+ });+ }+ }++ if (Schema::hasColumn('server_settings', 'docker_cleanup_next_run_at')) {+ Schema::table('server_settings', function (Blueprint $table) {+ $table->dropIndex(['docker_cleanup_next_run_at']);+ $table->dropColumn('docker_cleanup_next_run_at');+ });+ }++ if (! Schema::hasTable('scheduled_job_states')) {+ Schema::create('scheduled_job_states', function (Blueprint $table) {+ $table->id();+ $table->string('uuid')->unique();+ $table->string('schedule_key')->unique();+ $table->timestampTz('last_scheduled_for')->nullable();+ $table->timestamps();+ });+ }+ }++ public function down(): void+ {+ // The removed columns are not added again.+ }+};diff --git a/openapi.json b/openapi.jsonindex 8d2aefe1ad..9e6bf53c3e 100644--- a/openapi.json+++ b/openapi.json@@ -23565,12 +23565,6 @@"type": "integer","description": "The timeout of the scheduled task in seconds."},- "next_run_at": {- "type": "string",- "format": "date-time",- "nullable": true,- "description": "The next time the scheduled task runs, in UTC. Null when it is disabled or not calculated yet."- },"created_at": {"type": "string","format": "date-time",diff --git a/openapi.yaml b/openapi.yamlindex fd86ff91b1..68950d480a 100644--- a/openapi.yaml+++ b/openapi.yaml@@ -15066,11 +15066,6 @@ components:timeout:type: integerdescription: 'The timeout of the scheduled task in seconds.'- next_run_at:- type: string- format: date-time- nullable: true- description: 'The next time the scheduled task runs, in UTC. Null when it is disabled or not calculated yet.'created_at:type: stringformat: date-timediff --git a/resources/views/components/next-run.blade.php b/resources/views/components/next-run.blade.phpdeleted file mode 100644index f0db83ffda..0000000000--- a/resources/views/components/next-run.blade.php+++ /dev/null@@ -1,30 +0,0 @@-@props([- 'at' => null,- 'timezone' => null,- 'enabled' => true,- 'frequency' => null,-])--@php- // Same fallback as next_cron_run_at(): schedules without a valid server timezone run in the instance timezone.- $displayTimezone = filled($timezone) && validate_timezone((string) $timezone) ? $timezone : config('app.timezone');- $nextRunAt = $at ? \Illuminate\Support\Carbon::parse($at) : null;- // Without a next run, the frequency is either not calculated yet or cannot be calculated at all.- $isFrequencyInvalid = ! $nextRunAt && filled($frequency) && next_cron_run_at((string) $frequency, $displayTimezone, now()) === null;-@endphp--<p {{ $attributes->merge(['class' => 'text-xs leading-5 text-neutral-500 dark:text-fg-dim']) }}>- <span class="font-medium text-neutral-700 dark:text-fg">Next run</span>- @if (! $enabled)- <span>Disabled</span>- @elseif ($isFrequencyInvalid)- <span class="text-red-500">Invalid frequency. Change it to a valid cron expression, or this schedule never runs.</span>- @elseif (! $nextRunAt)- <span>Calculating…</span>- @else- <time datetime="{{ $nextRunAt->toIso8601String() }}"- title="{{ $nextRunAt->copy()->utc()->format('Y-m-d H:i') }} UTC">{{ $nextRunAt->copy()->setTimezone($displayTimezone)->format('Y-m-d H:i') }} ({{ $displayTimezone }})</time>- <span>·- {{ $nextRunAt->isFuture() ? 'in '.$nextRunAt->diffForHumans(syntax: \Carbon\CarbonInterface::DIFF_ABSOLUTE) : $nextRunAt->diffForHumans() }}</span>- @endif-</p>diff --git a/resources/views/livewire/project/database/backup-edit/general.blade.php b/resources/views/livewire/project/database/backup-edit/general.blade.phpindex 1dd6f28eb8..49385b0258 100644--- a/resources/views/livewire/project/database/backup-edit/general.blade.php+++ b/resources/views/livewire/project/database/backup-edit/general.blade.php@@ -96,7 +96,6 @@min="0" max="365" suffix="days" canGate="manageBackups" :canResource="$backup->database"helper="Notify through backup failure channels after this many days without an execution. Use 0 to disable." required /></div>- <x-next-run :at="$backup->next_run_at" :timezone="$timezone" :enabled="$backup->enabled" :frequency="$backup->frequency" /></div></x-application.settings-section></form>diff --git a/resources/views/livewire/project/shared/scheduled-task/show.blade.php b/resources/views/livewire/project/shared/scheduled-task/show.blade.phpindex 7849f35dcf..c5be9974ba 100644--- a/resources/views/livewire/project/shared/scheduled-task/show.blade.php+++ b/resources/views/livewire/project/shared/scheduled-task/show.blade.php@@ -44,7 +44,6 @@<x-forms.input canGate="update" :canResource="$resource" :disabled="!auth()->user()->can('update', $resource)"placeholder="php artisan schedule:run" id="command" label="Command" required /></div>- <x-next-run class="mt-4" :at="$task->next_run_at" :timezone="$this->serverTimezone" :enabled="$task->enabled" :frequency="$task->frequency" /></x-application.settings-section></form>diff --git a/resources/views/livewire/project/shared/storages/volume-backups/general.blade.php b/resources/views/livewire/project/shared/storages/volume-backups/general.blade.phpindex 48764b9c36..5b4ece52f1 100644--- a/resources/views/livewire/project/shared/storages/volume-backups/general.blade.php+++ b/resources/views/livewire/project/shared/storages/volume-backups/general.blade.php@@ -45,6 +45,5 @@<x-forms.input id="timeout" type="number" min="60" max="36000" label="Timeout"helper="Maximum backup runtime in seconds." required /></div>- <x-next-run class="mt-4" :at="$backup?->next_run_at" :timezone="$timezone" :enabled="(bool) $backup?->enabled" :frequency="$backup?->frequency" /></x-application.settings-section></form>diff --git a/resources/views/livewire/server/docker-cleanup.blade.php b/resources/views/livewire/server/docker-cleanup.blade.phpindex a530f7c746..e12f4cd3da 100644--- a/resources/views/livewire/server/docker-cleanup.blade.php+++ b/resources/views/livewire/server/docker-cleanup.blade.php@@ -33,8 +33,6 @@@if (!isCloud() && $this->isCleanupStale)<x-callout type="warning" title="Docker cleanup may be stalled">- A scheduled cleanup was due- {{ $server->settings->docker_cleanup_next_run_at?->diffForHumans() }} but has not started.The last cleanup ran {{ $this->lastExecutionTime ?? 'at an unknown time' }}.@if (!$this->isSchedulerHealthy)The scheduled job manager also appears inactive.@@ -54,8 +52,6 @@<p class="mt-1 text-xs leading-5 text-neutral-500 dark:text-fg-dim">Cleanup runs automatically using the schedule and threshold configured below.</p>- <x-next-run class="mt-1" :at="$server->settings->docker_cleanup_next_run_at"- :timezone="$server->settings->server_timezone" :frequency="$server->settings->docker_cleanup_frequency" /></div></div>@endifdiff --git a/tests/Feature/Jobs/ScheduledJobManagerStaleLockTest.php b/tests/Feature/Jobs/ScheduledJobManagerStaleLockTest.phpnew file mode 100644index 0000000000..dba5b6041a--- /dev/null+++ b/tests/Feature/Jobs/ScheduledJobManagerStaleLockTest.php@@ -0,0 +1,50 @@+<?php++use App\Jobs\ScheduledJobManager;+use Illuminate\Support\Facades\Redis;++function scheduledJobManagerLockKey(): string+{+ return config('cache.prefix').'laravel-queue-overlap:'.ScheduledJobManager::class.':scheduled-job-manager';+}++it('clears stale lock when TTL is -1', function () {+ $lockKey = scheduledJobManagerLockKey();++ $redis = Mockery::mock();+ $redis->shouldReceive('ttl')->once()->with($lockKey)->andReturn(-1);+ $redis->shouldReceive('del')->once()->with($lockKey)->andReturn(1);++ Redis::shouldReceive('connection')->with('default')->andReturn($redis);++ $job = new ScheduledJobManager;+ $job->middleware();+});++it('preserves valid lock with positive TTL', function () {+ $lockKey = scheduledJobManagerLockKey();++ $redis = Mockery::mock();+ $redis->shouldReceive('ttl')->once()->with($lockKey)->andReturn(60);+ $redis->shouldNotReceive('del');++ Redis::shouldReceive('connection')->with('default')->andReturn($redis);++ $job = new ScheduledJobManager;+ $job->middleware();+});++it('does not fail when no lock exists', function () {+ $lockKey = scheduledJobManagerLockKey();++ $redis = Mockery::mock();+ $redis->shouldReceive('ttl')->once()->with($lockKey)->andReturn(-2);+ $redis->shouldNotReceive('del');++ Redis::shouldReceive('connection')->with('default')->andReturn($redis);++ $job = new ScheduledJobManager;+ $middleware = $job->middleware();++ expect($middleware)->toBeArray()->toHaveCount(1);+});diff --git a/tests/Feature/QueueRoutingTest.php b/tests/Feature/QueueRoutingTest.phpindex a35d2d8555..1bb837be89 100644--- a/tests/Feature/QueueRoutingTest.php+++ b/tests/Feature/QueueRoutingTest.php@@ -4,6 +4,7 @@ use App\Actions\Database\StartDatabase;use App\Actions\Database\StartDatabaseProxy;use App\Actions\Service\StartService;use App\Jobs\DatabaseBackupJob;+use App\Jobs\ScheduledJobManager;use App\Models\ScheduledDatabaseBackup;describe('deployment_queue helper', function () {@@ -60,12 +61,14 @@ describe('scheduled job routing', function () {test('scheduled jobs use the crons queue on cloud', function () {config(['constants.coolify.self_hosted' => false]);+ expect((new ScheduledJobManager)->queue)->toBe('crons');expect((new DatabaseBackupJob(new ScheduledDatabaseBackup))->queue)->toBe('crons');});test('scheduled jobs use the high queue on self-hosted', function () {config(['constants.coolify.self_hosted' => true]);+ expect((new ScheduledJobManager)->queue)->toBe('high');expect((new DatabaseBackupJob(new ScheduledDatabaseBackup))->queue)->toBe('high');});});diff --git a/tests/Feature/ScheduleNextRunAtTest.php b/tests/Feature/ScheduleNextRunAtTest.phpdeleted file mode 100644index e05acc2fb3..0000000000--- a/tests/Feature/ScheduleNextRunAtTest.php+++ /dev/null@@ -1,179 +0,0 @@-<?php--use App\Models\Application;-use App\Models\Environment;-use App\Models\PrivateKey;-use App\Models\Project;-use App\Models\ScheduledTask;-use App\Models\Server;-use App\Models\StandaloneDocker;-use App\Models\Team;-use App\Services\ScheduleNextRunRecalculator;-use Carbon\CarbonImmutable;-use Illuminate\Database\Schema\Blueprint;-use Illuminate\Foundation\Testing\RefreshDatabase;-use Illuminate\Support\Carbon;-use Illuminate\Support\Facades\DB;-use Illuminate\Support\Facades\Schema;--uses(RefreshDatabase::class);--beforeEach(function () {- Server::flushIdentityMap();- Carbon::setTestNow(Carbon::create(2026, 9, 17, 10, 20, 30, 'UTC'));-});--it('calculates the next due time of a frequency in the server timezone', function (string $frequency, string $timezone, string $after, bool $includeCurrentMinute, ?string $expected) {- $next = next_cron_run_at($frequency, $timezone, CarbonImmutable::parse($after, 'UTC'), $includeCurrentMinute);-- expect($next?->toDateTimeString())->toBe($expected);-})->with([- 'alias' => ['daily', 'UTC', '2026-09-17 10:20:30', false, '2026-09-18 00:00:00'],- 'server timezone' => ['0 2 * * *', 'Europe/Budapest', '2026-09-17 10:20:30', false, '2026-09-18 00:00:00'],- 'strictly after' => ['20 10 * * *', 'UTC', '2026-09-17 10:20:00', false, '2026-09-18 10:20:00'],- 'current minute included' => ['20 10 * * *', 'UTC', '2026-09-17 10:20:30', true, '2026-09-17 10:20:00'],- 'a time that does not exist when daylight saving time starts' => ['30 2 * * *', 'Europe/Budapest', '2026-03-28 12:00:00', false, '2026-03-29 01:30:00'],- 'invalid timezone uses the app timezone' => ['0 2 * * *', 'Mars/Olympus', '2026-09-17 10:20:30', false, '2026-09-18 02:00:00'],- 'invalid frequency' => ['not a cron', 'UTC', '2026-09-17 10:20:30', false, null],-]);--it('calculates the next run of a schedule when it is created or its frequency or enabled flag changes', function () {- $task = nextRunAtTestTask(nextRunAtTestApplication('Europe/Budapest'), '0 2 * * *');-- expect($task->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00');-- $task->update(['frequency' => 'hourly']);- expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-17 11:00:00');-- $task->update(['enabled' => false]);- expect($task->fresh()->next_run_at)->toBeNull();-- Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 30, 0, 'UTC'));- $task->update(['enabled' => true]);- expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-17 13:00:00');-- $task->update(['name' => 'renamed']);- expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-17 13:00:00');-});--it('calculates the next Docker cleanup when its frequency or the server timezone changes', function () {- $settings = nextRunAtTestApplication()->destination->server->settings;-- $settings->update(['docker_cleanup_frequency' => '0 2 * * *']);- expect($settings->fresh()->docker_cleanup_next_run_at->toDateTimeString())->toBe('2026-09-18 02:00:00');-- $settings->update(['server_timezone' => 'Europe/Budapest']);- expect($settings->fresh()->docker_cleanup_next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00');-});--it('recalculates the next run of the server schedules when the server timezone changes', function () {- $application = nextRunAtTestApplication();- $task = nextRunAtTestTask($application, '0 2 * * *');- $otherTask = nextRunAtTestTask(nextRunAtTestApplication(), '0 2 * * *');-- $application->destination->server->settings->update(['server_timezone' => 'Europe/Budapest']);-- expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00')- ->and($otherTask->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 02:00:00');-});--it('keeps the timezone of the main server for applications that also run on the changed server', function () {- $application = nextRunAtTestApplication();- $task = nextRunAtTestTask($application, '0 2 * * *');- $additionalServer = nextRunAtTestApplication()->destination->server;- $application->additional_networks()->attach(- StandaloneDocker::where('server_id', $additionalServer->id)->value('id'),- ['server_id' => $additionalServer->id],- );-- $additionalServer->settings->update(['server_timezone' => 'Europe/Budapest']);-- expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 02:00:00');-});--it('does not run an occurrence again when the timezone changes in the minute it ran', function () {- Carbon::setTestNow(Carbon::create(2026, 9, 17, 11, 59, 0, 'UTC'));- $application = nextRunAtTestApplication();- $task = nextRunAtTestTask($application, '0 * * * *');-- Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 30, 'UTC'));- $application->destination->server->settings->update(['server_timezone' => 'Europe/Budapest']);-- expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-17 13:00:00');-});--it('recalculates the next run of the resource schedules with the timezone of its current server', function () {- $application = nextRunAtTestApplication();- $task = nextRunAtTestTask($application, '0 2 * * *');- $application->destination->server->settings()->update(['server_timezone' => 'Europe/Budapest']);-- app(ScheduleNextRunRecalculator::class)->forResource($application);-- expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00');-});--it('lists overdue schedules and schedules without a next run in the diagnostics', function () {- $overdue = nextRunAtTestTask(nextRunAtTestApplication(), '0 2 * * *');- ScheduledTask::query()->whereKey($overdue->id)->update(['next_run_at' => now()->subHour()]);- $invalid = nextRunAtTestTask(nextRunAtTestApplication(), '0 2 * * *');- ScheduledTask::query()->whereKey($invalid->id)->update(['frequency' => '60 * * * *', 'next_run_at' => null]);- nextRunAtTestTask(nextRunAtTestApplication(), '0 3 * * *');-- $this->artisan('scheduled:diagnostics')- ->expectsOutputToContain('Scheduled tasks: 3 enabled, 2 overdue or without a next run')- ->expectsOutputToContain('invalid frequency')- ->assertSuccessful();-});--it('continues each schedule after the occurrence that the previous dispatcher recorded last', function () {- $task = nextRunAtTestTask(nextRunAtTestApplication(), '0 2 * * *');- Schema::create('scheduled_job_states', function (Blueprint $table) {- $table->id();- $table->string('uuid')->unique();- $table->string('schedule_key')->unique();- $table->timestampTz('last_scheduled_for')->nullable();- $table->timestamps();- });- DB::table('scheduled_job_states')->insert([- 'uuid' => 'state-1',- 'schedule_key' => "scheduled-task:{$task->id}",- 'last_scheduled_for' => '2026-09-16 02:00:00',- ]);-- $migration = require database_path('migrations/2026_09_28_194652_add_next_run_at_to_scheduled_jobs.php');- $migration->continueAfterLastScheduledOccurrences();-- // The occurrence of 2026-09-17 02:00 was not recorded, so the dispatcher runs it on its next run.- expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-17 02:00:00');-});--function nextRunAtTestApplication(string $timezone = 'UTC'): Application-{- $team = Team::factory()->create();- $server = Server::factory()->create([- 'team_id' => $team->id,- 'private_key_id' => PrivateKey::factory()->create(['team_id' => $team->id])->id,- ]);- $server->settings()->update(['server_timezone' => $timezone]);- $destination = StandaloneDocker::where('server_id', $server->id)->first()- ?? StandaloneDocker::factory()->create(['server_id' => $server->id]);- $environment = Environment::factory()->create([- 'project_id' => Project::factory()->create(['team_id' => $team->id])->id,- ]);-- return Application::factory()->create([- 'environment_id' => $environment->id,- 'destination_id' => $destination->id,- 'destination_type' => StandaloneDocker::class,- ]);-}--function nextRunAtTestTask(Application $application, string $frequency): ScheduledTask-{- return ScheduledTask::factory()->create([- 'team_id' => $application->environment->project->team_id,- 'application_id' => $application->id,- 'frequency' => $frequency,- 'enabled' => true,- ])->fresh();-}diff --git a/tests/Feature/ScheduleNextRunDisplayTest.php b/tests/Feature/ScheduleNextRunDisplayTest.phpdeleted file mode 100644index d8910b1d6c..0000000000--- a/tests/Feature/ScheduleNextRunDisplayTest.php+++ /dev/null@@ -1,222 +0,0 @@-<?php--use App\Livewire\Project\Database\BackupEdit;-use App\Livewire\Project\Shared\ScheduledTask\Show;-use App\Livewire\Project\Shared\Storages\VolumeBackups;-use App\Livewire\Server\DockerCleanup;-use App\Models\Application;-use App\Models\Environment;-use App\Models\InstanceSettings;-use App\Models\LocalPersistentVolume;-use App\Models\Project;-use App\Models\ScheduledDatabaseBackup;-use App\Models\ScheduledTask;-use App\Models\Server;-use App\Models\ServerSetting;-use App\Models\StandaloneDocker;-use App\Models\StandalonePostgresql;-use App\Models\Team;-use App\Models\User;-use Illuminate\Foundation\Testing\RefreshDatabase;-use Illuminate\Routing\Events\RouteMatched;-use Illuminate\Support\Carbon;-use Illuminate\Support\Facades\Event;-use Livewire\Livewire;--uses(RefreshDatabase::class);--beforeEach(function () {- Server::flushIdentityMap();- Carbon::setTestNow(Carbon::create(2026, 9, 17, 10, 20, 30, 'UTC'));- InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0]));-- $this->team = Team::factory()->create();- $this->user = User::factory()->create();- $this->user->teams()->attach($this->team, ['role' => 'owner']);- $this->actingAs($this->user);- session(['currentTeam' => $this->team]);-- $this->server = Server::factory()->create(['team_id' => $this->team->id]);- $this->server->settings->update(['server_timezone' => 'Europe/Budapest']);- $this->destination = StandaloneDocker::where('server_id', $this->server->id)->firstOrFail();- $project = Project::factory()->create(['team_id' => $this->team->id]);- $this->environment = Environment::factory()->create(['project_id' => $project->id]);- $this->application = Application::factory()->create([- 'environment_id' => $this->environment->id,- 'destination_id' => $this->destination->id,- 'destination_type' => $this->destination->getMorphClass(),- ]);-});--afterEach(function () {- Event::forget(RouteMatched::class);- Carbon::setTestNow();-});--function nextRunDisplayBindTaskRoute(Application $application, ScheduledTask $task): void-{- $application->loadMissing('environment.project');-- Event::listen(RouteMatched::class, function (RouteMatched $event) use ($application, $task): void {- $event->route->setParameter('task_uuid', $task->uuid);- $event->route->setParameter('project_uuid', $application->environment->project->uuid);- $event->route->setParameter('environment_uuid', $application->environment->uuid);- $event->route->setParameter('application_uuid', $application->uuid);- });-}--function nextRunDisplaySetCleanupNextRun(Server $server, ?Carbon $nextRunAt): void-{- ServerSetting::query()->where('server_id', $server->id)->update(['docker_cleanup_next_run_at' => $nextRunAt]);-}--it('shows the next Docker cleanup in the server timezone and refreshes it after saving a new frequency', function () {- $this->server->settings->update(['docker_cleanup_frequency' => '0 2 * * *']);-- Livewire::test(DockerCleanup::class, ['server_uuid' => $this->server->uuid])- ->assertSee('Next run')- ->assertSee('2026-09-18 02:00 (Europe/Budapest)', false)- ->set('dockerCleanupFrequency', 'hourly')- ->call('submit')- ->assertSee('2026-09-17 13:00 (Europe/Budapest)', false)- ->assertSee('in 39 minutes');-});--it('shows Calculating when the Docker cleanup next run is not calculated yet', function () {- nextRunDisplaySetCleanupNextRun($this->server, null);-- Livewire::test(DockerCleanup::class, ['server_uuid' => $this->server->uuid])- ->assertSee('Next run')- ->assertSee('Calculating…');-});--it('marks the Docker cleanup as stale only when its next run is more than 10 minutes overdue', function (?int $minutesFromNow, bool $stale) {- nextRunDisplaySetCleanupNextRun($this->server, $minutesFromNow === null ? null : now()->addMinutes($minutesFromNow));-- $component = Livewire::test(DockerCleanup::class, ['server_uuid' => $this->server->uuid]);-- expect($component->instance()->isCleanupStale())->toBe($stale);- if ($stale) {- $component->assertSee('Docker cleanup may be stalled');- } else {- $component->assertDontSee('Docker cleanup may be stalled');- }-})->with([- '30 minutes overdue' => [-30, true],- '5 minutes overdue' => [-5, false],- 'in the future' => [30, false],- 'not calculated yet' => [null, false],-]);--it('does not mark the Docker cleanup as stale on a server without a real IP address', function () {- $this->server->update(['ip' => Server::PLACEHOLDER_IP]);- nextRunDisplaySetCleanupNextRun($this->server, now()->subMinutes(30));-- $component = Livewire::test(DockerCleanup::class, ['server_uuid' => $this->server->uuid]);-- expect($component->instance()->isCleanupStale())->toBeFalse();-});--it('shows the next scheduled task run in the server timezone and refreshes it on changes', function () {- $task = ScheduledTask::factory()->create([- 'team_id' => $this->team->id,- 'application_id' => $this->application->id,- 'frequency' => '0 2 * * *',- 'enabled' => true,- ]);- nextRunDisplayBindTaskRoute($this->application, $task);-- $component = Livewire::test(Show::class)- ->assertSee('Next run')- ->assertSee('2026-09-18 02:00 (Europe/Budapest)', false)- ->assertSee('in 13 hours');-- $component->call('toggleEnabled')- ->assertDontSee('2026-09-18 02:00 (Europe/Budapest)', false)- ->assertSeeInOrder(['Next run', 'Disabled']);-- $component->call('toggleEnabled')- ->set('frequency', 'hourly')- ->call('submit')- ->assertSee('2026-09-17 13:00 (Europe/Budapest)', false);-- expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-17 11:00:00');-});--it('shows Calculating for an enabled scheduled task without a next run', function () {- $task = ScheduledTask::factory()->create([- 'team_id' => $this->team->id,- 'application_id' => $this->application->id,- 'frequency' => '0 2 * * *',- 'enabled' => true,- ]);- ScheduledTask::query()->whereKey($task->id)->update(['next_run_at' => null]);- nextRunDisplayBindTaskRoute($this->application, $task);-- Livewire::test(Show::class)->assertSee('Calculating…');-});--it('shows an invalid frequency instead of Calculating when the next run cannot be calculated', function () {- $task = ScheduledTask::factory()->create([- 'team_id' => $this->team->id,- 'application_id' => $this->application->id,- 'frequency' => '0 2 * * *',- 'enabled' => true,- ]);- // The forms reject such a value, so only old data or a direct database change can have it.- ScheduledTask::query()->whereKey($task->id)->update(['frequency' => '60 * * * *', 'next_run_at' => null]);- nextRunDisplayBindTaskRoute($this->application, $task);-- Livewire::test(Show::class)- ->assertSee('Invalid frequency')- ->assertDontSee('Calculating…');-});--it('shows an invalid Docker cleanup frequency', function () {- ServerSetting::query()->where('server_id', $this->server->id)->update(['docker_cleanup_frequency' => '0 0 31 2 *', 'docker_cleanup_next_run_at' => null]);-- Livewire::test(DockerCleanup::class, ['server_uuid' => $this->server->uuid])- ->assertSee('Invalid frequency')- ->assertDontSee('Calculating…');-});--it('shows the next database backup run in the server timezone', function () {- $database = StandalonePostgresql::create([- 'name' => 'pg-next-run',- 'image' => 'postgres:16-alpine',- 'postgres_user' => 'postgres',- 'postgres_password' => 'password',- 'postgres_db' => 'postgres',- 'environment_id' => $this->environment->id,- 'destination_id' => $this->destination->id,- 'destination_type' => $this->destination->getMorphClass(),- ]);- $backup = ScheduledDatabaseBackup::create([- 'frequency' => '0 2 * * *',- 'enabled' => true,- 'database_type' => $database->getMorphClass(),- 'database_id' => $database->id,- 'team_id' => $this->team->id,- ]);-- Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => collect()])- ->assertSee('2026-09-18 02:00 (Europe/Budapest)', false)- ->call('toggleEnabled')- ->assertDontSee('2026-09-18 02:00 (Europe/Budapest)', false)- ->assertSeeInOrder(['Next run', 'Disabled']);-});--it('shows the next volume backup run in the server timezone', function () {- $volume = LocalPersistentVolume::create([- 'name' => 'app-data',- 'mount_path' => '/data',- 'resource_id' => $this->application->id,- 'resource_type' => $this->application->getMorphClass(),- ]);-- Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $this->application])- ->assertSeeInOrder(['Next run', 'Disabled'])- ->set('frequency', '0 2 * * *')- ->call('toggleEnabled')- ->assertSee('2026-09-18 02:00 (Europe/Budapest)', false);-});diff --git a/tests/Feature/ScheduledJobManagerDispatchTest.php b/tests/Feature/ScheduledJobManagerDispatchTest.phpindex 61ecfa66ef..eaff025bfc 100644--- a/tests/Feature/ScheduledJobManagerDispatchTest.php+++ b/tests/Feature/ScheduledJobManagerDispatchTest.php@@ -2,7 +2,6 @@use App\Events\BackupCreated;use App\Jobs\DatabaseBackupJob;-use App\Jobs\DockerCleanupJob;use App\Jobs\ScheduledJobManager;use App\Jobs\ScheduledTaskJob;use App\Models\Application;@@ -11,6 +10,7 @@ use App\Models\PrivateKey;use App\Models\Project;use App\Models\ScheduledDatabaseBackup;use App\Models\ScheduledJobDelivery;+use App\Models\ScheduledJobState;use App\Models\ScheduledTask;use App\Models\Server;use App\Models\ServerSetting;@@ -23,6 +23,7 @@ use App\Services\ScheduledJobDeliveryService;use Illuminate\Foundation\Testing\RefreshDatabase;use Illuminate\Support\Carbon;use Illuminate\Support\Facades\Bus;+use Illuminate\Support\Facades\Cache;use Illuminate\Support\Facades\DB;use Illuminate\Support\Facades\Event;use Illuminate\Support\Facades\Log;@@ -32,219 +33,173 @@ uses(RefreshDatabase::class);beforeEach(function () {Server::flushIdentityMap();+});++it('dispatches scheduled tasks across chunks', function () {config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));Queue::fake();-});-it('dispatches due scheduled tasks across chunks and moves their next run forward', function () {- Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 0, 30, 'UTC'));- $application = createScheduledTaskApplication();- foreach (range(1, 101) as $ignored) {- createScheduledApplicationTask($application, ['frequency' => '* * * * *']);- }+ $team = Team::factory()->create();+ $privateKey = PrivateKey::create([+ 'name' => 'Test Key',+ 'private_key' => '-----BEGIN OPENSSH PRIVATE KEY-----+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW+QyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevAAAAJi/QySHv0Mk+hwAAAAtzc2gtZWQyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevA+AAAECBQw4jg1WRT2IGHMncCiZhURCts2s24HoDS0thHnnRKVuGmoeGq/pojrsyP1pszcNV+uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA==+-----END OPENSSH PRIVATE KEY-----',+ 'team_id' => $team->id,+ ]);+ $server = Server::factory()->create([+ 'team_id' => $team->id,+ 'private_key_id' => $privateKey->id,+ ]);+ $server->settings()->update([+ 'is_reachable' => true,+ 'is_usable' => true,+ 'force_disabled' => false,+ 'docker_cleanup_frequency' => '0 * * * *',+ ]);++ $destination = StandaloneDocker::where('server_id', $server->id)->first()+ ?? StandaloneDocker::factory()->create(['server_id' => $server->id]);+ $project = Project::factory()->create(['team_id' => $team->id]);+ $environment = Environment::factory()->create(['project_id' => $project->id]);+ $application = Application::factory()->create([+ 'environment_id' => $environment->id,+ 'destination_id' => $destination->id,+ 'destination_type' => StandaloneDocker::class,+ 'status' => 'running',+ ]);++ ScheduledTask::factory()+ ->count(101)+ ->create([+ 'team_id' => $team->id,+ 'application_id' => $application->id,+ 'frequency' => '* * * * *',+ 'enabled' => true,+ ]);- Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));(new ScheduledJobManager)->handle();Queue::assertPushed(ScheduledTaskJob::class, 101);- expect(ScheduledTask::query()->pluck('next_run_at')->map->toDateTimeString()->unique()->all())- ->toBe(['2026-05-27 00:02:00']);});-it('runs a new schedule first at its next due time', function () {- Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 20, 'UTC'));- $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => '* * * * *']);+it('skips expensive dispatch for schedules outside the catch-up window', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));+ Queue::fake();++ $application = createScheduledTaskApplication();++ $task = ScheduledTask::factory()->create([+ 'team_id' => $application->environment->project->team_id,+ 'application_id' => $application->id,+ 'frequency' => '0 2 * * *',+ 'enabled' => true,+ ]);(new ScheduledJobManager)->handle();Queue::assertNotPushed(ScheduledTaskJob::class);- expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-05-27 00:02:00');+ expect(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-task:{$task->id}")->exists())->toBeFalse();});-it('does not query schedules that are not due', function () {- Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 0, 0, 'UTC'));- $application = createScheduledTaskApplication();- $database = createScheduledBackupDatabase();- $countQueries = function (): int {- DB::flushQueryLog();- DB::enableQueryLog();- (new ScheduledJobManager)->handle();- $count = count(DB::getQueryLog());- DB::disableQueryLog();-- return $count;- };- // Initialize the Docker cleanup schedules of both servers, so only the schedules below change.- Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));- $countQueries();- $withoutSchedules = $countQueries();-- foreach (range(1, 5) as $ignored) {- createScheduledApplicationTask($application, ['frequency' => '0 2 * * *']);- createScheduledDatabaseBackup($database, ['frequency' => '0 2 * * *']);- }+it('dispatches a recently missed daily task when deduplication cache is empty', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 10, 0, 'UTC'));+ Queue::fake();- expect($countQueries())->toBe($withoutSchedules);- Queue::assertNothingPushed();-});+ $application = createScheduledTaskApplication();-it('dispatches one job when several managers run for the same occurrence', function () {- Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));- $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);+ ScheduledTask::factory()->create([+ 'team_id' => $application->environment->project->team_id,+ 'application_id' => $application->id,+ 'frequency' => 'daily',+ 'enabled' => true,+ ]);- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 5, 'UTC'));- (new ScheduledJobManager)->handle();(new ScheduledJobManager)->handle();Queue::assertPushed(ScheduledTaskJob::class, 1);- expect(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-task:{$task->id}")->count())->toBe(1)- ->and($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00');});-it('does not dispatch an occurrence that another manager claims after this one read it', function () {- Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));- $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 5, 'UTC'));-- // Another node claims the occurrence between this manager's select and its update.- $claimedByOtherNode = false;- ScheduledTask::retrieved(function (ScheduledTask $retrieved) use (&$claimedByOtherNode) {- if (! $claimedByOtherNode) {- $claimedByOtherNode = true;- ScheduledTask::query()->whereKey($retrieved->id)->toBase()->update(['next_run_at' => '2026-09-18 00:00:00']);- }- });+it('dispatches one job when multiple managers evaluate the same occurrence', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));+ Queue::fake();+ $application = createScheduledTaskApplication();+ $task = ScheduledTask::factory()->create([+ 'team_id' => $application->environment->project->team_id,+ 'application_id' => $application->id,+ 'frequency' => 'daily',+ 'enabled' => true,+ ]);++ (new ScheduledJobManager)->handle();(new ScheduledJobManager)->handle();- Queue::assertNotPushed(ScheduledTaskJob::class);- expect($claimedByOtherNode)->toBeTrue()- ->and(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-task:{$task->id}")->exists())->toBeFalse();+ Queue::assertPushed(ScheduledTaskJob::class, 1);+ expect(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-task:{$task->id}")->count())->toBe(1)+ ->and(ScheduledJobState::query()->where('schedule_key', "scheduled-task:{$task->id}")->count())->toBe(1);});-it('calculates the next run of a schedule without one and runs it when it is due now', function (string $frequency, bool $runsNow, string $nextRunAt) {- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 20, 'UTC'));- $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => $frequency]);- ScheduledTask::query()->whereKey($task->id)->update(['next_run_at' => null]);-- (new ScheduledJobManager)->handle();-- Queue::assertPushed(ScheduledTaskJob::class, $runsNow ? 1 : 0);- expect($task->fresh()->next_run_at->toDateTimeString())->toBe($nextRunAt);-})->with([- 'due in the current minute' => ['0 0 * * *', true, '2026-09-18 00:00:00'],- 'due later' => ['30 * * * *', false, '2026-09-17 00:30:00'],-]);+it('does not retry an occurrence skipped while its server is not functional', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));+ Queue::fake();-it('moves the next run forward when an occurrence is skipped, and does not retry it', function () {- Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));- $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);+ $application = createScheduledTaskApplication();+ $task = createScheduledApplicationTask($application, ['frequency' => 'daily']);$server = $task->server();$server->settings()->update(['is_reachable' => false]);- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 5, 'UTC'));(new ScheduledJobManager)->handle();+$server->settings()->update(['is_reachable' => true]);- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 1, 0, 'UTC'));+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 6, 0, 'UTC'));(new ScheduledJobManager)->handle();Queue::assertNotPushed(ScheduledTaskJob::class);- expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00')- ->and(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-task:{$task->id}")->exists())->toBeFalse();+ expect(ScheduledJobState::query()->where('schedule_key', "scheduled-task:{$task->id}")->value('last_scheduled_for'))+ ->not->toBeNull()+ ->and(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-task:{$task->id}")->exists())+ ->toBeFalse();});-it('does not dispatch a task when its application is not running', function () {- Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));+it('does not publish a task occurrence when its application is not running', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));+ Queue::fake();+$application = createScheduledTaskApplication();$application->update(['status' => 'stopped']);$task = createScheduledApplicationTask($application, ['frequency' => 'daily']);- $logPath = captureScheduledJobManagerTestLog('scheduled');- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 5, 'UTC'));(new ScheduledJobManager)->handle();Queue::assertNotPushed(ScheduledTaskJob::class);- expect(file_get_contents($logPath))->toContain('Task skipped')->toContain('"skip_reason":"application_not_running"');- @unlink($logPath);+ expect(ScheduledJobState::query()->where('schedule_key', "scheduled-task:{$task->id}")->exists())->toBeTrue()+ ->and(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-task:{$task->id}")->exists())+ ->toBeFalse();});-it('runs late backups once, but logs late tasks outside the late-run window as missed', function (int $minutesLate, bool $taskRuns) {- Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));- $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);- $backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);- $logPath = captureScheduledJobManagerTestLog('scheduled');-- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC')->addMinutes($minutesLate));- (new ScheduledJobManager)->handle();-- Queue::assertPushed(ScheduledTaskJob::class, $taskRuns ? 1 : 0);- Queue::assertPushed(DatabaseBackupJob::class, 1);- expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00')- ->and($backup->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00')- ->and(str_contains(file_get_contents($logPath), 'Task missed'))->toBe(! $taskRuns);- @unlink($logPath);-})->with([- 'five minutes late' => [5, true],- 'three hours late' => [180, false],-]);+it('publishes a pending occurrence after a previous publisher interruption', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));+ Queue::fake();-it('runs a daily schedule once on the night the clocks go back', function () {- Carbon::setTestNow(Carbon::create(2026, 10, 24, 12, 0, 0, 'UTC'));$application = createScheduledTaskApplication();- $application->destination->server->settings->update(['server_timezone' => 'Europe/Budapest']);- $task = createScheduledApplicationTask($application, ['frequency' => '30 2 * * *']);-- // 02:30 CEST. At 03:00 CEST the clocks go back to 02:00, so 02:30 CET follows one hour later.- Carbon::setTestNow(Carbon::create(2026, 10, 25, 0, 30, 5, 'UTC'));- (new ScheduledJobManager)->handle();-- Queue::assertPushed(ScheduledTaskJob::class, 1);- expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-10-26 01:30:00');-});--it('dispatches due Docker cleanups with their settings', function () {- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 30, 0, 'UTC'));- $server = createScheduledTaskApplication()->destination->server;- $server->settings->update(['docker_cleanup_frequency' => '0 * * * *', 'delete_unused_volumes' => true]);-- Carbon::setTestNow(Carbon::create(2026, 9, 17, 1, 0, 10, 'UTC'));- (new ScheduledJobManager)->handle();-- Queue::assertPushed(DockerCleanupJob::class, fn (DockerCleanupJob $job) => $job->server->is($server) && $job->deleteUnusedVolumes);- expect($server->settings->fresh()->docker_cleanup_next_run_at->toDateTimeString())->toBe('2026-09-17 02:00:00');-});--it('does not dispatch an occurrence that already has a delivery', function () {- Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));- $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);- ScheduledJobDelivery::create([- 'schedule_key' => "scheduled-task:{$task->id}",- 'scheduled_for' => Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'),- 'job_type' => 'scheduled-task',- 'resource_id' => $task->id,- 'status' => 'claimed',+ $task = ScheduledTask::factory()->create([+ 'team_id' => $application->environment->project->team_id,+ 'application_id' => $application->id,+ 'frequency' => 'daily',+ 'enabled' => true,]);-- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 5, 'UTC'));- (new ScheduledJobManager)->handle();-- Queue::assertNotPushed(ScheduledTaskJob::class);- expect($task->fresh()->next_run_at->toDateTimeString())->toBe('2026-09-18 00:00:00');-});--it('dispatches a due backup of a service database', function () {- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 4, 30, 'UTC'));- $backup = createScheduledServiceDatabaseBackup(['frequency' => '* * * * *']);-- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));- (new ScheduledJobManager)->handle();-- Queue::assertPushed(DatabaseBackupJob::class, fn (DatabaseBackupJob $job) => $job->backup->is($backup));-});--it('publishes a pending occurrence after a previous publisher interruption', function () {- Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));- $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);$occurrence = ScheduledJobDelivery::create(['schedule_key' => "scheduled-task:{$task->id}",'scheduled_for' => Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'),@@ -258,37 +213,6 @@ it('publishes a pending occurrence after a previous publisher interruption', funexpect($occurrence->fresh()->status)->toBe('enqueued');});-it('keeps publishing pending occurrences after one of them fails', function () {- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));- $logPath = captureScheduledJobManagerTestLog('scheduled-errors');- $database = createScheduledBackupDatabase();- $occurrences = collect([createScheduledDatabaseBackup($database), createScheduledDatabaseBackup($database)])- ->map(fn (ScheduledDatabaseBackup $backup) => ScheduledJobDelivery::create([- 'schedule_key' => "scheduled-backup:{$backup->id}",- 'scheduled_for' => now()->startOfMinute(),- 'job_type' => 'database-backup',- 'resource_id' => $backup->id,- 'status' => 'pending',- ]));-- $dispatchAttempts = 0;- Bus::shouldReceive('dispatch')->andReturnUsing(function () use (&$dispatchAttempts) {- if (++$dispatchAttempts === 1) {- throw new RuntimeException('queue connection lost');- }- });-- app(ScheduledJobDeliveryService::class)->publishPending();-- $log = file_get_contents($logPath);- @unlink($logPath);-- expect($occurrences->first()->fresh()->status)->toBe('pending')- ->and($occurrences->last()->fresh()->status)->toBe('enqueued')- ->and($log)->toContain('Failed to publish pending scheduled occurrence')- ->toContain('queue connection lost');-});-it('allows only one worker to claim an occurrence', function () {$occurrence = ScheduledJobDelivery::create(['schedule_key' => 'scheduled-task:claim-test',@@ -352,8 +276,30 @@ it('marks stale claimed deliveries as failed', function () {expect($delivery->fresh()->status)->toBe('failed');});+it('dispatches the instance coolify-db backup even when its id is zero', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));+ Queue::fake();++ $database = createScheduledBackupDatabase();+ $backup = createScheduledDatabaseBackup($database, [+ 'id' => 0,+ 'frequency' => '* * * * *',+ ]);++ expect($backup->id)->toBe(0);++ (new ScheduledJobManager)->handle();++ Queue::assertPushed(DatabaseBackupJob::class, 1);+ Queue::assertPushed(DatabaseBackupJob::class, fn (DatabaseBackupJob $job) => $job->backup->id === 0);+});+it('dispatches zero-id schedules and continues with positive ids', function () {- Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 0, 30, 'UTC'));+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));+ Queue::fake();+$application = createScheduledTaskApplication();$database = StandalonePostgresql::create(['name' => 'coolify-db',@@ -366,14 +312,17 @@ it('dispatches zero-id schedules and continues with positive ids', function () {'destination_id' => $application->destination_id,'destination_type' => $application->destination_type,]);+$zeroIdBackup = createScheduledDatabaseBackup($database, ['id' => 0]);$positiveIdBackup = createScheduledDatabaseBackup($database);$zeroIdTask = createScheduledApplicationTask($application, ['id' => 0]);$positiveIdTask = createScheduledApplicationTask($application);- expect($zeroIdBackup->id)->toBe(0)->and($zeroIdTask->id)->toBe(0);+ expect($zeroIdBackup->id)->toBe(0)+ ->and($positiveIdBackup->id)->toBeGreaterThan(0)+ ->and($zeroIdTask->id)->toBe(0)+ ->and($positiveIdTask->id)->toBeGreaterThan(0);- Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));(new ScheduledJobManager)->handle();Queue::assertPushed(DatabaseBackupJob::class, 2);@@ -385,7 +334,14 @@ it('dispatches zero-id schedules and continues with positive ids', function () {});it('does not query relationships when constructing scheduled task jobs', function () {- $task = createScheduledApplicationTask(createScheduledTaskApplication());+ $application = createScheduledTaskApplication();++ $task = ScheduledTask::factory()->create([+ 'team_id' => $application->environment->project->team_id,+ 'application_id' => $application->id,+ 'frequency' => '* * * * *',+ 'enabled' => true,+ ])->fresh();DB::flushQueryLog();DB::enableQueryLog();@@ -397,32 +353,56 @@ it('does not query relationships when constructing scheduled task jobs', functio->and($job->timeout)->toBe(300);});-it('reads current server settings in each manager run of the same process', function () {- Carbon::setTestNow(Carbon::create(2026, 9, 15, 12, 0, 0, 'UTC'));+it('reads current server settings in each queued manager run of the same worker', function () {+ config(['constants.coolify.self_hosted' => true]);+ Queue::fakeExcept(ScheduledJobManager::class);+$database = createScheduledBackupDatabase();createScheduledDatabaseBackup($database, ['frequency' => 'daily']);$serverId = $database->destination->server_id;ServerSetting::query()->where('server_id', $serverId)->update(['is_reachable' => false]);- Carbon::setTestNow(Carbon::create(2026, 9, 16, 0, 0, 5, 'UTC'));- $this->artisan('scheduled:dispatch')->assertSuccessful();+ Carbon::setTestNow(Carbon::create(2026, 9, 16, 0, 5, 0, 'UTC'));+ dispatch(new ScheduledJobManager);Queue::assertNotPushed(DatabaseBackupJob::class);// Another worker marks the server reachable again before the next daily run.ServerSetting::query()->where('server_id', $serverId)->update(['is_reachable' => true]);- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 0, 5, 'UTC'));- $this->artisan('scheduled:dispatch')->assertSuccessful();+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));+ dispatch(new ScheduledJobManager);Queue::assertPushed(DatabaseBackupJob::class, 1);});-it('logs each step of a scheduled occurrence', function () {- Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));- $backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);+it('logs the current database server state when a backup is skipped', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));+ Queue::fake();$logPath = captureScheduledJobManagerTestLog('scheduled');+ $database = createScheduledBackupDatabase();+ $backup = createScheduledDatabaseBackup($database, ['frequency' => 'daily']);+ ServerSetting::query()->where('server_id', $database->destination->server_id)->update(['is_reachable' => false]);++ (new ScheduledJobManager)->handle();++ $log = file_get_contents($logPath);+ @unlink($logPath);++ expect($log)->toContain('Backup skipped')+ ->toContain('"skip_reason":"server_not_functional"')+ ->toContain("\"backup_id\":{$backup->id}")+ ->toContain('"db_is_reachable":false');+});++it('logs each step of a scheduled occurrence', function () {+ config(['constants.coolify.self_hosted' => true]);Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));+ Queue::fake();+ $logPath = captureScheduledJobManagerTestLog('scheduled');++ $backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);(new ScheduledJobManager)->handle();$occurrenceUuid = ScheduledJobDelivery::query()->where('schedule_key', "scheduled-backup:{$backup->id}")->value('uuid');$service = app(ScheduledJobDeliveryService::class);@@ -437,6 +417,7 @@ it('logs each step of a scheduled occurrence', function () {expect($log)->toContain('Backup dispatched')->toContain('Scheduled occurrence enqueued')->toContain('"occurrence_uuid":"'.$occurrenceUuid.'"')+ ->toContain('"host":"'.gethostname().'"')->toContain('Scheduled occurrence claimed')->toContain('Scheduled occurrence claim rejected')->toContain('Scheduled occurrence completed')@@ -444,83 +425,220 @@ it('logs each step of a scheduled occurrence', function () {->toContain('"seconds_since_due":300');});+it('warns when the manager starts after the catch-up window', function (int $minutesSincePreviousStart, bool $expectWarning) {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 30, 0, 'UTC'));+ Queue::fake();+ $logPath = captureScheduledJobManagerTestLog('scheduled-errors');+ Cache::put('scheduled-job-manager:last-started-at', now()->subMinutes($minutesSincePreviousStart)->toIso8601String());++ (new ScheduledJobManager)->handle();++ $log = file_get_contents($logPath);+ @unlink($logPath);++ expect(str_contains($log, 'ScheduledJobManager started late'))->toBe($expectWarning);+})->with([+ 'normal one-minute interval' => [1, false],+ 'twenty-minute gap' => [20, true],+]);++it('warns when an occurrence is enqueued while the previous one is still running', function () {+ config(['constants.coolify.self_hosted' => true]);+ Queue::fake();+ $logPath = captureScheduledJobManagerTestLog('scheduled');+ $backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);++ Carbon::setTestNow(Carbon::create(2026, 9, 16, 0, 5, 0, 'UTC'));+ (new ScheduledJobManager)->handle();+ $previousUuid = ScheduledJobDelivery::query()->where('schedule_key', "scheduled-backup:{$backup->id}")->value('uuid');+ app(ScheduledJobDeliveryService::class)->claim($previousUuid, 'worker-a');++ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));+ (new ScheduledJobManager)->handle();++ $log = file_get_contents($logPath);+ @unlink($logPath);++ expect($log)->toContain('Scheduled occurrence enqueued while the previous occurrence is still open')+ ->toContain('"previous_occurrence_uuid":"'.$previousUuid.'"')+ ->toContain('"previous_status":"claimed"');+});++it('reports occurrences that stay open after fifteen minutes', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 30, 0, 'UTC'));+ Queue::fake();+ $logPath = captureScheduledJobManagerTestLog('scheduled');++ foreach (['enqueued' => 20, 'claimed' => 20, 'failed' => 20] as $status => $minutesAgo) {+ ScheduledJobDelivery::create([+ 'schedule_key' => "scheduled-backup:{$status}",+ 'scheduled_for' => now()->subMinutes($minutesAgo),+ 'job_type' => 'database-backup',+ 'resource_id' => 1,+ 'status' => $status,+ ])->forceFill(['created_at' => now()->subMinutes($minutesAgo)])->save();+ }+ ScheduledJobDelivery::create([+ 'schedule_key' => 'scheduled-backup:recent',+ 'scheduled_for' => now()->subMinutes(2),+ 'job_type' => 'database-backup',+ 'resource_id' => 1,+ 'status' => 'enqueued',+ ]);++ (new ScheduledJobManager)->handle();++ $log = file_get_contents($logPath);+ @unlink($logPath);++ expect($log)->toContain('"open_occurrences_over_15_minutes":{"claimed":1,"enqueued":1}');+});++it('keeps publishing pending occurrences after one of them fails', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));+ $logPath = captureScheduledJobManagerTestLog('scheduled-errors');+ $database = createScheduledBackupDatabase();+ $occurrences = collect([createScheduledDatabaseBackup($database), createScheduledDatabaseBackup($database)])+ ->map(fn (ScheduledDatabaseBackup $backup) => ScheduledJobDelivery::create([+ 'schedule_key' => "scheduled-backup:{$backup->id}",+ 'scheduled_for' => now()->startOfMinute(),+ 'job_type' => 'database-backup',+ 'resource_id' => $backup->id,+ 'status' => 'pending',+ ]));++ $dispatchAttempts = 0;+ Bus::shouldReceive('dispatch')->andReturnUsing(function () use (&$dispatchAttempts) {+ if (++$dispatchAttempts === 1) {+ throw new RuntimeException('queue connection lost');+ }+ });++ app(ScheduledJobDeliveryService::class)->publishPending();++ $log = file_get_contents($logPath);+ @unlink($logPath);++ expect($occurrences->first()->fresh()->status)->toBe('pending')+ ->and($occurrences->last()->fresh()->status)->toBe('enqueued')+ ->and($log)->toContain('Failed to publish pending scheduled occurrence')+ ->toContain('queue connection lost');+});+it('logs a backup run that ends early, even when it completes before the enqueue log', function () {- Carbon::setTestNow(Carbon::create(2026, 9, 16, 12, 0, 0, 'UTC'));+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));Event::fake([BackupCreated::class]);- Queue::fakeExcept([DatabaseBackupJob::class]);+ Queue::fakeExcept([ScheduledJobManager::class, DatabaseBackupJob::class]);+ $logPath = captureScheduledJobManagerTestLog('scheduled');+$database = createScheduledBackupDatabase();$database->update(['status' => 'exited']);$backup = createScheduledDatabaseBackup($database, ['frequency' => 'daily']);- $logPath = captureScheduledJobManagerTestLog('scheduled');- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));- $this->artisan('scheduled:dispatch')->assertSuccessful();+ dispatch(new ScheduledJobManager);$log = file_get_contents($logPath);@unlink($logPath);expect($log)->toContain('Database backup job ended without a backup')->toContain('"skip_reason":"database_not_running"')+ ->toContain('"database_status":"exited:unhealthy"')->toContain('Scheduled occurrence completed')->toMatch('/Scheduled occurrence enqueued \{[^\n]*"schedule_key":"scheduled-backup:'.$backup->id.'"[^\n]*"status":"enqueued"/')->and(ScheduledJobDelivery::query()->where('schedule_key', "scheduled-backup:{$backup->id}")->exists())->toBeFalse();});-it('logs an invalid frequency once and runs the schedule again after the frequency is fixed', function () {- Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));- $logPath = captureScheduledJobManagerTestLog('scheduled-errors');- $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => '* * * * *']);- ScheduledTask::query()->whereKey($task->id)->update(['frequency' => 'not a cron', 'next_run_at' => now()->subMinute()]);+it('runs the manager in the scheduler process through the scheduled:dispatch command', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 5, 27, 0, 1, 0, 'UTC'));+ Queue::fakeExcept(ScheduledJobManager::class);++ $application = createScheduledTaskApplication();+ createScheduledApplicationTask($application, ['frequency' => '* * * * *']);++ $this->artisan('scheduled:dispatch')->assertSuccessful();- foreach (range(1, 3) as $minute) {- Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, $minute, 0, 'UTC'));- (new ScheduledJobManager)->handle();+ Queue::assertNotPushed(ScheduledJobManager::class);+ Queue::assertPushed(ScheduledTaskJob::class, 1);+});++it('loads the server chain of scheduled database backups without a query for each backup', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));+ Queue::fake();++ $database = createScheduledBackupDatabase();+ createScheduledDatabaseBackup($database, ['frequency' => '0 2 * * *']);+ foreach (range(1, 2) as $number) {+ $otherDatabase = StandalonePostgresql::create([+ ...$database->only(['image', 'postgres_user', 'postgres_password', 'postgres_db', 'status', 'environment_id', 'destination_id', 'destination_type']),+ 'name' => "database-{$number}",+ ]);+ createScheduledDatabaseBackup($otherDatabase, ['frequency' => '0 2 * * *']);}+ createScheduledServiceDatabaseBackup(['frequency' => '0 2 * * *'], Application::query()->first());- $log = file_get_contents($logPath);- @unlink($logPath);+ DB::flushQueryLog();+ DB::enableQueryLog();+ (new ScheduledJobManager)->handle();+ $queries = collect(DB::getQueryLog())->pluck('query');+ DB::disableQueryLog();- Queue::assertNotPushed(ScheduledTaskJob::class);- expect(substr_count($log, 'Task has an invalid frequency'))->toBe(1)- ->and($task->fresh()->next_run_at)->toBeNull()- ->and($task->fresh()->enabled)->toBeTrue();+ $perBackupLookups = $queries->filter(fn (string $query) => str_contains($query, '"standalone_dockers"."id" = ?')+ || str_contains($query, '"services"."id" = ?')+ || str_contains($query, '"server_settings"."server_id" = ?'));++ expect($perBackupLookups)->toBeEmpty();+});++it('dispatches a due backup of a service database', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));+ Queue::fake();++ $backup = createScheduledServiceDatabaseBackup(['frequency' => '* * * * *']);- $task->fresh()->update(['frequency' => '* * * * *']);- Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 4, 0, 'UTC'));(new ScheduledJobManager)->handle();- Queue::assertPushed(ScheduledTaskJob::class, 1);+ Queue::assertPushed(DatabaseBackupJob::class, fn (DatabaseBackupJob $job) => $job->backup->is($backup));});-it('logs an invalid frequency of a schedule without a next run only once', function () {- Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));+it('keeps dispatching other schedules when one schedule throws a PHP error', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));$logPath = captureScheduledJobManagerTestLog('scheduled-errors');$database = createScheduledBackupDatabase();- $backup = createScheduledDatabaseBackup($database, ['frequency' => 'daily']);- ScheduledDatabaseBackup::query()->whereKey($backup->id)->update(['frequency' => '61 * * * *', 'next_run_at' => null]);+ createScheduledDatabaseBackup($database);+ createScheduledDatabaseBackup($database);- foreach (range(1, 3) as $minute) {- Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, $minute, 0, 'UTC'));- (new ScheduledJobManager)->handle();- }+ $dispatchAttempts = 0;+ Bus::shouldReceive('dispatch')->andReturnUsing(function () use (&$dispatchAttempts) {+ if (++$dispatchAttempts === 1) {+ throw new Error('broken schedule');+ }+ });++ (new ScheduledJobManager)->handle();$log = file_get_contents($logPath);@unlink($logPath);- expect(substr_count($log, 'Backup has an invalid frequency'))->toBe(1);+ // The first backup stays pending for publishPending(); the second one is still enqueued.+ expect(ScheduledJobDelivery::query()->where('job_type', 'database-backup')->orderBy('id')->pluck('status')->all())->toBe(['pending', 'enqueued'])+ ->and($log)->toContain('Error processing backup')+ ->toContain('broken schedule');});it('republishes a backup occurrence whose queued job was lost, and the old job cannot run it again', function () {+ config(['constants.coolify.self_hosted' => true]);Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));+ Queue::fake();$backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);- $occurrence = ScheduledJobDelivery::create([- 'schedule_key' => "scheduled-backup:{$backup->id}",- 'scheduled_for' => Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'),- 'job_type' => 'database-backup',- 'resource_id' => $backup->id,- 'status' => 'enqueued',- 'enqueued_at' => now()->subMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1),- ]);+ $occurrence = createStaleEnqueuedOccurrence("scheduled-backup:{$backup->id}", 'database-backup', $backup->id, Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'));(new ScheduledJobManager)->handle();@@ -533,18 +651,32 @@ it('republishes a backup occurrence whose queued job was lost, and the old job c->and($service->claim($occurrence->uuid, 'original-job'))->toBeFalse();});+it('does not run a lost backup occurrence again when a newer occurrence exists', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));+ Queue::fake();+ $logPath = captureScheduledJobManagerTestLog('scheduled');+ $backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'hourly']);+ $lost = createStaleEnqueuedOccurrence("scheduled-backup:{$backup->id}", 'database-backup', $backup->id, Carbon::create(2026, 9, 17, 10, 0, 0, 'UTC'));+ createStaleEnqueuedOccurrence("scheduled-backup:{$backup->id}", 'database-backup', $backup->id, Carbon::create(2026, 9, 17, 11, 0, 0, 'UTC'), minutesAgo: 5);++ (new ScheduledJobManager)->handle();++ $log = file_get_contents($logPath);+ @unlink($logPath);++ Queue::assertNotPushed(DatabaseBackupJob::class, fn (DatabaseBackupJob $job) => $job->occurrenceUuid === $lost->uuid);+ expect($lost->fresh()->status)->toBe('failed')+ ->and($log)->toContain('Scheduled occurrence skipped: its queued job was not started and a newer occurrence exists');+});+it('logs a lost scheduled task occurrence as missed instead of running it late', function () {+ config(['constants.coolify.self_hosted' => true]);Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));+ Queue::fake();$logPath = captureScheduledJobManagerTestLog('scheduled');$task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);- $occurrence = ScheduledJobDelivery::create([- 'schedule_key' => "scheduled-task:{$task->id}",- 'scheduled_for' => Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'),- 'job_type' => 'scheduled-task',- 'resource_id' => $task->id,- 'status' => 'enqueued',- 'enqueued_at' => now()->subMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1),- ]);+ $occurrence = createStaleEnqueuedOccurrence("scheduled-task:{$task->id}", 'scheduled-task', $task->id, Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'));(new ScheduledJobManager)->handle();@@ -558,16 +690,11 @@ it('logs a lost scheduled task occurrence as missed instead of running it late',});it('leaves recently enqueued occurrences alone', function () {+ config(['constants.coolify.self_hosted' => true]);Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));+ Queue::fake();$backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);- $occurrence = ScheduledJobDelivery::create([- 'schedule_key' => "scheduled-backup:{$backup->id}",- 'scheduled_for' => Carbon::create(2026, 9, 17, 11, 0, 0, 'UTC'),- 'job_type' => 'database-backup',- 'resource_id' => $backup->id,- 'status' => 'enqueued',- 'enqueued_at' => now()->subMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES - 1),- ]);+ $occurrence = createStaleEnqueuedOccurrence("scheduled-backup:{$backup->id}", 'database-backup', $backup->id, Carbon::create(2026, 9, 17, 11, 0, 0, 'UTC'), minutesAgo: ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES - 1);(new ScheduledJobManager)->handle();@@ -575,6 +702,46 @@ it('leaves recently enqueued occurrences alone', function () {expect($occurrence->fresh()->enqueued_at->toDateTimeString())->toBe('2026-09-17 11:01:00');});+function createStaleEnqueuedOccurrence(string $scheduleKey, string $jobType, int $resourceId, Carbon $scheduledFor, int $minutesAgo = ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1): ScheduledJobDelivery+{+ return ScheduledJobDelivery::create([+ 'schedule_key' => $scheduleKey,+ 'scheduled_for' => $scheduledFor,+ 'job_type' => $jobType,+ 'resource_id' => $resourceId,+ 'status' => 'enqueued',+ 'enqueued_at' => now()->subMinutes($minutesAgo),+ ]);+}++function createScheduledServiceDatabaseBackup(array $overrides = [], ?Application $application = null): ScheduledDatabaseBackup+{+ $application ??= createScheduledTaskApplication();+ $service = Service::factory()->create([+ 'environment_id' => $application->environment_id,+ 'destination_id' => $application->destination_id,+ 'destination_type' => $application->destination_type,+ ]);+ $database = ServiceDatabase::create([+ 'name' => 'postgres',+ 'image' => 'postgres:16-alpine',+ 'service_id' => $service->id,+ ]);++ $backup = new ScheduledDatabaseBackup;+ $backup->forceFill(array_merge([+ 'enabled' => true,+ 'save_s3' => false,+ 'frequency' => '* * * * *',+ 'database_id' => $database->id,+ 'database_type' => $database->getMorphClass(),+ 'team_id' => $application->environment->project->team_id,+ ], $overrides));+ $backup->save();++ return $backup->fresh();+}+function captureScheduledJobManagerTestLog(string $channel): string{$path = tempnam(sys_get_temp_dir(), 'coolify-scheduled-log-');@@ -655,34 +822,6 @@ function createScheduledDatabaseBackup(StandalonePostgresql $database, array $ovreturn $backup->fresh();}-function createScheduledServiceDatabaseBackup(array $overrides = []): ScheduledDatabaseBackup-{- $application = createScheduledTaskApplication();- $service = Service::factory()->create([- 'environment_id' => $application->environment_id,- 'destination_id' => $application->destination_id,- 'destination_type' => $application->destination_type,- ]);- $database = ServiceDatabase::create([- 'name' => 'postgres',- 'image' => 'postgres:16-alpine',- 'service_id' => $service->id,- ]);-- $backup = new ScheduledDatabaseBackup;- $backup->forceFill(array_merge([- 'enabled' => true,- 'save_s3' => false,- 'frequency' => '* * * * *',- 'database_id' => $database->id,- 'database_type' => $database->getMorphClass(),- 'team_id' => $application->environment->project->team_id,- ], $overrides));- $backup->save();-- return $backup->fresh();-}-function createScheduledApplicationTask(Application $application, array $overrides = []): ScheduledTask{$task = new ScheduledTask;diff --git a/tests/Feature/VolumeBackupTest.php b/tests/Feature/VolumeBackupTest.phpindex a161864408..623e140dfb 100644--- a/tests/Feature/VolumeBackupTest.php+++ b/tests/Feature/VolumeBackupTest.php@@ -2519,9 +2519,9 @@ it('dispatches due scheduled volume backups', function () {'frequency' => '* * * * *','enabled' => true,]);+ Cache::forget("scheduled-volume-backup:{$backup->id}");expect($backup->server()?->isFunctional())->toBeTrue();- $this->travel(1)->minutes();(new ScheduledJobManager)->handle();Queue::assertPushed(@@ -2547,8 +2547,8 @@ it('dispatches due scheduled directory backups', function () {'frequency' => '* * * * *','enabled' => true,]);+ Cache::forget("scheduled-volume-backup:{$backup->id}");- $this->travel(1)->minutes();(new ScheduledJobManager)->handle();Queue::assertPushed(@@ -2577,7 +2577,6 @@ it('retains scheduled volume backups and archive metadata when the server is mis'destination_type' => null,]);- $this->travel(1)->minutes();(new ScheduledJobManager)->handle();expect($backup->fresh())->not->toBeNull()@@ -2607,7 +2606,6 @@ it('dispatches pending recovery without starting another volume backup', functio'stop_recovery_pending' => true,]);- $this->travel(1)->minutes();(new ScheduledJobManager)->handle();Queue::assertPushed(diff --git a/tests/Unit/ScheduledJobManagerLockTest.php b/tests/Unit/ScheduledJobManagerLockTest.phpnew file mode 100644index 0000000000..924f6f43cf--- /dev/null+++ b/tests/Unit/ScheduledJobManagerLockTest.php@@ -0,0 +1,63 @@+<?php++use App\Jobs\ScheduledJobManager;+use Illuminate\Queue\Middleware\WithoutOverlapping;+use Tests\TestCase;++uses(TestCase::class);++it('uses WithoutOverlapping middleware with expireAfter to prevent stale locks', function () {+ $job = new ScheduledJobManager;+ $middleware = $job->middleware();++ // Assert middleware exists+ expect($middleware)->toBeArray()+ ->and($middleware)->toHaveCount(1);++ $overlappingMiddleware = $middleware[0];++ // Assert it's a WithoutOverlapping instance+ expect($overlappingMiddleware)->toBeInstanceOf(WithoutOverlapping::class);++ // Use reflection to check private properties+ $reflection = new ReflectionClass($overlappingMiddleware);++ // Check expireAfter is set (should be 60 seconds - matches job frequency)+ $expiresAfterProperty = $reflection->getProperty('expiresAfter');+ $expiresAfterProperty->setAccessible(true);+ $expiresAfter = $expiresAfterProperty->getValue($overlappingMiddleware);++ expect($expiresAfter)->toBe(90)+ ->and($expiresAfter)->toBeGreaterThan(0, 'expireAfter must be set to prevent stale locks');++ // Check releaseAfter is NOT set (we use dontRelease)+ $releaseAfterProperty = $reflection->getProperty('releaseAfter');+ $releaseAfterProperty->setAccessible(true);+ $releaseAfter = $releaseAfterProperty->getValue($overlappingMiddleware);++ expect($releaseAfter)->toBeNull('releaseAfter should be null when using dontRelease()');++ // Check the lock key+ $keyProperty = $reflection->getProperty('key');+ $keyProperty->setAccessible(true);+ $key = $keyProperty->getValue($overlappingMiddleware);++ expect($key)->toBe('scheduled-job-manager');+});++it('prevents stale locks by ensuring expireAfter is always set', function () {+ $job = new ScheduledJobManager;+ $middleware = $job->middleware();++ $overlappingMiddleware = $middleware[0];+ $reflection = new ReflectionClass($overlappingMiddleware);++ $expiresAfterProperty = $reflection->getProperty('expiresAfter');+ $expiresAfterProperty->setAccessible(true);+ $expiresAfter = $expiresAfterProperty->getValue($overlappingMiddleware);++ // Critical check: expireAfter MUST be set to prevent GitHub issue #4539+ expect($expiresAfter)->not->toBeNull(+ 'expireAfter() is required to prevent stale locks (see GitHub #4539)'+ );+});diff --git a/tests/Unit/ServerPlaceholderIpTest.php b/tests/Unit/ServerPlaceholderIpTest.phpindex e7c0f25069..94787b4d47 100644--- a/tests/Unit/ServerPlaceholderIpTest.php+++ b/tests/Unit/ServerPlaceholderIpTest.php@@ -4,10 +4,8 @@ use App\Jobs\CleanupOrphanedPreviewContainersJob;use App\Jobs\ScheduledJobManager;use App\Models\PrivateKey;use App\Models\Server;-use App\Models\ServerSetting;use App\Models\Team;use Illuminate\Foundation\Testing\RefreshDatabase;-use Illuminate\Support\Facades\Queue;use Tests\TestCase;uses(TestCase::class, RefreshDatabase::class);@@ -101,13 +99,10 @@ it('excludes every placeholder address from scheduled Docker cleanup', function]);}- ServerSetting::query()->update(['docker_cleanup_next_run_at' => null]);- Queue::fake();+ $method = new ReflectionMethod(ScheduledJobManager::class, 'getServersForCleanupQuery');+ $servers = $method->invoke(new ScheduledJobManager)->get();- (new ScheduledJobManager)->handle();-- // The dispatcher calculates the next cleanup only for servers that it selects.- expect(ServerSetting::query()->whereNotNull('docker_cleanup_next_run_at')->pluck('server_id')->all())->toBe([$realServer->id]);+ expect($servers->modelKeys())->toBe([$realServer->id]);});it('excludes every placeholder address from orphaned preview cleanup', function () {