An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(scheduler): bound occurrence recovery and fail interrupted claims
Andras Bacsaiedf1ab03a722
diff --git a/.ai/lessons.md b/.ai/lessons.mdindex 948fbcda3f..7ec10c1506 100644--- a/.ai/lessons.md+++ b/.ai/lessons.md@@ -65,6 +65,7 @@- A Redis instance that holds queues must use `maxmemory-policy noeviction`; an evicting policy deletes queued jobs without an error. Before changing scheduler code for missed runs, check `INFO stats` `evicted_keys`.- Keep pending occurrences recoverable across publisher interruptions, and define an explicit bounded policy for late or offline schedules.- Horizon workers are long-lived: flush every static or `once()` cache (for example `Server::flushIdentityMap()`) in `Queue::before`, or later jobs decide with stale state.+- A job of a killed worker comes back after `retry_after` (one day) as attempt 2. With `tries > 1` it runs again a day late; queued jobs that must not run late need `tries = 1` or an attempt guard, and `failed()` must not guess its execution row from "the latest" record.## Fail closed at public webhook boundaries- Reject missing or blank secrets before signature verification, and return generic errors without logging secrets, signatures, or payloads.diff --git a/app/Jobs/ScheduledJobManager.php b/app/Jobs/ScheduledJobManager.phpindex c5d553d850..d3af90a996 100644--- a/app/Jobs/ScheduledJobManager.php+++ b/app/Jobs/ScheduledJobManager.php@@ -136,16 +136,21 @@ class ScheduledJobManager implements ShouldQueue$this->logStart();$jobTypes = $this->type === null ? null : self::TYPES[$this->type];- try {- app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued($jobTypes);- } catch (\Throwable $e) {- Log::channel('scheduled-errors')->error('Failed to recover stale enqueued occurrences', [- 'error' => $e->getMessage(),- ]);+ $deliveries = app(ScheduledJobDeliveryService::class);+ foreach ([+ 'Failed to recover stale enqueued occurrences' => fn () => $deliveries->recoverStaleEnqueued($jobTypes),+ 'Failed to recover interrupted occurrences' => fn () => $deliveries->failInterruptedClaims($jobTypes),+ 'Failed to publish pending occurrences' => fn () => $deliveries->publishPending($jobTypes),+ ] as $errorMessage => $recover) {+ try {+ $recover();+ } catch (\Throwable $e) {+ Log::channel('scheduled-errors')->error($errorMessage, [+ 'error' => $e->getMessage(),+ ]);+ }}- app(ScheduledJobDeliveryService::class)->publishPending($jobTypes);-// Process scheduled backups and tasks together so neither type starves the other.try {if ($this->includes('backups') || $this->includes('tasks')) {diff --git a/app/Jobs/ScheduledTaskJob.php b/app/Jobs/ScheduledTaskJob.phpindex ac8e3f5939..e3a352ebb1 100644--- a/app/Jobs/ScheduledTaskJob.php+++ b/app/Jobs/ScheduledTaskJob.php@@ -107,6 +107,15 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueuepublic function handle(): void{+ // maxExceptions = 1 fails the job on its first error, so a second attempt only happens+ // when Redis hands the job out again after its worker was killed (retry_after, about a+ // day later). The command may have run already, so it is not run again late.+ if ($this->attempts() > 1) {+ $this->fail(new \RuntimeException('The worker stopped while this task was running. The task is not run again late.'));++ return;+ }+if ($this->occurrenceUuid && ! app(ScheduledJobDeliveryService::class)->claim($this->occurrenceUuid, $this->job?->uuid() ?? $this->occurrenceUuid)) {return;}@@ -266,10 +275,12 @@ class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue$execution = ScheduledTaskExecution::find($this->executionId);}- // If no stored ID or not found, query for the most recent execution log for this task+ // If no stored ID or not found, query for the most recent unfinished execution log for this+ // task. A finished execution belongs to another run and must keep its result.if (! $execution) {$execution = ScheduledTaskExecution::query()->where('scheduled_task_id', $this->task->id)+ ->whereNull('finished_at')->orderBy('created_at', 'desc')->first();}diff --git a/app/Services/ScheduledJobDeliveryService.php b/app/Services/ScheduledJobDeliveryService.phpindex a2046484f0..8609b78b80 100644--- a/app/Services/ScheduledJobDeliveryService.php+++ b/app/Services/ScheduledJobDeliveryService.php@@ -39,6 +39,18 @@ class ScheduledJobDeliveryService*/public const ENQUEUED_STALE_AFTER_MINUTES = 60;+ /**+ * At most this many open occurrences are published again or recovered per scheduler run, so a+ * large backlog after an outage is worked off in steps instead of in one long scheduler tick.+ */+ public const RECOVERY_BATCH_SIZE = 500;++ /**+ * A claimed occurrence that started longer ago than the worker timeout plus this margin has no+ * live worker: Horizon kills every job at the worker timeout.+ */+ public const INTERRUPTED_CLAIM_MARGIN_MINUTES = 15;+public const MISSED_OCCURRENCE_NOTIFICATION = 'scheduled-occurrence-missed';public const MISSED_OCCURRENCE_NOTIFICATION_INTERVAL_MINUTES = 60;@@ -49,6 +61,12 @@ class ScheduledJobDeliveryService*/private const LATE_RUN_JOB_TYPES = ['database-backup', 'volume-backup'];+ /**+ * A lost backup occurrence runs late only while it is younger than this. Older work is not+ * replayed, for example after a long Redis or worker outage.+ */+ public const LATE_RUN_MAX_AGE_HOURS = 24;+public function recordAndPublish(string $scheduleKey,string $frequency,@@ -66,7 +84,7 @@ class ScheduledJobDeliveryServicereturn false;}- $delivery = DB::transaction(function () use ($scheduleKey, $scheduledFor, $jobType, $resourceId, $payload): ?ScheduledJobDelivery {+ $delivery = DB::transaction(function () use ($scheduleKey, $scheduledFor, $cron, $timezone, $jobType, $resourceId, $payload): ?ScheduledJobDelivery {ScheduledJobState::query()->insertOrIgnore(['uuid' => new_public_id(),'schedule_key' => $scheduleKey,@@ -79,7 +97,7 @@ class ScheduledJobDeliveryService->lockForUpdate()->firstOrFail();- if ($state->last_scheduled_for?->gte($scheduledFor)) {+ if ($this->isDuplicateOccurrence($state, $scheduledFor, $cron, $timezone)) {return null;}@@ -116,7 +134,7 @@ class ScheduledJobDeliveryServicereturn false;}- return DB::transaction(function () use ($scheduleKey, $scheduledFor): bool {+ return DB::transaction(function () use ($scheduleKey, $scheduledFor, $cron, $timezone): bool {ScheduledJobState::query()->insertOrIgnore(['uuid' => new_public_id(),'schedule_key' => $scheduleKey,@@ -129,7 +147,7 @@ class ScheduledJobDeliveryService->lockForUpdate()->firstOrFail();- if ($state->last_scheduled_for?->gte($scheduledFor)) {+ if ($this->isDuplicateOccurrence($state, $scheduledFor, $cron, $timezone)) {return false;}@@ -140,28 +158,64 @@ class ScheduledJobDeliveryService}/**+ * The same schedule can have the same local time twice when daylight saving time ends. A+ * schedule with fixed hours (for example `30 2 * * *`) then runs once, like cron does. Interval+ * schedules (`*` or `/` in the hour field) still run in both hours.+ */+ private function isDuplicateOccurrence(ScheduledJobState $state, Carbon $scheduledFor, CronExpression $cron, string $timezone): bool+ {+ $last = $state->last_scheduled_for;+ if ($last === null) {+ return false;+ }++ if ($last->gte($scheduledFor)) {+ return true;+ }++ if (preg_match('/^[\d,\-]+$/', (string) $cron->getExpression(CronExpression::HOUR)) !== 1) {+ return false;+ }++ $format = 'Y-m-d H:i';++ return $last->copy()->setTimezone($timezone)->format($format) === $scheduledFor->copy()->setTimezone($timezone)->format($format);+ }++ /**+ * Publish occurrences again whose publication did not finish, for example because Redis+ * rejected the job. An occurrence that stays pending longer than the stale limit follows the+ * same late-run policy as a lost queued job (see recoverStaleEnqueued()).+ ** @param array<int, string>|null $jobTypes Only these job types, or all when null.*/public function publishPending(?array $jobTypes = null): void{- ScheduledJobDelivery::query()+ $occurrences = ScheduledJobDelivery::query()->where('status', 'pending')->when($jobTypes !== null, fn ($query) => $query->whereIn('job_type', $jobTypes))->orderBy('id')- ->chunkById(100, function ($occurrences): void {- foreach ($occurrences as $occurrence) {- try {- $this->logOccurrence($occurrence, 'Scheduled occurrence publishing again', 'warning');- $this->publish($occurrence);- } catch (\Throwable $e) {- Log::channel('scheduled-errors')->error('Failed to publish pending scheduled occurrence', [- 'occurrence_uuid' => $occurrence->uuid,- 'schedule_key' => $occurrence->schedule_key,- 'error' => $e->getMessage(),- ]);- }+ ->limit(self::RECOVERY_BATCH_SIZE)+ ->get();++ foreach ($occurrences as $occurrence) {+ try {+ if ($occurrence->created_at->lt(now()->subMinutes(self::ENQUEUED_STALE_AFTER_MINUTES))) {+ $this->recoverUnstartedOccurrence($occurrence);++ continue;}- });++ $this->logOccurrence($occurrence, 'Scheduled occurrence publishing again', 'warning');+ $this->publish($occurrence);+ } catch (\Throwable $e) {+ Log::channel('scheduled-errors')->error('Failed to publish pending scheduled occurrence', [+ 'occurrence_uuid' => $occurrence->uuid,+ 'schedule_key' => $occurrence->schedule_key,+ 'error' => $e->getMessage(),+ ]);+ }+ }}/**@@ -191,27 +245,70 @@ class ScheduledJobDeliveryService*/public function recoverStaleEnqueued(?array $jobTypes = null): void{- ScheduledJobDelivery::query()+ $occurrences = ScheduledJobDelivery::query()->where('status', 'enqueued')->when($jobTypes !== null, fn ($query) => $query->whereIn('job_type', $jobTypes))->where('enqueued_at', '<', now()->subMinutes(self::ENQUEUED_STALE_AFTER_MINUTES))- ->chunkById(100, function ($occurrences): void {- foreach ($occurrences as $occurrence) {- try {- $this->recoverStaleEnqueuedOccurrence($occurrence);- } catch (\Throwable $e) {- Log::channel('scheduled-errors')->error('Failed to recover enqueued scheduled occurrence', [- 'occurrence_uuid' => $occurrence->uuid,- 'schedule_key' => $occurrence->schedule_key,- 'error' => $e->getMessage(),- ]);- }- }- });+ ->orderBy('id')+ ->limit(self::RECOVERY_BATCH_SIZE)+ ->get();++ foreach ($occurrences as $occurrence) {+ try {+ $this->recoverUnstartedOccurrence($occurrence);+ } catch (\Throwable $e) {+ Log::channel('scheduled-errors')->error('Failed to recover enqueued scheduled occurrence', [+ 'occurrence_uuid' => $occurrence->uuid,+ 'schedule_key' => $occurrence->schedule_key,+ 'error' => $e->getMessage(),+ ]);+ }+ }}- private function recoverStaleEnqueuedOccurrence(ScheduledJobDelivery $occurrence): void+ /**+ * Mark claimed occurrences as failed when no worker can still run them. Horizon kills a job at+ * the worker timeout, so a claim older than that lost its worker (for example a killed+ * container). The job is not run again: it can have done part of its remote work already.+ *+ * @param array<int, string>|null $jobTypes Only these job types, or all when null.+ */+ public function failInterruptedClaims(?array $jobTypes = null): void{+ $startedBefore = now()+ ->subSeconds((int) config('horizon.worker_timeout'))+ ->subMinutes(self::INTERRUPTED_CLAIM_MARGIN_MINUTES);++ $occurrences = ScheduledJobDelivery::query()+ ->where('status', 'claimed')+ ->when($jobTypes !== null, fn ($query) => $query->whereIn('job_type', $jobTypes))+ ->where('started_at', '<', $startedBefore)+ ->orderBy('id')+ ->limit(self::RECOVERY_BATCH_SIZE)+ ->get(['id', 'uuid', 'claim_token']);++ foreach ($occurrences as $occurrence) {+ $failed = ScheduledJobDelivery::query()+ ->whereKey($occurrence->id)+ ->where('status', 'claimed')+ ->where('claim_token', $occurrence->claim_token)+ ->update(['status' => 'failed', 'updated_at' => now()]) === 1;++ if ($failed) {+ $this->logOccurrence($occurrence->uuid, 'Scheduled occurrence interrupted: its worker stopped after it claimed the occurrence', 'warning');+ }+ }+ }++ /**+ * Decide what happens to an occurrence that never started: a pending row whose publication kept+ * failing, or an enqueued row whose queued job was lost. Backups run once late unless a newer+ * occurrence exists; tasks and Docker cleanups are reported as missed.+ */+ private function recoverUnstartedOccurrence(ScheduledJobDelivery $occurrence): void+ {+ $staleBefore = now()->subMinutes(self::ENQUEUED_STALE_AFTER_MINUTES);+// complete() deletes the row of a finished occurrence, so the schedule state also counts.$hasNewerOccurrence = ScheduledJobState::query()->where('schedule_key', $occurrence->schedule_key)@@ -221,13 +318,18 @@ class ScheduledJobDeliveryService->where('schedule_key', $occurrence->schedule_key)->where('scheduled_for', '>', $occurrence->scheduled_for)->exists();- $runLate = in_array($occurrence->job_type, self::LATE_RUN_JOB_TYPES, true) && ! $hasNewerOccurrence;+ $tooOld = $occurrence->scheduled_for->lt(now()->subHours(self::LATE_RUN_MAX_AGE_HOURS));+ $runLate = in_array($occurrence->job_type, self::LATE_RUN_JOB_TYPES, true) && ! $hasNewerOccurrence && ! $tooOld;// Only one scheduler node can move each occurrence.$moved = ScheduledJobDelivery::query()->whereKey($occurrence->id)- ->where('status', 'enqueued')- ->where('enqueued_at', '<', now()->subMinutes(self::ENQUEUED_STALE_AFTER_MINUTES))+ ->where('status', $occurrence->status)+ ->when(+ $occurrence->status === 'enqueued',+ fn ($query) => $query->where('enqueued_at', '<', $staleBefore),+ fn ($query) => $query->where('created_at', '<', $staleBefore),+ )->update(['status' => $runLate ? 'pending' : 'failed', 'updated_at' => now()]) === 1;if (! $moved) {@@ -235,9 +337,11 @@ class ScheduledJobDeliveryService}if (! $runLate) {- $this->logOccurrence($occurrence->uuid, $hasNewerOccurrence- ? 'Scheduled occurrence skipped: its queued job was not started and a newer occurrence exists'- : 'Scheduled occurrence missed: its queued job was not started', 'warning');+ $this->logOccurrence($occurrence->uuid, match (true) {+ $hasNewerOccurrence => 'Scheduled occurrence skipped: its queued job was not started and a newer occurrence exists',+ $tooOld && in_array($occurrence->job_type, self::LATE_RUN_JOB_TYPES, true) => 'Scheduled occurrence missed: its queued job was not started and it is too old to run late',+ default => 'Scheduled occurrence missed: its queued job was not started',+ }, 'warning', ['previous_status' => $occurrence->status]);$this->reportMissedOccurrence($occurrence);return;diff --git a/tests/Feature/ScheduledJobManagerDispatchTest.php b/tests/Feature/ScheduledJobManagerDispatchTest.phpindex 44201e0d4c..4e201b90be 100644--- a/tests/Feature/ScheduledJobManagerDispatchTest.php+++ b/tests/Feature/ScheduledJobManagerDispatchTest.php@@ -456,14 +456,14 @@ it('warns when an occurrence is enqueued while the previous one is still runningconfig(['constants.coolify.self_hosted' => true]);Queue::fake();$logPath = captureScheduledJobManagerTestLog('scheduled');- $backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);+ $backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'hourly']);Carbon::setTestNow(Carbon::create(2026, 9, 16, 0, 5, 0, 'UTC'));(new ScheduledJobManager)->handle();$previousUuid = ScheduledJobDelivery::query()->where('schedule_key', "scheduled-backup:{$backup->id}")->value('uuid');app(ScheduledJobDeliveryService::class)->claim($previousUuid, 'worker-a');- Carbon::setTestNow(Carbon::create(2026, 9, 17, 0, 5, 0, 'UTC'));+ Carbon::setTestNow(Carbon::create(2026, 9, 16, 1, 5, 0, 'UTC'));(new ScheduledJobManager)->handle();$log = file_get_contents($logPath);@@ -887,6 +887,172 @@ it('rejects an unknown schedule type', function () {Queue::assertNothingPushed();});+it('reports a task occurrence as missed when its publication kept failing for an hour', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));+ Queue::fake();+ Notification::fake();+ $task = createScheduledApplicationTask(createScheduledTaskApplication(), ['frequency' => 'daily']);+ $occurrence = ScheduledJobDelivery::create([+ 'schedule_key' => "scheduled-task:{$task->id}",+ 'scheduled_for' => Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'),+ 'job_type' => 'scheduled-task',+ 'resource_id' => $task->id,+ 'status' => 'pending',+ ]);+ $occurrence->forceFill(['created_at' => now()->subMinutes(ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1)])->save();++ app(ScheduledJobDeliveryService::class)->publishPending();++ Queue::assertNotPushed(ScheduledTaskJob::class);+ expect($occurrence->fresh()->status)->toBe('failed')+ ->and(ScheduledTaskExecution::query()->where('scheduled_task_id', $task->id)->value('status'))->toBe('failed');+});++it('publishes a backup occurrence once late when its publication kept failing and no newer occurrence exists', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));+ Queue::fake();+ $backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'daily']);+ $occurrence = ScheduledJobDelivery::create([+ 'schedule_key' => "scheduled-backup:{$backup->id}",+ 'scheduled_for' => Carbon::create(2026, 9, 17, 0, 0, 0, 'UTC'),+ 'job_type' => 'database-backup',+ 'resource_id' => $backup->id,+ 'status' => 'pending',+ ]);+ $occurrence->forceFill(['created_at' => now()->subHours(2)])->save();++ app(ScheduledJobDeliveryService::class)->publishPending();++ Queue::assertPushed(DatabaseBackupJob::class, fn (DatabaseBackupJob $job) => $job->occurrenceUuid === $occurrence->uuid);+ expect($occurrence->fresh()->status)->toBe('enqueued');+});++it('does not run a lost backup occurrence late when it is older than the late-run limit', function () {+ config(['constants.coolify.self_hosted' => true]);+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));+ Queue::fake();+ $logPath = captureScheduledJobManagerTestLog('scheduled');+ $backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => 'weekly']);+ $occurrence = createStaleEnqueuedOccurrence(+ "scheduled-backup:{$backup->id}",+ 'database-backup',+ $backup->id,+ now()->subHours(ScheduledJobDeliveryService::LATE_RUN_MAX_AGE_HOURS + 1),+ minutesAgo: (ScheduledJobDeliveryService::LATE_RUN_MAX_AGE_HOURS + 1) * 60,+ );++ app(ScheduledJobDeliveryService::class)->recoverStaleEnqueued();++ $log = file_get_contents($logPath);+ @unlink($logPath);++ Queue::assertNotPushed(DatabaseBackupJob::class);+ expect($occurrence->fresh()->status)->toBe('failed')+ ->and($log)->toContain('too old to run late');+});++it('fails claimed occurrences whose worker can no longer be running, and keeps recent claims', function () {+ Carbon::setTestNow(Carbon::create(2026, 9, 17, 12, 0, 0, 'UTC'));+ $deadAfterSeconds = (int) config('horizon.worker_timeout') + ScheduledJobDeliveryService::INTERRUPTED_CLAIM_MARGIN_MINUTES * 60;+ $create = fn (string $key, int $startedSecondsAgo) => ScheduledJobDelivery::create([+ 'schedule_key' => $key,+ 'scheduled_for' => now()->subSeconds($startedSecondsAgo),+ 'job_type' => 'scheduled-task',+ 'resource_id' => 1,+ 'status' => 'claimed',+ 'claim_token' => "token-{$key}",+ 'started_at' => now()->subSeconds($startedSecondsAgo),+ ]);+ $interrupted = $create('scheduled-task:interrupted', $deadAfterSeconds + 60);+ $running = $create('scheduled-task:running', $deadAfterSeconds - 60);++ $service = app(ScheduledJobDeliveryService::class);+ $service->failInterruptedClaims();++ expect($interrupted->fresh()->status)->toBe('failed')+ ->and($running->fresh()->status)->toBe('claimed')+ // The job copy that Redis hands out again later cannot claim the failed occurrence.+ ->and($service->claim($interrupted->uuid, 'token-scheduled-task:interrupted'))->toBeFalse();+});++it('does not run a scheduled task again when Redis hands it out after its worker was killed', function () {+ $task = createScheduledApplicationTask(createScheduledTaskApplication());+ $occurrence = ScheduledJobDelivery::create([+ 'schedule_key' => "scheduled-task:{$task->id}",+ 'scheduled_for' => now()->subDay(),+ 'job_type' => 'scheduled-task',+ 'resource_id' => $task->id,+ 'status' => 'claimed',+ 'claim_token' => 'killed-worker',+ 'started_at' => now()->subDay(),+ ]);++ $job = (new ScheduledTaskJob($task, $occurrence->uuid))->withFakeQueueInteractions();+ $job->job->attempts = 2;+ $job->handle();++ $job->assertFailed();+ expect(ScheduledTaskExecution::query()->where('scheduled_task_id', $task->id)->exists())->toBeFalse();+});++it('marks only the unfinished execution as failed when a scheduled task job fails without its execution id', function () {+ InstanceSettings::forceCreate(['id' => 0]);+ Notification::fake();+ $task = createScheduledApplicationTask(createScheduledTaskApplication());+ $interrupted = ScheduledTaskExecution::create(['scheduled_task_id' => $task->id, 'status' => 'running', 'started_at' => now()->subHours(2)]);+ $interrupted->forceFill(['created_at' => now()->subHours(2)])->save();+ $later = ScheduledTaskExecution::create(['scheduled_task_id' => $task->id, 'status' => 'success', 'started_at' => now()->subHour(), 'finished_at' => now()->subMinutes(59)]);++ (new ScheduledTaskJob($task))->failed(new RuntimeException('worker killed'));++ expect($interrupted->fresh()->status)->toBe('failed')+ ->and($later->fresh()->status)->toBe('success');+});++it('runs a fixed-time schedule once when daylight saving time repeats its local time', function () {+ $service = app(ScheduledJobDeliveryService::class);+ $record = fn (string $frequency, string $utc) => $service->recordSkipped("dst:{$frequency}", $frequency, 'Europe/Berlin', Carbon::parse($utc, 'UTC'));++ // 2026-10-25: Berlin goes from 03:00 CEST back to 02:00 CET, so 02:30 happens at 00:30Z and 01:30Z.+ expect($record('30 2 * * *', '2026-10-25 00:31:00'))->toBeTrue()+ ->and($record('30 2 * * *', '2026-10-25 01:31:00'))->toBeFalse()+ ->and($record('30 * * * *', '2026-10-25 00:31:00'))->toBeTrue()+ ->and($record('30 * * * *', '2026-10-25 01:31:00'))->toBeTrue();+});++it('publishes one backup when daylight saving time repeats the local backup time', function () {+ config(['constants.coolify.self_hosted' => true]);+ Queue::fake();+ $backup = createScheduledDatabaseBackup(createScheduledBackupDatabase(), ['frequency' => '30 2 * * *']);+ $service = app(ScheduledJobDeliveryService::class);++ foreach (['2026-10-25 00:31:00', '2026-10-25 01:31:00'] as $utc) {+ $service->recordAndPublish("scheduled-backup:{$backup->id}", '30 2 * * *', 'Europe/Berlin', 'database-backup', $backup->id, executionTime: Carbon::parse($utc, 'UTC'));+ }++ Queue::assertPushed(DatabaseBackupJob::class, 1);+});++it('does not run a fixed time that does not exist when daylight saving time starts', function () {+ $service = app(ScheduledJobDeliveryService::class);++ // 2026-03-29: Berlin skips from 02:00 CET to 03:00 CEST, so 02:30 does not exist that day.+ expect($service->recordSkipped('dst:spring', '30 2 * * *', 'Europe/Berlin', Carbon::parse('2026-03-29 01:01:00', 'UTC')))->toBeFalse()+ ->and($service->recordSkipped('dst:spring', '30 2 * * *', 'Europe/Berlin', Carbon::parse('2026-03-30 00:31:00', 'UTC')))->toBeTrue();+});++it('records a midnight schedule in the server timezone, not in UTC', function () {+ $service = app(ScheduledJobDeliveryService::class);++ // Midnight in New York (EDT) is 04:00Z; 00:05Z is 20:05 the day before in New York.+ expect($service->recordSkipped('midnight', '0 0 * * *', 'America/New_York', Carbon::parse('2026-10-03 00:05:00', 'UTC')))->toBeFalse()+ ->and($service->recordSkipped('midnight', '0 0 * * *', 'America/New_York', Carbon::parse('2026-10-03 04:05:00', 'UTC')))->toBeTrue()+ ->and($service->recordSkipped('midnight', '0 0 * * *', 'America/New_York', Carbon::parse('2026-10-03 04:09:00', 'UTC')))->toBeFalse()+ ->and(ScheduledJobState::query()->where('schedule_key', 'midnight')->value('last_scheduled_for')->toDateTimeString())->toBe('2026-10-03 04:00:00');+});+function createStaleEnqueuedOccurrence(string $scheduleKey, string $jobType, int $resourceId, Carbon $scheduledFor, int $minutesAgo = ScheduledJobDeliveryService::ENQUEUED_STALE_AFTER_MINUTES + 1): ScheduledJobDelivery{return ScheduledJobDelivery::create([