An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
feat(horizon): add fixed-size per-queue worker pools on Coolify Cloud
Andras Bacsaie144b990c30e
diff --git a/.env.development.example b/.env.development.exampleindex f53fcce5f7..4f4198ddb2 100644--- a/.env.development.example+++ b/.env.development.example@@ -31,6 +31,15 @@ DB_PORT=5432+# Coolify Cloud Horizon pools (only used when SELF_HOSTED=false and APP_ENV=production).+# Fixed worker count per node and queue. See docs/cloud-horizon-workers.md.+# HORIZON_DEPLOYMENTS_PROCESSES=60+# HORIZON_CRONS_PROCESSES=60+# HORIZON_HIGH_PROCESSES=60+# HORIZON_DEFAULT_PROCESSES=40+# Docker cleanups; small on purpose to bound concurrent slow remote prunes.+# HORIZON_MAINTENANCE_PROCESSES=10+# Server-Timing headers + on-screen HUD (defaults ON when APP_ENV=local).# Force on in any environment (including production): SERVER_TIMING_ENABLED=true# Force off even in local: SERVER_TIMING_ENABLED=falsediff --git a/app/Actions/Database/StartDatabaseImport.php b/app/Actions/Database/StartDatabaseImport.phpindex 5e113d163c..235399e652 100644--- a/app/Actions/Database/StartDatabaseImport.php+++ b/app/Actions/Database/StartDatabaseImport.php@@ -47,7 +47,7 @@ class StartDatabaseImportpublic static function jobTimeoutSeconds(): int{$timeout = (int) config('constants.ssh.command_timeout') + self::JOB_TIMEOUT_MARGIN_SECONDS;- $workerTimeout = (int) config('horizon.defaults.s6.timeout');+ $workerTimeout = (int) config('horizon.worker_timeout');return $workerTimeout > 0 ? min($timeout, $workerTimeout - self::JOB_TIMEOUT_MARGIN_SECONDS) : $timeout;}diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.phpindex cf620bb964..da6faa43ee 100644--- a/bootstrap/helpers/shared.php+++ b/bootstrap/helpers/shared.php@@ -1028,8 +1028,8 @@ function isCloud(): bool* is decided by `isCloud()` (config-based) rather than `HORIZON_QUEUES`, so the dispatching* process needs no special env — only the worker must be configured to drain `deployments`.*- * IMPORTANT: on cloud a worker MUST include `deployments` in its `HORIZON_QUEUES`, otherwise- * these jobs are never processed.+ * On cloud, config/horizon.php provisions a dedicated `deployments` pool in production+ * (see docs/cloud-horizon-workers.md).*/function deployment_queue(): string{@@ -1045,14 +1045,31 @@ function deployment_queue(): string* by `isCloud()` (config-based), so the dispatching process needs no special env — only the* worker must be configured to drain `crons`.*- * IMPORTANT: on cloud a worker MUST include `crons` in its `HORIZON_QUEUES`, otherwise these- * jobs are never processed.+ * On cloud, config/horizon.php provisions a dedicated `crons` pool in production+ * (see docs/cloud-horizon-workers.md).*/function crons_queue(): string{return isCloud() ? 'crons' : 'high';}+/**+ * Resolve the queue used for slow server maintenance — scheduled, manual and stop-triggered+ * Docker cleanups.+ *+ * On cloud these jobs run on a dedicated `maintenance` queue so a small, bounded Horizon pool+ * drains them and slow remote prunes cannot occupy the `high` workers; self-hosted keeps them+ * on the shared `high` queue, so a custom `HORIZON_QUEUES` does not need a new queue name. Routing is decided by `isCloud()` (config-based), so the dispatching+ * process needs no special env — only the worker must be configured to drain `maintenance`.+ *+ * On cloud, config/horizon.php provisions a dedicated `maintenance` pool in production+ * (see docs/cloud-horizon-workers.md).+ */+function maintenance_queue(): string+{+ return isCloud() ? 'maintenance' : 'high';+}+function translate_cron_expression($expression_to_validate): string{if (isset(VALID_CRON_STRINGS[$expression_to_validate])) {diff --git a/config/horizon.php b/config/horizon.phpindex 0c79f0eed3..cd532f936c 100644--- a/config/horizon.php+++ b/config/horizon.php@@ -3,6 +3,67 @@use App\Models\ScheduledVolumeBackup;use Illuminate\Support\Str;+/*+| Cloud mode is read from SELF_HOSTED directly instead of isCloud(): isCloud()+| reads the config repository, which is still being built while this file loads.+*/+$isCloud = ! env('SELF_HOSTED', true);++$workerOptions = [+ 'connection' => 'redis',+ 'maxTime' => env('HORIZON_MAX_TIME', 0),+ 'maxJobs' => 400,+ 'memory' => 128,+ 'tries' => 1,+ 'nice' => 0,+ 'sleep' => 3,+ 'timeout' => min(+ max((int) env('HORIZON_TIMEOUT', 39600), ScheduledVolumeBackup::DEFAULT_TIMEOUT + 600),+ 85800,+ ),+];++$selfHostedSupervisor = $workerOptions + [+ 'balance' => env('HORIZON_BALANCE', 'false'),+ 'queue' => env('HORIZON_QUEUES', 'high,default'),+ 'autoScalingStrategy' => 'size',+ 'minProcesses' => env('HORIZON_MIN_PROCESSES', 1),+ 'maxProcesses' => env('HORIZON_MAX_PROCESSES', 4),+ 'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1),+ 'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1),+];++/*+| Coolify Cloud: one fixed-size pool per queue, so a busy queue cannot starve+| the others. Process counts apply to each node. Invalid values (not a+| positive integer) fall back to the default.+|+| The `maintenance` pool (see maintenance_queue()) runs slow remote Docker+| cleanups. It is small on purpose: it bounds how many cleanups run at once+| per node, and it comes on top of the other pools, so cleanups never take+| deployment, cron or high workers.+*/+$cloudSupervisors = [];++foreach ([+ 'deployments' => ['HORIZON_DEPLOYMENTS_PROCESSES', 60],+ 'crons' => ['HORIZON_CRONS_PROCESSES', 60],+ 'high' => ['HORIZON_HIGH_PROCESSES', 60],+ 'default' => ['HORIZON_DEFAULT_PROCESSES', 40],+ 'maintenance' => ['HORIZON_MAINTENANCE_PROCESSES', 10],+] as $queue => [$processesEnv, $defaultProcesses]) {+ $processes = filter_var(env($processesEnv), FILTER_VALIDATE_INT, [+ 'options' => ['min_range' => 1, 'default' => $defaultProcesses],+ ]);++ $cloudSupervisors[$queue] = $workerOptions + [+ 'queue' => $queue,+ 'balance' => false,+ 'minProcesses' => $processes,+ 'maxProcesses' => $processes,+ ];+}+return [/*@@ -190,45 +251,21 @@ return [| in all environments. These supervisors and settings handle all your| queued jobs and will be provisioned by Horizon during deployment.|+ | "defaults" stays empty: Horizon merges every default supervisor into+ | every environment, so each environment defines complete options.+ |*/- 'defaults' => [- 's6' => [- 'connection' => 'redis',- 'balance' => env('HORIZON_BALANCE', 'false'),- 'queue' => env('HORIZON_QUEUES', 'high,default'),- 'maxTime' => env('HORIZON_MAX_TIME', 0),- 'maxJobs' => 400,- 'memory' => 128,- 'tries' => 1,- 'nice' => 0,- 'sleep' => 3,- 'timeout' => min(- max((int) env('HORIZON_TIMEOUT', 39600), ScheduledVolumeBackup::DEFAULT_TIMEOUT + 600),- 85800,- ),- ],+ 'defaults' => [],- ],+ /*+ | Worker timeout shared by every supervisor. Jobs that must finish before+ | the worker kills them (for example database imports) read this value.+ */+ 'worker_timeout' => $workerOptions['timeout'],'environments' => [- 'production' => [- 's6' => [- 'autoScalingStrategy' => 'size',- 'minProcesses' => env('HORIZON_MIN_PROCESSES', 1),- 'maxProcesses' => env('HORIZON_MAX_PROCESSES', 4),- 'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1),- 'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1),- ],- ],- 'local' => [- 's6' => [- 'autoScalingStrategy' => 'size',- 'minProcesses' => env('HORIZON_MIN_PROCESSES', 1),- 'maxProcesses' => env('HORIZON_MAX_PROCESSES', 4),- 'balanceMaxShift' => env('HORIZON_BALANCE_MAX_SHIFT', 1),- 'balanceCooldown' => env('HORIZON_BALANCE_COOLDOWN', 1),- ],- ],+ 'production' => $isCloud ? $cloudSupervisors : ['s6' => $selfHostedSupervisor],+ 'local' => ['s6' => $selfHostedSupervisor],],];diff --git a/docs/cloud-horizon-workers.md b/docs/cloud-horizon-workers.mdnew file mode 100644index 0000000000..11ee81ef5e--- /dev/null+++ b/docs/cloud-horizon-workers.md@@ -0,0 +1,157 @@+# Coolify Cloud Horizon worker pools++Coolify Cloud runs Horizon on two app nodes that share one Redis. On Cloud+(`SELF_HOSTED=false`, `APP_ENV=production`), `config/horizon.php` provisions+one fixed-size supervisor per queue. A busy queue cannot starve the others.++| Supervisor | Queue | Env variable | Default per node |+|---------------|---------------|---------------------------------|------------------|+| `deployments` | `deployments` | `HORIZON_DEPLOYMENTS_PROCESSES` | 60 |+| `crons` | `crons` | `HORIZON_CRONS_PROCESSES` | 60 |+| `high` | `high` | `HORIZON_HIGH_PROCESSES` | 60 |+| `default` | `default` | `HORIZON_DEFAULT_PROCESSES` | 40 |+| `maintenance` | `maintenance` | `HORIZON_MAINTENANCE_PROCESSES` | 10 |++- Counts apply to **each node**. Two nodes with the defaults run 460 workers.+- `maintenance` runs Docker cleanups (scheduled, manual, and the cleanup+ that runs after a resource stops). Remote prunes can be slow, so this pool+ is small on purpose: it limits how many cleanups run at the same time and+ keeps them away from the `high` workers. It is added on top of the other+ pools. Each cleanup has a 600 s timeout, and only one cleanup runs per+ server at a time. Self-hosted instances keep cleanups on `high`.+- Each pool has `minProcesses = maxProcesses` and `balance = false`. Horizon+ does not scale them.+- A value that is not a positive integer falls back to the default.+- `HORIZON_QUEUES`, `HORIZON_MIN_PROCESSES`, `HORIZON_MAX_PROCESSES`, and the+ `HORIZON_BALANCE*` variables have no effect on Cloud production. They still+ configure the self-hosted `s6` supervisor and the `local` environment.+- Timeout, retry, memory, and recycling settings (`HORIZON_TIMEOUT`,+ `HORIZON_MAX_TIME`, `maxJobs`, `memory`, `tries`) are the same as for+ self-hosted.+- The config reads `SELF_HOSTED` with `env()` and does not call `isCloud()`,+ because `isCloud()` reads the config repository, which is not complete while+ config files load.++## Sample Cloud configuration++Put these values in the `.env` of each node. The values are the defaults, so+set only the values that you change.++```dotenv+SELF_HOSTED=false+APP_ENV=production+HORIZON_DEPLOYMENTS_PROCESSES=60+HORIZON_CRONS_PROCESSES=60+HORIZON_HIGH_PROCESSES=60+HORIZON_DEFAULT_PROCESSES=40+HORIZON_MAINTENANCE_PROCESSES=10+# Worker timeout. Must stay above the longest job timeout (36000 s) and below+# retry_after (86400 s). The config clamps it to 36600..85800.+HORIZON_TIMEOUT=36600+```++Redis that holds queues must use `maxmemory-policy noeviction`. An evicting+policy deletes queued jobs without an error.++## Worker shutdown, retry_after, and interrupted jobs++- **Graceful (`php artisan horizon:terminate`)**: workers stop taking new+ jobs and finish the current job. `fast_termination` is `false`, so the+ master waits until the last worker exits. This can take up to the longest+ job timeout. Waiting jobs stay in Redis. The other node continues to+ process them.+- **Forced (SIGKILL, container stop after its grace period, OOM kill)**: the+ running job stays in `queues:<queue>:reserved`. Redis gives it to a worker+ again only after `retry_after` (86400 s). Then the job has attempt 2:+ - Jobs with `tries = 1` fail with `MaxAttemptsExceededException` and their+ `failed()` method runs.+ - A `ScheduledTaskJob` does not run its command again. It fails.+ - A scheduled occurrence that a killed worker claimed is marked `failed`+ ("interrupted") when it is older than the worker timeout plus 15 minutes.+ It is not run again, because part of its remote work can be done.+- Do not lower `retry_after` below the worker timeout. A running job would+ then be given to a second worker.++### Which settings need new workers++| Change | Needed action |+|-----------------------------------------------------------|--------------------------------------------------|+| `HORIZON_*_PROCESSES`, `HORIZON_TIMEOUT`, queue names | `config:cache` and `horizon:terminate` |+| Horizon `trim` values | `config:cache` and `horizon:terminate` |+| `retry_after` (`config/queue.php`) | `config:cache` and `horizon:terminate` |+| Application code (jobs, queues, middleware) | New image, then `horizon:terminate` |+| New database migration | `php artisan migrate` before new workers start |++## Rollout++Do these steps on one node at a time. Do not flush Redis or clear queues.++1. Run the database migrations once (the new volume backup recovery columns+ must exist before the new scheduler runs):++ ```bash+ docker exec coolify php artisan migrate --force+ ```++2. Optional: set the `HORIZON_*_PROCESSES` variables in the node's `.env`.+ Remove manual pool overrides from earlier hotfixes.+3. Rebuild the configuration cache:++ ```bash+ docker exec coolify php artisan config:cache+ ```++4. Restart Horizon on this node only. `horizon:terminate` lets the running+ jobs finish, then s6 starts Horizon again with the new configuration:++ ```bash+ docker exec coolify php artisan horizon:terminate+ ```++5. Make sure the five supervisors are running on this node:++ ```bash+ docker exec coolify php artisan horizon:supervisors+ ```++6. Make sure the `maintenance` queue drains (its depth goes down) and that+ `php artisan scheduled:diagnostics` shows no growing count of stale+ occurrences.+7. Do the same steps on the next node.++Jobs that the old code put on `high` (for example Docker cleanups) stay on+`high`, and the `high` pool still processes them.++## Rollback++1. Deploy the previous image, or remove the `HORIZON_*_PROCESSES` overrides.+2. Run `config:cache` and `horizon:terminate` on one node at a time.+3. Jobs that wait on the `maintenance` queue are not lost, but the old+ configuration has no `maintenance` pool. Keep one node with the new+ configuration until `maintenance` is empty, or start a temporary worker:+ `php artisan queue:work redis --queue=maintenance --tries=1 --timeout=600`.+4. Do not roll back the migration. The old code ignores the new columns.++## Metrics to collect before you change worker counts++The incident samples are short and show only completed jobs. Collect these+values before you change pool sizes:++- Arrival and completion rate per queue (per minute).+- Age of the oldest waiting job per queue (`LINDEX queues:<queue> -1`, then+ read `pushedAt` from the payload). Do not scan the full list.+- Queue depth (`LLEN queues:<queue>`) and reserved age+ (`ZRANGE queues:<queue>:reserved 0 0 WITHSCORES`).+- Active worker processes per supervisor (count processes, not the+ Horizon reserved count; reserved entries stay after forced restarts).+- Job duration distribution (p50/p95/max) and failure/retry rate per job+ class.+- Stale scheduled occurrences per status (`scheduled:diagnostics` and the+ `open_occurrences_over_15_minutes` value in `scheduled.log`).+- Redis `used_memory` / `maxmemory`, the delta of `evicted_keys` (`INFO+ stats`), and the delta of `errorstat_OOM` (`INFO errorstats`).+- Horizon metadata size: the count of `horizon:*` job hashes and the+ `horizon:recent_jobs` / `horizon:completed_jobs` sizes.++Label metrics only by queue and job class. Do not label by resource ID or+occurrence UUID.diff --git a/tests/Feature/DatabaseImportInterruptedCleanupTest.php b/tests/Feature/DatabaseImportInterruptedCleanupTest.phpindex 0ef66fed33..f82b8cc438 100644--- a/tests/Feature/DatabaseImportInterruptedCleanupTest.php+++ b/tests/Feature/DatabaseImportInterruptedCleanupTest.php@@ -289,7 +289,7 @@ test('an import task may run longer than the SSH command timeout while other rem$timeouts = Queue::pushed(CoolifyTask::class)->mapWithKeys(fn (CoolifyTask $job) => [(data_get($job->activity, 'properties.operation') ?? 'other') => $job->timeout,]);- $workerTimeout = (int) config('horizon.defaults.s6.timeout');+ $workerTimeout = (int) config('horizon.worker_timeout');expect($timeouts['other'])->toBe(600)->and($timeouts[ResourceStartActivity::DATABASE_IMPORT_OPERATION])->toBeGreaterThan(min($commandTimeout, $workerTimeout - 600))diff --git a/tests/Feature/HorizonConfigurationTest.php b/tests/Feature/HorizonConfigurationTest.phpnew file mode 100644index 0000000000..4f8a8d6fc3--- /dev/null+++ b/tests/Feature/HorizonConfigurationTest.php@@ -0,0 +1,162 @@+<?php++use Laravel\Horizon\ProvisioningPlan;+use Laravel\Horizon\SupervisorOptions;++/**+ * Evaluate config/horizon.php with the given environment variables, then restore them.+ *+ * @param array<string, string|null> $variables+ */+function loadHorizonConfigWithEnv(array $variables): array+{+ $previous = [];++ foreach ($variables as $name => $value) {+ $previous[$name] = getenv($name);+ $value === null ? putenv($name) : putenv("{$name}={$value}");+ if ($value === null) {+ unset($_ENV[$name], $_SERVER[$name]);+ } else {+ $_ENV[$name] = $_SERVER[$name] = $value;+ }+ }++ try {+ return require config_path('horizon.php');+ } finally {+ foreach ($previous as $name => $value) {+ $value === false ? putenv($name) : putenv("{$name}={$value}");+ if ($value === false) {+ unset($_ENV[$name], $_SERVER[$name]);+ } else {+ $_ENV[$name] = $_SERVER[$name] = $value;+ }+ }+ }+}++/**+ * Parse every environment through Horizon's ProvisioningPlan, as `php artisan horizon` does on startup.+ *+ * @return array<string, array<string, SupervisorOptions>>+ */+function horizonSupervisorsFor(array $config): array+{+ $parsed = array_map(+ fn ($supervisors) => collect($supervisors)->all(),+ (new ProvisioningPlan('master', $config['environments'], $config['defaults']))->toSupervisorOptions(),+ );++ foreach ($parsed as $supervisors) {+ foreach ($supervisors as $options) {+ expect($options->connection)->toBe('redis')+ ->and($options->timeout)->toBe($config['worker_timeout'])+ ->and($options->toSupervisorCommand())->toContain('horizon:supervisor');+ }+ }++ return $parsed;+}++test('self-hosted keeps a single auto-scaling s6 supervisor in every environment', function () {+ $parsed = horizonSupervisorsFor(loadHorizonConfigWithEnv([+ 'SELF_HOSTED' => 'true',+ 'HORIZON_QUEUES' => null,+ 'HORIZON_MIN_PROCESSES' => null,+ 'HORIZON_MAX_PROCESSES' => null,+ ]));++ expect(array_keys($parsed))->toBe(['production', 'local']);++ foreach ($parsed as $supervisors) {+ expect(array_keys($supervisors))->toBe(['s6']);++ $s6 = $supervisors['s6'];+ expect($s6->queue)->toBe('high,default')+ ->and($s6->minProcesses)->toBe(1)+ ->and($s6->maxProcesses)->toBe(4)+ ->and($s6->autoScalingStrategy)->toBe('size')+ ->and($s6->maxJobs)->toBe(400)+ ->and($s6->memory)->toBe(128)+ ->and($s6->maxTries)->toBe(1);+ }+});++test('cloud production runs one fixed-size pool per queue with default process counts', function () {+ $parsed = horizonSupervisorsFor(loadHorizonConfigWithEnv([+ 'SELF_HOSTED' => 'false',+ 'HORIZON_DEPLOYMENTS_PROCESSES' => null,+ 'HORIZON_CRONS_PROCESSES' => null,+ 'HORIZON_HIGH_PROCESSES' => null,+ 'HORIZON_DEFAULT_PROCESSES' => null,+ 'HORIZON_MAINTENANCE_PROCESSES' => null,+ ]));++ $production = $parsed['production'];+ expect(array_keys($production))->toBe(['deployments', 'crons', 'high', 'default', 'maintenance']);++ foreach (['deployments' => 60, 'crons' => 60, 'high' => 60, 'default' => 40, 'maintenance' => 10] as $queue => $processes) {+ $pool = $production[$queue];+ expect($pool->queue)->toBe($queue)+ ->and($pool->balancing())->toBeFalse()+ ->and($pool->minProcesses)->toBe($processes)+ ->and($pool->maxProcesses)->toBe($processes)+ ->and($pool->maxJobs)->toBe(400)+ ->and($pool->memory)->toBe(128)+ ->and($pool->maxTries)->toBe(1)+ ->and($pool->timeout)->toBe($parsed['local']['s6']->timeout);+ }++ expect(array_keys($parsed['local']))->toBe(['s6']);+});++test('cloud pool sizes come from env and invalid values fall back to defaults', function () {+ $production = horizonSupervisorsFor(loadHorizonConfigWithEnv([+ 'SELF_HOSTED' => 'false',+ 'HORIZON_DEPLOYMENTS_PROCESSES' => '25',+ 'HORIZON_CRONS_PROCESSES' => '0',+ 'HORIZON_HIGH_PROCESSES' => 'abc',+ 'HORIZON_DEFAULT_PROCESSES' => '-5',+ 'HORIZON_MAINTENANCE_PROCESSES' => '4',+ ]))['production'];++ expect($production['deployments']->maxProcesses)->toBe(25)+ ->and($production['deployments']->minProcesses)->toBe(25)+ ->and($production['crons']->maxProcesses)->toBe(60)+ ->and($production['high']->maxProcesses)->toBe(60)+ ->and($production['default']->maxProcesses)->toBe(40)+ ->and($production['maintenance']->minProcesses)->toBe(4)+ ->and($production['maintenance']->maxProcesses)->toBe(4);++ $fallback = horizonSupervisorsFor(loadHorizonConfigWithEnv([+ 'SELF_HOSTED' => 'false',+ 'HORIZON_MAINTENANCE_PROCESSES' => 'ten',+ ]))['production'];++ expect($fallback['maintenance']->maxProcesses)->toBe(10);+});++test('cloud queue helpers route to queues that have a dedicated pool', function () {+ config(['constants.coolify.self_hosted' => false]);++ $production = horizonSupervisorsFor(loadHorizonConfigWithEnv(['SELF_HOSTED' => 'false']))['production'];++ expect($production)->toHaveKeys([deployment_queue(), crons_queue(), maintenance_queue()]);+});++test('self-hosted queue helpers route to a queue the s6 supervisor drains', function () {+ config(['constants.coolify.self_hosted' => true]);++ $parsed = horizonSupervisorsFor(loadHorizonConfigWithEnv([+ 'SELF_HOSTED' => 'true',+ 'HORIZON_QUEUES' => null,+ 'HORIZON_MAINTENANCE_PROCESSES' => '3',+ ]));++ foreach ($parsed as $supervisors) {+ expect(array_keys($supervisors))->toBe(['s6'])+ ->and(explode(',', $supervisors['s6']->queue))+ ->toContain(deployment_queue(), crons_queue(), maintenance_queue());+ }+});diff --git a/tests/Feature/VolumeBackupTest.php b/tests/Feature/VolumeBackupTest.phpindex 1f733f72d4..2fc39486d5 100644--- a/tests/Feature/VolumeBackupTest.php+++ b/tests/Feature/VolumeBackupTest.php@@ -61,8 +61,8 @@ it('allows large volume backups to run for ten hours by default', function () {expect($job->timeout)->toBe(36000)->and((new VolumeBackups)->timeout)->toBe(36000)- ->and(config('horizon.defaults.s6.timeout'))->toBeGreaterThan($job->timeout)- ->and(config('queue.connections.redis.retry_after'))->toBeGreaterThan(config('horizon.defaults.s6.timeout'));+ ->and(config('horizon.worker_timeout'))->toBeGreaterThan($job->timeout)+ ->and(config('queue.connections.redis.retry_after'))->toBeGreaterThan(config('horizon.worker_timeout'));});it('changes the default volume backup timeout without changing existing timeouts', function () {