An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.
PHP coolifydatabasesdeploymentdockerdocker-composeinertiajslaravelmariadbmysqlnextjsnodejsphppostgresredisself-hostedself-hostingserverstatic-sitesveltesvelte5
fix(horizon): route webhook jobs to a dedicated cloud webhooks queue
Andras Bacsai083e71c48154
diff --git a/.env.development.example b/.env.development.exampleindex 4f4198ddb2..f7a01180e3 100644--- a/.env.development.example+++ b/.env.development.example@@ -39,6 +39,8 @@ DB_PORT=5432# HORIZON_DEFAULT_PROCESSES=40# Docker cleanups; small on purpose to bound concurrent slow remote prunes.# HORIZON_MAINTENANCE_PROCESSES=10+# GitHub pull request webhooks and Stripe events.+# HORIZON_WEBHOOKS_PROCESSES=10# Server-Timing headers + on-screen HUD (defaults ON when APP_ENV=local).# Force on in any environment (including production): SERVER_TIMING_ENABLED=truediff --git a/app/Jobs/ProcessGithubPullRequestWebhook.php b/app/Jobs/ProcessGithubPullRequestWebhook.phpindex 5186a58bbf..362ad0a2d8 100644--- a/app/Jobs/ProcessGithubPullRequestWebhook.php+++ b/app/Jobs/ProcessGithubPullRequestWebhook.php@@ -41,7 +41,7 @@ class ProcessGithubPullRequestWebhook implements ShouldBeEncrypted, ShouldQueuepublic string $fullName,public bool $isForkPullRequest = false,) {- $this->onQueue('high');+ $this->onQueue(webhooks_queue());}public function handle(): voiddiff --git a/app/Jobs/StripeProcessJob.php b/app/Jobs/StripeProcessJob.phpindex 0f56476e25..58e67de085 100644--- a/app/Jobs/StripeProcessJob.php+++ b/app/Jobs/StripeProcessJob.php@@ -23,7 +23,7 @@ class StripeProcessJob implements ShouldBeEncrypted, ShouldQueuepublic function __construct(public $event){- $this->onQueue('high');+ $this->onQueue(webhooks_queue());}public function handle(): voiddiff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.phpindex da6faa43ee..a3be2667ba 100644--- a/bootstrap/helpers/shared.php+++ b/bootstrap/helpers/shared.php@@ -1070,6 +1070,23 @@ function maintenance_queue(): stringreturn isCloud() ? 'maintenance' : 'high';}+/**+ * Resolve the queue used for incoming webhook processing — GitHub pull request webhooks and+ * Stripe events.+ *+ * On cloud these jobs run on a dedicated `webhooks` queue so a busy `high` queue cannot delay+ * them; self-hosted keeps them on the shared `high` queue, so a custom `HORIZON_QUEUES` does+ * not need a new queue name. Routing is decided by `isCloud()` (config-based), so the+ * dispatching process needs no special env — only the worker must be configured to drain `webhooks`.+ *+ * On cloud, config/horizon.php provisions a dedicated `webhooks` pool in production+ * (see docs/cloud-horizon-workers.md).+ */+function webhooks_queue(): string+{+ return isCloud() ? 'webhooks' : 'high';+}+function translate_cron_expression($expression_to_validate): string{if (isset(VALID_CRON_STRINGS[$expression_to_validate])) {diff --git a/config/horizon.php b/config/horizon.phpindex cd532f936c..c29ad15fc9 100644--- a/config/horizon.php+++ b/config/horizon.php@@ -42,6 +42,9 @@ $selfHostedSupervisor = $workerOptions + [| cleanups. It is small on purpose: it bounds how many cleanups run at once| per node, and it comes on top of the other pools, so cleanups never take| deployment, cron or high workers.+|+| The `webhooks` pool (see webhooks_queue()) processes GitHub pull request+| webhooks and Stripe events, so other jobs cannot delay them.*/$cloudSupervisors = [];@@ -51,6 +54,7 @@ foreach (['high' => ['HORIZON_HIGH_PROCESSES', 60],'default' => ['HORIZON_DEFAULT_PROCESSES', 40],'maintenance' => ['HORIZON_MAINTENANCE_PROCESSES', 10],+ 'webhooks' => ['HORIZON_WEBHOOKS_PROCESSES', 10],] as $queue => [$processesEnv, $defaultProcesses]) {$processes = filter_var(env($processesEnv), FILTER_VALIDATE_INT, ['options' => ['min_range' => 1, 'default' => $defaultProcesses],diff --git a/docs/cloud-horizon-workers.md b/docs/cloud-horizon-workers.mdindex 11ee81ef5e..fa494c65aa 100644--- a/docs/cloud-horizon-workers.md+++ b/docs/cloud-horizon-workers.md@@ -11,14 +11,18 @@ one fixed-size supervisor per queue. A busy queue cannot starve the others.| `high` | `high` | `HORIZON_HIGH_PROCESSES` | 60 || `default` | `default` | `HORIZON_DEFAULT_PROCESSES` | 40 || `maintenance` | `maintenance` | `HORIZON_MAINTENANCE_PROCESSES` | 10 |+| `webhooks` | `webhooks` | `HORIZON_WEBHOOKS_PROCESSES` | 10 |-- Counts apply to **each node**. Two nodes with the defaults run 460 workers.+- Counts apply to **each node**. Two nodes with the defaults run 480 workers.- `maintenance` runs Docker cleanups (scheduled, manual, and the cleanupthat runs after a resource stops). Remote prunes can be slow, so this poolis small on purpose: it limits how many cleanups run at the same time andkeeps them away from the `high` workers. It is added on top of the otherpools. Each cleanup has a 600 s timeout, and only one cleanup runs perserver at a time. Self-hosted instances keep cleanups on `high`.+- `webhooks` processes GitHub pull request webhooks and Stripe events, so a+ full `high` queue cannot delay them. These jobs are short (60 s timeout or+ less). Self-hosted instances keep them on `high`.- Each pool has `minProcesses = maxProcesses` and `balance = false`. Horizondoes not scale them.- A value that is not a positive integer falls back to the default.@@ -45,6 +49,7 @@ HORIZON_CRONS_PROCESSES=60HORIZON_HIGH_PROCESSES=60HORIZON_DEFAULT_PROCESSES=40HORIZON_MAINTENANCE_PROCESSES=10+HORIZON_WEBHOOKS_PROCESSES=10# Worker timeout. Must stay above the longest job timeout (36000 s) and below# retry_after (86400 s). The config clamps it to 36600..85800.HORIZON_TIMEOUT=36600@@ -108,28 +113,28 @@ Do these steps on one node at a time. Do not flush Redis or clear queues.docker exec coolify php artisan horizon:terminate```-5. Make sure the five supervisors are running on this node:+5. Make sure the six supervisors are running on this node:```bashdocker exec coolify php artisan horizon:supervisors```-6. Make sure the `maintenance` queue drains (its depth goes down) and that+6. Make sure the `maintenance` and `webhooks` queues drain (its depth goes down) and that`php artisan scheduled:diagnostics` shows no growing count of staleoccurrences.7. Do the same steps on the next node.-Jobs that the old code put on `high` (for example Docker cleanups) stay on-`high`, and the `high` pool still processes them.+Jobs that the old code put on `high` (for example Docker cleanups and webhook+jobs) stay on `high`, and the `high` pool still processes them.## Rollback1. Deploy the previous image, or remove the `HORIZON_*_PROCESSES` overrides.2. Run `config:cache` and `horizon:terminate` on one node at a time.-3. Jobs that wait on the `maintenance` queue are not lost, but the old- configuration has no `maintenance` pool. Keep one node with the new- configuration until `maintenance` is empty, or start a temporary worker:- `php artisan queue:work redis --queue=maintenance --tries=1 --timeout=600`.+3. Jobs that wait on the `maintenance` or `webhooks` queue are not lost, but+ the old configuration has no pool for them. Keep one node with the new+ configuration until both queues are empty, or start a temporary worker:+ `php artisan queue:work redis --queue=maintenance,webhooks --tries=1 --timeout=600`.4. Do not roll back the migration. The old code ignores the new columns.## Metrics to collect before you change worker countsdiff --git a/tests/Feature/HorizonConfigurationTest.php b/tests/Feature/HorizonConfigurationTest.phpindex 4f8a8d6fc3..70a2d4d6ea 100644--- a/tests/Feature/HorizonConfigurationTest.php+++ b/tests/Feature/HorizonConfigurationTest.php@@ -1,5 +1,7 @@<?php+use App\Jobs\ProcessGithubPullRequestWebhook;+use App\Jobs\StripeProcessJob;use Laravel\Horizon\ProvisioningPlan;use Laravel\Horizon\SupervisorOptions;@@ -91,12 +93,13 @@ test('cloud production runs one fixed-size pool per queue with default process c'HORIZON_HIGH_PROCESSES' => null,'HORIZON_DEFAULT_PROCESSES' => null,'HORIZON_MAINTENANCE_PROCESSES' => null,+ 'HORIZON_WEBHOOKS_PROCESSES' => null,]));$production = $parsed['production'];- expect(array_keys($production))->toBe(['deployments', 'crons', 'high', 'default', 'maintenance']);+ expect(array_keys($production))->toBe(['deployments', 'crons', 'high', 'default', 'maintenance', 'webhooks']);- foreach (['deployments' => 60, 'crons' => 60, 'high' => 60, 'default' => 40, 'maintenance' => 10] as $queue => $processes) {+ foreach (['deployments' => 60, 'crons' => 60, 'high' => 60, 'default' => 40, 'maintenance' => 10, 'webhooks' => 10] as $queue => $processes) {$pool = $production[$queue];expect($pool->queue)->toBe($queue)->and($pool->balancing())->toBeFalse()@@ -119,6 +122,7 @@ test('cloud pool sizes come from env and invalid values fall back to defaults','HORIZON_HIGH_PROCESSES' => 'abc','HORIZON_DEFAULT_PROCESSES' => '-5','HORIZON_MAINTENANCE_PROCESSES' => '4',+ 'HORIZON_WEBHOOKS_PROCESSES' => '6',]))['production'];expect($production['deployments']->maxProcesses)->toBe(25)@@ -127,7 +131,9 @@ test('cloud pool sizes come from env and invalid values fall back to defaults',->and($production['high']->maxProcesses)->toBe(60)->and($production['default']->maxProcesses)->toBe(40)->and($production['maintenance']->minProcesses)->toBe(4)- ->and($production['maintenance']->maxProcesses)->toBe(4);+ ->and($production['maintenance']->maxProcesses)->toBe(4)+ ->and($production['webhooks']->minProcesses)->toBe(6)+ ->and($production['webhooks']->maxProcesses)->toBe(6);$fallback = horizonSupervisorsFor(loadHorizonConfigWithEnv(['SELF_HOSTED' => 'false',@@ -142,7 +148,7 @@ test('cloud queue helpers route to queues that have a dedicated pool', function$production = horizonSupervisorsFor(loadHorizonConfigWithEnv(['SELF_HOSTED' => 'false']))['production'];- expect($production)->toHaveKeys([deployment_queue(), crons_queue(), maintenance_queue()]);+ expect($production)->toHaveKeys([deployment_queue(), crons_queue(), maintenance_queue(), webhooks_queue()]);});test('self-hosted queue helpers route to a queue the s6 supervisor drains', function () {@@ -157,6 +163,31 @@ test('self-hosted queue helpers route to a queue the s6 supervisor drains', funcforeach ($parsed as $supervisors) {expect(array_keys($supervisors))->toBe(['s6'])->and(explode(',', $supervisors['s6']->queue))- ->toContain(deployment_queue(), crons_queue(), maintenance_queue());+ ->toContain(deployment_queue(), crons_queue(), maintenance_queue(), webhooks_queue());}});++test('webhook jobs run on the webhooks queue on cloud and on high on self-hosted', function (bool $selfHosted, string $queue) {+ config(['constants.coolify.self_hosted' => $selfHosted]);++ $stripeJob = new StripeProcessJob(['type' => 'invoice.paid']);+ $githubJob = new ProcessGithubPullRequestWebhook(+ applicationId: 1,+ githubAppId: null,+ action: 'opened',+ pullRequestId: 1,+ pullRequestHtmlUrl: 'https://github.com/coollabsio/coolify/pull/1',+ pullRequestTitle: null,+ beforeSha: null,+ afterSha: null,+ commitSha: 'abc123',+ authorAssociation: null,+ fullName: 'coollabsio/coolify',+ );++ expect($stripeJob->queue)->toBe($queue)+ ->and($githubJob->queue)->toBe($queue);+})->with([+ 'cloud' => [false, 'webhooks'],+ 'self-hosted' => [true, 'high'],+]);