Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(horizon): route webhook jobs to a dedicated cloud webhooks queue

Andras Bacsai083e71c48154
diff --git a/.env.development.example b/.env.development.example
index 4f4198ddb2..f7a01180e3 100644
--- a/.env.development.example
+++ b/.env.development.example
@@ -39,6 +39,8 @@ DB_PORT=5432
# HORIZON_DEFAULT_PROCESSES=40
# Docker cleanups; small on purpose to bound concurrent slow remote prunes.
# HORIZON_MAINTENANCE_PROCESSES=10
+# GitHub pull request webhooks and Stripe events.
+# HORIZON_WEBHOOKS_PROCESSES=10
# Server-Timing headers + on-screen HUD (defaults ON when APP_ENV=local).
# Force on in any environment (including production): SERVER_TIMING_ENABLED=true
diff --git a/app/Jobs/ProcessGithubPullRequestWebhook.php b/app/Jobs/ProcessGithubPullRequestWebhook.php
index 5186a58bbf..362ad0a2d8 100644
--- a/app/Jobs/ProcessGithubPullRequestWebhook.php
+++ b/app/Jobs/ProcessGithubPullRequestWebhook.php
@@ -41,7 +41,7 @@ class ProcessGithubPullRequestWebhook implements ShouldBeEncrypted, ShouldQueue
public string $fullName,
public bool $isForkPullRequest = false,
) {
- $this->onQueue('high');
+ $this->onQueue(webhooks_queue());
}
public function handle(): void
diff --git a/app/Jobs/StripeProcessJob.php b/app/Jobs/StripeProcessJob.php
index 0f56476e25..58e67de085 100644
--- a/app/Jobs/StripeProcessJob.php
+++ b/app/Jobs/StripeProcessJob.php
@@ -23,7 +23,7 @@ class StripeProcessJob implements ShouldBeEncrypted, ShouldQueue
public function __construct(public $event)
{
- $this->onQueue('high');
+ $this->onQueue(webhooks_queue());
}
public function handle(): void
diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php
index da6faa43ee..a3be2667ba 100644
--- a/bootstrap/helpers/shared.php
+++ b/bootstrap/helpers/shared.php
@@ -1070,6 +1070,23 @@ function maintenance_queue(): string
return isCloud() ? 'maintenance' : 'high';
}
+/**
+ * Resolve the queue used for incoming webhook processing — GitHub pull request webhooks and
+ * Stripe events.
+ *
+ * On cloud these jobs run on a dedicated `webhooks` queue so a busy `high` queue cannot delay
+ * them; self-hosted keeps them on the shared `high` queue, so a custom `HORIZON_QUEUES` does
+ * not need a new queue name. Routing is decided by `isCloud()` (config-based), so the
+ * dispatching process needs no special env — only the worker must be configured to drain `webhooks`.
+ *
+ * On cloud, config/horizon.php provisions a dedicated `webhooks` pool in production
+ * (see docs/cloud-horizon-workers.md).
+ */
+function webhooks_queue(): string
+{
+ return isCloud() ? 'webhooks' : 'high';
+}
+
function translate_cron_expression($expression_to_validate): string
{
if (isset(VALID_CRON_STRINGS[$expression_to_validate])) {
diff --git a/config/horizon.php b/config/horizon.php
index cd532f936c..c29ad15fc9 100644
--- a/config/horizon.php
+++ b/config/horizon.php
@@ -42,6 +42,9 @@ $selfHostedSupervisor = $workerOptions + [
| cleanups. It is small on purpose: it bounds how many cleanups run at once
| per node, and it comes on top of the other pools, so cleanups never take
| deployment, cron or high workers.
+|
+| The `webhooks` pool (see webhooks_queue()) processes GitHub pull request
+| webhooks and Stripe events, so other jobs cannot delay them.
*/
$cloudSupervisors = [];
@@ -51,6 +54,7 @@ foreach ([
'high' => ['HORIZON_HIGH_PROCESSES', 60],
'default' => ['HORIZON_DEFAULT_PROCESSES', 40],
'maintenance' => ['HORIZON_MAINTENANCE_PROCESSES', 10],
+ 'webhooks' => ['HORIZON_WEBHOOKS_PROCESSES', 10],
] as $queue => [$processesEnv, $defaultProcesses]) {
$processes = filter_var(env($processesEnv), FILTER_VALIDATE_INT, [
'options' => ['min_range' => 1, 'default' => $defaultProcesses],
diff --git a/docs/cloud-horizon-workers.md b/docs/cloud-horizon-workers.md
index 11ee81ef5e..fa494c65aa 100644
--- a/docs/cloud-horizon-workers.md
+++ b/docs/cloud-horizon-workers.md
@@ -11,14 +11,18 @@ one fixed-size supervisor per queue. A busy queue cannot starve the others.
| `high` | `high` | `HORIZON_HIGH_PROCESSES` | 60 |
| `default` | `default` | `HORIZON_DEFAULT_PROCESSES` | 40 |
| `maintenance` | `maintenance` | `HORIZON_MAINTENANCE_PROCESSES` | 10 |
+| `webhooks` | `webhooks` | `HORIZON_WEBHOOKS_PROCESSES` | 10 |
-- Counts apply to **each node**. Two nodes with the defaults run 460 workers.
+- Counts apply to **each node**. Two nodes with the defaults run 480 workers.
- `maintenance` runs Docker cleanups (scheduled, manual, and the cleanup
that runs after a resource stops). Remote prunes can be slow, so this pool
is small on purpose: it limits how many cleanups run at the same time and
keeps them away from the `high` workers. It is added on top of the other
pools. Each cleanup has a 600 s timeout, and only one cleanup runs per
server at a time. Self-hosted instances keep cleanups on `high`.
+- `webhooks` processes GitHub pull request webhooks and Stripe events, so a
+ full `high` queue cannot delay them. These jobs are short (60 s timeout or
+ less). Self-hosted instances keep them on `high`.
- Each pool has `minProcesses = maxProcesses` and `balance = false`. Horizon
does not scale them.
- A value that is not a positive integer falls back to the default.
@@ -45,6 +49,7 @@ HORIZON_CRONS_PROCESSES=60
HORIZON_HIGH_PROCESSES=60
HORIZON_DEFAULT_PROCESSES=40
HORIZON_MAINTENANCE_PROCESSES=10
+HORIZON_WEBHOOKS_PROCESSES=10
# Worker timeout. Must stay above the longest job timeout (36000 s) and below
# retry_after (86400 s). The config clamps it to 36600..85800.
HORIZON_TIMEOUT=36600
@@ -108,28 +113,28 @@ Do these steps on one node at a time. Do not flush Redis or clear queues.
docker exec coolify php artisan horizon:terminate
```
-5. Make sure the five supervisors are running on this node:
+5. Make sure the six supervisors are running on this node:
```bash
docker exec coolify php artisan horizon:supervisors
```
-6. Make sure the `maintenance` queue drains (its depth goes down) and that
+6. Make sure the `maintenance` and `webhooks` queues drain (its depth goes down) and that
`php artisan scheduled:diagnostics` shows no growing count of stale
occurrences.
7. Do the same steps on the next node.
-Jobs that the old code put on `high` (for example Docker cleanups) stay on
-`high`, and the `high` pool still processes them.
+Jobs that the old code put on `high` (for example Docker cleanups and webhook
+jobs) stay on `high`, and the `high` pool still processes them.
## Rollback
1. Deploy the previous image, or remove the `HORIZON_*_PROCESSES` overrides.
2. Run `config:cache` and `horizon:terminate` on one node at a time.
-3. Jobs that wait on the `maintenance` queue are not lost, but the old
- configuration has no `maintenance` pool. Keep one node with the new
- configuration until `maintenance` is empty, or start a temporary worker:
- `php artisan queue:work redis --queue=maintenance --tries=1 --timeout=600`.
+3. Jobs that wait on the `maintenance` or `webhooks` queue are not lost, but
+ the old configuration has no pool for them. Keep one node with the new
+ configuration until both queues are empty, or start a temporary worker:
+ `php artisan queue:work redis --queue=maintenance,webhooks --tries=1 --timeout=600`.
4. Do not roll back the migration. The old code ignores the new columns.
## Metrics to collect before you change worker counts
diff --git a/tests/Feature/HorizonConfigurationTest.php b/tests/Feature/HorizonConfigurationTest.php
index 4f8a8d6fc3..70a2d4d6ea 100644
--- a/tests/Feature/HorizonConfigurationTest.php
+++ b/tests/Feature/HorizonConfigurationTest.php
@@ -1,5 +1,7 @@
<?php
+use App\Jobs\ProcessGithubPullRequestWebhook;
+use App\Jobs\StripeProcessJob;
use Laravel\Horizon\ProvisioningPlan;
use Laravel\Horizon\SupervisorOptions;
@@ -91,12 +93,13 @@ test('cloud production runs one fixed-size pool per queue with default process c
'HORIZON_HIGH_PROCESSES' => null,
'HORIZON_DEFAULT_PROCESSES' => null,
'HORIZON_MAINTENANCE_PROCESSES' => null,
+ 'HORIZON_WEBHOOKS_PROCESSES' => null,
]));
$production = $parsed['production'];
- expect(array_keys($production))->toBe(['deployments', 'crons', 'high', 'default', 'maintenance']);
+ expect(array_keys($production))->toBe(['deployments', 'crons', 'high', 'default', 'maintenance', 'webhooks']);
- foreach (['deployments' => 60, 'crons' => 60, 'high' => 60, 'default' => 40, 'maintenance' => 10] as $queue => $processes) {
+ foreach (['deployments' => 60, 'crons' => 60, 'high' => 60, 'default' => 40, 'maintenance' => 10, 'webhooks' => 10] as $queue => $processes) {
$pool = $production[$queue];
expect($pool->queue)->toBe($queue)
->and($pool->balancing())->toBeFalse()
@@ -119,6 +122,7 @@ test('cloud pool sizes come from env and invalid values fall back to defaults',
'HORIZON_HIGH_PROCESSES' => 'abc',
'HORIZON_DEFAULT_PROCESSES' => '-5',
'HORIZON_MAINTENANCE_PROCESSES' => '4',
+ 'HORIZON_WEBHOOKS_PROCESSES' => '6',
]))['production'];
expect($production['deployments']->maxProcesses)->toBe(25)
@@ -127,7 +131,9 @@ test('cloud pool sizes come from env and invalid values fall back to defaults',
->and($production['high']->maxProcesses)->toBe(60)
->and($production['default']->maxProcesses)->toBe(40)
->and($production['maintenance']->minProcesses)->toBe(4)
- ->and($production['maintenance']->maxProcesses)->toBe(4);
+ ->and($production['maintenance']->maxProcesses)->toBe(4)
+ ->and($production['webhooks']->minProcesses)->toBe(6)
+ ->and($production['webhooks']->maxProcesses)->toBe(6);
$fallback = horizonSupervisorsFor(loadHorizonConfigWithEnv([
'SELF_HOSTED' => 'false',
@@ -142,7 +148,7 @@ test('cloud queue helpers route to queues that have a dedicated pool', function
$production = horizonSupervisorsFor(loadHorizonConfigWithEnv(['SELF_HOSTED' => 'false']))['production'];
- expect($production)->toHaveKeys([deployment_queue(), crons_queue(), maintenance_queue()]);
+ expect($production)->toHaveKeys([deployment_queue(), crons_queue(), maintenance_queue(), webhooks_queue()]);
});
test('self-hosted queue helpers route to a queue the s6 supervisor drains', function () {
@@ -157,6 +163,31 @@ test('self-hosted queue helpers route to a queue the s6 supervisor drains', func
foreach ($parsed as $supervisors) {
expect(array_keys($supervisors))->toBe(['s6'])
->and(explode(',', $supervisors['s6']->queue))
- ->toContain(deployment_queue(), crons_queue(), maintenance_queue());
+ ->toContain(deployment_queue(), crons_queue(), maintenance_queue(), webhooks_queue());
}
});
+
+test('webhook jobs run on the webhooks queue on cloud and on high on self-hosted', function (bool $selfHosted, string $queue) {
+ config(['constants.coolify.self_hosted' => $selfHosted]);
+
+ $stripeJob = new StripeProcessJob(['type' => 'invoice.paid']);
+ $githubJob = new ProcessGithubPullRequestWebhook(
+ applicationId: 1,
+ githubAppId: null,
+ action: 'opened',
+ pullRequestId: 1,
+ pullRequestHtmlUrl: 'https://github.com/coollabsio/coolify/pull/1',
+ pullRequestTitle: null,
+ beforeSha: null,
+ afterSha: null,
+ commitSha: 'abc123',
+ authorAssociation: null,
+ fullName: 'coollabsio/coolify',
+ );
+
+ expect($stripeJob->queue)->toBe($queue)
+ ->and($githubJob->queue)->toBe($queue);
+})->with([
+ 'cloud' => [false, 'webhooks'],
+ 'self-hosted' => [true, 'high'],
+]);