Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(webhooks): each GitHub item event refreshes or deletes its own item

garrytande236b476990
diff --git a/docs/guides/github-source.md b/docs/guides/github-source.md
index 09ed21efb7..cb53317b60 100644
--- a/docs/guides/github-source.md
+++ b/docs/guides/github-source.md
@@ -156,7 +156,11 @@ pull requests, issue comments, PR reviews, PR review comments, labels,
milestones, assignees, check runs, check suites, workflow runs. Each event
submits a targeted `sync` job that refreshes exactly the item that changed
(check events resolve the linked PR from the payload; events without an
-item reference are acknowledged and skipped). Push events follow the
+item reference are acknowledged and skipped). When 50 item refreshes are
+already waiting for the source, the event is refused with 429
+`webhook_backlog_full`; redeliver it from the webhook's Recent Deliveries
+once the queue drains, or let the next sweep catch it. A refresh that finds
+another sync of the source running is retried after it. Push events follow the
git-source behavior.
Without a public URL, use a tunnel (Tailscale Funnel, ngrok, or any HTTPS
diff --git a/src/commands/serve-http-webhooks.ts b/src/commands/serve-http-webhooks.ts
index ac8b47d050..8125b26d18 100644
--- a/src/commands/serve-http-webhooks.ts
+++ b/src/commands/serve-http-webhooks.ts
@@ -151,6 +151,9 @@ export function createWebhookLimiters() {
return { ingestRateLimiter, githubWebhookLimiter };
}
+/** #8799: item refreshes one github-kind source may have waiting before an item event is refused with 429. */
+const GITHUB_ITEM_MAX_WAITING = 50;
+
/**
* v0.46: issue/PR event handling for github-kind sources. The payload
* names a single item (repo + number); we verify the per-source HMAC and
@@ -218,6 +221,11 @@ async function handleGitHubItemEvent(
return;
}
+ // #8799: the key names the item and its intent, so a deletion never reuses an edit's refresh. The waiting cap is per
+ // source, so a cap hit can return another item's job: unless that job covers this event (the same item and intent, or an
+ // explicit source-wide sync for an edit), it is a refusal, never this event's 202. A busy source defers the job (sync handler).
+ const deleted = eventName === 'issues' && parsed.action === 'deleted';
+ const idempotencyKey = `webhook:item:${source.id}:${ref.repo}:${ref.number}:${deleted ? 'delete' : 'refresh'}:${Math.floor(Date.now() / 30_000)}`;
try {
const queue = new MinionQueue(engine);
const job = await queue.add(
@@ -229,16 +237,27 @@ async function handleGitHubItemEvent(
repo: ref.repo,
number: ref.number,
kind: ref.kind,
- ...(eventName === 'issues' && parsed.action === 'deleted' ? { deleted: true } : {}),
+ ...(deleted ? { deleted: true } : {}),
},
embed_reason: 'webhook',
},
{
priority: -10,
- idempotency_key: `webhook:item:${source.id}:${ref.repo}:${ref.number}:${Math.floor(Date.now() / 30_000)}`,
- maxWaiting: 1,
+ idempotency_key: idempotencyKey,
+ maxWaiting: GITHUB_ITEM_MAX_WAITING,
},
);
+ const coalesced = job.data as { github_item?: { repo?: unknown; number?: unknown; deleted?: unknown }; embed_reason?: unknown } | null;
+ const coalescedItem = coalesced?.github_item;
+ const covered = job.idempotency_key === idempotencyKey || (coalescedItem
+ ? coalescedItem.repo === ref.repo && coalescedItem.number === ref.number && (coalescedItem.deleted === true) === deleted
+ : !deleted && coalesced?.embed_reason !== 'autopilot_freshness');
+ if (!covered) {
+ res.setHeader('Retry-After', '30');
+ res.status(429).json({ error: 'webhook_backlog_full', source_id: source.id,
+ message: `${GITHUB_ITEM_MAX_WAITING} item refreshes are already waiting for this source; redeliver this event once they drain, or run gbrain sync --source ${source.id}.` });
+ return;
+ }
res.status(202).json({ job_id: job.id, source_id: source.id, item: ref });
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
diff --git a/src/core/minions/handlers/sync.ts b/src/core/minions/handlers/sync.ts
index d068925e92..c2a7bfe4ac 100644
--- a/src/core/minions/handlers/sync.ts
+++ b/src/core/minions/handlers/sync.ts
@@ -4,9 +4,18 @@
import type { BrainEngine } from '../../engine.ts';
import type { MinionHandler } from '../types.ts';
import { resolveJobPull } from './job-pull.ts';
-import { UnrecoverableError } from '../errors.ts';
+import { JobDeferredError, UnrecoverableError } from '../errors.ts';
import type { SyncResult } from '../../../commands/sync.ts';
+/**
+ * #8799: a webhook's single-item refresh that finds its source busy is deferred instead of completing as a skip, so the
+ * queue runs it with its item and delete intent once the source is free, burning no attempt; a source-wide skip would
+ * record only a pull marker for it.
+ */
+function githubItemBusy(sourceId: string, item: { repo: string; number: number; deleted: boolean }): JobDeferredError {
+ return new JobDeferredError('sync_in_progress', `another sync holds source ${sourceId}; the ${item.deleted ? 'deletion' : 'refresh'} of ${item.repo}#${item.number} runs after it.`, 30_000);
+}
+
export function makeSyncHandler(engine: BrainEngine): MinionHandler {
return async (job) => {
const { performSync } = await import('../../../commands/sync.ts');
@@ -103,6 +112,7 @@ export function makeSyncHandler(engine: BrainEngine): MinionHandler {
AND (COALESCE(data->>'sourceId', data->>'source_id')=$2 OR (COALESCE(data->>'sourceId', data->>'source_id') IS NULL AND ($4::text IS NULL AND $2='default' OR data->>'repoPath'=$4))) LIMIT 1`,
[job.id, sourceId ?? 'default', typeof job.data.continuation_of === 'number' ? job.data.continuation_of : -1, repoPath ?? null]).catch(() => []);
if (live) {
+ if (githubItem) throw githubItemBusy(sourceId ?? 'default', githubItem);
console.error(`[sync] skipped: sync job #${live.id} is already draining ${sourceId ?? 'default'}.`);
return { skipped: true, reason: 'sync_in_progress', source_id: sourceId ?? 'default', running_job_id: Number(live.id),
next: { command: `gbrain jobs get ${live.id} --json`, safe_to_loop: true, retry_after_ms: 30_000, why: `Sync job #${live.id} is draining source ${sourceId ?? 'default'}; it resumes the cursor, so nothing else needs to run.` } };
@@ -132,6 +142,7 @@ export function makeSyncHandler(engine: BrainEngine): MinionHandler {
// advanced) anchor.
const { SyncLockBusyError } = await import('../../../commands/sync.ts');
if (err instanceof SyncLockBusyError) {
+ if (githubItem) throw githubItemBusy(sourceId ?? 'default', githubItem);
// #6712 PR 1 (folds #6706's live-job guard): one drainer per source is the drain lease; a job that finds it held completes as a skip.
const id = sourceId ?? 'default';
const { readDrainHolder, publicDrainHolder } = await import('../../persistence/sync-lease.ts');
diff --git a/test/serve-http-github-items-8799.test.ts b/test/serve-http-github-items-8799.test.ts
new file mode 100644
index 0000000000..64675d61e7
--- /dev/null
+++ b/test/serve-http-github-items-8799.test.ts
@@ -0,0 +1,117 @@
+/**
+ * #8799: a GitHub item webhook queues a refresh for exactly the item it names.
+ * Before the fix a waiting refresh for one item of a source absorbed every
+ * other item's event (a source-wide waiting cap of 1, answered 202 with the
+ * first item's job), and an issues/deleted event inside the same 30 s bucket
+ * as an edit reused the edit's job, so the page was refetched, not removed.
+ * A cap hit answers 429 unless the job it returns covers the event.
+ *
+ * Real PGLite queue, synthetic repository and secret.
+ */
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import express from 'express';
+import { createHmac } from 'node:crypto';
+import type { Server } from 'node:http';
+import type { AddressInfo } from 'node:net';
+import { PGLiteEngine } from '../src/core/pglite-engine.ts';
+import { MinionQueue } from '../src/core/minions/queue.ts';
+import { mountWebhooks } from '../src/commands/serve-http-webhooks.ts';
+import type { ServeHttpContext } from '../src/commands/serve-http.ts';
+
+const SECRET = 'example-webhook-secret';
+const REPO = 'acme-example/app';
+let engine: PGLiteEngine;
+let server: Server;
+let base: string;
+
+beforeAll(async () => {
+ engine = new PGLiteEngine();
+ await engine.connect({});
+ await engine.initSchema();
+ for (const [id, repo] of [['source-a', REPO], ['source-b', 'acme-example/other'], ['source-c', 'acme-example/third'], ['source-d', 'acme-example/fourth']]) {
+ await engine.executeRaw('INSERT INTO sources (id, name, config) VALUES ($1, $1, $2::jsonb)',
+ [id, JSON.stringify({ kind: 'github', gh_scope: 'repos', gh_repos: repo, webhook_secret: SECRET })]);
+ }
+ const app = express();
+ const pass: express.RequestHandler = (_req, _res, next) => next();
+ mountWebhooks(app, {
+ engine,
+ resourceVerifier: { verifyAccessToken: async () => { throw new Error('unused'); } },
+ resourceMetadataUrl: 'http://127.0.0.1/.well-known/oauth-protected-resource',
+ ingestRateLimiter: pass,
+ githubWebhookLimiter: pass,
+ broadcastEvent: () => {},
+ } as unknown as ServeHttpContext);
+ server = await new Promise<Server>(resolve => { const s = app.listen(0, '127.0.0.1', () => resolve(s)); });
+ base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
+}, 60_000);
+
+afterAll(async () => {
+ await new Promise<void>(resolve => server?.close(() => resolve()));
+ await engine?.disconnect();
+}, 60_000);
+
+async function issueEvent(number: number, action: string, repo = REPO) {
+ const payload = JSON.stringify({ action, repository: { full_name: repo }, issue: { number } });
+ const res = await fetch(`${base}/webhooks/github`, {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', 'x-github-event': 'issues',
+ 'x-hub-signature-256': `sha256=${createHmac('sha256', SECRET).update(payload).digest('hex')}` },
+ body: payload,
+ });
+ return { status: res.status, body: await res.json() as Record<string, unknown> };
+}
+
+const waitingItems = async (sourceId: string) => (await engine.executeRaw<{ item: { number: number; deleted?: boolean } }>(
+ "SELECT data->'github_item' AS item FROM minion_jobs WHERE name = 'sync' AND status = 'waiting' AND data->>'sourceId' = $1 ORDER BY id", [sourceId]))
+ .map(row => (typeof row.item === 'string' ? JSON.parse(row.item) : row.item) as { number: number; deleted?: boolean });
+
+describe('#8799 GitHub item webhooks', () => {
+ test('two items of one source each get their own refresh job', async () => {
+ const first = await issueEvent(1, 'edited');
+ const second = await issueEvent(2, 'edited');
+ expect(first.status).toBe(202);
+ expect(second.status).toBe(202);
+ expect(second.body.job_id).not.toBe(first.body.job_id);
+ expect((await waitingItems('source-a')).map(item => item.number)).toEqual([1, 2]);
+ });
+
+ test('a repeated event for the same item inside the bucket reuses its job', async () => {
+ const again = await issueEvent(1, 'labeled');
+ const [row] = await engine.executeRaw<{ id: number }>(
+ "SELECT id FROM minion_jobs WHERE name = 'sync' AND status = 'waiting' AND (data->'github_item'->>'number')::int = 1 ORDER BY id LIMIT 1");
+ expect(again.status).toBe(202);
+ expect(again.body.job_id).toBe(row!.id);
+ });
+
+ test('a deletion right after an edit queues a delete, not the edit refresh', async () => {
+ const deleted = await issueEvent(1, 'deleted');
+ expect(deleted.status).toBe(202);
+ const [job] = await engine.executeRaw<{ item: unknown }>("SELECT data->'github_item' AS item FROM minion_jobs WHERE id = $1", [deleted.body.job_id]);
+ const item = (typeof job!.item === 'string' ? JSON.parse(job!.item) : job!.item) as { number: number; deleted?: boolean };
+ expect(item).toMatchObject({ number: 1, deleted: true });
+ });
+
+ test('a full backlog refuses the event with 429 instead of answering with another item', async () => {
+ for (let number = 1; number <= 50; number++) expect((await issueEvent(number, 'edited', 'acme-example/other')).status).toBe(202);
+ const refused = await issueEvent(51, 'edited', 'acme-example/other');
+ expect(refused.status).toBe(429);
+ expect(refused.body).toMatchObject({ error: 'webhook_backlog_full', source_id: 'source-b' });
+ expect((await waitingItems('source-b')).some(item => item.number === 51)).toBe(false);
+ });
+
+ test('at a full backlog a waiting source-wide sync covers an edit but not a deletion', async () => {
+ for (let number = 1; number <= 49; number++) expect((await issueEvent(number, 'edited', 'acme-example/third')).status).toBe(202);
+ const sweep = await new MinionQueue(engine).add('sync', { sourceId: 'source-c', noExtract: false }, { priority: -10 });
+ const edit = await issueEvent(60, 'edited', 'acme-example/third');
+ expect(edit.status).toBe(202);
+ expect(edit.body.job_id).toBe(sweep.id);
+ expect((await issueEvent(61, 'deleted', 'acme-example/third')).status).toBe(429);
+ });
+
+ test('at a full backlog a waiting automatic freshness sync, which may skip, covers nothing', async () => {
+ for (let number = 1; number <= 49; number++) expect((await issueEvent(number, 'edited', 'acme-example/fourth')).status).toBe(202);
+ await new MinionQueue(engine).add('sync', { sourceId: 'source-d', embed_reason: 'autopilot_freshness' }, { priority: -10 });
+ expect((await issueEvent(60, 'edited', 'acme-example/fourth')).status).toBe(429);
+ });
+});
diff --git a/test/sync-handler-github-item-busy-8799.test.ts b/test/sync-handler-github-item-busy-8799.test.ts
new file mode 100644
index 0000000000..1526b1790d
--- /dev/null
+++ b/test/sync-handler-github-item-busy-8799.test.ts
@@ -0,0 +1,57 @@
+/**
+ * #8799: a GitHub webhook's single-item refresh that finds its source busy is deferred with its item and delete intent.
+ * Before the fix the sync handler completed it as a sync_in_progress skip, which records only a source-wide pull marker,
+ * so the item's refresh, or its deletion, was lost while another sync of the source ran. A plain sync job still skips.
+ * Real PGLite, the real sync handler, the per-source sync lock held by a stand-in holder; no network.
+ */
+import { afterAll, beforeAll, expect, test } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { PGLiteEngine } from '../src/core/pglite-engine.ts';
+import { syncLockId, tryAcquireDbLock } from '../src/core/db-lock.ts';
+import { makeSyncHandler } from '../src/core/minions/handlers/sync.ts';
+import { JobDeferredError } from '../src/core/minions/errors.ts';
+import type { MinionJobContext } from '../src/core/minions/types.ts';
+import { withEnv } from './helpers/with-env.ts';
+
+const SOURCE = 'source-a';
+const home = mkdtempSync(join(tmpdir(), 'gbrain-8799-busy-'));
+let engine: PGLiteEngine;
+
+beforeAll(async () => {
+ engine = new PGLiteEngine();
+ await engine.connect({});
+ await engine.initSchema();
+ await engine.executeRaw('INSERT INTO sources (id, name, local_path, config) VALUES ($1, $1, $2, $3::jsonb)', [SOURCE, join(home, SOURCE),
+ JSON.stringify({ kind: 'github', gh_scope: 'repos', gh_repos: 'acme-example/app', gh_token_env: 'CONNECTOR_TEST_TOKEN' })]);
+}, 60_000);
+
+afterAll(async () => {
+ await engine?.disconnect();
+ rmSync(home, { recursive: true, force: true });
+}, 60_000);
+
+const job = (data: Record<string, unknown>) => ({ id: 9201, name: 'sync', data: { sourceId: SOURCE, ...data }, attempts_made: 0,
+ signal: new AbortController().signal, shutdownSignal: new AbortController().signal, deadlineAtMs: Date.now() + 60_000,
+ updateProgress: async () => undefined, updateTokens: async () => undefined, log: async () => undefined, isActive: async () => true,
+ readInbox: async () => [] }) as unknown as MinionJobContext;
+
+const deferral = async (run: Promise<unknown>) => {
+ const err = await run.then(() => null, (e: unknown) => e);
+ expect(err).toBeInstanceOf(JobDeferredError);
+ return { reason: (err as JobDeferredError).reason, retryInMs: (err as JobDeferredError).retryInMs, message: (err as JobDeferredError).message };
+};
+
+test('an item refresh or deletion on a busy source is deferred, burning no attempt; a plain sync job still skips', async () => withEnv({ GBRAIN_HOME: home, CONNECTOR_TEST_TOKEN: 'synthetic-local-fixture' }, async () => {
+ const held = await tryAcquireDbLock(engine, syncLockId(SOURCE), 5);
+ expect(held).not.toBeNull();
+ try {
+ const handler = makeSyncHandler(engine);
+ expect(await deferral(handler(job({ github_item: { repo: 'acme-example/app', number: 1, kind: 'issue' } }))))
+ .toMatchObject({ reason: 'sync_in_progress', retryInMs: 30_000, message: expect.stringMatching(/refresh of acme-example\/app#1/) });
+ expect(await deferral(handler(job({ github_item: { repo: 'acme-example/app', number: 2, kind: 'issue', deleted: true } }))))
+ .toMatchObject({ reason: 'sync_in_progress', retryInMs: 30_000, message: expect.stringMatching(/deletion of acme-example\/app#2/) });
+ expect(await handler(job({ pull: false }))).toMatchObject({ skipped: true, reason: 'sync_in_progress', source_id: SOURCE });
+ } finally { await held!.release(); }
+}), 120_000);