Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(webhooks): each GitHub item event refreshes or deletes its own item
garrytande236b476990
diff --git a/docs/guides/github-source.md b/docs/guides/github-source.mdindex 09ed21efb7..cb53317b60 100644--- a/docs/guides/github-source.md+++ b/docs/guides/github-source.md@@ -156,7 +156,11 @@ pull requests, issue comments, PR reviews, PR review comments, labels,milestones, assignees, check runs, check suites, workflow runs. Each eventsubmits a targeted `sync` job that refreshes exactly the item that changed(check events resolve the linked PR from the payload; events without an-item reference are acknowledged and skipped). Push events follow the+item reference are acknowledged and skipped). When 50 item refreshes are+already waiting for the source, the event is refused with 429+`webhook_backlog_full`; redeliver it from the webhook's Recent Deliveries+once the queue drains, or let the next sweep catch it. A refresh that finds+another sync of the source running is retried after it. Push events follow thegit-source behavior.Without a public URL, use a tunnel (Tailscale Funnel, ngrok, or any HTTPSdiff --git a/src/commands/serve-http-webhooks.ts b/src/commands/serve-http-webhooks.tsindex ac8b47d050..8125b26d18 100644--- a/src/commands/serve-http-webhooks.ts+++ b/src/commands/serve-http-webhooks.ts@@ -151,6 +151,9 @@ export function createWebhookLimiters() {return { ingestRateLimiter, githubWebhookLimiter };}+/** #8799: item refreshes one github-kind source may have waiting before an item event is refused with 429. */+const GITHUB_ITEM_MAX_WAITING = 50;+/*** v0.46: issue/PR event handling for github-kind sources. The payload* names a single item (repo + number); we verify the per-source HMAC and@@ -218,6 +221,11 @@ async function handleGitHubItemEvent(return;}+ // #8799: the key names the item and its intent, so a deletion never reuses an edit's refresh. The waiting cap is per+ // source, so a cap hit can return another item's job: unless that job covers this event (the same item and intent, or an+ // explicit source-wide sync for an edit), it is a refusal, never this event's 202. A busy source defers the job (sync handler).+ const deleted = eventName === 'issues' && parsed.action === 'deleted';+ const idempotencyKey = `webhook:item:${source.id}:${ref.repo}:${ref.number}:${deleted ? 'delete' : 'refresh'}:${Math.floor(Date.now() / 30_000)}`;try {const queue = new MinionQueue(engine);const job = await queue.add(@@ -229,16 +237,27 @@ async function handleGitHubItemEvent(repo: ref.repo,number: ref.number,kind: ref.kind,- ...(eventName === 'issues' && parsed.action === 'deleted' ? { deleted: true } : {}),+ ...(deleted ? { deleted: true } : {}),},embed_reason: 'webhook',},{priority: -10,- idempotency_key: `webhook:item:${source.id}:${ref.repo}:${ref.number}:${Math.floor(Date.now() / 30_000)}`,- maxWaiting: 1,+ idempotency_key: idempotencyKey,+ maxWaiting: GITHUB_ITEM_MAX_WAITING,},);+ const coalesced = job.data as { github_item?: { repo?: unknown; number?: unknown; deleted?: unknown }; embed_reason?: unknown } | null;+ const coalescedItem = coalesced?.github_item;+ const covered = job.idempotency_key === idempotencyKey || (coalescedItem+ ? coalescedItem.repo === ref.repo && coalescedItem.number === ref.number && (coalescedItem.deleted === true) === deleted+ : !deleted && coalesced?.embed_reason !== 'autopilot_freshness');+ if (!covered) {+ res.setHeader('Retry-After', '30');+ res.status(429).json({ error: 'webhook_backlog_full', source_id: source.id,+ message: `${GITHUB_ITEM_MAX_WAITING} item refreshes are already waiting for this source; redeliver this event once they drain, or run gbrain sync --source ${source.id}.` });+ return;+ }res.status(202).json({ job_id: job.id, source_id: source.id, item: ref });} catch (err) {const msg = err instanceof Error ? err.message : String(err);diff --git a/src/core/minions/handlers/sync.ts b/src/core/minions/handlers/sync.tsindex d068925e92..c2a7bfe4ac 100644--- a/src/core/minions/handlers/sync.ts+++ b/src/core/minions/handlers/sync.ts@@ -4,9 +4,18 @@import type { BrainEngine } from '../../engine.ts';import type { MinionHandler } from '../types.ts';import { resolveJobPull } from './job-pull.ts';-import { UnrecoverableError } from '../errors.ts';+import { JobDeferredError, UnrecoverableError } from '../errors.ts';import type { SyncResult } from '../../../commands/sync.ts';+/**+ * #8799: a webhook's single-item refresh that finds its source busy is deferred instead of completing as a skip, so the+ * queue runs it with its item and delete intent once the source is free, burning no attempt; a source-wide skip would+ * record only a pull marker for it.+ */+function githubItemBusy(sourceId: string, item: { repo: string; number: number; deleted: boolean }): JobDeferredError {+ return new JobDeferredError('sync_in_progress', `another sync holds source ${sourceId}; the ${item.deleted ? 'deletion' : 'refresh'} of ${item.repo}#${item.number} runs after it.`, 30_000);+}+export function makeSyncHandler(engine: BrainEngine): MinionHandler {return async (job) => {const { performSync } = await import('../../../commands/sync.ts');@@ -103,6 +112,7 @@ export function makeSyncHandler(engine: BrainEngine): MinionHandler {AND (COALESCE(data->>'sourceId', data->>'source_id')=$2 OR (COALESCE(data->>'sourceId', data->>'source_id') IS NULL AND ($4::text IS NULL AND $2='default' OR data->>'repoPath'=$4))) LIMIT 1`,[job.id, sourceId ?? 'default', typeof job.data.continuation_of === 'number' ? job.data.continuation_of : -1, repoPath ?? null]).catch(() => []);if (live) {+ if (githubItem) throw githubItemBusy(sourceId ?? 'default', githubItem);console.error(`[sync] skipped: sync job #${live.id} is already draining ${sourceId ?? 'default'}.`);return { skipped: true, reason: 'sync_in_progress', source_id: sourceId ?? 'default', running_job_id: Number(live.id),next: { command: `gbrain jobs get ${live.id} --json`, safe_to_loop: true, retry_after_ms: 30_000, why: `Sync job #${live.id} is draining source ${sourceId ?? 'default'}; it resumes the cursor, so nothing else needs to run.` } };@@ -132,6 +142,7 @@ export function makeSyncHandler(engine: BrainEngine): MinionHandler {// advanced) anchor.const { SyncLockBusyError } = await import('../../../commands/sync.ts');if (err instanceof SyncLockBusyError) {+ if (githubItem) throw githubItemBusy(sourceId ?? 'default', githubItem);// #6712 PR 1 (folds #6706's live-job guard): one drainer per source is the drain lease; a job that finds it held completes as a skip.const id = sourceId ?? 'default';const { readDrainHolder, publicDrainHolder } = await import('../../persistence/sync-lease.ts');diff --git a/test/serve-http-github-items-8799.test.ts b/test/serve-http-github-items-8799.test.tsnew file mode 100644index 0000000000..64675d61e7--- /dev/null+++ b/test/serve-http-github-items-8799.test.ts@@ -0,0 +1,117 @@+/**+ * #8799: a GitHub item webhook queues a refresh for exactly the item it names.+ * Before the fix a waiting refresh for one item of a source absorbed every+ * other item's event (a source-wide waiting cap of 1, answered 202 with the+ * first item's job), and an issues/deleted event inside the same 30 s bucket+ * as an edit reused the edit's job, so the page was refetched, not removed.+ * A cap hit answers 429 unless the job it returns covers the event.+ *+ * Real PGLite queue, synthetic repository and secret.+ */+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';+import express from 'express';+import { createHmac } from 'node:crypto';+import type { Server } from 'node:http';+import type { AddressInfo } from 'node:net';+import { PGLiteEngine } from '../src/core/pglite-engine.ts';+import { MinionQueue } from '../src/core/minions/queue.ts';+import { mountWebhooks } from '../src/commands/serve-http-webhooks.ts';+import type { ServeHttpContext } from '../src/commands/serve-http.ts';++const SECRET = 'example-webhook-secret';+const REPO = 'acme-example/app';+let engine: PGLiteEngine;+let server: Server;+let base: string;++beforeAll(async () => {+ engine = new PGLiteEngine();+ await engine.connect({});+ await engine.initSchema();+ for (const [id, repo] of [['source-a', REPO], ['source-b', 'acme-example/other'], ['source-c', 'acme-example/third'], ['source-d', 'acme-example/fourth']]) {+ await engine.executeRaw('INSERT INTO sources (id, name, config) VALUES ($1, $1, $2::jsonb)',+ [id, JSON.stringify({ kind: 'github', gh_scope: 'repos', gh_repos: repo, webhook_secret: SECRET })]);+ }+ const app = express();+ const pass: express.RequestHandler = (_req, _res, next) => next();+ mountWebhooks(app, {+ engine,+ resourceVerifier: { verifyAccessToken: async () => { throw new Error('unused'); } },+ resourceMetadataUrl: 'http://127.0.0.1/.well-known/oauth-protected-resource',+ ingestRateLimiter: pass,+ githubWebhookLimiter: pass,+ broadcastEvent: () => {},+ } as unknown as ServeHttpContext);+ server = await new Promise<Server>(resolve => { const s = app.listen(0, '127.0.0.1', () => resolve(s)); });+ base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;+}, 60_000);++afterAll(async () => {+ await new Promise<void>(resolve => server?.close(() => resolve()));+ await engine?.disconnect();+}, 60_000);++async function issueEvent(number: number, action: string, repo = REPO) {+ const payload = JSON.stringify({ action, repository: { full_name: repo }, issue: { number } });+ const res = await fetch(`${base}/webhooks/github`, {+ method: 'POST',+ headers: { 'content-type': 'application/json', 'x-github-event': 'issues',+ 'x-hub-signature-256': `sha256=${createHmac('sha256', SECRET).update(payload).digest('hex')}` },+ body: payload,+ });+ return { status: res.status, body: await res.json() as Record<string, unknown> };+}++const waitingItems = async (sourceId: string) => (await engine.executeRaw<{ item: { number: number; deleted?: boolean } }>(+ "SELECT data->'github_item' AS item FROM minion_jobs WHERE name = 'sync' AND status = 'waiting' AND data->>'sourceId' = $1 ORDER BY id", [sourceId]))+ .map(row => (typeof row.item === 'string' ? JSON.parse(row.item) : row.item) as { number: number; deleted?: boolean });++describe('#8799 GitHub item webhooks', () => {+ test('two items of one source each get their own refresh job', async () => {+ const first = await issueEvent(1, 'edited');+ const second = await issueEvent(2, 'edited');+ expect(first.status).toBe(202);+ expect(second.status).toBe(202);+ expect(second.body.job_id).not.toBe(first.body.job_id);+ expect((await waitingItems('source-a')).map(item => item.number)).toEqual([1, 2]);+ });++ test('a repeated event for the same item inside the bucket reuses its job', async () => {+ const again = await issueEvent(1, 'labeled');+ const [row] = await engine.executeRaw<{ id: number }>(+ "SELECT id FROM minion_jobs WHERE name = 'sync' AND status = 'waiting' AND (data->'github_item'->>'number')::int = 1 ORDER BY id LIMIT 1");+ expect(again.status).toBe(202);+ expect(again.body.job_id).toBe(row!.id);+ });++ test('a deletion right after an edit queues a delete, not the edit refresh', async () => {+ const deleted = await issueEvent(1, 'deleted');+ expect(deleted.status).toBe(202);+ const [job] = await engine.executeRaw<{ item: unknown }>("SELECT data->'github_item' AS item FROM minion_jobs WHERE id = $1", [deleted.body.job_id]);+ const item = (typeof job!.item === 'string' ? JSON.parse(job!.item) : job!.item) as { number: number; deleted?: boolean };+ expect(item).toMatchObject({ number: 1, deleted: true });+ });++ test('a full backlog refuses the event with 429 instead of answering with another item', async () => {+ for (let number = 1; number <= 50; number++) expect((await issueEvent(number, 'edited', 'acme-example/other')).status).toBe(202);+ const refused = await issueEvent(51, 'edited', 'acme-example/other');+ expect(refused.status).toBe(429);+ expect(refused.body).toMatchObject({ error: 'webhook_backlog_full', source_id: 'source-b' });+ expect((await waitingItems('source-b')).some(item => item.number === 51)).toBe(false);+ });++ test('at a full backlog a waiting source-wide sync covers an edit but not a deletion', async () => {+ for (let number = 1; number <= 49; number++) expect((await issueEvent(number, 'edited', 'acme-example/third')).status).toBe(202);+ const sweep = await new MinionQueue(engine).add('sync', { sourceId: 'source-c', noExtract: false }, { priority: -10 });+ const edit = await issueEvent(60, 'edited', 'acme-example/third');+ expect(edit.status).toBe(202);+ expect(edit.body.job_id).toBe(sweep.id);+ expect((await issueEvent(61, 'deleted', 'acme-example/third')).status).toBe(429);+ });++ test('at a full backlog a waiting automatic freshness sync, which may skip, covers nothing', async () => {+ for (let number = 1; number <= 49; number++) expect((await issueEvent(number, 'edited', 'acme-example/fourth')).status).toBe(202);+ await new MinionQueue(engine).add('sync', { sourceId: 'source-d', embed_reason: 'autopilot_freshness' }, { priority: -10 });+ expect((await issueEvent(60, 'edited', 'acme-example/fourth')).status).toBe(429);+ });+});diff --git a/test/sync-handler-github-item-busy-8799.test.ts b/test/sync-handler-github-item-busy-8799.test.tsnew file mode 100644index 0000000000..1526b1790d--- /dev/null+++ b/test/sync-handler-github-item-busy-8799.test.ts@@ -0,0 +1,57 @@+/**+ * #8799: a GitHub webhook's single-item refresh that finds its source busy is deferred with its item and delete intent.+ * Before the fix the sync handler completed it as a sync_in_progress skip, which records only a source-wide pull marker,+ * so the item's refresh, or its deletion, was lost while another sync of the source ran. A plain sync job still skips.+ * Real PGLite, the real sync handler, the per-source sync lock held by a stand-in holder; no network.+ */+import { afterAll, beforeAll, expect, test } from 'bun:test';+import { mkdtempSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { PGLiteEngine } from '../src/core/pglite-engine.ts';+import { syncLockId, tryAcquireDbLock } from '../src/core/db-lock.ts';+import { makeSyncHandler } from '../src/core/minions/handlers/sync.ts';+import { JobDeferredError } from '../src/core/minions/errors.ts';+import type { MinionJobContext } from '../src/core/minions/types.ts';+import { withEnv } from './helpers/with-env.ts';++const SOURCE = 'source-a';+const home = mkdtempSync(join(tmpdir(), 'gbrain-8799-busy-'));+let engine: PGLiteEngine;++beforeAll(async () => {+ engine = new PGLiteEngine();+ await engine.connect({});+ await engine.initSchema();+ await engine.executeRaw('INSERT INTO sources (id, name, local_path, config) VALUES ($1, $1, $2, $3::jsonb)', [SOURCE, join(home, SOURCE),+ JSON.stringify({ kind: 'github', gh_scope: 'repos', gh_repos: 'acme-example/app', gh_token_env: 'CONNECTOR_TEST_TOKEN' })]);+}, 60_000);++afterAll(async () => {+ await engine?.disconnect();+ rmSync(home, { recursive: true, force: true });+}, 60_000);++const job = (data: Record<string, unknown>) => ({ id: 9201, name: 'sync', data: { sourceId: SOURCE, ...data }, attempts_made: 0,+ signal: new AbortController().signal, shutdownSignal: new AbortController().signal, deadlineAtMs: Date.now() + 60_000,+ updateProgress: async () => undefined, updateTokens: async () => undefined, log: async () => undefined, isActive: async () => true,+ readInbox: async () => [] }) as unknown as MinionJobContext;++const deferral = async (run: Promise<unknown>) => {+ const err = await run.then(() => null, (e: unknown) => e);+ expect(err).toBeInstanceOf(JobDeferredError);+ return { reason: (err as JobDeferredError).reason, retryInMs: (err as JobDeferredError).retryInMs, message: (err as JobDeferredError).message };+};++test('an item refresh or deletion on a busy source is deferred, burning no attempt; a plain sync job still skips', async () => withEnv({ GBRAIN_HOME: home, CONNECTOR_TEST_TOKEN: 'synthetic-local-fixture' }, async () => {+ const held = await tryAcquireDbLock(engine, syncLockId(SOURCE), 5);+ expect(held).not.toBeNull();+ try {+ const handler = makeSyncHandler(engine);+ expect(await deferral(handler(job({ github_item: { repo: 'acme-example/app', number: 1, kind: 'issue' } }))))+ .toMatchObject({ reason: 'sync_in_progress', retryInMs: 30_000, message: expect.stringMatching(/refresh of acme-example\/app#1/) });+ expect(await deferral(handler(job({ github_item: { repo: 'acme-example/app', number: 2, kind: 'issue', deleted: true } }))))+ .toMatchObject({ reason: 'sync_in_progress', retryInMs: 30_000, message: expect.stringMatching(/deletion of acme-example\/app#2/) });+ expect(await handler(job({ pull: false }))).toMatchObject({ skipped: true, reason: 'sync_in_progress', source_id: SOURCE });+ } finally { await held!.release(); }+}), 120_000);