Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Network ingestion for `gbrain serve --http`: the OAuth-authenticated
* POST /ingest webhook source (v0.38) and the HMAC-verified
* POST /webhooks/github push / item-event sync trigger (v0.40, v0.46).
*/
import express, { type Express, type Request, type Response } from 'express';
import rateLimit from 'express-rate-limit';
import { createHash, createHmac, randomBytes } from 'crypto';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { requireBearerAuth } from '@modelcontextprotocol/sdk/server/auth/middleware/bearerAuth.js';
import { safeHexEqual } from '../core/timing-safe.ts';
import { isValidRepoName } from '../core/github-source.ts';
import type { BrainEngine } from '../core/engine.ts';
import type { AuthInfo } from '../core/operations.ts';
import { executeRawJsonb } from '../core/sql-query.ts';
import { MinionQueue } from '../core/minions/queue.ts';
import { coalesceOnIdempotencyKey } from '../core/minions/idempotency-coalesce.ts';
import { APPLICATION_AUTHORITY } from '../core/minions/submission-authority.ts';
import type { MinionJob } from '../core/minions/types.ts';
import { OperationError } from '../core/ops/contract.ts';
import { ERROR_CATALOGUE } from '../core/error-catalogue.ts';
import {
computeContentHash,
validateIngestionEvent,
type IngestionContentType,
type IngestionEvent,
} from '../core/ingestion/types.ts';
import type { ServeHttpContext } from './serve-http.ts';
import { NO_SOURCES } from '../core/source-id.ts';
import { noSourceGrantError, sharedWriteSourceMemberError } from '../core/ops/context.ts';
import { renderAction } from '../core/agent-output.ts';
import { assertWebhookWriterMayReplace } from '../core/minions/handlers/ingest-capture.ts';
import { defaultSourceMemberWriter } from '../core/persistence/page-visibility.ts';
import type { WriteAuthority } from '../core/persistence/model.ts';
import { validateSlug } from '../core/utils.ts';
import { RevisionBackfillPendingError } from '../core/page-state/types.ts';
/**
* v0.46: normalize the per-event GitHub webhook payload shape into
* {repo, number, kind}. Events differ: issues/issue_comment/label/
* assignee/milestone carry a top-level `issue` (PRs appear there too,
* flagged by `issue.pull_request`), pull_request/review events carry
* top-level `pull_request`, and check events nest the linked PRs under
* check_run/check_suite/workflow_run. Returns null when the payload
* carries no item reference (ping, branch, non-PR checks).
*/
export function extractGitHubItemRef(parsed: Record<string, unknown>): { repo: string; number: number; kind: 'issue' | 'pr' } | null {
const repoObj = parsed.repository as { full_name?: string } | undefined;
const repo = repoObj?.full_name ?? '';
const issueObj = parsed.issue as { number?: number; pull_request?: unknown } | undefined;
const prObj = parsed.pull_request as { number?: number } | undefined;
const checkRun = parsed.check_run as { pull_requests?: Array<{ number?: number }> } | undefined;
const checkSuite = parsed.check_suite as { pull_requests?: Array<{ number?: number }> } | undefined;
const workflowRun = parsed.workflow_run as { pull_requests?: Array<{ number?: number }> } | undefined;
const nestedPrNumber =
checkRun?.pull_requests?.[0]?.number ??
checkSuite?.pull_requests?.[0]?.number ??
workflowRun?.pull_requests?.[0]?.number;
const number = prObj?.number ?? issueObj?.number ?? nestedPrNumber;
if (typeof number !== 'number' || !isValidRepoName(repo)) return null;
const kind = prObj !== undefined || issueObj?.pull_request !== undefined || nestedPrNumber !== undefined ? 'pr' : 'issue';
return { repo, number, kind };
}
/**
* True when a github-kind source covers `fullName`: explicit gh_repos list
* for scope=repos, the last-discovered state file for scope=auto (accept
* when no state exists yet — the sync engine re-checks scope). Repo names
* are case-insensitive on GitHub, so matching folds case on both sides
* (config and legacy state files may carry canonical-case entries).
*/
export function githubKindCoversRepo(
cfg: Record<string, unknown>,
localPath: string | null,
fullName: string,
): boolean {
const repo = fullName.toLowerCase();
if (cfg.gh_scope === 'repos') {
const repos = typeof cfg.gh_repos === 'string' ? cfg.gh_repos.split(',').map((s) => s.trim().toLowerCase()) : [];
return repos.includes(repo);
}
if (localPath) {
try {
const state = JSON.parse(readFileSync(join(localPath, '.github-source.json'), 'utf-8')) as {
repos?: unknown[];
};
if (Array.isArray(state.repos)) {
return state.repos.some((r) => typeof r === 'string' && r.toLowerCase() === repo);
}
} catch {
/* no state yet */
}
}
return true;
}
/**
* Partition signature-verified webhook sources for an item event. Only a
* github-kind source can service a github_item refresh — the sync core
* rejects github_item on any other kind, so enqueueing for a legacy
* github_repo push source would only mint a dead job. Legacy matches are
* reported so the handler can ACK-and-ignore them instead.
*/
export function selectGitHubItemSources<Row extends { local_path: string | null; config: unknown }>(
rows: Row[],
repo: string,
verify: (cfg: Record<string, unknown>) => boolean,
): { verified: Row[]; legacyMatched: boolean } {
const verified: Row[] = [];
let legacyMatched = false;
const candidates = rows.map((row) => ({
row,
cfg: (typeof row.config === 'string' ? JSON.parse(row.config) : (row.config ?? {})) as Record<string, unknown>,
}));
const isLegacy = (cfg: Record<string, unknown>) => !(cfg.kind === 'github' && cfg.webhook_secret != null);
// Every github-kind row is checked before scope decides anything, and the legacy rows share exactly one check (a
// throwaway one when the repo has none), so the work done for an unsigned request does not depend on whether or how
// many sources name the repository (#8768). The check goes to the lowest-id legacy row that has a secret, so a second
// legacy source with a different secret is not tried.
const legacyPick =
candidates.find((c) => isLegacy(c.cfg) && typeof c.cfg.webhook_secret === 'string' && c.cfg.webhook_secret !== '') ??
candidates.find((c) => isLegacy(c.cfg));
for (const candidate of candidates) {
const { row, cfg } = candidate;
if (isLegacy(cfg) && candidate !== legacyPick) continue;
const signed = verify(cfg);
if (cfg.kind === 'github' && githubKindCoversRepo(cfg, row.local_path, repo) && signed) {
verified.push(row);
continue;
}
if (cfg.github_repo === repo && signed) legacyMatched = true;
}
if (!legacyPick) verify({});
return { verified, legacyMatched };
}
/** Per-server limiters for the two ingestion routes (carried on ServeHttpContext). */
export function createWebhookLimiters() {
const ingestRateLimiter = rateLimit({
windowMs: 10_000, // 10 seconds
limit: 100, // 100 events per IP per window
standardHeaders: 'draft-7',
legacyHeaders: false,
message: { error: 'rate_limit_exceeded', message: 'too many /ingest events; backoff and retry' },
});
const githubWebhookLimiter = rateLimit({
windowMs: 60_000,
limit: 60,
standardHeaders: 'draft-7',
legacyHeaders: false,
message: { error: 'rate_limit_exceeded', message: 'too many GitHub webhook requests' },
});
return { ingestRateLimiter, githubWebhookLimiter };
}
/** #8799: item refreshes one github-kind source may have waiting before an item event is refused with 429. */
const GITHUB_ITEM_MAX_WAITING = 50;
/**
* v0.46: issue/PR event handling for github-kind sources. The payload
* names a single item (repo + number); we verify the per-source HMAC and
* submit a targeted `sync` job with github_item so exactly that item is
* refreshed. Out-of-scope repos are rejected at queue time by the sync
* engine's own scope check.
*/
async function handleGitHubItemEvent(
engine: BrainEngine,
parsed: Record<string, unknown>,
sigHeader: string,
payload: Buffer,
res: Response,
eventName: string,
deliveryId: string | undefined,
): Promise<void> {
const ref = extractGitHubItemRef(parsed);
if (ref === null) {
// Not an item-bearing payload (e.g. check events without a linked PR,
// ping, branch protection). Acknowledge so GitHub does not retry.
res.status(202).json({ status: 'ignored', reason: 'no_item_ref' });
return;
}
// Collect ALL candidate sources: exact github_repo matches (legacy
// webhook config) and github-kind sources with a webhook secret, then
// verify HMAC per candidate. Only github-kind sources may enqueue a
// github_item refresh; two verifying github-kind sources mean ambiguous
// configuration and must not pick silently.
let source: { id: string; local_path: string | null; config: unknown } | null = null;
try {
const rows = await engine.executeRaw<{ id: string; local_path: string | null; config: unknown }>(
`SELECT id, local_path, config FROM sources
WHERE archived = false
AND ((config->>'github_repo' = $1)
OR (config->>'kind' = 'github' AND config->>'webhook_secret' IS NOT NULL))
ORDER BY id`,
[ref.repo],
);
const { verified, legacyMatched } = selectGitHubItemSources(rows, ref.repo, (cfg) =>
verifyWebhookSig(cfg.webhook_secret, sigHeader, payload),
);
if (verified.length > 1) {
res.status(500).json({
error: 'ambiguous_webhook',
message: `multiple sources verified the signature for ${ref.repo}; configure one webhook secret per source`,
sources: verified.map((v) => v.id),
});
return;
}
if (verified.length === 0 && legacyMatched) {
// A legacy push-webhook source verified the signature but cannot
// service item events — ACK so GitHub doesn't retry, exactly like
// the pre-item-flow non-push behavior.
res.status(202).json({ status: 'ignored', reason: `event=${eventName}` });
return;
}
source = verified[0] ?? null;
} catch (err) {
console.error('webhook: github-kind source lookup error:', err);
res.status(500).json({ error: 'lookup_failed' });
return;
}
if (!source) {
res.status(404).json({ error: 'unknown_repo', repo: ref.repo });
return;
}
// #8799: the key names the item and its intent, so a deletion never reuses an edit's refresh. The waiting cap is per
// source, so a cap hit can return another item's job: unless that job covers this event (the same item and intent, or an
// explicit source-wide sync for an edit), it is a refusal, never this event's 202. A busy source defers the job (sync handler).
// An event joins a refresh of its item that has not started; otherwise its delivery id keys a new one, so an event
// after the refresh began is never dropped and a GitHub redelivery (same delivery id) maps to its first job.
const deleted = eventName === 'issues' && parsed.action === 'deleted';
const idempotencyKey = `webhook:item:${source.id}:${ref.repo}:${ref.number}:${deleted ? 'delete' : 'refresh'}:${deliveryId || Math.floor(Date.now() / 30_000)}`;
try {
const [waiting] = await engine.executeRaw<{ id: number }>(
`SELECT id FROM minion_jobs WHERE name = 'sync' AND status = 'waiting' AND data->>'sourceId' = $1 AND data->'github_item'->>'repo' = $2
AND data->'github_item'->>'number' = $3 AND COALESCE(data->'github_item'->>'deleted', 'false') = $4 ORDER BY id LIMIT 1`,
[source.id, ref.repo, String(ref.number), String(deleted)]);
if (waiting) {
res.status(202).json({ job_id: Number(waiting.id), source_id: source.id, item: ref });
return;
}
const queue = new MinionQueue(engine);
const job = await queue.add(
'sync',
{
sourceId: source.id,
noExtract: false,
github_item: {
repo: ref.repo,
number: ref.number,
kind: ref.kind,
...(deleted ? { deleted: true } : {}),
},
embed_reason: 'webhook',
},
{
priority: -10,
idempotency_key: idempotencyKey,
maxWaiting: GITHUB_ITEM_MAX_WAITING,
},
);
const coalesced = job.data as { github_item?: { repo?: unknown; number?: unknown; deleted?: unknown }; embed_reason?: unknown } | null;
const coalescedItem = coalesced?.github_item;
const covered = job.idempotency_key === idempotencyKey || (coalescedItem
? coalescedItem.repo === ref.repo && coalescedItem.number === ref.number && (coalescedItem.deleted === true) === deleted
: !deleted && coalesced?.embed_reason !== 'autopilot_freshness');
if (!covered) {
res.setHeader('Retry-After', '30');
res.status(429).json({ error: 'webhook_backlog_full', source_id: source.id,
message: `${GITHUB_ITEM_MAX_WAITING} item refreshes are already waiting for this source; redeliver this event once they drain, or run gbrain sync --source ${source.id}.` });
return;
}
res.status(202).json({ job_id: job.id, source_id: source.id, item: ref });
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
console.error('webhook: item queue submission error:', msg);
res.status(500).json({ error: 'queue_submission_failed', message: msg });
}
}
/** Slugs near 2.7 KB exceed the page key index's row size, so a capture with one would be accepted and never land; 1024 bytes leaves a wide margin. */
const MAX_INGEST_SLUG_BYTES = 1024;
/** The normalized target slug for an X-Gbrain-Slug header; throws on an invalid slug or one over MAX_INGEST_SLUG_BYTES. */
function ingestTargetSlug(callerSlug: string): string {
const bytes = Buffer.byteLength(callerSlug, 'utf8');
if (bytes > MAX_INGEST_SLUG_BYTES) throw new Error(`Invalid slug: X-Gbrain-Slug is ${bytes} bytes; the limit is ${MAX_INGEST_SLUG_BYTES}.`);
return validateSlug(callerSlug);
}
/**
* The job a server before v0.60.609.0 recorded for this capture, or null. Its key had no slug part
* (`ingest:webhook:<client>:<write source>:<content hash>`), and a job row keeps its key until `gbrain jobs prune`
* removes it, which nothing runs on a schedule, so such a row can answer a retry for as long as the brain keeps it. A
* retry that missed it queued a second job, and that job wrote the old body over any edit made since. The row answers
* only for the target slug it was queued with: the old key collapsed one body sent to two slugs, which is the dropped
* write #8769 fixed. A dead or cancelled row frees its key and answers nothing, as under the current key.
*/
async function legacyIngestJob(engine: BrainEngine, clientId: string, writeSourceId: string, contentHash: string, targetSlug: string): Promise<MinionJob | null> {
const key = `ingest:webhook:${clientId}:${writeSourceId}:${contentHash}`;
return engine.transaction(async (tx) => {
const [row] = await tx.executeRaw<{ data: unknown }>('SELECT data FROM minion_jobs WHERE idempotency_key = $1', [key]);
if (!row) return null;
const data = (typeof row.data === 'string' ? JSON.parse(row.data) : row.data) as { slug?: unknown } | null;
let rowSlug = '';
try {
if (typeof data?.slug === 'string' && data.slug) rowSlug = ingestTargetSlug(data.slug);
} catch {
return null;
}
return rowSlug === targetSlug ? coalesceOnIdempotencyKey(tx, key, APPLICATION_AUTHORITY) : null;
});
}
/** Stands in for a missing webhook secret; no source holds it and a match against it is still refused. */
const THROWAWAY_WEBHOOK_KEY = randomBytes(32);
/**
* True when `sigHeader` is GitHub's HMAC-SHA256 of `payload` under `secret`. The HMAC over the body is computed on
* every call, with a throwaway key when there is no source or no secret, so an unsigned caller cannot time responses
* to learn which repositories this brain syncs with a live secret (#8768). GitHub sends "sha256=<hex>": the prefix is
* stripped before safeHexEqual because Buffer.from('sha256=...', 'hex') truncates at the 's' and would make every
* signature "match", and the strict hex shape is checked first because a malformed 64-char signature makes
* safeHexEqual throw (500 instead of 401).
*/
function verifyWebhookSig(secret: unknown, sigHeader: string, payload: Buffer): boolean {
const keyed = typeof secret === 'string' && secret !== '';
const computedHex = createHmac('sha256', keyed ? secret : THROWAWAY_WEBHOOK_KEY).update(payload).digest('hex');
const matches = /^sha256=[0-9a-f]{64}$/.test(sigHeader) && safeHexEqual(sigHeader.slice('sha256='.length), computedHex);
return matches && keyed;
}
/**
* GST-2051: a member-bound credential writing to default or a federated source would publish where every no-grant credential
* reads; the capture job cannot keep the page to the member, so POST /ingest refuses it (answering res) and put_page, which
* does, stays open.
*/
async function refusedDefaultSourceMember(engine: BrainEngine, authInfo: AuthInfo, res: Response): Promise<boolean> {
const sourceId = authInfo.sourceId ?? 'default';
if (!authInfo.principal || !await defaultSourceMemberWriter(engine, { remote: true, principal: authInfo.principal as WriteAuthority['principal'], sourceId })) return false;
const refusal = sharedWriteSourceMemberError('POST /ingest', authInfo, sourceId);
res.status(403).json({ error: refusal.code, code: refusal.code, message: refusal.message, suggestion: refusal.suggestion, docs: refusal.docs,
fix: renderAction(refusal.fix!, { transport: 'http', isCallable: () => false, preapproved: () => false }) });
return true;
}
/** POST /ingest (OAuth write scope) and POST /webhooks/github (per-source HMAC), in registration order. */
export function mountWebhooks(app: Express, ctx: ServeHttpContext): void {
mountIngest(app, ctx);
mountGitHubWebhook(app, ctx);
}
function mountIngest(app: Express, ctx: ServeHttpContext): void {
const { engine, resourceVerifier, resourceMetadataUrl, ingestRateLimiter } = ctx;
// ---------------------------------------------------------------------------
// v0.38 ingestion substrate — POST /ingest (webhook source)
//
// The webhook ingestion source lives INSIDE serve --http (NOT in the
// ingestion daemon) per the /plan-eng-review E1 decision. This avoids
// cross-process IPC: the daemon supervises only daemon-side sources
// (file-watcher, inbox-folder, cron-scheduler) while serve --http hosts
// the network surface and submits Minion jobs directly.
//
// Auth: existing OAuth `write` scope. Rate limit: 100 events / 10s per
// IP (reuses the IP-keyed pattern from ccRateLimiter; a future tweak
// could key on authInfo.clientId for fairer per-agent fairness).
// Payload cap: 1 MB default. Content-type allowlist: markdown, plain,
// HTML, JSON. Binary content is REJECTED with HTTP 415 in v1 — the
// binary-upload flow ships as a separate route in a later wave when
// content-type processors land.
//
// Events always carry untrusted_payload: true because the input came
// over the network from an OAuth-authenticated but otherwise untrusted
// source (Zapier / IFTTT / Apple Shortcuts). The downstream
// ingest_capture handler logs the flag; a future v2 wave wires it
// through the put_page op to skip auto-link.
// ---------------------------------------------------------------------------
// Maximum payload bytes for POST /ingest. Configurable via env. Default 1 MB.
const ingestMaxBytes = (() => {
const fromEnv = process.env.GBRAIN_INGEST_MAX_BYTES;
if (!fromEnv) return 1_048_576;
const n = parseInt(fromEnv, 10);
return Number.isFinite(n) && n > 0 ? n : 1_048_576;
})();
// Content-type allowlist: text-shaped types only in v1. The handler
// routes binary content_types with HTTP 415; a future wave + skillpack
// processors will accept image/audio/video/pdf via a separate flow.
const INGEST_ALLOWED_CONTENT_TYPES: ReadonlySet<IngestionContentType> = new Set([
'text/markdown',
'text/plain',
'text/html',
'application/json',
]);
// Single MinionQueue instance shared across POST /ingest invocations
// (the queue is stateless beyond the engine handle; reusing avoids
// per-request construction).
const ingestQueue = new MinionQueue(engine);
app.post(
'/ingest',
ingestRateLimiter,
requireBearerAuth({ verifier: resourceVerifier, requiredScopes: ['write'], resourceMetadataUrl }),
express.raw({ type: '*/*', limit: ingestMaxBytes }),
async (req: Request, res: Response) => handleIngest(ctx, ingestQueue, INGEST_ALLOWED_CONTENT_TYPES, req, res),
);
}
async function handleIngest(
ctx: ServeHttpContext,
ingestQueue: MinionQueue,
INGEST_ALLOWED_CONTENT_TYPES: ReadonlySet<IngestionContentType>,
req: Request,
res: Response,
): Promise<void> {
const { engine, broadcastEvent } = ctx;
const startTime = Date.now();
const authInfo = (req as Request & { auth?: AuthInfo }).auth as AuthInfo;
const agentName = authInfo.clientName ?? authInfo.clientId;
// v0.39.3.0 BUG-2: outer try/catch ensures any unexpected throw
// returns a JSON envelope instead of leaking express's default HTML
// error page. Mirrors the MCP handler's F14 pattern (serve-http.ts
// F14 envelope around transport.handleRequest). The `!res.headersSent`
// guard (codex F#16) prevents a second-response attempt if the throw
// happens after the inner queue.add try/catch already responded.
try {
// The webhook queue bypasses MCP dispatch and does not carry an original
// operation grant through execution. A snapshot-bound client must use the
// shared MCP write path until ingestion has that same policy contract.
if (authInfo.sourceId === NO_SOURCES) {
const refusal = noSourceGrantError('POST /ingest', authInfo);
res.status(403).json({ error: refusal.code, message: refusal.message, detail: refusal.detail, hint: refusal.suggestion, docs_url: refusal.docs,
fix: renderAction(refusal.fix!, { transport: 'http', isCallable: () => false, preapproved: () => false }) });
return;
}
if (await refusedDefaultSourceMember(engine, authInfo, res)) return;
if (authInfo.allowedOperations != null || authInfo.grantProjectionDegraded) {
res.status(403).json({
error: 'permission_denied',
message: 'POST /ingest is unavailable to clients with operation snapshots. ' +
'Use an approved MCP capture, put_page, or remember operation so the current grant is enforced.',
});
return;
}
// v0.39.3.0 BUG-2: explicit null/undefined guard BEFORE body coercion.
// When the request has no body at all (no Content-Length header, no
// body-parser fed us anything), `req.body` is `undefined`. The pre-fix
// code's `else` branch called `Buffer.from(JSON.stringify(undefined),
// 'utf8')` — and `JSON.stringify(undefined) === undefined` (the
// literal, not the string), which makes `Buffer.from(undefined, 'utf8')`
// throw TypeError. Express's default error handler then served an HTML
// 500 page. Guard fires first to keep the response shape JSON.
if (req.body == null) {
res.status(400).json({
error: 'empty_body',
message: 'POST /ingest requires a non-empty body',
});
return;
}
// Express raw() returns a Buffer. Decode as UTF-8; reject non-UTF-8
// bytes loudly so callers know their payload was garbled.
let body: Buffer;
if (Buffer.isBuffer(req.body)) {
body = req.body;
} else if (typeof req.body === 'string') {
body = Buffer.from(req.body, 'utf8');
} else {
// express.json or urlencoded fired earlier in the chain and parsed
// for us. Re-serialize so we can hash and forward. The null/undefined
// case is already guarded above so JSON.stringify produces a real
// string here (objects round-trip, primitives become their JSON form).
body = Buffer.from(JSON.stringify(req.body), 'utf8');
}
if (body.length === 0) {
res.status(400).json({ error: 'empty_body', message: 'POST /ingest requires a non-empty body' });
return;
}
// Detect content_type. Caller can override via the X-Gbrain-Content-Type
// header for the JSON case (since the request's Content-Type would say
// application/json but the user might intend the body to be markdown).
const declared = (req.header('x-gbrain-content-type') || req.header('content-type') || '').toLowerCase();
let contentType: IngestionContentType;
if (declared.startsWith('text/markdown')) {
contentType = 'text/markdown';
} else if (declared.startsWith('text/html')) {
contentType = 'text/html';
} else if (declared.startsWith('text/plain')) {
contentType = 'text/plain';
} else if (declared.startsWith('application/json')) {
contentType = 'application/json';
} else if (declared.startsWith('text/')) {
// Unknown text/* sub-types pass through as text/plain.
contentType = 'text/plain';
} else {
// Binary or unknown — rejected in v1.
res.status(415).json({
error: 'unsupported_content_type',
message: `content_type '${declared}' not supported. Use one of: ${[...INGEST_ALLOWED_CONTENT_TYPES].join(', ')}. ` +
'Binary content (image/audio/video/pdf) is not yet supported via POST /ingest — install a content-type processor skillpack.',
});
return;
}
if (!INGEST_ALLOWED_CONTENT_TYPES.has(contentType)) {
res.status(415).json({
error: 'unsupported_content_type',
message: `content_type '${contentType}' is in the taxonomy but not currently accepted by POST /ingest`,
});
return;
}
const content = body.toString('utf8');
const contentHash = computeContentHash(content);
const sourceUri = (req.header('x-gbrain-source-uri') || `mcp-webhook:${authInfo.clientId}:${Date.now()}`).slice(0, 1024);
const sourceId = `webhook-${authInfo.clientId}`.slice(0, 256);
const callerSlug = req.header('x-gbrain-slug');
// Slug-bound clients cannot use /ingest at all. The route hands its
// payload to the ingest_capture minion handler, which deliberately
// bypasses the put_page op layer — so no OperationContext exists and
// enforceClientSlugFence never runs. The caller-supplied
// X-Gbrain-Source-Id is still never honored, but the write source is now
// resolved server-side from the client's own OAuth scope, so the write
// does land inside the client's granted source. The fence gap is
// therefore the SLUG axis alone: without this 403 a bound client could
// write any slug within its source, which is exactly the binding it was
// given. These clients have put_page over MCP, which enforces both the
// prefix fence and the source scope; webhook integrations use unbound
// clients.
const boundPrefixes = authInfo.boundSlugPrefixes;
if (boundPrefixes || authInfo.fenceProjectionDegraded) {
res.status(403).json({
error: 'permission_denied',
message: authInfo.fenceProjectionDegraded
? 'POST /ingest is unavailable: this brain\'s oauth_clients projection is missing ' +
'bound_slug_prefixes, so client write bindings cannot be evaluated. ' +
'Run `gbrain apply-migrations --yes` on the brain host.'
: 'POST /ingest is not available to clients restricted to slug prefixes ' +
`(bound_slug_prefixes: ${boundPrefixes!.join(', ')}). Write through the MCP put_page op, ` +
'which enforces the prefix fence and your source scope.',
});
return;
}
const event: IngestionEvent = {
source_id: sourceId,
source_kind: 'webhook',
source_uri: sourceUri,
received_at: new Date().toISOString(),
content_type: contentType,
content,
content_hash: contentHash,
untrusted_payload: true, // ALWAYS true for network input
metadata: {
ip: req.ip,
user_agent: req.header('user-agent') ?? '',
client_id: authInfo.clientId,
...(callerSlug ? { slug: callerSlug } : {}),
},
};
const validationErr = validateIngestionEvent(event);
if (validationErr) {
res.status(400).json({
error: 'invalid_event',
message: validationErr.message,
field: validationErr.field,
});
return;
}
// The job re-checks this writer at write time; refusing here answers the caller before anything is queued.
const writeSourceId = authInfo.sourceId ?? 'default';
const writer = authInfo.principal ? { principal: { kind: authInfo.principal.kind, id: authInfo.principal.id } } : undefined;
let targetSlug = '';
if (callerSlug) {
try {
targetSlug = ingestTargetSlug(callerSlug);
} catch (err) {
res.status(400).json({ error: 'invalid_slug', message: err instanceof Error ? err.message : String(err) });
return;
}
try {
await assertWebhookWriterMayReplace(engine, writer, writeSourceId, callerSlug, content);
} catch (err) {
if (!(err instanceof OperationError) && !(err instanceof RevisionBackfillPendingError)) throw err;
res.status(err.code === 'page_not_found' ? 404 : err.code === 'revision_backfill_pending' ? 409 : 403).json({ error: err.code, message: err.message });
return;
}
}
// The same body sent to two slugs is two writes, so the normalized target slug (empty when the job picks one) is part
// of the key next to the write source; dropping it handed the second caller the first job, a 202 with nothing
// written to its slug (#8769). The slug goes in hashed so the key stays well inside the key index's row size.
const slugKey = targetSlug ? createHash('sha256').update(targetSlug).digest('hex') : '';
const idempotencyKey = `ingest:webhook:${authInfo.clientId}:${writeSourceId}:${slugKey}:${contentHash}`;
try {
const legacyJob = await legacyIngestJob(engine, authInfo.clientId, writeSourceId, contentHash, targetSlug);
const job = legacyJob ?? await ingestQueue.add(
'ingest_capture',
{
event,
...(callerSlug ? { slug: callerSlug } : {}),
sourceId: writeSourceId,
...(writer ? { writer } : {}),
},
{
// Idempotency: the same content from the same client to the same
// slug is one capture. While its job row exists the queue hands
// back that job; the ingest_capture handler applies a capture once
// per key for 90 days, even after `gbrain jobs prune` removed its
// job row (GST-1856), so a replay never rewrites the page. Different
// content gets different jobs. Daemon-side dedup catches the 24h window.
// The effective write source is part of the key: a client rescoped
// from source X to Y must land a NEW capture in Y rather than being
// deduped against its old X-bound job.
idempotency_key: idempotencyKey,
// Cap waiting captures per write source so a runaway integration
// can't fill the queue.
maxWaiting: 50,
},
);
// At the cap the queue hands back the newest waiting job instead of inserting this capture. Answering 202 with
// that job id would drop this body without a trace, so the caller is told to retry once the backlog drains.
if (!legacyJob && job.idempotency_key !== idempotencyKey) {
res.setHeader('Retry-After', '30');
res.status(429).json({
error: 'ingest_backlog_full',
message: 'POST /ingest has 50 captures waiting for this source; nothing was queued. Retry after the backlog drains.',
});
return;
}
const latency = Date.now() - startTime;
try {
await executeRawJsonb(
engine,
`INSERT INTO mcp_request_log (token_name, agent_name, operation, latency_ms, status, params)
VALUES ($1, $2, $3, $4, $5, $6::jsonb)`,
[authInfo.clientId, agentName, 'webhook_ingest', latency, 'success'],
// write_source_id is the security-relevant part of this request:
// which partition the capture was routed to. Without it the audit
// trail cannot answer "where did this client's writes land".
[{ content_type: contentType, content_hash: contentHash, bytes: body.length, job_id: job.id, write_source_id: writeSourceId }],
);
} catch { /* best effort */ }
broadcastEvent({
agent: agentName,
operation: 'webhook_ingest',
scopes: authInfo.scopes.join(','),
latency_ms: latency,
status: 'success',
timestamp: new Date().toISOString(),
});
res.status(202).json({
job_id: job.id,
content_hash: contentHash,
// Emitter identity (`webhook-<clientId>`), kept for back-compat.
source_id: sourceId,
// The brain source this capture is routed to, resolved server-side
// from the client's OAuth scope. This is the routing decision the
// caller actually cares about; `source_id` above is NOT a partition.
// Enqueue-time intent: the write runs asynchronously after this 202,
// so a later source_fallback (see the ingest_capture job result) can
// still redirect it.
write_source_id: writeSourceId,
message: 'Accepted. Event queued for ingestion.',
});
} catch (err) {
// A job row left with SQL NULL authority by an upgrade across v0.50 holds
// this capture's key (or fills its waiting cap): a conflict the operator
// resolves with the hinted recovery, not a server fault.
if (err instanceof OperationError && err.docs === ERROR_CATALOGUE.legacy_job_authority.docs) {
res.status(409).json({ error: err.code, message: err.message, hint: err.suggestion, docs_url: err.docs });
return;
}
const msg = err instanceof Error ? err.message : String(err);
console.error('POST /ingest queue submission error:', msg);
res.status(500).json({
error: 'queue_submission_failed',
message: msg,
});
}
// v0.39.3.0 BUG-2: outer try/catch close — anything that throws BEFORE
// the inner queue.add try/catch lands here. The headersSent guard
// (codex F#16) skips the second-response attempt if the inner block
// already wrote a response and then threw on a downstream line (e.g.
// a logging side-effect after `res.status(202).json(...)`).
} catch (outerErr) {
const msg = outerErr instanceof Error ? outerErr.message : String(outerErr);
console.error('POST /ingest unexpected handler error:', msg);
if (!res.headersSent) {
res.status(500).json({
error: 'internal_error',
message: msg,
});
}
}
}
function mountGitHubWebhook(app: Express, ctx: ServeHttpContext): void {
const { engine, githubWebhookLimiter } = ctx;
// ---------------------------------------------------------------------------
// POST /webhooks/github — push-triggered sync (v0.40 Federated Sync v2)
// ---------------------------------------------------------------------------
// Anonymous endpoint by necessity (GitHub doesn't carry an OAuth token).
// Auth is via per-source HMAC-SHA256 in the X-Hub-Signature-256 header.
//
// D3: 60 req/min/IP rate limit + pre-DB short-circuit on missing
// signature, so probe traffic doesn't even touch the source-lookup
// query.
// D5: event=push AND ref-match against sources.config.tracked_branch.
// Other event types (ping, pull_request, etc.) return 202 'ignored'
// so GitHub doesn't retry.
// D15.5: HMAC compare uses the shared safeHexEqual helper.
// D18: submits 'sync' job with extraction + auto_embed_backfill enabled and
// priority -10 (above autopilot's 0). This opts normal incremental pushes
// into sync's inline extraction while pagesAffected still identifies the
// changed pages. The sync core can still defer large (>100) changes.
// ---------------------------------------------------------------------------
app.post(
'/webhooks/github',
githubWebhookLimiter,
express.raw({ type: '*/*', limit: '1mb' }),
async (req: Request, res: Response) => {
// D3 pre-DB short-circuit: missing signature → 401 without any
// source lookup. Bot probe traffic ends here.
const sigHeader = req.header('X-Hub-Signature-256');
if (!sigHeader) {
res.status(401).json({ error: 'missing_signature', message: 'X-Hub-Signature-256 header is required' });
return;
}
// D5: filter by event header. GitHub fires webhooks for every event
// type. Anything not in the handled set is acknowledged with 202 +
// reason so GitHub doesn't retry — but no source lookup or job
// submission. Push events drive git-source sync (below). Issue/PR
// events drive github-kind single-item refresh (itemFlow).
const event = req.header('X-GitHub-Event') ?? '';
const GH_ITEM_EVENTS = new Set([
'issues',
'pull_request',
'issue_comment',
'pull_request_review',
'pull_request_review_comment',
'label',
'assignee',
'milestone',
'check_run',
'check_suite',
'workflow_run',
]);
if (event !== 'push' && !GH_ITEM_EVENTS.has(event)) {
res.status(202).json({ status: 'ignored', reason: `event=${event || '(missing)'}` });
return;
}
const payload = Buffer.isBuffer(req.body) ? req.body : Buffer.from(JSON.stringify(req.body), 'utf8');
if (payload.length === 0) {
res.status(400).json({ error: 'empty_body' });
return;
}
let parsed: Record<string, unknown>;
try {
parsed = JSON.parse(payload.toString('utf8'));
} catch {
res.status(400).json({ error: 'malformed_json' });
return;
}
// GitHub-kind item refresh path (v0.46): issues / pull_request /
// comment / review / label / assignee / milestone / check events
// refresh exactly the item that changed.
if (GH_ITEM_EVENTS.has(event)) {
await handleGitHubItemEvent(engine, parsed, sigHeader, payload, res, event, req.header('X-GitHub-Delivery'));
return;
}
const pushParsed = parsed as { repository?: { full_name?: string }; ref?: string };
const fullName = pushParsed.repository?.full_name;
const ref = pushParsed.ref;
if (!fullName || !ref) {
res.status(400).json({ error: 'missing_fields', message: 'repository.full_name and ref are required' });
return;
}
// Source lookup via the v87 partial expression index on
// config->>'github_repo'. fast even on large brains.
let source: { id: string; config: Record<string, unknown> | string } | null = null;
try {
const rows = await engine.executeRaw<{ id: string; config: Record<string, unknown> | string }>(
`SELECT id, config FROM sources WHERE config->>'github_repo' = $1 LIMIT 1`,
[fullName],
);
source = rows[0] ?? null;
} catch (err) {
console.error('webhook: source lookup error:', err);
res.status(500).json({ error: 'lookup_failed' });
return;
}
const cfg = (source ? (typeof source.config === 'string' ? JSON.parse(source.config) : source.config) : {}) as {
webhook_secret?: string;
tracked_branch?: string;
};
// HMAC verify before anything about the repository is answered: an unknown repo, a source with no secret and a
// bad signature all get the same 401 after the same HMAC work, so an unsigned request cannot tell which
// repositories this brain syncs.
const secret = cfg.webhook_secret;
const verified = verifyWebhookSig(secret, sigHeader, payload);
if (!source || !verified) {
if (source && !(typeof secret === 'string' && secret)) console.error(`webhook: push for source ${source.id} refused: no webhook secret is set (run: gbrain sources webhook set ${source.id})`);
res.status(401).json({ error: 'signature_mismatch' });
return;
}
// D5: ref must match the configured tracked branch (default 'main').
const trackedBranch = cfg.tracked_branch ?? 'main';
const expectedRef = `refs/heads/${trackedBranch}`;
if (ref !== expectedRef) {
res.status(202).json({
status: 'ignored',
reason: `ref_mismatch`,
received_ref: ref,
tracked_branch: trackedBranch,
});
return;
}
try {
await submitGitHubPush(engine, source.id, (parsed as { after?: unknown }).after, res);
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
console.error('webhook: queue submission error:', msg);
res.status(500).json({ error: 'queue_submission_failed', message: msg });
}
},
);
}
const PUSHED_SHA = /^[0-9a-f]{40}([0-9a-f]{24})?$/;
/** The part of an Express response the push answer writes. */
export interface PushAnswerSink { status(code: number): { json(body: unknown): unknown } }
/**
* GBRA-133 PR 2a.3: a verified push for a tracked branch. The pushed commit is recorded as a wanted marker first, so
* it is never lost to coalescing: a classic source queues one sync (priority -10, single flight per source through
* maxPending 1; this replaced the 30 s idempotency bucket, which dropped a second push inside the bucket while the
* first job ran) and a later run re-checks the marker. A managed source never pulls, so instead of a 202 for a job
* that cannot import the push it answers refresh_required (409) with the gbrain sources refresh fix, keeping the
* marker until the source reaches that commit.
*/
export async function submitGitHubPush(engine: BrainEngine, sourceId: string, after: unknown, res: PushAnswerSink): Promise<void> {
const wantedHead = typeof after === 'string' && PUSHED_SHA.test(after) && !/^0+$/.test(after) ? after : null;
const { recordTriggerMarker } = await import('../core/notifier/nudge.ts');
const recorded = await recordTriggerMarker(engine, { sourceId, intent: 'pull', wantedHead, requestedBy: 'webhook' });
if (!recorded) { res.status(404).json({ error: 'unknown_source', source_id: sourceId }); return; }
const { marker, managed } = recorded;
const base = { source_id: sourceId, wanted_head: marker.wanted_head, marker_generation: marker.generation };
if (managed) {
const { opError } = await import('../core/ops/contract.ts');
const error = opError('refresh_required', `A push arrived for managed source ${sourceId}; a managed source never pulls, so the pushed commit waits for a refresh.`,
`Run gbrain sources refresh ${sourceId} on the brain host; the resident notifier then syncs the refreshed checkout.`,
{ fix: { argv: ['gbrain', 'sources', 'refresh', sourceId], consent: [], actor: 'user', requires_exclusive: false,
why: 'Fast-forwards the managed checkout to its upstream under the writer lock and runs the managed --no-pull sync.' } });
res.status(409).json({ ...error.toJSON(), ...base, status: 'refresh_required' });
return;
}
const queue = new MinionQueue(engine);
const job = await queue.add(
'sync',
{
sourceId,
noExtract: false,
auto_embed_backfill: true,
embed_reason: 'webhook',
},
{ priority: -10, maxPending: 1 },
);
const { attachMarkerJob } = await import('../core/notifier/store.ts');
await attachMarkerJob(engine, sourceId, marker.intent, marker.generation, job.id);
res.status(202).json({ job_id: job.id, ...base, action: job.coalesced ? 'coalesced' : 'queued' });
}