Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(agents): hash-chain every run step into a verifiable receipt and deliver developer webhooks from a durable retry queue
nirholas03ae7cf0d54f
diff --git a/api/_lib/agents-v1/automations.js b/api/_lib/agents-v1/automations.jsindex f7083b7c4..e8431e245 100644--- a/api/_lib/agents-v1/automations.js+++ b/api/_lib/agents-v1/automations.js@@ -15,6 +15,7 @@// object (source 'wallet_intent' / 'alert_rule') and can be deleted through it.// A stopped agent (agent_identities.status) never fires.+import { createHash } from 'node:crypto';import { CronExpressionParser } from 'cron-parser';import { sql } from '../db.js';import { apiError } from './http.js';@@ -22,8 +23,10 @@ import {createIntent,deleteIntent,fireIntentForAutomation,+ getIntent,listIntents,normalizeIntent,+ updateIntent,} from '../wallet-intents.js';import { fetchTokenPriceUsd } from '../market/token-market.js';import { getSolanaAddressBalances } from '../agent-wallet.js';@@ -31,6 +34,7 @@ import { recentPumpLaunches } from '../pump-launch-feed.js';import { fetchRecentTrades } from '../pump-alert-runner.js';import { solPriceUsd } from '../sol-price.js';import { resolveSolanaRecipient } from '../../../src/solana/sns.js';+import { dispatchWebhooks } from '../webhook-dispatch.js';export const TRIGGER_TYPES = Object.freeze(['price_threshold',@@ -331,12 +335,7 @@ export async function createAutomation({ agent, userId, body, source = 'api' })const [{ n }] = await sql`SELECT count(*)::int AS n FROM agent_automations WHERE agent_id = ${agent.id}`;if (n >= MAX_PER_AGENT) throw apiError(409, 'automation_limit', `An agent can hold at most ${MAX_PER_AGENT} automations.`);- if (a.action.type === 'transfer') {- const dest = await resolveSolanaRecipient(a.action.destination).catch(() => null);- const address = dest?.address || (BASE58_RE.test(a.action.destination) ? a.action.destination : null);- if (!address) throw bad('invalid_parameter', 'action.destination must be a Solana address or .sol name.', { parameter: 'action.destination' });- a.action.destination = address;- }+ await resolveTransferDestination(a);let intentId = null;if (a.action.type !== 'agent_prompt') {@@ -419,6 +418,185 @@ export async function deleteAutomation(userId, id) {throw apiError(404, 'not_found', 'No automation with that id.');}+/** Resolve a transfer destination (.sol name or address) in place, or throw. */+async function resolveTransferDestination(a) {+ if (a.action.type !== 'transfer') return;+ const dest = await resolveSolanaRecipient(a.action.destination).catch(() => null);+ const address = dest?.address || (BASE58_RE.test(a.action.destination) ? a.action.destination : null);+ if (!address) throw bad('invalid_parameter', 'action.destination must be a Solana address or .sol name.', { parameter: 'action.destination' });+ a.action.destination = address;+}++async function ownedAutomationRow(userId, id) {+ const [row] = await sql`SELECT * FROM agent_automations WHERE id = ${id} AND user_id = ${userId} LIMIT 1`;+ return row || null;+}++function limitsFromIntent(intent) {+ const l = intent?.limits || {};+ const out = {};+ if (l.per_action_usd != null) out.perActionUsd = Number(l.per_action_usd);+ if (l.daily_usd != null) out.dailyUsd = Number(l.daily_usd);+ if (l.total_usd != null) out.totalUsd = Number(l.total_usd);+ return out;+}++/**+ * One automation the caller owns, with its spend limits and (for agent_prompt+ * automations) the runs it started most recently. Wallet intents and alert+ * rules resolve too, so every id automation_list returns can be read back.+ */+export async function getAutomation(userId, id) {+ const row = await ownedAutomationRow(userId, id);+ if (row) {+ const [intent, runs] = await Promise.all([+ row.intent_id ? getIntent(row.agent_id, row.intent_id) : null,+ row.action_type === 'agent_prompt'+ ? sql`+ SELECT id, status, summary, created_at, finished_at FROM agent_runs+ WHERE automation_id = ${row.id} ORDER BY created_at DESC LIMIT 10+ `+ : [],+ ]);+ return {+ ...serializeAutomation(row),+ limits: limitsFromIntent(intent),+ nextFireAt: row.trigger_type === 'schedule' && row.enabled ? nextScheduledAt(row).toISOString() : null,+ recentRuns: runs.map((r) => ({ id: r.id, status: r.status, summary: r.summary, createdAt: r.created_at, finishedAt: r.finished_at })),+ };+ }+ const [w] = await sql`+ SELECT w.id, w.agent_id FROM agent_wallet_intents w+ JOIN agent_identities i ON i.id = w.agent_id+ WHERE w.id = ${id} AND i.user_id = ${userId}+ `;+ if (w) {+ const intent = await getIntent(w.agent_id, w.id);+ return { ...serializeIntent(intent), limits: limitsFromIntent(intent), nextFireAt: null, recentRuns: [] };+ }+ const [rule] = await sql`+ SELECT r.*, f.last_fired_at FROM pump_alert_rules r+ LEFT JOIN pump_alert_rule_fires f ON f.rule_id = r.id+ WHERE r.id = ${id} AND r.user_id = ${userId}+ `;+ if (rule) return { ...serializeAlertRule(rule), limits: {}, nextFireAt: null, recentRuns: [] };+ throw apiError(404, 'not_found', 'No automation with that id.');+}++/** Merge a partial trigger/action patch over the stored config; a new type replaces it. */+function mergePart(stored, patch) {+ if (patch == null) return stored;+ if (typeof patch !== 'object') return patch;+ if (patch.type && patch.type !== stored?.type) return patch;+ return { ...stored, ...patch };+}++/**+ * Update an automation the caller owns. The merged result is re-validated by+ * normalizeAutomation, the same validator create uses, so a patch can never+ * store a config create would refuse. Spend automations need `confirm: true`+ * on every change except switching them off.+ *+ * @param {{ userId: string, id: string, body: object }} o+ */+export async function updateAutomation({ userId, id, body }) {+ const patch = body && typeof body === 'object' ? body : {};+ const row = await ownedAutomationRow(userId, id);+ if (!row) {+ await getAutomation(userId, id);+ throw apiError(409, 'not_editable', 'Only automations created as automations can be edited here. Edit wallet rules from the agent wallet page and alert rules from /alerts, or delete and recreate them as an automation.');+ }+ const intent = row.intent_id ? await getIntent(row.agent_id, row.intent_id) : null;+ const storedTrigger = row.trigger_config;+ const storedAction = row.action_config;+ const customTitle = row.title !== defaultTitle(storedTrigger, storedAction);+ const raw = {+ title: typeof patch.title === 'string' ? patch.title : customTitle ? row.title : undefined,+ trigger: mergePart(storedTrigger, patch.trigger),+ action: mergePart(storedAction, patch.action),+ triggerOnce: typeof patch.triggerOnce === 'boolean' ? patch.triggerOnce : row.trigger_once,+ limits: patch.limits && typeof patch.limits === 'object' ? { ...limitsFromIntent(intent), ...patch.limits } : limitsFromIntent(intent),+ };+ const a = normalizeAutomation(raw);+ const enabled = typeof patch.enabled === 'boolean' ? patch.enabled : row.enabled;+ const onlyDisabling = enabled === false && ['trigger', 'action', 'limits', 'title', 'triggerOnce'].every((k) => patch[k] === undefined);+ if (SPEND_ACTIONS.has(a.action.type) && !onlyDisabling && patch.confirm !== true) {+ throw apiError(400, 'confirmation_required', `This ${a.action.type} automation spends from the agent wallet on its own. Resend with "confirm": true to authorize the change.`, {+ action: a.action,+ trigger: a.trigger,+ limits: a.limits,+ });+ }+ await resolveTransferDestination(a);++ let intentId = row.intent_id;+ if (a.action.type === 'agent_prompt') {+ if (intentId) await deleteIntent(row.agent_id, userId, intentId);+ intentId = null;+ } else {+ const norm = normalizeIntent(backingIntentPayload(a, a.title));+ if (!norm.ok) throw bad(norm.error, norm.message);+ if (intentId && intent) {+ const updated = await updateIntent(row.agent_id, userId, intentId, { intent: norm.intent, enabled });+ if (updated?.error) throw bad(updated.error, updated.message);+ } else {+ const created = await createIntent(row.agent_id, userId, norm.intent, { sourceText: `v1 automation: ${a.title}` });+ intentId = created.id;+ if (!enabled) await updateIntent(row.agent_id, userId, intentId, { enabled: false });+ }+ }+ const triggerChanged = canonicalTrigger(storedTrigger) !== canonicalTrigger(a.trigger);+ const [next] = await sql`+ UPDATE agent_automations SET+ title = ${a.title},+ trigger_type = ${a.trigger.type},+ trigger_config = ${JSON.stringify(a.trigger)}::jsonb,+ action_type = ${a.action.type},+ action_config = ${JSON.stringify(a.action)}::jsonb,+ trigger_once = ${a.triggerOnce},+ enabled = ${enabled},+ intent_id = ${intentId},+ state = CASE WHEN ${triggerChanged} THEN NULL ELSE state END,+ updated_at = now()+ WHERE id = ${row.id}+ RETURNING *+ `;+ return { ...serializeAutomation(next), limits: a.limits };+}++function canonicalTrigger(t) {+ return JSON.stringify(Object.keys(t || {}).sort().map((k) => [k, t[k]]));+}++/**+ * Fire an automation's action right now, outside its trigger. The fire key is+ * unique per call, so the action runs once per request; it still passes every+ * guard a triggered fire does (stopped agent, spend policy, intent caps).+ * Spend actions need `confirm: true`.+ */+export async function triggerAutomation({ userId, id, confirm = false }) {+ const row = await ownedAutomationRow(userId, id);+ if (!row) {+ await getAutomation(userId, id);+ throw apiError(409, 'not_triggerable', 'Only automations created as automations can be fired on demand.');+ }+ if (!row.enabled) throw apiError(409, 'automation_disabled', 'This automation is switched off. Enable it first, then trigger it.');+ const [agent] = await sql`SELECT status, deleted_at FROM agent_identities WHERE id = ${row.agent_id} LIMIT 1`;+ if (!agent || agent.deleted_at) throw apiError(404, 'agent_not_found', 'Agent not found.');+ if (agent.status === 'stopped') throw apiError(409, 'agent_stopped', 'This agent is stopped. Start it before triggering its automations.');+ if (row.action_config?.mintFromEvent && !row.action_config?.mint) {+ throw apiError(409, 'needs_event', 'This swap buys the coin its trigger event names, so it can only run when that event happens.');+ }+ if (SPEND_ACTIONS.has(row.action_type) && confirm !== true) {+ throw apiError(400, 'confirmation_required', `Triggering this ${row.action_type} automation spends from the agent wallet now. Resend with "confirm": true to authorize it.`, {+ action: row.action_config,+ });+ }+ const result = await fireAutomation(row, { manual: true }, `manual:${Date.now()}`);+ const [fresh] = await sql`SELECT * FROM agent_automations WHERE id = ${row.id}`;+ return { result, automation: serializeAutomation(fresh) };+}+// ── evaluation ───────────────────────────────────────────────────────────────function cooledDown(row, now) {@@ -474,6 +652,33 @@ function eventSummary(event) {* the same (automation, key) never executes twice.*/export async function fireAutomation(row, event, key) {+ const outcome = await runAutomationAction(row, event, key);+ if (outcome.fired) {+ // Keyed on the fire's own discriminator, so a sweep that retries the+ // same trigger delivers automation.fired once.+ await dispatchWebhooks({+ userId: row.user_id,+ agentId: row.agent_id,+ eventType: 'automation.fired',+ eventId: `evt_auto_${createHash('sha256').update(`${row.id}:${key}`).digest('hex').slice(0, 32)}`,+ data: {+ automation_id: row.id,+ agent_id: row.agent_id,+ title: row.title || null,+ trigger: { type: row.trigger_type, key },+ action: { type: row.action_type },+ status: outcome.status,+ run_id: outcome.runId || null,+ signature: outcome.signature || null,+ note: outcome.note || null,+ event: event && typeof event === 'object' ? event : null,+ },+ });+ }+ return outcome;+}++async function runAutomationAction(row, event, key) {const discriminator = `auto:${row.id}:${key}`;if (row.action_type === 'agent_prompt') {const { createRun } = await import('./runs.js');diff --git a/api/_lib/agents-v1/run-receipts.js b/api/_lib/agents-v1/run-receipts.jsnew file mode 100644index 000000000..8f1f831d3--- /dev/null+++ b/api/_lib/agents-v1/run-receipts.js@@ -0,0 +1,145 @@+// Run receipts and summaries: the audit layer over agent_run_steps.+//+// Every step a run writes carries a receipt, a sha256 digest chained to the+// step before it (migration 20261010163000_agent_run_receipts.sql). Anyone+// holding the step rows can recompute the chain with verifyReceiptChain and+// prove that no tool call or tool result was edited, dropped or reordered.+//+// JSON is hashed in canonical form (object keys sorted at every depth) because+// jsonb does not preserve key order: the bytes we hash at insert time must be+// reproducible from what the database hands back later.++import { createHash } from 'node:crypto';++/** JSON with object keys sorted at every depth; null for undefined. */+export function canonicalJson(value) {+ if (value === undefined || value === null) return 'null';+ if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`;+ if (typeof value === 'object') {+ const keys = Object.keys(value).filter((k) => value[k] !== undefined).sort();+ return `{${keys.map((k) => `${JSON.stringify(k)}:${canonicalJson(value[k])}`).join(',')}}`;+ }+ return JSON.stringify(value);+}++/**+ * The receipt for one step, chained to the receipt of the step before it.+ * @param {{ prev: string|null, runId: string, seq: number, kind: string, tool?: string|null, input?: unknown, output?: unknown }} s+ */+export function stepReceipt({ prev, runId, seq, kind, tool = null, input = null, output = null }) {+ return createHash('sha256')+ .update([prev || '', runId, String(seq), kind, tool || '', canonicalJson(input), canonicalJson(output)].join('|'))+ .digest('hex');+}++/**+ * Recompute a run's receipt chain from its serialized steps (seq ascending).+ * Steps written before receipts existed (receipt null) are skipped until the+ * chain starts. Returns { verified, checked, brokenAt }.+ */+export function verifyReceiptChain(runId, steps) {+ let prev = null;+ let checked = 0;+ for (const s of steps) {+ if (!s.receipt) {+ if (checked) return { verified: false, checked, brokenAt: s.seq };+ continue;+ }+ const expected = stepReceipt({ prev, runId, seq: s.seq, kind: s.kind, tool: s.tool, input: s.input, output: s.output });+ if (expected !== s.receipt) return { verified: false, checked, brokenAt: s.seq };+ prev = s.receipt;+ checked += 1;+ }+ return { verified: checked > 0, checked, brokenAt: null };+}++/**+ * Pair every tool call with its result (or its block) into one trace entry,+ * in call order. The receipt on each entry is the result's receipt, which+ * chains over the call before it, so one digest vouches for both halves.+ */+export function toolTraces(steps) {+ const traces = [];+ const open = [];+ for (const s of steps) {+ if (s.kind === 'tool_call') {+ const t = { tool: s.tool, input: s.input, callSeq: s.seq, callReceipt: s.receipt ?? null, status: 'pending', output: null, resultSeq: null, receipt: s.receipt ?? null, latencyMs: null, at: s.at };+ traces.push(t);+ open.push(t);+ } else if (s.kind === 'tool_result' || s.kind === 'tool_blocked') {+ const i = open.findIndex((t) => t.tool === s.tool);+ const t = i >= 0 ? open.splice(i, 1)[0] : { tool: s.tool, input: s.input, callSeq: null, callReceipt: null, at: s.at };+ if (i < 0) traces.push(t);+ const failed = s.kind === 'tool_result' && s.output && typeof s.output === 'object' && 'error' in s.output && Object.keys(s.output).length === 1;+ t.status = s.kind === 'tool_blocked' ? 'blocked' : failed ? 'error' : 'ok';+ t.output = s.output;+ t.resultSeq = s.seq;+ t.receipt = s.receipt ?? null;+ t.latencyMs = s.latencyMs ?? null;+ }+ }+ return traces;+}++const STATUS_LEAD = {+ completed: 'Completed',+ failed: 'Failed',+ cancelled: 'Cancelled',+ budget_exhausted: 'Stopped at its budget',+};++function money(n) {+ const v = Number(n) || 0;+ return v === 0 ? '$0' : v < 0.01 ? `$${v.toFixed(4)}` : `$${v.toFixed(2)}`;+}++function duration(ms) {+ if (!Number.isFinite(ms) || ms < 0) return null;+ const s = Math.round(ms / 1000);+ if (s < 60) return `${s}s`;+ const m = Math.floor(s / 60);+ if (m < 60) return `${m}m ${s % 60}s`;+ return `${Math.floor(m / 60)}h ${m % 60}m`;+}++function excerpt(text, n = 280) {+ const t = String(text || '').replace(/\s+/g, ' ').trim();+ return t.length > n ? `${t.slice(0, n - 1)}…` : t;+}++/**+ * A plain-language account of how a run ended.+ * @param {object} run the agent_runs row after finalize+ * @param {Array<{ kind: string, tool_name: string|null, n: number, failed: number }>} tally step counts by kind and tool+ */+export function summarizeRun(run, tally) {+ const lead = STATUS_LEAD[run.status] || run.status;+ const turns = tally.filter((t) => t.kind === 'model_call').reduce((a, t) => a + Number(t.n), 0);+ const byTool = new Map();+ let failedCalls = 0;+ let blocked = 0;+ for (const t of tally) {+ if (t.kind === 'tool_call' && t.tool_name) byTool.set(t.tool_name, (byTool.get(t.tool_name) || 0) + Number(t.n));+ if (t.kind === 'tool_result') failedCalls += Number(t.failed || 0);+ if (t.kind === 'tool_blocked') blocked += Number(t.n);+ }+ const started = run.started_at || run.created_at;+ const took = started && run.finished_at ? duration(new Date(run.finished_at) - new Date(started)) : null;++ const parts = [];+ parts.push(`${lead} after ${turns} model turn${turns === 1 ? '' : 's'}${took ? ` in ${took}` : ''}.`);+ const calls = [...byTool.values()].reduce((a, n) => a + n, 0);+ if (calls) {+ const list = [...byTool.entries()].sort((a, b) => b[1] - a[1]).map(([name, n]) => (n > 1 ? `${name} x${n}` : name)).join(', ');+ const extra = [failedCalls ? `${failedCalls} failed` : null, blocked ? `${blocked} blocked` : null].filter(Boolean).join(', ');+ parts.push(`Made ${calls} tool call${calls === 1 ? '' : 's'}: ${list}${extra ? ` (${extra})` : ''}.`);+ } else {+ parts.push('Made no tool calls.');+ }+ const budget = Number(run.budget_credits_usd) || 0;+ const spent = Number(run.spent_credits_usd) || 0;+ parts.push(budget > 0 ? `Spent ${money(spent)} of a ${money(budget)} budget.` : 'Ran on free model lanes only, with no credit spend.');+ if (run.status === 'completed' && run.result) parts.push(`Result: ${excerpt(run.result)}`);+ else if (run.error) parts.push(`Reason: ${excerpt(run.error)}`);+ return parts.join(' ');+}diff --git a/api/_lib/agents-v1/runs.js b/api/_lib/agents-v1/runs.jsindex 16e774cea..ed2897e58 100644--- a/api/_lib/agents-v1/runs.js+++ b/api/_lib/agents-v1/runs.js@@ -27,6 +27,8 @@ import { agentToolSchemas, agentToolHandlers } from '../agent-tools.js';import { debitCredits } from '../credits.js';import { assertAgentBudget, InferenceBillingError } from '../inference-billing.js';import { AGENT_SYSTEM_NOTE, createAgentLoop, finalAnswer, initialLoopState, loopFinished } from '../agent-loop.js';+import { stepReceipt, summarizeRun } from './run-receipts.js';+import { dispatchWebhooks } from '../webhook-dispatch.js';const LEASE_SECONDS = 120;const TERMINAL = new Set(['completed', 'failed', 'cancelled', 'budget_exhausted']);@@ -61,6 +63,7 @@ export function serializeRun(r) {scheduledFor: r.scheduled_for,result: r.result,error: r.error,+ summary: r.summary ?? null,source: r.source,automationId: r.automation_id,cancelRequested: Boolean(r.cancel_requested_at),@@ -83,6 +86,7 @@ export function serializeStep(s) {usage: s.input_tokens == null && s.output_tokens == null ? null : { input: s.input_tokens, output: s.output_tokens },costUsd: s.cost_micro_usd == null ? null : Number(s.cost_micro_usd) / 1e6,latencyMs: s.latency_ms,+ receipt: s.receipt ?? null,at: s.created_at,};}@@ -220,17 +224,28 @@ async function lastSeq(runId) {async function insertSteps(runId, fromSeq, events) {let seq = fromSeq;+ let prev = null;+ if (fromSeq > 0) {+ const [p] = await sql`SELECT receipt FROM agent_run_steps WHERE run_id = ${runId} AND seq = ${fromSeq} LIMIT 1`;+ prev = p?.receipt || null;+ }for (const e of events) {seq += 1;+ // Hash exactly what jsonb will hand back: a JSON round trip drops+ // undefined and turns Dates into strings before the digest is taken.+ const input = e.input == null ? null : JSON.parse(JSON.stringify(e.input));+ const output = e.output == null ? null : JSON.parse(JSON.stringify(e.output));+ const receipt = stepReceipt({ prev, runId, seq, kind: e.kind, tool: e.tool || null, input, output });await sql`INSERT INTO agent_run_steps- (run_id, seq, kind, provider, model, tool_name, input, output, input_tokens, output_tokens, cost_micro_usd, latency_ms)+ (run_id, seq, kind, provider, model, tool_name, input, output, input_tokens, output_tokens, cost_micro_usd, latency_ms, receipt)VALUES(${runId}, ${seq}, ${e.kind}, ${e.provider || null}, ${e.model || null}, ${e.tool || null},- ${e.input == null ? null : JSON.stringify(e.input)}::jsonb, ${e.output == null ? null : JSON.stringify(e.output)}::jsonb,- ${e.inputTokens ?? null}, ${e.outputTokens ?? null}, ${e.costMicroUsd ?? null}, ${e.latencyMs ?? null})+ ${input == null ? null : JSON.stringify(input)}::jsonb, ${output == null ? null : JSON.stringify(output)}::jsonb,+ ${e.inputTokens ?? null}, ${e.outputTokens ?? null}, ${e.costMicroUsd ?? null}, ${e.latencyMs ?? null}, ${receipt})ON CONFLICT (run_id, seq) DO NOTHING`;+ prev = receipt;}return seq;}@@ -245,7 +260,24 @@ async function finalize(run, status, { result = null, error = null, note = nullawait insertSteps(run.id, await lastSeq(run.id), [{ kind: status === 'completed' ? 'final' : status === 'failed' ? 'error' : 'status', output: { status, result, error, note } },]);- return row;+ const tally = await sql`+ SELECT kind, tool_name, count(*)::int AS n,+ count(*) FILTER (WHERE kind = 'tool_result' AND output ? 'error' AND jsonb_typeof(output) = 'object')::int AS failed+ FROM agent_run_steps WHERE run_id = ${run.id}+ GROUP BY kind, tool_name+ `;+ const summary = summarizeRun(row, tally);+ const [done] = await sql`UPDATE agent_runs SET summary = ${summary} WHERE id = ${run.id} RETURNING *`;+ // The event ID is derived from the run, so a finalize that races another+ // still delivers run.finished once per endpoint.+ await dispatchWebhooks({+ userId: done.user_id,+ agentId: done.agent_id,+ eventType: 'run.finished',+ eventId: `evt_run_finished_${done.id}`,+ data: serializeRun(done),+ });+ return done;}async function acquireLease(runId, owner) {diff --git a/api/_lib/migrations/20261010163000_agent_run_receipts.sql b/api/_lib/migrations/20261010163000_agent_run_receipts.sqlnew file mode 100644index 000000000..5c29758d0--- /dev/null+++ b/api/_lib/migrations/20261010163000_agent_run_receipts.sql@@ -0,0 +1,20 @@+-- Migration: agent run receipts and end-of-run summaries.+-- Apply: node scripts/apply-migrations.mjs --apply --file 20261010163000_agent_run_receipts.sql+-- Idempotent.+--+-- agent_run_steps.receipt+-- A sha256 hex digest that chains every step of a run to the one before it:+-- sha256(prev_receipt | run_id | seq | kind | tool | input | output). A replay+-- or an MCP client can recompute the chain from the step rows and prove that+-- no tool call or tool result was edited, dropped or reordered after the+-- fact. Rows written before this migration keep a null receipt; the chain+-- starts at the first step written after it.+--+-- agent_runs.summary+-- A plain-language account of how a run ended (status, turns, tool calls by+-- name, spend against budget, duration and the result or error), written once+-- when the run reaches a terminal status. The agent page replay view and the+-- MCP run tools show it verbatim.++alter table agent_run_steps add column if not exists receipt text;+alter table agent_runs add column if not exists summary text;diff --git a/api/_lib/migrations/20261010170000_agent_webhook_queue.sql b/api/_lib/migrations/20261010170000_agent_webhook_queue.sqlnew file mode 100644index 000000000..539ee73b9--- /dev/null+++ b/api/_lib/migrations/20261010170000_agent_webhook_queue.sql@@ -0,0 +1,65 @@+-- Durable webhook delivery for the v1 agents API.+--+-- Developer webhooks used to retry three times inside the request that raised+-- the event (1s, 4s, 16s), writing one webhook_deliveries row per attempt. A+-- receiver that was down for a minute lost the event for good, and a process+-- that recycled mid-retry dropped it silently.+--+-- From here on a delivery is ONE row per (endpoint, event) that moves through+-- pending -> delivering -> succeeded | failed. Every HTTP attempt is appended+-- to `attempts`. A failed attempt reschedules the row with backoff in+-- `next_attempt_at`; the every-minute cron (api/cron/webhook-deliveries.js)+-- picks up whatever is due, and `lease_until` keeps two workers from posting+-- the same row at once. A replay is a fresh row pointing at its original+-- through `replay_of`. Rows written before this migration keep status NULL+-- and still read as one-row-per-attempt history.+--+-- Service layer: api/_lib/webhook-dispatch.js. Routes: /api/v1/webhooks+-- (api/v1/rest.js) and /api/developer/webhooks.++-- An endpoint can be narrowed to one agent: agent-scoped events (run.finished,+-- automation.fired, message.received, approval.needed) for any other agent are+-- not delivered to it. NULL means every agent the account owns.+ALTER TABLE developer_webhooks ADD COLUMN IF NOT EXISTS agent_id uuid REFERENCES agent_identities(id) ON DELETE CASCADE;+ALTER TABLE developer_webhooks ADD COLUMN IF NOT EXISTS consecutive_failures int NOT NULL DEFAULT 0;+ALTER TABLE developer_webhooks ADD COLUMN IF NOT EXISTS disabled_reason text;+ALTER TABLE developer_webhooks ADD COLUMN IF NOT EXISTS last_success_at timestamptz;+ALTER TABLE developer_webhooks ADD COLUMN IF NOT EXISTS last_failure_at timestamptz;++CREATE INDEX IF NOT EXISTS developer_webhooks_user_created+ ON developer_webhooks (user_id, created_at DESC, id DESC);++ALTER TABLE webhook_deliveries ADD COLUMN IF NOT EXISTS status text;+ALTER TABLE webhook_deliveries ADD COLUMN IF NOT EXISTS next_attempt_at timestamptz;+ALTER TABLE webhook_deliveries ADD COLUMN IF NOT EXISTS lease_until timestamptz;+ALTER TABLE webhook_deliveries ADD COLUMN IF NOT EXISTS attempts jsonb NOT NULL DEFAULT '[]'::jsonb;+ALTER TABLE webhook_deliveries ADD COLUMN IF NOT EXISTS delivered_at timestamptz;+ALTER TABLE webhook_deliveries ADD COLUMN IF NOT EXISTS duration_ms int;+ALTER TABLE webhook_deliveries ADD COLUMN IF NOT EXISTS replay_of uuid REFERENCES webhook_deliveries(id) ON DELETE SET NULL;+ALTER TABLE webhook_deliveries ADD COLUMN IF NOT EXISTS updated_at timestamptz NOT NULL DEFAULT now();++DO $$ BEGIN+ ALTER TABLE webhook_deliveries ADD CONSTRAINT webhook_deliveries_status_check+ CHECK (status IS NULL OR status IN ('pending', 'delivering', 'succeeded', 'failed'));+EXCEPTION WHEN duplicate_object THEN NULL; END $$;++-- One original delivery per endpoint per event: an event raised twice (a+-- retried cron, a re-polled approval) can never post twice. Replays and the+-- legacy per-attempt rows are outside the constraint.+CREATE UNIQUE INDEX IF NOT EXISTS webhook_deliveries_event_once+ ON webhook_deliveries (webhook_id, event_id)+ WHERE status IS NOT NULL AND replay_of IS NULL;++-- The retry sweep reads only what is due.+CREATE INDEX IF NOT EXISTS webhook_deliveries_due+ ON webhook_deliveries (next_attempt_at)+ WHERE status = 'pending';++-- A worker that died holding a lease is found by this.+CREATE INDEX IF NOT EXISTS webhook_deliveries_leased+ ON webhook_deliveries (lease_until)+ WHERE status = 'delivering';++-- The delivery log pages newest first by (created_at, id).+CREATE INDEX IF NOT EXISTS webhook_deliveries_webhook_page+ ON webhook_deliveries (webhook_id, created_at DESC, id DESC);diff --git a/api/_lib/webhook-dispatch.js b/api/_lib/webhook-dispatch.jsindex 8451ffd13..3dfdbebdb 100644--- a/api/_lib/webhook-dispatch.js+++ b/api/_lib/webhook-dispatch.js@@ -1,20 +1,49 @@-// Outgoing webhook dispatch — delivers events to developer-registered endpoints.+// Outgoing webhooks: a durable, signed, retried delivery queue for+// developer-registered endpoints.//-// Uses the Standard Webhooks signature format:-// webhook-id: unique event ID-// webhook-timestamp: unix epoch seconds-// webhook-signature: v1,{base64 HMAC-SHA256}+// Signature (Standard Webhooks format):+// webhook-id: the event ID, stable across retries and replays+// webhook-timestamp: unix epoch seconds of THIS attempt+// webhook-signature: v1,{base64url HMAC-SHA256 of `${id}.${timestamp}.${body}`}//-// Retry: 3 attempts with exponential backoff (1s, 4s, 16s).-// Fire-and-forget — callers don't await delivery results.+// Delivery model (migration 20261010170000_agent_webhook_queue.sql):+// • Raising an event writes one webhook_deliveries row per matching endpoint+// (status 'pending'), then attempts each one immediately, fire-and-forget.+// Callers never wait on a receiver.+// • A failed attempt is appended to the row's `attempts` log and the row is+// rescheduled with backoff (RETRY_SCHEDULE_MS). The every-minute cron+// (api/cron/webhook-deliveries.js) runs `sweepWebhookDeliveries`, which+// posts whatever is due, so a retry survives a recycled process.+// • A row is claimed with a lease before it is posted, so the inline attempt+// and the sweep can never deliver the same row concurrently.+// • An endpoint that keeps failing is switched off (DISABLE_AFTER_FAILURES).+// • `replayDelivery` re-sends a past delivery as a new row linked through+// `replay_of`, with the original event ID so receivers can deduplicate.import { sql } from './db.js';import { randomToken, hmacSha256 } from './crypto.js';import { validatePublicUrl, resolvePublicHost, pinnedAgent, SsrfError } from './ssrf.js';-const MAX_ATTEMPTS = 3;-const BACKOFF_BASE_MS = 1000;const DELIVERY_TIMEOUT_MS = 10_000;+const LEASE_SECONDS = 30;+const RESPONSE_BODY_MAX = 1024;++// Delay before attempt n+1, after attempt n failed. Eight attempts spread over+// roughly twelve hours, then the delivery is marked failed and can be replayed.+export const RETRY_SCHEDULE_MS = Object.freeze([+ 30_000, // 30 s+ 2 * 60_000, // 2 min+ 10 * 60_000, // 10 min+ 30 * 60_000, // 30 min+ 60 * 60_000, // 1 h+ 3 * 60 * 60_000, // 3 h+ 8 * 60 * 60_000, // 8 h+]);+export const MAX_ATTEMPTS = RETRY_SCHEDULE_MS.length + 1;++// Failed attempts in a row, with no success in the last 24 hours, before an+// endpoint is set inactive. Re-enable with PATCH { active: true }.+export const DISABLE_AFTER_FAILURES = 50;const EVENT_TYPES = ['avatar.created',@@ -34,8 +63,18 @@ const EVENT_TYPES = [// all deliver the same event exactly once.'forge.completed','forge.failed',+ // Agent activity (the v1 agents API, docs/api-reference.md). Each carries+ // `agent_id`, so an endpoint narrowed to one agent only hears that agent.+ 'run.finished', // a run reached completed, failed, cancelled or budget_exhausted+ 'automation.fired', // an automation's trigger matched and its action ran+ 'message.received', // a user message was added to an agent's thread+ 'approval.needed', // an agent action is waiting for the owner's approval];+// Sent only by POST /api/v1/webhooks/:id/test, to that one endpoint, whatever+// it subscribes to. Not subscribable.+export const TEST_EVENT_TYPE = 'webhook.test';+export { EVENT_TYPES };/**@@ -67,84 +106,278 @@ export function selectEventTypes(raw) {return { events: events.length ? events : [...EVENT_TYPES] };}-export async function dispatchWebhooks({ userId, eventType, data }) {- if (!EVENT_TYPES.includes(eventType)) return;+/**+ * Loopback and plain-http receivers are refused in production. Outside it, the+ * operator can opt in with WEBHOOKS_ALLOW_PRIVATE_TARGETS=1 to point an+ * endpoint at a receiver on their own machine.+ */+export function privateTargetsAllowed() {+ return process.env.NODE_ENV !== 'production' && process.env.WEBHOOKS_ALLOW_PRIVATE_TARGETS === '1';+}++/** A fresh signing secret. */+export function newWebhookSecret() {+ return `whsec_${randomToken(24)}`;+}- let webhooks;+/** A fresh event ID. */+export function newEventId() {+ return `evt_${randomToken(16)}`;+}++function envelope(eventId, eventType, data) {+ return { id: eventId, type: eventType, created_at: new Date().toISOString(), data };+}++/**+ * Raise an event for every matching endpoint the user owns.+ *+ * Resolves once the delivery rows are written; the HTTP attempts run in the+ * background. Never throws: a webhook outage must not fail the caller's work.+ *+ * @param {object} o+ * @param {string} o.userId+ * @param {string} o.eventType one of EVENT_TYPES+ * @param {object} o.data+ * @param {string} [o.agentId] the agent the event concerns; endpoints narrowed+ * to a different agent are skipped+ * @param {string} [o.eventId] a deterministic ID, so raising the same event+ * twice delivers it once per endpoint+ * @returns {Promise<string[]>} the delivery IDs written+ */+export async function dispatchWebhooks({ userId, eventType, data, agentId = null, eventId = null }) {+ if (!userId || !EVENT_TYPES.includes(eventType)) return [];+ const id = eventId || newEventId();+ const payload = envelope(id, eventType, data ?? {});+ let rows;try {- webhooks = await sql`- select id, url, secret, events- from developer_webhooks- where user_id = ${userId}- and active = true- and (events @> ARRAY[${eventType}]::text[] or cardinality(events) = 0)+ rows = await sql`+ insert into webhook_deliveries+ (webhook_id, event_type, event_id, payload, attempt, status, next_attempt_at)+ select w.id, ${eventType}, ${id}, ${JSON.stringify(payload)}::jsonb, 0, 'pending', now()+ from developer_webhooks w+ where w.user_id = ${userId}+ and w.active = true+ and (w.events @> ARRAY[${eventType}]::text[] or cardinality(w.events) = 0)+ and (w.agent_id is null or w.agent_id = ${agentId})+ on conflict (webhook_id, event_id) where status is not null and replay_of is null do nothing+ returning id`;- } catch {- return;+ } catch (err) {+ console.warn('[webhooks] enqueue failed:', eventType, err?.message);+ return [];}+ const ids = rows.map((r) => r.id);+ for (const deliveryId of ids) processDelivery(deliveryId).catch(() => {});+ return ids;+}- if (!webhooks.length) return;-- const eventId = `evt_${randomToken(16)}`;- const timestamp = Math.floor(Date.now() / 1000);- const payload = JSON.stringify({- id: eventId,- type: eventType,- created_at: new Date().toISOString(),- data,+/**+ * Queue a `webhook.test` ping to one endpoint and attempt it now. Awaited, so+ * the caller can show the outcome.+ */+export async function sendTestEvent(webhook) {+ const id = newEventId();+ const payload = envelope(id, TEST_EVENT_TYPE, {+ webhook_id: webhook.id,+ message: 'This is a test delivery from three.ws. Verify its signature with your whsec_ secret.',});-- for (const wh of webhooks) {- deliverWithRetry(wh, eventId, timestamp, payload, eventType).catch(() => {});- }+ const [row] = await sql`+ insert into webhook_deliveries+ (webhook_id, event_type, event_id, payload, attempt, status, next_attempt_at)+ values (${webhook.id}, ${TEST_EVENT_TYPE}, ${id}, ${JSON.stringify(payload)}::jsonb, 0, 'pending', now())+ returning id+ `;+ await processDelivery(row.id);+ return row.id;}-async function deliverWithRetry(webhook, eventId, timestamp, payload, eventType) {- for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {- const {- statusCode,- responseBody,- error: deliveryError,- } = await deliver(webhook.url, webhook.secret, eventId, timestamp, payload);+/**+ * Re-send a past delivery to its endpoint as a new delivery row. The event ID+ * and payload are the original's; the attempt history starts fresh.+ */+export async function replayDelivery(delivery) {+ const original = delivery.replay_of || delivery.id;+ const [row] = await sql`+ insert into webhook_deliveries+ (webhook_id, event_type, event_id, payload, attempt, status, next_attempt_at, replay_of)+ select webhook_id, event_type, event_id, payload, 0, 'pending', now(), ${original}+ from webhook_deliveries where id = ${delivery.id}+ returning id+ `;+ if (!row) return null;+ await processDelivery(row.id);+ return row.id;+}- try {- await sql`- insert into webhook_deliveries (webhook_id, event_type, event_id, payload, status_code, response_body, error, attempt)- values (${webhook.id}, ${eventType}, ${eventId}, ${payload}::jsonb, ${statusCode}, ${responseBody}, ${deliveryError}, ${attempt})- `;- } catch {- // Best-effort logging+/**+ * Post every due delivery. Called by the every-minute cron. Rows whose lease+ * expired (a worker died mid-attempt) go back to pending first.+ *+ * @returns {Promise<{ reclaimed: number, attempted: number, succeeded: number }>}+ */+export async function sweepWebhookDeliveries({ limit = 100, concurrency = 8, deadlineMs = 50_000 } = {}) {+ const started = Date.now();+ const reclaimed = await sql`+ update webhook_deliveries+ set status = 'pending', lease_until = null, updated_at = now()+ where status = 'delivering' and lease_until < now()+ returning id+ `;+ const due = await sql`+ select id from webhook_deliveries+ where status = 'pending' and next_attempt_at <= now()+ order by next_attempt_at+ limit ${limit}+ `;+ const queue = due.map((r) => r.id);+ let attempted = 0;+ let succeeded = 0;+ async function worker() {+ while (queue.length && Date.now() - started < deadlineMs) {+ const id = queue.shift();+ const outcome = await processDelivery(id).catch(() => null);+ if (!outcome) continue;+ attempted += 1;+ if (outcome.status === 'succeeded') succeeded += 1;}+ }+ await Promise.all(Array.from({ length: Math.min(concurrency, queue.length) }, worker));+ return { reclaimed: reclaimed.length, attempted, succeeded };+}- if (statusCode && statusCode >= 200 && statusCode < 300) return;+/**+ * Claim one due delivery, post it, and record the attempt.+ * Returns null when the row was not claimable (not due, or another worker has it).+ */+export async function processDelivery(deliveryId) {+ const [row] = await sql`+ update webhook_deliveries d+ set status = 'delivering',+ lease_until = now() + make_interval(secs => ${LEASE_SECONDS}),+ updated_at = now()+ from developer_webhooks w+ where d.id = ${deliveryId}+ and d.webhook_id = w.id+ and d.status = 'pending'+ and d.next_attempt_at <= now()+ returning d.id, d.event_id, d.event_type, d.payload, d.attempt,+ w.id as webhook_id, w.url, w.secret, w.active+ `;+ if (!row) return null;- if (attempt < MAX_ATTEMPTS) {- const delay = BACKOFF_BASE_MS * Math.pow(4, attempt - 1);- await sleep(delay);- }+ if (!row.active) {+ const attempt = { at: new Date().toISOString(), status_code: null, error: 'endpoint_disabled', duration_ms: 0 };+ await sql`+ update webhook_deliveries+ set status = 'failed', error = 'endpoint_disabled', lease_until = null, next_attempt_at = null,+ attempts = attempts || ${JSON.stringify([attempt])}::jsonb, updated_at = now()+ where id = ${row.id}+ `;+ return { status: 'failed', attempt };}+ const body = typeof row.payload === 'string' ? row.payload : JSON.stringify(row.payload);+ const t0 = Date.now();+ const result = await deliver(row.url, row.secret, row.event_id, body);+ const durationMs = Date.now() - t0;+ const attemptNo = Number(row.attempt || 0) + 1;+ const ok = result.statusCode >= 200 && result.statusCode < 300;+ const attempt = {+ n: attemptNo,+ at: new Date(t0).toISOString(),+ status_code: result.statusCode,+ error: result.error,+ duration_ms: durationMs,+ response_body: result.responseBody,+ };+ const exhausted = !ok && attemptNo >= MAX_ATTEMPTS;+ const status = ok ? 'succeeded' : exhausted ? 'failed' : 'pending';+ const nextAt = status === 'pending' ? new Date(Date.now() + backoffMs(attemptNo)) : null;++ await sql`+ update webhook_deliveries+ set status = ${status},+ attempt = ${attemptNo},+ status_code = ${result.statusCode},+ response_body = ${result.responseBody},+ error = ${result.error},+ duration_ms = ${durationMs},+ delivered_at = ${ok ? new Date() : null},+ next_attempt_at = ${nextAt},+ lease_until = null,+ attempts = attempts || ${JSON.stringify([attempt])}::jsonb,+ updated_at = now()+ where id = ${row.id}+ `;+ await recordEndpointHealth(row.webhook_id, ok);+ return { status, attempt, nextAttemptAt: nextAt };+}++/** Delay after failed attempt n, with up to 10% jitter so retries do not stampede. */+export function backoffMs(attemptNo) {+ const base = RETRY_SCHEDULE_MS[Math.min(attemptNo, RETRY_SCHEDULE_MS.length) - 1] ?? RETRY_SCHEDULE_MS[0];+ return Math.round(base * (1 + Math.random() * 0.1));+}++async function recordEndpointHealth(webhookId, ok) {try {- const [{ failure_count }] = await sql`- select count(*)::int as failure_count- from webhook_deliveries- where webhook_id = ${webhook.id}- and (status_code is null or status_code >= 400)- and created_at > now() - interval '24 hours'- `;- if (failure_count >= 50) {+ if (ok) {await sql`- update developer_webhooks set active = false, updated_at = now()- where id = ${webhook.id}+ update developer_webhooks+ set consecutive_failures = 0, last_success_at = now()+ where id = ${webhookId}`;+ return;}+ await sql`+ update developer_webhooks+ set consecutive_failures = consecutive_failures + 1,+ last_failure_at = now(),+ active = case+ when consecutive_failures + 1 >= ${DISABLE_AFTER_FAILURES}+ and (last_success_at is null or last_success_at < now() - interval '24 hours')+ then false else active end,+ disabled_reason = case+ when consecutive_failures + 1 >= ${DISABLE_AFTER_FAILURES}+ and (last_success_at is null or last_success_at < now() - interval '24 hours')+ then 'consecutive_failures' else disabled_reason end,+ updated_at = now()+ where id = ${webhookId}+ `;+ } catch (err) {+ console.warn('[webhooks] endpoint health update failed:', err?.message);+ }+}++/**+ * Validate a receiver URL for registration: HTTPS to a public address in+ * production. Returns an error string, or null when the URL is acceptable.+ */+export async function webhookUrlProblem(raw) {+ if (typeof raw !== 'string' || !raw.trim()) return 'url is required';+ const url = raw.trim();+ if (url.length > 2048) return 'url exceeds 2048 characters';+ let parsed;+ try {+ parsed = new URL(url);+ } catch {+ return 'url is not a valid URL';+ }+ if (privateTargetsAllowed() && (parsed.protocol === 'http:' || parsed.protocol === 'https:')) return null;+ if (parsed.protocol !== 'https:') return 'Webhook URL must use HTTPS';+ try {+ validatePublicUrl(url, { allowHttp: false });+ await resolvePublicHost(parsed.hostname);} catch {- // Best-effort deactivation+ return 'Webhook URL must resolve to a public address';}+ return null;}-async function deliver(url, secret, eventId, timestamp, payload) {- const signature = await sign(secret, eventId, timestamp, payload);+async function deliver(url, secret, eventId, body) {+ const timestamp = Math.floor(Date.now() / 1000);+ const signature = await hmacSha256(secret, `${eventId}.${timestamp}.${body}`);// SSRF guard: the URL is developer-supplied, so resolve it and pin the// connection to the validated public address(es). Without this, a webhook@@ -153,11 +386,15 @@ async function deliver(url, secret, eventId, timestamp, payload) {// target and persist the status/error as a probing oracle. Redirects are NOT// followed (manual) so a public host can't bounce the request internally.let target;- let agent;+ let agent = null;try {- target = validatePublicUrl(url);- const addrs = await resolvePublicHost(target.hostname);- agent = pinnedAgent(target.hostname, addrs);+ if (privateTargetsAllowed()) {+ target = new URL(url);+ } else {+ target = validatePublicUrl(url, { allowHttp: false });+ const addrs = await resolvePublicHost(target.hostname);+ agent = pinnedAgent(target.hostname, addrs);+ }} catch (err) {const reason = err instanceof SsrfError ? `blocked_url:${err.code}` : 'invalid_url';return { statusCode: null, responseBody: null, error: reason };@@ -172,7 +409,7 @@ async function deliver(url, secret, eventId, timestamp, payload) {res = await fetch(target, {method: 'POST',redirect: 'manual',- dispatcher: agent,+ ...(agent ? { dispatcher: agent } : {}),headers: {'content-type': 'application/json','webhook-id': eventId,@@ -180,15 +417,15 @@ async function deliver(url, secret, eventId, timestamp, payload) {'webhook-signature': `v1,${signature}`,'user-agent': 'three.ws-webhooks/1.0',},- body: payload,+ body,signal: controller.signal,});} finally {clearTimeout(timeout);- await agent.close().catch(() => {});+ if (agent) await agent.close().catch(() => {});}- // A redirect is a misconfigured (or hostile) endpoint — record it as a+ // A redirect is a misconfigured (or hostile) endpoint: record it as a// failure instead of following it to a potentially internal target.if (res.status >= 300 && res.status < 400) {return { statusCode: res.status, responseBody: null, error: 'redirect_not_followed' };@@ -197,22 +434,14 @@ async function deliver(url, secret, eventId, timestamp, payload) {let responseBody = null;try {responseBody = await res.text();- if (responseBody.length > 1024) responseBody = responseBody.slice(0, 1024);+ if (responseBody.length > RESPONSE_BODY_MAX) responseBody = responseBody.slice(0, RESPONSE_BODY_MAX);} catch {- // Body read failed+ responseBody = null;}-- return { statusCode: res.status, responseBody, error: null };+ const error = res.status >= 200 && res.status < 300 ? null : `http_${res.status}`;+ return { statusCode: res.status, responseBody, error };} catch (err) {- return { statusCode: null, responseBody: null, error: err?.message || 'delivery_failed' };+ const error = err?.name === 'AbortError' ? 'timeout' : err?.cause?.code || err?.message || 'delivery_failed';+ return { statusCode: null, responseBody: null, error };}}--async function sign(secret, eventId, timestamp, payload) {- const message = `${eventId}.${timestamp}.${payload}`;- return hmacSha256(secret, message);-}--function sleep(ms) {- return new Promise((resolve) => setTimeout(resolve, ms));-}diff --git a/api/developer/webhooks.js b/api/developer/webhooks.jsindex a0fc3b0bd..fa61af015 100644--- a/api/developer/webhooks.js+++ b/api/developer/webhooks.js@@ -5,12 +5,9 @@ import { cors, error, json, method, readJson, wrap } from '../_lib/http.js';import { getSessionUser } from '../_lib/auth.js';import { requireCsrf } from '../_lib/csrf.js';import { sql } from '../_lib/db.js';-import { randomToken } from '../_lib/crypto.js';-import { EVENT_TYPES, selectEventTypes } from '../_lib/webhook-dispatch.js';-import { assertPublicHttpsUrl } from '../_lib/ssrf.js';+import { EVENT_TYPES, newWebhookSecret, selectEventTypes, webhookUrlProblem } from '../_lib/webhook-dispatch.js';const MAX_WEBHOOKS_PER_USER = 10;-const URL_MAX_LENGTH = 2048;export default wrap(async function handler(req, res) {if (cors(req, res, { methods: 'GET,POST,OPTIONS', credentials: true })) return;@@ -22,14 +19,18 @@ export default wrap(async function handler(req, res) {// One round trip for the list AND its 7-day delivery stats: the lateral// join keeps a 10-webhook dashboard at a single query instead of 1 + N.const rows = await sql`- select w.id, w.url, w.events, w.active, w.description, w.created_at, w.updated_at,- s.total, s.succeeded, s.failed, s.last_delivery_at+ select w.id, w.url, w.events, w.active, w.description, w.agent_id, w.disabled_reason,+ w.consecutive_failures, w.created_at, w.updated_at,+ s.total, s.succeeded, s.failed, s.pending, s.last_delivery_atfrom developer_webhooks wleft join lateral (selectcount(*)::int as total,- count(*) filter (where status_code between 200 and 299)::int as succeeded,- count(*) filter (where status_code is null or status_code >= 400)::int as failed,+ -- Queue rows (status set) count by their final state; legacy+ -- one-row-per-attempt rows (status null) count by HTTP code.+ count(*) filter (where status = 'succeeded' or (status is null and status_code between 200 and 299))::int as succeeded,+ count(*) filter (where status = 'failed' or (status is null and (status_code is null or status_code >= 400)))::int as failed,+ count(*) filter (where status in ('pending', 'delivering'))::int as pending,max(created_at) as last_delivery_atfrom webhook_deliveries dwhere d.webhook_id = w.id and d.created_at > now() - interval '7 days'@@ -38,9 +39,9 @@ export default wrap(async function handler(req, res) {order by w.created_at desc`;- const webhooks = rows.map(({ total, succeeded, failed, last_delivery_at, ...wh }) => ({+ const webhooks = rows.map(({ total, succeeded, failed, pending, last_delivery_at, ...wh }) => ({...wh,- stats_7d: { total, succeeded, failed, last_delivery_at },+ stats_7d: { total, succeeded, failed, pending, last_delivery_at },}));return json(res, 200, { webhooks, event_types: EVENT_TYPES });@@ -58,25 +59,10 @@ export default wrap(async function handler(req, res) {}const url = typeof body.url === 'string' ? body.url.trim() : '';- if (!url) return error(res, 400, 'bad_request', 'url is required');- if (url.length > URL_MAX_LENGTH)- return error(res, 400, 'bad_request', `url exceeds ${URL_MAX_LENGTH} characters`);-- try {- const parsed = new URL(url);- if (parsed.protocol !== 'https:') {- return error(res, 400, 'bad_request', 'Webhook URL must use HTTPS');- }- } catch {- return error(res, 400, 'bad_request', 'Invalid URL');- }- // Reject URLs that resolve to a private/loopback/link-local/metadata address- // at registration time (SSRF). Delivery re-validates and pins per attempt.- try {- await assertPublicHttpsUrl(url);- } catch {- return error(res, 400, 'bad_request', 'Webhook URL must resolve to a public address');- }+ // HTTPS to a public address (SSRF): checked here at registration, and again+ // with a pinned connection on every delivery attempt.+ const urlProblem = await webhookUrlProblem(url);+ if (urlProblem) return error(res, 400, 'bad_request', urlProblem);const selection = selectEventTypes(body.events);if (selection.error) return error(res, 400, 'bad_request', selection.error);@@ -96,7 +82,7 @@ export default wrap(async function handler(req, res) {);}- const secret = `whsec_${randomToken(24)}`;+ const secret = newWebhookSecret();const [webhook] = await sql`insert into developer_webhooks (user_id, url, secret, events, description)diff --git a/api/developer/webhooks/[id].js b/api/developer/webhooks/[id].jsindex d42f271c1..062e67e55 100644--- a/api/developer/webhooks/[id].js+++ b/api/developer/webhooks/[id].js@@ -6,12 +6,9 @@ import { cors, error, json, method, readJson, wrap } from '../../_lib/http.js';import { getSessionUser } from '../../_lib/auth.js';import { requireCsrf } from '../../_lib/csrf.js';import { sql } from '../../_lib/db.js';-import { selectEventTypes } from '../../_lib/webhook-dispatch.js';-import { assertPublicHttpsUrl } from '../../_lib/ssrf.js';+import { selectEventTypes, webhookUrlProblem } from '../../_lib/webhook-dispatch.js';import { isUuid } from '../../_lib/validate.js';-const URL_MAX_LENGTH = 2048;-export default wrap(async function handler(req, res) {if (cors(req, res, { methods: 'GET,PATCH,DELETE,OPTIONS', credentials: true })) return;@@ -36,7 +33,8 @@ export default wrap(async function handler(req, res) {if (req.method === 'GET') {const deliveries = await sql`- select id, event_type, event_id, status_code, error, attempt, created_at+ select id, event_type, event_id, status, status_code, error, attempt,+ next_attempt_at, delivered_at, duration_ms, replay_of, created_atfrom webhook_deliverieswhere webhook_id = ${id}order by created_at desc@@ -58,27 +56,10 @@ export default wrap(async function handler(req, res) {const updates = {};if (typeof body.url === 'string') {const trimmed = body.url.trim();- if (trimmed.length > URL_MAX_LENGTH)- return error(res, 400, 'bad_request', 'URL too long');- try {- const parsed = new URL(trimmed);- if (parsed.protocol !== 'https:')- return error(res, 400, 'bad_request', 'Must use HTTPS');- } catch {- return error(res, 400, 'bad_request', 'Invalid URL');- }- // SSRF: reject a URL that resolves to a non-public address (delivery- // also re-validates and pins per attempt).- try {- await assertPublicHttpsUrl(trimmed);- } catch {- return error(- res,- 400,- 'bad_request',- 'Webhook URL must resolve to a public address',- );- }+ // SSRF: HTTPS to a public address (delivery also re-validates and+ // pins per attempt).+ const urlProblem = await webhookUrlProblem(trimmed);+ if (urlProblem) return error(res, 400, 'bad_request', urlProblem);updates.url = trimmed;}if (body.events !== undefined) {@@ -89,6 +70,9 @@ export default wrap(async function handler(req, res) {if (typeof body.active === 'boolean') {updates.active = body.active;}+ // Re-enabling an endpoint the failure breaker switched off clears the+ // breaker, so it is not disabled again by the very next failure.+ const reenable = updates.active === true && !webhook.active;if (typeof body.description === 'string') {updates.description = body.description.trim().slice(0, 200);}@@ -103,6 +87,8 @@ export default wrap(async function handler(req, res) {events = ${updates.events ?? webhook.events},active = ${updates.active ?? webhook.active},description = ${updates.description !== undefined ? updates.description : webhook.description},+ consecutive_failures = case when ${reenable} then 0 else consecutive_failures end,+ disabled_reason = case when ${reenable} then null else disabled_reason end,updated_at = now()where id = ${id} and user_id = ${user.id}returning id, url, events, active, description, created_at, updated_at