Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(agents): hash-chain every run step into a verifiable receipt and deliver developer webhooks from a durable retry queue

nirholas03ae7cf0d54f
diff --git a/api/_lib/agents-v1/automations.js b/api/_lib/agents-v1/automations.js
index f7083b7c4..e8431e245 100644
--- a/api/_lib/agents-v1/automations.js
+++ b/api/_lib/agents-v1/automations.js
@@ -15,6 +15,7 @@
// object (source 'wallet_intent' / 'alert_rule') and can be deleted through it.
// A stopped agent (agent_identities.status) never fires.
+import { createHash } from 'node:crypto';
import { CronExpressionParser } from 'cron-parser';
import { sql } from '../db.js';
import { apiError } from './http.js';
@@ -22,8 +23,10 @@ import {
createIntent,
deleteIntent,
fireIntentForAutomation,
+ getIntent,
listIntents,
normalizeIntent,
+ updateIntent,
} from '../wallet-intents.js';
import { fetchTokenPriceUsd } from '../market/token-market.js';
import { getSolanaAddressBalances } from '../agent-wallet.js';
@@ -31,6 +34,7 @@ import { recentPumpLaunches } from '../pump-launch-feed.js';
import { fetchRecentTrades } from '../pump-alert-runner.js';
import { solPriceUsd } from '../sol-price.js';
import { resolveSolanaRecipient } from '../../../src/solana/sns.js';
+import { dispatchWebhooks } from '../webhook-dispatch.js';
export const TRIGGER_TYPES = Object.freeze([
'price_threshold',
@@ -331,12 +335,7 @@ export async function createAutomation({ agent, userId, body, source = 'api' })
const [{ n }] = await sql`SELECT count(*)::int AS n FROM agent_automations WHERE agent_id = ${agent.id}`;
if (n >= MAX_PER_AGENT) throw apiError(409, 'automation_limit', `An agent can hold at most ${MAX_PER_AGENT} automations.`);
- if (a.action.type === 'transfer') {
- const dest = await resolveSolanaRecipient(a.action.destination).catch(() => null);
- const address = dest?.address || (BASE58_RE.test(a.action.destination) ? a.action.destination : null);
- if (!address) throw bad('invalid_parameter', 'action.destination must be a Solana address or .sol name.', { parameter: 'action.destination' });
- a.action.destination = address;
- }
+ await resolveTransferDestination(a);
let intentId = null;
if (a.action.type !== 'agent_prompt') {
@@ -419,6 +418,185 @@ export async function deleteAutomation(userId, id) {
throw apiError(404, 'not_found', 'No automation with that id.');
}
+/** Resolve a transfer destination (.sol name or address) in place, or throw. */
+async function resolveTransferDestination(a) {
+ if (a.action.type !== 'transfer') return;
+ const dest = await resolveSolanaRecipient(a.action.destination).catch(() => null);
+ const address = dest?.address || (BASE58_RE.test(a.action.destination) ? a.action.destination : null);
+ if (!address) throw bad('invalid_parameter', 'action.destination must be a Solana address or .sol name.', { parameter: 'action.destination' });
+ a.action.destination = address;
+}
+
+async function ownedAutomationRow(userId, id) {
+ const [row] = await sql`SELECT * FROM agent_automations WHERE id = ${id} AND user_id = ${userId} LIMIT 1`;
+ return row || null;
+}
+
+function limitsFromIntent(intent) {
+ const l = intent?.limits || {};
+ const out = {};
+ if (l.per_action_usd != null) out.perActionUsd = Number(l.per_action_usd);
+ if (l.daily_usd != null) out.dailyUsd = Number(l.daily_usd);
+ if (l.total_usd != null) out.totalUsd = Number(l.total_usd);
+ return out;
+}
+
+/**
+ * One automation the caller owns, with its spend limits and (for agent_prompt
+ * automations) the runs it started most recently. Wallet intents and alert
+ * rules resolve too, so every id automation_list returns can be read back.
+ */
+export async function getAutomation(userId, id) {
+ const row = await ownedAutomationRow(userId, id);
+ if (row) {
+ const [intent, runs] = await Promise.all([
+ row.intent_id ? getIntent(row.agent_id, row.intent_id) : null,
+ row.action_type === 'agent_prompt'
+ ? sql`
+ SELECT id, status, summary, created_at, finished_at FROM agent_runs
+ WHERE automation_id = ${row.id} ORDER BY created_at DESC LIMIT 10
+ `
+ : [],
+ ]);
+ return {
+ ...serializeAutomation(row),
+ limits: limitsFromIntent(intent),
+ nextFireAt: row.trigger_type === 'schedule' && row.enabled ? nextScheduledAt(row).toISOString() : null,
+ recentRuns: runs.map((r) => ({ id: r.id, status: r.status, summary: r.summary, createdAt: r.created_at, finishedAt: r.finished_at })),
+ };
+ }
+ const [w] = await sql`
+ SELECT w.id, w.agent_id FROM agent_wallet_intents w
+ JOIN agent_identities i ON i.id = w.agent_id
+ WHERE w.id = ${id} AND i.user_id = ${userId}
+ `;
+ if (w) {
+ const intent = await getIntent(w.agent_id, w.id);
+ return { ...serializeIntent(intent), limits: limitsFromIntent(intent), nextFireAt: null, recentRuns: [] };
+ }
+ const [rule] = await sql`
+ SELECT r.*, f.last_fired_at FROM pump_alert_rules r
+ LEFT JOIN pump_alert_rule_fires f ON f.rule_id = r.id
+ WHERE r.id = ${id} AND r.user_id = ${userId}
+ `;
+ if (rule) return { ...serializeAlertRule(rule), limits: {}, nextFireAt: null, recentRuns: [] };
+ throw apiError(404, 'not_found', 'No automation with that id.');
+}
+
+/** Merge a partial trigger/action patch over the stored config; a new type replaces it. */
+function mergePart(stored, patch) {
+ if (patch == null) return stored;
+ if (typeof patch !== 'object') return patch;
+ if (patch.type && patch.type !== stored?.type) return patch;
+ return { ...stored, ...patch };
+}
+
+/**
+ * Update an automation the caller owns. The merged result is re-validated by
+ * normalizeAutomation, the same validator create uses, so a patch can never
+ * store a config create would refuse. Spend automations need `confirm: true`
+ * on every change except switching them off.
+ *
+ * @param {{ userId: string, id: string, body: object }} o
+ */
+export async function updateAutomation({ userId, id, body }) {
+ const patch = body && typeof body === 'object' ? body : {};
+ const row = await ownedAutomationRow(userId, id);
+ if (!row) {
+ await getAutomation(userId, id);
+ throw apiError(409, 'not_editable', 'Only automations created as automations can be edited here. Edit wallet rules from the agent wallet page and alert rules from /alerts, or delete and recreate them as an automation.');
+ }
+ const intent = row.intent_id ? await getIntent(row.agent_id, row.intent_id) : null;
+ const storedTrigger = row.trigger_config;
+ const storedAction = row.action_config;
+ const customTitle = row.title !== defaultTitle(storedTrigger, storedAction);
+ const raw = {
+ title: typeof patch.title === 'string' ? patch.title : customTitle ? row.title : undefined,
+ trigger: mergePart(storedTrigger, patch.trigger),
+ action: mergePart(storedAction, patch.action),
+ triggerOnce: typeof patch.triggerOnce === 'boolean' ? patch.triggerOnce : row.trigger_once,
+ limits: patch.limits && typeof patch.limits === 'object' ? { ...limitsFromIntent(intent), ...patch.limits } : limitsFromIntent(intent),
+ };
+ const a = normalizeAutomation(raw);
+ const enabled = typeof patch.enabled === 'boolean' ? patch.enabled : row.enabled;
+ const onlyDisabling = enabled === false && ['trigger', 'action', 'limits', 'title', 'triggerOnce'].every((k) => patch[k] === undefined);
+ if (SPEND_ACTIONS.has(a.action.type) && !onlyDisabling && patch.confirm !== true) {
+ throw apiError(400, 'confirmation_required', `This ${a.action.type} automation spends from the agent wallet on its own. Resend with "confirm": true to authorize the change.`, {
+ action: a.action,
+ trigger: a.trigger,
+ limits: a.limits,
+ });
+ }
+ await resolveTransferDestination(a);
+
+ let intentId = row.intent_id;
+ if (a.action.type === 'agent_prompt') {
+ if (intentId) await deleteIntent(row.agent_id, userId, intentId);
+ intentId = null;
+ } else {
+ const norm = normalizeIntent(backingIntentPayload(a, a.title));
+ if (!norm.ok) throw bad(norm.error, norm.message);
+ if (intentId && intent) {
+ const updated = await updateIntent(row.agent_id, userId, intentId, { intent: norm.intent, enabled });
+ if (updated?.error) throw bad(updated.error, updated.message);
+ } else {
+ const created = await createIntent(row.agent_id, userId, norm.intent, { sourceText: `v1 automation: ${a.title}` });
+ intentId = created.id;
+ if (!enabled) await updateIntent(row.agent_id, userId, intentId, { enabled: false });
+ }
+ }
+ const triggerChanged = canonicalTrigger(storedTrigger) !== canonicalTrigger(a.trigger);
+ const [next] = await sql`
+ UPDATE agent_automations SET
+ title = ${a.title},
+ trigger_type = ${a.trigger.type},
+ trigger_config = ${JSON.stringify(a.trigger)}::jsonb,
+ action_type = ${a.action.type},
+ action_config = ${JSON.stringify(a.action)}::jsonb,
+ trigger_once = ${a.triggerOnce},
+ enabled = ${enabled},
+ intent_id = ${intentId},
+ state = CASE WHEN ${triggerChanged} THEN NULL ELSE state END,
+ updated_at = now()
+ WHERE id = ${row.id}
+ RETURNING *
+ `;
+ return { ...serializeAutomation(next), limits: a.limits };
+}
+
+function canonicalTrigger(t) {
+ return JSON.stringify(Object.keys(t || {}).sort().map((k) => [k, t[k]]));
+}
+
+/**
+ * Fire an automation's action right now, outside its trigger. The fire key is
+ * unique per call, so the action runs once per request; it still passes every
+ * guard a triggered fire does (stopped agent, spend policy, intent caps).
+ * Spend actions need `confirm: true`.
+ */
+export async function triggerAutomation({ userId, id, confirm = false }) {
+ const row = await ownedAutomationRow(userId, id);
+ if (!row) {
+ await getAutomation(userId, id);
+ throw apiError(409, 'not_triggerable', 'Only automations created as automations can be fired on demand.');
+ }
+ if (!row.enabled) throw apiError(409, 'automation_disabled', 'This automation is switched off. Enable it first, then trigger it.');
+ const [agent] = await sql`SELECT status, deleted_at FROM agent_identities WHERE id = ${row.agent_id} LIMIT 1`;
+ if (!agent || agent.deleted_at) throw apiError(404, 'agent_not_found', 'Agent not found.');
+ if (agent.status === 'stopped') throw apiError(409, 'agent_stopped', 'This agent is stopped. Start it before triggering its automations.');
+ if (row.action_config?.mintFromEvent && !row.action_config?.mint) {
+ throw apiError(409, 'needs_event', 'This swap buys the coin its trigger event names, so it can only run when that event happens.');
+ }
+ if (SPEND_ACTIONS.has(row.action_type) && confirm !== true) {
+ throw apiError(400, 'confirmation_required', `Triggering this ${row.action_type} automation spends from the agent wallet now. Resend with "confirm": true to authorize it.`, {
+ action: row.action_config,
+ });
+ }
+ const result = await fireAutomation(row, { manual: true }, `manual:${Date.now()}`);
+ const [fresh] = await sql`SELECT * FROM agent_automations WHERE id = ${row.id}`;
+ return { result, automation: serializeAutomation(fresh) };
+}
+
// ── evaluation ───────────────────────────────────────────────────────────────
function cooledDown(row, now) {
@@ -474,6 +652,33 @@ function eventSummary(event) {
* the same (automation, key) never executes twice.
*/
export async function fireAutomation(row, event, key) {
+ const outcome = await runAutomationAction(row, event, key);
+ if (outcome.fired) {
+ // Keyed on the fire's own discriminator, so a sweep that retries the
+ // same trigger delivers automation.fired once.
+ await dispatchWebhooks({
+ userId: row.user_id,
+ agentId: row.agent_id,
+ eventType: 'automation.fired',
+ eventId: `evt_auto_${createHash('sha256').update(`${row.id}:${key}`).digest('hex').slice(0, 32)}`,
+ data: {
+ automation_id: row.id,
+ agent_id: row.agent_id,
+ title: row.title || null,
+ trigger: { type: row.trigger_type, key },
+ action: { type: row.action_type },
+ status: outcome.status,
+ run_id: outcome.runId || null,
+ signature: outcome.signature || null,
+ note: outcome.note || null,
+ event: event && typeof event === 'object' ? event : null,
+ },
+ });
+ }
+ return outcome;
+}
+
+async function runAutomationAction(row, event, key) {
const discriminator = `auto:${row.id}:${key}`;
if (row.action_type === 'agent_prompt') {
const { createRun } = await import('./runs.js');
diff --git a/api/_lib/agents-v1/run-receipts.js b/api/_lib/agents-v1/run-receipts.js
new file mode 100644
index 000000000..8f1f831d3
--- /dev/null
+++ b/api/_lib/agents-v1/run-receipts.js
@@ -0,0 +1,145 @@
+// Run receipts and summaries: the audit layer over agent_run_steps.
+//
+// Every step a run writes carries a receipt, a sha256 digest chained to the
+// step before it (migration 20261010163000_agent_run_receipts.sql). Anyone
+// holding the step rows can recompute the chain with verifyReceiptChain and
+// prove that no tool call or tool result was edited, dropped or reordered.
+//
+// JSON is hashed in canonical form (object keys sorted at every depth) because
+// jsonb does not preserve key order: the bytes we hash at insert time must be
+// reproducible from what the database hands back later.
+
+import { createHash } from 'node:crypto';
+
+/** JSON with object keys sorted at every depth; null for undefined. */
+export function canonicalJson(value) {
+ if (value === undefined || value === null) return 'null';
+ if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`;
+ if (typeof value === 'object') {
+ const keys = Object.keys(value).filter((k) => value[k] !== undefined).sort();
+ return `{${keys.map((k) => `${JSON.stringify(k)}:${canonicalJson(value[k])}`).join(',')}}`;
+ }
+ return JSON.stringify(value);
+}
+
+/**
+ * The receipt for one step, chained to the receipt of the step before it.
+ * @param {{ prev: string|null, runId: string, seq: number, kind: string, tool?: string|null, input?: unknown, output?: unknown }} s
+ */
+export function stepReceipt({ prev, runId, seq, kind, tool = null, input = null, output = null }) {
+ return createHash('sha256')
+ .update([prev || '', runId, String(seq), kind, tool || '', canonicalJson(input), canonicalJson(output)].join('|'))
+ .digest('hex');
+}
+
+/**
+ * Recompute a run's receipt chain from its serialized steps (seq ascending).
+ * Steps written before receipts existed (receipt null) are skipped until the
+ * chain starts. Returns { verified, checked, brokenAt }.
+ */
+export function verifyReceiptChain(runId, steps) {
+ let prev = null;
+ let checked = 0;
+ for (const s of steps) {
+ if (!s.receipt) {
+ if (checked) return { verified: false, checked, brokenAt: s.seq };
+ continue;
+ }
+ const expected = stepReceipt({ prev, runId, seq: s.seq, kind: s.kind, tool: s.tool, input: s.input, output: s.output });
+ if (expected !== s.receipt) return { verified: false, checked, brokenAt: s.seq };
+ prev = s.receipt;
+ checked += 1;
+ }
+ return { verified: checked > 0, checked, brokenAt: null };
+}
+
+/**
+ * Pair every tool call with its result (or its block) into one trace entry,
+ * in call order. The receipt on each entry is the result's receipt, which
+ * chains over the call before it, so one digest vouches for both halves.
+ */
+export function toolTraces(steps) {
+ const traces = [];
+ const open = [];
+ for (const s of steps) {
+ if (s.kind === 'tool_call') {
+ const t = { tool: s.tool, input: s.input, callSeq: s.seq, callReceipt: s.receipt ?? null, status: 'pending', output: null, resultSeq: null, receipt: s.receipt ?? null, latencyMs: null, at: s.at };
+ traces.push(t);
+ open.push(t);
+ } else if (s.kind === 'tool_result' || s.kind === 'tool_blocked') {
+ const i = open.findIndex((t) => t.tool === s.tool);
+ const t = i >= 0 ? open.splice(i, 1)[0] : { tool: s.tool, input: s.input, callSeq: null, callReceipt: null, at: s.at };
+ if (i < 0) traces.push(t);
+ const failed = s.kind === 'tool_result' && s.output && typeof s.output === 'object' && 'error' in s.output && Object.keys(s.output).length === 1;
+ t.status = s.kind === 'tool_blocked' ? 'blocked' : failed ? 'error' : 'ok';
+ t.output = s.output;
+ t.resultSeq = s.seq;
+ t.receipt = s.receipt ?? null;
+ t.latencyMs = s.latencyMs ?? null;
+ }
+ }
+ return traces;
+}
+
+const STATUS_LEAD = {
+ completed: 'Completed',
+ failed: 'Failed',
+ cancelled: 'Cancelled',
+ budget_exhausted: 'Stopped at its budget',
+};
+
+function money(n) {
+ const v = Number(n) || 0;
+ return v === 0 ? '$0' : v < 0.01 ? `$${v.toFixed(4)}` : `$${v.toFixed(2)}`;
+}
+
+function duration(ms) {
+ if (!Number.isFinite(ms) || ms < 0) return null;
+ const s = Math.round(ms / 1000);
+ if (s < 60) return `${s}s`;
+ const m = Math.floor(s / 60);
+ if (m < 60) return `${m}m ${s % 60}s`;
+ return `${Math.floor(m / 60)}h ${m % 60}m`;
+}
+
+function excerpt(text, n = 280) {
+ const t = String(text || '').replace(/\s+/g, ' ').trim();
+ return t.length > n ? `${t.slice(0, n - 1)}…` : t;
+}
+
+/**
+ * A plain-language account of how a run ended.
+ * @param {object} run the agent_runs row after finalize
+ * @param {Array<{ kind: string, tool_name: string|null, n: number, failed: number }>} tally step counts by kind and tool
+ */
+export function summarizeRun(run, tally) {
+ const lead = STATUS_LEAD[run.status] || run.status;
+ const turns = tally.filter((t) => t.kind === 'model_call').reduce((a, t) => a + Number(t.n), 0);
+ const byTool = new Map();
+ let failedCalls = 0;
+ let blocked = 0;
+ for (const t of tally) {
+ if (t.kind === 'tool_call' && t.tool_name) byTool.set(t.tool_name, (byTool.get(t.tool_name) || 0) + Number(t.n));
+ if (t.kind === 'tool_result') failedCalls += Number(t.failed || 0);
+ if (t.kind === 'tool_blocked') blocked += Number(t.n);
+ }
+ const started = run.started_at || run.created_at;
+ const took = started && run.finished_at ? duration(new Date(run.finished_at) - new Date(started)) : null;
+
+ const parts = [];
+ parts.push(`${lead} after ${turns} model turn${turns === 1 ? '' : 's'}${took ? ` in ${took}` : ''}.`);
+ const calls = [...byTool.values()].reduce((a, n) => a + n, 0);
+ if (calls) {
+ const list = [...byTool.entries()].sort((a, b) => b[1] - a[1]).map(([name, n]) => (n > 1 ? `${name} x${n}` : name)).join(', ');
+ const extra = [failedCalls ? `${failedCalls} failed` : null, blocked ? `${blocked} blocked` : null].filter(Boolean).join(', ');
+ parts.push(`Made ${calls} tool call${calls === 1 ? '' : 's'}: ${list}${extra ? ` (${extra})` : ''}.`);
+ } else {
+ parts.push('Made no tool calls.');
+ }
+ const budget = Number(run.budget_credits_usd) || 0;
+ const spent = Number(run.spent_credits_usd) || 0;
+ parts.push(budget > 0 ? `Spent ${money(spent)} of a ${money(budget)} budget.` : 'Ran on free model lanes only, with no credit spend.');
+ if (run.status === 'completed' && run.result) parts.push(`Result: ${excerpt(run.result)}`);
+ else if (run.error) parts.push(`Reason: ${excerpt(run.error)}`);
+ return parts.join(' ');
+}
diff --git a/api/_lib/agents-v1/runs.js b/api/_lib/agents-v1/runs.js
index 16e774cea..ed2897e58 100644
--- a/api/_lib/agents-v1/runs.js
+++ b/api/_lib/agents-v1/runs.js
@@ -27,6 +27,8 @@ import { agentToolSchemas, agentToolHandlers } from '../agent-tools.js';
import { debitCredits } from '../credits.js';
import { assertAgentBudget, InferenceBillingError } from '../inference-billing.js';
import { AGENT_SYSTEM_NOTE, createAgentLoop, finalAnswer, initialLoopState, loopFinished } from '../agent-loop.js';
+import { stepReceipt, summarizeRun } from './run-receipts.js';
+import { dispatchWebhooks } from '../webhook-dispatch.js';
const LEASE_SECONDS = 120;
const TERMINAL = new Set(['completed', 'failed', 'cancelled', 'budget_exhausted']);
@@ -61,6 +63,7 @@ export function serializeRun(r) {
scheduledFor: r.scheduled_for,
result: r.result,
error: r.error,
+ summary: r.summary ?? null,
source: r.source,
automationId: r.automation_id,
cancelRequested: Boolean(r.cancel_requested_at),
@@ -83,6 +86,7 @@ export function serializeStep(s) {
usage: s.input_tokens == null && s.output_tokens == null ? null : { input: s.input_tokens, output: s.output_tokens },
costUsd: s.cost_micro_usd == null ? null : Number(s.cost_micro_usd) / 1e6,
latencyMs: s.latency_ms,
+ receipt: s.receipt ?? null,
at: s.created_at,
};
}
@@ -220,17 +224,28 @@ async function lastSeq(runId) {
async function insertSteps(runId, fromSeq, events) {
let seq = fromSeq;
+ let prev = null;
+ if (fromSeq > 0) {
+ const [p] = await sql`SELECT receipt FROM agent_run_steps WHERE run_id = ${runId} AND seq = ${fromSeq} LIMIT 1`;
+ prev = p?.receipt || null;
+ }
for (const e of events) {
seq += 1;
+ // Hash exactly what jsonb will hand back: a JSON round trip drops
+ // undefined and turns Dates into strings before the digest is taken.
+ const input = e.input == null ? null : JSON.parse(JSON.stringify(e.input));
+ const output = e.output == null ? null : JSON.parse(JSON.stringify(e.output));
+ const receipt = stepReceipt({ prev, runId, seq, kind: e.kind, tool: e.tool || null, input, output });
await sql`
INSERT INTO agent_run_steps
- (run_id, seq, kind, provider, model, tool_name, input, output, input_tokens, output_tokens, cost_micro_usd, latency_ms)
+ (run_id, seq, kind, provider, model, tool_name, input, output, input_tokens, output_tokens, cost_micro_usd, latency_ms, receipt)
VALUES
(${runId}, ${seq}, ${e.kind}, ${e.provider || null}, ${e.model || null}, ${e.tool || null},
- ${e.input == null ? null : JSON.stringify(e.input)}::jsonb, ${e.output == null ? null : JSON.stringify(e.output)}::jsonb,
- ${e.inputTokens ?? null}, ${e.outputTokens ?? null}, ${e.costMicroUsd ?? null}, ${e.latencyMs ?? null})
+ ${input == null ? null : JSON.stringify(input)}::jsonb, ${output == null ? null : JSON.stringify(output)}::jsonb,
+ ${e.inputTokens ?? null}, ${e.outputTokens ?? null}, ${e.costMicroUsd ?? null}, ${e.latencyMs ?? null}, ${receipt})
ON CONFLICT (run_id, seq) DO NOTHING
`;
+ prev = receipt;
}
return seq;
}
@@ -245,7 +260,24 @@ async function finalize(run, status, { result = null, error = null, note = null
await insertSteps(run.id, await lastSeq(run.id), [
{ kind: status === 'completed' ? 'final' : status === 'failed' ? 'error' : 'status', output: { status, result, error, note } },
]);
- return row;
+ const tally = await sql`
+ SELECT kind, tool_name, count(*)::int AS n,
+ count(*) FILTER (WHERE kind = 'tool_result' AND output ? 'error' AND jsonb_typeof(output) = 'object')::int AS failed
+ FROM agent_run_steps WHERE run_id = ${run.id}
+ GROUP BY kind, tool_name
+ `;
+ const summary = summarizeRun(row, tally);
+ const [done] = await sql`UPDATE agent_runs SET summary = ${summary} WHERE id = ${run.id} RETURNING *`;
+ // The event ID is derived from the run, so a finalize that races another
+ // still delivers run.finished once per endpoint.
+ await dispatchWebhooks({
+ userId: done.user_id,
+ agentId: done.agent_id,
+ eventType: 'run.finished',
+ eventId: `evt_run_finished_${done.id}`,
+ data: serializeRun(done),
+ });
+ return done;
}
async function acquireLease(runId, owner) {
diff --git a/api/_lib/migrations/20261010163000_agent_run_receipts.sql b/api/_lib/migrations/20261010163000_agent_run_receipts.sql
new file mode 100644
index 000000000..5c29758d0
--- /dev/null
+++ b/api/_lib/migrations/20261010163000_agent_run_receipts.sql
@@ -0,0 +1,20 @@
+-- Migration: agent run receipts and end-of-run summaries.
+-- Apply: node scripts/apply-migrations.mjs --apply --file 20261010163000_agent_run_receipts.sql
+-- Idempotent.
+--
+-- agent_run_steps.receipt
+-- A sha256 hex digest that chains every step of a run to the one before it:
+-- sha256(prev_receipt | run_id | seq | kind | tool | input | output). A replay
+-- or an MCP client can recompute the chain from the step rows and prove that
+-- no tool call or tool result was edited, dropped or reordered after the
+-- fact. Rows written before this migration keep a null receipt; the chain
+-- starts at the first step written after it.
+--
+-- agent_runs.summary
+-- A plain-language account of how a run ended (status, turns, tool calls by
+-- name, spend against budget, duration and the result or error), written once
+-- when the run reaches a terminal status. The agent page replay view and the
+-- MCP run tools show it verbatim.
+
+alter table agent_run_steps add column if not exists receipt text;
+alter table agent_runs add column if not exists summary text;
diff --git a/api/_lib/migrations/20261010170000_agent_webhook_queue.sql b/api/_lib/migrations/20261010170000_agent_webhook_queue.sql
new file mode 100644
index 000000000..539ee73b9
--- /dev/null
+++ b/api/_lib/migrations/20261010170000_agent_webhook_queue.sql
@@ -0,0 +1,65 @@
+-- Durable webhook delivery for the v1 agents API.
+--
+-- Developer webhooks used to retry three times inside the request that raised
+-- the event (1s, 4s, 16s), writing one webhook_deliveries row per attempt. A
+-- receiver that was down for a minute lost the event for good, and a process
+-- that recycled mid-retry dropped it silently.
+--
+-- From here on a delivery is ONE row per (endpoint, event) that moves through
+-- pending -> delivering -> succeeded | failed. Every HTTP attempt is appended
+-- to `attempts`. A failed attempt reschedules the row with backoff in
+-- `next_attempt_at`; the every-minute cron (api/cron/webhook-deliveries.js)
+-- picks up whatever is due, and `lease_until` keeps two workers from posting
+-- the same row at once. A replay is a fresh row pointing at its original
+-- through `replay_of`. Rows written before this migration keep status NULL
+-- and still read as one-row-per-attempt history.
+--
+-- Service layer: api/_lib/webhook-dispatch.js. Routes: /api/v1/webhooks
+-- (api/v1/rest.js) and /api/developer/webhooks.
+
+-- An endpoint can be narrowed to one agent: agent-scoped events (run.finished,
+-- automation.fired, message.received, approval.needed) for any other agent are
+-- not delivered to it. NULL means every agent the account owns.
+ALTER TABLE developer_webhooks ADD COLUMN IF NOT EXISTS agent_id uuid REFERENCES agent_identities(id) ON DELETE CASCADE;
+ALTER TABLE developer_webhooks ADD COLUMN IF NOT EXISTS consecutive_failures int NOT NULL DEFAULT 0;
+ALTER TABLE developer_webhooks ADD COLUMN IF NOT EXISTS disabled_reason text;
+ALTER TABLE developer_webhooks ADD COLUMN IF NOT EXISTS last_success_at timestamptz;
+ALTER TABLE developer_webhooks ADD COLUMN IF NOT EXISTS last_failure_at timestamptz;
+
+CREATE INDEX IF NOT EXISTS developer_webhooks_user_created
+ ON developer_webhooks (user_id, created_at DESC, id DESC);
+
+ALTER TABLE webhook_deliveries ADD COLUMN IF NOT EXISTS status text;
+ALTER TABLE webhook_deliveries ADD COLUMN IF NOT EXISTS next_attempt_at timestamptz;
+ALTER TABLE webhook_deliveries ADD COLUMN IF NOT EXISTS lease_until timestamptz;
+ALTER TABLE webhook_deliveries ADD COLUMN IF NOT EXISTS attempts jsonb NOT NULL DEFAULT '[]'::jsonb;
+ALTER TABLE webhook_deliveries ADD COLUMN IF NOT EXISTS delivered_at timestamptz;
+ALTER TABLE webhook_deliveries ADD COLUMN IF NOT EXISTS duration_ms int;
+ALTER TABLE webhook_deliveries ADD COLUMN IF NOT EXISTS replay_of uuid REFERENCES webhook_deliveries(id) ON DELETE SET NULL;
+ALTER TABLE webhook_deliveries ADD COLUMN IF NOT EXISTS updated_at timestamptz NOT NULL DEFAULT now();
+
+DO $$ BEGIN
+ ALTER TABLE webhook_deliveries ADD CONSTRAINT webhook_deliveries_status_check
+ CHECK (status IS NULL OR status IN ('pending', 'delivering', 'succeeded', 'failed'));
+EXCEPTION WHEN duplicate_object THEN NULL; END $$;
+
+-- One original delivery per endpoint per event: an event raised twice (a
+-- retried cron, a re-polled approval) can never post twice. Replays and the
+-- legacy per-attempt rows are outside the constraint.
+CREATE UNIQUE INDEX IF NOT EXISTS webhook_deliveries_event_once
+ ON webhook_deliveries (webhook_id, event_id)
+ WHERE status IS NOT NULL AND replay_of IS NULL;
+
+-- The retry sweep reads only what is due.
+CREATE INDEX IF NOT EXISTS webhook_deliveries_due
+ ON webhook_deliveries (next_attempt_at)
+ WHERE status = 'pending';
+
+-- A worker that died holding a lease is found by this.
+CREATE INDEX IF NOT EXISTS webhook_deliveries_leased
+ ON webhook_deliveries (lease_until)
+ WHERE status = 'delivering';
+
+-- The delivery log pages newest first by (created_at, id).
+CREATE INDEX IF NOT EXISTS webhook_deliveries_webhook_page
+ ON webhook_deliveries (webhook_id, created_at DESC, id DESC);
diff --git a/api/_lib/webhook-dispatch.js b/api/_lib/webhook-dispatch.js
index 8451ffd13..3dfdbebdb 100644
--- a/api/_lib/webhook-dispatch.js
+++ b/api/_lib/webhook-dispatch.js
@@ -1,20 +1,49 @@
-// Outgoing webhook dispatch — delivers events to developer-registered endpoints.
+// Outgoing webhooks: a durable, signed, retried delivery queue for
+// developer-registered endpoints.
//
-// Uses the Standard Webhooks signature format:
-// webhook-id: unique event ID
-// webhook-timestamp: unix epoch seconds
-// webhook-signature: v1,{base64 HMAC-SHA256}
+// Signature (Standard Webhooks format):
+// webhook-id: the event ID, stable across retries and replays
+// webhook-timestamp: unix epoch seconds of THIS attempt
+// webhook-signature: v1,{base64url HMAC-SHA256 of `${id}.${timestamp}.${body}`}
//
-// Retry: 3 attempts with exponential backoff (1s, 4s, 16s).
-// Fire-and-forget — callers don't await delivery results.
+// Delivery model (migration 20261010170000_agent_webhook_queue.sql):
+// • Raising an event writes one webhook_deliveries row per matching endpoint
+// (status 'pending'), then attempts each one immediately, fire-and-forget.
+// Callers never wait on a receiver.
+// • A failed attempt is appended to the row's `attempts` log and the row is
+// rescheduled with backoff (RETRY_SCHEDULE_MS). The every-minute cron
+// (api/cron/webhook-deliveries.js) runs `sweepWebhookDeliveries`, which
+// posts whatever is due, so a retry survives a recycled process.
+// • A row is claimed with a lease before it is posted, so the inline attempt
+// and the sweep can never deliver the same row concurrently.
+// • An endpoint that keeps failing is switched off (DISABLE_AFTER_FAILURES).
+// • `replayDelivery` re-sends a past delivery as a new row linked through
+// `replay_of`, with the original event ID so receivers can deduplicate.
import { sql } from './db.js';
import { randomToken, hmacSha256 } from './crypto.js';
import { validatePublicUrl, resolvePublicHost, pinnedAgent, SsrfError } from './ssrf.js';
-const MAX_ATTEMPTS = 3;
-const BACKOFF_BASE_MS = 1000;
const DELIVERY_TIMEOUT_MS = 10_000;
+const LEASE_SECONDS = 30;
+const RESPONSE_BODY_MAX = 1024;
+
+// Delay before attempt n+1, after attempt n failed. Eight attempts spread over
+// roughly twelve hours, then the delivery is marked failed and can be replayed.
+export const RETRY_SCHEDULE_MS = Object.freeze([
+ 30_000, // 30 s
+ 2 * 60_000, // 2 min
+ 10 * 60_000, // 10 min
+ 30 * 60_000, // 30 min
+ 60 * 60_000, // 1 h
+ 3 * 60 * 60_000, // 3 h
+ 8 * 60 * 60_000, // 8 h
+]);
+export const MAX_ATTEMPTS = RETRY_SCHEDULE_MS.length + 1;
+
+// Failed attempts in a row, with no success in the last 24 hours, before an
+// endpoint is set inactive. Re-enable with PATCH { active: true }.
+export const DISABLE_AFTER_FAILURES = 50;
const EVENT_TYPES = [
'avatar.created',
@@ -34,8 +63,18 @@ const EVENT_TYPES = [
// all deliver the same event exactly once.
'forge.completed',
'forge.failed',
+ // Agent activity (the v1 agents API, docs/api-reference.md). Each carries
+ // `agent_id`, so an endpoint narrowed to one agent only hears that agent.
+ 'run.finished', // a run reached completed, failed, cancelled or budget_exhausted
+ 'automation.fired', // an automation's trigger matched and its action ran
+ 'message.received', // a user message was added to an agent's thread
+ 'approval.needed', // an agent action is waiting for the owner's approval
];
+// Sent only by POST /api/v1/webhooks/:id/test, to that one endpoint, whatever
+// it subscribes to. Not subscribable.
+export const TEST_EVENT_TYPE = 'webhook.test';
+
export { EVENT_TYPES };
/**
@@ -67,84 +106,278 @@ export function selectEventTypes(raw) {
return { events: events.length ? events : [...EVENT_TYPES] };
}
-export async function dispatchWebhooks({ userId, eventType, data }) {
- if (!EVENT_TYPES.includes(eventType)) return;
+/**
+ * Loopback and plain-http receivers are refused in production. Outside it, the
+ * operator can opt in with WEBHOOKS_ALLOW_PRIVATE_TARGETS=1 to point an
+ * endpoint at a receiver on their own machine.
+ */
+export function privateTargetsAllowed() {
+ return process.env.NODE_ENV !== 'production' && process.env.WEBHOOKS_ALLOW_PRIVATE_TARGETS === '1';
+}
+
+/** A fresh signing secret. */
+export function newWebhookSecret() {
+ return `whsec_${randomToken(24)}`;
+}
- let webhooks;
+/** A fresh event ID. */
+export function newEventId() {
+ return `evt_${randomToken(16)}`;
+}
+
+function envelope(eventId, eventType, data) {
+ return { id: eventId, type: eventType, created_at: new Date().toISOString(), data };
+}
+
+/**
+ * Raise an event for every matching endpoint the user owns.
+ *
+ * Resolves once the delivery rows are written; the HTTP attempts run in the
+ * background. Never throws: a webhook outage must not fail the caller's work.
+ *
+ * @param {object} o
+ * @param {string} o.userId
+ * @param {string} o.eventType one of EVENT_TYPES
+ * @param {object} o.data
+ * @param {string} [o.agentId] the agent the event concerns; endpoints narrowed
+ * to a different agent are skipped
+ * @param {string} [o.eventId] a deterministic ID, so raising the same event
+ * twice delivers it once per endpoint
+ * @returns {Promise<string[]>} the delivery IDs written
+ */
+export async function dispatchWebhooks({ userId, eventType, data, agentId = null, eventId = null }) {
+ if (!userId || !EVENT_TYPES.includes(eventType)) return [];
+ const id = eventId || newEventId();
+ const payload = envelope(id, eventType, data ?? {});
+ let rows;
try {
- webhooks = await sql`
- select id, url, secret, events
- from developer_webhooks
- where user_id = ${userId}
- and active = true
- and (events @> ARRAY[${eventType}]::text[] or cardinality(events) = 0)
+ rows = await sql`
+ insert into webhook_deliveries
+ (webhook_id, event_type, event_id, payload, attempt, status, next_attempt_at)
+ select w.id, ${eventType}, ${id}, ${JSON.stringify(payload)}::jsonb, 0, 'pending', now()
+ from developer_webhooks w
+ where w.user_id = ${userId}
+ and w.active = true
+ and (w.events @> ARRAY[${eventType}]::text[] or cardinality(w.events) = 0)
+ and (w.agent_id is null or w.agent_id = ${agentId})
+ on conflict (webhook_id, event_id) where status is not null and replay_of is null do nothing
+ returning id
`;
- } catch {
- return;
+ } catch (err) {
+ console.warn('[webhooks] enqueue failed:', eventType, err?.message);
+ return [];
}
+ const ids = rows.map((r) => r.id);
+ for (const deliveryId of ids) processDelivery(deliveryId).catch(() => {});
+ return ids;
+}
- if (!webhooks.length) return;
-
- const eventId = `evt_${randomToken(16)}`;
- const timestamp = Math.floor(Date.now() / 1000);
- const payload = JSON.stringify({
- id: eventId,
- type: eventType,
- created_at: new Date().toISOString(),
- data,
+/**
+ * Queue a `webhook.test` ping to one endpoint and attempt it now. Awaited, so
+ * the caller can show the outcome.
+ */
+export async function sendTestEvent(webhook) {
+ const id = newEventId();
+ const payload = envelope(id, TEST_EVENT_TYPE, {
+ webhook_id: webhook.id,
+ message: 'This is a test delivery from three.ws. Verify its signature with your whsec_ secret.',
});
-
- for (const wh of webhooks) {
- deliverWithRetry(wh, eventId, timestamp, payload, eventType).catch(() => {});
- }
+ const [row] = await sql`
+ insert into webhook_deliveries
+ (webhook_id, event_type, event_id, payload, attempt, status, next_attempt_at)
+ values (${webhook.id}, ${TEST_EVENT_TYPE}, ${id}, ${JSON.stringify(payload)}::jsonb, 0, 'pending', now())
+ returning id
+ `;
+ await processDelivery(row.id);
+ return row.id;
}
-async function deliverWithRetry(webhook, eventId, timestamp, payload, eventType) {
- for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
- const {
- statusCode,
- responseBody,
- error: deliveryError,
- } = await deliver(webhook.url, webhook.secret, eventId, timestamp, payload);
+/**
+ * Re-send a past delivery to its endpoint as a new delivery row. The event ID
+ * and payload are the original's; the attempt history starts fresh.
+ */
+export async function replayDelivery(delivery) {
+ const original = delivery.replay_of || delivery.id;
+ const [row] = await sql`
+ insert into webhook_deliveries
+ (webhook_id, event_type, event_id, payload, attempt, status, next_attempt_at, replay_of)
+ select webhook_id, event_type, event_id, payload, 0, 'pending', now(), ${original}
+ from webhook_deliveries where id = ${delivery.id}
+ returning id
+ `;
+ if (!row) return null;
+ await processDelivery(row.id);
+ return row.id;
+}
- try {
- await sql`
- insert into webhook_deliveries (webhook_id, event_type, event_id, payload, status_code, response_body, error, attempt)
- values (${webhook.id}, ${eventType}, ${eventId}, ${payload}::jsonb, ${statusCode}, ${responseBody}, ${deliveryError}, ${attempt})
- `;
- } catch {
- // Best-effort logging
+/**
+ * Post every due delivery. Called by the every-minute cron. Rows whose lease
+ * expired (a worker died mid-attempt) go back to pending first.
+ *
+ * @returns {Promise<{ reclaimed: number, attempted: number, succeeded: number }>}
+ */
+export async function sweepWebhookDeliveries({ limit = 100, concurrency = 8, deadlineMs = 50_000 } = {}) {
+ const started = Date.now();
+ const reclaimed = await sql`
+ update webhook_deliveries
+ set status = 'pending', lease_until = null, updated_at = now()
+ where status = 'delivering' and lease_until < now()
+ returning id
+ `;
+ const due = await sql`
+ select id from webhook_deliveries
+ where status = 'pending' and next_attempt_at <= now()
+ order by next_attempt_at
+ limit ${limit}
+ `;
+ const queue = due.map((r) => r.id);
+ let attempted = 0;
+ let succeeded = 0;
+ async function worker() {
+ while (queue.length && Date.now() - started < deadlineMs) {
+ const id = queue.shift();
+ const outcome = await processDelivery(id).catch(() => null);
+ if (!outcome) continue;
+ attempted += 1;
+ if (outcome.status === 'succeeded') succeeded += 1;
}
+ }
+ await Promise.all(Array.from({ length: Math.min(concurrency, queue.length) }, worker));
+ return { reclaimed: reclaimed.length, attempted, succeeded };
+}
- if (statusCode && statusCode >= 200 && statusCode < 300) return;
+/**
+ * Claim one due delivery, post it, and record the attempt.
+ * Returns null when the row was not claimable (not due, or another worker has it).
+ */
+export async function processDelivery(deliveryId) {
+ const [row] = await sql`
+ update webhook_deliveries d
+ set status = 'delivering',
+ lease_until = now() + make_interval(secs => ${LEASE_SECONDS}),
+ updated_at = now()
+ from developer_webhooks w
+ where d.id = ${deliveryId}
+ and d.webhook_id = w.id
+ and d.status = 'pending'
+ and d.next_attempt_at <= now()
+ returning d.id, d.event_id, d.event_type, d.payload, d.attempt,
+ w.id as webhook_id, w.url, w.secret, w.active
+ `;
+ if (!row) return null;
- if (attempt < MAX_ATTEMPTS) {
- const delay = BACKOFF_BASE_MS * Math.pow(4, attempt - 1);
- await sleep(delay);
- }
+ if (!row.active) {
+ const attempt = { at: new Date().toISOString(), status_code: null, error: 'endpoint_disabled', duration_ms: 0 };
+ await sql`
+ update webhook_deliveries
+ set status = 'failed', error = 'endpoint_disabled', lease_until = null, next_attempt_at = null,
+ attempts = attempts || ${JSON.stringify([attempt])}::jsonb, updated_at = now()
+ where id = ${row.id}
+ `;
+ return { status: 'failed', attempt };
}
+ const body = typeof row.payload === 'string' ? row.payload : JSON.stringify(row.payload);
+ const t0 = Date.now();
+ const result = await deliver(row.url, row.secret, row.event_id, body);
+ const durationMs = Date.now() - t0;
+ const attemptNo = Number(row.attempt || 0) + 1;
+ const ok = result.statusCode >= 200 && result.statusCode < 300;
+ const attempt = {
+ n: attemptNo,
+ at: new Date(t0).toISOString(),
+ status_code: result.statusCode,
+ error: result.error,
+ duration_ms: durationMs,
+ response_body: result.responseBody,
+ };
+ const exhausted = !ok && attemptNo >= MAX_ATTEMPTS;
+ const status = ok ? 'succeeded' : exhausted ? 'failed' : 'pending';
+ const nextAt = status === 'pending' ? new Date(Date.now() + backoffMs(attemptNo)) : null;
+
+ await sql`
+ update webhook_deliveries
+ set status = ${status},
+ attempt = ${attemptNo},
+ status_code = ${result.statusCode},
+ response_body = ${result.responseBody},
+ error = ${result.error},
+ duration_ms = ${durationMs},
+ delivered_at = ${ok ? new Date() : null},
+ next_attempt_at = ${nextAt},
+ lease_until = null,
+ attempts = attempts || ${JSON.stringify([attempt])}::jsonb,
+ updated_at = now()
+ where id = ${row.id}
+ `;
+ await recordEndpointHealth(row.webhook_id, ok);
+ return { status, attempt, nextAttemptAt: nextAt };
+}
+
+/** Delay after failed attempt n, with up to 10% jitter so retries do not stampede. */
+export function backoffMs(attemptNo) {
+ const base = RETRY_SCHEDULE_MS[Math.min(attemptNo, RETRY_SCHEDULE_MS.length) - 1] ?? RETRY_SCHEDULE_MS[0];
+ return Math.round(base * (1 + Math.random() * 0.1));
+}
+
+async function recordEndpointHealth(webhookId, ok) {
try {
- const [{ failure_count }] = await sql`
- select count(*)::int as failure_count
- from webhook_deliveries
- where webhook_id = ${webhook.id}
- and (status_code is null or status_code >= 400)
- and created_at > now() - interval '24 hours'
- `;
- if (failure_count >= 50) {
+ if (ok) {
await sql`
- update developer_webhooks set active = false, updated_at = now()
- where id = ${webhook.id}
+ update developer_webhooks
+ set consecutive_failures = 0, last_success_at = now()
+ where id = ${webhookId}
`;
+ return;
}
+ await sql`
+ update developer_webhooks
+ set consecutive_failures = consecutive_failures + 1,
+ last_failure_at = now(),
+ active = case
+ when consecutive_failures + 1 >= ${DISABLE_AFTER_FAILURES}
+ and (last_success_at is null or last_success_at < now() - interval '24 hours')
+ then false else active end,
+ disabled_reason = case
+ when consecutive_failures + 1 >= ${DISABLE_AFTER_FAILURES}
+ and (last_success_at is null or last_success_at < now() - interval '24 hours')
+ then 'consecutive_failures' else disabled_reason end,
+ updated_at = now()
+ where id = ${webhookId}
+ `;
+ } catch (err) {
+ console.warn('[webhooks] endpoint health update failed:', err?.message);
+ }
+}
+
+/**
+ * Validate a receiver URL for registration: HTTPS to a public address in
+ * production. Returns an error string, or null when the URL is acceptable.
+ */
+export async function webhookUrlProblem(raw) {
+ if (typeof raw !== 'string' || !raw.trim()) return 'url is required';
+ const url = raw.trim();
+ if (url.length > 2048) return 'url exceeds 2048 characters';
+ let parsed;
+ try {
+ parsed = new URL(url);
+ } catch {
+ return 'url is not a valid URL';
+ }
+ if (privateTargetsAllowed() && (parsed.protocol === 'http:' || parsed.protocol === 'https:')) return null;
+ if (parsed.protocol !== 'https:') return 'Webhook URL must use HTTPS';
+ try {
+ validatePublicUrl(url, { allowHttp: false });
+ await resolvePublicHost(parsed.hostname);
} catch {
- // Best-effort deactivation
+ return 'Webhook URL must resolve to a public address';
}
+ return null;
}
-async function deliver(url, secret, eventId, timestamp, payload) {
- const signature = await sign(secret, eventId, timestamp, payload);
+async function deliver(url, secret, eventId, body) {
+ const timestamp = Math.floor(Date.now() / 1000);
+ const signature = await hmacSha256(secret, `${eventId}.${timestamp}.${body}`);
// SSRF guard: the URL is developer-supplied, so resolve it and pin the
// connection to the validated public address(es). Without this, a webhook
@@ -153,11 +386,15 @@ async function deliver(url, secret, eventId, timestamp, payload) {
// target and persist the status/error as a probing oracle. Redirects are NOT
// followed (manual) so a public host can't bounce the request internally.
let target;
- let agent;
+ let agent = null;
try {
- target = validatePublicUrl(url);
- const addrs = await resolvePublicHost(target.hostname);
- agent = pinnedAgent(target.hostname, addrs);
+ if (privateTargetsAllowed()) {
+ target = new URL(url);
+ } else {
+ target = validatePublicUrl(url, { allowHttp: false });
+ const addrs = await resolvePublicHost(target.hostname);
+ agent = pinnedAgent(target.hostname, addrs);
+ }
} catch (err) {
const reason = err instanceof SsrfError ? `blocked_url:${err.code}` : 'invalid_url';
return { statusCode: null, responseBody: null, error: reason };
@@ -172,7 +409,7 @@ async function deliver(url, secret, eventId, timestamp, payload) {
res = await fetch(target, {
method: 'POST',
redirect: 'manual',
- dispatcher: agent,
+ ...(agent ? { dispatcher: agent } : {}),
headers: {
'content-type': 'application/json',
'webhook-id': eventId,
@@ -180,15 +417,15 @@ async function deliver(url, secret, eventId, timestamp, payload) {
'webhook-signature': `v1,${signature}`,
'user-agent': 'three.ws-webhooks/1.0',
},
- body: payload,
+ body,
signal: controller.signal,
});
} finally {
clearTimeout(timeout);
- await agent.close().catch(() => {});
+ if (agent) await agent.close().catch(() => {});
}
- // A redirect is a misconfigured (or hostile) endpoint — record it as a
+ // A redirect is a misconfigured (or hostile) endpoint: record it as a
// failure instead of following it to a potentially internal target.
if (res.status >= 300 && res.status < 400) {
return { statusCode: res.status, responseBody: null, error: 'redirect_not_followed' };
@@ -197,22 +434,14 @@ async function deliver(url, secret, eventId, timestamp, payload) {
let responseBody = null;
try {
responseBody = await res.text();
- if (responseBody.length > 1024) responseBody = responseBody.slice(0, 1024);
+ if (responseBody.length > RESPONSE_BODY_MAX) responseBody = responseBody.slice(0, RESPONSE_BODY_MAX);
} catch {
- // Body read failed
+ responseBody = null;
}
-
- return { statusCode: res.status, responseBody, error: null };
+ const error = res.status >= 200 && res.status < 300 ? null : `http_${res.status}`;
+ return { statusCode: res.status, responseBody, error };
} catch (err) {
- return { statusCode: null, responseBody: null, error: err?.message || 'delivery_failed' };
+ const error = err?.name === 'AbortError' ? 'timeout' : err?.cause?.code || err?.message || 'delivery_failed';
+ return { statusCode: null, responseBody: null, error };
}
}
-
-async function sign(secret, eventId, timestamp, payload) {
- const message = `${eventId}.${timestamp}.${payload}`;
- return hmacSha256(secret, message);
-}
-
-function sleep(ms) {
- return new Promise((resolve) => setTimeout(resolve, ms));
-}
diff --git a/api/developer/webhooks.js b/api/developer/webhooks.js
index a0fc3b0bd..fa61af015 100644
--- a/api/developer/webhooks.js
+++ b/api/developer/webhooks.js
@@ -5,12 +5,9 @@ import { cors, error, json, method, readJson, wrap } from '../_lib/http.js';
import { getSessionUser } from '../_lib/auth.js';
import { requireCsrf } from '../_lib/csrf.js';
import { sql } from '../_lib/db.js';
-import { randomToken } from '../_lib/crypto.js';
-import { EVENT_TYPES, selectEventTypes } from '../_lib/webhook-dispatch.js';
-import { assertPublicHttpsUrl } from '../_lib/ssrf.js';
+import { EVENT_TYPES, newWebhookSecret, selectEventTypes, webhookUrlProblem } from '../_lib/webhook-dispatch.js';
const MAX_WEBHOOKS_PER_USER = 10;
-const URL_MAX_LENGTH = 2048;
export default wrap(async function handler(req, res) {
if (cors(req, res, { methods: 'GET,POST,OPTIONS', credentials: true })) return;
@@ -22,14 +19,18 @@ export default wrap(async function handler(req, res) {
// One round trip for the list AND its 7-day delivery stats: the lateral
// join keeps a 10-webhook dashboard at a single query instead of 1 + N.
const rows = await sql`
- select w.id, w.url, w.events, w.active, w.description, w.created_at, w.updated_at,
- s.total, s.succeeded, s.failed, s.last_delivery_at
+ select w.id, w.url, w.events, w.active, w.description, w.agent_id, w.disabled_reason,
+ w.consecutive_failures, w.created_at, w.updated_at,
+ s.total, s.succeeded, s.failed, s.pending, s.last_delivery_at
from developer_webhooks w
left join lateral (
select
count(*)::int as total,
- count(*) filter (where status_code between 200 and 299)::int as succeeded,
- count(*) filter (where status_code is null or status_code >= 400)::int as failed,
+ -- Queue rows (status set) count by their final state; legacy
+ -- one-row-per-attempt rows (status null) count by HTTP code.
+ count(*) filter (where status = 'succeeded' or (status is null and status_code between 200 and 299))::int as succeeded,
+ count(*) filter (where status = 'failed' or (status is null and (status_code is null or status_code >= 400)))::int as failed,
+ count(*) filter (where status in ('pending', 'delivering'))::int as pending,
max(created_at) as last_delivery_at
from webhook_deliveries d
where d.webhook_id = w.id and d.created_at > now() - interval '7 days'
@@ -38,9 +39,9 @@ export default wrap(async function handler(req, res) {
order by w.created_at desc
`;
- const webhooks = rows.map(({ total, succeeded, failed, last_delivery_at, ...wh }) => ({
+ const webhooks = rows.map(({ total, succeeded, failed, pending, last_delivery_at, ...wh }) => ({
...wh,
- stats_7d: { total, succeeded, failed, last_delivery_at },
+ stats_7d: { total, succeeded, failed, pending, last_delivery_at },
}));
return json(res, 200, { webhooks, event_types: EVENT_TYPES });
@@ -58,25 +59,10 @@ export default wrap(async function handler(req, res) {
}
const url = typeof body.url === 'string' ? body.url.trim() : '';
- if (!url) return error(res, 400, 'bad_request', 'url is required');
- if (url.length > URL_MAX_LENGTH)
- return error(res, 400, 'bad_request', `url exceeds ${URL_MAX_LENGTH} characters`);
-
- try {
- const parsed = new URL(url);
- if (parsed.protocol !== 'https:') {
- return error(res, 400, 'bad_request', 'Webhook URL must use HTTPS');
- }
- } catch {
- return error(res, 400, 'bad_request', 'Invalid URL');
- }
- // Reject URLs that resolve to a private/loopback/link-local/metadata address
- // at registration time (SSRF). Delivery re-validates and pins per attempt.
- try {
- await assertPublicHttpsUrl(url);
- } catch {
- return error(res, 400, 'bad_request', 'Webhook URL must resolve to a public address');
- }
+ // HTTPS to a public address (SSRF): checked here at registration, and again
+ // with a pinned connection on every delivery attempt.
+ const urlProblem = await webhookUrlProblem(url);
+ if (urlProblem) return error(res, 400, 'bad_request', urlProblem);
const selection = selectEventTypes(body.events);
if (selection.error) return error(res, 400, 'bad_request', selection.error);
@@ -96,7 +82,7 @@ export default wrap(async function handler(req, res) {
);
}
- const secret = `whsec_${randomToken(24)}`;
+ const secret = newWebhookSecret();
const [webhook] = await sql`
insert into developer_webhooks (user_id, url, secret, events, description)
diff --git a/api/developer/webhooks/[id].js b/api/developer/webhooks/[id].js
index d42f271c1..062e67e55 100644
--- a/api/developer/webhooks/[id].js
+++ b/api/developer/webhooks/[id].js
@@ -6,12 +6,9 @@ import { cors, error, json, method, readJson, wrap } from '../../_lib/http.js';
import { getSessionUser } from '../../_lib/auth.js';
import { requireCsrf } from '../../_lib/csrf.js';
import { sql } from '../../_lib/db.js';
-import { selectEventTypes } from '../../_lib/webhook-dispatch.js';
-import { assertPublicHttpsUrl } from '../../_lib/ssrf.js';
+import { selectEventTypes, webhookUrlProblem } from '../../_lib/webhook-dispatch.js';
import { isUuid } from '../../_lib/validate.js';
-const URL_MAX_LENGTH = 2048;
-
export default wrap(async function handler(req, res) {
if (cors(req, res, { methods: 'GET,PATCH,DELETE,OPTIONS', credentials: true })) return;
@@ -36,7 +33,8 @@ export default wrap(async function handler(req, res) {
if (req.method === 'GET') {
const deliveries = await sql`
- select id, event_type, event_id, status_code, error, attempt, created_at
+ select id, event_type, event_id, status, status_code, error, attempt,
+ next_attempt_at, delivered_at, duration_ms, replay_of, created_at
from webhook_deliveries
where webhook_id = ${id}
order by created_at desc
@@ -58,27 +56,10 @@ export default wrap(async function handler(req, res) {
const updates = {};
if (typeof body.url === 'string') {
const trimmed = body.url.trim();
- if (trimmed.length > URL_MAX_LENGTH)
- return error(res, 400, 'bad_request', 'URL too long');
- try {
- const parsed = new URL(trimmed);
- if (parsed.protocol !== 'https:')
- return error(res, 400, 'bad_request', 'Must use HTTPS');
- } catch {
- return error(res, 400, 'bad_request', 'Invalid URL');
- }
- // SSRF: reject a URL that resolves to a non-public address (delivery
- // also re-validates and pins per attempt).
- try {
- await assertPublicHttpsUrl(trimmed);
- } catch {
- return error(
- res,
- 400,
- 'bad_request',
- 'Webhook URL must resolve to a public address',
- );
- }
+ // SSRF: HTTPS to a public address (delivery also re-validates and
+ // pins per attempt).
+ const urlProblem = await webhookUrlProblem(trimmed);
+ if (urlProblem) return error(res, 400, 'bad_request', urlProblem);
updates.url = trimmed;
}
if (body.events !== undefined) {
@@ -89,6 +70,9 @@ export default wrap(async function handler(req, res) {
if (typeof body.active === 'boolean') {
updates.active = body.active;
}
+ // Re-enabling an endpoint the failure breaker switched off clears the
+ // breaker, so it is not disabled again by the very next failure.
+ const reenable = updates.active === true && !webhook.active;
if (typeof body.description === 'string') {
updates.description = body.description.trim().slice(0, 200);
}
@@ -103,6 +87,8 @@ export default wrap(async function handler(req, res) {
events = ${updates.events ?? webhook.events},
active = ${updates.active ?? webhook.active},
description = ${updates.description !== undefined ? updates.description : webhook.description},
+ consecutive_failures = case when ${reenable} then 0 else consecutive_failures end,
+ disabled_reason = case when ${reenable} then null else disabled_reason end,
updated_at = now()
where id = ${id} and user_id = ${user.id}
returning id, url, events, active, description, created_at, updated_at