Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// GET /api/ops/health — comprehensive internal platform health check.
//
// Returns live probe results for every critical subsystem + cron heartbeats
// from Redis. Auth is authorizeOps (api/_lib/ops-auth.js): an admin session, or
// x-ops-secret / Authorization Bearer matching OPS_SECRET. It is deliberately
// never CRON_SECRET, the credential the crons that move real funds carry, so a
// leaked ops password cannot be escalated into triggering a payment job. With no
// OPS_SECRET configured the gate is open off-production and denies in production.
import { readFileSync } from 'node:fs';
import { CronExpressionParser } from 'cron-parser';
import { cors, error, json, method, wrap, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { cacheGet } from '../_lib/cache.js';
import { env } from '../_lib/env.js';
import { sql } from '../_lib/db.js';
import { authorizeOps } from '../_lib/ops-auth.js';
const PROBE_TIMEOUT_MS = 8_000;
const ORIGIN = env.APP_ORIGIN || 'https://three.ws';
// ── Subsystems to probe live ─────────────────────────────────────────────────
// Each probe hits a real endpoint from outside. Grouped by category.
// A POST probe may carry `headers` + `body`: a handler that gates on
// content-type answers 415 to a bare POST, which is a probe bug, not a
// subsystem fault (see the x402_pay entry).
const PROBES = [
// Core
{ id: 'site', cat: 'core', label: 'Website', path: '/', expect: 200 },
{ id: 'api', cat: 'core', label: 'Platform API', path: '/api/healthz', expect: 200 },
{ id: 'explore', cat: 'core', label: 'Explore feed', path: '/api/explore', expect: [200, 401] },
// x402
{ id: 'x402_discovery', cat: 'x402', label: 'x402 discovery', path: '/.well-known/x402.json', expect: 200 },
{ id: 'x402_dance_tip', cat: 'x402', label: 'dance-tip endpoint', path: '/api/x402/dance-tip', expect: 402 },
{ id: 'x402_mint_batch', cat: 'x402', label: 'mint-to-mesh-batch', path: '/api/x402/mint-to-mesh-batch', method: 'POST', expect: 402 },
// A bodyless POST is answered 415 ("content-type must be application/json")
// before the handler runs, so the probe measured its own missing header
// forever. An empty JSON object reaches the dispatcher and comes back 400
// (invalid_tool): proof the handler is alive and validating, and it invokes
// no tool, so the probe stays read-only.
{ id: 'x402_pay', cat: 'x402', label: 'x402-pay handler', path: '/api/x402-pay', method: 'POST', headers: { 'content-type': 'application/json' }, body: '{}', expect: [200, 400, 401] },
// Pumpfun / trading
{ id: 'pump_action', cat: 'trading', label: 'Pump action API', path: '/api/pump/launches', expect: [200, 400, 401, 404] },
{ id: 'oracle_feed', cat: 'trading', label: 'Oracle feed', path: '/api/oracle/feed', expect: [200, 401] },
{ id: 'oracle_signal', cat: 'trading', label: 'Oracle signal', path: '/api/oracle/signal', expect: [200, 401] },
// Agents
{ id: 'agents_public', cat: 'agents', label: 'Public agents list', path: '/api/agents/public', expect: [200, 401] },
{ id: 'agents_featured', cat: 'agents', label: 'Featured agents', path: '/api/agents/featured', expect: [200, 401] },
// Marketplace
{ id: 'marketplace', cat: 'marketplace', label: 'Marketplace API', path: '/api/marketplace/agents', expect: [200, 401] },
// Forge / AI
{ id: 'forge_gallery', cat: 'ai', label: 'Forge gallery', path: '/api/forge-gallery', expect: [200, 401] },
// Tips / social
{ id: 'club_tips', cat: 'social', label: 'Tips stream (probe)', path: '/api/club/tips-stream', method: 'HEAD', expect: [200, 400, 401, 426] },
// Avatars
{ id: 'avatars_featured', cat: 'avatars', label: 'Featured avatars', path: '/api/avatars/featured', expect: [200, 401] },
// Auth
{ id: 'auth', cat: 'auth', label: 'Auth endpoint', path: '/api/auth', expect: [200, 400, 401, 404, 405] },
// Payments
{ id: 'pay_session', cat: 'payments', label: 'Payment sessions', path: '/api/pay/session', expect: [200, 401] },
// Notifications
{ id: 'notifications', cat: 'core', label: 'Notifications', path: '/api/notifications', expect: [200, 401] },
];
// ── Crons we expect to have heartbeats ──────────────────────────────────────
// Every cron's staleness window is DERIVED from the schedule it actually runs
// on (vercel.json `crons`, the same array Cloud Scheduler is seeded from), never
// hand-set. Hand-set windows drift the moment a schedule changes, and three of
// them had: oracle-digest (daily 08:00) and gmgn-seed (daily 03:00) carried a
// 70-minute window and dead-man-switch (daily 05:00) a 15-minute one, so all
// three read as failing for ~23 hours of every day and /api/ops/health could
// never answer ok:true. A board that is always red is a board nobody reads.
//
// Window = one and a half cadences plus 30 minutes of slack: a single late tick
// is tolerated, a stopped cron is caught within half a cadence of its next miss.
const STALE_SLACK_MS = 30 * 60 * 1000;
// Used only when a schedule cannot be read (vercel.json unreadable, expression
// unparseable): generous enough not to invent failures, tight enough to notice.
const DEFAULT_STALE_AFTER_MS = 70 * 60 * 1000;
/** Cron id (the path segment under /api/cron/) → its schedule expression. */
function loadCronSchedules() {
try {
const raw = readFileSync(new URL('../../vercel.json', import.meta.url), 'utf8');
const crons = JSON.parse(raw).crons || [];
return new Map(
crons.map((c) => [String(c.path || '').replace(/^\/api\/cron\//, '').split('?')[0], c.schedule]),
);
} catch {
return new Map();
}
}
// The widest gap between consecutive fires, so a schedule with uneven spacing
// (`0 9,17 * * *`) is measured by its longest quiet stretch, not its shortest.
export function cadenceMs(schedule) {
if (!schedule) return null;
try {
const it = CronExpressionParser.parse(schedule, { tz: 'UTC' });
const fires = [it.next().getTime(), it.next().getTime(), it.next().getTime(), it.next().getTime()];
let widest = 0;
for (let i = 1; i < fires.length; i++) widest = Math.max(widest, fires[i] - fires[i - 1]);
return widest > 0 ? widest : null;
} catch {
return null;
}
}
export function deriveStaleAfterMs(schedule) {
const cadence = cadenceMs(schedule);
if (!cadence) return DEFAULT_STALE_AFTER_MS;
return Math.round(cadence * 1.5) + STALE_SLACK_MS;
}
const CRON_SCHEDULES = loadCronSchedules();
// `driven_by` names the cron that fires this one as a fan-out target rather than
// a scheduler job of its own (api/cron/economy-tick.js): its cadence is the
// driver's cadence, and it has no `crons` entry to read.
export const CRONS = [
{ id: 'uptime-check', label: 'Uptime monitor' },
{ id: 'pulse-tick', label: 'Pulse tick', driven_by: 'economy-tick' },
{ id: 'oracle-score', label: 'Oracle score' },
{ id: 'oracle-digest', label: 'Oracle digest' },
{ id: 'three-holders-snapshot', label: 'THREE holders snapshot' },
{ id: 'payment-session-sweep', label: 'Payment session sweep' },
{ id: 'flush-usage-events', label: 'Flush usage events' },
{ id: 'reflect-sweep', label: 'Reflect sweep' },
{ id: 'forge-seed-cron', label: 'Forge seed' },
{ id: 'avaturn-seed-cron', label: 'Avaturn seed' },
{ id: 'smart-money-rollup', label: 'Smart money rollup' },
{ id: 'smart-money-graph', label: 'Smart money graph' },
{ id: 'dead-man-switch', label: 'Dead man switch' },
{ id: 'world-health', label: 'World health' },
{ id: 'quota-check', label: 'Quota check' },
{ id: 'recompute-reputation', label: 'Reputation recompute' },
{ id: 'copy-fanout', label: 'Copy fanout' },
{ id: 'mirror-fanout', label: 'Mirror fanout' },
{ id: 'signal-fanout', label: 'Signal fanout' },
{ id: 'strategy-fanout', label: 'Strategy fanout' },
{ id: 'gmgn-seed', label: 'GMGN seed' },
{ id: 'intel-learn', label: 'Intel learn' },
{ id: 'wallet-intents', label: 'Wallet intents' },
{ id: 'auto-rig-sweep', label: 'Auto rig sweep' },
{ id: 'radar-watchlist', label: 'Radar watchlist' },
// Preview-coverage crons. Every agent card, forge card, and marketplace
// listing renders a real image only because one of these keeps draining its
// backlog; when one stops, the surfaces degrade to gradient placeholders
// slowly enough that nobody notices for weeks. They all wrote heartbeats
// already (wrapCron does it for every cron), but nothing surfaced them, so
// the only way to tell a stalled one from a healthy one was gcloud.
{ id: 'agent-avatar-backfill', label: 'Agent avatar backfill' },
{ id: 'avatar-thumbnail-backfill', label: 'Avatar thumbnail backfill' },
{ id: 'avatar-thumbnail-render', label: 'Avatar thumbnail render' },
{ id: 'forge-thumbnail-backfill', label: 'Forge thumbnail backfill' },
{ id: 'coin-intel-observe', label: 'Coin intel observe' },
{ id: 'unstoppable-tick', label: 'Unstoppable agent tick' },
{ id: 'sketchfab-showcase', label: 'Sketchfab showcase' },
// Storage and job-recovery crons. Both write heartbeats already, but neither
// was surfaced here, so a stall was invisible until it showed up as a symptom
// somewhere else: db-retention is the only thing holding the Neon branch under
// its size cap (when it stops, every write-heavy cron latches into the
// storage-pressure skip), and reconstruct-sweep is what finalizes or reaps a
// reconstruct job whose worker callback never arrived (when it stops, avatars
// sit in 'rigging' forever).
{ id: 'db-retention', label: 'DB retention' },
{ id: 'reconstruct-sweep', label: 'Reconstruct sweep' },
// Community delivery: posts every shipped release to Telegram, and the
// reviewed @trythreews queue. Each answers 502 on failure, red here.
{ id: 'changelog-push', label: 'Changelog push (Telegram)' },
{ id: 'x-content', label: 'X content queue' },
].map((c) => {
const schedule = CRON_SCHEDULES.get(c.driven_by || c.id) || null;
return { ...c, schedule, stale_after_ms: deriveStaleAfterMs(schedule) };
});
async function runProbe(probe) {
const url = `${ORIGIN}${probe.path}`;
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), PROBE_TIMEOUT_MS);
const t0 = Date.now();
try {
const res = await fetch(url, {
method: probe.method || 'GET',
redirect: 'manual',
signal: controller.signal,
headers: { 'user-agent': 'threews-ops-health/1.0', ...(probe.headers || {}) },
...(probe.body ? { body: probe.body } : {}),
});
const ms = Date.now() - t0;
const expected = Array.isArray(probe.expect) ? probe.expect : [probe.expect];
const ok = expected.includes(res.status);
return { id: probe.id, cat: probe.cat, label: probe.label, ok, status: res.status, ms };
} catch (err) {
return { id: probe.id, cat: probe.cat, label: probe.label, ok: false, status: 0, ms: Date.now() - t0, err: err?.message };
} finally {
clearTimeout(timer);
}
}
export default wrap(async (req, res) => {
// Same-origin is what cors() does when `origins` is omitted (it falls back to the
// first-party allow-list). A sentinel string was not a supported value: it reached
// `allowed.some(...)` and threw, 500ing any request that carried an Origin header,
// preflights included. Same fix as api/irl/analytics.js.
if (cors(req, res, { methods: 'GET,OPTIONS' })) return;
// The shared gate, like every neighbouring ops handler: it answers HEAD as
// GET (RFC 9110) and sends the Allow header a bare 405 was missing.
if (!method(req, res, ['GET'])) return;
// `authedReadIp` (300/5m), not the strict `authIp` credential bucket. This is a
// polled read board, and every poll used to spend the same 50/10m budget that
// gates logins from that IP, so watching the dashboard could 429 an operator's
// sign-in. Same reason `authedReadIp` was split out for authed reads at large.
const rl = await limits.authedReadIp(clientIp(req));
if (!rl.success) return rateLimited(res, rl);
// Hardened ops gate: admin session or a dedicated
// OPS_SECRET, fail-closed in production, never CRON_SECRET.
const auth = await authorizeOps(req);
if (!auth.ok) return error(res, 401, 'unauthorized', 'admin session or x-ops-secret required');
const t0 = Date.now();
// Run all probes in parallel
const probeResults = await Promise.all(PROBES.map(runProbe));
// Load all cron heartbeats in parallel
const heartbeats = await Promise.all(
CRONS.map(async (cron) => {
const hb = await cacheGet(`cron:heartbeat:${cron.id}`).catch(() => null);
const now = Date.now();
const lastRan = hb?.t ?? null;
const msSince = lastRan ? now - lastRan : null;
const stale = msSince === null || msSince > cron.stale_after_ms;
return {
id: cron.id,
label: cron.label,
// Both the schedule and the window it produced, so an operator
// reading a stale row can tell a stopped cron from a bad window.
schedule: cron.schedule,
staleAfterMs: cron.stale_after_ms,
ok: hb?.ok ?? null,
lastRan,
msSince,
stale,
lastErr: hb?.ok === false ? hb.err : null,
durationMs: hb?.ms ?? null,
};
}),
);
// Cross-network payment circuit breaker — latest per-network status written
// hourly by the autonomous loop (api/_lib/x402/circuit-breaker.js). A tripped
// route or failed Solana settlement means the payment stack is degraded.
const circuitBreaker = await loadCircuitBreaker();
// Agent wallet balance — latest sample written every 10 min by the autonomous
// loop (api/_lib/x402/wallet-balance-monitor.js). A low/unconfigured wallet
// means autonomous calls are about to start failing on insufficient funds.
const walletBalance = await loadWalletBalance();
// Payment-proof idempotency audit — latest verdict written daily by the
// autonomous loop (api/_lib/x402/pipelines/payment-proof-idempotency-audit.js).
// A confirmed double-settlement means the x402 anti-replay guard is broken.
const idempotencyAudit = await loadIdempotencyAudit();
// API-key bypass security test — latest verdict written daily by the autonomous
// loop (api/_lib/x402/pipelines/api-key-bypass-audit.js). A confirmed leak means
// the X-API-Key bypass lane is granting free access to a missing/invalid key —
// paid endpoints can be drained for free.
const apiKeyBypassAudit = await loadApiKeyBypassAudit();
// Builder-code attribution — latest per-endpoint verdict written every 6h by the
// autonomous loop (api/_lib/x402/pipelines/builder-code-attribution.js). A gap
// (a priced endpoint that stopped declaring three_d_agent, or a failed attributed
// settlement) means on-chain volume is settling UNATTRIBUTED — lost builder rewards.
const builderAttribution = await loadBuilderAttribution();
// Spend-reservation leak rate — swept count written every 15 min by the
// autonomous loop (api/_lib/x402/pipelines/spend-reservation-leak-detector.js).
// A sustained rate of orphaned reservations means a reserve→finalize path is
// crashing and silently starving agent spend caps.
const reservationLeaks = await loadReservationLeaks();
// Categorize probe results
const byCategory = {};
for (const r of probeResults) {
if (!byCategory[r.cat]) byCategory[r.cat] = [];
byCategory[r.cat].push(r);
}
const allProbesOk = probeResults.every((r) => r.ok);
const allCronsOk = heartbeats.every((h) => !h.stale && h.ok !== false);
// A missing breaker row (never run / table absent) is unknown, not failing —
// don't fail health on a feature that hasn't booted. Only a present-and-degraded
// breaker folds into the verdict.
const breakerOk = circuitBreaker.ok !== false;
// A low or unconfigured wallet degrades health (autonomous spend will fail);
// a never-run monitor (ok:null) is unknown, not failing.
const walletOk = walletBalance.ok !== false;
// A confirmed double-settlement (ok:false) degrades health; a never-run audit
// (ok:null) is unknown, not failing.
const idempotencyOk = idempotencyAudit.ok !== false;
// An attribution gap (ok:false) degrades health — lost builder rewards; a
// never-run/stale tracker (ok:null) is unknown, not failing.
const attributionOk = builderAttribution.ok !== false;
// A confirmed bypass leak / broken bypass (ok:false) degrades health — the
// paywall is granting free access or rejecting valid keys; a never-run/stale
// audit (ok:null) is unknown, not failing.
const apiKeyBypassOk = apiKeyBypassAudit.ok !== false;
// A systemic reservation-leak rate (ok:false) degrades health — agent spend caps
// are being starved by orphaned reservations; a never-run/clean sweep (ok:null/
// ok:true) is fine.
const reservationLeaksOk = reservationLeaks.ok !== false;
const overallOk = allProbesOk && allCronsOk && breakerOk && walletOk && idempotencyOk && attributionOk && apiKeyBypassOk && reservationLeaksOk;
const failingProbes = probeResults.filter((r) => !r.ok);
const failingCrons = heartbeats.filter((h) => h.stale || h.ok === false);
return json(res, overallOk ? 200 : 207, {
ok: overallOk,
checkedAt: t0,
durationMs: Date.now() - t0,
summary: {
probes: { total: probeResults.length, ok: probeResults.filter((r) => r.ok).length, failing: failingProbes.length },
crons: { total: heartbeats.length, ok: heartbeats.filter((h) => !h.stale && h.ok !== false).length, failing: failingCrons.length },
},
failing: {
probes: failingProbes,
crons: failingCrons,
},
probes: byCategory,
crons: heartbeats,
circuitBreaker,
walletBalance,
idempotencyAudit,
apiKeyBypassAudit,
builderAttribution,
reservationLeaks,
});
});
// Read the latest builder-code attribution verdict written every 6h by the
// autonomous loop (api/_lib/x402/pipelines/builder-code-attribution.js). ok:false
// when a priced endpoint is currently missing/mismatched on its three_d_agent
// declaration (a gap, so its settled volume earns no rewards) or the attributed
// settlement proof failed. A stale or never-run tracker is unknown (ok:null) so
// health doesn't trip on a cold or paused feature.
const ATTRIBUTION_STALE_AFTER_MS = 8 * 60 * 60 * 1000; // 8h (6h cadence + slack)
async function loadBuilderAttribution() {
try {
const rows = await sql`
SELECT endpoint, challenged, matches, gap, declared_code, expected_code,
settled, echo_accepted, tx_signature, error,
extract(epoch FROM checked_at) * 1000 AS checked_at_ms
FROM builder_code_attribution
ORDER BY checked_at DESC
`;
if (!rows.length) return { ok: null, reason: 'no_data' };
const now = Date.now();
const newest = Math.max(...rows.map((r) => Number(r.checked_at_ms) || 0));
const stale = !newest || now - newest > ATTRIBUTION_STALE_AFTER_MS;
const gaps = rows.filter((r) => r.gap);
// A settlement proof row (settled column true on the endpoint we pay) that
// flipped echo_accepted=false means an attributed payment failed to settle.
// The settlement-proof target is the dance-tip row. It FAILED only when the
// tracker matched its attribution (so it actually attempted an attributed
// payment) yet the echo was not accepted for a reason other than a missing
// wallet — i.e. the attributed payment was rejected/never settled on-chain.
const settleRow = rows.find((r) => /dance-tip/.test(r.endpoint));
const settleFailed = !!(settleRow && settleRow.matches && settleRow.echo_accepted === false
&& !/^wallet_unconfigured/.test(settleRow.error || ''));
const ok = stale ? null : gaps.length === 0 && !settleFailed;
return {
ok,
expectedCode: rows[0]?.expected_code || null,
endpointsChallenged: rows.filter((r) => r.challenged).length,
attributed: rows.filter((r) => r.matches).length,
gaps: gaps.map((g) => ({ endpoint: g.endpoint, declaredCode: g.declared_code, error: g.error })),
settleTx: settleRow?.tx_signature || null,
lastChecked: newest || null,
stale,
};
} catch (err) {
if (/does not exist|relation .* does not exist/i.test(err?.message || '')) {
return { ok: null, reason: 'table_absent' };
}
return { ok: null, reason: `query_failed:${err?.message || 'unknown'}` };
}
}
// Read the latest payment-proof idempotency audit verdict written daily by the
// autonomous loop (api/_lib/x402/pipelines/payment-proof-idempotency-audit.js).
// ok:false ONLY on a confirmed double-settlement (the anti-replay guard failed);
// a stale or never-run audit is unknown (ok:null), not failing, so health doesn't
// trip on a cold or paused feature.
const IDEMPOTENCY_STALE_AFTER_MS = 48 * 60 * 60 * 1000; // 48h (daily cadence + slack)
async function loadIdempotencyAudit() {
try {
const [row] = await sql`
SELECT route, verdict, double_settled, pass, first_tx, second_tx,
second_marker, payment_id,
extract(epoch FROM ts) * 1000 AS checked_at_ms
FROM x402_idempotency_audit
ORDER BY ts DESC
LIMIT 1
`;
if (!row) return { ok: null, reason: 'no_data' };
const checkedAt = row.checked_at_ms ? Number(row.checked_at_ms) : null;
const stale = checkedAt === null || Date.now() - checkedAt > IDEMPOTENCY_STALE_AFTER_MS;
// Only a confirmed double-settlement fails health. A stale audit is unknown
// (loop stopped writing), and an inconclusive run (first call never settled)
// is not evidence of a broken guard → ok:null.
const ok = row.double_settled ? false : stale ? null : row.verdict === 'inconclusive' ? null : true;
return {
ok,
verdict: row.verdict,
doubleSettled: row.double_settled,
pass: row.pass,
route: row.route,
firstTx: row.first_tx,
secondTx: row.second_tx,
secondMarker: row.second_marker,
paymentId: row.payment_id,
lastChecked: checkedAt,
stale,
};
} catch (err) {
if (/does not exist|relation .* does not exist/i.test(err?.message || '')) {
return { ok: null, reason: 'table_absent' };
}
return { ok: null, reason: `query_failed:${err?.message || 'unknown'}` };
}
}
// Read the latest API-key bypass security-test verdict written daily by the
// autonomous loop (api/_lib/x402/pipelines/api-key-bypass-audit.js). ok:false ONLY
// on a confirmed bypass leak (free access granted to a missing/invalid key) or a
// broken bypass (valid keys rejected); a stale, never-run, or inconclusive audit
// is unknown (ok:null), not failing, so health doesn't trip on a cold/paused feature.
const API_KEY_BYPASS_STALE_AFTER_MS = 48 * 60 * 60 * 1000; // 48h (daily cadence + slack)
async function loadApiKeyBypassAudit() {
try {
const [row] = await sql`
SELECT route, verdict, pass, leak, leaks, valid_key_source,
paid_settled, amount_atomic,
extract(epoch FROM ts) * 1000 AS checked_at_ms
FROM x402_api_key_bypass_audit
ORDER BY ts DESC
LIMIT 1
`;
if (!row) return { ok: null, reason: 'no_data' };
const checkedAt = row.checked_at_ms ? Number(row.checked_at_ms) : null;
const stale = checkedAt === null || Date.now() - checkedAt > API_KEY_BYPASS_STALE_AFTER_MS;
// A confirmed leak or broken bypass fails health. A stale/inconclusive audit
// is unknown (loop paused, or a valid key couldn't be acquired) → ok:null.
const ok =
row.verdict === 'bypass_leak' || row.verdict === 'bypass_broken'
? false
: stale || row.verdict === 'inconclusive'
? null
: true;
return {
ok,
verdict: row.verdict,
pass: row.pass,
leak: row.leak,
leaks: row.leaks,
route: row.route,
validKeySource: row.valid_key_source,
paidSettled: row.paid_settled,
lastChecked: checkedAt,
stale,
};
} catch (err) {
if (/does not exist|relation .* does not exist/i.test(err?.message || '')) {
return { ok: null, reason: 'table_absent' };
}
return { ok: null, reason: `query_failed:${err?.message || 'unknown'}` };
}
}
// Read the latest agent-wallet balance sample written every 10 min by the
// autonomous loop (api/_lib/x402/wallet-balance-monitor.js). A sample older than
// the staleness window means the monitor stopped writing; ok:null when it has
// never run (table absent / no rows) so health doesn't fail on a cold feature.
const WALLET_STALE_AFTER_MS = 30 * 60 * 1000; // 30m (10-min cadence + slack)
async function loadWalletBalance() {
try {
const [row] = await sql`
SELECT address, configured, usdc, sol, low_balance, threshold_usdc,
usdc_delta, spend_rate_usdc_hr,
extract(epoch FROM ts) * 1000 AS checked_at_ms
FROM agent_wallet_balance_log
ORDER BY ts DESC
LIMIT 1
`;
if (!row) return { ok: null, reason: 'no_data' };
const checkedAt = row.checked_at_ms ? Number(row.checked_at_ms) : null;
const stale = checkedAt === null || Date.now() - checkedAt > WALLET_STALE_AFTER_MS;
// Degraded when the wallet is below threshold or unconfigured. A stale
// sample is unknown (monitor stopped), not a balance failure → ok:null.
const ok = stale ? null : !(row.low_balance || row.configured === false);
return {
ok,
configured: row.configured,
address: row.address,
usdc: row.usdc != null ? Number(row.usdc) : null,
sol: row.sol != null ? Number(row.sol) : null,
lowBalance: row.low_balance,
thresholdUsdc: row.threshold_usdc != null ? Number(row.threshold_usdc) : null,
usdcDelta: row.usdc_delta != null ? Number(row.usdc_delta) : null,
spendRateUsdcHr: row.spend_rate_usdc_hr != null ? Number(row.spend_rate_usdc_hr) : null,
lastChecked: checkedAt,
stale,
};
} catch (err) {
if (/does not exist|relation .* does not exist/i.test(err?.message || '')) {
return { ok: null, reason: 'table_absent' };
}
return { ok: null, reason: `query_failed:${err?.message || 'unknown'}` };
}
}
// Read the spend-reservation leak rate written every 15 min by the autonomous
// loop (api/_lib/x402/pipelines/spend-reservation-leak-detector.js). A reservation
// claims agent spend-cap headroom that a crash between reserve and finalize/release
// can orphan; the detector sweeps and frees them. A few swept leaks is normal noise
// (an occasional crashed request), but a SUSTAINED rate means a reserve→finalize
// path is breaking and silently starving agent spend caps — that degrades health.
// Never-run / table-absent → ok:null so health doesn't fail on a cold feature.
const LEAK_WINDOW_MS = 60 * 60 * 1000; // count leaks swept in the last hour
// At/above this many leaks swept in the window, treat it as a systemic leak.
const LEAK_DEGRADE_THRESHOLD = Math.max(
1,
Number(process.env.X402_RESERVATION_LEAK_HEALTH_THRESHOLD || 25),
);
async function loadReservationLeaks() {
try {
const [row] = await sql`
SELECT count(*)::int AS swept_recent,
coalesce(sum(usd), 0)::float8 AS usd_freed,
coalesce(sum(sol_amount), 0)::float8 AS sol_freed,
count(DISTINCT agent_id)::int AS agents_affected,
extract(epoch FROM max(swept_at)) * 1000 AS last_swept_ms
FROM spend_reservation_leaks
WHERE swept_at > now() - ${`${Math.round(LEAK_WINDOW_MS / 1000)} seconds`}::interval
`;
const sweptRecent = Number(row?.swept_recent || 0);
const lastSwept = row?.last_swept_ms ? Number(row.last_swept_ms) : null;
// Below threshold (including zero) is healthy; at/above is a systemic leak.
const ok = sweptRecent >= LEAK_DEGRADE_THRESHOLD ? false : true;
return {
ok,
sweptRecent,
windowHours: Math.round(LEAK_WINDOW_MS / 3600000),
threshold: LEAK_DEGRADE_THRESHOLD,
usdFreed: Number(row?.usd_freed || 0),
solFreed: Number(row?.sol_freed || 0),
agentsAffected: Number(row?.agents_affected || 0),
lastSwept,
};
} catch (err) {
if (/does not exist|relation .* does not exist/i.test(err?.message || '')) {
return { ok: null, reason: 'table_absent' };
}
return { ok: null, reason: `query_failed:${err?.message || 'unknown'}` };
}
}
// Read the latest per-network circuit-breaker snapshot. The hourly breaker
// upserts one row per network into x402_circuit_breaker; a row older than the
// staleness window means the loop stopped writing. Returns ok:null when the
// feature has never run (table absent / no rows) so health doesn't fail on it.
const BREAKER_STALE_AFTER_MS = 2 * 60 * 60 * 1000; // 2h (hourly cadence + slack)
async function loadCircuitBreaker() {
try {
const rows = await sql`
SELECT network, label, scheme, advertised, route_ok, settled,
receipt_valid, tx_signature, error,
extract(epoch FROM checked_at) * 1000 AS checked_at_ms
FROM x402_circuit_breaker
ORDER BY label ASC
`;
if (!rows.length) return { ok: null, reason: 'no_data', networks: [] };
const now = Date.now();
const networks = rows.map((r) => {
const checkedAt = r.checked_at_ms ? Number(r.checked_at_ms) : null;
const stale = checkedAt === null || now - checkedAt > BREAKER_STALE_AFTER_MS;
return {
network: r.network,
label: r.label,
scheme: r.scheme,
advertised: r.advertised,
routeOk: r.route_ok,
settled: r.settled,
receiptValid: r.receipt_valid,
txSignature: r.tx_signature,
error: r.error,
lastChecked: checkedAt,
stale,
};
});
const routesOk = networks.every((n) => n.routeOk && !n.stale);
// Solana is the network we actually settle on; its settlement is the
// end-to-end proof. Base/BSC are route-verify only.
const solana = networks.find((n) => /solana/i.test(n.label) || /solana/i.test(n.network));
const settlementOk = solana ? solana.settled && !solana.stale : false;
const ok = routesOk && settlementOk;
return {
ok,
routesOk,
settlementOk,
solanaTx: solana?.txSignature || null,
networks,
};
} catch (err) {
// Table not yet created (first boot before the loop runs) → unknown, not failing.
if (/does not exist|relation .* does not exist/i.test(err?.message || '')) {
return { ok: null, reason: 'table_absent', networks: [] };
}
return { ok: null, reason: `query_failed:${err?.message || 'unknown'}`, networks: [] };
}
}