Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(upstreams): harden market, Solana RPC, rate-limit and x402 loop paths against degraded providers, with tests
nirholas7925f1580507
diff --git a/api/_lib/coinpaprika.js b/api/_lib/coinpaprika.jsindex 9c2f20897..72e4549b9 100644--- a/api/_lib/coinpaprika.js+++ b/api/_lib/coinpaprika.js@@ -1,3 +1,5 @@+import { recordSource } from './brownout/provenance.js';+import { applyFault, faultFor } from './brownout/chaos.js';// Shared CoinPaprika client — one place that owns the free tier's budget.//// **The free tier allows sixty requests per HOUR** (hard cap 25k/month). Past@@ -65,24 +67,58 @@ export const PAPRIKA_BASE = 'https://api.coinpaprika.com/v1';* @returns {Promise<any|null>} parsed body, or null on miss/budget/error*/export async function paprikaGet(url, timeoutMs = 8000) {- if (isPaprikaBenched()) return null;+ // `fallback` tier, always: nothing calls CoinPaprika first. It answers a+ // question CoinGecko was supposed to answer, which is a different provider+ // giving a different-but-comparable reading, and a reader deserves to know+ // the number in front of them came from the understudy.+ const startedAt = Date.now();+ if (isPaprikaBenched()) {+ recordSource({ name: 'coinpaprika', outcome: 'skip', ms: 0, tier: 'fallback', detail: 'benched' });+ return null;+ }try {- const res = await fetch(url, {- headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },- signal: AbortSignal.timeout(timeoutMs),- });- if (notePaprikaStatus(res.status)) return null;- if (!res.ok) return null;+ const fault = faultFor('coinpaprika');+ let res;+ if (fault) {+ const injected = await applyFault(fault, url);+ res = injected;+ }+ if (!res) {+ res = await fetch(url, {+ headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },+ signal: AbortSignal.timeout(timeoutMs),+ });+ }+ if (notePaprikaStatus(res.status)) {+ recordSource({ name: 'coinpaprika', outcome: 'fail', ms: Date.now() - startedAt, tier: 'fallback', detail: res.status });+ return null;+ }+ if (!res.ok) {+ recordSource({ name: 'coinpaprika', outcome: 'fail', ms: Date.now() - startedAt, tier: 'fallback', detail: res.status });+ return null;+ }const body = await res.json();// The budget reply is JSON carrying this marker; honour it whatever status// delivered it.if (body?.type === 'payment_required') {benchPaprika();console.warn('[coinpaprika] payment_required — free-tier budget spent; benching for 1h');+ recordSource({ name: 'coinpaprika', outcome: 'fail', ms: Date.now() - startedAt, tier: 'fallback', detail: 'budget' });return null;}- return body ?? null;- } catch {+ if (body != null) {+ recordSource({ name: 'coinpaprika', outcome: 'ok', ms: Date.now() - startedAt, tier: 'fallback' });+ return body;+ }+ return null;+ } catch (err) {+ recordSource({+ name: 'coinpaprika',+ outcome: 'fail',+ ms: Date.now() - startedAt,+ tier: 'fallback',+ detail: err?.name === 'TimeoutError' ? 'timeout' : 'network',+ });return null;}}diff --git a/api/_lib/rate-limit.js b/api/_lib/rate-limit.jsindex fa67b92ef..47a8e6d41 100644--- a/api/_lib/rate-limit.js+++ b/api/_lib/rate-limit.js@@ -1022,6 +1022,27 @@ export const limits = {window: '1 h',critical: true,}).limit('global'),+ // Portal (api/portal): fetches a caller-supplied website once and builds a+ // walkable world from it. The spend is other people's bandwidth rather than+ // ours, which is exactly why it is capped: a build is one origin request plus+ // a robots.txt read, and an uncapped endpoint would turn three.ws into a+ // crawler someone else pays for. Cached worlds do not reach this limiter, so+ // the cap governs distinct pages, not shares of the same link. Non-critical:+ // a Redis blip degrades to the per-instance limiter rather than refusing a+ // visitor a world we can build for free from cache.+ portalBuildIp: (ip) =>+ getLimiter('portal:build:ip', { limit: 30, window: '10 m' }).limit(ip),+ // Fleet-wide brake, so no distribution of IPs can point Portal at one origin+ // hard enough to look like an attack from three.ws.+ portalBuildGlobal: () =>+ getLimiter('portal:build:global', {+ limit: Math.max(200, Number(process.env.PORTAL_BUILD_GLOBAL_HOURLY) || 1200),+ window: '1 h',+ }).limit('global'),+ // GLB export of an already-built world: pure CPU on our side plus a download,+ // so it gets its own, smaller cap.+ portalExportIp: (ip) =>+ getLimiter('portal:export:ip', { limit: 12, window: '10 m' }).limit(ip),// Diorama save (api/diorama action:save) — persists a forged world to the// public gallery table. Anonymous write, so cap per IP to stop one caller// carpeting the gallery; non-critical so an infra hiccup never blocks a save.@@ -1808,6 +1829,21 @@ export const limits = {// Preference-center writes — debounced client, generous ceiling.notifPrefsWrite: (userId) =>getLimiter('notif:prefs:write', { limit: 60, window: '1 h' }).limit(userId),+ // Herald delivery rail (/api/herald/*). Announces are machine-written (CI,+ // crons, agents holding an API key) and land as a physical interruption on+ // the owner's screen, so the ceiling is a spam guard on their own attention+ // as much as on our Redis: 60/min is far above any sane integration and far+ // below a runaway loop. NOT local: the whole point is a global cap on how+ // often anything can interrupt one person, and a per-instance counter would+ // multiply that by the instance count.+ heraldAnnounce: (userId) =>+ getLimiter('herald:announce', { limit: 60, window: '1 m' }).limit(userId),+ // The SSE listener. One connection per open tab, re-established every ~5+ // minutes when the stream hits its duration cap, so this only has to stop a+ // reconnect storm. local: a connection budget per instance is the thing that+ // actually protects that instance.+ heraldStream: (userId) =>+ getLimiter('herald:stream', { limit: 60, window: '5 m', local: true }).limit(userId),// Companion (/api/companion/*). Reads are a page load plus a poll while the// setup page is open, so they get the same generous local bucket the bell// inbox uses. Writes touch encrypted credentials and are human-paced.@@ -1836,6 +1872,22 @@ export const limits = {// a loop does not.companionIngest: (token) =>getLimiter('companion:ingest', { limit: 120, window: '5 m' }).limit(token),+ // Knock (/api/knock/*). The owner's own reads are a page load plus polling+ // while the inbox is open.+ knockRead: (userId) =>+ getLimiter('knock:read', { limit: 120, window: '1 m', local: true }).limit(userId),+ knockWrite: (userId) =>+ getLimiter('knock:write', { limit: 90, window: '10 m' }).limit(userId),+ // Public door + directory reads, keyed by IP. A door page is one read; a+ // crawler walking the directory stays inside this.+ knockPublic: (ip) =>+ getLimiter('knock:public', { limit: 120, window: '5 m' }).limit(ip),+ // Free-door knocks (POST /api/knock/send), keyed by IP. Free means there is+ // no price to make a flood expensive, so the ceiling has to do that job:+ // a person writing to a few people is fine, a script is not. Priced doors+ // go through the x402 lane and are limited by their own cost.+ knockSendIp: (ip) =>+ getLimiter('knock:send:ip', { limit: 8, window: '1 h' }).limit(ip),// Funnel tracking (opened/returned) — high local ceiling; one ping per// notification interaction, deduped server-side anyway.notifTrack: (userId) =>diff --git a/api/_lib/solana/connection.js b/api/_lib/solana/connection.jsindex d4f7a924b..02776049d 100644--- a/api/_lib/solana/connection.js+++ b/api/_lib/solana/connection.js@@ -1366,6 +1366,44 @@ export function rpcBatchCaps(now = Date.now()) {// chatter (the source of the recurring "[solana-rpc] … 429, cooling" warnings).// The genuinely actionable condition: every provider in the chain failing within// one request, so the caller gets nothing back: is the only WARN.+// Brownout: report each lane attempt against the request that caused it, so a+// response assembled through RPC failover can say which node answered and how+// many refused first. Loaded the same lazy way as the shared cache above, for+// the same reason: this module is in the browser bundle and node:async_hooks is+// not. In a browser both hooks are inert.+let _brownoutMod = null;+let _brownoutPromise;+function brownoutModule() {+ if (_brownoutPromise !== undefined) return _brownoutPromise;+ _brownoutPromise = null;+ if (isServer()) {+ _brownoutPromise = import('../brownout/index.js')+ .then((m) => {+ _brownoutMod = m;+ return m;+ })+ .catch(() => null);+ }+ return _brownoutPromise;+}+brownoutModule();++/** Record one lane attempt. Synchronous once the module is in hand, so a record can never arrive after its own response. */+function noteRpc(url, outcome, ms, tier, detail) {+ if (!_brownoutMod) return;+ _brownoutMod.recordSource?.({ name: `solana-rpc:${maskUrl(url).replace(/^https?:\/\//, '')}`, outcome, ms, tier, detail });+}++/** A fault declared for the Solana RPC lane, or for one specific host in it. */+function rpcFault(url) {+ if (!_brownoutMod) return null;+ return (+ _brownoutMod.faultFor?.('solana-rpc') ??+ _brownoutMod.faultFor?.(`solana-rpc:${maskUrl(url).replace(/^https?:\/\//, '')}`) ??+ null+ );+}+export function makeRotatingFetch(endpoints) {return async function rotatingFetch(_info, init) {// The call shapes in this request. Empty for an unreadable body, which makes@@ -1443,6 +1481,7 @@ export function makeRotatingFetch(endpoints) {};const tryEndpoint = async (url) => {+ const attemptStartedAt = Date.now();// Bound every attempt so one hanging provider can never absorb the whole// request budget (undici's default timeouts run to minutes): the attempt// aborts, cools briefly, and the rotation moves on. A caller-supplied@@ -1463,7 +1502,12 @@ export function makeRotatingFetch(endpoints) {// probe request that only re-learns it and go straight to singles.if (splittable && isBatchCapped(url, methods)) return { response: await sendUnbatched(url, signal) };- const resp = await fetch(url, { ...init, signal });+ // A declared fault stands in for the lane failing, inside the same try+ // the real failure would land in, so cooldowns, method demotion and+ // the re-sweep all behave exactly as they would in an outage.+ const fault = rpcFault(url);+ const injected = fault ? await _brownoutMod.applyFault(fault, url) : null;+ const resp = injected || (await fetch(url, { ...init, signal }));// A per-batch cap is the lane's own policy, not a caller error and not a// lane fault, so it neither rotates nor cools: unroll and re-send here.if (splittable && !resp.ok) {@@ -1495,6 +1539,7 @@ export function makeRotatingFetch(endpoints) {.catch(() => ''): '';penalise(url, resp.status, bodyText, isMethodRefusal(bodyText), String(resp.status));+ noteRpc(url, 'fail', Date.now() - attemptStartedAt, 'live', resp.status);return { error: new Error(`solana rpc ${resp.status} @ ${maskUrl(url)}`) };}const okBody = await resp.text();@@ -1516,6 +1561,7 @@ export function makeRotatingFetch(endpoints) {// the same payload. Only content-type is preserved; copying// content-encoding/content-length would mislead the consumer since the// transport already decoded the body into `okBody`.+ noteRpc(url, 'ok', Date.now() - attemptStartedAt, 'live');return {response: new Response(okBody, {status: resp.status,@@ -1541,6 +1587,7 @@ export function makeRotatingFetch(endpoints) {// message, so the exhausted-chain warning names the real fault// (EAI_AGAIN is a resolver problem, ECONNRESET a socket one) instead of// undici's bare "fetch failed". The original stays on `cause`.+ noteRpc(url, 'fail', Date.now() - attemptStartedAt, 'live', err?.name === 'TimeoutError' ? 'timeout' : 'network');const cause = disposition === 'cool-lane' ? transportCause(err) : '';if (cause) {const tagged = new Error(`${err?.message || 'fetch failed'} (${cause}) @ ${maskUrl(url)}`, { cause: err });@@ -1620,6 +1667,7 @@ export function makeRotatingFetch(endpoints) {// through to the error, because a stale answer to those is worse than none.const stale = await serveLastGood(staleKey, init?.body);if (stale) {+ noteRpc('last-good', 'ok', Number(stale.headers.get('x-solana-rpc-stale')) || 0, 'stale', 'stale');logStaleServe(methods[0] || 'read', Number(stale.headers.get('x-solana-rpc-stale')) || 0, lastErr?.message || 'unknown error');return stale;}diff --git a/api/_lib/token-market.js b/api/_lib/token-market.jsindex 03e946602..a52a8e7c3 100644--- a/api/_lib/token-market.js+++ b/api/_lib/token-market.js@@ -95,7 +95,16 @@ export async function fetchTokenMarket(ca, opts = {}) {{ maxAgeMs: 15 * 60_000 },);data = out.value;- } catch {+ } catch (err) {+ // Null keeps the contract every existing caller was written against, but+ // the distinction is preserved on the error for the ones that must not+ // confuse "no market exists" with "the indexer could not be reached".+ // fetchTokenMarketResult below is how a caller asks for it.+ if (opts.__surfaceUnavailable) throw Object.assign(new Error('market indexer unreachable'), {+ code: 'market_unavailable',+ status: 503,+ cause: err,+ });return null;}let pairs = Array.isArray(data?.pairs) ? data.pairs : [];@@ -378,3 +387,26 @@ export function buildTokenSignal(m) {const confidence = Math.min(0.93, 0.64 + Math.min(Math.abs(change) / 20, 0.29));return { signal, headline, rationale, confidence };}++/**+ * Like fetchTokenMarket, but tells apart the two ways it can come back empty.+ *+ * `fetchTokenMarket` answers null both for a token with no indexed pair and for+ * an indexer that could not be reached at all, which is fine for a decorative+ * price chip and wrong for anything that states a conclusion about the token.+ * Reporting "no such token" during an outage denies real tokens; two endpoints+ * shipped that bug before this existed.+ *+ * @param {string} ca+ * @param {object} [opts] same options as fetchTokenMarket+ * @returns {Promise<{ market: object | null, unavailable: boolean }>}+ */+export async function fetchTokenMarketResult(ca, opts = {}) {+ try {+ const market = await fetchTokenMarket(ca, { ...opts, __surfaceUnavailable: true });+ return { market, unavailable: false };+ } catch (err) {+ if (err?.code === 'market_unavailable') return { market: null, unavailable: true };+ throw err;+ }+}diff --git a/api/ca2x402/resolve.js b/api/ca2x402/resolve.jsindex 4a46faad9..43e696ee9 100644--- a/api/ca2x402/resolve.js+++ b/api/ca2x402/resolve.js@@ -23,6 +23,7 @@ import { priceFor } from '../_lib/x402-prices.js';import { env } from '../_lib/env.js';import {fetchTokenMarket,+ fetchTokenMarketResult,buildTokenSignal,buildTokenRisk,isResolvableAddress,@@ -87,11 +88,25 @@ async function handler(req, res) {);}- let market = null;- try {- market = await fetchTokenMarket(mint);- } catch {- /* upstream hiccup — fall through to the not-found envelope below */+ // "The indexer says this token has no market" and "the indexer could not be+ // reached" are different answers, and only the first one is the caller's+ // business. fetchTokenMarket serves a remembered payload when it has one, so+ // a throw here means a cold instance during an outage: answering the designed+ // 404 there tells someone their real token does not exist, which is the exact+ // claim the last-good tier was added to stop making.+ const { market, unavailable } = await fetchTokenMarketResult(mint);+ if (unavailable) {+ res.setHeader('cache-control', 'no-store');+ res.setHeader('Retry-After', '15');+ return json(res, 503, {+ ok: false,+ mint,+ chain: chainOf(mint),+ error: 'market_unavailable',+ error_description:+ 'The market indexer could not be reached, so this address could not be checked. Retry shortly.',+ retry_after: 15,+ });}if (!market) {diff --git a/src/avatar-page.js b/src/avatar-page.jsindex 43e971ac0..55f0665f2 100644--- a/src/avatar-page.js+++ b/src/avatar-page.js@@ -1211,6 +1211,9 @@ function renderEmbedPanel(glbUrl) {allow="autoplay; xr-spatial-tracking"></iframe>`;const linkSnippet = fullUrl;+ // The terminal viewer resolves an agent to its bound body through+ // /api/agents/:id, so the agent id is the right handle in agent mode.+ const ttySnippet = `npx @three-ws/tty-avatar ${mode === 'agent' ? `agent:${entityId}` : avatarId}`;const wizardUrl = `/embed?${new URLSearchParams(mode === 'agent' ? { agent: entityId, ...(avatarId ? { avatar: avatarId } : {}) } : { avatar: avatarId },)}`;@@ -1240,6 +1243,14 @@ function renderEmbedPanel(glbUrl) {</div><pre class="av-embed-code" id="embed-link">${esc(linkSnippet)}</pre></div>+ <div class="av-embed-section">+ <div class="av-embed-label">+ <span>Terminal</span>+ <button class="av-embed-copy" data-copy="tty">Copy</button>+ </div>+ <pre class="av-embed-code" id="embed-tty">${esc(ttySnippet)}</pre>+ <p class="av-embed-intro" style="margin:8px 0 0;font-size:12px;color:var(--ink-dim)">Runs this ${mode} live in any terminal, no browser or GPU. Add <code>install-hooks --write</code> and it becomes your Claude Code agent's face. <a href="/docs/tty-avatar" target="_blank" rel="noopener">How it works ↗</a></p>+ </div>`;}@@ -1509,7 +1520,7 @@ function bindTabs() {const btn = e.target.closest('.av-embed-copy');if (!btn) return;const which = btn.dataset.copy;- const sourceMap = { wc: 'embed-wc', iframe: 'embed-iframe', link: 'embed-link' };+ const sourceMap = { wc: 'embed-wc', iframe: 'embed-iframe', link: 'embed-link', tty: 'embed-tty' };const src = $(sourceMap[which]);if (!src) return;try {diff --git a/tests/brownout-core.test.js b/tests/brownout-core.test.jsindex ba621b382..95b6daf44 100644--- a/tests/brownout-core.test.js+++ b/tests/brownout-core.test.js@@ -87,6 +87,27 @@ describe('provenance ledger', () => {});});+ it('does NOT call an ordinary cache hit degraded', () => {+ // Almost every response on a healthy site is a cache hit. Counting those as+ // degraded would mark the whole platform degraded and train every reader to+ // ignore the flag, which is worse than not having one.+ withProvenance(() => {+ recordSource({ name: 'coingecko', outcome: 'ok', ms: 0, tier: 'cache' });+ const s = provenanceSummary();+ expect(s.tier).toBe('cache');+ expect(s.degraded).toBe(false);+ });+ });++ it('calls a stale or fallback answer degraded, because it is worse than intended', () => {+ for (const tier of ['stale', 'fallback']) {+ withProvenance(() => {+ recordSource({ name: 'src', outcome: 'ok', ms: 1, tier });+ expect(provenanceSummary().degraded, tier).toBe(true);+ });+ }+ });+it('counts a failure as degraded even when the answer that came back was live', () => {// A ladder that failed over still tells the reader the chain was exercised.withProvenance(() => {diff --git a/tests/upstream-degradation.test.js b/tests/upstream-degradation.test.jsnew file mode 100644index 000000000..82d39f991--- /dev/null+++ b/tests/upstream-degradation.test.js@@ -0,0 +1,193 @@+// Degradation contracts, proven by breaking the upstream.+//+// Every endpoint below declares what it is ALLOWED to say when the services it+// depends on are entirely unreachable. The test then makes that true: it swaps+// global fetch for one that fails the way a dead host fails, calls the real+// handler, and judges the real response.+//+// This exists because "does it have a fallback" is the wrong review question. In+// one week this repo shipped four fallbacks that were individually reasonable+// and collectively dangerous: a payment retry that re-signed the blockhash the+// facilitator had just refused, a token security verdict assembled from+// remembered on-chain state while the chain was unreadable, an access gate+// answering from a cached verdict, and an alert feed that went silent in a way+// that looked like a calm market. Each passed review. None had a stated answer+// to "when this is down, what may this endpoint claim?".+//+// Adding an endpoint here is one line. See scripts/resilience/contracts.mjs for+// the vocabulary and docs/resilience.md for the generated upstream map.++import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';+import { CONTRACT, judge, failingFetch } from '../scripts/resilience/contracts.mjs';++vi.mock('../api/_lib/zauth.js', () => ({ instrument: () => {}, drain: async () => {} }));+vi.mock('../api/_lib/sentry.js', () => ({ captureException: () => {} }));+vi.mock('../api/_lib/rate-limit.js', () => ({+ limits: new Proxy({}, { get: () => async () => ({ success: true, remaining: 100 }) }),+ clientIp: () => '203.0.113.9',+}));++function makeRes() {+ return {+ statusCode: 200,+ _h: {},+ setHeader(k, v) { this._h[String(k).toLowerCase()] = v; },+ getHeader(k) { return this._h[String(k).toLowerCase()]; },+ status(code) { this.statusCode = code; return this; },+ json(payload) { this._body = JSON.stringify(payload); return this; },+ end(body) { if (body !== undefined) this._body = body; },+ };+}++/**+ * Load a handler with a genuinely cold module registry, so a value remembered+ * by a previous case cannot answer this one and make it pass for the wrong+ * reason. Every module in this area keeps per-instance state on purpose.+ */+async function loadHandler(path) {+ vi.resetModules();+ const mod = await import(path);+ return mod.default ?? mod.handler;+}++async function invoke(handler, url, { method = 'GET' } = {}) {+ const res = makeRes();+ let threw;+ try {+ await handler({ url, method, headers: { host: 'three.ws' }, query: {}, socket: { remoteAddress: '203.0.113.9' } }, res);+ } catch (err) {+ threw = err;+ }+ let body = null;+ if (res._body) {+ try {+ body = JSON.parse(res._body);+ } catch {+ body = res._body;+ }+ }+ return { status: res.statusCode, body, headers: res._h, threw };+}++// The declared contracts. `warm` marks endpoints whose contract only makes sense+// once something has been remembered; those are exercised twice, first with a+// working upstream so there is a remembered value to serve.+const ENDPOINTS = [+ {+ name: 'GET /api/v1/token/security',+ module: '../api/v1/token/security.js',+ url: '/api/v1/token/security?address=FeMbDoX7R1Psc4GEcvJdsbNbZA3bfztcyDCatJVJpump',+ contract: CONTRACT.MUST_NOT_SERVE_STALE,+ why: 'mint and freeze authority and holder concentration are the whole verdict, so a remembered copy served as live is a clean bill of health for a chain nobody can see',+ },+ {+ name: 'GET /api/coin/fear-greed',+ module: '../api/coin/fear-greed.js',+ url: '/api/coin/fear-greed',+ contract: CONTRACT.MAY_SERVE_STALE,+ why: 'a sentiment reading minutes old is a better answer than a blank panel',+ },+ {+ name: 'GET /api/defi/chains',+ module: '../api/defi/chains.js',+ url: '/api/defi/chains',+ contract: CONTRACT.MAY_SERVE_STALE,+ why: 'TVL by chain moves slowly and the page is decorative if it is empty',+ },+ {+ name: 'GET /api/defi/protocols',+ module: '../api/defi/protocols.js',+ url: '/api/defi/protocols',+ contract: CONTRACT.MAY_SERVE_STALE,+ why: 'same board, same reasoning as chains',+ },+ {+ name: 'GET /api/ca2x402/resolve',+ module: '../api/ca2x402/resolve.js',+ url: '/api/ca2x402/resolve?mint=FeMbDoX7R1Psc4GEcvJdsbNbZA3bfztcyDCatJVJpump',+ contract: CONTRACT.MAY_SERVE_STALE,+ why: 'a throttled indexer used to make a live token read as "not found", which denied a real token downstream',+ },+];++describe('upstream degradation contracts', () => {+ let realFetch;++ beforeEach(() => {+ realFetch = globalThis.fetch;+ });++ afterEach(() => {+ globalThis.fetch = realFetch;+ vi.unstubAllGlobals();+ });++ for (const ep of ENDPOINTS) {+ for (const mode of ['network', 'status']) {+ it(`${ep.name} honours "${ep.contract}" when every upstream fails (${mode})`, async () => {+ const handler = await loadHandler(ep.module);+ globalThis.fetch = failingFetch(mode);++ const observed = await invoke(handler, ep.url);+ const verdict = judge(ep.contract, observed);++ expect(+ verdict.ok,+ `${ep.name} declares ${ep.contract} because ${ep.why}.\n` ++ `Observed: status=${observed.status} ` ++ `body=${JSON.stringify(observed.body)?.slice(0, 200)} ` ++ `threw=${observed.threw ? observed.threw.message : 'no'}\n` ++ `Verdict: ${verdict.reason}`,+ ).toBe(true);+ });+ }+ }++ it('never answers a total outage with an untyped 500', async () => {+ // The one rule every endpoint shares regardless of its contract: a caller+ // must be able to tell "try again shortly" from "your request was wrong".+ // An untyped 500 says neither, and is what an unhandled rejection looks+ // like from the outside.+ for (const ep of ENDPOINTS) {+ const handler = await loadHandler(ep.module);+ globalThis.fetch = failingFetch('network');+ const observed = await invoke(handler, ep.url);+ expect(observed.status, `${ep.name} answered an untyped 500`).not.toBe(500);+ if (observed.threw) {+ const typed = observed.threw.status ?? observed.threw.statusCode ?? observed.threw.code;+ expect(typed, `${ep.name} threw an untyped error out of the handler`).toBeTruthy();+ }+ }+ });+});++describe('contract judgement', () => {+ // The judge decides whether a real endpoint passes, so its own reasoning is+ // worth pinning: a bug here would quietly pass every endpoint above.+ it('rejects an unmarked 200 under may-serve-stale', () => {+ expect(judge(CONTRACT.MAY_SERVE_STALE, { status: 200, body: { total: 1 } }).ok).toBe(false);+ });++ it('accepts a marked 200 under may-serve-stale', () => {+ expect(judge(CONTRACT.MAY_SERVE_STALE, { status: 200, body: { total: 1, stale: true } }).ok).toBe(true);+ expect(+ judge(CONTRACT.MAY_SERVE_STALE, { status: 200, body: { total: 1 }, headers: { 'x-three-stale': '1' } }).ok,+ ).toBe(true);+ });++ it('rejects any 2xx under must-refuse and must-not-serve-stale', () => {+ expect(judge(CONTRACT.MUST_REFUSE, { status: 200, body: {} }).ok).toBe(false);+ expect(judge(CONTRACT.MUST_NOT_SERVE_STALE, { status: 200, body: { stale: true } }).ok).toBe(false);+ });++ it('accepts a typed refusal and rejects an untyped one', () => {+ expect(judge(CONTRACT.MUST_REFUSE, { status: 503, body: { error: 'sources_unavailable' } }).ok).toBe(true);+ expect(judge(CONTRACT.MUST_REFUSE, { status: 500, body: {} }).ok).toBe(false);+ expect(judge(CONTRACT.MUST_REFUSE, { status: 404, body: {} }).ok).toBe(false);+ });++ it('treats a thrown typed error as a refusal but an untyped throw as a failure', () => {+ expect(judge(CONTRACT.MUST_REFUSE, { threw: Object.assign(new Error('x'), { status: 503 }) }).ok).toBe(true);+ expect(judge(CONTRACT.MUST_REFUSE, { threw: new Error('boom') }).ok).toBe(false);+ });+});