Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(forge): cap the seeder circuit breaker so one outage cannot park the cron for a day

nirholascdd9e2488498
diff --git a/api/_lib/forge-scale.js b/api/_lib/forge-scale.js
index 63d7f8fc7..b463d906d 100644
--- a/api/_lib/forge-scale.js
+++ b/api/_lib/forge-scale.js
@@ -323,21 +323,37 @@ export async function circuitState(name) {
}
// Record one failure; opens the breaker for (failures × baseMs) once threshold is
-// reached. Returns the new consecutive-failure count.
-export async function circuitRecordFailure(name, { threshold, baseMs }) {
+// reached, clamped to maxMs. Returns the new consecutive-failure count.
+//
+// The clamp is load-bearing. `failures` only resets on a success, and the counter
+// key is refreshed on every recorded failure, so a provider outage that outlives
+// one retry cycle keeps extending the window: 12 consecutive failures already park
+// a per-minute cron for two hours, and it grows from there with no natural
+// ceiling. That turns a transient upstream blip into a seeder that has gone quiet
+// for most of a day, which reads from the outside as a dead cron rather than a
+// backed-off one. Capping the window keeps the retry cadence bounded, so the cron
+// re-probes the provider at least once per CIRCUIT_MAX_OPEN_MS and resumes on its
+// own the moment the outage clears.
+const CIRCUIT_MAX_OPEN_MS = 60 * 60_000;
+
+const openWindowMs = (failures, baseMs, maxMs) => Math.min(failures * baseMs, maxMs);
+
+export async function circuitRecordFailure(name, { threshold, baseMs, maxMs = CIRCUIT_MAX_OPEN_MS }) {
if (!redis) {
const c = memCircuit(name);
c.failures++;
- if (c.failures >= threshold) c.openUntil = Date.now() + c.failures * baseMs;
+ if (c.failures >= threshold) c.openUntil = Date.now() + openWindowMs(c.failures, baseMs, maxMs);
return c.failures;
}
try {
const failures = await redis.incr(`${CIRCUIT_PREFIX}${name}:failures`);
await redis.expire(`${CIRCUIT_PREFIX}${name}:failures`, CIRCUIT_TTL_S);
if (failures >= threshold) {
- await redis.set(`${CIRCUIT_PREFIX}${name}:openUntil`, String(Date.now() + failures * baseMs), {
- ex: CIRCUIT_TTL_S,
- });
+ await redis.set(
+ `${CIRCUIT_PREFIX}${name}:openUntil`,
+ String(Date.now() + openWindowMs(failures, baseMs, maxMs)),
+ { ex: CIRCUIT_TTL_S },
+ );
}
return failures;
} catch {
diff --git a/api/cron/forge-seed-cron.js b/api/cron/forge-seed-cron.js
index 5e3dc9333..9f49585a9 100644
--- a/api/cron/forge-seed-cron.js
+++ b/api/cron/forge-seed-cron.js
@@ -117,7 +117,9 @@ const gateBatchSize = () => intEnv('SEED_CRON_GATE', 2, { min: 1, max: 10 });
// every cron lambda sees the same open/closed decision; without that, each instance
// would rediscover a provider outage on its own and keep submitting into a dead
// lane. When CIRCUIT_THRESHOLD consecutive forge submits fail the circuit opens for
-// (failures × CIRCUIT_BASE_MS) so the cron goes quiet during provider outages.
+// (failures × CIRCUIT_BASE_MS), capped by forge-scale's CIRCUIT_MAX_OPEN_MS, so the
+// cron goes quiet during a provider outage but still re-probes the lane at least
+// once an hour instead of backing off into silence for the rest of the day.
const CIRCUIT_NAME = 'forge-seed';
const CIRCUIT_THRESHOLD = 3;
const CIRCUIT_BASE_MS = 10 * 60_000; // 10 min × consecutive failures
diff --git a/scripts/audit-console.mjs b/scripts/audit-console.mjs
index 7cb6cb6f4..536a230c7 100644
--- a/scripts/audit-console.mjs
+++ b/scripts/audit-console.mjs
@@ -105,6 +105,14 @@ function freePort() {
const PROBE_BASE = 'http://127.0.0.1:3000';
+// Dev-server readiness budget. A shared/loaded box (several agents running
+// browser fleets and test workers at once) starves Vite's boot: it answers
+// eventually, just not inside a short window. Waiting longer only delays the
+// sweep on a busy machine; it never changes what counts as a console error.
+// Raise SERVER_BOOT_MS if a run still dies with "vite did not become ready".
+const SERVER_BOOT_MS = Number(process.env.SERVER_BOOT_MS || 240_000);
+const REUSE_PROBE_MS = Number(process.env.REUSE_PROBE_MS || 20_000);
+
async function warmupDeps(base) {
// Hit a few dep-heavy pages so Vite pre-bundles before the timed sweep,
// otherwise the first real navigation eats the optimizer's reload.
@@ -119,7 +127,7 @@ async function warmupDeps(base) {
}
async function startServer() {
- if (await probe(`${PROBE_BASE}/`, 5000)) {
+ if (await probe(`${PROBE_BASE}/`, REUSE_PROBE_MS)) {
console.log(C.d(' reusing dev server on :3000'));
// localhost (not 127.0.0.1) — some CDN CORS configs allow it, fewer spurious errors.
const navBase = 'http://localhost:3000';
@@ -135,11 +143,11 @@ async function startServer() {
});
const probeBase = `http://127.0.0.1:${port}`;
const navBase = `http://localhost:${port}`;
- const deadline = Date.now() + 90_000;
+ const deadline = Date.now() + SERVER_BOOT_MS;
process.stdout.write(` starting Vite on :${port} `);
while (Date.now() < deadline) {
if (child.exitCode != null) throw new Error(`vite exited early (code ${child.exitCode})`);
- if (await probe(`${probeBase}/`, 2000)) {
+ if (await probe(`${probeBase}/`, 10_000)) {
process.stdout.write('\n');
break;
}
@@ -148,7 +156,7 @@ async function startServer() {
}
if (Date.now() >= deadline) {
child.kill('SIGKILL');
- throw new Error('vite did not become ready within 90s');
+ throw new Error(`vite did not become ready within ${Math.round(SERVER_BOOT_MS / 1000)}s`);
}
await warmupDeps(navBase);
return { base: navBase, stop: async () => child.kill('SIGTERM') };
diff --git a/scripts/audit-csp.mjs b/scripts/audit-csp.mjs
index a27af70b8..adfa62112 100644
--- a/scripts/audit-csp.mjs
+++ b/scripts/audit-csp.mjs
@@ -31,12 +31,21 @@
// documented transform (drop 'unsafe-inline', add 'sha256-…'); every other
// directive and every other security header must match the route table exactly.
//
+// `--headers-only` answers the narrower question ("does this origin serve the
+// headers the route table declares?") with a plain fetch and no browser. It is
+// the only mode that can sweep production honestly: a real browser loading 200
+// live pages is minutes of work and is at the mercy of whatever else the
+// machine is doing, while the header comparison is deterministic. It proves
+// strictly less than the full run (nothing evaluates the policy), so it is an
+// addition to the browser sweep, never a substitute for it.
+//
// Usage:
// node server/index.mjs & # or any origin serving dist/
// node scripts/audit-csp.mjs # sweeps the default page set
// node scripts/audit-csp.mjs --base http://127.0.0.1:8099
// node scripts/audit-csp.mjs --base https://three.ws # audit production
// node scripts/audit-csp.mjs --all # every page in data/pages.json
+// node scripts/audit-csp.mjs --headers-only --base https://three.ws --all
// node scripts/audit-csp.mjs /pay /studio # only these paths
import { readFileSync } from 'node:fs';
@@ -44,6 +53,7 @@ import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { chromium } from 'playwright';
import { loadRouteTable, resolvePhase1 } from '../server/route-resolve.mjs';
+import { headerProblems as compareHeaders } from './lib/csp-headers.mjs';
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
@@ -82,6 +92,7 @@ const args = process.argv.slice(2);
const baseIdx = args.indexOf('--base');
const BASE = baseIdx === -1 ? 'http://127.0.0.1:8099' : args[baseIdx + 1];
const wantAll = args.includes('--all');
+const headersOnly = args.includes('--headers-only');
const explicit = args.filter((a) => a.startsWith('/'));
function allCataloguedPaths() {
@@ -103,16 +114,10 @@ const SETTLE_MS = Number(process.env.CSP_AUDIT_SETTLE_MS) || 3500;
const { phase1Routes } = loadRouteTable(path.join(ROOT, 'vercel.json'));
-// The security headers vercel.json is expected to put on a document. Anything
-// the route table declares for a path is checked; this list is what makes a
-// MISSING declaration visible too, so deleting the global header rule fails
-// here instead of shipping a bare-headed site.
-const REQUIRED_ON_HTML = [
- 'content-security-policy',
- 'strict-transport-security',
- 'x-content-type-options',
- 'referrer-policy',
-];
+// What counts as a header problem lives in scripts/lib/csp-headers.mjs, next to
+// the tests that pin it. It checks the headers a path declares AND that the
+// document ones are declared at all, so deleting the global header rule from
+// vercel.json fails here instead of shipping a bare-headed site.
/** The header bag vercel.json declares for `pathname`, per the server's resolver. */
function declaredHeaders(pathname, base) {
@@ -125,93 +130,41 @@ function declaredHeaders(pathname, base) {
return bag;
}
-/** "a 'b' c; d 'e'" -> Map{ a => Set{'b','c'}, d => Set{'e'} } */
-function parsePolicy(value) {
- const directives = new Map();
- for (const chunk of String(value).split(';')) {
- const parts = chunk.trim().split(/\s+/).filter(Boolean);
- if (!parts.length) continue;
- directives.set(parts[0].toLowerCase(), new Set(parts.slice(1)));
- }
- return directives;
-}
-
-const setsEqual = (a, b) => a.size === b.size && [...a].every((v) => b.has(v));
-
-/**
- * Differences between the policy the route table declares and the one the
- * response carried, allowing only the rewrite server/csp-hashes.mjs performs:
- * within script-src / script-src-elem, `'unsafe-inline'` is replaced by any
- * number of `'sha256-…'` sources. Everything else must survive byte-for-byte.
- */
-function policyDiff(declared, served) {
- const want = parsePolicy(declared);
- const got = parsePolicy(served);
- const problems = [];
- for (const [name, wantSources] of want) {
- const gotSources = got.get(name);
- if (!gotSources) {
- problems.push(`directive "${name}" is missing from the served policy`);
- continue;
- }
- const hashable = name === 'script-src' || name === 'script-src-elem';
- const expected = new Set(wantSources);
- const actual = new Set(gotSources);
- if (hashable) {
- expected.delete("'unsafe-inline'");
- for (const src of [...actual]) if (src.startsWith("'sha256-")) actual.delete(src);
- }
- if (setsEqual(expected, actual)) continue;
- const missing = [...expected].filter((s) => !actual.has(s));
- const extra = [...actual].filter((s) => !expected.has(s));
- problems.push(
- `directive "${name}" differs${missing.length ? ` (missing ${missing.join(' ')})` : ''}` +
- `${extra.length ? ` (unexpected ${extra.join(' ')})` : ''}`,
- );
- }
- return problems;
-}
-
/**
* Every way the response's security headers fail to be what the repo says it
* serves for this path.
*/
function headerProblems(pathname, base, servedHeaders) {
- const want = declaredHeaders(pathname, base);
- const problems = [];
-
- for (const name of REQUIRED_ON_HTML) {
- if (!want[name]) problems.push(`vercel.json declares no ${name} for this path`);
- }
+ return compareHeaders(declaredHeaders(pathname, base), servedHeaders);
+}
- for (const [name, expected] of Object.entries(want)) {
- // Only security headers are the subject here. Cache and CORS values are
- // legitimately rewritten by the CDN and by conditional-request handling.
- if (!name.startsWith('x-') && !REQUIRED_ON_HTML.includes(name) && name !== 'permissions-policy') {
- continue;
- }
- const served = servedHeaders[name];
- if (served === undefined) {
- problems.push(`${name} was declared but the response did not carry it`);
- continue;
- }
- if (name === 'content-security-policy') {
- problems.push(...policyDiff(expected, served));
- continue;
- }
- if (served.trim() !== expected.trim()) {
- problems.push(`${name} is "${served}" but vercel.json declares "${expected}"`);
+/**
+ * The header half of the audit for one path, over a plain request. No policy is
+ * evaluated: a clean result here means the response carried what vercel.json
+ * declares, nothing more.
+ */
+async function auditHeaders(p) {
+ const violations = [];
+ const errors = [];
+ const headers = [];
+ let status = 0;
+ try {
+ const res = await fetch(BASE + p, { redirect: 'follow' });
+ await res.arrayBuffer();
+ status = res.status;
+ if (status < 400) {
+ const bag = {};
+ for (const [k, v] of res.headers) bag[k.toLowerCase()] = v;
+ headers.push(...headerProblems(new URL(res.url).pathname, BASE, bag));
}
+ } catch (err) {
+ errors.push(`request: ${err.message}`);
}
- return problems;
+ return { status, violations, errors, headers };
}
-const browser = await chromium.launch({
- args: ['--no-sandbox', '--disable-dev-shm-usage'],
-});
-
-const results = [];
-for (const p of paths) {
+/** The full audit for one path: load it in a real browser and collect what the policy blocked. */
+async function auditInBrowser(browser, p) {
const context = await browser.newContext();
const page = await context.newPage();
const violations = [];
@@ -253,6 +206,18 @@ for (const p of paths) {
errors.push(`navigation: ${err.message}`);
}
await context.close();
+ return { status, violations, errors, headers };
+}
+
+const browser = headersOnly
+ ? null
+ : await chromium.launch({ args: ['--no-sandbox', '--disable-dev-shm-usage'] });
+
+const results = [];
+for (const p of paths) {
+ const { status, violations, errors, headers } = headersOnly
+ ? await auditHeaders(p)
+ : await auditInBrowser(browser, p);
// A page that never loaded, or answered with an error status, proves
// nothing: it has no inline scripts to block. Counting it as clean is how a
// sweep reports success over a wiped dist/ or a server that died halfway.
@@ -275,7 +240,7 @@ for (const p of paths) {
console.log(`[${mark}] ${String(status).padEnd(3)} ${p}${note}`);
}
-await browser.close();
+if (browser) await browser.close();
const failed = results.filter((r) => r.violations.length > 0);
const unreachable = results.filter((r) => r.unreachable);
@@ -305,7 +270,11 @@ if (misheaded.length) {
if (failed.length === 0) {
console.log(`\nCSP audit: clean across ${results.length} page(s) on ${BASE}`);
- console.log('Each page loaded, carried the policy vercel.json declares, and fired no violation.');
+ console.log(
+ headersOnly
+ ? 'Each response carried the headers vercel.json declares. No policy was evaluated: rerun without --headers-only for that.'
+ : 'Each page loaded, carried the policy vercel.json declares, and fired no violation.',
+ );
process.exit(0);
}
diff --git a/scripts/audit-links.mjs b/scripts/audit-links.mjs
index be9ebf906..c6baba128 100644
--- a/scripts/audit-links.mjs
+++ b/scripts/audit-links.mjs
@@ -74,9 +74,17 @@ const ASSET_CATCHALL = '/(.*\\.(?:svg|png|jpg|jpeg|gif|webp|avif|ico|woff2|woff|
const isCatchAll = (src) => src === '/(.*)' || src === ASSET_CATCHALL;
const isRedirect = (r) => typeof r.status === 'number' && r.status >= 300 && r.status < 400;
+// The terminal `{"src":"/(?!_vercel/).*","status":404,"dest":"/404.html"}` rewrite
+// is the miss handler, not a destination. It carries a dest and its src is not the
+// literal `/(.*)` catch-all, so without this it entered the route table, matched
+// every path, and made every root-absolute link resolve: the broken-link count
+// read 0 because nothing could ever be counted, not because nothing was broken.
+const isErrorFallback = (r) => typeof r.status === 'number' && r.status >= 400;
// Specific routes that resolve a request to something: a rewrite (dest) OR a
// 3xx redirect to a real page. Both are valid link destinations.
-const destRoutes = (vercel.routes || []).filter((r) => (r.dest || isRedirect(r)) && !isCatchAll(r.src));
+const destRoutes = (vercel.routes || []).filter(
+ (r) => (r.dest || isRedirect(r)) && !isCatchAll(r.src) && !isErrorFallback(r)
+);
const compiledRoutes = destRoutes.map((r) => {
let s = r.src;
@@ -364,6 +372,21 @@ const htmlInertRe = /<(pre|code)\b[^>]*>[\s\S]*?<\/\1>|<!--[\s\S]*?-->/gi;
// comments, and `a.href = '#' + doc` is the head of a computed anchor.
const scriptRe = /<script\b[^>]*>([\s\S]*?)<\/script>/gi;
+// `<link rel="preconnect" href="https://fonts.gstatic.com">` names an origin to
+// open a socket to, not a document to navigate to. Probing it as a link reports
+// the font API root as a dead 404 forever, on pages whose fonts load fine. The
+// stylesheet request itself is a separate <link> and stays audited. These hints
+// also appear inside JS template strings (api/discover-detail.js renders one),
+// so both mask paths apply it.
+const linkHintRe = /<link\b[^>]*\brel\s*=\s*["']?(?:preconnect|dns-prefetch)\b[^>]*>/gi;
+
+function maskLinkHints(content, mask) {
+ linkHintRe.lastIndex = 0;
+ let m;
+ while ((m = linkHintRe.exec(content))) mask.fill(1, m.index, m.index + m[0].length);
+ return mask;
+}
+
function htmlMask(content) {
const mask = new Uint8Array(content.length);
let m;
@@ -383,7 +406,7 @@ function scanFile(file) {
if (VENDOR_RE.test(rel)) return; // skip vendored libs — not our link surface
const content = readFileSync(file, 'utf8');
const isJs = /\.m?js$/.test(file);
- const mask = isJs ? commentMask(content) : htmlMask(content);
+ const mask = maskLinkHints(content, isJs ? commentMask(content) : htmlMask(content));
const inComment = (index) => mask[index] === 1;
findings.scanned++;
let m;
@@ -436,31 +459,85 @@ function danglingRoutes() {
}
// ── External liveness (opt-in) ───────────────────────────────────────────────
+
+// A bare `fetch` sends no User-Agent that any CDN recognizes, so Cloudflare and
+// friends answer with a TLS reset or an interstitial. That read back as a dead
+// link for pages a browser opens fine (t.me, solflare.com, ibm.com), which is
+// worse than no report: a list that is mostly false positives gets ignored, and
+// the real 404 hiding in it ships. Ask for the page the way a browser does.
+const PROBE_HEADERS = {
+ 'user-agent':
+ 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
+ accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+ 'accept-language': 'en-US,en;q=0.9',
+};
+
+// x.com serves a logged-out `/status/<id>` page as a 404 to every client that
+// does not run its JS, including a real browser with cookies cleared. The probe
+// therefore cannot tell a live post from a deleted one there, so it must not
+// claim either. Profiles and every other path still get probed normally.
+const UNPROBEABLE_RE = /^https?:\/\/(?:www\.)?(?:x|twitter)\.com\/[^/]+\/status\//i;
+
+// 401/403/429/405 mean the host answered but bot-blocks automated probes, so the
+// page exists. Only 404/410/5xx and DNS/timeout failures are truly dead.
+const aliveStatus = (s) => s < 400 || [401, 403, 405, 429].includes(s);
+
+// A HEAD is cheap but some hosts route it differently from a GET: alibabacloud
+// sends one to a geo-router that 500s while the GET loads the page. A 404, an
+// explicit method rejection, or any 5xx therefore earns one GET before the link
+// is called dead. 403 is deliberately absent: it already reads as alive, and
+// re-asking a bot-blocking CDN with a GET trades that answer for a reset.
+const retryWithGet = (s) => s === 404 || s === 405 || s === 501 || s >= 500;
+
+// undici caps response headers at 16 KB and throws past it. The host answered,
+// so the page is there; only the probe could not read the preamble.
+const ANSWERED_ANYWAY = new Set(['UND_ERR_HEADERS_OVERFLOW']);
+
+function errorCode(e) {
+ if (e.name === 'AbortError') return 'timeout';
+ const code = e.cause?.code || e.cause?.name;
+ return code ? `error:${code}` : 'error';
+}
+
async function probeExternal(urls) {
const dead = [];
- const queue = [...urls];
- const CONCURRENCY = 12;
+ const queue = urls.filter((u) => !UNPROBEABLE_RE.test(u));
+ // Eight at a time, not twelve: the probe shares one egress with whatever else
+ // this machine is doing, and a saturated pool shows up as ETIMEDOUT on live
+ // hosts, which is exactly the false positive this whole pass exists to kill.
+ const CONCURRENCY = 8;
async function worker() {
while (queue.length) {
const url = queue.shift();
- let ok = false;
- let status = 0;
- try {
+ const once = async (method) => {
const ctrl = new AbortController();
- const t = setTimeout(() => ctrl.abort(), 10000);
- let res = await fetch(url, { method: 'HEAD', redirect: 'follow', signal: ctrl.signal });
- if (res.status === 405 || res.status === 403 || res.status === 501) {
- res = await fetch(url, { method: 'GET', redirect: 'follow', signal: ctrl.signal });
+ const t = setTimeout(() => ctrl.abort(), 15000);
+ try {
+ const res = await fetch(url, { method, redirect: 'follow', signal: ctrl.signal, headers: PROBE_HEADERS });
+ return { status: res.status };
+ } catch (e) {
+ if (ANSWERED_ANYWAY.has(e.cause?.code)) return { status: 200 };
+ return { error: errorCode(e) };
+ } finally {
+ clearTimeout(t);
+ }
+ };
+ let res = await once('HEAD');
+ // A network error is as often this machine's egress as the host, and a
+ // method rejection says nothing about the page, so both earn a GET. The
+ // GET answers the actual question, but a GET that itself fails to connect
+ // must not overwrite an answer the host already gave. A failed retry gets
+ // one more attempt after a pause, since a saturated pool clears on its own.
+ if (res.error || retryWithGet(res.status)) {
+ let retry = await once('GET');
+ if (retry.error) {
+ await new Promise((r) => setTimeout(r, 1500));
+ retry = await once('GET');
}
- clearTimeout(t);
- status = res.status;
- // 401/403/429/405 mean the host answered but bot-blocks automated probes —
- // the page exists. Only 404/410/5xx and DNS/timeout failures are truly dead.
- ok = res.status < 400 || [401, 403, 405, 429].includes(res.status);
- } catch (e) {
- status = e.name === 'AbortError' ? 'timeout' : 'error';
+ if (!retry.error) res = retry;
}
- if (!ok) dead.push({ url, status });
+ if (res.error) dead.push({ url, status: res.error });
+ else if (!aliveStatus(res.status)) dead.push({ url, status: res.status });
}
}
await Promise.all(Array.from({ length: CONCURRENCY }, worker));
diff --git a/scripts/audit-play-failure-modes.mjs b/scripts/audit-play-failure-modes.mjs
index 563e74645..3c535c2f4 100644
--- a/scripts/audit-play-failure-modes.mjs
+++ b/scripts/audit-play-failure-modes.mjs
@@ -96,6 +96,27 @@ const ATTEMPTS = Math.max(1, Number(process.env.ATTEMPTS) || 4);
const HOST_TRANSPORT_ERROR =
/ERR_NETWORK_CHANGED|ERR_CONNECTION_(RESET|CLOSED|ABORTED)|ERR_EMPTY_RESPONSE|ERR_SOCKET_NOT_CONNECTED/i;
+// Playwright bounds navigations and actions, but NOT page.evaluate() or a
+// context/browser close, and on this box those are exactly the calls that stop
+// coming back: /play runs a rAF loop over two WebGL contexts, and when the host
+// is oversubscribed several hundred deep the renderer stops servicing the CDP
+// channel altogether. The sweep then sat on one scenario forever, printing
+// nothing, which is worse than either a pass or a finding because it never
+// reaches the other seventeen. Bound both, and treat a breach as infra so the
+// scenario is retried and, failing that, counted NOT RUN.
+const OBSERVE_TIMEOUT_MS = Number(process.env.OBSERVE_TIMEOUT_MS) || 90_000;
+const TEARDOWN_TIMEOUT_MS = 15_000;
+
+function withTimeout(promise, ms, label) {
+ let timer;
+ return Promise.race([
+ Promise.resolve(promise).finally(() => clearTimeout(timer)),
+ new Promise((_, reject) => {
+ timer = setTimeout(() => reject(new Error(`${label} did not return within ${ms}ms`)), ms);
+ }),
+ ]);
+}
+
// ── Hostile inputs ───────────────────────────────────────────────────────────
// Every payload calls the same sentinel, so one flag proves script execution
// regardless of which vector fired.
@@ -295,6 +316,21 @@ async function baseAlive(base, timeoutMs = 5000) {
}
}
+// Wait for the origin to answer again before spending an attempt on it. A dev
+// server in this worktree restarts itself every time another agent edits .env or
+// .env.local, and every request inside that window is refused. Retrying straight
+// into the gap burned attempts on a server that was seconds away from being back,
+// and a scenario that ran out of them was reported NOT RUN for a reason that had
+// nothing to do with /play.
+async function waitForBase(base, timeoutMs = 90_000) {
+ const deadline = Date.now() + timeoutMs;
+ while (Date.now() < deadline) {
+ if (await baseAlive(base, 5000)) return true;
+ await new Promise((r) => setTimeout(r, 2000));
+ }
+ return false;
+}
+
function freePort() {
return new Promise((resolve, reject) => {
const srv = createServer();
@@ -356,9 +392,13 @@ async function runScenario(base, sc) {
const browser = await chromium.launch({ headless: !process.env.HEADFUL });
const context = await browser.newContext({ viewport: { width: 1440, height: 900 } });
// Close the process, not just the context. Every exit path goes through this.
+ // A wedged renderer can swallow the graceful close too, and an unbounded close
+ // hangs the whole sweep on a scenario that already produced its verdict. Bound
+ // it and move on; Playwright kills the browser processes it launched when this
+ // process exits, so the worst case is one idle chromium until the run ends.
const teardown = async () => {
- await context.close().catch(() => {});
- await browser.close().catch(() => {});
+ await withTimeout(context.close(), TEARDOWN_TIMEOUT_MS, 'context.close').catch(() => {});
+ await withTimeout(browser.close(), TEARDOWN_TIMEOUT_MS, 'browser.close').catch(() => {});
};
const consoleIssues = [];
const pageErrors = [];
@@ -440,6 +480,19 @@ async function runScenario(base, sc) {
|| HOST_TRANSPORT_ERROR.test(m)) {
return { infra: m, findings: [], consoleIssues: [], pageErrors: [] };
}
+ // A navigation that ran out its budget is the ambiguous case: either the
+ // document genuinely never reaches DOMContentLoaded, or this box (load in
+ // the hundreds, memory exhausted) never gave the renderer enough CPU to get
+ // there. Ask the origin directly, then retry rather than ruling on one
+ // sample. A real hang reproduces on every attempt and still ends the
+ // scenario, so nothing is excused here; what is avoided is one starved
+ // attempt being published as "/play never loaded".
+ if (/Timeout .*exceeded/i.test(m)) {
+ const t0 = Date.now();
+ const served = await baseAlive(base, 10_000);
+ const how = served ? `origin served / in ${Date.now() - t0}ms` : 'origin did not answer either';
+ return { infra: `${m.split('\n')[0]} (${how})`, findings: [], consoleIssues: [], pageErrors: [] };
+ }
findings.push(`navigation failed: ${m}`);
return { findings, consoleIssues, pageErrors };
}
@@ -455,7 +508,7 @@ async function runScenario(base, sc) {
return { infra: `host transport failure mid-scenario (${transportHit})`, findings: [], consoleIssues: [], pageErrors: [] };
}
- const o = await page.evaluate(() => {
+ const observe = () => page.evaluate(() => {
const loader = document.getElementById('kx-loading');
const hidden = !loader || loader.classList.contains('kx-hidden');
const errorCard = !!document.querySelector('.kx-boot-error');
@@ -499,6 +552,16 @@ async function runScenario(base, sc) {
};
});
+ let o;
+ try {
+ o = await withTimeout(observe(), OBSERVE_TIMEOUT_MS, 'page.evaluate');
+ } catch (err) {
+ // The renderer stopped answering (see OBSERVE_TIMEOUT_MS). Nothing was
+ // observed, so there is nothing to judge /play on.
+ await teardown();
+ return { infra: String(err?.message || err), findings: [], consoleIssues: [], pageErrors: [] };
+ }
+
// ── Universal assertions ────────────────────────────────────────────────
if (o.xss) findings.push('SCRIPT EXECUTED from a query parameter (window.__xss set)');
if (o.injectedScripts) findings.push(`${o.injectedScripts} injected script/img[onerror] node(s) in the DOM`);
@@ -554,6 +617,7 @@ async function main() {
// A network-change storm usually settles within seconds, so back off a
// little further each time rather than landing in the same window twice.
await new Promise((r) => setTimeout(r, 3000 * (attempt - 1)));
+ await waitForBase(server.base);
res = await runScenario(server.base, sc);
}
if (res.infra) {
diff --git a/scripts/lib/csp-headers.mjs b/scripts/lib/csp-headers.mjs
new file mode 100644
index 000000000..e09c900d9
--- /dev/null
+++ b/scripts/lib/csp-headers.mjs
@@ -0,0 +1,114 @@
+// Comparing the security headers a response carried against the ones the route
+// table declares for it.
+//
+// The comparison is subtle enough to deserve its own home and its own tests:
+// `server/csp-hashes.mjs` rewrites `script-src` per response, so a served
+// policy is allowed to differ from the declared one in exactly one way, and
+// only on the responses that get rewritten. Everything else about a security
+// header has to arrive byte-for-byte or the site is not serving what the repo
+// says it serves.
+//
+// Used by scripts/audit-csp.mjs. Pure functions: no network, no route table,
+// no filesystem.
+
+/** The security headers a document is expected to carry, whatever else it declares. */
+export const REQUIRED_ON_HTML = [
+ 'content-security-policy',
+ 'strict-transport-security',
+ 'x-content-type-options',
+ 'referrer-policy',
+];
+
+/** "a 'b' c; d 'e'" -> Map{ a => Set{'b','c'}, d => Set{'e'} } */
+export function parsePolicy(value) {
+ const directives = new Map();
+ for (const chunk of String(value).split(';')) {
+ const parts = chunk.trim().split(/\s+/).filter(Boolean);
+ if (!parts.length) continue;
+ directives.set(parts[0].toLowerCase(), new Set(parts.slice(1)));
+ }
+ return directives;
+}
+
+const setsEqual = (a, b) => a.size === b.size && [...a].every((v) => b.has(v));
+
+/**
+ * Differences between the policy the route table declares and the one the
+ * response carried, allowing only the rewrite server/csp-hashes.mjs performs:
+ * within script-src / script-src-elem, `'unsafe-inline'` is replaced by any
+ * number of `'sha256-…'` sources. Everything else must survive byte-for-byte.
+ *
+ * The rewrite is only applied to documents, so it is only required of them.
+ * `sha256Rewritten` says whether this response is one the server hardens: for a
+ * document it is demanded (a page that shipped the raw `'unsafe-inline'` policy
+ * is exactly the regression this comparison exists to catch), and for
+ * everything else (robots.txt, llms.txt, the .well-known JSON, an API response)
+ * the declared policy has to arrive untouched instead.
+ */
+export function policyDiff(declared, served, sha256Rewritten) {
+ const want = parsePolicy(declared);
+ const got = parsePolicy(served);
+ const problems = [];
+ for (const [name, wantSources] of want) {
+ const gotSources = got.get(name);
+ if (!gotSources) {
+ problems.push(`directive "${name}" is missing from the served policy`);
+ continue;
+ }
+ const hashable = name === 'script-src' || name === 'script-src-elem';
+ const expected = new Set(wantSources);
+ const actual = new Set(gotSources);
+ if (hashable && sha256Rewritten) {
+ expected.delete("'unsafe-inline'");
+ for (const src of [...actual]) if (src.startsWith("'sha256-")) actual.delete(src);
+ }
+ if (setsEqual(expected, actual)) continue;
+ const missing = [...expected].filter((s) => !actual.has(s));
+ const extra = [...actual].filter((s) => !expected.has(s));
+ problems.push(
+ `directive "${name}" differs${missing.length ? ` (missing ${missing.join(' ')})` : ''}` +
+ `${extra.length ? ` (unexpected ${extra.join(' ')})` : ''}`,
+ );
+ }
+ return problems;
+}
+
+/**
+ * Every way a response's security headers fail to be what the route table says
+ * this path serves. `declared` is the resolved header bag from vercel.json,
+ * `served` the response's own headers; both keyed lowercase.
+ */
+export function headerProblems(declared, served) {
+ const problems = [];
+
+ for (const name of REQUIRED_ON_HTML) {
+ if (!declared[name]) problems.push(`vercel.json declares no ${name} for this path`);
+ }
+
+ // Only documents get their script-src rewritten into hashes; a text or JSON
+ // response carries the declared policy verbatim, and a response that carries
+ // hashes proves the rewrite ran even if its content type is unusual.
+ const isDocument = /text\/html/i.test(served['content-type'] || '');
+ const carriesHashes = /'sha256-/.test(served['content-security-policy'] || '');
+
+ for (const [name, expected] of Object.entries(declared)) {
+ // Only security headers are the subject here. Cache and CORS values are
+ // legitimately rewritten by the CDN and by conditional-request handling.
+ if (!name.startsWith('x-') && !REQUIRED_ON_HTML.includes(name) && name !== 'permissions-policy') {
+ continue;
+ }
+ const value = served[name];
+ if (value === undefined) {
+ problems.push(`${name} was declared but the response did not carry it`);
+ continue;
+ }
+ if (name === 'content-security-policy') {
+ problems.push(...policyDiff(expected, value, isDocument || carriesHashes));
+ continue;
+ }
+ if (value.trim() !== expected.trim()) {
+ problems.push(`${name} is "${value}" but vercel.json declares "${expected}"`);
+ }
+ }
+ return problems;
+}
diff --git a/src/marketplace.js b/src/marketplace.js
index 0872e0de0..1f797d7c0 100644
--- a/src/marketplace.js
+++ b/src/marketplace.js
@@ -781,6 +781,14 @@ function renderTheme() {
// to carry a public avatar GLB, so each card shows a live rotating model. We
// reshuffle client-side too, so revisiting the page varies the order even when
// the API response is served from cache.
+//
+// Each viewer ships `data-src` rather than `src`, the same contract the grid
+// cards and the hero carousel already use: observeCardModelViewers() promotes
+// it on first intersect and adds `auto-rotate` there, so a strip nobody has
+// scrolled to never downloads a GLB, never stands up a WebGL context, and never
+// runs a raf loop. Shipping `src` here left this the last eager model-viewer
+// surface on the page, and model-viewer was the single most expensive script on
+// it: 7,924ms of evaluation in a Lighthouse desktop trace of /marketplace.
function renderThemePicks() {
const row = $('market-theme-picks');
if (!row) return;
@@ -802,10 +810,9 @@ function renderThemePicks() {
${views > 0 ? `<span class="market-theme-pick-views">⊙ ${fmtNumber(views)}</span>` : ''}
<span class="market-theme-pick-placeholder" aria-hidden="true">${escapeHtml(initial(a.name || 'A'))}</span>
<model-viewer
- src="${escapeHtml(a.avatar_glb_url)}"
+ data-src="${escapeHtml(a.avatar_glb_url)}"
alt="${escapeHtml(a.name || 'Agent')}"
${a.thumbnail_url ? `poster="${escapeHtml(a.thumbnail_url)}"` : ''}
- auto-rotate
autoplay
rotation-per-second="22deg"
interaction-prompt="none"
@@ -844,6 +851,9 @@ function renderThemePicks() {
);
}
});
+ // Hand the freshly built viewers to the shared observer so their data-src is
+ // promoted when, and only when, the strip is actually near the viewport.
+ observeCardModelViewers();
}
// Fisher–Yates shuffle. Browser-only path, so Math.random is fine here.
@@ -1522,7 +1532,9 @@ function observeCardModelViewers() {
{ rootMargin: '200px 0px', threshold: 0.01 },
);
}
- document.querySelectorAll('.market-card-avatar model-viewer, .market-grid model-viewer').forEach((mv) => {
+ const LAZY_VIEWERS =
+ '.market-card-avatar model-viewer, .market-grid model-viewer, .market-theme-pick model-viewer';
+ document.querySelectorAll(LAZY_VIEWERS).forEach((mv) => {
if (mv.dataset.observed) return;
mv.dataset.observed = '1';
cardObserver.observe(mv);
diff --git a/src/viewer.js b/src/viewer.js
index b1507ea0d..953b7a798 100644
--- a/src/viewer.js
+++ b/src/viewer.js
@@ -290,6 +290,16 @@ export class Viewer {
this.scene = new Scene();
this.scene.background = this.backgroundColor;
+ // The host element can still be unlaid-out at construction (hidden tab,
+ // CSS not applied yet, flex parent that sizes on the next frame). A 0-px
+ // renderer/composer allocates zero-dimension GL textures, which the
+ // driver rejects (glTexStorage2D: dimensions must be > 0) and every
+ // subsequent clear/blit/draw then fails against an incomplete
+ // framebuffer. Start at 1 px and let the ResizeObserver below re-size to
+ // the real box on the first frame that has one.
+ const initialWidth = Math.max(1, el.clientWidth);
+ const initialHeight = Math.max(1, el.clientHeight);
+
const fov = options.preset === Preset.ASSET_GENERATOR ? (0.8 * 180) / Math.PI : 60;
const aspect = el.clientHeight > 0 ? el.clientWidth / el.clientHeight : 1;
this.defaultCamera = new PerspectiveCamera(fov, aspect, 0.01, 1000);
@@ -323,7 +333,7 @@ export class Viewer {
// biggest lever after MSAA on weak mobile GPUs.
const dprCap = this._lowPower ? 1 : (options.maxPixelRatio ?? (options.kiosk ? 1.5 : 2));
this.renderer.setPixelRatio(Math.min(window.devicePixelRatio, dprCap));
- this.renderer.setSize(el.clientWidth, el.clientHeight);
+ this.renderer.setSize(initialWidth, initialHeight);
// Ground contact shadow (the one rendering-quality gap vs the
// model-viewer surfaces). Cheap when nothing casts; skipped entirely on
@@ -372,8 +382,8 @@ export class Viewer {
// into a richer look. Screenshot capture bypasses this — see screenshot.js.
this.state.cinematicPreset = options.cinematicPreset ?? DEFAULT_CINEMATIC_PRESET;
this._cinematic = new CinematicPipeline(this.renderer, this.scene, this.activeCamera, {
- width: el.clientWidth,
- height: el.clientHeight,
+ width: initialWidth,
+ height: initialHeight,
preset: this.state.cinematicPreset,
});
this._composer = this._cinematic.composer;
diff --git a/tests/csp-headers-compare.test.js b/tests/csp-headers-compare.test.js
new file mode 100644
index 000000000..80a5ba8c9
--- /dev/null
+++ b/tests/csp-headers-compare.test.js
@@ -0,0 +1,163 @@
+// The rule this file pins: a document has to arrive with its `script-src`
+// rewritten into hashes, and everything else has to arrive exactly as
+// vercel.json declares it.
+//
+// Both halves have been wrong in production. A page that ships the raw
+// `'unsafe-inline'` policy has lost the whole point of per-response hashing and
+// must fail. A robots.txt or a .well-known JSON is never rewritten (the server
+// only hardens HTML), so demanding hashes of it reports a defect on every
+// non-document route the site serves, which is how a real header regression
+// gets lost in the noise.
+
+import { describe, it, expect } from 'vitest';
+import { headerProblems, policyDiff, parsePolicy } from '../scripts/lib/csp-headers.mjs';
+
+const DECLARED_CSP =
+ "base-uri 'self'; object-src 'none'; script-src 'self' 'unsafe-inline' 'unsafe-eval' blob: https://esm.sh; worker-src 'self' blob:; frame-ancestors 'self'";
+
+const HASH = "'sha256-K7gNU3sdo+OL0wNhqoVWhr3g6s1xYv72ol/pe/Unols='";
+
+const declaredBag = () => ({
+ 'content-security-policy': DECLARED_CSP,
+ 'strict-transport-security': 'max-age=63072000; includeSubDomains; preload',
+ 'x-content-type-options': 'nosniff',
+ 'referrer-policy': 'strict-origin-when-cross-origin',
+ 'cache-control': 'public, max-age=0, must-revalidate',
+});
+
+const hardened = DECLARED_CSP.replace("'unsafe-inline'", HASH);
+
+const servedDocument = (csp = hardened) => ({
+ ...declaredBag(),
+ 'content-security-policy': csp,
+ 'content-type': 'text/html; charset=utf-8',
+});
+
+const servedText = (csp = DECLARED_CSP) => ({
+ ...declaredBag(),
+ 'content-security-policy': csp,
+ 'content-type': 'text/plain; charset=utf-8',
+});
+
+describe('parsePolicy', () => {
+ it('splits directives into a source set each', () => {
+ const parsed = parsePolicy("script-src 'self' blob:; object-src 'none'");
+ expect([...parsed.get('script-src')]).toEqual(["'self'", 'blob:']);
+ expect([...parsed.get('object-src')]).toEqual(["'none'"]);
+ });
+
+ it('ignores empty segments and trailing semicolons', () => {
+ expect([...parsePolicy("base-uri 'self';;").keys()]).toEqual(['base-uri']);
+ });
+});
+
+describe('policyDiff on a rewritten response', () => {
+ it('accepts unsafe-inline replaced by hashes', () => {
+ expect(policyDiff(DECLARED_CSP, hardened, true)).toEqual([]);
+ });
+
+ it('accepts several hashes in place of the one unsafe-inline', () => {
+ const many = DECLARED_CSP.replace("'unsafe-inline'", `${HASH} ${HASH.replace('K7g', 'Zzz')}`);
+ expect(policyDiff(DECLARED_CSP, many, true)).toEqual([]);
+ });
+
+ it('accepts a document with no inline script at all, which gets no hashes', () => {
+ const noInline = DECLARED_CSP.replace(" 'unsafe-inline'", '');
+ expect(policyDiff(DECLARED_CSP, noInline, true)).toEqual([]);
+ });
+
+ it('rejects a document that kept unsafe-inline instead of hashing', () => {
+ const problems = policyDiff(DECLARED_CSP, DECLARED_CSP, true);
+ expect(problems).toHaveLength(1);
+ expect(problems[0]).toContain("unexpected 'unsafe-inline'");
+ });
+
+ it('rejects unsafe-inline smuggled in alongside the hashes', () => {
+ const both = DECLARED_CSP.replace("'unsafe-inline'", `'unsafe-inline' ${HASH}`);
+ expect(policyDiff(DECLARED_CSP, both, true)[0]).toContain("unexpected 'unsafe-inline'");
+ });
+
+ it('rejects a widened source list even when the hashes are right', () => {
+ const widened = hardened.replace("script-src 'self'", "script-src 'self' https://evil.example");
+ expect(policyDiff(DECLARED_CSP, widened, true)[0]).toContain('https://evil.example');
+ });
+
+ it('rejects a dropped directive', () => {
+ const dropped = hardened.replace("; object-src 'none'", '');
+ expect(policyDiff(DECLARED_CSP, dropped, true)[0]).toContain('"object-src" is missing');
+ });
+
+ it('rejects a loosened frame-ancestors', () => {
+ const framed = hardened.replace("frame-ancestors 'self'", 'frame-ancestors *');
+ expect(policyDiff(DECLARED_CSP, framed, true)[0]).toContain('frame-ancestors');
+ });
+});
+
+describe('policyDiff on a response the server never rewrites', () => {
+ it('accepts the declared policy verbatim', () => {
+ expect(policyDiff(DECLARED_CSP, DECLARED_CSP, false)).toEqual([]);
+ });
+
+ it('still rejects a policy that differs in any other way', () => {
+ const widened = DECLARED_CSP.replace("object-src 'none'", "object-src 'self'");
+ expect(policyDiff(DECLARED_CSP, widened, false)[0]).toContain('object-src');
+ });
+});
+
+describe('headerProblems', () => {
+ it('passes a document that carries exactly what is declared', () => {
+ expect(headerProblems(declaredBag(), servedDocument())).toEqual([]);
+ });
+
+ it('passes a text response that carries the unrewritten policy', () => {
+ expect(headerProblems(declaredBag(), servedText())).toEqual([]);
+ });
+
+ it('demands the rewrite of a document, not of a text response', () => {
+ expect(headerProblems(declaredBag(), servedDocument(DECLARED_CSP))).toHaveLength(1);
+ expect(headerProblems(declaredBag(), servedText(DECLARED_CSP))).toEqual([]);
+ });
+
+ it('demands the rewrite of any response that already carries hashes', () => {
+ const partly = servedText(`${DECLARED_CSP} ${HASH}`);
+ expect(headerProblems(declaredBag(), partly)).not.toEqual([]);
+ });
+
+ it('reports a security header the response dropped', () => {
+ const served = servedDocument();
+ delete served['strict-transport-security'];
+ expect(headerProblems(declaredBag(), served)).toEqual([
+ 'strict-transport-security was declared but the response did not carry it',
+ ]);
+ });
+
+ it('reports a weakened HSTS max-age', () => {
+ const served = { ...servedDocument(), 'strict-transport-security': 'max-age=300' };
+ expect(headerProblems(declaredBag(), served)[0]).toContain('strict-transport-security');
+ });
+
+ it('reports a missing declaration, so deleting the rule from vercel.json fails', () => {
+ const declared = declaredBag();
+ delete declared['x-content-type-options'];
+ expect(headerProblems(declared, servedDocument())).toEqual([
+ 'vercel.json declares no x-content-type-options for this path',
+ ]);
+ });
+
+ it('ignores cache-control, which the CDN legitimately rewrites', () => {
+ const served = { ...servedDocument(), 'cache-control': 'public, max-age=31536000, immutable' };
+ expect(headerProblems(declaredBag(), served)).toEqual([]);
+ });
+
+ it('checks every x- header the route table declares', () => {
+ const declared = { ...declaredBag(), 'x-frame-options': 'SAMEORIGIN' };
+ const served = { ...servedDocument(), 'x-frame-options': 'ALLOWALL' };
+ expect(headerProblems(declared, served)[0]).toContain('x-frame-options');
+ });
+
+ it('checks permissions-policy', () => {
+ const declared = { ...declaredBag(), 'permissions-policy': 'camera=()' };
+ const served = { ...servedDocument(), 'permissions-policy': 'camera=(self)' };
+ expect(headerProblems(declared, served)[0]).toContain('permissions-policy');
+ });
+});
diff --git a/tests/forge-scale.test.js b/tests/forge-scale.test.js
index 4156f9114..485dc5654 100644
--- a/tests/forge-scale.test.js
+++ b/tests/forge-scale.test.js
@@ -119,6 +119,37 @@ describe('shared circuit breaker (in-memory fallback)', () => {
await circuitRecordSuccess(name);
expect((await circuitState(name)).open).toBe(false);
});
+
+ it('caps the open window so a long outage cannot silence a cron for hours', async () => {
+ // failures only reset on a success, so an uncapped (failures × baseMs) window
+ // grows without bound: the every-minute seed crons would back off past the
+ // point where they still look alive. The clamp keeps the retry cadence
+ // bounded no matter how long the provider stays down.
+ const name = 'test-breaker-cap';
+ const baseMs = 10 * 60_000;
+ const maxMs = 60 * 60_000;
+ for (let i = 0; i < 40; i++) {
+ await circuitRecordFailure(name, { threshold: 3, baseMs, maxMs });
+ }
+ const state = await circuitState(name);
+ expect(state.failures).toBe(40);
+ expect(state.open).toBe(true);
+ // 40 × 10min would be 400 minutes without the clamp.
+ expect(state.openUntil - Date.now()).toBeLessThanOrEqual(maxMs);
+ expect(state.openUntil - Date.now()).toBeGreaterThan(maxMs - 60_000);
+ await circuitRecordSuccess(name);
+ expect((await circuitState(name)).open).toBe(false);
+ });
+
+ it('defaults the cap when a caller passes only threshold and baseMs', async () => {
+ const name = 'test-breaker-default-cap';
+ for (let i = 0; i < 40; i++) {
+ await circuitRecordFailure(name, { threshold: 3, baseMs: 10 * 60_000 });
+ }
+ const state = await circuitState(name);
+ expect(state.open).toBe(true);
+ expect(state.openUntil - Date.now()).toBeLessThanOrEqual(60 * 60_000);
+ });
});
describe('SCALE_LIMITS', () => {