Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(forge): cap the seeder circuit breaker so one outage cannot park the cron for a day
nirholascdd9e2488498
diff --git a/api/_lib/forge-scale.js b/api/_lib/forge-scale.jsindex 63d7f8fc7..b463d906d 100644--- a/api/_lib/forge-scale.js+++ b/api/_lib/forge-scale.js@@ -323,21 +323,37 @@ export async function circuitState(name) {}// Record one failure; opens the breaker for (failures × baseMs) once threshold is-// reached. Returns the new consecutive-failure count.-export async function circuitRecordFailure(name, { threshold, baseMs }) {+// reached, clamped to maxMs. Returns the new consecutive-failure count.+//+// The clamp is load-bearing. `failures` only resets on a success, and the counter+// key is refreshed on every recorded failure, so a provider outage that outlives+// one retry cycle keeps extending the window: 12 consecutive failures already park+// a per-minute cron for two hours, and it grows from there with no natural+// ceiling. That turns a transient upstream blip into a seeder that has gone quiet+// for most of a day, which reads from the outside as a dead cron rather than a+// backed-off one. Capping the window keeps the retry cadence bounded, so the cron+// re-probes the provider at least once per CIRCUIT_MAX_OPEN_MS and resumes on its+// own the moment the outage clears.+const CIRCUIT_MAX_OPEN_MS = 60 * 60_000;++const openWindowMs = (failures, baseMs, maxMs) => Math.min(failures * baseMs, maxMs);++export async function circuitRecordFailure(name, { threshold, baseMs, maxMs = CIRCUIT_MAX_OPEN_MS }) {if (!redis) {const c = memCircuit(name);c.failures++;- if (c.failures >= threshold) c.openUntil = Date.now() + c.failures * baseMs;+ if (c.failures >= threshold) c.openUntil = Date.now() + openWindowMs(c.failures, baseMs, maxMs);return c.failures;}try {const failures = await redis.incr(`${CIRCUIT_PREFIX}${name}:failures`);await redis.expire(`${CIRCUIT_PREFIX}${name}:failures`, CIRCUIT_TTL_S);if (failures >= threshold) {- await redis.set(`${CIRCUIT_PREFIX}${name}:openUntil`, String(Date.now() + failures * baseMs), {- ex: CIRCUIT_TTL_S,- });+ await redis.set(+ `${CIRCUIT_PREFIX}${name}:openUntil`,+ String(Date.now() + openWindowMs(failures, baseMs, maxMs)),+ { ex: CIRCUIT_TTL_S },+ );}return failures;} catch {diff --git a/api/cron/forge-seed-cron.js b/api/cron/forge-seed-cron.jsindex 5e3dc9333..9f49585a9 100644--- a/api/cron/forge-seed-cron.js+++ b/api/cron/forge-seed-cron.js@@ -117,7 +117,9 @@ const gateBatchSize = () => intEnv('SEED_CRON_GATE', 2, { min: 1, max: 10 });// every cron lambda sees the same open/closed decision; without that, each instance// would rediscover a provider outage on its own and keep submitting into a dead// lane. When CIRCUIT_THRESHOLD consecutive forge submits fail the circuit opens for-// (failures × CIRCUIT_BASE_MS) so the cron goes quiet during provider outages.+// (failures × CIRCUIT_BASE_MS), capped by forge-scale's CIRCUIT_MAX_OPEN_MS, so the+// cron goes quiet during a provider outage but still re-probes the lane at least+// once an hour instead of backing off into silence for the rest of the day.const CIRCUIT_NAME = 'forge-seed';const CIRCUIT_THRESHOLD = 3;const CIRCUIT_BASE_MS = 10 * 60_000; // 10 min × consecutive failuresdiff --git a/scripts/audit-console.mjs b/scripts/audit-console.mjsindex 7cb6cb6f4..536a230c7 100644--- a/scripts/audit-console.mjs+++ b/scripts/audit-console.mjs@@ -105,6 +105,14 @@ function freePort() {const PROBE_BASE = 'http://127.0.0.1:3000';+// Dev-server readiness budget. A shared/loaded box (several agents running+// browser fleets and test workers at once) starves Vite's boot: it answers+// eventually, just not inside a short window. Waiting longer only delays the+// sweep on a busy machine; it never changes what counts as a console error.+// Raise SERVER_BOOT_MS if a run still dies with "vite did not become ready".+const SERVER_BOOT_MS = Number(process.env.SERVER_BOOT_MS || 240_000);+const REUSE_PROBE_MS = Number(process.env.REUSE_PROBE_MS || 20_000);+async function warmupDeps(base) {// Hit a few dep-heavy pages so Vite pre-bundles before the timed sweep,// otherwise the first real navigation eats the optimizer's reload.@@ -119,7 +127,7 @@ async function warmupDeps(base) {}async function startServer() {- if (await probe(`${PROBE_BASE}/`, 5000)) {+ if (await probe(`${PROBE_BASE}/`, REUSE_PROBE_MS)) {console.log(C.d(' reusing dev server on :3000'));// localhost (not 127.0.0.1) — some CDN CORS configs allow it, fewer spurious errors.const navBase = 'http://localhost:3000';@@ -135,11 +143,11 @@ async function startServer() {});const probeBase = `http://127.0.0.1:${port}`;const navBase = `http://localhost:${port}`;- const deadline = Date.now() + 90_000;+ const deadline = Date.now() + SERVER_BOOT_MS;process.stdout.write(` starting Vite on :${port} `);while (Date.now() < deadline) {if (child.exitCode != null) throw new Error(`vite exited early (code ${child.exitCode})`);- if (await probe(`${probeBase}/`, 2000)) {+ if (await probe(`${probeBase}/`, 10_000)) {process.stdout.write('\n');break;}@@ -148,7 +156,7 @@ async function startServer() {}if (Date.now() >= deadline) {child.kill('SIGKILL');- throw new Error('vite did not become ready within 90s');+ throw new Error(`vite did not become ready within ${Math.round(SERVER_BOOT_MS / 1000)}s`);}await warmupDeps(navBase);return { base: navBase, stop: async () => child.kill('SIGTERM') };diff --git a/scripts/audit-csp.mjs b/scripts/audit-csp.mjsindex a27af70b8..adfa62112 100644--- a/scripts/audit-csp.mjs+++ b/scripts/audit-csp.mjs@@ -31,12 +31,21 @@// documented transform (drop 'unsafe-inline', add 'sha256-…'); every other// directive and every other security header must match the route table exactly.//+// `--headers-only` answers the narrower question ("does this origin serve the+// headers the route table declares?") with a plain fetch and no browser. It is+// the only mode that can sweep production honestly: a real browser loading 200+// live pages is minutes of work and is at the mercy of whatever else the+// machine is doing, while the header comparison is deterministic. It proves+// strictly less than the full run (nothing evaluates the policy), so it is an+// addition to the browser sweep, never a substitute for it.+//// Usage:// node server/index.mjs & # or any origin serving dist/// node scripts/audit-csp.mjs # sweeps the default page set// node scripts/audit-csp.mjs --base http://127.0.0.1:8099// node scripts/audit-csp.mjs --base https://three.ws # audit production// node scripts/audit-csp.mjs --all # every page in data/pages.json+// node scripts/audit-csp.mjs --headers-only --base https://three.ws --all// node scripts/audit-csp.mjs /pay /studio # only these pathsimport { readFileSync } from 'node:fs';@@ -44,6 +53,7 @@ import path from 'node:path';import { fileURLToPath } from 'node:url';import { chromium } from 'playwright';import { loadRouteTable, resolvePhase1 } from '../server/route-resolve.mjs';+import { headerProblems as compareHeaders } from './lib/csp-headers.mjs';const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');@@ -82,6 +92,7 @@ const args = process.argv.slice(2);const baseIdx = args.indexOf('--base');const BASE = baseIdx === -1 ? 'http://127.0.0.1:8099' : args[baseIdx + 1];const wantAll = args.includes('--all');+const headersOnly = args.includes('--headers-only');const explicit = args.filter((a) => a.startsWith('/'));function allCataloguedPaths() {@@ -103,16 +114,10 @@ const SETTLE_MS = Number(process.env.CSP_AUDIT_SETTLE_MS) || 3500;const { phase1Routes } = loadRouteTable(path.join(ROOT, 'vercel.json'));-// The security headers vercel.json is expected to put on a document. Anything-// the route table declares for a path is checked; this list is what makes a-// MISSING declaration visible too, so deleting the global header rule fails-// here instead of shipping a bare-headed site.-const REQUIRED_ON_HTML = [- 'content-security-policy',- 'strict-transport-security',- 'x-content-type-options',- 'referrer-policy',-];+// What counts as a header problem lives in scripts/lib/csp-headers.mjs, next to+// the tests that pin it. It checks the headers a path declares AND that the+// document ones are declared at all, so deleting the global header rule from+// vercel.json fails here instead of shipping a bare-headed site./** The header bag vercel.json declares for `pathname`, per the server's resolver. */function declaredHeaders(pathname, base) {@@ -125,93 +130,41 @@ function declaredHeaders(pathname, base) {return bag;}-/** "a 'b' c; d 'e'" -> Map{ a => Set{'b','c'}, d => Set{'e'} } */-function parsePolicy(value) {- const directives = new Map();- for (const chunk of String(value).split(';')) {- const parts = chunk.trim().split(/\s+/).filter(Boolean);- if (!parts.length) continue;- directives.set(parts[0].toLowerCase(), new Set(parts.slice(1)));- }- return directives;-}--const setsEqual = (a, b) => a.size === b.size && [...a].every((v) => b.has(v));--/**- * Differences between the policy the route table declares and the one the- * response carried, allowing only the rewrite server/csp-hashes.mjs performs:- * within script-src / script-src-elem, `'unsafe-inline'` is replaced by any- * number of `'sha256-…'` sources. Everything else must survive byte-for-byte.- */-function policyDiff(declared, served) {- const want = parsePolicy(declared);- const got = parsePolicy(served);- const problems = [];- for (const [name, wantSources] of want) {- const gotSources = got.get(name);- if (!gotSources) {- problems.push(`directive "${name}" is missing from the served policy`);- continue;- }- const hashable = name === 'script-src' || name === 'script-src-elem';- const expected = new Set(wantSources);- const actual = new Set(gotSources);- if (hashable) {- expected.delete("'unsafe-inline'");- for (const src of [...actual]) if (src.startsWith("'sha256-")) actual.delete(src);- }- if (setsEqual(expected, actual)) continue;- const missing = [...expected].filter((s) => !actual.has(s));- const extra = [...actual].filter((s) => !expected.has(s));- problems.push(- `directive "${name}" differs${missing.length ? ` (missing ${missing.join(' ')})` : ''}` +- `${extra.length ? ` (unexpected ${extra.join(' ')})` : ''}`,- );- }- return problems;-}-/*** Every way the response's security headers fail to be what the repo says it* serves for this path.*/function headerProblems(pathname, base, servedHeaders) {- const want = declaredHeaders(pathname, base);- const problems = [];-- for (const name of REQUIRED_ON_HTML) {- if (!want[name]) problems.push(`vercel.json declares no ${name} for this path`);- }+ return compareHeaders(declaredHeaders(pathname, base), servedHeaders);+}- for (const [name, expected] of Object.entries(want)) {- // Only security headers are the subject here. Cache and CORS values are- // legitimately rewritten by the CDN and by conditional-request handling.- if (!name.startsWith('x-') && !REQUIRED_ON_HTML.includes(name) && name !== 'permissions-policy') {- continue;- }- const served = servedHeaders[name];- if (served === undefined) {- problems.push(`${name} was declared but the response did not carry it`);- continue;- }- if (name === 'content-security-policy') {- problems.push(...policyDiff(expected, served));- continue;- }- if (served.trim() !== expected.trim()) {- problems.push(`${name} is "${served}" but vercel.json declares "${expected}"`);+/**+ * The header half of the audit for one path, over a plain request. No policy is+ * evaluated: a clean result here means the response carried what vercel.json+ * declares, nothing more.+ */+async function auditHeaders(p) {+ const violations = [];+ const errors = [];+ const headers = [];+ let status = 0;+ try {+ const res = await fetch(BASE + p, { redirect: 'follow' });+ await res.arrayBuffer();+ status = res.status;+ if (status < 400) {+ const bag = {};+ for (const [k, v] of res.headers) bag[k.toLowerCase()] = v;+ headers.push(...headerProblems(new URL(res.url).pathname, BASE, bag));}+ } catch (err) {+ errors.push(`request: ${err.message}`);}- return problems;+ return { status, violations, errors, headers };}-const browser = await chromium.launch({- args: ['--no-sandbox', '--disable-dev-shm-usage'],-});--const results = [];-for (const p of paths) {+/** The full audit for one path: load it in a real browser and collect what the policy blocked. */+async function auditInBrowser(browser, p) {const context = await browser.newContext();const page = await context.newPage();const violations = [];@@ -253,6 +206,18 @@ for (const p of paths) {errors.push(`navigation: ${err.message}`);}await context.close();+ return { status, violations, errors, headers };+}++const browser = headersOnly+ ? null+ : await chromium.launch({ args: ['--no-sandbox', '--disable-dev-shm-usage'] });++const results = [];+for (const p of paths) {+ const { status, violations, errors, headers } = headersOnly+ ? await auditHeaders(p)+ : await auditInBrowser(browser, p);// A page that never loaded, or answered with an error status, proves// nothing: it has no inline scripts to block. Counting it as clean is how a// sweep reports success over a wiped dist/ or a server that died halfway.@@ -275,7 +240,7 @@ for (const p of paths) {console.log(`[${mark}] ${String(status).padEnd(3)} ${p}${note}`);}-await browser.close();+if (browser) await browser.close();const failed = results.filter((r) => r.violations.length > 0);const unreachable = results.filter((r) => r.unreachable);@@ -305,7 +270,11 @@ if (misheaded.length) {if (failed.length === 0) {console.log(`\nCSP audit: clean across ${results.length} page(s) on ${BASE}`);- console.log('Each page loaded, carried the policy vercel.json declares, and fired no violation.');+ console.log(+ headersOnly+ ? 'Each response carried the headers vercel.json declares. No policy was evaluated: rerun without --headers-only for that.'+ : 'Each page loaded, carried the policy vercel.json declares, and fired no violation.',+ );process.exit(0);}diff --git a/scripts/audit-links.mjs b/scripts/audit-links.mjsindex be9ebf906..c6baba128 100644--- a/scripts/audit-links.mjs+++ b/scripts/audit-links.mjs@@ -74,9 +74,17 @@ const ASSET_CATCHALL = '/(.*\\.(?:svg|png|jpg|jpeg|gif|webp|avif|ico|woff2|woff|const isCatchAll = (src) => src === '/(.*)' || src === ASSET_CATCHALL;const isRedirect = (r) => typeof r.status === 'number' && r.status >= 300 && r.status < 400;+// The terminal `{"src":"/(?!_vercel/).*","status":404,"dest":"/404.html"}` rewrite+// is the miss handler, not a destination. It carries a dest and its src is not the+// literal `/(.*)` catch-all, so without this it entered the route table, matched+// every path, and made every root-absolute link resolve: the broken-link count+// read 0 because nothing could ever be counted, not because nothing was broken.+const isErrorFallback = (r) => typeof r.status === 'number' && r.status >= 400;// Specific routes that resolve a request to something: a rewrite (dest) OR a// 3xx redirect to a real page. Both are valid link destinations.-const destRoutes = (vercel.routes || []).filter((r) => (r.dest || isRedirect(r)) && !isCatchAll(r.src));+const destRoutes = (vercel.routes || []).filter(+ (r) => (r.dest || isRedirect(r)) && !isCatchAll(r.src) && !isErrorFallback(r)+);const compiledRoutes = destRoutes.map((r) => {let s = r.src;@@ -364,6 +372,21 @@ const htmlInertRe = /<(pre|code)\b[^>]*>[\s\S]*?<\/\1>|<!--[\s\S]*?-->/gi;// comments, and `a.href = '#' + doc` is the head of a computed anchor.const scriptRe = /<script\b[^>]*>([\s\S]*?)<\/script>/gi;+// `<link rel="preconnect" href="https://fonts.gstatic.com">` names an origin to+// open a socket to, not a document to navigate to. Probing it as a link reports+// the font API root as a dead 404 forever, on pages whose fonts load fine. The+// stylesheet request itself is a separate <link> and stays audited. These hints+// also appear inside JS template strings (api/discover-detail.js renders one),+// so both mask paths apply it.+const linkHintRe = /<link\b[^>]*\brel\s*=\s*["']?(?:preconnect|dns-prefetch)\b[^>]*>/gi;++function maskLinkHints(content, mask) {+ linkHintRe.lastIndex = 0;+ let m;+ while ((m = linkHintRe.exec(content))) mask.fill(1, m.index, m.index + m[0].length);+ return mask;+}+function htmlMask(content) {const mask = new Uint8Array(content.length);let m;@@ -383,7 +406,7 @@ function scanFile(file) {if (VENDOR_RE.test(rel)) return; // skip vendored libs — not our link surfaceconst content = readFileSync(file, 'utf8');const isJs = /\.m?js$/.test(file);- const mask = isJs ? commentMask(content) : htmlMask(content);+ const mask = maskLinkHints(content, isJs ? commentMask(content) : htmlMask(content));const inComment = (index) => mask[index] === 1;findings.scanned++;let m;@@ -436,31 +459,85 @@ function danglingRoutes() {}// ── External liveness (opt-in) ───────────────────────────────────────────────++// A bare `fetch` sends no User-Agent that any CDN recognizes, so Cloudflare and+// friends answer with a TLS reset or an interstitial. That read back as a dead+// link for pages a browser opens fine (t.me, solflare.com, ibm.com), which is+// worse than no report: a list that is mostly false positives gets ignored, and+// the real 404 hiding in it ships. Ask for the page the way a browser does.+const PROBE_HEADERS = {+ 'user-agent':+ 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',+ accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',+ 'accept-language': 'en-US,en;q=0.9',+};++// x.com serves a logged-out `/status/<id>` page as a 404 to every client that+// does not run its JS, including a real browser with cookies cleared. The probe+// therefore cannot tell a live post from a deleted one there, so it must not+// claim either. Profiles and every other path still get probed normally.+const UNPROBEABLE_RE = /^https?:\/\/(?:www\.)?(?:x|twitter)\.com\/[^/]+\/status\//i;++// 401/403/429/405 mean the host answered but bot-blocks automated probes, so the+// page exists. Only 404/410/5xx and DNS/timeout failures are truly dead.+const aliveStatus = (s) => s < 400 || [401, 403, 405, 429].includes(s);++// A HEAD is cheap but some hosts route it differently from a GET: alibabacloud+// sends one to a geo-router that 500s while the GET loads the page. A 404, an+// explicit method rejection, or any 5xx therefore earns one GET before the link+// is called dead. 403 is deliberately absent: it already reads as alive, and+// re-asking a bot-blocking CDN with a GET trades that answer for a reset.+const retryWithGet = (s) => s === 404 || s === 405 || s === 501 || s >= 500;++// undici caps response headers at 16 KB and throws past it. The host answered,+// so the page is there; only the probe could not read the preamble.+const ANSWERED_ANYWAY = new Set(['UND_ERR_HEADERS_OVERFLOW']);++function errorCode(e) {+ if (e.name === 'AbortError') return 'timeout';+ const code = e.cause?.code || e.cause?.name;+ return code ? `error:${code}` : 'error';+}+async function probeExternal(urls) {const dead = [];- const queue = [...urls];- const CONCURRENCY = 12;+ const queue = urls.filter((u) => !UNPROBEABLE_RE.test(u));+ // Eight at a time, not twelve: the probe shares one egress with whatever else+ // this machine is doing, and a saturated pool shows up as ETIMEDOUT on live+ // hosts, which is exactly the false positive this whole pass exists to kill.+ const CONCURRENCY = 8;async function worker() {while (queue.length) {const url = queue.shift();- let ok = false;- let status = 0;- try {+ const once = async (method) => {const ctrl = new AbortController();- const t = setTimeout(() => ctrl.abort(), 10000);- let res = await fetch(url, { method: 'HEAD', redirect: 'follow', signal: ctrl.signal });- if (res.status === 405 || res.status === 403 || res.status === 501) {- res = await fetch(url, { method: 'GET', redirect: 'follow', signal: ctrl.signal });+ const t = setTimeout(() => ctrl.abort(), 15000);+ try {+ const res = await fetch(url, { method, redirect: 'follow', signal: ctrl.signal, headers: PROBE_HEADERS });+ return { status: res.status };+ } catch (e) {+ if (ANSWERED_ANYWAY.has(e.cause?.code)) return { status: 200 };+ return { error: errorCode(e) };+ } finally {+ clearTimeout(t);+ }+ };+ let res = await once('HEAD');+ // A network error is as often this machine's egress as the host, and a+ // method rejection says nothing about the page, so both earn a GET. The+ // GET answers the actual question, but a GET that itself fails to connect+ // must not overwrite an answer the host already gave. A failed retry gets+ // one more attempt after a pause, since a saturated pool clears on its own.+ if (res.error || retryWithGet(res.status)) {+ let retry = await once('GET');+ if (retry.error) {+ await new Promise((r) => setTimeout(r, 1500));+ retry = await once('GET');}- clearTimeout(t);- status = res.status;- // 401/403/429/405 mean the host answered but bot-blocks automated probes —- // the page exists. Only 404/410/5xx and DNS/timeout failures are truly dead.- ok = res.status < 400 || [401, 403, 405, 429].includes(res.status);- } catch (e) {- status = e.name === 'AbortError' ? 'timeout' : 'error';+ if (!retry.error) res = retry;}- if (!ok) dead.push({ url, status });+ if (res.error) dead.push({ url, status: res.error });+ else if (!aliveStatus(res.status)) dead.push({ url, status: res.status });}}await Promise.all(Array.from({ length: CONCURRENCY }, worker));diff --git a/scripts/audit-play-failure-modes.mjs b/scripts/audit-play-failure-modes.mjsindex 563e74645..3c535c2f4 100644--- a/scripts/audit-play-failure-modes.mjs+++ b/scripts/audit-play-failure-modes.mjs@@ -96,6 +96,27 @@ const ATTEMPTS = Math.max(1, Number(process.env.ATTEMPTS) || 4);const HOST_TRANSPORT_ERROR =/ERR_NETWORK_CHANGED|ERR_CONNECTION_(RESET|CLOSED|ABORTED)|ERR_EMPTY_RESPONSE|ERR_SOCKET_NOT_CONNECTED/i;+// Playwright bounds navigations and actions, but NOT page.evaluate() or a+// context/browser close, and on this box those are exactly the calls that stop+// coming back: /play runs a rAF loop over two WebGL contexts, and when the host+// is oversubscribed several hundred deep the renderer stops servicing the CDP+// channel altogether. The sweep then sat on one scenario forever, printing+// nothing, which is worse than either a pass or a finding because it never+// reaches the other seventeen. Bound both, and treat a breach as infra so the+// scenario is retried and, failing that, counted NOT RUN.+const OBSERVE_TIMEOUT_MS = Number(process.env.OBSERVE_TIMEOUT_MS) || 90_000;+const TEARDOWN_TIMEOUT_MS = 15_000;++function withTimeout(promise, ms, label) {+ let timer;+ return Promise.race([+ Promise.resolve(promise).finally(() => clearTimeout(timer)),+ new Promise((_, reject) => {+ timer = setTimeout(() => reject(new Error(`${label} did not return within ${ms}ms`)), ms);+ }),+ ]);+}+// ── Hostile inputs ───────────────────────────────────────────────────────────// Every payload calls the same sentinel, so one flag proves script execution// regardless of which vector fired.@@ -295,6 +316,21 @@ async function baseAlive(base, timeoutMs = 5000) {}}+// Wait for the origin to answer again before spending an attempt on it. A dev+// server in this worktree restarts itself every time another agent edits .env or+// .env.local, and every request inside that window is refused. Retrying straight+// into the gap burned attempts on a server that was seconds away from being back,+// and a scenario that ran out of them was reported NOT RUN for a reason that had+// nothing to do with /play.+async function waitForBase(base, timeoutMs = 90_000) {+ const deadline = Date.now() + timeoutMs;+ while (Date.now() < deadline) {+ if (await baseAlive(base, 5000)) return true;+ await new Promise((r) => setTimeout(r, 2000));+ }+ return false;+}+function freePort() {return new Promise((resolve, reject) => {const srv = createServer();@@ -356,9 +392,13 @@ async function runScenario(base, sc) {const browser = await chromium.launch({ headless: !process.env.HEADFUL });const context = await browser.newContext({ viewport: { width: 1440, height: 900 } });// Close the process, not just the context. Every exit path goes through this.+ // A wedged renderer can swallow the graceful close too, and an unbounded close+ // hangs the whole sweep on a scenario that already produced its verdict. Bound+ // it and move on; Playwright kills the browser processes it launched when this+ // process exits, so the worst case is one idle chromium until the run ends.const teardown = async () => {- await context.close().catch(() => {});- await browser.close().catch(() => {});+ await withTimeout(context.close(), TEARDOWN_TIMEOUT_MS, 'context.close').catch(() => {});+ await withTimeout(browser.close(), TEARDOWN_TIMEOUT_MS, 'browser.close').catch(() => {});};const consoleIssues = [];const pageErrors = [];@@ -440,6 +480,19 @@ async function runScenario(base, sc) {|| HOST_TRANSPORT_ERROR.test(m)) {return { infra: m, findings: [], consoleIssues: [], pageErrors: [] };}+ // A navigation that ran out its budget is the ambiguous case: either the+ // document genuinely never reaches DOMContentLoaded, or this box (load in+ // the hundreds, memory exhausted) never gave the renderer enough CPU to get+ // there. Ask the origin directly, then retry rather than ruling on one+ // sample. A real hang reproduces on every attempt and still ends the+ // scenario, so nothing is excused here; what is avoided is one starved+ // attempt being published as "/play never loaded".+ if (/Timeout .*exceeded/i.test(m)) {+ const t0 = Date.now();+ const served = await baseAlive(base, 10_000);+ const how = served ? `origin served / in ${Date.now() - t0}ms` : 'origin did not answer either';+ return { infra: `${m.split('\n')[0]} (${how})`, findings: [], consoleIssues: [], pageErrors: [] };+ }findings.push(`navigation failed: ${m}`);return { findings, consoleIssues, pageErrors };}@@ -455,7 +508,7 @@ async function runScenario(base, sc) {return { infra: `host transport failure mid-scenario (${transportHit})`, findings: [], consoleIssues: [], pageErrors: [] };}- const o = await page.evaluate(() => {+ const observe = () => page.evaluate(() => {const loader = document.getElementById('kx-loading');const hidden = !loader || loader.classList.contains('kx-hidden');const errorCard = !!document.querySelector('.kx-boot-error');@@ -499,6 +552,16 @@ async function runScenario(base, sc) {};});+ let o;+ try {+ o = await withTimeout(observe(), OBSERVE_TIMEOUT_MS, 'page.evaluate');+ } catch (err) {+ // The renderer stopped answering (see OBSERVE_TIMEOUT_MS). Nothing was+ // observed, so there is nothing to judge /play on.+ await teardown();+ return { infra: String(err?.message || err), findings: [], consoleIssues: [], pageErrors: [] };+ }+// ── Universal assertions ────────────────────────────────────────────────if (o.xss) findings.push('SCRIPT EXECUTED from a query parameter (window.__xss set)');if (o.injectedScripts) findings.push(`${o.injectedScripts} injected script/img[onerror] node(s) in the DOM`);@@ -554,6 +617,7 @@ async function main() {// A network-change storm usually settles within seconds, so back off a// little further each time rather than landing in the same window twice.await new Promise((r) => setTimeout(r, 3000 * (attempt - 1)));+ await waitForBase(server.base);res = await runScenario(server.base, sc);}if (res.infra) {diff --git a/scripts/lib/csp-headers.mjs b/scripts/lib/csp-headers.mjsnew file mode 100644index 000000000..e09c900d9--- /dev/null+++ b/scripts/lib/csp-headers.mjs@@ -0,0 +1,114 @@+// Comparing the security headers a response carried against the ones the route+// table declares for it.+//+// The comparison is subtle enough to deserve its own home and its own tests:+// `server/csp-hashes.mjs` rewrites `script-src` per response, so a served+// policy is allowed to differ from the declared one in exactly one way, and+// only on the responses that get rewritten. Everything else about a security+// header has to arrive byte-for-byte or the site is not serving what the repo+// says it serves.+//+// Used by scripts/audit-csp.mjs. Pure functions: no network, no route table,+// no filesystem.++/** The security headers a document is expected to carry, whatever else it declares. */+export const REQUIRED_ON_HTML = [+ 'content-security-policy',+ 'strict-transport-security',+ 'x-content-type-options',+ 'referrer-policy',+];++/** "a 'b' c; d 'e'" -> Map{ a => Set{'b','c'}, d => Set{'e'} } */+export function parsePolicy(value) {+ const directives = new Map();+ for (const chunk of String(value).split(';')) {+ const parts = chunk.trim().split(/\s+/).filter(Boolean);+ if (!parts.length) continue;+ directives.set(parts[0].toLowerCase(), new Set(parts.slice(1)));+ }+ return directives;+}++const setsEqual = (a, b) => a.size === b.size && [...a].every((v) => b.has(v));++/**+ * Differences between the policy the route table declares and the one the+ * response carried, allowing only the rewrite server/csp-hashes.mjs performs:+ * within script-src / script-src-elem, `'unsafe-inline'` is replaced by any+ * number of `'sha256-…'` sources. Everything else must survive byte-for-byte.+ *+ * The rewrite is only applied to documents, so it is only required of them.+ * `sha256Rewritten` says whether this response is one the server hardens: for a+ * document it is demanded (a page that shipped the raw `'unsafe-inline'` policy+ * is exactly the regression this comparison exists to catch), and for+ * everything else (robots.txt, llms.txt, the .well-known JSON, an API response)+ * the declared policy has to arrive untouched instead.+ */+export function policyDiff(declared, served, sha256Rewritten) {+ const want = parsePolicy(declared);+ const got = parsePolicy(served);+ const problems = [];+ for (const [name, wantSources] of want) {+ const gotSources = got.get(name);+ if (!gotSources) {+ problems.push(`directive "${name}" is missing from the served policy`);+ continue;+ }+ const hashable = name === 'script-src' || name === 'script-src-elem';+ const expected = new Set(wantSources);+ const actual = new Set(gotSources);+ if (hashable && sha256Rewritten) {+ expected.delete("'unsafe-inline'");+ for (const src of [...actual]) if (src.startsWith("'sha256-")) actual.delete(src);+ }+ if (setsEqual(expected, actual)) continue;+ const missing = [...expected].filter((s) => !actual.has(s));+ const extra = [...actual].filter((s) => !expected.has(s));+ problems.push(+ `directive "${name}" differs${missing.length ? ` (missing ${missing.join(' ')})` : ''}` ++ `${extra.length ? ` (unexpected ${extra.join(' ')})` : ''}`,+ );+ }+ return problems;+}++/**+ * Every way a response's security headers fail to be what the route table says+ * this path serves. `declared` is the resolved header bag from vercel.json,+ * `served` the response's own headers; both keyed lowercase.+ */+export function headerProblems(declared, served) {+ const problems = [];++ for (const name of REQUIRED_ON_HTML) {+ if (!declared[name]) problems.push(`vercel.json declares no ${name} for this path`);+ }++ // Only documents get their script-src rewritten into hashes; a text or JSON+ // response carries the declared policy verbatim, and a response that carries+ // hashes proves the rewrite ran even if its content type is unusual.+ const isDocument = /text\/html/i.test(served['content-type'] || '');+ const carriesHashes = /'sha256-/.test(served['content-security-policy'] || '');++ for (const [name, expected] of Object.entries(declared)) {+ // Only security headers are the subject here. Cache and CORS values are+ // legitimately rewritten by the CDN and by conditional-request handling.+ if (!name.startsWith('x-') && !REQUIRED_ON_HTML.includes(name) && name !== 'permissions-policy') {+ continue;+ }+ const value = served[name];+ if (value === undefined) {+ problems.push(`${name} was declared but the response did not carry it`);+ continue;+ }+ if (name === 'content-security-policy') {+ problems.push(...policyDiff(expected, value, isDocument || carriesHashes));+ continue;+ }+ if (value.trim() !== expected.trim()) {+ problems.push(`${name} is "${value}" but vercel.json declares "${expected}"`);+ }+ }+ return problems;+}diff --git a/src/marketplace.js b/src/marketplace.jsindex 0872e0de0..1f797d7c0 100644--- a/src/marketplace.js+++ b/src/marketplace.js@@ -781,6 +781,14 @@ function renderTheme() {// to carry a public avatar GLB, so each card shows a live rotating model. We// reshuffle client-side too, so revisiting the page varies the order even when// the API response is served from cache.+//+// Each viewer ships `data-src` rather than `src`, the same contract the grid+// cards and the hero carousel already use: observeCardModelViewers() promotes+// it on first intersect and adds `auto-rotate` there, so a strip nobody has+// scrolled to never downloads a GLB, never stands up a WebGL context, and never+// runs a raf loop. Shipping `src` here left this the last eager model-viewer+// surface on the page, and model-viewer was the single most expensive script on+// it: 7,924ms of evaluation in a Lighthouse desktop trace of /marketplace.function renderThemePicks() {const row = $('market-theme-picks');if (!row) return;@@ -802,10 +810,9 @@ function renderThemePicks() {${views > 0 ? `<span class="market-theme-pick-views">⊙ ${fmtNumber(views)}</span>` : ''}<span class="market-theme-pick-placeholder" aria-hidden="true">${escapeHtml(initial(a.name || 'A'))}</span><model-viewer- src="${escapeHtml(a.avatar_glb_url)}"+ data-src="${escapeHtml(a.avatar_glb_url)}"alt="${escapeHtml(a.name || 'Agent')}"${a.thumbnail_url ? `poster="${escapeHtml(a.thumbnail_url)}"` : ''}- auto-rotateautoplayrotation-per-second="22deg"interaction-prompt="none"@@ -844,6 +851,9 @@ function renderThemePicks() {);}});+ // Hand the freshly built viewers to the shared observer so their data-src is+ // promoted when, and only when, the strip is actually near the viewport.+ observeCardModelViewers();}// Fisher–Yates shuffle. Browser-only path, so Math.random is fine here.@@ -1522,7 +1532,9 @@ function observeCardModelViewers() {{ rootMargin: '200px 0px', threshold: 0.01 },);}- document.querySelectorAll('.market-card-avatar model-viewer, .market-grid model-viewer').forEach((mv) => {+ const LAZY_VIEWERS =+ '.market-card-avatar model-viewer, .market-grid model-viewer, .market-theme-pick model-viewer';+ document.querySelectorAll(LAZY_VIEWERS).forEach((mv) => {if (mv.dataset.observed) return;mv.dataset.observed = '1';cardObserver.observe(mv);diff --git a/src/viewer.js b/src/viewer.jsindex b1507ea0d..953b7a798 100644--- a/src/viewer.js+++ b/src/viewer.js@@ -290,6 +290,16 @@ export class Viewer {this.scene = new Scene();this.scene.background = this.backgroundColor;+ // The host element can still be unlaid-out at construction (hidden tab,+ // CSS not applied yet, flex parent that sizes on the next frame). A 0-px+ // renderer/composer allocates zero-dimension GL textures, which the+ // driver rejects (glTexStorage2D: dimensions must be > 0) and every+ // subsequent clear/blit/draw then fails against an incomplete+ // framebuffer. Start at 1 px and let the ResizeObserver below re-size to+ // the real box on the first frame that has one.+ const initialWidth = Math.max(1, el.clientWidth);+ const initialHeight = Math.max(1, el.clientHeight);+const fov = options.preset === Preset.ASSET_GENERATOR ? (0.8 * 180) / Math.PI : 60;const aspect = el.clientHeight > 0 ? el.clientWidth / el.clientHeight : 1;this.defaultCamera = new PerspectiveCamera(fov, aspect, 0.01, 1000);@@ -323,7 +333,7 @@ export class Viewer {// biggest lever after MSAA on weak mobile GPUs.const dprCap = this._lowPower ? 1 : (options.maxPixelRatio ?? (options.kiosk ? 1.5 : 2));this.renderer.setPixelRatio(Math.min(window.devicePixelRatio, dprCap));- this.renderer.setSize(el.clientWidth, el.clientHeight);+ this.renderer.setSize(initialWidth, initialHeight);// Ground contact shadow (the one rendering-quality gap vs the// model-viewer surfaces). Cheap when nothing casts; skipped entirely on@@ -372,8 +382,8 @@ export class Viewer {// into a richer look. Screenshot capture bypasses this — see screenshot.js.this.state.cinematicPreset = options.cinematicPreset ?? DEFAULT_CINEMATIC_PRESET;this._cinematic = new CinematicPipeline(this.renderer, this.scene, this.activeCamera, {- width: el.clientWidth,- height: el.clientHeight,+ width: initialWidth,+ height: initialHeight,preset: this.state.cinematicPreset,});this._composer = this._cinematic.composer;diff --git a/tests/csp-headers-compare.test.js b/tests/csp-headers-compare.test.jsnew file mode 100644index 000000000..80a5ba8c9--- /dev/null+++ b/tests/csp-headers-compare.test.js@@ -0,0 +1,163 @@+// The rule this file pins: a document has to arrive with its `script-src`+// rewritten into hashes, and everything else has to arrive exactly as+// vercel.json declares it.+//+// Both halves have been wrong in production. A page that ships the raw+// `'unsafe-inline'` policy has lost the whole point of per-response hashing and+// must fail. A robots.txt or a .well-known JSON is never rewritten (the server+// only hardens HTML), so demanding hashes of it reports a defect on every+// non-document route the site serves, which is how a real header regression+// gets lost in the noise.++import { describe, it, expect } from 'vitest';+import { headerProblems, policyDiff, parsePolicy } from '../scripts/lib/csp-headers.mjs';++const DECLARED_CSP =+ "base-uri 'self'; object-src 'none'; script-src 'self' 'unsafe-inline' 'unsafe-eval' blob: https://esm.sh; worker-src 'self' blob:; frame-ancestors 'self'";++const HASH = "'sha256-K7gNU3sdo+OL0wNhqoVWhr3g6s1xYv72ol/pe/Unols='";++const declaredBag = () => ({+ 'content-security-policy': DECLARED_CSP,+ 'strict-transport-security': 'max-age=63072000; includeSubDomains; preload',+ 'x-content-type-options': 'nosniff',+ 'referrer-policy': 'strict-origin-when-cross-origin',+ 'cache-control': 'public, max-age=0, must-revalidate',+});++const hardened = DECLARED_CSP.replace("'unsafe-inline'", HASH);++const servedDocument = (csp = hardened) => ({+ ...declaredBag(),+ 'content-security-policy': csp,+ 'content-type': 'text/html; charset=utf-8',+});++const servedText = (csp = DECLARED_CSP) => ({+ ...declaredBag(),+ 'content-security-policy': csp,+ 'content-type': 'text/plain; charset=utf-8',+});++describe('parsePolicy', () => {+ it('splits directives into a source set each', () => {+ const parsed = parsePolicy("script-src 'self' blob:; object-src 'none'");+ expect([...parsed.get('script-src')]).toEqual(["'self'", 'blob:']);+ expect([...parsed.get('object-src')]).toEqual(["'none'"]);+ });++ it('ignores empty segments and trailing semicolons', () => {+ expect([...parsePolicy("base-uri 'self';;").keys()]).toEqual(['base-uri']);+ });+});++describe('policyDiff on a rewritten response', () => {+ it('accepts unsafe-inline replaced by hashes', () => {+ expect(policyDiff(DECLARED_CSP, hardened, true)).toEqual([]);+ });++ it('accepts several hashes in place of the one unsafe-inline', () => {+ const many = DECLARED_CSP.replace("'unsafe-inline'", `${HASH} ${HASH.replace('K7g', 'Zzz')}`);+ expect(policyDiff(DECLARED_CSP, many, true)).toEqual([]);+ });++ it('accepts a document with no inline script at all, which gets no hashes', () => {+ const noInline = DECLARED_CSP.replace(" 'unsafe-inline'", '');+ expect(policyDiff(DECLARED_CSP, noInline, true)).toEqual([]);+ });++ it('rejects a document that kept unsafe-inline instead of hashing', () => {+ const problems = policyDiff(DECLARED_CSP, DECLARED_CSP, true);+ expect(problems).toHaveLength(1);+ expect(problems[0]).toContain("unexpected 'unsafe-inline'");+ });++ it('rejects unsafe-inline smuggled in alongside the hashes', () => {+ const both = DECLARED_CSP.replace("'unsafe-inline'", `'unsafe-inline' ${HASH}`);+ expect(policyDiff(DECLARED_CSP, both, true)[0]).toContain("unexpected 'unsafe-inline'");+ });++ it('rejects a widened source list even when the hashes are right', () => {+ const widened = hardened.replace("script-src 'self'", "script-src 'self' https://evil.example");+ expect(policyDiff(DECLARED_CSP, widened, true)[0]).toContain('https://evil.example');+ });++ it('rejects a dropped directive', () => {+ const dropped = hardened.replace("; object-src 'none'", '');+ expect(policyDiff(DECLARED_CSP, dropped, true)[0]).toContain('"object-src" is missing');+ });++ it('rejects a loosened frame-ancestors', () => {+ const framed = hardened.replace("frame-ancestors 'self'", 'frame-ancestors *');+ expect(policyDiff(DECLARED_CSP, framed, true)[0]).toContain('frame-ancestors');+ });+});++describe('policyDiff on a response the server never rewrites', () => {+ it('accepts the declared policy verbatim', () => {+ expect(policyDiff(DECLARED_CSP, DECLARED_CSP, false)).toEqual([]);+ });++ it('still rejects a policy that differs in any other way', () => {+ const widened = DECLARED_CSP.replace("object-src 'none'", "object-src 'self'");+ expect(policyDiff(DECLARED_CSP, widened, false)[0]).toContain('object-src');+ });+});++describe('headerProblems', () => {+ it('passes a document that carries exactly what is declared', () => {+ expect(headerProblems(declaredBag(), servedDocument())).toEqual([]);+ });++ it('passes a text response that carries the unrewritten policy', () => {+ expect(headerProblems(declaredBag(), servedText())).toEqual([]);+ });++ it('demands the rewrite of a document, not of a text response', () => {+ expect(headerProblems(declaredBag(), servedDocument(DECLARED_CSP))).toHaveLength(1);+ expect(headerProblems(declaredBag(), servedText(DECLARED_CSP))).toEqual([]);+ });++ it('demands the rewrite of any response that already carries hashes', () => {+ const partly = servedText(`${DECLARED_CSP} ${HASH}`);+ expect(headerProblems(declaredBag(), partly)).not.toEqual([]);+ });++ it('reports a security header the response dropped', () => {+ const served = servedDocument();+ delete served['strict-transport-security'];+ expect(headerProblems(declaredBag(), served)).toEqual([+ 'strict-transport-security was declared but the response did not carry it',+ ]);+ });++ it('reports a weakened HSTS max-age', () => {+ const served = { ...servedDocument(), 'strict-transport-security': 'max-age=300' };+ expect(headerProblems(declaredBag(), served)[0]).toContain('strict-transport-security');+ });++ it('reports a missing declaration, so deleting the rule from vercel.json fails', () => {+ const declared = declaredBag();+ delete declared['x-content-type-options'];+ expect(headerProblems(declared, servedDocument())).toEqual([+ 'vercel.json declares no x-content-type-options for this path',+ ]);+ });++ it('ignores cache-control, which the CDN legitimately rewrites', () => {+ const served = { ...servedDocument(), 'cache-control': 'public, max-age=31536000, immutable' };+ expect(headerProblems(declaredBag(), served)).toEqual([]);+ });++ it('checks every x- header the route table declares', () => {+ const declared = { ...declaredBag(), 'x-frame-options': 'SAMEORIGIN' };+ const served = { ...servedDocument(), 'x-frame-options': 'ALLOWALL' };+ expect(headerProblems(declared, served)[0]).toContain('x-frame-options');+ });++ it('checks permissions-policy', () => {+ const declared = { ...declaredBag(), 'permissions-policy': 'camera=()' };+ const served = { ...servedDocument(), 'permissions-policy': 'camera=(self)' };+ expect(headerProblems(declared, served)[0]).toContain('permissions-policy');+ });+});diff --git a/tests/forge-scale.test.js b/tests/forge-scale.test.jsindex 4156f9114..485dc5654 100644--- a/tests/forge-scale.test.js+++ b/tests/forge-scale.test.js@@ -119,6 +119,37 @@ describe('shared circuit breaker (in-memory fallback)', () => {await circuitRecordSuccess(name);expect((await circuitState(name)).open).toBe(false);});++ it('caps the open window so a long outage cannot silence a cron for hours', async () => {+ // failures only reset on a success, so an uncapped (failures × baseMs) window+ // grows without bound: the every-minute seed crons would back off past the+ // point where they still look alive. The clamp keeps the retry cadence+ // bounded no matter how long the provider stays down.+ const name = 'test-breaker-cap';+ const baseMs = 10 * 60_000;+ const maxMs = 60 * 60_000;+ for (let i = 0; i < 40; i++) {+ await circuitRecordFailure(name, { threshold: 3, baseMs, maxMs });+ }+ const state = await circuitState(name);+ expect(state.failures).toBe(40);+ expect(state.open).toBe(true);+ // 40 × 10min would be 400 minutes without the clamp.+ expect(state.openUntil - Date.now()).toBeLessThanOrEqual(maxMs);+ expect(state.openUntil - Date.now()).toBeGreaterThan(maxMs - 60_000);+ await circuitRecordSuccess(name);+ expect((await circuitState(name)).open).toBe(false);+ });++ it('defaults the cap when a caller passes only threshold and baseMs', async () => {+ const name = 'test-breaker-default-cap';+ for (let i = 0; i < 40; i++) {+ await circuitRecordFailure(name, { threshold: 3, baseMs: 10 * 60_000 });+ }+ const state = await circuitState(name);+ expect(state.open).toBe(true);+ expect(state.openUntil - Date.now()).toBeLessThanOrEqual(60 * 60_000);+ });});describe('SCALE_LIMITS', () => {