Official Codebase of 3DCodeVerse
Python
render: cure the wedged-shared-browser 100s tail
gaoypeng8862caa9ce51
diff --git a/harness/docs/COST.md b/harness/docs/COST.mdindex 18a28ea3..f71d6243 100644--- a/harness/docs/COST.md+++ b/harness/docs/COST.md@@ -1212,14 +1212,22 @@ storm: `gemini-3.7-flash` answered on **15/22, 18/22 and 21/22 keys** while **5,keys returned 503 at the same instant (successful latencies 1.3–38 s; the slow ones were thesame few keys). So at any moment most keys work and the next key is the cure.-`rotate_with_retries` now adds the 503'd key to the call's failed set and rotates to a fresh-key for free (no sleep, no budget); only once `storm_quorum` (6, capped at the pool size)-distinct keys have 503'd within one call is it a storm and the old wait budget applies. What-this buys per call is the whole storm wait it used to pay first (median 5 s × the storm-streak, up to 900 s); what it costs is one more round-trip on a fresh key. Follow-ups worth-measuring: rank keys by recent latency (the 30 s keys are consistent), and record the key-index in `telemetry/usage.jsonl` so the distribution of calls per key can be read instead of-probed.+`rotate_with_retries` adds the 503'd key to the call's failed set and rotates to a fresh key+for free (no sleep, no budget, no `max_attempts`) — **exactly the 429 rule: while an untried+key remains, rotation is free**. It is a storm, and the bounded wait budget applies, only+once **every** key in the pool has 503'd inside one logical call (a 22-key pool rotates 21+times first). What this buys per call is the whole storm wait it used to pay first (median+5 s × the storm streak, up to 900 s); what it costs is one more round-trip on a fresh key.+Follow-ups worth measuring: rank keys by recent latency (the 30 s keys are consistent), and+record the key index in `telemetry/usage.jsonl` so the distribution of calls per key can be+read instead of probed.++**Amended 2026-08-27 (owner's rule):** the first cut of this gated the rotation behind a+`storm_quorum` of 6 distinct 503'd keys — so a 22-key pool stopped rotating and started+sleeping with 16 keys untried. The probes above say only **1–7 of 22 keys** are 503 at any+instant, and the 2026-08-26 logs carry **548 "capacity storm" lines with 0 429s**: the pool+had spare quota the whole time and the waits were pure loss. `storm_quorum` is gone; the+guard is now `len(failed_keys) < len(pool)`.@@ -1296,6 +1304,19 @@ remaining session / round budget (~1 430 s/run) — (2) hedge a 503 retry acrossthe ceiling inside a round (837 s sooner on killed runs) — (7) flash as the loop judge (~60 sbaseline) — (8) 30 s render cap (≤ 100 s). (1)–(3) and the key/attempt ledger fields landed the same afternoon (§27's follow-up paragraphs).+**Addendum 2026-08-27 — the render tail's root cause.** The 30–100 s renders (and the 330 s+timeouts) were never rendering: node's own `timing_ms` reported ~0.55 s for the same GLBs while the+Python wall clock read 100+ s, and 8 concurrent renders finished in 1.0 s total. The cost was a+**wedged shared browser**: the daemon's Chrome degrades after hours alive (WSL GPU decay + WebGL+pages leaked by SIGKILLed clients — every render timeout leaks one), and then *every* CDP roundtrip+(`newPage`, `page.close`) stalls ~100 s while the heartbeat keeps it alive forever. Fixed in+`runtime_js/`: (1) `gpu_launch.cjs` connect canary — one bounded `pages()` (6 s) per connect; an+unresponsive or overgrown (>12 pages) browser is poisoned and a fresh daemon spawned (~1 s once,+measured: leak 14 pages → next render 3.5 s, then 0.7 s steady); (2) `browser_daemon.cjs` reaps+pages older than `CV3D_PAGE_TTL_MS` (default 8 min > every harness ceiling); (3) cleanup steps in+`render_glb.mjs` / `host_page.mjs` are time-bounded (≤3 s each) — the record is on disk before+cleanup runs. Step-level stderr breadcrumbs (`[render_glb] <step> +ms`) stay in for the next audit.+## 29. Fewer turns — first turn read out (`CV3D_FEWER_TURNS`, `turns_v1`, 2026-08-26)Six blender prompts, plan-pinned pairs, flash, 3 rounds, storm afternoon; variant = `build` foldsdiff --git a/harness/runtime_js/browser_daemon.cjs b/harness/runtime_js/browser_daemon.cjsindex 617c7fb2..2932119a 100644--- a/harness/runtime_js/browser_daemon.cjs+++ b/harness/runtime_js/browser_daemon.cjs@@ -19,6 +19,10 @@ const { _internal } = require('./gpu_launch.cjs');const IDLE_REAP_MS = 90 * 1000;const HARD_REAP_MS = 10 * 60 * 1000;const POLL_MS = 15 * 1000;+// A page older than this belongs to nobody: every harness render/probe finishes or+// times out well under it (ceilings <= 330 s), and a SIGKILLed client never closes+// its page. Leaked pages hold WebGL contexts that eventually wedge the browser.+const PAGE_TTL_MS = Number(process.env.CV3D_PAGE_TTL_MS || 8 * 60 * 1000);function arg(name, dflt) {const i = process.argv.indexOf(`--${name}`);@@ -71,6 +75,7 @@ async function main() {process.exit(0);});+ const firstSeen = new Map(); // Page -> first-seen ms (puppeteer caches Page objects per target)setInterval(async () => {let st = null;try {@@ -80,6 +85,16 @@ async function main() {}const cur = (() => { try { return JSON.parse(fs.readFileSync(epPath, 'utf8')); } catch (_e) { return null; } })();if (!cur || cur.ws !== record.ws) return bail(0); // superseded by another daemon+ try { // reap pages leaked by killed clients (never the initial about:blank)+ const open = await browser.pages();+ const now = Date.now();+ const live = new Set(open);+ for (const p of open.slice(1)) if (!firstSeen.has(p)) firstSeen.set(p, now);+ for (const [p, at] of firstSeen) {+ if (!live.has(p)) firstSeen.delete(p);+ else if (now - at > PAGE_TTL_MS) { firstSeen.delete(p); await p.close().catch(() => {}); }+ }+ } catch (_e) { /* browser mid-shutdown; the checks below handle it */ }const idle = Date.now() - st.mtimeMs;if (idle < IDLE_REAP_MS) return;let pages = 2;diff --git a/harness/runtime_js/gpu_launch.cjs b/harness/runtime_js/gpu_launch.cjsindex 315e6f03..0f472481 100644--- a/harness/runtime_js/gpu_launch.cjs+++ b/harness/runtime_js/gpu_launch.cjs@@ -33,6 +33,8 @@ const CACHE_PATH = path.join(CACHE_DIR, 'gpu_probe.json');const NEGATIVE_TTL_MS = 20 * 60 * 1000; // how long a "no GPU" verdict is trustedconst PROTOCOL_TIMEOUT_MS = 15 * 60 * 1000; // big frame batches exceed puppeteer's 180 s defaultconst DAEMON_WAIT_MS = 25000; // max wait for a spawned daemon to advertise its endpoint+const CANARY_TIMEOUT_MS = 6000; // a healthy browser answers pages() in ~5 ms+const MAX_SHARED_PAGES = 12; // more open pages than any sane moment = leaked pages piling up // max wait for a spawned daemon to advertise its endpointconst SPAWN_LOCK_STALE_MS = 30000;const BASE_ARGS = ['--no-sandbox', '--disable-setuid-sandbox', '--enable-webgl', '--ignore-gpu-blocklist'];@@ -164,6 +166,26 @@ async function connectShared(puppeteer, backend) {if (!info || !info.ws) return null;try {const browser = await puppeteer.connect({ browserWSEndpoint: info.ws, protocolTimeout: PROTOCOL_TIMEOUT_MS });+ // Health canary: a long-lived shared browser can wedge (hours of WSL GPU decay+ // plus pages leaked by SIGKILLed clients) — then every CDP roundtrip stalls for+ // ~100 s and every render pays it. One bounded pages() catches both failure+ // modes: unresponsive, or overgrown with leaked pages. A bad verdict poisons+ // the endpoint (the daemon sees it vanish, bails and closes the browser) and+ // the caller spawns a fresh daemon (~1 s once, instead of minutes per render).+ let pages = null;+ try {+ let timer = null;+ pages = await Promise.race([+ browser.pages(),+ new Promise((_res, rej) => { timer = setTimeout(() => rej(new Error('canary timeout')), CANARY_TIMEOUT_MS); if (timer.unref) timer.unref(); }),+ ]);+ if (timer) clearTimeout(timer);+ } catch (_e) { pages = null; }+ if (!pages || pages.length > MAX_SHARED_PAGES) {+ try { await browser.disconnect(); } catch (_e) { /* ignore */ }+ try { fs.rmSync(endpointPath(backend), { force: true }); } catch (_e2) { /* ignore */ }+ return null;+ }heartbeat(backend);return {browser,diff --git a/harness/runtime_js/lib/host_page.mjs b/harness/runtime_js/lib/host_page.mjsindex 096db40f..985785d3 100644--- a/harness/runtime_js/lib/host_page.mjs+++ b/harness/runtime_js/lib/host_page.mjs@@ -105,10 +105,11 @@ export async function openHost(wsRoot, { width = 1024, height = 576, gpu = 'auto);const close = async () => {// page first, then release: a SHARED browser (connect-first reuse) must get- // its pages closed by us and be disconnected, never closed.- try { await page.close(); } catch (e) { /* ignore */ }- try { await releaseBrowser(launched); } catch (e) { /* ignore */ }- await srv.close();+ // its pages closed by us and be disconnected, never closed. Each step is+ // time-bounded: the result is already in hand when close() runs.+ await bounded(page.close(), 3000);+ await bounded(releaseBrowser(launched), 3000);+ await bounded(srv.close(), 2000);};return { page, browser: launched.browser, base: srv.base, gpu: launched.gpu, renderer: launched.renderer, errors, boot, close };} catch (e) {@@ -119,6 +120,17 @@ export async function openHost(wsRoot, { width = 1024, height = 576, gpu = 'auto}}+/** Await `work`, but at most `ms` — cleanup must never dominate a driver's wall time+ * (a wedged shared browser once cost 100 s per render in page.close alone; the daemon's+ * page reaper picks up anything a bounded close leaves behind). */+export function bounded(work, ms) {+ let timer = null;+ return Promise.race([+ Promise.resolve(work).catch(() => {}),+ new Promise((res) => { timer = setTimeout(res, ms); if (timer.unref) timer.unref(); }),+ ]).finally(() => { if (timer) clearTimeout(timer); });+}+/** Release a launchBrowser handle: release() when present (shared-aware), else close(). */export async function releaseBrowser(launched) {if (!launched || !launched.browser) return;diff --git a/harness/runtime_js/render_glb.mjs b/harness/runtime_js/render_glb.mjsindex e991f940..31c13e30 100644--- a/harness/runtime_js/render_glb.mjs+++ b/harness/runtime_js/render_glb.mjs@@ -19,7 +19,7 @@ import path from 'node:path';import { ensureDir, finish, parseCli, writeJson } from './lib/cli.mjs';import { launchBrowser, importMapHtml, runtimeMount, serveDirs } from './lib/host_env.mjs';-import { releaseBrowser } from './lib/host_page.mjs';+import { bounded, releaseBrowser } from './lib/host_page.mjs';const MODES = ['shaded', 'wire', 'normals', 'silhouette', 'clay'];@@ -76,6 +76,7 @@ renderGlbViews(${JSON.stringify(config_)}).then(async function main() {const t0 = Date.now();+ const step = (name) => process.stderr.write(`[render_glb] ${name} +${Date.now() - t0}ms\n`);const cfg = config();if (!fs.existsSync(cfg.glb)) throw new Error(`GLB not found: ${cfg.glb}`);ensureDir(cfg.out);@@ -86,16 +87,20 @@ async function main() {let page = null;let record = null;try {+ step('connect');launched = await launchBrowser({ gpu: cfg.gpu });const timing = { launch_ms: Date.now() - t0 };+ step('newPage');page = await launched.browser.newPage();await page.setViewport({ width: cfg.width, height: cfg.height, deviceScaleFactor: 1 });page.on('console', (msg) => {if (msg.type() === 'error' || msg.type() === 'warning') consoleErrors.push(`${msg.type()}: ${msg.text()}`);});page.on('pageerror', (e) => consoleErrors.push(`pageerror: ${e.message}`));+ step('goto');await page.goto(srv.url('/__render.html'), { waitUntil: 'load', timeout: cfg.timeoutMs });+ step('waitResult');await page.waitForFunction('window.__c3v_result !== null', { timeout: cfg.timeoutMs, polling: 100 });const result = await page.evaluate(() => window.__c3v_result);if (!result || !result.ok) {@@ -125,9 +130,13 @@ async function main() {writeJson(path.join(cfg.out, 'views.json'), record);} finally {// page first, then release: a shared browser is disconnected, never closed- if (page) await page.close().catch(() => {});- await releaseBrowser(launched).catch(() => {});- await srv.close();+ step('cleanup');+ if (page) await bounded(page.close(), 3000);+ step('page.closed');+ await bounded(releaseBrowser(launched), 3000);+ step('released');+ await bounded(srv.close(), 2000);+ step('srv.closed');}return finish(record, 0);}