Skip to content

Official Codebase of 3DCodeVerse

Python

docs: a single report page — architecture, implementation, and every measured result with its confidence interval

gaoypengacc07894919b
diff --git a/harness/docs/report.html b/harness/docs/report.html
new file mode 100644
index 00000000..a04c8bec
--- /dev/null
+++ b/harness/docs/report.html
@@ -0,0 +1,562 @@
+<title>3dcodeverse Harness Report</title>
+<link rel="preconnect" href="https://fonts.googleapis.com">
+<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;500;600&family=IBM+Plex+Sans:wght@400;500;600;700&display=swap">
+<style>
+:root{
+ --bg:#F4F6F8; --surface:#FFFFFF; --surface-2:#EBEFF3; --line:#D4DBE3; --line-soft:#E3E9EF;
+ --ink:#111821; --ink-2:#3D4956; --ink-3:#68757F;
+ --accent:#00786F; --accent-soft:#D3ECE8; --accent-line:#00786F;
+ --good:#1F6E4A; --good-bg:#DDF0E4; --warn:#8A5A08; --warn-bg:#FBEBCF; --null:#9B2C24; --null-bg:#F8DEDA;
+ --grid:#DFE5EB;
+ --mono:'IBM Plex Mono',ui-monospace,SFMono-Regular,Menlo,monospace;
+ --sans:'IBM Plex Sans',system-ui,-apple-system,Segoe UI,Roboto,sans-serif;
+ --rail:250px;
+}
+@media (prefers-color-scheme:dark){:root:not([data-theme="light"]){
+ --bg:#0C1016; --surface:#141A22; --surface-2:#1B232D; --line:#2A3542; --line-soft:#212B36;
+ --ink:#E6ECF2; --ink-2:#A8B5C2; --ink-3:#76848F;
+ --accent:#3ED8C6; --accent-soft:#123330; --accent-line:#2AB5A6;
+ --good:#5CD69A; --good-bg:#12301F; --warn:#F0B846; --warn-bg:#32250C; --null:#F08A80; --null-bg:#34161A;
+ --grid:#232E3A;
+}}
+:root[data-theme="dark"]{
+ --bg:#0C1016; --surface:#141A22; --surface-2:#1B232D; --line:#2A3542; --line-soft:#212B36;
+ --ink:#E6ECF2; --ink-2:#A8B5C2; --ink-3:#76848F;
+ --accent:#3ED8C6; --accent-soft:#123330; --accent-line:#2AB5A6;
+ --good:#5CD69A; --good-bg:#12301F; --warn:#F0B846; --warn-bg:#32250C; --null:#F08A80; --null-bg:#34161A;
+ --grid:#232E3A;
+}
+*{box-sizing:border-box}
+body{margin:0;background:var(--bg);color:var(--ink);font-family:var(--sans);font-size:15px;line-height:1.6;
+ -webkit-font-smoothing:antialiased;font-variant-numeric:tabular-nums}
+.wrap{display:grid;grid-template-columns:var(--rail) minmax(0,1fr);gap:0;max-width:1400px;margin:0 auto}
+/* ---------- rail ---------- */
+.rail{position:sticky;top:0;height:100vh;overflow-y:auto;padding:34px 20px 34px 28px;border-right:1px solid var(--line);
+ background:var(--bg)}
+.brand{font-family:var(--mono);font-weight:600;font-size:14px;letter-spacing:.02em;color:var(--ink);margin-bottom:2px}
+.brand span{color:var(--accent)}
+.brand-sub{font-family:var(--mono);font-size:11px;color:var(--ink-3);margin-bottom:26px;line-height:1.5}
+.rail a{display:flex;gap:9px;text-decoration:none;color:var(--ink-2);font-size:13px;padding:5px 0;border-radius:4px}
+.rail a:hover{color:var(--accent)}
+.rail a .n{font-family:var(--mono);font-size:11px;color:var(--ink-3);min-width:15px;padding-top:2px}
+.rail a:hover .n{color:var(--accent)}
+.rail hr{border:0;border-top:1px solid var(--line-soft);margin:18px 0}
+/* ---------- main ---------- */
+main{padding:56px 48px 120px;min-width:0}
+h1{font-family:var(--mono);font-size:30px;font-weight:600;letter-spacing:-.01em;margin:0 0 10px;text-wrap:balance}
+.lede{font-size:17px;color:var(--ink-2);max-width:68ch;margin:0 0 12px}
+.stamp{font-family:var(--mono);font-size:11.5px;color:var(--ink-3);margin-bottom:36px}
+section{margin:0 0 56px;scroll-margin-top:24px}
+h2{font-family:var(--mono);font-size:12px;font-weight:600;letter-spacing:.1em;text-transform:uppercase;color:var(--accent);
+ margin:0 0 6px;display:flex;align-items:baseline;gap:10px}
+h2 .n{color:var(--ink-3);font-weight:400}
+h3{font-size:19px;font-weight:600;margin:0 0 12px;letter-spacing:-.01em;text-wrap:balance}
+h4{font-family:var(--mono);font-size:13px;font-weight:600;margin:26px 0 10px;color:var(--ink)}
+p{max-width:72ch;margin:0 0 14px;color:var(--ink-2)}
+p strong,li strong{color:var(--ink);font-weight:600}
+code{font-family:var(--mono);font-size:.88em;background:var(--surface-2);padding:1px 5px;border-radius:3px;color:var(--ink)}
+ul{max-width:72ch;color:var(--ink-2);padding-left:18px;margin:0 0 14px}
+li{margin-bottom:6px}
+/* ---------- verdict ---------- */
+.verdicts{display:grid;grid-template-columns:repeat(auto-fit,minmax(230px,1fr));gap:12px;margin:0 0 20px}
+.vcard{background:var(--surface);border:1px solid var(--line);border-radius:8px;padding:16px 18px;border-top:3px solid var(--tone)}
+.vcard.good{--tone:var(--good)} .vcard.warn{--tone:var(--warn)} .vcard.null{--tone:var(--null)}
+.vcard .k{font-family:var(--mono);font-size:10.5px;letter-spacing:.09em;text-transform:uppercase;color:var(--tone);font-weight:600}
+.vcard .v{font-family:var(--mono);font-size:25px;font-weight:600;margin:7px 0 4px;letter-spacing:-.02em}
+.vcard .d{font-size:13px;color:var(--ink-3);line-height:1.45}
+/* ---------- tables ---------- */
+.scroll{overflow-x:auto;margin:0 0 16px;border:1px solid var(--line);border-radius:8px;background:var(--surface)}
+table{border-collapse:collapse;width:100%;font-size:13.5px}
+th{font-family:var(--mono);font-size:10.5px;letter-spacing:.07em;text-transform:uppercase;color:var(--ink-3);
+ text-align:left;padding:11px 14px;border-bottom:1px solid var(--line);white-space:nowrap;font-weight:600}
+td{padding:9px 14px;border-bottom:1px solid var(--line-soft);color:var(--ink-2);white-space:nowrap}
+tr:last-child td{border-bottom:0}
+td.num,th.num{text-align:right;font-family:var(--mono)}
+td.arm{font-family:var(--mono);font-size:12.5px;color:var(--ink);white-space:nowrap}
+tr.hl td{background:var(--accent-soft)}
+tr.hl td.arm{font-weight:600}
+.chip{display:inline-block;font-family:var(--mono);font-size:10.5px;font-weight:600;padding:2px 7px;border-radius:99px;letter-spacing:.04em}
+.chip.good{background:var(--good-bg);color:var(--good)} .chip.warn{background:var(--warn-bg);color:var(--warn)}
+.chip.null{background:var(--null-bg);color:var(--null)} .chip.mute{background:var(--surface-2);color:var(--ink-3)}
+/* ---------- CI plot ---------- */
+.ci{display:grid;grid-template-columns:var(--cilbl) 1fr var(--cival);gap:0 16px;align-items:center;
+ background:var(--surface);border:1px solid var(--line);border-radius:8px;padding:18px 20px}
+/* fixed columns, not auto: each row is its own grid, so an auto label column made the
+ zero line land at a different x on every row — the one thing this plot must get right */
+:root{--cilbl:250px;--cival:158px}
+.ci .lbl{font-family:var(--mono);font-size:12px;color:var(--ink);padding:7px 0}
+.ci .track{position:relative;height:34px}
+.ci .zero{position:absolute;top:0;bottom:0;width:1px;background:var(--ink-3);opacity:.55}
+.ci .bar{position:absolute;top:50%;transform:translateY(-50%);height:5px;border-radius:3px;background:var(--tone);opacity:.35}
+.ci .pt{position:absolute;top:50%;transform:translate(-50%,-50%);width:11px;height:11px;border-radius:50%;background:var(--tone);
+ border:2px solid var(--surface)}
+.ci .val{font-family:var(--mono);font-size:11.5px;color:var(--ink-3);text-align:right;padding:7px 0}
+.ci .val b{color:var(--ink);font-weight:600}
+.ci.g{--tone:var(--good)} .ci.w{--tone:var(--warn)} .ci.n{--tone:var(--null)}
+.axis{display:grid;grid-template-columns:var(--cilbl) 1fr var(--cival);gap:0 16px;padding:0 20px;
+ font-family:var(--mono);font-size:10.5px;color:var(--ink-3)}
+.axis .ticks{display:flex;justify-content:space-between}
+/* ---------- pipeline ---------- */
+.pipe{display:flex;flex-wrap:wrap;align-items:stretch;gap:0;margin:0 0 18px;border:1px solid var(--line);border-radius:8px;
+ background:var(--surface);overflow:hidden}
+.stage{flex:1 1 130px;min-width:130px;padding:14px 15px;border-right:1px solid var(--line-soft)}
+.stage:last-child{border-right:0}
+.stage .t{font-family:var(--mono);font-size:11px;font-weight:600;color:var(--accent);letter-spacing:.05em;text-transform:uppercase}
+.stage .b{font-size:12.5px;color:var(--ink-2);margin-top:5px;line-height:1.45}
+.stage.gate{background:var(--surface-2)}
+.stage.gate .t{color:var(--ink-2)}
+/* ---------- package map ---------- */
+.pkgs{display:grid;grid-template-columns:repeat(auto-fill,minmax(220px,1fr));gap:9px}
+.pkg{background:var(--surface);border:1px solid var(--line);border-radius:7px;padding:12px 14px}
+.pkg .n{font-family:var(--mono);font-size:12.5px;color:var(--ink);font-weight:600}
+.pkg .m{font-family:var(--mono);font-size:10.5px;color:var(--ink-3);margin:3px 0 6px}
+.pkg .d{font-size:12.5px;color:var(--ink-2);line-height:1.45}
+.pkg .barwrap{height:3px;background:var(--surface-2);border-radius:2px;margin-top:9px;overflow:hidden}
+.pkg .barwrap i{display:block;height:100%;background:var(--accent-line);opacity:.65}
+/* ---------- callout ---------- */
+.call{border-left:3px solid var(--tone,var(--accent));background:var(--surface);padding:15px 18px;border-radius:0 7px 7px 0;
+ margin:0 0 18px;border-top:1px solid var(--line);border-right:1px solid var(--line);border-bottom:1px solid var(--line)}
+.call .h{font-family:var(--mono);font-size:11px;font-weight:600;letter-spacing:.07em;text-transform:uppercase;
+ color:var(--tone,var(--accent));margin-bottom:6px}
+.call p{margin:0;max-width:none;font-size:14px}
+.call p + p{margin-top:9px}
+.call.warn{--tone:var(--warn)} .call.null{--tone:var(--null)} .call.good{--tone:var(--good)}
+.sev{display:grid;grid-template-columns:repeat(auto-fit,minmax(120px,1fr));gap:9px;margin:0 0 18px}
+.sevc{background:var(--surface);border:1px solid var(--line);border-radius:7px;padding:12px 14px;text-align:center}
+.sevc .v{font-family:var(--mono);font-size:24px;font-weight:600;color:var(--tone,var(--ink))}
+.sevc .k{font-family:var(--mono);font-size:10px;letter-spacing:.07em;text-transform:uppercase;color:var(--ink-3);margin-top:3px}
+.foot{border-top:1px solid var(--line);padding-top:18px;font-family:var(--mono);font-size:11.5px;color:var(--ink-3);max-width:none}
+@media (max-width:960px){
+ .wrap{grid-template-columns:1fr}
+ .rail{position:static;height:auto;border-right:0;border-bottom:1px solid var(--line);padding:22px 22px 14px}
+ .rail nav{display:flex;flex-wrap:wrap;gap:2px 16px}
+ main{padding:30px 22px 80px}
+ .ci,.axis{grid-template-columns:1fr;gap:2px}
+ :root{--cilbl:1fr;--cival:auto}
+ .ci .track{height:26px}
+ .ci .val{text-align:left}
+}
+</style>
+<div class="wrap">
+<aside class="rail">
+ <div class="brand">3dcodeverse<span>/</span>harness</div>
+ <div class="brand-sub">data flywheel for<br>LLM-written raw 3D code</div>
+ <nav>
+ <a href="#verdict"><span class="n">1</span><span>Verdict</span></a>
+ <a href="#what"><span class="n">2</span><span>What it is</span></a>
+ <a href="#arch"><span class="n">3</span><span>Architecture</span></a>
+ <a href="#impl"><span class="n">4</span><span>Implementation</span></a>
+ <a href="#noise"><span class="n">5</span><span>The noise floor</span></a>
+ <a href="#stats"><span class="n">6</span><span>Paired results</span></a>
+ <a href="#arms"><span class="n">7</span><span>Every arm</span></a>
+ <a href="#cells"><span class="n">8</span><span>Per-prompt</span></a>
+ <a href="#stability"><span class="n">9</span><span>Stability pass</span></a>
+ <a href="#open"><span class="n">10</span><span>Known open</span></a>
+ </nav>
+ <hr>
+ <div class="brand-sub" id="railstamp"></div>
+</aside>
+
+<main>
+<h1>3dcodeverse harness</h1>
+<p class="lede">A measurement instrument for LLM-authored <strong>raw</strong> 3D code — bpy, CadQuery, three.js, URDF, GLSL — across four tracks and eight backends, with deterministic gates ahead of every VLM judgement.</p>
+<p class="stamp" id="stamp"></p>
+
+<!-- ============================== 1 VERDICT -->
+<section id="verdict">
+ <h2><span class="n">01</span> Verdict</h2>
+ <h3>What the measurements actually support</h3>
+ <div class="verdicts" id="verdictcards"></div>
+ <div class="call">
+ <div class="h">Read this first</div>
+ <p>Every number below is a <strong>paired</strong> comparison: the same prompt, the same fixed judge (<code>gemini-3.1-pro-preview</code>, n_samples 2), the two arms run in the same weather. Cells lost to a provider outage are <code>infra_failed</code> and excluded from every rate — never scored 0. Unpaired means over different prompt sets are not comparable and are not used for any claim here.</p>
+ </div>
+</section>
+
+<!-- ============================== 2 WHAT -->
+<section id="what">
+ <h2><span class="n">02</span> What it is</h2>
+ <h3>A flywheel, not a generator</h3>
+ <p>The harness takes a one-line prompt and a chosen <em>track</em>, drives a model to write raw source in the target language, then builds, measures, renders and judges the artifact — repeating for a bounded number of rounds. Every round is a git commit of <code>src/</code>, every model call is a priced <code>Usage</code> row, and every run ends as a <code>record.json</code> that the flywheel turns into dataset samples, preference pairs and captions.</p>
+ <h4>Tracks and languages</h4>
+ <div class="scroll"><table>
+ <thead><tr><th>Track</th><th>Languages</th><th>Deterministic gates before the judge</th></tr></thead>
+ <tbody>
+ <tr><td class="arm">static_object</td><td class="arm">blender · cadquery · threejs</td><td>lint → build → measure → connectivity → contract → 8-view render</td></tr>
+ <tr><td class="arm">articulated_object</td><td class="arm">urdf_blender</td><td>+ joint sweep, per-pose collision, link-hierarchy part count</td></tr>
+ <tr><td class="arm">scene</td><td class="arm">scene_threejs</td><td>+ scene census, camera framing, inter-frame motion, shader preflight</td></tr>
+ <tr><td class="arm">graphics</td><td class="arm">glsl_shader · opengl_python</td><td>+ shader compile, GL frame capture, frame statistics</td></tr>
+ </tbody>
+ </table></div>
+ <h4>Backends it drives</h4>
+ <p>Five agent kinds — <code>api-agent</code> (in-process, tool-calling), <code>gemini-cli</code>, <code>claude-code</code>, <code>codex</code>, <code>agy</code> — plus <code>single-shot:&lt;provider&gt;:&lt;model&gt;</code> for one-call generation. Providers: Gemini, Anthropic, OpenAI. The same <code>Spec</code> runs on any of them, which is what makes the comparisons in §6 possible at all.</p>
+ <div class="call">
+ <div class="h">The law that shapes everything</div>
+ <p>Generated code is <strong>raw language, never an SDK</strong>. The model writes the same <code>bpy</code> a human would; the harness owns the wrappers, exporters and render rigs. That is the difference between measuring a model's 3D reasoning and measuring how well it can call a helper library.</p>
+ </div>
+</section>
+
+<!-- ============================== 3 ARCHITECTURE -->
+<section id="arch">
+ <h2><span class="n">03</span> Architecture</h2>
+ <h3>One run, end to end</h3>
+ <div class="pipe">
+ <div class="stage"><div class="t">Spec</div><div class="b">prompt + track + language + dims + must-haves, validated as a typed contract</div></div>
+ <div class="stage"><div class="t">Brief</div><div class="b">optional: one cheap call expands the prompt into real dimensions and sub-assemblies</div></div>
+ <div class="stage"><div class="t">Plan</div><div class="b">typed part list with bboxes, materials, attachment; budget derived from the request</div></div>
+ <div class="stage"><div class="t">Generate</div><div class="b">agent writes raw source; per-part scoping for large objects</div></div>
+ <div class="stage gate"><div class="t">Gates</div><div class="b">lint → build → measure → connectivity → contract — all deterministic, all in code</div></div>
+ <div class="stage gate"><div class="t">Render</div><div class="b">8 canonical views + montage, GPU headless Chrome or Blender</div></div>
+ <div class="stage"><div class="t">Judge</div><div class="b">VLM scores a rubric; the score is computed in code from weights, caps and floors</div></div>
+ <div class="stage"><div class="t">Refine</div><div class="b">repair / refine / detail rounds until pass, plateau or budget</div></div>
+ <div class="stage"><div class="t">Record</div><div class="b">record.json → dataset samples, preference pairs, captions, gallery</div></div>
+ </div>
+ <h4>The six laws</h4>
+ <ul>
+ <li><strong>Raw language only</strong> — no SDK or helper import in generated code.</li>
+ <li><strong>One source of truth for frames</strong> — <code>conventions.py</code> alone states units, axes and naming; every package imports, none restates.</li>
+ <li><strong>Deterministic first, VLM last</strong> — code decides geometry; the model is used only for perception. Scores are computed from rubric weights, with explicit caps and floors.</li>
+ <li><strong>Cheap first</strong> — lint before build before gates before renders before the judge. Budgets are hard ceilings.</li>
+ <li><strong>Everything is a record</strong> — every round a git commit, every call a priced <code>Usage</code>, every run a <code>record.json</code>.</li>
+ <li><strong>Export as authored</strong> — no wrapper silently re-centres or grounds an object; the gates catch it instead.</li>
+ </ul>
+ <h4>Package map</h4>
+ <div class="pkgs" id="pkgs"></div>
+</section>
+
+<!-- ============================== 4 IMPLEMENTATION -->
+<section id="impl">
+ <h2><span class="n">04</span> Implementation</h2>
+ <h3>The parts that carry the weight</h3>
+
+ <h4>Deterministic gates — where the harness earns its lift</h4>
+ <p>The judge never decides whether geometry is correct. <code>spatial/</code> measures it: part count over the real link hierarchy, pairwise interpenetration depth and buried-surface fraction, floating-component detection against declared attachment, joint sweeps over poses with per-pose collision, camera framing and inter-frame motion for scenes, GL frame statistics for shaders. A finding carries a severity, a target, a fix hint and structured data, and the round loop feeds those back as the repair prompt.</p>
+
+ <h4>Key pool — 22 keys, scheduled on tokens not requests</h4>
+ <p>Measured on this machine: a generator call averages ~42k prompt tokens, so 1M TPM per key is ~24 calls/min — the token quota binds long before the request quota. <code>KeyPool</code> reserves estimated prompt tokens on acquire and reconciles on report. The measured concurrency knee is <strong>64 in-flight</strong> (72.9 calls/min, against 43.9 at 32 and a fall back to 47.2 at 128). Because the pool is per-process, the operational rule is that the <em>sum</em> of in-flight caps across every harness process stays at the knee.</p>
+
+ <h4>Retry policy — bounded in time, not in attempts</h4>
+ <p>No single wait exceeds <strong>5 s</strong>; with 22 keys, rotating beats sitting out a backoff. A capacity storm gets its own patience budget, and the whole retry sequence for one logical call is bounded by a <strong>900 s deadline</strong> — before that bound existed, 60 storm attempts × a 300 s read timeout meant one call could hold for 5.6 hours.</p>
+
+ <h4>Cost ledger</h4>
+ <p>Every model call is priced into <code>telemetry/cost.jsonl</code> with its price provenance, attributed to a stage and a role, nested per run so a parallel battery still attributes correctly. Three profiles — <code>economy</code>, <code>balanced</code>, <code>quality</code> — move models, judge samples, rounds, candidates and ceilings together on one dial.</p>
+
+ <h4>Honest failure accounting</h4>
+ <p>A cell can end without a score for reasons that say nothing about the model. <code>infra_failed</code> (provider outage) is excluded from score, build rate <em>and</em> wall-clock. <code>budget_exhausted</code> (ran out of time with no artifact) has no score but still counts against build rate. A model that answers with prose, unparseable code or a genuine API mistake keeps its zero. Getting this wrong is not cosmetic: it once put five hard zeros on a one-shot arm while dropping the harness cells hit by the same outage.</p>
+
+ <h4>Scale</h4>
+ <div class="scroll"><table>
+ <thead><tr><th>Component</th><th class="num">Files</th><th class="num">Lines</th></tr></thead>
+ <tbody id="scaletable"></tbody>
+ </table></div>
+</section>
+
+<!-- ============================== 5 NOISE -->
+<section id="noise">
+ <h2><span class="n">05</span> The noise floor</h2>
+ <h3>What a single comparison can and cannot tell you</h3>
+ <div class="call null">
+ <div class="h">A/A calibration — the most important result here</div>
+ <p>Two <strong>A/A</strong> runs — identical arms, same prompt, same judge, differing only in run-to-run randomness — measured <strong>+0.344</strong> and <strong>−0.100</strong>. A rig with a ±0.02 keep/revert threshold would have concluded <em>keep</em> from one and <em>revert</em> from the other, <strong>from nothing at all</strong>.</p>
+ <p>So a single prompt pair carries no signal. Every claim in §6 is therefore reported with its paired standard error, a 95% confidence interval and an exact sign test, and anything whose interval crosses zero is labelled unsupported however good the mean looks.</p>
+ </div>
+ <p>The practical consequence: an 8-prompt battery resolves a real difference of roughly <strong>0.25</strong>, not 0.02. Differences smaller than that need either many more prompts or a within-prompt design.</p>
+</section>
+
+<!-- ============================== 6 STATS -->
+<section id="stats">
+ <h2><span class="n">06</span> Paired results</h2>
+ <h3>Every headline, with its confidence interval</h3>
+ <p>Bars show the 95% CI of the paired difference; the dot is the mean. A bar that crosses the zero line is <strong>not separated from noise</strong> — the ordering may be real, but this data does not establish it.</p>
+ <div id="ciplot"></div>
+ <div class="scroll" style="margin-top:16px"><table>
+ <thead><tr><th>Comparison</th><th class="num">n</th><th class="num">Δ</th><th class="num">95% CI</th><th class="num">t</th><th class="num">W/L/T</th><th class="num">sign p</th><th>Verdict</th></tr></thead>
+ <tbody id="stattable"></tbody>
+ </table></div>
+ <div class="call good" style="margin-top:18px">
+ <div class="h">The finding worth keeping</div>
+ <p>The harness lift is <strong>largest on the weakest model</strong> and shrinks as the model gets stronger: luna <strong>+0.291</strong> (solid, 10W/0L), terra <strong>+0.191</strong>, sol <strong>+0.155</strong>. The loop compresses the gap between models rather than scaling with them — which is exactly what a good scaffold should do, and it is why a cheap model inside the harness reaches a strong model's one-shot quality.</p>
+ </div>
+ <div class="call warn">
+ <div class="h">What is not yet true</div>
+ <p>Paired on the 8 shared prompts, <strong>harness+flash − one-shot gpt-5.6-sol = −0.002</strong> (CI −0.118…+0.114). That is a <strong>dead heat</strong>, not a win. The goal of beating sol with flash alone is currently <em>at parity</em>. The remaining gap to close is the <strong>+0.169</strong> that harness+codex-sol still holds over harness+flash (8W/0L, solid).</p>
+ </div>
+</section>
+
+<!-- ============================== 7 ARMS -->
+<section id="arms">
+ <h2><span class="n">07</span> Every arm</h2>
+ <h3>Per battery, evaluated cells only</h3>
+ <p><code>drop</code> counts cells lost to a provider outage and excluded from every rate. <code>$gen</code> is generation cost for the whole arm — for subscription CLIs it is imputed from recorded tokens at list price, not money actually billed.</p>
+ <div id="armtables"></div>
+</section>
+
+<!-- ============================== 8 CELLS -->
+<section id="cells">
+ <h2><span class="n">08</span> Per-prompt</h2>
+ <h3>The hard battery, cell by cell</h3>
+ <p>Eight prompts on the <code>compare_v2</code> battery — two medium controls and six hard, chosen across decomposition, curved-organic, thin-feature, mechanism and architectural axes. Every one of the one-shot zeros below is an <em>earned</em> zero: a real Blender API mistake (<code>calc_normals</code>, <code>calc_normals_split</code>, <code>Matrix.Euler</code> — all removed in 4.1+), not an outage.</p>
+ <div id="celltable"></div>
+</section>
+
+<!-- ============================== 9 STABILITY -->
+<section id="stability">
+ <h2><span class="n">09</span> Stability pass</h2>
+ <h3>45 candidate defects, 42 verified, 50 fixes</h3>
+ <p>Six independent sweeps hunted the tree; a seventh set of agents tried to <em>refute</em> every finding before it counted; three fixed what survived. Each fix carries a regression test proven to fail on the old code.</p>
+ <div class="sev" id="sevcards"></div>
+ <h4>The ones that mattered</h4>
+ <div class="scroll"><table>
+ <thead><tr><th>Defect</th><th>Why it mattered</th></tr></thead>
+ <tbody>
+ <tr><td class="arm">connectivity blind to<br>non-watertight parts</td><td>Interpenetration was never detected between open meshes — which is most LLM-authored geometry. Two sibling defects: a model authoring below the floor disabled the floating check, and every ground-touching component counted as supported.</td></tr>
+ <tr><td class="arm">write_json_atomic<br>shared temp name</td><td>One fixed <code>.tmp</code> per path meant concurrent writers published truncated JSON. It is the primitive under every record, plan and ledger; the same race silently disabled the three.js asset gate.</td></tr>
+ <tr><td class="arm">claude-code recorded<br>the wrong model</td><td><code>claude -p</code> bills two models per session and lists the CLI's housekeeping model first. Every alias — <code>sonnet</code>, <code>opus</code>, the default — was recorded as <code>claude-haiku-4-5</code>, the model that did 0.9% of the tokens. A sonnet-vs-default comparison was really haiku-vs-haiku.</td></tr>
+ <tr><td class="arm">resume corrupted<br>finished runs</td><td>No terminal-state guard: resuming a completed run re-billed the plan stage and overwrote <code>passed</code> with <code>planning</code>, leaving it stuck. Separately, an unreadable cached stage bricked resume permanently.</td></tr>
+ <tr><td class="arm">parquet dropped the<br>complexity columns</td><td>The flywheel's dataset silently shipped without the objective complexity vector — the metrics the whole complexity study depends on.</td></tr>
+ <tr><td class="arm">one CV3D_JUDGE__* var<br>disabled the profile</td><td>Setting a single judge field discarded the profile's entire judge block.</td></tr>
+ <tr><td class="arm">a NUL from the model<br>reached an argv</td><td>A planner wrote <code></code> where it meant <code>×</code> and <code>±</code>. Legal JSON, so it passed the parser, pydantic and <code>plan.json</code> — then killed all four generation tasks of a cell inside <code>subprocess.Popen</code> with an error naming no file, no field and no value.</td></tr>
+ </tbody>
+ </table></div>
+ <h4>Smoke matrix</h4>
+ <p><strong>47 settings executed, 0 failed, 8 skipped.</strong> All seven track × language pairs create spec and workspace correctly; all 14 CLI commands run, including MCP driven over real stdio by an <code>mcp.ClientSession</code>; all three profiles resolve to genuinely distinct settings. The 8 skips were concurrency-budget exhaustion, not functional gaps — the health probe showed the provider would have served.</p>
+ <h4>Clean-clone check</h4>
+ <p><code>git clone</code> to a temp dir, fresh Python <strong>3.10.21</strong> venv, every credential unset, CI's exact command: <strong>1295 passed, 0 failed</strong>. Main tree at sign-off: <strong>2257 passed, 0 failed</strong>, ruff clean.</p>
+</section>
+
+<!-- ============================== 10 OPEN -->
+<section id="open">
+ <h2><span class="n">10</span> Known open</h2>
+ <h3>Stated, not buried</h3>
+ <ul>
+ <li><strong><code>CodingAgent.available()</code> is a false green light.</strong> It returns available for <code>agy:gemini-3.7-flash</code> (cannot launch) and <code>codex:gpt-5.1-codex</code> (backend rejects with HTTP 400). Usability, not correctness — it cannot corrupt an artifact.</li>
+ <li><strong>Two pre-existing flaky tests</strong> in <code>test_cost_latency</code> and <code>test_prompts_assets</code>. They predate this pass and were deliberately left alone so their attribution stays clean.</li>
+ <li><strong>The scene track has no scored after-measurement.</strong> Four deterministic gate defects were found and fixed — a judge shown the harness's diagnostic rig instead of the scene's authored cameras, two false-positive gates, and unverifiable planner must-haves — and two are proven offline on recorded frames. Their effect on scores is not yet measured.</li>
+ <li><strong>The interpenetration study needs recomputing.</strong> Its correlations were measured with the connectivity gate that was blind to non-watertight parts.</li>
+ <li><strong><code>max_minutes</code> is enforced at billing points, not on a timer.</strong> The retry deadline makes every call terminate, so the ceiling is evaluated again — but a run whose calls never complete is still not checked against it.</li>
+ </ul>
+ <p class="foot" id="foot"></p>
+</section>
+</main>
+</div>
+<script>
+const DATA={"batteries":{"ab_aa_noise":{"control":{"n":1,"eval":1,"dropped":0,"mean":0.7,"median":0.7,"pass":0,"build":1,"gen_usd":1.21,"judge_usd":0.108,"min_per_run":26.2,"cells":{"ctrl_med_dining_chair":{"status":"scored","score":0.7,"build":true,"passed":false}}},"variant":{"n":1,"eval":1,"dropped":0,"mean":0.6,"median":0.6,"pass":0,"build":1,"gen_usd":0.952,"judge_usd":0.094,"min_per_run":47.9,"cells":{"ctrl_med_dining_chair":{"status":"scored","score":0.6,"build":true,"passed":false}}}},"ab_smoke_features":{"control":{"n":1,"eval":1,"dropped":0,"mean":0.5905,"median":0.5905,"pass":0,"build":1,"gen_usd":1.343,"judge_usd":0.145,"min_per_run":46.5,"cells":{"ctrl_med_dining_chair":{"status":"scored","score":0.5905,"build":true,"passed":false}}},"variant":{"n":1,"eval":1,"dropped":0,"mean":0.934,"median":0.934,"pass":1,"build":1,"gen_usd":1.685,"judge_usd":0.105,"min_per_run":31.7,"cells":{"ctrl_med_dining_chair":{"status":"scored","score":0.934,"build":true,"passed":true}}}},"codex_tiers_v2":{"oneshot:codex:gpt-5.6-sol":{"n":8,"eval":8,"dropped":0,"mean":0.6269,"median":0.6,"pass":1,"build":8,"gen_usd":2.737,"judge_usd":0.952,"min_per_run":3.9,"cells":{"cmp_med_dining_chair":{"status":"scored","score":0.5845,"build":true,"passed":false},"cmp_med_toaster":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_coffee_grinder":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_garden_arch":{"status":"scored","score":0.9095,"build":true,"passed":true},"cmp_hard_bonsai_rock":{"status":"scored","score":0.67,"build":true,"passed":false},"cmp_hard_violin":{"status":"scored","score":0.4515,"build":true,"passed":false},"cmp_hard_bay_window":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_drafting_table":{"status":"scored","score":0.6,"build":true,"passed":false}}},"oneshot:codex:gpt-5.6-terra":{"n":8,"eval":8,"dropped":0,"mean":0.4805,"median":0.5513,"pass":2,"build":7,"gen_usd":0.22,"judge_usd":0.82,"min_per_run":2.8,"cells":{"cmp_med_dining_chair":{"status":"scored","score":0.2955,"build":true,"passed":false},"cmp_hard_coffee_grinder":{"status":"scored","score":0.9,"build":true,"passed":true},"cmp_med_toaster":{"status":"scored","score":0.846,"build":true,"passed":true},"cmp_hard_garden_arch":{"status":"scored","score":0.5025,"build":true,"passed":false},"cmp_hard_violin":{"status":"scored","score":0.0,"build":true,"passed":false},"cmp_hard_bonsai_rock":{"status":"scored","score":0.7,"build":true,"passed":false},"cmp_hard_bay_window":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_drafting_table":{"status":"build_failed","score":0.0,"build":false,"passed":false}}},"harness:codex:gpt-5.6-sol":{"n":2,"eval":2,"dropped":0,"mean":0.928,"median":0.928,"pass":2,"build":2,"gen_usd":14.421,"judge_usd":0.206,"min_per_run":27.1,"cells":{"cmp_med_toaster":{"status":"scored","score":0.987,"build":true,"passed":true},"cmp_med_dining_chair":{"status":"scored","score":0.869,"build":true,"passed":true}}},"oneshot:codex:gpt-5.6-luna":{"n":8,"eval":8,"dropped":0,"mean":0.3444,"median":0.3243,"pass":1,"build":6,"gen_usd":0.113,"judge_usd":0.767,"min_per_run":3.1,"cells":{"cmp_hard_bay_window":{"status":"scored","score":0.4095,"build":true,"passed":false},"cmp_hard_bonsai_rock":{"status":"scored","score":0.2725,"build":true,"passed":false},"cmp_hard_coffee_grinder":{"status":"build_failed","score":0.0,"build":false,"passed":false},"cmp_hard_drafting_table":{"status":"build_failed","score":0.0,"build":false,"passed":false},"cmp_hard_garden_arch":{"status":"scored","score":0.522,"build":true,"passed":false},"cmp_hard_violin":{"status":"scored","score":0.231,"build":true,"passed":false},"cmp_med_dining_chair":{"status":"scored","score":0.376,"build":true,"passed":false},"cmp_med_toaster":{"status":"scored","score":0.9445,"build":true,"passed":true}}},"oneshot:claude-code":{"n":8,"eval":8,"dropped":0,"mean":0.4723,"median":0.6,"pass":0,"build":6,"gen_usd":9.336,"judge_usd":0.722,"min_per_run":12.8,"cells":{"cmp_hard_bay_window":{"status":"no_code","score":0.0,"build":false,"passed":false},"cmp_hard_bonsai_rock":{"status":"scored","score":0.607,"build":true,"passed":false},"cmp_hard_coffee_grinder":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_drafting_table":{"status":"scored","score":0.5715,"build":true,"passed":false},"cmp_hard_garden_arch":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_violin":{"status":"no_code","score":0.0,"build":false,"passed":false},"cmp_med_dining_chair":{"status":"scored","score":0.7,"build":true,"passed":false},"cmp_med_toaster":{"status":"scored","score":0.7,"build":true,"passed":false}}},"oneshot:codex":{"n":8,"eval":8,"dropped":0,"mean":0.6366,"median":0.6,"pass":1,"build":8,"gen_usd":2.604,"judge_usd":0.991,"min_per_run":4.0,"cells":{"cmp_hard_bay_window":{"status":"scored","score":0.7,"build":true,"passed":false},"cmp_hard_bonsai_rock":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_coffee_grinder":{"status":"scored","score":0.901,"build":true,"passed":true},"cmp_hard_drafting_table":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_garden_arch":{"status":"scored","score":0.497,"build":true,"passed":false},"cmp_hard_violin":{"status":"scored","score":0.5315,"build":true,"passed":false},"cmp_med_dining_chair":{"status":"scored","score":0.5635,"build":true,"passed":false},"cmp_med_toaster":{"status":"scored","score":0.7,"build":true,"passed":false}}},"oneshot:gemini:gemini-3.7-flash":{"n":8,"eval":7,"dropped":1,"mean":0.1464,"median":0.0,"pass":0,"build":4,"gen_usd":0.296,"judge_usd":0.453,"min_per_run":9.2,"cells":{"cmp_hard_bay_window":{"status":"no_code","score":null},"cmp_hard_bonsai_rock":{"status":"scored","score":0.302,"build":true,"passed":false},"cmp_hard_coffee_grinder":{"status":"scored","score":0.0,"build":true,"passed":false},"cmp_hard_drafting_table":{"status":"build_failed","score":0.0,"build":false,"passed":false},"cmp_hard_garden_arch":{"status":"scored","score":0.5295,"build":true,"passed":false},"cmp_hard_violin":{"status":"scored","score":0.193,"build":true,"passed":false},"cmp_med_dining_chair":{"status":"build_failed","score":0.0,"build":false,"passed":false},"cmp_med_toaster":{"status":"build_failed","score":0.0,"build":false,"passed":false}}},"harness:api-agent:gemini:gemini-3.7-flash":{"n":3,"eval":2,"dropped":1,"mean":0.6,"median":0.6,"pass":0,"build":2,"gen_usd":1.273,"judge_usd":0.232,"min_per_run":52.6,"cells":{"cmp_hard_coffee_grinder":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_med_dining_chair":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_med_toaster":{"status":"error","score":null}}}},"codex_tiers_v3":{"oneshot:codex:gpt-5.6-sol":{"n":12,"eval":12,"dropped":0,"mean":0.6026,"median":0.6,"pass":1,"build":12,"gen_usd":3.529,"judge_usd":1.429,"min_per_run":4.4,"cells":{"cmp_med_dining_chair":{"status":"scored","score":0.5845,"build":true,"passed":false},"cmp_med_toaster":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_coffee_grinder":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_garden_arch":{"status":"scored","score":0.9095,"build":true,"passed":true},"cmp_hard_bonsai_rock":{"status":"scored","score":0.67,"build":true,"passed":false},"cmp_hard_violin":{"status":"scored","score":0.4515,"build":true,"passed":false},"cmp_hard_bay_window":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_drafting_table":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_med_hand_drill":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_penny_farthing":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_seated_cat":{"status":"scored","score":0.47,"build":true,"passed":false},"cmp_hard_telescope_tripod":{"status":"scored","score":0.5455,"build":true,"passed":false}}},"oneshot:codex:gpt-5.6-terra":{"n":12,"eval":12,"dropped":0,"mean":0.4406,"median":0.4302,"pass":2,"build":11,"gen_usd":0.576,"judge_usd":1.312,"min_per_run":6.1,"cells":{"cmp_med_dining_chair":{"status":"scored","score":0.2955,"build":true,"passed":false},"cmp_hard_coffee_grinder":{"status":"scored","score":0.9,"build":true,"passed":true},"cmp_med_toaster":{"status":"scored","score":0.846,"build":true,"passed":true},"cmp_hard_garden_arch":{"status":"scored","score":0.5025,"build":true,"passed":false},"cmp_hard_violin":{"status":"scored","score":0.0,"build":true,"passed":false},"cmp_hard_bonsai_rock":{"status":"scored","score":0.7,"build":true,"passed":false},"cmp_hard_bay_window":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_drafting_table":{"status":"build_failed","score":0.0,"build":false,"passed":false},"cmp_hard_penny_farthing":{"status":"scored","score":0.2905,"build":true,"passed":false},"cmp_hard_seated_cat":{"status":"scored","score":0.358,"build":true,"passed":false},"cmp_med_hand_drill":{"status":"scored","score":0.5165,"build":true,"passed":false},"cmp_hard_telescope_tripod":{"status":"scored","score":0.2785,"build":true,"passed":false}}},"harness:codex:gpt-5.6-sol":{"n":12,"eval":11,"dropped":1,"mean":0.7577,"median":0.869,"pass":6,"build":11,"gen_usd":80.692,"judge_usd":1.238,"min_per_run":23.2,"cells":{"cmp_med_toaster":{"status":"scored","score":0.987,"build":true,"passed":true},"cmp_med_dining_chair":{"status":"scored","score":0.869,"build":true,"passed":true},"cmp_hard_coffee_grinder":{"status":"scored","score":0.9795,"build":true,"passed":true},"cmp_hard_garden_arch":{"status":"scored","score":0.948,"build":true,"passed":true},"cmp_hard_bonsai_rock":{"status":"scored","score":0.7,"build":true,"passed":false},"cmp_hard_violin":{"status":"scored","score":0.303,"build":true,"passed":false},"cmp_hard_bay_window":{"status":"scored","score":0.963,"build":true,"passed":true},"cmp_med_hand_drill":{"status":"error","score":null},"cmp_hard_drafting_table":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_penny_farthing":{"status":"scored","score":0.9725,"build":true,"passed":true},"cmp_hard_seated_cat":{"status":"scored","score":0.599,"build":true,"passed":false},"cmp_hard_telescope_tripod":{"status":"scored","score":0.4135,"build":true,"passed":false}}},"oneshot:codex:gpt-5.6-luna":{"n":12,"eval":12,"dropped":0,"mean":0.2889,"median":0.308,"pass":1,"build":9,"gen_usd":0.166,"judge_usd":1.147,"min_per_run":3.7,"cells":{"cmp_hard_bay_window":{"status":"scored","score":0.4095,"build":true,"passed":false},"cmp_hard_bonsai_rock":{"status":"scored","score":0.2725,"build":true,"passed":false},"cmp_hard_coffee_grinder":{"status":"build_failed","score":0.0,"build":false,"passed":false},"cmp_hard_drafting_table":{"status":"build_failed","score":0.0,"build":false,"passed":false},"cmp_hard_garden_arch":{"status":"scored","score":0.522,"build":true,"passed":false},"cmp_hard_violin":{"status":"scored","score":0.231,"build":true,"passed":false},"cmp_med_dining_chair":{"status":"scored","score":0.376,"build":true,"passed":false},"cmp_med_toaster":{"status":"scored","score":0.9445,"build":true,"passed":true},"cmp_med_hand_drill":{"status":"build_failed","score":0.0,"build":false,"passed":false},"cmp_hard_penny_farthing":{"status":"scored","score":0.0,"build":true,"passed":false},"cmp_hard_seated_cat":{"status":"scored","score":0.3675,"build":true,"passed":false},"cmp_hard_telescope_tripod":{"status":"scored","score":0.3435,"build":true,"passed":false}}},"oneshot:claude-code":{"n":8,"eval":8,"dropped":0,"mean":0.4723,"median":0.6,"pass":0,"build":6,"gen_usd":9.336,"judge_usd":0.722,"min_per_run":12.8,"cells":{"cmp_hard_bay_window":{"status":"no_code","score":0.0,"build":false,"passed":false},"cmp_hard_bonsai_rock":{"status":"scored","score":0.607,"build":true,"passed":false},"cmp_hard_coffee_grinder":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_drafting_table":{"status":"scored","score":0.5715,"build":true,"passed":false},"cmp_hard_garden_arch":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_violin":{"status":"no_code","score":0.0,"build":false,"passed":false},"cmp_med_dining_chair":{"status":"scored","score":0.7,"build":true,"passed":false},"cmp_med_toaster":{"status":"scored","score":0.7,"build":true,"passed":false}}},"oneshot:codex":{"n":8,"eval":8,"dropped":0,"mean":0.6366,"median":0.6,"pass":1,"build":8,"gen_usd":2.604,"judge_usd":0.991,"min_per_run":4.0,"cells":{"cmp_hard_bay_window":{"status":"scored","score":0.7,"build":true,"passed":false},"cmp_hard_bonsai_rock":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_coffee_grinder":{"status":"scored","score":0.901,"build":true,"passed":true},"cmp_hard_drafting_table":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_garden_arch":{"status":"scored","score":0.497,"build":true,"passed":false},"cmp_hard_violin":{"status":"scored","score":0.5315,"build":true,"passed":false},"cmp_med_dining_chair":{"status":"scored","score":0.5635,"build":true,"passed":false},"cmp_med_toaster":{"status":"scored","score":0.7,"build":true,"passed":false}}},"oneshot:gemini:gemini-3.7-flash":{"n":7,"eval":7,"dropped":0,"mean":0.1464,"median":0.0,"pass":0,"build":4,"gen_usd":0.296,"judge_usd":0.453,"min_per_run":9.2,"cells":{"cmp_hard_bonsai_rock":{"status":"scored","score":0.302,"build":true,"passed":false},"cmp_hard_coffee_grinder":{"status":"scored","score":0.0,"build":true,"passed":false},"cmp_hard_drafting_table":{"status":"build_failed","score":0.0,"build":false,"passed":false},"cmp_hard_garden_arch":{"status":"scored","score":0.5295,"build":true,"passed":false},"cmp_hard_violin":{"status":"scored","score":0.193,"build":true,"passed":false},"cmp_med_dining_chair":{"status":"build_failed","score":0.0,"build":false,"passed":false},"cmp_med_toaster":{"status":"build_failed","score":0.0,"build":false,"passed":false}}},"harness:api-agent:gemini:gemini-3.7-flash":{"n":2,"eval":2,"dropped":0,"mean":0.6,"median":0.6,"pass":0,"build":2,"gen_usd":1.273,"judge_usd":0.232,"min_per_run":52.6,"cells":{"cmp_hard_coffee_grinder":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_med_dining_chair":{"status":"scored","score":0.6,"build":true,"passed":false}}},"harness:codex:gpt-5.6-terra":{"n":12,"eval":11,"dropped":1,"mean":0.6461,"median":0.6,"pass":2,"build":11,"gen_usd":47.924,"judge_usd":1.242,"min_per_run":25.9,"cells":{"cmp_med_toaster":{"status":"scored","score":0.9755,"build":true,"passed":true},"cmp_med_dining_chair":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_coffee_grinder":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_garden_arch":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_bonsai_rock":{"status":"scored","score":0.7,"build":true,"passed":false},"cmp_hard_violin":{"status":"scored","score":0.596,"build":true,"passed":false},"cmp_hard_bay_window":{"status":"scored","score":0.7,"build":true,"passed":false},"cmp_hard_drafting_table":{"status":"scored","score":0.509,"build":true,"passed":false},"cmp_med_hand_drill":{"status":"scored","score":0.402,"build":true,"passed":false},"cmp_hard_penny_farthing":{"status":"scored","score":0.9565,"build":true,"passed":true},"cmp_hard_telescope_tripod":{"status":"error","score":null},"cmp_hard_seated_cat":{"status":"scored","score":0.4685,"build":true,"passed":false}}},"harness:codex:gpt-5.6-luna":{"n":12,"eval":10,"dropped":2,"mean":0.6006,"median":0.6,"pass":1,"build":10,"gen_usd":11.523,"judge_usd":1.141,"min_per_run":40.0,"cells":{"cmp_med_toaster":{"status":"scored","score":0.951,"build":true,"passed":true},"cmp_med_dining_chair":{"status":"scored","score":0.5805,"build":true,"passed":false},"cmp_hard_coffee_grinder":{"status":"scored","score":0.544,"build":true,"passed":false},"cmp_hard_garden_arch":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_violin":{"status":"scored","score":0.338,"build":true,"passed":false},"cmp_hard_bonsai_rock":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_bay_window":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_drafting_table":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_med_hand_drill":{"status":"error","score":null},"cmp_hard_seated_cat":{"status":"infra_failed","score":null},"cmp_hard_penny_farthing":{"status":"scored","score":0.7,"build":true,"passed":false},"cmp_hard_telescope_tripod":{"status":"scored","score":0.4925,"build":true,"passed":false}}}},"compare_v1_full":{"oneshot:gemini:gemini-3.7-flash":{"n":8,"eval":8,"dropped":0,"mean":0.1622,"median":0.0,"pass":0,"build":3,"gen_usd":0.295,"judge_usd":0.32,"min_per_run":3.9,"cells":{"cmp_easy_stool":{"status":"build_failed","score":0.0,"build":false,"passed":false},"cmp_med_wheelbarrow":{"status":"build_failed","score":0.0,"build":false,"passed":false},"cmp_med_espresso_machine":{"status":"scored","score":0.4235,"build":true,"passed":false},"cmp_hard_bicycle":{"status":"build_failed","score":0.0,"build":false,"passed":false},"cmp_hard_acoustic_guitar":{"status":"build_failed","score":0.0,"build":false,"passed":false},"cmp_hard_sailboat":{"status":"scored","score":0.589,"build":true,"passed":false},"cmp_easy_desk_lamp":{"status":"build_failed","score":0.0,"build":false,"passed":false},"cmp_med_dining_chair":{"status":"scored","score":0.285,"build":true,"passed":false}}},"harness:api-agent:gemini:gemini-3.7-flash":{"n":8,"eval":8,"dropped":0,"mean":0.6852,"median":0.675,"pass":4,"build":8,"gen_usd":15.296,"judge_usd":0.812,"min_per_run":24.5,"cells":{"cmp_easy_stool":{"status":"scored","score":0.7805,"build":true,"passed":true},"cmp_med_dining_chair":{"status":"scored","score":0.8695,"build":true,"passed":true},"cmp_med_espresso_machine":{"status":"scored","score":0.75,"build":true,"passed":true},"cmp_easy_desk_lamp":{"status":"scored","score":0.7685,"build":true,"passed":true},"cmp_med_wheelbarrow":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_bicycle":{"status":"scored","score":0.562,"build":true,"passed":false},"cmp_hard_acoustic_guitar":{"status":"scored","score":0.553,"build":true,"passed":false},"cmp_hard_sailboat":{"status":"scored","score":0.598,"build":true,"passed":false}}}},"compare_v1_live2":{"oneshot:codex":{"n":2,"eval":2,"dropped":0,"mean":0.786,"median":0.786,"pass":1,"build":2,"gen_usd":0.404,"judge_usd":0.183,"min_per_run":2.1,"cells":{"cmp_easy_stool":{"status":"scored","score":0.9045,"build":true,"passed":true},"cmp_easy_desk_lamp":{"status":"scored","score":0.6675,"build":true,"passed":false}}},"oneshot:claude-code":{"n":2,"eval":2,"dropped":0,"mean":0.673,"median":0.673,"pass":0,"build":2,"gen_usd":1.899,"judge_usd":0.176,"min_per_run":4.1,"cells":{"cmp_easy_stool":{"status":"scored","score":0.646,"build":true,"passed":false},"cmp_easy_desk_lamp":{"status":"scored","score":0.7,"build":true,"passed":false}}},"oneshot:gemini:gemini-3.7-flash":{"n":2,"eval":2,"dropped":0,"mean":0.35,"median":0.35,"pass":0,"build":1,"gen_usd":0.043,"judge_usd":0.077,"min_per_run":1.8,"cells":{"cmp_easy_stool":{"status":"scored","score":0.7,"build":true,"passed":false},"cmp_easy_desk_lamp":{"status":"build_failed","score":0.0,"build":false,"passed":false}}},"harness:api-agent:gemini:gemini-3.7-flash":{"n":2,"eval":2,"dropped":0,"mean":0.8345,"median":0.8345,"pass":2,"build":2,"gen_usd":0.878,"judge_usd":0.192,"min_per_run":7.2,"cells":{"cmp_easy_desk_lamp":{"status":"scored","score":0.81,"build":true,"passed":true},"cmp_easy_stool":{"status":"scored","score":0.859,"build":true,"passed":true}}},"harness:gemini-cli:gemini-3.7-flash":{"n":2,"eval":2,"dropped":0,"mean":0.8265,"median":0.8265,"pass":1,"build":2,"gen_usd":1.999,"judge_usd":0.19,"min_per_run":15.9,"cells":{"cmp_easy_stool":{"status":"scored","score":0.953,"build":true,"passed":true},"cmp_easy_desk_lamp":{"status":"scored","score":0.7,"build":true,"passed":false}}}},"compare_v2_full":{"oneshot:gemini:gemini-3.7-flash":{"n":8,"eval":8,"dropped":0,"mean":0.2244,"median":0.0,"pass":0,"build":3,"gen_usd":0.292,"judge_usd":0.344,"min_per_run":9.1,"cells":{"cmp_med_dining_chair":{"status":"scored","score":0.7,"build":true,"passed":false},"cmp_hard_coffee_grinder":{"status":"build_failed","score":0.0,"build":false,"passed":false},"cmp_med_toaster":{"status":"build_failed","score":0.0,"build":false,"passed":false},"cmp_hard_violin":{"status":"build_failed","score":0.0,"build":false,"passed":false},"cmp_hard_garden_arch":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_drafting_table":{"status":"build_failed","score":0.0,"build":false,"passed":false},"cmp_hard_bonsai_rock":{"status":"build_failed","score":0.0,"build":false,"passed":false},"cmp_hard_bay_window":{"status":"scored","score":0.495,"build":true,"passed":false}}},"oneshot:codex":{"n":8,"eval":8,"dropped":0,"mean":0.6366,"median":0.6,"pass":1,"build":8,"gen_usd":2.604,"judge_usd":0.991,"min_per_run":4.0,"cells":{"cmp_med_dining_chair":{"status":"scored","score":0.5635,"build":true,"passed":false},"cmp_med_toaster":{"status":"scored","score":0.7,"build":true,"passed":false},"cmp_hard_coffee_grinder":{"status":"scored","score":0.901,"build":true,"passed":true},"cmp_hard_garden_arch":{"status":"scored","score":0.497,"build":true,"passed":false},"cmp_hard_violin":{"status":"scored","score":0.5315,"build":true,"passed":false},"cmp_hard_bonsai_rock":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_drafting_table":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_bay_window":{"status":"scored","score":0.7,"build":true,"passed":false}}},"oneshot:claude-code":{"n":8,"eval":8,"dropped":0,"mean":0.4723,"median":0.6,"pass":0,"build":6,"gen_usd":9.336,"judge_usd":0.722,"min_per_run":12.8,"cells":{"cmp_hard_coffee_grinder":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_med_dining_chair":{"status":"scored","score":0.7,"build":true,"passed":false},"cmp_med_toaster":{"status":"scored","score":0.7,"build":true,"passed":false},"cmp_hard_garden_arch":{"status":"scored","score":0.6,"build":true,"passed":false},"cmp_hard_violin":{"status":"no_code","score":0.0,"build":false,"passed":false},"cmp_hard_bonsai_rock":{"status":"scored","score":0.607,"build":true,"passed":false},"cmp_hard_bay_window":{"status":"no_code","score":0.0,"build":false,"passed":false},"cmp_hard_drafting_table":{"status":"scored","score":0.5715,"build":true,"passed":false}}},"harness:api-agent:gemini:gemini-3.7-flash":{"n":8,"eval":8,"dropped":0,"mean":0.6249,"median":0.7,"pass":2,"build":8,"gen_usd":19.273,"judge_usd":0.908,"min_per_run":38.6,"cells":{"cmp_med_dining_chair":{"status":"scored","score":0.7,"build":true,"passed":false},"cmp_med_toaster":{"status":"scored","score":0.8095,"build":true,"passed":true},"cmp_hard_coffee_grinder":{"status":"scored","score":0.7,"build":true,"passed":false},"cmp_hard_garden_arch":{"status":"scored","score":0.923,"build":true,"passed":true},"cmp_hard_violin":{"status":"scored","score":0.207,"build":true,"passed":false},"cmp_hard_bonsai_rock":{"status":"scored","score":0.587,"build":true,"passed":false},"cmp_hard_bay_window":{"status":"scored","score":0.7,"build":true,"passed":false},"cmp_hard_drafting_table":{"status":"scored","score":0.373,"build":true,"passed":false}}}}},"paired":[{"label":"harness+flash \u2212 one-shot flash","n":8,"delta":0.4006,"sd":0.2764,"se":0.0977,"t":4.1,"ci":[0.2091,0.5921],"wins":7,"losses":0,"ties":1,"p":0.0156,"separated":true,"cells":[{"id":"cmp_hard_bay_window","a":0.7,"b":0.495,"d":0.205},{"id":"cmp_hard_bonsai_rock","a":0.587,"b":0.0,"d":0.587},{"id":"cmp_hard_coffee_grinder","a":0.7,"b":0.0,"d":0.7},{"id":"cmp_hard_drafting_table","a":0.373,"b":0.0,"d":0.373},{"id":"cmp_hard_garden_arch","a":0.923,"b":0.6,"d":0.323},{"id":"cmp_hard_violin","a":0.207,"b":0.0,"d":0.207},{"id":"cmp_med_dining_chair","a":0.7,"b":0.7,"d":0.0},{"id":"cmp_med_toaster","a":0.809,"b":0.0,"d":0.809}]},{"label":"one-shot sol \u2212 terra","n":12,"delta":0.162,"sd":0.2757,"se":0.0796,"t":2.03,"ci":[0.006,0.318],"wins":8,"losses":3,"ties":1,"p":0.2266,"separated":true,"cells":[{"id":"cmp_hard_bay_window","a":0.6,"b":0.6,"d":0.0},{"id":"cmp_hard_bonsai_rock","a":0.67,"b":0.7,"d":-0.03},{"id":"cmp_hard_coffee_grinder","a":0.6,"b":0.9,"d":-0.3},{"id":"cmp_hard_drafting_table","a":0.6,"b":0.0,"d":0.6},{"id":"cmp_hard_garden_arch","a":0.909,"b":0.502,"d":0.407},{"id":"cmp_hard_penny_farthing","a":0.6,"b":0.29,"d":0.309},{"id":"cmp_hard_seated_cat","a":0.47,"b":0.358,"d":0.112},{"id":"cmp_hard_telescope_tripod","a":0.545,"b":0.279,"d":0.267},{"id":"cmp_hard_violin","a":0.452,"b":0.0,"d":0.452},{"id":"cmp_med_dining_chair","a":0.585,"b":0.295,"d":0.289},{"id":"cmp_med_hand_drill","a":0.6,"b":0.516,"d":0.084},{"id":"cmp_med_toaster","a":0.6,"b":0.846,"d":-0.246}]},{"label":"one-shot terra \u2212 luna","n":12,"delta":0.1517,"sd":0.3273,"se":0.0945,"t":1.61,"ci":[-0.0334,0.3369],"wins":5,"losses":6,"ties":1,"p":1.0,"separated":false,"cells":[{"id":"cmp_hard_bay_window","a":0.6,"b":0.409,"d":0.191},{"id":"cmp_hard_bonsai_rock","a":0.7,"b":0.273,"d":0.427},{"id":"cmp_hard_coffee_grinder","a":0.9,"b":0.0,"d":0.9},{"id":"cmp_hard_drafting_table","a":0.0,"b":0.0,"d":0.0},{"id":"cmp_hard_garden_arch","a":0.502,"b":0.522,"d":-0.02},{"id":"cmp_hard_penny_farthing","a":0.29,"b":0.0,"d":0.29},{"id":"cmp_hard_seated_cat","a":0.358,"b":0.367,"d":-0.01},{"id":"cmp_hard_telescope_tripod","a":0.279,"b":0.344,"d":-0.065},{"id":"cmp_hard_violin","a":0.0,"b":0.231,"d":-0.231},{"id":"cmp_med_dining_chair","a":0.295,"b":0.376,"d":-0.081},{"id":"cmp_med_hand_drill","a":0.516,"b":0.0,"d":0.516},{"id":"cmp_med_toaster","a":0.846,"b":0.945,"d":-0.099}]},{"label":"one-shot sol \u2212 luna","n":12,"delta":0.3137,"sd":0.2805,"se":0.081,"t":3.87,"ci":[0.155,0.4724],"wins":11,"losses":1,"ties":0,"p":0.0063,"separated":true,"cells":[{"id":"cmp_hard_bay_window","a":0.6,"b":0.409,"d":0.191},{"id":"cmp_hard_bonsai_rock","a":0.67,"b":0.273,"d":0.398},{"id":"cmp_hard_coffee_grinder","a":0.6,"b":0.0,"d":0.6},{"id":"cmp_hard_drafting_table","a":0.6,"b":0.0,"d":0.6},{"id":"cmp_hard_garden_arch","a":0.909,"b":0.522,"d":0.387},{"id":"cmp_hard_penny_farthing","a":0.6,"b":0.0,"d":0.6},{"id":"cmp_hard_seated_cat","a":0.47,"b":0.367,"d":0.102},{"id":"cmp_hard_telescope_tripod","a":0.545,"b":0.344,"d":0.202},{"id":"cmp_hard_violin","a":0.452,"b":0.231,"d":0.221},{"id":"cmp_med_dining_chair","a":0.585,"b":0.376,"d":0.209},{"id":"cmp_med_hand_drill","a":0.6,"b":0.0,"d":0.6},{"id":"cmp_med_toaster","a":0.6,"b":0.945,"d":-0.345}]},{"label":"harness lift on codex sol","n":11,"delta":0.1549,"sd":0.2097,"se":0.0632,"t":2.45,"ci":[0.0309,0.2788],"wins":8,"losses":2,"ties":1,"p":0.1094,"separated":true,"cells":[{"id":"cmp_hard_bay_window","a":0.963,"b":0.6,"d":0.363},{"id":"cmp_hard_bonsai_rock","a":0.7,"b":0.67,"d":0.03},{"id":"cmp_hard_coffee_grinder","a":0.98,"b":0.6,"d":0.38},{"id":"cmp_hard_drafting_table","a":0.6,"b":0.6,"d":0.0},{"id":"cmp_hard_garden_arch","a":0.948,"b":0.909,"d":0.038},{"id":"cmp_hard_penny_farthing","a":0.973,"b":0.6,"d":0.373},{"id":"cmp_hard_seated_cat","a":0.599,"b":0.47,"d":0.129},{"id":"cmp_hard_telescope_tripod","a":0.413,"b":0.545,"d":-0.132},{"id":"cmp_hard_violin","a":0.303,"b":0.452,"d":-0.149},{"id":"cmp_med_dining_chair","a":0.869,"b":0.585,"d":0.284},{"id":"cmp_med_toaster","a":0.987,"b":0.6,"d":0.387}]},{"label":"harness lift on codex terra","n":11,"delta":0.1908,"sd":0.3003,"se":0.0905,"t":2.11,"ci":[0.0133,0.3682],"wins":8,"losses":2,"ties":1,"p":0.1094,"separated":true,"cells":[{"id":"cmp_hard_bay_window","a":0.7,"b":0.6,"d":0.1},{"id":"cmp_hard_bonsai_rock","a":0.7,"b":0.7,"d":0.0},{"id":"cmp_hard_coffee_grinder","a":0.6,"b":0.9,"d":-0.3},{"id":"cmp_hard_drafting_table","a":0.509,"b":0.0,"d":0.509},{"id":"cmp_hard_garden_arch","a":0.6,"b":0.502,"d":0.098},{"id":"cmp_hard_penny_farthing","a":0.957,"b":0.29,"d":0.666},{"id":"cmp_hard_seated_cat","a":0.469,"b":0.358,"d":0.111},{"id":"cmp_hard_violin","a":0.596,"b":0.0,"d":0.596},{"id":"cmp_med_dining_chair","a":0.6,"b":0.295,"d":0.304},{"id":"cmp_med_hand_drill","a":0.402,"b":0.516,"d":-0.114},{"id":"cmp_med_toaster","a":0.976,"b":0.846,"d":0.13}]},{"label":"harness lift on codex luna","n":10,"delta":0.2907,"sd":0.2418,"se":0.0764,"t":3.8,"ci":[0.1409,0.4405],"wins":10,"losses":0,"ties":0,"p":0.002,"separated":true,"cells":[{"id":"cmp_hard_bay_window","a":0.6,"b":0.409,"d":0.191},{"id":"cmp_hard_bonsai_rock","a":0.6,"b":0.273,"d":0.327},{"id":"cmp_hard_coffee_grinder","a":0.544,"b":0.0,"d":0.544},{"id":"cmp_hard_drafting_table","a":0.6,"b":0.0,"d":0.6},{"id":"cmp_hard_garden_arch","a":0.6,"b":0.522,"d":0.078},{"id":"cmp_hard_penny_farthing","a":0.7,"b":0.0,"d":0.7},{"id":"cmp_hard_telescope_tripod","a":0.492,"b":0.344,"d":0.149},{"id":"cmp_hard_violin","a":0.338,"b":0.231,"d":0.107},{"id":"cmp_med_dining_chair","a":0.581,"b":0.376,"d":0.205},{"id":"cmp_med_toaster","a":0.951,"b":0.945,"d":0.006}]},{"label":"harness+flash \u2212 one-shot codex sol","n":8,"delta":-0.002,"sd":0.1672,"se":0.0591,"t":-0.03,"ci":[-0.1178,0.1138],"wins":5,"losses":3,"ties":0,"p":0.7266,"separated":false,"cells":[{"id":"cmp_hard_bay_window","a":0.7,"b":0.6,"d":0.1},{"id":"cmp_hard_bonsai_rock","a":0.587,"b":0.67,"d":-0.083},{"id":"cmp_hard_coffee_grinder","a":0.7,"b":0.6,"d":0.1},{"id":"cmp_hard_drafting_table","a":0.373,"b":0.6,"d":-0.227},{"id":"cmp_hard_garden_arch","a":0.923,"b":0.909,"d":0.014},{"id":"cmp_hard_violin","a":0.207,"b":0.452,"d":-0.245},{"id":"cmp_med_dining_chair","a":0.7,"b":0.585,"d":0.115},{"id":"cmp_med_toaster","a":0.809,"b":0.6,"d":0.21}]},{"label":"harness+flash \u2212 one-shot claude-code","n":8,"delta":0.1526,"sd":0.2706,"se":0.0957,"t":1.6,"ci":[-0.0349,0.3401],"wins":5,"losses":2,"ties":1,"p":0.4531,"separated":false,"cells":[{"id":"cmp_hard_bay_window","a":0.7,"b":0.0,"d":0.7},{"id":"cmp_hard_bonsai_rock","a":0.587,"b":0.607,"d":-0.02},{"id":"cmp_hard_coffee_grinder","a":0.7,"b":0.6,"d":0.1},{"id":"cmp_hard_drafting_table","a":0.373,"b":0.572,"d":-0.199},{"id":"cmp_hard_garden_arch","a":0.923,"b":0.6,"d":0.323},{"id":"cmp_hard_violin","a":0.207,"b":0.0,"d":0.207},{"id":"cmp_med_dining_chair","a":0.7,"b":0.7,"d":0.0},{"id":"cmp_med_toaster","a":0.809,"b":0.7,"d":0.11}]},{"label":"harness+flash \u2212 harness+codex sol","n":8,"delta":-0.1688,"sd":0.0876,"se":0.031,"t":-5.45,"ci":[-0.2294,-0.1081],"wins":0,"losses":8,"ties":0,"p":0.0078,"separated":true,"cells":[{"id":"cmp_hard_bay_window","a":0.7,"b":0.963,"d":-0.263},{"id":"cmp_hard_bonsai_rock","a":0.587,"b":0.7,"d":-0.113},{"id":"cmp_hard_coffee_grinder","a":0.7,"b":0.98,"d":-0.28},{"id":"cmp_hard_drafting_table","a":0.373,"b":0.6,"d":-0.227},{"id":"cmp_hard_garden_arch","a":0.923,"b":0.948,"d":-0.025},{"id":"cmp_hard_violin","a":0.207,"b":0.303,"d":-0.096},{"id":"cmp_med_dining_chair","a":0.7,"b":0.869,"d":-0.169},{"id":"cmp_med_toaster","a":0.809,"b":0.987,"d":-0.177}]}]};
+const ARCH={"packages":{"agents":{"files":14,"loc":2389},"cli":{"files":13,"loc":2346},"contracts":{"files":9,"loc":1252},"cost":{"files":12,"loc":2945},"flywheel":{"files":16,"loc":3451},"gallery":{"files":16,"loc":3259},"judges":{"files":14,"loc":2673},"languages":{"files":47,"loc":6662},"models":{"files":18,"loc":3727},"orchestrator":{"files":7,"loc":1223},"prompts":{"files":1,"loc":43},"reference":{"files":11,"loc":1245},"spatial":{"files":35,"loc":7187},"texturing":{"files":12,"loc":2744},"tracks":{"files":26,"loc":6885},"(top level)":{"files":8,"loc":1301}},"totals":{"codeverse_files":259,"codeverse_loc":49332,"tests_files":219,"tests_loc":29105,"bench_files":14,"bench_loc":3264,"runtime_js_files":34,"runtime_js_loc":4006,"prompt_files":40,"prompt_loc":6547},"commits":91,"head":"8ec95bb"};
+</script>
+<script>
+(function(){
+const $ = s => document.querySelector(s);
+const el = (t,c,h) => { const n=document.createElement(t); if(c) n.className=c; if(h!=null) n.innerHTML=h; return n; };
+const f3 = v => (v==null?'—':(v>=0?'+':'')+v.toFixed(3));
+const p3 = v => (v==null?'—':v.toFixed(3));
+
+// ---------- stamps
+const stamp = `HEAD ${ARCH.head} · ${ARCH.commits} commits · ${ARCH.totals.codeverse_files} modules / ${ARCH.totals.codeverse_loc.toLocaleString()} lines · 2257 offline tests · generated 2026-08-24`;
+$('#stamp').textContent = stamp;
+$('#railstamp').innerHTML = `HEAD ${ARCH.head}<br>${ARCH.commits} commits<br>2257 tests green`;
+$('#foot').textContent = `Generated from bench/out/*/results.jsonl at HEAD ${ARCH.head}. Every figure is recomputed from the recorded cells, not transcribed.`;
+
+// ---------- verdict cards
+const P = {}; DATA.paired.forEach(r => P[r.label] = r);
+const grade = r => !r ? 'null' : (r.separated && r.p < 0.05 ? 'good' : (r.separated ? 'warn' : 'null'));
+const cards = [
+ ['harness+flash − one-shot flash', 'The harness is worth this much', 'on the same model, hard prompts'],
+ ['harness lift on codex luna', 'Lift on the weakest model', 'largest of the three tiers'],
+ ['harness+flash − one-shot codex sol', 'Flash vs sol, paired', 'dead heat — the open goal'],
+ ['harness+flash − harness+codex sol', 'Gap still to close', 'harness+sol over harness+flash'],
+];
+const vc = $('#verdictcards');
+cards.forEach(([k, title, sub]) => {
+ const r = P[k]; if(!r) return;
+ const g = grade(r);
+ const c = el('div','vcard '+g);
+ c.appendChild(el('div','k',title));
+ c.appendChild(el('div','v',f3(r.delta)));
+ c.appendChild(el('div','d',`${sub} · n=${r.n} · ${r.separated&&r.p<0.05?'separated from noise':(r.separated?'marginal':'not separated')}`));
+ vc.appendChild(c);
+});
+
+// ---------- package map
+const PKGDESC = {
+ 'spatial':'measurement, gates, renderers, MCP tool server',
+ 'tracks':'per-track lifecycles, planner, brief, generation, detail pass',
+ 'languages':'runtime/lint/skeleton/wrappers per target language',
+ 'models':'ChatModel, key pool, retry, storm, health probe, pricing',
+ 'flywheel':'dataset export, preference pairs, captions, index',
+ 'gallery':'the local run gallery served on localhost',
+ 'cost':'ledger, price provenance, budget guard, profiles',
+ 'texturing':'text-to-image material pass for objects and scenes',
+ 'judges':'rubrics, montage, scoring, caps, pairwise',
+ 'agents':'CodingAgent implementations for the five CLI/API kinds',
+ 'cli':'the 3dcv command surface',
+ 'contracts':'typed, data-only, shared by every package',
+ 'reference':'synthesized reference images + the plausibility gate',
+ 'orchestrator':'round policy, candidates, fan-out, budget',
+ 'prompts':'system prompts, per-language contracts and cookbooks',
+ '(top level)':'conventions, workspace, events, proc, config, fanout',
+};
+const pk = Object.entries(ARCH.packages).sort((a,b)=>b[1].loc-a[1].loc);
+const maxloc = pk[0][1].loc;
+const pkgs = $('#pkgs');
+pk.forEach(([name,v]) => {
+ const d = el('div','pkg');
+ d.appendChild(el('div','n', name==='(top level)' ? 'codeverse/*.py' : 'codeverse/'+name+'/'));
+ d.appendChild(el('div','m', `${v.files} files · ${v.loc.toLocaleString()} lines`));
+ d.appendChild(el('div','d', PKGDESC[name] || ''));
+ const w = el('div','barwrap'); const i = el('i'); i.style.width = (100*v.loc/maxloc)+'%'; w.appendChild(i); d.appendChild(w);
+ pkgs.appendChild(d);
+});
+
+// ---------- scale table
+const T = ARCH.totals;
+const rows = [
+ ['codeverse/ — the package', T.codeverse_files, T.codeverse_loc],
+ ['tests/', T.tests_files, T.tests_loc],
+ ['prompts (.j2 / .md)', T.prompt_files, T.prompt_loc],
+ ['runtime_js/ (node side)', T.runtime_js_files, T.runtime_js_loc],
+ ['bench/', T.bench_files, T.bench_loc],
+];
+const st = $('#scaletable');
+rows.forEach(([k,f,l]) => {
+ const tr = el('tr');
+ tr.appendChild(el('td','arm',k));
+ tr.appendChild(el('td','num',String(f)));
+ tr.appendChild(el('td','num',l.toLocaleString()));
+ st.appendChild(tr);
+});
+{
+ const tot = rows.reduce((a,r)=>a+r[2],0), totf = rows.reduce((a,r)=>a+r[1],0);
+ const tr = el('tr'); tr.className='hl';
+ tr.appendChild(el('td','arm','total'));
+ tr.appendChild(el('td','num',String(totf)));
+ tr.appendChild(el('td','num',tot.toLocaleString()));
+ st.appendChild(tr);
+}
+
+// ---------- CI plot
+const ORDER = ['harness+flash − one-shot flash','one-shot sol − luna','harness lift on codex luna',
+ 'harness+flash − harness+codex sol','harness lift on codex terra','one-shot sol − terra',
+ 'harness lift on codex sol','one-shot terra − luna','harness+flash − one-shot claude-code',
+ 'harness+flash − one-shot codex sol'];
+const shown = ORDER.map(k=>P[k]).filter(Boolean);
+const lo = Math.min(...shown.map(r=>r.ci[0])), hi = Math.max(...shown.map(r=>r.ci[1]));
+const pad = 0.06, LO = Math.min(lo,0)-pad, HI = Math.max(hi,0)+pad;
+const pct = v => ((v-LO)/(HI-LO))*100;
+const plot = $('#ciplot');
+const box = el('div','ci ' + 'g');
+box.className='ci'; box.style.display='block'; box.style.padding='18px 20px';
+shown.forEach(r => {
+ const g = grade(r)==='good'?'g':(grade(r)==='warn'?'w':'n');
+ const row = el('div','ci '+g);
+ row.style.border='0'; row.style.padding='0'; row.style.background='transparent';
+ row.appendChild(el('div','lbl', r.label.replace('−','&minus;')));
+ const tr = el('div','track');
+ const z = el('div','zero'); z.style.left = pct(0)+'%'; tr.appendChild(z);
+ const b = el('div','bar'); b.style.left = pct(r.ci[0])+'%'; b.style.width = (pct(r.ci[1])-pct(r.ci[0]))+'%'; tr.appendChild(b);
+ const pt = el('div','pt'); pt.style.left = pct(r.delta)+'%'; tr.appendChild(pt);
+ row.appendChild(tr);
+ row.appendChild(el('div','val', `<b>${f3(r.delta)}</b> &nbsp;n=${r.n} &nbsp;${r.wins}W/${r.losses}L`));
+ box.appendChild(row);
+});
+plot.appendChild(box);
+const ax = el('div','axis');
+ax.appendChild(el('div','',''));
+const ticks = el('div','ticks');
+[LO, (LO+HI)/2, HI].forEach(v=>ticks.appendChild(el('span','',v.toFixed(2))));
+ax.appendChild(ticks); ax.appendChild(el('div','',''));
+plot.appendChild(ax);
+
+// ---------- stat table
+const stt = $('#stattable');
+shown.forEach(r => {
+ const g = grade(r);
+ const tr = el('tr');
+ tr.appendChild(el('td','arm', r.label));
+ tr.appendChild(el('td','num', String(r.n)));
+ tr.appendChild(el('td','num', f3(r.delta)));
+ tr.appendChild(el('td','num', `${f3(r.ci[0])} … ${f3(r.ci[1])}`));
+ tr.appendChild(el('td','num', (r.t>=0?'+':'')+r.t.toFixed(2)));
+ tr.appendChild(el('td','num', `${r.wins}/${r.losses}/${r.ties}`));
+ tr.appendChild(el('td','num', r.p.toFixed(3)));
+ const label = g==='good'?'solid':(g==='warn'?'marginal':'not separated');
+ tr.appendChild(el('td','', `<span class="chip ${g}">${label}</span>`));
+ stt.appendChild(tr);
+});
+
+// ---------- arm tables
+const BATT = [
+ ['compare_v2_full','compare_v2 — 8 hard/medium prompts, blender'],
+ ['codex_tiers_v3','compare_v3 — 12 prompts, the codex tier matrix'],
+ ['compare_v1_full','compare_v1 — 8 prompts, the original harness-vs-one-shot battery'],
+];
+const at = $('#armtables');
+BATT.forEach(([key,title]) => {
+ const arms = DATA.batteries[key]; if(!arms) return;
+ at.appendChild(el('h4','',title));
+ const sc = el('div','scroll');
+ const tb = el('table');
+ tb.innerHTML = '<thead><tr><th>Arm</th><th class="num">n</th><th class="num">eval</th><th class="num">drop</th>'+
+ '<th class="num">mean</th><th class="num">median</th><th class="num">pass</th><th class="num">build</th>'+
+ '<th class="num">$gen</th><th class="num">min/run</th></tr></thead>';
+ const body = el('tbody');
+ Object.entries(arms).filter(([,g])=>g.eval>0).sort((a,b)=>(b[1].mean??-1)-(a[1].mean??-1)).forEach(([a,g])=>{
+ const tr = el('tr');
+ if(a.startsWith('harness:api-agent:gemini')) tr.className='hl';
+ tr.appendChild(el('td','arm',a));
+ tr.appendChild(el('td','num',String(g.n)));
+ tr.appendChild(el('td','num',String(g.eval)));
+ tr.appendChild(el('td','num', g.dropped? String(g.dropped):'·'));
+ tr.appendChild(el('td','num', p3(g.mean)));
+ tr.appendChild(el('td','num', p3(g.median)));
+ tr.appendChild(el('td','num', `${g.pass}/${g.eval}`));
+ tr.appendChild(el('td','num', `${g.build}/${g.eval}`));
+ tr.appendChild(el('td','num', '$'+g.gen_usd.toFixed(2)));
+ tr.appendChild(el('td','num', g.min_per_run.toFixed(0)));
+ body.appendChild(tr);
+ });
+ tb.appendChild(body); sc.appendChild(tb); at.appendChild(sc);
+});
+
+// ---------- per-cell table for compare_v2
+const v2 = DATA.batteries['compare_v2_full'];
+if(v2){
+ const H='harness:api-agent:gemini:gemini-3.7-flash', O='oneshot:gemini:gemini-3.7-flash',
+ C='oneshot:claude-code', X='oneshot:codex';
+ const ids = Object.keys(v2[H].cells).sort();
+ const sc = el('div','scroll'); const tb = el('table');
+ tb.innerHTML = '<thead><tr><th>Prompt</th><th class="num">harness+flash</th><th class="num">one-shot flash</th>'+
+ '<th class="num">Δ</th><th class="num">one-shot codex</th><th class="num">one-shot claude-code</th></tr></thead>';
+ const body = el('tbody');
+ const cell = (arm,id) => { const a = v2[arm] && v2[arm].cells[id]; if(!a) return null; return a; };
+ ids.forEach(id=>{
+ const h=cell(H,id), o=cell(O,id), x=cell(X,id), c=cell(C,id);
+ const tr = el('tr');
+ tr.appendChild(el('td','arm', id.replace('cmp_','')));
+ const sfmt = a => a==null ? '—' : (a.score==null ? `<span class="chip mute">${a.status}</span>` : a.score.toFixed(3));
+ tr.appendChild(el('td','num', sfmt(h)));
+ tr.appendChild(el('td','num', sfmt(o)));
+ const d = (h&&o&&h.score!=null&&o.score!=null) ? h.score-o.score : null;
+ const dt = el('td','num', d==null?'—':f3(d));
+ if(d!=null && d>0.02) dt.style.color='var(--good)';
+ if(d!=null && d<-0.02) dt.style.color='var(--null)';
+ tr.appendChild(dt);
+ tr.appendChild(el('td','num', sfmt(x)));
+ tr.appendChild(el('td','num', sfmt(c)));
+ body.appendChild(tr);
+ });
+ tb.appendChild(body); sc.appendChild(tb); $('#celltable').appendChild(sc);
+}
+
+// ---------- severity cards
+const SEV = [['crash',8,'null'],['wrong-result',19,'null'],['data-loss',2,'null'],['hang',2,'warn'],['usability',10,'warn'],['style',1,'mute']];
+const sv = $('#sevcards');
+SEV.forEach(([k,v,tone])=>{
+ const c = el('div','sevc');
+ c.style.setProperty('--tone', tone==='null'?'var(--null)':(tone==='warn'?'var(--warn)':'var(--ink-3)'));
+ c.appendChild(el('div','v',String(v)));
+ c.appendChild(el('div','k',k));
+ sv.appendChild(c);
+});
+})();
+</script>