Official Codebase of 3DCodeVerse
Python
tracks: D68 counts corrected; a hero whose session died in the storm gets one cheap repair; a storm-dead repair session ends the loop
gaoypeng5d12a750ba78
diff --git a/harness/codeverse/tracks/generation.py b/harness/codeverse/tracks/generation.pyindex 193d09a6..360304f3 100644--- a/harness/codeverse/tracks/generation.py+++ b/harness/codeverse/tracks/generation.py@@ -340,11 +340,15 @@ class GenerationResult(BaseModel):turn_capped: bool = Field(default=False, description="a session hit the turn budget and was wrapped up")+ transient: bool = Field(+ default=False,+ description="the last session died on a transient-failure streak with no final answer "+ "(AgentResult.transient) — whatever it wrote before the wall is in files_changed",+ )storm: bool = Field(default=False,- description="every session died on a transient-failure streak (AgentResult.transient) and "- "nothing was written — the agent route is down, not the task; tracks fall back to the "- "hedged single-shot path (tracks.common.generate_for)",+ description="transient AND nothing was written — the agent route is down, not the task; "+ "tracks fall back to the hedged single-shot path (tracks.common.generate_for)",)@@ -661,6 +665,7 @@ def run_agent_task(turns=acc.turns,sessions=acc.sessions,turn_capped=acc.wrapped,+ transient=bool(res.transient),storm=not changes and bool(res.transient),)diff --git a/harness/codeverse/tracks/repair.py b/harness/codeverse/tracks/repair.pyindex a2cb9c56..664c4175 100644--- a/harness/codeverse/tracks/repair.py+++ b/harness/codeverse/tracks/repair.py@@ -212,6 +212,11 @@ def build_with_repair(ctx: RunContext, *, round_index: int, label: str, files_hioutcome.build, outcome.lint = build, lintctx.events.emit("build.done", round=round_index, ok=build.ok, lint_errors=len(lint.errors), attempt=attempt,error=build.error_message[:200], duration_ms=build.duration_ms)+ if res.transient and not outcome.ok:+ # the repair session died in a 503 storm (D68): whatever it wrote is built above;+ # another session would spend its whole window at the same wall+ ctx.events.emit("repair.storm", round=round_index, attempt=attempt, notes=res.notes[:200])+ breakoutcome.repaired = outcome.ok and attempt > 0return outcomediff --git a/harness/codeverse/tracks/scene_assets.py b/harness/codeverse/tracks/scene_assets.pyindex 7dc44d63..a8d72d80 100644--- a/harness/codeverse/tracks/scene_assets.py+++ b/harness/codeverse/tracks/scene_assets.py@@ -325,7 +325,29 @@ def _ladder(ctx: RunContext, asset: AssetPlan, rel: str, *, language: Language,res = _generate_asset(ctx, asset, rel, language=language, attempt=0, files=files, timeout_s=timeout_s)notes = res.notesstrategy = "escalated" if chk is not None else "agent"- chk = check(ctx) if res.ok else chk+ # a session that died in the storm mid-work (D68) left partial files: build and check+ # them, but with the CHEAP context — the check's own one build repair then goes+ # through the single-shot path, not another agent session at the wall+ chk = check(sub if res.transient and sub is not None else ctx) if res.ok else chk+ if res.transient and sub is not None and chk is not None and not chk.ok:+ # the session died in a 503 storm mid-work (D68): its partial files failed the+ # check, and another 12 minutes at the wall would too — the cheap rung answers.+ ctx.events.emit("asset.storm_repair", asset=asset.name, errors=chk.errors[:3])+ try:+ res2 = _generate_asset(sub, asset, rel, language=language, attempt=2, files=files,+ feedback=repair_feedback(chk, rel))+ except Exception as e: # noqa: BLE001 — same rule as the ladder's own single-shots+ from codeverse.orchestrator import BudgetExceeded++ if isinstance(e, BudgetExceeded):+ raise+ log.warning("storm repair of %s failed: %s", asset.name, e)+ return strategy, chk, notes+ if res2.ok:+ chk2 = check(sub)+ if chk2.ok:+ return "escalated+repair", chk2, res2.notes+ chk = chk2return strategy, chk, notesdiff --git a/harness/docs/DECISIONS.md b/harness/docs/DECISIONS.mdindex 08b2dda8..0d8a030d 100644--- a/harness/docs/DECISIONS.md+++ b/harness/docs/DECISIONS.md@@ -1071,21 +1071,27 @@ written) that were accepted because the code works that way and the tests pin itlighthouse did exactly that.* **D68 A session that died in a 503 storm is retried through the single-shot path- (2026-09-07).** Loops 10–12 ran in an evening-long Gemini 503 storm: 23 of 24 gemini-cli- sessions (env, zones, escalated heroes, refines) ended `timeout / 0 turns / $0` after- 8–17 consecutive "Attempt N failed with status 503" inside the CLI's own retry loop —- gemini-cli never gives up before the harness's wall — while every single-shot call in the- same minutes got through (hedged across the 22 keys) and `codex:gpt-6-astra` sessions- completed. The stages that lost their session shipped the skeleton env and empty zones- and judged 0.00–0.14 (clockmaker: "the hero LongcaseClock is completely missing"). The- session result now says why it died (`AgentResult.transient`: timed out, nothing produced,- a transient streak in stderr), the generation result carries `storm`, and the ONE call- every stage makes — `tracks.common.generate_for(ctx, task)`, which replaced seven identical- eleven-argument `generate(...)` sites — retries a storm-dead task through- `single_shot_ctx`, the same degradation the soft budget already used. The owner's rule for- the storm (2026-09-07 evening): keep querying, keep retrying. Not a second agent session:- that is another 12 minutes at the wall while the storm lasts; the cheap path is the retry- that answers. `single_shot_agent_id` / `single_shot_ctx` moved from `scene_assets` to+ (2026-09-07; counts corrected the same night).** Loops 10–13 ran in an evening-long+ Gemini 503 storm. Counted over their 40 gemini-cli sessions that ended `timeout`+ (env, zones, escalated heroes, refines): 0 produced a final answer, 14 wrote nothing at+ all, 26 wrote partial files and died at the wall mid-work — 1–15 "Attempt N failed with+ status 503" each inside the CLI's own retry loop, which never gives up before the+ harness's wall (gemini-cli exposes no turn count, so "0 turns" in the events says+ nothing). Every single-shot call in the same minutes got through (hedged across the 22+ keys) and `codex:gpt-6-astra` sessions completed. The stages whose session wrote+ nothing shipped the skeleton env and empty zones and judged 0.00–0.14 (clockmaker: "the+ hero LongcaseClock is completely missing"). The session result now says why it died+ (`AgentResult.transient`: timed out, no final answer, a transient streak in stderr),+ the generation result carries `transient` and `storm` (transient AND nothing written),+ and the ONE call every stage makes — `tracks.common.generate_for(ctx, task)`, which+ replaced seven identical eleven-argument `generate(...)` sites — retries a storm-dead+ task through `single_shot_ctx`, the degradation the soft budget already used. A session+ that died with partial files keeps them: the build gates and repairs judge that work as+ they judge any other. A hero's escalated session that died transient with a failing+ check gets ONE more single-shot repair with that check's feedback (`asset.storm_repair`)+ before the hero is given up — the cheap rung that answers, not a second 12-minute+ session at the wall. The owner's rule for the storm (2026-09-07 evening): keep querying,+ keep retrying. `single_shot_agent_id` / `single_shot_ctx` moved from `scene_assets` to`tracks.common` with it (they were never scene-specific).## Rejected / deferreddiff --git a/harness/tests/orchestrator_tracks/test_harness_failure_no_repair.py b/harness/tests/orchestrator_tracks/test_harness_failure_no_repair.pyindex f6a4f0fe..4a7c4822 100644--- a/harness/tests/orchestrator_tracks/test_harness_failure_no_repair.py+++ b/harness/tests/orchestrator_tracks/test_harness_failure_no_repair.py@@ -94,7 +94,7 @@ def test_a_real_build_failure_is_still_repaired(monkeypatch):def fake_generate(ws, **kw):seen.append(1)- return SimpleNamespace(ok=True, files_changed=[], usage=Usage(), notes="", storm=False)+ return SimpleNamespace(ok=True, files_changed=[], usage=Usage(), notes="", storm=False, transient=False)monkeypatch.setattr(C, "generate", fake_generate)monkeypatch.setattr(R, "make_repair_task", lambda *a, **k: SimpleNamespace(label="t"))@@ -111,6 +111,30 @@ def test_a_real_build_failure_is_still_repaired(monkeypatch):assert out.ok is True and len(seen) == 1, "a real error still gets one repair"+def test_a_repair_session_that_died_in_the_storm_ends_the_loop(monkeypatch):+ """D68: a repair session that timed out in a 503 streak (partial or no files) is built once+ and the loop stops — the next attempt would spend its whole window at the same wall."""+ seen: list[int] = []++ from codeverse.contracts.common import Usage++ def fake_generate(ws, **kw):+ seen.append(1)+ return SimpleNamespace(ok=False, files_changed=[], usage=Usage(), notes="exit=timeout", storm=False, transient=True)++ monkeypatch.setattr(C, "generate", fake_generate)+ monkeypatch.setattr(R, "make_repair_task", lambda *a, **k: SimpleNamespace(label="t"))+ calls = {"n": 0}++ def fake_build_once(ctx):+ calls["n"] += 1+ return _build(ok=False, msg="TypeError: x is not a function"), GateReport(gate="lint", passed=True, findings=[])++ monkeypatch.setattr(R, "build_once", fake_build_once)+ out = R.build_with_repair(_ctx(), round_index=0, label="r00", max_attempts=3)+ assert out.ok is False and len(seen) == 1 and calls["n"] == 2, "one storm-dead repair, one rebuild, then stop"++def test_a_harness_failure_is_rebuilt_even_with_no_repair_budget(monkeypatch, no_generate):"""The rebuilds are bounded by MAX_HARNESS_REBUILDS, not by the repair budget: abaseline arm that runs with max_repair_attempts=0 must still get its retries, or adiff --git a/harness/tests/orchestrator_tracks/test_scene_hero_ladder.py b/harness/tests/orchestrator_tracks/test_scene_hero_ladder.pyindex 3f3ceb47..77e07da6 100644--- a/harness/tests/orchestrator_tracks/test_scene_hero_ladder.py+++ b/harness/tests/orchestrator_tracks/test_scene_hero_ladder.py@@ -153,6 +153,54 @@ def test_hero_glb_soft_findings_trigger_one_feedback_repair_then_escalate(tmp_wsassert any(e["event"] == "asset.escalated" for e in events) and any(e["event"] == "asset.plan_failed" for e in events)+class _StormAgent(FakeAgent):+ """A CLI session that wrote partial files and died at the wall in a 503 streak with no final+ answer (D68: 26 of the evening's 40 timed-out sessions)."""++ def run(self, job):+ from codeverse.contracts.agent import AgentResult+ from codeverse.workspace import Workspace++ self.jobs.append(job)+ ws = Workspace(job.workspace)+ before = ws.head()+ for rel, content in (self.writer(job, ws) or {}).items():+ (ws.root / rel).parent.mkdir(parents=True, exist_ok=True)+ (ws.root / rel).write_text(content)+ return AgentResult(ok=False, exit_reason="timeout", transient=True, files_changed=ws.changed_files(before),+ errors=["killed by watchdog (hard_timeout) after 720s", "7 x 503 inside the CLI's own retry loop before the wall; nothing produced"])+++def test_a_hero_session_that_died_in_the_storm_gets_one_more_cheap_repair(tmp_ws, settings, monkeypatch):+ """Loops 10 and 13 (2026-09-07): the clockmaker's LongcaseClock escalated, the session wrote+ eleven part files and died at the wall after 7 x 503, the partial parts failed the check and+ the scene shipped without its hero. The ladder now takes ONE more single-shot repair with+ that check's feedback (asset.storm_repair) before giving the hero up."""+ calls = {"n": 0}++ class FailTwiceThenGood(SingleShotServices):+ def _answer(self, req):+ if req.response_schema is not None:+ return plan_example(Track.STATIC_OBJECT)+ calls["n"] += 1+ body = GOOD_MODEL if calls["n"] >= 3 else f"import bpy\n{FAIL_MARK}\n" # two shots fail, the rung answers+ return f"=== FILE: src/model.py ===\n{body}=== END FILE ==="++ services = FailTwiceThenGood()+ agent = _StormAgent(_writer(f"import bpy\n{FAIL_MARK}\n")) # partial, broken parts+ monkeypatch.setattr(SA, "hero_plan", lambda sub, asset: _crate_plan(asset))+ ctx, hero = _scene(tmp_ws, settings, services=services, agent=agent)+ res = SA.build_blender_asset(ctx, hero, judge=False)+ assert res.ok and res.strategy == "escalated+repair", (res.strategy, res.notes)+ # ONE agent session: the partial files were checked on the cheap context (no agent repair) and the rung went single-shot+ assert calls["n"] == 3 and [j.label for j in agent.jobs] == [f"asset_{SA.to_snake(hero.name)}"]+ texts = [" ".join(p.text for m in r.messages for p in m.parts if getattr(p, "text", "")) for r in services.chat.requests+ if r.response_schema is None]+ assert "did NOT pass the deterministic asset check" in texts[2]+ events = [json.loads(line) for line in tmp_ws.events_path.read_text().splitlines()]+ assert [e["event"] for e in events if e["event"] in ("asset.escalated", "asset.storm_repair")] == ["asset.escalated", "asset.storm_repair"]++def test_hero_repair_sessions_are_clipped_to_the_asset_window_and_capped_at_one(tmp_ws, settings):agent = FakeAgent(_writer(f"import bpy\n# {FAIL_MARK}\n")) # every session leaves a build that failsctx, hero = _scene(tmp_ws, settings, services=CardedServices(), agent=agent)