Skip to content

Official Codebase of 3DCodeVerse

Python

tracks: D68 counts corrected; a hero whose session died in the storm gets one cheap repair; a storm-dead repair session ends the loop

gaoypeng5d12a750ba78
diff --git a/harness/codeverse/tracks/generation.py b/harness/codeverse/tracks/generation.py
index 193d09a6..360304f3 100644
--- a/harness/codeverse/tracks/generation.py
+++ b/harness/codeverse/tracks/generation.py
@@ -340,11 +340,15 @@ class GenerationResult(BaseModel):
turn_capped: bool = Field(
default=False, description="a session hit the turn budget and was wrapped up"
)
+ transient: bool = Field(
+ default=False,
+ description="the last session died on a transient-failure streak with no final answer "
+ "(AgentResult.transient) — whatever it wrote before the wall is in files_changed",
+ )
storm: bool = Field(
default=False,
- description="every session died on a transient-failure streak (AgentResult.transient) and "
- "nothing was written — the agent route is down, not the task; tracks fall back to the "
- "hedged single-shot path (tracks.common.generate_for)",
+ description="transient AND nothing was written — the agent route is down, not the task; "
+ "tracks fall back to the hedged single-shot path (tracks.common.generate_for)",
)
@@ -661,6 +665,7 @@ def run_agent_task(
turns=acc.turns,
sessions=acc.sessions,
turn_capped=acc.wrapped,
+ transient=bool(res.transient),
storm=not changes and bool(res.transient),
)
diff --git a/harness/codeverse/tracks/repair.py b/harness/codeverse/tracks/repair.py
index a2cb9c56..664c4175 100644
--- a/harness/codeverse/tracks/repair.py
+++ b/harness/codeverse/tracks/repair.py
@@ -212,6 +212,11 @@ def build_with_repair(ctx: RunContext, *, round_index: int, label: str, files_hi
outcome.build, outcome.lint = build, lint
ctx.events.emit("build.done", round=round_index, ok=build.ok, lint_errors=len(lint.errors), attempt=attempt,
error=build.error_message[:200], duration_ms=build.duration_ms)
+ if res.transient and not outcome.ok:
+ # the repair session died in a 503 storm (D68): whatever it wrote is built above;
+ # another session would spend its whole window at the same wall
+ ctx.events.emit("repair.storm", round=round_index, attempt=attempt, notes=res.notes[:200])
+ break
outcome.repaired = outcome.ok and attempt > 0
return outcome
diff --git a/harness/codeverse/tracks/scene_assets.py b/harness/codeverse/tracks/scene_assets.py
index 7dc44d63..a8d72d80 100644
--- a/harness/codeverse/tracks/scene_assets.py
+++ b/harness/codeverse/tracks/scene_assets.py
@@ -325,7 +325,29 @@ def _ladder(ctx: RunContext, asset: AssetPlan, rel: str, *, language: Language,
res = _generate_asset(ctx, asset, rel, language=language, attempt=0, files=files, timeout_s=timeout_s)
notes = res.notes
strategy = "escalated" if chk is not None else "agent"
- chk = check(ctx) if res.ok else chk
+ # a session that died in the storm mid-work (D68) left partial files: build and check
+ # them, but with the CHEAP context — the check's own one build repair then goes
+ # through the single-shot path, not another agent session at the wall
+ chk = check(sub if res.transient and sub is not None else ctx) if res.ok else chk
+ if res.transient and sub is not None and chk is not None and not chk.ok:
+ # the session died in a 503 storm mid-work (D68): its partial files failed the
+ # check, and another 12 minutes at the wall would too — the cheap rung answers.
+ ctx.events.emit("asset.storm_repair", asset=asset.name, errors=chk.errors[:3])
+ try:
+ res2 = _generate_asset(sub, asset, rel, language=language, attempt=2, files=files,
+ feedback=repair_feedback(chk, rel))
+ except Exception as e: # noqa: BLE001 — same rule as the ladder's own single-shots
+ from codeverse.orchestrator import BudgetExceeded
+
+ if isinstance(e, BudgetExceeded):
+ raise
+ log.warning("storm repair of %s failed: %s", asset.name, e)
+ return strategy, chk, notes
+ if res2.ok:
+ chk2 = check(sub)
+ if chk2.ok:
+ return "escalated+repair", chk2, res2.notes
+ chk = chk2
return strategy, chk, notes
diff --git a/harness/docs/DECISIONS.md b/harness/docs/DECISIONS.md
index 08b2dda8..0d8a030d 100644
--- a/harness/docs/DECISIONS.md
+++ b/harness/docs/DECISIONS.md
@@ -1071,21 +1071,27 @@ written) that were accepted because the code works that way and the tests pin it
lighthouse did exactly that.
* **D68 A session that died in a 503 storm is retried through the single-shot path
- (2026-09-07).** Loops 10–12 ran in an evening-long Gemini 503 storm: 23 of 24 gemini-cli
- sessions (env, zones, escalated heroes, refines) ended `timeout / 0 turns / $0` after
- 8–17 consecutive "Attempt N failed with status 503" inside the CLI's own retry loop —
- gemini-cli never gives up before the harness's wall — while every single-shot call in the
- same minutes got through (hedged across the 22 keys) and `codex:gpt-6-astra` sessions
- completed. The stages that lost their session shipped the skeleton env and empty zones
- and judged 0.00–0.14 (clockmaker: "the hero LongcaseClock is completely missing"). The
- session result now says why it died (`AgentResult.transient`: timed out, nothing produced,
- a transient streak in stderr), the generation result carries `storm`, and the ONE call
- every stage makes — `tracks.common.generate_for(ctx, task)`, which replaced seven identical
- eleven-argument `generate(...)` sites — retries a storm-dead task through
- `single_shot_ctx`, the same degradation the soft budget already used. The owner's rule for
- the storm (2026-09-07 evening): keep querying, keep retrying. Not a second agent session:
- that is another 12 minutes at the wall while the storm lasts; the cheap path is the retry
- that answers. `single_shot_agent_id` / `single_shot_ctx` moved from `scene_assets` to
+ (2026-09-07; counts corrected the same night).** Loops 10–13 ran in an evening-long
+ Gemini 503 storm. Counted over their 40 gemini-cli sessions that ended `timeout`
+ (env, zones, escalated heroes, refines): 0 produced a final answer, 14 wrote nothing at
+ all, 26 wrote partial files and died at the wall mid-work — 1–15 "Attempt N failed with
+ status 503" each inside the CLI's own retry loop, which never gives up before the
+ harness's wall (gemini-cli exposes no turn count, so "0 turns" in the events says
+ nothing). Every single-shot call in the same minutes got through (hedged across the 22
+ keys) and `codex:gpt-6-astra` sessions completed. The stages whose session wrote
+ nothing shipped the skeleton env and empty zones and judged 0.00–0.14 (clockmaker: "the
+ hero LongcaseClock is completely missing"). The session result now says why it died
+ (`AgentResult.transient`: timed out, no final answer, a transient streak in stderr),
+ the generation result carries `transient` and `storm` (transient AND nothing written),
+ and the ONE call every stage makes — `tracks.common.generate_for(ctx, task)`, which
+ replaced seven identical eleven-argument `generate(...)` sites — retries a storm-dead
+ task through `single_shot_ctx`, the degradation the soft budget already used. A session
+ that died with partial files keeps them: the build gates and repairs judge that work as
+ they judge any other. A hero's escalated session that died transient with a failing
+ check gets ONE more single-shot repair with that check's feedback (`asset.storm_repair`)
+ before the hero is given up — the cheap rung that answers, not a second 12-minute
+ session at the wall. The owner's rule for the storm (2026-09-07 evening): keep querying,
+ keep retrying. `single_shot_agent_id` / `single_shot_ctx` moved from `scene_assets` to
`tracks.common` with it (they were never scene-specific).
## Rejected / deferred
diff --git a/harness/tests/orchestrator_tracks/test_harness_failure_no_repair.py b/harness/tests/orchestrator_tracks/test_harness_failure_no_repair.py
index f6a4f0fe..4a7c4822 100644
--- a/harness/tests/orchestrator_tracks/test_harness_failure_no_repair.py
+++ b/harness/tests/orchestrator_tracks/test_harness_failure_no_repair.py
@@ -94,7 +94,7 @@ def test_a_real_build_failure_is_still_repaired(monkeypatch):
def fake_generate(ws, **kw):
seen.append(1)
- return SimpleNamespace(ok=True, files_changed=[], usage=Usage(), notes="", storm=False)
+ return SimpleNamespace(ok=True, files_changed=[], usage=Usage(), notes="", storm=False, transient=False)
monkeypatch.setattr(C, "generate", fake_generate)
monkeypatch.setattr(R, "make_repair_task", lambda *a, **k: SimpleNamespace(label="t"))
@@ -111,6 +111,30 @@ def test_a_real_build_failure_is_still_repaired(monkeypatch):
assert out.ok is True and len(seen) == 1, "a real error still gets one repair"
+def test_a_repair_session_that_died_in_the_storm_ends_the_loop(monkeypatch):
+ """D68: a repair session that timed out in a 503 streak (partial or no files) is built once
+ and the loop stops — the next attempt would spend its whole window at the same wall."""
+ seen: list[int] = []
+
+ from codeverse.contracts.common import Usage
+
+ def fake_generate(ws, **kw):
+ seen.append(1)
+ return SimpleNamespace(ok=False, files_changed=[], usage=Usage(), notes="exit=timeout", storm=False, transient=True)
+
+ monkeypatch.setattr(C, "generate", fake_generate)
+ monkeypatch.setattr(R, "make_repair_task", lambda *a, **k: SimpleNamespace(label="t"))
+ calls = {"n": 0}
+
+ def fake_build_once(ctx):
+ calls["n"] += 1
+ return _build(ok=False, msg="TypeError: x is not a function"), GateReport(gate="lint", passed=True, findings=[])
+
+ monkeypatch.setattr(R, "build_once", fake_build_once)
+ out = R.build_with_repair(_ctx(), round_index=0, label="r00", max_attempts=3)
+ assert out.ok is False and len(seen) == 1 and calls["n"] == 2, "one storm-dead repair, one rebuild, then stop"
+
+
def test_a_harness_failure_is_rebuilt_even_with_no_repair_budget(monkeypatch, no_generate):
"""The rebuilds are bounded by MAX_HARNESS_REBUILDS, not by the repair budget: a
baseline arm that runs with max_repair_attempts=0 must still get its retries, or a
diff --git a/harness/tests/orchestrator_tracks/test_scene_hero_ladder.py b/harness/tests/orchestrator_tracks/test_scene_hero_ladder.py
index 3f3ceb47..77e07da6 100644
--- a/harness/tests/orchestrator_tracks/test_scene_hero_ladder.py
+++ b/harness/tests/orchestrator_tracks/test_scene_hero_ladder.py
@@ -153,6 +153,54 @@ def test_hero_glb_soft_findings_trigger_one_feedback_repair_then_escalate(tmp_ws
assert any(e["event"] == "asset.escalated" for e in events) and any(e["event"] == "asset.plan_failed" for e in events)
+class _StormAgent(FakeAgent):
+ """A CLI session that wrote partial files and died at the wall in a 503 streak with no final
+ answer (D68: 26 of the evening's 40 timed-out sessions)."""
+
+ def run(self, job):
+ from codeverse.contracts.agent import AgentResult
+ from codeverse.workspace import Workspace
+
+ self.jobs.append(job)
+ ws = Workspace(job.workspace)
+ before = ws.head()
+ for rel, content in (self.writer(job, ws) or {}).items():
+ (ws.root / rel).parent.mkdir(parents=True, exist_ok=True)
+ (ws.root / rel).write_text(content)
+ return AgentResult(ok=False, exit_reason="timeout", transient=True, files_changed=ws.changed_files(before),
+ errors=["killed by watchdog (hard_timeout) after 720s", "7 x 503 inside the CLI's own retry loop before the wall; nothing produced"])
+
+
+def test_a_hero_session_that_died_in_the_storm_gets_one_more_cheap_repair(tmp_ws, settings, monkeypatch):
+ """Loops 10 and 13 (2026-09-07): the clockmaker's LongcaseClock escalated, the session wrote
+ eleven part files and died at the wall after 7 x 503, the partial parts failed the check and
+ the scene shipped without its hero. The ladder now takes ONE more single-shot repair with
+ that check's feedback (asset.storm_repair) before giving the hero up."""
+ calls = {"n": 0}
+
+ class FailTwiceThenGood(SingleShotServices):
+ def _answer(self, req):
+ if req.response_schema is not None:
+ return plan_example(Track.STATIC_OBJECT)
+ calls["n"] += 1
+ body = GOOD_MODEL if calls["n"] >= 3 else f"import bpy\n{FAIL_MARK}\n" # two shots fail, the rung answers
+ return f"=== FILE: src/model.py ===\n{body}=== END FILE ==="
+
+ services = FailTwiceThenGood()
+ agent = _StormAgent(_writer(f"import bpy\n{FAIL_MARK}\n")) # partial, broken parts
+ monkeypatch.setattr(SA, "hero_plan", lambda sub, asset: _crate_plan(asset))
+ ctx, hero = _scene(tmp_ws, settings, services=services, agent=agent)
+ res = SA.build_blender_asset(ctx, hero, judge=False)
+ assert res.ok and res.strategy == "escalated+repair", (res.strategy, res.notes)
+ # ONE agent session: the partial files were checked on the cheap context (no agent repair) and the rung went single-shot
+ assert calls["n"] == 3 and [j.label for j in agent.jobs] == [f"asset_{SA.to_snake(hero.name)}"]
+ texts = [" ".join(p.text for m in r.messages for p in m.parts if getattr(p, "text", "")) for r in services.chat.requests
+ if r.response_schema is None]
+ assert "did NOT pass the deterministic asset check" in texts[2]
+ events = [json.loads(line) for line in tmp_ws.events_path.read_text().splitlines()]
+ assert [e["event"] for e in events if e["event"] in ("asset.escalated", "asset.storm_repair")] == ["asset.escalated", "asset.storm_repair"]
+
+
def test_hero_repair_sessions_are_clipped_to_the_asset_window_and_capped_at_one(tmp_ws, settings):
agent = FakeAgent(_writer(f"import bpy\n# {FAIL_MARK}\n")) # every session leaves a build that fails
ctx, hero = _scene(tmp_ws, settings, services=CardedServices(), agent=agent)