Skip to content

Official Codebase of 3DCodeVerse

Python

tracks: a session that died in a 503 storm is retried through the single-shot path; one generate_for() for every stage

gaoypeng7e7f5a0f4ee3
diff --git a/harness/codeverse/agents/backends.py b/harness/codeverse/agents/backends.py
index 118ea2a9..4d20f6f9 100644
--- a/harness/codeverse/agents/backends.py
+++ b/harness/codeverse/agents/backends.py
@@ -264,6 +264,7 @@ class GeminiCliAgent(_CliAgent):
return finish_session(
s, ok=outcome["ok"], exit_reason=outcome["exit_reason"], text=outcome["text"],
usage=usage_total, tool_calls=usage_total.tool_calls, errors=outcome["errors"],
+ transient=bool(outcome["transient"]) and not outcome["ok"],
attempts=attempts, rc=proc.rc, killed_reason=proc.killed_reason, session_id=outcome.get("session_id", ""),
)
finally:
@@ -283,6 +284,12 @@ class GeminiCliAgent(_CliAgent):
if proc.timed_out:
out["exit_reason"] = "timeout"
errors.append(watchdog_error(proc))
+ # the CLI retries 503s itself, with backoff, and never gives up before our wall:
+ # measured 2026-09-07, 8-17 "Attempt N failed with status 503" per 12-minute
+ # session, nothing produced. That is a storm, not the task.
+ out["transient"] = not text.strip() and is_transient_failure(proc.stderr)
+ if out["transient"]:
+ errors.append(f"{proc.stderr.count('status 503')} x 503 inside the CLI's own retry loop before the wall; nothing produced")
return out
if served and self.model not in served:
out["exit_reason"] = "model_substituted"
@@ -415,9 +422,11 @@ class ClaudeCodeAgent(_CliAgent):
text = str((env or {}).get("result") or "")
turns = int((env or {}).get("num_turns") or 0)
errors: list[str] = []
+ transient = False
if proc.timed_out:
reason, ok = "timeout", False
errors.append(watchdog_error(proc))
+ transient = not text.strip() and is_transient_failure(proc.stderr)
elif env is None or proc.rc != 0:
reason, ok = "error", False
errors.append(f"rc={proc.rc}; no result envelope; stderr tail: {tail(proc.stderr, 1500)}")
@@ -432,7 +441,7 @@ class ClaudeCodeAgent(_CliAgent):
reason, ok = "completed", True
return finish_session(
s, ok=ok, exit_reason=reason, text=text, usage=usage, tool_calls=max(turns - 1, 0), turns=turns,
- errors=errors, rc=proc.rc, killed_reason=proc.killed_reason, num_turns=turns,
+ errors=errors, transient=transient, rc=proc.rc, killed_reason=proc.killed_reason, num_turns=turns,
session_id=(env or {}).get("session_id", ""), subtype=(env or {}).get("subtype", ""),
model_usage=(env or {}).get("modelUsage", {}),
)
diff --git a/harness/codeverse/agents/cli_common.py b/harness/codeverse/agents/cli_common.py
index 213baffb..1dc9e490 100644
--- a/harness/codeverse/agents/cli_common.py
+++ b/harness/codeverse/agents/cli_common.py
@@ -493,6 +493,7 @@ def finish_session(
tool_calls: int = 0,
turns: int = 0,
errors: list[str] | None = None,
+ transient: bool = False,
**extra: Any,
) -> AgentResult:
"""Commit the agent's work, compute ``files_changed`` via git (attributed to this
@@ -520,7 +521,7 @@ def finish_session(
ok=ok, exit_reason=exit_reason, text=text, files_changed=files,
transcript_path=str(s.traj.transcript_path if s.traj.transcript_path.exists() else s.traj.dir),
usage=usage, duration_s=round(time.monotonic() - s.t0, 3), tool_calls=tool_calls,
- turns=turns, errors=errors,
+ turns=turns, errors=errors, transient=transient,
)
s.traj.write_result(res, kind=s.kind, label=s.label, round=s.round_index, attempt=s.attempt, job_label=s.job.label,
head_before=s.head_before, head_after=s.ws.head(), notes=s.notes, **extra)
diff --git a/harness/codeverse/contracts/agent.py b/harness/codeverse/contracts/agent.py
index 7008e01e..e7a538cd 100644
--- a/harness/codeverse/contracts/agent.py
+++ b/harness/codeverse/contracts/agent.py
@@ -84,3 +84,10 @@ class AgentResult(BaseModel):
"turn.completed events, agy num_turns); 0 for gemini-cli, which exposes no turn count",
)
errors: list[str] = Field(default_factory=list)
+ transient: bool = Field(
+ default=False,
+ description="the session died on a transient-failure streak (503 / UNAVAILABLE / 429 in the CLI's "
+ "own retry loop) and produced nothing — a 503 storm, not the task; the cheap single-shot "
+ "path, hedged across keys, may still get through (measured 2026-09-07: 23 of 24 gemini-cli "
+ "sessions in one evening)",
+ )
diff --git a/harness/codeverse/tracks/common.py b/harness/codeverse/tracks/common.py
index 2df1eb78..7ab63ca5 100644
--- a/harness/codeverse/tracks/common.py
+++ b/harness/codeverse/tracks/common.py
@@ -15,7 +15,7 @@ from __future__ import annotations
import importlib
import logging
from collections.abc import Sequence
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
from pathlib import Path
from typing import Any
@@ -30,7 +30,13 @@ from codeverse.orchestrator import BudgetGuard, RoundPolicy, RunState
from codeverse.proc import EventLog
from codeverse.prompts import load_text, prompt_hash
from codeverse.prompts.catalog import prompt_dir_for
-from codeverse.tracks.generation import is_single_shot
+from codeverse.tracks.generation import (
+ SINGLE_SHOT_PREFIX,
+ GenerationResult,
+ GenerationTask,
+ generate,
+ is_single_shot,
+)
from codeverse.workspace import Workspace
log = logging.getLogger(__name__)
@@ -194,6 +200,69 @@ class RunContext:
self.prompt_hashes[name] = prompt_hash(text)
+# ----------------------------------------------------------------------------- strategy
+def single_shot_agent_id(agent_id: str, chat_model_id: str = "") -> str:
+ """The single-shot strategy id for this run, or "" when there is no chat model.
+
+ Single-shot is ONE api call that returns the asset file — the cheap path the asset
+ stage tries before escalating to a full agent session. It needs a chat model, and
+ the coding agent is always a vendor CLI (which exposes none), so the model comes
+ from ``chat_model_id`` — the run's planner backend, which is always an API model.
+ Until 2026-08-28 it was derived from the in-process ``api-agent`` generator id;
+ that backend is gone."""
+ if is_single_shot(agent_id):
+ return agent_id
+ # a SHAPE check only, deliberately not models.registry.parse_model_id: whether the
+ # id resolves is the SERVICES' call (tests run fake providers like "fake:planner"),
+ # and single_shot_ctx already degrades to the agent path when chat_model() raises
+ return SINGLE_SHOT_PREFIX + chat_model_id if ":" in chat_model_id else ""
+
+
+def single_shot_ctx(ctx: RunContext) -> RunContext | None:
+ """A copy of ``ctx`` bound to the single-shot strategy, or None when the
+ generator has no usable chat model (CLI agents, tests with fake services)."""
+ cached = ctx.extra.get("_single_shot_ctx")
+ if cached is not None:
+ return cached or None # False = known-unavailable
+ sid = single_shot_agent_id(ctx.agent_id, ctx.spec.backends.planner)
+ sub: RunContext | None = None
+ if sid:
+ try:
+ model = ctx.model if is_single_shot(ctx.agent_id) else ctx.services.chat_model(sid[len(SINGLE_SHOT_PREFIX):])
+ except Exception as e: # noqa: BLE001 — no chat model → keep the agent path
+ log.info("single-shot generation unavailable for %s: %s", ctx.agent_id, e)
+ model = None
+ if model is not None:
+ sub = replace(ctx, agent_id=sid, model=model, agent=None)
+ ctx.extra["_single_shot_ctx"] = sub or False
+ return sub
+
+
+def generate_for(ctx: RunContext, task: GenerationTask) -> GenerationResult:
+ """``generate()`` with everything ``ctx`` knows — the one call every stage makes — plus the
+ storm fallback: a session that died on a transient-failure streak and wrote nothing
+ (``GenerationResult.storm``) is retried through the single-shot path, one hedged,
+ key-rotating API call. Measured 2026-09-07: 23 of 24 gemini-cli sessions of an evening
+ ended ``timeout / 0 turns / $0`` after 8-17 consecutive 503s inside the CLI's own retry
+ loop, while every single-shot in the same minutes got through; the stages that lost
+ their session shipped the skeleton env and empty zones and judged 0.00-0.14."""
+ res = generate(ctx.ws, agent_id=ctx.agent_id, task=task, agent=ctx.agent, model=ctx.model, settings=ctx.settings,
+ budget=ctx.budget, events=ctx.events,
+ max_turns=ctx.policy.agent_max_turns, wrapup_turns=ctx.policy.agent_wrapup_turns)
+ if not res.storm or ctx.single_shot:
+ return res
+ sub = single_shot_ctx(ctx)
+ if sub is None:
+ return res
+ ctx.events.emit("generate.storm_fallback", label=task.label, sessions=res.sessions, notes=res.notes[:300])
+ again = generate(sub.ws, agent_id=sub.agent_id, task=task, agent=sub.agent, model=sub.model, settings=sub.settings,
+ budget=sub.budget, events=sub.events,
+ max_turns=sub.policy.agent_max_turns, wrapup_turns=sub.policy.agent_wrapup_turns)
+ again.usage = res.usage + again.usage
+ again.notes = f"agent session died in a 503 storm ({res.notes[:120]}) → single-shot: {again.notes}"
+ return again
+
+
# ----------------------------------------------------------------------------- prompt helpers
def load_prompt_or(rel: str, fallback: str) -> str:
"""``prompts/<rel>`` if it exists, else ``fallback``."""
diff --git a/harness/codeverse/tracks/generation.py b/harness/codeverse/tracks/generation.py
index f77488ec..193d09a6 100644
--- a/harness/codeverse/tracks/generation.py
+++ b/harness/codeverse/tracks/generation.py
@@ -340,6 +340,12 @@ class GenerationResult(BaseModel):
turn_capped: bool = Field(
default=False, description="a session hit the turn budget and was wrapped up"
)
+ storm: bool = Field(
+ default=False,
+ description="every session died on a transient-failure streak (AgentResult.transient) and "
+ "nothing was written — the agent route is down, not the task; tracks fall back to the "
+ "hedged single-shot path (tracks.common.generate_for)",
+ )
# ----------------------------------------------------------------------------- strategies
@@ -655,6 +661,7 @@ def run_agent_task(
turns=acc.turns,
sessions=acc.sessions,
turn_capped=acc.wrapped,
+ storm=not changes and bool(res.transient),
)
diff --git a/harness/codeverse/tracks/repair.py b/harness/codeverse/tracks/repair.py
index bd397dd9..a2cb9c56 100644
--- a/harness/codeverse/tracks/repair.py
+++ b/harness/codeverse/tracks/repair.py
@@ -20,8 +20,8 @@ from codeverse.contracts.artifacts import BuildResult, GateReport
from codeverse.contracts.common import Usage
from codeverse.prompts import render
from codeverse.tracks import skills_hook
-from codeverse.tracks.common import RunContext
-from codeverse.tracks.generation import GenerationResult, GenerationTask, generate
+from codeverse.tracks.common import RunContext, generate_for
+from codeverse.tracks.generation import GenerationResult, GenerationTask
from codeverse.tracks.prompting import base_prompt_context, language_system_prompt
log = logging.getLogger(__name__)
@@ -203,9 +203,7 @@ def build_with_repair(ctx: RunContext, *, round_index: int, label: str, files_hi
task = make_repair_task(ctx, build, lint, round_index=round_index, attempt=attempt, repeats=repeats,
label=f"{label}_repair{attempt}", files_hint=files_hint, timeout_s=timeout_s)
ctx.events.emit("repair.attempt", round=round_index, attempt=attempt, repeats=repeats, signature=sig[:200])
- res = generate(ctx.ws, agent_id=ctx.agent_id, task=task, agent=ctx.agent, model=ctx.model,
- settings=ctx.settings, budget=ctx.budget, events=ctx.events,
- max_turns=ctx.policy.agent_max_turns, wrapup_turns=ctx.policy.agent_wrapup_turns)
+ res = generate_for(ctx, task)
outcome.attempts.append(res)
outcome.usage = outcome.usage + res.usage
if not res.ok:
diff --git a/harness/codeverse/tracks/scene.py b/harness/codeverse/tracks/scene.py
index 65502963..ce4ae074 100644
--- a/harness/codeverse/tracks/scene.py
+++ b/harness/codeverse/tracks/scene.py
@@ -46,8 +46,8 @@ from codeverse.spatial.frame_motion import motion_text_for
from codeverse.spatial.render_scene import JUDGE_MAX_VIEWS, perf_detail
from codeverse.texturing.plan import texture_pack_prompt
from codeverse.tracks import skills_hook
-from codeverse.tracks.common import RunContext, ServiceUnavailable
-from codeverse.tracks.generation import GenerationResult, GenerationTask, generate
+from codeverse.tracks.common import RunContext, ServiceUnavailable, generate_for, single_shot_ctx
+from codeverse.tracks.generation import GenerationResult, GenerationTask
from codeverse.tracks.lifecycle import BaseTrack
from codeverse.tracks.prompting import (
SCENE_FILES,
@@ -66,7 +66,6 @@ from codeverse.tracks.scene_assets import (
read_dedupe_note,
run_asset_stage,
select_assets,
- single_shot_ctx,
)
from codeverse.tracks.zone_layout import layout_block, layout_zones
from codeverse.workspace import Workspace
@@ -346,9 +345,7 @@ class SceneTrack(BaseTrack):
temperature=0.5, timeout_s=ctx.budget.timeout_s(ENV_TIMEOUT_S, floor_s=120),
images=reference_images(ctx))
task = self._deliver_skills(gen, "env", [task])[0]
- res = generate(ctx.ws, agent_id=gen.agent_id, task=task, agent=gen.agent, model=gen.model, settings=ctx.settings,
- budget=ctx.budget, events=ctx.events,
- max_turns=ctx.policy.agent_max_turns, wrapup_turns=ctx.policy.agent_wrapup_turns)
+ res = generate_for(gen, task)
self._record_skills(gen, "env")
ctx.ws.commit("env")
return {"ok": res.ok, "files": [c.path for c in res.files_changed], "notes": res.notes}
@@ -368,9 +365,7 @@ class SceneTrack(BaseTrack):
def _one(batch: list[ZonePlan]) -> GenerationResult:
task = skills_hook.with_inlined_skill(zone_gen, [self._zone_task(zone_gen, batch)])[0]
- return generate(ctx.ws, agent_id=zone_gen.agent_id, task=task, agent=zone_gen.agent, model=zone_gen.model, settings=ctx.settings,
- budget=ctx.budget, events=ctx.events,
- max_turns=ctx.policy.agent_max_turns, wrapup_turns=ctx.policy.agent_wrapup_turns)
+ return generate_for(zone_gen, task)
results = fan_out(batches, _one, max_workers=ctx.settings.limits.max_parallel_agents, label="zones",
item_name=lambda b: "+".join(z.name for z in b))
@@ -442,9 +437,7 @@ class SceneTrack(BaseTrack):
task = GenerationTask(label="compose", prompt=prompt, system=self.system_prompt(ctx), files_hint=["src/scene.js"], round=0,
kind="compose", temperature=0.4, owns_entry=True, images=reference_images(ctx))
task = self._deliver_skills(ctx, "compose", [task])[0]
- res = generate(ctx.ws, agent_id=ctx.agent_id, task=task, agent=ctx.agent, model=ctx.model, settings=ctx.settings,
- budget=ctx.budget, events=ctx.events,
- max_turns=ctx.policy.agent_max_turns, wrapup_turns=ctx.policy.agent_wrapup_turns)
+ res = generate_for(ctx, task)
self._record_skills(ctx, "compose")
ctx.ws.commit("compose")
return {"ok": res.ok, "deterministic": False, "files": [c.path for c in res.files_changed], "notes": res.notes}
diff --git a/harness/codeverse/tracks/scene_assets.py b/harness/codeverse/tracks/scene_assets.py
index bf5609be..7dc44d63 100644
--- a/harness/codeverse/tracks/scene_assets.py
+++ b/harness/codeverse/tracks/scene_assets.py
@@ -38,8 +38,14 @@ from codeverse.conventions import OBJECT_VIEWS_QUICK, to_pascal, to_snake
from codeverse.judges.rubrics import is_degraded
from codeverse.proc import fan_out, write_json_atomic, write_text_atomic
from codeverse.prompts import render
-from codeverse.tracks.common import RunContext, language_contract, load_prompt_or
-from codeverse.tracks.generation import SINGLE_SHOT_PREFIX, GenerationTask, generate, is_single_shot
+from codeverse.tracks.common import (
+ RunContext,
+ generate_for,
+ language_contract,
+ load_prompt_or,
+ single_shot_ctx,
+)
+from codeverse.tracks.generation import GenerationTask, is_single_shot
from codeverse.tracks.planner import plan as run_planner
from codeverse.tracks.prompting import (
base_prompt_context,
@@ -337,9 +343,7 @@ def _generate_asset(ctx: RunContext, asset: AssetPlan, rel: str, *, language: La
# threejs assets share the scene workspace (a stray write would hit
# zones/env); blender heroes own their whole sub-workspace
edit_only=language is Language.SCENE_THREEJS)
- return generate(ctx.ws, agent_id=ctx.agent_id, task=task, agent=ctx.agent, model=ctx.model, settings=ctx.settings,
- budget=ctx.budget, events=ctx.events,
- max_turns=ctx.policy.agent_max_turns, wrapup_turns=ctx.policy.agent_wrapup_turns)
+ return generate_for(ctx, task)
def _inline(ctx: RunContext, files: list[str]) -> str:
@@ -663,9 +667,7 @@ def _judge_and_fix(ctx: RunContext, asset: AssetPlan, result: AssetResult, rende
files_hint=files, round=1, kind="asset_fix", temperature=0.4,
edit_only=language is Language.SCENE_THREEJS, timeout_s=asset_timeout_s(ctx, 120))
restore = snapshot() if snapshot is not None else None # BEFORE the fix rewrites the files
- res = generate(gen.ws, agent_id=gen.agent_id, task=task, agent=gen.agent, model=gen.model, settings=gen.settings,
- budget=gen.budget, events=gen.events,
- max_turns=gen.policy.agent_max_turns, wrapup_turns=gen.policy.agent_wrapup_turns)
+ res = generate_for(gen, task)
if not res.ok:
return result
result.fixed = after_fix(gen) if after_fix is not None else True
@@ -771,44 +773,6 @@ def _read(ws: Workspace, rel: str, limit: int = 30_000) -> str:
ASSET_MAX_TRIS = 15_000
-# ----------------------------------------------------------------------------- strategy
-def single_shot_agent_id(agent_id: str, chat_model_id: str = "") -> str:
- """The single-shot strategy id for this run, or "" when there is no chat model.
-
- Single-shot is ONE api call that returns the asset file — the cheap path the asset
- stage tries before escalating to a full agent session. It needs a chat model, and
- the coding agent is always a vendor CLI (which exposes none), so the model comes
- from ``chat_model_id`` — the run's planner backend, which is always an API model.
- Until 2026-08-28 it was derived from the in-process ``api-agent`` generator id;
- that backend is gone."""
- if is_single_shot(agent_id):
- return agent_id
- # a SHAPE check only, deliberately not models.registry.parse_model_id: whether the
- # id resolves is the SERVICES' call (tests run fake providers like "fake:planner"),
- # and single_shot_ctx already degrades to the agent path when chat_model() raises
- return SINGLE_SHOT_PREFIX + chat_model_id if ":" in chat_model_id else ""
-
-
-def single_shot_ctx(ctx: RunContext) -> RunContext | None:
- """A copy of ``ctx`` bound to the single-shot strategy, or None when the
- generator has no usable chat model (CLI agents, tests with fake services)."""
- cached = ctx.extra.get("_single_shot_ctx")
- if cached is not None:
- return cached or None # False = known-unavailable
- sid = single_shot_agent_id(ctx.agent_id, ctx.spec.backends.planner)
- sub: RunContext | None = None
- if sid:
- try:
- model = ctx.model if is_single_shot(ctx.agent_id) else ctx.services.chat_model(sid[len(SINGLE_SHOT_PREFIX):])
- except Exception as e: # noqa: BLE001 — no chat model → keep the agent path
- log.info("single-shot generation unavailable for %s: %s", ctx.agent_id, e)
- model = None
- if model is not None:
- sub = replace(ctx, agent_id=sid, model=model, agent=None)
- ctx.extra["_single_shot_ctx"] = sub or False
- return sub
-
-
# ----------------------------------------------------------------------------- deterministic check
class AssetCheck(BaseModel):
"""Result of importing the asset module and calling its builder."""
diff --git a/harness/codeverse/tracks/steps.py b/harness/codeverse/tracks/steps.py
index 4a157bc9..6028f372 100644
--- a/harness/codeverse/tracks/steps.py
+++ b/harness/codeverse/tracks/steps.py
@@ -37,8 +37,8 @@ from codeverse.orchestrator import BudgetExceeded, usage_delta
from codeverse.proc import fan_out
from codeverse.spatial.render import RenderError
from codeverse.tracks import skills_hook
-from codeverse.tracks.common import RunContext
-from codeverse.tracks.generation import GenerationResult, GenerationTask, generate
+from codeverse.tracks.common import RunContext, generate_for
+from codeverse.tracks.generation import GenerationResult, GenerationTask
from codeverse.tracks.repair import RepairOutcome, build_with_repair
from codeverse.workspace import Workspace
@@ -149,9 +149,7 @@ def _run_phase(ctx: RunContext, tasks: Sequence[GenerationTask]) -> list[Generat
"""One parallel batch of generation tasks (the pre-phase behaviour, unchanged)."""
def _one(task: GenerationTask) -> GenerationResult:
- return generate(ctx.ws, agent_id=ctx.agent_id, task=task, agent=ctx.agent, model=ctx.model,
- settings=ctx.settings, budget=ctx.budget, events=ctx.events,
- max_turns=ctx.policy.agent_max_turns, wrapup_turns=ctx.policy.agent_wrapup_turns)
+ return generate_for(ctx, task)
# fan_out also for ONE task: a crashing generator (503 storm, parse error) becomes a failed
# result → RoundFailed (refine rounds treat that as a plateau) instead of killing the run.
diff --git a/harness/docs/DECISIONS.md b/harness/docs/DECISIONS.md
index b334a48e..08b2dda8 100644
--- a/harness/docs/DECISIONS.md
+++ b/harness/docs/DECISIONS.md
@@ -1070,6 +1070,24 @@ written) that were accepted because the code works that way and the tests pin it
add a second DirectionalLight on top of the rig the starter builds (D65) — the loop-9
lighthouse did exactly that.
+* **D68 A session that died in a 503 storm is retried through the single-shot path
+ (2026-09-07).** Loops 10–12 ran in an evening-long Gemini 503 storm: 23 of 24 gemini-cli
+ sessions (env, zones, escalated heroes, refines) ended `timeout / 0 turns / $0` after
+ 8–17 consecutive "Attempt N failed with status 503" inside the CLI's own retry loop —
+ gemini-cli never gives up before the harness's wall — while every single-shot call in the
+ same minutes got through (hedged across the 22 keys) and `codex:gpt-6-astra` sessions
+ completed. The stages that lost their session shipped the skeleton env and empty zones
+ and judged 0.00–0.14 (clockmaker: "the hero LongcaseClock is completely missing"). The
+ session result now says why it died (`AgentResult.transient`: timed out, nothing produced,
+ a transient streak in stderr), the generation result carries `storm`, and the ONE call
+ every stage makes — `tracks.common.generate_for(ctx, task)`, which replaced seven identical
+ eleven-argument `generate(...)` sites — retries a storm-dead task through
+ `single_shot_ctx`, the same degradation the soft budget already used. The owner's rule for
+ the storm (2026-09-07 evening): keep querying, keep retrying. Not a second agent session:
+ that is another 12 minutes at the wall while the storm lasts; the cheap path is the retry
+ that answers. `single_shot_agent_id` / `single_shot_ctx` moved from `scene_assets` to
+ `tracks.common` with it (they were never scene-specific).
+
## Rejected / deferred
* A versioned `Spec`/`RunRecord`/`RunState` load-normaliser (rejected 2026-08-30: of the seven
diff --git a/harness/docs/INTERFACES.md b/harness/docs/INTERFACES.md
index 9c56f01d..52eab04a 100644
--- a/harness/docs/INTERFACES.md
+++ b/harness/docs/INTERFACES.md
@@ -465,6 +465,12 @@ GenerationTask.phase: int = 0 # tasks run in parallel WITHIN a phase, phases i
# part group (`baseline_<parts>`, own files only), phase 1 = ONE `assemble` session that owns the
# entry file and the placement gates. Every other caller is phase 0, i.e. unchanged.
generate(ws, *, agent_id, task, ..., budget=BudgetGuard, max_turns=0, wrapup_turns=6) -> GenerationResult
+ # GenerationResult.storm: every session died on a transient streak (AgentResult.transient) and wrote nothing
+tracks.common.generate_for(ctx: RunContext, task: GenerationTask) -> GenerationResult
+ # THE call every stage makes (env, zones, compose, rounds, repairs, asset ladder, judged fix): generate()
+ # with everything ctx knows, and a storm-dead task retried through single_shot_ctx(ctx) (D68)
+tracks.common.single_shot_agent_id(agent_id, chat_model_id='') -> str · single_shot_ctx(ctx) -> RunContext | None
+ # moved from tracks.scene_assets 2026-09-07 (never scene-specific)
# GenerationResult adds turns / sessions / turn_capped. A turn cap is applied ONLY if a caller
# asks: task.max_turns > max_turns > $CV3D_AGENT_MAX_TURNS > settings.limits.agent_max_turns >
# DEFAULT_AGENT_MAX_TURNS (0 = leave AgentJob.max_turns at the backend's own default — a 28-turn
diff --git a/harness/tests/agents/test_gemini_cli.py b/harness/tests/agents/test_gemini_cli.py
index f794169c..9ecb99a8 100644
--- a/harness/tests/agents/test_gemini_cli.py
+++ b/harness/tests/agents/test_gemini_cli.py
@@ -33,6 +33,10 @@ if mode in ("fail_once", "fail_once_503", "fail_always", "fail_twice_429"):
sys.exit(1)
if mode == "hang":
time.sleep(60)
+if mode == "storm": # what gemini-cli 0.53 prints, 8-17 times per session, in a 503 storm
+ for i in range(4):
+ print(f"Attempt {i + 1} failed with status 503. Retrying with backoff... _ApiError: UNAVAILABLE", file=sys.stderr)
+ time.sleep(60)
served = "gemini-9-pro" if mode == "substitute" else model
os.makedirs("src", exist_ok=True)
open("src/hello.txt", "w").write(prompt[:20])
@@ -193,6 +197,19 @@ def test_timeout_is_reported(tmp_ws: Workspace, agent: GeminiCliAgent, monkeypat
monkeypatch.setattr("codeverse.agents.cli_common.IDLE_GRACE_S", 1.0)
res = agent.run(_job(tmp_ws, timeout_s=1))
assert not res.ok and res.exit_reason == "timeout" and res.duration_s < 30
+ assert res.transient is False # a plain hang is not a storm
+
+
+def test_a_timeout_after_a_503_streak_is_marked_transient(tmp_ws: Workspace, agent: GeminiCliAgent, monkeypatch):
+ """Measured 2026-09-07: 23 of 24 gemini-cli sessions of an evening ended timeout / 0 turns
+ / $0 after 8-17 consecutive 503s inside the CLI's own retry loop. The wall still ends the
+ session (the CLI never gives up first), but the result says WHY, so a track can fall back
+ to the single-shot path instead of shipping the skeleton (tracks.common.generate_for)."""
+ monkeypatch.setenv("FAKE_MODE", "storm")
+ monkeypatch.setattr("codeverse.agents.cli_common.IDLE_GRACE_S", 1.0)
+ res = agent.run(_job(tmp_ws, timeout_s=1))
+ assert not res.ok and res.exit_reason == "timeout" and res.transient is True
+ assert any("4 x 503" in e for e in res.errors), res.errors
def test_unavailable_when_no_keys(tmp_ws: Workspace, agent: GeminiCliAgent, monkeypatch):
diff --git a/harness/tests/orchestrator_tracks/test_cost_latency.py b/harness/tests/orchestrator_tracks/test_cost_latency.py
index 0466bea6..1a72bff9 100644
--- a/harness/tests/orchestrator_tracks/test_cost_latency.py
+++ b/harness/tests/orchestrator_tracks/test_cost_latency.py
@@ -8,12 +8,14 @@ import shutil
import pytest
from codeverse.config import get_settings
+from codeverse.contracts.agent import AgentResult
from codeverse.contracts.common import Budget, Language, Track
from codeverse.contracts.plan import AssetPlan, ScenePlan, ZonePlan
from codeverse.contracts.run import RunStatus
from codeverse.orchestrator import BudgetExceeded, BudgetGuard, RoundPolicy, RunState
from codeverse.proc import EventLog
-from codeverse.tracks.common import RunContext
+from codeverse.tracks.common import RunContext, generate_for, single_shot_agent_id, single_shot_ctx
+from codeverse.tracks.generation import GenerationTask
from codeverse.tracks.planner import plan_example
from codeverse.tracks.scene import SceneTrack, plan_zone_batches, zone_file
from codeverse.tracks.scene_assets import (
@@ -22,8 +24,6 @@ from codeverse.tracks.scene_assets import (
dedupe_assets,
run_asset_stage,
select_assets,
- single_shot_agent_id,
- single_shot_ctx,
variant_index,
)
from codeverse.workspace import Workspace
@@ -264,6 +264,45 @@ def test_single_shot_ctx_is_none_for_cli_backends(tmp_path, settings):
assert single_shot_ctx(ctx2) is None # FakeServices has no chat model → agent path stays
+class _StormAgent(FakeAgent):
+ """A CLI session that died at the wall after a 503 streak: nothing written, transient."""
+
+ def run(self, job):
+ self.jobs.append(job)
+ return AgentResult(ok=False, exit_reason="timeout", transient=True,
+ errors=["killed by watchdog (hard_timeout) after 720s", "11 x 503 inside the CLI's own retry loop before the wall; nothing produced"])
+
+
+def test_a_storm_dead_session_falls_back_to_single_shot(tmp_path, settings):
+ """Loops 10-11 (2026-09-07): 23 of 24 gemini-cli sessions ended timeout / 0 turns / $0 in
+ a 503 storm while every single-shot got through; env and zones shipped the skeleton and
+ judged 0.00-0.14. The same task goes once more through the hedged single-shot path."""
+ def respond(req):
+ return _envelope("src/env.js", "export function buildEnv() { return {}; }\n")
+
+ services = _ChatServices(FakeChatModel(respond))
+ agent = _StormAgent(lambda j, w: None)
+ ctx = _scene_ctx(tmp_path, settings, services=services, agent=agent, agent_id="fake-agent:gemini:x")
+ task = GenerationTask(label="env", prompt="write `src/env.js`", files_hint=["src/env.js"], round=0, kind="env")
+ res = generate_for(ctx, task)
+ assert res.ok and [c.path for c in res.files_changed] == ["src/env.js"]
+ assert len(agent.jobs) == 1 and services.chat_ids == [ctx.spec.backends.planner] # the planner backend, always an API model
+ assert "503 storm" in res.notes and "single-shot" in res.notes
+ kinds = [e["event"] for e in _events(ctx)]
+ assert "generate.storm_fallback" in kinds
+
+
+def test_a_plain_failed_session_does_not_fall_back(tmp_path, settings):
+ """An agent that ran and produced nothing (not a storm) keeps its own verdict: the
+ fallback is for the transport being down, not for the task being hard."""
+ services = _ChatServices(FakeChatModel(lambda req: _envelope("src/env.js", "x")))
+ agent = FakeAgent(lambda j, w: None)
+ ctx = _scene_ctx(tmp_path, settings, services=services, agent=agent, agent_id="fake-agent:gemini:x")
+ task = GenerationTask(label="env", prompt="write `src/env.js`", files_hint=["src/env.js"], round=0, kind="env")
+ res = generate_for(ctx, task)
+ assert not res.ok and not res.storm and services.chat_ids == []
+
+
# ----------------------------------------------------------------------------- salvage
def _threejs_scene_plan() -> ScenePlan:
"""The scene example narrowed to its three.js assets — the only kind these fakes build."""
diff --git a/harness/tests/orchestrator_tracks/test_generation_depth.py b/harness/tests/orchestrator_tracks/test_generation_depth.py
index e0c77a6f..363118e9 100644
--- a/harness/tests/orchestrator_tracks/test_generation_depth.py
+++ b/harness/tests/orchestrator_tracks/test_generation_depth.py
@@ -284,15 +284,15 @@ def test_phases_run_in_order_and_a_failed_phase_does_not_kill_the_round(tmp_path
order.append(task.label)
return GenerationResult(ok=task.label != "b", label=task.label, notes="")
- import codeverse.tracks.steps as steps
- orig = steps.generate
- steps.generate = fake_generate
+ import codeverse.tracks.common as common # the one seam every stage generates through
+ orig = common.generate
+ common.generate = fake_generate
try:
tasks = [GenerationTask(label="a", prompt="p", phase=0), GenerationTask(label="b", prompt="p", phase=0),
GenerationTask(label="z", prompt="p", phase=1)]
out = run_generation_tasks(ctx, tasks)
finally:
- steps.generate = orig
+ common.generate = orig
assert order.index("z") == 2 # phase 1 ran last
assert [r.label for r in out] == ["a", "b", "z"] # results keep the caller's order
assert [r.ok for r in out] == [True, False, True]
diff --git a/harness/tests/orchestrator_tracks/test_harness_failure_no_repair.py b/harness/tests/orchestrator_tracks/test_harness_failure_no_repair.py
index 081cd733..f6a4f0fe 100644
--- a/harness/tests/orchestrator_tracks/test_harness_failure_no_repair.py
+++ b/harness/tests/orchestrator_tracks/test_harness_failure_no_repair.py
@@ -18,6 +18,7 @@ from types import SimpleNamespace
import pytest
from codeverse.contracts.artifacts import BuildResult, GateReport
+from codeverse.tracks import common as C
from codeverse.tracks import repair as R
@@ -46,7 +47,7 @@ def no_generate(monkeypatch):
calls.append("generate")
raise AssertionError("the agent must not be asked to repair a harness failure")
- monkeypatch.setattr(R, "generate", boom)
+ monkeypatch.setattr(C, "generate", boom)
return calls
@@ -93,9 +94,9 @@ def test_a_real_build_failure_is_still_repaired(monkeypatch):
def fake_generate(ws, **kw):
seen.append(1)
- return SimpleNamespace(ok=True, files_changed=[], usage=Usage(), notes="")
+ return SimpleNamespace(ok=True, files_changed=[], usage=Usage(), notes="", storm=False)
- monkeypatch.setattr(R, "generate", fake_generate)
+ monkeypatch.setattr(C, "generate", fake_generate)
monkeypatch.setattr(R, "make_repair_task", lambda *a, **k: SimpleNamespace(label="t"))
seq = [(_build(ok=False, msg="TypeError: x is not a function"), GateReport(gate="lint", passed=True, findings=[])),
(_build(ok=True), GateReport(gate="lint", passed=True, findings=[]))]
diff --git a/harness/tests/orchestrator_tracks/test_write_scope.py b/harness/tests/orchestrator_tracks/test_write_scope.py
index d79b9123..55be7933 100644
--- a/harness/tests/orchestrator_tracks/test_write_scope.py
+++ b/harness/tests/orchestrator_tracks/test_write_scope.py
@@ -225,6 +225,7 @@ def test_scene_constructors_enforce_their_file_sets(tmp_path, settings, monkeypa
assert track._refine_task(ctx, group, last, 1, parallel=False).edit_only is False
# threejs asset tasks share the scene workspace → scoped; blender heroes own a sub-workspace
+ import codeverse.tracks.common as common
import codeverse.tracks.scene_assets as scene_assets
captured: dict[str, GenerationTask] = {}
@@ -233,7 +234,7 @@ def test_scene_constructors_enforce_their_file_sets(tmp_path, settings, monkeypa
captured["task"] = kw["task"]
return GenerationResult(ok=True, label=kw["task"].label)
- monkeypatch.setattr(scene_assets, "generate", fake_generate)
+ monkeypatch.setattr(common, "generate", fake_generate) # the one seam: tracks.common.generate_for
assert plan.assets, "plan_example(SCENE) is expected to plan assets"
asset = plan.assets[0]
scene_assets._generate_asset(ctx, asset, "src/assets/koi.js", language=Language.SCENE_THREEJS, attempt=0)