Skip to content

Official Codebase of 3DCodeVerse

Python

cleanup: tracks/orchestrator/languages/agents — knobs nobody turns, the single-file bpy skeleton, three ladder/pool/stop bugs

gaoypengc136c20280df
diff --git a/harness/codeverse/agents/backends.py b/harness/codeverse/agents/backends.py
index f10f49ca..2060a542 100644
--- a/harness/codeverse/agents/backends.py
+++ b/harness/codeverse/agents/backends.py
@@ -31,7 +31,6 @@ from codeverse.agents.cli_common import (
Session,
begin_session,
deliver_prompt,
- estimate_cost_safe,
exists_on_path,
failed,
find_json_object,
@@ -49,6 +48,7 @@ from codeverse.agents.materialize import MCP_SERVER_NAME, MCP_TOOL_TIMEOUT_MS, c
from codeverse.config import get_settings
from codeverse.contracts.agent import AgentJob, AgentResult
from codeverse.contracts.common import Usage
+from codeverse.models.pricing import estimate_cost
from codeverse.models.retry import KeyPool, KeyPoolExhausted
from codeverse.proc import run_subprocess
@@ -128,9 +128,9 @@ def usage_from_stats(stats: dict[str, Any], model: str) -> Usage:
cost = 0.0
for name, m in (stats.get("models") or {}).items():
part = _model_usage((m or {}).get("tokens") or {}, str(name))
- c = estimate_cost_safe("gemini", str(name), part)
+ c = estimate_cost("gemini", str(name), part)
if c == 0.0 and name != model and (part.input_tokens or part.output_tokens):
- c = estimate_cost_safe("gemini", model, part) # utility model missing from the price table
+ c = estimate_cost("gemini", model, part) # utility model missing from the price table
cost += c
u.input_tokens += part.input_tokens
u.output_tokens += part.output_tokens
@@ -234,12 +234,16 @@ class GeminiCliAgent(_CliAgent):
while True:
attempts += 1
used.add(key)
- proc = invoke(s, self.build_argv(prompt), self.build_env(s, key), prompt=prompt, attempt=attempts,
- soft_timeout_s=next_soft, model=self.model, key_tail=key[-4:])
- outcome = self._interpret(s, proc)
- usage_total = usage_total + outcome["usage"]
- pool.report(key, "429" if outcome["quota"] else ("ok" if outcome["ok"] else "5xx"))
- pool.release() # one acquire per attempt: keep the pool's in-flight gauge honest
+ try:
+ proc = invoke(s, self.build_argv(prompt), self.build_env(s, key), prompt=prompt, attempt=attempts,
+ soft_timeout_s=next_soft, model=self.model, key_tail=key[-4:])
+ outcome = self._interpret(s, proc)
+ usage_total = usage_total + outcome["usage"]
+ pool.report(key, "429" if outcome["quota"] else ("ok" if outcome["ok"] else "5xx"))
+ finally:
+ # one acquire per attempt: keep the pool's in-flight gauge honest — also when
+ # invoke raises, or one slot of the PROCESS-WIDE pool is gone for good
+ pool.release()
# a per-minute 429 on the single key a CLI process holds clears within the
# rotated retry's wait; ab_fewer_turns (2026-08-29) lost 4 of 10 cells to
# two 429s in a row, so a quota failure gets one more rotated attempt
@@ -533,7 +537,7 @@ class CodexEvents:
cached_tokens=self.usage_raw["cached_input_tokens"], tool_calls=self.tool_calls,
thoughts_tokens=reasoning,
)
- u.cost_usd = estimate_cost_safe("openai", model, u)
+ u.cost_usd = estimate_cost("openai", model, u)
return u
diff --git a/harness/codeverse/agents/cli_common.py b/harness/codeverse/agents/cli_common.py
index 1dc9e490..9409572a 100644
--- a/harness/codeverse/agents/cli_common.py
+++ b/harness/codeverse/agents/cli_common.py
@@ -23,7 +23,6 @@ from pydantic import BaseModel
from codeverse.contracts.agent import AgentJob, AgentResult, FileChange
from codeverse.contracts.common import Usage, is_harness_owned
-from codeverse.models.pricing import estimate_cost
from codeverse.proc import ManagedProcess, append_jsonl_line, read_jsonl_lenient, scrub_secrets
from codeverse.workspace import Workspace
@@ -686,16 +685,6 @@ def find_json_object(stdout: str, accept: Callable[[dict[str, Any]], bool]) -> d
return None
-# --------------------------------------------------------------------------- pricing
-def estimate_cost_safe(provider: str, model: str, usage: Usage) -> float:
- """``estimate_cost`` that never raises: an unknown model prices as 0.0 + a warning."""
- try:
- return float(estimate_cost(provider, model, usage))
- except Exception as e: # unknown model in the price table must not sink the run
- log.warning("estimate_cost(%s, %s) failed: %s", provider, model, e)
- return 0.0
-
-
def exists_on_path(binary: str) -> bool:
from shutil import which
diff --git a/harness/codeverse/languages/blender/__init__.py b/harness/codeverse/languages/blender/__init__.py
index 919cceea..73bca7de 100644
--- a/harness/codeverse/languages/blender/__init__.py
+++ b/harness/codeverse/languages/blender/__init__.py
@@ -223,8 +223,6 @@ def _rules(c: _Collector, source: str, *, target: str, expect_names: bool, expec
out.append(_f(E, "shade_smooth(use_auto_smooth=...) was removed in 4.1", call.lineno, "use `bpy.ops.object.shade_smooth_by_angle(angle=0.523599)` or `bpy.ops.object.shade_auto_smooth()`"))
if name.endswith(".calc_normals"):
out.append(_f(E, "Mesh.calc_normals() was removed in 4.0 (normals are computed automatically)", call.lineno, "delete the call (use me.update() if needed)"))
- if name == "bpy.ops.wm.redraw_timer":
- pass
for target, line in c.attr_stores:
if target.endswith(".use_auto_smooth") or target.endswith(".auto_smooth_angle"):
out.append(_f(E, f"`{target}` was removed in Blender 4.1", line, "use bpy.ops.object.shade_smooth_by_angle(angle=...) or mark sharp edges; or just shade_smooth()"))
@@ -581,11 +579,11 @@ def _part_function(p: PartPlan) -> str:
return "\n".join(lines) + "\n"
-def _plan_header(plan: StaticPlan, *, multi_file: bool) -> str:
+def _plan_header(plan: StaticPlan) -> str:
ob = plan.overall_bbox
parts_doc = "\n".join(
f" - {to_pascal(p.name)}{'' if p.instances == 1 else f' x{p.instances}'}: {p.role}; bbox center {_fmt(p.bbox.center)} "
- f"extents {_fmt(p.bbox.extents)}" + (f" [{part_file_rel(p.name)}]" if multi_file else "")
+ f"extents {_fmt(p.bbox.extents)} [{part_file_rel(p.name)}]"
for p in plan.parts
)
accept = "\n".join(f" - [{a.id}] {a.text}" for a in plan.acceptance) or " (none listed)"
@@ -593,8 +591,6 @@ def _plan_header(plan: StaticPlan, *, multi_file: bool) -> str:
f" * LAYOUT: this file is the ENTRY. Each part lives in src/{PARTS_DIR}/<snake>.py and exports\n"
f" build_<snake>() -> bpy.types.Object; main() below imports and calls them in order.\n"
" Edit geometry in the part files; keep this file to imports + calls + self-check.\n"
- if multi_file else
- " * LAYOUT: single file (small object). One build_<snake>() per part, called from main().\n"
)
return f'''"""{plan.object_name} — Blender (bpy) model.
@@ -651,7 +647,7 @@ def model_file_source(plan: StaticPlan, *, ground_tol_m: float = 0.002) -> str:
imports = "\n".join(f"from {PARTS_DIR}.{to_snake(p.name)} import {build_fn_name(p.name)}" for p in plan.parts)
calls = "\n".join(f" {build_fn_name(p.name)}()" for p in plan.parts)
return (
- _plan_header(plan, multi_file=True)
+ _plan_header(plan)
+ "import bpy\nfrom mathutils import Vector\n\n"
+ imports + "\n\n"
+ selfcheck_source(ground_tol_m)
@@ -660,20 +656,7 @@ def model_file_source(plan: StaticPlan, *, ground_tol_m: float = 0.002) -> str:
)
-def blender_skeleton_source(plan: StaticPlan, *, ground_tol_m: float = 0.002) -> str:
- """The same model as ONE ``src/model.py`` (small objects may use a single file)."""
- body = "\n# ----------------------------------------------------------------------------- parts\n"
- for p in plan.parts:
- body += _constants(p) + "\n\n" + _part_function(p) + "\n\n"
- calls = "\n".join(f" {build_fn_name(p.name)}()" for p in plan.parts)
- return (
- _plan_header(plan, multi_file=False) + IMPORTS + HELPERS + body + selfcheck_source(ground_tol_m)
- + "\ndef main():\n" + calls + "\n _selfcheck()\n\n\nmain()\n"
- )
-
-
-def write_blender_skeleton(ws: Workspace, plan: StaticPlan, *, multi_file: bool = True,
- ground_tol_m: float = 0.002) -> list[Path]:
+def write_blender_skeleton(ws: Workspace, plan: StaticPlan, *, ground_tol_m: float = 0.002) -> list[Path]:
"""Write the starter files (overwrites) and return the written paths (entry first).
``ground_tol_m`` is the self-check's "stands on z=0" tolerance: 2 mm for an object
@@ -681,9 +664,6 @@ def write_blender_skeleton(ws: Workspace, plan: StaticPlan, *, multi_file: bool
the scene seats anyway (`lib/place.js seat`) and whose module twin is allowed 20 mm."""
ws.src.mkdir(parents=True, exist_ok=True)
entry = ws.src / "model.py"
- if not multi_file:
- entry.write_text(blender_skeleton_source(plan, ground_tol_m=ground_tol_m))
- return [entry]
entry.write_text(model_file_source(plan, ground_tol_m=ground_tol_m))
written = [entry]
for p in plan.parts:
@@ -741,9 +721,6 @@ class BlenderRuntime(RuntimeDocs):
(the tracks call this via ``getattr`` to fan out per-part refinement)."""
return part_file_rel(part_name)
- def part_file(self, ws: Workspace, part_name: str) -> Path:
- return ws.root / self.file_for_part(part_name)
-
def build_command(
self, ws: Workspace, *, stl: bool = True, blend: bool = False, seed: int = 0,
tri_limit: int = MAX_TRIS_OBJECT, rlimit_gb: float | None = None,
diff --git a/harness/codeverse/languages/cadquery/__init__.py b/harness/codeverse/languages/cadquery/__init__.py
index 41c4c230..54eee98a 100644
--- a/harness/codeverse/languages/cadquery/__init__.py
+++ b/harness/codeverse/languages/cadquery/__init__.py
@@ -23,7 +23,7 @@ from codeverse.languages._ast_lint import (
)
from codeverse.languages._common import MISSING_ENTRY, compose_build_result
from codeverse.languages._docs import RuntimeDocs
-from codeverse.languages.blender import finish_for, instance_centers
+from codeverse.languages.blender import _fmt, finish_for, instance_centers
from codeverse.proc import run_subprocess, scrub_secrets
from codeverse.workspace import Workspace
@@ -220,10 +220,6 @@ def lint_cadquery_file(path: Path, *, target: str = "src/model.py") -> GateRepor
# ===================================================================== skeleton
-def _fmt(v: tuple[float, float, float]) -> str:
- return "(" + ", ".join(f"{x:.3f}" for x in v) + ")"
-
-
def _part_function(p: PartPlan) -> str:
pascal, snake = to_pascal(p.name), to_snake(p.name)
mn, mx = p.bbox.min, p.bbox.max
diff --git a/harness/codeverse/orchestrator.py b/harness/codeverse/orchestrator.py
index 4ad8256a..4258bfc2 100644
--- a/harness/codeverse/orchestrator.py
+++ b/harness/codeverse/orchestrator.py
@@ -159,15 +159,13 @@ class StageRunner:
fn: Callable[[], T],
*,
inputs: Any = "",
- force: bool = False,
model: type[BaseModel] | None = None,
- list_of: type[BaseModel] | None = None,
) -> T:
"""Run ``fn`` unless a cached result for the same ``inputs`` exists."""
h = hash_inputs(inputs)
path = self.result_path(name)
prior = self.state.stages.get(name)
- if not force and prior is not None and prior.inputs_hash == h and path.is_file():
+ if prior is not None and prior.inputs_hash == h and path.is_file():
# A cached result that cannot be read back is a cache MISS, not a dead run.
# ``inputs_hash`` covers the INPUTS only, never the result model's schema, so a
# contract that gained a field invalidates nothing — and a clobbered file
@@ -177,7 +175,7 @@ class StageRunner:
# out (there is no flag to drop a cached stage). Re-run instead and
# overwrite the file — the same tolerance load_ledger and _read_jsonl apply.
try:
- result = _revive(json.loads(path.read_text()), model, list_of)
+ result = _revive(json.loads(path.read_text()), model)
except (OSError, ValueError) as e: # ValidationError is a ValueError
self.events.emit("stage.cache_invalid", stage=name, inputs_hash=h, path=str(path),
error=f"{type(e).__name__}: {e}")
@@ -208,12 +206,10 @@ class StageRunner:
return prior is not None and prior.inputs_hash == hash_inputs(inputs) and self.result_path(name).is_file()
-def _revive(data: dict[str, Any], model: type[BaseModel] | None, list_of: type[BaseModel] | None) -> Any:
+def _revive(data: dict[str, Any], model: type[BaseModel] | None) -> Any:
result = data.get("result")
if model is not None and isinstance(result, dict):
return model.model_validate(result)
- if list_of is not None and isinstance(result, list):
- return [list_of.model_validate(r) for r in result]
return result
@@ -489,7 +485,6 @@ class RoundPolicy:
plateau_window: int = 2 # consecutive scored rounds without min_delta gain
min_delta: float = 0.02
target: float = 0.8 # rubric pass threshold (overridden from the rubric when known)
- judge_on_gate_errors: bool = True # still judge when gates (not build) fail
max_refine_tasks: int = 6
max_instructions_per_task: int = 6 # lines handed to ONE generation task (grouped by target)
parallel_min_tasks: int = 2 # fan out only when >= this many file-disjoint groups
@@ -500,7 +495,6 @@ class RoundPolicy:
# ---- money stops (docs/COST.md §5); thresholds are multiples of the judge's measured σ
judge_model: str = "" # judge backend id → σ via judge_sigma(); "" = the default judge's σ
regression_sigma: float = 1.0 # a round below (best − this × σ) regressed: change shape or stop
- regression_allow_switch: bool = True # False = a regression stops the run outright
marginal_sigma: float = 1.5 # from marginal_from_round on, the last gain must beat this × σ
marginal_from_round: int = 3 # first refine round index the marginal-value test applies to
# ---- agent session shape. A 28-turn cap was TESTED AND REJECTED: a controlled
@@ -633,8 +627,6 @@ class StopPolicy:
drop = best_before - score
detail = (f"r{last.index:02d} scored {score:.3f} vs best {best_before:.3f} "
f"({-drop:+.3f}, judge σ {self.policy.sigma:.3f})")
- if not self.policy.regression_allow_switch:
- return StopDecision("regression", detail=detail)
if last.kind == REWRITE_KIND or self._regressions(history) > 1:
return StopDecision("regression", detail=detail + " — a changed strategy did not recover it")
return StopDecision("continue", strategy="switch", detail=detail) if switch else None
@@ -653,7 +645,7 @@ class StopPolicy:
if gain <= need:
return StopDecision("diminishing_returns",
detail=f"r{nxt:02d} not started: last gain {gain:+.3f} ≤ {need:.3f} "
- f"(1.5 × judge σ {self.policy.sigma:.3f})")
+ f"({self.policy.marginal_sigma:g} × judge σ {self.policy.sigma:.3f})")
if best is not None and best >= self.policy.target:
return StopDecision("diminishing_returns",
detail=f"r{nxt:02d} not started: best {best:.3f} already at target "
diff --git a/harness/codeverse/tracks/common.py b/harness/codeverse/tracks/common.py
index 7ab63ca5..004ec6ff 100644
--- a/harness/codeverse/tracks/common.py
+++ b/harness/codeverse/tracks/common.py
@@ -114,10 +114,6 @@ class Services:
"""``scene_frames`` gate from a scene RenderSet's metrics.json (missing metrics → passing empty report)."""
return _import("codeverse.spatial.frame_metrics", "frame_gate_from_renders")(renders)
- def select_judge_views(self, renders: RenderSet, max_n: int = 10) -> RenderSet:
- """The ≤ ``max_n`` scene views a judge should see."""
- return _import("codeverse.spatial.render_scene", "select_judge_views")(renders, max_n=max_n)
-
def silhouette(self, render_png: Path | str, reference_png: Path | str) -> dict[str, Any]:
"""Outline IoU of a render vs a reference image (``{iou, reliable, ...}``)."""
return _import("codeverse.spatial.silhouette", "compare_silhouette")(render_png, reference_png)
diff --git a/harness/codeverse/tracks/depth.py b/harness/codeverse/tracks/depth.py
index fd14c7dd..bf706c92 100644
--- a/harness/codeverse/tracks/depth.py
+++ b/harness/codeverse/tracks/depth.py
@@ -117,14 +117,9 @@ DEFAULT_PARTS_PER_SCOPE = 3
def scoped_generation_enabled(default: bool = True) -> bool:
"""``CV3D_SCOPED_PARTS=off|0|false`` turns per-part scoped baselines off (A/B, debugging)."""
- import os
-
- raw = os.environ.get("CV3D_SCOPED_PARTS", "").strip().lower()
- if raw in ("off", "0", "false", "no"):
- return False
- if raw in ("on", "1", "true", "yes"):
- return True
- return default
+ from codeverse.config import env_flag
+
+ return env_flag("CV3D_SCOPED_PARTS", default)
@dataclass(frozen=True)
diff --git a/harness/codeverse/tracks/generation.py b/harness/codeverse/tracks/generation.py
index ac072fba..3d843dc0 100644
--- a/harness/codeverse/tracks/generation.py
+++ b/harness/codeverse/tracks/generation.py
@@ -549,7 +549,6 @@ def run_agent_task(
settings: Any | None = None,
budget: Any | None = None,
events: Any | None = None,
- retry_silent_bail: bool = True,
max_turns: int = 0,
wrapup_turns: int = DEFAULT_WRAPUP_TURNS,
) -> GenerationResult:
@@ -620,8 +619,7 @@ def run_agent_task(
changes = res.files_changed or changes or _attributed_fallback(ws, task, before)
if (
- retry_silent_bail
- and not changes
+ not changes
and not acc.wrapped
and res.exit_reason not in ("timeout", "budget")
):
diff --git a/harness/codeverse/tracks/plan_checks.py b/harness/codeverse/tracks/plan_checks.py
index faff1530..333a1f17 100644
--- a/harness/codeverse/tracks/plan_checks.py
+++ b/harness/codeverse/tracks/plan_checks.py
@@ -52,10 +52,6 @@ class Box:
c, e = bbox.center, bbox.extents
return cls(tuple(c[i] - e[i] / 2 for i in range(3)), tuple(c[i] + e[i] / 2 for i in range(3))) # type: ignore[arg-type]
- @classmethod
- def of_points(cls, pts: list[Vec]) -> Box:
- return cls(tuple(min(p[i] for p in pts) for i in range(3)), tuple(max(p[i] for p in pts) for i in range(3))) # type: ignore[arg-type]
-
def corners(self) -> list[Vec]:
return [(x, y, z) for x in (self.lo[0], self.hi[0]) for y in (self.lo[1], self.hi[1]) for z in (self.lo[2], self.hi[2])]
diff --git a/harness/codeverse/tracks/planner.py b/harness/codeverse/tracks/planner.py
index f444231a..48ac3562 100644
--- a/harness/codeverse/tracks/planner.py
+++ b/harness/codeverse/tracks/planner.py
@@ -55,9 +55,9 @@ BRIEF_MAX_TOKENS = 65_536
# ----------------------------------------------------------------------------- brief
def brief_enabled(spec: Spec, *, default: bool = True) -> bool:
"""Whether to expand the brief for this spec (env switch wins; object tracks only)."""
- raw = (os.environ.get(BRIEF_ENV) or "").strip().lower()
- on = default if not raw else raw not in ("0", "off", "false", "no")
- return on and spec.track in BRIEF_TRACKS
+ from codeverse.config import env_flag # ONE flag vocabulary (review-3 S4)
+
+ return env_flag(BRIEF_ENV, default) and spec.track in BRIEF_TRACKS
def brief_cache_dir() -> Path:
@@ -888,7 +888,10 @@ def plan_with_usage[P: BaseModel](spec: Spec, model_id: str, plan_model: type[P]
if guard is not None:
# booked where it is paid: a later attempt that raises cannot erase this dollar
guard.add(resp.usage, stage="plan")
- raw = resp.parsed if resp.parsed is not None else _parse_json(resp.text)
+ try:
+ raw = resp.parsed if resp.parsed is not None else parse_json_lenient(resp.text)
+ except ValueError: # JsonParseError: nothing parsed — the re-ask below says so
+ raw = None
try:
if not isinstance(raw, dict):
raise ValueError(f"planner returned {type(raw).__name__}, expected a JSON object")
@@ -1016,23 +1019,6 @@ def build_user_prompt(spec: Spec, *, brief: Any | None = None, budget: PlanBudge
return "\n".join(lines)
-def _parse_json(text: str) -> Any:
- t = (text or "").strip()
- if t.startswith("```"):
- t = t.strip("`")
- t = t[t.find("\n") + 1:] if "\n" in t else t
- try:
- return json.loads(t)
- except json.JSONDecodeError:
- start, end = t.find("{"), t.rfind("}")
- if 0 <= start < end:
- try:
- return json.loads(t[start:end + 1])
- except json.JSONDecodeError:
- return None
- return None
-
-
# ----------------------------------------------------------------------------- deterministic acceptance
def add_acceptance_item(items: list[AcceptanceItem], prefix: str, text: str, how: str) -> None:
"""Append a framework-derived *must* acceptance item unless an item with the
diff --git a/harness/codeverse/tracks/scene.py b/harness/codeverse/tracks/scene.py
index e172073a..1adecf4a 100644
--- a/harness/codeverse/tracks/scene.py
+++ b/harness/codeverse/tracks/scene.py
@@ -39,11 +39,12 @@ from codeverse.contracts.common import TRACK_INFO, Track
from codeverse.contracts.plan import Plan, ScenePlan, ZonePlan
from codeverse.contracts.run import RoundRecord
from codeverse.conventions import to_snake
+from codeverse.judges.base import judged_subset
from codeverse.orchestrator import StageRunner, TaskGroup, compact_instructions
from codeverse.proc import fan_out
from codeverse.prompts import render
from codeverse.spatial.frame_motion import motion_text_for
-from codeverse.spatial.render_scene import JUDGE_MAX_VIEWS, perf_detail
+from codeverse.spatial.render_scene import perf_detail
from codeverse.texturing.plan import texture_pack_prompt
from codeverse.tracks import skills_hook
from codeverse.tracks.common import RunContext, ServiceUnavailable, generate_for, single_shot_ctx
@@ -158,12 +159,9 @@ class ScenePipeline:
return out
def judge_views(self, ctx: RunContext, renders: RenderSet) -> RenderSet:
- """The ≤ 10 views the judge sees (authored@t0 first); the full set stays on disk.
- Prefers the per-view ``judge`` flags stamped at render time; legacy render
- sets (no flags) fall back to ``select_judge_views``."""
- if any(v.judge is not None for v in renders.views):
- return renders.model_copy(update={"views": [v for v in renders.views if v.judge]})
- return ctx.services.select_judge_views(renders, max_n=JUDGE_MAX_VIEWS)
+ """The ≤ 10 views the judge sees (authored@t0 first); the full set stays on disk:
+ the per-view ``judge`` flags ``render_scene`` stamps on every set it returns."""
+ return judged_subset(renders) # type: ignore[return-value]
def plan_summary(self, ctx: RunContext) -> str:
plan: ScenePlan = ctx.plan # type: ignore[assignment]
diff --git a/harness/codeverse/tracks/scene_assets.py b/harness/codeverse/tracks/scene_assets.py
index 705b9dc4..2118cd0c 100644
--- a/harness/codeverse/tracks/scene_assets.py
+++ b/harness/codeverse/tracks/scene_assets.py
@@ -290,9 +290,12 @@ def _ladder(ctx: RunContext, asset: AssetPlan, rel: str, *, language: Language,
outage = False
if sub is not None:
for attempt in range(2): # first shot + ONE error-feedback repair
+ # after an OUTAGE the model never wrote a file: the second shot is the first shot
+ # again, not a "repair" of the skeleton stub against a 503 it cannot fix
+ feedback = repair_feedback(chk, rel) if chk is not None and not outage else ""
try:
res = _generate_asset(sub, asset, rel, language=language, attempt=attempt, files=files,
- feedback=repair_feedback(chk, rel) if chk is not None else "")
+ feedback=feedback)
except Exception as e: # noqa: BLE001 — a bad answer escalates; a dead model does not
from codeverse.orchestrator import BudgetExceeded
@@ -311,7 +314,7 @@ def _ladder(ctx: RunContext, asset: AssetPlan, rel: str, *, language: Language,
chk = (check(sub) if res.ok
else AssetCheck(ok=False, ran=True, fatal=True, errors=[f"no file was written ({res.notes or 'empty answer'})"]))
if chk.ok:
- strategy = "single-shot" if attempt == 0 else "single-shot+repair"
+ strategy = "single-shot+repair" if feedback else "single-shot"
break
if not strategy and not outage:
ctx.events.emit("asset.escalated", asset=asset.name, errors=(chk.errors[:3] if chk else []))
diff --git a/harness/codeverse/tracks/steps.py b/harness/codeverse/tracks/steps.py
index 36b49ed3..8959d3de 100644
--- a/harness/codeverse/tracks/steps.py
+++ b/harness/codeverse/tracks/steps.py
@@ -308,7 +308,7 @@ def _run_round(
tri_count=rec.measurement.tri_count if rec.measurement else None)
# a candidate's verdict is the ONLY thing that picks the code r00 starts from, so it is
# bought even past the ceiling — unlike a refine verdict, which could promote nothing
- skip = skip_judge_reason(ctx, gates=gates, renders=rec.renders, ignore_budget=kind == "candidate")
+ skip = skip_judge_reason(ctx, renders=rec.renders, ignore_budget=kind == "candidate")
if skip:
notes.append(f"judge skipped ({skip})")
ctx.events.emit("judge.skipped", round=index, reason=skip)
@@ -345,8 +345,7 @@ def _run_round(
# ----------------------------------------------------------------------------- cost of a round
-def skip_judge_reason(ctx: RunContext, *, gates: Sequence[GateReport], renders: RenderSet | None,
- ignore_budget: bool = False) -> str:
+def skip_judge_reason(ctx: RunContext, *, renders: RenderSet | None, ignore_budget: bool = False) -> str:
"""Why this round must NOT be judged (``""`` = judge it).
Only states in which the verdict is never bought at all — a skip that the
@@ -359,8 +358,6 @@ def skip_judge_reason(ctx: RunContext, *, gates: Sequence[GateReport], renders:
$0.09 in the audit were bought past the run's wall clock (docs/COST.md §5).
``ignore_budget`` exempts the best-of-N candidates: their verdict picks the
code r00 starts from, so skipping it wastes the N generations already paid for.
- * ``gate errors`` — only when the caller set ``judge_on_gate_errors=False``;
- ``rejudge_round`` honours that too, so the verdict is not re-bought.
A broken build never gets here (``run_round`` judges only when the build is
ok) and a round in which nothing changed never gets here either
@@ -371,16 +368,9 @@ def skip_judge_reason(ctx: RunContext, *, gates: Sequence[GateReport], renders:
return "no renders"
if not ctx.budget.ok() and not ignore_budget:
return "budget already exceeded"
- if judge_blocked_by_gates(ctx, gates):
- return "gate errors"
return ""
-def judge_blocked_by_gates(ctx: RunContext, gates: Sequence[GateReport]) -> bool:
- """The caller asked not to judge rounds with gate errors, and this round has some."""
- return bool(not ctx.policy.judge_on_gate_errors and any(g.errors for g in gates))
-
-
def emit_round_cost(
ctx: RunContext,
*,
@@ -526,16 +516,9 @@ def rejudge_round(ctx: RunContext, pipeline: RoundPipeline, rec: RoundRecord, pr
No regeneration, no rebuild: the same commit is judged again. On success the
round record is updated in place and re-persisted. Returns True when the
- round now has a usable judgment.
-
- A verdict :func:`skip_judge_reason` deliberately did not buy is NOT bought
- here — otherwise the skip only moves the same dollar one loop iteration later
- (the verifier's reproduction: three rounds "skipped", two of them re-judged on
- the next iteration and the last one left without a score)."""
+ round now has a usable judgment."""
if ctx.judge is None or rec.build is None or not rec.build.ok or rec.renders is None or not rec.renders.views:
return False
- if judge_blocked_by_gates(ctx, rec.gates):
- return False
ctx.events.emit("judge.retry", round=rec.index)
notes: list[str] = [rec.notes] if rec.notes else []
judgment, judge_usage = _judge(ctx, pipeline, rec.index, rec.build, list(rec.gates), rec, previous, notes)
@@ -575,10 +558,8 @@ def failed_acceptance(ctx: RunContext, judgment: Judgment | None) -> list[Accept
return [items[k] for k, ok in judgment.acceptance_results.items() if not ok and k in items]
-def sum_usage(rounds: Sequence[RoundRecord], *extra: Usage) -> Usage:
+def sum_usage(rounds: Sequence[RoundRecord]) -> Usage:
total = Usage()
for r in rounds:
total = total + r.usage
- for u in extra:
- total = total + u
return total
diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.md
index bee6113c..62851202 100644
--- a/harness/docs/ARCHITECTURE.md
+++ b/harness/docs/ARCHITECTURE.md
@@ -446,7 +446,7 @@ repeat while StopPolicy says continue (≤ max_rounds refine rounds, plateau_win
a cap a caller sets (CV3D_AGENT_MAX_TURNS / task; no profile sets one) still buys a wrap-up session
that lands a final build + summary instead of being killed) → build+repair (error-focused,
escalates on identical signatures) → gates → … → judge (SKIPPED only where the verdict is never
- bought at all: no judge/renders, budget already exceeded, or judge_on_gate_errors=False)
+ bought at all: no judge/renders or budget already exceeded)
BestSelector: highest score, tie → fewer gate errors; |Δ| < pairwise_margin (0.03) → position-swapped
PairwiseJudge decides (replace only at confidence ≥ 0.6; note persisted in rNN.json)
every round emits cost.round {stage → $, judge $, agent turns, wasted flag}; a round that raises mid-way
diff --git a/harness/docs/COST.md b/harness/docs/COST.md
index f71f9fe4..ef676da6 100644
--- a/harness/docs/COST.md
+++ b/harness/docs/COST.md
@@ -804,7 +804,7 @@ out of tree (no such test ships here):
| `no file change` | **removed** | unreachable: a generation result is `ok` only when a file changed, so `run_generation_tasks` raises `RoundFailed` (→ the loop's `no_change` plateau) before any judge question is asked |
| `build not repaired within the repair budget` | **removed** | a broken build never reaches the judge at all (`run_round` judges only when `build.ok`), so it only ever fired on a round that BUILT but still lint-failed — and the loop's `rejudge_round` then bought the same verdict one iteration later. Measured on the reproduction: 3 rounds "skipped", 2 verdicts re-bought as `judge.retry`, and the last round left **without a score — it was the best of the three (0.7)** and could not be promoted |
| `budget already exceeded` | **kept** | the loop's next `budget_ok` check ends the run, so the verdict is never bought later. Guard, not a measured saving: 0 of the 108 recorded rounds bought a verdict after the budget ended, so this branch has saved $0 so far — it is kept because the state is knowable in advance and the run is over either way (§5) |
-| `no judge` / `no renders` / `gate errors` | **kept** | pre-existing; the first two are guards (nothing to buy), the third only fires when a caller sets `judge_on_gate_errors=False` |
+| `no judge` / `no renders` | **kept** | pre-existing guards (nothing to buy). The third pre-existing reason, `gate errors` behind `judge_on_gate_errors=False`, went 2026-09-21: no caller ever set it |
Two changes were needed to make the kept branches real:
@@ -1196,7 +1196,7 @@ The harness had exactly one number and used it for both.
**Measured 2026-08-25**, `tsr_scn_temple_night`, `codex:gpt-5.6-sol`, `--profile
quality`. The two Blender hero-asset sessions were priced at OpenAI list rates
-(`agents/codex.py` → `estimate_cost_safe("openai", …)`) and the run crossed the profile's
+(`agents/codex.py` → `estimate_cost("openai", …)`) and the run crossed the profile's
soft cap 6.8 minutes in:
```
diff --git a/harness/docs/INTERFACES.md b/harness/docs/INTERFACES.md
index 1ef0e111..07beb254 100644
--- a/harness/docs/INTERFACES.md
+++ b/harness/docs/INTERFACES.md
@@ -31,7 +31,7 @@ from codeverse.contracts import ENTRY_FILE, code_file, LANGUAGE_LABEL # {Langu
from codeverse.contracts import RunOptions # Spec.options: candidates (int|None, ≥1), texture (bool)
GateFinding.as_line(with_gate=False, with_severity=False, with_target=False, with_hint=True) -> str
# "GATE <gate>: [<sev>] <message> [<target>] FIX: <hint>" — flags opt in; no leading "- "
-RenderView.judge: bool | None # stamped True/False at render time; None = legacy round (fall back to select_judge_views)
+RenderView.judge: bool | None # stamped True/False at render time; None = legacy round (every stored view is judged)
RenderSet.out_dir: str # directory the views (+ views.json/metrics.json) were written to ("" on old rounds)
from codeverse.config import get_settings, env_flag # env_flag(env, fallback) -> bool: on/off/1/0/true/false/yes/no;
# unset/empty -> fallback; garbage -> warning + fallback (Settings value), never a silent switch
@@ -391,10 +391,10 @@ rec = get_track(spec.track, **options).run(spec, ws, resume=False) -> RunRecord
TrackPipeline.run(spec, ws, *, resume=False, force=False) -> RunRecord
from codeverse.orchestrator import RoundPolicy, StopPolicy, StopDecision, BestSelector, judge_sigma, \
best_score, last_gain, best_index, REWRITE_KIND, build_refine_instructions, compact_instructions
-RoundPolicy(max_rounds=4, plateau_window=2, min_delta=0.02, target=0.8, judge_on_gate_errors=True, max_refine_tasks=6,
+RoundPolicy(max_rounds=4, plateau_window=2, min_delta=0.02, target=0.8, max_refine_tasks=6,
max_instructions_per_task=6, parallel_min_tasks=2, n_candidates=1, pairwise_margin=0.03,
pairwise_min_confidence=0.6, judge_samples=1,
- judge_model="", regression_sigma=1.0, regression_allow_switch=True, # money stops (docs/COST.md §5)
+ judge_model="", regression_sigma=1.0, # money stops (docs/COST.md §5)
marginal_sigma=1.5, marginal_from_round=3, # r03+ must beat 1.5σ
agent_max_turns=0, agent_wrapup_turns=6, # 0 = the backend's own AgentJob.max_turns (claude-code 60
# +6 wrap-up; gemini-cli/codex/agy no turn cap); a 28-turn cap was A/B'd and rejected (+$0.02, −0.21
@@ -488,12 +488,11 @@ tracks.common.single_shot_agent_id(agent_id, chat_model_id='') -> str · single_
# EVERY session (attempt 1, <label>.a2 retry, <label>.wrapup) is charged as it ends.
from codeverse.tracks.repair import build_with_repair # RepairOutcome(.ok/.repaired/.max_attempts, attempts, usage); build_with_repair(ctx, *, round_index, label, files_hint=None, max_attempts=None, timeout_s=None) — timeout_s clips every repair session (a scene asset's window)
# scene assets (tracks/scene_assets.py): build_threejs_asset / build_blender_asset climb ONE ladder (_ladder: single-shot → check → one feedback repair → agent session); a hero's parts come from the static planner (hero_plan); SceneThreeJsRuntime.render_asset(ws, name, out_dir) renders a module on the hero's quick rig; the assembled scene.js plays every GLB clone's clips (clone.userData.clipOffset de-phases a copy)
-# languages/blender.write_blender_skeleton(ws, plan, *, multi_file=True, ground_tol_m=0.002) — the self-check's stands-on-z=0 tolerance (a scene hero passes 0.02)
+# languages/blender.write_blender_skeleton(ws, plan, *, ground_tol_m=0.002) — the self-check's stands-on-z=0 tolerance (a scene hero passes 0.02)
from codeverse.tracks.steps import run_round, skip_judge_reason, emit_round_cost, record_aborted_round
-skip_judge_reason(ctx, *, gates, renders) -> str # "" = judge it. ONLY states where the verdict is
- # never bought at all: no judge / no renders / budget already exceeded / gate errors with
- # policy.judge_on_gate_errors=False. rejudge_round honours the last one too, so a policy skip is
- # never re-bought (docs/COST.md §17 — "no file change" and "build not repaired" were removed)
+skip_judge_reason(ctx, *, renders, ignore_budget=False) -> str # "" = judge it. ONLY states where the
+ # verdict is never bought at all: no judge / no renders / budget already exceeded
+ # (docs/COST.md §17 — "no file change" and "build not repaired" were removed)
run_round(ctx, *, index, kind, tasks, pipeline, ..., previous_best=None, render=None, geometry_views=True) -> RoundRecord
# render: RenderFn | None swaps the pipeline's render (candidates pass quick_render); geometry_views=False skips
# the clay/normals views
diff --git a/harness/tests/agents/test_gemini_cli.py b/harness/tests/agents/test_gemini_cli.py
index 9ecb99a8..6976c02f 100644
--- a/harness/tests/agents/test_gemini_cli.py
+++ b/harness/tests/agents/test_gemini_cli.py
@@ -192,6 +192,19 @@ def test_pool_exhausted_before_first_attempt_is_a_budget_result(tmp_ws: Workspac
assert not res.ok and res.exit_reason == "budget" and "exhausted" in res.errors[0]
+def test_a_raising_invoke_gives_its_pool_slot_back(tmp_ws: Workspace, agent: GeminiCliAgent, monkeypatch):
+ """The pool is process-wide: a slot leaked here is gone for every later session."""
+ from codeverse.agents import backends as gc
+
+ def boom(*a, **k):
+ raise OSError("cannot spawn")
+
+ monkeypatch.setattr(gc, "invoke", boom)
+ with pytest.raises(OSError, match="cannot spawn"):
+ agent.run(_job(tmp_ws))
+ assert gc._key_pool(get_settings().gemini_api_keys).stats()["in_flight"] == 0 # noqa: SLF001
+
+
def test_timeout_is_reported(tmp_ws: Workspace, agent: GeminiCliAgent, monkeypatch):
monkeypatch.setenv("FAKE_MODE", "hang")
monkeypatch.setattr("codeverse.agents.cli_common.IDLE_GRACE_S", 1.0)
diff --git a/harness/tests/blender_cadquery/test_blender_skeleton.py b/harness/tests/blender_cadquery/test_blender_skeleton.py
index bf13ef10..53c218f7 100644
--- a/harness/tests/blender_cadquery/test_blender_skeleton.py
+++ b/harness/tests/blender_cadquery/test_blender_skeleton.py
@@ -1,4 +1,4 @@
-"""Offline tests for the bpy skeleton generator (multi-file default + single-file variant)."""
+"""Offline tests for the bpy skeleton generator."""
from __future__ import annotations
@@ -6,7 +6,6 @@ import ast
from codeverse.contracts.plan import BBox
from codeverse.languages.blender import (
- blender_skeleton_source,
finish_for,
instance_centers,
lint_blender_source,
@@ -17,19 +16,6 @@ from codeverse.languages.blender import (
)
-def test_single_file_skeleton_parses_lints_and_names_parts(table_plan) -> None:
- src = blender_skeleton_source(table_plan)
- ast.parse(src)
- assert "import codeverse" not in src and "from codeverse" not in src and "from parts" not in src
- for fn in ("def build_table_top(", "def build_leg(", "def build_shelf(", "def main():", "def _selfcheck("):
- assert fn in src
- assert 'add_box("TableTop"' in src and 'add_box(f"Leg_{i}"' in src
- assert "add_empty" not in src and ".parent" not in src # instances stay TOP-LEVEL (measured as parts)
- assert "Z is up, -Y is the FRONT" in src and "[a1]" in src and "LEG_EXTENTS = (0.040, 0.040, 0.560)" in src
- r = lint_blender_source(src)
- assert r.passed, [f.message for f in r.findings]
-
-
def test_part_file_source_is_self_contained(table_plan) -> None:
leg = table_plan.parts[1]
src = part_file_source(leg)
@@ -68,12 +54,6 @@ def test_write_skeleton_multi_file_and_workspace_lint(tmp_ws, table_plan) -> Non
assert not [f for f in rep.findings if f.severity.value == "warn"], [f.message for f in rep.findings]
-def test_write_skeleton_single_file(tmp_ws, table_plan) -> None:
- paths = write_blender_skeleton(tmp_ws, table_plan, multi_file=False)
- assert paths == [tmp_ws.src / "model.py"] and paths[0].is_file()
- assert lint_workspace(tmp_ws).passed
-
-
def test_instance_centers_symmetry() -> None:
bb = BBox(center=(0.2, 0.3, 0.5), extents=(0.1, 0.1, 1.0))
assert instance_centers(bb, 1, "none") == [(0.2, 0.3, 0.5)]
diff --git a/harness/tests/orchestrator_tracks/test_cost_latency.py b/harness/tests/orchestrator_tracks/test_cost_latency.py
index 1a72bff9..1011e4f9 100644
--- a/harness/tests/orchestrator_tracks/test_cost_latency.py
+++ b/harness/tests/orchestrator_tracks/test_cost_latency.py
@@ -486,6 +486,29 @@ def test_a_model_outage_escalates_the_asset_instead_of_losing_it(tmp_path, setti
assert failed and "503" in failed[0]["error"]
+def test_the_shot_after_an_outage_is_a_plain_generation_not_a_repair(tmp_path, settings):
+ from codeverse.models.base import ModelError
+
+ plan = ScenePlan.model_validate(plan_example(Track.SCENE))
+ plan = plan.model_copy(update={"assets": [_asset("Bollard", (0.3, 0.5, 0.3))], "zones": []})
+ prompts: list[str] = []
+
+ def respond(req):
+ prompts.append(req.messages[0].text)
+ if len(prompts) == 1:
+ raise ModelError("Gemini API error 503: high demand", retryable=True, status=503)
+ return _envelope("src/assets/bollard.js", _module("Bollard", 0.3, 0.5, 0.3))
+
+ agent = FakeAgent(lambda job, ws: {"src/nope.js": "// should not run\n"})
+ services = _ChatServices(FakeChatModel(respond, cost=0.02), judge=FakeJudge(scores=(0.9,)))
+ ctx = _scene_ctx(tmp_path, settings, services=services, agent=agent, plan=plan)
+ results = run_asset_stage(ctx)
+ assert results["Bollard"].ok and results["Bollard"].strategy == "single-shot" and not agent.jobs
+ assert len(prompts) == 2 and prompts[1] == prompts[0], "the model wrote nothing: there is nothing to repair"
+ assert "did NOT pass" not in prompts[1] and "rewrite COMPLETELY" not in prompts[1]
+
+
def test_a_generation_session_never_outlives_the_wall_budget():
from codeverse.contracts.spec import Budget
from codeverse.orchestrator import BudgetGuard
diff --git a/harness/tests/orchestrator_tracks/test_fix_generation.py b/harness/tests/orchestrator_tracks/test_fix_generation.py
index 3e313d4b..5ab71932 100644
--- a/harness/tests/orchestrator_tracks/test_fix_generation.py
+++ b/harness/tests/orchestrator_tracks/test_fix_generation.py
@@ -131,5 +131,5 @@ def test_run_agent_task_that_wrote_nothing_is_not_ok(tmp_path):
(ws.root / "events.jsonl").write_text("{}\n") # harness noise only
agent = NonReportingAgent({})
task = GenerationTask(label="asset_stone_lantern", prompt="p", files_hint=["src/assets/stone_lantern.js"])
- res = run_agent_task(ws, agent=agent, task=task, retry_silent_bail=False)
+ res = run_agent_task(ws, agent=agent, task=task)
assert not res.ok and res.files_changed == []
diff --git a/harness/tests/orchestrator_tracks/test_integrations2.py b/harness/tests/orchestrator_tracks/test_integrations2.py
index e57fb981..196acf51 100644
--- a/harness/tests/orchestrator_tracks/test_integrations2.py
+++ b/harness/tests/orchestrator_tracks/test_integrations2.py
@@ -6,7 +6,6 @@ from __future__ import annotations
from types import SimpleNamespace
-from codeverse.contracts.artifacts import RenderSet, RenderView
from codeverse.contracts.common import Language, Track, Usage
from codeverse.contracts.plan import ScenePlan
from codeverse.contracts.run import RunStatus
@@ -16,7 +15,7 @@ from codeverse.tracks import get_track
from codeverse.tracks.graphics import GraphicsTrack
from codeverse.tracks.planner import plan_example
from codeverse.tracks.prompting import file_for_target_factory
-from codeverse.tracks.scene import JUDGE_MAX_VIEWS, ScenePipeline, SceneTrack
+from codeverse.tracks.scene import SceneTrack
from codeverse.tracks.static_object import StaticObjectTrack, expected_files
from codeverse.workspace import Workspace
from tests.orchestrator_tracks.conftest import make_spec
@@ -73,19 +72,6 @@ def test_scene_judge_gets_no_geometry_views(tmp_path, settings):
assert services.geometry_renders == []
-# --------------------------------------------------------------------- (a) judge view selection for scenes
-def test_scene_pipeline_selects_at_most_ten_judge_views():
- views = [RenderView(name=f"cam{i}", path=f"/tmp/x{i}_{t}.png", time_s=t)
- for i in range(7) for t in (0.0, 1.5)]
- rs = RenderSet(views=views, renderer="fake")
- ctx = SimpleNamespace(services=FakeServices())
- out = ScenePipeline().judge_views(ctx, rs)
- assert len(out.views) == 9 <= JUDGE_MAX_VIEWS == 10 # 7 authored at t=0 + the first two at t=1.5
- # authored views at t=0 all survive
- kept_t0 = [v.name for v in out.views if v.time_s == 0.0]
- assert set(kept_t0) >= {f"cam{i}" for i in range(7)}
-
-
# --------------------------------------------------------------------- (d) connectivity hints in the author's frame
def test_connectivity_gate_receives_spec_language(tmp_path, chair_plan, settings):
rec, ws, judge, services = _static_run(tmp_path, chair_plan, settings)
diff --git a/harness/tests/orchestrator_tracks/test_orchestrator.py b/harness/tests/orchestrator_tracks/test_orchestrator.py
index c5333623..b6ae8a7f 100644
--- a/harness/tests/orchestrator_tracks/test_orchestrator.py
+++ b/harness/tests/orchestrator_tracks/test_orchestrator.py
@@ -100,14 +100,12 @@ def test_stage_runner_caches_by_input_hash(tmp_ws):
assert len(calls) == 1
assert runner.stage("s1", fn, inputs={"a": 2}) == {"x": 1}
assert len(calls) == 2
- runner.stage("s1", fn, inputs={"a": 2}, force=True)
- assert len(calls) == 3
kinds = [e["event"] for e in events.read()]
- assert kinds.count("stage.cached") == 1 and kinds.count("stage.done") == 3
+ assert kinds.count("stage.cached") == 1 and kinds.count("stage.done") == 2
# resume from disk with a fresh runner + state
st = RunState.load(tmp_ws)
r2 = StageRunner(tmp_ws, events, st)
- assert r2.stage("s1", fn, inputs={"a": 2}) == {"x": 1} and len(calls) == 3
+ assert r2.stage("s1", fn, inputs={"a": 2}) == {"x": 1} and len(calls) == 2
assert runner.result_path("a:b/c").name == "a_b_c.json"
@@ -146,11 +144,8 @@ def test_an_unreadable_cached_stage_is_a_miss_not_a_dead_run(tmp_ws):
assert [e["event"] for e in events.read()].count("stage.cache_invalid") == 2
-def test_stage_runner_revives_models_and_raises(tmp_ws):
+def test_stage_runner_raises_and_records_nothing(tmp_ws):
runner = StageRunner(tmp_ws, EventLog(tmp_ws.events_path))
- r = runner.stage("rounds", lambda: [RoundRecord(index=0, kind="baseline")], inputs="x", list_of=RoundRecord)
- r2 = runner.stage("rounds", lambda: None, inputs="x", list_of=RoundRecord)
- assert isinstance(r2[0], RoundRecord) and r2[0].kind == r[0].kind
def boom():
raise RuntimeError("nope")
diff --git a/harness/tests/orchestrator_tracks/test_refactor_batch2.py b/harness/tests/orchestrator_tracks/test_refactor_batch2.py
index 379a2b8b..eb34d98b 100644
--- a/harness/tests/orchestrator_tracks/test_refactor_batch2.py
+++ b/harness/tests/orchestrator_tracks/test_refactor_batch2.py
@@ -121,10 +121,6 @@ def test_judge_view_flags_and_path_reconstruction(tmp_path):
rs = _rs([True, False, True])
out = pipe.judge_views(ctx, rs)
assert [v.name for v in out.views] == ["v0", "v2"]
- # legacy render set (no flags) falls back to select_judge_views (identity here: few views)
- legacy = _rs([None, None])
- out2 = pipe.judge_views(ctx, legacy)
- assert len(out2.views) == 2
rs = _rs([True, False, None])
sub = judged_subset(rs)
diff --git a/harness/tests/orchestrator_tracks/test_round_economics.py b/harness/tests/orchestrator_tracks/test_round_economics.py
index cad96852..29eefe0e 100644
--- a/harness/tests/orchestrator_tracks/test_round_economics.py
+++ b/harness/tests/orchestrator_tracks/test_round_economics.py
@@ -93,12 +93,6 @@ def test_a_sub_noise_dip_never_burns_the_strategy_switch():
assert not meaningful_regression(None, 0.60, pol) and not meaningful_regression(0.5, None, pol)
-def test_regression_can_be_configured_to_stop_outright():
- sp = StopPolicy(RoundPolicy(max_rounds=4, target=0.9, marginal_from_round=99, regression_allow_switch=False))
- d = sp.evaluate([_round(0, 0.60), _round(1, 0.52)])
- assert d.reason == "regression" and d.strategy == "same"
-
-
def test_a_noisy_judge_widens_the_regression_band():
flash = StopPolicy(RoundPolicy(max_rounds=4, target=0.9, marginal_from_round=99,
judge_model="gemini:gemini-3.7-flash"))
@@ -131,6 +125,11 @@ def test_the_marginal_round_is_configurable():
sp = StopPolicy(RoundPolicy(max_rounds=4, target=0.9, marginal_from_round=2,
judge_model="gemini:gemini-3.1-pro-preview"))
assert sp.evaluate([_round(0, 0.40), _round(1, 0.42)]).reason == "diminishing_returns"
+ # the detail states the multiple that was APPLIED, not the default one
+ wide = StopPolicy(RoundPolicy(max_rounds=4, target=0.9, marginal_sigma=3.0,
+ judge_model="gemini:gemini-3.1-pro-preview"))
+ d = wide.evaluate([_round(0, 0.40), _round(1, 0.50), _round(2, 0.57)])
+ assert d.reason == "diminishing_returns" and "3 × judge σ" in d.detail and "1.5 ×" not in d.detail
def test_the_stop_order_is_exhausted_regression_then_marginal_then_switch_then_plateau():
@@ -292,23 +291,18 @@ def test_skip_judge_reasons_are_only_states_where_the_verdict_is_never_bought(tm
from codeverse.tracks.steps import skip_judge_reason
ctx = _ctx(tmp_path, spec, settings)
- assert skip_judge_reason(ctx, gates=_gates(), renders=_renders()) == ""
- # a round with gate errors IS judged by default — the gates say what is broken,
- # the verdict says whether the shape is right, and only the verdict can promote it
- assert skip_judge_reason(ctx, gates=_gates(3), renders=_renders()) == ""
- assert skip_judge_reason(ctx, gates=_gates(), renders=None) == "no renders"
- # the caller can switch that off; then the SAME rule holds in the rejudge path
- strict = _ctx(tmp_path, spec, settings, policy=RoundPolicy(judge_on_gate_errors=False), name="strict")
- assert skip_judge_reason(strict, gates=_gates(3), renders=_renders()) == "gate errors"
- assert skip_judge_reason(strict, gates=_gates(0), renders=_renders()) == ""
+ # gate errors are no reason: the gates say what is broken, the verdict says whether
+ # the shape is right, and only the verdict can promote the round
+ assert skip_judge_reason(ctx, renders=_renders()) == ""
+ assert skip_judge_reason(ctx, renders=None) == "no renders"
# ... the budget/clock stop: the loop's next budget_ok check ends the run, so this
# verdict cannot promote anything (audit: 2 verdicts / $0.09 bought past the clock)
ctx.budget.add(Usage(cost_usd=99.0), stage="refine")
ctx.budget._active_s = (ctx.budget.budget.max_minutes + 1) * 60 # noqa: SLF001
- assert skip_judge_reason(ctx, gates=_gates(), renders=_renders()) == "budget already exceeded"
+ assert skip_judge_reason(ctx, renders=_renders()) == "budget already exceeded"
assert StopPolicy(ctx.policy).evaluate([_round(0, 0.5)], budget_ok=ctx.budget.ok()).reason == "budget"
ctx.judge = None
- assert skip_judge_reason(ctx, gates=_gates(), renders=_renders()) == "no judge configured"
+ assert skip_judge_reason(ctx, renders=_renders()) == "no judge configured"
def test_a_round_that_changed_no_file_never_reaches_the_judge_question(tmp_path, spec, settings):
@@ -325,18 +319,13 @@ def test_a_round_that_changed_no_file_never_reaches_the_judge_question(tmp_path,
run_generation_tasks(ctx, [GenerationTask(label="refine", prompt="p", round=1, kind="refine")])
-def test_a_skipped_verdict_is_not_bought_back_by_the_rejudge_path(tmp_path, spec, settings):
+def test_a_round_with_gate_errors_whose_verdict_was_lost_is_rejudged(tmp_path, spec, settings):
from codeverse.tracks.steps import rejudge_round
- ctx = _ctx(tmp_path, spec, settings, policy=RoundPolicy(judge_on_gate_errors=False), name="norebuy")
+ ctx = _ctx(tmp_path, spec, settings, name="rebuy")
rec = RoundRecord(index=1, kind="refine", build=BuildResult(ok=True, language="l"),
gates=_gates(2), renders=_renders())
- assert rejudge_round(ctx, _pipeline(), rec) is False
- assert ctx.judge.calls == [] and ctx.budget.spent.cost_usd == 0.0
- # the same round with clean gates (an outage, not a policy skip) IS re-judged
- rec2 = RoundRecord(index=2, kind="refine", build=BuildResult(ok=True, language="l"),
- gates=_gates(0), renders=_renders())
- assert rejudge_round(ctx, _pipeline(), rec2) is True and len(ctx.judge.calls) == 1
+ assert rejudge_round(ctx, _pipeline(), rec) is True and len(ctx.judge.calls) == 1
def test_a_round_that_broke_the_gates_does_not_displace_a_clean_one():