Skip to content

Official Codebase of 3DCodeVerse

Python

languages/judges/models: honest prices, honest limits, honest lint

gaoypengb83f909b0bea
diff --git a/harness/codeverse/conventions.py b/harness/codeverse/conventions.py
index f1258c3c..ea1b9234 100644
--- a/harness/codeverse/conventions.py
+++ b/harness/codeverse/conventions.py
@@ -134,6 +134,10 @@ def to_pascal(name: str) -> str:
return "".join(w[:1].upper() + w[1:] for w in to_snake(name).split("_") if w) or "Part"
+#: a part / assembly name as the contracts require it: ``SeatCushion``, instances ``SeatCushion_3``
+PASCAL_RE = re.compile(r"^[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]*)*(?:_\d+)?$")
+
+
def slugify(text: str, max_len: int = 48) -> str:
"""Filesystem/URL-safe slug for run directories."""
s = to_snake(text)[:max_len].strip("_")
diff --git a/harness/codeverse/judges/base.py b/harness/codeverse/judges/base.py
index 2198af56..abad941e 100644
--- a/harness/codeverse/judges/base.py
+++ b/harness/codeverse/judges/base.py
@@ -1,8 +1,8 @@
-"""Judge protocol + inputs."""
+"""``JudgeInput`` + the pure round-replay helpers ``3dcv judge`` and calibration share."""
from __future__ import annotations
-from typing import Any, Protocol
+from typing import Any
from pydantic import BaseModel, Field
@@ -27,13 +27,7 @@ class JudgeInput(BaseModel):
)
-class Judge(Protocol):
- name: str
-
- def judge(self, inp: JudgeInput) -> Judgment: ...
-
-
-# ===================================================================== replay_input
+# ===================================================================== round replay
def plan_digest(plan: dict[str, Any]) -> str:
"""A track-agnostic plan summary (parts / joints / zones / assets / cameras) from ``plan.json``."""
bits: list[str] = []
diff --git a/harness/codeverse/judges/calibration.py b/harness/codeverse/judges/calibration.py
index 77254726..95bcc75a 100644
--- a/harness/codeverse/judges/calibration.py
+++ b/harness/codeverse/judges/calibration.py
@@ -34,6 +34,7 @@ from codeverse.contracts.spec import Spec
from codeverse.judges.base import JudgeInput, judged_subset, plan_digest, resolve_paths
from codeverse.judges.rubrics import is_degraded
from codeverse.judges.vlm_judge import VlmJudge
+from codeverse.proc import read_json_or_none
from codeverse.workspace import Workspace
log = logging.getLogger(__name__)
@@ -150,16 +151,12 @@ class CalibrationTable(BaseModel):
# --------------------------------------------------------------------------- loading runs
-def _read_json(path: Path) -> dict[str, Any]:
- return json.loads(path.read_text())
-
-
def load_run_cases(run_dir: Path, *, rounds: list[int] | None = None) -> list[RoundCase]:
"""Rebuild judge inputs for every judgeable round of a run (rounds with renders)."""
run_dir = Path(run_dir)
spec = Spec.model_validate_json((run_dir / "spec.json").read_text())
- plan = _read_json(run_dir / "plan.json") if (run_dir / "plan.json").is_file() else {}
- record = _read_json(run_dir / "record.json") if (run_dir / "record.json").is_file() else {}
+ plan = read_json_or_none(run_dir / "plan.json") or {}
+ record = read_json_or_none(run_dir / "record.json") or {}
best = record.get("best_round")
acceptance = plan.get("acceptance") or []
digest = plan_digest(plan)
@@ -237,7 +234,7 @@ def spearman(xs: list[float], ys: list[float]) -> float | None:
# --------------------------------------------------------------------------- main entry
def _judge_case(case: RoundCase, judge: VlmJudge, geometry: RenderSet | None, out: Path) -> CalibrationRow:
t0 = time.time()
- j = judge.judge(case.inp, geometry_views=geometry)
+ j = judge.judge(case.inp.model_copy(update={"geometry_views": geometry}) if geometry is not None else case.inp)
raw = json.loads(j.raw) if j.raw else {}
jdir = out / "judgments"
jdir.mkdir(parents=True, exist_ok=True)
diff --git a/harness/codeverse/judges/pairwise.py b/harness/codeverse/judges/pairwise.py
index 7072a4b5..3a1ab876 100644
--- a/harness/codeverse/judges/pairwise.py
+++ b/harness/codeverse/judges/pairwise.py
@@ -95,14 +95,14 @@ class PairwiseJudge:
model_id: str | None = None,
*,
temperature: float = 0.2,
- max_px: int = 768,
+ max_px: int = 1024,
views_per_side: int = 4,
chat_model: ChatModel | None = None,
cache_dir: Path | None = None,
):
self.model_id = model_id or get_settings().default_judge
self.temperature = temperature
- self.max_px = max(max_px, 1024) # a 2×2 montage needs the resolution
+ self.max_px = max(max_px, 1024) # a 2×2 montage needs the resolution: 1024 is the floor, not a hint
self.views_per_side = min(4, views_per_side)
self._model = chat_model
self.cache_dir = cache_dir
diff --git a/harness/codeverse/judges/prompt_builder.py b/harness/codeverse/judges/prompt_builder.py
index 5311a6b0..ee22396f 100644
--- a/harness/codeverse/judges/prompt_builder.py
+++ b/harness/codeverse/judges/prompt_builder.py
@@ -1,10 +1,14 @@
-"""Judge image preparation: downscale, label strips, caching, view geometry.
-
-Every image sent to a judge goes through ``prepare_image`` so the token cost is
-bounded (≤ ``max_px`` on the long side) and every view carries a burnt-in label
-(``VIEW 3/8 — front · az 0° el 8°``) that the model can cite as evidence.
-Prepared files are cached under ``<cache_dir>/<sha>.png`` keyed by source path,
-mtime, size, max_px and label text.
+"""What the judge SEES: image prep, montage planning and message assembly.
+
+* image prep (``prepare_image``): every image sent to a judge is downscaled
+ (≤ ``max_px`` on the long side) and carries a burnt-in label (``VIEW 3/8 — front ·
+ az 0° el 8°``) the model can cite as evidence; prepared files are cached under
+ ``<cache_dir>/<sha>.png`` keyed by source path, mtime, size, max_px and label text.
+* montage planning (``plan_montages`` / ``render_montage``): ranked views packed into
+ ≤2×2 sheets — shaded, geometry (clay/normals), poses, detail crops.
+* message assembly (``build_judge_messages``): rubric + montages + measurements +
+ acceptance items → the ``ChatRequest`` system/messages; ``judge_prompt_hash`` is the
+ verdict's provenance stamp.
"""
from __future__ import annotations
@@ -22,6 +26,7 @@ from PIL import Image, ImageDraw
from codeverse.config import get_settings
from codeverse.contracts.artifacts import (
+ RENDER_MODES,
GateReport,
Judgment,
Measurement,
@@ -124,8 +129,8 @@ def image_part(path: Path, label: str) -> ImagePart:
# ===================================================================== montage
MontageKind = Literal["shaded", "geometry", "poses", "pose_sheet", "detail"]
-#: modes that show geometry without material/lighting noise
-GEOMETRY_MODES = ("clay", "normals", "wire", "silhouette", "depth")
+#: modes that show geometry without material/lighting noise: every render mode but shaded
+GEOMETRY_MODES = tuple(m for m in RENDER_MODES if m != "shaded")
#: most-informative-first order for the object rig (names from conventions.OBJECT_VIEWS)
OBJECT_RANK = ("front_right_34", "back_left_34", "top", "low_front_left", "front", "right", "back", "left")
#: scene rig: authored cameras first (graded for composition), then the overview rig.
diff --git a/harness/codeverse/judges/rubrics.py b/harness/codeverse/judges/rubrics.py
index e19b5113..0329f9e5 100644
--- a/harness/codeverse/judges/rubrics.py
+++ b/harness/codeverse/judges/rubrics.py
@@ -7,7 +7,7 @@ the weighted mean) and a ``kind`` (``visual`` = scored by the VLM,
``measured`` = scored in code and merely *shown* to the VLM).
``caps`` are rules that bound the overall score from deterministic gate
-findings (build error → 0.0, floating part → ≤ 0.6 ...) — see ``caps.py``.
+findings (build error → 0.0, floating part → ≤ 0.6 ...) — see ``apply_caps`` below.
``defects`` is a fixed **binary checklist** the VLM answers (present / absent)
— "a part floats", "lying on its back", "flat untextured" … — and the code
@@ -720,7 +720,7 @@ def aggregate_samples(
if tied:
tie_broken.append(did)
# a checklist defect that a passed gate measured as absent neither penalises nor caps
- # (judges/caps.measured_absent); it is still named in the verdict tail
+ # (``veto_measured_defects``); it is still named in the verdict tail
defects, overridden = veto_measured_defects(rubric, defects, gates)
penalty = round(sum(rubric.defect(did).penalty for did, on in defects.items() if on), 4)
after_defects = max(0.0, overall_uncapped - penalty)
diff --git a/harness/codeverse/judges/vlm_judge.py b/harness/codeverse/judges/vlm_judge.py
index a82837c8..41bcb3d1 100644
--- a/harness/codeverse/judges/vlm_judge.py
+++ b/harness/codeverse/judges/vlm_judge.py
@@ -5,8 +5,8 @@ Per sample: one ``ChatRequest`` with a per-rubric JSON schema (criteria scored
plan, acceptance verdicts); samples differ by montage/tile order (shuffle seed)
so n-sample mean/std measures judge noise. With ``n_samples > 1`` the model
calls run in parallel (``codeverse.proc``); results accumulate in sample order. Score/defect penalties/floors/caps
-/pass are computed in code (``scoring.py``). Images are ≤2×2 montages
-(``montage.py``); tracks may pass a clay/normals ``geometry_views`` RenderSet. Retries: up to
+/pass are computed in code (``rubrics.aggregate_samples``). Images are ≤2×2 montages
+(``prompt_builder``); tracks may pass a clay/normals ``JudgeInput.geometry_views`` RenderSet. Retries: up to
``max_attempts`` per sample on ``ModelError`` / parse failure, all of them inside one
``sample_budget_s`` (:data:`SAMPLE_BUDGET_S`) that also clips the model's own retry
deadline (``ChatRequest.max_wait_s``); if no sample
@@ -27,7 +27,7 @@ from pathlib import Path
from typing import Any, Literal
from codeverse.config import get_settings
-from codeverse.contracts.artifacts import Judgment, RenderSet, RenderView
+from codeverse.contracts.artifacts import Judgment, RenderView
from codeverse.contracts.chat import ChatRequest, ChatResponse
from codeverse.contracts.common import Usage
from codeverse.judges.base import JudgeInput
@@ -64,6 +64,9 @@ class JudgeContext:
measured_scores: dict[str, float] = field(default_factory=dict)
extra_text: str = ""
extra_images: list[tuple[str, str]] = field(default_factory=list)
+ #: what building the context cost (ReferenceJudge's mismatch pass is a real model
+ #: call); folded into the verdict so BudgetGuard and record.total_usage see it
+ usage: Usage = field(default_factory=Usage)
class VlmJudge:
@@ -93,7 +96,7 @@ class VlmJudge:
self.n_samples = max(1, int(n_samples))
if self.n_samples % 2 == 0:
log.warning("judge n_samples=%d is even: exact vote ties on defects / acceptance items are decided by the "
- "representative sample (scoring.py); an odd n gives a true majority", self.n_samples)
+ "representative sample (rubrics.aggregate_samples); an odd n gives a true majority", self.n_samples)
self.temperature = temperature
self.thinking = thinking
self.max_attempts = max(1, int(max_attempts))
@@ -123,20 +126,15 @@ class VlmJudge:
return self._model
# ------------------------------------------------------------------ API
- def judge(self, inp: JudgeInput, *, geometry_views: RenderSet | None = None) -> Judgment:
- """Judge one round.
-
- ``geometry_views``: optional clay/normals RenderSet for the geometry montage
- (falls back to ``inp.geometry_views`` if the input model carries that field;
- clay/normals views embedded in ``inp.renders`` by ``RenderView.mode`` are
- always routed to the geometry montage).
- """
- geometry_views = geometry_views or getattr(inp, "geometry_views", None)
+ def judge(self, inp: JudgeInput) -> Judgment:
+ """Judge one round. ``inp.geometry_views`` feeds the geometry montage (clay/normals
+ views embedded in ``inp.renders`` by ``RenderView.mode`` are always routed there too)."""
+ geometry_views = inp.geometry_views
acceptance_ids = [a.id for a in inp.acceptance]
schema = wire_schema(self.rubric, acceptance_ids)
ctx = self.context(inp)
measured, extra_text, extra_images = ctx.measured_scores, ctx.extra_text, ctx.extra_images
- usage = Usage()
+ usage = ctx.usage
samples: list[JudgeOutput] = []
errors: list[str] = []
reqs: list[ChatRequest] = []
@@ -223,7 +221,7 @@ class VlmJudge:
usage = usage + (getattr(e, "usage", None) or Usage()) # the provider billed it
last = f"ModelError(attempt {attempt}): {e}"
log.warning("judge %s: %s", req.label, last)
- if not e.retryable and attempt >= 2:
+ if not e.retryable: # same request again cannot fix a 400 / a blocked prompt
break
continue
usage = usage + resp.usage
@@ -311,21 +309,6 @@ class ReferenceJudge(VlmJudge):
self.diff = diff
self._diff_model = diff_model
self.diff_model_id = diff_model_id
- #: the last diff computed (tests / callers that want the mismatches themselves)
- self.last_diff: Any | None = None
-
- # ------------------------------------------------------------------ API
- def judge(self, inp: JudgeInput, **kwargs: Any) -> Any:
- """``VlmJudge.judge`` plus the mismatch pass's own spend.
-
- The diff is a real model call made from :meth:`context`; folding its
- ``Usage`` into the verdict keeps ``BudgetGuard`` and ``record.total_usage``
- honest (the ledger already sees it through the metered chat model)."""
- verdict = super().judge(inp, **kwargs)
- diff = self.last_diff
- if diff is not None and getattr(diff, "usage", None) is not None:
- verdict.usage = verdict.usage + diff.usage
- return verdict
# ------------------------------------------------------------------ hook
def context(self, inp: JudgeInput) -> JudgeContext:
@@ -363,18 +346,17 @@ class ReferenceJudge(VlmJudge):
elif refs:
info = self.measure_silhouette(inp)
diff = self.reference_diff(inp, refs, info, synthesized=synth) if refs else None
- self.last_diff = diff
if diff is not None and diff.as_text():
blocks.append(diff.as_text())
- return JudgeContext(measured_scores=measured_scores, extra_text="\n\n".join(blocks), extra_images=images)
+ return JudgeContext(measured_scores=measured_scores, extra_text="\n\n".join(blocks), extra_images=images,
+ usage=diff.usage if diff is not None else Usage())
# ------------------------------------------------------------------ proportion guard
def dimension_conflict(self, inp: JudgeInput, refs: list[Any]) -> dict[str, Any]:
"""Does the target reference's own outline contradict the brief's dimensions?
When it does, no numeric proportion signal derived from that picture may be
- used against the object — the brief wins (see
- :mod:`codeverse.reference.proportions`).
+ used against the object — the brief wins (``reference.dimension_conflict``).
"""
targets = [r for r in refs if r.role == "target"] or refs
if not targets:
@@ -497,7 +479,7 @@ class ReferenceJudge(VlmJudge):
def _plan_part_names(inp: JudgeInput) -> list[str]:
"""Part names from the plan digest the judge already receives (best effort).
- ``judges.replay_input.plan_digest`` writes them as one ``Parts: A, B×2, C`` segment;
+ ``judges.base.plan_digest`` writes them as one ``Parts: A, B×2, C`` segment;
a multi-line digest lists one ``- Name · role · …`` per line. Both are handled, and
an unrecognised digest simply yields no names (the diff prompt then says so).
"""
diff --git a/harness/codeverse/languages/_common.py b/harness/codeverse/languages/_common.py
index 80b6eac8..b0495f68 100644
--- a/harness/codeverse/languages/_common.py
+++ b/harness/codeverse/languages/_common.py
@@ -6,11 +6,7 @@ and the runtime turns those into a :class:`BuildResult`. This module owns:
* :func:`compose_build_result` — wrapper json + process outcome → BuildResult,
failing loud when the wrapper did not report;
-* :func:`read_json_file` / :func:`remove_stale` / :func:`strip_blender_noise`.
-
-The subprocess + atomic-JSON primitives (:class:`ProcResult`,
-:func:`run_subprocess`, :func:`tail`, :func:`write_json_atomic`) live in
-:mod:`codeverse.proc` and are re-exported here for the runtime callers.
+* :func:`read_json_file` / :func:`strip_blender_noise`.
Wrappers themselves are standalone scripts (they never import ``codeverse``;
Blender's bundled python cannot see this package).
@@ -24,12 +20,13 @@ from pathlib import Path
from typing import Any
from codeverse.contracts.artifacts import BuildResult
-from codeverse.proc import ( # noqa: F401 — re-exported
- ProcResult,
- run_subprocess,
- tail,
- write_json_atomic,
-)
+from codeverse.proc import ProcResult, tail
+
+#: The one spelling of every runtime's typed build failures — recorded in build.json /
+#: record.json / bench cells, so the strings stay as first recorded (2026-08-29: threejs,
+#: urdf said "Timeout" and the GL runtimes "MissingEntry"; nothing keyed on either).
+MISSING_ENTRY = "MissingEntryFile"
+BUILD_TIMEOUT = "BuildTimeout"
# lines Blender prints on every headless run that carry no signal for the agent
_BLENDER_NOISE_PREFIXES = (
@@ -78,12 +75,6 @@ def read_json_file(path: Path) -> dict[str, Any]:
return data
-def remove_stale(*paths: Path) -> None:
- """Delete previous wrapper outputs so a crashed run cannot be mistaken for a fresh one."""
- for p in paths:
- p.unlink(missing_ok=True)
-
-
def compose_build_result(
*,
language: str,
@@ -98,7 +89,7 @@ def compose_build_result(
Contract: success requires the wrapper to say ``ok`` AND a non-empty GLB on
disk. Timeouts / crashes before ``build.json`` exists become a failed
- BuildResult with a typed error (``BuildTimeout`` / ``WrapperCrash``) — never
+ BuildResult with a typed error (``BUILD_TIMEOUT`` / ``WrapperCrash``) — never
an exception — so the orchestrator can route them to repair.
"""
stdout_tail = tail(output_filter(proc.stdout) if output_filter else proc.stdout)
@@ -108,7 +99,7 @@ def compose_build_result(
if proc.timed_out:
return BuildResult(
ok=False,
- error_type="BuildTimeout",
+ error_type=BUILD_TIMEOUT,
error_message=f"build exceeded the time limit ({proc.duration_ms // 1000}s); "
"reduce geometry (subdivisions, array counts, boolean ops) so the script finishes quickly",
**base,
diff --git a/harness/codeverse/languages/_docs.py b/harness/codeverse/languages/_docs.py
index fb5f7bd8..d506fa2c 100644
--- a/harness/codeverse/languages/_docs.py
+++ b/harness/codeverse/languages/_docs.py
@@ -16,26 +16,16 @@ for a fact is a second answer, which is what law 2 exists to prevent.
from __future__ import annotations
-from pathlib import Path
-
from codeverse.contracts.common import Language
-from codeverse.prompts import PROMPTS_DIR, load_text
-from codeverse.prompts.catalog import PROMPT_DIRS as _PROMPT_DIRS
-from codeverse.prompts.catalog import prompt_dir_for as _prompt_dir_for
-
-#: re-exported for the callers that already import it from here; the mapping itself
-#: now lives with the rest of the prompt-resolution policy in prompts/catalog.py.
-PROMPT_DIRS = _PROMPT_DIRS
+from codeverse.prompts import load_text
+from codeverse.prompts.catalog import prompt_dir_for
#: only reachable if the wheel shipped without its prompt data
MINIMAL_CONTRACT = "Write raw code in the language's native frame; meters; named parts."
-prompt_dir_for = _prompt_dir_for
-
-
class RuntimeDocs:
- """``contract_doc`` / ``cookbook_path`` for every ``LanguageRuntime``."""
+ """``contract_doc`` for every ``LanguageRuntime``."""
language: Language
@@ -48,6 +38,3 @@ class RuntimeDocs:
return load_text(f"{self.prompt_dir}/contract.md")
except FileNotFoundError:
return MINIMAL_CONTRACT
-
- def cookbook_path(self) -> Path:
- return PROMPTS_DIR / self.prompt_dir / "cookbook.md"
diff --git a/harness/codeverse/languages/_gl_common.py b/harness/codeverse/languages/_gl_common.py
index 17b65b50..4cb2dfe4 100644
--- a/harness/codeverse/languages/_gl_common.py
+++ b/harness/codeverse/languages/_gl_common.py
@@ -49,7 +49,7 @@ def invalidate_stale_outputs(ws: Workspace) -> None:
"""Wipe the previous build's GL artifacts at the TOP of ``build()``.
Hoisted from ``GlHost._run`` (which still wipes its own out_dir) so the
- ``MissingEntry`` early returns — which never reach the host — also clear
+ ``MISSING_ENTRY`` early returns — which never reach the host — also clear
``frames/`` and ``gl_result.json``; sheet/gif/metrics/build.json were never
cleared anywhere, so a failed build left the previous round's sheet looking
current to every bare-existence reader (``read_metrics``, the gallery)."""
diff --git a/harness/codeverse/languages/base.py b/harness/codeverse/languages/base.py
index d355f0eb..7db8f558 100644
--- a/harness/codeverse/languages/base.py
+++ b/harness/codeverse/languages/base.py
@@ -40,8 +40,6 @@ class LanguageRuntime(Protocol):
"""Prompt text: the authoring contract for this language (from prompts/<lang>/)."""
...
- def cookbook_path(self) -> Path: ...
-
#: Language → (module, class). Adding a language is a row, and the import stays lazy
#: (a runtime pulls in bpy / moderngl / node bindings the other six do not need).
diff --git a/harness/codeverse/languages/blender/__init__.py b/harness/codeverse/languages/blender/__init__.py
index 6757ffd4..2ca56509 100644
--- a/harness/codeverse/languages/blender/__init__.py
+++ b/harness/codeverse/languages/blender/__init__.py
@@ -5,7 +5,6 @@ from __future__ import annotations
import ast
import math
import os
-import re
import time
from pathlib import Path
@@ -13,21 +12,17 @@ from codeverse.config import Settings, get_settings
from codeverse.contracts.artifacts import BuildResult, GateFinding, GateReport, Severity
from codeverse.contracts.common import ENTRY_FILE, Language
from codeverse.contracts.plan import BBox, PartPlan, Plan, StaticPlan
-from codeverse.conventions import MAX_TRIS_OBJECT, to_pascal, to_snake
-from codeverse.languages._ast_lint import ( # noqa: F401 — dotted re-exported
+from codeverse.conventions import MAX_TRIS_OBJECT, PASCAL_RE, to_pascal, to_snake
+from codeverse.languages._ast_lint import (
BASE_FORBIDDEN_IMPORTS,
check_imports,
describe_parse_failure,
dotted,
safe_parse,
)
-from codeverse.languages._common import (
- compose_build_result,
- run_subprocess,
- strip_blender_noise,
-)
+from codeverse.languages._common import MISSING_ENTRY, compose_build_result, strip_blender_noise
from codeverse.languages._docs import RuntimeDocs
-from codeverse.proc import scrub_secrets
+from codeverse.proc import run_subprocess, scrub_secrets
from codeverse.workspace import Workspace
# ===================================================================== lint
@@ -76,7 +71,6 @@ REMOVED_BSDF_INPUTS = {
"Transmission Roughness": "(removed) use Roughness",
}
KNOWN_BINDINGS = ("Vector", "Matrix", "Euler", "Quaternion", "bmesh", "math", "random", "np", "numpy", "bpy")
-PASCAL_RE = re.compile(r"^[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]*)*(?:_\d+)?$")
class _Collector(ast.NodeVisitor):
@@ -310,7 +304,6 @@ def lint_blender_file(path: Path, *, target: str = "src/model.py") -> GateReport
# ===================================================================== layout
ENTRY_REL = ENTRY_FILE[Language.BLENDER] # "src/model.py"
PARTS_DIR = "parts"
-PARTS_PKG = "parts"
def part_file_rel(part_name: str) -> str:
@@ -357,7 +350,7 @@ def _imported_part_modules(tree: ast.Module) -> set[str]:
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom) and node.module:
dotted = node.module.split(".")
- if dotted[0] == PARTS_PKG:
+ if dotted[0] == PARTS_DIR:
if len(dotted) > 1:
out.add(dotted[1])
else: # ``from parts import seat, leg``
@@ -365,7 +358,7 @@ def _imported_part_modules(tree: ast.Module) -> set[str]:
elif isinstance(node, ast.Import):
for a in node.names:
dotted = a.name.split(".")
- if dotted[0] == PARTS_PKG and len(dotted) > 1:
+ if dotted[0] == PARTS_DIR and len(dotted) > 1:
out.add(dotted[1])
return out
@@ -391,7 +384,7 @@ def _layout_rules(ws: Workspace, parts: list[Path], entry_tree: ast.Module | Non
if fn not in _exported_functions(tree):
out.append(_finding(E, target, f"{target} does not define `def {fn}()`",
f"every part file exports `def {fn}() -> bpy.types.Object` returning the named object "
- f"at its world pose; model.py calls it (`from {PARTS_PKG}.{stem} import {fn}`)"))
+ f"at its world pose; model.py calls it (`from {PARTS_DIR}.{stem} import {fn}`)"))
for node in tree.body:
if isinstance(node, ast.Expr) and isinstance(node.value, ast.Call):
name = getattr(node.value.func, "id", "")
@@ -401,7 +394,7 @@ def _layout_rules(ws: Workspace, parts: list[Path], entry_tree: ast.Module | Non
"call here builds the part twice → auto-suffixed 'Name.001')", node.lineno))
if entry_tree is not None and stem not in imported:
out.append(_finding(W, target, f"{target} is never imported by src/model.py → its part is not built",
- f"add `from {PARTS_PKG}.{stem} import {fn}` to model.py and call `{fn}()` in main()"))
+ f"add `from {PARTS_DIR}.{stem} import {fn}` to model.py and call `{fn}()` in main()"))
return out
@@ -621,7 +614,7 @@ CONTRACT (the harness runs this file in an EMPTY scene with `blender -b --factor
each TOP-LEVEL (never parented under an Empty — that merges them into ONE measured part).
{layout} * Materials: Principled BSDF (Base Color / Roughness / Metallic). GLB keeps flat PBR + image
textures only (procedural node textures are NOT exported) — rely on geometry + flat PBR.
- * Modifiers may stay unapplied (the exporter applies them). Keep < 500k triangles, < 120 s.
+ * Modifiers may stay unapplied (the exporter applies them). Keep < {MAX_TRIS_OBJECT // 1000}k triangles.
* NEVER: cameras, lights, world, render settings, export/import, file IO, bpy.ops.wm.*.
* Only bpy / bmesh / mathutils / math / random (seeded). No other imports.
@@ -655,7 +648,7 @@ def part_file_source(p: PartPlan) -> str:
def model_file_source(plan: StaticPlan) -> str:
"""Complete multi-file entry ``src/model.py``: imports, ordered calls, self-check."""
- imports = "\n".join(f"from {PARTS_PKG}.{to_snake(p.name)} import {build_fn_name(p.name)}" for p in plan.parts)
+ imports = "\n".join(f"from {PARTS_DIR}.{to_snake(p.name)} import {build_fn_name(p.name)}" for p in plan.parts)
calls = "\n".join(f" {build_fn_name(p.name)}()" for p in plan.parts)
return (
_plan_header(plan, multi_file=True)
@@ -746,14 +739,6 @@ class BlenderRuntime(RuntimeDocs):
def part_file(self, ws: Workspace, part_name: str) -> Path:
return ws.root / self.file_for_part(part_name)
- @staticmethod
- def file_for_target(target: str) -> list[str]:
- """Refine target → files: whole-object targets (``overall``/``assembly``/``object``/'')
- map to the entry ``src/model.py``; anything else is treated as a part name."""
- if target.strip().lower() in ("", "overall", "assembly", "object", "model"):
- return [ENTRY_REL]
- return [part_file_rel(target)]
-
def build_command(
self, ws: Workspace, *, stl: bool = True, blend: bool = False, seed: int = 0,
tri_limit: int = MAX_TRIS_OBJECT, rlimit_gb: float | None = None,
@@ -796,7 +781,7 @@ class BlenderRuntime(RuntimeDocs):
ws.stage_artifacts("build.json", "census.json", "object.glb", "object.stl", "object.blend").invalidate()
entry = self.entry_file(ws)
if not entry.is_file():
- result = BuildResult(ok=False, language=self.language.value, error_type="MissingEntryFile",
+ result = BuildResult(ok=False, language=self.language.value, error_type=MISSING_ENTRY,
error_message=f"{ENTRY_REL} does not exist", error_file=ENTRY_REL)
ws.write_json(build_json, result)
return result
diff --git a/harness/codeverse/languages/cadquery/__init__.py b/harness/codeverse/languages/cadquery/__init__.py
index 980f75e1..41c4c230 100644
--- a/harness/codeverse/languages/cadquery/__init__.py
+++ b/harness/codeverse/languages/cadquery/__init__.py
@@ -13,18 +13,18 @@ from codeverse.config import Settings, get_settings
from codeverse.contracts.artifacts import BuildResult, GateFinding, GateReport, Severity
from codeverse.contracts.common import ENTRY_FILE, Language
from codeverse.contracts.plan import PartPlan, Plan, StaticPlan
-from codeverse.conventions import MAX_TRIS_OBJECT, to_pascal, to_snake
-from codeverse.languages._ast_lint import ( # noqa: F401 — dotted re-exported
+from codeverse.conventions import MAX_TRIS_OBJECT, PASCAL_RE, to_pascal, to_snake
+from codeverse.languages._ast_lint import (
BASE_FORBIDDEN_IMPORTS,
check_imports,
describe_parse_failure,
dotted,
safe_parse,
)
-from codeverse.languages._common import compose_build_result, run_subprocess
+from codeverse.languages._common import MISSING_ENTRY, compose_build_result
from codeverse.languages._docs import RuntimeDocs
from codeverse.languages.blender import finish_for, instance_centers
-from codeverse.proc import scrub_secrets
+from codeverse.proc import run_subprocess, scrub_secrets
from codeverse.workspace import Workspace
# ===================================================================== lint
@@ -48,9 +48,6 @@ FORBIDDEN_CALLS: tuple[tuple[str, str], ...] = (
)
FORBIDDEN_METHODS = {"save": "no .save()/.export() on the assembly — the harness exports", "export": "no .export() — the harness exports",
"exportStep": "no export calls", "exportStl": "no export calls", "exportSvg": "no export calls"}
-PASCAL_RE = re.compile(r"^[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]*)*(?:_\d+)?$")
-
-
RADIAN_CONSTS = {"pi", "tau"}
TO_RADIANS_FUNCS = {"radians", "deg2rad"}
TO_DEGREES_FUNCS = {"degrees", "rad2deg"}
@@ -373,7 +370,7 @@ class CadQueryRuntime(RuntimeDocs):
ws.stage_artifacts("build.json", "census.json", "object.glb", "object.step", "object.stl").invalidate()
entry = self.entry_file(ws)
if not entry.is_file():
- result = BuildResult(ok=False, language=self.language.value, error_type="MissingEntryFile",
+ result = BuildResult(ok=False, language=self.language.value, error_type=MISSING_ENTRY,
error_message="src/model.py does not exist", error_file="src/model.py")
ws.write_json(build_json, result)
return result
diff --git a/harness/codeverse/languages/glsl_shader/__init__.py b/harness/codeverse/languages/glsl_shader/__init__.py
index 2aed803e..18aefd7a 100644
--- a/harness/codeverse/languages/glsl_shader/__init__.py
+++ b/harness/codeverse/languages/glsl_shader/__init__.py
@@ -10,6 +10,7 @@ from pathlib import Path
from codeverse.contracts.artifacts import BuildResult, GateFinding, GateReport, Severity
from codeverse.contracts.common import ENTRY_FILE, Language
from codeverse.contracts.plan import GraphicsPlan, Plan
+from codeverse.languages._common import MISSING_ENTRY
from codeverse.languages._docs import RuntimeDocs
from codeverse.languages._gl_common import ( # noqa: F401 — re-exported
GlslMessage,
@@ -379,12 +380,6 @@ def write_skeleton(ws: Workspace, plan: Plan | None) -> list[Path]:
# ===================================================================== runtime
-CONTRACT_FALLBACK = """src/shader.frag — GLSL 330 fragment shader body (NO #version line, NO uniform declarations):
-write `void mainImage(out vec4 fragColor, in vec2 fragCoord)` using the harness uniforms u_time, u_resolution,
-u_mouse, u_frame, u_prev (previous frame), u_noise (256² noise), u_buffer_a (src/buffer_a.frag output).
-Optional src/common.glsl (helpers, pasted in first) and src/buffer_a.frag (one feedback pass). Animate with u_time."""
-
-
class GlslShaderRuntime(RuntimeDocs):
language = Language.GLSL_SHADER
entry_globs: tuple[str, ...] = (ENTRY_FILE[Language.GLSL_SHADER], COMMON, BUFFER_A)
@@ -392,8 +387,6 @@ class GlslShaderRuntime(RuntimeDocs):
def __init__(self, *, host: GlHost | None = None):
self._host = host
- # ------------------------------------------------------------------ contract
-
# ------------------------------------------------------------------ skeleton / lint
def skeleton(self, ws: Workspace, plan: Plan | None) -> list[Path]:
return write_skeleton(ws, plan)
@@ -420,9 +413,9 @@ class GlslShaderRuntime(RuntimeDocs):
def build(self, ws: Workspace, *, timeout_s: int | None = None, times: list[float] | None = None,
preview: bool = True, width: int | None = None, height: int | None = None) -> BuildResult:
ws.artifacts.mkdir(parents=True, exist_ok=True)
- invalidate_stale_outputs(ws) # BEFORE the MissingEntry return, so it also clears
+ invalidate_stale_outputs(ws) # BEFORE the MISSING_ENTRY return, so it also clears
if not (ws.root / SHADER).is_file():
- res = GlResult(ok=False, mode="shader", stage="lint", error_type="MissingEntry", error_message=f"{SHADER} is missing")
+ res = GlResult(ok=False, mode="shader", stage="lint", error_type=MISSING_ENTRY, error_message=f"{SHADER} is missing")
return finish_build(ws, res, language=self.language.value, error_file=SHADER)
plan = load_plan(ws)
w, h = resolution_for(plan)
diff --git a/harness/codeverse/languages/opengl_python/__init__.py b/harness/codeverse/languages/opengl_python/__init__.py
index e4516e51..f0208cf4 100644
--- a/harness/codeverse/languages/opengl_python/__init__.py
+++ b/harness/codeverse/languages/opengl_python/__init__.py
@@ -15,6 +15,7 @@ from codeverse.languages._ast_lint import (
describe_parse_failure,
safe_parse,
)
+from codeverse.languages._common import MISSING_ENTRY
from codeverse.languages._docs import RuntimeDocs
from codeverse.languages._gl_common import (
finish_build,
@@ -336,12 +337,6 @@ def write_skeleton(ws: Workspace, plan: Plan | None) -> list[Path]:
# ===================================================================== runtime
-CONTRACT_FALLBACK = """src/program.py — raw moderngl program: `def setup(ctx, width, height) -> state` (programs, VAOs, textures,
-FBOs; GLSL 330 core inline strings) and `def render(ctx, state, t, frame, fbo)` (draw the frame at time t into the given,
-already bound framebuffer; multi-pass via your own FBOs, finish with fbo.use()). No window, no context creation, no clock,
-no file IO (except src/*.glsl next to program.py). Imports: moderngl, numpy, math, random, struct, array, pathlib."""
-
-
class OpenGLPythonRuntime(RuntimeDocs):
language = Language.OPENGL_PYTHON
entry_globs: tuple[str, ...] = (ENTRY_FILE[Language.OPENGL_PYTHON], "src/*.glsl")
@@ -349,8 +344,6 @@ class OpenGLPythonRuntime(RuntimeDocs):
def __init__(self, *, host: GlHost | None = None):
self._host = host
- # ------------------------------------------------------------------ contract
-
# ------------------------------------------------------------------ skeleton / lint
def skeleton(self, ws: Workspace, plan: Plan | None) -> list[Path]:
return write_skeleton(ws, plan)
@@ -365,10 +358,10 @@ class OpenGLPythonRuntime(RuntimeDocs):
def build(self, ws: Workspace, *, timeout_s: int | None = None, times: list[float] | None = None,
preview: bool = True, width: int | None = None, height: int | None = None) -> BuildResult:
ws.artifacts.mkdir(parents=True, exist_ok=True)
- invalidate_stale_outputs(ws) # BEFORE the MissingEntry return, so it also clears
+ invalidate_stale_outputs(ws) # BEFORE the MISSING_ENTRY return, so it also clears
program = ws.root / PROGRAM
if not program.is_file():
- res = GlResult(ok=False, mode="program", stage="lint", error_type="MissingEntry", error_message=f"{PROGRAM} is missing")
+ res = GlResult(ok=False, mode="program", stage="lint", error_type=MISSING_ENTRY, error_message=f"{PROGRAM} is missing")
return finish_build(ws, res, language=self.language.value, error_file=PROGRAM)
plan = load_plan(ws)
w, h = resolution_for(plan)
diff --git a/harness/codeverse/languages/threejs/__init__.py b/harness/codeverse/languages/threejs/__init__.py
index c65f26cf..341bb8da 100644
--- a/harness/codeverse/languages/threejs/__init__.py
+++ b/harness/codeverse/languages/threejs/__init__.py
@@ -14,8 +14,13 @@ from codeverse.contracts.artifacts import BuildResult, GateFinding, GateReport,
from codeverse.contracts.common import ENTRY_FILE, Language
from codeverse.contracts.plan import Plan, StaticPlan
from codeverse.conventions import to_pascal, to_snake
+from codeverse.languages._common import BUILD_TIMEOUT
from codeverse.languages._docs import RuntimeDocs
-from codeverse.languages._js_lint import ImportKind, ImportVerdict, check_imports
+from codeverse.languages._js_lint import (
+ ImportKind,
+ ImportVerdict,
+ check_imports,
+)
from codeverse.languages._js_lint import (
node_check_syntax as check_syntax, # module-level name: tests monkeypatch it
)
@@ -87,43 +92,6 @@ export function build{pascal}(THREE_) {{
PLACEHOLDER_COLORS = ("0x9aa5b1", "0xb08968", "0x7f8c8d", "0xc0a080", "0x6c7a89", "0xa0522d", "0x8fa3ad", "0xbfa27a")
-# ===================================================================== contract
-CONTRACT_FALLBACK = """\
-# three.js static-object contract (raw ESM, no SDK)
-
-Files (all under `src/`):
-* `src/parts/<snake_name>.js` — one file per part:
- `export function build<PascalName>(THREE) { ...; return group; }`
- The returned `THREE.Group` is named `<PascalName>` and already sits at its WORLD pose
- (no re-positioning happens in object.js).
-* `src/object.js` — `export function build(THREE) { ... return root; }` imports every part
- builder, adds each part group to one root `THREE.Group` whose `.name` is the object name,
- and returns it. Optional: `root.userData.tick = (t, dt) => { ... }` for idle animation.
-
-Frame & units: Y is UP, +Z is the FRONT (towards the default camera), +X is the right.
-Meters. The object stands on the ground: lowest point at y = 0, footprint centred on
-the Y axis. Real-world sizes (a chair seat is ~0.45 m high, not 45).
-
-Imports allowed: `import * as THREE from 'three'`, `three/addons/...` (e.g.
-`three/addons/utils/BufferGeometryUtils.js`, `three/addons/geometries/RoundedBoxGeometry.js`),
-and relative `./` files inside `src/`. Nothing else: no CDN/http URLs, no npm packages,
-no `fetch`, no `document`/`window`, no WebGLRenderer/cameras/lights/scenes in object code —
-the harness renders and exports for you.
-
-Geometry & materials: BufferGeometry only; `MeshStandardMaterial`/`MeshPhysicalMaterial`
-with explicit `color`, `roughness`, `metalness`. Textures are NOT exportable from node —
-use materials and vertex detail. Every mesh gets a meaningful `.name`. Keep total
-triangles well under 600k. Parts must physically touch (no floating pieces) and must not
-interpenetrate visibly.
-
-Build: the harness runs `build(THREE)` in node, validates the group (≥1 mesh, finite
-bbox, no NaN), calls an optional exported `selfcheck(THREE, root)` (a throw fails the
-build with your message), bakes `InstancedMesh` copies into named meshes, and exports
-`artifacts/object.glb` with one named node per part — exactly where you placed it (no
-automatic drop-to-ground / re-centring; off-placement is warned and gated instead).
-"""
-
-
# ===================================================================== lint
GATE = "lint:threejs"
@@ -375,8 +343,6 @@ class ThreeJsRuntime(RuntimeDocs):
language = Language.THREEJS
entry_globs: tuple[str, ...] = (ENTRY_FILE[Language.THREEJS], "src/parts/*.js")
- # ------------------------------------------------------------------ contract
-
# ------------------------------------------------------------------ skeleton / lint
def skeleton(self, ws: Workspace, plan: Plan) -> list[Path]:
return write_skeleton(ws, plan)
@@ -413,7 +379,7 @@ class ThreeJsRuntime(RuntimeDocs):
except NodeError as e:
if e.result is None:
raise # harness problem (no node binary / script)
- result = self._failure(e.result, "Timeout" if e.result.timed_out else "NodeError", str(e))
+ result = self._failure(e.result, BUILD_TIMEOUT if e.result.timed_out else "NodeError", str(e))
self._write_build_json(ws, result)
return result
diff --git a/harness/codeverse/languages/threejs/wrappers/README.md b/harness/codeverse/languages/threejs/wrappers/README.md
deleted file mode 100644
index 289eb0a6..00000000
--- a/harness/codeverse/languages/threejs/wrappers/README.md
+++ /dev/null
@@ -1,5 +0,0 @@
-The three.js build/export wrappers are node scripts and live in `runtime_js/`
-(`export_glb.mjs`, `render_glb.mjs`, `lib/*.mjs`). They are invoked through
-`codeverse.spatial.node.run_node` with the `--import lib/resolve_three.mjs`
-hook so agent code can `import 'three'` without a node_modules of its own.
-Agent code never imports anything from here.
diff --git a/harness/codeverse/languages/urdf/__init__.py b/harness/codeverse/languages/urdf/__init__.py
index b3d98c2c..d6fca341 100644
--- a/harness/codeverse/languages/urdf/__init__.py
+++ b/harness/codeverse/languages/urdf/__init__.py
@@ -1,8 +1,5 @@
-"""``urdf_blender``: bpy link meshes + hand-written URDF.
-
-Modules: ``runtime`` (UrdfBlenderRuntime), ``lint``, ``skeleton``, ``consistency``
-(FK ↔ authored geometry), ``wrappers/run_bpy_links.py`` (Blender-side build),
-``wrappers/render_glb_bpy.py`` (fallback renderer). Contract: ``CONTRACT.md``."""
+"""``urdf_blender``: bpy link meshes + hand-written URDF — lint, skeleton, FK ↔ geometry
+consistency and the ``UrdfBlenderRuntime`` around ``wrappers/run_bpy_links.py``."""
from __future__ import annotations
@@ -25,9 +22,16 @@ from codeverse.contracts.artifacts import BuildResult, GateFinding, GateReport,
from codeverse.contracts.common import ENTRY_FILE, Language
from codeverse.contracts.plan import ArticulatedPlan, JointPlan, PartPlan, Plan
from codeverse.conventions import to_snake
-from codeverse.languages._ast_lint import describe_parse_failure, safe_parse
-from codeverse.languages._common import ProcResult, read_json_file, run_subprocess
+from codeverse.languages._ast_lint import (
+ BASE_FORBIDDEN_IMPORTS,
+ check_imports,
+ describe_parse_failure,
+ dotted,
+ safe_parse,
+)
+from codeverse.languages._common import BUILD_TIMEOUT, ProcResult, read_json_file
from codeverse.languages._docs import RuntimeDocs
+from codeverse.proc import run_subprocess, tail
from codeverse.spatial.joints import (
UrdfError,
load_urdf,
@@ -117,7 +121,17 @@ FORBIDDEN_BPY_PREFIXES: tuple[tuple[str, str], ...] = (
("bpy.ops.import_scene.", "importing external files — build geometry procedurally"),
("bpy.ops.import_mesh.", "importing external files — build geometry procedurally"),
)
-FORBIDDEN_MODULES: tuple[str, ...] = ("subprocess", "socket", "urllib", "requests", "http", "shutil", "ctypes", "multiprocessing")
+#: the shared floor (subprocess/network/pickle/threading/importlib …): the wrapper execs
+#: model.py inside the same Blender interpreter as the blender language, so no looser list
+FORBIDDEN_IMPORTS: frozenset[str] = frozenset(BASE_FORBIDDEN_IMPORTS)
+#: model.py runs inside the SAME Blender python as blender/model.py, so the pure-computation
+#: stdlib it allows is allowed here too; anything else is an "unexpected" WARN (os/sys stay out:
+#: a link-mesh script has no business in the filesystem, and FORBIDDEN_CALLS catches the uses)
+ALLOWED_IMPORTS: frozenset[str] = frozenset({
+ "bpy", "bmesh", "mathutils", "math", "random", "numpy", "np", "bpy_extras", "__future__",
+ "typing", "dataclasses", "itertools", "functools", "collections", "enum", "copy",
+ "colorsys", "statistics", "operator",
+})
FORBIDDEN_CALLS: dict[str, str] = {
"os.system": "shell access", "os.remove": "file deletion", "os.unlink": "file deletion", "os.rmdir": "file deletion",
"sys.exit": "exits Blender before the export — just return/raise instead", "exit": "exits Blender", "quit": "exits Blender",
@@ -343,42 +357,20 @@ def _lint_joint(el: ET.Element, jname: str, link_names: list[str], parent_of: di
class _Visitor(ast.NodeVisitor):
def __init__(self) -> None:
self.findings: list[GateFinding] = []
- self.has_bpy = False
+ self.imports: dict[str, int] = {} # top-level module → first line
self.strings: set[str] = set()
- def _chain(self, node: ast.AST) -> str:
- parts: list[str] = []
- while isinstance(node, ast.Attribute):
- parts.append(node.attr)
- node = node.value
- if isinstance(node, ast.Name):
- parts.append(node.id)
- return ".".join(reversed(parts))
-
def visit_Import(self, node: ast.Import) -> None:
for a in node.names:
- top = a.name.split(".")[0]
- if top == "bpy":
- self.has_bpy = True
- if top in FORBIDDEN_MODULES:
- self.findings.append(_f(Severity.ERROR, f"line {node.lineno}: import of '{a.name}' is not allowed in model.py",
- target=MODEL_REL, fix="Build geometry with bpy/bmesh/mathutils/math only.", line=node.lineno))
+ self.imports.setdefault(a.name.split(".")[0], node.lineno)
self.generic_visit(node)
def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
- top = (node.module or "").split(".")[0]
- if top == "bpy":
- self.has_bpy = True
- if top in FORBIDDEN_MODULES:
- self.findings.append(_f(Severity.ERROR, f"line {node.lineno}: import from '{node.module}' is not allowed in model.py",
- target=MODEL_REL, fix="Build geometry with bpy/bmesh/mathutils/math only.", line=node.lineno))
- if top == "codeverse":
- self.findings.append(_f(Severity.ERROR, f"line {node.lineno}: model.py must not import the harness", target=MODEL_REL,
- fix="Raw bpy only.", line=node.lineno))
+ self.imports.setdefault((node.module or "").split(".")[0], node.lineno)
self.generic_visit(node)
def visit_Call(self, node: ast.Call) -> None:
- chain = self._chain(node.func)
+ chain = dotted(node.func)
for prefix, why in FORBIDDEN_BPY_PREFIXES:
if chain.startswith(prefix):
self.findings.append(_f(Severity.ERROR, f"line {node.lineno}: {chain}() is forbidden: {why}", target=MODEL_REL,
@@ -412,7 +404,18 @@ def lint_model_text(text: str, link_names: list[str], *, label: str = MODEL_REL)
v = _Visitor()
v.visit(tree)
out = v.findings
- if not v.has_bpy:
+
+ def _import_finding(kind: str, mod: str, line: int) -> GateFinding:
+ if mod == "codeverse":
+ return _f(Severity.ERROR, f"line {line}: model.py must not import the harness", target=label, fix="Raw bpy only.", line=line)
+ if kind == "forbidden":
+ return _f(Severity.ERROR, f"line {line}: import of '{mod}' is not allowed in model.py", target=label,
+ fix="Build geometry with bpy/bmesh/mathutils/math only.", line=line)
+ return _f(Severity.WARN, f"line {line}: import of '{mod}' is unexpected in model.py (outside the contract's list)",
+ target=label, fix="Build geometry with bpy/bmesh/mathutils/math only.", line=line)
+
+ out.extend(check_imports(v.imports, forbidden=FORBIDDEN_IMPORTS, allowed=ALLOWED_IMPORTS, make_finding=_import_finding))
+ if "bpy" not in v.imports:
out.append(_f(Severity.ERROR, f"{label} never imports bpy", target=label, fix="import bpy"))
for link in link_names:
if link not in v.strings and not any(link in s for s in v.strings):
@@ -651,10 +654,6 @@ FK_TOL_M = 0.001
STAGED_OUTPUTS = ("build.json", "census.json", "meshes", "robot.urdf", "articulation.json", "object.glb")
-def _tail(s: str, n: int = 3000) -> str:
- return s[-n:] if s else ""
-
-
class UrdfBlenderRuntime(RuntimeDocs):
language = Language.URDF_BLENDER
entry_globs = (ENTRY_FILE[Language.URDF_BLENDER], "src/robot.urdf")
@@ -723,11 +722,11 @@ class UrdfBlenderRuntime(RuntimeDocs):
build_json, census_json = stage.path("build.json"), stage.path("census.json")
proc = _run_blender(blender, ws, stage.staging_dir, timeout_s, settings.limits.bpy_rlimit_gb)
if proc.timed_out:
- return fail("Timeout", f"Blender build exceeded {timeout_s}s (killed)", file="src/model.py", census=census,
- stdout_tail=_tail(proc.stdout), stderr_tail=_tail(proc.stderr))
+ return fail(BUILD_TIMEOUT, f"Blender build exceeded {timeout_s}s (killed)", file="src/model.py", census=census,
+ stdout_tail=tail(proc.stdout), stderr_tail=tail(proc.stderr))
if not build_json.is_file():
return fail("WrapperCrash", f"wrapper produced no build.json (exit {proc.returncode})", file="src/model.py",
- census=census, stdout_tail=_tail(proc.stdout), stderr_tail=_tail(proc.stderr))
+ census=census, stdout_tail=tail(proc.stdout), stderr_tail=tail(proc.stderr))
wb = read_json_file(build_json)
wcensus = read_json_file(census_json) if census_json.is_file() else {}
census.update({k: wcensus.get(k) for k in ("objects", "links", "unmatched_objects", "missing_links", "hints") if k in wcensus})
@@ -735,8 +734,8 @@ class UrdfBlenderRuntime(RuntimeDocs):
hints = "\n".join(f" hint: {h}" for h in (wcensus.get("hints") or {}).values())
return fail(wb.get("error_type") or "ScriptError", (wb.get("error_message") or "") + ("\n" + hints if hints else ""),
file=wb.get("error_file") or "src/model.py", line=wb.get("error_line"), census=census,
- stdout_tail=_tail(wb.get("stdout_tail", "") or proc.stdout),
- stderr_tail=_tail(wb.get("stderr_tail", "") or proc.stderr))
+ stdout_tail=tail(wb.get("stdout_tail", "") or proc.stdout),
+ stderr_tail=tail(wb.get("stderr_tail", "") or proc.stderr))
# 3. URDF copy + load (staged files; extra_paths name the canonical homes)
urdf_staged = stage.path("robot.urdf")
@@ -770,7 +769,7 @@ class UrdfBlenderRuntime(RuntimeDocs):
urdf_to_glb(robot, stage.path("object.glb"), None)
extra["object_glb"] = str(art / "object.glb")
res = BuildResult(ok=True, language=self.language.value, glb_path=str(art / "object.glb"), extra_paths=extra,
- stdout_tail=_tail(wb.get("stdout_tail", "")), duration_ms=int((time.time() - t0) * 1000), census=census)
+ stdout_tail=tail(wb.get("stdout_tail", "")), duration_ms=int((time.time() - t0) * 1000), census=census)
if report.summary.rest_max_penetration_m > REST_PENETRATION_MAX_M:
worst = [f for f in findings if f.data.get("pose") == {} and f.severity == "error"]
res.ok = False
diff --git a/harness/codeverse/models/anthropic.py b/harness/codeverse/models/anthropic.py
index 96024018..25321e86 100644
--- a/harness/codeverse/models/anthropic.py
+++ b/harness/codeverse/models/anthropic.py
@@ -38,7 +38,7 @@ from codeverse.models.parts import (
with_logged_retries,
)
from codeverse.models.pricing import cache_write_surcharge, estimate_cost
-from codeverse.models.retry import MAX_WAIT_S
+from codeverse.models.retry import MAX_WAIT_S, cause_for
from codeverse.models.schema_utils import (
JsonParseError,
parse_json_lenient,
@@ -309,11 +309,13 @@ class AnthropicModel:
try:
return self._once(kwargs, request)
except Exception as exc: # noqa: BLE001 - classified
- raise classify_exception(exc) from exc
+ err = classify_exception(exc)
+ raise err from cause_for(err, exc)
return with_logged_retries(attempt, label="anthropic", model=self.model,
attempts=self.max_attempts, base_delay=self.base_delay,
- max_delay=self.max_delay, sleep=self._sleep, log=log)
+ max_delay=self.max_delay, sleep=self._sleep, log=log,
+ max_wait_s=request.max_wait_s)
def _once(self, kwargs: dict[str, Any], request: ChatRequest) -> ChatResponse:
client = self.client()
diff --git a/harness/codeverse/models/gemini.py b/harness/codeverse/models/gemini.py
index 967162fb..f0c9bfea 100644
--- a/harness/codeverse/models/gemini.py
+++ b/harness/codeverse/models/gemini.py
@@ -1,7 +1,13 @@
-"""ChatRequest ⇄ google-genai types (contents, config, tools, response parsing).
-
-Kept separate from ``gemini.py`` (which owns clients, keys, retries) so each
-module stays small and the conversions are unit-testable offline.
+"""The whole Gemini stack in one module (merged 2026-08-28, d512ccc):
+
+* conversion — ``ChatRequest`` ⇄ google-genai types (``to_contents``, ``build_config``,
+ ``extract_candidate``, ``parse_usage``); pure, unit-testable offline;
+* ``GeminiModel`` — the ``ChatModel``: a shared ``KeyPool`` over the configured keys,
+ ``rotate_with_retries`` with the storm gate / hedge, streaming with stall detection,
+ per-attempt ledger rows;
+* ``GeminiImageModel`` — text(+reference images) → PIL images on the same pool and
+ retry machine, priced per generated image (``pricing.per_image_usd``), with a
+ fallback model when the primary is missing.
"""
from __future__ import annotations
@@ -36,7 +42,7 @@ from codeverse.contracts.chat import (
from codeverse.contracts.common import Usage
from codeverse.models.base import ModelError
from codeverse.models.parts import BoundedCache, Stopwatch, image_bytes
-from codeverse.models.pricing import estimate_cost
+from codeverse.models.pricing import estimate_cost, per_image_usd
from codeverse.models.retry import (
MAX_WAIT_S,
RETRY_DEADLINE_S,
@@ -231,6 +237,20 @@ def parse_usage(resp: types.GenerateContentResponse, model: str) -> Usage:
)
+def _guard_candidates(resp: types.GenerateContentResponse, usage: Usage | None, what: str) -> Any:
+ """The first candidate, or the two billed failures every Gemini reply can carry:
+ a blocked prompt (non-retryable) and no candidates at all (retryable)."""
+ pf = resp.prompt_feedback
+ if pf is not None and pf.block_reason:
+ raise ModelError(
+ f"{what} prompt blocked: {pf.block_reason} {pf.block_reason_message or ''}".strip(),
+ retryable=False, usage=usage,
+ )
+ if not resp.candidates:
+ raise ModelError(f"{what} returned no candidates", retryable=True, usage=usage)
+ return resp.candidates[0]
+
+
def extract_candidate(
resp: types.GenerateContentResponse, usage: Usage | None = None
) -> tuple[str, list[ToolCallPart], str]:
@@ -238,15 +258,7 @@ def extract_candidate(
prompts (non-retryable) or empty candidates (retryable); ``usage`` (what the caller
already parsed off ``resp``) rides on the error, because every one of these
failures was billed."""
- pf = resp.prompt_feedback
- if pf is not None and pf.block_reason:
- raise ModelError(
- f"prompt blocked by Gemini: {pf.block_reason} {pf.block_reason_message or ''}".strip(),
- retryable=False, usage=usage,
- )
- if not resp.candidates:
- raise ModelError("Gemini returned no candidates", retryable=True, usage=usage)
- cand = resp.candidates[0]
+ cand = _guard_candidates(resp, usage, "Gemini")
finish = str(cand.finish_reason.value if cand.finish_reason else "") or "UNKNOWN"
texts: list[str] = []
calls: list[ToolCallPart] = []
@@ -362,6 +374,27 @@ def _default_keys() -> list[str]:
return list(get_settings().gemini_api_keys)
+def _keys_or_raise(keys: list[str] | None, pool: KeyPool | None) -> list[str]:
+ keys = list(keys) if keys is not None else _default_keys()
+ if pool is None and not keys:
+ raise ModelError("no Gemini API keys configured (GEMINI_API_KEYS / ~/.config/astra3d/gemini_keys.env)")
+ return keys
+
+
+def _client_for(key: str, timeout_s: float, factory: Callable[[str], Any] | None) -> Any:
+ """One ``genai.Client`` per (key, timeout), process-wide; ``factory`` (tests) bypasses the cache."""
+ if factory is not None:
+ return factory(key)
+ ck = (key, int(timeout_s * 1000))
+ with _registry_lock:
+ client = _clients.get(ck)
+ if client is None:
+ client = genai.Client(api_key=key, http_options=types.HttpOptions(
+ timeout=ck[1], client_args=_ipv4_client_args() or None))
+ _clients[ck] = client
+ return client
+
+
def classify_exception(exc: BaseException) -> ModelError:
"""Map provider / transport exceptions onto ``ModelError``."""
if isinstance(exc, ModelError):
@@ -406,15 +439,16 @@ def is_dead_key_error(err: ModelError) -> bool:
def failure_outcome(err: ModelError) -> Outcome:
- """``KeyPool.report`` outcome for a failed call (content-level failures such as
- bad JSON / empty candidates carry no status and are not the key's fault)."""
+ """``KeyPool.report`` outcome for a failed call. Content-level failures (bad JSON,
+ empty candidates) carry no status and are not the key's fault: ``skip`` reconciles
+ the tokens they were billed and leaves health / counters alone."""
if err.status == 429:
return "429"
if (err.status or 0) >= 500:
return "5xx"
if is_dead_key_error(err):
return "dead"
- return "error" if err.status else "ok"
+ return "error" if err.status else "skip"
def _retry_after_s(exc: BaseException) -> float | None:
@@ -496,7 +530,9 @@ def _merge_stream_chunks(
class GeminiModel:
- """ChatModel for ``gemini:<model>``. See module docstring."""
+ """ChatModel for ``gemini:<model>``: a shared ``KeyPool``, ``rotate_with_retries``
+ (storm gate, hedge, ``ChatRequest.max_wait_s`` deadline), streaming with stall
+ detection and per-attempt ledger rows."""
provider = "gemini"
@@ -526,11 +562,7 @@ class GeminiModel:
#: keys a retry is raced on after the call's first 503 (``Settings.rate.hedge``,
#: 2; 1 = off) — see ``rotate_with_retries`` and docs/COST.md §27
self.hedge = max(1, int(_rate().hedge if hedge is None else hedge))
- keys = list(keys) if keys is not None else _default_keys()
- if pool is None and not keys:
- raise ModelError(
- "no Gemini API keys configured (GEMINI_API_KEYS / ~/.config/astra3d/gemini_keys.env)"
- )
+ keys = _keys_or_raise(keys, pool)
self.pool = pool or shared_pool(keys, rpm_per_key=rpm_per_key, tpm_per_key=tpm_per_key)
self.storm_gate = storm_gate if storm_gate is not None else (
storm_gate_for(f"gemini:{model}") if _rate().storm_gate else None
@@ -552,19 +584,6 @@ class GeminiModel:
def supports_vision(self) -> bool:
return True
- # ---------------------------------------------------------------- clients
- def _client(self, key: str) -> Any:
- if self._client_factory is not None:
- return self._client_factory(key)
- ck = (key, int(self.timeout_s * 1000))
- with _registry_lock:
- client = _clients.get(ck)
- if client is None:
- client = genai.Client(api_key=key, http_options=types.HttpOptions(
- timeout=ck[1], client_args=_ipv4_client_args() or None))
- _clients[ck] = client
- return client
-
# --------------------------------------------------------------- generate
def generate(self, request: ChatRequest) -> ChatResponse:
contents = to_contents(request.messages)
@@ -680,7 +699,7 @@ class GeminiModel:
request: ChatRequest,
warnings: list[str],
) -> ChatResponse:
- client = self._client(key)
+ client = _client_for(key, self.timeout_s, self._client_factory)
if not _streaming_enabled():
with Stopwatch() as sw:
resp = client.models.generate_content(
@@ -746,21 +765,11 @@ class GeminiModel:
DEFAULT_IMAGE_MODEL = "gemini-3.1-flash-image"
FALLBACK_IMAGE_MODEL = "gemini-2.5-flash-image"
-#: approximate USD per generated 1024² image (floor for cost accounting)
-IMAGE_USD: dict[str, float] = {
- "gemini-3.1-flash-image": 0.067,
- "gemini-2.5-flash-image": 0.039,
-}
-
-#: requested pixel size → Gemini ``image_size`` token
+#: requested pixel size → Gemini ``image_size`` token; a 512 request GENERATES a 1K
+#: image (resized locally), so it is billed as one — ``_GENERATED_PX`` is the size
+#: ``pricing.per_image_usd`` is asked for
_IMAGE_SIZE_TOKEN: dict[int, str] = {512: "1K", 1024: "1K", 2048: "2K", 4096: "4K"}
-
-
-def image_cost(model: str, usage: Usage, n_images: int) -> float:
- """Token-priced cost floored by the per-image price (approximate)."""
- token_cost = estimate_cost("gemini", model, usage)
- per_image = IMAGE_USD.get(model.strip().lower(), 0.0)
- return max(token_cost, per_image * max(0, n_images))
+_GENERATED_PX: dict[str, int] = {"1K": 1024, "2K": 2048, "4K": 4096}
def _to_part(ref: Image.Image | Path | str) -> types.Part:
@@ -768,11 +777,8 @@ def _to_part(ref: Image.Image | Path | str) -> types.Part:
buf = io.BytesIO()
ref.convert("RGB").save(buf, format="PNG")
return types.Part.from_bytes(data=buf.getvalue(), mime_type="image/png")
- p = Path(ref)
- if not p.is_file():
- raise ModelError(f"reference image not found: {p}")
- mime = "image/jpeg" if p.suffix.lower() in (".jpg", ".jpeg") else "image/png"
- return types.Part.from_bytes(data=p.read_bytes(), mime_type=mime)
+ data, mime = image_bytes(ImagePart(path=str(ref)))
+ return types.Part.from_bytes(data=data, mime_type=mime)
def _is_model_missing(err: ModelError) -> bool:
@@ -794,7 +800,9 @@ def _record(usage: Usage, n_images: int) -> None:
class GeminiImageModel:
- """See module docstring. ``model`` is the primary; ``fallback`` the second try."""
+ """Text (+ reference images) → PIL images on the shared key pool and retry machine.
+ ``model`` is the primary; ``fallback`` the second try once the primary is missing
+ or fails retryably. Not a ``ChatModel``: it meters itself (``_record``)."""
provider = "gemini"
@@ -818,18 +826,15 @@ class GeminiImageModel:
#: no 503 hedge by default: an image is billed per image, so a hedge that
#: lands twice pays for two of them (the chat models default to 2)
self.hedge = max(1, int(hedge))
- keys = list(keys) if keys is not None else _default_keys()
- if pool is None and not keys:
- raise ModelError("no Gemini API keys configured (GEMINI_API_KEYS / ~/.config/astra3d/gemini_keys.env)")
+ keys = _keys_or_raise(keys, pool)
self.pool = pool or shared_pool(keys)
+ self.storm_gate = storm_gate_for(f"gemini:{model}") if _rate().storm_gate else None
self.timeout_s = timeout_s
self.max_attempts = max(1, max_attempts)
self.base_delay = base_delay
self.max_delay = max_delay
self._sleep = sleep
- # reuse GeminiModel's cached clients (same key → same genai.Client)
- self._clients = GeminiModel(model, pool=self.pool, timeout_s=timeout_s, client_factory=client_factory)
- self.storm_gate = self._clients.storm_gate
+ self._client_factory = client_factory
self._lock = threading.Lock()
self._primary_dead = False
@@ -838,20 +843,6 @@ class GeminiImageModel:
return f"gemini-image:{self.model}"
# ------------------------------------------------------------------ API
- def generate(
- self,
- prompt: str,
- *,
- size: int = 1024,
- n: int = 1,
- seed: int | None = None,
- reference_images: Sequence[Image.Image | Path | str] = (),
- max_wait_s: float | None = None,
- ) -> list[Image.Image]:
- images, _ = self.generate_with_usage(prompt, size=size, n=n, seed=seed,
- reference_images=reference_images, max_wait_s=max_wait_s)
- return images
-
def generate_with_usage(
self,
prompt: str,
@@ -865,8 +856,7 @@ class GeminiImageModel:
"""``max_wait_s`` is the caller's retry budget for ONE image (each of the
``n`` images gets its own window, and a fallback model too), exactly like
``ChatRequest.max_wait_s`` clips ``GeminiModel.generate``; ``None`` = the
- full ``retry.RETRY_DEADLINE_S`` (900 s). Kept off the ``ImageModel``
- protocol so fakes stay conformant; callers that hold a deadline pass it."""
+ full ``retry.RETRY_DEADLINE_S`` (1800 s)."""
if not prompt.strip():
raise ModelError("empty image prompt")
if n < 1:
@@ -922,9 +912,10 @@ class GeminiImageModel:
parts: list[types.Part] = [_to_part(r) for r in refs]
parts.append(types.Part.from_text(text=prompt))
contents = [types.Content(role="user", parts=parts)]
+ token = _IMAGE_SIZE_TOKEN.get(int(size), "1K")
config = types.GenerateContentConfig(
response_modalities=["IMAGE"],
- image_config=types.ImageConfig(aspect_ratio="1:1", image_size=_IMAGE_SIZE_TOKEN.get(int(size), "1K")),
+ image_config=types.ImageConfig(aspect_ratio="1:1", image_size=token),
seed=seed,
http_options=types.HttpOptions(timeout=int(self.timeout_s * 1000)),
)
@@ -942,7 +933,7 @@ class GeminiImageModel:
return rotate_with_retries(
self.pool,
- lambda key: self._call(key, model, contents, _attempt_config(), size),
+ lambda key: self._call(key, model, contents, _attempt_config(), size, _GENERATED_PX[token]),
classify=classify_exception,
outcome_of=failure_outcome,
max_attempts=self.max_attempts,
@@ -958,20 +949,16 @@ class GeminiImageModel:
)
def _call(
- self, key: str, model: str, contents: list[types.Content], config: types.GenerateContentConfig, size: int
+ self, key: str, model: str, contents: list[types.Content], config: types.GenerateContentConfig,
+ size: int, generated_px: int,
) -> tuple[list[Image.Image], Usage]:
- client = self._clients._client(key)
+ client = _client_for(key, self.timeout_s, self._client_factory)
with Stopwatch() as sw:
resp = client.models.generate_content(model=model, contents=contents, config=config)
usage = parse_usage(resp, model) # parsed first: the raises below were billed too
usage.backend = "gemini-image"
usage.latency_ms = sw.ms
- pf = resp.prompt_feedback
- if pf is not None and pf.block_reason:
- raise ModelError(f"image prompt blocked: {pf.block_reason}", retryable=False, usage=usage)
- if not resp.candidates:
- raise ModelError("image model returned no candidates", retryable=True, usage=usage)
- cand = resp.candidates[0]
+ cand = _guard_candidates(resp, usage, "image model")
finish = str(cand.finish_reason.name if cand.finish_reason is not None else "")
images: list[Image.Image] = []
for part in (cand.content.parts if cand.content and cand.content.parts else []):
@@ -986,7 +973,9 @@ class GeminiImageModel:
retryable=False, usage=usage)
raise ModelError(f"image model returned no image (finish_reason={finish or 'unknown'})",
retryable=True, usage=usage)
- usage.cost_usd = image_cost(model, usage, len(images))
+ # token-priced cost floored by the per-image price of the size that was GENERATED
+ usage.cost_usd = max(estimate_cost("gemini", model, usage),
+ per_image_usd("gemini", model, size=generated_px) * len(images))
if int(size) not in (0, 1024) and images[0].size != (size, size):
images = [im.resize((int(size), int(size)), Image.LANCZOS) for im in images]
return images, usage
diff --git a/harness/codeverse/models/health.py b/harness/codeverse/models/health.py
index b905298c..dbd942de 100644
--- a/harness/codeverse/models/health.py
+++ b/harness/codeverse/models/health.py
@@ -72,7 +72,7 @@ def probe(
sample: int = DEFAULT_SAMPLE,
timeout_s: float = DEFAULT_TIMEOUT_S,
) -> Health:
- """One tiny call per sampled key, in parallel, no retries. Never raises."""
+ """One workload-sized call (``PROBE_TOKENS``) per sampled key, in parallel, no retries. Never raises."""
from codeverse.contracts.chat import ChatMessage, ChatRequest
prompt = _probe_prompt()
@@ -121,9 +121,13 @@ def _bare_model(model_id: str, timeout_s: float):
from codeverse.models.gemini import GeminiModel
return GeminiModel(name, timeout_s=timeout_s, max_attempts=1, storm_attempts=0)
- from codeverse.models import get_chat_model # other providers: no bare constructor needed yet
+ if provider == "anthropic":
+ from codeverse.models.anthropic import AnthropicModel
- return get_chat_model(model_id)
+ return AnthropicModel(name, timeout_s=timeout_s, max_attempts=1)
+ from codeverse.models.openai import OpenAIModel # everything else, as models.registry does
+
+ return OpenAIModel(name, timeout_s=timeout_s, max_attempts=1)
# --------------------------------------------------------------------------- siblings
diff --git a/harness/codeverse/models/openai.py b/harness/codeverse/models/openai.py
index 7143eefe..508f0e23 100644
--- a/harness/codeverse/models/openai.py
+++ b/harness/codeverse/models/openai.py
@@ -28,7 +28,7 @@ from codeverse.contracts.common import Usage
from codeverse.models.base import ModelError
from codeverse.models.parts import Stopwatch, classify_sdk_exception, image_b64, with_logged_retries
from codeverse.models.pricing import estimate_cost
-from codeverse.models.retry import MAX_WAIT_S
+from codeverse.models.retry import MAX_WAIT_S, cause_for
from codeverse.models.schema_utils import (
JsonParseError,
inline_refs,
@@ -290,12 +290,14 @@ class OpenAIModel:
try:
return self._once(kwargs, request)
except Exception as exc2: # noqa: BLE001
- raise classify_exception(exc2) from exc2
- raise err from exc
+ err2 = classify_exception(exc2)
+ raise err2 from cause_for(err2, exc2)
+ raise err from cause_for(err, exc)
return with_logged_retries(attempt, label="openai", model=self.model,
attempts=self.max_attempts, base_delay=self.base_delay,
- max_delay=self.max_delay, sleep=self._sleep, log=log)
+ max_delay=self.max_delay, sleep=self._sleep, log=log,
+ max_wait_s=request.max_wait_s)
def _once(self, kwargs: dict[str, Any], request: ChatRequest) -> ChatResponse:
client = self.client()
diff --git a/harness/codeverse/models/parts.py b/harness/codeverse/models/parts.py
index 097958d0..87458c3e 100644
--- a/harness/codeverse/models/parts.py
+++ b/harness/codeverse/models/parts.py
@@ -65,15 +65,10 @@ class Stopwatch:
class BoundedCache[T]:
- """Thread-safe bounded LRU ``str -> T`` map for per-call-id provider state.
-
- Replaces ``gemini_convert._SignatureCache`` (thought signatures, ``bytes``)
- and ``anthropic_convert._ThinkingCache`` (thinking blocks, ``list[dict]``):
- both had the identical ``__init__``/``put``/``get`` — an ``OrderedDict``
- under a lock, ``put`` skipping falsy values, ``move_to_end`` then
- ``popitem(last=False)`` until under capacity — and differed only in value
- type and capacity. ``get`` returns ``None`` on a miss; a caller that needs
- the old copy/empty-default shape wraps it: ``list(cache.get(k) or [])``.
+ """Thread-safe bounded LRU ``str -> T`` map for per-call-id provider state
+ (gemini thought signatures, anthropic thinking blocks). ``put`` skips falsy
+ values; ``get`` returns ``None`` on a miss — a caller that needs a copy /
+ empty default wraps it: ``list(cache.get(k) or [])``.
"""
def __init__(self, capacity: int) -> None:
@@ -108,8 +103,6 @@ def classify_sdk_exception(exc: BaseException, sdk: Any, label: str,
out" and "connection error" to tell a provider outage from a model failure, so the
f"{label} …" forms below must stay byte-identical to what each adapter emitted.
"""
- from codeverse.models.base import ModelError
-
if isinstance(exc, ModelError):
return exc
if isinstance(exc, sdk.APIStatusError):
@@ -126,19 +119,21 @@ def classify_sdk_exception(exc: BaseException, sdk: Any, label: str,
def with_logged_retries(attempt: Any, *, label: str, model: str, attempts: int,
- base_delay: float, max_delay: float, sleep: Any, log: Any) -> Any:
+ base_delay: float, max_delay: float, sleep: Any, log: Any,
+ max_wait_s: float | None = None) -> Any:
"""``with_retries`` plus the one log line both SDK adapters write.
- ChatRequest.max_wait_s is not honoured here: with_retries has no deadline, and its
- 6 attempts x <= 5 s backoff bound one call to ~20 s of waiting plus the round-trips.
+ ``max_wait_s`` is the request's ``ChatRequest.max_wait_s`` deadline (retries and
+ their waits included; ``None`` = ``RETRY_DEADLINE_S``); the backoff itself is
+ ``attempts`` x <= ``MAX_WAIT_S`` (3 s), the round-trips are the real cost.
"""
- from codeverse.models.base import ModelError
- from codeverse.models.retry import with_retries
+ from codeverse.models.retry import RETRY_DEADLINE_S, with_retries
def on_retry(n: int, exc: BaseException, delay: float) -> None:
log.warning("%s %s attempt %d/%d failed (%s); retrying in %.1fs",
label, model, n, attempts, exc, delay)
+ budget = RETRY_DEADLINE_S if max_wait_s is None else min(RETRY_DEADLINE_S, float(max_wait_s))
return with_retries(attempt, is_retryable=lambda e: isinstance(e, ModelError) and e.retryable,
- attempts=attempts, base_delay=base_delay, max_delay=max_delay,
+ attempts=attempts, base_delay=base_delay, max_delay=max_delay, max_total_s=budget,
on_retry=on_retry, sleep=sleep)
diff --git a/harness/codeverse/models/retry.py b/harness/codeverse/models/retry.py
index 90117509..763f94d7 100644
--- a/harness/codeverse/models/retry.py
+++ b/harness/codeverse/models/retry.py
@@ -1,6 +1,17 @@
-"""Multi-key pool with per-key rate limiting, 429 cooldown and a health score.
+"""The scheduling machine under every model call (merged 2026-08-28, d512ccc):
-Used by ``GeminiModel`` (22 keys on the owner's box) but provider-neutral.
+1. ``KeyPool`` (+ ``TokenBucket``): a multi-key pool with per-key RPM/TPM buckets,
+ 429 cooldown and a health score — used by ``GeminiModel`` (22 keys on the owner's
+ box) but provider-neutral;
+2. ``StormGate`` + ``storm_gate()`` / ``all_gates()``: the process-wide 503 back-pressure
+ one model's callers share (ships OFF, ``Settings.rate.storm_gate``; docs/COST.md §21);
+3. the two retry loops — ``with_retries`` (single key, SDK adapters) and
+ ``rotate_with_retries`` (the pool, the gate, hedging, the ``RETRY_DEADLINE_S`` deadline);
+4. ``request_parts`` / ``request_tokens``: the prompt-token estimate the pool reserves.
+
+Every wait anywhere in here clips to ``MAX_WAIT_S``.
+
+The pool:
* ``acquire()`` picks the next healthy key round-robin, honouring per-key
RPM / TPM token buckets and 429 cool-downs; it blocks (bounded) when every
@@ -11,8 +22,10 @@ Used by ``GeminiModel`` (22 keys on the owner's box) but provider-neutral.
may be out at once, and :meth:`KeyPool.release` (a ``finally`` in
``rotate_with_retries``) hands the slot back. ``try_acquire()`` is the
never-waiting variant a hedged retry uses for its extra keys.
-* ``report(key, outcome)`` feeds back ``ok | 429 | 5xx | error | dead`` so the
- pool can cool a key down and adjust its health score. ``dead`` is for
+* ``report(key, outcome)`` feeds back ``ok | 429 | 5xx | error | dead | skip`` so the
+ pool can cool a key down and adjust its health score. ``skip`` is a content
+ failure the key did not cause (bad JSON, empty candidates): only the token
+ reconciliation runs, health and counters are untouched. ``dead`` is for
key-scoped auth/permission failures (revoked / suspended / invalid key): the
key is benched for ``dead_cooldown_s`` (default one hour) and re-probed once
that elapses — a dead key must never keep failing its share of calls.
@@ -38,7 +51,7 @@ from typing import TYPE_CHECKING, Any, Literal
from codeverse.contracts.chat import ChatRequest, ImagePart, TextPart, ToolResultPart
-Outcome = Literal["ok", "429", "5xx", "error", "dead"]
+Outcome = Literal["ok", "429", "5xx", "error", "dead", "skip"]
class KeyPoolExhausted(RuntimeError):
@@ -352,7 +365,7 @@ class KeyPool:
st.health = 0.0
st.dead_until = max(st.dead_until, now + self._dead_cooldown_s)
st.cooldown_until = max(st.cooldown_until, st.dead_until)
- else:
+ elif outcome != "skip":
st.n_error += 1
st.health *= 0.9
if st.tpm is not None and (tokens or reserved):
@@ -434,7 +447,7 @@ class KeyPool:
# ===================================================================== storm
class StormGate:
- """Shared 503 back-pressure for one model. See the module docstring."""
+ """Shared 503 back-pressure for one model (section 2 of the module docstring)."""
def __init__(
self,
@@ -590,7 +603,7 @@ log = logging.getLogger(__name__)
OnRetry = Callable[[int, BaseException, float], None]
-def _cause(err: BaseException, exc: BaseException | None) -> BaseException | None:
+def cause_for(err: BaseException, exc: BaseException | None) -> BaseException | None:
"""The cause to raise ``err`` from — ``None`` when it would be ``err`` itself.
Every adapter's ``classify()`` returns an already-classified ``ModelError``
@@ -630,29 +643,38 @@ def with_retries[T](
attempts: int = 6,
base_delay: float = 1.0,
max_delay: float = MAX_WAIT_S,
+ max_total_s: float | None = None,
on_retry: OnRetry | None = None,
sleep: Callable[[float], None] = time.sleep,
jitter: bool = True,
) -> T:
- """Call ``fn`` up to ``attempts`` times.
-
- Re-raises the last exception when it is not retryable (per ``is_retryable``)
- or when attempts are exhausted. ``on_retry(attempt, exc, delay)`` is called
- before each sleep (attempt is the 1-based index of the attempt that failed).
+ """Call ``fn`` up to ``attempts`` times, within ``max_total_s`` when given.
+
+ Re-raises the last exception when it is not retryable (per ``is_retryable``),
+ when attempts are exhausted, or when the next backoff would cross the deadline
+ (``ChatRequest.max_wait_s`` on the SDK adapters). ``on_retry(attempt, exc, delay)``
+ is called before each sleep (attempt is the 1-based index of the attempt that
+ failed). A raised exception carrying an ``attempts`` attribute (``ModelError``)
+ is stamped with the number of round-trips issued.
"""
if attempts < 1:
raise ValueError("attempts must be >= 1")
+ deadline = None if max_total_s is None else time.monotonic() + max_total_s
last: BaseException | None = None
for attempt in range(1, attempts + 1):
try:
return fn()
except BaseException as exc: # noqa: BLE001 - classification is delegated
last = exc
+ if hasattr(exc, "attempts"):
+ exc.attempts = attempt
if attempt >= attempts or not is_retryable(exc):
raise
delay = backoff_delay(
attempt, base_delay=base_delay, max_delay=max_delay, jitter=jitter
)
+ if deadline is not None and time.monotonic() + delay >= deadline:
+ raise
if on_retry is not None:
on_retry(attempt, exc, delay)
sleep(delay)
@@ -712,12 +734,13 @@ def rotate_with_retries[T](
call is it a
* **capacity storm** (the whole pool is out of capacity, the backstop): that gets
its own patience budget — up to ``storm_attempts`` waits with
- backoff capped at ``storm_max_delay`` = ``MAX_WAIT_S`` per wait (the house rule), so
- patience comes from the NUMBER of waits (60 x <=5 s ~ 5 min) rather than
- from long sleeps that do NOT consume ``max_attempts``. Observed 2026-08-23: a
+ backoff capped at ``storm_max_delay`` = ``MAX_WAIT_S`` (3 s) per wait (the house
+ rule), so patience comes from the NUMBER of waits (60 x <=3 s) rather than from
+ long sleeps that do NOT consume ``max_attempts`` — all inside the
+ ``RETRY_DEADLINE_S`` / ``max_total_s`` deadline. Observed 2026-08-23: a
multi-minute gemini-3.7-flash "high demand" outage killed 8 bench runs
- under the plain 6-attempt budget. A ``storm_gate`` (see
- :mod:`codeverse.models.retry`) shares that discovery across the process:
+ under the plain 6-attempt budget. A ``storm_gate`` (:class:`StormGate`
+ above) shares that discovery across the process:
workers park at the gate instead of each spending a round-trip to learn
the model is out of capacity, and one probe at a time reopens it.
* other retryable errors → exponential backoff + jitter until
@@ -786,12 +809,9 @@ def rotate_with_retries[T](
err = classify(exc)
outcome = outcome_of(err)
# a key that looks dead is reported "error" now and benched only once a
- # sibling proves the request itself is fine (bench())
- # a charged-but-invalid reply (bad JSON with real usage) DID consume
- # its prompt tokens: report them so the pool does not refund a spent
- # reservation (a 42k-token invalid reply used to hand the key 42k TPM
- # back). Failures billed nothing (429 / 503 / transport) carry an
- # empty ``ModelError.usage`` and are refunded exactly as before.
+ # sibling proves the request itself is fine (bench()). ``tokens`` is what
+ # the failure was billed: a 42k-token invalid reply ("skip") used to be
+ # refunded as if it had never reached the model.
consumed = getattr(err, "usage", None)
pool.report(
key,
@@ -866,7 +886,7 @@ def rotate_with_retries[T](
# sleeps: a free rotation must not out-live the caller's budget
log.warning("%s giving up after %.0f s of retrying (%s)",
label, max_total_s, last_err)
- raise last_err from _cause(last_err, last_exc)
+ raise last_err from cause_for(last_err, last_exc)
attempt += 1
# never go back to a key that looked dead this call; throttled keys are
# excluded while an untried one remains, else acquire() waits for a cooldown
@@ -880,7 +900,7 @@ def rotate_with_retries[T](
log.warning("%s budget of %.0f s spent waiting at the storm gate", label, max_total_s)
gate_err = last_err or classify(TimeoutError(
f"{label}: retry budget spent waiting for capacity"))
- raise gate_err from _cause(gate_err, last_exc)
+ raise gate_err from cause_for(gate_err, last_exc)
budget_left = None if deadline == float("inf") else max(0.0, deadline - monotonic())
try:
# the key/slot wait must fit the remaining budget, never outlive it
@@ -933,11 +953,11 @@ def rotate_with_retries[T](
if out_of_time():
log.warning("%s giving up after %.0f s of retrying (%s)",
label, max_total_s, err)
- raise err from _cause(err, exc)
+ raise err from cause_for(err, exc)
log.warning("%s key …%s looks dead (%s); rotating", label, key[-4:], err)
attempt -= 1
continue
- raise err from _cause(err, exc) # every key failed the same way: not the keys' fault
+ raise err from cause_for(err, exc) # every key failed the same way: not the keys' fault
if is_storm(err) and storm < storm_attempts and not out_of_time():
failed_keys.update(t.key for t in tries)
if len(failed_keys) < len(pool):
@@ -978,7 +998,7 @@ def rotate_with_retries[T](
if out_of_time():
log.warning("%s giving up after %.0f s of retrying (%s)",
label, max_total_s, err)
- raise err from _cause(err, exc)
+ raise err from cause_for(err, exc)
# an untried key remains: rotation is free, only a courtesy pause
attempt -= 1
log.warning(
@@ -988,10 +1008,10 @@ def rotate_with_retries[T](
continue
if not err.retryable or attempt >= max_attempts:
bench()
- raise err from _cause(err, exc)
+ raise err from cause_for(err, exc)
if out_of_time():
log.warning("%s giving up after %.0f s of retrying (%s)", label, max_total_s, err)
- raise err from _cause(err, exc)
+ raise err from cause_for(err, exc)
delay = backoff_delay(attempt, base_delay=base_delay, max_delay=max_delay)
log.warning(
"%s attempt %d/%d failed (%s); retrying in %.1fs",
diff --git a/harness/codeverse/models/schema_utils.py b/harness/codeverse/models/schema_utils.py
index 65c3eff7..cc296743 100644
--- a/harness/codeverse/models/schema_utils.py
+++ b/harness/codeverse/models/schema_utils.py
@@ -1,4 +1,5 @@
-"""JSON-schema adapters for the three providers + a fence-tolerant JSON parser.
+"""JSON-schema adapters for the three providers, a fence-tolerant JSON parser and
+``ask_structured`` (one schema-validated call → object | None, usage, error).
Pydantic v2 emits ``$defs``/``$ref``, ``title``, ``default``, ``const``,
``prefixItems`` (tuples), ``anyOf [.., {"type":"null"}]`` and so on. Each
@@ -24,8 +25,15 @@ from __future__ import annotations
import copy
import json
import re
+from collections.abc import Sequence
from typing import Any
+from pydantic import BaseModel, ValidationError
+
+from codeverse.contracts.chat import ChatMessage, ChatRequest, ImagePart
+from codeverse.contracts.common import Usage
+from codeverse.models.base import ModelError
+
# keys Gemini's Schema type rejects or ignores
_GEMINI_DROP = {
"title",
@@ -355,3 +363,30 @@ def parse_json_lenient(text: str) -> Any:
except json.JSONDecodeError:
pass
raise JsonParseError(f"no JSON value found: {last_err} — head={s[:120]!r}")
+
+
+def ask_structured(model: Any, schema: type[BaseModel], *, system: str, text: str,
+ images: Sequence[ImagePart] = (), temperature: float,
+ label: str) -> tuple[Any, Usage, str]:
+ """One structured call → ``(validated object | None, usage, error)``.
+
+ Five callers (reference gate / image-prompt plan / mismatch diff, the texture
+ material plan and the scene texture pack) built the same ChatRequest
+ (thinking="low", the 65 536 ceiling, the 900 s wait), caught the same two failure
+ families and parsed the same two ways. They differ only in what they RETURN on
+ failure, which is why this hands the error back rather than raising or deciding.
+ Not for callers that need the failed call's own ``Usage`` or separate parse
+ errors from call errors (``judges.pairwise``, ``tracks.planner``).
+ """
+ req = ChatRequest(messages=[ChatMessage.user(text, images=list(images) or None)], system=system,
+ response_schema=schema.model_json_schema(), temperature=temperature,
+ thinking="low", max_output_tokens=65_536, max_wait_s=900.0, label=label)
+ try:
+ resp = model.generate(req)
+ except ModelError as e:
+ return None, Usage(), f"call failed: {e}"
+ payload = resp.parsed if resp.parsed is not None else parse_json_lenient(resp.text)
+ try:
+ return schema.model_validate(payload), resp.usage, ""
+ except (ValidationError, TypeError) as e:
+ return None, resp.usage, f"answer unparsable: {e}"
diff --git a/harness/codeverse/prompts/__init__.py b/harness/codeverse/prompts/__init__.py
index ebdae4bb..ebcdef2e 100644
--- a/harness/codeverse/prompts/__init__.py
+++ b/harness/codeverse/prompts/__init__.py
@@ -66,7 +66,3 @@ def render(rel_path: str, **ctx: Any) -> str:
def prompt_hash(text: str) -> str:
return hashlib.sha256(text.encode()).hexdigest()[:12]
-
-
-def list_prompts() -> list[str]:
- return sorted(str(p.relative_to(PROMPTS_DIR)) for p in PROMPTS_DIR.rglob("*") if p.suffix in (".md", ".j2", ".txt", ".py", ".js"))
diff --git a/harness/codeverse/prompts/blender/system_v0.md b/harness/codeverse/prompts/blender/system_v0.md
deleted file mode 100644
index 778aa3d1..00000000
--- a/harness/codeverse/prompts/blender/system_v0.md
+++ /dev/null
@@ -1 +0,0 @@
-You are an expert blender 3D modeller writing RAW code (no SDKs, no helper libraries). Follow the contract exactly; exact numbers beat adjectives.
diff --git a/harness/codeverse/prompts/cadquery/contract.md b/harness/codeverse/prompts/cadquery/contract.md
index 82f944b7..6cc44738 100644
--- a/harness/codeverse/prompts/cadquery/contract.md
+++ b/harness/codeverse/prompts/cadquery/contract.md
@@ -29,7 +29,7 @@ No `OCP`/`OCC` direct calls, no numpy, no file I/O, no `cq.exporters`, no
## Forbidden / limits
* No `show_object`, `exporters.export`, `open(`, `os`, `sys`, `subprocess`, network.
* ≤ 40 parts; each part a valid solid (`.val().isValid()`), no empty Workplanes; fillets
- < 0.45 × thinnest adjacent wall; feature sizes ≥ 0.5 mm; build < 120 s.
+ < 0.45 × thinnest adjacent wall; feature sizes ≥ 0.5 mm; build < 300 s.
* Do not union parts across plan boundaries (keep one solid per part); small intentional
overlap of **0.5–2 mm** between touching parts is required — that is how they "weld";
deeper than 2 mm and the connectivity gate calls it interpenetration.
diff --git a/harness/codeverse/prompts/cadquery/system_v0.md b/harness/codeverse/prompts/cadquery/system_v0.md
deleted file mode 100644
index 00fbefdc..00000000
--- a/harness/codeverse/prompts/cadquery/system_v0.md
+++ /dev/null
@@ -1 +0,0 @@
-You are an expert cadquery 3D modeller writing RAW code (no SDKs, no helper libraries). Follow the contract exactly; exact numbers beat adjectives.
diff --git a/harness/codeverse/prompts/glsl_shader/system_v0.md b/harness/codeverse/prompts/glsl_shader/system_v0.md
deleted file mode 100644
index b031a4d8..00000000
--- a/harness/codeverse/prompts/glsl_shader/system_v0.md
+++ /dev/null
@@ -1 +0,0 @@
-You are an expert GLSL fragment-shader artist (Shadertoy style) writing RAW code for a headless harness. Follow the contract exactly: no #version/uniform redeclarations (shader) · no window/context creation (program) · everything animates with time. Render and LOOK at your frames before finishing.
diff --git a/harness/codeverse/prompts/opengl_python/system_v0.md b/harness/codeverse/prompts/opengl_python/system_v0.md
deleted file mode 100644
index 372c8502..00000000
--- a/harness/codeverse/prompts/opengl_python/system_v0.md
+++ /dev/null
@@ -1 +0,0 @@
-You are an expert raw OpenGL (moderngl) graphics programmer writing RAW code for a headless harness. Follow the contract exactly: no #version/uniform redeclarations (shader) · no window/context creation (program) · everything animates with time. Render and LOOK at your frames before finishing.
diff --git a/harness/codeverse/prompts/scene_threejs/system_v0.md b/harness/codeverse/prompts/scene_threejs/system_v0.md
deleted file mode 100644
index 776d0923..00000000
--- a/harness/codeverse/prompts/scene_threejs/system_v0.md
+++ /dev/null
@@ -1 +0,0 @@
-You are an expert three.js + GLSL graphics programmer writing RAW ESM modules for a multi-file scene. No SDKs, no DOM, no fetch, no CDN imports: `import * as THREE from 'three'` only. Exact numbers beat adjectives.
diff --git a/harness/codeverse/prompts/threejs/system_v0.md b/harness/codeverse/prompts/threejs/system_v0.md
deleted file mode 100644
index f7645a7f..00000000
--- a/harness/codeverse/prompts/threejs/system_v0.md
+++ /dev/null
@@ -1 +0,0 @@
-You are an expert threejs 3D modeller writing RAW code (no SDKs, no helper libraries). Follow the contract exactly; exact numbers beat adjectives.
diff --git a/harness/codeverse/prompts/urdf/system_v0.md b/harness/codeverse/prompts/urdf/system_v0.md
deleted file mode 100644
index 6e8bd516..00000000
--- a/harness/codeverse/prompts/urdf/system_v0.md
+++ /dev/null
@@ -1 +0,0 @@
-You are an expert in Blender bpy and URDF writing RAW code (no SDKs). Links are bpy objects named exactly as the URDF links; joints are native URDF with origins/axes in the parent link frame. Exact numbers beat adjectives.
diff --git a/harness/codeverse/reference.py b/harness/codeverse/reference.py
index fcf9bd7e..2cc2f2b2 100644
--- a/harness/codeverse/reference.py
+++ b/harness/codeverse/reference.py
@@ -2,13 +2,13 @@
Today a prompt goes to a planner that has never SEEN the object. This module
synthesizes a neutral studio product shot of the brief with the image model,
-**validates** it (:mod:`.gate`), and hands it to the rest of the harness as the
+**validates** it (``check_plausible`` + ``decide``), and hands it to the rest of the harness as the
fidelity anchor: the planner sees it, the generator's ``compare_reference`` tool
sees it, the silhouette gate measures against it and the judge scores against it
(:class:`codeverse.judges.vlm_judge.ReferenceJudge`).
A synthesized reference is never ground truth and is marked as such everywhere
-(:mod:`.attach`): the user's own ``--image`` wins, the brief's dimensions win,
+(``attach`` / ``SYNTH_NOTE``): the user's own ``--image`` wins, the brief's dimensions win,
and a picture that fails the gate is discarded rather than chased.
Entry points::
@@ -36,11 +36,10 @@ from typing import Any, Literal
from pydantic import BaseModel, Field, ValidationError
from codeverse.config import get_settings
-from codeverse.contracts.chat import ChatMessage, ChatRequest, ImagePart
+from codeverse.contracts.chat import ImagePart
from codeverse.contracts.common import Usage
from codeverse.contracts.spec import ReferenceImage, Spec
-from codeverse.models.base import ModelError
-from codeverse.models.schema_utils import parse_json_lenient
+from codeverse.models.schema_utils import ask_structured
from codeverse.prompts import prompt_hash
log = logging.getLogger(__name__)
@@ -71,7 +70,7 @@ class PlausibilityVerdict(BaseModel):
"""One vision call's answer about ONE candidate reference image.
``ok`` is computed in code from the booleans + ``contradictions`` (see
- :func:`codeverse.reference.gate.decide`), never taken from the model.
+ :func:`decide`), never taken from the model.
"""
ok: bool = False
@@ -121,7 +120,7 @@ class ReferenceView(BaseModel):
class ReferenceSet(BaseModel):
- """The result of :func:`codeverse.reference.synth.synth_reference`.
+ """The result of :func:`synth_reference`.
``accepted`` is what may be attached to a spec; ``rejected`` is kept only so
the run can record *why* the fallback to no-reference happened.
@@ -308,7 +307,7 @@ VIEW_CLAUSE: dict[str, str] = {
#: order views are requested in as ``n_views`` grows. The straight-on ``front``
#: elevation comes second on purpose: it is the SILHOUETTE TARGET, and the harness
-#: measures IoU against the run's ``front`` render (``tracks.reference.silhouette_gate``),
+#: measures IoU against the run's ``front`` render (``tracks.static_object.silhouette_gate``),
#: so the two cameras must agree. The 3/4 shot carries the part inventory.
VIEW_ORDER: tuple[str, ...] = ("three_quarter", "front", "side", "back")
@@ -442,8 +441,6 @@ def image_dir(key: str, *, cache_dir: Path | None = None) -> Path:
#: relative aspect disagreement above which the picture's proportions are not a target
ASPECT_TOL = 0.25
-#: the measured silhouette criterion's score when the reference contradicts the brief
-NEUTRAL_SCORE = 0.5
#: brief keys that describe a HORIZONTAL extent of the object
_WIDTH_KEYS = ("width", "length", "depth", "diameter")
@@ -558,31 +555,6 @@ def decide(ans: GateAnswer, *, model_id: str = "") -> PlausibilityVerdict:
)
-def _ask(model: Any, schema: type[BaseModel], *, system: str, text: str,
- images: list[ImagePart] | None = None, temperature: float,
- label: str) -> tuple[Any, Usage, str]:
- """One structured call → ``(validated object | None, usage, error)``.
-
- The three callers below built the same ChatRequest (thinking="low", the 65 536
- ceiling, the 900 s wait), caught the same two failure families and parsed the same
- two ways — each was a copy carried in from its own pre-merge file. They differ only
- in what they RETURN on failure, which is why this hands the error back rather than
- raising or deciding.
- """
- req = ChatRequest(messages=[ChatMessage.user(text, images=images)], system=system,
- response_schema=schema.model_json_schema(), temperature=temperature,
- thinking="low", max_output_tokens=65_536, max_wait_s=900.0, label=label)
- try:
- resp = model.generate(req)
- except ModelError as e:
- return None, Usage(), f"call failed: {e}"
- payload = resp.parsed if resp.parsed is not None else parse_json_lenient(resp.text)
- try:
- return schema.model_validate(payload), resp.usage, ""
- except (ValidationError, TypeError) as e:
- return None, resp.usage, f"answer unparsable: {e}"
-
-
def check_plausible(
image: Path | str,
spec: Spec,
@@ -597,7 +569,7 @@ def check_plausible(
if not path.is_file():
return PlausibilityVerdict(reason=f"image missing: {path.name}", model_id=model_id), Usage()
text = GATE_USER.format(brief=brief_text(spec), constraints=visual_constraints(spec))
- ans, usage, err = _ask(model, GateAnswer, system=GATE_SYSTEM, text=text,
+ ans, usage, err = ask_structured(model, GateAnswer, system=GATE_SYSTEM, text=text,
images=[ImagePart(path=str(path), label="CANDIDATE REFERENCE")],
temperature=temperature, label="reference_gate")
if err:
@@ -698,7 +670,7 @@ def image_prompt_plan(spec: Spec, view_names: list[str], *, model: Any, temperat
if model is None:
return _fallback_plan(spec, view_names), Usage()
text = IMAGE_PROMPT_USER.format(brief=brief_text(spec), n_views=len(view_names), views=", ".join(view_names))
- plan, usage, err = _ask(model, ImagePromptPlan, system=IMAGE_PROMPT_SYSTEM, text=text,
+ plan, usage, err = ask_structured(model, ImagePromptPlan, system=IMAGE_PROMPT_SYSTEM, text=text,
temperature=temperature, label="reference_prompt")
if err:
log.warning("reference prompt writer %s; using the brief verbatim", err)
@@ -812,8 +784,6 @@ def _publish(rs: ReferenceSet, out_dir: Path | None) -> ReferenceSet:
# ===================================================================== mismatch
MAX_MISMATCHES = 6
-#: how many mismatches become refine tasks
-TOP_TASKS = 3
_SYNTH_LINE = ("The REFERENCE was SYNTHESIZED from the brief by an image model — it is a shape target, "
"not ground truth. Where it disagrees with the brief, the BRIEF wins and it is not a mismatch.")
@@ -860,7 +830,7 @@ def compare(
)
images = [ImagePart(path=str(p), label=f"REFERENCE {i + 1}") for i, p in enumerate(refs)]
images += [ImagePart(path=str(p), label=f"RENDER {i + 1} ({p.stem})") for i, p in enumerate(rens)]
- ans, usage, err = _ask(model, DiffAnswer, system=DIFF_SYSTEM, text=text, images=images,
+ ans, usage, err = ask_structured(model, DiffAnswer, system=DIFF_SYSTEM, text=text, images=images,
temperature=temperature, label="reference_diff")
diff.usage = usage
if err:
@@ -890,24 +860,6 @@ def _measured_block(diff: ReferenceDiff) -> str:
return line + ". This is a fact; do not re-estimate it.\n"
-def refine_tasks(diff: ReferenceDiff, *, top: int = TOP_TASKS) -> list[Any]:
- """The top mismatches as priority-1 ``RefineTask``s (``source='gate'`` so the
- round policy protects them from being trimmed as judge chatter)."""
- from codeverse.orchestrator import RefineTask
-
- out = []
- for m in diff.top(top):
- out.append(RefineTask(
- target=m.target or "overall",
- kind="reference",
- instruction=(f"Reference mismatch ({m.kind.replace('_', ' ')}, {m.severity}): {m.detail} "
- f"Look at the reference image again and fix this specifically."),
- priority=1,
- source="gate",
- ))
- return out
-
-
# ===================================================================== run
ARTIFACT_SUBDIR = "reference"
@@ -979,6 +931,6 @@ __all__ = [
"SYNTH_NOTE", "SYNTH_TAG", "Mismatch", "PlausibilityVerdict", "ReferenceDiff", "ReferenceSet",
"ReferenceView", "attach", "check_plausible", "compare", "compose_image_prompt", "conflict_note",
"decide", "dimension_conflict", "expected_aspect_range",
- "ground_spec", "has_user_references", "is_synthetic", "reference_images", "refine_tasks",
+ "ground_spec", "has_user_references", "is_synthetic", "reference_images",
"synth_reference", "views_for",
]
diff --git a/harness/codeverse/texturing/__init__.py b/harness/codeverse/texturing/__init__.py
index aad0e2a9..c465079e 100644
--- a/harness/codeverse/texturing/__init__.py
+++ b/harness/codeverse/texturing/__init__.py
@@ -1,34 +1,3 @@
"""Text-to-image texturing: a language-agnostic pass on the canonical GLB
-(objects) and a scene texture pack (scenes). See ``run.texture_pass`` and
-``scene_pack.scene_texture_pack``."""
-
-from codeverse.texturing.apply import ApplyReport, apply_textures
-from codeverse.texturing.generate import (
- FakeImageModel,
- GateResult,
- SeamGateResult,
- TextureAsset,
- TextureSet,
- generate_textures,
- judge_gate,
- make_tileable,
- seam_gate,
- seam_score,
-)
-from codeverse.texturing.plan import (
- ScenePack,
- TexturePart,
- TexturePlan,
- default_plan,
- material_plan,
- scene_texture_pack,
- texture_pack_prompt,
-)
-from codeverse.texturing.run import TextureReport, texture_pass
-
-__all__ = [
- "ApplyReport", "apply_textures", "GateResult", "SeamGateResult", "judge_gate", "seam_gate",
- "FakeImageModel", "TextureAsset", "TextureSet", "generate_textures", "TexturePart", "TexturePlan",
- "default_plan", "material_plan", "TextureReport", "texture_pass", "ScenePack", "scene_texture_pack",
- "texture_pack_prompt", "make_tileable", "seam_score",
-]
+(objects, ``run.texture_pass``) and a scene texture pack (scenes,
+``plan.scene_texture_pack``). Import from the submodules."""
diff --git a/harness/codeverse/texturing/apply.py b/harness/codeverse/texturing/apply.py
index 4ebefd86..13ac186f 100644
--- a/harness/codeverse/texturing/apply.py
+++ b/harness/codeverse/texturing/apply.py
@@ -354,8 +354,8 @@ def _material(
) -> trimesh.visual.material.PBRMaterial:
"""PBR material for one texture id.
- Beyond the albedo the material carries two maps DERIVED from that albedo (see
- ``texturing.maps``): a metallic/roughness texture so the grain also modulates
+ Beyond the albedo the material carries two maps DERIVED from that albedo
+ (``derived_maps`` below): a metallic/roughness texture so the grain also modulates
the specular lobe, and a normal map so it catches light in relief. They cost
no extra image call and cannot disagree with the colour. ``derived_maps=False``
reproduces the albedo-only material the pass shipped before.
@@ -489,21 +489,6 @@ def verify_textured_glb(glb_in: Path, glb_out: Path, *, expected_textured: int)
return warnings
-def textured_summary(glb: Path | str) -> dict[str, int]:
- """Counts for reports / tests: geometries, textured geometries, distinct images."""
- s = load_scene(glb)
- textured = 0
- imgs: set[int] = set()
- for g in s.geometry.values():
- vis = getattr(g, "visual", None)
- mat = getattr(vis, "material", None)
- tex = getattr(mat, "baseColorTexture", None)
- if tex is not None:
- textured += 1
- imgs.add(id(tex))
- return {"geometries": len(s.geometry), "textured": textured, "images": len(imgs)}
-
-
# ===================================================================== normalise
diff --git a/harness/codeverse/texturing/generate.py b/harness/codeverse/texturing/generate.py
index 021807f3..12860ca1 100644
--- a/harness/codeverse/texturing/generate.py
+++ b/harness/codeverse/texturing/generate.py
@@ -6,8 +6,6 @@
* ``make_tileable(img, border_frac)``: cross-fades each edge band with the mirrored
opposite band (a smoothstep ramp over ``border_frac`` of the width) so the wrap
seam disappears; content in the middle is untouched.
-* ``offset_check(img)``: half-offset the image (seams move to the centre) — the
- classic visual check; returns the offset image for inspection.
* ``fit_size`` / ``save_texture``: power-of-two downscale + PNG or JPEG q90.
"""
@@ -33,6 +31,8 @@ from codeverse.contracts.common import Usage
from codeverse.contracts.plan import AcceptanceItem, StaticPlan
from codeverse.contracts.spec import Spec
from codeverse.conventions import OBJECT_VIEWS_QUICK, ViewPreset
+from codeverse.judges.base import JudgeInput, plan_digest
+from codeverse.judges.rubrics import is_degraded
#: seam score above which a texture is considered NOT tileable (after make_tileable)
SEAM_MAX = 0.08
@@ -94,14 +94,6 @@ def make_tileable(img: Image.Image, border_frac: float = 0.12) -> Image.Image:
return Image.fromarray((np.clip(out, 0.0, 1.0) * 255.0 + 0.5).astype(np.uint8), "RGB")
-def offset_check(img: Image.Image) -> Image.Image:
- """Roll the image by half its size so the wrap seams sit in the centre."""
- a = _arr(img)
- h, w = a.shape[:2]
- rolled = np.roll(np.roll(a, w // 2, axis=1), h // 2, axis=0)
- return Image.fromarray((rolled * 255.0 + 0.5).astype(np.uint8), "RGB")
-
-
def fit_size(img: Image.Image, size: int) -> Image.Image:
"""Square, power-of-two-ish downscale to ``size`` (never upscales)."""
w, h = img.size
@@ -124,16 +116,6 @@ def save_texture(img: Image.Image, path: Path, *, fmt: str | None = None, qualit
return path
-def tile_preview(img: Image.Image, reps: int = 2, size: int = 512) -> Image.Image:
- """``reps × reps`` repetition for eyeballing tileability."""
- t = img.convert("RGB").resize((size // reps, size // reps), Image.LANCZOS)
- out = Image.new("RGB", (t.size[0] * reps, t.size[1] * reps))
- for i in range(reps):
- for j in range(reps):
- out.paste(t, (i * t.size[0], j * t.size[1]))
- return out
-
-
# ===================================================================== generate
log = logging.getLogger(__name__)
@@ -195,10 +177,6 @@ class FakeImageModel:
def id(self) -> str:
return f"fake-image:{self.model}"
- def generate(self, prompt: str, *, size: int = 1024, n: int = 1, seed: int | None = None,
- reference_images: Sequence[Any] = ()) -> list[Image.Image]:
- return self.generate_with_usage(prompt, size=size, n=n, seed=seed, reference_images=reference_images)[0]
-
def generate_with_usage(self, prompt: str, *, size: int = 1024, n: int = 1, seed: int | None = None,
reference_images: Sequence[Any] = ()) -> tuple[list[Image.Image], Usage]:
with self._lock:
@@ -384,20 +362,6 @@ def material_criterion(scores: dict[str, float], rubric_hint: str = "") -> str:
return ""
-def _plan_summary(plan: StaticPlan | None) -> str:
- if plan is None:
- return ""
- parts = ", ".join(f"{p.name}×{p.instances}" if p.instances > 1 else p.name for p in plan.parts)
- e = plan.overall_bbox.extents
- return f"{plan.object_name}: {plan.summary} Overall {e[0]:.2f}×{e[1]:.2f}×{e[2]:.2f} m. Parts: {parts}."
-
-
-def _degraded(j: Judgment) -> bool:
- from codeverse.judges.rubrics import is_degraded
-
- return is_degraded(j)
-
-
def judge_gate(
spec: Spec,
plan: StaticPlan | None,
@@ -415,8 +379,6 @@ def judge_gate(
) -> GateResult:
"""Render + judge both GLBs; decide. ``judge`` is any object with
``.judge(JudgeInput) -> Judgment`` (``VlmJudge`` or a fake)."""
- from codeverse.judges.base import JudgeInput
-
t0 = time.time()
if render is None:
from codeverse.spatial.render import render_glb
@@ -426,7 +388,7 @@ def judge_gate(
rs_before = render(glb_before, out_dir / "before", views=list(views), width=size, height=size)
rs_after = render(glb_after, out_dir / "after", views=list(views), width=size, height=size)
acceptance: list[AcceptanceItem] = list(getattr(plan, "acceptance", []) or [])
- summary = _plan_summary(plan)
+ summary = plan_digest(plan.model_dump()) if plan is not None else ""
res = GateResult(shipped=False, renders_before=rs_before, renders_after=rs_after)
jb = judge.judge(JudgeInput(spec=spec, renders=rs_before, measurement=measurement, acceptance=acceptance,
plan_summary=summary, round_index=0,
@@ -437,7 +399,7 @@ def judge_gate(
res.judgment_before, res.judgment_after = jb, ja
res.usage = jb.usage + ja.usage
res.duration_s = round(time.time() - t0, 2)
- if _degraded(jb) or _degraded(ja):
+ if is_degraded(jb) or is_degraded(ja):
res.reason = "judge degraded on one side — not shipped"
return res
res.overall_before, res.overall_after = float(jb.overall), float(ja.overall)
@@ -450,8 +412,6 @@ def judge_gate(
res.materials_delta = round(res.materials_after - res.materials_before, 4)
ok_overall = res.delta >= min_overall_delta
ok_mat = (res.materials_delta is None) or (res.materials_delta > min_materials_delta)
- if crit and res.materials_delta is None:
- ok_mat = False
res.shipped = bool(ok_overall and ok_mat)
if res.shipped:
res.reason = f"Δoverall {res.delta:+.3f} ≥ {min_overall_delta:+.2f} and {crit or 'overall'} {res.materials_delta:+.3f}" if crit \
diff --git a/harness/codeverse/texturing/materials.py b/harness/codeverse/texturing/materials.py
index 9f939574..69e8a99e 100644
--- a/harness/codeverse/texturing/materials.py
+++ b/harness/codeverse/texturing/materials.py
@@ -34,7 +34,7 @@ from typing import NamedTuple
from pydantic import BaseModel, Field
#: glTF/Blender/three framework defaults. A material sitting exactly on one of
-#: these pairs was never authored — see ``normalise.is_untouched``.
+#: these pairs was never authored — see ``is_framework_default``.
FRAMEWORK_DEFAULTS: tuple[tuple[float, float], ...] = (
(1.0, 1.0), # glTF 2.0 material default (metallic 1, roughness 1) — renders as dark mud
(0.0, 1.0), # three.js MeshStandardMaterial default
diff --git a/harness/codeverse/texturing/plan.py b/harness/codeverse/texturing/plan.py
index 9b3d463e..07bc8055 100644
--- a/harness/codeverse/texturing/plan.py
+++ b/harness/codeverse/texturing/plan.py
@@ -19,12 +19,12 @@ from pydantic import BaseModel, Field, ValidationError
from codeverse.config import get_settings
from codeverse.contracts.artifacts import RenderSet
-from codeverse.contracts.chat import ChatMessage, ChatRequest, ImagePart
+from codeverse.contracts.chat import ImagePart
from codeverse.contracts.common import Usage
from codeverse.contracts.plan import PartPlan, ScenePlan, StaticPlan
from codeverse.contracts.spec import Spec
from codeverse.conventions import to_snake
-from codeverse.models.schema_utils import parse_json_lenient
+from codeverse.models.schema_utils import ask_structured
from codeverse.prompts import load_text, prompt_hash, render
from codeverse.texturing.generate import TextureSet, generate_textures
@@ -316,19 +316,13 @@ def material_plan(
model = get_chat_model(model_id)
text = render(PLAN_TEMPLATE, spec_prompt=spec.prompt, style_notes=getattr(plan, "style_notes", ""),
n_parts=len(plan.parts), parts_table=parts_table(plan))
- req = ChatRequest(
- messages=[ChatMessage.user(text, images=[ImagePart(path=str(p), label=p.stem) for p in images])],
- system="You plan PBR textures for 3D assets. Answer with JSON only.",
- response_schema=PlannerOutput.model_json_schema(), temperature=temperature, thinking="low",
- max_output_tokens=65_536, max_wait_s=900.0, label="texture_plan",
+ out, usage, err = ask_structured(
+ model, PlannerOutput, system="You plan PBR textures for 3D assets. Answer with JSON only.", text=text,
+ images=[ImagePart(path=str(p), label=p.stem) for p in images], temperature=temperature, label="texture_plan",
)
- resp = model.generate(req)
- payload = resp.parsed if resp.parsed is not None else parse_json_lenient(resp.text)
- try:
- out = PlannerOutput.model_validate(payload)
- except ValidationError as e:
- raise ValueError(f"texture planner returned invalid JSON for the schema: {e}") from e
- tp = finalize_plan(out, plan, model_id=model_id, usage=resp.usage, source="vlm")
+ if out is None:
+ raise ValueError(f"texture planner: {err}")
+ tp = finalize_plan(out, plan, model_id=model_id, usage=usage, source="vlm")
if use_cache:
cache_root.mkdir(parents=True, exist_ok=True)
cached.write_text(tp.model_dump_json(indent=2))
@@ -458,15 +452,10 @@ def scene_pack_plan(
assets = "\n".join(f"- {a.name} ({a.kind}): {a.description}" for a in plan.assets) or "- (none)"
text = render(PACK_TEMPLATE, n_min=n_min, n_max=n_max, title=plan.title, setting=plan.setting, mood=plan.mood,
environment=plan.environment, zones=zones, assets=assets)
- req = ChatRequest(messages=[ChatMessage.user(text)], system="You plan texture packs for 3D scenes. JSON only.",
- response_schema=PackOutput.model_json_schema(), temperature=temperature, thinking="low",
- max_output_tokens=65_536, max_wait_s=900.0, label="scene_texture_pack")
- resp = model.generate(req)
- payload = resp.parsed if resp.parsed is not None else parse_json_lenient(resp.text)
- try:
- out = PackOutput.model_validate(payload)
- except ValidationError as e:
- raise ValueError(f"scene pack planner returned invalid JSON: {e}") from e
+ out, usage, err = ask_structured(model, PackOutput, system="You plan texture packs for 3D scenes. JSON only.",
+ text=text, temperature=temperature, label="scene_texture_pack")
+ if out is None:
+ raise ValueError(f"scene pack planner: {err}")
entries: list[PackEntry] = []
seen: set[str] = set()
for e in out.textures:
@@ -477,7 +466,7 @@ def scene_pack_plan(
entries.append(fe)
if not entries:
raise ValueError("scene pack planner returned no textures")
- return entries[:n_max], out.notes, resp.usage
+ return entries[:n_max], out.notes, usage
# --------------------------------------------------------------------------- generation
diff --git a/harness/codeverse/texturing/run.py b/harness/codeverse/texturing/run.py
index f5806f4b..c1733806 100644
--- a/harness/codeverse/texturing/run.py
+++ b/harness/codeverse/texturing/run.py
@@ -292,15 +292,8 @@ def texture_pass(
def _make_judge(spec: Spec, rubric: str | None, judge_model_id: str | None) -> Any:
from codeverse.judges.vlm_judge import VlmJudge
- info = TRACK_INFO.get(spec.track)
- if rubric:
- name = rubric
- elif info is not None:
- name = info.rubric
- else:
- name = "static_object_v1"
- log.warning("no TRACK_INFO row for track %s; texture gate falls back to %s", spec.track, name)
- return VlmJudge(rubric=name, model_id=judge_model_id or spec.backends.judge, n_samples=1, label="texture_gate")
+ return VlmJudge(rubric=rubric or TRACK_INFO[spec.track].rubric, model_id=judge_model_id or spec.backends.judge,
+ n_samples=1, label="texture_gate")
def _finish(ws: Workspace, report: TextureReport, t0: float, events: EventLog, update_record: bool) -> TextureReport:
diff --git a/harness/tests/blender_cadquery/test_artifact_stage.py b/harness/tests/blender_cadquery/test_artifact_stage.py
index 102ac88d..ef160e49 100644
--- a/harness/tests/blender_cadquery/test_artifact_stage.py
+++ b/harness/tests/blender_cadquery/test_artifact_stage.py
@@ -83,7 +83,7 @@ def test_promote_replaces_an_existing_dir(tmp_ws: Workspace) -> None:
def test_invalidate_standalone_and_name_validation(tmp_ws: Workspace) -> None:
_seed(tmp_ws)
- tmp_ws.stage_artifacts("build.json", "object.glb", "meshes").invalidate() # the remove_stale replacement
+ tmp_ws.stage_artifacts("build.json", "object.glb", "meshes").invalidate()
for name in ("build.json", "object.glb", "meshes"):
assert not (tmp_ws.artifacts / name).exists(), name
stage = tmp_ws.stage_artifacts("build.json")
diff --git a/harness/tests/blender_cadquery/test_blender_runtime.py b/harness/tests/blender_cadquery/test_blender_runtime.py
index 9e8050f8..a1de41df 100644
--- a/harness/tests/blender_cadquery/test_blender_runtime.py
+++ b/harness/tests/blender_cadquery/test_blender_runtime.py
@@ -45,7 +45,6 @@ def test_protocol_and_registry() -> None:
assert rt.entry_globs == ("src/model.py", "src/parts/*.py") and rt.language.value == "blender"
assert rt.file_for_part("Seat Cushion") == "src/parts/seat_cushion.py" # tracks call this via getattr
assert "bpy" in rt.contract_doc() and "Z is up" in rt.contract_doc()
- assert rt.cookbook_path().name == "cookbook.md"
def test_missing_binary_raises(tmp_ws) -> None:
@@ -55,12 +54,6 @@ def test_missing_binary_raises(tmp_ws) -> None:
rt.build(tmp_ws)
-def test_missing_entry_file(tmp_ws, tmp_path) -> None:
- rt = BlenderRuntime(blender=_fake_blender(tmp_path))
- r = rt.build(tmp_ws)
- assert not r.ok and r.error_type == "MissingEntryFile"
-
-
def test_missing_entry_invalidates_previous_outputs(tmp_ws, tmp_path) -> None:
"""Deleting model.py between rounds must not leave round N-1's object.glb +
build.json (ok: true) looking current — the invalidation runs BEFORE the
diff --git a/harness/tests/blender_cadquery/test_common.py b/harness/tests/blender_cadquery/test_common.py
index beca7afe..a627da1c 100644
--- a/harness/tests/blender_cadquery/test_common.py
+++ b/harness/tests/blender_cadquery/test_common.py
@@ -3,41 +3,14 @@
from __future__ import annotations
import json
-import sys
-import time
from pathlib import Path
from codeverse.languages._common import (
ProcResult,
compose_build_result,
- run_subprocess,
strip_blender_noise,
- tail,
- write_json_atomic,
)
-
-
-def test_run_subprocess_captures_output(tmp_path: Path) -> None:
- r = run_subprocess([sys.executable, "-c", "import sys; print('out'); print('err', file=sys.stderr); sys.exit(3)"],
- cwd=tmp_path, timeout_s=10)
- assert r.returncode == 3 and r.stdout.strip() == "out" and r.stderr.strip() == "err" and not r.timed_out
-
-
-def test_run_subprocess_timeout_kills_group(tmp_path: Path) -> None:
- code = "import subprocess, sys, time; p = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)']); print(p.pid, flush=True); time.sleep(30)"
- t0 = time.monotonic()
- r = run_subprocess([sys.executable, "-c", code], cwd=tmp_path, timeout_s=1.5)
- assert r.timed_out and time.monotonic() - t0 < 6
- child_pid = int(r.stdout.strip())
- time.sleep(0.3)
- assert not Path(f"/proc/{child_pid}").exists() or "zombie" in (Path(f"/proc/{child_pid}/status").read_text().lower())
-
-
-def test_tail_limits() -> None:
- text = "\n".join(str(i) for i in range(100))
- t = tail(text, max_lines=5)
- assert t.splitlines() == ["95", "96", "97", "98", "99"]
- assert len(tail("x" * 10_000, max_chars=100)) == 100
+from codeverse.proc import write_json_atomic
def test_strip_blender_noise() -> None:
diff --git a/harness/tests/graphics/test_stale_invalidation.py b/harness/tests/graphics/test_stale_invalidation.py
index bab0de96..1e3f42e3 100644
--- a/harness/tests/graphics/test_stale_invalidation.py
+++ b/harness/tests/graphics/test_stale_invalidation.py
@@ -1,7 +1,7 @@
"""GL builds must not leave the previous build's artifacts looking current (offline).
frames_sheet.png / preview.gif / metrics.json are written only on ok and were never
-cleared; the MissingEntry early returns never reached GlHost._run's frames wipe.
+cleared; the MissingEntryFile early returns never reached GlHost._run's frames wipe.
Both wipes now happen at the top of build() (``_gl_common.invalidate_stale_outputs``)
plus a failure-path invalidation in ``finish_build``.
"""
@@ -49,11 +49,11 @@ def test_failed_build_invalidates_sheet_gif_metrics(tmp_ws: Workspace) -> None:
def test_missing_entry_clears_frames_and_result(tmp_ws: Workspace) -> None:
- """The MissingEntry early return never reaches the host — the hoisted wipe must
+ """The MissingEntryFile early return never reaches the host — the hoisted wipe must
still clear frames/ + gl_result.json (and the sheet/gif/metrics trio)."""
_seed_stale(tmp_ws)
br = GlslShaderRuntime(host=_FailingHost()).build(tmp_ws, preview=False)
- assert not br.ok and br.error_type == "MissingEntry"
+ assert not br.ok and br.error_type == "MissingEntryFile"
assert not list((tmp_ws.artifacts / "frames").glob("*.png"))
for name in ("gl_result.json", "frames_sheet.png", "preview.gif", "metrics.json"):
assert not (tmp_ws.artifacts / name).exists(), name
diff --git a/harness/tests/graphics/test_track.py b/harness/tests/graphics/test_track.py
index 1e5e4c27..b0990888 100644
--- a/harness/tests/graphics/test_track.py
+++ b/harness/tests/graphics/test_track.py
@@ -84,10 +84,7 @@ class FakeGlRuntime:
return finish_build(ws, res, language="glsl_shader", census={"convention": "mainImage"})
def contract_doc(self) -> str:
- return "FAKE GLSL CONTRACT"
-
- def cookbook_path(self) -> Path:
- return Path("/dev/null")
+ return "FAKE glsl_shader authoring contract" # the runtime's contract_doc is what the prompt sees
def _writer(job, ws):
diff --git a/harness/tests/judges/test_vlm_judge.py b/harness/tests/judges/test_vlm_judge.py
index 420dae8e..8180a14f 100644
--- a/harness/tests/judges/test_vlm_judge.py
+++ b/harness/tests/judges/test_vlm_judge.py
@@ -138,10 +138,10 @@ def test_partial_samples_still_score(judge_input, cache_dir):
assert raw["n_requested"] == 2 and raw["n_used"] == 1 and len(raw["sample_errors"]) == 1
-def test_non_retryable_error_stops_early(judge_input, cache_dir):
+def test_non_retryable_error_stops_at_once(judge_input, cache_dir):
model = FakeChatModel(default=ModelError("bad request", retryable=False))
j = _judge(model, cache_dir=cache_dir).judge(judge_input)
- assert is_degraded(j) and len(model.requests) == 2
+ assert is_degraded(j) and len(model.requests) == 1
def test_base_judge_rejects_measured_rubric(judge_input, cache_dir):
diff --git a/harness/tests/languages/test_ast_lint.py b/harness/tests/languages/test_ast_lint.py
index 079769ce..25df4179 100644
--- a/harness/tests/languages/test_ast_lint.py
+++ b/harness/tests/languages/test_ast_lint.py
@@ -7,6 +7,7 @@ import ast
import codeverse.languages.blender as blender_lint
import codeverse.languages.cadquery as cadquery_lint
import codeverse.languages.opengl_python as opengl_lint
+import codeverse.languages.urdf as urdf_lint
from codeverse.contracts.artifacts import GateFinding, Severity
from codeverse.languages._ast_lint import BASE_FORBIDDEN_IMPORTS, check_imports, dotted
@@ -24,6 +25,7 @@ def test_language_forbidden_sets_are_supersets_of_base() -> None:
assert BASE_FORBIDDEN_IMPORTS <= blender_lint.FORBIDDEN_IMPORTS
assert BASE_FORBIDDEN_IMPORTS <= cadquery_lint.FORBIDDEN_IMPORTS
assert BASE_FORBIDDEN_IMPORTS <= opengl_lint.DANGEROUS_MODULES
+ assert BASE_FORBIDDEN_IMPORTS <= urdf_lint.FORBIDDEN_IMPORTS # was a hand-copied subset until 2026-08-29
def test_language_specific_extras_survive() -> None:
@@ -51,6 +53,10 @@ def test_base_modules_error_in_every_language_lint() -> None:
)
assert any(f.severity == Severity.ERROR and "webbrowser" in f.message for f in findings)
+ findings = urdf_lint.lint_model_text("import bpy\nimport webbrowser\nimport os\n", [])
+ assert any(f.severity == Severity.ERROR and "webbrowser" in f.message for f in findings)
+ assert any(f.severity == Severity.WARN and "'os'" in f.message for f in findings)
+
# ----------------------------------------------------------------- check_imports / dotted
diff --git a/harness/tests/models/test_gemini.py b/harness/tests/models/test_gemini.py
index a4dbd17b..088dc15f 100644
--- a/harness/tests/models/test_gemini.py
+++ b/harness/tests/models/test_gemini.py
@@ -28,15 +28,18 @@ from codeverse.models.gemini import (
failure_outcome,
to_contents,
)
-from codeverse.models.retry import MAX_WAIT_S, KeyPool
+from codeverse.models.retry import KeyPool
PNG_1PX = base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="
)
#: the one tool spec every tool-call test uses.
-MEASURE_TOOL = ToolSpec(name="measure", description="m",
- parameters={"type": "object", "properties": {"part": {"type": "string"}}})
+MEASURE_TOOL = ToolSpec(
+ name="measure",
+ description="m",
+ parameters={"type": "object", "properties": {"part": {"type": "string"}}},
+)
def text_response(
@@ -170,18 +173,12 @@ def test_structured_output_parsed_and_schema_sanitised():
)
-def test_bad_json_is_retried_then_raises():
+def test_bad_json_is_retried():
m, log, _ = make_model([text_response("not json"), text_response('{"ok": true}')])
r = m.generate(
ChatRequest(messages=[ChatMessage.user("x")], response_schema={"type": "object"})
)
assert r.parsed == {"ok": True} and len(log) == 2
- m2, _, _ = make_model([text_response("nope")] * 6)
- with pytest.raises(ModelError) as ei:
- m2.generate(
- ChatRequest(messages=[ChatMessage.user("x")], response_schema={"type": "object"})
- )
- assert ei.value.retryable
def test_429_rotates_keys_and_cools_down():
@@ -209,38 +206,6 @@ def _api_error(code: int, message: str, status: str) -> genai_errors.APIError:
return genai_errors.APIError(code, {"error": {"message": message, "status": status}})
-class _KeyedClient:
- """Client whose behaviour depends on the key: ``fail[key]`` raises, others answer."""
-
- def __init__(self, key: str, fail: dict[str, BaseException], log: list[dict]):
- self.key, self.fail, self.log = key, fail, log
- self.models = self
-
- def generate_content(self, *, model: str, contents, config):
- self.log.append({"key": self.key})
- if self.key in self.fail:
- raise self.fail[self.key]
- return text_response(f"ok from {self.key}")
-
- def generate_content_stream(self, *, model: str, contents, config):
- yield self.generate_content(model=model, contents=contents, config=config)
-
-
-def _keyed_model(
- keys: list[str], fail: dict[str, BaseException], pool: KeyPool | None = None, **kw
-):
- log: list[dict] = []
- pool = pool or KeyPool(keys, cooldown_s=30)
- m = GeminiModel(
- "gemini-3.7-flash",
- pool=pool,
- sleep=lambda s: None,
- client_factory=lambda key: _KeyedClient(key, fail, log),
- **kw,
- )
- return m, log, pool
-
-
SUSPENDED = _api_error(
403, "PERMISSION_DENIED: Consumer 'api_key:xxx' has been suspended.", "PERMISSION_DENIED"
)
@@ -256,87 +221,16 @@ THROTTLED = genai_errors.APIError(
)
-def test_429_rotation_does_not_consume_the_retry_budget():
- # 8 throttled keys > max_attempts=6, yet fresh keys remain → must still succeed
- keys = [f"k{i}" for i in range(1, 23)]
- m, log, pool = _keyed_model(keys, {k: THROTTLED for k in keys[:8]}, max_attempts=6)
- r = m.generate(ChatRequest(messages=[ChatMessage.user("x")]))
- assert r.text == "ok from k9" and [e["key"] for e in log] == keys[:9]
- assert pool.stats()["n_cooling"] == 8
-
-
-def test_429_on_every_key_waits_for_cooldown_then_counts_against_budget():
- clock = {"t": 1000.0}
- pool = KeyPool(
- ["k1", "k2", "k3"],
- clock=lambda: clock["t"],
- sleep=lambda s: clock.__setitem__("t", clock["t"] + s),
- )
- keys = ["k1", "k2", "k3"]
- m, log, _ = _keyed_model(keys, {k: THROTTLED for k in keys}, pool=pool, max_attempts=3)
- with pytest.raises(ModelError) as ei:
- m.generate(ChatRequest(messages=[ChatMessage.user("x")]))
- assert ei.value.retryable and ei.value.status == 429
- # 2 free rotations + 3 budgeted attempts; the pool waited out the cooldowns in between
- assert len(log) == 5
- # …but no single wait exceeded the house limit, even though the provider's
- # own retryDelay in THROTTLED asks for 7 s (MAX_WAIT_S caps it).
- assert 1000.0 < clock["t"] <= 1000.0 + 5 * MAX_WAIT_S
-
-
-def test_no_single_wait_exceeds_the_house_limit():
- """Every backoff path — 429 cooldown, provider retryDelay, 5xx, capacity
- storm — sleeps at most MAX_WAIT_S at a time (owner rule, 2026-08-24)."""
- # the sleep must ADVANCE the clock, like the neighbour test's: with a recording
- # sleep and the real monotonic clock the cooldown loop never gets past "still
- # cooling" and busy-spins — it logged 1.9 M naps over 3 s of wall time, of which
- # 4 were real waits, so `max(naps)` was sampling the spin, not the backoff.
- clock = {"t": 1000.0}
- naps: list[float] = []
-
- def nap(s: float) -> None:
- naps.append(s)
- clock["t"] += s
-
- pool = KeyPool(["k1", "k2"], clock=lambda: clock["t"], sleep=nap)
- m, _log, _ = _keyed_model(["k1", "k2"], {k: THROTTLED for k in ("k1", "k2")}, pool=pool, max_attempts=3)
- m._sleep = nap
- with pytest.raises(ModelError):
- m.generate(ChatRequest(messages=[ChatMessage.user("x")]))
- assert naps, "expected the throttled path to sleep at least once"
- assert max(naps) <= MAX_WAIT_S, f"a wait exceeded {MAX_WAIT_S}s: {sorted(naps)[-3:]}"
-
-
def test_api_key_invalid_400_is_treated_as_dead_key():
bad = _api_error(400, "API key not valid. Please pass a valid API key.", "INVALID_ARGUMENT")
- m, log, pool = _keyed_model(["k1", "k2"], {"k1": bad})
- assert m.generate(ChatRequest(messages=[ChatMessage.user("x")])).text == "ok from k2"
+ m, log, pool = make_model([bad, text_response("ok")], keys=("k1", "k2"))
+ assert m.generate(ChatRequest(messages=[ChatMessage.user("x")])).text == "ok"
+ assert [event["key"] for event in log] == ["k1", "k2"]
assert pool.stats()["dead"] == 1
assert failure_outcome(classify_exception(bad)) == "dead"
assert failure_outcome(classify_exception(SUSPENDED)) == "dead"
assert failure_outcome(classify_exception(THROTTLED)) == "429"
- assert failure_outcome(ModelError("bad json", retryable=True)) == "ok"
-
-
-def test_5xx_retries_then_gives_up_with_retryable_error():
- # 500 = plain 5xx (no storm budget): the classic 6-attempt backoff applies
- err = genai_errors.APIError(500, {"error": {"message": "internal", "status": "INTERNAL"}})
- slept: list[float] = []
- m, log, _ = make_model([err] * 6)
- m._sleep = slept.append
- with pytest.raises(ModelError) as ei:
- m.generate(ChatRequest(messages=[ChatMessage.user("x")]))
- assert ei.value.retryable and ei.value.status == 500
- assert len(log) == 6 and len(slept) == 5 and slept[1] > slept[0] / 2
-
-
-def test_503_storm_survives_beyond_the_attempt_budget():
- # 503 = capacity storm: its own patience budget on top of max_attempts
- err = genai_errors.APIError(503, {"error": {"message": "high demand", "status": "UNAVAILABLE"}})
- m, log, _ = make_model([err] * 8 + [text_response("recovered")])
- m._sleep = lambda d: None
- resp = m.generate(ChatRequest(messages=[ChatMessage.user("x")]))
- assert resp.text == "recovered" and len(log) == 9
+ assert failure_outcome(ModelError("bad json", retryable=True)) == "skip" # not the key's fault: no health nudge
def test_400_is_not_retried():
@@ -499,31 +393,34 @@ def test_no_keys_is_loud():
GeminiModel("gemini-3.7-flash", keys=[])
-def test_truncated_json_is_not_retried():
- m, log, _ = make_model(
- [text_response('{"a": [1, 2', finish="MAX_TOKENS"), text_response("never")]
+def test_max_token_failures_are_not_retried():
+ cases = (
+ (
+ text_response('{"a": [1, 2', finish="MAX_TOKENS"),
+ ChatRequest(messages=[ChatMessage.user("x")], response_schema={"type": "object"}),
+ "max_output_tokens",
+ ),
+ (
+ text_response("", finish="MAX_TOKENS", thoughts=49),
+ ChatRequest(messages=[ChatMessage.user("x")], max_output_tokens=50),
+ "thinking",
+ ),
)
- with pytest.raises(ModelError) as ei:
- m.generate(
- ChatRequest(messages=[ChatMessage.user("x")], response_schema={"type": "object"})
- )
- assert not ei.value.retryable and len(log) == 1 and "max_output_tokens" in str(ei.value)
+ for response, request, message in cases:
+ model, log, _ = make_model([response, text_response("never")])
+ with pytest.raises(ModelError) as error:
+ model.generate(request)
+ assert not error.value.retryable and len(log) == 1
+ assert message in str(error.value)
-def test_empty_max_tokens_is_not_retried():
- m, log, _ = make_model(
- [text_response("", finish="MAX_TOKENS", thoughts=49), text_response("never")]
- )
- with pytest.raises(ModelError) as ei:
- m.generate(ChatRequest(messages=[ChatMessage.user("x")], max_output_tokens=50))
- assert not ei.value.retryable and len(log) == 1 and "thinking" in str(ei.value)
def test_a_charged_but_invalid_reply_carries_its_usage_on_the_error():
- """``ModelError.usage``: what the provider billed for the failed call — a
- bad-JSON reply is charged like a success (mirrors tracks.planner.PlanningError),
- so the ledger's error row and the pool's TPM accounting can see real money."""
+ """A failed structured reply preserves its billed token usage."""
m, _log, _ = make_model([text_response("nope")] * 6)
with pytest.raises(ModelError) as ei:
- m.generate(ChatRequest(messages=[ChatMessage.user("x")], response_schema={"type": "object"}))
+ m.generate(
+ ChatRequest(messages=[ChatMessage.user("x")], response_schema={"type": "object"})
+ )
assert ei.value.usage.input_tokens == 100 and ei.value.usage.output_tokens == 20
assert ei.value.usage.cost_usd > 0
@@ -540,11 +437,12 @@ def _resp_with_calls(parts: list[types.Part]) -> types.GenerateContentResponse:
def test_synthetic_call_ids_are_never_echoed_to_the_provider():
- """When the provider sent NO id we mint one for local pairing — and it must
- never travel back to Gemini (an id the provider did not issue)."""
+ """Locally synthesized tool ids never travel back to Gemini."""
from codeverse.models.gemini import is_synthetic_call_id
- tool = ToolSpec(name="measure", description="m", parameters={"type": "object", "properties": {}})
+ tool = ToolSpec(
+ name="measure", description="m", parameters={"type": "object", "properties": {}}
+ )
m, log, _ = make_model([call_response("measure", {"part": "all"}), text_response("ok")])
req = ChatRequest(messages=[ChatMessage.user("x")], tools=[tool])
r = m.generate(req)
@@ -553,7 +451,9 @@ def test_synthetic_call_ids_are_never_echoed_to_the_provider():
msgs = [
*req.messages,
ChatMessage(role="assistant", parts=[call]),
- ChatMessage(role="tool", parts=[ToolResultPart(call_id=call.id, name="measure", content="{}")]),
+ ChatMessage(
+ role="tool", parts=[ToolResultPart(call_id=call.id, name="measure", content="{}")]
+ ),
]
m.generate(ChatRequest(messages=msgs, tools=[tool]))
contents = log[1]["contents"]
@@ -562,14 +462,14 @@ def test_synthetic_call_ids_are_never_echoed_to_the_provider():
def test_two_parallel_same_name_calls_keep_their_provider_ids():
- """Two calls to the SAME tool in one turn: a GENUINE provider id is echoed verbatim on
- BOTH the function_call replay and the function_response (both used to drop it), and only
- those ids pair a response with its call — each keeping its own thought_signature."""
+ """Parallel same-name calls preserve provider ids and signatures."""
from codeverse.models.gemini import is_synthetic_call_id
parts = []
for cid, target in (("id-a", "seat"), ("id-b", "leg")):
- p = types.Part(function_call=types.FunctionCall(name="measure", args={"part": target}, id=cid))
+ p = types.Part(
+ function_call=types.FunctionCall(name="measure", args={"part": target}, id=cid)
+ )
p.thought_signature = f"sig-{cid}".encode()
parts.append(p)
tool = MEASURE_TOOL
@@ -596,22 +496,26 @@ def test_two_parallel_same_name_calls_keep_their_provider_ids():
assert [fp.function_call.id for fp in model_parts] == ["id-a", "id-b"]
assert [fp.thought_signature for fp in model_parts] == [b"sig-id-a", b"sig-id-b"]
responses = log[1]["contents"][2].parts
- assert [(fp.function_response.id, fp.function_response.response["part"]) for fp in responses] == [
+ assert [
+ (fp.function_response.id, fp.function_response.response["part"]) for fp in responses
+ ] == [
("id-b", "leg"),
("id-a", "seat"),
]
def test_max_output_tokens_eaten_by_thinking_carries_the_thought_tokens():
- """The most expensive failure on Gemini 3: the reply is empty because thinking ate
- the whole budget, and every thought token was billed. ``extract_candidate`` read
- ``thoughts_token_count`` only to put it in the message and dropped the usage."""
+ """A MAX_TOKENS failure reports the thinking tokens it consumed."""
resp = types.GenerateContentResponse(
model_version="gemini-3.7-flash",
- candidates=[types.Candidate(content=types.Content(role="model", parts=[]),
- finish_reason="MAX_TOKENS")],
+ candidates=[
+ types.Candidate(
+ content=types.Content(role="model", parts=[]), finish_reason="MAX_TOKENS"
+ )
+ ],
usage_metadata=types.GenerateContentResponseUsageMetadata(
- prompt_token_count=1000, candidates_token_count=0, thoughts_token_count=8000),
+ prompt_token_count=1000, candidates_token_count=0, thoughts_token_count=8000
+ ),
)
m, _log, _ = make_model([resp])
with pytest.raises(ModelError) as e:
@@ -620,23 +524,7 @@ def test_max_output_tokens_eaten_by_thinking_carries_the_thought_tokens():
assert e.value.usage.thoughts_tokens == 8000 and e.value.usage.cost_usd > 0
-
# --------------------------------------------------------------------- streaming
-def test_merge_stream_chunks_reassembles_the_reply():
- from codeverse.models.gemini import _merge_stream_chunks, extract_candidate
-
- c1 = types.GenerateContentResponse(
- candidates=[types.Candidate(content=types.Content(role="model", parts=[types.Part.from_text(text="hel")]))])
- c2 = types.GenerateContentResponse(
- candidates=[types.Candidate(content=types.Content(role="model", parts=[types.Part.from_text(text="lo")]),
- finish_reason="STOP")],
- usage_metadata=types.GenerateContentResponseUsageMetadata(prompt_token_count=5, candidates_token_count=2))
- merged = _merge_stream_chunks([c1, c2])
- text, calls, finish = extract_candidate(merged)
- assert (text, calls, finish) == ("hello", [], "STOP")
- assert merged.usage_metadata.prompt_token_count == 5
-
-
def test_merge_stream_chunks_empty_stream_is_retryable():
from codeverse.models.gemini import _merge_stream_chunks
@@ -646,8 +534,7 @@ def test_merge_stream_chunks_empty_stream_is_retryable():
def test_drain_stream_cuts_a_stream_past_its_attempt_budget():
- """The wire timeout under streaming bounds only inter-chunk silence, so a
- healthy-but-endless stream must be cut by the attempt deadline instead."""
+ """The attempt deadline also bounds a healthy but endless stream."""
from codeverse.models.gemini import _drain_stream
ticks = iter([1.0, 10.0, 20.0])
@@ -679,26 +566,32 @@ def test_ipv4_transport_toggle(monkeypatch):
def test_merge_stream_chunks_carries_tool_calls_and_signatures():
- """Tool calls stream as function_call parts; the merge must keep them in arrival
- order next to the text parts, and extract_candidate must still mint/pair ids and
- stash thought signatures — the text-only tests above never exercised this."""
- from codeverse.models.gemini import _merge_stream_chunks
+ """Stream merging preserves ordered tool calls and thought signatures."""
+ from codeverse.models.gemini import _merge_stream_chunks, extract_candidate
fc = types.Part(function_call=types.FunctionCall(name="measure", args={"part": "Hull"}))
fc.thought_signature = b"sig-bytes"
c1 = types.GenerateContentResponse(
- candidates=[types.Candidate(content=types.Content(role="model", parts=[types.Part.from_text(text="checking ")]))])
+ candidates=[
+ types.Candidate(
+ content=types.Content(role="model", parts=[types.Part.from_text(text="hel")])
+ )
+ ]
+ )
c2 = types.GenerateContentResponse(
- candidates=[types.Candidate(content=types.Content(role="model", parts=[fc]), finish_reason="STOP")],
- usage_metadata=types.GenerateContentResponseUsageMetadata(prompt_token_count=9, candidates_token_count=4))
+ candidates=[
+ types.Candidate(
+ content=types.Content(role="model", parts=[types.Part.from_text(text="lo"), fc]),
+ finish_reason="STOP",
+ )
+ ],
+ usage_metadata=types.GenerateContentResponseUsageMetadata(
+ prompt_token_count=9, candidates_token_count=4
+ ),
+ )
merged = _merge_stream_chunks([c1, c2])
- text, calls, finish = extract_candidate_for_test(merged)
- assert text == "checking " and finish == "STOP"
+ text, calls, finish = extract_candidate(merged)
+ assert text == "hello" and finish == "STOP"
+ assert merged.usage_metadata.prompt_token_count == 9
assert len(calls) == 1 and calls[0].name == "measure" and calls[0].arguments == {"part": "Hull"}
assert SIGNATURES.get(calls[0].id) == b"sig-bytes"
-
-
-def extract_candidate_for_test(resp):
- from codeverse.models.gemini import extract_candidate
-
- return extract_candidate(resp)
diff --git a/harness/tests/models/test_keypool.py b/harness/tests/models/test_keypool.py
index ee2f8c2e..a305cc55 100644
--- a/harness/tests/models/test_keypool.py
+++ b/harness/tests/models/test_keypool.py
@@ -79,9 +79,9 @@ def test_all_cooling_blocks_until_cooldown_ends():
t0 = clock.t
assert pool.acquire(timeout_s=60) in ("a", "b")
assert MAX_WAIT_S - 1e-6 <= clock.t - t0 <= 5 + 1e-6
+ pool.report("a", "429")
+ pool.report("b", "429")
with pytest.raises(KeyPoolExhausted):
- pool.report("a", "429")
- pool.report("b", "429")
pool.acquire(timeout_s=1.0)
@@ -188,3 +188,15 @@ def test_the_in_flight_slot_wait_is_bounded_by_timeout():
assert time.monotonic() - t0 < 2.0, "the slot wait must honour the timeout"
pool.release()
assert pool.acquire(timeout_s=0.5) == "a", "the slot came back; nothing leaked"
+
+
+def test_skip_reconciles_tokens_but_leaves_health_and_counters_alone():
+ """A charged-but-invalid reply (bad JSON) is not the key's fault: it used to be
+ reported "ok", boosting health and counting as a success in doctor --live."""
+ pool, _ = make(keys=("a",), tpm_per_key=100_000)
+ pool.report("a", "429")
+ before = {k["key"]: k for k in pool.stats()["keys"]}["…a"]
+ pool.report("a", "skip", tokens=7_500, reserved=2_000)
+ after = {k["key"]: k for k in pool.stats()["keys"]}["…a"]
+ assert after["health"] == before["health"] and after["ok"] == before["ok"] and after["error"] == before["error"]
+ assert pool.stats()["tokens_used"] == 7_500
diff --git a/harness/tests/models/test_retry.py b/harness/tests/models/test_retry.py
index 4383f7f0..7ae636b9 100644
--- a/harness/tests/models/test_retry.py
+++ b/harness/tests/models/test_retry.py
@@ -58,25 +58,50 @@ def test_retries_until_success_and_reports():
assert [n for n, _ in seen] == [1, 2]
-def test_non_retryable_raises_immediately():
- n = 0
+def test_with_retries_stops_on_non_retryable_and_exhaustion():
+ calls = 0
- def fn():
- nonlocal n
- n += 1
+ def non_retryable():
+ nonlocal calls
+ calls += 1
raise ValueError("bad")
with pytest.raises(ValueError):
- with_retries(fn, is_retryable=lambda e: False, attempts=5, sleep=lambda s: None)
- assert n == 1
-
-
-def test_exhausted_raises_last():
- def fn():
+ with_retries(
+ non_retryable, is_retryable=lambda error: False, attempts=5, sleep=lambda _: None
+ )
+ assert calls == 1
+
+ def exhausted():
+ nonlocal calls
+ calls += 1
raise TimeoutError("always")
with pytest.raises(TimeoutError):
- with_retries(fn, is_retryable=lambda e: True, attempts=3, sleep=lambda s: None)
+ with_retries(exhausted, is_retryable=lambda error: True, attempts=3, sleep=lambda _: None)
+ assert calls == 4
+
+
+def test_with_retries_honours_a_deadline_and_stamps_attempts():
+ """The SDK adapters' loop: ``ChatRequest.max_wait_s`` bounds the whole call and the
+ raised ModelError says how many round-trips were issued (the ledger's error row)."""
+ from codeverse.models.base import ModelError
+
+ calls = 0
+
+ def flaky():
+ nonlocal calls
+ calls += 1
+ raise ModelError("busy", retryable=True, status=503)
+
+ with pytest.raises(ModelError) as ei:
+ with_retries(flaky, is_retryable=lambda e: True, attempts=6, base_delay=1.0, max_delay=1.0,
+ max_total_s=0.0, sleep=lambda _: None, jitter=False)
+ assert calls == 1 and ei.value.attempts == 1 # the first backoff would already cross the deadline
+ calls = 0
+ with pytest.raises(ModelError) as ei:
+ with_retries(flaky, is_retryable=lambda e: True, attempts=3, sleep=lambda _: None)
+ assert calls == 3 and ei.value.attempts == 3
# --------------------------------------------------------------------------- rotate_with_retries
@@ -123,8 +148,7 @@ def _rotate503(pool, call, **kw):
class Clock:
- """Fake monotonic clock: pass it as ``monotonic``, its ``sleep`` records the nap and
- advances it, and a test may advance ``.t`` itself to simulate time inside a call."""
+ """Fake monotonic clock whose sleeps advance recorded time."""
def __init__(self, t: float = 0.0) -> None:
self.t = t
@@ -140,6 +164,7 @@ class Clock:
def _boom(msg: str = "503 high demand"):
"""A ``call`` that always raises ``msg``."""
+
def call(_key):
raise RuntimeError(msg)
@@ -185,9 +210,10 @@ def test_rotate_every_key_dead_raises_without_benching():
def test_rotate_429_is_free_while_untried_keys_remain():
- pool = KeyPool(["k1", "k2", "k3"], cooldown_s=30)
- slept: list[float] = []
+ clock = Clock()
+ pool = KeyPool(["k1", "k2", "k3"], cooldown_s=30, clock=clock, sleep=clock.sleep)
calls: list[str] = []
+ hinted: list[BaseException] = []
def call(key: str) -> str:
calls.append(key)
@@ -195,10 +221,18 @@ def test_rotate_429_is_free_while_untried_keys_remain():
raise ModelError("quota", retryable=True, status=429)
return key
- out = _rotate(pool, call, max_attempts=1, sleep=slept.append)
+ out = _rotate(
+ pool,
+ call,
+ max_attempts=1,
+ sleep=clock.sleep,
+ retry_after=lambda error: hinted.append(error) or 7.0,
+ )
assert out == "k3" and calls == ["k1", "k2", "k3"]
- assert slept == [0.5, 0.5] # courtesy pauses only, no backoff
- assert pool.stats()["n_cooling"] == 2
+ assert clock.naps == [0.5, 0.5]
+ stats = pool.stats()
+ assert len(hinted) == 2 and stats["n_cooling"] == 2
+ assert max(key["cooldown_s"] for key in stats["keys"]) <= MAX_WAIT_S
def test_rotate_on_free_retry_hook_consumes_no_budget():
@@ -222,55 +256,39 @@ def test_rotate_on_free_retry_hook_consumes_no_budget():
assert len(calls) == 2
-def test_rotate_retryable_backoff_then_exhausted_raises_last():
+def test_rotate_retryable_backoff_and_nonretryable_failure_modes():
pool = KeyPool(["k1"])
slept: list[float] = []
- def call(key: str) -> str:
+ def retryable(key: str) -> str:
raise ModelError("unavailable", retryable=True, status=500)
with pytest.raises(ModelError) as ei:
- _rotate(pool, call, max_attempts=3, base_delay=1.0, max_delay=4.0, sleep=slept.append)
+ _rotate(
+ pool,
+ retryable,
+ max_attempts=3,
+ base_delay=1.0,
+ max_delay=4.0,
+ sleep=slept.append,
+ )
assert ei.value.status == 500 and len(slept) == 2 # backoff between the 3 attempts
-
-def test_rotate_nonretryable_raises_immediately():
- pool = KeyPool(["k1", "k2"])
calls: list[str] = []
- def call(key: str) -> str:
+ def nonretryable(key: str) -> str:
calls.append(key)
raise ModelError("bad request", retryable=False, status=400)
with pytest.raises(ModelError):
- _rotate(pool, call, max_attempts=5)
+ _rotate(KeyPool(["k1", "k2"]), nonretryable, max_attempts=5)
assert calls == ["k1"]
-def test_rotate_retry_after_hint_reaches_the_pool():
- clock = Clock()
- pool = KeyPool(["k1", "k2"], cooldown_s=30, clock=clock, sleep=clock.sleep)
- seen: list[BaseException] = []
-
- def call(key: str) -> str:
- if key == "k1":
- raise ModelError("quota retryDelay 7s", retryable=True, status=429)
- return key
-
- out = _rotate(pool, call, max_attempts=1, retry_after=lambda e: seen.append(e) or 7.0)
- assert out == "k2" and len(seen) == 1
- cooling = {k["key"]: k["cooldown_s"] for k in pool.stats()["keys"]}
- assert 0 < cooling["…k1"] <= 7.0
-
-
# --------------------------------------------------------------------------- 503 storms
@pytest.mark.parametrize(("hedge", "want_calls", "want_naps"), [(1, {9}, 7), (2, {9, 10}, 4)])
def test_503_storm_has_its_own_patience_budget(hedge, want_calls, want_naps):
- """A model-wide 503 storm must not burn the normal attempt budget. hedge=1: eight
- sequential 503s then ok — seven storm waits, more than max_attempts=3 could survive.
- hedge=2 (the default, audit 2026-08-26 §5.2) races both keys from the second 503 on, so
- the same eight failures cost FOUR waits and the winner's sibling may land (a 10th call,
- discarded)."""
+ """A 503 storm gets a separate budget; hedging reduces its waits."""
lock = threading.Lock()
calls = {"n": 0}
naps: list[float] = []
@@ -283,57 +301,45 @@ def test_503_storm_has_its_own_patience_budget(hedge, want_calls, want_naps):
raise RuntimeError("503 storm")
return "ok"
- out = _rotate503(_pool(2), call, max_attempts=3, base_delay=0.01, storm_attempts=10,
- storm_max_delay=0.05, sleep=naps.append, hedge=hedge)
+ out = _rotate503(
+ _pool(2),
+ call,
+ max_attempts=3,
+ base_delay=0.01,
+ storm_attempts=10,
+ storm_max_delay=0.05,
+ sleep=naps.append,
+ hedge=hedge,
+ )
assert out == "ok" and calls["n"] in want_calls
# the first 503 rotates to k2 for free (no nap); k2's 503 leaves no untried key in the
# 2-key pool, so that one is the storm
assert len(naps) == want_naps and all(d <= 0.05 for d in naps) # the cap is the cap
-def test_storm_waits_never_exceed_the_house_limit_at_production_defaults():
- """Jitter must be applied BEFORE the cap. Regression: the storm branch capped first and
- then multiplied by up to 1.25, so a documented "<=5 s" wait was observed at 6 s in a live
- bench run (2026-08-24). 50 seeds, so this is not one lucky jitter draw."""
+def test_storm_waits_never_exceed_the_house_limit_at_production_defaults(monkeypatch):
+ monkeypatch.setattr(random, "random", lambda: 1.0)
naps: list[float] = []
- for seed in range(50):
- random.seed(seed)
- with pytest.raises(ModelError):
- # storm_attempts deep enough that the exponent saturates the cap
- _rotate503(_pool(2), _boom(), max_attempts=2, storm_attempts=12, sleep=naps.append)
- assert naps, "expected the storm path to sleep"
- assert max(naps) <= MAX_WAIT_S, f"a storm wait exceeded {MAX_WAIT_S}s: {max(naps)}"
- assert max(naps) > MAX_WAIT_S * 0.9, "the cap should still be reached, not just respected"
+ with pytest.raises(ModelError):
+ _rotate503(_pool(2), _boom(), max_attempts=2, storm_attempts=12, sleep=naps.append)
+ assert naps and max(naps) == MAX_WAIT_S
def test_503_storm_budget_exhausts_then_normal_budget_applies():
- """The storm patience is finite: once ``storm_attempts`` is spent the normal budget
- applies again and the call gives up instead of looping forever."""
+ """Exhausted storm patience falls back to the finite normal budget."""
with pytest.raises(ModelError):
- _rotate503(_pool(1), _boom("503 forever"), max_attempts=2, base_delay=0.0,
- storm_attempts=3, storm_max_delay=0.0)
-
-
-def test_live_tests_are_opt_in_by_default():
- """A bare `pytest` must never fire real API calls: on 2026-08-24 a plain run on a keyed
- machine hung for the whole timeout because the live suite was selected by default."""
- import tomllib
- from pathlib import Path
-
- root = Path(__file__).resolve().parents[2]
- cfg = tomllib.loads((root / "pyproject.toml").read_text())
- addopts = cfg["tool"]["pytest"]["ini_options"].get("addopts", [])
- joined = " ".join(addopts) if isinstance(addopts, list) else str(addopts)
- assert "not live" in joined, "live tests must be deselected by default (see pyproject addopts)"
+ _rotate503(
+ _pool(1),
+ _boom("503 forever"),
+ max_attempts=2,
+ base_delay=0.0,
+ storm_attempts=3,
+ storm_max_delay=0.0,
+ )
def test_one_call_cannot_retry_for_hours():
- """The storm branch was bounded in ATTEMPTS, not in time: 60 storm attempts x (a 300 s
- read timeout + a <=5 s wait) is 5.1 hours for ONE logical call while the docstring
- claimed "~5 min" — and the wall-clock ceiling is only checked when a call is BILLED, so
- (measured 2026-08-24) a run with --max-minutes 90 reached 200 minutes. The deadline is
- now threaded through every branch and ``GeminiModel`` clips each attempt's HTTP read
- timeout to the remaining budget, floored at ``HTTP_TIMEOUT_FLOOR_S`` (simulated below)."""
+ """The wall-clock deadline bounds even a long 503 storm."""
clock = Clock()
def call(_key):
@@ -343,12 +349,20 @@ def test_one_call_cannot_retry_for_hours():
with pytest.raises(ModelError):
# hedge=1: the fake clock is serial, so two hedged 300 s calls would add 600 s
- _rotate503(_pool(2), call, max_attempts=6, storm_attempts=60, sleep=clock.sleep,
- monotonic=clock, hedge=1)
+ _rotate503(
+ _pool(2),
+ call,
+ max_attempts=6,
+ storm_attempts=60,
+ sleep=clock.sleep,
+ monotonic=clock,
+ hedge=1,
+ )
# bounded by the clock: the ONLY legal overshoot is the one floored attempt that was
# already in flight when the deadline passed
assert clock.t <= RETRY_DEADLINE_S + HTTP_TIMEOUT_FLOOR_S, (
- f"one call burned {clock.t / 3600:.2f} h; the deadline is {RETRY_DEADLINE_S / 60:.0f} min")
+ f"one call burned {clock.t / 3600:.2f} h; the deadline is {RETRY_DEADLINE_S / 60:.0f} min"
+ )
assert clock.t < 5 * 3600, "this is the 5.1-hour regression"
@@ -359,24 +373,26 @@ def test_the_deadline_does_not_cut_a_call_that_is_making_progress():
def call(_key):
calls["n"] += 1
- clock.t += 280.0 # slow, but under the per-attempt timeout
+ clock.t += 280.0 # slow, but under the per-attempt timeout
if calls["n"] < 3:
raise RuntimeError("503 high demand")
return "ok"
# hedge=1: serial fake clock and call counter (see the hedge tests for the raced form)
- out = _rotate503(_pool(2), call, max_attempts=6, storm_attempts=60, sleep=clock.sleep,
- monotonic=clock, hedge=1)
+ out = _rotate503(
+ _pool(2),
+ call,
+ max_attempts=6,
+ storm_attempts=60,
+ sleep=clock.sleep,
+ monotonic=clock,
+ hedge=1,
+ )
assert out == "ok" and calls["n"] == 3
def test_503_rotates_through_every_untried_key_before_it_is_a_storm():
- """Owner's rule (2026-08-27): a 503 on one key is simply replaced by the next key — free
- while an untried key remains, exactly like the 429 path. Measured 2026-08-26, one tiny
- request per key in parallel during the day's storm: flash answered on 15/22, 18/22, 21/22
- keys while only 5/4/1 keys said 503 at the same instant. So with a 10-key pool NINE
- consecutive 503s still rotate: no sleep, no storm, no ``max_attempts`` spent
- (``max_attempts=3`` here proves the budget is free)."""
+ """A 503 rotates freely while the pool still has an untried key."""
keys = [f"k{i}" for i in range(1, 11)]
gate = StormGate()
lock = threading.Lock()
@@ -391,24 +407,30 @@ def test_503_rotates_through_every_untried_key_before_it_is_a_storm():
raise RuntimeError("503 high demand")
return "ok"
- out = _rotate503(_pool(10), call, max_attempts=3, storm_attempts=10, storm_max_delay=0.05,
- sleep=naps.append, storm_gate=gate)
+ out = _rotate503(
+ _pool(10),
+ call,
+ max_attempts=3,
+ storm_attempts=10,
+ storm_max_delay=0.05,
+ sleep=naps.append,
+ storm_gate=gate,
+ )
# one key per round-trip, never twice: 9 free rotations, then the 10th key lands
assert out == "ok" and len(calls) == len(keys) and set(calls) == set(keys)
assert naps == [], "rotation to a fresh key costs no sleep"
assert gate.n_storms == 0 and not gate.storming, "an untried key remains: not a storm"
[email protected](("hedge", "want_calls", "want_hits", "want_stats"), [
- (1, {6}, 3, {"attempts": 6, "hedged": 0, "storm": 3}),
- (2, {6, 7}, 2, {"attempts": 7, "hedged": 3, "storm": 2}),
-])
+ ("hedge", "want_calls", "want_hits", "want_stats"),
+ [
+ (1, {6}, 3, {"attempts": 6, "hedged": 0, "storm": 3}),
+ (2, {6, 7}, 2, {"attempts": 7, "hedged": 3, "storm": 2}),
+ ],
+)
def test_503_on_every_key_is_still_a_storm(hedge, want_calls, want_hits, want_stats):
- """The backstop: once EVERY key in the pool has 503'd inside this one call there is no
- untried key left to rotate to, so the bounded storm wait engages — one wait per hedged
- ATTEMPT, not per key. hedge=1: k1, k2, k3 fail one by one (the whole pool), then two
- storm waits, then ok. hedge=2: k1 fails alone, k2+k3 fail together (pool exhausted:
- storm 1), two more together (storm 2), then the raced pair lands."""
+ """Exhausting every key engages one bounded wait per hedged attempt."""
gate = StormGate("t", base_delay=0.0, max_wait_s=0.0, probe_lease_s=0.0) # no real waits
lock = threading.Lock()
n = {"c": 0}
@@ -422,10 +444,20 @@ def test_503_on_every_key_is_still_a_storm(hedge, want_calls, want_hits, want_st
raise RuntimeError("503 high demand")
return "ok"
- out = _rotate503(_pool(3), call, max_attempts=3, storm_attempts=10, storm_max_delay=0.05,
- storm_gate=gate, hedge=hedge, stats=stats)
+ out = _rotate503(
+ _pool(3),
+ call,
+ max_attempts=3,
+ storm_attempts=10,
+ storm_max_delay=0.05,
+ storm_gate=gate,
+ hedge=hedge,
+ stats=stats,
+ )
assert out == "ok" and n["c"] in want_calls
- assert gate.n_storms == 1 and gate.n_hits == want_hits, "declared only after all three keys failed, then waited out"
+ assert gate.n_storms == 1 and gate.n_hits == want_hits, (
+ "declared only after all three keys failed, then waited out"
+ )
assert stats == want_stats
@@ -439,11 +471,7 @@ def _wait_idle(pool, timeout: float = 5.0) -> None:
def test_after_the_first_503_the_next_attempt_is_hedged_and_the_first_success_wins():
- """Measured 2026-08-26: a failed 503 costs the 21-50 s round-trip the provider holds
- before rejecting, not the <= 5 s sleep (logged sleep was 13 % of the wait), and storm
- streaks average 4.7 attempts. So from the first 503 on, the next attempt goes out on
- two fresh keys at once and the first success is returned while the other is still in
- flight; the loser reports its own outcome and releases its own slot when it lands."""
+ """After a 503, fresh keys race and the first success wins."""
pool = KeyPool(["k1", "k2", "k3", "k4"], rpm_per_key=10_000, max_in_flight=8)
release_k2 = threading.Event()
lock = threading.Lock()
@@ -461,14 +489,27 @@ def test_after_the_first_503_the_next_attempt_is_hedged_and_the_first_success_wi
raise RuntimeError("503 high demand")
return f"ok:{key}"
- out = rotate_with_retries(pool, call, classify=_503, outcome_of=lambda e: "5xx", max_attempts=3,
- storm_attempts=10, sleep=naps.append, tokens_of=lambda r: 7, stats=stats)
+ out = rotate_with_retries(
+ pool,
+ call,
+ classify=_503,
+ outcome_of=lambda e: "5xx",
+ max_attempts=3,
+ storm_attempts=10,
+ sleep=naps.append,
+ tokens_of=lambda r: 7,
+ stats=stats,
+ )
assert out == "ok:k3"
assert naps == [], "a hedged retry sleeps for nothing"
- assert calls[0] == "k1" and sorted(calls[1:]) == ["k2", "k3"], "two distinct fresh keys, k1 excluded"
+ assert calls[0] == "k1" and sorted(calls[1:]) == ["k2", "k3"], (
+ "two distinct fresh keys, k1 excluded"
+ )
assert stats == {"attempts": 3, "hedged": 1, "storm": 0}
st = pool.stats()
- assert st["in_flight"] == 1 and st["peak_in_flight"] == 2, "the loser still holds its max_in_flight slot"
+ assert st["in_flight"] == 1 and st["peak_in_flight"] == 2, (
+ "the loser still holds its max_in_flight slot"
+ )
assert st["ok"] == 1 and st["5xx"] == 1
release_k2.set()
_wait_idle(pool)
@@ -477,9 +518,7 @@ def test_after_the_first_503_the_next_attempt_is_hedged_and_the_first_success_wi
def test_a_loser_that_succeeds_later_is_discarded_but_its_tokens_are_reported():
- """A success-then-success is the one case the hedge is not free: the second answer is
- thrown away, its tokens still counted against the key (a 503 bills nothing, so while it
- storms the hedge costs nothing)."""
+ """A late successful loser is discarded but still charged to its key."""
pool = KeyPool(["k1", "k2", "k3"], rpm_per_key=10_000, tpm_per_key=1_000_000)
release_k2 = threading.Event()
@@ -491,7 +530,13 @@ def test_a_loser_that_succeeds_later_is_discarded_but_its_tokens_are_reported():
return "late:k2"
return "ok:k3"
- out = _rotate(pool, call, max_attempts=3, storm_attempts=10, tokens_of=lambda r: 900 if r.endswith("k2") else 100)
+ out = _rotate(
+ pool,
+ call,
+ max_attempts=3,
+ storm_attempts=10,
+ tokens_of=lambda r: 900 if r.endswith("k2") else 100,
+ )
assert out == "ok:k3"
release_k2.set()
_wait_idle(pool)
@@ -512,11 +557,19 @@ def test_hedge_1_disables_and_a_full_pool_of_slots_degrades_to_one_key():
if i <= 3:
raise ModelError("503 high demand", retryable=True, status=503)
return "ok"
+
return call
stats: dict = {}
- _rotate(_pool(3), make_call(), max_attempts=3, storm_attempts=10,
- storm_max_delay=0.0, hedge=1, stats=stats)
+ _rotate(
+ _pool(3),
+ make_call(),
+ max_attempts=3,
+ storm_attempts=10,
+ storm_max_delay=0.0,
+ hedge=1,
+ stats=stats,
+ )
assert stats["hedged"] == 0 and stats["attempts"] == 4
# max_in_flight=1: the primary holds the only slot, so a partner is never waited for
pool = _pool(3, max_in_flight=1)
@@ -532,8 +585,7 @@ def test_hedge_1_disables_and_a_full_pool_of_slots_degrades_to_one_key():
def test_the_worst_error_decides_a_hedged_attempt():
- """One key says 503 and the other says the request itself is bad: raise the 400 at once
- instead of storming on a request that can never succeed."""
+ """A nonretryable hedge error outranks a sibling 503."""
pool = _pool(3)
stats: dict = {}
@@ -559,13 +611,14 @@ def test_a_dead_key_among_the_hedge_is_benched_once_the_sibling_wins():
assert _rotate(pool, call, max_attempts=3, storm_attempts=10) == "ok:k3"
_wait_idle(pool)
- assert pool.stats()["n_dead"] == 1, "k3 proved the request fine, so k2's 403 was the key's fault"
+ assert pool.stats()["n_dead"] == 1, (
+ "k3 proved the request fine, so k2's 403 was the key's fault"
+ )
# --------------------------------------------------------------------------- deadline threading (2026-08-27)
def test_the_key_wait_is_clipped_to_the_remaining_budget():
- """pool.acquire used to be called with its own 120 s default no matter how little
- budget the call had left; now it gets min(ACQUIRE_TIMEOUT_S, remaining)."""
+ """Pool acquisition cannot outwait the remaining call budget."""
seen: list[float | None] = []
class SpyPool(KeyPool):
@@ -584,15 +637,24 @@ def test_a_storm_wait_is_clipped_to_the_remaining_budget():
"""Sleeping past the deadline only to give up on waking helps nobody."""
clock = Clock()
with pytest.raises(ModelError):
- _rotate503(_pool(1), _boom(), max_attempts=6, storm_attempts=60, base_delay=4.0,
- storm_max_delay=5.0, max_total_s=3.0, sleep=clock.sleep, monotonic=clock, hedge=1)
+ _rotate503(
+ _pool(1),
+ _boom(),
+ max_attempts=6,
+ storm_attempts=60,
+ base_delay=4.0,
+ storm_max_delay=5.0,
+ max_total_s=3.0,
+ sleep=clock.sleep,
+ monotonic=clock,
+ hedge=1,
+ )
assert clock.naps, "expected the storm path to sleep"
assert clock.t <= 3.0 + 1e-6, f"slept past the deadline: {clock.t}"
def test_free_429_rotation_cannot_outlive_the_deadline():
- """Free rotation costs no attempts but DOES cost wall clock (0.5 s per hop):
- with many keys it used to spin long past the budget."""
+ """Free key rotation still consumes wall-clock budget."""
clock = Clock()
pool = _pool(10, cooldown_s=0.0, clock=clock, sleep=clock.sleep)
calls: list[str] = []
@@ -609,9 +671,7 @@ def test_free_429_rotation_cannot_outlive_the_deadline():
def test_the_pool_is_not_refunded_for_a_charged_but_invalid_reply():
- """retry.py used to report tokens=0 for a bad-JSON failure, refunding a TPM
- reservation the provider had actually consumed — a 42k-token invalid reply
- handed the key 42k TPM back. ``ModelError.usage`` now carries the real bill."""
+ """A billed invalid response must consume its actual TPM usage."""
clock = Clock(1000.0)
pool = _pool(1, tpm_per_key=100_000, clock=clock, sleep=clock.sleep)
calls = {"n": 0}
@@ -619,8 +679,11 @@ def test_the_pool_is_not_refunded_for_a_charged_but_invalid_reply():
def call(key):
calls["n"] += 1
if calls["n"] == 1:
- raise ModelError("structured output is not valid JSON", retryable=True,
- usage=Usage(input_tokens=42_000))
+ raise ModelError(
+ "structured output is not valid JSON",
+ retryable=True,
+ usage=Usage(input_tokens=42_000),
+ )
return "ok"
out = _rotate(pool, call, max_attempts=3, tokens_hint=1_000, tokens_of=lambda r: 100)
@@ -631,10 +694,7 @@ def test_the_pool_is_not_refunded_for_a_charged_but_invalid_reply():
# --------------------------------------------------------------------------- on_attempt hook (2026-08-27)
def test_on_attempt_hears_every_round_trip_and_a_late_hedge_loser():
- """The per-attempt hook: the winner flushes as not-discarded, every failed try
- as discarded, and a hedge loser still in flight reports (discarded — a late
- success included) when it lands. That is how a loser's paid tokens reach the
- cost ledger at all."""
+ """The attempt hook reports winners, failures, and late hedge losers."""
pool = _pool(4)
release_k2 = threading.Event()
lock = threading.Lock()
@@ -681,14 +741,13 @@ def test_a_broken_on_attempt_hook_never_breaks_the_call():
assert _rotate(pool, lambda key: "ok", on_attempt=hook) == "ok"
-def test_the_error_it_gives_up_with_is_never_its_own_cause():
- """`classify()` returns an already-classified ModelError unchanged, so the give-up
- `raise err from exc` was `raise e from e` — and CPython does not check `from` for a
- cycle. bench/_infra.py had to grow a seen-set after that chain took run_cell to a
- RecursionError and a compare matrix lost 11 finished cells (2026-08-27)."""
+def test_error_causes_avoid_cycles_and_preserve_real_wrapped_errors():
+ """Classified errors avoid self-cycles while preserving genuine causes."""
+
def call(key):
- raise ModelError("structured output unavailable (finish_reason=MAX_TOKENS)",
- retryable=False, status=None)
+ raise ModelError(
+ "structured output unavailable (finish_reason=MAX_TOKENS)", retryable=False, status=None
+ )
with pytest.raises(ModelError) as ei:
_rotate(_pool(), call, max_attempts=1)
@@ -702,22 +761,16 @@ def test_the_error_it_gives_up_with_is_never_its_own_cause():
node = node.__cause__ if node.__cause__ is not None else node.__context__
assert node is None, "the cause chain closes into a cycle"
-
-def test_a_real_wrapped_cause_still_survives_the_self_cause_guard():
- """The guard drops the cause only when it IS the error — a genuine wrapped cause
- (the classifier built a NEW ModelError) must still reach the caller."""
- def call(key):
+ def timeout(key):
raise TimeoutError("read timed out")
with pytest.raises(ModelError) as ei:
- _rotate(_pool(), call, max_attempts=1)
+ _rotate(_pool(), timeout, max_attempts=1)
assert isinstance(ei.value.__cause__, TimeoutError)
def test_a_budget_spent_at_the_storm_gate_issues_no_round_trip():
- """A worker parked at a gate ANOTHER thread closed carries no ``last_err``, so the
- guard at the top of the loop (which requires one) let it through: it woke past its
- deadline and still spent a full round-trip (measured: 25 s on a 5 s budget)."""
+ """A worker waking past its gate deadline issues no request."""
clock = Clock(1000.0)
calls: list[str] = []
@@ -733,7 +786,13 @@ def test_a_budget_spent_at_the_storm_gate_issues_no_round_trip():
pass
with pytest.raises(ModelError) as e:
- _rotate(_pool(1), lambda key: calls.append(key),
- max_total_s=5.0, monotonic=clock, storm_gate=Gate(), label="t")
+ _rotate(
+ _pool(1),
+ lambda key: calls.append(key),
+ max_total_s=5.0,
+ monotonic=clock,
+ storm_gate=Gate(),
+ label="t",
+ )
assert calls == [], "a round-trip was issued after the budget was already spent"
assert "capacity" in str(e.value)
diff --git a/harness/tests/prompts/test_files.py b/harness/tests/prompts/test_files.py
index 27775fba..69600ff4 100644
--- a/harness/tests/prompts/test_files.py
+++ b/harness/tests/prompts/test_files.py
@@ -9,26 +9,17 @@ from tests.prompts.conftest import PROMPT_FILES, PROMPTS_DIR, read_prompt
@pytest.mark.parametrize("rel", PROMPT_FILES)
-def test_file_exists_and_nonempty(rel: str) -> None:
+def test_prompt_file_contract(rel: str) -> None:
p = PROMPTS_DIR / rel
assert p.is_file(), f"missing prompt file {rel}"
assert len(p.read_text()) > 500, f"{rel} suspiciously small"
-
-
[email protected]("rel", PROMPT_FILES)
-def test_loads_and_renders(rel: str) -> None:
raw = load_text(rel)
assert prompt_hash(raw)
# markdown prompts must be jinja-inert: render() with no context must be a no-op
rendered = render(rel)
assert rendered == raw, f"{rel} contains live jinja syntax; keep prompt .md files static"
-
-
[email protected]("rel", PROMPT_FILES)
-def test_jinja_safe(rel: str) -> None:
- text = read_prompt(rel)
for seq in ("{{", "{%", "{#"):
- assert seq not in text, f"{rel} contains {seq!r} which breaks jinja rendering"
+ assert seq not in raw, f"{rel} contains {seq!r} which breaks jinja rendering"
@pytest.mark.parametrize(
@@ -91,7 +82,7 @@ def test_every_language_ships_a_system_prompt() -> None:
A missing file here means a language silently falls back to nothing.
"""
from codeverse.contracts.common import Language
- from codeverse.languages._docs import prompt_dir_for
+ from codeverse.prompts.catalog import prompt_dir_for
from codeverse.tracks.prompting import language_system_prompt
for lang in Language:
diff --git a/harness/tests/reference/test_mismatch.py b/harness/tests/reference/test_mismatch.py
index f9a20bbe..7a96337f 100644
--- a/harness/tests/reference/test_mismatch.py
+++ b/harness/tests/reference/test_mismatch.py
@@ -1,4 +1,4 @@
-"""The render-vs-reference diff and the refine tasks derived from it."""
+"""The render-vs-reference diff."""
from __future__ import annotations
@@ -7,7 +7,7 @@ from pathlib import Path
from PIL import Image
from codeverse.models.base import ModelError
-from codeverse.reference import Mismatch, ReferenceDiff, compare, refine_tasks
+from codeverse.reference import compare
from tests.reference.conftest import FakeChat, make_spec
ANSWER = {
@@ -45,14 +45,10 @@ def test_compare_names_mismatches_and_builds_text(tmp_path: Path):
assert "the BRIEF wins" in req.messages[0].parts[0].text # synthesized caveat
-def test_top_orders_by_severity_and_becomes_three_tasks(tmp_path: Path):
+def test_top_orders_by_severity(tmp_path: Path):
ref, ren = _png(tmp_path / "ref.png"), _png(tmp_path / "ren.png")
d = compare(make_spec(), [ref], [ren], model=FakeChat({"reference_diff": [ANSWER]}))
assert [m.severity for m in d.top(3)] == ["critical", "major", "minor"]
- tasks = refine_tasks(d)
- assert len(tasks) == 3
- assert tasks[0].target == "Hopper" and tasks[0].priority == 1 and tasks[0].source == "gate"
- assert "missing feature" in tasks[0].instruction and "conical hopper" in tasks[0].instruction
def test_model_failure_is_soft(tmp_path: Path):
@@ -64,9 +60,3 @@ def test_model_failure_is_soft(tmp_path: Path):
def test_missing_images_short_circuit(tmp_path: Path):
d = compare(make_spec(), [tmp_path / "gone.png"], [_png(tmp_path / "r.png")], model=FakeChat())
assert "nothing to compare" in d.error
-
-
-def test_no_mismatches_produces_no_tasks():
- assert refine_tasks(ReferenceDiff()) == []
- d = ReferenceDiff(mismatches=[Mismatch(kind="wrong_shape", detail="x")])
- assert len(refine_tasks(d, top=3)) == 1
diff --git a/harness/tests/texturing/test_apply.py b/harness/tests/texturing/test_apply.py
index 9dcaaa77..ca0b1ef0 100644
--- a/harness/tests/texturing/test_apply.py
+++ b/harness/tests/texturing/test_apply.py
@@ -8,7 +8,7 @@ import numpy as np
import trimesh
from codeverse.spatial.measure import measure_glb
-from codeverse.texturing.apply import apply_textures, node_part_lookup, textured_summary
+from codeverse.texturing.apply import apply_textures, node_part_lookup
from codeverse.texturing.generate import FakeImageModel, generate_textures
from codeverse.texturing.plan import default_plan
@@ -30,8 +30,6 @@ def test_apply_textures_end_to_end(tmp_path: Path, chair_glb: Path, chair_plan):
assert rep.parts_skipped == ["Knob"] and rep.parts_unmatched == []
assert rep.projections["Leg_0"] == "cylinder" and rep.projections["Seat"] == "box"
assert rep.n_materials == 2
- summ = textured_summary(out)
- assert summ["textured"] == 6 and summ["geometries"] == 7 and summ["images"] == 2
# names + geometry preserved; measurement unchanged
a, b = measure_glb(chair_glb), measure_glb(out)
assert {p.name for p in a.parts} == {p.name for p in b.parts}
diff --git a/harness/tests/texturing/test_tile.py b/harness/tests/texturing/test_tile.py
index 9cbd591d..ee9ed570 100644
--- a/harness/tests/texturing/test_tile.py
+++ b/harness/tests/texturing/test_tile.py
@@ -9,11 +9,9 @@ from codeverse.texturing.generate import (
SEAM_MAX,
fit_size,
make_tileable,
- offset_check,
procedural_texture,
save_texture,
seam_score,
- tile_preview,
)
@@ -52,5 +50,3 @@ def test_fit_size_save_and_helpers(tmp_path):
assert p.is_file() and Image.open(p).format == "JPEG"
p2 = save_texture(sq, tmp_path / "t.png")
assert Image.open(p2).format == "PNG"
- assert offset_check(sq).size == sq.size
- assert tile_preview(sq, reps=2, size=64).size == (64, 64)