Skip to content

Official Codebase of 3DCodeVerse

Python

Review fixes: budget-bound planner, frozen stages never re-run, a round's code only from its commit, fewer retries, honest accounting and probes; delete variant knobs and the lenient prompt fallback

gaoypeng41bf8962198a
diff --git a/harness/codeverse3d/addons/dataset/sample.py b/harness/codeverse3d/addons/dataset/sample.py
index 5a2a5932..d2d728e0 100644
--- a/harness/codeverse3d/addons/dataset/sample.py
+++ b/harness/codeverse3d/addons/dataset/sample.py
@@ -26,7 +26,7 @@ from codeverse3d.contracts.common import (
)
from codeverse3d.contracts.run import RoundRecord, RunRecord
from codeverse3d.record import git_history
-from codeverse3d.record.deliverable import round_outputs
+from codeverse3d.record.deliverable import load_deliverable, round_outputs
from codeverse3d.record.record import effective_judgment, round_summary, stop_reason
from codeverse3d.workspace import Workspace
@@ -123,12 +123,16 @@ def exported_round(ws: Workspace, record: RunRecord) -> RoundRecord | None:
def code_files_for_round(ws: Workspace, rnd: RoundRecord | None) -> tuple[dict[str, bytes], str]:
- if rnd is not None and rnd.commit and ws.has_commit(rnd.commit):
+ """The round's code at its commit; a packaged ``deliverable/`` only when its manifest names that round."""
+ if rnd is None:
+ return git_history.read_working_tree(ws), "working_tree"
+ if rnd.commit and ws.has_commit(rnd.commit):
return git_history.read_tree_at(ws, rnd.commit), "commit"
- packaged = _deliverable_code(ws)
- if packaged:
- return packaged, "deliverable"
- return git_history.read_working_tree(ws), "working_tree"
+ manifest = load_deliverable(ws)
+ if manifest is not None and manifest.round == rnd.index and manifest.commit == rnd.commit:
+ return _deliverable_code(ws), "deliverable"
+ raise SampleError(f"round {rnd.index}: commit {rnd.commit[:10] or '(none)'} is unavailable and deliverable/ holds "
+ + (f"round {manifest.round}" if manifest is not None else "no manifest"))
def _deliverable_code(ws: Workspace) -> dict[str, bytes]:
diff --git a/harness/codeverse3d/addons/select.py b/harness/codeverse3d/addons/select.py
index f6702183..0a201ac7 100644
--- a/harness/codeverse3d/addons/select.py
+++ b/harness/codeverse3d/addons/select.py
@@ -114,8 +114,6 @@ def round_file(ws: Workspace, rnd: RoundRecord | None, name: str = "object.glb")
def round_complexity_block(ws: Workspace, rnd: RoundRecord | None) -> dict | None:
block = round_complexity(rnd) if rnd is not None else None
- if block is None and rnd is not None and round_outputs(ws, rnd) == ws.artifacts:
- block = ((read_json_or_none(ws.artifacts / "measurement.json") or {}).get("extra") or {}).get("complexity")
return block if isinstance(block, dict) else None
diff --git a/harness/codeverse3d/models/health.py b/harness/codeverse3d/models/health.py
index 2ec5cb4b..70304233 100644
--- a/harness/codeverse3d/models/health.py
+++ b/harness/codeverse3d/models/health.py
@@ -7,8 +7,11 @@ from __future__ import annotations
import concurrent.futures as cf
import logging
+import random
from dataclasses import dataclass
+from typing import Any
+from codeverse3d.config import get_settings
from codeverse3d.contracts.common import Backends
from codeverse3d.models.registry import build_chat_model
@@ -52,10 +55,9 @@ def probe(
prompt = _probe_prompt()
- def one(_: int) -> str | None:
+ def one(key: str | None) -> str | None:
try:
- # built directly, not via get_chat_model: a probe must NOT retry or back off
- m = _bare_model(model, timeout_s)
+ m = _bare_model(model, timeout_s, key)
m.generate(
ChatRequest(
messages=[ChatMessage.user(prompt)],
@@ -63,15 +65,17 @@ def probe(
max_output_tokens=256,
thinking="off",
label="health",
+ max_wait_s=timeout_s,
)
)
return None
except Exception as e: # noqa: BLE001 — a probe reports, it does not propagate
return f"{type(e).__name__}: {str(e)[:100]}"
- n = max(1, sample)
+ keys = _sampled_keys(model, max(1, sample))
+ n = len(keys)
with cf.ThreadPoolExecutor(max_workers=n) as pool:
- out = list(pool.map(one, range(n)))
+ out = list(pool.map(one, keys))
reasons = tuple(dict.fromkeys(r for r in out if r)) # de-duplicated, order kept
return Health(model=model, n_ok=sum(r is None for r in out), n_tried=n, reasons=reasons)
@@ -82,9 +86,24 @@ def _probe_prompt(tokens: int = PROBE_TOKENS) -> str:
return f"{filler}\n\nIgnore the text above. Reply with the single word: pong"
-def _bare_model(model_id: str, timeout_s: float):
- """The model with retries off; a bare name is a gemini model."""
+def _sampled_keys(model_id: str, n: int) -> list[str | None]:
+ """One pool key per probe for a gemini model (distinct while the pool allows); ``[None] * n`` otherwise."""
+ if ":" in model_id and not model_id.startswith("gemini:"):
+ return [None] * n
+ keys = list(get_settings().gemini_api_keys)
+ if not keys:
+ return [None] * n
+ chosen = random.sample(keys, min(n, len(keys)))
+ return [chosen[i % len(chosen)] for i in range(n)]
+
+
+def _bare_model(model_id: str, timeout_s: float, key: str | None = None):
+ """The model with every retry off, on ONE key when given; a bare name is a gemini model."""
if ":" not in model_id:
model_id = f"gemini:{model_id}"
- extra = {"storm_attempts": 0} if model_id.startswith("gemini:") else {}
+ extra: dict[str, Any] = {}
+ if model_id.startswith("gemini:"):
+ extra = {"storm_attempts": 0, "hedge": 1}
+ if key is not None:
+ extra["keys"] = [key]
return build_chat_model(model_id, timeout_s=timeout_s, max_attempts=1, **extra)
diff --git a/harness/codeverse3d/models/parts.py b/harness/codeverse3d/models/parts.py
index 5b015c09..0f99efa7 100644
--- a/harness/codeverse3d/models/parts.py
+++ b/harness/codeverse3d/models/parts.py
@@ -12,6 +12,7 @@ from typing import Any
from codeverse3d.config import get_settings
from codeverse3d.contracts.chat import ChatMessage, ImagePart, TextPart
+from codeverse3d.contracts.common import Usage
from codeverse3d.models.base import ModelError
@@ -151,6 +152,20 @@ def retry_one_key(attempt: Any, *, label: str, classify: Any, attempts: int, sle
storm patience or hedge, inside the request's ``max_wait_s`` deadline."""
from codeverse3d.models.retry import KeyPool, rotate_with_retries
- return rotate_with_retries(KeyPool([label], cooldown_s=0.0), lambda _key: attempt(), classify=classify,
+ billed = Usage() # what failed attempts were billed for (ModelError.usage): joins the winner's row
+
+ def once() -> Any:
+ nonlocal billed
+ try:
+ return attempt()
+ except ModelError as e:
+ if e.usage is not None:
+ billed = billed + e.usage
+ raise
+
+ resp = rotate_with_retries(KeyPool([label], cooldown_s=0.0), lambda _key: once(), classify=classify,
max_attempts=attempts, max_total_s=retry_budget_s(max_wait_s),
storm_attempts=0, hedge=1, sleep=sleep, label=label)
+ if billed.input_tokens or billed.output_tokens:
+ resp = resp.model_copy(update={"usage": resp.usage + billed})
+ return resp
diff --git a/harness/codeverse3d/orchestrator.py b/harness/codeverse3d/orchestrator.py
index a1fcc727..793e4e08 100644
--- a/harness/codeverse3d/orchestrator.py
+++ b/harness/codeverse3d/orchestrator.py
@@ -136,6 +136,9 @@ class StageRunner:
try:
result = _revive(json.loads(path.read_text()), model)
except (OSError, ValueError) as e: # ValidationError is a ValueError
+ if self.frozen:
+ raise RuntimeError(f"stage {name}: its recorded result {path} is unreadable ({e}) and rounds exist, "
+ "so it is never re-run; `3dcode resume --force` regenerates the stages") from e
self.events.emit("stage.cache_invalid", stage=name, inputs_hash=h, path=str(path),
error=f"{type(e).__name__}: {e}")
log.warning("stage %s: cached result at %s is unusable (%s); re-running", name, path, e)
diff --git a/harness/codeverse3d/prompts/__init__.py b/harness/codeverse3d/prompts/__init__.py
index 9f7edbfd..d1e3b663 100644
--- a/harness/codeverse3d/prompts/__init__.py
+++ b/harness/codeverse3d/prompts/__init__.py
@@ -12,8 +12,7 @@ from functools import lru_cache
from pathlib import Path
from typing import Any
-from jinja2 import ChainableUndefined, ChoiceLoader, Environment, FileSystemLoader, StrictUndefined
-from jinja2.exceptions import UndefinedError
+from jinja2 import ChoiceLoader, Environment, FileSystemLoader, StrictUndefined
from codeverse3d.prompts.variants import active_variant, variant_root
@@ -26,18 +25,14 @@ _FLAGS: dict[str, Any] = {"undefined": StrictUndefined, "trim_blocks": True, "ls
_env = Environment(loader=FileSystemLoader(str(PROMPTS_DIR)), **_FLAGS)
-#: lenient environment: an unknown name renders empty; the fallback in :func:`render` only
-_lenient_env = _env.overlay(undefined=ChainableUndefined)
-
@lru_cache(maxsize=16)
-def _env_for(variant: str) -> tuple[Environment, Environment]:
- """``(strict, lenient)`` environments: the base pair for ``""``, else overlay-first then base."""
+def _env_for(variant: str) -> Environment:
+ """The base environment for ``""``, else overlay-first then base."""
if not variant:
- return _env, _lenient_env
- env = Environment(loader=ChoiceLoader([FileSystemLoader(str(variant_root(variant))),
- FileSystemLoader(str(PROMPTS_DIR))]), **_FLAGS)
- return env, env.overlay(undefined=ChainableUndefined)
+ return _env
+ return Environment(loader=ChoiceLoader([FileSystemLoader(str(variant_root(variant))),
+ FileSystemLoader(str(PROMPTS_DIR))]), **_FLAGS)
def _resolve(variant: str, rel_path: str) -> Path:
@@ -62,19 +57,8 @@ def load_text(rel_path: str) -> str:
def render(rel_path: str, **ctx: Any) -> str:
- """Render a jinja template under prompts/ with ``ctx``.
-
- StrictUndefined raises on a missing name, except that a name missing at render time degrades
- that one section to empty (logged) so an edit made while a run is in flight cannot kill it.
- """
- env, lenient = _env_for(active_variant())
- template = env.get_template(rel_path)
- try:
- return template.render(**ctx)
- except UndefinedError as e:
- log.error("prompt %s wants a variable this caller does not pass (%s); rendering it "
- "leniently — the prompt loses that section, the run continues", rel_path, e)
- return lenient.get_template(rel_path).render(**ctx)
+ """Render a jinja template under prompts/ with ``ctx``; a missing name raises (StrictUndefined)."""
+ return _env_for(active_variant()).get_template(rel_path).render(**ctx)
def prompt_hash(text: str) -> str:
diff --git a/harness/codeverse3d/prompts/variants.py b/harness/codeverse3d/prompts/variants.py
index b352256d..126fa5a9 100644
--- a/harness/codeverse3d/prompts/variants.py
+++ b/harness/codeverse3d/prompts/variants.py
@@ -25,9 +25,6 @@ log = logging.getLogger(__name__)
VARIANTS_DIR: Path = Path(__file__).resolve().parent / "variants"
VARIANT_FILE = "variant.toml"
-#: ``[knobs]`` may set only these ``Settings`` fields; applied as defaults, user-stated values win
-KNOB_FIELDS: frozenset[str] = frozenset({"skills_max", "skills_only"})
-_KNOB_TYPES: dict[str, type] = {"skills_max": int, "skills_only": str}
HASH_EXCLUDED: frozenset[str] = frozenset({"README.md"})
_ACTIVE: ContextVar[str | None] = ContextVar("c3d_variant", default=None)
@@ -43,7 +40,6 @@ class VariantInfo(BaseModel):
generators: tuple[str, ...] = Field(default=(), description="informative: the generator ids it was tuned "
"for; selection is Settings.variants / --variant")
description: str = ""
- knobs: dict[str, Any] = Field(default_factory=dict, description="Settings fields to apply (KNOB_FIELDS only)")
def _plain_name(name: str) -> bool:
@@ -80,9 +76,9 @@ def _parse(path: Path, dir_name: str) -> VariantInfo:
head = data.get("variant")
if not isinstance(head, dict):
raise ValueError(f"{path}: missing the [variant] table")
- unknown_tables = sorted(set(data) - {"variant", "knobs"})
+ unknown_tables = sorted(set(data) - {"variant"})
if unknown_tables:
- raise ValueError(f"{path}: unknown table(s) {unknown_tables}; variant.toml holds [variant] and [knobs]")
+ raise ValueError(f"{path}: unknown table(s) {unknown_tables}; variant.toml holds the [variant] table")
unknown = sorted(set(head) - {"name", "version", "generators", "description"})
if unknown:
raise ValueError(f"{path}: unknown [variant] key(s) {unknown}")
@@ -98,18 +94,7 @@ def _parse(path: Path, dir_name: str) -> VariantInfo:
description = head.get("description", "")
if not isinstance(description, str):
raise ValueError(f"{path}: description must be a string")
- knobs = data.get("knobs", {})
- if not isinstance(knobs, dict):
- raise ValueError(f"{path}: [knobs] must be a table")
- for key, value in knobs.items():
- if key not in KNOB_FIELDS:
- raise ValueError(f"{path}: [knobs] {key!r} is not a variant knob (allowed: {sorted(KNOB_FIELDS)})")
- want = _KNOB_TYPES[key]
- if (isinstance(value, bool) is not (want is bool)) or not isinstance(value, want) or (want is int and value < 0):
- raise ValueError(f"{path}: [knobs] {key} must be a {want.__name__}"
- f"{' >= 0' if want is int else ''}, got {value!r}")
- return VariantInfo(name=name, version=version, generators=tuple(generators), description=description,
- knobs=dict(knobs))
+ return VariantInfo(name=name, version=version, generators=tuple(generators), description=description)
def served_id(generator_id: str) -> str:
@@ -201,35 +186,6 @@ def variant_block(name: str) -> dict[str, Any]:
return {"name": name, "version": 0, "hash": ""}
-def apply_variant_knobs(settings: Settings, info: VariantInfo) -> dict[str, Any]:
- """Set ``info.knobs`` on ``settings`` as defaults: fields the user stated are untouched and the
- field-set is restored. Returns what was set."""
- stated = set(settings.model_fields_set)
- applied: dict[str, Any] = {}
- for field_name, value in info.knobs.items():
- if field_name in stated:
- continue
- setattr(settings, field_name, value)
- applied[field_name] = value
- settings.__pydantic_fields_set__ = stated
- return applied
-
-
-@contextmanager
-def knobs_applied(settings: Settings, info: VariantInfo | None) -> Iterator[dict[str, Any]]:
- """:func:`apply_variant_knobs` for the block only: every knob field returns to its prior value and
- field-set membership on exit. Yields what was set."""
- prior = {field_name: getattr(settings, field_name) for field_name in KNOB_FIELDS}
- stated = set(settings.model_fields_set) & KNOB_FIELDS
- applied = apply_variant_knobs(settings, info) if info is not None else {}
- try:
- yield applied
- finally:
- for field_name, value in prior.items():
- setattr(settings, field_name, value)
- settings.__pydantic_fields_set__ = (set(settings.model_fields_set) - KNOB_FIELDS) | stated
-
-
-__all__ = ["HASH_EXCLUDED", "KNOB_FIELDS", "VARIANTS_DIR", "VARIANT_FILE", "VariantInfo", "activate",
- "active_variant", "apply_variant_knobs", "knobs_applied", "list_variants", "resolve_variant", "served_id",
- "variant_block", "variant_hash", "variant_info", "variant_root"]
+__all__ = ["HASH_EXCLUDED", "VARIANTS_DIR", "VARIANT_FILE", "VariantInfo", "activate", "active_variant",
+ "list_variants", "resolve_variant", "served_id", "variant_block", "variant_hash", "variant_info",
+ "variant_root"]
diff --git a/harness/codeverse3d/prompts/variants/argon/variant.toml b/harness/codeverse3d/prompts/variants/argon/variant.toml
index 9887b28f..8fd63fc0 100644
--- a/harness/codeverse3d/prompts/variants/argon/variant.toml
+++ b/harness/codeverse3d/prompts/variants/argon/variant.toml
@@ -15,6 +15,3 @@ name = "argon"
version = 2
generators = ["agy:gemini-4-argon"]
description = "gemini-4-argon on the scene track (threejs + blender)"
-
-# [knobs] # optional; only Settings.skills_max / skills_only, applied like a profile
-# skills_max = 4 # (a field the user stated in config.yaml / C3D_* wins)
diff --git a/harness/codeverse3d/record/deliverable.py b/harness/codeverse3d/record/deliverable.py
index f8aa482c..e4d10b9c 100644
--- a/harness/codeverse3d/record/deliverable.py
+++ b/harness/codeverse3d/record/deliverable.py
@@ -72,18 +72,12 @@ class _Writer:
def _code_tree(ws: Workspace, rnd: RoundRecord | None) -> tuple[dict[str, bytes], str]:
- """``src/**`` (+ ``public/**``) at the round's commit, else the working tree."""
- if rnd is not None and rnd.commit:
- try:
- if ws.has_commit(rnd.commit):
- return git_history.read_tree_at(ws, rnd.commit), "commit"
- except git_history.GitReadError as e:
- log.warning("deliverable: git read failed in %s: %s", ws.root, e)
- try:
+ """``src/**`` (+ ``public/**``) at the round's commit; the working tree only when no round is named."""
+ if rnd is None:
return git_history.read_working_tree(ws), "working_tree"
- except OSError as e: # pragma: no cover - defensive
- log.warning("deliverable: working tree unreadable in %s: %s", ws.root, e)
- return {}, "working_tree"
+ if not (rnd.commit and ws.has_commit(rnd.commit)):
+ raise git_history.GitReadError(f"round {rnd.index}: commit {rnd.commit[:10] or '(none)'} is not in {ws.root}")
+ return git_history.read_tree_at(ws, rnd.commit), "commit"
def round_outputs(ws: Workspace, rnd: RoundRecord | None) -> Path | None:
diff --git a/harness/codeverse3d/spatial/gl_render.py b/harness/codeverse3d/spatial/gl_render.py
index 886b3b02..72cbd421 100644
--- a/harness/codeverse3d/spatial/gl_render.py
+++ b/harness/codeverse3d/spatial/gl_render.py
@@ -123,7 +123,6 @@ class GlHost:
# The subprocess starts inside out_dir: a relative job path would be resolved twice.
out_dir = Path(out_dir).resolve()
out_dir.mkdir(parents=True, exist_ok=True)
- (out_dir / RESULT_NAME).unlink(missing_ok=True)
for p in (out_dir / "frames").glob("*.png"):
p.unlink()
job.update({"out_dir": str(out_dir), "fps": self.fps, "max_steps": self.max_steps, "backend": "egl"})
@@ -145,6 +144,7 @@ class GlHost:
def _run_once(self, job_path: Path, out_dir: Path, *, use_gpu: bool, mode: str) -> GlResult:
env = _base_env(GPU_ENV if use_gpu else CPU_ENV)
+ (out_dir / RESULT_NAME).unlink(missing_ok=True) # never read a previous attempt's result
t0 = time.time()
proc = run_subprocess([self.python, str(RUNNER), "--job", str(job_path)], cwd=out_dir, timeout_s=self.timeout_s, env=env)
result_path = out_dir / RESULT_NAME
diff --git a/harness/codeverse3d/spatial/joints_collide.py b/harness/codeverse3d/spatial/joints_collide.py
index 14cadfa4..2622676f 100644
--- a/harness/codeverse3d/spatial/joints_collide.py
+++ b/harness/codeverse3d/spatial/joints_collide.py
@@ -84,7 +84,6 @@ class LinkBody:
self.islands = oriented_islands(mesh)
self.watertight = all(bool(i.is_watertight) for i in self.islands)
self._island_pq = [trimesh.proximity.ProximityQuery(i) for i in self.islands]
- self._pq = trimesh.proximity.ProximityQuery(mesh) # unsigned distances, LOCAL frame
self.obj = fcl_collision_object(mesh)
self.T = np.eye(4)
self.T_inv = np.eye(4)
@@ -94,12 +93,6 @@ class LinkBody:
self.T_inv = np.linalg.inv(T)
self.obj.setTransform(_fcl.Transform(np.ascontiguousarray(T[:3, :3]), np.ascontiguousarray(T[:3, 3])))
- @property
- def posed(self) -> trimesh.Trimesh:
- m = trimesh.util.concatenate(self.islands) if len(self.islands) > 1 else self.islands[0].copy()
- m.apply_transform(self.T)
- return m
-
@property
def points(self) -> np.ndarray:
return trimesh.transform_points(self.local_points, self.T)
diff --git a/harness/codeverse3d/tracks/candidates.py b/harness/codeverse3d/tracks/candidates.py
index 934de6f1..bd5c19cb 100644
--- a/harness/codeverse3d/tracks/candidates.py
+++ b/harness/codeverse3d/tracks/candidates.py
@@ -63,17 +63,6 @@ def run_best_of_n(track: Any, ctx: RunContext, tasks: Sequence[GenerationTask],
with timed("candidates", ctx.state.steps, round_index=0):
results = fan_out(list(range(n)), _one, max_workers=ctx.settings.limits.max_parallel_agents,
label="candidates", item_name=lambda k: f"c{k}")
- # one retry for candidates that crashed (transient 503s, parse failures) — not for budget stops
- retry = [k for k, r in enumerate(results) if isinstance(r, Exception) and not isinstance(r, BudgetExceeded)]
- if retry and ctx.budget.ok():
- for k in retry:
- ctx.events.emit("candidate.retry", candidate=k, error=f"{type(results[k]).__name__}: {results[k]}"[:300])
- subs[k] = make_candidate_context(track, ctx, k)
- with timed("candidates", ctx.state.steps, round_index=0):
- again = fan_out(retry, _one, max_workers=ctx.settings.limits.max_parallel_agents, label="candidates-retry",
- item_name=lambda k: f"c{k}")
- for k, r in zip(retry, again, strict=True):
- results[k] = r
records: list[CandidateRecord] = []
budget_stop: BudgetExceeded | None = None
for k, r in enumerate(results):
diff --git a/harness/codeverse3d/tracks/lifecycle.py b/harness/codeverse3d/tracks/lifecycle.py
index a45847fc..9109b4f5 100644
--- a/harness/codeverse3d/tracks/lifecycle.py
+++ b/harness/codeverse3d/tracks/lifecycle.py
@@ -5,8 +5,7 @@ from __future__ import annotations
import logging
import platform
import traceback
-from collections.abc import Callable, Iterator, Sequence
-from contextlib import ExitStack, contextmanager
+from collections.abc import Callable, Sequence
from dataclasses import dataclass, replace
from datetime import UTC, datetime
from pathlib import Path
@@ -41,10 +40,8 @@ from codeverse3d.prompts.catalog import language_text
from codeverse3d.prompts.variants import (
activate,
active_variant,
- knobs_applied,
resolve_variant,
variant_block,
- variant_info,
)
from codeverse3d.record.record import previous_verdict
from codeverse3d.tracks.candidates import run_best_of_n
@@ -274,8 +271,7 @@ class BaseTrack:
def run(self, spec: Spec, ws: Workspace, *, resume: bool = False, force: bool = False) -> RunRecord:
ws.create()
variant = self.variant_for(spec)
- with activate(variant), run_ledger(ws.root, run=run_binding().run or ws.root.name), \
- self._apply_variant(variant) as knobs:
+ with activate(variant), run_ledger(ws.root, run=run_binding().run or ws.root.name):
if not ws.spec_path.is_file() or not resume:
ws.write_json(ws.spec_path, spec)
events = EventLog(ws.events_path)
@@ -287,7 +283,7 @@ class BaseTrack:
agent=ctx.agent_id, planner=spec.backends.planner, judge=spec.backends.judge,
variant=variant)
if variant:
- events.emit("variant.active", **variant_block(variant), knobs=knobs)
+ events.emit("variant.active", **variant_block(variant))
stop = RunStatus.FAILED
error = ""
try:
@@ -318,19 +314,6 @@ class BaseTrack:
"""The variant this run activates: the spec's frozen choice, else the one resolved for its generator."""
return spec.options.variant or resolve_variant(get_settings(), spec.backends.generator)
- @contextmanager
- def _apply_variant(self, variant: str) -> Iterator[dict[str, Any]]:
- """Apply the variant's ``[knobs]`` as defaults on the process settings for the duration of the run."""
- if not variant:
- yield {}
- return
- info = variant_info(variant)
- with ExitStack() as stack:
- applied = stack.enter_context(knobs_applied(get_settings(), info))
- if self._settings is not None and self._settings is not get_settings():
- stack.enter_context(knobs_applied(self._settings, info))
- yield applied
-
# ------------------------------------------------------------------ resume reconciliation
def reconcile_resume(self, spec: Spec, ws: Workspace, events: EventLog, state: RunState,
*, resume: bool, force: bool) -> list[RoundRecord]:
diff --git a/harness/codeverse3d/tracks/planner.py b/harness/codeverse3d/tracks/planner.py
index 0acd4321..9998922c 100644
--- a/harness/codeverse3d/tracks/planner.py
+++ b/harness/codeverse3d/tracks/planner.py
@@ -536,7 +536,7 @@ def plan_wait_s(tokens: int, guard: object | None = None, *, scale: float = 1.0)
"""How long one planner call may take, given the answer size it asked for."""
want = max(PLAN_MAX_WAIT_S, tokens / PLAN_SLOW_TOKENS_PER_S + PLAN_WAIT_OVERHEAD_S) * scale
fn = getattr(guard, "timeout_s", None)
- return float(fn(want, floor_s=PLAN_MAX_WAIT_S)) if callable(fn) else want
+ return float(fn(want)) if callable(fn) else want
def plan_tokens(budget: PlanBudget, floor: int) -> int:
@@ -699,6 +699,8 @@ def plan[P: BaseModel](spec: Spec, model_id: str, plan_model: type[P], ws: Works
model = get_chat_model(model_id)
brief, usage = (None, Usage())
if brief_enabled(spec):
+ if guard is not None:
+ guard.check()
brief, usage = expand_brief(spec, model_id, model=model, events=events)
budget = plan_budget(spec, brief)
unit = "passes" if spec.track is Track.GRAPHICS else "parts"
@@ -709,11 +711,13 @@ def plan[P: BaseModel](spec: Spec, model_id: str, plan_model: type[P], ws: Works
base_messages = list(messages)
schema = plan_schema(spec, plan_model)
last_error = ""
- invalid = requeried = grown = restarts = 0
+ invalid = requeried = restarts = 0
tokens = plan_tokens(budget, PLAN_TOKENS_MAX if spec.track is Track.GRAPHICS else PLAN_OUTPUT_FLOOR)
thinking = "medium"
wait_scale = 1.0
for attempt in range(4 + MAX_VALIDATION_REASKS + MAX_QUALITY_REASKS + MAX_PLAN_RESTARTS):
+ if guard is not None:
+ guard.check()
req = ChatRequest(messages=messages, system=system, response_schema=schema, temperature=plan_temperature(spec.track),
thinking=thinking, max_output_tokens=tokens, label=f"planner{'-retry' if attempt else ''}",
max_wait_s=plan_wait_s(tokens, guard, scale=wait_scale))
@@ -722,7 +726,6 @@ def plan[P: BaseModel](spec: Spec, model_id: str, plan_model: type[P], ws: Works
except Exception as e: # noqa: BLE001 — a truncated plan is retryable; anything else is not
if not (_truncated(e) and tokens < PLAN_TOKENS_MAX):
raise
- grown += 1
tokens = min(PLAN_TOKENS_MAX, int(tokens * TRUNCATION_GROWTH))
thinking = "off"
wait_scale = TRUNCATION_WAIT_SCALE
diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.md
index 9e6dceb7..e3f33bf6 100644
--- a/harness/docs/ARCHITECTURE.md
+++ b/harness/docs/ARCHITECTURE.md
@@ -184,7 +184,7 @@ codeverse3d/
variants.py: per-model OVERLAYS — variants/<name>/<rel> shadows <rel> for load_text /
render / language_text while that variant is active (a ContextVar BaseTrack.run sets from
RunOptions.variant, else C3D_VARIANT / C3D_VARIANTS, resolved by exact generator id);
- variant.toml = name/version/generators/knobs, variant_hash lands in the record.
+ variant.toml = name/version/generators, variant_hash lands in the record.
codeverse3d/skills/ is what an AGENT chooses to read; a file that tries to be both is the
bug this split prevents
cli/ main.py (app wiring, make/resume/mcp + the doctor/tools/gallery commands), inspect_cmd.py
diff --git a/harness/docs/RUNBOOK.md b/harness/docs/RUNBOOK.md
index b2c94a6b..3f3edf92 100644
--- a/harness/docs/RUNBOOK.md
+++ b/harness/docs/RUNBOOK.md
@@ -223,8 +223,7 @@ storm's rows (`--redo-status error,infra_failed,budget`) rather than reading the
Every new template variable is guarded `{% if name is defined and name %}` for at least one
wave: templates are read at render time, so a live worker on old python renders the new file.
* **A per-model variant**: `codeverse3d/prompts/variants/<name>/variant.toml` (`[variant]` name =
- the directory name, version, generators, description; optional `[knobs]` — `skills_max`,
- `skills_only` — applied as defaults, a user-stated field wins) beside the prompt
+ the directory name, version, generators, description) beside the prompt
files that shadow the base ones by relative path (a template there may `include` a base
file), and `codeverse3d/skills/variants/<name>/` for bundles that shadow base bundles by name,
their `_claims/`, and `routes.toml` (`[[boost]]` skill + delta, `[[disable]]` rule, `[[route]]`
diff --git a/harness/pyproject.toml b/harness/pyproject.toml
index 04546bbb..5148a464 100644
--- a/harness/pyproject.toml
+++ b/harness/pyproject.toml
@@ -45,12 +45,7 @@ mesh = ["shapely>=2.0", "networkx>=3.2", "matplotlib>=3.8"] # cross_section are
mcp = ["mcp>=1.0"] # `3dcode mcp` spatial-tool server for the agentic CLIs
flywheel = ["pyarrow>=15"] # `3dcode flywheel export --pack` (parquet)
all = ["3dcodeverse[cad,urdf,graphics,mesh,mcp,flywheel]"]
-dev = ["pytest>=8", "pytest-timeout>=2.3", "pytest-xdist>=3.6", "ruff>=0.5", "pyarrow>=15",
- # the Agent Skills reference validator (CLI: `agentskills`). Pure python,
- # click + strictyaml. tests/skills/test_spec_compliance.py runs it over every
- # bundle and skips cleanly when it is absent, so it is dev-only on purpose:
- # the spec is the arbiter of our library, not a runtime dependency of it.
- "skills-ref>=0.1.1"]
+dev = ["pytest>=8", "pytest-timeout>=2.3", "pytest-xdist>=3.6", "ruff>=0.5", "pyarrow>=15"]
[project.scripts]
3dcode = "codeverse3d.cli.main:app"
diff --git a/harness/tests/orchestrator_tracks/test_candidates_reference_motion.py b/harness/tests/orchestrator_tracks/test_candidates_reference_motion.py
index 415c1b61..4c814a4b 100644
--- a/harness/tests/orchestrator_tracks/test_candidates_reference_motion.py
+++ b/harness/tests/orchestrator_tracks/test_candidates_reference_motion.py
@@ -148,14 +148,14 @@ def test_articulated_motion_direction_gate_feeds_refine_and_judge(tmp_path, sett
assert any(v.name == "pose_DrawerSlide_upper" for v in rec.rounds[0].renders.views)
-def test_crashed_candidate_is_retried_once(tmp_path, chair_plan, settings):
+def test_a_crashed_candidate_is_recorded_failed_and_never_retried_on_its_own(tmp_path, chair_plan, settings):
spec = make_spec(language=Language.BLENDER, max_rounds=0)
ws = Workspace(tmp_path / "runs" / "retry")
seen: dict[str, int] = {}
def writer(job, ws_):
seen[job.label] = seen.get(job.label, 0) + 1
- if job.label == "baseline_c0" and seen[job.label] == 1:
+ if job.label == "baseline_c0":
raise RuntimeError("503 high demand")
return _writer_by_candidate(job, ws_)
@@ -163,10 +163,10 @@ def test_crashed_candidate_is_retried_once(tmp_path, chair_plan, settings):
track = StaticObjectTrack(services=FakeServices(), judge=judge, agent=FakeAgent(writer), planner_model=_planner(chair_plan.model_dump(mode="json")),
settings=settings, runtime=FakeRuntime(Language.BLENDER), n_candidates=2)
rec = track.run(spec, ws)
- assert seen == {"baseline_c0": 2, "baseline_c1": 1}
+ assert seen == {"baseline_c0": 1, "baseline_c1": 1} # one session per candidate: no candidate-level retry
kinds = [e["event"] for e in EventLog(ws.events_path).read()]
- assert "candidate.retry" in kinds and "candidate.failed" not in kinds
- assert "best-of-2: selected c0" in rec.rounds[0].notes and "_cand/c0" in (ws.src / "model.py").read_text()
+ assert "candidate.failed" in kinds and "candidate.retry" not in kinds
+ assert "best-of-2: selected c1" in rec.rounds[0].notes and "_cand/c1" in (ws.src / "model.py").read_text()
def test_default_motion_checks_on_real_urdf(tmp_path):
diff --git a/harness/tests/orchestrator_tracks/test_fix_generation.py b/harness/tests/orchestrator_tracks/test_generation_parsing.py
similarity index 100%
rename from harness/tests/orchestrator_tracks/test_fix_generation.py
rename to harness/tests/orchestrator_tracks/test_generation_parsing.py
diff --git a/harness/tests/orchestrator_tracks/test_generation_planner_repair.py b/harness/tests/orchestrator_tracks/test_generation_planner_repair.py
index 50b672e2..ec9203e2 100644
--- a/harness/tests/orchestrator_tracks/test_generation_planner_repair.py
+++ b/harness/tests/orchestrator_tracks/test_generation_planner_repair.py
@@ -128,9 +128,10 @@ def test_planner_validates_retries_and_writes(tmp_ws):
assert isinstance(p, StaticPlan) and p.object_name == _valid_plan_dict()["object_name"] and tmp_ws.plan_path.is_file()
assert len(model.requests) == 2 and "failed validation" in model.requests[1].messages[-1].text
assert model.requests[0].response_schema is not None and "PascalCase" in model.requests[0].system
- from codeverse3d.tracks.planner import PLAN_MAX_WAIT_S, plan_wait_s
+ from codeverse3d.tracks.planner import plan_wait_s
- assert all(PLAN_MAX_WAIT_S <= r.max_wait_s <= plan_wait_s(r.max_output_tokens) for r in model.requests)
+ # every planner call waits at most what the clock has left (never past the budget), and never more than its size asks
+ assert all(60 <= r.max_wait_s <= min(plan_wait_s(r.max_output_tokens), budget.timeout_s(float("inf")) + 2) for r in model.requests)
# deterministic acceptance items from constraints were appended
texts = " ".join(a.text for a in p.acceptance)
assert "height = 0.820" in texts and "Includes: armrests" in texts
diff --git a/harness/tests/orchestrator_tracks/test_fix_batch2.py b/harness/tests/orchestrator_tracks/test_lifecycle_regressions.py
similarity index 100%
rename from harness/tests/orchestrator_tracks/test_fix_batch2.py
rename to harness/tests/orchestrator_tracks/test_lifecycle_regressions.py
diff --git a/harness/tests/orchestrator_tracks/test_prompts_assets.py b/harness/tests/orchestrator_tracks/test_prompts_assets.py
index cfa60925..5542cc3e 100644
--- a/harness/tests/orchestrator_tracks/test_prompts_assets.py
+++ b/harness/tests/orchestrator_tracks/test_prompts_assets.py
@@ -89,7 +89,7 @@ def test_scene_templates_render_and_asset_stage_with_blender(tmp_ws, settings):
out = render(f"tracks/{tpl}.j2", **st._ctx(ctx, **extra))
assert "Harbour at dusk" in out and "DOM" in out
sa = render("tracks/scene_asset.j2", **base_prompt_context(ctx, asset_name="Bollard", asset_kind="threejs", asset_description="d", asset_size=(0.3, 0.5, 0.3),
- asset_file="src/assets/bollard.js", asset_language="scene_threejs", fix_instructions=["- x"], current_code=""))
+ asset_file="src/assets/bollard.js", asset_language="scene_threejs", fix_instructions=["- x"], current_code="", library_rows=""))
assert "buildBollard" in sa and "FIX PASS" in sa
# D100: a three.js asset whose sheet names a library factory gets that factory's catalog row
from codeverse3d.tracks.scene_assets import _asset_prompt
diff --git a/harness/tests/orchestrator_tracks/test_refactor_batch2.py b/harness/tests/orchestrator_tracks/test_refine_groups_and_judge_views.py
similarity index 100%
rename from harness/tests/orchestrator_tracks/test_refactor_batch2.py
rename to harness/tests/orchestrator_tracks/test_refine_groups_and_judge_views.py
diff --git a/harness/tests/orchestrator_tracks/test_write_scope.py b/harness/tests/orchestrator_tracks/test_write_scope.py
index 308b7d20..867d4e3c 100644
--- a/harness/tests/orchestrator_tracks/test_write_scope.py
+++ b/harness/tests/orchestrator_tracks/test_write_scope.py
@@ -20,7 +20,7 @@ from codeverse3d.tracks.planner import plan_example
from codeverse3d.tracks.scene import SceneTrack
from codeverse3d.tracks.static_object import StaticObjectTrack
from codeverse3d.workspace import Workspace
-from tests.orchestrator_tracks.test_fix_generation import scripted
+from tests.orchestrator_tracks.test_generation_parsing import scripted
from .conftest import make_spec
from .fakes import FakeAgent, FakeRuntime, FakeServices
diff --git a/harness/tests/prompts/test_variants.py b/harness/tests/prompts/test_variants.py
index 5cacc067..8a0121a3 100644
--- a/harness/tests/prompts/test_variants.py
+++ b/harness/tests/prompts/test_variants.py
@@ -1,4 +1,4 @@
-"""The variant layer (D105): selection, the prompt overlay, caches, activation, knobs, the record."""
+"""The variant layer (D105): selection, the prompt overlay, caches, activation, the record."""
from __future__ import annotations
@@ -15,14 +15,13 @@ from codeverse3d.prompts import variants as V
from codeverse3d.prompts.variants import (
VARIANTS_DIR,
activate,
- knobs_applied,
resolve_variant,
variant_info,
)
from tests.prompts.conftest import PROMPTS_DIR
TV_TOML = ('[variant]\nname = "tv"\nversion = 3\ngenerators = ["agy:gemini-4-argon"]\n'
- 'description = "a test variant"\n\n[knobs]\nskills_max = 4\nskills_only = "c3d-part-contact"\n')
+ 'description = "a test variant"\n')
def _clear() -> None:
prompts._load.cache_clear()
@@ -97,22 +96,5 @@ def test_an_overlay_file_shadows_the_base_and_a_missing_one_falls_back(overlay):
# --------------------------------------------------------------------------- provenance
-# --------------------------------------------------------------------------- knobs
-
# --------------------------------------------------------------------------- the run
-def test_knobs_apply_as_defaults_a_stated_field_wins_and_nothing_outlives_the_block(overlay, monkeypatch):
- """Regression: a variant's knobs leaked onto the process Settings and the next base run inherited them."""
- monkeypatch.delenv("C3D_SKILLS_MAX", raising=False)
- monkeypatch.delenv("C3D_SKILLS_ONLY", raising=False)
- info = variant_info("tv")
- default_max = Settings().skills_max
- s = Settings(skills_only="c3d-scene-water") # stated by the user: never touched
- with knobs_applied(s, info) as applied:
- assert applied == {"skills_max": 4} and (s.skills_max, s.skills_only) == (4, "c3d-scene-water")
- assert (s.skills_max, s.skills_only) == (default_max, "c3d-scene-water")
- s = Settings()
- with pytest.raises(RuntimeError), knobs_applied(s, info):
- assert (s.skills_max, s.skills_only) == (4, "c3d-part-contact")
- raise RuntimeError("boom")
- assert (s.skills_max, s.skills_only) == (default_max, "") and not set(s.model_fields_set) & V.KNOB_FIELDS
diff --git a/harness/tests/scene_prompts/test_quality_teaching.py b/harness/tests/scene_prompts/test_quality_teaching.py
index 3193ba67..bbfeb101 100644
--- a/harness/tests/scene_prompts/test_quality_teaching.py
+++ b/harness/tests/scene_prompts/test_quality_teaching.py
@@ -32,7 +32,7 @@ def test_the_env_and_zone_briefs_hand_the_enclosure_to_env_for_an_interior() ->
base = dict(title="t", spec_prompt="p", setting="s", mood="m", bounds="b", environment="e", frame_doc="", contract="",
zones_table="", cameras="", effects="", animation="", asset_api="", textures="", constraints_text="",
references="", skills="", tool_cards="", zone_name="Z", zone_description="d", zone_bbox="bb", zone_contents=[],
- zone_file="src/zones/z.js", neighbours=[], layout="")
+ zone_file="src/zones/z.js", neighbours=[], layout="", output_format="")
env_in = render("tracks/scene_env.j2", **base, interior=True, glazed=False)
env_out = render("tracks/scene_env.j2", **base, interior=False, glazed=False)
assert "This is an INTERIOR" in env_in and "roomShell" in env_in and "openings" in env_in