Skip to content

Official Codebase of 3DCodeVerse

Python

tests: core, install, gallery, flywheel_cli, third pass — route and runs_dir twins fold, three slow tests get cheap (298 → 195, −305 lines)

gaoypeng24accccb982a
diff --git a/harness/tests/core/test_cli_render_size.py b/harness/tests/core/test_cli_render_size.py
index 47d6c57c..7ebc9796 100644
--- a/harness/tests/core/test_cli_render_size.py
+++ b/harness/tests/core/test_cli_render_size.py
@@ -49,39 +49,28 @@ def captured(monkeypatch) -> dict[str, Any]:
return seen
-def _run(tmp_path: Path, language: Language, track: Track) -> Path:
- runs = tmp_path / "runs"
- ws = Workspace(runs / "r1").create()
- ws.write_json(ws.spec_path, Spec(id="r1", track=track, language=language, prompt="a stool",
+def _run(runs: Path, slug: str, language: Language, track: Track) -> None:
+ ws = Workspace(runs / slug).create()
+ ws.write_json(ws.spec_path, Spec(id=slug, track=track, language=language, prompt="a stool",
backends=Backends(generator="gemini-cli:gemini-3.6-flash")))
(ws.artifacts / "object.glb").write_bytes(b"glTF\x02\x00\x00\x00")
- return runs
-def test_object_render_uses_configured_size_and_explicit_flags_win(tmp_path, captured, monkeypatch):
- runs = _run(tmp_path, Language.BLENDER, Track.STATIC_OBJECT)
- monkeypatch.setenv("C3D_RENDER__WIDTH", "1600")
- monkeypatch.setenv("C3D_RENDER__HEIGHT", "1200")
+def test_render_uses_the_configured_object_and_scene_sizes_and_explicit_flags_win(tmp_path, captured, monkeypatch):
+ runs = tmp_path / "runs"
+ _run(runs, "obj", Language.BLENDER, Track.STATIC_OBJECT)
+ _run(runs, "scn", Language.SCENE_THREEJS, Track.SCENE)
+ for name, value in (("WIDTH", "1600"), ("HEIGHT", "1200"), ("SCENE_WIDTH", "1920"), ("SCENE_HEIGHT", "1080")):
+ monkeypatch.setenv(f"C3D_RENDER__{name}", value)
from codeverse3d.config import get_settings
get_settings.cache_clear()
- res = runner.invoke(app, ["render", "r1", "--runs-dir", str(runs)])
-
+ res = runner.invoke(app, ["render", "obj", "--runs-dir", str(runs)])
assert res.exit_code == 0, res.output
assert (captured.get("width"), captured.get("height")) == (1600, 1200)
- res = runner.invoke(app, ["render", "r1", "--runs-dir", str(runs), "--width", "320", "--height", "240"])
+ res = runner.invoke(app, ["render", "obj", "--runs-dir", str(runs), "--width", "320", "--height", "240"])
assert res.exit_code == 0, res.output
assert (captured.get("width"), captured.get("height")) == (320, 240)
-
-
-def test_scene_render_uses_the_scene_size(tmp_path, captured, monkeypatch):
- runs = _run(tmp_path, Language.SCENE_THREEJS, Track.SCENE)
- monkeypatch.setenv("C3D_RENDER__SCENE_WIDTH", "1920")
- monkeypatch.setenv("C3D_RENDER__SCENE_HEIGHT", "1080")
- from codeverse3d.config import get_settings
-
- get_settings.cache_clear()
- res = runner.invoke(app, ["render", "r1", "--runs-dir", str(runs)])
-
+ res = runner.invoke(app, ["render", "scn", "--runs-dir", str(runs)])
assert res.exit_code == 0, res.output
assert (captured.get("width"), captured.get("height")) == (1920, 1080)
diff --git a/harness/tests/core/test_config_env_aliases.py b/harness/tests/core/test_config_env_aliases.py
index ac615768..916f1841 100644
--- a/harness/tests/core/test_config_env_aliases.py
+++ b/harness/tests/core/test_config_env_aliases.py
@@ -30,7 +30,10 @@ def test_every_switch_speaks_one_grammar(monkeypatch, caplog):
assert env in caplog.text
-def test_flat_and_nested_max_in_flight_aliases(monkeypatch):
+def test_the_in_flight_cap_spellings_empty_zero_and_garbage(monkeypatch):
+ """Flat beats nested; empty is unset; 0 means unlimited; a cap is not a switch — garbage stops the run."""
+ from codeverse3d.config import Rate
+
monkeypatch.delenv("C3D_RATE__MAX_IN_FLIGHT", raising=False)
monkeypatch.setenv("C3D_MAX_IN_FLIGHT", "16")
assert Settings().rate.max_in_flight == 16
@@ -42,26 +45,15 @@ def test_flat_and_nested_max_in_flight_aliases(monkeypatch):
monkeypatch.delenv("C3D_RENDER__GPU", raising=False)
monkeypatch.setenv("C3D_RENDER_GPU", " ON ") # the documented flat spelling, case- and space-tolerant
assert Settings().render.gpu == "on"
-
-
-def test_empty_is_unset_and_garbage_in_a_sizing_knob_is_loud(monkeypatch):
- """A cap is not a switch: an unparsable one stops the run before it starts."""
monkeypatch.delenv("C3D_RATE__MAX_IN_FLIGHT", raising=False)
monkeypatch.setenv("C3D_MAX_IN_FLIGHT", "")
assert Settings().rate.max_in_flight == 64
+ assert Rate(max_in_flight=0).max_in_flight == 0, "0 stays legal: it means unlimited"
monkeypatch.setenv("C3D_MAX_IN_FLIGHT", "sixteen")
with pytest.raises(ValueError, match="max_in_flight"):
Settings()
-
-def test_zero_in_flight_is_unlimited_and_a_bad_turn_cap_asks_for_none():
- from codeverse3d.config import Limits, Rate
-
- assert Rate(max_in_flight=0).max_in_flight == 0, "0 stays legal: it means unlimited"
- assert Limits(agent_max_turns=-1).agent_max_turns == 0, "a switch: a bad cap warns and asks for none"
-
-
# --------------------------------------------------------------------------- yaml layering
def test_the_environment_beats_the_config_files_per_setting(monkeypatch):
diff --git a/harness/tests/core/test_contract_additions.py b/harness/tests/core/test_contract_additions.py
index 7bfd6c4d..9441935d 100644
--- a/harness/tests/core/test_contract_additions.py
+++ b/harness/tests/core/test_contract_additions.py
@@ -7,19 +7,6 @@ from codeverse3d.contracts.common import Language, Track
from codeverse3d.contracts.spec import RunOptions, Spec
-# ------------------------------------------------------------------ RunOptions
-def test_spec_options_legacy_roundtrip():
- s = Spec(id="x", track=Track.STATIC_OBJECT, language=Language.BLENDER, prompt="a chair")
- d = s.model_dump(mode="json")
- d.pop("options")
- s2 = Spec(**d) # old spec.json without options still parses
- assert s2.options.candidates is None and s2.options.texture is False
- s = Spec(id="x", track=Track.SCENE, language=Language.SCENE_THREEJS, prompt="p",
- options=RunOptions(candidates=3, texture=True))
- s2 = Spec(**s.model_dump(mode="json"))
- assert s2.options == RunOptions(candidates=3, texture=True)
-
-
def test_spec_options_do_not_change_the_plan_stage_hash():
from codeverse3d.orchestrator import hash_inputs
from codeverse3d.tracks.lifecycle import plan_stage_inputs
@@ -30,10 +17,15 @@ def test_spec_options_do_not_change_the_plan_stage_hash():
def test_records_written_before_the_2026_08_30_field_retirements_still_load():
- """Stored runs carry keys since retired; every model here must keep ignoring them."""
+ """Stored runs carry keys since retired (or lack ones added since); every model here must keep loading them."""
from codeverse3d.contracts.agent import AgentJob
from codeverse3d.contracts.run import RoundRecord, SkillRead
+ d = Spec(id="x", track=Track.STATIC_OBJECT, language=Language.BLENDER, prompt="a chair").model_dump(mode="json")
+ d.pop("options")
+ s2 = Spec(**d) # old spec.json without options still parses
+ assert s2.options.candidates is None and s2.options.texture is False
+
rs = RenderSet.model_validate({"renderer": "blender", "turntable": "renders/turntable.mp4",
"contact_sheet": "renders/sheet.png"})
assert rs.contact_sheet == "renders/sheet.png" and not hasattr(rs, "turntable")
diff --git a/harness/tests/core/test_contracts.py b/harness/tests/core/test_contracts.py
index bc0ec359..3f1a0851 100644
--- a/harness/tests/core/test_contracts.py
+++ b/harness/tests/core/test_contracts.py
@@ -7,7 +7,7 @@ import math
import pytest
from pydantic import ValidationError
-from codeverse3d.contracts.common import Budget, Language, Track
+from codeverse3d.contracts.common import Budget
from codeverse3d.contracts.plan import (
ArticulatedPlan,
AssetPlan,
@@ -50,11 +50,6 @@ def test_view_presets_unique():
assert clay_top.elevation_deg == 88.0
-def test_spec_rejects_language_outside_track():
- with pytest.raises(ValidationError):
- Spec(id="x", track=Track.SCENE, language=Language.BLENDER, prompt="p")
-
-
def test_static_plan_validates_attach_and_duplicates():
with pytest.raises(ValidationError):
StaticPlan(object_name="o", summary="s", overall_bbox=_box(),
diff --git a/harness/tests/core/test_names.py b/harness/tests/core/test_names.py
index 1ef03b3e..3e693caa 100644
--- a/harness/tests/core/test_names.py
+++ b/harness/tests/core/test_names.py
@@ -8,8 +8,6 @@ import ast
import re
from pathlib import Path
-import pytest
-
from codeverse3d.contracts.common import Track
from codeverse3d.contracts.plan import StaticPlan
from codeverse3d.conventions import PASCAL_RE, to_pascal, to_snake
@@ -46,40 +44,25 @@ def _wrapper_snake():
return ns["_snake"]
[email protected]("name", VALID)
-def test_acronym_names_are_valid_pascal(name):
- assert PASCAL_RE.match(name)
-
-
[email protected]("name", INVALID)
-def test_lowercase_first_spaces_hyphens_and_empty_are_rejected(name):
- assert not PASCAL_RE.match(name)
-
-
-def test_the_cadquery_wrapper_copy_is_the_conventions_rule():
- """run_cq.py cannot import codeverse3d, so it keeps a copy — the same pattern, same verdicts."""
+def test_the_naming_rule_and_its_stdlib_wrapper_copies():
+ """One rule; run_cq.py / run_bpy_links.py cannot import codeverse3d, so they keep copies — same verdicts."""
+ assert [n for n in VALID if not PASCAL_RE.match(n)] == []
+ assert [n for n in INVALID if PASCAL_RE.match(n)] == []
wrap = _wrapper_pascal_re()
assert wrap.pattern == PASCAL_RE.pattern
assert [bool(wrap.match(n)) for n in VALID + INVALID] == [bool(PASCAL_RE.match(n)) for n in VALID + INVALID]
-
-
-def test_the_links_wrapper_snake_copy_is_to_snake():
snake = _wrapper_snake()
for n in [*VALID, *INVALID[:-3], "LPCompressor", "Seat2Cushion"]:
if n.strip():
assert snake(n) == to_snake(n), n
[email protected]("name", list(SNAKE))
-def test_snake_round_trip_of_acronym_names(name):
- snake, rebuilt = SNAKE[name]
- assert to_snake(name) == snake
- assert to_pascal(snake) == rebuilt # rebuilding from the key loses the acronym
- if "_" not in name: # ... but for a plan name never the key itself
- assert to_snake(to_pascal(name)) == snake == to_snake(rebuilt)
-
-
-def test_to_pascal_keeps_a_valid_name_and_still_normalises_the_rest():
+def test_snake_round_trip_and_to_pascal():
+ for name, (snake, rebuilt) in SNAKE.items():
+ assert to_snake(name) == snake, name
+ assert to_pascal(snake) == rebuilt, name # rebuilding from the key loses the acronym
+ if "_" not in name: # ... but for a plan name never the key itself
+ assert to_snake(to_pascal(name)) == snake == to_snake(rebuilt), name
assert [to_pascal(n) for n in ("TVStand", "LED", "A", "HDMIPort1", "SeatCushion")] == ["TVStand", "LED", "A", "HDMIPort1", "SeatCushion"]
# an instance suffix / underscores / spaces / hyphens / lowercase still go through to_snake
# (an instance name is not a plan name: to_pascal folds its suffix, CPU_2 -> Cpu2 -> key cpu2,
diff --git a/harness/tests/core/test_plan_normalise.py b/harness/tests/core/test_plan_normalise.py
index ea136f54..e14e15e9 100644
--- a/harness/tests/core/test_plan_normalise.py
+++ b/harness/tests/core/test_plan_normalise.py
@@ -49,7 +49,7 @@ def _joint(p: ArticulatedPlan, j: dict):
return next(x for x in p.joints if x.name == j["name"])
-def test_degree_shaped_limits_and_rest_are_converted_together():
+def test_degree_shaped_limits_convert_together_and_a_two_turn_range_becomes_continuous():
d = _raw()
j = _revolute(d, 0.0, 90.0, 90.0)
p = ArticulatedPlan.model_validate(d)
@@ -59,13 +59,10 @@ def test_degree_shaped_limits_and_rest_are_converted_together():
assert sum("looked like degrees" in n for n in p.normalisations) == 1
assert not any("continuous" in n for n in p.normalisations)
-
-def test_a_two_turn_degree_range_falls_through_to_continuous():
d = _raw()
j = _revolute(d, -360.0, 360.0)
p = ArticulatedPlan.model_validate(d)
- fixed = _joint(p, j)
- assert fixed.type == "continuous"
+ assert _joint(p, j).type == "continuous"
assert any("looked like degrees" in n for n in p.normalisations) and any("> 2π → continuous" in n for n in p.normalisations)
diff --git a/harness/tests/core/test_proc.py b/harness/tests/core/test_proc.py
index bdc73cc5..f96708fa 100644
--- a/harness/tests/core/test_proc.py
+++ b/harness/tests/core/test_proc.py
@@ -50,11 +50,11 @@ def test_a_detached_descendant_holding_the_pipes_cannot_extend_the_timeout(tmp_p
"""A detached descendant holding stdout cannot extend the caller's timeout."""
import codeverse3d.proc as proc_mod
- monkeypatch.setattr(proc_mod, "DRAIN_TIMEOUT_S", 1.0)
+ monkeypatch.setattr(proc_mod, "DRAIN_TIMEOUT_S", 0.3)
t0 = time.monotonic()
r = run_subprocess(["bash", "-c", "setsid sleep 20 & echo started; sleep 20"],
- cwd=tmp_path, timeout_s=1.0)
+ cwd=tmp_path, timeout_s=0.3)
elapsed = time.monotonic() - t0
assert r.timed_out and r.returncode != 0
@@ -150,23 +150,14 @@ def test_keyboard_interrupt_kills_the_group(tmp_path: Path):
assert_pid_gone(int(pid_file.read_text()), group=True)
-def test_invalid_utf8_replaces_instead_of_raising(tmp_path: Path):
- """One bad byte is replaced instead of discarding the subprocess result."""
- r = run_subprocess(
- [sys.executable, "-c", "import sys; sys.stdout.buffer.write(b'ok \\xff\\xfe end')"],
- cwd=tmp_path, timeout_s=30,
- )
- assert r.returncode == 0 and not r.timed_out
- assert r.stdout == "ok �� end"
-
-
@pytest.mark.timeout(120)
def test_huge_output_is_bounded_head_and_tail(tmp_path: Path):
- """Large output is bounded while preserving its head, tail, and truncation marker."""
+ """Large output is bounded while preserving its head, tail, and truncation marker; one bad byte
+ is replaced instead of discarding the subprocess result."""
code = (
"import sys\n"
"w = sys.stdout.buffer.write\n"
- "w(b'HEADSTART\\n')\n"
+ "w(b'HEADSTART ok \\xff\\xfe end\\n')\n"
"chunk = b'x' * (1 << 20)\n"
"for _ in range(48):\n"
" w(chunk)\n"
@@ -175,7 +166,7 @@ def test_huge_output_is_bounded_head_and_tail(tmp_path: Path):
r = run_subprocess([sys.executable, "-c", code], cwd=tmp_path, timeout_s=110)
assert r.returncode == 0 and not r.timed_out
assert len(r.stdout) <= STREAM_BUDGET_BYTES + 200 # the budget plus the truncation marker
- assert r.stdout.startswith("HEADSTART") # head survived
+ assert r.stdout.startswith("HEADSTART ok �� end") # head survived, invalid utf-8 replaced
assert r.stdout.rstrip("\n").endswith("TAILEND") # tail survived
assert "bytes dropped" in r.stdout # marker sits between them
cut = r.stdout.index("bytes dropped")
diff --git a/harness/tests/core/test_run_layout.py b/harness/tests/core/test_run_layout.py
index a5696529..ed83a8b7 100644
--- a/harness/tests/core/test_run_layout.py
+++ b/harness/tests/core/test_run_layout.py
@@ -123,14 +123,17 @@ def test_a_copied_run_reads_its_own_files_while_the_original_still_exists(tmp_pa
assert Workspace(a).rebase(str(sheet_a)) == sheet_a
-def test_a_stale_index_lock_names_the_remedy(tmp_path):
+def test_a_stale_index_lock_names_the_remedy(tmp_path, monkeypatch):
"""A stale git index lock is surfaced with its exact safe remedy."""
import subprocess
import pytest
+ import codeverse3d.workspace as W
from codeverse3d.workspace import Workspace, WorkspaceGitError
+ monkeypatch.setattr(W.time, "sleep", lambda _s: None) # the live-holder retry backoff: nothing to wait for
+
ws = Workspace(tmp_path / "run").create()
ws.src.mkdir(parents=True, exist_ok=True)
(ws.src / "model.py").write_text("x = 1\n")
diff --git a/harness/tests/core/test_runlock.py b/harness/tests/core/test_runlock.py
index 6449e2c6..04ef5af8 100644
--- a/harness/tests/core/test_runlock.py
+++ b/harness/tests/core/test_runlock.py
@@ -37,13 +37,17 @@ def _child_script(run_root: Path, ready: Path, go: Path) -> str:
def test_a_second_process_is_refused_and_a_killed_holder_never_wedges_the_run(tmp_path: Path):
- """Two processes cannot mutate one workspace; the kernel drops the flock when the holder dies."""
+ """Two processes cannot mutate one workspace; ``--force`` may overwrite a dead run, never evict a
+ live holder; the kernel drops the flock when the holder dies."""
import signal
import subprocess
import sys
+ from codeverse3d.cli._common import CliError, mutating
+
run_root = tmp_path / "runs" / "slug"
run_root.mkdir(parents=True)
+ (run_root / "spec.json").write_text("{}") # non-empty, so --force would rmtree it
ready, go = tmp_path / "ready", tmp_path / "go"
child = subprocess.Popen([sys.executable, "-c", _child_script(run_root, ready, go)])
try:
@@ -57,6 +61,10 @@ def test_a_second_process_is_refused_and_a_killed_holder_never_wedges_the_run(tm
msg = str(got.value)
assert str(child.pid) in msg # names the holder, so a human can kill THAT pid
assert "pkill" in msg, "and must warn against the pattern kill that took out 13 runs"
+ with pytest.raises(CliError) as ei, mutating(run_root, what="3dcode make slug"):
+ pytest.fail("the mutation boundary must not be entered")
+ assert ei.value.exit_code == 2
+ assert (run_root / "spec.json").exists(), "the live holder's workspace must survive"
finally:
child.send_signal(signal.SIGKILL)
child.wait(timeout=10)
@@ -105,30 +113,3 @@ def test_the_lock_file_lives_outside_the_run_directory(tmp_path: Path):
assert lock.is_file()
assert run_root not in lock.parents
assert lock.parent == run_root.parent / LOCKS_DIR
-
-
-def test_force_refuses_to_wipe_a_run_another_holder_is_using(tmp_path: Path):
- """``--force`` may overwrite a dead run, never evict a live holder."""
- import subprocess
- import sys
-
- from codeverse3d.cli._common import CliError, mutating
-
- run_root = tmp_path / "runs" / "held_run"
- run_root.mkdir(parents=True)
- (run_root / "spec.json").write_text("{}") # non-empty, so --force would rmtree it
- ready, go = tmp_path / "ready", tmp_path / "go"
- child = subprocess.Popen([sys.executable, "-c", _child_script(run_root, ready, go)])
- try:
- for _ in range(500):
- if ready.exists():
- break
- time.sleep(0.02)
- assert ready.read_text() == "held"
- with pytest.raises(CliError) as ei, mutating(run_root, what="3dcode make held_run"):
- pytest.fail("the mutation boundary must not be entered")
- assert ei.value.exit_code == 2
- assert (run_root / "spec.json").exists(), "the live holder's workspace must survive"
- finally:
- go.write_text("release")
- child.wait(timeout=10)
diff --git a/harness/tests/flywheel_cli/conftest.py b/harness/tests/flywheel_cli/conftest.py
index e74c5449..3410ae6e 100644
--- a/harness/tests/flywheel_cli/conftest.py
+++ b/harness/tests/flywheel_cli/conftest.py
@@ -56,7 +56,7 @@ def _judgment(score: float, passed: bool, plan: list[str]) -> Judgment:
def make_fake_run(
runs_dir: Path, slug: str = "wooden_chair_ab12cd34", *, prompt: str = "a wooden dining chair",
language: Language = Language.BLENDER, generator: str = "gemini-cli:gemini-3.6-flash",
- scores: tuple[float, float] = (0.55, 0.80), with_repair: bool = False, code_v2: str | None = None,
+ scores: tuple[float, float] = (0.55, 0.80), with_repair: bool = False,
) -> tuple[Workspace, RunRecord]:
ws = Workspace(runs_dir / slug).create()
track = Track.SCENE if language is Language.SCENE_THREEJS else (
@@ -76,7 +76,7 @@ def make_fake_run(
rounds = []
for i, sc in enumerate(scores):
if i == 1:
- e.write_text(code_v2 or "# round 1\nimport bpy\n\nbpy.ops.mesh.primitive_cube_add(size=2.0)\n")
+ e.write_text("# round 1\nimport bpy\n\nbpy.ops.mesh.primitive_cube_add(size=2.0)\n")
(ws.src / "parts").mkdir(exist_ok=True)
(ws.src / "parts" / "leg.py").write_text("LEG = 0.04\n")
commit = c0 if i == 0 else ws.commit(f"round {i}")
diff --git a/harness/tests/flywheel_cli/test_captions.py b/harness/tests/flywheel_cli/test_captions.py
index 6c9cf799..4ef1204f 100644
--- a/harness/tests/flywheel_cli/test_captions.py
+++ b/harness/tests/flywheel_cli/test_captions.py
@@ -24,7 +24,7 @@ GOOD = {
}
-def test_caption_sample_captions_the_picked_round_and_records_provenance(fake_run):
+def test_caption_sample_captions_the_picked_round_records_provenance_and_retries_once(fake_run):
ws, rec = fake_run
m = FakeChatModel([GOOD])
caption_sample(ws, rec, "fake:fake", model=m)
@@ -34,6 +34,14 @@ def test_caption_sample_captions_the_picked_round_and_records_provenance(fake_ru
prov = load_record(ws).extra["captions"]["provenance"]
assert prov["captioner"] == "fake:fake" and prov["cost_usd"] == pytest.approx(0.002)
+ bad = dict(GOOD, detailed="Made with bpy.ops primitives: a chair with four legs and a backrest.")
+ m = FakeChatModel([bad, GOOD])
+ caps = caption_sample(ws, rec, "fake:fake", model=m)
+ assert caps.detailed == GOOD["detailed"] and len(m.requests) == 2
+ assert "bpy" in m.requests[1].messages[-1].text
+ with pytest.raises(CaptionError):
+ caption_sample(ws, rec, "fake:fake", model=FakeChatModel([bad, bad]))
+
def test_caption_cmd_writes_the_captioner_row_into_the_run_ledger(fake_run, monkeypatch):
"""The captioner's priced call joins the run's ledger."""
@@ -57,18 +65,6 @@ def test_caption_cmd_writes_the_captioner_row_into_the_run_ledger(fake_run, monk
assert (ws.deliverable / "captions.json").is_file()
-def test_caption_retry_then_fail(fake_run):
- ws, rec = fake_run
- bad = dict(GOOD, detailed="Made with bpy.ops primitives: a chair with four legs and a backrest.")
- m = FakeChatModel([bad, GOOD])
- caps = caption_sample(ws, rec, "fake:fake", model=m)
- assert caps.detailed == GOOD["detailed"] and len(m.requests) == 2
- assert "bpy" in m.requests[1].messages[-1].text
- m2 = FakeChatModel([bad, bad])
- with pytest.raises(CaptionError):
- caption_sample(ws, rec, "fake:fake", model=m2)
-
-
@pytest.mark.live
def test_caption_live_gemini(fake_run):
from codeverse3d.config import get_settings
@@ -125,7 +121,9 @@ def _old_urdf_run(tmp_path):
def test_an_old_runs_caption_then_export_still_ships_its_glb_gif_and_meshes(tmp_path):
- """The caption's record rewrite once dropped the raw best_round, and with it the old run's outputs."""
+ """The caption's record rewrite once dropped the raw best_round, and with it the old run's outputs;
+ a record already rewritten without ``best_round`` falls back to its packaged deliverable."""
+ from codeverse3d.addons import select
from codeverse3d.addons.dataset.export import export_one
ws, rec = _old_urdf_run(tmp_path)
@@ -137,12 +135,6 @@ def test_an_old_runs_caption_then_export_still_ships_its_glb_gif_and_meshes(tmp_
sample = export_one(ws, load_record(ws), tmp_path / "ds")
assert {"renders/object.glb", "renders/preview.gif", "meshes/base.glb"} <= set(sample.file_hashes)
-
-def test_an_old_run_whose_record_lost_best_round_falls_back_to_its_deliverable(tmp_path):
- """A record already rewritten without ``best_round``: its packaged deliverable names the round."""
- from codeverse3d.addons import select
-
- ws, rec = _old_urdf_run(tmp_path)
data = json.loads(ws.record_path.read_text())
del data["best_round"]
ws.write_json(ws.record_path, data)
diff --git a/harness/tests/flywheel_cli/test_cli.py b/harness/tests/flywheel_cli/test_cli.py
index 74c8ce37..fae892a8 100644
--- a/harness/tests/flywheel_cli/test_cli.py
+++ b/harness/tests/flywheel_cli/test_cli.py
@@ -78,17 +78,6 @@ def test_make_no_run_creates_valid_workspace(tmp_path: Path):
assert r3.exit_code == 0
-def test_flywheel_commands(runs_dir: Path, tmp_path: Path):
- out = tmp_path / "ds"
- r = runner.invoke(app, ["flywheel", "export", str(runs_dir), str(out), "--pack"])
- assert r.exit_code == 0, r.output
- assert (out / "metadata.parquet").is_file() and (out / "samples-000.tar").is_file()
- r = runner.invoke(app, ["flywheel", "pairs", str(runs_dir), str(tmp_path / "p.jsonl")])
- assert r.exit_code == 0 and "pairs" in r.output
- r = runner.invoke(app, ["flywheel", "index", str(runs_dir), str(tmp_path / "i.sqlite")])
- assert r.exit_code == 0 and "indexed 3 runs" in r.output
-
-
def test_tools_list_handles_an_optional_spatial_install():
r = runner.invoke(app, ["tools", "list"])
# spatial tools may or may not be installed yet; either a table or a clear message, never a traceback
@@ -141,11 +130,33 @@ def _write_stool_plan(ws):
ws.write_json(ws.plan_path, plan)
-def test_judge_rejudges_with_in_run_inputs(runs_dir: Path, monkeypatch):
+def test_commands_over_a_runs_dir(runs_dir: Path, tmp_path: Path, monkeypatch):
+ """flywheel export/pairs/index/caption, judge (in-run inputs, the reference judge, a typed failure), status."""
+ import codeverse3d.addons.dataset.captions as cap
import codeverse3d.judges.vlm_judge as vj
+ from codeverse3d.config import get_settings
from codeverse3d.workspace import Workspace
from tests.flywheel_cli.conftest import tiny_png
+ out = tmp_path / "ds"
+ r = runner.invoke(app, ["flywheel", "export", str(runs_dir), str(out), "--pack"])
+ assert r.exit_code == 0, r.output
+ assert (out / "metadata.parquet").is_file() and (out / "samples-000.tar").is_file()
+ r = runner.invoke(app, ["flywheel", "pairs", str(runs_dir), str(tmp_path / "p.jsonl")])
+ assert r.exit_code == 0 and "pairs" in r.output
+ r = runner.invoke(app, ["flywheel", "index", str(runs_dir), str(tmp_path / "i.sqlite")])
+ assert r.exit_code == 0 and "indexed 3 runs" in r.output
+
+ # no --model: Settings.default_captioner, not a literal the CLI kept (a configured one was ignored)
+ seen: list[str] = []
+ monkeypatch.setattr(get_settings(), "default_captioner", "fake:configured")
+ monkeypatch.setattr(cap, "caption_sample", lambda ws, rec, model, **kw: seen.append(model) or SimpleNamespace(instruction="x"))
+ r = runner.invoke(app, ["flywheel", "caption", "wooden_chair_ab12cd34", "--runs-dir", str(runs_dir), "--out",
+ str(tmp_path / "side")])
+ assert r.exit_code == 0, r.output
+ assert seen == ["fake:configured"]
+
+ # judge: re-judges with the run's own inputs
ws = Workspace(runs_dir / "wooden_chair_ab12cd34")
_write_stool_plan(ws)
tiny_png(ws.renders_dir(1) / "clay" / "view_top.png", (128, 128, 128))
@@ -164,24 +175,17 @@ def test_judge_rejudges_with_in_run_inputs(runs_dir: Path, monkeypatch):
assert (ws.artifacts / "judge" / "r01_cli.json").is_file()
assert "prompt images" in r.output
-
-def test_judge_uses_reference_judge_for_measured_rubrics(runs_dir: Path, monkeypatch):
- import codeverse3d.judges.vlm_judge as ref
-
+ # a measured rubric goes to the reference judge
class _FakeRef(_FakeVlm):
pass
- monkeypatch.setattr(ref, "ReferenceJudge", _FakeRef)
+ monkeypatch.setattr(vj, "ReferenceJudge", _FakeRef)
_FakeRef.captured.clear()
- r = runner.invoke(app, ["judge", "wooden_chair_ab12cd34", "--runs-dir", str(runs_dir),
- "--rubric", "reference_v1"])
+ r = runner.invoke(app, ["judge", "wooden_chair_ab12cd34", "--runs-dir", str(runs_dir), "--rubric", "reference_v1"])
assert r.exit_code == 0, r.output # no ValueError traceback any more
assert _FakeRef.captured["rubric"] == "reference_v1"
-
-def test_judge_wraps_value_error_as_cli_error(runs_dir: Path, monkeypatch):
- import codeverse3d.judges.vlm_judge as vj
-
+ # a judge ValueError is a CLI error
class _Boom(_FakeVlm):
def judge(self, inp):
raise ValueError("rubric has measured criteria but VlmJudge computes none")
@@ -189,8 +193,23 @@ def test_judge_wraps_value_error_as_cli_error(runs_dir: Path, monkeypatch):
monkeypatch.setattr(vj, "VlmJudge", _Boom)
r = runner.invoke(app, ["judge", "wooden_chair_ab12cd34", "--runs-dir", str(runs_dir)])
assert r.exit_code == 1
- out = r.output + str(getattr(r, "stderr", "") or "")
- assert "judge failed" in out and "Traceback" not in r.output
+ assert "judge failed" in r.output + str(getattr(r, "stderr", "") or "") and "Traceback" not in r.output
+
+ # status shows candidates and texturing
+ (ws.root / "rounds").mkdir(exist_ok=True)
+ (ws.root / "rounds" / "candidates.json").write_text(json.dumps({
+ "n": 2, "selected": 1,
+ "candidates": [{"index": 0, "label": "c0", "score": 0.51, "build_ok": True},
+ {"index": 1, "label": "c1", "score": 0.63, "build_ok": True}],
+ "pairwise": {"a": "c0", "b": "c1", "winner": "b", "confidence": 0.8}}))
+ rec = json.loads(ws.record_path.read_text())
+ rec.setdefault("extra", {})["texturing"] = {"shipped": True, "delta": 0.01, "reason": "", "n_textures": 3,
+ "glb_textured": "artifacts/object_textured.glb"}
+ ws.record_path.write_text(json.dumps(rec))
+ r = runner.invoke(app, ["status", "wooden_chair_ab12cd34", "--runs-dir", str(runs_dir)])
+ assert r.exit_code == 0, r.output
+ assert "candidates" in r.output and "c1" in r.output and "pairwise" in r.output
+ assert "texturing" in r.output and "shipped" in r.output
def test_calibration_rubric_map_includes_graphics(tmp_path: Path):
@@ -218,22 +237,9 @@ def test_make_invalid_combo_leaves_no_orphan_workspace(tmp_path: Path):
assert not (runs / "bad").exists(), "invalid spec must not leave an orphan run dir"
-def test_resume_budget_flags_rewrite_spec_and_emit_event(made_run, stub_track):
- runs, ws = made_run("--max-minutes", "1.0", "--rounds", "1")
- seen = {}
- stub_track(lambda spec, resume, force: seen.__setitem__("spec", spec))
- r2 = runner.invoke(app, ["resume", ws.name, "--runs-dir", str(runs), "--max-minutes", "60", "--rounds", "3"])
- assert r2.exit_code == 130
- spec = Spec.model_validate_json((ws / "spec.json").read_text())
- assert spec.budget.max_rounds == 3 and spec.budget.max_minutes == 60.0
- assert seen["spec"].budget.max_minutes == 60.0, "the resumed run must see the raised ceiling"
- events = [json.loads(line) for line in (ws / "events.jsonl").read_text().splitlines()]
- raised = [e for e in events if e.get("event") == "budget.raised"]
- assert raised and raised[0]["max_rounds"] == 3
-
-
def test_resume_refuses_a_finished_run_and_never_re_enters_it(made_run, stub_track):
- """A finished run never re-enters the track unless forced."""
+ """A finished run never re-enters the track unless forced; a raised cap is written to spec.json
+ and seen by the resumed run; spec drift is a clean CLI error."""
from codeverse3d.contracts.run import RunStatus
from codeverse3d.orchestrator import RunState
from codeverse3d.workspace import Workspace
@@ -245,8 +251,8 @@ def test_resume_refuses_a_finished_run_and_never_re_enters_it(made_run, stub_tra
"best_commit": "abc", "best_score": 0.7436}))
assert RunState.load(ws).status is RunStatus.STOPPED
- entered = []
- stub_track(lambda spec, resume, force: entered.append((resume, force)))
+ entered, specs = [], []
+ stub_track(lambda spec, resume, force: (entered.append((resume, force)), specs.append(spec)))
r = runner.invoke(app, ["resume", ws.root.name, "--runs-dir", str(runs)])
assert r.exit_code == 1 and "already finished" in r.output and "stop_reason='pass'" in r.output
assert entered == [], "the pipeline must not be re-entered"
@@ -261,51 +267,28 @@ def test_resume_refuses_a_finished_run_and_never_re_enters_it(made_run, stub_tra
assert r.exit_code == 1 and "status=max_rounds" in r.output
assert runner.invoke(app, ["resume", ws.root.name, "--runs-dir", str(runs), "--max-minutes", "90"]).exit_code == 1
assert runner.invoke(app, ["resume", ws.root.name, "--runs-dir", str(runs), "--rounds", "5"]).exit_code == 130
+ assert Spec.model_validate_json(ws.spec_path.read_text()).budget.max_rounds == 5
# a budget stop is the documented exception: it resumes when a cap is raised
RunState(status=RunStatus.BUDGET, stop_reason="budget").save(ws)
assert runner.invoke(app, ["resume", ws.root.name, "--runs-dir", str(runs)]).exit_code == 1
assert runner.invoke(app, ["resume", ws.root.name, "--runs-dir", str(runs), "--max-minutes", "60"]).exit_code == 130
+ assert Spec.model_validate_json(ws.spec_path.read_text()).budget.max_minutes == 60.0
+ assert specs[-1].budget.max_minutes == 60.0, "the resumed run must see the raised ceiling"
# an interrupted run is untouched by the guard
RunState(status=RunStatus.REFINING).save(ws)
assert runner.invoke(app, ["resume", ws.root.name, "--runs-dir", str(runs)]).exit_code == 130
-
-def test_a_spec_change_refusal_is_a_clean_cli_error(made_run, stub_track):
- """Spec drift is a typed CLI refusal, not a traceback."""
+ # spec drift is a typed CLI refusal, not a traceback
from codeverse3d.tracks.lifecycle import SpecChanged
- runs, ws = made_run()
stub_track(exc=SpecChanged("spec.json changed under this run; fork a new run or resume with --force"))
- r = runner.invoke(app, ["resume", ws.name, "--runs-dir", str(runs)])
- out = r.output + str(getattr(r, "stderr", "") or "")
- assert r.exit_code == 2 and "--force" in out
+ r = runner.invoke(app, ["resume", ws.root.name, "--runs-dir", str(runs)])
+ assert r.exit_code == 2 and "--force" in r.output + str(getattr(r, "stderr", "") or "")
assert "Traceback" not in r.output
-def test_status_shows_candidates_and_texturing(runs_dir: Path):
- import json as _json
-
- from codeverse3d.workspace import Workspace
-
- ws = Workspace(runs_dir / "wooden_chair_ab12cd34")
- (ws.root / "rounds").mkdir(exist_ok=True)
- (ws.root / "rounds" / "candidates.json").write_text(_json.dumps({
- "n": 2, "selected": 1,
- "candidates": [{"index": 0, "label": "c0", "score": 0.51, "build_ok": True},
- {"index": 1, "label": "c1", "score": 0.63, "build_ok": True}],
- "pairwise": {"a": "c0", "b": "c1", "winner": "b", "confidence": 0.8}}))
- rec = _json.loads(ws.record_path.read_text())
- rec.setdefault("extra", {})["texturing"] = {"shipped": True, "delta": 0.01, "reason": "", "n_textures": 3,
- "glb_textured": "artifacts/object_textured.glb"}
- ws.record_path.write_text(_json.dumps(rec))
- r = runner.invoke(app, ["status", "wooden_chair_ab12cd34", "--runs-dir", str(runs_dir)])
- assert r.exit_code == 0, r.output
- assert "candidates" in r.output and "c1" in r.output and "pairwise" in r.output
- assert "texturing" in r.output and "shipped" in r.output
-
-
def test_render_graphics_regenerates_frames(tmp_path: Path, monkeypatch):
import codeverse3d.languages as langs
from codeverse3d.contracts.artifacts import BuildResult
@@ -453,21 +436,6 @@ def test_render_only_labels_the_working_tree_round(tmp_path: Path):
assert _render_round_or_refuse(_round_guard_ws(tmp_path / "b", tree=1), None) == 1
-def test_flywheel_caption_defaults_to_the_configured_captioner(runs_dir: Path, monkeypatch):
- """No --model: Settings.default_captioner, not a literal the CLI kept (a configured one was ignored)."""
- import codeverse3d.addons.dataset.captions as cap
- from codeverse3d.config import get_settings
-
- seen: list[str] = []
- monkeypatch.setattr(get_settings(), "default_captioner", "fake:configured")
- monkeypatch.setattr(cap, "caption_sample", lambda ws, rec, model, **kw: seen.append(model) or SimpleNamespace(instruction="x"))
- r = runner.invoke(app, ["flywheel", "caption", "wooden_chair_ab12cd34", "--runs-dir", str(runs_dir), "--out",
- str(runs_dir.parent / "side")])
- assert r.exit_code == 0, r.output
- assert seen == ["fake:configured"]
-
-
-# --------------------------------------------------- the configuration sweep (2026-09-23): refused up front, cleanly
@pytest.mark.parametrize("flag,value", [
("--generator", "antigravity:gemini-3.7-flash"), ("--generator", "nocolon"), ("--generator", "single-shot:nope"),
("--generator", "single-shot:gemini:"), ("--planner", "gemini"), ("--judge", "bogus:model"),
diff --git a/harness/tests/flywheel_cli/test_flywheel.py b/harness/tests/flywheel_cli/test_flywheel.py
index 6c63fcfa..d061f94a 100644
--- a/harness/tests/flywheel_cli/test_flywheel.py
+++ b/harness/tests/flywheel_cli/test_flywheel.py
@@ -35,39 +35,12 @@ def test_finalize_and_load_record(fake_run):
assert loaded.total_usage.cost_usd == pytest.approx(0.06), "the total is the ledger's"
-def test_a_missing_runtime_js_does_not_lose_a_finished_runs_record(fake_run, monkeypatch):
- """The version probe raised through finalize_record, so a shader run (no node needed) with a
- bad C3D_RUNTIME_JS paid for every round and ended with no record.json (sweep 2026-09-23)."""
- from codeverse3d import config
- from codeverse3d.record import record as R
-
- ws, rec = fake_run
- rec.environment = {}
- monkeypatch.setenv("C3D_RUNTIME_JS", str(ws.root / "nowhere"))
- config.get_settings.cache_clear()
- R.environment_versions.cache_clear()
- try:
- finalize_record(ws, rec)
- assert load_record(ws).environment["three"] == ""
- finally:
- config.get_settings.cache_clear()
- R.environment_versions.cache_clear()
-
-
-def test_iter_runs_and_errors(runs_dir: Path):
- (runs_dir / "broken").mkdir()
- (runs_dir / "broken" / "record.json").write_text("{not json")
- with pytest.raises(RecordError):
- list(iter_runs(runs_dir))
- bad = []
- runs = list(iter_runs(runs_dir, on_error=lambda d, e: bad.append(d.name)))
- assert len(runs) == 3 and bad == ["broken"]
-
-
# --------------------------------------------------------------------------- git helpers
-def test_git_tree_at_commit(fake_run):
+def test_git_reads_at_a_commit(fake_run):
+ """read_tree_at reads a commit's tree (never a symlink as a file of its target); a sha the repository
+ no longer holds raises rather than diffing against an empty tree."""
ws, rec = fake_run
c0, c1 = rec.rounds[0].commit, rec.rounds[1].commit
t0 = git_history.read_tree_at(ws, c0)
@@ -77,78 +50,56 @@ def test_git_tree_at_commit(fake_run):
assert b"round 0" in t0["src/model.py"] and b"round 1" in t1["src/model.py"]
with pytest.raises(git_history.GitReadError):
git_history.read_tree_at(ws, "deadbeef")
+ for a, b in (("0" * 40, c1), (c1, "0" * 40)):
+ with pytest.raises(git_history.GitReadError):
+ git_history.diff_between(ws, a, b)
+ with pytest.raises(git_history.GitReadError):
+ git_history.changed_files_between(ws, c1, "0" * 40)
-
-def test_a_planted_diff_driver_never_runs(fake_run, tmp_path):
- """An agent-planted diff driver or textconv (a program git RUNS) never runs on the export read."""
- ws, _rec = fake_run
- fired = tmp_path / "fired"
- payload = f"sh -c 'echo pwned >> {fired}; cat'"
- (ws.src / "blob.bin").write_bytes(b"\x00\x01before\n")
- before = ws.commit("binary before")
- (ws.src / "blob.bin").write_bytes(b"\x00\x01after\n")
- after = ws.commit("binary after")
- # planted AFTER the last commit: ws.commit() sanitises .git/config on the way past
- (ws.root / ".gitattributes").write_text("*.bin diff=evil\n*.py diff=evil\n")
- for key in ("textconv", "command"):
- ws._git("config", "--local", f"diff.evil.{key}", payload)
- ws._git("config", "--local", "diff.external", payload)
-
- text, _total, _truncated = git_history.diff_between(ws, before, after)
- files = git_history.changed_files_between(ws, before, after)
-
- assert not fired.exists(), f"a planted diff driver ran: {fired.read_text()!r}"
- assert files == ["src/blob.bin"] and "src/blob.bin" in text
-
-
-def test_a_symlink_is_not_exported_as_a_file_of_its_target(fake_run) -> None:
- """ls-tree lists a symlink as a blob holding its target — never a file of the sample."""
- from codeverse3d.record.git_history import read_tree_at
-
- ws, _rec = fake_run
(ws.src / "model.py").write_text("import bpy\n")
(ws.src / "link.py").symlink_to("model.py")
- commit = ws.commit("with a symlink")
-
- tree = read_tree_at(ws, commit)
+ tree = git_history.read_tree_at(ws, ws.commit("with a symlink"))
assert "src/model.py" in tree
assert "src/link.py" not in tree, "a symlink is not a file the agent wrote"
-def test_a_planted_smudge_filter_never_runs(fake_run, tmp_path):
- """read_tree_at reads the object database directly, so a planted smudge filter never runs."""
+def test_a_planted_smudge_filter_or_diff_driver_never_runs(fake_run, tmp_path):
+ """Programs git RUNS (a smudge filter, a diff driver / textconv) never run on the export read."""
ws, _rec = fake_run
- fired = tmp_path / "smudged"
+ fired = tmp_path / "fired"
+ payload = f"sh -c 'echo pwned >> {fired}; cat'"
(ws.src / "model.py").write_text("# real content\n")
commit = ws.commit("content")
(ws.root / ".git" / "info").mkdir(parents=True, exist_ok=True)
(ws.root / ".git" / "info" / "attributes").write_text("* filter=evil\n")
(ws.root / ".gitattributes").write_text("* filter=evil\n")
- ws._git("config", "--local", "filter.evil.smudge", f"sh -c 'echo pwned >> {fired}; cat'")
+ ws._git("config", "--local", "filter.evil.smudge", payload)
ws._git("config", "--local", "filter.evil.required", "false")
-
files = git_history.read_tree_at(ws, commit)
-
assert not fired.exists(), f"a planted smudge filter ran: {fired.read_text()!r}"
assert files["src/model.py"] == b"# real content\n" # the raw blob, unfiltered
-
-def test_diff_between_refuses_a_sha_the_repo_does_not_have(fake_run):
- """A sha the repository no longer holds raises rather than diffing against an empty tree."""
- ws, rec = fake_run
- good = rec.rounds[1].commit
- with pytest.raises(git_history.GitReadError):
- git_history.diff_between(ws, "0" * 40, good)
- with pytest.raises(git_history.GitReadError):
- git_history.diff_between(ws, good, "0" * 40)
- with pytest.raises(git_history.GitReadError):
- git_history.changed_files_between(ws, good, "0" * 40)
+ (ws.root / ".git" / "info" / "attributes").unlink()
+ (ws.root / ".gitattributes").unlink()
+ (ws.src / "blob.bin").write_bytes(b"\x00\x01before\n")
+ before = ws.commit("binary before")
+ (ws.src / "blob.bin").write_bytes(b"\x00\x01after\n")
+ after = ws.commit("binary after")
+ # planted AFTER the last commit: ws.commit() sanitises .git/config on the way past
+ (ws.root / ".gitattributes").write_text("*.bin diff=evil\n*.py diff=evil\n")
+ for key in ("textconv", "command"):
+ ws._git("config", "--local", f"diff.evil.{key}", payload)
+ ws._git("config", "--local", "diff.external", payload)
+ text, _total, _truncated = git_history.diff_between(ws, before, after)
+ files = git_history.changed_files_between(ws, before, after)
+ assert not fired.exists(), f"a planted diff driver ran: {fired.read_text()!r}"
+ assert files == ["src/blob.bin"] and "src/blob.bin" in text
# --------------------------------------------------------------------------- export + pack
-def test_export_samples(runs_dir: Path, tmp_path: Path):
+def test_export_pairs_and_a_core_only_install_over_a_runs_dir(runs_dir: Path, tmp_path: Path, monkeypatch):
out = tmp_path / "dataset"
rep = export_samples(runs_dir, out)
assert rep.n_runs == 3 and rep.n_exported == 3 and not rep.skipped
@@ -181,12 +132,65 @@ def test_export_samples(runs_dir: Path, tmp_path: Path):
rep3 = export_samples(runs_dir, tmp_path / "d3", only_passed=True)
assert rep3.n_exported == 1
+ # pairs
+ pairs_out = tmp_path / "pairs.jsonl"
+ n = build_pairs(runs_dir, pairs_out, min_delta=0.05)
+ pairs = [json.loads(ln) for ln in pairs_out.read_text().splitlines()]
+ assert n == len(pairs)
+ assert {p["kind"] for p in pairs} == {"preference", "repair", "cross_backend"}
+ pref = [p for p in pairs if p["kind"] == "preference"]
+ assert len(pref) == 2 # chair (0.25) + codex chair (0.10); lamp delta 0.02 < min
+ p = next(x for x in pref if x["run"] == "wooden_chair_ab12cd34")
+ assert p["delta"] == 0.25 and "thicken the legs" in p["reason"]
+ assert p["chosen"]["files"]["src/model.py"].startswith("# round 1")
+ assert p["rejected"]["files"]["src/model.py"].startswith("# round 0")
+ xb = next(x for x in pairs if x["kind"] == "cross_backend")
+ assert xb["chosen"]["generator"].startswith("gemini-cli") and xb["rejected"]["generator"].startswith("codex")
+ assert xb["delta"] == pytest.approx(0.2)
+
+ # a core-only install (no pyarrow) always exports JSONL, makes parquet optional, and --pack refuses
+ from typer.testing import CliRunner
+
+ from codeverse3d.cli.main import app
+
+ _block_pyarrow(monkeypatch)
+ core = tmp_path / "ds_core"
+ rep = export_samples(runs_dir, core) # must NOT raise
+ assert rep.n_exported == 3 and rep.n_indexed == 3
+ assert len((core / "metadata.jsonl").read_text().splitlines()) == 3
+ # the parquet is skipped, loudly and by name — never silently
+ assert not (core / "metadata.parquet").exists() and rep.parquet == ""
+ assert rep.notes and "pyarrow" in rep.notes[0] and "harness[flywheel]" in rep.notes[0]
+ packed = tmp_path / "ds_pack"
+ res = CliRunner().invoke(app, ["flywheel", "export", str(runs_dir), str(packed), "--pack"])
+ assert res.exit_code == 1
+ assert "harness[flywheel]" in res.output
+ assert not packed.exists(), "no partial dataset may be left behind"
+
+ # a broken record: iter_runs raises, or reports it through on_error
+ (runs_dir / "broken").mkdir()
+ (runs_dir / "broken" / "record.json").write_text("{not json")
+ with pytest.raises(RecordError):
+ list(iter_runs(runs_dir))
+ bad = []
+ runs = list(iter_runs(runs_dir, on_error=lambda d, e: bad.append(d.name)))
+ assert len(runs) == 3 and bad == ["broken"]
+
+
+def test_export_of_one_run_with_captions_and_a_texture_pass_and_reexport(fake_run, tmp_path: Path):
+ from codeverse3d.texturing.run import report_path
+ from tests.flywheel_cli.conftest import tiny_png
-def test_export_with_captions_and_reexport(fake_run, tmp_path: Path):
ws, rec = fake_run
rec.extra["captions"] = {"detailed": "A chair.", "instruction": "Write a Blender Python script for a chair.",
"factory": "Build seat then legs.", "provenance": {"captioner": "gemini:x"}}
+ tiny_png(ws.artifacts / "textures" / "wood.png")
+ (ws.artifacts / "object_textured.glb").write_bytes(b"glTF\x02\x00\x00\x00" + b"\0" * 8)
+ rec.extra["texturing"] = {"shipped": True, "glb_textured": "artifacts/object_textured.glb",
+ "textures_dir": "artifacts/textures"}
ws.write_json(ws.record_path, rec)
+ # the pass's report names the GLB it started from: the exported round's (r01, the pick)
+ ws.write_json(report_path(ws), {**rec.extra["texturing"], "glb_in": "artifacts/r01/object.glb"})
out = tmp_path / "ds"
export_samples(ws.root.parent, out)
rep = export_samples(ws.root.parent, out) # re-export overwrites, index stays at 1 row
@@ -194,27 +198,18 @@ def test_export_with_captions_and_reexport(fake_run, tmp_path: Path):
row = json.loads((out / "metadata.jsonl").read_text().splitlines()[0])
assert row["captions"]["factory"] == "Build seat then legs."
assert json.loads(row["meta_json"])["has_captions"] is True
-
-
-# --------------------------------------------------------------------------- pairs
-
-
-def test_build_pairs(runs_dir: Path, tmp_path: Path):
- out = tmp_path / "pairs.jsonl"
- n = build_pairs(runs_dir, out, min_delta=0.05)
- pairs = [json.loads(ln) for ln in out.read_text().splitlines()]
- assert n == len(pairs)
- kinds = {p["kind"] for p in pairs}
- assert kinds == {"preference", "repair", "cross_backend"}
- pref = [p for p in pairs if p["kind"] == "preference"]
- assert len(pref) == 2 # chair (0.25) + codex chair (0.10); lamp delta 0.02 < min
- p = next(x for x in pref if x["run"] == "wooden_chair_ab12cd34")
- assert p["delta"] == 0.25 and "thicken the legs" in p["reason"]
- assert p["chosen"]["files"]["src/model.py"].startswith("# round 1")
- assert p["rejected"]["files"]["src/model.py"].startswith("# round 0")
- xb = next(x for x in pairs if x["kind"] == "cross_backend")
- assert xb["chosen"]["generator"].startswith("gemini-cli") and xb["rejected"]["generator"].startswith("codex")
- assert xb["delta"] == pytest.approx(0.2)
+ sdir = next(out.rglob("meta.json")).parent
+ assert (sdir / "textures" / "wood.png").is_file()
+ assert (sdir / "renders" / "object_textured.glb").is_file()
+ meta = json.loads((sdir / "meta.json").read_text())
+ assert "textures/wood.png" in meta["files"] and "renders/object_textured.glb" in meta["files"]
+ # not shipped → nothing copied
+ rec.extra["texturing"]["shipped"] = False
+ ws.write_json(ws.record_path, rec)
+ ws.write_json(report_path(ws), {**rec.extra["texturing"], "glb_in": "artifacts/r01/object.glb"})
+ export_samples(ws.root.parent, tmp_path / "ds2")
+ sdir2 = next((tmp_path / "ds2").rglob("meta.json")).parent
+ assert not (sdir2 / "textures").exists()
# --------------------------------------------------------------------------- finding: repair pairs (pairs.py:115)
@@ -313,51 +308,6 @@ def test_a_run_with_only_degraded_verdicts_exports_unscored(tmp_path: Path):
assert meta["acceptance_results"] == {}
-def test_export_includes_textured_assets_when_shipped(fake_run, tmp_path: Path):
- from codeverse3d.texturing.run import report_path
- from tests.flywheel_cli.conftest import tiny_png
-
- ws, rec = fake_run
- tex_dir = ws.artifacts / "textures"
- tiny_png(tex_dir / "wood.png")
- (ws.artifacts / "object_textured.glb").write_bytes(b"glTF\x02\x00\x00\x00" + b"\0" * 8)
- rec.extra["texturing"] = {"shipped": True, "glb_textured": "artifacts/object_textured.glb",
- "textures_dir": "artifacts/textures"}
- ws.write_json(ws.record_path, rec)
- # the pass's report names the GLB it started from: the exported round's (r01, the pick)
- ws.write_json(report_path(ws), {**rec.extra["texturing"], "glb_in": "artifacts/r01/object.glb"})
- out = tmp_path / "ds"
- export_samples(ws.root.parent, out)
- sdir = next(out.rglob("meta.json")).parent
- assert (sdir / "textures" / "wood.png").is_file()
- assert (sdir / "renders" / "object_textured.glb").is_file()
- meta = json.loads((sdir / "meta.json").read_text())
- assert "textures/wood.png" in meta["files"] and "renders/object_textured.glb" in meta["files"]
- # not shipped → nothing copied
- rec.extra["texturing"]["shipped"] = False
- ws.write_json(ws.record_path, rec)
- ws.write_json(report_path(ws), {**rec.extra["texturing"], "glb_in": "artifacts/r01/object.glb"})
- export_samples(ws.root.parent, tmp_path / "ds2")
- sdir2 = next((tmp_path / "ds2").rglob("meta.json")).parent
- assert not (sdir2 / "textures").exists()
-
-
-def test_an_ab_plan_cell_is_one_run_and_its_eval_sibling_is_none(tmp_path):
- """The ab_plan layout resolves to its run; the compare layout is pinned in gallery/test_index."""
- from codeverse3d.addons.gallery.index import scan_root
- from codeverse3d.record.record import find_run_dirs
-
- ab = tmp_path / "ab_aa_noise"
- cell = ab / "arms" / "control" / "cells" / "ctrl_med_chair" / "harness_api-agent"
- (cell / "run").mkdir(parents=True)
- (cell / "run" / "record.json").write_text("{}")
- (cell / "eval").mkdir()
- (cell / "eval" / "spec.json").write_text("{}")
- assert find_run_dirs(ab) == [cell / "run"]
- section = scan_root(ab)
- assert [e.slug for e in section.entries] == ["control__ctrl_med_chair__harness_api-agent"]
-
-
def test_a_run_reached_twice_is_found_once_at_its_physical_path(tmp_path):
"""Batteries symlink each other's cells: export, index, pairs and the gallery counted them twice."""
from codeverse3d.record.record import find_run_dirs
@@ -421,29 +371,20 @@ def _block_pyarrow(monkeypatch) -> None:
monkeypatch.setattr(builtins, "__import__", no_pyarrow)
-def test_a_core_only_install_still_gets_a_complete_dataset(runs_dir: Path, tmp_path: Path, monkeypatch):
- """Core installs always export JSONL and make parquet optional."""
- _block_pyarrow(monkeypatch)
- out = tmp_path / "ds_core"
- rep = export_samples(runs_dir, out) # must NOT raise
- assert rep.n_exported == 3 and rep.n_indexed == 3
- # the index the operator can actually use is there, and complete
- assert (out / "metadata.jsonl").is_file()
- assert len((out / "metadata.jsonl").read_text().splitlines()) == 3
- # the parquet is skipped, loudly and by name — never silently
- assert not (out / "metadata.parquet").exists() and rep.parquet == ""
- assert rep.notes and "pyarrow" in rep.notes[0] and "harness[flywheel]" in rep.notes[0]
-
-
-def test_pack_refuses_before_writing_anything_when_pyarrow_is_missing(runs_dir: Path, tmp_path: Path, monkeypatch):
- """Packing refuses before writing when its optional dependency is absent."""
- from typer.testing import CliRunner
-
- from codeverse3d.cli.main import app
+def test_a_missing_runtime_js_does_not_lose_a_finished_runs_record(fake_run, monkeypatch):
+ """The version probe raised through finalize_record, so a shader run (no node needed) with a
+ bad C3D_RUNTIME_JS paid for every round and ended with no record.json (sweep 2026-09-23)."""
+ from codeverse3d import config
+ from codeverse3d.record import record as R
- _block_pyarrow(monkeypatch)
- out = tmp_path / "ds_pack"
- res = CliRunner().invoke(app, ["flywheel", "export", str(runs_dir), str(out), "--pack"])
- assert res.exit_code == 1
- assert "harness[flywheel]" in res.output
- assert not out.exists(), "no partial dataset may be left behind"
+ ws, rec = fake_run
+ rec.environment = {}
+ monkeypatch.setenv("C3D_RUNTIME_JS", str(ws.root / "nowhere"))
+ config.get_settings.cache_clear()
+ R.environment_versions.cache_clear()
+ try:
+ finalize_record(ws, rec)
+ assert load_record(ws).environment["three"] == ""
+ finally:
+ config.get_settings.cache_clear()
+ R.environment_versions.cache_clear()
diff --git a/harness/tests/flywheel_cli/test_manifest_pack.py b/harness/tests/flywheel_cli/test_manifest_pack.py
index 272446c1..ae98fd53 100644
--- a/harness/tests/flywheel_cli/test_manifest_pack.py
+++ b/harness/tests/flywheel_cli/test_manifest_pack.py
@@ -27,13 +27,6 @@ def exported_ds(runs_dir: Path, tmp_path: Path) -> Path:
return out
-def packed_ds(exported_ds: Path) -> Path:
- """...and packed once — the previous GOOD dataset every failure window must leave intact."""
- pack_samples(exported_ds)
- return exported_ds
-
-
def _tar_member_names(out: Path) -> list[str]:
names: list[str] = []
for tp in sorted(out.glob("samples-*.tar")):
@@ -42,8 +35,6 @@ def _tar_member_names(out: Path) -> list[str]:
return names
-# --------------------------------------------------------------------------- manifest
-
# --------------------------------------------------------------------------- drop-duplicates → pack
def test_drop_duplicates_then_pack_ships_no_duplicate(runs_dir: Path, tmp_path: Path):
out = tmp_path / "ds"
@@ -81,29 +72,6 @@ def _packed_state(out: Path) -> dict[str, bytes]:
for p in [*out.glob("samples-*.tar"), out / "metadata.jsonl", out / "metadata.parquet"]}
-def test_a_failure_after_the_pack_loop_publishes_nothing(packed_ds: Path, monkeypatch):
- """Every tar written, then the index write fails: the old dataset stays whole, tars and index."""
- import codeverse3d.addons.dataset.pack as P
-
- before = _packed_state(packed_ds)
- monkeypatch.setattr(P, "write_parquet", lambda *a, **k: (_ for _ in ()).throw(OSError("No space left on device")))
- with pytest.raises(OSError, match="No space left"):
- pack_samples(packed_ds)
- assert _packed_state(packed_ds) == before
- assert not list(packed_ds.glob("*.tmp"))
- assert verify_locators(packed_ds) == 3
-
-
-def test_repacking_over_an_existing_archive_succeeds_and_leaves_no_orphan_tar(packed_ds: Path):
- """A repack into fewer shards leaves no orphan tar the new index never names."""
- assert pack_samples(packed_ds, max_tar_bytes=1).tars == ["samples-000.tar", "samples-001.tar", "samples-002.tar"]
- prep = pack_samples(packed_ds)
- assert prep.tars == ["samples-000.tar"] and prep.n_samples == 3
- assert sorted(p.name for p in packed_ds.glob("samples-*.tar")) == ["samples-000.tar"]
- assert verify_locators(packed_ds) == 3
- assert not list(packed_ds.glob("*.tmp"))
-
-
def test_a_tampered_sample_file_fails_fast_and_names_it(exported_ds: Path):
(exported_ds / "static_object" / "blender" / "wooden_chair_ab12cd34" / "code.py").write_text("tampered")
with pytest.raises(PackError, match="code.py"):
@@ -111,8 +79,12 @@ def test_a_tampered_sample_file_fails_fast_and_names_it(exported_ds: Path):
assert not list(exported_ds.glob("samples-*.tar")) and not list(exported_ds.glob("*.tmp"))
-# --------------------------------------------------------------------------- phantom rows
-def test_planted_src_meta_json_is_not_a_row_and_not_packed(exported_ds: Path):
+# --------------------------------------------------------------------------- phantom rows, failed and repeated packs
+def test_a_planted_row_a_failed_publish_and_a_repack(exported_ds: Path, monkeypatch):
+ """A planted src/meta.json is not a row and not packed; every tar written then the index write
+ fails: the old dataset stays whole; a repack into fewer shards leaves no orphan tar."""
+ import codeverse3d.addons.dataset.pack as P
+
sdir = exported_ds / "static_object" / "blender" / "wooden_chair_ab12cd34"
(sdir / "src" / "meta.json").write_text(json.dumps({"id": "phantom", "key": "phantom"}))
pack_samples(exported_ds) # and pack never looks at the directory tree at all
@@ -120,3 +92,19 @@ def test_planted_src_meta_json_is_not_a_row_and_not_packed(exported_ds: Path):
assert "wooden_chair_ab12cd34/src/meta.json" not in _tar_member_names(exported_ds)
keys = {json.loads(ln)["key"] for ln in (exported_ds / "metadata.jsonl").read_text().splitlines()}
assert keys == {"wooden_chair_ab12cd34", "wooden_chair_codex", "lamp_three"}
+
+ before = _packed_state(exported_ds)
+ with monkeypatch.context() as mp:
+ mp.setattr(P, "write_parquet", lambda *a, **k: (_ for _ in ()).throw(OSError("No space left on device")))
+ with pytest.raises(OSError, match="No space left"):
+ pack_samples(exported_ds)
+ assert _packed_state(exported_ds) == before
+ assert not list(exported_ds.glob("*.tmp"))
+ assert verify_locators(exported_ds) == 3
+
+ assert pack_samples(exported_ds, max_tar_bytes=1).tars == ["samples-000.tar", "samples-001.tar", "samples-002.tar"]
+ prep = pack_samples(exported_ds)
+ assert prep.tars == ["samples-000.tar"] and prep.n_samples == 3
+ assert sorted(p.name for p in exported_ds.glob("samples-*.tar")) == ["samples-000.tar"]
+ assert verify_locators(exported_ds) == 3
+ assert not list(exported_ds.glob("*.tmp"))
diff --git a/harness/tests/flywheel_cli/test_quality_gallery.py b/harness/tests/flywheel_cli/test_quality_gallery.py
index cbf85b29..e783e254 100644
--- a/harness/tests/flywheel_cli/test_quality_gallery.py
+++ b/harness/tests/flywheel_cli/test_quality_gallery.py
@@ -14,7 +14,6 @@ from codeverse3d.addons.dataset.quality import (
quality_tier,
)
from codeverse3d.record.record import load_record
-from tests.flywheel_cli.conftest import make_fake_run
from tests.flywheel_cli.test_captions import GOOD
from tests.orchestrator_tracks.fakes import FakeChatModel
@@ -49,31 +48,25 @@ def test_only_raw_hash_duplicates_are_dropped_normalised_ones_are_only_marked():
assert prompt_id(" a chair ") == prompt_id("a chair") and len(prompt_id("x")) == 16
-def test_whitespace_inside_a_string_is_a_near_duplicate_not_a_duplicate(runs_dir: Path, tmp_path: Path):
- """LABEL = "a b" and LABEL="ab" normalise alike but are different programs: kept, only marked."""
- common = '# round 1\nimport bpy\nLABEL={}\nbpy.ops.mesh.primitive_cube_add(size=2.0)\n'
- make_fake_run(runs_dir, "stool_spaced", prompt="a stool", code_v2=common.format('"a b"'))
- make_fake_run(runs_dir, "stool_tight", prompt="a stool", code_v2=common.format('"ab"'))
- out = tmp_path / "ds"
- rep = export_samples(runs_dir, out, drop_duplicates=True)
- rows = {json.loads(ln)["key"]: json.loads(ln) for ln in (out / "metadata.jsonl").read_text().splitlines()}
- assert {"stool_spaced", "stool_tight"} <= set(rows)
- assert rows["stool_spaced"]["code_sha256"] != rows["stool_tight"]["code_sha256"]
- assert rows["stool_spaced"]["code_fingerprint"] == rows["stool_tight"]["code_fingerprint"]
- assert [rows[k]["duplicate_of"] for k in ("stool_spaced", "stool_tight")] == ["", ""]
- assert rows["stool_tight"]["near_duplicate_of"] == "3dcodeverse/static_object/blender/stool_spaced"
- assert "wooden_chair_codex" not in rows and rep.n_duplicates == 1 # byte-identical → dropped
-
-
# --------------------------------------------------------------------------- export extras
-def test_export_scene_views_repeating_names(fake_run, tmp_path: Path):
+def test_a_side_car_caption_leaves_the_run_untouched_and_export_keeps_repeating_view_names(fake_run, tmp_path: Path):
"""Scene renders repeat a camera name per capture time; the sample must keep every file."""
from codeverse3d.contracts.artifacts import RenderView
from tests.flywheel_cli.conftest import tiny_png
ws, rec = fake_run
+ before = ws.record_path.read_text()
+ side = tmp_path / "caps"
+ caps = caption_sample(ws, rec, "fake:fake", model=FakeChatModel([GOOD]), out_dir=side)
+ assert caps.factory == GOOD["factory"]
+ assert ws.record_path.read_text() == before and not (ws.root / "captions.json").exists()
+ data = json.loads((side / f"{ws.root.name}.json").read_text())
+ assert data["provenance"]["captioner"] == "fake:fake" and data["provenance"]["images_used"][0].startswith("artifacts/")
+ assert load_captions(ws, load_record(ws), side)["detailed"] == GOOD["detailed"]
+ assert load_captions(ws, load_record(ws), None) == {}
+
rnd = rec.rounds[1]
rd = ws.renders_dir(1)
assert rnd.renders is not None
@@ -85,16 +78,3 @@ def test_export_scene_views_repeating_names(fake_run, tmp_path: Path):
meta = json.loads(next(out.rglob("meta.json")).read_text())
assert {"renders/view_Establishing_t0.png", "renders/view_Establishing_t1p5.png"} <= set(meta["renders"])
assert len(meta["files"]) == len(set(meta["files"]))
-
-
-def test_caption_out_dir_leaves_run_untouched(fake_run, tmp_path: Path):
- ws, rec = fake_run
- before = ws.record_path.read_text()
- side = tmp_path / "caps"
- caps = caption_sample(ws, rec, "fake:fake", model=FakeChatModel([GOOD]), out_dir=side)
- assert caps.factory == GOOD["factory"]
- assert ws.record_path.read_text() == before and not (ws.root / "captions.json").exists()
- data = json.loads((side / f"{ws.root.name}.json").read_text())
- assert data["provenance"]["captioner"] == "fake:fake" and data["provenance"]["images_used"][0].startswith("artifacts/")
- assert load_captions(ws, load_record(ws), side)["detailed"] == GOOD["detailed"]
- assert load_captions(ws, load_record(ws), None) == {}
diff --git a/harness/tests/flywheel_cli/test_refine.py b/harness/tests/flywheel_cli/test_refine.py
index 36a91bee..5318820d 100644
--- a/harness/tests/flywheel_cli/test_refine.py
+++ b/harness/tests/flywheel_cli/test_refine.py
@@ -33,6 +33,8 @@ def _rows(root: Path, **kw):
def test_a_refine_round_carries_its_brief_its_diff_and_its_delta(tmp_path):
+ """One refine transition; --with-code inlines the answer and the tree it started from; the jsonl is
+ written atomically."""
_corpus(tmp_path)
rows, drops = _rows(tmp_path)
assert len(rows) == 1 and not drops
@@ -48,9 +50,6 @@ def test_a_refine_round_carries_its_brief_its_diff_and_its_delta(tmp_path):
assert t.before_commit and t.after_commit and t.before_commit != t.after_commit
assert t.before_files is None and t.after_files is None # --with-code is opt-in
-
-def test_with_code_inlines_the_answer_and_the_tree_it_started_from(tmp_path):
- _corpus(tmp_path)
rows, _ = _rows(tmp_path, with_code=True)
t = rows[0]
assert set(t.after_files) == {"src/model.py", "src/parts/leg.py"} # what changed
@@ -58,6 +57,12 @@ def test_with_code_inlines_the_answer_and_the_tree_it_started_from(tmp_path):
assert "src/model.py" in t.before_files and "size=1.0" in t.before_files["src/model.py"]
assert "src/parts/leg.py" not in t.before_files # it did not exist yet
+ out = tmp_path / "nested" / "refine.jsonl"
+ n, drops = build_refine(tmp_path, out)
+ assert n == 1 and not drops and not list(out.parent.glob("*.part"))
+ row = json.loads(out.read_text().splitlines()[0])
+ assert row["outcome"] == "improved" and "before_files" not in row # exclude_none keeps rows small
+
@pytest.mark.parametrize("delta,label", [(0.05, "improved"), (0.04, "unchanged"), (-0.05, "regressed"),
(None, "unscored")])
@@ -89,12 +94,3 @@ def test_every_unexportable_round_is_counted_under_a_named_reason(tmp_path, brea
rows, drops = _rows(tmp_path)
assert rows == []
assert drops == ({} if reason is None else {reason: 1}), f"{break_it}: {dict(drops)}"
-
-
-def test_build_refine_writes_jsonl_atomically(tmp_path):
- _corpus(tmp_path)
- out = tmp_path / "nested" / "refine.jsonl"
- n, drops = build_refine(tmp_path, out)
- assert n == 1 and not drops and not list(out.parent.glob("*.part"))
- row = json.loads(out.read_text().splitlines()[0])
- assert row["outcome"] == "improved" and "before_files" not in row # exclude_none keeps rows small
diff --git a/harness/tests/flywheel_cli/test_run_identity.py b/harness/tests/flywheel_cli/test_run_identity.py
index f47f7b2b..2c7e744e 100644
--- a/harness/tests/flywheel_cli/test_run_identity.py
+++ b/harness/tests/flywheel_cli/test_run_identity.py
@@ -15,22 +15,18 @@ from codeverse3d.record.record import unique_files
from tests.flywheel_cli.conftest import make_fake_run
[email protected]("battery, rel, slug", [
- ("runs", "wooden_chair_ab12cd34", "wooden_chair_ab12cd34"),
- ("h2h_scene_v1", "h2h_b01_neon_alley.attempt1", "h2h_b01_neon_alley.attempt1"), # a retry keeps its suffix
- ("articulated_v1_flash", "runs/art_easy_laptop", "art_easy_laptop"), # runs/ is noise
- ("compare_v1_full", "cells/cmp_easy_stool/harness_x/run", "cmp_easy_stool__harness_x"),
-])
-def test_the_slug_rule(battery, rel, slug):
- assert RunId(battery=battery, rel=rel).slug == slug
-
-
-def test_compare_backends_layout_arm_and_cell():
+def test_the_slug_rule_and_the_battery_layouts():
+ for battery, rel, slug in [
+ ("runs", "wooden_chair_ab12cd34", "wooden_chair_ab12cd34"),
+ ("h2h_scene_v1", "h2h_b01_neon_alley.attempt1", "h2h_b01_neon_alley.attempt1"), # a retry keeps its suffix
+ ("articulated_v1_flash", "runs/art_easy_laptop", "art_easy_laptop"), # runs/ is noise
+ ("compare_v1_full", "cells/cmp_easy_stool/harness_x/run", "cmp_easy_stool__harness_x"),
+ ]:
+ assert RunId(battery=battery, rel=rel).slug == slug, rel
+ # compare_backends: cell then arm
rid = RunId(battery="compare_v1_full", rel="cells/cmp_easy_stool/harness_x/run")
assert (rid.cell, rid.arm) == ("cmp_easy_stool", "harness_x")
-
-
-def test_ab_plan_layout_slug_arm_cell_and_attempt_suffix():
+ # ab_plan: slug, arm, cell and the attempt suffix
rid = RunId(battery="plan_loop",
rel="C0/arms/control/cells/arch_hard_bay_window/"
"harness_api-agent_gemini_gemini-3.7-flash/run")
diff --git a/harness/tests/flywheel_cli/test_run_layout_cli.py b/harness/tests/flywheel_cli/test_run_layout_cli.py
index f5ab9e95..502252ba 100644
--- a/harness/tests/flywheel_cli/test_run_layout_cli.py
+++ b/harness/tests/flywheel_cli/test_run_layout_cli.py
@@ -109,10 +109,7 @@ def test_telemetry_summarises_the_runs_own_ledger(fake_run):
assert tele.cost is not None and tele.cost.by_stage[0].stage == "plan"
assert {s.stage for s in tele.cost.by_stage} <= set(STAGE_ORDER)
assert not (ws.telemetry / "usage.jsonl").exists(), "one ledger, one name"
-
-
-def test_telemetry_by_round_scores_a_degraded_verdict_as_unscored(fake_run):
- ws, rec = fake_run
+ # a degraded verdict is unscored in the per-round cost table, never a 0.0
rec.rounds[1].judgment = rec.rounds[1].judgment.model_copy(update={"summary": "judge_error: outage"})
by_round = build_telemetry(ws, rec, write=False).cost.by_round
assert [r["score"] for r in by_round] == [rec.rounds[0].judgment.overall, None]
@@ -271,16 +268,11 @@ def test_export_and_gallery_on_both_layouts(tmp_path: Path):
assert old_links["glb"] == "artifacts/object.glb"
assert new_links["glb"] == "deliverable/object.glb"
-
-def test_export_falls_back_to_the_packaged_snapshot_without_git(tmp_path: Path):
- runs = tmp_path / "runs"
- ws, rec = make_fake_run(runs, "run_new")
- package_run(ws, rec)
- ws.write_json(ws.record_path, rec)
- select.package(ws.root, 1)
- shutil.rmtree(ws.root / ".git") # a run copied without its history
- rep = export_samples(runs, tmp_path / "ds")
+ # a packaged run copied without its history exports from its deliverable snapshot
+ shutil.rmtree(ws_new.root / ".git")
+ shutil.rmtree(ws_old.root)
+ rep = export_samples(runs, tmp_path / "ds_nogit")
assert rep.n_exported == 1, rep.skipped
- meta = json.loads(next((tmp_path / "ds").rglob("meta.json")).read_text())
- assert meta["code_source"] == "deliverable"
- assert (next((tmp_path / "ds").rglob("meta.json")).parent / "src" / "model.py").is_file()
+ meta_path = next((tmp_path / "ds_nogit").rglob("meta.json"))
+ assert json.loads(meta_path.read_text())["code_source"] == "deliverable"
+ assert (meta_path.parent / "src" / "model.py").is_file()
diff --git a/harness/tests/flywheel_cli/test_select.py b/harness/tests/flywheel_cli/test_select.py
index 3779d73d..eea354b8 100644
--- a/harness/tests/flywheel_cli/test_select.py
+++ b/harness/tests/flywheel_cli/test_select.py
@@ -24,7 +24,6 @@ from codeverse3d.contracts.artifacts import (
from codeverse3d.contracts.common import Backends, Language, Track, Usage
from codeverse3d.contracts.run import RoundRecord, RunRecord, RunStatus
from codeverse3d.contracts.spec import Spec
-from codeverse3d.proc import EventLog
from codeverse3d.workspace import Workspace
from .conftest import make_fake_run, tiny_png
@@ -119,7 +118,6 @@ def test_pick_by_pairwise_asks_only_inside_the_margin_and_never_buys_a_verdict_t
assert judge.calls and "/r01/" in judge.calls[0][0] and judge.calls[0][2] == "static_object_v1"
assert select.pick(ws.root, by="pairwise", judge=judge) == 0 and len(judge.calls) == 1 # cached, not re-bought
assert ws.judge_path(1, "_vs_r00_pairwise").is_file()
- assert [e for e in EventLog(ws.events_path).read() if e["event"] == "pick.pairwise"]
# an unsure verdict, an outage or a clear gap keep the top score
assert select.pick(_run(tmp_path / "u", [(0.70, 0), (0.72, 0)]).root, by="pairwise", judge=_Pairwise("b", 0.5)) == 1
assert select.pick(_run(tmp_path / "f", [(0.70, 0), (0.72, 0)]).root, by="pairwise",
@@ -183,7 +181,7 @@ def _fake_texture_pass(calls: list, *, shipped: bool = True, fail: bool = False)
return run
-def test_package_textures_the_rounds_own_glb_once(tmp_path, monkeypatch):
+def test_package_textures_the_rounds_own_glb_once_and_a_failed_pass_still_hands_over(tmp_path, monkeypatch):
import codeverse3d.texturing.run as trun
ws = _run(tmp_path / "r", [(0.5, 0), (0.8, 0)])
@@ -199,6 +197,11 @@ def test_package_textures_the_rounds_own_glb_once(tmp_path, monkeypatch):
# another round's hand-over does not carry round 0's pack
select.package(ws.root, 1)
assert not (ws.deliverable / "object_textured.glb").exists()
+ # a failed pass does not stop the hand-over
+ monkeypatch.setattr(trun, "texture_pass", _fake_texture_pass([], fail=True))
+ select.package(ws.root, 1, texture=True)
+ assert (ws.deliverable / "object.glb").is_file() and not (ws.deliverable / "object_textured.glb").exists()
+ assert json.loads((ws.root / select.SELECTION_NAME).read_text())["textured"] is False
def test_an_export_of_another_round_never_ships_the_pack_a_texture_pass_made_for_r02(tmp_path, monkeypatch):
@@ -219,17 +222,6 @@ def test_an_export_of_another_round_never_ships_the_pack_a_texture_pass_made_for
assert "renders/object_textured.glb" in export_one(ws, load_record(ws), tmp_path / "ds2").file_hashes
-def test_a_failed_texture_pass_does_not_stop_the_hand_over(tmp_path, monkeypatch):
- import codeverse3d.texturing.run as trun
-
- ws = _run(tmp_path / "r", [(0.5, 0)])
- monkeypatch.setattr(trun, "texture_pass", _fake_texture_pass([], fail=True))
- select.package(ws.root, 0, texture=True)
- assert (ws.deliverable / "object.glb").is_file() and not (ws.deliverable / "object_textured.glb").exists()
- assert json.loads((ws.root / select.SELECTION_NAME).read_text())["textured"] is False
- assert "texture.failed" in [e["event"] for e in EventLog(ws.events_path).read()]
-
-
# --------------------------------------------------------------------------- CLI
def test_pick_command_by_score_by_round_and_by_pairwise(tmp_path, monkeypatch):
runs = tmp_path / "runs"
diff --git a/harness/tests/gallery/test_cli.py b/harness/tests/gallery/test_cli.py
index 90d7fbf0..38163fe1 100644
--- a/harness/tests/gallery/test_cli.py
+++ b/harness/tests/gallery/test_cli.py
@@ -13,7 +13,7 @@ from codeverse3d.cli.main import app, resolve_roots
runner = CliRunner()
-def test_build_writes_a_page(gallery_tree: dict[str, Path], tmp_path: Path):
+def test_build_writes_a_linked_page_and_embed_inlines_the_images(gallery_tree: dict[str, Path], tmp_path: Path):
out = tmp_path / "g.html"
r = runner.invoke(app, ["gallery", "build", str(gallery_tree["runs"]), str(gallery_tree["battery"]),
"--out", str(out), "--title", "batch 9"])
@@ -22,10 +22,6 @@ def test_build_writes_a_page(gallery_tree: dict[str, Path], tmp_path: Path):
markup = out.read_text()
assert "batch 9" in markup and "wooden_chair_ab12cd34" in markup
assert "data:image/jpeg" not in markup # link mode by default
-
-
-def test_build_embed(gallery_tree: dict[str, Path], tmp_path: Path):
- out = tmp_path / "e.html"
r = runner.invoke(app, ["gallery", "build", str(gallery_tree["runs"]), "--out", str(out), "--embed"])
assert r.exit_code == 0, r.output
assert "data:image/jpeg;base64," in out.read_text()
diff --git a/harness/tests/gallery/test_hardening.py b/harness/tests/gallery/test_hardening.py
index 590f5cdc..35766853 100644
--- a/harness/tests/gallery/test_hardening.py
+++ b/harness/tests/gallery/test_hardening.py
@@ -34,14 +34,11 @@ def test_llm_html_and_svg_file_responses_are_sandboxed(gallery_tree: dict[str, P
# --------------------------------------------------------------------------- csv injection
-def test_csv_safe_neutralises_formula_prefixes():
+def test_csv_cells_never_reach_a_spreadsheet_as_formulas(tmp_path: Path):
assert csv_safe("=HYPERLINK(1)") == "'=HYPERLINK(1)"
assert csv_safe("+1") == "'+1" and csv_safe("-1") == "'-1" and csv_safe("@cmd") == "'@cmd"
assert csv_safe("a chair") == "a chair"
assert csv_safe(3) == 3 and csv_safe(None) is None
-
-
-def test_a_formula_prompt_round_trips_csv_escaped(tmp_path: Path):
runs = tmp_path / "runs"
make_fake_run(runs, "formula_prompt", prompt="=2+5|cmd", scores=(0.5, 0.9))
text = export_csv(build_index([runs]).entries())
diff --git a/harness/tests/gallery/test_index.py b/harness/tests/gallery/test_index.py
index c29c1942..ad2dd31b 100644
--- a/harness/tests/gallery/test_index.py
+++ b/harness/tests/gallery/test_index.py
@@ -5,23 +5,10 @@ from __future__ import annotations
import json
from pathlib import Path
-from codeverse3d.addons.gallery.index import build_index, entry_for_dir
+from codeverse3d.addons.gallery.index import build_index
from codeverse3d.addons.gallery.model import match, sort_entries, summarize
-def test_a_card_links_the_picked_rounds_own_glb(tmp_path: Path):
- """artifacts/object.glb is the LAST round's: until a hand-over the card links the picked round's copy."""
- from codeverse3d.addons import select
- from tests.flywheel_cli.conftest import make_fake_run
-
- ws, _ = make_fake_run(tmp_path / "runs", "kept_rounds", scores=(0.9, 0.6)) # the pick is r00
- links = {ln.label: ln.rel for ln in entry_for_dir("runs", ws.root).links}
- assert links["glb"] == "artifacts/r00/object.glb"
- select.package(ws.root, 0)
- links = {ln.label: ln.rel for ln in entry_for_dir("runs", ws.root).links}
- assert links["glb"] == "deliverable/object.glb"
-
-
def test_filter_and_sort_and_summary(gallery_tree: dict[str, Path]):
index = build_index([gallery_tree["runs"], gallery_tree["battery"]])
entries = index.entries()
@@ -43,13 +30,17 @@ def test_filter_and_sort_and_summary(gallery_tree: dict[str, Path]):
# --------------------------------------------------------------------------- run identity
-def test_nested_battery_runs_get_distinct_findable_slugs(tmp_path: Path):
- """Two compare-style runs whose dirs are both named ``run`` keep distinct, findable slugs."""
+def test_nested_battery_runs_get_distinct_findable_slugs_and_eval_workspaces_are_not_runs(tmp_path: Path):
+ """Two compare-style runs whose dirs are both named ``run`` keep distinct, findable slugs; a cell's
+ ``eval/`` judge workspace (spec.json, no record.json) is not a run."""
from tests.flywheel_cli.conftest import make_fake_run
root = tmp_path / "compare_v9"
make_fake_run(root / "cells" / "cmp_a_stool" / "armx", "run", prompt="a stool")
make_fake_run(root / "cells" / "cmp_b_lamp" / "armx", "run", prompt="a lamp")
+ eval_dir = root / "cells" / "cmp_a_stool" / "armx" / "eval"
+ eval_dir.mkdir()
+ (eval_dir / "spec.json").write_text(json.dumps({"prompt": "judge ws"}))
index = build_index([root])
(section,) = index.sections
assert sorted(e.slug for e in section.entries) == ["cmp_a_stool__armx", "cmp_b_lamp__armx"]
@@ -60,19 +51,6 @@ def test_nested_battery_runs_get_distinct_findable_slugs(tmp_path: Path):
assert a.key != b.key
-def test_nested_eval_workspaces_are_not_counted_as_runs(tmp_path: Path):
- """A cell's ``eval/`` judge workspace (spec.json, no record.json) is not a run."""
- from tests.flywheel_cli.conftest import make_fake_run
-
- root = tmp_path / "compare_v9"
- make_fake_run(root / "cells" / "cmp_a_stool" / "armx", "run", prompt="a stool")
- eval_dir = root / "cells" / "cmp_a_stool" / "armx" / "eval"
- eval_dir.mkdir()
- (eval_dir / "spec.json").write_text(json.dumps({"prompt": "judge ws"}))
- (section,) = build_index([root]).sections
- assert [e.slug for e in section.entries] == ["cmp_a_stool__armx"]
-
-
def test_duplicate_slugs_within_a_root_are_disambiguated(tmp_path: Path):
"""Two rels that reduce to one slug follow the label#2 precedent — the viewer shows both."""
from tests.flywheel_cli.conftest import make_fake_run
diff --git a/harness/tests/gallery/test_page.py b/harness/tests/gallery/test_page.py
index 433049e6..4a5f147a 100644
--- a/harness/tests/gallery/test_page.py
+++ b/harness/tests/gallery/test_page.py
@@ -66,7 +66,9 @@ def _assert_offline(markup: str) -> _Checker:
return c
-def test_static_build_parses_and_is_self_contained(gallery_tree: dict[str, Path], tmp_path: Path):
+def test_static_and_server_pages(gallery_tree: dict[str, Path], tmp_path: Path):
+ """The static build parses and is self-contained; the server page filters server-side and offers
+ detail links only where the target has them; the static form never advertises server-only routes."""
out = tmp_path / "out" / "gallery.html"
path, n, index = build_static([gallery_tree["runs"], gallery_tree["battery"]], out, embed=True)
assert path == out and n == 6 and out.is_file()
@@ -81,9 +83,6 @@ def test_static_build_parses_and_is_self_contained(gallery_tree: dict[str, Path]
assert slug in markup
assert "broken:" in markup and "pending:" in markup # the two non-ok cards say why
-
-def test_server_page_applies_the_filter_server_side(gallery_tree: dict[str, Path]):
- index = build_index([gallery_tree["runs"], gallery_tree["battery"]])
markup = render_index(index, UrlMaker(), flt={"lang": "threejs"}, sort="score")
_assert_offline(markup)
# every card is still in the DOM (so the client can widen the filter without a reload)
@@ -94,15 +93,11 @@ def test_server_page_applies_the_filter_server_side(gallery_tree: dict[str, Path
assert "<option value='threejs' selected>" in markup
assert "vc-passed" not in markup and "PASS RATE" not in markup.upper() # no pass/fail since 2026-09-22
-
-def test_detail_links_are_offered_only_when_the_target_has_them(gallery_tree: dict[str, Path]):
- index = build_index([gallery_tree["runs"]])
markup = render_index(index, UrlMaker())
assert "/run/runs/wooden_chair_ab12cd34" in markup
assert "/file/runs/wooden_chair_ab12cd34/artifacts/renders/r01/sheet.png" in markup
assert "/code/runs/wooden_chair_ab12cd34/src" in markup
assert "/viewer/runs/wooden_chair_ab12cd34/artifacts/r01/object.glb" in markup # the picked round's
- # the static form never advertises server-only routes
static = render_static(index, embed=False)
assert "/run/runs/" not in static and "/viewer/" not in static
diff --git a/harness/tests/gallery/test_server.py b/harness/tests/gallery/test_server.py
index fc9b3fd2..5db80777 100644
--- a/harness/tests/gallery/test_server.py
+++ b/harness/tests/gallery/test_server.py
@@ -29,7 +29,7 @@ def app(gallery_tree: dict[str, Path]) -> GalleryApp:
# --------------------------------------------------------------------------- routing
-def test_index_and_api_routes(app: GalleryApp):
+def test_routes(app: GalleryApp):
r = app.route("/")
assert r.status == 200 and r.content_type.startswith("text/html")
assert b"3dcode gallery" in r.body
@@ -41,15 +41,13 @@ def test_index_and_api_routes(app: GalleryApp):
assert summary["n"] == 2
assert app.route("/api/nope").status == 404
-
-def test_detail_of_a_broken_run_does_not_crash(app: GalleryApp):
+ # the detail of a broken / pending run does not crash
r = app.route("/run/static_v9/half_written")
assert r.status == 200 and b"broken" in r.body
r = app.route("/run/static_v9/not_started")
assert r.status == 200 and b"pending" in r.body
-
-def test_file_dir_and_code_routes(app: GalleryApp):
+ # file, dir and code routes
base = "/file/runs/wooden_chair_ab12cd34"
rec = app.route(f"{base}/record.json")
assert rec.status == 200 and rec.content_type == "application/json; charset=utf-8"
@@ -67,48 +65,12 @@ def test_file_dir_and_code_routes(app: GalleryApp):
assert app.route(f"{base}/artifacts").status == 200 # a file route onto a dir lists it
assert app.route(f"{base}/does/not/exist.png").status == 404
-
- "../../../../etc/passwd",
- "..%2f..%2fetc/passwd",
- "src/../../lamp_three/record.json",
-])
-def test_path_traversal_is_refused(app: GalleryApp, bad: str):
- r = app.route(f"/file/runs/wooden_chair_ab12cd34/{bad}")
- assert r.status in (403, 404), f"{bad} → {r.status}"
- assert b"root:x:" not in r.body
-
-
-def test_absolute_paths_are_refused(app: GalleryApp):
- # '//etc/passwd' collapses to 'etc/passwd' inside the run (404, not a read)
- assert app.route("/file/runs/wooden_chair_ab12cd34//etc/passwd").status == 404
- assert safe_join(app.roots[0], "wooden_chair_ab12cd34").is_dir()
- with pytest.raises(PathError):
- safe_join(app.roots[0], "/etc/passwd")
- with pytest.raises(PathError):
- safe_join(app.roots[0], "../..")
- with pytest.raises(PathError):
- safe_join(app.roots[0], "a/../../b")
- with pytest.raises(PathError):
- safe_join(app.roots[0], "x\x00y")
-
-
-def test_symlink_out_of_the_run_is_refused(app: GalleryApp, tmp_path: Path):
- run = app.roots[0] / "wooden_chair_ab12cd34"
- outside = tmp_path / "secret.txt"
- outside.write_text("classified")
- (run / "escape").symlink_to(outside)
- r = app.route("/file/runs/wooden_chair_ab12cd34/escape")
- assert r.status == 403 and b"classified" not in r.body
-
-
-def test_unknown_run_and_unknown_route(app: GalleryApp):
+ # unknown run / unknown route
assert app.route("/file/nope/nope/record.json").status == 404
assert app.route("/wat").status == 404
assert app.route("/file").status == 404
-
-def test_vendor_and_viewer(app: GalleryApp):
+ # vendor and viewer
from codeverse3d.addons.gallery.viewer import viewer_available
v = app.route("/viewer/runs/wooden_chair_ab12cd34/artifacts/object.glb")
@@ -122,6 +84,25 @@ def test_vendor_and_viewer(app: GalleryApp):
assert v.status == 501
+def test_traversal_absolute_paths_and_symlinks_out_of_the_run_are_refused(app: GalleryApp, tmp_path: Path):
+ for bad in ("../../../../etc/passwd", "..%2f..%2fetc/passwd", "src/../../lamp_three/record.json"):
+ r = app.route(f"/file/runs/wooden_chair_ab12cd34/{bad}")
+ assert r.status in (403, 404), f"{bad} → {r.status}"
+ assert b"root:x:" not in r.body
+ # '//etc/passwd' collapses to 'etc/passwd' inside the run (404, not a read)
+ assert app.route("/file/runs/wooden_chair_ab12cd34//etc/passwd").status == 404
+ assert safe_join(app.roots[0], "wooden_chair_ab12cd34").is_dir()
+ for bad in ("/etc/passwd", "../..", "a/../../b", "x\x00y"):
+ with pytest.raises(PathError):
+ safe_join(app.roots[0], bad)
+ run = app.roots[0] / "wooden_chair_ab12cd34"
+ outside = tmp_path / "secret.txt"
+ outside.write_text("classified")
+ (run / "escape").symlink_to(outside)
+ r = app.route("/file/runs/wooden_chair_ab12cd34/escape")
+ assert r.status == 403 and b"classified" not in r.body
+
+
def test_content_type_table():
assert content_type("a.glb") == "model/gltf-binary"
assert content_type("a.frag") == "text/plain; charset=utf-8"
@@ -169,6 +150,8 @@ def test_unknown_compare_keys_cost_one_rescan_not_one_each(gallery_tree: dict[st
# --------------------------------------------------------------------------- over a real socket
def test_over_a_loopback_socket(app: GalleryApp):
+ with pytest.raises(GalleryError):
+ make_server(app, "203.0.113.1", 0) # TEST-NET-3: never assigned to this host
httpd = make_server(app, DEFAULT_HOST, 0)
port = httpd.server_address[1]
thread = threading.Thread(target=httpd.serve_forever, daemon=True)
@@ -199,8 +182,3 @@ def test_over_a_loopback_socket(app: GalleryApp):
httpd.shutdown()
httpd.server_close()
thread.join(timeout=5)
-
-
-def test_make_server_reports_a_bad_bind(app: GalleryApp):
- with pytest.raises(GalleryError):
- make_server(app, "203.0.113.1", 0) # TEST-NET-3: never assigned to this host
diff --git a/harness/tests/gallery/test_triage.py b/harness/tests/gallery/test_triage.py
index 80699d7b..ad149d90 100644
--- a/harness/tests/gallery/test_triage.py
+++ b/harness/tests/gallery/test_triage.py
@@ -6,12 +6,10 @@ from __future__ import annotations
from pathlib import Path
from urllib.parse import parse_qs
-import pytest
-
from codeverse3d.addons.gallery.compare import (
CSV_COLUMNS,
)
-from codeverse3d.addons.gallery.index import build_index, hero_view
+from codeverse3d.addons.gallery.index import hero_view
from codeverse3d.addons.gallery.model import (
VERDICTS,
humanize_view,
@@ -22,9 +20,10 @@ from codeverse3d.addons.gallery.server import GalleryApp
from codeverse3d.addons.gallery.urls import UrlMaker
-# --------------------------------------------------------------------------- breakdown
-def test_the_buckets_are_disjoint_and_sum_to_n(gallery_tree: dict[str, Path]):
- entries = build_index([gallery_tree["runs"], gallery_tree["battery"]]).entries()
+def test_breakdown_compare_and_csv_over_both_roots(gallery_tree: dict[str, Path]):
+ app = GalleryApp([gallery_tree["runs"], gallery_tree["battery"]])
+ # the breakdown: disjoint and exhaustive
+ entries = app.index.entries()
s = summarize(entries)
assert s.n == 6
assert set(s.breakdown) == set(VERDICTS)
@@ -38,20 +37,53 @@ def test_the_buckets_are_disjoint_and_sum_to_n(gallery_tree: dict[str, Path]):
assert by_slug["half_written"].verdict == by_slug["not_started"].verdict == "error"
assert by_slug["half_written"].score is None
+ # multi-select: the compare route renders only the selected runs
+ r = app.route("/compare", {"runs": "runs/wooden_chair_ab12cd34,static_v9/ctrl_med_toaster"})
+ assert r.status == 200
+ page = r.body.decode()
+ assert page.count("<td class='head'>") == 2
+ assert "wooden_chair_ab12cd34" in page and "ctrl_med_toaster" in page
+ assert "lamp_three" not in page
+ assert "Δ vs baseline" in page
+ assert "never starts, resumes or deletes" in page # the read-only stance is stated
+ r = app.route("/compare", {"runs": "static_v9/half_written,static_v9/not_started"})
+ assert r.status == 200 and b"half_written" in r.body # survives runs with no record
+
+ # csv export without a selection exports the current filter
+ r = app.route("/export.csv", {"verdict": "error"})
+ assert r.content_type.startswith("text/csv") and "attachment" in r.headers["Content-Disposition"]
+ body = r.body.decode()
+ assert body.splitlines()[0] == ",".join(CSV_COLUMNS)
+ assert len(body.strip().splitlines()) == 3 # header + the two unusable runs
+ assert "half_written" in body and "not_started" in body
-# --------------------------------------------------------------------------- hero view
-def test_the_card_shows_one_hero_view_not_the_contact_sheet(gallery_tree: dict[str, Path]):
- entry = next(e for e in build_index([gallery_tree["runs"]]).entries()
- if e.slug == "wooden_chair_ab12cd34")
+def test_hero_view_empty_compare_and_neighbours_over_one_root(gallery_tree: dict[str, Path]):
+ app = GalleryApp([gallery_tree["runs"]])
+ # the card shows one hero view, not the contact sheet
+ entry = app.index.find("runs", "wooden_chair_ab12cd34")
assert entry.hero.endswith("view_front.png") # "front" is in HERO_PREFERENCE
assert entry.hero_label == "Front"
assert entry.n_views == 2
assert entry.sheet.endswith("sheet.png") and entry.card_image == entry.hero
- markup = render_index(build_index([gallery_tree["runs"]]), UrlMaker())
+ markup = render_index(app.index, UrlMaker())
assert "view_front.png" in markup
assert "⊞ 2" in markup # the sheet is one disclosure away
+ page = app.route("/compare").body.decode()
+ assert "nothing to compare" in page and app.route("/compare").status == 200
+
+ # detail pages link to the neighbouring runs
+ ordered = app.neighbours(entry)
+ assert ordered[0] is None # highest score in the battery
+ assert ordered[1].slug == "lamp_three"
+ prev, nxt = app.neighbours(app.index.find("runs", "lamp_three"))
+ assert prev.slug == "wooden_chair_ab12cd34" and nxt is None
+ page = app.route("/run/runs/wooden_chair_ab12cd34").body.decode()
+ assert "/run/runs/lamp_three" in page and "next →" in page
+ outside = app.index.find("runs", "lamp_three").model_copy(update={"battery": "nope"})
+ assert app.neighbours(outside) == (None, None)
+
def test_hero_falls_back_to_the_sheet_when_a_round_has_no_views(tmp_path: Path):
from tests.flywheel_cli.conftest import make_fake_run
@@ -62,38 +94,18 @@ def test_hero_falls_back_to_the_sheet_when_a_round_has_no_views(tmp_path: Path):
assert hero_view(ws, rec, 1) == ("", "", 0) and hero_view(ws, rec, None) == ("", "", 0)
-# --------------------------------------------------------------------------- labels
[email protected](("raw", "human"), [
- ("front_right_high", "Front Right High"),
- ("front_right_34", "Front Right ¾"),
- ("view_front_right_34.png", "Front Right ¾"),
- ("t=2.5s", "t = 2.5 s"),
- ("pose_rest", "Pose · rest"),
- ("articulation_sheet", "Articulation poses"),
- ("preview.gif", "Animated preview"),
- ("", ""),
-])
-def test_view_names_are_humanised(raw: str, human: str):
- assert humanize_view(raw) == human
-
-
-# --------------------------------------------------------------------------- multi-select
-def test_compare_route_renders_only_the_selected_runs(gallery_tree: dict[str, Path]):
- app = GalleryApp([gallery_tree["runs"], gallery_tree["battery"]])
- r = app.route("/compare", {"runs": "runs/wooden_chair_ab12cd34,static_v9/ctrl_med_toaster"})
- assert r.status == 200
- page = r.body.decode()
- assert page.count("<td class='head'>") == 2
- assert "wooden_chair_ab12cd34" in page and "ctrl_med_toaster" in page
- assert "lamp_three" not in page
- assert "Δ vs baseline" in page
- assert "never starts, resumes or deletes" in page # the read-only stance is stated
-
-
-def test_compare_of_nothing_explains_itself(gallery_tree: dict[str, Path]):
- app = GalleryApp([gallery_tree["runs"]])
- page = app.route("/compare").body.decode()
- assert "nothing to compare" in page and app.route("/compare").status == 200
+def test_view_names_are_humanised():
+ for raw, human in [
+ ("front_right_high", "Front Right High"),
+ ("front_right_34", "Front Right ¾"),
+ ("view_front_right_34.png", "Front Right ¾"),
+ ("t=2.5s", "t = 2.5 s"),
+ ("pose_rest", "Pose · rest"),
+ ("articulation_sheet", "Articulation poses"),
+ ("preview.gif", "Animated preview"),
+ ("", ""),
+ ]:
+ assert humanize_view(raw) == human, raw
def test_query_links_survive_a_duplicate_root_label(gallery_tree: dict[str, Path]):
@@ -105,38 +117,3 @@ def test_query_links_survive_a_duplicate_root_label(gallery_tree: dict[str, Path
assert "#" not in href and parse_qs(href) == {"runs": [key]}
detail = app.route("/run/runs#2/wooden_chair_ab12cd34").body.decode()
assert "href='/?battery=runs%232'" in detail
-
-
-def test_compare_survives_a_run_with_no_record(gallery_tree: dict[str, Path]):
- app = GalleryApp([gallery_tree["battery"]])
- r = app.route("/compare", {"runs": "static_v9/half_written,static_v9/not_started"})
- assert r.status == 200 and b"half_written" in r.body
-
-
-# --------------------------------------------------------------------------- csv export
-def test_export_csv_without_a_selection_exports_the_current_filter(gallery_tree: dict[str, Path]):
- app = GalleryApp([gallery_tree["runs"], gallery_tree["battery"]])
- r = app.route("/export.csv", {"verdict": "error"})
- assert r.content_type.startswith("text/csv") and "attachment" in r.headers["Content-Disposition"]
- body = r.body.decode()
- assert body.splitlines()[0] == ",".join(CSV_COLUMNS)
- assert len(body.strip().splitlines()) == 3 # header + the two unusable runs
- assert "half_written" in body and "not_started" in body
-
-
-# --------------------------------------------------------------------------- neighbours
-def test_detail_pages_link_to_the_neighbouring_runs(gallery_tree: dict[str, Path]):
- app = GalleryApp([gallery_tree["runs"]])
- ordered = app.neighbours(app.index.find("runs", "wooden_chair_ab12cd34"))
- assert ordered[0] is None # highest score in the battery
- assert ordered[1].slug == "lamp_three"
- prev, nxt = app.neighbours(app.index.find("runs", "lamp_three"))
- assert prev.slug == "wooden_chair_ab12cd34" and nxt is None
- page = app.route("/run/runs/wooden_chair_ab12cd34").body.decode()
- assert "/run/runs/lamp_three" in page and "next →" in page
-
-
-def test_neighbours_of_a_run_outside_the_index_are_empty(gallery_tree: dict[str, Path]):
- app = GalleryApp([gallery_tree["runs"]])
- entry = app.index.find("runs", "lamp_three").model_copy(update={"battery": "nope"})
- assert app.neighbours(entry) == (None, None)
diff --git a/harness/tests/install/test_docs.py b/harness/tests/install/test_docs.py
index 71e94a2b..ceff7944 100644
--- a/harness/tests/install/test_docs.py
+++ b/harness/tests/install/test_docs.py
@@ -148,14 +148,6 @@ def test_every_pyproject_extra_is_documented() -> None:
assert not undocumented, f"extras missing from docs/INSTALL.md: {undocumented}"
-def test_bare_pytest_keeps_live_tests_opt_in() -> None:
- """A default local run must never spend API quota or wait on a provider."""
- with (HARNESS / "pyproject.toml").open("rb") as fh:
- addopts = tomllib.load(fh)["tool"]["pytest"]["ini_options"]["addopts"]
- assert "-m" in addopts
- assert addopts[addopts.index("-m") + 1] == "not live"
-
-
#: `pytest ... -m "<expr>"` as published in the docs
_PYTEST_MARKER_EXPR = re.compile(r'pytest[^\n]*?-m\s+"([^"]+)"')
#: every file that publishes a pytest command a reader will paste
@@ -163,14 +155,18 @@ DOCS_WITH_TEST_COMMANDS = [INSTALL, HARNESS / "docs" / "RUNBOOK.md", HARNESS / "
HARNESS / "README.md", REPO / "README.md"]
[email protected]("doc", DOCS_WITH_TEST_COMMANDS, ids=lambda p: p.name)
-def test_documented_pytest_subsets_never_re_enable_the_live_tests(doc: Path) -> None:
- """Every documented marker override must keep live tests opt-in."""
- if not doc.is_file(): # the harness can be checked out without the repo README
- pytest.skip(f"{doc} not present")
- for expr in _PYTEST_MARKER_EXPR.findall(doc.read_text()):
- assert "not live" in expr or expr.strip() == "live", (
- f"{doc.name} publishes pytest -m \"{expr}\", which re-enables the live tests")
+def test_live_tests_stay_opt_in_for_bare_pytest_and_every_documented_subset() -> None:
+ """A default local run — or a documented marker override — must never spend API quota or wait on a provider."""
+ with (HARNESS / "pyproject.toml").open("rb") as fh:
+ addopts = tomllib.load(fh)["tool"]["pytest"]["ini_options"]["addopts"]
+ assert "-m" in addopts
+ assert addopts[addopts.index("-m") + 1] == "not live"
+ for doc in DOCS_WITH_TEST_COMMANDS:
+ if not doc.is_file(): # the harness can be checked out without the repo README
+ continue
+ for expr in _PYTEST_MARKER_EXPR.findall(doc.read_text()):
+ assert "not live" in expr or expr.strip() == "live", (
+ f"{doc.name} publishes pytest -m \"{expr}\", which re-enables the live tests")
def test_doctor_checks_every_module_of_every_optional_extra(monkeypatch) -> None:
@@ -333,14 +329,19 @@ def test_setup_verifies_the_interpreter_it_installed_into(tmp_path) -> None:
impostor = fake / "3dcodeverse"
impostor.write_text("#!/bin/sh\necho WRONG_INSTALL_ON_PATH\n")
impostor.chmod(0o755)
+ # the --python interpreter answers the doctor call itself (the real doctor is ~10 s of probes)
+ py = tmp_path / "python-shim"
+ py.write_text('#!/bin/sh\n[ "$1 $2" = "-m codeverse3d.cli.main" ] && { echo "DOCTOR_VIA_PYTHON $3"; exit 0; }\n'
+ f'exec {sys.executable} "$@"\n')
+ py.chmod(0o755)
env = dict(os.environ, PATH=f"{fake}{os.pathsep}{os.environ['PATH']}")
proc = subprocess.run(
- ["bash", str(SETUP), "--python", sys.executable, "--no-node", "--no-chrome", "--no-gpu", "--no-python"],
+ ["bash", str(SETUP), "--python", str(py), "--no-node", "--no-chrome", "--no-gpu", "--no-python"],
capture_output=True, text=True, timeout=300, env=env)
assert "WRONG_INSTALL_ON_PATH" not in proc.stdout + proc.stderr, (
"setup.sh verified the 3dcodeverse first on PATH instead of the --python interpreter")
- assert "doctor" in proc.stdout
+ assert "DOCTOR_VIA_PYTHON doctor" in proc.stdout, proc.stdout + proc.stderr
def test_setup_names_the_remedy_when_the_interpreter_has_no_pip(tmp_path) -> None: