Skip to content

Official Codebase of 3DCodeVerse

Python

tests: flywheel_cli, agents — 43 tests go that cover no line another test does not (−973 lines)

gaoypeng163f3dc46c70
diff --git a/harness/tests/agents/test_agent_job_wiring.py b/harness/tests/agents/test_agent_job_wiring.py
index b44e439c..576f55a0 100644
--- a/harness/tests/agents/test_agent_job_wiring.py
+++ b/harness/tests/agents/test_agent_job_wiring.py
@@ -14,26 +14,19 @@ from codeverse3d.workspace import Workspace
class StubAgent:
kind, model = "stub", "m"
- def __init__(self, turns: int = 0):
+ def __init__(self):
self.jobs: list[AgentJob] = []
- self.turns = turns
def run(self, job: AgentJob) -> AgentResult:
self.jobs.append(job)
ws = Workspace(job.workspace)
(ws.src / "a.txt").write_text("x\n")
- return AgentResult(ok=True, exit_reason="completed", turns=self.turns,
+ return AgentResult(ok=True, exit_reason="completed", turns=0,
usage=Usage(backend="stub", input_tokens=10, cost_usd=0.01))
def test_images_reach_the_agent_as_paths_in_the_prompt(tmp_ws: Workspace, tmp_path):
- """No vendor CLI takes an image on argv: the contact sheet the judge scored (and the
- reference photos) are copied into the workspace and listed for the agent's own file
- tools. The copy is what makes the read possible: a host path is outside every CLI's
- workspace, and the sheet's own home (``artifacts/renders/``) is hidden by
- ``.geminiignore``. ``AgentJob.images`` used to be filled beside this and read by
- nobody; it was deleted 2026-08-30, so the prompt block and the staged bytes below are
- the whole delivery path — nothing else carries an image into a session."""
+ """Images are staged into the workspace and listed in the prompt — no CLI takes one on argv."""
sheet = tmp_path / "contact_sheet.png"
sheet.write_bytes(b"png")
task = GenerationTask(label="refine", prompt="fix the horn", round=1, kind="refine",
@@ -54,8 +47,7 @@ def test_images_reach_the_agent_as_paths_in_the_prompt(tmp_ws: Workspace, tmp_pa
def test_session_window_is_clipped_to_the_hard_clock_not_the_soft_share(tmp_ws: Workspace):
- """A scene refine asks for its window with soft=False (after the 0.55 soft share is
- spent); re-clipping it against the SOFT share handed it the 120 s floor (2026-08-29)."""
+ """Clipping against the soft share handed a late refine the 120 s floor (2026-08-29)."""
guard = BudgetGuard(Budget(max_minutes=100), soft_fraction=0.5)
guard.start_time -= 60 * 60 # 60 of 100 minutes gone: the soft share is spent, 40 min remain
assert not guard.soft_ok()
@@ -63,7 +55,3 @@ def test_session_window_is_clipped_to_the_hard_clock_not_the_soft_share(tmp_ws:
run_agent_task(tmp_ws, agent=agent, task=GenerationTask(label="refine", prompt="p", timeout_s=900), budget=guard)
assert agent.jobs[0].timeout_s == 900
-
-def test_turns_come_from_the_typed_result(tmp_ws: Workspace):
- res = run_agent_task(tmp_ws, agent=StubAgent(turns=7), task=GenerationTask(label="baseline", prompt="p"))
- assert res.turns == 7
diff --git a/harness/tests/agents/test_antigravity.py b/harness/tests/agents/test_antigravity.py
index d8f63913..df0766ee 100644
--- a/harness/tests/agents/test_antigravity.py
+++ b/harness/tests/agents/test_antigravity.py
@@ -11,9 +11,7 @@ import pytest
from codeverse3d.agents.backends import (
AntigravityAgent,
available_models,
- parse_agy_json,
resolve_model,
- usage_from_agy,
)
from codeverse3d.agents.cli_common import begin_session
from codeverse3d.contracts.agent import AgentJob
@@ -79,29 +77,19 @@ print(""" + repr(json.dumps(ENVELOPE)) + r""")
"""
-def test_parse_and_usage():
- env = parse_agy_json(json.dumps(ENVELOPE))
- assert env["status"] == "SUCCESS"
- assert parse_agy_json("just text") is None
- u = usage_from_agy(env, "gemini-3.6-flash-high")
- assert (u.input_tokens, u.output_tokens, u.thoughts_tokens, u.cached_tokens) == (18589, 494, 413, 12199)
- assert u.cost_usd == 0.0 and u.latency_ms == 3000
-
-
-def test_argv(tmp_ws: Workspace):
+def test_argv_sets_the_print_timeout_a_minute_past_the_session(tmp_ws: Workspace):
a = AntigravityAgent("gemini-3.6-flash-high", binary="agy")
s = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="p", timeout_s=1800), "agy")
argv = a.build_argv(s)
- assert "--print" not in argv and "p" not in argv and argv[argv.index("--print-timeout") + 1] == "31m"
- assert argv[argv.index("--log-file") + 1] == str(s.traj.dir / "agy.log") # its retries, kept with the session
- assert argv[argv.index("--model") + 1] == "gemini-3.6-flash-high" and argv[argv.index("--add-dir") + 1] == str(tmp_ws.root)
- assert argv[argv.index("--output-format") + 1] == "json"
+ assert argv[argv.index("--print-timeout") + 1] == "31m"
def test_fake_run_json_and_text(tmp_ws: Workspace, fake_bin, monkeypatch):
a = AntigravityAgent("gemini-3.6-flash-high", binary=fake_bin("agy", FAKE_AGY))
res = a.run(AgentJob(workspace=str(tmp_ws.root), prompt="hello", label="a", timeout_s=30))
- assert res.ok and res.text.strip() == "DONE" and res.usage.input_tokens == 18589, res.errors
+ assert res.ok and res.text.strip() == "DONE", res.errors
+ u = res.usage
+ assert (u.input_tokens, u.output_tokens, u.thoughts_tokens, u.cached_tokens) == (18589, 494, 413, 12199)
assert [f.path for f in res.files_changed] == ["src/hello.txt", "src/prompt.txt"]
assert (tmp_ws.src / "prompt.txt").read_text().endswith("\n\nhello") # on stdin, after the workspace header
monkeypatch.setenv("FAKE_MODE", "text")
@@ -149,44 +137,19 @@ def test_live_agy_tiny(tmp_ws: Workspace):
assert (tmp_ws.src / "hello.txt").read_text().strip() == "hi"
-# --------------------------------------------------------------------------- SMOKE3
-def test_a_bare_model_id_gains_the_effort_agy_requires(tmp_ws: Workspace, fake_bin):
- """SMOKE3: build_argv appended --model but never an effort, so `agy:gemini-3.7-flash`
- died with rc=1 'invalid model selection ... requires --effort'. The effort cannot be
- a blanket --effort flag: agy rejects that outright for a model that has none, so it
- has to go into the model ID — the only spelling `agy models` lists."""
- available_models.cache_clear()
- a = AntigravityAgent("gemini-3.7-flash", binary=fake_bin("agy", FAKE_AGY_MODELS))
- s = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="p", timeout_s=60), "agy")
-
- argv = a.build_argv(s)
-
- assert argv[argv.index("--model") + 1] == "gemini-3.7-flash-medium"
- assert "--effort" not in argv
-
-
-def test_an_effort_suffixed_or_effortless_id_is_left_alone(tmp_ws: Workspace, fake_bin):
- """Both forms already worked and must not be touched: a suffixed id is passed
- through without a lookup, and claude-sonnet-4-6 has no effort at all."""
- available_models.cache_clear()
- binary = fake_bin("agy", FAKE_AGY_MODELS)
- assert resolve_model("gemini-3.7-flash-low", binary) == "gemini-3.7-flash-low"
- assert resolve_model("claude-sonnet-4-6", binary) == "claude-sonnet-4-6"
- assert resolve_model("", binary) == ""
-
-
def test_an_unknown_model_and_an_unreachable_agy_are_passed_through(tmp_ws: Workspace, fake_bin):
"""Resolution can only turn a guaranteed failure into a run, never the reverse."""
available_models.cache_clear()
binary = fake_bin("agy", FAKE_AGY_MODELS)
+ assert resolve_model("gemini-3.7-flash-low", binary) == "gemini-3.7-flash-low"
+ assert resolve_model("claude-sonnet-4-6", binary) == "claude-sonnet-4-6" # has no effort
assert resolve_model("some-future-model", binary) == "some-future-model"
available_models.cache_clear()
assert resolve_model("gemini-3.7-flash", "/nonexistent/agy") == "gemini-3.7-flash"
def test_the_run_uses_and_records_the_resolved_id(tmp_ws: Workspace, fake_bin):
- """The fake refuses the bare id exactly as agy 1.1.19 does, so a green run here
- means the resolved id really was the one sent."""
+ """agy 1.1.19 refuses a bare id that has efforts: the run sends the effort-suffixed id."""
available_models.cache_clear()
a = AntigravityAgent("gemini-3.7-flash", binary=fake_bin("agy", FAKE_AGY_MODELS))
diff --git a/harness/tests/agents/test_claude_code.py b/harness/tests/agents/test_claude_code.py
index b6ab8ad3..b26adf0e 100644
--- a/harness/tests/agents/test_claude_code.py
+++ b/harness/tests/agents/test_claude_code.py
@@ -84,67 +84,22 @@ def test_parse_envelope_variants():
assert u.input_tokens == 5210
-def test_argv_includes_mcp_when_materialized(tmp_ws: Workspace):
+def test_the_mcp_config_is_the_typed_jobs_never_the_workspace_file(tmp_ws: Workspace):
+ cmd = ["python", "-m", "codeverse3d.spatial.mcp_server", "--workspace", str(tmp_ws.root)]
materialize_workspace(tmp_ws, agent_kind="claude-code", contract_md="c", cookbook_text="", spatial_tools=True,
- mcp_command=["python", "-m", "codeverse3d.spatial.mcp_server", "--workspace", str(tmp_ws.root)])
+ mcp_command=cmd)
a = ClaudeCodeAgent("sonnet", binary="claude")
- s = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="p", system_append="S", max_turns=7,
- mcp_command=["python", "-m", "codeverse3d.spatial.mcp_server", "--workspace", str(tmp_ws.root)]),
- "claude-code")
+ s = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="p", mcp_command=cmd), "claude-code")
+ (tmp_ws.root / ".mcp.json").write_text(json.dumps({"mcpServers": {"3dcode": {"command": "/tmp/evil", "args": []}}}))
argv = a.build_argv(s)
- assert argv[:3] == ["claude", "-p", "--output-format"] and "p" not in argv # the prompt goes on stdin
- # the config is written fresh into THIS session's trajectory dir from the typed job,
- # never the workspace .mcp.json the agent can rewrite between rounds
cfg = Path(argv[argv.index("--mcp-config") + 1])
- assert cfg.name == "mcp.json" and cfg.parent == s.traj.dir
- server = json.loads(cfg.read_text())["mcpServers"]["3dcode"]
- assert server["command"] == "python" and "--workspace" in server["args"] # the TYPED job command, verbatim
- (tmp_ws.root / ".mcp.json").write_text(json.dumps(
- {"mcpServers": {"3dcode": {"command": "/tmp/evil", "args": []}}}))
- argv2 = a.build_argv(s) # a tampered workspace file changes nothing
- assert json.loads(Path(argv2[argv2.index("--mcp-config") + 1]).read_text(
- ))["mcpServers"]["3dcode"]["command"].endswith("python")
- assert "--strict-mcp-config" in argv and argv[argv.index("--max-turns") + 1] == "7"
- assert argv[argv.index("--append-system-prompt") + 1] == "S" and argv[argv.index("--model") + 1] == "sonnet"
+ assert cfg.parent == s.traj.dir
+ assert json.loads(cfg.read_text())["mcpServers"]["3dcode"]["command"] == "python"
assert "mcp__3dcode__*" in argv[argv.index("--allowedTools") + 1]
s2 = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="p", spatial_tools=False), "claude-code")
assert "--mcp-config" not in a.build_argv(s2)
-def test_a_session_sees_only_the_routed_skills(tmp_ws: Workspace):
- """claude-code 2.1.280 listed 26 skills besides the routed bundles (2026-09-22 rig). No
- "user" setting source drops the account-synced skills and plugins; the env switch drops the
- bundled ones; the two that survive it by design are hidden by name. Checked against a local
- fake API: the init event then lists exactly the routed bundles."""
- from codeverse3d.agents.backends import CLAUDE_SETTINGS
-
- a = ClaudeCodeAgent("sonnet", binary="claude")
- s = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="p", spatial_tools=False), "claude-code")
- argv = a.build_argv(s)
- assert argv[argv.index("--setting-sources") + 1] == "project"
- assert json.loads(argv[argv.index("--settings") + 1]) == CLAUDE_SETTINGS == {
- "skillOverrides": {"design": "off", "doctor": "off"}}
- assert a.build_env(s)["CLAUDE_CODE_DISABLE_BUNDLED_SKILLS"] == "1"
- assert "--disable-slash-commands" not in argv # claude-code: "Disable all skills" — the routed ones too
-
-
-def test_the_effort_is_the_harness_setting_not_the_users(tmp_ws: Workspace, switch):
- """No user setting source means no inherited effortLevel: the harness states it."""
- s = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="p", spatial_tools=False), "claude-code")
- argv = ClaudeCodeAgent("sonnet", binary="claude").build_argv(s)
- assert argv[argv.index("--effort") + 1] == "xhigh"
- switch("C3D_AGENTS__CLAUDE_EFFORT", "high")
- argv = ClaudeCodeAgent("sonnet", binary="claude").build_argv(s)
- assert argv[argv.index("--effort") + 1] == "high"
-
-
-def test_the_usage_limit_is_quota_not_a_transient_death(tmp_ws: Workspace, fake_bin, monkeypatch):
- a = ClaudeCodeAgent("sonnet", binary=fake_bin("claude", FAKE_CLAUDE))
- monkeypatch.setenv("FAKE_MODE", "usage_limit")
- res = a.run(AgentJob(workspace=str(tmp_ws.root), prompt="hello", label="u", timeout_s=30))
- assert not res.ok and res.quota and not res.transient and res.exit_reason == "budget"
-
-
def test_fake_run_success_and_error(tmp_ws: Workspace, fake_bin, monkeypatch):
a = ClaudeCodeAgent("sonnet", binary=fake_bin("claude", FAKE_CLAUDE))
res = a.run(AgentJob(workspace=str(tmp_ws.root), prompt="hello", label="c", system_append="SYS", timeout_s=30))
@@ -161,16 +116,16 @@ def test_fake_run_success_and_error(tmp_ws: Workspace, fake_bin, monkeypatch):
assert res2.transient is False # the task's own failure
-def test_an_overloaded_exit_is_transient(tmp_ws: Workspace, fake_bin, monkeypatch):
- """A 529 exit is the provider's, not the task's: it must say so (AgentResult.transient) —
- the storm fallback and the repair loop's stop both key on it, and it was always False."""
[email protected]("mode, reason, transient, quota", [
+ ("usage_limit", "budget", False, True), # the subscription is spent
+ ("overloaded", "error", True, False), # a 529 is the provider's, not the task's
+ ("rate_limited", "budget", True, False),
+])
+def test_a_provider_exit_is_typed(tmp_ws: Workspace, fake_bin, monkeypatch, mode, reason, transient, quota):
a = ClaudeCodeAgent("sonnet", binary=fake_bin("claude", FAKE_CLAUDE))
- monkeypatch.setenv("FAKE_MODE", "overloaded")
+ monkeypatch.setenv("FAKE_MODE", mode)
res = a.run(AgentJob(workspace=str(tmp_ws.root), prompt="hello", label="o", timeout_s=30))
- assert not res.ok and res.exit_reason == "error" and res.transient is True and not res.quota
- monkeypatch.setenv("FAKE_MODE", "rate_limited")
- res = a.run(AgentJob(workspace=str(tmp_ws.root), prompt="hello", label="q", timeout_s=30))
- assert not res.ok and res.exit_reason == "budget" and res.transient is True and not res.quota
+ assert not res.ok and res.exit_reason == reason and res.transient is transient and res.quota is quota
@pytest.mark.live
@@ -185,35 +140,12 @@ def test_live_claude_tiny(tmp_ws: Workspace):
assert res.usage.output_tokens > 0
-# --------------------------------------------------------------------------- CC-1
-def test_an_alias_records_the_model_that_did_the_work_not_the_housekeeping_one():
- """`claude -p` bills two models and lists the AUXILIARY one first, so
- `served[0]` recorded haiku — which did 0.9% of the tokens — for every
- --model ALIAS ('sonnet', 'opus', and the default arm). A sonnet-vs-default
- comparison was therefore labelled haiku-vs-haiku."""
- for alias in ("sonnet", "opus", ""):
- u = usage_from_envelope(TWO_MODEL_ENVELOPE, alias)
- assert u.model == "claude-sonnet-5", alias
- # the top-level usage block reports the MAIN conversation only — that is the signal
- assert primary_served_model(TWO_MODEL_ENVELOPE, "sonnet") == "claude-sonnet-5"
-
-
-def test_a_full_id_we_passed_verbatim_is_kept():
- """Full ids were never affected; the bug bit only the form the owner asked for."""
- assert primary_served_model(TWO_MODEL_ENVELOPE, "claude-sonnet-5") == "claude-sonnet-5"
+def test_the_served_model_is_the_work_model_not_the_housekeeping_one():
+ """`claude -p` bills two models and lists the auxiliary one first (CC-1)."""
+ assert usage_from_envelope(TWO_MODEL_ENVELOPE, "").model == "claude-sonnet-5"
assert primary_served_model(TWO_MODEL_ENVELOPE, "claude-haiku-4-5-20251001") == "claude-haiku-4-5-20251001"
-
-
-def test_the_dearest_entry_wins_when_the_token_counts_do_not_match():
- """Fallback for an envelope whose top-level usage block does not line up with any
- modelUsage row: the work model is the one that cost the money, never the side-call."""
- env = {**TWO_MODEL_ENVELOPE, "usage": {"input_tokens": 0, "output_tokens": 0}}
- assert primary_served_model(env, "sonnet") == "claude-sonnet-5"
-
-
-def test_no_model_usage_block_keeps_what_we_asked_for():
- assert primary_served_model({"usage": {"input_tokens": 1}}, "sonnet") == "sonnet"
- assert usage_from_envelope({"usage": {"input_tokens": 1}}, "sonnet").model == "sonnet"
+ env = {**TWO_MODEL_ENVELOPE, "usage": {"input_tokens": 0, "output_tokens": 0}} # no row lines up
+ assert primary_served_model(env, "sonnet") == "claude-sonnet-5" # the dearest wins
# --------------------------------------------------------------------------- killed sessions
@@ -240,10 +172,7 @@ KILLED_STREAM = [
def test_a_killed_session_is_booked_from_its_messages_and_its_retries_are_provider_wait(tmp_ws: Workspace,
fake_bin, monkeypatch):
- """claude prints no result event when it is killed (checked 2026-09-22: SIGTERM mid-request →
- nothing), so the envelope's usage never comes. Each message's own usage block does: the input
- side exact, the output a floor. Its api_retry events (claude's own, with the announced
- back-off) are the time the provider cost — here until the kill, since nothing followed."""
+ """A killed claude prints no result event: usage comes from each message, wait from api_retry."""
from codeverse3d.models.pricing import cache_write_surcharge, estimate_cost
monkeypatch.setenv("FAKE_EVENTS", json.dumps(KILLED_STREAM))
diff --git a/harness/tests/agents/test_cli_common.py b/harness/tests/agents/test_cli_common.py
index 9cc91c73..985978ac 100644
--- a/harness/tests/agents/test_cli_common.py
+++ b/harness/tests/agents/test_cli_common.py
@@ -6,7 +6,6 @@ import json
import os
from codeverse3d.agents.cli_common import (
- Trajectory,
attribute_changes,
begin_session,
clean_env,
@@ -33,28 +32,10 @@ def test_clean_env_drops_secrets_but_the_kept_ones(monkeypatch):
assert env["C3D_AGENT_CONTEXT"] == "1" and "GIT_CEILING_DIRECTORIES" not in env
-def test_hardened_env_strips_secrets_and_adds_guards(tmp_ws: Workspace, monkeypatch):
- monkeypatch.setenv("OPENAI_API_KEY", "sk-x")
- monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-y")
- monkeypatch.setenv("KEEPME", "1")
- job = AgentJob(workspace=str(tmp_ws.root), prompt="p", env={"EXTRA": "2"})
- env = hardened_env(tmp_ws, job, keep={"ANTHROPIC_API_KEY"})
- assert "OPENAI_API_KEY" not in env and "FAKE_SERVICE_API_KEY" not in env
- assert env["ANTHROPIC_API_KEY"] == "sk-y" and env["KEEPME"] == "1" and env["EXTRA"] == "2"
- assert env["C3D_AGENT_CONTEXT"] == "1" and env["GIT_CEILING_DIRECTORIES"] == str(tmp_ws.root.parent)
-
-
-def test_session_roundtrip_tracks_files_and_writes_result(tmp_ws: Workspace):
- job = AgentJob(workspace=str(tmp_ws.root), prompt="do it", label="baseline", round=2)
- s = begin_session(job, "fake")
- assert s.traj.dir.name == "baseline_r02" and s.traj.prompt_path.read_text() == "do it"
- (tmp_ws.src / "model.py").write_text("print(1)\n")
- res = finish_session(s, ok=True, exit_reason="completed", text="done", usage=Usage(input_tokens=5), tool_calls=1)
- assert res.ok and [f.path for f in res.files_changed] == ["src/model.py"]
- assert res.files_changed[0].status == "added" and res.files_changed[0].lines_added == 1
- data = json.loads(s.traj.result_path.read_text())
- assert data["exit_reason"] == "completed" and data["head_before"] != data["head_after"]
- assert tmp_ws.head() == data["head_after"]
+def test_hardened_env_adds_the_job_env_and_the_git_ceiling(tmp_ws: Workspace):
+ env = hardened_env(tmp_ws, AgentJob(workspace=str(tmp_ws.root), prompt="p", env={"EXTRA": "2"}), keep=set())
+ assert env["EXTRA"] == "2" and env["GIT_CEILING_DIRECTORIES"] == str(tmp_ws.root.parent)
+ assert "FAKE_SERVICE_API_KEY" not in env
def test_retry_same_label_round_keeps_first_attempt_trajectory(tmp_ws: Workspace):
@@ -87,9 +68,7 @@ def test_attribute_changes_pure():
def test_the_prompt_goes_on_stdin_byte_for_byte_and_never_on_argv(tmp_ws: Workspace):
- """A 2 500-line, 200 kB prompt — past argv's 128 KiB per-argument cap and past gemini-cli's
- 2 000-line read_file window, which is where the old file stub dropped a scene's last zones —
- reaches the CLI whole, and the transcript records its size, not its text."""
+ """A 200 kB, 2 500-line prompt (past argv's 128 KiB cap) reaches the CLI whole."""
import sys
s = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="x", label="p"), "fake")
@@ -104,9 +83,7 @@ def test_the_prompt_goes_on_stdin_byte_for_byte_and_never_on_argv(tmp_ws: Worksp
def test_one_failure_vocabulary():
- """The backends classify what the CLI said about the call that ended its session; the round
- loop reads only the typed flags. Provider failures a retry may get through are transient; a
- spent usage limit is quota and never transient; the task's own failure is neither."""
+ """Transient = a retry may get through; quota = a spent usage limit, never transient."""
for text in ("Attempt 3 failed with status 503", "API Error: 529 overloaded_error", "got status: UNAVAILABLE",
"Error: 429 RESOURCE_EXHAUSTED", "TypeError: fetch failed sending request", "read ECONNRESET",
"stream disconnected before completion", "got an empty response from the model"):
@@ -146,16 +123,6 @@ def test_finish_session_carries_the_typed_failure_and_the_wait(tmp_ws: Workspace
assert 0.0 <= res.provider_wait_s <= res.duration_s # never more than the session lasted
-def test_trajectory_jsonl_and_result(tmp_path):
- t = Trajectory(tmp_path / "traj")
- t.append("a", x=1)
- t.append("b", y="z")
- recs = t.read_transcript()
- assert [r["kind"] for r in recs] == ["a", "b"] and recs[1]["y"] == "z" and "t" in recs[0]
- t.write_result(Usage(input_tokens=3), note="n")
- assert json.loads(t.result_path.read_text()) == {**Usage(input_tokens=3).model_dump(), "note": "n"}
-
-
def test_mcp_command_resolution(tmp_ws: Workspace):
from codeverse3d.agents.cli_common import default_mcp_command, mcp_command_for
from codeverse3d.agents.materialize import materialize_workspace
diff --git a/harness/tests/agents/test_cli_write_scope.py b/harness/tests/agents/test_cli_write_scope.py
index 2fde8e9a..dcf9d779 100644
--- a/harness/tests/agents/test_cli_write_scope.py
+++ b/harness/tests/agents/test_cli_write_scope.py
@@ -1,20 +1,11 @@
-"""CLI write-scope enforcement: per-workspace serialisation + post-hoc edit_only restore.
-
-No backend has a write-time file gate (the in-process one that did was deleted
-2026-08-28), and both session snapshots run
-``git add -A``, so two concurrent CLI sessions in one workspace make provenance (and
-any per-path rollback) unfixable. ``begin_session`` therefore serialises
-``EXCLUSIVE_KINDS`` per workspace, and ``finish_session`` reverts writes outside
-``job.write_roots`` (always) and outside ``files_hint`` (``edit_only``) to the session's
-own ``pre:`` commit and fails the session.
-"""
+"""CLI write scope: per-workspace serialisation and the post-hoc restore in finish_session."""
from __future__ import annotations
import threading
import time
-from codeverse3d.agents.cli_common import EXCLUSIVE_KINDS, begin_session, finish_session
+from codeverse3d.agents.cli_common import begin_session, finish_session
from codeverse3d.contracts.agent import AgentJob
from codeverse3d.contracts.common import Usage
from codeverse3d.workspace import Workspace
@@ -29,11 +20,6 @@ def _finish(s, *, ok: bool = True):
# --------------------------------------------------------------------------- serialisation
-def test_every_vendor_cli_kind_is_exclusive():
- assert {"claude-code", "codex", "gemini-cli", "agy"} == EXCLUSIVE_KINDS
- assert "fake" not in EXCLUSIVE_KINDS # test doubles stay lock-free
-
-
def test_two_cli_sessions_on_one_workspace_are_serialized(tmp_ws: Workspace):
"""Two threads, one workspace: the begin→finish windows must not overlap."""
windows: dict[str, tuple[float, float]] = {}
@@ -59,23 +45,6 @@ def test_two_cli_sessions_on_one_workspace_are_serialized(tmp_ws: Workspace):
assert a1 <= b0 or b1 <= a0, f"sessions interleaved: a={windows['a']} b={windows['b']}"
-def test_an_unknown_kind_takes_no_lock(tmp_ws: Workspace):
- s1 = begin_session(_job(tmp_ws, "s1"), "fake")
- assert s1.ws_lock is None
- s2 = begin_session(_job(tmp_ws, "s2"), "fake") # would deadlock if locked
- _finish(s2)
- _finish(s1)
-
-
-def test_same_thread_can_begin_again_without_finishing(tmp_ws: Workspace):
- """The lock is re-entrant: argv-building tests begin sessions they never finish."""
- s1 = begin_session(_job(tmp_ws, "s1"), "codex")
- assert s1.ws_lock is not None
- s2 = begin_session(_job(tmp_ws, "s2"), "codex")
- _finish(s2)
- _finish(s1) # double release stays idempotent via release_session
-
-
# --------------------------------------------------------------------------- post-hoc scope
def test_out_of_scope_cli_write_is_restored_and_fails_the_session(scoped_ws: Workspace):
ws = scoped_ws
@@ -94,10 +63,7 @@ def test_out_of_scope_cli_write_is_restored_and_fails_the_session(scoped_ws: Wor
def test_write_roots_are_enforced_without_edit_only(scoped_ws: Workspace):
- """``write_roots`` was enforced for the api-agent (api_tools) and single-shot but NOT
- for CLI agents: ``_enforce_scope`` returned early unless edit_only+files_hint, so a
- plain codex session wrote and COMMITTED root-level evil.py / conftest.py and
- ``attribute_changes`` then hid them from files_changed (audit 2026-08-27)."""
+ """A plain codex session once committed root-level evil.py / conftest.py (audit 2026-08-27)."""
ws = scoped_ws
(ws.root / "notes.md").write_text("original\n")
s = begin_session(_job(ws, "baseline"), "codex") # edit_only=False, write_roots=["src", "public"]
@@ -116,12 +82,7 @@ def test_write_roots_are_enforced_without_edit_only(scoped_ws: Workspace):
def test_harness_control_files_are_never_agent_writable(scoped_ws: Workspace):
- """Control-file tampering used to survive silently: HARNESS_OWNED_FILES was a skip
- (never reverted, never reported) and the gitignored run_state.json / record.json
- were invisible to git entirely — yet reconcile_resume and _prior_record_fields
- trust both blindly. Now: tracked control files (AGENTS.md, .mcp.json) revert to
- the pre: commit, the gitignored two restore from a byte snapshot, an agent-planted
- record.json is unlinked, and the session FAILS with each path named."""
+ """Tracked control files revert, the gitignored ones restore from a snapshot, the session fails."""
import json
ws = scoped_ws
@@ -146,8 +107,7 @@ def test_harness_control_files_are_never_agent_writable(scoped_ws: Workspace):
def test_control_files_are_enforced_even_with_an_empty_write_scope(scoped_ws: Workspace):
- """The empty-scope early-out used to skip enforcement entirely: a job with no
- write_roots, no files_hint and no read_only left every control file writable."""
+ """The empty-scope early-out once left every control file writable."""
ws = scoped_ws
(ws.root / "AGENTS.md").write_text("# original rules\n")
s = begin_session(_job(ws, "free"), "codex") # no write_roots at all
@@ -174,18 +134,8 @@ def test_entry_file_writes_need_ownership(scoped_ws: Workspace):
assert res2.ok and (ws.src / "model.py").read_text() == "# entry + import\n"
-def test_unscoped_cli_sessions_keep_the_old_behaviour(scoped_ws: Workspace):
- ws = scoped_ws
- s = begin_session(_job(ws, "baseline"), "codex") # edit_only=False
- (ws.src / "parts" / "leg.py").write_text("# fine\n")
- res = _finish(s)
- assert res.ok and (ws.src / "parts" / "leg.py").read_text() == "# fine\n"
-
-
def test_a_harness_owned_file_is_restored_when_the_agent_rewrites_it(tmp_ws, monkeypatch):
- """src/recipes.glsl is written by the harness for the agent to CALL. FileTools used
- to refuse the write outright; with the in-process agent deleted (2026-08-28) the
- post-hoc check is the only guard, and without this layer the file was unprotected."""
+ """A read_only harness file (src/recipes.glsl) is restored and the session fails."""
owned = "src/recipes.glsl"
(tmp_ws.root / owned).parent.mkdir(parents=True, exist_ok=True)
(tmp_ws.root / owned).write_text("float aurora(vec2 p){return 0.0;}\n")
diff --git a/harness/tests/agents/test_codex.py b/harness/tests/agents/test_codex.py
index 66121050..6b4c61e3 100644
--- a/harness/tests/agents/test_codex.py
+++ b/harness/tests/agents/test_codex.py
@@ -67,21 +67,14 @@ def test_reasoning_tokens_are_not_billed_twice():
assert u.cost_usd == pytest.approx(1000 * 4.0 / 1e6 + 9000 * 20.0 / 1e6)
-def test_argv_with_mcp_overrides(tmp_ws: Workspace):
+def test_argv_sandboxes_the_workspace_and_approves_mcp_calls(tmp_ws: Workspace):
a = CodexAgent("gpt-5.6-sol", binary="codex")
- s = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="p"), "codex")
- argv = a.build_argv(s)
- assert argv[:3] == ["codex", "exec", "--json"] and argv[argv.index("-C") + 1] == str(tmp_ws.root)
- assert argv[argv.index("--sandbox") + 1] == "workspace-write" and "--skip-git-repo-check" in argv
- joined = " ".join(argv)
- assert 'mcp_servers.3dcode.command="' in joined and "mcp_servers.3dcode.args=[" in joined and argv[-1] == "-"
- # the routed bundles only: codex's five bundled .system skills are switched off (codex debug prompt-input)
- assert argv[argv.index("skills.bundled.enabled=false") - 1] == "-c"
+ argv = a.build_argv(begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="p"), "codex"))
+ assert argv[argv.index("--sandbox") + 1] == "workspace-write"
# codex exec has nobody to answer the per-tool approval elicitation → every MCP call would be cancelled
assert 'mcp_servers.3dcode.default_tools_approval_mode="approve"' in argv
- assert argv[argv.index("--model") + 1] == "gpt-5.6-sol"
s2 = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="p", spatial_tools=False), "codex")
- assert "mcp_servers" not in " ".join(a.build_argv(s2)) and a.build_argv(s2)[-1] == "-"
+ assert "mcp_servers" not in " ".join(a.build_argv(s2))
def test_reasoning_effort_is_explicit(tmp_ws: Workspace, monkeypatch):
@@ -115,8 +108,6 @@ def test_fake_run(tmp_ws: Workspace, fake_bin, monkeypatch):
def test_a_codex_failure_is_typed_quota_or_transient(tmp_ws: Workspace, fake_bin, monkeypatch):
- """codex had no classification at all: its usage limit reached the round loop only because
- the loop's own list said "usage limit", and a 503 death was never transient."""
a = CodexAgent("gpt-5.6-sol", binary=fake_bin("codex", FAKE_CODEX))
monkeypatch.setenv("FAKE_MODE", "usage_limit")
res = a.run(AgentJob(workspace=str(tmp_ws.root), prompt="hello", label="q", timeout_s=30))
diff --git a/harness/tests/agents/test_gemini_cli.py b/harness/tests/agents/test_gemini_cli.py
index c8bbe0a9..a6e8e610 100644
--- a/harness/tests/agents/test_gemini_cli.py
+++ b/harness/tests/agents/test_gemini_cli.py
@@ -104,7 +104,7 @@ def agent(fake_bin, monkeypatch, tmp_path):
def test_retry_window_never_restarts():
- """The retry rule itself: what is LEFT of the window, or None when < min(120, T/4)."""
+ """What is LEFT of the window, or None when < min(120, T/4)."""
from codeverse3d.agents.backends import retry_window_left
assert retry_window_left(1800, 1700) is None # 100s left < 120 floor
@@ -119,41 +119,9 @@ def _job(ws: Workspace, **kw) -> AgentJob:
return AgentJob(**base)
-def test_the_prompt_arrives_on_stdin_whole(tmp_ws: Workspace, agent: GeminiCliAgent):
- """No -p: a piped stdin makes gemini-cli 0.53 headless and IS the input. A 5-zone scene
- prompt runs past 2 000 lines — the window of the read_file tool the old >100 kB file stub
- relied on — and its last zone must still arrive (checked live 2026-09-22, docs/SKILLS.md)."""
- prompt = "\n".join(f"zone {i % 5} requirement line {i}" for i in range(2600)) + "\nLAST-ZONE-SENTINEL"
- res = agent.run(_job(tmp_ws, prompt=prompt))
- assert res.ok, res.errors
- assert (tmp_ws.artifacts / "stdin.txt").read_text() == prompt
-
-
-def test_a_session_killed_in_a_storm_is_booked_from_its_chat_record(tmp_ws: Workspace, agent: GeminiCliAgent,
- monkeypatch):
- """The watchdog killed it, so the CLI printed no envelope — and until 2026-09-22 the session
- was booked at $0 while its chat record held every reply's tokens (live: 6.47 M prompt tokens,
- ≈ $1.2, in one baseline). The record is the usage now, and its tool calls the tool count."""
- from codeverse3d.models.pricing import estimate_cost
-
- monkeypatch.setenv("FAKE_MODE", "storm")
- monkeypatch.setattr("codeverse3d.agents.cli_common.IDLE_GRACE_S", 1.0)
- res = agent.run(_job(tmp_ws, timeout_s=1))
- assert not res.ok and res.exit_reason == "timeout" and res.transient
- u = res.usage
- assert (u.input_tokens, u.cached_tokens, u.output_tokens, u.thoughts_tokens) == (4000, 2400, 70, 20) # g1 once
- assert u.cost_usd == pytest.approx(estimate_cost("gemini", "gemini-3.7-flash", u), abs=1e-6) and u.cost_usd > 0
- assert res.tool_calls == u.tool_calls == 1
- # the storm ran from the last reply until the kill: all of it lost to the provider
- assert 0.0 < res.provider_wait_s <= res.duration_s
- rec = json.loads((Path(res.transcript_path).parent / "result.json").read_text())
- assert rec["usage_from"] == ["chat record"]
-
-
def test_the_metered_proxy_books_a_killed_sessions_record_once(tmp_ws: Workspace, agent: GeminiCliAgent, monkeypatch,
tmp_path):
- """Filling AgentResult.usage is the whole fix: MeteredAgent writes the one session row from
- it (it wrote NONE for a $0 session) and the round's BudgetGuard is charged the same usage."""
+ """A watchdog-killed session is booked from its chat record (it was booked at $0 until 2026-09-22)."""
from codeverse3d.cost import load_ledger
from codeverse3d.cost.instrument import MeteredAgent, run_ledger
@@ -168,8 +136,7 @@ def test_the_metered_proxy_books_a_killed_sessions_record_once(tmp_ws: Workspace
def test_a_session_that_gave_up_is_booked_from_its_record_and_is_typed_transient(tmp_ws: Workspace,
agent: GeminiCliAgent, monkeypatch):
- """0.53 writes its give-up object to STDERR with no stats and exits 503 & 255 = 247: every
- gemini-cli session of the 2026-09-22 live runs that "completed a round" ended this way."""
+ """0.53's give-up: an error object on STDERR, no stats, exit 503 & 255 = 247."""
monkeypatch.setattr(get_settings(), "gemini_api_keys", ["only"])
from codeverse3d.models import gemini as gm
for sig in [x for x in list(gm._pools) if x and x[0] == "only"]: # noqa: SLF001
@@ -184,9 +151,7 @@ def test_a_session_that_gave_up_is_booked_from_its_record_and_is_typed_transient
def test_the_envelope_wins_over_the_record_and_nothing_is_booked_twice(tmp_ws: Workspace, agent: GeminiCliAgent,
monkeypatch):
- """A completed session leaves an envelope AND a chat record; the record is read (it is the
- tool trace) but its tokens are used only when the envelope has none — the metered proxy
- books AgentResult.usage once, so adding them would double the bill."""
+ """The chat record's tokens count only when the envelope has none — adding them doubles the bill."""
monkeypatch.setenv("FAKE_MODE", "ok_with_record")
res = agent.run(_job(tmp_ws))
assert res.ok and res.usage.input_tokens == 100 and res.usage.output_tokens == 20 # the envelope's, alone
@@ -196,28 +161,8 @@ def test_the_envelope_wins_over_the_record_and_nothing_is_booked_twice(tmp_ws: W
assert [r["tool"] for r in rows if r["kind"] == "tool_call"] == ["write_file"] # the record still traced
-def test_success_path(tmp_ws: Workspace, agent: GeminiCliAgent):
- res = agent.run(_job(tmp_ws))
- assert res.ok and res.exit_reason == "completed", res.errors
- assert res.text.startswith("DONE")
- assert [f.path for f in res.files_changed] == ["src/hello.txt"]
- assert res.usage.input_tokens == 100 and res.usage.output_tokens == 20 and res.usage.thoughts_tokens == 7
- assert res.usage.tool_calls == 1 and res.tool_calls == 1
- traj = Path(res.transcript_path).parent
- assert (traj / "stdout.json").exists() and (traj / "prompt.md").exists() and (traj / "result.json").exists()
- rec = json.loads((traj / "result.json").read_text())
- assert rec["attempts"] == 1 and rec["session_id"] == "s1" and rec["usage_from"] == ["envelope"]
- assert res.provider_wait_s == 0.0
- # commits: pre + agent
- assert "agent:t" in tmp_ws._git("log", "--oneline").stdout
-
-
def test_agent_planted_mcp_server_never_reaches_the_cli(tmp_ws: Workspace, agent: GeminiCliAgent):
- """``ws/.gemini/settings.json`` is agent-writable and gemini-cli merged its mcpServers,
- so an agent could choose what the NEXT round's CLI launched (`gemini mcp list` in a
- poisoned workspace tried to start it). 3dcode + ``mcp.allowed`` now live in the
- per-session system settings, which is applied LAST and whose ``mcp.allowed``
- REPLACES rather than merges (audit 2026-08-27)."""
+ """3dcode + mcp.allowed live in the per-session system settings, applied last (audit 2026-08-27)."""
from codeverse3d.agents.cli_common import begin_session, default_mcp_command, release_session
from codeverse3d.agents.materialize import materialize_workspace
@@ -249,18 +194,6 @@ def test_model_substitution_detected(tmp_ws: Workspace, agent: GeminiCliAgent, m
assert "gemini-9-pro" in res.errors[0]
-def test_transient_failure_retries_once_with_other_key(tmp_ws: Workspace, agent: GeminiCliAgent, monkeypatch):
- monkeypatch.setenv("FAKE_MODE", "fail_once")
- res = agent.run(_job(tmp_ws))
- assert res.ok, res.errors
- assert (tmp_ws.artifacts / "attempts.txt").read_text() == "2"
- rec = json.loads((Path(res.transcript_path).parent / "result.json").read_text())
- assert rec["attempts"] == 2 and (Path(res.transcript_path).parent / "stdout.2.json").exists()
- lines = [json.loads(ln) for ln in Path(res.transcript_path).read_text().splitlines()]
- keys = [ln["key_tail"] for ln in lines if ln["kind"] == "invoke"]
- assert keys == ["k1", "k2"]
-
-
def test_single_key_transient_failure_retries_same_key_and_never_raises(tmp_ws: Workspace, agent: GeminiCliAgent, monkeypatch):
"""Settings with ONE key: the retry must reuse it (no KeyPoolExhausted out of run())."""
@@ -310,10 +243,7 @@ def test_pool_exhausted_before_first_attempt_is_a_budget_result(tmp_ws: Workspac
def test_a_cli_session_holds_no_in_flight_slot(tmp_ws: Workspace, agent: GeminiCliAgent, monkeypatch):
- """The slots are machine-wide and a session is minutes long: one held per session would
- let a few dozen sessions starve every planner and judge call on the box. The session
- takes its KEY from the shared pool (rotation, cooldowns) and nothing else — so nothing
- can leak either, even when invoke raises."""
+ """The machine-wide in-flight slots are for API calls; a session takes only its key."""
from codeverse3d.agents import backends as gc
pool = gc._key_pool(get_settings().gemini_api_keys) # noqa: SLF001
@@ -329,19 +259,8 @@ def test_a_cli_session_holds_no_in_flight_slot(tmp_ws: Workspace, agent: GeminiC
assert seen == [0] and pool.stats()["in_flight"] == 0
-def test_timeout_is_reported(tmp_ws: Workspace, agent: GeminiCliAgent, monkeypatch):
- monkeypatch.setenv("FAKE_MODE", "hang")
- monkeypatch.setattr("codeverse3d.agents.cli_common.IDLE_GRACE_S", 1.0)
- res = agent.run(_job(tmp_ws, timeout_s=1))
- assert not res.ok and res.exit_reason == "timeout" and res.duration_s < 30
- assert res.transient is False # a plain hang is not a storm
-
-
def test_a_timeout_after_a_503_streak_is_marked_transient(tmp_ws: Workspace, agent: GeminiCliAgent, monkeypatch):
- """Measured 2026-09-07: 23 of 24 gemini-cli sessions of an evening ended timeout / 0 turns
- / $0 after 8-17 consecutive 503s inside the CLI's own retry loop. The wall still ends the
- session (the CLI never gives up first), but the result says WHY, so a track can fall back
- to the single-shot path instead of shipping the skeleton (tracks.common.generate_for)."""
+ """A 503 storm inside the CLI's own retry loop ends on the wall: the result says why (2026-09-07)."""
monkeypatch.setenv("FAKE_MODE", "storm")
monkeypatch.setattr("codeverse3d.agents.cli_common.IDLE_GRACE_S", 1.0)
res = agent.run(_job(tmp_ws, timeout_s=1))
@@ -352,15 +271,15 @@ def test_a_timeout_after_a_503_streak_is_marked_transient(tmp_ws: Workspace, age
def test_the_key_pool_hears_a_5xx_only_when_the_failure_was_transient(tmp_ws: Workspace, agent: GeminiCliAgent,
monkeypatch):
- """The pool is shared with every API call and a "5xx" costs the key 20 % of its health: a
- hang, a substituted model or the agent's own crash is not the KEY's fault."""
+ """A hang, a substituted model or the agent's own crash is not transient and not the key's fault."""
from codeverse3d.agents import backends as gc
monkeypatch.setattr("codeverse3d.agents.cli_common.IDLE_GRACE_S", 1.0)
pool = gc._key_pool(get_settings().gemini_api_keys) # noqa: SLF001
for mode, timeout_s in (("substitute", 20), ("crash", 20), ("hang", 1)):
monkeypatch.setenv("FAKE_MODE", mode)
- assert not agent.run(_job(tmp_ws, timeout_s=timeout_s, label=mode)).ok
+ res = agent.run(_job(tmp_ws, timeout_s=timeout_s, label=mode))
+ assert not res.ok and res.transient is False, mode
st = pool.stats()
assert (st["5xx"], st["ok"], st["error"]) == (0, 0, 0), st
monkeypatch.setenv("FAKE_MODE", "fail_once_503") # a real 503, then a rotated success
@@ -389,7 +308,7 @@ def test_parse_helpers():
def test_usage_input_is_total_prompt_and_cost_reprices_cache():
- """gemini-cli reports tokens.input = prompt - cached; pricing wants the TOTAL prompt (regression: ~4x under-billing)."""
+ """tokens.input is prompt - cached; pricing wants the TOTAL prompt (was ~4x under-billing)."""
from codeverse3d.models.pricing import estimate_cost, lookup_price
tok = {"input": 720_753, "prompt": 14_190_170, "cached": 13_469_417, "candidates": 62_012, "thoughts": 66_809}
diff --git a/harness/tests/agents/test_materialize.py b/harness/tests/agents/test_materialize.py
index f52539dc..7aafaee1 100644
--- a/harness/tests/agents/test_materialize.py
+++ b/harness/tests/agents/test_materialize.py
@@ -4,10 +4,9 @@ from __future__ import annotations
import json
import logging
-import sys
from codeverse3d.agents.cli_common import default_mcp_command
-from codeverse3d.agents.materialize import C3D_DIR, codex_mcp_overrides, materialize_workspace
+from codeverse3d.agents.materialize import C3D_DIR, materialize_workspace
from codeverse3d.workspace import Workspace
CONTRACT = "## blender contract\nWrite pure bpy into src/model.py."
@@ -24,15 +23,13 @@ def test_writes_three_bodies_same_content(tmp_ws: Workspace):
assert bodies[0] == bodies[1] == bodies[2]
body = bodies[0]
assert "ONLY under `src/` and `public/`" in body
- assert "never import from `codeverse3d`" in body.lower() or "Never import from `codeverse3d`" in body
+ assert "never import from `codeverse3d`" in body.lower()
assert CONTRACT.splitlines()[0] in body
assert "Spatial tools (3dcode)" in body
def test_bodies_document_only_this_tracks_tools(tmp_ws: Workspace):
- """AGENTS.md used to list all 19 tools + the object workflow for every track — for
- agy (no MCP) this file is the ONLY tool documentation and its shell fallback is
- unscoped, so scene/graphics agents were taught dead-end object-GLB tools (V11a)."""
+ """Scene/graphics agents were once taught dead-end object-GLB tools (V11a)."""
tmp_ws.spec_path.write_text(json.dumps({"id": "t", "track": "scene", "language": "scene_threejs", "prompt": "p"}))
_mat(tmp_ws, kind="agy")
body = (tmp_ws.root / "AGENTS.md").read_text()
@@ -66,9 +63,7 @@ def test_no_mcp_server_is_written_into_the_workspace(tmp_ws: Workspace):
def test_ignore_files_keep_agent_facing_paths_readable(tmp_ws: Workspace):
- """gemini-cli refuses read_file on ignored paths: the build census and tool renders must NOT
- be ignored. trajectories/ is, whole: the prompt reaches every CLI on stdin since 2026-09-22,
- so nothing there is the agent's to read."""
+ """gemini-cli refuses read_file on ignored paths: census and tool renders stay readable."""
import fnmatch
from codeverse3d.agents.materialize import IGNORE_LINES
@@ -113,10 +108,7 @@ def test_cookbook_copied_when_found(tmp_ws: Workspace, caplog):
def test_missing_cookbook_warns_where_someone_can_see_it(tmp_ws: Workspace, caplog):
- """The unresolved cookbook is the failure the cookbook path lookup (`cookbook_rel_for`, gone)
- was fixed for (an articulated run told the agent "No cookbook is available" while its 24 kB cookbook sat
- on disk). It used to land in a `Materialized.warnings` list every caller threw away; the
- log line is the whole signal now, so it has to fire."""
+ """The log line is the only signal of an unresolved cookbook."""
with caplog.at_level(logging.WARNING, logger="codeverse3d.agents.materialize"):
_mat(tmp_ws, cookbook="")
assert any("cookbook not found" in r.getMessage() for r in caplog.records)
@@ -128,21 +120,3 @@ def test_kind_specific_tool_hint(tmp_ws: Workspace):
assert "mcp__3dcode__" in (tmp_ws.root / "CLAUDE.md").read_text()
_mat(tmp_ws, kind="agy")
assert "codeverse3d.cli.main tools" in (tmp_ws.root / "AGENTS.md").read_text()
-
-
-def test_codex_overrides_are_valid_toml_fragments():
- ov = codex_mcp_overrides(["python", "-m", "x", "--workspace", "/a b/c"])
- assert ov[0] == "-c" and ov[1] == 'mcp_servers.3dcode.command="python"'
- assert ov[3] == 'mcp_servers.3dcode.args=["-m", "x", "--workspace", "/a b/c"]'
- assert ov[::2] == ["-c"] * (len(ov) // 2)
- keys = {kv.split("=", 1)[0]: kv.split("=", 1)[1] for kv in ov[1::2]}
- # codex exec cannot answer the per-tool approval elicitation: without this every 3dcode call is cancelled
- assert keys["mcp_servers.3dcode.default_tools_approval_mode"] == '"approve"'
-
-
-def test_default_mcp_command_is_the_backends_interpreter(tmp_ws: Workspace):
- # one place knows the command: the body + codex overrides quote sys.executable, never bare "python"
- materialize_workspace(tmp_ws, agent_kind="codex", contract_md="c", cookbook_text="", spatial_tools=True)
- body = (tmp_ws.root / "AGENTS.md").read_text()
- assert default_mcp_command(tmp_ws)[0] == sys.executable
- assert f"command: `{' '.join(default_mcp_command(tmp_ws))}`" in body
diff --git a/harness/tests/agents/test_tool_trace.py b/harness/tests/agents/test_tool_trace.py
index 10802591..94e43b87 100644
--- a/harness/tests/agents/test_tool_trace.py
+++ b/harness/tests/agents/test_tool_trace.py
@@ -1,16 +1,8 @@
-"""Every CLI backend folds its session's tool calls into ``transcript.jsonl`` — the skill
-read probe's ground truth (docs/SKILLS.md §4, 2026-09-22).
-
-The fixtures are excerpts of the four live sessions recorded 2026-09-22 (claude-code
-2.1.280 stream-json, gemini-cli 0.53.0 chat record, codex 0.155.1 ``exec --json``, agy
-1.2.2 conversation database), trimmed to the fields the parsers read. A vendor that
-renames one of them breaks a test here instead of silently reporting "read nothing".
-"""
+"""Each CLI backend folds its tool calls into transcript.jsonl; fixtures are trimmed live sessions (2026-09-22)."""
from __future__ import annotations
import json
-import os
import sqlite3
import time
from pathlib import Path
@@ -19,10 +11,8 @@ from codeverse3d.agents.backends import (
SYSTEM_SETTINGS,
ClaudeCodeAgent,
ClaudeStream,
- CodexEvents,
agy_tool_calls,
gemini_chat_records,
- parse_claude_json,
read_gemini_chats,
write_system_settings,
)
@@ -86,17 +76,9 @@ def test_claude_stream_folds_the_index_the_skill_calls_and_their_failures():
assert len(s.calls) == 3
-def test_the_stream_json_envelope_is_the_result_line_and_an_unfinished_stream_has_none():
- assert parse_claude_json(_stream_text(CLAUDE_STREAM))["num_turns"] == 25
- assert parse_claude_json(_stream_text(CLAUDE_STREAM[:-1])) is None # killed before its result
- single = json.dumps({"num_turns": 2, "result": "ok"}) # one document still parses
- assert parse_claude_json(single)["num_turns"] == 2
-
-
def test_claude_is_launched_streaming(tmp_ws: Workspace):
s = begin_session(AgentJob(workspace=str(tmp_ws.root), prompt="p", spatial_tools=False), "claude-code")
argv = ClaudeCodeAgent("sonnet", binary="claude").build_argv(s)
- assert argv[argv.index("--output-format") + 1] == "stream-json" and "--verbose" in argv
assert "Skill" in argv[argv.index("--allowedTools") + 1].split(",")
assert "--disable-slash-commands" not in argv # claude-code: "Disable all skills"
@@ -147,7 +129,7 @@ GEMINI_RECORD = [
]
-def _gemini_home(tmp_path: Path, ws: Path, *, since_offset: float = 0.0) -> Path:
+def _gemini_home(tmp_path: Path, ws: Path) -> Path:
home = tmp_path / "home"
proj = home / ".gemini" / "tmp" / ws.name
(proj / "chats").mkdir(parents=True)
@@ -158,9 +140,6 @@ def _gemini_home(tmp_path: Path, ws: Path, *, since_offset: float = 0.0) -> Path
(other / "chats" / "session-2026-09-22T19-31-aaaaaaaa.jsonl").write_text(json.dumps(GEMINI_RECORD[2]) + "\n")
rec = proj / "chats" / "session-2026-09-22T19-31-da5c91db.jsonl"
rec.write_text("".join(json.dumps(r) + "\n" for r in GEMINI_RECORD))
- if since_offset:
- t = time.time() + since_offset
- os.utime(rec, (t, t))
return home
@@ -180,81 +159,13 @@ def test_gemini_chat_record_is_found_by_workspace_and_time_and_folded_by_call_id
def test_gemini_system_settings_pin_skills_on_and_keep_the_workspace_trusted(tmp_path):
- """0.53 merges the system file LAST, so a user's (or an agent-planted workspace)
- ``skills.enabled: false`` cannot hide the bundles; ``folderTrust`` off is what makes
- ``isTrustedFolder()`` true, without which ``discoverSkills`` skips both workspace roots."""
+ """0.53 merges the system file last; folderTrust off makes discoverSkills read the workspace roots."""
data = json.loads(write_system_settings(tmp_path / "s.json").read_text())
assert data["skills"]["enabled"] is True and data["skills"] == SYSTEM_SETTINGS["skills"]
assert data["security"]["folderTrust"] == {"enabled": False}
assert data["security"]["auth"]["selectedType"] == "gemini-api-key"
assert data["experimental"]["dynamicModelConfiguration"] is True # else 0.53 substitutes models
-
-
-def test_gemini_sessions_see_the_routed_bundles_and_not_the_clis_own(tmp_path):
- """0.53 lists its two built-in skills (bundle/builtin/) to the model next to the routed
- bundles — seen in the request body a local fake API received, 2026-09-22 — and
- ``skills.disabled`` (by name) is the only switch; the fake API then saw neither."""
- data = json.loads(write_system_settings(tmp_path / "s.json").read_text())
- assert data["skills"]["disabled"] == ["skill-creator", "antigravity-support"]
-
-
-def test_the_chat_record_is_the_usage_of_a_session_that_printed_no_envelope(tmp_path):
- """Every reply once (the record re-writes a message as it progresses), its tokens in the
- envelope's ``stats.models`` shape — the sum reproduced two live envelopes to the token."""
- ws = tmp_path / "gemini1"
- ws.mkdir()
- home = _gemini_home(tmp_path, ws)
- rec = next((home / ".gemini" / "tmp" / ws.name / "chats").glob("session-*"))
- tok = {"input": 5000, "output": 40, "cached": 3000, "thoughts": 9, "tool": 0, "total": 5049}
- with rec.open("a") as f:
- for row in ({"id": "fb86", "type": "gemini", "model": "gemini-3.7-flash", "timestamp": "2026-09-22T19:31:40.000Z",
- "content": "", "tokens": tok},
- {"id": "fb86", "type": "gemini", "model": "gemini-3.7-flash", "timestamp": "2026-09-22T19:31:41.000Z",
- "content": "", "tokens": tok}, # the same reply, re-written
- {"id": "fb99", "type": "gemini", "model": "gemini-3.7-flash", "content": ""}): # died mid-reply
- f.write(json.dumps(row) + "\n")
- chat = read_gemini_chats(gemini_chat_records(ws, time.time() - 60, {"GEMINI_CLI_HOME": str(home)}))
- assert chat.stats() == {"models": {"gemini-3.7-flash": {"tokens": {"prompt": 5000, "cached": 3000,
- "candidates": 40, "thoughts": 9}}}}
- assert len(chat.times) == 2 and chat.times[1] - chat.times[0] == 1.0
-
-
-# --------------------------------------------------------------------------- codex
-CODEX_EVENTS = [
- {"type": "thread.started", "thread_id": "01a0ca9a"},
- {"type": "turn.started"},
- {"type": "item.completed", "item": {"id": "item_0", "type": "agent_message",
- "text": "I'm using the chair-relevant 3D skills"}},
- # the model expanded the r1 skill-root alias as r0 (the system root): the read failed
- {"type": "item.completed", "item": {
- "id": "item_1", "type": "command_execution", "status": "failed", "exit_code": 2,
- "command": "/bin/bash -lc \"sed -n '1,240p' AGENTS.md && sed -n '1,240p' "
- "/home/u/.codex/skills/.system/c3d-blender-forms/SKILL.md\"",
- "aggregated_output": "# 3dcode workspace — rules for the coding agent ..."}},
- {"type": "item.completed", "item": {
- "id": "item_3", "type": "command_execution", "status": "completed", "exit_code": 0,
- "command": "/bin/bash -lc \"sed -n '1,280p' .agents/skills/c3d-blender-forms/SKILL.md && "
- "sed -n '1,260p' .agents/skills/c3d-part-contact/SKILL.md\"",
- "aggregated_output": "--- name: c3d-blender-forms ..."}},
- {"type": "item.completed", "item": {"id": "item_6", "type": "file_change", "status": "completed",
- "changes": [{"path": f"{WS}/src/model.py", "kind": "add"}]}},
- {"type": "item.completed", "item": {"id": "item_7", "type": "mcp_tool_call", "server": "3dcode",
- "tool": "build", "arguments": {}, "status": "completed"}},
- {"type": "turn.completed", "usage": {"input_tokens": 2292081, "cached_input_tokens": 2196992,
- "output_tokens": 18551, "reasoning_output_tokens": 7544}},
-]
-
-
-def test_codex_events_keep_what_was_run_and_whether_it_worked():
- ev = CodexEvents()
- for e in CODEX_EVENTS:
- ev.feed(json.dumps(e))
- assert [(c.tool, c.failed) for c in ev.calls] == [
- ("command_execution", True), ("command_execution", False), ("file_change", False), ("mcp_tool_call", False)]
- assert "aggregated_output" not in ev.calls[0].args # what ran, never what came back
- assert ev.calls[1].args["command"].count("SKILL.md") == 2
- assert ev.calls[3].args == {"server": "3dcode", "tool": "build", "arguments": {}}
- assert ev.tool_calls == 4
+ assert data["skills"]["disabled"] == ["skill-creator", "antigravity-support"] # the CLI's own bundles
# --------------------------------------------------------------------------- agy
diff --git a/harness/tests/agents/test_watchdog.py b/harness/tests/agents/test_watchdog.py
index 88913984..bc54d1e2 100644
--- a/harness/tests/agents/test_watchdog.py
+++ b/harness/tests/agents/test_watchdog.py
@@ -1,9 +1,4 @@
-"""run_with_watchdog's CLOCKS: idle kill, activity extension, hard timeout.
-
-The process lifecycle underneath (pipes, stdin writer thread, group kill,
-KeyboardInterrupt cleanup, drain/reap) is ``proc.ManagedProcess`` and is owned by
-tests/core/test_proc.py; this module keeps one integration smoke plus the clocks.
-"""
+"""run_with_watchdog's clocks; the process lifecycle underneath is tests/core/test_proc.py's."""
from __future__ import annotations
@@ -14,20 +9,9 @@ from pathlib import Path
import pytest
from codeverse3d.agents.cli_common import run_with_watchdog
-from tests.conftest import assert_pid_gone
PY = sys.executable
-#: spawns a grandchild (pid to argv[1]), prints nothing, then hangs — the shape that
-#: proves a timeout kill takes the whole session, not just the leader.
-_SPAWN_THEN_HANG = (
- "import subprocess, sys, time\n"
- "p = subprocess.Popen(['sleep', '300'])\n"
- "open(sys.argv[1], 'w').write(str(p.pid))\n"
- "time.sleep(300)\n"
-)
-
-
def test_normal_run_streams_lines_and_delivers_stdin(tmp_path: Path):
"""The integration smoke: stdin reaches the child, every line reaches ``on_line``
and the bounded texts, and stdout/stderr stay separate."""
@@ -43,26 +27,6 @@ def test_normal_run_streams_lines_and_delivers_stdin(tmp_path: Path):
assert ("stdout", "HELLO") in seen and ("stderr", "b") in seen
-def test_idle_kill_after_soft_timeout_takes_the_whole_group(tmp_path: Path):
- """Silent past soft+grace → ``killed_reason="idle"``, and the grandchild dies with
- it (``start_new_session`` means only a group kill can reach it)."""
- marker = tmp_path / "child.pid"
- t0 = time.time()
- res = run_with_watchdog([PY, "-c", _SPAWN_THEN_HANG, str(marker)], cwd=tmp_path, env=None,
- soft_timeout_s=1, idle_grace_s=1, hard_timeout_s=30, poll_s=0.2)
- assert res.timed_out and res.killed_reason == "idle"
- assert time.time() - t0 < 15
- assert_pid_gone(int(marker.read_text()))
-
-
-def test_activity_extends_past_soft_timeout(tmp_path: Path):
- # prints every 0.3s for ~2.5s: soft=1, grace=1 → must NOT be killed (activity keeps it alive)
- code = "import time,sys\nfor i in range(8):\n print(i); sys.stdout.flush(); time.sleep(0.3)\nprint('done')"
- res = run_with_watchdog([PY, "-c", code], cwd=tmp_path, env=None, soft_timeout_s=1, idle_grace_s=1,
- hard_timeout_s=30, poll_s=0.2)
- assert not res.timed_out and res.stdout.strip().endswith("done")
-
-
def test_hard_timeout_kills_active_process(tmp_path: Path):
code = "import time,sys\nwhile True:\n print('tick'); sys.stdout.flush(); time.sleep(0.2)"
t0 = time.time()
@@ -88,12 +52,10 @@ def test_file_mtime_counts_as_activity(tmp_path: Path):
@pytest.mark.timeout(60, method="thread")
def test_child_that_never_reads_a_big_stdin_still_times_out(tmp_path: Path):
- """The prompt used to be written synchronously before the poll loop: a child that
- never reads >64 KiB of stdin blocked ``stdin.write`` forever and BOTH clocks were
- dead (codex pipes prompts over 100 kB this way). 1 MB in, must still die."""
+ """A child that never reads >64 KiB of stdin once blocked stdin.write and both clocks."""
t0 = time.monotonic()
res = run_with_watchdog([PY, "-c", "import time; time.sleep(300)"], cwd=tmp_path, env=None,
soft_timeout_s=0.5, idle_grace_s=0.5, hard_timeout_s=8, poll_s=0.2,
stdin="x" * 1_000_000)
- assert res.timed_out and res.killed_reason in ("idle", "hard_timeout")
+ assert res.timed_out and res.killed_reason == "idle"
assert time.monotonic() - t0 < 30
diff --git a/harness/tests/flywheel_cli/test_captions.py b/harness/tests/flywheel_cli/test_captions.py
index 2cc73e18..4f90ea3d 100644
--- a/harness/tests/flywheel_cli/test_captions.py
+++ b/harness/tests/flywheel_cli/test_captions.py
@@ -24,36 +24,19 @@ GOOD = {
}
-def test_caption_sample_stores_and_writes(fake_run):
- from codeverse3d.addons import select
- from codeverse3d.record.deliverable import load_deliverable
-
+def test_caption_sample_captions_the_picked_round_and_records_provenance(fake_run):
ws, rec = fake_run
- select.package(ws.root, 1) # a round was handed over: its deliverable/ must follow the caption
m = FakeChatModel([GOOD])
- caps = caption_sample(ws, rec, "fake:fake", model=m)
- assert caps.detailed.startswith("A four-legged")
+ caption_sample(ws, rec, "fake:fake", model=m)
req = m.requests[0]
- assert req.response_schema is not None and "Blender Python script" in req.system
- imgs = [p for p in req.messages[0].parts if isinstance(p, ImagePart)]
- assert len(imgs) == 3 # sheet + 2 views
- assert "round 1" in req.messages[0].text # best-round code
- assert (ws.root / "captions.json").is_file()
- again = load_record(ws)
- assert again.extra["captions"]["factory"] == GOOD["factory"]
- assert again.extra["captions"]["provenance"]["captioner"] == "fake:fake"
- assert again.extra["captions"]["provenance"]["cost_usd"] == pytest.approx(0.002)
- # a caption changes the hand-over folder too — it used to leave deliverable/,
- # the manifest and telemetry/ describing an uncaptioned run
- assert json.loads((ws.deliverable / "captions.json").read_text())["factory"] == GOOD["factory"]
- handed = load_deliverable(ws)
- assert handed is not None and handed.round == 1 and again.telemetry is not None
- assert "deliverable/captions.json" in {f.path for f in handed.files}
+ assert "Blender Python script" in req.system and "round 1" in req.messages[0].text # best-round code
+ assert len([p for p in req.messages[0].parts if isinstance(p, ImagePart)]) == 3 # sheet + 2 views
+ prov = load_record(ws).extra["captions"]["provenance"]
+ assert prov["captioner"] == "fake:fake" and prov["cost_usd"] == pytest.approx(0.002)
def test_caption_cmd_writes_the_captioner_row_into_the_run_ledger(fake_run, monkeypatch):
- """`3dcode flywheel caption` joins the run's ledger the way `3dcode judge` does —
- the captioner's priced call used to go to the per-process log instead."""
+ """The captioner's priced call joins the run's ledger."""
from typer.testing import CliRunner
import codeverse3d.models.registry as R
@@ -86,15 +69,6 @@ def test_caption_retry_then_fail(fake_run):
caption_sample(ws, rec, "fake:fake", model=m2)
-def test_validate_rules():
- from codeverse3d.contracts.common import Language
-
- c = Captions(**GOOD)
- assert validate_captions(c, Language.BLENDER) == []
- c2 = Captions(**dict(GOOD, instruction="Please model a wooden chair with four legs for me."))
- assert any("target language" in p for p in validate_captions(c2, Language.BLENDER))
-
-
@pytest.mark.live
def test_caption_live_gemini(fake_run):
from codeverse3d.config import get_settings
@@ -106,81 +80,29 @@ def test_caption_live_gemini(fake_run):
assert len(caps.detailed) > 20 and "blender" in caps.instruction.lower()
-# --------------------------------------------------------------------------- finding: Three.js phrase vs forbidden THREE.
-GOOD_JS = {
- "detailed": "A desk lamp with a round base, an arched arm and a conical shade.",
- "instruction": "Write a Three.js module that builds a desk lamp with a round base and conical shade.",
- "factory": "Build the base cylinder, sweep the arm along an arc, add the cone shade, then merge the groups.",
-}
-
-
-def test_threejs_instruction_naming_threejs_is_valid():
- """The required 'Three.js' phrase must NOT trip the forbidden-API check (both js languages)."""
- from codeverse3d.addons.dataset.captions import Captions, validate_captions
+GOOD_JS = dict(GOOD, instruction="Write a Three.js module that builds a desk lamp with a round base and conical shade.",
+ factory="Build the base cylinder, sweep the arm along an arc, add the cone shade, then merge the groups.")
+SHADER = dict(GOOD, instruction="Write a GLSL fragment shader showing neon rain running down a window at night.",
+ factory="Layer a bokeh background pass, a grid-cell rain pass, then grade with a vignette.")
+
+
[email protected]("fields, language, problem", [
+ (GOOD, "blender", None),
+ (dict(GOOD, instruction="Please model a wooden chair with four legs for me."), "blender", "target language"),
+ (GOOD_JS, "threejs", None), # the required 'Three.js' phrase is not the forbidden API
+ (GOOD_JS, "scene_threejs", None),
+ (dict(GOOD_JS, factory="Build a THREE.Group holding THREE.Mesh boxes for every part."), "threejs", "THREE."),
+ (SHADER, "glsl_shader", None),
+ (dict(SHADER, instruction="Write a shader with rain."), "glsl_shader", "target language"),
+ (dict(SHADER, instruction="Write an OpenGL Python program showing neon rain on a window."), "opengl_python", None),
+ (dict(SHADER, instruction="Write an OpenGL Python program showing neon rain on a window.",
+ factory="Uses moderngl FBOs for the feedback pass."), "opengl_python", "moderngl"),
+])
+def test_validate_captions(fields, language, problem):
from codeverse3d.contracts.common import Language
- c = Captions(**GOOD_JS)
- assert validate_captions(c, Language.THREEJS) == []
- assert validate_captions(c, Language.SCENE_THREEJS) == []
- # the API namespace itself stays forbidden (case-sensitive THREE.<Symbol>)
- bad = Captions(**dict(GOOD_JS, factory="Build a THREE.Group holding THREE.Mesh boxes for every part."))
- probs = validate_captions(bad, Language.THREEJS)
- assert probs and "THREE." in probs[0]
- # and omitting the platform name still fails the phrase check
- off = Captions(**dict(GOOD_JS, instruction="Write a module that builds a desk lamp with a round base."))
- assert any("target language" in p for p in validate_captions(off, Language.THREEJS))
-
-
-def test_caption_sample_threejs_run(tmp_path):
- """End to end: a threejs run captions successfully with a 'Three.js' instruction."""
- from codeverse3d.contracts.common import Language
- from tests.flywheel_cli.conftest import make_fake_run
-
- ws, rec = make_fake_run(tmp_path / "runs", "lamp_js", prompt="a desk lamp", language=Language.THREEJS)
- m = FakeChatModel([GOOD_JS])
- caps = caption_sample(ws, rec, "fake:fake", model=m)
- assert "Three.js" in caps.instruction and len(m.requests) == 1 # no retry needed
-
-
-def test_caption_graphics_and_scene_phrases():
- """Graphics languages have phrases + phrase words (no KeyError, sane rules)."""
- from codeverse3d.addons.dataset.captions import Captions, validate_captions
- from codeverse3d.contracts.common import Language
-
- shader = Captions(
- detailed="Neon rain streaks down a dark window while blurred city lights pulse behind the glass.",
- instruction="Write a GLSL fragment shader showing neon rain running down a window at night.",
- factory="Layer a bokeh background pass, a grid-cell rain pass with per-cell offsets, then grade with a vignette.",
- )
- assert validate_captions(shader, Language.GLSL_SHADER) == []
- assert any("target language" in p
- for p in validate_captions(shader.model_copy(update={"instruction": "Write a shader with rain."}),
- Language.GLSL_SHADER))
- gl = shader.model_copy(update={"instruction": "Write an OpenGL Python program showing neon rain on a window."})
- assert validate_captions(gl, Language.OPENGL_PYTHON) == []
- leaked = shader.model_copy(update={"factory": "Uses moderngl FBOs for the feedback pass."})
- assert any("moderngl" in p for p in validate_captions(leaked, Language.OPENGL_PYTHON))
-
-
-def test_caption_graphics_run_from_frames(tmp_path):
- """(f) graphics runs (no GLB) caption from the sheet/frames renders."""
- from codeverse3d.contracts.common import Language
- from tests.flywheel_cli.conftest import make_fake_run
-
- ws, rec = make_fake_run(tmp_path / "runs", "rain_glsl", prompt="neon rain", language=Language.GLSL_SHADER)
- reply = {
- "detailed": "Bright neon streaks slide down a dark pane while soft coloured discs drift behind it.",
- "instruction": "Write a GLSL fragment shader with neon rain streaking down a dark window.",
- "factory": "Hash-place bokeh discs in three depth layers, add per-cell rain trails, then tonemap and vignette.",
- }
- m = FakeChatModel([reply])
- caps = caption_sample(ws, rec, "fake:fake", model=m)
- assert caps.instruction.lower().count("glsl")
- req = m.requests[0]
- from codeverse3d.contracts.chat import ImagePart as IP
-
- imgs = [p for p in req.messages[0].parts if isinstance(p, IP)]
- assert len(imgs) == 3 # sheet + 2 frame views
+ probs = validate_captions(Captions(**fields), Language(language))
+ assert (probs == []) if problem is None else any(problem in p for p in probs), probs
# --------------------------------------------------------------------------- a run recorded before artifacts/rNN/
@@ -205,9 +127,7 @@ def _old_urdf_run(tmp_path):
def test_an_old_runs_caption_then_export_still_ships_its_glb_gif_and_meshes(tmp_path):
- """`flywheel caption` rewrote record.json through RunRecord, which dropped the raw `best_round`
- — the only pointer to the round artifacts/ holds — so the re-packaged deliverable and the
- export lost the old run's object.glb, preview.gif and meshes/."""
+ """The caption's record rewrite once dropped the raw best_round, and with it the old run's outputs."""
from codeverse3d.addons.dataset.export import export_one
ws, rec = _old_urdf_run(tmp_path)
diff --git a/harness/tests/flywheel_cli/test_cli.py b/harness/tests/flywheel_cli/test_cli.py
index 041ae672..851d7aac 100644
--- a/harness/tests/flywheel_cli/test_cli.py
+++ b/harness/tests/flywheel_cli/test_cli.py
@@ -78,15 +78,6 @@ def test_make_no_run_creates_valid_workspace(tmp_path: Path):
assert r3.exit_code == 0
-def test_status_on_fake_run(runs_dir: Path):
- r = runner.invoke(app, ["status", "wooden_chair_ab12cd34", "--runs-dir", str(runs_dir)])
- assert r.exit_code == 0, r.output
- assert "passed" in r.output and "0.800" in r.output and "rounds" in r.output
- assert "1*" in r.output # best round marked
- r = runner.invoke(app, ["status", "nope", "--runs-dir", str(runs_dir)])
- assert r.exit_code == 1
-
-
def test_flywheel_commands(runs_dir: Path, tmp_path: Path):
out = tmp_path / "ds"
r = runner.invoke(app, ["flywheel", "export", str(runs_dir), str(out), "--pack"])
@@ -127,7 +118,6 @@ def test_doctor_json(monkeypatch):
]
-# --------------------------------------------------------------------------- finding: `3dcode judge` inputs (main.py:204)
class _FakeVlm:
captured: dict = {}
@@ -203,26 +193,8 @@ def test_judge_wraps_value_error_as_cli_error(runs_dir: Path, monkeypatch):
assert "judge failed" in out and "Traceback" not in r.output
-def test_judge_rubric_map_includes_graphics():
- from codeverse3d.cli._judge import rubric_for
- from codeverse3d.contracts.common import TRACK_INFO, Track
- from codeverse3d.contracts.run import RoundRecord
-
- assert TRACK_INFO[Track.GRAPHICS].rubric == "shader_v2"
-
- class _R: # minimal record stub
- class spec:
- track = Track.GRAPHICS
-
- rnd = RoundRecord(index=0, kind="baseline")
- assert rubric_for(_R, rnd, None) == "shader_v2"
- assert rubric_for(_R, rnd, "asset_v1") == "asset_v1"
-
-
def test_calibration_rubric_map_includes_graphics(tmp_path: Path):
- """Same default as `3dcode judge`: a graphics round with no stored judgment used to be
- re-judged with static_object_v1 because calibration kept its own three-track
- TRACK_RUBRIC instead of reading TRACK_INFO."""
+ """A graphics round with no stored judgment is re-judged with the track's rubric, not static_object_v1."""
from codeverse3d.addons.calibration import load_run_cases
from codeverse3d.contracts.artifacts import RenderSet, RenderView
from codeverse3d.contracts.common import Language, Track
@@ -238,27 +210,6 @@ def test_calibration_rubric_map_includes_graphics(tmp_path: Path):
assert [c.rubric for c in load_run_cases(run)] == ["shader_v2"]
-# --------------------------------------------------------------------------- make --texture / status extras / render graphics
-def test_make_texture_flag_sets_spec_options(tmp_path: Path):
- runs = tmp_path / "runs"
- r = runner.invoke(app, ["make", "a clay pot", "--runs-dir", str(runs), "--no-run", "--texture", "--candidates", "3"])
- assert r.exit_code == 0, r.output
- ws = next(d for d in runs.iterdir() if d.is_dir() and not d.name.startswith("."))
- spec = Spec.model_validate_json((ws / "spec.json").read_text())
- # run-shape options are frozen on the spec (no more magic 'texture' tag)
- assert spec.options.texture is True and spec.options.candidates == 3
- assert "texture" not in spec.tags
-
-
-def test_make_language_defaults_to_the_tracks_first(tmp_path: Path):
- runs = tmp_path / "runs"
- r = runner.invoke(app, ["make", "a harbour at night", "--track", "scene", "--runs-dir", str(runs), "--no-run"])
- assert r.exit_code == 0, r.output
- ws = next(d for d in runs.iterdir() if d.is_dir() and not d.name.startswith("."))
- spec = Spec.model_validate_json((ws / "spec.json").read_text())
- assert spec.language.value == "scene_threejs"
-
-
def test_make_invalid_combo_leaves_no_orphan_workspace(tmp_path: Path):
runs = tmp_path / "runs"
r = runner.invoke(app, ["make", "x", "--track", "scene", "--language", "blender",
@@ -295,14 +246,14 @@ def test_resume_refuses_a_finished_run_and_never_re_enters_it(made_run, stub_tra
assert RunState.load(ws).status is RunStatus.STOPPED
entered = []
- stub_track(lambda spec, resume, force: entered.append(resume))
+ stub_track(lambda spec, resume, force: entered.append((resume, force)))
r = runner.invoke(app, ["resume", ws.root.name, "--runs-dir", str(runs)])
assert r.exit_code == 1 and "already finished" in r.output and "stop_reason='pass'" in r.output
assert entered == [], "the pipeline must not be re-entered"
assert RunState.load(ws).status is RunStatus.STOPPED, "the terminal state must survive"
# ... and the escape hatch still works, loudly
assert runner.invoke(app, ["resume", ws.root.name, "--runs-dir", str(runs), "--force"]).exit_code == 130
- assert entered == [True]
+ assert entered == [(True, True)]
# every round ran: raising --rounds is how a max_rounds stop goes on (from its last round)
RunState(status=RunStatus.MAX_ROUNDS, stop_reason="max_rounds").save(ws)
@@ -321,17 +272,6 @@ def test_resume_refuses_a_finished_run_and_never_re_enters_it(made_run, stub_tra
assert runner.invoke(app, ["resume", ws.root.name, "--runs-dir", str(runs)]).exit_code == 130
-def test_resume_threads_force_through_to_the_track(made_run, stub_track):
- """The CLI forwards resume's force flag to the track."""
- runs, ws = made_run()
- seen = {}
- stub_track(lambda spec, resume, force: seen.update(resume=resume, force=force))
- assert runner.invoke(app, ["resume", ws.name, "--runs-dir", str(runs)]).exit_code == 130
- assert seen == {"resume": True, "force": False}
- assert runner.invoke(app, ["resume", ws.name, "--runs-dir", str(runs), "--force"]).exit_code == 130
- assert seen == {"resume": True, "force": True}
-
-
def test_a_spec_change_refusal_is_a_clean_cli_error(made_run, stub_track):
"""Spec drift is a typed CLI refusal, not a traceback."""
from codeverse3d.tracks.lifecycle import SpecChanged
@@ -438,15 +378,6 @@ def test_a_negative_budget_is_rejected_before_the_workspace_exists(tmp_path: Pat
r = runner.invoke(app, ["make", "a chair", "--max-minutes", "0", "--no-run",
"--runs-dir", str(runs), "--slug", "zero"])
assert r.exit_code == 0 and (runs / "zero" / "spec.json").is_file()
- # ... and the contract refuses a negative ceiling even when built directly
- import pytest as _pytest
- from pydantic import ValidationError
-
- from codeverse3d.contracts.common import Budget
-
- with _pytest.raises(ValidationError):
- Budget(max_minutes=-3.0)
- assert Budget(max_minutes=0.0).max_minutes == 0.0
def test_texture_is_not_offered_on_tracks_that_have_no_glb(tmp_path: Path):
@@ -484,8 +415,7 @@ def test_texture_is_not_offered_on_tracks_that_have_no_glb(tmp_path: Path):
def test_best_of_n_is_not_offered_on_a_scene(tmp_path: Path):
- """F1: a scene writes its baseline in its stages, so --candidates N was a silent no-op that
- recorded n_candidates=N. Explicit: refused. Profile-implied: 1. The record: the width that ran."""
+ """F1: explicit --candidates N on a scene is refused, a profile's is 1, a stale spec runs one."""
from codeverse3d.config import get_settings
from codeverse3d.contracts.common import Track
from codeverse3d.tracks.scene import SceneTrack
@@ -524,8 +454,7 @@ def _round_guard_ws(tmp_path: Path, *, tree: int, best: int | None = None):
def test_render_only_labels_the_working_tree_round(tmp_path: Path):
- """The tree ends at the last round (2026-09-22), so a plain render IS that round; a
- --round that names another one would label another round's code, and is refused."""
+ """A plain render is the working tree's round; a --round naming another one is refused."""
from codeverse3d.cli._common import CliError
from codeverse3d.cli.inspect_cmd import _render_round_or_refuse
diff --git a/harness/tests/flywheel_cli/test_complexity_record.py b/harness/tests/flywheel_cli/test_complexity_record.py
index 7cceac23..c6451cff 100644
--- a/harness/tests/flywheel_cli/test_complexity_record.py
+++ b/harness/tests/flywheel_cli/test_complexity_record.py
@@ -68,9 +68,3 @@ def test_record_carries_the_last_rounds_complexity_and_the_trail() -> None:
def test_complexity_block_falls_back_to_the_last_measured_round() -> None:
rec = _record([_round(0, 0.6, 0.42), _round(1, 0.5, None)])
assert complexity_block(rec)["index"] == 0.42
-
-
-def test_no_measurement_means_no_block() -> None:
- rec = fill_derived(_record([_round(0, 0.6, None)]))
- assert "complexity" not in rec.extra
- assert round_summary(rec.rounds[0])["complexity"] is None
diff --git a/harness/tests/flywheel_cli/test_flywheel.py b/harness/tests/flywheel_cli/test_flywheel.py
index 75500a9f..bbcef03a 100644
--- a/harness/tests/flywheel_cli/test_flywheel.py
+++ b/harness/tests/flywheel_cli/test_flywheel.py
@@ -2,23 +2,14 @@
from __future__ import annotations
-import io
import json
-import sqlite3
-import tarfile
from pathlib import Path
import pytest
from codeverse3d.addons import select
from codeverse3d.addons.dataset.export import export_samples
-from codeverse3d.addons.dataset.index import build_index, summary
-from codeverse3d.addons.dataset.pack import pack_samples, verify_locators
from codeverse3d.addons.dataset.pairs import build_pairs
-from codeverse3d.addons.dataset.quality import (
- code_fingerprint,
- normalise_code,
-)
from codeverse3d.record import _git
from codeverse3d.record.record import (
RecordError,
@@ -70,14 +61,7 @@ def test_git_tree_at_commit(fake_run):
def test_a_planted_diff_driver_never_runs(fake_run, tmp_path):
- """``.git/config`` and ``.gitattributes`` in a run workspace were writable by the
- agent, and a git diff driver is a program git RUNS.
-
- ``--no-ext-diff`` does not cover textconv: with only that flag, a ``.gitattributes``
- entry ``*.bin diff=evil`` plus ``[diff "evil"] textconv = sh -c …`` executes the
- command while the exporter merely reads the repository (reproduced on git 2.34).
- ``-c`` cannot unset a LOCAL named driver, so the flag is the fix, not the config.
- """
+ """An agent-planted diff driver or textconv (a program git RUNS) never runs on the export read."""
ws, _rec = fake_run
fired = tmp_path / "fired"
payload = f"sh -c 'echo pwned >> {fired}; cat'"
@@ -99,10 +83,7 @@ def test_a_planted_diff_driver_never_runs(fake_run, tmp_path):
def test_a_symlink_is_not_exported_as_a_file_of_its_target(fake_run) -> None:
- """``git archive`` skipped symlinks (a tar member, not a file); ``ls-tree`` lists one
- as a blob whose content IS the link target, so ``src/link.py -> model.py`` came back
- as a one-line file saying ``model.py`` — and with ``--with-code`` that goes into a
- training sample."""
+ """ls-tree lists a symlink as a blob holding its target — never a file of the sample."""
from codeverse3d.record._git import read_tree_at
ws, _rec = fake_run
@@ -116,14 +97,7 @@ def test_a_symlink_is_not_exported_as_a_file_of_its_target(fake_run) -> None:
def test_a_planted_smudge_filter_never_runs(fake_run, tmp_path):
- """The same window, the other content-rendering command: ``git archive`` renders
- blobs through ``convert_to_working_tree``, so a planted ``filter.<name>.smudge``
- RUNS — and archive has no ``--no-filters``, while ``-c core.attributesFile`` only
- silences the GLOBAL attributes file. ``read_tree_at`` therefore reads the object
- database directly (``ls-tree`` + ``cat-file --batch``), which applies no filter.
-
- Planted with ``.git/info/attributes`` — no commit needed, nothing the harness
- sanitises on the read path."""
+ """read_tree_at reads the object database directly, so a planted smudge filter never runs."""
ws, _rec = fake_run
fired = tmp_path / "smudged"
(ws.src / "model.py").write_text("# real content\n")
@@ -141,8 +115,7 @@ def test_a_planted_smudge_filter_never_runs(fake_run, tmp_path):
def test_diff_between_refuses_a_sha_the_repo_does_not_have(fake_run):
- """The recorded round shas are the only usable handles, and one the repository no
- longer holds must raise rather than diff against an empty tree."""
+ """A sha the repository no longer holds raises rather than diffing against an empty tree."""
ws, rec = fake_run
good = rec.rounds[1].commit
with pytest.raises(_git.GitReadError):
@@ -204,23 +177,6 @@ def test_export_with_captions_and_reexport(fake_run, tmp_path: Path):
assert json.loads(row["meta_json"])["has_captions"] is True
-def test_pack_samples_roundtrip(runs_dir: Path, tmp_path: Path):
- out = tmp_path / "ds"
- export_samples(runs_dir, out)
- rep = pack_samples(out, tar_prefix="3dcodeverse/test/", max_tar_bytes=200_000)
- assert rep.n_samples == 3 and rep.tars
- assert verify_locators(out) == 3
- rows = [json.loads(ln) for ln in (out / "metadata.jsonl").read_text().splitlines()]
- r = rows[0]
- assert r["tar"].startswith("3dcodeverse/test/samples-")
- with (out / Path(r["tar"]).name).open("rb") as fh:
- fh.seek(r["byte_start"])
- blob = fh.read(r["byte_len"])
- with tarfile.open(fileobj=io.BytesIO(blob)) as tf:
- names = [m.name for m in tf.getmembers()]
- assert all(n.startswith(r["key"] + "/") for n in names) and f"{r['key']}/meta.json" in names
-
-
# --------------------------------------------------------------------------- pairs
@@ -237,40 +193,11 @@ def test_build_pairs(runs_dir: Path, tmp_path: Path):
assert p["delta"] == 0.25 and "thicken the legs" in p["reason"]
assert p["chosen"]["files"]["src/model.py"].startswith("# round 1")
assert p["rejected"]["files"]["src/model.py"].startswith("# round 0")
- rep = next(x for x in pairs if x["kind"] == "repair")
- assert rep["error"]["type"] == "SyntaxError" and rep["chosen"]["build_ok"] is True
- assert "size=3.0" in rep["chosen"]["files"]["src/model.py"]
xb = next(x for x in pairs if x["kind"] == "cross_backend")
assert xb["chosen"]["generator"].startswith("gemini-cli") and xb["rejected"]["generator"].startswith("codex")
assert xb["delta"] == pytest.approx(0.2)
-# --------------------------------------------------------------------------- dedupe
-
-
-def test_code_fingerprint_normalises():
- a = {"src/model.py": "import bpy # hi\n\nx = 1\n# c\n"}
- b = {"src/model.py": "import bpy\nx=1\n"}
- c = {"src/model.py": "import bpy\nx=2\n"}
- assert code_fingerprint(a) == code_fingerprint(b) != code_fingerprint(c)
- assert normalise_code("// a\nlet x = 1; // t\n") == "letx=1;"
-
-
-# --------------------------------------------------------------------------- index
-
-
-def test_build_index_and_queries(runs_dir: Path, tmp_path: Path):
- db = tmp_path / "idx.sqlite"
- assert build_index(runs_dir, db) == 3
- con = sqlite3.connect(db)
- assert con.execute("SELECT COUNT(*) FROM runs").fetchone()[0] == 3
- assert con.execute("SELECT COUNT(*) FROM rounds").fetchone()[0] == 2 + 4 + 2
- assert con.execute("SELECT COUNT(*) FROM usage WHERE role='judge'").fetchone()[0] == 6
- con.close()
- s = summary(db)
- assert {r["language"] for r in s} == {"blender", "threejs"}
-
-
# --------------------------------------------------------------------------- finding: repair pairs (pairs.py:115)
def test_repair_pairs_match_structurally_not_by_kind(tmp_path: Path):
"""Lifecycle labels the fixing round 'refine' — repair pairs must not need kind='repair'."""
@@ -346,7 +273,6 @@ def test_degraded_round_is_not_a_zero_score(tmp_path: Path):
ws.write_json(ws.record_path, rec)
assert select.pick(ws.root) == 2 # 0.64 beats 0.62; the degraded 0.0 never competes
pairs = preference_pairs(ws, rec, min_delta=0.05)
- # old behaviour: chosen=r2 rejected=r1(degraded) with delta 0.64 — a pure noise pair
assert pairs == []
pairs = preference_pairs(ws, rec, min_delta=0.01)
assert len(pairs) == 1 and pairs[0]["rejected"]["round"] == 0 and pairs[0]["chosen"]["round"] == 2
@@ -368,25 +294,6 @@ def test_a_run_with_only_degraded_verdicts_exports_unscored(tmp_path: Path):
assert meta["acceptance_results"] == {}
-# --------------------------------------------------------------------------- graphics + textured exports
-def test_export_graphics_sample(tmp_path: Path):
- from codeverse3d.contracts.common import Language
- from tests.flywheel_cli.conftest import make_fake_run, tiny_png
-
- ws, rec = make_fake_run(tmp_path / "runs", "rain_glsl", prompt="neon rain", language=Language.GLSL_SHADER)
- for r in rec.rounds: # each round keeps its own preview (gif magic irrelevant here — copied by name)
- tiny_png(ws.round_artifacts(r.index) / "preview.gif")
- tiny_png(ws.artifacts / "preview.gif") # the canonical one is the LAST build's: never exported for another round
- out = tmp_path / "ds"
- rep = export_samples(ws.root.parent, out)
- assert rep.n_exported == 1, rep.skipped
- sdir = out / "graphics" / "glsl_shader" / "rain_glsl"
- assert (sdir / "code.frag").is_file() and (sdir / "src" / "shader.frag").is_file()
- meta = json.loads((sdir / "meta.json").read_text())
- assert meta["entry"] == "code.frag" and meta["type"] == "Procedural Graphics"
- assert "renders/preview.gif" in meta["renders"]
-
-
def test_export_includes_textured_assets_when_shipped(fake_run, tmp_path: Path):
from codeverse3d.texturing.run import report_path
from tests.flywheel_cli.conftest import tiny_png
@@ -417,7 +324,7 @@ def test_export_includes_textured_assets_when_shipped(fake_run, tmp_path: Path):
def test_battery_layouts_are_discovered_by_flywheel_and_gallery(tmp_path):
- """All three battery layouts resolve runs; eval siblings never become gallery runs."""
+ """All three eval battery layouts resolve runs; eval siblings never become gallery runs."""
from codeverse3d.addons.gallery.index import scan_root
from codeverse3d.record.record import find_run_dirs, iter_runs
diff --git a/harness/tests/flywheel_cli/test_gemini_image.py b/harness/tests/flywheel_cli/test_gemini_image.py
index 317589be..8a983cd0 100644
--- a/harness/tests/flywheel_cli/test_gemini_image.py
+++ b/harness/tests/flywheel_cli/test_gemini_image.py
@@ -1,9 +1,4 @@
-"""GeminiImageModel key rotation: dead-key and free-429 semantics adopted from
-``models/gemini.py`` (offline; fake clients + a 2-key pool).
-
-Lives with the flywheel/CLI tests because this group owns
-``codeverse3d/models/gemini_image.py``'s rotation behaviour.
-"""
+"""GeminiImageModel: key rotation, the per-attempt timeout and per-image pricing (fake clients)."""
from __future__ import annotations
@@ -56,24 +51,6 @@ def _model(script: list, used: list[str], sleeps: list[float], *, max_attempts:
return m, pool
-def test_dead_key_rotates_free_and_is_benched(monkeypatch: pytest.MonkeyPatch):
- used: list[str] = []
- sleeps: list[float] = []
- script = [ModelError("API key expired", retryable=False, status=403), _image_response()]
- m, pool = _model(script, used, sleeps, max_attempts=1)
- reports: list[tuple[str, str]] = []
- orig = pool.report
- monkeypatch.setattr(pool, "report", lambda key, outcome, **kw: (reports.append((key, outcome)), orig(key, outcome, **kw))[1])
- images, usage = m.generate_with_usage("a wooden crate texture")
- # the 403 did not consume the single attempt: a second key was tried and won
- assert len(images) == 1 and len(used) == 2 and used[0] != used[1]
- assert sleeps == [] # dead rotation never sleeps
- dead_key = used[0]
- assert (dead_key, "error") in reports # reported at rotation time
- assert reports[-1] == (dead_key, "dead") # benched once the sibling key succeeded
- assert (used[1], "ok") in reports
-
-
def test_429_rotation_is_free_while_untried_keys_remain():
used: list[str] = []
sleeps: list[float] = []
@@ -122,8 +99,7 @@ def test_image_budget_bounds_the_per_attempt_http_timeout():
def test_images_are_priced_by_the_generated_size():
- """A 2K request is billed at the 2K per-image price (it was billed at the 1K rate,
- 0.067 instead of 0.134); a 512 request generates a 1K image and is billed as one."""
+ """A 2K request is billed at the 2K price (it was billed at the 1K rate); 512 is billed as 1K."""
from codeverse3d.models.pricing import per_image_usd
def cost(size: int) -> float:
diff --git a/harness/tests/flywheel_cli/test_manifest_pack.py b/harness/tests/flywheel_cli/test_manifest_pack.py
index 89c50d60..b187edc6 100644
--- a/harness/tests/flywheel_cli/test_manifest_pack.py
+++ b/harness/tests/flywheel_cli/test_manifest_pack.py
@@ -1,31 +1,19 @@
-"""The dataset generation manifest + the manifest-driven pack.
-
-Export used to filter duplicates only in memory while ``pack_samples`` re-scanned
-the directory tree — so ``--drop-duplicates --pack`` shipped every dropped
-duplicate, a re-pack truncated the previous valid archive at open, and a planted
-``src/meta.json`` (sample folders carry the run's LLM-written src/** tree) could
-inject a phantom row. The manifest is now the single source of truth: pack tars
-ONLY manifest entries, verifies every byte against the recorded sha256, and
-publishes the archives AND the index in one replace loop after the whole pack
-succeeded (a failure in either window leaves the previous dataset untouched).
-"""
+"""The dataset manifest is the single source of truth: pack tars only its entries, verifies every
+byte against it, and publishes archives and index together only after the whole pack succeeded."""
from __future__ import annotations
import hashlib
import json
-import re
import tarfile
from pathlib import Path
import pytest
from codeverse3d.addons.dataset.export import (
- MANIFEST_NAME,
ManifestError,
export_samples,
load_manifest,
- write_manifest,
)
from codeverse3d.addons.dataset.pack import PackError, pack_samples, verify_locators
@@ -56,33 +44,15 @@ def _tar_member_names(out: Path) -> list[str]:
# --------------------------------------------------------------------------- manifest
-def test_export_writes_a_hashed_manifest(runs_dir: Path, tmp_path: Path):
- out = tmp_path / "ds"
- rep = export_samples(runs_dir, out)
- assert rep.manifest == str(out / MANIFEST_NAME)
- m = load_manifest(out)
- assert m.generated_at is not None
- assert m.filters.drop_duplicates is False and m.filters.min_score is None
- assert len(m.entries) == 3 and m.dropped == []
+def test_export_hashes_every_byte_it_wrote(exported_ds: Path):
+ m = load_manifest(exported_ds)
e = next(x for x in m.entries if x.sample_id == CHAIR_ID)
- assert e.run.battery == "runs" and e.run.rel == "wooden_chair_ab12cd34"
- assert e.sample_rel_dir == "static_object/blender/wooden_chair_ab12cd34"
assert {"meta.json", "captions.json", "code.py", "src/model.py"} <= set(e.files)
- for rel, digest in e.files.items(): # every byte the export wrote is hashed
- assert hashlib.sha256((out / e.sample_rel_dir / rel).read_bytes()).hexdigest() == digest
- assert len(e.code_sha256) == 64 and len(e.code_fingerprint) == 64
+ for rel, digest in e.files.items():
+ assert hashlib.sha256((exported_ds / e.sample_rel_dir / rel).read_bytes()).hexdigest() == digest
assert e.code_sha256 != e.code_fingerprint # raw-exact vs normalised
-def test_filtered_runs_are_recorded_as_dropped(runs_dir: Path, tmp_path: Path):
- export_samples(runs_dir, tmp_path / "ds", min_score=0.75)
- m = load_manifest(tmp_path / "ds")
- assert len(m.entries) == 1 and len(m.dropped) == 2
- assert m.filters.min_score == 0.75
- assert all("min_score" in d.reason for d in m.dropped)
- assert {d.run.rel for d in m.dropped} == {"wooden_chair_codex", "lamp_three"}
-
-
# --------------------------------------------------------------------------- drop-duplicates → pack
def test_drop_duplicates_then_pack_ships_no_duplicate(runs_dir: Path, tmp_path: Path):
out = tmp_path / "ds"
@@ -120,27 +90,8 @@ def _packed_state(out: Path) -> dict[str, bytes]:
for p in [*out.glob("samples-*.tar"), out / "metadata.jsonl", out / "metadata.parquet"]}
-def test_a_failure_inside_the_pack_loop_leaves_the_old_tar_and_index_intact(packed_ds: Path):
- before = _packed_state(packed_ds)
- # corrupt the recorded hash of a file in the LAST entry: the mismatch fires
- # mid-pack, after earlier samples were already added to the new tmp archive
- m = load_manifest(packed_ds)
- victim = m.entries[-1]
- rel = sorted(victim.files)[0]
- victim.files[rel] = "0" * 64
- write_manifest(m, packed_ds)
- with pytest.raises(PackError, match=re.escape(rel)):
- pack_samples(packed_ds)
- assert _packed_state(packed_ds) == before # old archives AND index untouched
- assert not list(packed_ds.glob("*.tmp")) # the half-written tmp tar is cleaned up
- assert verify_locators(packed_ds) == 3 # the previous pack still round-trips
-
-
def test_a_failure_after_the_pack_loop_publishes_nothing(packed_ds: Path, monkeypatch):
- """The window the tar loop does NOT cover: every tar is written, then the index
- write fails. Publishing the tars alone left the OLD index resolving to the new
- tars' byte offsets — six rows pointing at the wrong bytes, and verify_locators
- happily passing a corrupt dataset."""
+ """Every tar written, then the index write fails: the old dataset stays whole, tars and index."""
import codeverse3d.addons.dataset.pack as P
before = _packed_state(packed_ds)
@@ -153,8 +104,7 @@ def test_a_failure_after_the_pack_loop_publishes_nothing(packed_ds: Path, monkey
def test_repacking_over_an_existing_archive_succeeds_and_leaves_no_orphan_tar(packed_ds: Path):
- """Re-packing used to truncate the previous archive at open; and a repack into FEWER
- shards left 001/002 on disk as dead bytes the new index never names."""
+ """A repack into fewer shards leaves no orphan tar the new index never names."""
assert pack_samples(packed_ds, max_tar_bytes=1).tars == ["samples-000.tar", "samples-001.tar", "samples-002.tar"]
prep = pack_samples(packed_ds)
assert prep.tars == ["samples-000.tar"] and prep.n_samples == 3
diff --git a/harness/tests/flywheel_cli/test_quality_gallery.py b/harness/tests/flywheel_cli/test_quality_gallery.py
index 2b295fbd..f6bb74b2 100644
--- a/harness/tests/flywheel_cli/test_quality_gallery.py
+++ b/harness/tests/flywheel_cli/test_quality_gallery.py
@@ -13,7 +13,6 @@ from codeverse3d.addons.dataset.quality import (
prompt_hash,
quality_tier,
)
-from codeverse3d.contracts.common import Language
from codeverse3d.record.record import load_record
from tests.flywheel_cli.conftest import make_fake_run
from tests.flywheel_cli.test_captions import GOOD
@@ -32,9 +31,7 @@ def test_quality_tier_rule():
def test_only_raw_hash_duplicates_are_dropped_normalised_ones_are_only_marked():
- """``duplicate_of`` (the DROP set) is keyed on the RAW code_sha256; the normalised
- fingerprint only ever stamps ``near_duplicate_of`` — row c differs from a/b/e in
- bytes alone and must survive."""
+ """duplicate_of (the drop set) is keyed on the raw code_sha256; the fingerprint only marks."""
rows = [
{"id": "a", "key": "a", "code_sha256": "R", "code_fingerprint": "X", "prompt_hash": "p", "quality_tier": "B", "score": 0.8},
{"id": "b", "key": "b", "code_sha256": "R", "code_fingerprint": "X", "prompt_hash": "p", "quality_tier": "A", "score": 0.8},
@@ -53,9 +50,7 @@ def test_only_raw_hash_duplicates_are_dropped_normalised_ones_are_only_marked():
def test_whitespace_inside_a_string_is_a_near_duplicate_not_a_duplicate(runs_dir: Path, tmp_path: Path):
- """``LABEL = "a b"`` and ``LABEL="ab"`` normalise identically but are different
- programs: --drop-duplicates must keep both and only mark them, while the
- byte-identical chair/codex pair is still dropped."""
+ """LABEL = "a b" and LABEL="ab" normalise alike but are different programs: kept, only marked."""
common = '# round 1\nimport bpy\nLABEL={}\nbpy.ops.mesh.primitive_cube_add(size=2.0)\n'
make_fake_run(runs_dir, "stool_spaced", prompt="a stool", code_v2=common.format('"a b"'))
make_fake_run(runs_dir, "stool_tight", prompt="a stool", code_v2=common.format('"ab"'))
@@ -73,49 +68,6 @@ def test_whitespace_inside_a_string_is_a_near_duplicate_not_a_duplicate(runs_dir
# --------------------------------------------------------------------------- export extras
-def test_export_meta_tiers_duplicates_and_captions_sidecar(runs_dir: Path, tmp_path: Path):
- # a fourth run = exact duplicate (same code + prompt) of the first, and a urdf run with link meshes
- make_fake_run(runs_dir, "wooden_chair_again")
- ws_u, rec_u = make_fake_run(runs_dir, "cabinet_urdf", prompt="a cabinet", language=Language.URDF_BLENDER)
- for r in rec_u.rounds: # each round keeps its own link meshes (artifacts/rNN/meshes/)
- (ws_u.round_artifacts(r.index) / "meshes").mkdir()
- (ws_u.round_artifacts(r.index) / "meshes" / "Body.glb").write_bytes(b"glTF" + b"\0" * 8)
- side = tmp_path / "caps"
- side.mkdir()
- (side / "lamp_three.json").write_text(json.dumps({**GOOD, "provenance": {"captioner": "gemini:x"}}))
-
- out = tmp_path / "ds"
- rep = export_samples(runs_dir, out, captions_dir=side)
- # the codex run shares prompt + round-1 code with the chair → also an exact duplicate
- assert rep.n_exported == 5 and rep.n_indexed == 5 and rep.n_duplicates == 2
- assert rep.duplicates[0].canonical == "3dcodeverse/static_object/blender/wooden_chair_ab12cd34"
- assert set(rep.duplicates[0].duplicates) == {"3dcodeverse/static_object/blender/wooden_chair_again",
- "3dcodeverse/static_object/blender/wooden_chair_codex"}
- assert rep.tiers == {"A": 3, "C": 2} # codex run 0.6 (not passed) and lamp 0.72 → C
- meta = json.loads((out / "static_object" / "blender" / "wooden_chair_ab12cd34" / "meta.json").read_text())
- assert meta["quality_tier"] == "A" and meta["gate_errors"] == 0 and meta["gate_summary"] == {"lint": 0}
- assert meta["acceptance"] == [{"id": "a1", "text": "", "how": "", "priority": "", "passed": True}]
- assert len(meta["code_fingerprint"]) == 64 and meta["prompt_hash"] == prompt_hash("a wooden dining chair")
- assert [r["index"] for r in meta["rounds_summary"]] == [0, 1] and meta["round"] == 1
- u_meta = json.loads((out / "articulated_object" / "urdf_blender" / "cabinet_urdf" / "meta.json").read_text())
- assert "meshes/Body.glb" in u_meta["files"] and (out / "articulated_object" / "urdf_blender" / "cabinet_urdf" / "meshes" / "Body.glb").is_file()
- # side-car captions are picked up for lamp_three only
- rows = {json.loads(line)["key"]: json.loads(line) for line in (out / "metadata.jsonl").read_text().splitlines()}
- assert rows["lamp_three"]["has_captions"] is True and rows["lamp_three"]["captions"]["factory"] == GOOD["factory"]
- assert rows["wooden_chair_ab12cd34"]["has_captions"] is False
- assert rows["wooden_chair_again"]["duplicate_of"] == "3dcodeverse/static_object/blender/wooden_chair_ab12cd34"
- assert (out / "duplicates.json").is_file()
- import pyarrow.parquet as pq
-
- table = pq.read_table(out / "metadata.parquet")
- assert {"quality_tier", "gate_errors", "cost_usd", "rounds", "status", "code_fingerprint", "code_sha256",
- "prompt_hash", "duplicate_of", "near_duplicate_of", "has_captions"} <= set(table.column_names)
- # drop duplicates from the index (folder stays)
- rep2 = export_samples(runs_dir, out, captions_dir=side, drop_duplicates=True)
- assert rep2.n_indexed == 3 and rep2.n_duplicates == 2 and rep2.tiers == {"A": 2, "C": 1}
- assert (out / "static_object" / "blender" / "wooden_chair_again").is_dir()
-
-
def test_export_scene_views_repeating_names(fake_run, tmp_path: Path):
"""Scene renders repeat a camera name per capture time; the sample must keep every file."""
from codeverse3d.contracts.artifacts import RenderView
diff --git a/harness/tests/flywheel_cli/test_record_env.py b/harness/tests/flywheel_cli/test_record_env.py
index d93731f9..6da8078e 100644
--- a/harness/tests/flywheel_cli/test_record_env.py
+++ b/harness/tests/flywheel_cli/test_record_env.py
@@ -1,35 +1,18 @@
"""D38: ``record.environment.harness_git_sha`` names the checkout the harness ran from."""
-import re
import shutil
import subprocess
from pathlib import Path
import pytest
-from codeverse3d.record import record as record_mod
-from codeverse3d.record.record import _harness_git_sha, environment_versions
-
-SHA_RE = re.compile(r"^[0-9a-f]{40}(-dirty)?$")
+from codeverse3d.record.record import _harness_git_sha
def _git(*args: str, cwd: Path) -> subprocess.CompletedProcess[str]:
return subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True, check=False)
-def _in_a_checkout() -> bool:
- here = Path(record_mod.__file__).resolve()
- return shutil.which("git") is not None and _git("ls-files", "--error-unmatch", here.name, cwd=here.parent).returncode == 0
-
-
[email protected](not _in_a_checkout(), reason="harness is not running from a git checkout (wheel / sdist install)")
-def test_harness_git_sha_is_populated_inside_the_repo():
- # the repo's .git lives ABOVE harness/ — the old probe (harness/.git) never matched
- sha = _harness_git_sha()
- assert SHA_RE.match(sha), sha
- assert environment_versions()["harness_git_sha"] == sha
-
-
def test_harness_git_sha_from_a_tracked_file_and_dirty_marker(tmp_path):
if shutil.which("git") is None:
pytest.skip("git not installed")
diff --git a/harness/tests/flywheel_cli/test_refine.py b/harness/tests/flywheel_cli/test_refine.py
index 46bdc9fd..36a91bee 100644
--- a/harness/tests/flywheel_cli/test_refine.py
+++ b/harness/tests/flywheel_cli/test_refine.py
@@ -59,22 +59,12 @@ def test_with_code_inlines_the_answer_and_the_tree_it_started_from(tmp_path):
assert "src/parts/leg.py" not in t.before_files # it did not exist yet
[email protected]("delta,label", [(0.5, "improved"), (0.05, "improved"), (0.04, "unchanged"),
- (0.0, "unchanged"), (-0.05, "regressed"), (None, "unscored")])
[email protected]("delta,label", [(0.05, "improved"), (0.04, "unchanged"), (-0.05, "regressed"),
+ (None, "unscored")])
def test_the_outcome_rule_is_the_one_pairs_uses(delta, label):
assert outcome_of(delta, MIN_PREFERENCE_DELTA) == label
-def test_a_run_reached_through_a_symlink_is_exported_once_under_its_physical_battery(tmp_path):
- _corpus(tmp_path, battery="zz_physical")
- alias = tmp_path / "aa_alias" / "runs"
- alias.mkdir(parents=True)
- (alias / "chair").symlink_to(tmp_path / "zz_physical" / "runs" / "chair")
- rows, drops = _rows(tmp_path)
- assert len(rows) == 1 and not drops, "a run on disk is found once"
- assert rows[0].battery == "zz_physical", "labelled by the battery it was reached through first"
-
-
@pytest.mark.parametrize("break_it,reason", [
("kind", None), ("commit", "no_commit"), ("build", "predecessor_build_failed"),
# RoundRecord.build is optional: a round that never reached the build stage carries
diff --git a/harness/tests/flywheel_cli/test_run_identity.py b/harness/tests/flywheel_cli/test_run_identity.py
index 6c3df4a4..f47f7b2b 100644
--- a/harness/tests/flywheel_cli/test_run_identity.py
+++ b/harness/tests/flywheel_cli/test_run_identity.py
@@ -1,48 +1,33 @@
-"""Run identity (RunId): the slug rule, and that nested batteries no longer collide.
-
-The real eval/bench/out tree holds 396 run dirs with only 169 distinct basenames — 211 are
-literally ``run`` (compare_backends ``cells/<id>/<arm>/run``, ab_plan
-``arms/<arm>/cells/<id>/<slug>/run``). Keying anything on ``ws.root.name`` therefore
-made exports rmtree each other, blew the SQLite PRIMARY KEY, collapsed gallery entries
-and bled captions/pairs across runs. These tests pin the fix end to end.
-"""
+"""Run identity (RunId): the slug rule — 211 of 396 recorded run dirs are literally named ``run``."""
from __future__ import annotations
-import json
import sqlite3
from pathlib import Path
import pytest
-from codeverse3d.addons.dataset.export import export_samples, load_captions
+from codeverse3d.addons.dataset.export import export_samples
from codeverse3d.addons.dataset.index import build_index as build_sqlite_index
-from codeverse3d.addons.dataset.pairs import build_pairs
from codeverse3d.addons.dataset.sample import SampleError
from codeverse3d.contracts.run import RunId
-from codeverse3d.record.record import FoundRun, battery_label, iter_runs, run_id_for, unique_files
+from codeverse3d.record.record import unique_files
from tests.flywheel_cli.conftest import make_fake_run
-# --------------------------------------------------------------------------- slug rule
-def test_flat_layout_keeps_the_plain_basename():
- """Back-compat: a rel with no parent directories is the basename, unchanged."""
- assert RunId(battery="runs", rel="wooden_chair_ab12cd34").slug == "wooden_chair_ab12cd34"
- # retry cells keep their .attemptN suffix — it distinguishes them from the original
- assert RunId(battery="h2h_scene_v1",
- rel="h2h_b01_neon_alley.attempt1").slug == "h2h_b01_neon_alley.attempt1"
-
-
-def test_a_runs_wrapper_is_structural_noise():
- assert RunId(battery="articulated_v1_flash", rel="runs/art_easy_laptop").slug == "art_easy_laptop"
[email protected]("battery, rel, slug", [
+ ("runs", "wooden_chair_ab12cd34", "wooden_chair_ab12cd34"),
+ ("h2h_scene_v1", "h2h_b01_neon_alley.attempt1", "h2h_b01_neon_alley.attempt1"), # a retry keeps its suffix
+ ("articulated_v1_flash", "runs/art_easy_laptop", "art_easy_laptop"), # runs/ is noise
+ ("compare_v1_full", "cells/cmp_easy_stool/harness_x/run", "cmp_easy_stool__harness_x"),
+])
+def test_the_slug_rule(battery, rel, slug):
+ assert RunId(battery=battery, rel=rel).slug == slug
-def test_compare_backends_layout_slug_arm_cell():
- rid = RunId(battery="compare_v1_full",
- rel="cells/cmp_easy_stool/harness_api-agent_gemini_gemini-3.7-flash/run")
- assert rid.slug == "cmp_easy_stool__harness_api-agent_gemini_gemini-3.7-flash"
- assert rid.cell == "cmp_easy_stool"
- assert rid.arm == "harness_api-agent_gemini_gemini-3.7-flash"
+def test_compare_backends_layout_arm_and_cell():
+ rid = RunId(battery="compare_v1_full", rel="cells/cmp_easy_stool/harness_x/run")
+ assert (rid.cell, rid.arm) == ("cmp_easy_stool", "harness_x")
def test_ab_plan_layout_slug_arm_cell_and_attempt_suffix():
@@ -57,16 +42,8 @@ def test_ab_plan_layout_slug_arm_cell_and_attempt_suffix():
assert retry.slug.endswith("harness_api-agent_gemini_gemini-3.7-flash.attempt1")
-def test_battery_label_and_run_id_for(tmp_path: Path):
- runs = tmp_path / "bench" / "out" / "static_v9" / "runs"
- (runs / "stool_a1").mkdir(parents=True)
- assert battery_label(runs) == "static_v9"
- rid = run_id_for(runs, runs / "stool_a1")
- assert rid.battery == "static_v9" and rid.rel == "stool_a1" and rid.slug == "stool_a1"
-
-
def test_unique_files_skips_subruns_below_the_root_not_above_it(tmp_path: Path):
- """A scan rooted INSIDE ``_assets`` used to return [] — the skip read absolute parts."""
+ """A scan rooted inside ``_assets`` once returned [] — the skip read absolute parts."""
root = tmp_path / "scene" / "_assets" / "lamp"
(root / "_cand" / "c1").mkdir(parents=True)
(root / "record.json").write_text("{}")
@@ -75,14 +52,6 @@ def test_unique_files_skips_subruns_below_the_root_not_above_it(tmp_path: Path):
# --------------------------------------------------------------------------- shared tree
-def _nested_battery(tmp_path: Path) -> Path:
- """ONE root, two real runs whose directories are BOTH named ``run``."""
- root = tmp_path / "compare_v9"
- make_fake_run(root / "cells" / "cmp_a_stool" / "armx", "run", prompt="a stool")
- make_fake_run(root / "cells" / "cmp_b_lamp" / "armx", "run", prompt="a lamp")
- return root
-
-
def _colliding_root(tmp_path: Path) -> Path:
"""Two rels that legitimately reduce to the SAME slug ``x``."""
root = tmp_path / "mixed"
@@ -91,62 +60,6 @@ def _colliding_root(tmp_path: Path) -> Path:
return root
-def test_nested_run_identity_survives_every_flywheel_consumer(tmp_path: Path):
- """Build the colliding-basename corpus once, then exercise every reader."""
- root = _nested_battery(tmp_path)
- found = list(iter_runs(root))
- assert all(isinstance(f, FoundRun) for f in found)
- assert all(f.ws.root.name == "run" for f in found) # the basename really is degenerate
- assert sorted(f.run_id.slug for f in found) == ["cmp_a_stool__armx", "cmp_b_lamp__armx"]
-
- # Export keeps both identities and their distinct payloads.
- out = tmp_path / "ds"
- rep = export_samples(root, out)
- assert rep.n_exported == 2 and rep.n_indexed == 2, rep.skipped
- a = out / "static_object" / "blender" / "cmp_a_stool__armx"
- b = out / "static_object" / "blender" / "cmp_b_lamp__armx"
- assert a.is_dir() and b.is_dir()
- assert json.loads((a / "meta.json").read_text())["prompt"] == "a stool"
- assert json.loads((b / "meta.json").read_text())["prompt"] == "a lamp"
- keys = {json.loads(line)["key"] for line in (out / "metadata.jsonl").read_text().splitlines()}
- assert keys == {"cmp_a_stool__armx", "cmp_b_lamp__armx"}
-
- # SQLite keeps the same rel/arm/cell mapping.
- db = tmp_path / "idx.sqlite"
- assert build_sqlite_index(root, db) == 2
- con = sqlite3.connect(db)
- rows = con.execute("SELECT slug, rel, arm, cell FROM runs ORDER BY slug").fetchall()
- con.close()
- assert rows == [
- ("cmp_a_stool__armx", "cells/cmp_a_stool/armx/run", "armx", "cmp_a_stool"),
- ("cmp_b_lamp__armx", "cells/cmp_b_lamp/armx/run", "armx", "cmp_b_lamp"),
- ]
-
- # Pair rows use the resolved slug, never the degenerate directory basename.
- out = tmp_path / "pairs.jsonl"
- n = build_pairs(root, out, min_delta=0.05)
- assert n == 2 # one preference pair per run (0.55 → 0.80)
- runs = {json.loads(line)["run"] for line in out.read_text().splitlines()}
- assert runs == {"cmp_a_stool__armx", "cmp_b_lamp__armx"}
-
- # Caption sidecars are equally isolated.
- from codeverse3d.addons.dataset.captions import caption_sample
- from tests.flywheel_cli.test_captions import GOOD
- from tests.orchestrator_tracks.fakes import FakeChatModel
-
- side = tmp_path / "caps"
- by_slug = {f.run_id.slug: f for f in found}
- a = by_slug["cmp_a_stool__armx"]
- b = by_slug["cmp_b_lamp__armx"]
- caption_sample(a.ws, a.record, "fake:fake", model=FakeChatModel([GOOD]),
- out_dir=side, slug=a.run_id.slug)
- assert (side / "cmp_a_stool__armx.json").is_file()
- assert not (side / "run.json").exists(), "the degenerate basename side-car is the bleed"
- assert load_captions(a.ws, a.record, side, slug=a.run_id.slug)["detailed"] == GOOD["detailed"]
- assert load_captions(b.ws, b.record, side, slug=b.run_id.slug) == {}, \
- "the un-captioned run must not pick up its neighbour's side-car"
-
-
def test_slug_collisions_fail_before_export_or_index_publication(tmp_path: Path):
root = _colliding_root(tmp_path)
out = tmp_path / "ds"
diff --git a/harness/tests/flywheel_cli/test_run_layout_cli.py b/harness/tests/flywheel_cli/test_run_layout_cli.py
index de7ebb98..5bf69f9a 100644
--- a/harness/tests/flywheel_cli/test_run_layout_cli.py
+++ b/harness/tests/flywheel_cli/test_run_layout_cli.py
@@ -67,34 +67,22 @@ def test_package_run_builds_a_stable_independent_and_loadable_handover(fake_run)
d = load_deliverable(ws)
assert d is not None
roles = {f.role for f in d.files}
- # the manifest does NOT list itself: no file can carry its own hash, and the
- # returned object used to have one more file (and a smaller total) than the disk
- assert {"code", "model", "sheet"} <= roles and "manifest" not in roles
+ assert {"code", "model", "sheet"} <= roles and "manifest" not in roles # it cannot carry its own hash
assert load_deliverable(ws) == d and d.total_bytes == sum(f.bytes for f in d.files)
- assert d.round == 1 and d.code_source == "commit"
- assert d.entry == "deliverable/src/model.py"
+ assert d.round == 1 and d.entry == "deliverable/src/model.py"
assert (ws.deliverable / "src" / "model.py").read_text().startswith("# round 1")
- assert (ws.deliverable / "object.glb").is_file() and (ws.deliverable / "sheet.png").is_file()
- for f in d.files:
- assert (ws.root / f.path).is_file() and f.bytes > 0 and len(f.sha256) == 64
- assert ws.settings_path.is_file() and ws.cost_path.is_file()
cost = json.loads(ws.cost_path.read_text())
assert cost["total_usd"] == rec.total_usage.cost_usd
assert cost["by_round"]
settings = json.loads(ws.settings_path.read_text())
assert {r["role"] for r in settings["roles"]} == {"planner", "generator", "judge", "captioner"}
- assert settings["rubric"] and settings["rubric_hash"] and settings["price_table_version"]
assert deliverable_path(ws, "object.glb") == ws.deliverable / "object.glb"
assert deliverable_path(ws, "nope.glb") is None
# Rebuilding unchanged content is byte-for-byte stable.
- first = d
- files_before = sorted(p.relative_to(ws.deliverable).as_posix() for p in ws.deliverable.rglob("*") if p.is_file())
second = build_deliverable(ws, rec, 1)
- files_after = sorted(p.relative_to(ws.deliverable).as_posix() for p in ws.deliverable.rglob("*") if p.is_file())
- assert files_before == files_after
- assert first.generated_at == second.generated_at # unchanged content → no diff
- assert [(f.path, f.sha256) for f in first.files] == [(f.path, f.sha256) for f in second.files]
+ assert d.generated_at == second.generated_at # unchanged content → no diff
+ assert [(f.path, f.sha256) for f in d.files] == [(f.path, f.sha256) for f in second.files]
# Package metadata can be recovered from files even before the caller saves rec.
bare = load_record(ws)
@@ -131,10 +119,7 @@ def test_telemetry_by_round_scores_a_degraded_verdict_as_unscored(fake_run):
def test_the_settings_snapshot_records_the_temperature_the_track_planned_at(tmp_path):
- """The planner row reported ``planner.plan()``'s default 0.4 for every run — a default
- no track uses: each passes its own ``plan_temperature``, and graphics plans at 0.5. The
- track stamps what it ran with (``record.extra["sampling"]``), the judge's knobs too; a
- record without the stamp leaves them empty instead of guessing."""
+ """The planner row names the temperature the track planned at (graphics 0.5), never a default."""
from types import SimpleNamespace
from codeverse3d.contracts.common import Language
diff --git a/harness/tests/flywheel_cli/test_select.py b/harness/tests/flywheel_cli/test_select.py
index 91bb262c..7b038608 100644
--- a/harness/tests/flywheel_cli/test_select.py
+++ b/harness/tests/flywheel_cli/test_select.py
@@ -75,13 +75,6 @@ def test_pick_takes_the_best_score_then_fewer_gate_errors_then_the_earlier_round
assert select.pick(_run(tmp_path / "r", rounds).root) == want
-def test_round_rows_carry_the_judges_own_verdict_per_round(tmp_path):
- rows = select.round_rows(_run(tmp_path / "r", [(0.5, 0), (0.8, 2), ("degraded", 0)]).root)
- assert [(r.index, r.score, r.passed, r.gate_errors) for r in rows] == [
- (0, 0.5, False, 0), (1, 0.8, True, 2), (2, None, None, 0)]
- assert rows[1].commit and rows[1].minutes == 1.5 and rows[1].cost_usd == pytest.approx(0.2)
-
-
def test_summarise_is_baseline_pick_delta_rounds_and_the_stop_reason(tmp_path):
ws = _run(tmp_path / "r", [(0.5, 0), (0.8, 0), (0.6, 0)], status=RunStatus.BUDGET)
s = select.summarise(ws.root)
@@ -97,8 +90,7 @@ def test_summarise_is_baseline_pick_delta_rounds_and_the_stop_reason(tmp_path):
def test_an_old_record_with_best_fields_still_summarises(tmp_path):
- """A record.json written before 2026-09-22 (best_round / final_score / status 'passed')
- loads — the extra keys are ignored, the status reads 'stopped', the stop reason stays."""
+ """A pre-2026-09-22 record (best_round / final_score / status 'passed') still loads and summarises."""
ws, _ = make_fake_run(tmp_path / "runs")
data = json.loads(ws.record_path.read_text())
data.update(status="passed", best_round=0, baseline_score=0.55, final_score=0.55)
@@ -138,23 +130,8 @@ def test_pick_by_pairwise_asks_only_inside_the_margin_and_never_buys_a_verdict_t
# --------------------------------------------------------------------------- package
-def test_package_hands_over_one_round_and_says_why(tmp_path):
- ws = _run(tmp_path / "r", [(0.5, 0), (0.8, 0), (0.6, 0)])
- out = select.package(ws.root, 1, method="score")
- assert out == ws.deliverable and (out / "src" / "model.py").read_text() == "import bpy # round 1\n"
- assert (out / "object.glb").read_bytes() == (ws.round_artifacts(1) / "object.glb").read_bytes()
- sel = json.loads((ws.root / select.SELECTION_NAME).read_text())
- assert (sel["round"], sel["method"], sel["textured"]) == (1, "score", False)
- assert sel["scores"] == {"0": 0.5, "1": 0.8, "2": 0.6}
- assert json.loads(ws.deliverable_manifest_path.read_text())["round"] == 1
- with pytest.raises(ValueError, match="no round 7"):
- select.package(ws.root, 7)
-
-
def test_a_scene_or_graphics_round_packages_from_its_own_commit_and_renders(tmp_path):
- """A scene round's hand-over is its commit (src/ + public/, GLB assets included); a
- graphics round's is its commit + its judged frames (renders/rNN) + the sheet and GIF it
- kept under artifacts/rNN/. Round 0 of two packages round 0's bytes, never the last's."""
+ """A scene or graphics hand-over of round 0 of two is round 0's bytes, never the last's."""
for track, language, entry in ((Track.SCENE, Language.SCENE_THREEJS, "src/scene.js"),
(Track.GRAPHICS, Language.GLSL_SHADER, "src/shader.frag")):
ws = Workspace(tmp_path / language.value).create()
@@ -225,8 +202,7 @@ def test_package_textures_the_rounds_own_glb_once(tmp_path, monkeypatch):
def test_an_export_of_another_round_never_ships_the_pack_a_texture_pass_made_for_r02(tmp_path, monkeypatch):
- """copy_textured shipped whatever pack the record called shipped: texture r02, hand over r00
- (`pick --round 0`), export — and the r00 sample carried r02's textured GLB."""
+ """Texture r02, hand over r00, export: the r00 sample once carried r02's textured GLB."""
import codeverse3d.texturing.run as trun
from codeverse3d.addons.dataset.export import export_one
from codeverse3d.record.record import load_record
@@ -302,8 +278,7 @@ def test_make_hands_over_the_picked_round_unless_no_pick(tmp_path, monkeypatch):
# --------------------------------------------------------------------------- a round's own outputs
def test_round_outputs_are_the_rounds_own_never_the_last_builds(tmp_path: Path):
- """artifacts/object.glb and measurement.json are the LAST build's: a reader asking for round N's
- GLB or complexity got the last round's (calibration, `3dcode judge`, the export, gallery, dataset)."""
+ """A reader asking for round N's GLB or complexity once got the last build's (B4, B6)."""
ws = _run(tmp_path / "r", [(0.8, 0), (0.6, 0)])
rec = RunRecord.model_validate_json(ws.record_path.read_text())
(ws.artifacts / "object.glb").write_bytes(b"last")
@@ -323,8 +298,7 @@ def test_round_outputs_are_the_rounds_own_never_the_last_builds(tmp_path: Path):
def test_cli_texture_pass_textures_the_picked_round_not_the_last_build(tmp_path: Path, monkeypatch):
- """`3dcode texture pass` handed texture_pass no GLB, so it textured artifacts/object.glb — the
- LAST round's since D80 — whichever round the run hands over."""
+ """B5: `3dcode texture pass` textures the handed-over round's GLB, not the last build's."""
ws, _ = make_fake_run(tmp_path / "runs", "chair", scores=(0.90, 0.50)) # r00 is picked, r01 built last
seen: dict = {}
monkeypatch.setattr("codeverse3d.texturing.run.texture_pass", lambda *a, **kw: seen.update(kw))