Official Codebase of 3DCodeVerse
Python
tools: a FAIL verdict is no longer an MCP error — 63% of joint_sweep calls were
ziyaobebe0f8d76dd
diff --git a/harness/codeverse/cli/_common.py b/harness/codeverse/cli/_common.pyindex 3123c9ee..c9c5fa8c 100644--- a/harness/codeverse/cli/_common.py+++ b/harness/codeverse/cli/_common.py@@ -108,8 +108,10 @@ def print_observation(obs: Any, *, as_json: bool) -> None:if as_json:console.print_json(obs.model_dump_json())return- style = "green" if obs.ok else "red"- console.print(Panel(obs.text, title=f"[{style}]{'ok' if obs.ok else 'error'}[/{style}] ({obs.duration_ms} ms)", expand=False))+ # three states, not two: a FAIL verdict is an answer the tool computed, an error is a+ # tool that could not run (the same split the MCP server reports as is_error)+ state, style = ("ok", "green") if obs.ok else (("error", "red") if obs.failed else ("FAIL", "yellow"))+ console.print(Panel(obs.text, title=f"[{style}]{state}[/{style}] ({obs.duration_ms} ms)", expand=False))if obs.numbers:console.print(kv_table("numbers", obs.numbers))for img in obs.images:diff --git a/harness/codeverse/spatial/mcp_server.py b/harness/codeverse/spatial/mcp_server.pyindex 2415e368..e7d474a5 100644--- a/harness/codeverse/spatial/mcp_server.py+++ b/harness/codeverse/spatial/mcp_server.py@@ -6,6 +6,8 @@ One MCP tool per registry tool (name, description, JSON schema from``ToolDef.schema()``). Results are content blocks: one ``TextContent`` with theobservation text (+ compact numbers when small) and one ``ImageContent`` perobservation image (base64 PNG, long side ≤ 1024 px, at most ``MAX_IMAGES``).+``is_error`` is ``Observation.failed`` (the tool could not run) — never a+negative verdict, which is an ordinary result whose text leads with FAIL.Track / language default to ``<ws>/spec.json`` so agents need no flags."""@@ -90,7 +92,10 @@ def make_server(ctx: ToolContext):if tdef is None:return types.CallToolResult(content=[types.TextContent(type="text", text=f"unknown tool {params.name!r}; known: {sorted(defs)}")], is_error=True)obs = await asyncio.to_thread(tdef.call, ctx, params.arguments or {})- return types.CallToolResult(content=observation_content(obs), is_error=not obs.ok)+ # is_error is Observation.failed, NOT `not ok`: a tool that RAN and answered FAIL+ # is a result the model must read, and an MCP error is a call it retries instead+ # (~117k prompt tokens a retry). The FAIL verdict leads the text.+ return types.CallToolResult(content=observation_content(obs), is_error=obs.failed)return Server(SERVER_NAME, version="0.1.0",instructions="3dcodeverse spatial tools: build, measure, render and check the 3D object in this workspace.",diff --git a/harness/codeverse/spatial/observe.py b/harness/codeverse/spatial/observe.pyindex 2fce8da7..0ba5470b 100644--- a/harness/codeverse/spatial/observe.py+++ b/harness/codeverse/spatial/observe.py@@ -105,6 +105,7 @@ def text_observation(lines: Sequence[str] | str,*,ok: bool = True,+ failed: bool = False,numbers: dict[str, Any] | None = None,images: Sequence[str | Path] = (),limit: int = MAX_TEXT,@@ -114,10 +115,12 @@ def text_observation(The plain-text counterpart of :func:`gate_observation` / :func:`render_observation`— every tool that assembles its own report (build, gl_probe, gl_frames,texture_pass, …) goes through here so truncation and the image budget are- applied exactly once, in one place.+ applied exactly once, in one place. ``ok=False`` is a verdict (LINT/BUILD+ FAILED is the answer, and the text says so first); ``failed`` is only for a+ tool that could not run."""body = lines if isinstance(lines, str) else "\n".join(str(x) for x in lines)- return Observation(ok=ok, text=truncate(body, limit), numbers=dict(numbers or {}),+ return Observation(ok=ok, failed=failed, text=truncate(body, limit), numbers=dict(numbers or {}),images=image_budget([str(i) for i in images]))@@ -168,11 +171,16 @@ def build_failure_lines(br: BuildResult, root: Path, lint_warns: Sequence[str],def gate_observation(report: GateReport, *, title: str = "", max_findings: int = 20, images: Iterable[str] = ()) -> Observation:- """Errors first, each with its fix hint; counts in ``numbers``."""+ """Errors first, each with its fix hint; counts in ``numbers``.++ The verdict leads the text for every gate, ``title`` or not: a failing gate is a+ result, not a tool error (``Observation.failed`` stays False), so PASS/FAIL is all+ the model has to go on. ``title`` names the gate, never the verdict.+ """findings = sorted(report.findings, key=lambda f: _SEV_ORDER.get(f.severity, 3))n_err = sum(1 for f in findings if f.severity == Severity.ERROR)n_warn = sum(1 for f in findings if f.severity == Severity.WARN)- head = title or f"{report.gate}: {'PASS' if report.passed else 'FAIL'}"+ head = f"{title or report.gate}: {'PASS' if report.passed else 'FAIL'}"lines = [f"{head} — {n_err} error(s), {n_warn} warning(s)"]for f in findings[:max_findings]:tgt = f" [{f.target}]" if f.target else ""diff --git a/harness/codeverse/spatial/probes.py b/harness/codeverse/spatial/probes.pyindex fffe87e0..7cd8ccf0 100644--- a/harness/codeverse/spatial/probes.py+++ b/harness/codeverse/spatial/probes.py@@ -33,10 +33,11 @@ _BOOT_HINTS = {class SceneProbeResult(BaseModel):"""``probe_scene`` result: unpacks as ``gate, census = probe_scene(ws)`` and alsodumps to a dict generic tool adapters can read. Tool semantics: ``ok`` /- ``errors`` describe whether the PROBE TOOL ran (``is_error`` in MCP terms) —- a gate that fails on agent-fixable findings keeps ``ok=True`` with the- finding lines under ``findings``; ``errors`` holds only harness/driver- failures. Gate truth stays on ``gate.passed``."""+ ``errors`` describe whether the PROBE TOOL ran — a gate that fails on+ agent-fixable findings keeps ``ok=True`` with the finding lines under+ ``findings``; ``errors`` holds only harness/driver failures and is what the+ tool maps to ``Observation.failed`` (MCP ``is_error``). Gate truth stays on+ ``gate.passed``, which is the observation's verdict."""gate: GateReportcensus: dict[str, Any] = Field(default_factory=dict)diff --git a/harness/codeverse/spatial/registry.py b/harness/codeverse/spatial/registry.pyindex 2278fc07..5d9fc7e3 100644--- a/harness/codeverse/spatial/registry.py+++ b/harness/codeverse/spatial/registry.py@@ -13,7 +13,9 @@ Usage::Observation, ``ToolUnavailable`` (a missing sibling package) an "unavailable"one, anything else a failed Observation — a tool body never needs try/except.Observations: ``text`` (what the agent reads), ``numbers`` (machine-readable),-``images`` (paths, small PNGs the agent may view), ``ok``.+``images`` (paths, small PNGs the agent may view), ``ok`` (the VERDICT) and+``failed`` (the tool could not run). The two are different answers and only+``failed`` may reach the model as a protocol error — see :class:`Observation`."""from __future__ import annotations@@ -33,7 +35,20 @@ class NoArgs(BaseModel):class Observation(BaseModel):+ """A tool's answer. ``ok`` is the VERDICT (the gate passed, the build compiled,+ nothing penetrates); ``failed`` says the tool could not run at all.++ Only ``failed`` becomes MCP ``is_error`` (``spatial.mcp_server``). Measured over+ 217 recorded gemini-cli sessions: with ``is_error = not ok``, 63% of 1404+ joint_sweep calls, 23% of 2073 builds and ~15% of the connectivity/contract calls+ reached the model as broken calls, and the model retries a broken call — ~117k+ prompt tokens ≈ $0.034 each. A negative verdict is a result: it keeps+ ``failed=False`` and leads its ``text`` with the FAIL verdict instead.+ """+ok: bool = True+ failed: bool = Field(default=False, description="the tool could not run: exception, missing "+ "artefact, unusable arguments (NOT a negative verdict)")text: str = Field(description="human/LLM-readable summary (≤ ~2k chars)")numbers: dict[str, Any] = Field(default_factory=dict)images: list[str] = Field(default_factory=list, description="PNG paths (small, labelled)")@@ -41,7 +56,8 @@ class Observation(BaseModel):@classmethoddef error(cls, text: str, **numbers: Any) -> Observation:- return cls(ok=False, text=text, numbers=numbers)+ """The tool could not run — the ONE constructor for a genuine failure."""+ return cls(ok=False, failed=True, text=text, numbers=numbers)class ToolUnavailable(RuntimeError):diff --git a/harness/codeverse/spatial/tools.py b/harness/codeverse/spatial/tools.pyindex 86089f39..2a23e71a 100644--- a/harness/codeverse/spatial/tools.py+++ b/harness/codeverse/spatial/tools.py@@ -28,7 +28,12 @@ from codeverse.spatial.frame_metrics import frame_summary_textfrom codeverse.spatial.joints_export import ARTICULATION_SHEET_NAME, render_posesfrom codeverse.spatial.joints_model import UrdfError, load_urdffrom codeverse.spatial.joints_poses import limit_poses, pose_samples-from codeverse.spatial.joints_sweep import report_numbers, summary_text, sweep_collisions+from codeverse.spatial.joints_sweep import (+ SweepReport,+ report_numbers,+ summary_text,+ sweep_collisions,+)from codeverse.spatial.measure import GlbLoadError, measure_glb, measure_summary_tablefrom codeverse.spatial.observe import (build_failure_lines,@@ -113,6 +118,20 @@ def _no_glb_summary(ctx: ToolContext, br: BuildResult, language: str) -> tuple[breturn True, lines, {"census": keep}+def _gl_gate_verdict(numbers: dict[str, Any]) -> list[str]:+ """The FRAME GATE line that goes ABOVE the '… OK' headline, or [] when it passed.++ The shader compiled and the frames rendered, so the gate verdict is the answer and+ not an error to retry (``Observation.failed`` stays False) — it has to be readable+ at the top of the text, where the '… OK' headline would otherwise be the only+ verdict the model sees. Shared by ``build`` (graphics), ``gl_probe``, ``gl_frames``.+ """+ n = int(numbers.get("gate_errors", 0))+ if not n:+ return []+ return [f"FRAME GATE: FAIL — {n} error(s) listed below (the code ran; fix the frames, do not re-run blind)"]++#: what `build` says about itself when the gates ride along (docs/COST.md §29): the agent#: must not spend two more round trips asking for what the build observation already holds._BUILD_INCLUDES_CHECKS = ("On success it ALSO runs check_connectivity and check_contract and reports them "@@ -133,8 +152,7 @@ def _folded_checks(ctx: ToolContext, glb: Path, m: Measurement, language: str) -def fold(name: str, report: GateReport) -> None:ws.write_json(ws.gates_dir(ctx.round_index) / f"{name.lower()}_tool.json", report)- obs = gate_observation(report, title=f"{name}: {'PASS' if report.passed else 'FAIL'}",- max_findings=_FOLDED_MAX_FINDINGS)+ obs = gate_observation(report, title=name, max_findings=_FOLDED_MAX_FINDINGS)n_err = int(obs.numbers["errors"])lines.append(truncate(obs.text, _FOLDED_SECTION_CHARS))numbers[f"{name.lower()}_errors"] = n_err@@ -205,19 +223,25 @@ def build(ctx: ToolContext, args: NoArgs) -> Observation:if not br.ok:return _build_failed(ctx, br, lint_warns)numbers: dict[str, Any] = {"stage": "build", "ok": br.ok, "duration_ms": br.duration_ms}+ broken = Falseif not br.glb_path and language in _SCENE_LANGS + _GL_LANGS:# languages without a GLB deliverable: report the language's own artifactsok, extra_lines, extra_numbers = _no_glb_summary(ctx, br, language)- lines = [f"BUILD OK ({br.duration_ms} ms)"] + extra_lines+ lines = _gl_gate_verdict(extra_numbers) + [f"BUILD OK ({br.duration_ms} ms)"] + extra_linesnumbers.update(extra_numbers)m = Noneelse:m, table = _measure_after_build(ctx, br)ok = m is not None- lines = [f"BUILD OK ({br.duration_ms} ms) → {rel_path(br.glb_path, ws.root)}"]+ # the runtime reported success and left nothing measurable: the tool could not+ # answer (not a verdict on the code), and the headline must not read BUILD OK+ broken = m is None+ lines = [f"BUILD OK ({br.duration_ms} ms) → {rel_path(br.glb_path, ws.root)}" if ok+ else f"BUILD PRODUCED NO USABLE GLB ({br.duration_ms} ms): {table}"]if br.extra_paths:lines.append("extras: " + ", ".join(f"{k}={rel_path(v, ws.root)}" for k, v in br.extra_paths.items()))- lines.append(table)+ if ok:+ lines.append(table)if m is not None:numbers.update({"extents_m": list(m.extents), "tri_count": m.tri_count, "n_parts": len(m.parts),"n_islands": m.n_islands, "ground_gap_m": m.ground_gap_m,@@ -238,7 +262,7 @@ def build(ctx: ToolContext, args: NoArgs) -> Observation:# the head/tail truncation eats the middle of the measurement table, never a gate errorlines[1:1] = foldednumbers.update(folded_numbers)- return text_observation(lines, ok=ok, numbers=numbers, limit=3000)+ return text_observation(lines, ok=ok, failed=broken, numbers=numbers, limit=3000)#: the object-GLB toolset (everything that reads artifacts/object.glb): scene and@@ -441,6 +465,24 @@ class JointSweepArgs(BaseModel):n_samples: int = Field(default=8, ge=2, le=32, description="poses per joint across its range")+def _sweep_verdict(report: SweepReport) -> tuple[bool, str]:+ """(passed, headline) of a collision sweep — the ONE place that decides both.++ The headline leads the observation because a sweep that finds a penetration is a+ RESULT, not a tool error (63% of 1404 recorded joint_sweep calls answered FAIL):+ without it the model reads 'pose sweep: 12 poses…' and has to infer the verdict.+ """+ s = report.summary+ reasons = []+ if s.max_penetration_m > report.tol_m:+ reasons.append(f"penetration {s.max_penetration_m * 1000:.1f} mm > tolerance {report.tol_m * 1000:.1f} mm")+ if s.floating_links:+ reasons.append(f"{len(s.floating_links)} floating link(s): {', '.join(s.floating_links[:6])}")+ if reasons:+ return False, "JOINT SWEEP: FAIL — " + "; ".join(reasons) + " (the sweep ran; fix the links below)"+ return True, "JOINT SWEEP: PASS — no penetration beyond tolerance, every link attached"++def _poses_for(robot, joints: list[str] | None) -> list[tuple[str, dict[str, float]]] | None:"""``limit_poses`` narrowed to ``joints`` (rest kept); ``None`` = the full sheet.@@ -475,8 +517,8 @@ def joint_sweep(ctx: ToolContext, args: JointSweepArgs) -> Observation:return Observation.error(f"joint_sweep: {e}")out_dir = tool_out_dir(ctx, "joints")render_poses(robot, out_dir, poses=_poses_for(robot, args.joints or None))- ok = report.summary.max_penetration_m <= report.tol_m and not report.summary.floating_links- return Observation(ok=ok, text=summary_text(report), numbers=report_numbers(report),+ ok, headline = _sweep_verdict(report)+ return Observation(ok=ok, text=f"{headline}\n{summary_text(report)}", numbers=report_numbers(report),images=[str(out_dir / ARTICULATION_SHEET_NAME)])@@ -491,10 +533,10 @@ def shader_probe(ctx: ToolContext, args: NoArgs) -> Observation:languages=(Language.SCENE_THREEJS.value,), cost_hint="slow")def scene_probe(ctx: ToolContext, args: NoArgs) -> Observation:res = probe_scene(ctx.workspace)- # ok = the probe TOOL ran (SceneProbeResult semantics): a gate that fails on- # agent-fixable findings stays ok; only harness/driver failures flip it+ # ok = the gate verdict; failed = the probe TOOL could not run (SceneProbeResult+ # semantics): agent-fixable findings are a FAIL the agent must read, not an errorobs = gate_observation(res.gate, title="scene probe")- obs.ok = not res.errors+ obs.failed = bool(res.errors)if res.errors:obs.text += "\nerrors:\n" + "\n".join(f" ! {e[:200]}" for e in res.errors[:10])if res.census:@@ -628,7 +670,7 @@ def gl_probe(ctx: ToolContext, args: GlProbeArgs) -> Observation:assert br is not Nonetext, numbers = _stats_text(ctx)frames = sorted(Path(br.extra_paths["frames"]).glob("f*_t*.png")) if br.extra_paths.get("frames") else []- lines = [f"PROBE OK ({br.duration_ms} ms, {br.census.get('renderer', 'GL')}) — frame at t={args.t:g}s", text]+ lines = _gl_gate_verdict(numbers) + [f"PROBE OK ({br.duration_ms} ms, {br.census.get('renderer', 'GL')}) — frame at t={args.t:g}s", text]if warns:lines.append("lint warnings:\n" + "\n".join(warns[:_LINT_WARNS_SHOWN]))numbers.update({"stage": "probe", "t": args.t, "duration_ms": br.duration_ms})@@ -655,7 +697,7 @@ def gl_frames(ctx: ToolContext, args: GlFramesArgs) -> Observation:dst = out_dir / "sheet.png"shutil.copy2(sheet, dst)images.append(str(dst))- lines = [f"FRAMES OK ({br.duration_ms} ms, {br.census.get('renderer', 'GL')}) — sheet tiles labelled t=<s>; compare them for motion", text]+ lines = _gl_gate_verdict(numbers) + [f"FRAMES OK ({br.duration_ms} ms, {br.census.get('renderer', 'GL')}) — sheet tiles labelled t=<s>; compare them for motion", text]if warns:lines.append("lint warnings:\n" + "\n".join(warns[:_LINT_WARNS_SHOWN]))numbers.update({"stage": "frames", "times": args.times, "duration_ms": br.duration_ms, "sheet": rel_path(images[0], ctx.workspace.root) if images else ""})diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.mdindex 30248ac4..6875f981 100644--- a/harness/docs/ARCHITECTURE.md+++ b/harness/docs/ARCHITECTURE.md@@ -336,6 +336,14 @@ server (name `3dcv`) for the vendor CLIs, (c) a native tool schema for any embed`joint_sweep` (articulated), `shader_probe`, `scene_probe`, `scene_views` (scene),`gl_probe`, `gl_frames` (graphics), `texture_pass`, `texture_preview` (object tracks).+An `Observation` carries two different answers: `ok` is the **verdict** and `failed`+says the tool **could not run** (exception, missing artefact, unusable arguments —+`Observation.error` is the only constructor that sets it). MCP `is_error` is `failed`+alone: a negative verdict is a result whose text leads with FAIL, because a vendor CLI+retries an errored call, and over 217 recorded gemini-cli sessions `is_error = not ok`+made 63 % of 1 404 `joint_sweep` calls, 23 % of 2 073 `build`s and ~15 % of the+connectivity/contract calls look broken (~117 k prompt tokens ≈ $0.034 a retry).+## 6. Judging (protocol v2)`VlmJudge(rubric, model_id, n_samples)` sends **montages, not loose views**: ≤ 5diff --git a/harness/docs/COST.md b/harness/docs/COST.mdindex 70a4037c..f5182428 100644--- a/harness/docs/COST.md+++ b/harness/docs/COST.md@@ -1373,3 +1373,34 @@ the write verdict) — they cost nothing and remove a class of turn; the next tucaps refine fan-out per round and inlines the whole part set under a size budget, and A/Bs theturn-discipline prompt on its own, since "finish in fewer turns" is where the must-item missescome from.++## 30. A tool's FAIL verdict is not an MCP error (2026-09-02)++`spatial/mcp_server.py` returned `is_error = not obs.ok`, and `obs.ok` was the tool's+**verdict** — so every gate that ran and answered FAIL reached the model as a broken call.+From the tool stats of 217 recorded gemini-cli sessions (`run/trajectories/*/stdout.json`):++| tool | calls | reported as an error | the harness's own gate, for comparison |+|---|---|---|---|+| `joint_sweep` | 1 404 | 63 % | end-of-round sweep gate fails 16 % of 243 rounds |+| `build` | 2 073 | 23 % | — (a build that does not compile IS the answer) |+| `check_contract` | 877 | 16 % | — |+| `check_connectivity` | 888 | 15 % | — |++A vendor CLI treats an errored tool call as a call that did not happen and retries it; a+retried request carries the whole session context, ~117 000 prompt tokens ≈ **$0.034**.++The fix is a second flag, not a threshold: `Observation.failed` (set only by+`Observation.error` — exception, missing artefact, unusable arguments, `ToolUnavailable` /+`ToolUsageError`) is what the MCP server reports as `is_error`; `ok` stays the verdict, and+every negative verdict now LEADS its text with `… FAIL` (`gate_observation` for the gates,+`JOINT SWEEP: FAIL — penetration …`, `FRAME GATE: FAIL — …`, `BUILD FAILED` as before) so the+model reads the answer instead of retrying the question.++What to watch on the next battery: in `tools stats`, `joint_sweep` / `build` /+`check_*` error counts → near zero (only genuine failures — no URDF, unreadable GLB,+bad arguments), calls per run down by the retries that used to follow each of them, and+`telemetry/cost.jsonl` prompt tokens per session down accordingly. The 63 % vs 16 % gap+between the sweep TOOL's verdict and the round's sweep GATE is a separate question — the tool+flags any overlap past `tol_m` and any floating link, while `sweep_findings` downgrades small+rest overlaps and hinge gaps to WARN.diff --git a/harness/docs/INTERFACES.md b/harness/docs/INTERFACES.mdindex 1a355f63..e409a5d4 100644--- a/harness/docs/INTERFACES.md+++ b/harness/docs/INTERFACES.md@@ -310,6 +310,8 @@ GlHost(gpu="auto|on|off", timeout_s=240, fps=30, max_steps=240)from codeverse.spatial.frame_stats import sequence_stats, frame_gate # gate "gl_frames"; data.kind ∈ nan | black | blown |# static | flicker | low_detail | duplicate | no_framesfrom codeverse.spatial.registry import tool, get_tool, list_tools, tool_cards, ToolContext, Observation+Observation{ok: VERDICT, failed: the tool could not run, text, numbers, images, duration_ms} # is_error == failed,+ # never `not ok`; Observation.error(...) is the ONE constructor that sets failedimport codeverse.spatial.tools # registers: build, measure, render_views, render_sheet, isolate, cross_section,# check_connectivity, check_contract, compare_silhouette, joint_sweep [articulated], shader_probe, scene_probe,# scene_views + check_placement [scene], gl_probe + gl_frames [graphics], texture_pass + texture_preview [object tracks]diff --git a/harness/tests/spatial_tools/test_mcp_server.py b/harness/tests/spatial_tools/test_mcp_server.pyindex b81b81d5..a3f12329 100644--- a/harness/tests/spatial_tools/test_mcp_server.py+++ b/harness/tests/spatial_tools/test_mcp_server.py@@ -64,14 +64,19 @@ def test_stdio_roundtrip(stool_ctx: ToolContext, tmp_path: Path) -> None:assert by_name["render_views"].input_schema["properties"]["views"]["type"] == "array"conn = await s.call_tool("check_connectivity", {})assert conn.content and conn.content[0].type == "text"- assert "connectivity" in conn.content[0].text # the gate's own header+ # the gate ran and answered FAIL: a RESULT, not a protocol error the model retries+ assert not conn.is_error+ assert conn.content[0].text.startswith("connectivity: FAIL") # the verdict leadsassert "Leg_3" in conn.content[0].text # the fixture's floating legmeas = await s.call_tool("measure", {})assert not meas.is_error and "Leg ×4" in meas.content[0].textsec = await s.call_tool("cross_section", {"axis": "y", "at": 0.5})assert [c.type for c in sec.content] == ["text", "image"]+ # ... and a tool that could NOT run is still an error, whichever way it brokebad = await s.call_tool("measure", {"parts": 3})assert bad.is_error and "invalid arguments" in bad.content[0].text+ unusable = await s.call_tool("isolate", {"part": "Nope"})+ assert unusable.is_error and "unknown part" in unusable.content[0].textunknown = await s.call_tool("nope", {})assert unknown.is_errordiff --git a/harness/tests/spatial_tools/test_tools.py b/harness/tests/spatial_tools/test_tools.pyindex 1105e804..492ed9b6 100644--- a/harness/tests/spatial_tools/test_tools.py+++ b/harness/tests/spatial_tools/test_tools.py@@ -74,7 +74,8 @@ def test_tools_without_glb(tmp_ws: Workspace) -> None:ctx = ToolContext(workspace=tmp_ws, language="blender")for name in ("measure", "check_connectivity", "cross_section", "render_views", "isolate"):obs = get_tool(name).call(ctx, {"part": "x"} if name == "isolate" else {})- assert not obs.ok and "run `build` first" in obs.text, name+ # a missing artefact IS a failure: the tool could not run at all+ assert not obs.ok and obs.failed and "run `build` first" in obs.text, namedef test_connectivity_and_section_tools(stool_ctx: ToolContext) -> None:@@ -159,6 +160,49 @@ def test_build_tool_lint_blocks(stool_ctx: ToolContext, monkeypatch: pytest.Monkassert obs.numbers["stage"] == "lint"+def test_a_negative_verdict_is_not_a_tool_failure(stool_ctx: ToolContext, monkeypatch: pytest.MonkeyPatch) -> None:+ """``ok`` is the VERDICT, ``failed`` is 'the tool could not run', and only ``failed``+ reaches the model as an MCP error. Over 217 recorded gemini-cli sessions the old+ ``is_error = not ok`` reported 63% of 1404 joint_sweep calls and 23% of 2073 builds as+ broken calls, and the model retries a broken call at ~117k prompt tokens each. Both+ directions here, plus the rule that makes it safe: the FAIL verdict LEADS the text."""+ import codeverse.spatial.tools as ts++ ws = stool_ctx.workspace+ ws.write_json(ws.plan_path, _stool_plan_with_missing_backrest())+ for name, head in (("check_connectivity", "connectivity: FAIL"), ("check_contract", "contract: FAIL")):+ obs = get_tool(name).call(stool_ctx, {})+ assert not obs.ok and not obs.failed, name # the gate ran and answered FAIL+ assert obs.text.startswith(head), obs.text++ # the other direction: a tool that could not run stays an error+ obs = get_tool("measure").call(stool_ctx, {"parts": ["Nope"]}) # ToolUsageError+ assert not obs.ok and obs.failed and "Seat" in obs.text+ def boom(*_a, **_k):+ raise RuntimeError("boom")++ monkeypatch.setattr(ts, "measure_glb", boom)+ obs = get_tool("measure").call(stool_ctx, {})+ assert not obs.ok and obs.failed and "measure failed: RuntimeError: boom" in obs.text+ monkeypatch.undo()++ _patch_runtime(monkeypatch, _FakeRuntime(build_ok=False)) # code that will not run+ obs = get_tool("build").call(stool_ctx, {})+ assert not obs.ok and not obs.failed and obs.text.startswith("BUILD FAILED")+ _patch_runtime(monkeypatch, _FakeRuntime(lint_errors=True))+ obs = get_tool("build").call(stool_ctx, {})+ assert not obs.ok and not obs.failed and obs.text.startswith("LINT FAILED")+++def test_build_that_leaves_no_readable_glb_is_a_failure(stool_ctx: ToolContext, monkeypatch: pytest.MonkeyPatch) -> None:+ """The runtime reported success and left nothing measurable: the tool could not+ answer (a real error), and the headline must not read BUILD OK."""+ _patch_runtime(monkeypatch, _FakeRuntime(glb=stool_ctx.workspace.artifacts / "gone.glb"))+ obs = get_tool("build").call(stool_ctx, {})+ assert not obs.ok and obs.failed+ assert obs.text.startswith("BUILD PRODUCED NO USABLE GLB") and "GLB unreadable" in obs.text++def test_build_failure_refuses_stale_glb(stool_ctx: ToolContext, monkeypatch: pytest.MonkeyPatch) -> None:"""A failed build after a success: the old object.glb survives as evidence butglb_path refuses to treat it as current (build_last.json says ok:false)."""@@ -319,15 +363,19 @@ def test_scene_tools_with_fake_siblings(stool_ctx: ToolContext, monkeypatch: pytmonkeypatch.setattr(ts, "check_shaders", fake_check_shaders)monkeypatch.setattr(ts, "probe_scene", fake_probe_scene)obs = get_tool("shader_probe").call(stool_ctx, {})- assert not obs.ok and "vUv" in obs.text and "declare varying" in obs.text+ assert not obs.ok and not obs.failed and "vUv" in obs.text and "declare varying" in obs.text+ assert obs.text.startswith("shader probe: FAIL") # a shader that will not compile is a verdictobs = get_tool("scene_probe").call(stool_ctx, {})- # ok = the probe TOOL ran; the failed gate is a result, not a tool error (SceneProbeResult)- assert obs.ok and "meshes=12" in obs.text and "boom" in obs.text and obs.numbers["census"]["meshes"] == 12+ # ok = the gate verdict; failed = the probe TOOL could not run. The gate fails on+ # agent-fixable findings here, so the observation is a FAIL, not a tool error+ assert not obs.ok and not obs.failed+ assert obs.text.startswith("scene probe: FAIL")+ assert "meshes=12" in obs.text and "boom" in obs.text and obs.numbers["census"]["meshes"] == 12monkeypatch.setattr(ts, "probe_scene", lambda ws, **kw: SceneProbeResult(gate=failed, errors=["driver died"], ok=False))obs = get_tool("scene_probe").call(stool_ctx, {})- assert not obs.ok and "driver died" in obs.text+ assert not obs.ok and obs.failed and "driver died" in obs.textobs = get_tool("joint_sweep").call(stool_ctx, {})- assert not obs.ok and "run `build`" in obs.text # no robot.urdf in a static workspace+ assert not obs.ok and obs.failed and "run `build`" in obs.text # no robot.urdf in a static workspace# --------------------------------------------------------------------------- build without a GLB (scene / graphics)diff --git a/harness/tests/urdf_joints/test_export.py b/harness/tests/urdf_joints/test_export.pyindex 8dd257b5..0f73de66 100644--- a/harness/tests/urdf_joints/test_export.py+++ b/harness/tests/urdf_joints/test_export.py@@ -120,11 +120,33 @@ def test_joint_sweep_tool_offline(tmp_path, monkeypatch):monkeypatch.setattr(ts, "render_poses", fake_render_poses)obs = get_tool("joint_sweep").call(ctx, {"joints": ["hinge"], "n_samples": 5})assert obs.ok and obs.numbers["max_penetration_m"] == 0.0+ assert obs.text.startswith("JOINT SWEEP: PASS")assert rendered == [["rest", "hinge@upper"]] and obs.images[0].endswith(ARTICULATION_SHEET_NAME) # lower=0 dedupes into restobs = get_tool("joint_sweep").call(ctx, {})assert rendered[-1] is None # empty joints = the full sheet (render_poses default)+def test_joint_sweep_penetration_is_a_verdict_not_an_mcp_error(tmp_path, monkeypatch):+ """A sweep that finds a penetration RAN: ``failed`` stays False (63% of 1404 recorded+ joint_sweep calls answered FAIL, and each one reported as an MCP error bought a retry+ at ~117k prompt tokens), and the FAIL verdict leads the text."""+ import codeverse.spatial.tools as ts+ from codeverse.spatial.registry import ToolContext, get_tool+ from codeverse.workspace import Workspace+ from tests.urdf_joints.conftest import write_prims_robot++ ws = Workspace(tmp_path / "ws").create()+ write_prims_robot(ws.artifacts / "robot.urdf", axis_z=+1) # the door swings into the body+ monkeypatch.setattr(ts, "render_poses", lambda robot, out_dir, poses=None, **kw: [+ Image.new("RGB", (8, 8), "gray").save(out_dir / ARTICULATION_SHEET_NAME)])+ ctx = ToolContext(workspace=ws, language="urdf_blender", track="articulated_object")+ obs = get_tool("joint_sweep").call(ctx, {})+ assert not obs.ok and not obs.failed+ assert obs.text.startswith("JOINT SWEEP: FAIL — penetration ") and "tolerance" in obs.text+ assert "pose sweep:" in obs.text # the detail the agent acts on is still there+ assert obs.numbers["max_penetration_m"] > 0.1++def test_robot_named_like_a_link_keeps_frame_and_placement(tmp_path):"""``<robot name="body">`` with a root link ``body``: scene-graph node names must beunique, so the root node gets a ``__root`` suffix instead of aliasing the link node