Skip to content

Official Codebase of 3DCodeVerse

Python

tools: a FAIL verdict is no longer an MCP error — 63% of joint_sweep calls were

ziyaobebe0f8d76dd
diff --git a/harness/codeverse/cli/_common.py b/harness/codeverse/cli/_common.py
index 3123c9ee..c9c5fa8c 100644
--- a/harness/codeverse/cli/_common.py
+++ b/harness/codeverse/cli/_common.py
@@ -108,8 +108,10 @@ def print_observation(obs: Any, *, as_json: bool) -> None:
if as_json:
console.print_json(obs.model_dump_json())
return
- style = "green" if obs.ok else "red"
- console.print(Panel(obs.text, title=f"[{style}]{'ok' if obs.ok else 'error'}[/{style}] ({obs.duration_ms} ms)", expand=False))
+ # three states, not two: a FAIL verdict is an answer the tool computed, an error is a
+ # tool that could not run (the same split the MCP server reports as is_error)
+ state, style = ("ok", "green") if obs.ok else (("error", "red") if obs.failed else ("FAIL", "yellow"))
+ console.print(Panel(obs.text, title=f"[{style}]{state}[/{style}] ({obs.duration_ms} ms)", expand=False))
if obs.numbers:
console.print(kv_table("numbers", obs.numbers))
for img in obs.images:
diff --git a/harness/codeverse/spatial/mcp_server.py b/harness/codeverse/spatial/mcp_server.py
index 2415e368..e7d474a5 100644
--- a/harness/codeverse/spatial/mcp_server.py
+++ b/harness/codeverse/spatial/mcp_server.py
@@ -6,6 +6,8 @@ One MCP tool per registry tool (name, description, JSON schema from
``ToolDef.schema()``). Results are content blocks: one ``TextContent`` with the
observation text (+ compact numbers when small) and one ``ImageContent`` per
observation image (base64 PNG, long side ≤ 1024 px, at most ``MAX_IMAGES``).
+``is_error`` is ``Observation.failed`` (the tool could not run) — never a
+negative verdict, which is an ordinary result whose text leads with FAIL.
Track / language default to ``<ws>/spec.json`` so agents need no flags.
"""
@@ -90,7 +92,10 @@ def make_server(ctx: ToolContext):
if tdef is None:
return types.CallToolResult(content=[types.TextContent(type="text", text=f"unknown tool {params.name!r}; known: {sorted(defs)}")], is_error=True)
obs = await asyncio.to_thread(tdef.call, ctx, params.arguments or {})
- return types.CallToolResult(content=observation_content(obs), is_error=not obs.ok)
+ # is_error is Observation.failed, NOT `not ok`: a tool that RAN and answered FAIL
+ # is a result the model must read, and an MCP error is a call it retries instead
+ # (~117k prompt tokens a retry). The FAIL verdict leads the text.
+ return types.CallToolResult(content=observation_content(obs), is_error=obs.failed)
return Server(SERVER_NAME, version="0.1.0",
instructions="3dcodeverse spatial tools: build, measure, render and check the 3D object in this workspace.",
diff --git a/harness/codeverse/spatial/observe.py b/harness/codeverse/spatial/observe.py
index 2fce8da7..0ba5470b 100644
--- a/harness/codeverse/spatial/observe.py
+++ b/harness/codeverse/spatial/observe.py
@@ -105,6 +105,7 @@ def text_observation(
lines: Sequence[str] | str,
*,
ok: bool = True,
+ failed: bool = False,
numbers: dict[str, Any] | None = None,
images: Sequence[str | Path] = (),
limit: int = MAX_TEXT,
@@ -114,10 +115,12 @@ def text_observation(
The plain-text counterpart of :func:`gate_observation` / :func:`render_observation`
— every tool that assembles its own report (build, gl_probe, gl_frames,
texture_pass, …) goes through here so truncation and the image budget are
- applied exactly once, in one place.
+ applied exactly once, in one place. ``ok=False`` is a verdict (LINT/BUILD
+ FAILED is the answer, and the text says so first); ``failed`` is only for a
+ tool that could not run.
"""
body = lines if isinstance(lines, str) else "\n".join(str(x) for x in lines)
- return Observation(ok=ok, text=truncate(body, limit), numbers=dict(numbers or {}),
+ return Observation(ok=ok, failed=failed, text=truncate(body, limit), numbers=dict(numbers or {}),
images=image_budget([str(i) for i in images]))
@@ -168,11 +171,16 @@ def build_failure_lines(br: BuildResult, root: Path, lint_warns: Sequence[str],
def gate_observation(report: GateReport, *, title: str = "", max_findings: int = 20, images: Iterable[str] = ()) -> Observation:
- """Errors first, each with its fix hint; counts in ``numbers``."""
+ """Errors first, each with its fix hint; counts in ``numbers``.
+
+ The verdict leads the text for every gate, ``title`` or not: a failing gate is a
+ result, not a tool error (``Observation.failed`` stays False), so PASS/FAIL is all
+ the model has to go on. ``title`` names the gate, never the verdict.
+ """
findings = sorted(report.findings, key=lambda f: _SEV_ORDER.get(f.severity, 3))
n_err = sum(1 for f in findings if f.severity == Severity.ERROR)
n_warn = sum(1 for f in findings if f.severity == Severity.WARN)
- head = title or f"{report.gate}: {'PASS' if report.passed else 'FAIL'}"
+ head = f"{title or report.gate}: {'PASS' if report.passed else 'FAIL'}"
lines = [f"{head} — {n_err} error(s), {n_warn} warning(s)"]
for f in findings[:max_findings]:
tgt = f" [{f.target}]" if f.target else ""
diff --git a/harness/codeverse/spatial/probes.py b/harness/codeverse/spatial/probes.py
index fffe87e0..7cd8ccf0 100644
--- a/harness/codeverse/spatial/probes.py
+++ b/harness/codeverse/spatial/probes.py
@@ -33,10 +33,11 @@ _BOOT_HINTS = {
class SceneProbeResult(BaseModel):
"""``probe_scene`` result: unpacks as ``gate, census = probe_scene(ws)`` and also
dumps to a dict generic tool adapters can read. Tool semantics: ``ok`` /
- ``errors`` describe whether the PROBE TOOL ran (``is_error`` in MCP terms) —
- a gate that fails on agent-fixable findings keeps ``ok=True`` with the
- finding lines under ``findings``; ``errors`` holds only harness/driver
- failures. Gate truth stays on ``gate.passed``."""
+ ``errors`` describe whether the PROBE TOOL ran — a gate that fails on
+ agent-fixable findings keeps ``ok=True`` with the finding lines under
+ ``findings``; ``errors`` holds only harness/driver failures and is what the
+ tool maps to ``Observation.failed`` (MCP ``is_error``). Gate truth stays on
+ ``gate.passed``, which is the observation's verdict."""
gate: GateReport
census: dict[str, Any] = Field(default_factory=dict)
diff --git a/harness/codeverse/spatial/registry.py b/harness/codeverse/spatial/registry.py
index 2278fc07..5d9fc7e3 100644
--- a/harness/codeverse/spatial/registry.py
+++ b/harness/codeverse/spatial/registry.py
@@ -13,7 +13,9 @@ Usage::
Observation, ``ToolUnavailable`` (a missing sibling package) an "unavailable"
one, anything else a failed Observation — a tool body never needs try/except.
Observations: ``text`` (what the agent reads), ``numbers`` (machine-readable),
-``images`` (paths, small PNGs the agent may view), ``ok``.
+``images`` (paths, small PNGs the agent may view), ``ok`` (the VERDICT) and
+``failed`` (the tool could not run). The two are different answers and only
+``failed`` may reach the model as a protocol error — see :class:`Observation`.
"""
from __future__ import annotations
@@ -33,7 +35,20 @@ class NoArgs(BaseModel):
class Observation(BaseModel):
+ """A tool's answer. ``ok`` is the VERDICT (the gate passed, the build compiled,
+ nothing penetrates); ``failed`` says the tool could not run at all.
+
+ Only ``failed`` becomes MCP ``is_error`` (``spatial.mcp_server``). Measured over
+ 217 recorded gemini-cli sessions: with ``is_error = not ok``, 63% of 1404
+ joint_sweep calls, 23% of 2073 builds and ~15% of the connectivity/contract calls
+ reached the model as broken calls, and the model retries a broken call — ~117k
+ prompt tokens ≈ $0.034 each. A negative verdict is a result: it keeps
+ ``failed=False`` and leads its ``text`` with the FAIL verdict instead.
+ """
+
ok: bool = True
+ failed: bool = Field(default=False, description="the tool could not run: exception, missing "
+ "artefact, unusable arguments (NOT a negative verdict)")
text: str = Field(description="human/LLM-readable summary (≤ ~2k chars)")
numbers: dict[str, Any] = Field(default_factory=dict)
images: list[str] = Field(default_factory=list, description="PNG paths (small, labelled)")
@@ -41,7 +56,8 @@ class Observation(BaseModel):
@classmethod
def error(cls, text: str, **numbers: Any) -> Observation:
- return cls(ok=False, text=text, numbers=numbers)
+ """The tool could not run — the ONE constructor for a genuine failure."""
+ return cls(ok=False, failed=True, text=text, numbers=numbers)
class ToolUnavailable(RuntimeError):
diff --git a/harness/codeverse/spatial/tools.py b/harness/codeverse/spatial/tools.py
index 86089f39..2a23e71a 100644
--- a/harness/codeverse/spatial/tools.py
+++ b/harness/codeverse/spatial/tools.py
@@ -28,7 +28,12 @@ from codeverse.spatial.frame_metrics import frame_summary_text
from codeverse.spatial.joints_export import ARTICULATION_SHEET_NAME, render_poses
from codeverse.spatial.joints_model import UrdfError, load_urdf
from codeverse.spatial.joints_poses import limit_poses, pose_samples
-from codeverse.spatial.joints_sweep import report_numbers, summary_text, sweep_collisions
+from codeverse.spatial.joints_sweep import (
+ SweepReport,
+ report_numbers,
+ summary_text,
+ sweep_collisions,
+)
from codeverse.spatial.measure import GlbLoadError, measure_glb, measure_summary_table
from codeverse.spatial.observe import (
build_failure_lines,
@@ -113,6 +118,20 @@ def _no_glb_summary(ctx: ToolContext, br: BuildResult, language: str) -> tuple[b
return True, lines, {"census": keep}
+def _gl_gate_verdict(numbers: dict[str, Any]) -> list[str]:
+ """The FRAME GATE line that goes ABOVE the '… OK' headline, or [] when it passed.
+
+ The shader compiled and the frames rendered, so the gate verdict is the answer and
+ not an error to retry (``Observation.failed`` stays False) — it has to be readable
+ at the top of the text, where the '… OK' headline would otherwise be the only
+ verdict the model sees. Shared by ``build`` (graphics), ``gl_probe``, ``gl_frames``.
+ """
+ n = int(numbers.get("gate_errors", 0))
+ if not n:
+ return []
+ return [f"FRAME GATE: FAIL — {n} error(s) listed below (the code ran; fix the frames, do not re-run blind)"]
+
+
#: what `build` says about itself when the gates ride along (docs/COST.md §29): the agent
#: must not spend two more round trips asking for what the build observation already holds.
_BUILD_INCLUDES_CHECKS = ("On success it ALSO runs check_connectivity and check_contract and reports them "
@@ -133,8 +152,7 @@ def _folded_checks(ctx: ToolContext, glb: Path, m: Measurement, language: str) -
def fold(name: str, report: GateReport) -> None:
ws.write_json(ws.gates_dir(ctx.round_index) / f"{name.lower()}_tool.json", report)
- obs = gate_observation(report, title=f"{name}: {'PASS' if report.passed else 'FAIL'}",
- max_findings=_FOLDED_MAX_FINDINGS)
+ obs = gate_observation(report, title=name, max_findings=_FOLDED_MAX_FINDINGS)
n_err = int(obs.numbers["errors"])
lines.append(truncate(obs.text, _FOLDED_SECTION_CHARS))
numbers[f"{name.lower()}_errors"] = n_err
@@ -205,19 +223,25 @@ def build(ctx: ToolContext, args: NoArgs) -> Observation:
if not br.ok:
return _build_failed(ctx, br, lint_warns)
numbers: dict[str, Any] = {"stage": "build", "ok": br.ok, "duration_ms": br.duration_ms}
+ broken = False
if not br.glb_path and language in _SCENE_LANGS + _GL_LANGS:
# languages without a GLB deliverable: report the language's own artifacts
ok, extra_lines, extra_numbers = _no_glb_summary(ctx, br, language)
- lines = [f"BUILD OK ({br.duration_ms} ms)"] + extra_lines
+ lines = _gl_gate_verdict(extra_numbers) + [f"BUILD OK ({br.duration_ms} ms)"] + extra_lines
numbers.update(extra_numbers)
m = None
else:
m, table = _measure_after_build(ctx, br)
ok = m is not None
- lines = [f"BUILD OK ({br.duration_ms} ms) → {rel_path(br.glb_path, ws.root)}"]
+ # the runtime reported success and left nothing measurable: the tool could not
+ # answer (not a verdict on the code), and the headline must not read BUILD OK
+ broken = m is None
+ lines = [f"BUILD OK ({br.duration_ms} ms) → {rel_path(br.glb_path, ws.root)}" if ok
+ else f"BUILD PRODUCED NO USABLE GLB ({br.duration_ms} ms): {table}"]
if br.extra_paths:
lines.append("extras: " + ", ".join(f"{k}={rel_path(v, ws.root)}" for k, v in br.extra_paths.items()))
- lines.append(table)
+ if ok:
+ lines.append(table)
if m is not None:
numbers.update({"extents_m": list(m.extents), "tri_count": m.tri_count, "n_parts": len(m.parts),
"n_islands": m.n_islands, "ground_gap_m": m.ground_gap_m,
@@ -238,7 +262,7 @@ def build(ctx: ToolContext, args: NoArgs) -> Observation:
# the head/tail truncation eats the middle of the measurement table, never a gate error
lines[1:1] = folded
numbers.update(folded_numbers)
- return text_observation(lines, ok=ok, numbers=numbers, limit=3000)
+ return text_observation(lines, ok=ok, failed=broken, numbers=numbers, limit=3000)
#: the object-GLB toolset (everything that reads artifacts/object.glb): scene and
@@ -441,6 +465,24 @@ class JointSweepArgs(BaseModel):
n_samples: int = Field(default=8, ge=2, le=32, description="poses per joint across its range")
+def _sweep_verdict(report: SweepReport) -> tuple[bool, str]:
+ """(passed, headline) of a collision sweep — the ONE place that decides both.
+
+ The headline leads the observation because a sweep that finds a penetration is a
+ RESULT, not a tool error (63% of 1404 recorded joint_sweep calls answered FAIL):
+ without it the model reads 'pose sweep: 12 poses…' and has to infer the verdict.
+ """
+ s = report.summary
+ reasons = []
+ if s.max_penetration_m > report.tol_m:
+ reasons.append(f"penetration {s.max_penetration_m * 1000:.1f} mm > tolerance {report.tol_m * 1000:.1f} mm")
+ if s.floating_links:
+ reasons.append(f"{len(s.floating_links)} floating link(s): {', '.join(s.floating_links[:6])}")
+ if reasons:
+ return False, "JOINT SWEEP: FAIL — " + "; ".join(reasons) + " (the sweep ran; fix the links below)"
+ return True, "JOINT SWEEP: PASS — no penetration beyond tolerance, every link attached"
+
+
def _poses_for(robot, joints: list[str] | None) -> list[tuple[str, dict[str, float]]] | None:
"""``limit_poses`` narrowed to ``joints`` (rest kept); ``None`` = the full sheet.
@@ -475,8 +517,8 @@ def joint_sweep(ctx: ToolContext, args: JointSweepArgs) -> Observation:
return Observation.error(f"joint_sweep: {e}")
out_dir = tool_out_dir(ctx, "joints")
render_poses(robot, out_dir, poses=_poses_for(robot, args.joints or None))
- ok = report.summary.max_penetration_m <= report.tol_m and not report.summary.floating_links
- return Observation(ok=ok, text=summary_text(report), numbers=report_numbers(report),
+ ok, headline = _sweep_verdict(report)
+ return Observation(ok=ok, text=f"{headline}\n{summary_text(report)}", numbers=report_numbers(report),
images=[str(out_dir / ARTICULATION_SHEET_NAME)])
@@ -491,10 +533,10 @@ def shader_probe(ctx: ToolContext, args: NoArgs) -> Observation:
languages=(Language.SCENE_THREEJS.value,), cost_hint="slow")
def scene_probe(ctx: ToolContext, args: NoArgs) -> Observation:
res = probe_scene(ctx.workspace)
- # ok = the probe TOOL ran (SceneProbeResult semantics): a gate that fails on
- # agent-fixable findings stays ok; only harness/driver failures flip it
+ # ok = the gate verdict; failed = the probe TOOL could not run (SceneProbeResult
+ # semantics): agent-fixable findings are a FAIL the agent must read, not an error
obs = gate_observation(res.gate, title="scene probe")
- obs.ok = not res.errors
+ obs.failed = bool(res.errors)
if res.errors:
obs.text += "\nerrors:\n" + "\n".join(f" ! {e[:200]}" for e in res.errors[:10])
if res.census:
@@ -628,7 +670,7 @@ def gl_probe(ctx: ToolContext, args: GlProbeArgs) -> Observation:
assert br is not None
text, numbers = _stats_text(ctx)
frames = sorted(Path(br.extra_paths["frames"]).glob("f*_t*.png")) if br.extra_paths.get("frames") else []
- lines = [f"PROBE OK ({br.duration_ms} ms, {br.census.get('renderer', 'GL')}) — frame at t={args.t:g}s", text]
+ lines = _gl_gate_verdict(numbers) + [f"PROBE OK ({br.duration_ms} ms, {br.census.get('renderer', 'GL')}) — frame at t={args.t:g}s", text]
if warns:
lines.append("lint warnings:\n" + "\n".join(warns[:_LINT_WARNS_SHOWN]))
numbers.update({"stage": "probe", "t": args.t, "duration_ms": br.duration_ms})
@@ -655,7 +697,7 @@ def gl_frames(ctx: ToolContext, args: GlFramesArgs) -> Observation:
dst = out_dir / "sheet.png"
shutil.copy2(sheet, dst)
images.append(str(dst))
- lines = [f"FRAMES OK ({br.duration_ms} ms, {br.census.get('renderer', 'GL')}) — sheet tiles labelled t=<s>; compare them for motion", text]
+ lines = _gl_gate_verdict(numbers) + [f"FRAMES OK ({br.duration_ms} ms, {br.census.get('renderer', 'GL')}) — sheet tiles labelled t=<s>; compare them for motion", text]
if warns:
lines.append("lint warnings:\n" + "\n".join(warns[:_LINT_WARNS_SHOWN]))
numbers.update({"stage": "frames", "times": args.times, "duration_ms": br.duration_ms, "sheet": rel_path(images[0], ctx.workspace.root) if images else ""})
diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.md
index 30248ac4..6875f981 100644
--- a/harness/docs/ARCHITECTURE.md
+++ b/harness/docs/ARCHITECTURE.md
@@ -336,6 +336,14 @@ server (name `3dcv`) for the vendor CLIs, (c) a native tool schema for any embed
`joint_sweep` (articulated), `shader_probe`, `scene_probe`, `scene_views` (scene),
`gl_probe`, `gl_frames` (graphics), `texture_pass`, `texture_preview` (object tracks).
+An `Observation` carries two different answers: `ok` is the **verdict** and `failed`
+says the tool **could not run** (exception, missing artefact, unusable arguments —
+`Observation.error` is the only constructor that sets it). MCP `is_error` is `failed`
+alone: a negative verdict is a result whose text leads with FAIL, because a vendor CLI
+retries an errored call, and over 217 recorded gemini-cli sessions `is_error = not ok`
+made 63 % of 1 404 `joint_sweep` calls, 23 % of 2 073 `build`s and ~15 % of the
+connectivity/contract calls look broken (~117 k prompt tokens ≈ $0.034 a retry).
+
## 6. Judging (protocol v2)
`VlmJudge(rubric, model_id, n_samples)` sends **montages, not loose views**: ≤ 5
diff --git a/harness/docs/COST.md b/harness/docs/COST.md
index 70a4037c..f5182428 100644
--- a/harness/docs/COST.md
+++ b/harness/docs/COST.md
@@ -1373,3 +1373,34 @@ the write verdict) — they cost nothing and remove a class of turn; the next tu
caps refine fan-out per round and inlines the whole part set under a size budget, and A/Bs the
turn-discipline prompt on its own, since "finish in fewer turns" is where the must-item misses
come from.
+
+## 30. A tool's FAIL verdict is not an MCP error (2026-09-02)
+
+`spatial/mcp_server.py` returned `is_error = not obs.ok`, and `obs.ok` was the tool's
+**verdict** — so every gate that ran and answered FAIL reached the model as a broken call.
+From the tool stats of 217 recorded gemini-cli sessions (`run/trajectories/*/stdout.json`):
+
+| tool | calls | reported as an error | the harness's own gate, for comparison |
+|---|---|---|---|
+| `joint_sweep` | 1 404 | 63 % | end-of-round sweep gate fails 16 % of 243 rounds |
+| `build` | 2 073 | 23 % | — (a build that does not compile IS the answer) |
+| `check_contract` | 877 | 16 % | — |
+| `check_connectivity` | 888 | 15 % | — |
+
+A vendor CLI treats an errored tool call as a call that did not happen and retries it; a
+retried request carries the whole session context, ~117 000 prompt tokens ≈ **$0.034**.
+
+The fix is a second flag, not a threshold: `Observation.failed` (set only by
+`Observation.error` — exception, missing artefact, unusable arguments, `ToolUnavailable` /
+`ToolUsageError`) is what the MCP server reports as `is_error`; `ok` stays the verdict, and
+every negative verdict now LEADS its text with `… FAIL` (`gate_observation` for the gates,
+`JOINT SWEEP: FAIL — penetration …`, `FRAME GATE: FAIL — …`, `BUILD FAILED` as before) so the
+model reads the answer instead of retrying the question.
+
+What to watch on the next battery: in `tools stats`, `joint_sweep` / `build` /
+`check_*` error counts → near zero (only genuine failures — no URDF, unreadable GLB,
+bad arguments), calls per run down by the retries that used to follow each of them, and
+`telemetry/cost.jsonl` prompt tokens per session down accordingly. The 63 % vs 16 % gap
+between the sweep TOOL's verdict and the round's sweep GATE is a separate question — the tool
+flags any overlap past `tol_m` and any floating link, while `sweep_findings` downgrades small
+rest overlaps and hinge gaps to WARN.
diff --git a/harness/docs/INTERFACES.md b/harness/docs/INTERFACES.md
index 1a355f63..e409a5d4 100644
--- a/harness/docs/INTERFACES.md
+++ b/harness/docs/INTERFACES.md
@@ -310,6 +310,8 @@ GlHost(gpu="auto|on|off", timeout_s=240, fps=30, max_steps=240)
from codeverse.spatial.frame_stats import sequence_stats, frame_gate # gate "gl_frames"; data.kind ∈ nan | black | blown |
# static | flicker | low_detail | duplicate | no_frames
from codeverse.spatial.registry import tool, get_tool, list_tools, tool_cards, ToolContext, Observation
+Observation{ok: VERDICT, failed: the tool could not run, text, numbers, images, duration_ms} # is_error == failed,
+ # never `not ok`; Observation.error(...) is the ONE constructor that sets failed
import codeverse.spatial.tools # registers: build, measure, render_views, render_sheet, isolate, cross_section,
# check_connectivity, check_contract, compare_silhouette, joint_sweep [articulated], shader_probe, scene_probe,
# scene_views + check_placement [scene], gl_probe + gl_frames [graphics], texture_pass + texture_preview [object tracks]
diff --git a/harness/tests/spatial_tools/test_mcp_server.py b/harness/tests/spatial_tools/test_mcp_server.py
index b81b81d5..a3f12329 100644
--- a/harness/tests/spatial_tools/test_mcp_server.py
+++ b/harness/tests/spatial_tools/test_mcp_server.py
@@ -64,14 +64,19 @@ def test_stdio_roundtrip(stool_ctx: ToolContext, tmp_path: Path) -> None:
assert by_name["render_views"].input_schema["properties"]["views"]["type"] == "array"
conn = await s.call_tool("check_connectivity", {})
assert conn.content and conn.content[0].type == "text"
- assert "connectivity" in conn.content[0].text # the gate's own header
+ # the gate ran and answered FAIL: a RESULT, not a protocol error the model retries
+ assert not conn.is_error
+ assert conn.content[0].text.startswith("connectivity: FAIL") # the verdict leads
assert "Leg_3" in conn.content[0].text # the fixture's floating leg
meas = await s.call_tool("measure", {})
assert not meas.is_error and "Leg ×4" in meas.content[0].text
sec = await s.call_tool("cross_section", {"axis": "y", "at": 0.5})
assert [c.type for c in sec.content] == ["text", "image"]
+ # ... and a tool that could NOT run is still an error, whichever way it broke
bad = await s.call_tool("measure", {"parts": 3})
assert bad.is_error and "invalid arguments" in bad.content[0].text
+ unusable = await s.call_tool("isolate", {"part": "Nope"})
+ assert unusable.is_error and "unknown part" in unusable.content[0].text
unknown = await s.call_tool("nope", {})
assert unknown.is_error
diff --git a/harness/tests/spatial_tools/test_tools.py b/harness/tests/spatial_tools/test_tools.py
index 1105e804..492ed9b6 100644
--- a/harness/tests/spatial_tools/test_tools.py
+++ b/harness/tests/spatial_tools/test_tools.py
@@ -74,7 +74,8 @@ def test_tools_without_glb(tmp_ws: Workspace) -> None:
ctx = ToolContext(workspace=tmp_ws, language="blender")
for name in ("measure", "check_connectivity", "cross_section", "render_views", "isolate"):
obs = get_tool(name).call(ctx, {"part": "x"} if name == "isolate" else {})
- assert not obs.ok and "run `build` first" in obs.text, name
+ # a missing artefact IS a failure: the tool could not run at all
+ assert not obs.ok and obs.failed and "run `build` first" in obs.text, name
def test_connectivity_and_section_tools(stool_ctx: ToolContext) -> None:
@@ -159,6 +160,49 @@ def test_build_tool_lint_blocks(stool_ctx: ToolContext, monkeypatch: pytest.Monk
assert obs.numbers["stage"] == "lint"
+def test_a_negative_verdict_is_not_a_tool_failure(stool_ctx: ToolContext, monkeypatch: pytest.MonkeyPatch) -> None:
+ """``ok`` is the VERDICT, ``failed`` is 'the tool could not run', and only ``failed``
+ reaches the model as an MCP error. Over 217 recorded gemini-cli sessions the old
+ ``is_error = not ok`` reported 63% of 1404 joint_sweep calls and 23% of 2073 builds as
+ broken calls, and the model retries a broken call at ~117k prompt tokens each. Both
+ directions here, plus the rule that makes it safe: the FAIL verdict LEADS the text."""
+ import codeverse.spatial.tools as ts
+
+ ws = stool_ctx.workspace
+ ws.write_json(ws.plan_path, _stool_plan_with_missing_backrest())
+ for name, head in (("check_connectivity", "connectivity: FAIL"), ("check_contract", "contract: FAIL")):
+ obs = get_tool(name).call(stool_ctx, {})
+ assert not obs.ok and not obs.failed, name # the gate ran and answered FAIL
+ assert obs.text.startswith(head), obs.text
+
+ # the other direction: a tool that could not run stays an error
+ obs = get_tool("measure").call(stool_ctx, {"parts": ["Nope"]}) # ToolUsageError
+ assert not obs.ok and obs.failed and "Seat" in obs.text
+ def boom(*_a, **_k):
+ raise RuntimeError("boom")
+
+ monkeypatch.setattr(ts, "measure_glb", boom)
+ obs = get_tool("measure").call(stool_ctx, {})
+ assert not obs.ok and obs.failed and "measure failed: RuntimeError: boom" in obs.text
+ monkeypatch.undo()
+
+ _patch_runtime(monkeypatch, _FakeRuntime(build_ok=False)) # code that will not run
+ obs = get_tool("build").call(stool_ctx, {})
+ assert not obs.ok and not obs.failed and obs.text.startswith("BUILD FAILED")
+ _patch_runtime(monkeypatch, _FakeRuntime(lint_errors=True))
+ obs = get_tool("build").call(stool_ctx, {})
+ assert not obs.ok and not obs.failed and obs.text.startswith("LINT FAILED")
+
+
+def test_build_that_leaves_no_readable_glb_is_a_failure(stool_ctx: ToolContext, monkeypatch: pytest.MonkeyPatch) -> None:
+ """The runtime reported success and left nothing measurable: the tool could not
+ answer (a real error), and the headline must not read BUILD OK."""
+ _patch_runtime(monkeypatch, _FakeRuntime(glb=stool_ctx.workspace.artifacts / "gone.glb"))
+ obs = get_tool("build").call(stool_ctx, {})
+ assert not obs.ok and obs.failed
+ assert obs.text.startswith("BUILD PRODUCED NO USABLE GLB") and "GLB unreadable" in obs.text
+
+
def test_build_failure_refuses_stale_glb(stool_ctx: ToolContext, monkeypatch: pytest.MonkeyPatch) -> None:
"""A failed build after a success: the old object.glb survives as evidence but
glb_path refuses to treat it as current (build_last.json says ok:false)."""
@@ -319,15 +363,19 @@ def test_scene_tools_with_fake_siblings(stool_ctx: ToolContext, monkeypatch: pyt
monkeypatch.setattr(ts, "check_shaders", fake_check_shaders)
monkeypatch.setattr(ts, "probe_scene", fake_probe_scene)
obs = get_tool("shader_probe").call(stool_ctx, {})
- assert not obs.ok and "vUv" in obs.text and "declare varying" in obs.text
+ assert not obs.ok and not obs.failed and "vUv" in obs.text and "declare varying" in obs.text
+ assert obs.text.startswith("shader probe: FAIL") # a shader that will not compile is a verdict
obs = get_tool("scene_probe").call(stool_ctx, {})
- # ok = the probe TOOL ran; the failed gate is a result, not a tool error (SceneProbeResult)
- assert obs.ok and "meshes=12" in obs.text and "boom" in obs.text and obs.numbers["census"]["meshes"] == 12
+ # ok = the gate verdict; failed = the probe TOOL could not run. The gate fails on
+ # agent-fixable findings here, so the observation is a FAIL, not a tool error
+ assert not obs.ok and not obs.failed
+ assert obs.text.startswith("scene probe: FAIL")
+ assert "meshes=12" in obs.text and "boom" in obs.text and obs.numbers["census"]["meshes"] == 12
monkeypatch.setattr(ts, "probe_scene", lambda ws, **kw: SceneProbeResult(gate=failed, errors=["driver died"], ok=False))
obs = get_tool("scene_probe").call(stool_ctx, {})
- assert not obs.ok and "driver died" in obs.text
+ assert not obs.ok and obs.failed and "driver died" in obs.text
obs = get_tool("joint_sweep").call(stool_ctx, {})
- assert not obs.ok and "run `build`" in obs.text # no robot.urdf in a static workspace
+ assert not obs.ok and obs.failed and "run `build`" in obs.text # no robot.urdf in a static workspace
# --------------------------------------------------------------------------- build without a GLB (scene / graphics)
diff --git a/harness/tests/urdf_joints/test_export.py b/harness/tests/urdf_joints/test_export.py
index 8dd257b5..0f73de66 100644
--- a/harness/tests/urdf_joints/test_export.py
+++ b/harness/tests/urdf_joints/test_export.py
@@ -120,11 +120,33 @@ def test_joint_sweep_tool_offline(tmp_path, monkeypatch):
monkeypatch.setattr(ts, "render_poses", fake_render_poses)
obs = get_tool("joint_sweep").call(ctx, {"joints": ["hinge"], "n_samples": 5})
assert obs.ok and obs.numbers["max_penetration_m"] == 0.0
+ assert obs.text.startswith("JOINT SWEEP: PASS")
assert rendered == [["rest", "hinge@upper"]] and obs.images[0].endswith(ARTICULATION_SHEET_NAME) # lower=0 dedupes into rest
obs = get_tool("joint_sweep").call(ctx, {})
assert rendered[-1] is None # empty joints = the full sheet (render_poses default)
+def test_joint_sweep_penetration_is_a_verdict_not_an_mcp_error(tmp_path, monkeypatch):
+ """A sweep that finds a penetration RAN: ``failed`` stays False (63% of 1404 recorded
+ joint_sweep calls answered FAIL, and each one reported as an MCP error bought a retry
+ at ~117k prompt tokens), and the FAIL verdict leads the text."""
+ import codeverse.spatial.tools as ts
+ from codeverse.spatial.registry import ToolContext, get_tool
+ from codeverse.workspace import Workspace
+ from tests.urdf_joints.conftest import write_prims_robot
+
+ ws = Workspace(tmp_path / "ws").create()
+ write_prims_robot(ws.artifacts / "robot.urdf", axis_z=+1) # the door swings into the body
+ monkeypatch.setattr(ts, "render_poses", lambda robot, out_dir, poses=None, **kw: [
+ Image.new("RGB", (8, 8), "gray").save(out_dir / ARTICULATION_SHEET_NAME)])
+ ctx = ToolContext(workspace=ws, language="urdf_blender", track="articulated_object")
+ obs = get_tool("joint_sweep").call(ctx, {})
+ assert not obs.ok and not obs.failed
+ assert obs.text.startswith("JOINT SWEEP: FAIL — penetration ") and "tolerance" in obs.text
+ assert "pose sweep:" in obs.text # the detail the agent acts on is still there
+ assert obs.numbers["max_penetration_m"] > 0.1
+
+
def test_robot_named_like_a_link_keeps_frame_and_placement(tmp_path):
"""``<robot name="body">`` with a root link ``body``: scene-graph node names must be
unique, so the root node gets a ``__root`` suffix instead of aliasing the link node