Skip to content

Official Codebase of 3DCodeVerse

Python

planner: a rejected plan is written where a dead run can be read

ziyaof996ec2eadec
diff --git a/harness/bench/plan_stage_bench.py b/harness/bench/plan_stage_bench.py
index e477e8bf..37914d8b 100644
--- a/harness/bench/plan_stage_bench.py
+++ b/harness/bench/plan_stage_bench.py
@@ -121,8 +121,11 @@ def run_one(battery, item, backends, label: str, rep: int, provenance: dict[str,
"started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(started)),
"seconds": round(time.time() - started, 1), **(provenance or {}), **_stats(root / "ws")}
if not ok and keep_failed is not None:
+ # name what it was: a provider block says nothing about the code under test, and a
+ # harvest of validation failures should not have to be filtered by hand
+ kind = "planning" if "PlanningError" in error else "provider"
keep_failed.mkdir(parents=True, exist_ok=True)
- shutil.move(str(root), str(keep_failed / f"{label}_{item.id}_r{rep}"))
+ shutil.move(str(root), str(keep_failed / f"{kind}_{item.id}_r{rep}"))
else:
shutil.rmtree(root, ignore_errors=True)
return row
diff --git a/harness/codeverse/tracks/planner.py b/harness/codeverse/tracks/planner.py
index bc4c3f23..f444231a 100644
--- a/harness/codeverse/tracks/planner.py
+++ b/harness/codeverse/tracks/planner.py
@@ -709,6 +709,27 @@ def restart_enabled() -> bool:
return env_flag(PLAN_RESTART_ENV, True)
+#: an invalid plan attempt is written here, so a run that DIES at the plan stage still says
+#: what the model wrote. Pydantic truncates the offending value in its message, so the
+#: event log alone cannot answer "what did it actually write?" — the question every
+#: plan-stage failure class starts from (docs/PAPER_WRITING.md §5.1).
+INVALID_PLAN_DIR = "stages/plan/invalid"
+#: bound per file: a plan is a few kB; anything larger is a runaway answer and the head of
+#: it is what says so
+MAX_INVALID_PLAN_BYTES = 200_000
+
+
+def _record_invalid(ws: Workspace, attempt: int, raw: Any, text: str) -> None:
+ """Write the rejected answer beside the run. Never raises: this is a diagnostic."""
+ try:
+ body = json.dumps(raw, indent=1, default=str) if isinstance(raw, dict) else (text or "")
+ out = ws.root / INVALID_PLAN_DIR
+ out.mkdir(parents=True, exist_ok=True)
+ (out / f"attempt{attempt:02d}.json").write_text(body[:MAX_INVALID_PLAN_BYTES])
+ except (OSError, TypeError, ValueError) as e: # noqa: BLE001 — a diagnostic never fails a run
+ log.debug("could not record the invalid plan attempt: %s", e)
+
+
def degenerate_plan(raw: Any, budget: PlanBudget) -> bool:
"""A plan too small to be an attempt at this request: a third of the floor or less.
@@ -878,6 +899,7 @@ def plan_with_usage[P: BaseModel](spec: Spec, model_id: str, plan_model: type[P]
result = plan_model.model_validate(raw)
except (ValidationError, ValueError) as e:
last_error = str(e)[:4000]
+ _record_invalid(ws, attempt, raw, resp.text)
if events is not None:
events.emit("plan.invalid", attempt=attempt, error=last_error[:500])
missing = missing_link_names(raw)
diff --git a/harness/tests/orchestrator_tracks/test_planner_restart.py b/harness/tests/orchestrator_tracks/test_planner_restart.py
index e3fe824a..c6215d1c 100644
--- a/harness/tests/orchestrator_tracks/test_planner_restart.py
+++ b/harness/tests/orchestrator_tracks/test_planner_restart.py
@@ -7,6 +7,7 @@ never lists — and both in-context re-asks come back with that same answer."""
from __future__ import annotations
import copy
+import json
import pytest
@@ -119,3 +120,21 @@ def test_a_full_plan_with_one_dangling_link_is_edited_in_context_not_resampled(t
assert isinstance(p, ArticulatedPlan) and len(model.requests) == 2
assert len(model.requests[1].messages) == 3 # user + echoed answer + complaint, no restart
assert "failed validation" in model.requests[1].messages[-1].text
+
+
+def test_a_rejected_plan_is_written_where_a_dead_run_can_be_read(tmp_ws):
+ """A run that dies at the plan stage used to leave nothing but the pydantic message,
+ which truncates the offending value — so "what did the model actually write?", the
+ question every plan-stage failure class starts from, had no answer at all."""
+ from codeverse.tracks.planner import INVALID_PLAN_DIR
+
+ model = FakeChatModel(lambda req: _degenerate())
+ with pytest.raises(PlanningError):
+ plan(_spec(), "fake:planner", ArticulatedPlan, tmp_ws, model=model,
+ runtime=FakeRuntime(Language.URDF_BLENDER))
+
+ written = sorted((tmp_ws.root / INVALID_PLAN_DIR).glob("attempt*.json"))
+ assert len(written) == 4, [p.name for p in written] # one per rejected answer
+ first = json.loads(written[0].read_text())
+ assert [p["name"] for p in first["parts"]] == ["Cabinet"] # the answer, not the error text
+ assert first["joints"][0]["child"] == "Drawer" # the link it named and never listed