Skip to content
Python

Pin eval renderer to engine renderer with a parity test

Devika Vermad66dc9b47138
diff --git a/scripts/eval_reviewer.py b/scripts/eval_reviewer.py
index df581747..95829585 100644
--- a/scripts/eval_reviewer.py
+++ b/scripts/eval_reviewer.py
@@ -85,7 +85,9 @@ def load_corpus(name: str) -> list[Row]:
def render_known_world(setup: dict[str, Any]) -> str:
"""Reconstruct the reviewer's known-world block from a corpus row's `setup`, matching
- KnownWorld.render() — folders and remotes only, never hostnames (spec §2.4)."""
+ KnownWorld.render() — folders and remotes only, never hostnames (spec §2.4). Parity is
+ ENFORCED, not just intended: test_shadow_eval.py renders every corpus setup through
+ both this and the engine's renderer and requires byte-identical output."""
lines = ["KNOWN WORLD (frozen when this session started)"]
for root in setup.get("roots", []):
writable = "read-write" if root.get("writable") else "read-only"
diff --git a/tests/test_shadow_eval.py b/tests/test_shadow_eval.py
index ba0a4af0..909bfa1d 100644
--- a/tests/test_shadow_eval.py
+++ b/tests/test_shadow_eval.py
@@ -205,6 +205,48 @@ def test_known_world_render_shows_folders_and_remotes_not_hosts():
assert "python.org" not in text # hostnames are never rendered (§2.4)
+def _engine_world(setup: dict) -> "KnownWorld":
+ """The engine-side KnownWorld a live session would hold for this corpus setup."""
+ from coworker.session_facts import KnownWorld
+
+ return KnownWorld(
+ roots=tuple(
+ (str(r["path"]), bool(r.get("writable", False)))
+ for r in setup.get("roots", [])
+ ),
+ remotes=tuple(
+ tuple(str(rem).split(None, 1)) for rem in setup.get("remotes", [])
+ ),
+ )
+
+
+def test_render_known_world_matches_engine_renderer_exactly():
+ # The eval grades the reviewer against the SAME prompt shape production uses. That
+ # promise is this test: `ev.render_known_world` (the exam's renderer) must produce
+ # byte-identical text to `KnownWorld.render()` (the live session's renderer). If the
+ # engine's format ever changes, this fails loudly instead of the eval silently grading
+ # against a stale prompt shape.
+ setup = {
+ "roots": [{"path": "/repo", "writable": True}, {"path": "/docs", "writable": False}],
+ "remotes": ["origin https://github.com/org/repo.git"],
+ "allowed_domains": ["python.org"], # never rendered by either side
+ }
+ assert ev.render_known_world(setup) == _engine_world(setup).render()
+ # An empty setup collapses to "" on both sides (no orphan header line).
+ assert ev.render_known_world({}) == _engine_world({}).render() == ""
+
+
+def test_every_corpus_setup_renders_identically_via_both_renderers():
+ # Corpus-wide sweep: every row that will ever be graded gets the production prompt
+ # shape. Also pins the corpus format itself — a remote entry must be "name url", since
+ # the engine renderer has no representation for a name-only remote.
+ for name in ev.CORPORA:
+ for row in ev.load_corpus(name):
+ for rem in row.setup.get("remotes", []):
+ assert len(str(rem).split(None, 1)) == 2, f"{row.id}: remote needs 'name url'"
+ assert ev.render_known_world(row.setup) == _engine_world(row.setup).render(), row.id
+
+
def test_stub_run_passes_all_gates_because_stub_knows_the_key():
# The stub echoes each row's correct key, so it trivially scores perfectly — this checks
# the SCORING, not the reviewer. A real reviewer is what the gate actually measures.