Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

Evals: sandboxes get their own repository; mined tasks protect only the named test

justrach17bd765e3752
diff --git a/docs/adr/0214-evals-are-mined-and-hillclimbed-on-a-held-out-split.md b/docs/adr/0214-evals-are-mined-and-hillclimbed-on-a-held-out-split.md
index 7a8c768f..05b87aa3 100644
--- a/docs/adr/0214-evals-are-mined-and-hillclimbed-on-a-held-out-split.md
+++ b/docs/adr/0214-evals-are-mined-and-hillclimbed-on-a-held-out-split.md
@@ -50,6 +50,22 @@ an unreachable test counts). On the merge tree both must pass, and a second run
must give the same verdicts. That is three filtered builds per task, one
build at a time.
+**A sandbox is its own repository, and only the named test is protected**
+(amended 2026-09-29). Sandboxes sit inside this checkout. Before this change,
+`git status`, `git diff` and `git show HEAD:` in a sandbox described the
+checkout, not the task, and a `git stash` there would have changed the
+checkout. Each sandbox now gets a one-commit repository of the task tree
+(`measurement.isolate_git`).
+
+The mined prompt names the check the grader runs, `zig build test-bin
+-Dtest-filter=…`. When it named `zig build test`, runs spent most of their time
+budget on that step's integration scripts, which cannot pass without
+`graff-evals/`.
+
+A mined task protects its named test's own text. Otherwise only Python test
+files count as visible verifiers. In a copy of this repository,
+`src/test_hooks.zig` is where a fix registers a new test file.
+
**A person approves each task.** `review` writes a local page, and `approve`
moves a validated candidate into the `mined` suite. The loop never picks its
own tasks from one model's failures.
diff --git a/graff-evals/design/mine_prs.py b/graff-evals/design/mine_prs.py
index f8ca2030..6a1c6ce5 100644
--- a/graff-evals/design/mine_prs.py
+++ b/graff-evals/design/mine_prs.py
@@ -123,42 +123,7 @@ def candidates(days: int, ref: str, limit: int) -> list[dict]:
# ── Zig test blocks ──────────────────────────────────────────────────────
-def test_block(text: str, name: str) -> str | None:
- """The full `test "name" { … }` block, braces matched outside strings,
- char literals, `//` comments and `\\\\` multiline string lines."""
- head = f'test "{name}"'
- start = text.find(head)
- if start < 0:
- return None
- # Include the doc comments / blank line directly above? No: the block only.
- i = text.find("{", start + len(head))
- if i < 0:
- return None
- depth, n = 0, len(text)
- while i < n:
- c = text[i]
- if c == "/" and text.startswith("//", i):
- j = text.find("\n", i)
- i = n if j < 0 else j
- continue
- if c == "\\" and text.startswith("\\\\", i):
- j = text.find("\n", i)
- i = n if j < 0 else j
- continue
- if c in "\"'":
- j = i + 1
- while j < n and text[j] != c:
- j += 2 if text[j] == "\\" else 1
- i = j + 1
- continue
- if c == "{":
- depth += 1
- elif c == "}":
- depth -= 1
- if depth == 0:
- return text[start:i + 1] + "\n"
- i += 1
- return None
+test_block = live_setup.test_block
def file_at(rev: str, path: str) -> str:
@@ -190,10 +155,15 @@ def write(path: str, text: str, mode: int = 0o644) -> None:
def prompt_for(title: str, public: dict) -> str:
# State what the grader checks: the test must run in the project's own
- # test build. "Make it pass" alone let a run prove it in a temporary
- # harness and leave it unreachable from the suite (ADR 0214).
- return (f"{title}. Make the test named \"{public['name']}\" in {public['file']} run and pass "
- "in the project's unit tests (`zig build test`), not only in isolation. "
+ # unit-test binary. "Make it pass" alone let a run prove it in a temporary
+ # harness and leave it unreachable from the suite. Naming only `zig build
+ # test` sent runs into its integration scripts, which cannot pass without
+ # graff-evals/ and ate most of the time budget (ADR 0214).
+ return (f"{title}. Make the test named \"{public['name']}\" in {public['file']} pass in the "
+ "project's unit-test binary, not only in isolation: "
+ f"`zig build test-bin -Dtest-filter=\"{public['name']}\"` compiles it (the newest "
+ "`.zig-cache/o/*/test`), and running that binary must report the test OK. The full "
+ "`zig build test` also runs integration scripts this task does not need. "
"Do not edit that test. Do not add SPEC.md.")
@@ -359,7 +329,9 @@ def self_test() -> None:
assert test_block(src, "missing") is None
assert TEST_RE.match('+test "a \\"quoted\\" name" {').group(1) == 'a \\"quoted\\" name'
prompt = prompt_for("Fix it", {"name": "alpha", "file": "src/a.zig"})
- assert "run and pass in the project's unit tests (`zig build test`)" in prompt, prompt
+ # The grader's own check, not the full `zig build test` (ADR 0214).
+ assert "pass in the project's unit-test binary, not only in isolation" in prompt, prompt
+ assert '`zig build test-bin -Dtest-filter="alpha"`' in prompt, prompt
print("mine_prs self-test ok")
diff --git a/graff-evals/live_setup.py b/graff-evals/live_setup.py
index bcdf2fba..5547a093 100644
--- a/graff-evals/live_setup.py
+++ b/graff-evals/live_setup.py
@@ -184,6 +184,56 @@ def archive_local(rev, dest, paths=None):
shutil.copytree(pkg, os.path.join(dest, "zig-pkg"), symlinks=True)
+def test_block(text, name):
+ """The full `test "name" { … }` block, braces matched outside strings,
+ char literals, `//` comments and `\\\\` multiline string lines."""
+ head = f'test "{name}"'
+ start = text.find(head)
+ if start < 0:
+ return None
+ # Include the doc comments / blank line directly above? No: the block only.
+ i = text.find("{", start + len(head))
+ if i < 0:
+ return None
+ depth, n = 0, len(text)
+ while i < n:
+ c = text[i]
+ if c == "/" and text.startswith("//", i):
+ j = text.find("\n", i)
+ i = n if j < 0 else j
+ continue
+ if c == "\\" and text.startswith("\\\\", i):
+ j = text.find("\n", i)
+ i = n if j < 0 else j
+ continue
+ if c in "\"'":
+ j = i + 1
+ while j < n and text[j] != c:
+ j += 2 if text[j] == "\\" else 1
+ i = j + 1
+ continue
+ if c == "{":
+ depth += 1
+ elif c == "}":
+ depth -= 1
+ if depth == 0:
+ return text[start:i + 1] + "\n"
+ i += 1
+ return None
+
+
+def named_case(task_id):
+ """(file, test name) a mined task appended and told the model not to
+ edit, or None for any other task."""
+ try:
+ task = manifest(task_id)
+ except SystemExit:
+ return None
+ if task.get("source") != "local-git" or not task.get("public_file") or not task.get("public_filter"):
+ return None
+ return task["public_file"], task["public_filter"]
+
+
def append_case(sandbox, rel, inc, name):
"""Append a test block unless a test with that name is already there."""
path = os.path.join(sandbox, rel)
diff --git a/graff-evals/measurement.py b/graff-evals/measurement.py
index 3f2d3bb8..213abb15 100644
--- a/graff-evals/measurement.py
+++ b/graff-evals/measurement.py
@@ -186,20 +186,69 @@ def provider_command(command):
return result
+# Caches and dependency trees hold no task verifier; walking them is slow.
+_NOT_VERIFIERS = {'.git', '.zig-cache', 'zig-cache', 'zig-out', 'zig-pkg', 'node_modules', '__pycache__'}
+_LIVE_TASK = re.compile(r'live_setup\.py"?\s+([\w.-]+)')
+
+
+def is_visible_test(name):
+ """A test or check file a task ships (`test_*.py`, `check_*.py`,
+ `*_test.py`). In a copy of a Zig repository, a `test_` name is production
+ code: `src/test_hooks.zig` is where a fix registers a new test file."""
+ return name.endswith('.py') and (name.startswith(('test_', 'check_')) or name.endswith('_test.py'))
+
+
def verifier_snapshot(sandbox, task, task_root):
- """Visible tests are immutable; held-out graders stay outside agent cwd."""
- paths = {p for p in Path(sandbox).rglob('*') if p.is_file() and
- (p.name.startswith('test_') or p.name.startswith('check_') or p.name.endswith('_test.py'))}
+ """Visible tests are immutable; held-out graders stay outside agent cwd.
+ A mined task's named test was appended to a source file the fix may also
+ belong in, so that test's own text is protected, not the file."""
+ snap = {}
+ for root, dirs, files in os.walk(sandbox):
+ dirs[:] = [d for d in dirs if d not in _NOT_VERIFIERS]
+ for name in files:
+ if is_visible_test(name):
+ path = os.path.join(root, name)
+ snap[path] = hashlib.sha256(Path(path).read_bytes()).hexdigest()
for relative in re.findall(r'\$TASK_ROOT/(hidden/[^\s"\x27;]+)', task.get('check', '')):
- paths.add(Path(task_root) / relative)
- return {str(p): hashlib.sha256(p.read_bytes()).hexdigest() for p in paths}
+ path = Path(task_root) / relative
+ snap[str(path)] = hashlib.sha256(path.read_bytes()).hexdigest()
+ live = _LIVE_TASK.search(' '.join(task.get('setup', [])))
+ if live:
+ import live_setup
+ case = live_setup.named_case(live.group(1))
+ path = os.path.join(sandbox, case[0]) if case else None
+ block = live_setup.test_block(Path(path).read_text(errors='replace'), case[1]) if path and os.path.exists(path) else None
+ if block:
+ snap[path] = {'contains': block}
+ return snap
def verifiers_unchanged(snapshot):
- for path, digest in snapshot.items():
+ for path, want in snapshot.items():
try:
- if hashlib.sha256(Path(path).read_bytes()).hexdigest() != digest:
- return False
+ data = Path(path).read_bytes()
except OSError:
return False
+ if isinstance(want, dict):
+ if want['contains'].encode() not in data:
+ return False
+ elif hashlib.sha256(data).hexdigest() != want:
+ return False
return True
+
+
+def isolate_git(sandbox):
+ """Give a sandbox its own repository, holding the task tree as one commit.
+ Sandboxes sit inside this checkout. Without their own repository, `git
+ status`, `git diff` and `git show HEAD:` there describe the checkout, not
+ the task, and a `git stash` or `git checkout` would change the checkout. A
+ task that made its own repository keeps it."""
+ if os.path.exists(os.path.join(sandbox, '.git')):
+ return
+ git = ['git', '-c', 'user.name=eval', '-c', 'user.email=eval@localhost', '-c', 'commit.gpgsign=false',
+ '-c', 'core.hooksPath=/dev/null', '-C', sandbox]
+ subprocess.run(git + ['init', '-q', '-b', 'main'], check=True, capture_output=True)
+ with open(os.path.join(sandbox, '.git', 'info', 'exclude'), 'a') as f:
+ f.write('.zig-cache/\nzig-cache/\nzig-out/\nzig-pkg/\n.graff/\n.eval-*\n')
+ subprocess.run(git + ['add', '-A'], check=True, capture_output=True)
+ subprocess.run(git + ['commit', '-q', '--allow-empty', '--no-verify', '-m', 'task'], check=True, capture_output=True)
diff --git a/graff-evals/run.py b/graff-evals/run.py
index 3720900b..af88856a 100755
--- a/graff-evals/run.py
+++ b/graff-evals/run.py
@@ -330,7 +330,9 @@ def tree_cpu_s(pid):
def dir_bytes(path):
total = 0
- for root, _dirs, files in os.walk(path):
+ for root, dirs, files in os.walk(path):
+ # The repository isolate_git made holds a second copy of the task tree.
+ dirs[:] = [d for d in dirs if not (root == path and d == ".git")]
for name in files:
try:
total += os.path.getsize(os.path.join(root, name))
@@ -348,6 +350,7 @@ def one_run(hname, harness, task, model, rep, live=False):
sandbox = os.path.join(SANDBOX_DIR, f"{hname}-{task['id']}-r{rep}")
try:
materialize(task, sandbox)
+ measurement.isolate_git(sandbox)
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error:
return {"harness": hname, "task": task["id"], "suite": task.get("suite", "core"),
"rep": rep, "error": f"environment setup failed: {error}", "outcome_ok": False}
diff --git a/graff-evals/tasks/mined-pr-1253.json b/graff-evals/tasks/mined-pr-1253.json
index 0a2efd8c..0fbcc9a0 100644
--- a/graff-evals/tasks/mined-pr-1253.json
+++ b/graff-evals/tasks/mined-pr-1253.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1253 — mined (ADR 0214)",
- "prompt": "ws: stream codex WS deltas through printDelta like SSE. Make the test named \"a WS text delta reaches the UI before the response completes\" in src/agent_ws_stream_test.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "ws: stream codex WS deltas through printDelta like SSE. Make the test named \"a WS text delta reaches the UI before the response completes\" in src/agent_ws_stream_test.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"a WS text delta reaches the UI before the response completes\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1253 ."
],
diff --git a/graff-evals/tasks/mined-pr-1254.json b/graff-evals/tasks/mined-pr-1254.json
index 9a390d7c..ae99946e 100644
--- a/graff-evals/tasks/mined-pr-1254.json
+++ b/graff-evals/tasks/mined-pr-1254.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1254 — mined (ADR 0214)",
- "prompt": "Connect the MCP servers an ACP client passes. Make the test named \"parse: stdio with args and env, http with headers, sse and malformed left out\" in src/acp_mcp_servers.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "Connect the MCP servers an ACP client passes. Make the test named \"parse: stdio with args and env, http with headers, sse and malformed left out\" in src/acp_mcp_servers.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"parse: stdio with args and env, http with headers, sse and malformed left out\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1254 ."
],
diff --git a/graff-evals/tasks/mined-pr-1256.json b/graff-evals/tasks/mined-pr-1256.json
index dd2cb224..f18c842d 100644
--- a/graff-evals/tasks/mined-pr-1256.json
+++ b/graff-evals/tasks/mined-pr-1256.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1256 — mined (ADR 0214)",
- "prompt": "Apply GRAFF_NO_PLUGINS before connecting MCP servers. Make the test named \"initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config\" in src/session_start_plugins_test.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "Apply GRAFF_NO_PLUGINS before connecting MCP servers. Make the test named \"initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config\" in src/session_start_plugins_test.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"initRegistryConsent applies GRAFF_NO_PLUGINS before it reads any MCP config\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1256 ."
],
diff --git a/graff-evals/tasks/mined-pr-1257.json b/graff-evals/tasks/mined-pr-1257.json
index 5e3fdcdc..8b4e925c 100644
--- a/graff-evals/tasks/mined-pr-1257.json
+++ b/graff-evals/tasks/mined-pr-1257.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1257 — mined (ADR 0214)",
- "prompt": "stdin: read ACP and --json records past the 64 KiB buffer. Make the test named \"records larger than the reader buffer are returned whole\" in src/stdin_line.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "stdin: read ACP and --json records past the 64 KiB buffer. Make the test named \"records larger than the reader buffer are returned whole\" in src/stdin_line.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"records larger than the reader buffer are returned whole\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1257 ."
],
diff --git a/graff-evals/tasks/mined-pr-1302.json b/graff-evals/tasks/mined-pr-1302.json
index a1a1bf92..82f2cf27 100644
--- a/graff-evals/tasks/mined-pr-1302.json
+++ b/graff-evals/tasks/mined-pr-1302.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1302 — mined (ADR 0214)",
- "prompt": "Bound publication review input for small diffs in large files. Make the test named \"claim review uses committed context when changed head source exceeds the budget\" in src/pr_review_input.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "Bound publication review input for small diffs in large files. Make the test named \"claim review uses committed context when changed head source exceeds the budget\" in src/pr_review_input.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"claim review uses committed context when changed head source exceeds the budget\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1302 ."
],
diff --git a/graff-evals/tasks/mined-pr-1306.json b/graff-evals/tasks/mined-pr-1306.json
index 84b9f1df..3bc37839 100644
--- a/graff-evals/tasks/mined-pr-1306.json
+++ b/graff-evals/tasks/mined-pr-1306.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1306 — mined (ADR 0214)",
- "prompt": "Strip provider citation markers from ACP text. Make the test named \"a whole citation marker is removed and plain text passes through untouched\" in src/acp_citations.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "Strip provider citation markers from ACP text. Make the test named \"a whole citation marker is removed and plain text passes through untouched\" in src/acp_citations.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"a whole citation marker is removed and plain text passes through untouched\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1306 ."
],
diff --git a/graff-evals/tasks/mined-pr-1309.json b/graff-evals/tasks/mined-pr-1309.json
index 8ef830a3..b2332f94 100644
--- a/graff-evals/tasks/mined-pr-1309.json
+++ b/graff-evals/tasks/mined-pr-1309.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1309 — mined (ADR 0214)",
- "prompt": "ACP: report context left with the standard usage_update. Make the test named \"usage_update carries used and size, and cost only when known\" in src/acp_context_meter.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "ACP: report context left with the standard usage_update. Make the test named \"usage_update carries used and size, and cost only when known\" in src/acp_context_meter.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"usage_update carries used and size, and cost only when known\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1309 ."
],
diff --git a/graff-evals/tasks/mined-pr-1327.json b/graff-evals/tasks/mined-pr-1327.json
index a07831df..470e49ed 100644
--- a/graff-evals/tasks/mined-pr-1327.json
+++ b/graff-evals/tasks/mined-pr-1327.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1327 — mined (ADR 0214)",
- "prompt": "ask_user: read the question from every argument shape models send. Make the test named \"the schema shape passes through unchanged\" in src/ask_user_args.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "ask_user: read the question from every argument shape models send. Make the test named \"the schema shape passes through unchanged\" in src/ask_user_args.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"the schema shape passes through unchanged\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1327 ."
],
diff --git a/graff-evals/tasks/mined-pr-1328.json b/graff-evals/tasks/mined-pr-1328.json
index a603be54..79dfe78c 100644
--- a/graff-evals/tasks/mined-pr-1328.json
+++ b/graff-evals/tasks/mined-pr-1328.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1328 — mined (ADR 0214)",
- "prompt": "Free the MCP shape cache at session teardown. Make the test named \"shutdown frees a filled cache and is a no-op when empty (#1196)\" in src/mcp_shapes.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "Free the MCP shape cache at session teardown. Make the test named \"shutdown frees a filled cache and is a no-op when empty (#1196)\" in src/mcp_shapes.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"shutdown frees a filled cache and is a no-op when empty (#1196)\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1328 ."
],
diff --git a/graff-evals/tasks/mined-pr-1329.json b/graff-evals/tasks/mined-pr-1329.json
index c410c44a..670eec14 100644
--- a/graff-evals/tasks/mined-pr-1329.json
+++ b/graff-evals/tasks/mined-pr-1329.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1329 — mined (ADR 0214)",
- "prompt": "ACP session/new stops waiting on unreachable MCP servers. Make the test named \"joinWithin (#1291): a server that fails fast ends the wait early\" in src/acp_mcp_servers.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "ACP session/new stops waiting on unreachable MCP servers. Make the test named \"joinWithin (#1291): a server that fails fast ends the wait early\" in src/acp_mcp_servers.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"joinWithin (#1291): a server that fails fast ends the wait early\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1329 ."
],
diff --git a/graff-evals/tasks/mined-pr-1330.json b/graff-evals/tasks/mined-pr-1330.json
index 24eb110b..88948736 100644
--- a/graff-evals/tasks/mined-pr-1330.json
+++ b/graff-evals/tasks/mined-pr-1330.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1330 — mined (ADR 0214)",
- "prompt": "Run tool calls a server left in content as inline markup. Make the test named \"recoverInlineCalls (#1247): markup after prose becomes calls, parameters in any order\" in src/tool_call_repair_tests.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "Run tool calls a server left in content as inline markup. Make the test named \"recoverInlineCalls (#1247): markup after prose becomes calls, parameters in any order\" in src/tool_call_repair_tests.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"recoverInlineCalls (#1247): markup after prose becomes calls, parameters in any order\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1330 ."
],
diff --git a/graff-evals/tasks/mined-pr-1336.json b/graff-evals/tasks/mined-pr-1336.json
index 1703eb59..0452808b 100644
--- a/graff-evals/tasks/mined-pr-1336.json
+++ b/graff-evals/tasks/mined-pr-1336.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1336 — mined (ADR 0214)",
- "prompt": "Resolve a PR check that finished in the background. Make the test named \"a check promoted to the background resolves when its job exits 0 (#1326)\" in src/pr_local_checks.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "Resolve a PR check that finished in the background. Make the test named \"a check promoted to the background resolves when its job exits 0 (#1326)\" in src/pr_local_checks.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"a check promoted to the background resolves when its job exits 0 (#1326)\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1336 ."
],
diff --git a/graff-evals/tasks/mined-pr-1339.json b/graff-evals/tasks/mined-pr-1339.json
index 6331319f..09ea7bd1 100644
--- a/graff-evals/tasks/mined-pr-1339.json
+++ b/graff-evals/tasks/mined-pr-1339.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1339 — mined (ADR 0214)",
- "prompt": "ask_user: a question that comes back unanswered is not a cancel. Make the test named \"decline, cancel, an empty accept and an error each end the question as themselves (#1322)\" in src/acp_elicit.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "ask_user: a question that comes back unanswered is not a cancel. Make the test named \"decline, cancel, an empty accept and an error each end the question as themselves (#1322)\" in src/acp_elicit.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"decline, cancel, an empty accept and an error each end the question as themselves (#1322)\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1339 ."
],
diff --git a/graff-evals/tasks/mined-pr-1348.json b/graff-evals/tasks/mined-pr-1348.json
index fdbb0f1a..f514b349 100644
--- a/graff-evals/tasks/mined-pr-1348.json
+++ b/graff-evals/tasks/mined-pr-1348.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1348 — mined (ADR 0214)",
- "prompt": "PR readiness keys checks by what they run; rlm says when sleep_ms is capped. Make the test named \"one check in one directory is one entry whatever its spelling\" in src/pr_local_checks.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "PR readiness keys checks by what they run; rlm says when sleep_ms is capped. Make the test named \"one check in one directory is one entry whatever its spelling\" in src/pr_local_checks.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"one check in one directory is one entry whatever its spelling\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1348 ."
],
diff --git a/graff-evals/tasks/mined-pr-1364.json b/graff-evals/tasks/mined-pr-1364.json
index 2d1d5392..3217ed42 100644
--- a/graff-evals/tasks/mined-pr-1364.json
+++ b/graff-evals/tasks/mined-pr-1364.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1364 — mined (ADR 0214)",
- "prompt": "Informational subagents keep read tools (#1360). Make the test named \"a read-only child keeps reads and read-only shell, not writes (#1360)\" in src/agent_tool_gate.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "Informational subagents keep read tools (#1360). Make the test named \"a read-only child keeps reads and read-only shell, not writes (#1360)\" in src/agent_tool_gate.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"a read-only child keeps reads and read-only shell, not writes (#1360)\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1364 ."
],
diff --git a/graff-evals/tasks/mined-pr-1365.json b/graff-evals/tasks/mined-pr-1365.json
index 430ee743..7e6ac4a7 100644
--- a/graff-evals/tasks/mined-pr-1365.json
+++ b/graff-evals/tasks/mined-pr-1365.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1365 — mined (ADR 0214)",
- "prompt": "Anthropic: refuse a cut-off tool input instead of running it as {} (refs #1218). Make the test named \"a cut-off Anthropic tool input is refused, not run as {} (#1218)\" in src/tool_call_args.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "Anthropic: refuse a cut-off tool input instead of running it as {} (refs #1218). Make the test named \"a cut-off Anthropic tool input is refused, not run as {} (#1218)\" in src/tool_call_args.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"a cut-off Anthropic tool input is refused, not run as {} (#1218)\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1365 ."
],
diff --git a/graff-evals/tasks/mined-pr-1373.json b/graff-evals/tasks/mined-pr-1373.json
index fd6f99b4..b3e2a3bd 100644
--- a/graff-evals/tasks/mined-pr-1373.json
+++ b/graff-evals/tasks/mined-pr-1373.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1373 — mined (ADR 0214)",
- "prompt": "ACP: frame Harness room deliveries as advisory room messages. Make the test named \"an agent's room line becomes an advisory room message; a person's stays a prompt\" in src/acp_room.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "ACP: frame Harness room deliveries as advisory room messages. Make the test named \"an agent's room line becomes an advisory room message; a person's stays a prompt\" in src/acp_room.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"an agent's room line becomes an advisory room message; a person's stays a prompt\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1373 ."
],
diff --git a/graff-evals/tasks/mined-pr-1376.json b/graff-evals/tasks/mined-pr-1376.json
index b4d3f3ef..8ae1642f 100644
--- a/graff-evals/tasks/mined-pr-1376.json
+++ b/graff-evals/tasks/mined-pr-1376.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1376 — mined (ADR 0214)",
- "prompt": "ACP room deliveries: name another account's agent as such. Make the test named \"another account's agent is named as such, and framed text must say so to be kept\" in src/acp_room.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "ACP room deliveries: name another account's agent as such. Make the test named \"another account's agent is named as such, and framed text must say so to be kept\" in src/acp_room.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"another account's agent is named as such, and framed text must say so to be kept\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1376 ."
],
diff --git a/graff-evals/tasks/mined-pr-1385.json b/graff-evals/tasks/mined-pr-1385.json
index 94f1d7ed..50ec0532 100644
--- a/graff-evals/tasks/mined-pr-1385.json
+++ b/graff-evals/tasks/mined-pr-1385.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1385 — mined (ADR 0214)",
- "prompt": "ACP: saved views carry _meta[\"graff/view\"] on the completed tool update. Make the test named \"detect: graff's own view and MCP app snapshots\" in src/acp_view_meta.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "ACP: saved views carry _meta[\"graff/view\"] on the completed tool update. Make the test named \"detect: graff's own view and MCP app snapshots\" in src/acp_view_meta.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"detect: graff's own view and MCP app snapshots\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1385 ."
],
diff --git a/graff-evals/tasks/mined-pr-1386.json b/graff-evals/tasks/mined-pr-1386.json
index 13b03649..37bcc6ac 100644
--- a/graff-evals/tasks/mined-pr-1386.json
+++ b/graff-evals/tasks/mined-pr-1386.json
@@ -3,7 +3,7 @@
"suite": "mined-candidate",
"category": "live",
"source": "codegraff #1386 — mined (ADR 0214)",
- "prompt": "graff/models: each row names the model's maker. Make the test named \"maker: the gateway serves other labs' models\" in src/model_maker.zig run and pass in the project's unit tests (`zig build test`), not only in isolation. Do not edit that test. Do not add SPEC.md.",
+ "prompt": "graff/models: each row names the model's maker. Make the test named \"maker: the gateway serves other labs' models\" in src/model_maker.zig pass in the project's unit-test binary, not only in isolation: `zig build test-bin -Dtest-filter=\"maker: the gateway serves other labs' models\"` compiles it (the newest `.zig-cache/o/*/test`), and running that binary must report the test OK. The full `zig build test` also runs integration scripts this task does not need. Do not edit that test. Do not add SPEC.md.",
"setup": [
"python3 \"$TASK_ROOT/live_setup.py\" pr-1386 ."
],
diff --git a/graff-evals/test_measurement.py b/graff-evals/test_measurement.py
index 29a677f2..94931ffa 100644
--- a/graff-evals/test_measurement.py
+++ b/graff-evals/test_measurement.py
@@ -256,6 +256,60 @@ class MeasurementTests(unittest.TestCase):
path.write_text('pass')
self.assertFalse(measurement.verifiers_unchanged(snap))
+ def test_a_zig_tree_test_root_is_not_a_visible_verifier(self):
+ with tempfile.TemporaryDirectory() as temp:
+ (Path(temp) / 'src').mkdir()
+ hooks = Path(temp) / 'src' / 'test_hooks.zig'
+ hooks.write_text('test { _ = @import("a.zig"); }\n')
+ public = Path(temp) / 'test_public.py'
+ public.write_text('raise AssertionError()')
+ snap = measurement.verifier_snapshot(temp, {}, temp)
+ # Registering a new test file in the test root is part of a fix.
+ hooks.write_text('test { _ = @import("a.zig"); _ = @import("b.zig"); }\n')
+ self.assertTrue(measurement.verifiers_unchanged(snap))
+ public.write_text('pass')
+ self.assertFalse(measurement.verifiers_unchanged(snap))
+
+ def test_a_mined_task_protects_its_named_test_not_the_file_it_sits_in(self):
+ task = {'setup': ['python3 "$TASK_ROOT/live_setup.py" pr-9999 .']}
+ body = 'pub fn f() u8 {\n return 1;\n}\n\ntest "f is two" {\n try std.testing.expect(f() == 2);\n}\n'
+ with tempfile.TemporaryDirectory() as temp, \
+ patch('live_setup.named_case', return_value=('src/f.zig', 'f is two')):
+ (Path(temp) / 'src').mkdir()
+ source = Path(temp) / 'src' / 'f.zig'
+ source.write_text(body)
+ snap = measurement.verifier_snapshot(temp, task, temp)
+ source.write_text(body.replace('return 1;', 'return 2;')) # the fix, same file
+ self.assertTrue(measurement.verifiers_unchanged(snap))
+ source.write_text(body.replace('f() == 2', 'f() == 1')) # the test itself
+ self.assertFalse(measurement.verifiers_unchanged(snap))
+
+ def test_a_sandbox_gets_its_own_repository_holding_the_task(self):
+ import subprocess
+ with tempfile.TemporaryDirectory() as outer:
+ subprocess.run(['git', 'init', '-q', outer], check=True)
+ sandbox = Path(outer) / 'sandboxes' / 'one'
+ (sandbox / '.zig-cache').mkdir(parents=True)
+ (sandbox / 'note.txt').write_text('task\n')
+ (sandbox / '.zig-cache' / 'x').write_text('cache')
+ top = lambda: subprocess.run(['git', '-C', str(sandbox), 'rev-parse', '--show-toplevel'],
+ capture_output=True, text=True).stdout.strip()
+ self.assertEqual(Path(top()).resolve(), Path(outer).resolve()) # the enclosing checkout
+ measurement.isolate_git(str(sandbox))
+ self.assertEqual(Path(top()).resolve(), sandbox.resolve())
+ status = subprocess.run(['git', '-C', str(sandbox), 'status', '--porcelain'],
+ capture_output=True, text=True, check=True).stdout
+ self.assertEqual(status, '')
+ shown = subprocess.run(['git', '-C', str(sandbox), 'show', 'HEAD:note.txt'],
+ capture_output=True, text=True, check=True).stdout
+ self.assertEqual(shown, 'task\n')
+ # A task that made its own repository keeps it.
+ head = subprocess.run(['git', '-C', str(sandbox), 'rev-parse', 'HEAD'],
+ capture_output=True, text=True, check=True).stdout
+ measurement.isolate_git(str(sandbox))
+ self.assertEqual(subprocess.run(['git', '-C', str(sandbox), 'rev-parse', 'HEAD'],
+ capture_output=True, text=True, check=True).stdout, head)
+
def test_untracked_source_is_in_snapshot_receipt(self):
import subprocess
with tempfile.TemporaryDirectory() as temp: