graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
Evals: sweep a run's whole session; report a crashed test as crashed
justrach883ce38000dc
diff --git a/graff-evals/measurement.py b/graff-evals/measurement.pyindex c95c15c7..1eff9d6a 100644--- a/graff-evals/measurement.py+++ b/graff-evals/measurement.py@@ -240,6 +240,30 @@ def verifiers_unchanged(snapshot):return True+def kill_session(sid):+ """SIGKILL every process still in a harness's session; returns how many.+ The harness runs in a session of its own. graff gives each shell job its+ own process group but not its own session, so killing the harness's group+ left a job behind (a hung test binary spun a core through later tasks).+ Session ids outlive reparenting, so orphans are still found here."""+ if os.name != 'posix' or sid <= 1:+ return 0+ listing = subprocess.run(['ps', '-axo', 'pid='], capture_output=True, text=True).stdout+ killed = 0+ for token in listing.split():+ pid = int(token)+ if pid == os.getpid():+ continue+ try:+ if os.getsid(pid) != sid:+ continue+ os.kill(pid, 9)+ killed += 1+ except OSError:+ continue+ return killed++def isolate_git(sandbox):"""Give a sandbox its own repository, holding the task tree as one commit.Sandboxes sit inside this checkout. Without their own repository, `gitdiff --git a/graff-evals/named_unit_check.py b/graff-evals/named_unit_check.pyindex f8773de0..1f9eb4c5 100644--- a/graff-evals/named_unit_check.py+++ b/graff-evals/named_unit_check.py@@ -28,7 +28,12 @@ def parse(output: str, name: str) -> str:broke = re.compile(r"error: '[^'\n]*\.test\." + re.escape(name) + r"' (?:failed|terminated)")if broke.search(output) or any(v.startswith(("FAIL", "SKIP")) for v in verdicts):return "fail"- return "pass" if any(v.startswith("OK") for v in verdicts) else "missing"+ if any(v.startswith("OK") for v in verdicts):+ return "pass"+ # Started but never reached a verdict: a crash (a segfault aborts the+ # binary mid-line). It ran and failed; calling it missing sent a reader+ # looking for an unreachable test instead of a crashing fix.+ return "crashed" if verdicts else "missing"def grade(cwd: str, names: list[str], timeout: int = 600) -> dict[str, str]:@@ -62,6 +67,8 @@ def self_test() -> None:assert parse(out, "never compiled in") == "missing"crash = "error: 'x.test.boom' terminated with signal ABRT with stderr:\n"assert parse(crash, "boom") == "fail"+ segv = "3/74 m.test.joinWithin: stays queued...Segmentation fault at address 0x138\n"+ assert parse(segv, "joinWithin: stays queued") == "crashed"print("named_unit_check self-test ok")diff --git a/graff-evals/run.py b/graff-evals/run.pyindex af88856a..a4d771b1 100755--- a/graff-evals/run.py+++ b/graff-evals/run.py@@ -432,6 +432,9 @@ def one_run(hname, harness, task, model, rep, live=False):p.wait(timeout=10)else:timed_out = False+ # Whatever the run left in its session (background jobs keep their own+ # process groups) must not run on into the next task.+ measurement.kill_session(p.pid)sel.close()p.stdout.close()p.stderr.close()diff --git a/graff-evals/test_measurement.py b/graff-evals/test_measurement.pyindex 658b78ca..498f0707 100644--- a/graff-evals/test_measurement.py+++ b/graff-evals/test_measurement.py@@ -313,6 +313,35 @@ class MeasurementTests(unittest.TestCase):self.assertEqual(subprocess.run(['git', '-C', str(sandbox), 'rev-parse', 'HEAD'],capture_output=True, text=True, check=True).stdout, head)+ @unittest.skipIf(os.name != 'posix', 'POSIX sessions')+ def test_a_run_s_leftover_jobs_are_swept_with_its_session(self):+ import subprocess, time+ # A harness in its own session starts a job in its own process group+ # (as graff does), then dies: the job is orphaned but keeps the session.+ with tempfile.TemporaryDirectory() as temp:+ pidfile = Path(temp) / 'job.pid'+ harness = subprocess.Popen(+ ['/bin/sh', '-c', f'set -m; sleep 60 & echo $! > {pidfile}; wait'],+ start_new_session=True)+ for _ in range(100):+ if pidfile.exists() and pidfile.read_text().strip():+ break+ time.sleep(0.05)+ job = int(pidfile.read_text())+ self.assertNotEqual(os.getpgid(job), harness.pid) # not in the harness's group+ os.killpg(harness.pid, 9) # what the harness used to do+ harness.wait()+ os.kill(job, 0) # still running+ self.assertGreaterEqual(measurement.kill_session(harness.pid), 1)+ for _ in range(100):+ try:+ os.kill(job, 0)+ except ProcessLookupError:+ break+ time.sleep(0.05)+ else:+ self.fail('the orphaned job survived the session sweep')+def test_untracked_source_is_in_snapshot_receipt(self):import subprocesswith tempfile.TemporaryDirectory() as temp: