Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

Evals: sweep a run's whole session; report a crashed test as crashed

justrach883ce38000dc
diff --git a/graff-evals/measurement.py b/graff-evals/measurement.py
index c95c15c7..1eff9d6a 100644
--- a/graff-evals/measurement.py
+++ b/graff-evals/measurement.py
@@ -240,6 +240,30 @@ def verifiers_unchanged(snapshot):
return True
+def kill_session(sid):
+ """SIGKILL every process still in a harness's session; returns how many.
+ The harness runs in a session of its own. graff gives each shell job its
+ own process group but not its own session, so killing the harness's group
+ left a job behind (a hung test binary spun a core through later tasks).
+ Session ids outlive reparenting, so orphans are still found here."""
+ if os.name != 'posix' or sid <= 1:
+ return 0
+ listing = subprocess.run(['ps', '-axo', 'pid='], capture_output=True, text=True).stdout
+ killed = 0
+ for token in listing.split():
+ pid = int(token)
+ if pid == os.getpid():
+ continue
+ try:
+ if os.getsid(pid) != sid:
+ continue
+ os.kill(pid, 9)
+ killed += 1
+ except OSError:
+ continue
+ return killed
+
+
def isolate_git(sandbox):
"""Give a sandbox its own repository, holding the task tree as one commit.
Sandboxes sit inside this checkout. Without their own repository, `git
diff --git a/graff-evals/named_unit_check.py b/graff-evals/named_unit_check.py
index f8773de0..1f9eb4c5 100644
--- a/graff-evals/named_unit_check.py
+++ b/graff-evals/named_unit_check.py
@@ -28,7 +28,12 @@ def parse(output: str, name: str) -> str:
broke = re.compile(r"error: '[^'\n]*\.test\." + re.escape(name) + r"' (?:failed|terminated)")
if broke.search(output) or any(v.startswith(("FAIL", "SKIP")) for v in verdicts):
return "fail"
- return "pass" if any(v.startswith("OK") for v in verdicts) else "missing"
+ if any(v.startswith("OK") for v in verdicts):
+ return "pass"
+ # Started but never reached a verdict: a crash (a segfault aborts the
+ # binary mid-line). It ran and failed; calling it missing sent a reader
+ # looking for an unreachable test instead of a crashing fix.
+ return "crashed" if verdicts else "missing"
def grade(cwd: str, names: list[str], timeout: int = 600) -> dict[str, str]:
@@ -62,6 +67,8 @@ def self_test() -> None:
assert parse(out, "never compiled in") == "missing"
crash = "error: 'x.test.boom' terminated with signal ABRT with stderr:\n"
assert parse(crash, "boom") == "fail"
+ segv = "3/74 m.test.joinWithin: stays queued...Segmentation fault at address 0x138\n"
+ assert parse(segv, "joinWithin: stays queued") == "crashed"
print("named_unit_check self-test ok")
diff --git a/graff-evals/run.py b/graff-evals/run.py
index af88856a..a4d771b1 100755
--- a/graff-evals/run.py
+++ b/graff-evals/run.py
@@ -432,6 +432,9 @@ def one_run(hname, harness, task, model, rep, live=False):
p.wait(timeout=10)
else:
timed_out = False
+ # Whatever the run left in its session (background jobs keep their own
+ # process groups) must not run on into the next task.
+ measurement.kill_session(p.pid)
sel.close()
p.stdout.close()
p.stderr.close()
diff --git a/graff-evals/test_measurement.py b/graff-evals/test_measurement.py
index 658b78ca..498f0707 100644
--- a/graff-evals/test_measurement.py
+++ b/graff-evals/test_measurement.py
@@ -313,6 +313,35 @@ class MeasurementTests(unittest.TestCase):
self.assertEqual(subprocess.run(['git', '-C', str(sandbox), 'rev-parse', 'HEAD'],
capture_output=True, text=True, check=True).stdout, head)
+ @unittest.skipIf(os.name != 'posix', 'POSIX sessions')
+ def test_a_run_s_leftover_jobs_are_swept_with_its_session(self):
+ import subprocess, time
+ # A harness in its own session starts a job in its own process group
+ # (as graff does), then dies: the job is orphaned but keeps the session.
+ with tempfile.TemporaryDirectory() as temp:
+ pidfile = Path(temp) / 'job.pid'
+ harness = subprocess.Popen(
+ ['/bin/sh', '-c', f'set -m; sleep 60 & echo $! > {pidfile}; wait'],
+ start_new_session=True)
+ for _ in range(100):
+ if pidfile.exists() and pidfile.read_text().strip():
+ break
+ time.sleep(0.05)
+ job = int(pidfile.read_text())
+ self.assertNotEqual(os.getpgid(job), harness.pid) # not in the harness's group
+ os.killpg(harness.pid, 9) # what the harness used to do
+ harness.wait()
+ os.kill(job, 0) # still running
+ self.assertGreaterEqual(measurement.kill_session(harness.pid), 1)
+ for _ in range(100):
+ try:
+ os.kill(job, 0)
+ except ProcessLookupError:
+ break
+ time.sleep(0.05)
+ else:
+ self.fail('the orphaned job survived the session sweep')
+
def test_untracked_source_is_in_snapshot_receipt(self):
import subprocess
with tempfile.TemporaryDirectory() as temp: