graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
Make offline evaluation checks portable on Windows
justrach6c3280330784
diff --git a/graff-evals/test_measurement.py b/graff-evals/test_measurement.pyindex 81c2a60f..9191b0bf 100644--- a/graff-evals/test_measurement.py+++ b/graff-evals/test_measurement.py@@ -246,7 +246,8 @@ class MeasurementTests(unittest.TestCase):env = {'CODEGRAFF_API_KEY': 'fixture'}request_capture.configure(env, first)self.assertEqual(env['GRAFF_REQ_STATS'], '1')- self.assertEqual(Path(env['GRAFF_REQ_DUMP_DIR']).stat().st_mode & 0o777, 0o700)+ if os.name != 'nt': # Windows mode bits do not represent inherited ACLs.+ self.assertEqual(Path(env['GRAFF_REQ_DUMP_DIR']).stat().st_mode & 0o777, 0o700)request_capture.configure({}, second)self.assertNotEqual(request_capture.directory(first), request_capture.directory(second))run = request_capture.directory(first) / 'run-fixture'@@ -318,7 +319,8 @@ class MeasurementTests(unittest.TestCase):with self.assertRaises(FileExistsError):measurement.isolated_paths(target)measurement.private_logs(sandboxes, 'fixture stdout', 'fixture stderr')- self.assertEqual((Path(sandboxes) / '.eval-stdout.txt').stat().st_mode & 0o777, 0o600)+ if os.name != 'nt': # Windows mode bits do not represent inherited ACLs.+ self.assertEqual((Path(sandboxes) / '.eval-stdout.txt').stat().st_mode & 0o777, 0o600)if __name__ == '__main__':diff --git a/scripts/eval-cache-affinity.py b/scripts/eval-cache-affinity.pyindex 5f3c178f..ade8cb6c 100644--- a/scripts/eval-cache-affinity.py+++ b/scripts/eval-cache-affinity.py@@ -18,6 +18,7 @@ import shutilimport signalimport statisticsimport subprocess+import sysimport timeHELPERS = Path(__file__).resolve().parents[1] / 'graff-evals' / 'measurement.py'@@ -134,7 +135,7 @@ def run(binary, cwd, artifacts, model, prompt, timeout, source_env):env = measurement.provider_environment('codegraff', str(artifacts), source_env, model)env['PWD'] = str(cwd)env['GRAFF_LEARNING_PRIVACY'] = 'local'- argv = [str(binary), '--no-local-tools', '--yolo', '--max-model-calls', '2',+ argv = ([sys.executable] if binary.suffix == '.py' else []) + [str(binary), '--no-local-tools', '--yolo', '--max-model-calls', '2','--no-telemetry', '-p', prompt]expected_hash = digest(binary)started = time.monotonic_ns()@@ -224,7 +225,7 @@ def main():if not source.is_file() or not os.access(source, os.X_OK):ap.error(f'{arm} must be an executable file')before_hash = digest(source)- dest = binaries / arm+ dest = binaries / f'{arm}{source.suffix}'shutil.copyfile(source, dest)dest.chmod(0o500)if digest(dest) != before_hash or digest(source) != before_hash:diff --git a/scripts/test-eval-cache-affinity.py b/scripts/test-eval-cache-affinity.pyindex 5f2ea687..bc51c705 100644--- a/scripts/test-eval-cache-affinity.py+++ b/scripts/test-eval-cache-affinity.py@@ -44,7 +44,7 @@ class ProbeTests(unittest.TestCase):with tempfile.TemporaryDirectory() as raw:root = Path(raw)for arm in ('before', 'after'):- binary = root / arm+ binary = root / f'{arm}.py'binary.write_text(f'''#!{sys.executable}import json, osfrom pathlib import Path@@ -61,8 +61,8 @@ print({FOOTER!r}, file=__import__('sys').stderr)binary.chmod(0o700)out = root / 'result'env = dict(os.environ, CODEGRAFF_API_KEY='offline-fixture-only', OPENAI_API_KEY='must-not-inherit')- p = subprocess.run([sys.executable, str(SCRIPT), '--before', str(root/'before'),- '--after', str(root/'after'), '--model', 'fixture', '--repeats', '2',+ p = subprocess.run([sys.executable, str(SCRIPT), '--before', str(root/'before.py'),+ '--after', str(root/'after.py'), '--model', 'fixture', '--repeats', '2','--out', str(out)], env=env, capture_output=True, text=True, timeout=30)self.assertEqual(p.returncode, 0, p.stderr + p.stdout)pairs = json.loads((out/'pairs.json').read_text())@@ -71,9 +71,11 @@ print({FOOTER!r}, file=__import__('sys').stderr)self.assertEqual(json.loads((out/'summary.json').read_text())['fixture']['valid_pairs'], 2)for pair in pairs:self.assertEqual(pair['before']['linked']['prompt_sha256'], pair['after']['linked']['prompt_sha256'])- self.assertEqual(out.stat().st_mode & 0o777, 0o700)- for path in out.rglob('*'):- self.assertEqual(path.stat().st_mode & 0o077, 0, str(path))+ # Windows reports synthetic mode bits; ACLs inherit from the private temp root.+ if os.name != 'nt':+ self.assertEqual(out.stat().st_mode & 0o777, 0o700)+ for path in out.rglob('*'):+ self.assertEqual(path.stat().st_mode & 0o077, 0, str(path))if __name__ == '__main__':diff --git a/scripts/test-workflow-isolation.py b/scripts/test-workflow-isolation.pyindex 3e8b72a2..75feeac5 100644--- a/scripts/test-workflow-isolation.py+++ b/scripts/test-workflow-isolation.py@@ -75,8 +75,23 @@ def check(binary, phases):trees = list((root/'.graff/worktrees').glob('agent-workflow-*'))assert len(trees) == len(expected), treesassert sorted((tree/'result.txt').read_text() for tree in trees) == expected+ def values(item):+ if isinstance(item, str):+ yield item+ elif isinstance(item, dict):+ for value in item.values():+ yield from values(value)+ elif isinstance(item, list):+ for value in item:+ yield from values(value)++ def path_text(value):+ return value.replace(chr(92) * 2, chr(92)).replace(chr(92), '/').casefold()++ delivered = path_text('\n'.join(values(tool_results)))for tree in trees:- assert str(tree) in text, 'retained path was not delivered'+ expected_path = path_text(str(tree))+ assert expected_path in delivered, 'retained path was not delivered'branch = subprocess.check_output(['git', '-C', str(tree), 'branch', '--show-current'], text=True).strip()assert branch in text, 'retained branch was not delivered'print(f'PASS {"phases" if phases else "pipeline"}: dependent reads, isolated items, committed trees retained and delivered')diff --git a/scripts/test-workspace-lifecycle.py b/scripts/test-workspace-lifecycle.pyindex 8486dc34..8ce04b60 100644--- a/scripts/test-workspace-lifecycle.py+++ b/scripts/test-workspace-lifecycle.py@@ -47,9 +47,18 @@ with tempfile.TemporaryDirectory(prefix="graff-workspace-lifecycle-") as temporaassert "setup ok" in graff("create", "task", "main")tree = root / ".graff/worktrees/task"assert git("config", "branch.worktree-task.graff-base") == "main"- assert Path((tree / ".graff/setup-cwd").read_text().strip()).resolve() == tree.resolve()+ def recorded_cwd(name):+ value = (tree / f'.graff/{name}-cwd').read_text().strip()+ # Git Bash pwd uses /d/... while Python on Windows uses D:\\... .+ if os.name == 'nt' and value.startswith('/'):+ drive, separator, rest = value[1:].partition('/')+ if separator and len(drive) == 1 and drive.isalpha():+ value = f'{drive}:/{rest}'+ return Path(value).resolve()++ assert recorded_cwd('setup') == tree.resolve()assert "run finished" in graff("run", "task")- assert Path((tree / ".graff/run-cwd").read_text().strip()).resolve() == tree.resolve()+ assert recorded_cwd('run') == tree.resolve()(tree / "file").write_text("task\n")git("add", "file", cwd=tree)git("commit", "-qm", "task", cwd=tree)