Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

Make offline evaluation checks portable on Windows

justrach6c3280330784
diff --git a/graff-evals/test_measurement.py b/graff-evals/test_measurement.py
index 81c2a60f..9191b0bf 100644
--- a/graff-evals/test_measurement.py
+++ b/graff-evals/test_measurement.py
@@ -246,7 +246,8 @@ class MeasurementTests(unittest.TestCase):
env = {'CODEGRAFF_API_KEY': 'fixture'}
request_capture.configure(env, first)
self.assertEqual(env['GRAFF_REQ_STATS'], '1')
- self.assertEqual(Path(env['GRAFF_REQ_DUMP_DIR']).stat().st_mode & 0o777, 0o700)
+ if os.name != 'nt': # Windows mode bits do not represent inherited ACLs.
+ self.assertEqual(Path(env['GRAFF_REQ_DUMP_DIR']).stat().st_mode & 0o777, 0o700)
request_capture.configure({}, second)
self.assertNotEqual(request_capture.directory(first), request_capture.directory(second))
run = request_capture.directory(first) / 'run-fixture'
@@ -318,7 +319,8 @@ class MeasurementTests(unittest.TestCase):
with self.assertRaises(FileExistsError):
measurement.isolated_paths(target)
measurement.private_logs(sandboxes, 'fixture stdout', 'fixture stderr')
- self.assertEqual((Path(sandboxes) / '.eval-stdout.txt').stat().st_mode & 0o777, 0o600)
+ if os.name != 'nt': # Windows mode bits do not represent inherited ACLs.
+ self.assertEqual((Path(sandboxes) / '.eval-stdout.txt').stat().st_mode & 0o777, 0o600)
if __name__ == '__main__':
diff --git a/scripts/eval-cache-affinity.py b/scripts/eval-cache-affinity.py
index 5f3c178f..ade8cb6c 100644
--- a/scripts/eval-cache-affinity.py
+++ b/scripts/eval-cache-affinity.py
@@ -18,6 +18,7 @@ import shutil
import signal
import statistics
import subprocess
+import sys
import time
HELPERS = Path(__file__).resolve().parents[1] / 'graff-evals' / 'measurement.py'
@@ -134,7 +135,7 @@ def run(binary, cwd, artifacts, model, prompt, timeout, source_env):
env = measurement.provider_environment('codegraff', str(artifacts), source_env, model)
env['PWD'] = str(cwd)
env['GRAFF_LEARNING_PRIVACY'] = 'local'
- argv = [str(binary), '--no-local-tools', '--yolo', '--max-model-calls', '2',
+ argv = ([sys.executable] if binary.suffix == '.py' else []) + [str(binary), '--no-local-tools', '--yolo', '--max-model-calls', '2',
'--no-telemetry', '-p', prompt]
expected_hash = digest(binary)
started = time.monotonic_ns()
@@ -224,7 +225,7 @@ def main():
if not source.is_file() or not os.access(source, os.X_OK):
ap.error(f'{arm} must be an executable file')
before_hash = digest(source)
- dest = binaries / arm
+ dest = binaries / f'{arm}{source.suffix}'
shutil.copyfile(source, dest)
dest.chmod(0o500)
if digest(dest) != before_hash or digest(source) != before_hash:
diff --git a/scripts/test-eval-cache-affinity.py b/scripts/test-eval-cache-affinity.py
index 5f2ea687..bc51c705 100644
--- a/scripts/test-eval-cache-affinity.py
+++ b/scripts/test-eval-cache-affinity.py
@@ -44,7 +44,7 @@ class ProbeTests(unittest.TestCase):
with tempfile.TemporaryDirectory() as raw:
root = Path(raw)
for arm in ('before', 'after'):
- binary = root / arm
+ binary = root / f'{arm}.py'
binary.write_text(f'''#!{sys.executable}
import json, os
from pathlib import Path
@@ -61,8 +61,8 @@ print({FOOTER!r}, file=__import__('sys').stderr)
binary.chmod(0o700)
out = root / 'result'
env = dict(os.environ, CODEGRAFF_API_KEY='offline-fixture-only', OPENAI_API_KEY='must-not-inherit')
- p = subprocess.run([sys.executable, str(SCRIPT), '--before', str(root/'before'),
- '--after', str(root/'after'), '--model', 'fixture', '--repeats', '2',
+ p = subprocess.run([sys.executable, str(SCRIPT), '--before', str(root/'before.py'),
+ '--after', str(root/'after.py'), '--model', 'fixture', '--repeats', '2',
'--out', str(out)], env=env, capture_output=True, text=True, timeout=30)
self.assertEqual(p.returncode, 0, p.stderr + p.stdout)
pairs = json.loads((out/'pairs.json').read_text())
@@ -71,9 +71,11 @@ print({FOOTER!r}, file=__import__('sys').stderr)
self.assertEqual(json.loads((out/'summary.json').read_text())['fixture']['valid_pairs'], 2)
for pair in pairs:
self.assertEqual(pair['before']['linked']['prompt_sha256'], pair['after']['linked']['prompt_sha256'])
- self.assertEqual(out.stat().st_mode & 0o777, 0o700)
- for path in out.rglob('*'):
- self.assertEqual(path.stat().st_mode & 0o077, 0, str(path))
+ # Windows reports synthetic mode bits; ACLs inherit from the private temp root.
+ if os.name != 'nt':
+ self.assertEqual(out.stat().st_mode & 0o777, 0o700)
+ for path in out.rglob('*'):
+ self.assertEqual(path.stat().st_mode & 0o077, 0, str(path))
if __name__ == '__main__':
diff --git a/scripts/test-workflow-isolation.py b/scripts/test-workflow-isolation.py
index 3e8b72a2..75feeac5 100644
--- a/scripts/test-workflow-isolation.py
+++ b/scripts/test-workflow-isolation.py
@@ -75,8 +75,23 @@ def check(binary, phases):
trees = list((root/'.graff/worktrees').glob('agent-workflow-*'))
assert len(trees) == len(expected), trees
assert sorted((tree/'result.txt').read_text() for tree in trees) == expected
+ def values(item):
+ if isinstance(item, str):
+ yield item
+ elif isinstance(item, dict):
+ for value in item.values():
+ yield from values(value)
+ elif isinstance(item, list):
+ for value in item:
+ yield from values(value)
+
+ def path_text(value):
+ return value.replace(chr(92) * 2, chr(92)).replace(chr(92), '/').casefold()
+
+ delivered = path_text('\n'.join(values(tool_results)))
for tree in trees:
- assert str(tree) in text, 'retained path was not delivered'
+ expected_path = path_text(str(tree))
+ assert expected_path in delivered, 'retained path was not delivered'
branch = subprocess.check_output(['git', '-C', str(tree), 'branch', '--show-current'], text=True).strip()
assert branch in text, 'retained branch was not delivered'
print(f'PASS {"phases" if phases else "pipeline"}: dependent reads, isolated items, committed trees retained and delivered')
diff --git a/scripts/test-workspace-lifecycle.py b/scripts/test-workspace-lifecycle.py
index 8486dc34..8ce04b60 100644
--- a/scripts/test-workspace-lifecycle.py
+++ b/scripts/test-workspace-lifecycle.py
@@ -47,9 +47,18 @@ with tempfile.TemporaryDirectory(prefix="graff-workspace-lifecycle-") as tempora
assert "setup ok" in graff("create", "task", "main")
tree = root / ".graff/worktrees/task"
assert git("config", "branch.worktree-task.graff-base") == "main"
- assert Path((tree / ".graff/setup-cwd").read_text().strip()).resolve() == tree.resolve()
+ def recorded_cwd(name):
+ value = (tree / f'.graff/{name}-cwd').read_text().strip()
+ # Git Bash pwd uses /d/... while Python on Windows uses D:\\... .
+ if os.name == 'nt' and value.startswith('/'):
+ drive, separator, rest = value[1:].partition('/')
+ if separator and len(drive) == 1 and drive.isalpha():
+ value = f'{drive}:/{rest}'
+ return Path(value).resolve()
+
+ assert recorded_cwd('setup') == tree.resolve()
assert "run finished" in graff("run", "task")
- assert Path((tree / ".graff/run-cwd").read_text().strip()).resolve() == tree.resolve()
+ assert recorded_cwd('run') == tree.resolve()
(tree / "file").write_text("task\n")
git("add", "file", cwd=tree)
git("commit", "-qm", "task", cwd=tree)