graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
graff-evals: keep the task's MCP on for --provider runs
Rach Pradhanc0ea1f9d31a9
diff --git a/graff-evals/harnesses.json b/graff-evals/harnesses.jsonindex 8f69f4b8..3e3fbc9a 100644--- a/graff-evals/harnesses.json+++ b/graff-evals/harnesses.json@@ -353,6 +353,14 @@"model_prefix": "codegraff","note": "scripted graff repl on the Codegraff gateway (CODEGRAFF_API_KEY); pass --model"},+ "graff-anthropic-repl": {+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],+ "stdin": "\u001b[200~{prompt}\u001b[201~\n",+ "answer": "stdout",+ "usage": "graff-stderr",+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_NO_TELEMETRY": "1"},+ "note": "scripted graff repl on the Anthropic API (ANTHROPIC_API_KEY); pass --provider anthropic and a claude-* --model"+ },"pi-codex": {"cmd": ["pi", "-p", "--mode", "json", "--provider", "openai-codex", "--model", "{model}", "--thinking", "low", "--no-session", "{prompt}"],"answer": "pi-json",diff --git a/graff-evals/measurement.py b/graff-evals/measurement.pyindex 1eff9d6a..8955305a 100644--- a/graff-evals/measurement.py+++ b/graff-evals/measurement.py@@ -136,13 +136,14 @@ def provider_environment(provider, sandbox, source=None, model=None):if "\n" in model or "\r" in model:raise ValueError("model name must fit one saved-selection line")(home / '.simple-harness-model').write_text(f'{provider}\n{model}\n')- empty_mcp = home / 'empty-mcp.json'- empty_mcp.write_text('{"mcpServers":{}}')env = {k: source[k] for k in ('PATH', 'LANG', 'LC_ALL', 'SDKROOT', 'DEVELOPER_DIR', 'SYSTEMROOT') if k in source}value = str(Path(source[credential]).expanduser().resolve()) if provider == 'codex' else source[credential]env.update({credential: value, 'HOME': str(home), 'USERPROFILE': str(home),'XDG_CONFIG_HOME': str(home / '.config'), 'XDG_CACHE_HOME': str(home / '.cache'),- 'PWD': sandbox, 'GRAFF_MCP_CONFIG': str(empty_mcp), 'GRAFF_FLEET': 'off',+ # The global MCP config points inside the isolated home and does not+ # exist: the user's servers stay out, and the task's .mcp.json still+ # merges. An existing empty override would switch MCP off (#549).+ 'PWD': sandbox, 'GRAFF_MCP_CONFIG': str(home / '.codegraff' / 'mcp.json'), 'GRAFF_FLEET': 'off','GRAFF_NO_TELEMETRY': '1', 'GRAFF_NO_SMOLIFY': '1', 'GRAFF_BEHAVIOR_UPLOAD': 'off', 'NO_COLOR': '1'})return env@@ -158,7 +159,7 @@ def trace_routing(sandbox, provider, model):except ValueError:continueapi = [row for row in rows if row.get('ev') == 'api']- roots = [row for row in api if row.get('agent') == 'main']+ roots = [row for row in api if row.get('agent') in ('main', 'repl')]pairs = sorted({(r['provider'], r['model']) for r in rowsif isinstance(r.get('provider'), str) and isinstance(r.get('model'), str)})valid = bool(roots) and all(r.get('model') == model for r in roots)diff --git a/graff-evals/test_measurement.py b/graff-evals/test_measurement.pyindex f564cc50..a59adda8 100644--- a/graff-evals/test_measurement.py+++ b/graff-evals/test_measurement.py@@ -213,7 +213,10 @@ class MeasurementTests(unittest.TestCase):self.assertNotIn('AWS_SECRET_ACCESS_KEY', env)self.assertNotIn('GRAFF_CODEX_URL', env)self.assertTrue(env['HOME'].startswith(temp))- self.assertTrue(Path(env['GRAFF_MCP_CONFIG']).is_file())+ # Inside the isolated home and absent: no user servers, and not the+ # empty-file off-switch that would drop the task's .mcp.json (#549).+ self.assertTrue(env['GRAFF_MCP_CONFIG'].startswith(temp))+ self.assertFalse(Path(env['GRAFF_MCP_CONFIG']).exists())with self.assertRaises(ValueError):measurement.provider_environment('codegraff', '/unused', {})