Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python

graff-evals: keep the task's MCP on for --provider runs

Rach Pradhanc0ea1f9d31a9
diff --git a/graff-evals/harnesses.json b/graff-evals/harnesses.json
index 8f69f4b8..3e3fbc9a 100644
--- a/graff-evals/harnesses.json
+++ b/graff-evals/harnesses.json
@@ -353,6 +353,14 @@
"model_prefix": "codegraff",
"note": "scripted graff repl on the Codegraff gateway (CODEGRAFF_API_KEY); pass --model"
},
+ "graff-anthropic-repl": {
+ "cmd": ["{repo}/zig-out/bin/graff", "repl", "--model", "{model}", "--yolo"],
+ "stdin": "\u001b[200~{prompt}\u001b[201~\n",
+ "answer": "stdout",
+ "usage": "graff-stderr",
+ "env": {"GRAFF_POST_DEADLINE_SECS": "90", "GRAFF_NO_TELEMETRY": "1"},
+ "note": "scripted graff repl on the Anthropic API (ANTHROPIC_API_KEY); pass --provider anthropic and a claude-* --model"
+ },
"pi-codex": {
"cmd": ["pi", "-p", "--mode", "json", "--provider", "openai-codex", "--model", "{model}", "--thinking", "low", "--no-session", "{prompt}"],
"answer": "pi-json",
diff --git a/graff-evals/measurement.py b/graff-evals/measurement.py
index 1eff9d6a..8955305a 100644
--- a/graff-evals/measurement.py
+++ b/graff-evals/measurement.py
@@ -136,13 +136,14 @@ def provider_environment(provider, sandbox, source=None, model=None):
if "\n" in model or "\r" in model:
raise ValueError("model name must fit one saved-selection line")
(home / '.simple-harness-model').write_text(f'{provider}\n{model}\n')
- empty_mcp = home / 'empty-mcp.json'
- empty_mcp.write_text('{"mcpServers":{}}')
env = {k: source[k] for k in ('PATH', 'LANG', 'LC_ALL', 'SDKROOT', 'DEVELOPER_DIR', 'SYSTEMROOT') if k in source}
value = str(Path(source[credential]).expanduser().resolve()) if provider == 'codex' else source[credential]
env.update({credential: value, 'HOME': str(home), 'USERPROFILE': str(home),
'XDG_CONFIG_HOME': str(home / '.config'), 'XDG_CACHE_HOME': str(home / '.cache'),
- 'PWD': sandbox, 'GRAFF_MCP_CONFIG': str(empty_mcp), 'GRAFF_FLEET': 'off',
+ # The global MCP config points inside the isolated home and does not
+ # exist: the user's servers stay out, and the task's .mcp.json still
+ # merges. An existing empty override would switch MCP off (#549).
+ 'PWD': sandbox, 'GRAFF_MCP_CONFIG': str(home / '.codegraff' / 'mcp.json'), 'GRAFF_FLEET': 'off',
'GRAFF_NO_TELEMETRY': '1', 'GRAFF_NO_SMOLIFY': '1', 'GRAFF_BEHAVIOR_UPLOAD': 'off', 'NO_COLOR': '1'})
return env
@@ -158,7 +159,7 @@ def trace_routing(sandbox, provider, model):
except ValueError:
continue
api = [row for row in rows if row.get('ev') == 'api']
- roots = [row for row in api if row.get('agent') == 'main']
+ roots = [row for row in api if row.get('agent') in ('main', 'repl')]
pairs = sorted({(r['provider'], r['model']) for r in rows
if isinstance(r.get('provider'), str) and isinstance(r.get('model'), str)})
valid = bool(roots) and all(r.get('model') == model for r in roots)
diff --git a/graff-evals/test_measurement.py b/graff-evals/test_measurement.py
index f564cc50..a59adda8 100644
--- a/graff-evals/test_measurement.py
+++ b/graff-evals/test_measurement.py
@@ -213,7 +213,10 @@ class MeasurementTests(unittest.TestCase):
self.assertNotIn('AWS_SECRET_ACCESS_KEY', env)
self.assertNotIn('GRAFF_CODEX_URL', env)
self.assertTrue(env['HOME'].startswith(temp))
- self.assertTrue(Path(env['GRAFF_MCP_CONFIG']).is_file())
+ # Inside the isolated home and absent: no user servers, and not the
+ # empty-file off-switch that would drop the task's .mcp.json (#549).
+ self.assertTrue(env['GRAFF_MCP_CONFIG'].startswith(temp))
+ self.assertFalse(Path(env['GRAFF_MCP_CONFIG']).exists())
with self.assertRaises(ValueError):
measurement.provider_environment('codegraff', '/unused', {})