graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs
Python
#!/usr/bin/env python3
"""A scripted OpenAI-compatible model, for tier-2 behavioral evals.
scripts/openai_mock.py answers everything with "done", which is enough to prove
plumbing but cannot drive the harness through a checklist. This one replays a
script: each entry is the reply to the next model call in the run, in order, so
a case can say "call todo_write with these three items, then finish them, then
claim completion" and the assertions can check what the harness did with it.
A reply is either:
{"text": "..."}
{"tool": "todo_write", "arguments": {...}}
{"tool": "todo_write", "arguments": {...}, "text": "..."}
{"text": "...", "stream_error": {"type": "api_error", "message": "..."}}
The last shape streams its text, then a terminal error frame instead of a
finish - a provider that died mid-response - so a case can prove the harness
retries rather than trims or dies.
Past the end of the script every call answers with `exhausted_text` (default
"done"), so a run that takes more turns than the case scripted still ends
instead of hanging.
Requests are recorded in order so an assertion can look at what the harness
actually sent - that is how "compaction restated the checklist" is checked.
"""
from __future__ import annotations
import http.server
import json
import socketserver
import threading
import time
from typing import Any
class LoopbackHTTPServer(http.server.ThreadingHTTPServer):
def server_bind(self) -> None:
# HTTPServer resolves its own hostname during bind. This offline fixture
# needs only the numeric loopback address; DNS can stall runner startup.
socketserver.TCPServer.server_bind(self)
self.server_name, self.server_port = self.server_address[:2]
class ScriptedModel:
def __init__(self, script: list[dict[str, Any]], exhausted_text: str = "done") -> None:
self.script = script
self.exhausted_text = exhausted_text
self.requests: list[dict[str, Any]] = []
self.request_headers: list[dict[str, str]] = []
self._lock = threading.Lock()
self._server: http.server.ThreadingHTTPServer | None = None
def next_reply(self, body: dict[str, Any]) -> dict[str, Any]:
with self._lock:
self.requests.append(body)
index = len(self.requests) - 1
if index < len(self.script):
return self.script[index]
return {"text": self.exhausted_text}
def start(self, port: int = 1234) -> int:
model = self
class Handler(http.server.BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def _body(self) -> dict[str, Any]:
length = int(self.headers.get("content-length", 0))
if not length:
return {}
try:
return json.loads(self.rfile.read(length))
except (ValueError, UnicodeDecodeError):
return {}
def _send(self, payload: bytes, content_type: str) -> None:
self.send_response(200)
self.send_header("content-type", content_type)
self.send_header("content-length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
def do_GET(self) -> None: # noqa: N802 - /v1/models probes
self._send(json.dumps({"data": [{"id": "mock"}]}).encode(), "application/json")
def do_POST(self) -> None: # noqa: N802
body = self._body()
model.request_headers.append({k.lower(): v for k, v in self.headers.items()})
reply = model.next_reply(body)
if reply.get("http_status"):
payload = reply.get("raw_error_body", json.dumps({"error": {"message": reply.get("error", "Scripted request failure")}})).encode()
self.send_response(reply["http_status"])
self.send_header("content-type", "application/json")
self.send_header("content-length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
return
if body.get("stream"):
self._stream(reply)
else:
self._whole(reply)
def _message(self, reply: dict[str, Any]) -> dict[str, Any]:
message: dict[str, Any] = {"role": "assistant", "content": reply.get("text", "")}
calls = reply.get("tools") or ([reply] if reply.get("tool") else [])
if calls:
message["tool_calls"] = [{
"id": f"call_{len(model.requests)}" + (f"_{index}" if len(calls) > 1 else ""),
"type": "function",
"function": {
"name": call["tool"],
"arguments": json.dumps(call.get("arguments", {})),
},
} for index, call in enumerate(calls)]
return message
def _whole(self, reply: dict[str, Any]) -> None:
if error := reply.get("stream_error"):
self._send(json.dumps({"type": "error", "error": error}).encode(), "application/json")
return
message = self._message(reply)
finish = "tool_calls" if "tool_calls" in message else "stop"
self._send(json.dumps({
"id": "mock", "object": "chat.completion", "model": "mock",
"choices": [{"index": 0, "message": message, "finish_reason": finish}],
"usage": reply.get("usage", {"prompt_tokens": 8, "completion_tokens": 4, "total_tokens": 12}),
}).encode(), "application/json")
def _stream(self, reply: dict[str, Any]) -> None:
self._fragment_bytes = reply.get("stream_fragment_bytes", 0)
self._fragment_delay = reply.get("stream_fragment_delay_ms", 0) / 1000
self.send_response(200)
self.send_header("content-type", "text/event-stream")
self.send_header("cache-control", "no-cache")
self.send_header("connection", "close")
self.end_headers()
message = self._message(reply)
deltas: list[dict[str, Any]] = []
if message["content"]:
deltas.append({"role": "assistant", "content": message["content"]})
for position, call in enumerate(message.get("tool_calls", [])):
function = call["function"]
size = reply.get("argument_chunk_size", 0)
arguments = function["arguments"]
fragments = [arguments[i:i + size] for i in range(0, len(arguments), size)] if size else [arguments]
for number, fragment in enumerate(fragments):
fn = {"arguments": fragment}
if number == 0: fn["name"] = function["name"]
deltas.append({"tool_calls": [{
"index": position, "id": call["id"], "type": "function", "function": fn,
}]})
if "stream_deltas" in reply:
deltas = reply["stream_deltas"]
finish = reply.get("finish") or ("tool_calls" if "tool_calls" in message or "stream_deltas" in reply else "stop")
for delta in deltas or [{"role": "assistant", "content": ""}]:
self._chunk({"choices": [{"index": 0, "delta": delta, "finish_reason": None}]})
if hook := getattr(model, "after_stream_delta", None):
self.wfile.flush()
hook(delta)
if error := reply.get("stream_error"):
# A mid-stream server failure: the provider streamed output,
# then sent a terminal {"type":"error"} frame (xAI's
# "Internal error during token parsing" shape, ADR 0148).
self.wfile.write(b"data: " + json.dumps({"type": "error", "error": error}).encode() + b"\n\n")
self.wfile.write(b"data: [DONE]\n\n")
self.wfile.flush()
self.close_connection = True
return
self._chunk({
"choices": [{"index": 0, "delta": {}, "finish_reason": finish}],
"usage": reply.get("usage", {"prompt_tokens": 8, "completion_tokens": 4, "total_tokens": 12}),
})
self.wfile.write(b"data: [DONE]\n\n")
self.wfile.flush()
self.close_connection = True
def _chunk(self, payload: dict[str, Any]) -> None:
framed = dict(payload, id="mock", object="chat.completion.chunk", model="mock")
wire = b"data: " + json.dumps(framed).encode() + b"\n\n"
if not self._fragment_bytes:
self.wfile.write(wire)
return
for start in range(0, len(wire), self._fragment_bytes):
self.wfile.write(wire[start:start + self._fragment_bytes])
self.wfile.flush()
time.sleep(self._fragment_delay)
def log_message(self, *_args) -> None:
pass
self._server = LoopbackHTTPServer(("127.0.0.1", port), Handler)
threading.Thread(target=self._server.serve_forever, daemon=True).start()
return self._server.server_address[1]
def stop(self) -> None:
if self._server is not None:
self._server.shutdown()
self._server.server_close()
self._server = None
def main() -> None:
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--port", type=int, default=1234)
parser.add_argument("--script", help="JSON file holding the reply list")
args = parser.parse_args()
script = json.load(open(args.script, encoding="utf-8")) if args.script else []
model = ScriptedModel(script)
port = model.start(args.port)
print(f"scripted model on 127.0.0.1:{port} ({len(script)} scripted replies)", flush=True)
try:
threading.Event().wait()
except KeyboardInterrupt:
model.stop()
if __name__ == "__main__":
main()