Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
11.1 KB
Raw Blame
#!/usr/bin/env python3
"""Opt-in offline JSON harness latency probe; see docs/harness-latency.md."""
from __future__ import annotations

import argparse
import hashlib
import http.server
import json
import math
import os
from pathlib import Path
import platform
import queue
import statistics
import subprocess
import sys
import tempfile
import threading
import time

sys.path.insert(0, str(Path(__file__).resolve().parent / "eval"))
from mock_model import LoopbackHTTPServer

clock = time.perf_counter_ns


class Model:
    """Records measurements only; does not retain request content or headers."""
    def __init__(self, tool_steps=0):
        self.calls = []
        self.lock = threading.Lock()
        self.tool_steps = tool_steps

    def start(self):
        model = self

        class Handler(http.server.BaseHTTPRequestHandler):
            def log_message(self, *_args):
                pass

            def do_GET(self):
                payload = b'{"data":[{"id":"lmstudio"}]}'
                self.send_response(200)
                self.send_header("Content-Length", str(len(payload)))
                self.end_headers()
                self.wfile.write(payload)

            def do_POST(self):
                arrived = clock()
                length = int(self.headers["Content-Length"])
                body = json.loads(self.rfile.read(length))
                record = {"received_ns": arrived, "request_bytes": length,
                          "messages": len(body.get("messages", []))}
                with model.lock:
                    step = len(model.calls) % (model.tool_steps + 1)
                    model.calls.append(record)
                self.send_response(200)
                self.send_header("Content-Type", "text/event-stream")
                self.send_header("Connection", "close")
                self.end_headers()
                chunks = [
                    {"choices": [{"index": 0, "delta": {"role": "assistant", "content": "Acknowledged."}, "finish_reason": None}]},
                    {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
                     "usage": {"prompt_tokens": 8, "completion_tokens": 4, "total_tokens": 12}},
                ]
                if step < model.tool_steps:
                    chunks[0]["choices"][0]["delta"] = {"tool_calls": [{
                        "index": 0, "id": f"fixture_{len(model.calls)}", "type": "function",
                        "function": {"name": "read_file", "arguments": '{"path":"fixture.txt"}'}}]}
                    chunks[1]["choices"][0]["finish_reason"] = "tool_calls"
                payload = b"".join(b"data: " + json.dumps(c).encode() + b"\n\n" for c in chunks)
                # Timestamp before write: an upper bound including local send/flush,
                # not a claim to observe when the client's final byte arrived.
                with model.lock:
                    record["response_write_ns"] = clock()
                self.wfile.write(payload + b"data: [DONE]\n\n")
                self.wfile.flush()
                self.close_connection = True

        # lmstudio's built-in endpoint is fixed. Fail if occupied; never reuse it.
        self.server = LoopbackHTTPServer(("127.0.0.1", 1234), Handler)
        threading.Thread(target=self.server.serve_forever, daemon=True).start()

    def stop(self):
        self.server.shutdown()
        self.server.server_close()


def read_events(pipe, events):
    for line in pipe:
        observed = clock()
        try:
            events.put((observed, json.loads(line)))
        except ValueError:
            continue
    events.put((clock(), {"type": "eof"}))


def wait_event(events, kind, timeout):
    deadline = time.monotonic() + timeout
    tools = []
    while True:
        timestamp, event = events.get(timeout=max(0, deadline - time.monotonic()))
        if event.get("type") == "eof":
            raise RuntimeError(f"harness exited before {kind}")
        if event.get("type") == "tool_result":
            tools.append(event)
        if event.get("type") == kind:
            return timestamp, event, tools


def run_session(binary, model, turns, payload_bytes, timeout, payload_kind):
    rows = []
    pattern = 'def example():\n    return {"path": "a\\\\b", "ok": True}\n' if payload_kind == "code" else "x"
    payload = (pattern * (payload_bytes // len(pattern) + 1))[:payload_bytes]
    with tempfile.TemporaryDirectory(prefix="graff-latency-") as workspace:
        Path(workspace, "fixture.txt").write_text("Synthetic benchmark evidence.\n")
        # A clean child environment prevents inherited provider/config overrides.
        env = {key: os.environ[key] for key in ("PATH", "TMPDIR", "SYSTEMROOT") if key in os.environ}
        env.update(HOME=workspace, LMSTUDIO_API_KEY="local", GRAFF_NO_TELEMETRY="1",
                   GRAFF_FLEET="off", GRAFF_NO_SMOLIFY="1", GRAFF_LEARN_AUTO="off",
                   GRAFF_CONTEXT="2000000", NO_COLOR="1")
        events = queue.Queue()
        with tempfile.TemporaryFile(mode="w+") as errors:
            started = clock()
            process = subprocess.Popen([str(binary), "--json", "--yolo", "--model", "lmstudio"],
                cwd=workspace, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                stderr=errors, text=True, bufsize=1)
            reader = threading.Thread(target=read_events, args=(process.stdout, events), daemon=True)
            reader.start()
            try:
                for turn in range(1, turns + 1):
                    with model.lock:
                        before = len(model.calls)
                    sent = clock()
                    instruction = "Read fixture.txt, then acknowledge." if model.tool_steps else "Acknowledge only; no tools."
                    prompt = f"Synthetic latency fixture {turn}. {instruction} Data: " + payload
                    process.stdin.write(json.dumps({"type": "user", "text": prompt}) + "\n")
                    process.stdin.flush()
                    completed, event, tools = wait_event(events, "turn", timeout)
                    with model.lock:
                        calls = [dict(c) for c in model.calls[before:]]
                    if len(calls) != model.tool_steps + 1 or event.get("text", "").strip() != "Acknowledged.":
                        raise RuntimeError(f"fixture drift at turn {turn}: unexpected request count or final reply ({len(calls)} requests)")
                    if len(tools) != model.tool_steps or any(t.get("name") != "read_file" or
                            t.get("is_error") or "Synthetic benchmark evidence." not in t.get("text", "") for t in tools):
                        raise RuntimeError(f"fixture tool execution failed at turn {turn}")
                    call = calls[-1]
                    if "response_write_ns" not in call:
                        raise RuntimeError("turn event preceded response write")
                    rows.append({"turn": turn, "request_bytes": call["request_bytes"],
                        "messages": call["messages"],
                        "prompt_to_request_ms": (calls[0]["received_ns"] - sent) / 1e6,
                        "response_to_turn_ms": (completed - call["response_write_ns"]) / 1e6,
                        "tool_roundtrip_ms": [(b["received_ns"] - a["response_write_ns"]) / 1e6
                                              for a, b in zip(calls, calls[1:])],
                        "prompt_to_turn_ms": (completed - sent) / 1e6})
                process.stdin.close()
                process.wait(timeout=timeout)
                if process.returncode:
                    raise RuntimeError(f"harness exited {process.returncode}")
                exited = clock()
                shutdown_ms = (exited - completed) / 1e6
            finally:
                if process.poll() is None:
                    process.kill()
                    process.wait()
                reader.join(timeout=2)
                process.stdout.close()
                if not process.stdin.closed:
                    process.stdin.close()
    return {"samples": rows, "shutdown_ms": shutdown_ms, "session_ms": (exited - started) / 1e6}


def summarize(values):
    values = sorted(values)
    return {"n": len(values), "median": statistics.median(values),
            "p95": values[max(0, math.ceil(len(values) * .95) - 1)], "max": max(values)}


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--graff", type=Path, default=Path(__file__).resolve().parents[1] / "zig-out/bin/graff")
    parser.add_argument("--repeats", type=int, default=3)
    parser.add_argument("--turns", type=int, default=100)
    parser.add_argument("--payload-bytes", type=int, default=4096)
    parser.add_argument("--payload-kind", choices=("text", "code"), default="text",
                        help="plain ASCII or code with quotes, backslashes, and newlines")
    parser.add_argument("--tool-steps", type=int, default=0, help="read_file steps before each final reply")
    parser.add_argument("--timeout", type=float, default=60)
    parser.add_argument("--out", type=Path, required=True)
    args = parser.parse_args()
    if min(args.repeats, args.turns, args.payload_bytes, args.timeout) <= 0:
        parser.error("counts, payload size, and timeout must be positive")
    if not 0 <= args.tool_steps <= 3:
        parser.error("tool-steps must be between 0 and 3")
    binary = args.graff.resolve(strict=True)
    digest = hashlib.sha256(binary.read_bytes()).hexdigest()
    model = Model(args.tool_steps)
    model.start()
    try:
        runs = []
        for repeat in range(args.repeats):
            runs.append(run_session(binary, model, args.turns, args.payload_bytes, args.timeout, args.payload_kind))
            print(f"Completed session {repeat + 1}/{args.repeats}", file=sys.stderr, flush=True)
    finally:
        model.stop()
    if hashlib.sha256(binary.read_bytes()).hexdigest() != digest:
        raise RuntimeError("binary changed during measurement; rerun with a stable copy")
    summary = {}
    # Compare identical history positions across independent sessions, not a mix
    # of different workloads masquerading as repeated samples.
    for turn in sorted({1, min(10, args.turns), min(50, args.turns), args.turns}):
        samples = [run["samples"][turn - 1] for run in runs]
        summary[str(turn)] = {key: summarize([s[key] for s in samples]) for key in
            ("prompt_to_request_ms", "response_to_turn_ms", "prompt_to_turn_ms", "request_bytes")}
    report = {"schema_version": 1, "binary_sha256": digest,
        "platform": platform.system(), "machine": platform.machine(),
        "python": platform.python_version(), "turns": args.turns,
        "payload_bytes": args.payload_bytes, "repeats": args.repeats,
        "payload_kind": args.payload_kind,
        "tool_steps": args.tool_steps,
        "summary_by_turn": summary, "runs": runs}
    args.out.parent.mkdir(parents=True, exist_ok=True)
    args.out.write_text(json.dumps(report, indent=2) + "\n")
    print(json.dumps(summary, indent=2))


if __name__ == "__main__":
    try:
        main()
    except (RuntimeError, OSError, queue.Empty, subprocess.TimeoutExpired) as exc:
        sys.exit(f"latency probe failed: {exc}")