Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
19.6 KB
Raw Blame
#!/usr/bin/env python3
"""Offline ACP load: model precedence, replay, context, and stale shell handles."""
import json
import os
import queue
import re
import signal
import subprocess
import sys
import tempfile
import threading
import time
from pathlib import Path

REPO = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO / "scripts/eval"))
from mock_model import ScriptedModel


class Acp:
    def __init__(self, binary, cwd, home, port, extra_env=None, extra_args=()):
        env = {k: v for k, v in os.environ.items() if not k.endswith("_API_KEY") and not k.startswith(("GRAFF_", "HARNESS_"))}
        env.update(HOME=str(home), AI_GATEWAY_API_KEY="local", GRAFF_FLEET="off",
                   GRAFF_VERCEL_URL=f"http://127.0.0.1:{port}/v1/chat/completions",
                   GRAFF_NO_TELEMETRY="1", GRAFF_NO_SMOLIFY="1",
                   GRAFF_BEHAVIOR_UPLOAD="off", NO_COLOR="1")
        if extra_env:
            env.update(extra_env)
        self.err = open(cwd / f"acp-{time.time_ns()}.stderr", "w")
        try:
            self.proc = subprocess.Popen([str(binary), "acp", "--yolo", "--old", "--model", "vercel", *extra_args],
                                         cwd=cwd, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                                         stderr=self.err, start_new_session=True)
        except BaseException:
            self.err.close()
            raise
        self.chunks = queue.SimpleQueue()
        self.reader = threading.Thread(target=self._read_stdout, daemon=True)
        self.reader.start()
        self.pending = b""
        self.next_id = 0
        self.events = []

    def _read_stdout(self):
        try:
            while chunk := os.read(self.proc.stdout.fileno(), 65536):
                self.chunks.put(chunk)
        except OSError:
            pass
        finally:
            self.chunks.put(None)

    def exit_detail(self):
        """How graff ended: its exit status (a signal by name) and the tail of
        its stderr, so a crash in CI names its cause instead of only that it
        happened."""
        try:
            code = self.proc.wait(timeout=2)
        except subprocess.TimeoutExpired:
            code = None
        how = f"exit {code}"
        if code is not None and code < 0:
            try:
                how += f" ({signal.Signals(-code).name})"
            except ValueError:
                pass
        self.err.flush()
        try:
            with open(self.err.name, errors="replace") as handle:
                tail = handle.read()[-2000:]
        except OSError:
            tail = ""
        return f"{how}; stderr tail:\n{tail}"

    def request(self, method, params=None, timeout=25):
        self.next_id += 1
        target = self.next_id
        self.proc.stdin.write((json.dumps({"jsonrpc": "2.0", "id": target, "method": method,
                                           "params": params or {}}) + "\n").encode())
        self.proc.stdin.flush()
        deadline = time.monotonic() + timeout
        while time.monotonic() < deadline:
            if b"\n" not in self.pending:
                try:
                    chunk = self.chunks.get(timeout=.1)
                except queue.Empty:
                    continue
                if chunk is None:
                    raise AssertionError(f"ACP exited during {method}: {self.exit_detail()}")
                self.pending += chunk
            while b"\n" in self.pending:
                line, self.pending = self.pending.split(b"\n", 1)
                item = json.loads(line)
                self.events.append(item)
                if item.get("id") == target:
                    return item
        raise AssertionError(f"ACP {method} timed out")

    def close(self):
        try:
            if self.proc.stdin and not self.proc.stdin.closed:
                try:
                    self.proc.stdin.close()
                except OSError:
                    pass
            try:
                self.proc.wait(timeout=3)
            except subprocess.TimeoutExpired:
                if os.name == "nt":
                    self.proc.terminate()
                else:
                    os.killpg(self.proc.pid, signal.SIGTERM)
                try:
                    self.proc.wait(timeout=3)
                except subprocess.TimeoutExpired:
                    if os.name == "nt":
                        self.proc.kill()
                    else:
                        os.killpg(self.proc.pid, signal.SIGKILL)
                    self.proc.wait(timeout=3)
        finally:
            try:
                self.reader.join(timeout=3)
                if self.proc.stdout:
                    self.proc.stdout.close()
            finally:
                self.err.close()


def tool_text(body):
    return [m.get("content", "") for m in body.get("messages", []) if m.get("role") == "tool"]


class FollowingModel(ScriptedModel):
    def __init__(self, old_handle):
        super().__init__([])
        self.old_handle = old_handle
        self.new_handle = None
        self.stale_output = ""
        self.stale_kill = ""
        self.new_output = ""

    def next_reply(self, body):
        with self._lock:
            self.requests.append(body)
            n = len(self.requests)
        if n == 1:
            return {"tool": "shell", "arguments": {"action": "run", "command": "sleep 5", "run_in_background": True}}
        if n == 2:
            found = re.search(r"\[job (\d+) started:", tool_text(body)[-1])
            assert found, "new background job handle missing"
            self.new_handle = int(found.group(1))
            return {"tool": "bash_output", "arguments": {"id": self.old_handle, "wait_ms": 0}}
        if n == 3:
            self.stale_output = tool_text(body)[-1]
            return {"tool": "bash_kill", "arguments": {"id": self.old_handle}}
        if n == 4:
            self.stale_kill = tool_text(body)[-1]
            return {"tool": "bash_output", "arguments": {"id": self.new_handle, "wait_ms": 0}}
        if n == 5:
            self.new_output = tool_text(body)[-1]
            return {"text": "Loaded context remains available."}
        return {"text": "done"}



def claude_import(binary):
    with tempfile.TemporaryDirectory(prefix="graff-acp-import-") as temporary:
        cwd = Path(temporary).resolve()
        home = cwd / "home"
        config = home / ".claude"
        slug = "".join(c if c.isascii() and c.isalnum() else "-" for c in str(cwd))
        project = config / "projects" / slug
        project.mkdir(parents=True)
        records = [
            {"type": "user", "message": {"content": "Imported human request."}},
            {"type": "assistant", "message": {"model": "claude-source-model", "content": [
                {"type": "thinking", "thinking": "private-import-thinking", "signature": "signed"},
                {"type": "text", "text": "Imported assistant answer."},
                {"type": "tool_use", "id": "import-call", "name": "Bash",
                 "input": {"command": "touch imported-tool-must-not-run"}},
            ]}},
            {"type": "user", "message": {"content": [
                {"type": "tool_result", "tool_use_id": "import-call", "content": "Imported tool receipt."},
            ]}},
        ]
        source = project / "chosen.jsonl"
        original = "\n".join(json.dumps(record) for record in records) + "\n"
        source.write_text(original)
        (project / "other.jsonl").write_text(original)
        env = dict(os.environ, HOME=str(home), CLAUDE_CONFIG_DIR=str(config), ANTHROPIC_API_KEY="")
        env.pop("CLAUDE_CODE_PROJECT_DIR_NAME", None)
        imported = subprocess.run([str(binary), "mcp", "import-session", "chosen"],
                                  cwd=cwd, env=env, text=True, capture_output=True, timeout=15)
        assert imported.returncode == 0, imported.stderr
        assert imported.stdout.strip() == "claude-chosen", imported.stdout
        assert not (cwd / ".graff/sessions/claude-other.session.json").exists()
        model = ScriptedModel([{"text": "Continued through graff."}])
        port = model.start(0)
        client = Acp(binary, cwd, home, port, extra_env={"ANTHROPIC_API_KEY": ""})
        try:
            client.request("initialize", {"protocolVersion": 1})
            client.request("session/new", {"cwd": str(cwd), "mcpServers": []})
            loaded = client.request("session/load", {"sessionId": "claude-chosen", "cwd": str(cwd), "mcpServers": []})
            assert "result" in loaded, loaded
            reply = client.request("session/prompt", {"sessionId": "claude-chosen",
                "prompt": [{"type": "text", "text": "Continue the imported task."}]})
            assert reply["result"]["stopReason"] == "end_turn", reply
            body = json.dumps(model.requests[0])
            for text in ["Imported human request.", "Imported assistant answer.", "Imported tool receipt."]:
                assert text in body, text
            assert "private-import-thinking" not in body
            assert model.requests[0]["model"] != "claude-source-model"
            assert not (cwd / "imported-tool-must-not-run").exists()
            assert source.read_text() == original
            saved = cwd / ".graff/sessions/claude-chosen.session.json"
            deadline = time.monotonic() + 5
            while json.loads(saved.read_text()).get("provider") != "vercel" and time.monotonic() < deadline:
                time.sleep(.05)
            snapshot = json.loads(saved.read_text())
            assert snapshot["provider"] == "vercel", snapshot
            assert "imported_from" not in snapshot
        finally:
            client.close()
            model.stop()
    print("ACP Claude history import uses graff credentials and retains tool receipts: ok")


def plan_login_restore(binary):
    """#1556: a host switches models by respawning with --model and calling
    session/load. A save on a plan-login provider outside that --model's scope
    must still restore."""
    if os.name == "nt":
        return
    record = {"email": "[email protected]", "issuer": "https://auth.openai.com", "subject": "user-1",
              "client_id": "oaiapp_x", "ext_agent_host_id": "urn:uuid:1", "id_token": "a.b.c",
              "access_token": "plan-token", "refresh_token": "ref-1", "token_type": "Bearer", "expires_in": 3600,
              "expires_at": 4102444800, "earliest_refresh_at": 4102444000,
              "scopes": ["chatgpt.tokens.use.direct", "email", "offline_access", "openid", "profile", "resource.invoke"],
              "saved_at": "2026-09-30T00:00:00Z"}
    with tempfile.TemporaryDirectory(prefix="graff-acp-plan-load-") as temporary:
        cwd = Path(temporary).resolve()
        home = cwd / "home"
        credentials = home / ".graff" / "credentials"
        credentials.mkdir(parents=True, mode=0o700)
        os.chmod(home / ".graff", 0o700)
        (credentials / "chatgpt-new.json").write_text(json.dumps(record))
        os.chmod(credentials / "chatgpt-new.json", 0o600)
        sessions = cwd / ".graff" / "sessions"
        sessions.mkdir(parents=True)
        (sessions / "plan-saved.session.json").write_text(json.dumps({
            "provider": "chatgpt-new", "model": "gpt-6.1-sol", "workspace": str(cwd),
            "messages": [{"role": "user", "content": "Plan-saved request."}, {"role": "assistant", "content": "ok"}]}))
        model = ScriptedModel([{"text": "unused"}])
        port = model.start(0)
        client = Acp(binary, cwd, home, port)  # --model vercel: the plan provider is out of scope
        try:
            client.request("initialize", {"protocolVersion": 1})
            loaded = client.request("session/load", {"sessionId": "plan-saved", "cwd": str(cwd), "mcpServers": []})
            assert "result" in loaded, loaded
        finally:
            client.close()
            model.stop()
    print("ACP session/load restores a plan-login save after a cross-provider --model respawn: ok")


def peer_session(binary, cwd, home, pid):
    """The session id `graff peer list` shows for the graff process `pid`."""
    env = {k: v for k, v in os.environ.items() if not k.startswith(("GRAFF_", "HARNESS_"))}
    env.update(HOME=str(home), GRAFF_NO_TELEMETRY="1", NO_COLOR="1")
    out = subprocess.run([str(binary), "peer", "list", "--json"], cwd=cwd, env=env, capture_output=True, text=True, timeout=30)
    rows = json.loads(out.stdout or "[]")
    return next((r.get("session_id") for r in rows if r.get("pid") == pid), None)


def run(binary):
    claude_import(binary)
    plan_login_restore(binary)
    with tempfile.TemporaryDirectory(prefix="graff-acp-load-") as temporary:
        cwd = Path(temporary)
        home = cwd / "home"
        home.mkdir()
        first_model = ScriptedModel([
            {"tool": "shell", "arguments": {"action": "run", "command": "sleep 5", "run_in_background": True}},
            {"text": "Original assistant answer."},
        ])
        first_port = first_model.start(0)
        a = Acp(binary, cwd, home, first_port)
        try:
            init = a.request("initialize", {"protocolVersion": 1})
            assert init["result"]["agentCapabilities"]["loadSession"] is True
            sid = a.request("session/new", {"cwd": str(cwd), "mcpServers": []})["result"]["sessionId"]
            assert "/" not in sid and "\\" not in sid
            changed = a.request("session/set_config_option", {"sessionId": sid, "configId": "thought_level", "value": "high"})
            assert changed["result"]["configOptions"][0]["currentValue"] == "high"
            duplicate = a.request("session/new", {"cwd": str(cwd), "mcpServers": []})
            assert duplicate["error"]["code"] == -32000
            turn = a.request("session/prompt", {"sessionId": sid, "prompt": [{"type": "text", "text": "Original human request."}]}, 35)
            assert turn["result"]["stopReason"] == "end_turn"
            old_result = tool_text(first_model.requests[1])[-1]
            old = int(re.search(r"\[job (\d+) started:", old_result).group(1))
            saved = cwd / ".graff" / "sessions" / f"{sid}.session.json"
            end = time.monotonic() + 5
            while not saved.exists() and time.monotonic() < end:
                time.sleep(.05)
            assert saved.exists(), "ACP session ID is not the durable save basename"
        finally:
            a.close()
            first_model.stop()

        # Give the saved conversation a different model from the explicit
        # --model vercel used by the process that will load it. The fixture's
        # local gateway accepts this saved model without catalog discovery.
        snapshot = json.loads(saved.read_text())
        assert snapshot["provider"] == "vercel", snapshot
        snapshot["model"] = "saved/model-a"
        saved.write_text(json.dumps(snapshot))

        # Simulate a compacted provider window: the transcript still retains
        # an older user turn that no longer appears in the model snapshot.
        transcript = cwd / ".graff" / "sessions" / f"{sid}.transcript.jsonl"
        assert transcript.exists()
        transcript.write_text(json.dumps({"role": "user", "content": "Older retained request."}) + "\n" + transcript.read_text())

        next_model = FollowingModel(old)
        second_port = next_model.start(0)
        b = Acp(binary, cwd, home, second_port)
        try:
            b.request("initialize", {"protocolVersion": 1})
            created = b.request("session/new", {"cwd": str(cwd), "mcpServers": []})["result"]
            fresh = created["sessionId"]
            assert fresh != sid
            lowered = b.request("session/set_config_option", {"sessionId": fresh, "configId": "thought_level", "value": "low"})
            assert lowered["result"]["configOptions"][0]["currentValue"] == "low"
            bad = b.request("session/load", {"sessionId": "../escape", "cwd": str(cwd), "mcpServers": []})
            assert bad["error"]["code"] == -32602
            missing = b.request("session/load", {"sessionId": "missing", "cwd": str(cwd), "mcpServers": []})
            assert missing["error"]["code"] == -32602
            corrupt = cwd / ".graff" / "sessions" / "corrupt.session.json"
            corrupt.write_text('{"provider":42,"model":"mock","messages":[]}')
            invalid = b.request("session/load", {"sessionId": "corrupt", "cwd": str(cwd), "mcpServers": []})
            assert invalid["error"]["code"] == -32602
            wrong = b.request("session/load", {"sessionId": sid, "cwd": str(home), "mcpServers": []})
            assert wrong["error"]["code"] == -32602
            before = len(b.events)
            loaded = b.request("session/load", {"sessionId": sid, "cwd": str(cwd), "mcpServers": []})
            # Peers (and Harness's send_message) address a chat by its ACP session id.
            assert peer_session(binary, cwd, home, b.proc.pid) == sid
            selected = b.request("graff/models")["result"]["current"]
            assert selected["provider"] == "vercel", selected
            assert selected["model"] == "alibaba/qwen3.8-27b", selected
            created_config = created["configOptions"]
            loaded_config = loaded["result"]["configOptions"]
            assert len(created_config) == len(loaded_config) == 1, (created, loaded)
            assert created_config[0]["category"] == loaded_config[0]["category"] == "thought_level"
            assert created_config[0]["currentValue"] == "high", created_config
            assert loaded_config[0]["currentValue"] == "low", loaded_config
            duplicate_after_load = b.request("session/new", {"cwd": str(cwd), "mcpServers": []})
            assert duplicate_after_load["error"]["code"] == -32000
            replay = b.events[before:]
            updates = [event["params"]["update"] for event in replay if event.get("method") == "session/update"]
            assert any(u.get("sessionUpdate") == "user_message_chunk" and u["content"].get("text") == "Original human request." for u in updates)
            assert any(u.get("sessionUpdate") == "user_message_chunk" and u["content"].get("text") == "Older retained request." for u in updates)
            assert any(u.get("sessionUpdate") == "agent_message_chunk" and u["content"].get("text") == "Original assistant answer." for u in updates)
            assert any(u.get("sessionUpdate") == "tool_call" for u in updates)
            assert any(u.get("sessionUpdate") == "tool_call_update" and u.get("status") == "completed" for u in updates)
            stale_session = b.request("session/prompt", {"sessionId": fresh, "prompt": [{"type": "text", "text": "Wrong session."}]})
            assert stale_session["error"]["code"] == -32602
            continued = b.request("session/prompt", {"sessionId": sid, "prompt": [{"type": "text", "text": "Continue with original context."}]}, 35)
            assert continued["result"]["stopReason"] == "end_turn"
            assert next_model.requests[0]["reasoning"]["effort"] == "low"
            assert next_model.requests[0]["model"] == "alibaba/qwen3.8-27b", next_model.requests[0]
            assert next_model.new_handle != old
            assert "Original human request." in json.dumps(next_model.requests[0])
            assert "Original assistant answer." in json.dumps(next_model.requests[0])
            assert re.search(r"unknown|stale|no background job|interrupted", next_model.stale_output, re.I)
            assert re.search(r"unknown|stale|no background job|interrupted", next_model.stale_kill, re.I)
            assert str(next_model.new_handle) not in next_model.stale_output
            assert not re.search(r"unknown|stale|no background job|interrupted", next_model.new_output, re.I)
        finally:
            b.close()
            next_model.stop()
    print("ACP session/load model precedence, replay, and stale shell isolation: ok")


if __name__ == "__main__":
    run(Path(sys.argv[1] if len(sys.argv) > 1 else REPO / "zig-out/bin/graff").resolve())