Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
4.9 KB
Raw Blame
"""Routed offline replies for parent/child feedback lifecycle evals."""
import json
import shlex
import threading

from mock_model import ScriptedModel

NOTE = "FEEDBACK_SCOPE_UPDATE: include the revised scope in your report."


def bash(code):
    return {"tool": "bash", "arguments": {"command": "python3 -c " + shlex.quote(code)}}


def wait_file(name):
    return (
        "from pathlib import Path\nimport time\n"
        "deadline = time.monotonic() + 15\n"
        f"while not Path({name!r}).exists():\n"
        " if time.monotonic() > deadline: raise RuntimeError('fixture timed out')\n"
        " time.sleep(.01)\n"
    )


class FeedbackModel(ScriptedModel):
    def __init__(self, scenario, load_tools=False):
        super().__init__([])
        if scenario not in ("tool", "final"):
            raise ValueError(f"unknown feedback scenario: {scenario}")
        self.scenario = scenario
        self.load_tools = load_tools
        self.parent_calls = 0
        self.child_calls = 0
        self.child_started = threading.Event()
        self.feedback_acked = threading.Event()
        self.checked = False
        self.problem = "child never consumed feedback"

    def next_reply(self, body):
        super().next_reply(body)  # retain the same request evidence as tier 2
        messages = body.get("messages", [])
        first_user = next((m for m in messages if m.get("role") == "user"), {})
        if "CHILD_FEEDBACK_FIXTURE" in json.dumps(first_user):
            return self.child_reply(messages)
        return self.parent_reply(messages)

    def child_reply(self, messages):
        step = self.child_calls
        self.child_calls += 1
        if step == 0:
            self.child_started.set()
            if self.scenario == "final":
                if not self.feedback_acked.wait(15):
                    return {"text": "fixture timed out waiting for feedback acknowledgement"}
                return {"text": "EARLY_REPORT"}
            setup = "from pathlib import Path\nPath('child-ready').write_text('ready')\n"
            return bash(setup + wait_file("child-release") + "print('TOOL_FINISHED')\n")
        notes = [i for i, m in enumerate(messages) if m.get("role") == "user" and NOTE in json.dumps(m)]
        tool_results = [i for i, m in enumerate(messages) if m.get("role") == "tool"]
        valid = step == 1 and len(notes) == 1 and "[Parent task feedback]" in json.dumps(messages[notes[0]])
        if self.scenario == "tool":
            valid = valid and bool(tool_results) and notes[0] > max(tool_results) and "TOOL_FINISHED" in json.dumps(messages)
        else:
            valid = valid and any(m.get("role") == "assistant" and "EARLY_REPORT" in json.dumps(m) for m in messages[:notes[0]])
        self.checked = valid
        self.problem = "feedback missing, duplicated, or delivered before the current operation finished"
        return {"text": "UPDATED_REPORT" if valid else "FEEDBACK_NOT_APPLIED"}

    def parent_reply(self, messages):
        step = self.parent_calls
        self.parent_calls += 1
        if self.load_tools:
            if step == 0:
                return {"tool": "load_tool_schemas", "arguments": {"tools": ["subagent", "agent_message", "agent_output"]}}
            step -= 1
        spawn = {"tool": "subagent", "arguments": {
            "description": "Feedback fixture child", "prompt": "CHILD_FEEDBACK_FIXTURE: complete the task and incorporate parent feedback.",
            "run_in_background": True,
        }}
        send = {"tool": "agent_message", "arguments": {"id": 1, "message": NOTE}}
        collect = {"tool": "agent_output", "arguments": {"id": 1, "wait_ms": 1}}
        late = {"tool": "agent_message", "arguments": {"id": 1, "message": "late feedback must be rejected"}}
        missing = {"tool": "agent_message", "arguments": {"id": 999999, "message": "unknown target"}}
        if self.scenario == "tool":
            replies = [spawn, bash(wait_file("child-ready")), send,
                       bash("from pathlib import Path; Path('child-release').write_text('release')"),
                       collect, late, missing]
        else:
            if step == 1 and not self.child_started.wait(15):
                return {"text": "fixture timed out waiting for the child's request"}
            if step == 2:
                self.feedback_acked.set()
            replies = [spawn, send, collect, late, missing]
        if step < len(replies):
            return replies[step]
        # The final verifier checks actual tool results in the parent's request,
        # rather than assuming a scripted request succeeded.
        outputs = [m.get("content", "") for m in messages if m.get("role") == "tool"]
        text = json.dumps(outputs)
        valid = self.checked and self.child_calls == 2 and all(s in text for s in
            ("feedback queued", "UPDATED_REPORT", "agent has already finished", "agent is not live"))
        return {"text": "FEEDBACK_TEST_DONE" if valid else "FEEDBACK_TEST_FAILED: " + self.problem}