Skip to content

graff — a fast agentic coding harness in Zig: multi-provider, MCP, workflows, DGM evolution loop, TS/Python SDKs

Python
8.2 KB
Raw Blame
#!/usr/bin/env python3
"""Issue #692 interactive Codex WS error-frame regression scenarios."""

from __future__ import annotations

import json
import os
from pathlib import Path

from codex_ws_mock import CodexMock, RecordedRequest
import codex_ws_test as base

GENERIC_ERROR_CODE = "invalid_request_error"
GENERIC_ERROR_SECRET = "raw-envelope-secret-must-not-leak"
GENERIC_ERROR_TAIL = "generic-error-tail-must-be-truncated"
GENERIC_ERROR_MESSAGE = "mock bad request\r\n" + "x" * 500 + GENERIC_ERROR_TAIL
CHAIN_ERROR_CODE = "previous_response_not_found"
CHAIN_FINAL = "recovered after automatic websocket re-anchor"


def generic_error_events(_request: RecordedRequest) -> list[dict]:
    """One generic terminal error with raw envelope data that must not leak."""
    return [
        {
            "type": "error",
            "error": {
                "code": GENERIC_ERROR_CODE,
                "message": GENERIC_ERROR_MESSAGE,
            },
            "debug": {"authorization": GENERIC_ERROR_SECRET},
        }
    ]


def chain_error_events(request: RecordedRequest) -> list[dict]:
    """Tool call, stale chained delta, then success after a full-input redial."""
    if request.ordinal == 1:
        call = {
            "type": "function_call",
            "id": "fc_chain_1",
            "call_id": "call_chain_1",
            "name": "todo_read",
            "arguments": "{}",
            "status": "completed",
        }
        return base.response_events(call, "resp_chain_1", 1_200)
    if request.ordinal == 2:
        return [
            {
                "type": "error",
                "error": {
                    "code": CHAIN_ERROR_CODE,
                    "message": "Previous response resp_chain_1 was not found",
                },
            }
        ]
    return base.response_events(
        base.message_item(CHAIN_FINAL, "msg_chain_final"),
        "resp_chain_final",
        1_300,
    )


def _env(tmp: str, codex_home: str, port: int) -> tuple[dict[str, str], tuple[str, ...]]:
    env = {
        "HOME": tmp,
        "CODEX_HOME": codex_home,
        "CODEGRAFF_API_KEY": "local-pty-test",
        "GRAFF_FLEET": "off",
        "GRAFF_NO_TELEMETRY": "1",
        "GRAFF_CODEX_URL": f"http://127.0.0.1:{port}/backend-api/codex/responses",
        "GRAFF_SERVER_COMPACT": "0",
    }
    ambient = tuple(
        key
        for key in os.environ
        if (key.startswith("GRAFF_") or key.startswith("CODEX_") or key == "NO_COLOR")
        and key not in env
    )
    return env, ambient


def _new_trace(tmp: str, before: set[str]) -> list[dict]:
    trace_dir = Path(tmp, ".graff", "traces")
    created = sorted(path for path in trace_dir.iterdir() if path.name not in before)
    if len(created) != 1:
        raise AssertionError(f"expected one new trace, got {created!r}")
    return [json.loads(line) for line in created[0].read_text().splitlines() if line]


def _assert_ws_stays_primary(session: base.PtySession) -> None:
    cursor = len(session.raw)
    session.send_line("/models health")
    session.wait_for_literal(
        "Codex transport: WebSocket primary with automatic SSE fallback",
        start=cursor,
    )
    session.wait_for_prompt(start=cursor)


def _exit(session: base.PtySession, label: str) -> None:
    session.send_key("ctrl-d")
    result = session.read_until_exit(5.0)
    if result.timed_out or result.exit_code != 0:
        raise SystemExit(
            f"{label}: REPL did not exit cleanly: "
            f"exit={result.exit_code} timed_out={result.timed_out}"
        )


def run_generic_error_scenario(
    tmp: str, codex_home: str, port: int, mock: CodexMock
) -> None:
    """A deterministic API error returns once with bounded safe diagnostics."""
    env, ambient = _env(tmp, codex_home, port)
    trace_dir = Path(tmp, ".graff", "traces")
    before = {path.name for path in trace_dir.iterdir()} if trace_dir.is_dir() else set()
    with base.PtySession(
        base.GRAFF,
        ["--model", "codex", "--no-telemetry"],
        cwd=tmp,
        env=env,
        unset_env=ambient,
        timeout=20.0,
    ) as session:
        session.wait_for_prompt()
        cursor = len(session.raw)
        session.send_line("trigger the generic websocket rejection")
        expected = f"codex api error [{GENERIC_ERROR_CODE}]: mock bad request"
        session.wait_for_literal(expected, start=cursor)
        session.wait_for_prompt(start=cursor)
        rendered = base.terminal_text(bytes(session.raw[cursor:]))
        if GENERIC_ERROR_SECRET in rendered or GENERIC_ERROR_TAIL in rendered:
            raise AssertionError(f"generic-error: raw/unbounded diagnostics leaked:\n{rendered}")
        if mock.ws_turns != 1 or mock.sse_turns != 0 or mock.ws_connections != 1:
            raise AssertionError(
                "generic-error: deterministic body was retried/fallen back: "
                f"connections={mock.ws_connections} ws={mock.ws_turns} sse={mock.sse_turns}"
            )
        _assert_ws_stays_primary(session)
        _exit(session, "generic-error")

    events = _new_trace(tmp, before)
    diagnostics = [event for event in events if event.get("ev") == "ws_api_error"]
    if len(diagnostics) != 1:
        raise AssertionError(f"generic-error: expected one ws_api_error trace: {events!r}")
    detail = diagnostics[0].get("detail", "")
    if (
        GENERIC_ERROR_CODE not in detail
        or GENERIC_ERROR_SECRET in detail
        or GENERIC_ERROR_TAIL in detail
        or len(detail.encode("utf-8")) >= 560
    ):
        raise AssertionError(f"generic-error: unsafe trace diagnostic: {detail!r}")
    ws_details = [event.get("detail", "") for event in events if event.get("ev") == "ws"]
    if any("transport error" in detail or "fallback" in detail for detail in ws_details):
        raise AssertionError(f"generic-error: API response burned transport ladder: {ws_details!r}")


def run_chain_reanchor_scenario(
    tmp: str, codex_home: str, port: int, mock: CodexMock
) -> None:
    """A recognized stale chain rebuilds once without requiring `continue`."""
    env, ambient = _env(tmp, codex_home, port)
    with base.PtySession(
        base.GRAFF,
        ["--model", "codex", "--no-telemetry"],
        cwd=tmp,
        env=env,
        unset_env=ambient,
        timeout=20.0,
    ) as session:
        session.wait_for_prompt()
        cursor = len(session.raw)
        session.send_line("exercise automatic stale-chain recovery")
        session.wait_for_literal(CHAIN_FINAL, start=cursor)
        session.wait_for_prompt(start=cursor)
        requests = mock.recorded_requests()
        if mock.ws_turns != 3 or mock.sse_turns != 0 or mock.ws_connections != 2:
            raise AssertionError(
                "chain-error: expected delta rejection then one fresh WS rebuild: "
                f"connections={mock.ws_connections} ws={mock.ws_turns} sse={mock.sse_turns}"
            )
        if len(requests) != 3:
            raise AssertionError(f"chain-error: expected 3 requests, got {requests!r}")
        first, rejected, rebuilt = requests
        if (
            first.connection_id != rejected.connection_id
            or rejected.connection_id == rebuilt.connection_id
            or rejected.body.get("previous_response_id") != "resp_chain_1"
            # A fresh socket may chain only from its observed prewarm,
            # never from the rejected inference response or another socket.
            or not mock.has_fresh_parent(rebuilt)
        ):
            raise AssertionError(
                "chain-error: rejected delta was not rebuilt as full input on a fresh WS: "
                f"connections={(first.connection_id, rejected.connection_id, rebuilt.connection_id)!r}, "
                f"rejected_parent={rejected.body.get('previous_response_id')!r}, "
                f"rebuilt_parent={rebuilt.body.get('previous_response_id')!r}"
            )
        mock.assert_fresh_anchor(rebuilt)
        rebuilt_types = [
            item.get("type")
            for item in rebuilt.body.get("input", [])
            if isinstance(item, dict)
        ]
        if "function_call" not in rebuilt_types or "function_call_output" not in rebuilt_types:
            raise AssertionError(f"chain-error: full rebuild lost tool history: {rebuilt.body!r}")
        _assert_ws_stays_primary(session)
        _exit(session, "chain-error")