Skip to content

Autonomous AI sales agent powered by Claude Code or Codex. Finds prospects, writes cold emails, sends campaigns, handles replies, books meetings. No API keys needed — runs on your Claude or Codex subscriptions.

9.4 KB
Raw Blame
"""Deterministic soak and fault injection through the real Brain and Codex
adapter against the scripted CLI (tests/fake_codex.py).

Many calls from concurrent clients, mixed with every failure class Harvey
must survive: crashes, partial/malformed/oversized output, hangs, attempted
tool calls, usage limits, auth/isolation/version denial and cancellation.
Afterwards: every call ended within its bound, no runtime process or temp
folder is left, the native login is byte-for-byte unchanged, and only
successful calls were counted.
"""

import asyncio
import json
import os
import sys
import tempfile
import time
from pathlib import Path

import pytest

import harvey.brain as brain_mod
import harvey.codex as hc

# The Codex provider supports macOS and Linux only (see harvey.codex).
pytestmark = pytest.mark.skipif(sys.platform == "win32",
                                reason="the Codex provider isn't supported on Windows")
from harvey.brain import Brain
from harvey.config import BrainConfig
from harvey.state import StateManager
from tests.test_codex import _write_login

FAKE = str(Path(__file__).with_name("fake_codex.py"))
# A call's deadline includes waiting for its Brain's runtime lock, and the
# mixed soak queues 8 calls per Brain. Healthy runtime start-ups take ~0.1 s
# each natively but seconds under CPU emulation (x86_64 on arm64 measured:
# a 4 s deadline expired while still queued). 30 s keeps the test about
# bounding and cleanup, not host speed. It doesn't slow fast hosts: no
# healthy or fault call here waits for its deadline except the dedicated
# hang job (2 s). Short deadlines (lock wait, blocked write, timeout) have
# their own tests in test_codex.py.
CALL_TIMEOUT = 30.0
# One call may take its timeout plus bounded teardown (kill + exit wait).
CALL_BOUND = CALL_TIMEOUT + hc.KILL_WAIT_SECONDS + 1.0

TURN_FAULTS = ["crash_turn", "partial", "malformed", "oversized", "toolcall"]


@pytest.fixture
def soak(monkeypatch, tmp_path):
    runtimes = tmp_path / "runtimes"
    runtimes.mkdir()
    monkeypatch.setattr(tempfile, "tempdir", str(runtimes))
    monkeypatch.setattr(brain_mod, "RETRY_BASE_DELAY", 0)
    native = tmp_path / "native"
    native.mkdir()
    _write_login(native)
    logs = tmp_path / "logs"
    logs.mkdir()
    real_exec = asyncio.create_subprocess_exec

    async def exec_routed(program, *args, **kwargs):
        # "codex@<scenario>" selects the fake's scenario per client.
        assert program.startswith("codex@"), program
        scenario = program.split("@", 1)[1]
        return await real_exec(sys.executable, FAKE, scenario, str(logs / f"{scenario}.log"),
                               *args, **kwargs)

    monkeypatch.setattr(hc.asyncio, "create_subprocess_exec", exec_routed)

    class Soak:
        def __init__(self):
            self.native = native
            self.runtimes = runtimes

        def entries(self, scenario=None):
            files = [logs / f"{scenario}.log"] if scenario else sorted(logs.glob("*.log"))
            out = []
            for f in files:
                if f.exists():
                    out += [json.loads(line) for line in f.read_text().splitlines()]
            return out

        def pids(self):
            return [e["pid"] for e in self.entries() if e.get("cmd") == "app-server"]

        def brain(self, state, scenario):
            brain = Brain(state, BrainConfig(provider="codex", model="gpt-5.5"))
            brain.codex._codex_path = f"codex@{scenario}"
            brain.codex.native_home = native
            return brain

        def client(self, scenario):
            return hc.CodexClient(model="gpt-5.5", codex_path=f"codex@{scenario}",
                                  native_home=native)

    return Soak()


def _native_identity(home: Path):
    st = (home / "auth.json").stat()
    return ((home / "auth.json").read_bytes(), st.st_ino, st.st_mode, st.st_mtime_ns)


def _alive(pid: int) -> bool:
    try:
        os.kill(pid, 0)
    except ProcessLookupError:
        return False
    except PermissionError:
        return True
    try:
        with open(f"/proc/{pid}/stat") as f:
            return f.read().split()[2] != "Z"
    except OSError:
        return True


async def _timed(coro):
    started = time.monotonic()
    try:
        return await coro, time.monotonic() - started
    except BaseException as e:  # noqa: BLE001 - recorded for assertions
        return e, time.monotonic() - started


@pytest.mark.asyncio
async def test_soak_mixed_faults_across_concurrent_brains(soak, tmp_path):
    state = StateManager(str(tmp_path / "soak.db"))
    await state.init_db()
    before = _native_identity(soak.native)

    healthy = soak.brain(state, "smart")
    faulty = soak.brain(state, "smart")         # healthy CLI, faults per prompt
    isolation = soak.brain(state, "hooks")      # persistent: unsafe config layer
    auth = soak.brain(state, "logged_out")      # persistent: no ChatGPT account
    version = soak.brain(state, "old_version")  # persistent: unverified CLI
    limited = soak.brain(state, "smart")
    # Its own Brain: the tool-call fault above (correctly) pauses `faulty`.
    hanger = soak.brain(state, "smart")

    rounds = 8
    jobs = []
    for i in range(rounds):
        jobs.append(("ok", healthy.think(f"Classify this email reply #{i}: pricing?",
                                         timeout=CALL_TIMEOUT, max_retries=1)))
        fault = TURN_FAULTS[i % len(TURN_FAULTS)]
        jobs.append((fault, faulty.think(f"FAULT:{fault} #{i}", timeout=CALL_TIMEOUT,
                                         max_retries=1)))
        jobs.append(("isolation", isolation.think(f"q{i}", timeout=CALL_TIMEOUT)))
        jobs.append(("auth", auth.think(f"q{i}", timeout=CALL_TIMEOUT)))
        jobs.append(("version", version.think(f"q{i}", timeout=CALL_TIMEOUT)))
        jobs.append(("limit", limited.think("FAULT:usage_limit" if i == 0 else f"q{i}",
                                            timeout=CALL_TIMEOUT)))
    # One hang, bounded by its own deadline.
    jobs.append(("hang", hanger.think("FAULT:hang", timeout=2.0, max_retries=0)))

    started = time.monotonic()
    results = await asyncio.gather(*(_timed(c) for _, c in jobs))
    elapsed = time.monotonic() - started

    outcomes = {}
    for (kind, _), (value, seconds) in zip(jobs, results):
        assert not isinstance(value, BaseException), (kind, value)
        # Retries double the worst case for retried kinds.
        assert seconds < 2 * CALL_BOUND, (kind, seconds)
        outcomes.setdefault(kind, []).append(value)
        if kind == "hang":
            # Really hung, and was cut off by its own 2 s deadline.
            assert 2.0 <= seconds < 2.0 + hc.KILL_WAIT_SECONDS + 1.0, seconds

    assert outcomes["ok"] == ["interested"] * rounds
    for kind in TURN_FAULTS + ["isolation", "auth", "version", "limit", "hang"]:
        assert all(v == "" for v in outcomes[kind]), kind
    assert elapsed < rounds * 3 * CALL_BOUND  # serialized per brain, bounded overall

    # Persistent failures: one runtime attempt, then no more launches.
    assert isolation.unavailable and auth.unavailable and version.unavailable
    assert sum(1 for e in soak.entries("hooks") if e.get("method") == "hooks/list") == 1
    assert sum(1 for e in soak.entries("logged_out") if e.get("method") == "account/read") == 1
    assert sum(1 for e in soak.entries("old_version") if e.get("cmd") == "--version") == 1
    # Usage limit: the first turn was refused, the rest never started.
    assert limited.paused_reason

    # Nothing left behind, login untouched, only successes counted.
    if os.name == "posix":
        assert not [p for p in soak.pids() if _alive(p)]
    assert list(soak.runtimes.iterdir()) == []
    assert _native_identity(soak.native) == before
    assert await state.get_usage_today("codex") == rounds
    rows = await state.usage_by_provider(days=1)
    assert [(r["provider"], r["calls"]) for r in rows] == [("codex", rounds)]


@pytest.mark.asyncio
async def test_soak_many_concurrent_clients(soak):
    """Six independent clients, five calls each, all at once."""
    clients = [soak.client("smart") for _ in range(6)]
    before = _native_identity(soak.native)
    started = time.monotonic()
    results = await asyncio.gather(*(
        c.run(f"Classify this email reply {i}-{n}: pricing?", timeout=30)
        for i, c in enumerate(clients) for n in range(5)))
    elapsed = time.monotonic() - started
    assert [r.text for r in results] == ["interested"] * 30
    assert len({r.thread_id for r in results}) == 30  # one runtime per call
    assert elapsed < 120
    if os.name == "posix":
        assert not [p for p in soak.pids() if _alive(p)]
    assert list(soak.runtimes.iterdir()) == []
    assert _native_identity(soak.native) == before


@pytest.mark.asyncio
async def test_soak_cancellation_storm(soak):
    """Cancel calls at many different points; nothing may leak."""
    client = soak.client("smart")
    before = _native_identity(soak.native)
    for delay in (0.0, 0.05, 0.1, 0.2, 0.3, 0.5, 0.8, 1.2):
        task = asyncio.ensure_future(client.run("FAULT:hang", timeout=30))
        await asyncio.sleep(delay)
        task.cancel()
        with pytest.raises(asyncio.CancelledError):
            await task
    # The client still works after all that.
    assert (await client.run("Classify this email reply: pricing?", timeout=30)).text == "interested"
    if os.name == "posix":
        assert not [p for p in soak.pids() if _alive(p)]
    assert list(soak.runtimes.iterdir()) == []
    assert _native_identity(soak.native) == before