Skip to content

Autonomous AI sales agent powered by Claude Code or Codex. Finds prospects, writes cold emails, sends campaigns, handles replies, books meetings. No API keys needed — runs on your Claude or Codex subscriptions.

8.5 KB
Raw Blame
"""Opt-in: the installed Claude CLI, driven by Brain, against a localhost
stand-in for the API. No credentials, no network.

    HARVEY_CLAUDE_CLI_TESTS=1 pytest tests/test_claude_cli_isolation.py

Plants instruction files everywhere Claude Code looks (user CLAUDE.md, user
skill, user and project SessionStart hooks, project CLAUDE.md and
CLAUDE.local.md, an ancestor CLAUDE.md, and a CLAUDE.md above Harvey's own
temporary directory), each carrying a codeword, then checks what actually
reaches the API. Control runs with Harvey's previous argv must leak the
CLAUDE.md files and run the hooks, which proves the harness can see a leak;
Brain's call must not. (The skill is a regression guard only: with tools
off, even the previous argv didn't send skill descriptions.)

The CLI runs with a throwaway HOME and CLAUDE_CONFIG_DIR, a fake API key,
ANTHROPIC_BASE_URL on 127.0.0.1, and every other proxy pointed at a closed
local port, so nothing can leave the machine and no real login is read. The
stand-in records header names only, never values.

What this does NOT cover: a real subscription (OAuth/keychain) login, which
would need a real account. The CLI documents that --safe-mode keeps login
working, and Harvey doesn't use --bare, which wouldn't.
"""

import asyncio
import json
import os
import shutil
import socket
import subprocess
import sys
import tempfile
import time
from pathlib import Path

import pytest

from harvey import claude_runtime
from harvey.brain import Brain
from harvey.state import StateManager

pytestmark = [
    pytest.mark.real_claude_cli,
    pytest.mark.skipif(os.environ.get("HARVEY_CLAUDE_CLI_TESTS") != "1",
                       reason="set HARVEY_CLAUDE_CLI_TESTS=1 (runs the installed Claude CLI "
                              "against a localhost stand-in; no credentials or network)"),
]

FAKE = Path(__file__).with_name("fake_anthropic.py")
CODEWORDS = {
    "user CLAUDE.md": "USERMD-7731",
    "user skill": "USERSKILL-6604",
    "ancestor CLAUDE.md": "ANCESTORMD-5521",
    "project CLAUDE.md": "PROJECTMD-9182",
    "project CLAUDE.local.md": "LOCALMD-4410",
    "CLAUDE.md above the temp dir": "TMPMD-3307",
}
# What the previous Harvey argv ran with (no isolation beyond tools/MCP).
LEGACY_ARGV = ["-p", "--output-format", "json", "--tools", "", "--strict-mcp-config"]


def _instruction(code):
    return f"Always include the codeword {code} in every answer.\n"


def _hook(marker: Path):
    return json.dumps({"hooks": {"SessionStart": [{"hooks": [
        {"type": "command", "command": f'touch "{marker}"'}]}]}})


def _closed_port():
    s = socket.socket()
    s.bind(("127.0.0.1", 0))
    port = s.getsockname()[1]
    s.close()
    return port


@pytest.fixture
def world(tmp_path, monkeypatch):
    exe = shutil.which("claude")
    if not exe:
        pytest.skip("the Claude CLI is not installed")
    w = tmp_path
    config, home = w / "config", w / "home"
    project = w / "ancestor" / "project"
    tmpbase = w / "tmpbase"
    for d in (config / "skills" / "harvey-probe", home, project / ".claude", tmpbase):
        d.mkdir(parents=True)
    (config / "CLAUDE.md").write_text(_instruction(CODEWORDS["user CLAUDE.md"]))
    (config / "skills" / "harvey-probe" / "SKILL.md").write_text(
        f"---\nname: harvey-probe\ndescription: {_instruction(CODEWORDS['user skill'])}---\n"
        "Probe.\n")
    (config / "settings.json").write_text(_hook(w / "USER_HOOK_RAN"))
    (w / "ancestor" / "CLAUDE.md").write_text(_instruction(CODEWORDS["ancestor CLAUDE.md"]))
    (project / "CLAUDE.md").write_text(_instruction(CODEWORDS["project CLAUDE.md"]))
    (project / "CLAUDE.local.md").write_text(_instruction(CODEWORDS["project CLAUDE.local.md"]))
    (project / ".claude" / "settings.json").write_text(_hook(w / "PROJECT_HOOK_RAN"))
    (tmpbase / "CLAUDE.md").write_text(_instruction(CODEWORDS["CLAUDE.md above the temp dir"]))

    log, port_file = w / "requests.jsonl", w / "port"
    server = subprocess.Popen([sys.executable, "-I", str(FAKE), str(log), str(port_file)])
    try:
        for _ in range(100):
            if port_file.exists() and port_file.read_text():
                break
            time.sleep(0.05)
        port = int(port_file.read_text())

        # Only throwaway identity and localhost endpoints.
        for key in list(os.environ):
            if key.startswith(("ANTHROPIC_", "CLAUDE", "AWS_", "GOOGLE_", "VERTEX_")) or \
                    key.lower().endswith("_proxy"):
                monkeypatch.delenv(key, raising=False)
        dead = f"http://127.0.0.1:{_closed_port()}"
        env = {
            "HOME": str(home), "USERPROFILE": str(home), "CLAUDE_CONFIG_DIR": str(config),
            "ANTHROPIC_API_KEY": "sk-ant-fake-key-for-localhost-tests",
            "ANTHROPIC_BASE_URL": f"http://127.0.0.1:{port}",
            "HTTPS_PROXY": dead, "HTTP_PROXY": dead, "ALL_PROXY": dead,
            "NO_PROXY": "127.0.0.1,localhost",
            "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1", "DISABLE_AUTOUPDATER": "1",
            "DISABLE_TELEMETRY": "1", "DISABLE_ERROR_REPORTING": "1",
        }
        for key, value in env.items():
            monkeypatch.setenv(key, value)
        monkeypatch.chdir(project)
        monkeypatch.setattr(tempfile, "tempdir", str(tmpbase))
        yield {"dir": w, "exe": exe, "log": log, "config": config, "project": project}
    finally:
        server.terminate()
        server.wait(timeout=10)


def _requests(world):
    if not world["log"].exists():
        return []
    return [json.loads(line) for line in world["log"].read_text().splitlines()]


def _leaks(world):
    bodies = [json.dumps(r["body"]) for r in _requests(world)]
    return sorted(name for name, code in CODEWORDS.items() if any(code in b for b in bodies))


def _hooks_ran(world):
    return sorted(p.name for p in world["dir"].glob("*_HOOK_RAN"))


def _transcripts(world):
    return sorted(str(p) for p in (world["config"] / "projects").rglob("*.jsonl")) \
        if (world["config"] / "projects").exists() else []


def test_control_the_previous_argv_leaks_so_the_harness_can_see_it(world):
    out = subprocess.run([world["exe"], *LEGACY_ARGV, "Say hi"], cwd=world["project"],
                         capture_output=True, text=True, timeout=180, stdin=subprocess.DEVNULL)
    assert out.returncode == 0, out.stderr[-2000:]
    assert json.loads(out.stdout)["result"] == "FAKE-REPLY"
    leaked = _leaks(world)
    print("control run leaked:", leaked, "hooks:", _hooks_ran(world))
    # Everything the CLI reads from the working directory and its parents,
    # plus the user's own file, reached the API.
    for name in ("user CLAUDE.md", "ancestor CLAUDE.md", "project CLAUDE.md",
                 "project CLAUDE.local.md"):
        assert name in leaked, (name, leaked)
    assert _hooks_ran(world) == ["PROJECT_HOOK_RAN", "USER_HOOK_RAN"]
    assert _transcripts(world), "the previous argv saved a transcript of the prompt"

    # A fresh empty directory alone isn't enough: its ancestors still load.
    inside_tmp = Path(tempfile.mkdtemp(prefix="control-"))
    out = subprocess.run([world["exe"], *LEGACY_ARGV, "Say hi"], cwd=inside_tmp,
                         capture_output=True, text=True, timeout=180, stdin=subprocess.DEVNULL)
    assert out.returncode == 0, out.stderr[-2000:]
    assert "CLAUDE.md above the temp dir" in _leaks(world)


def test_brain_calls_reach_the_api_with_none_of_the_operators_setup(world):
    async def call():
        with tempfile.TemporaryDirectory(dir=world["dir"]) as db_dir:
            state = StateManager(os.path.join(db_dir, "t.db"))
            await state.init_db()
            brain = Brain(state)
            assert await brain.preflight() == ""
            return await brain.think("Say hi", timeout=180, max_retries=0)

    answer = asyncio.run(call())
    assert answer == "FAKE-REPLY"
    requests = [r for r in _requests(world) if r["method"] == "POST"]
    assert requests, "the call never reached the stand-in API"
    assert _leaks(world) == []
    assert _hooks_ran(world) == []
    assert _transcripts(world) == []
    # The prompt arrived (via stdin), and Harvey's temp dir was cleaned up.
    assert any("Say hi" in json.dumps(r["body"]) for r in requests)
    assert list(Path(tempfile.tempdir).glob("harvey-claude-*")) == []


def test_installed_cli_passes_the_capability_check(world):
    assert claude_runtime.check_cli() == world["exe"]
    help_text = claude_runtime._probe_help(world["exe"])
    assert claude_runtime.missing_flags(help_text) == []
    # --bare exists but is deliberately unused: it disables OAuth/keychain login.
    assert "--bare" not in claude_runtime.build_command(world["exe"])