Skip to content

"Vibe-Trading: Your Personal Trading Agent"

20.1 KB
Raw Blame
"""Coverage for the local Claude Code CLI provider (``claude -p`` as a decision-only model)."""

from __future__ import annotations

import json
import subprocess
import threading
import time
from pathlib import Path
from types import SimpleNamespace

import pytest

import api_server  # noqa: F401
from src.api import settings_routes
from src.providers import claude_cli
from src.providers.capabilities import get_provider_capabilities
from src.providers.chat import ChatLLM
from src.swarm.models import public_provider_metadata
from tests.claude_cli_helpers import install_fake_cli, payload, tool, use_provider

_PROVIDERS_JSON = (
    Path(__file__).resolve().parents[1] / "src" / "providers" / "llm_providers.json"
)


@pytest.fixture
def fake_cli(tmp_path, monkeypatch):
    return install_fake_cli(tmp_path, monkeypatch)


# --- registration -----------------------------------------------------------


def test_claude_cli_is_registered_in_catalog_and_capabilities() -> None:
    entries = {
        item["name"]: item
        for item in json.loads(_PROVIDERS_JSON.read_text(encoding="utf-8"))
    }
    entry = entries["claude-cli"]
    assert entry["api_key_required"] is False
    assert entry["auth_type"] == "claude_cli"
    assert (
        get_provider_capabilities("claude-cli", "claude-opus-5-5").name == "claude-cli"
    )


def test_claude_cli_label_survives_public_run_metadata() -> None:
    assert public_provider_metadata("claude-cli") == "claude-cli"


def test_sync_provider_env_clears_openai_transport(monkeypatch) -> None:
    import src.providers.llm as llm

    use_provider(monkeypatch)
    monkeypatch.setenv("OPENAI_API_KEY", "sk-should-be-removed")
    monkeypatch.setenv("OPENAI_BASE_URL", "https://example.invalid/v1")
    llm._sync_provider_env()
    import os

    assert "OPENAI_API_KEY" not in os.environ
    assert "OPENAI_BASE_URL" not in os.environ


def test_build_llm_returns_cli_wrapper(monkeypatch) -> None:
    import src.providers.llm as llm

    use_provider(monkeypatch)
    model = llm.build_llm()
    assert isinstance(model, claude_cli.ClaudeCliLLM)
    assert model.model == "claude-opus-5-5"


# --- decision mapping ---------------------------------------------------------


def test_tool_calls_map_to_vibe_tool_call_requests(fake_cli, monkeypatch) -> None:
    set_replies, _ = fake_cli
    set_replies(
        {
            "payload": payload(
                {
                    "action": "tool_calls",
                    "tool_calls": [
                        {
                            "name": "get_market_data",
                            "arguments": {"codes": ["03988.HK"]},
                        }
                    ],
                }
            )
        }
    )
    use_provider(monkeypatch)

    response = ChatLLM().stream_chat(
        [{"role": "system", "content": "SYS"}, {"role": "user", "content": "price?"}],
        tools=[tool("get_market_data")],
    )

    assert response.has_tool_calls
    call = response.tool_calls[0]
    assert call.name == "get_market_data"
    assert call.arguments == {"codes": ["03988.HK"]}
    assert call.id.startswith("ccli_")
    assert response.usage_metadata["input_tokens"] == 10


def test_final_answer_maps_to_content(fake_cli, monkeypatch) -> None:
    set_replies, _ = fake_cli
    set_replies({"payload": payload({"action": "final", "content": "close 6.125"})})
    use_provider(monkeypatch)

    response = ChatLLM().chat([{"role": "user", "content": "hi"}])

    assert response.content == "close 6.125"
    assert not response.has_tool_calls


def test_final_without_content_is_fed_back_and_retried(fake_cli) -> None:
    set_replies, calls = fake_cli
    set_replies(
        {"payload": payload({"action": "final"})},
        {"payload": payload({"action": "final", "content": "order 42 cancelled"})},
    )

    message = claude_cli.ClaudeCliLLM(tools=[tool("get_market_data")]).invoke(
        [{"role": "user", "content": "cancel"}]
    )

    assert message.content == "order 42 cancelled"
    assert "no tool calls and no content" in calls()[1]["stdin"]
    # Billing includes both attempts, while compaction sees the last prompt.
    assert message.usage_metadata["input_tokens"] == 20
    assert message.usage_metadata["context_input_tokens"] == 10
    assert message.usage_metadata["calls"] == 2
    assert message.usage_metadata["output_tokens"] == 10


def test_native_attempt_is_fed_back_with_the_original_prompt(fake_cli) -> None:
    first = payload({"action": "final", "content": "No such tool available"})
    first["num_turns"] = 3
    set_replies, calls = fake_cli
    set_replies(
        {"payload": first},
        {
            "payload": payload(
                {
                    "action": "tool_calls",
                    "tool_calls": [
                        {
                            "name": "get_market_data",
                            "arguments": {"codes": ["HYPE-USDT"]},
                        }
                    ],
                }
            )
        },
    )

    message = claude_cli.ClaudeCliLLM(tools=[tool("get_market_data")]).invoke(
        [{"role": "user", "content": "run the backtest"}]
    )

    assert message.tool_calls[0]["name"] == "get_market_data"
    stdin = calls()[1]["stdin"]
    assert "run the backtest" in stdin
    assert "Native tool calls" in stdin
    assert "action=tool_calls" in stdin


def test_permission_denials_are_fed_back_instead_of_accepting_final(fake_cli) -> None:
    first = payload({"action": "final", "content": "tools are unavailable"})
    first["permission_denials"] = [{"tool_name": "bash"}]
    set_replies, calls = fake_cli
    set_replies(
        {"payload": first},
        {"payload": payload({"action": "final", "content": "recovered"})},
    )

    message = claude_cli.ClaudeCliLLM(tools=[tool("get_market_data")]).invoke(
        [{"role": "user", "content": "x"}]
    )

    assert message.content == "recovered"
    assert "Native tool calls" in calls()[1]["stdin"]


def test_repeated_native_attempts_exhaust_the_existing_retry_budget(fake_cli) -> None:
    reply = payload({"action": "final", "content": "tools are broken"})
    reply["num_turns"] = 3
    set_replies, calls = fake_cli
    set_replies({"payload": reply})

    with pytest.raises(RuntimeError, match="native tool calls kept bypassing"):
        claude_cli.ClaudeCliLLM(tools=[tool("get_market_data")]).invoke(
            [{"role": "user", "content": "x"}]
        )

    assert len(calls()) == claude_cli._MAX_RETRIES + 1


def test_valid_structured_calls_are_not_retried_after_native_metadata(fake_cli) -> None:
    reply = payload(
        {
            "action": "tool_calls",
            "tool_calls": [{"name": "get_market_data", "arguments": {}}],
        }
    )
    reply["permission_denials"] = [{"tool_name": "bash"}]
    set_replies, calls = fake_cli
    set_replies({"payload": reply})

    message = claude_cli.ClaudeCliLLM(tools=[tool("get_market_data")]).invoke(
        [{"role": "user", "content": "x"}]
    )

    assert message.tool_calls[0]["name"] == "get_market_data"
    assert len(calls()) == 1


def test_single_turn_final_answer_does_not_trigger_native_retry(fake_cli) -> None:
    reply = payload({"action": "final", "content": "done"})
    reply["num_turns"] = 1
    reply["permission_denials"] = []
    set_replies, calls = fake_cli
    set_replies({"payload": reply})

    message = claude_cli.ClaudeCliLLM(tools=[tool("get_market_data")]).invoke(
        [{"role": "user", "content": "x"}]
    )

    assert message.content == "done"
    assert len(calls()) == 1


def test_structured_output_turn_final_answer_does_not_trigger_native_retry(
    fake_cli,
) -> None:
    reply = payload({"action": "final", "content": "done"})
    reply["num_turns"] = 2
    reply["permission_denials"] = []
    set_replies, calls = fake_cli
    set_replies({"payload": reply})

    message = claude_cli.ClaudeCliLLM(tools=[tool("get_market_data")]).invoke(
        [{"role": "user", "content": "x"}]
    )

    assert message.content == "done"
    assert len(calls()) == 1


def test_final_never_echoes_the_structured_output_as_content(fake_cli) -> None:
    set_replies, _ = fake_cli
    set_replies({"payload": payload({"action": "final", "content": "  "})})

    with pytest.raises(RuntimeError, match="without content"):
        claude_cli.ClaudeCliLLM().invoke([{"role": "user", "content": "x"}])


def test_vibe_system_prompt_is_a_verbatim_prefix_and_lone_user_is_verbatim(
    fake_cli,
) -> None:
    set_replies, calls = fake_cli
    set_replies({"payload": payload({"action": "final", "content": "ok"})})
    system = "## Role\n\nexact text {with braces}\n"
    user = "Report the latest close of 03988.HK."

    claude_cli.ClaudeCliLLM(tools=[tool("get_market_data")]).invoke(
        [{"role": "system", "content": system}, {"role": "user", "content": user}]
    )

    record = calls()[0]
    assert record["system"].startswith(system)
    assert "Tool-calling protocol" in record["system"]
    assert record["stdin"] == user


def test_turn_without_tools_only_allows_final(fake_cli) -> None:
    set_replies, calls = fake_cli
    set_replies({"payload": payload({"action": "final", "content": "ok"})})

    claude_cli.ClaudeCliLLM().invoke([{"role": "user", "content": "x"}])

    args = calls()[0]["args"]
    schema = json.loads(args[args.index("--json-schema") + 1])
    assert schema["properties"]["action"]["enum"] == ["final"]
    assert args[args.index("--tools") + 1] == ""
    assert "--strict-mcp-config" in args and "--restricted" in args


def test_history_renders_tool_calls_and_results(fake_cli) -> None:
    set_replies, calls = fake_cli
    set_replies({"payload": payload({"action": "final", "content": "done"})})
    messages = [
        {"role": "system", "content": "SYS"},
        {"role": "user", "content": "price?"},
        {
            "role": "assistant",
            "content": "",
            "tool_calls": [
                {
                    "id": "c1",
                    "type": "function",
                    "function": {"name": "get_market_data", "arguments": "{}"},
                }
            ],
        },
        {
            "role": "tool",
            "tool_call_id": "c1",
            "name": "get_market_data",
            "content": "close=6.125",
        },
    ]

    claude_cli.ClaudeCliLLM(tools=[tool("get_market_data")]).invoke(messages)

    stdin = calls()[0]["stdin"]
    assert "[assistant tool_calls]" in stdin
    assert "[tool result: get_market_data id=c1]\nclose=6.125" in stdin


# --- error paths --------------------------------------------------------------


@pytest.mark.parametrize(
    "reply, message",
    [
        ({"raw": "not json"}, "no JSON"),
        ({"payload": payload({"action": "final"}, is_error=True)}, "failed"),
        ({"payload": payload(None)}, "structured_output"),
    ],
)
def test_bad_cli_output_raises(fake_cli, reply, message) -> None:
    set_replies, _ = fake_cli
    set_replies(reply)
    with pytest.raises(RuntimeError, match=message):
        claude_cli.ClaudeCliLLM().invoke([{"role": "user", "content": "x"}])


def test_timeout_kills_the_child(fake_cli, monkeypatch) -> None:
    set_replies, _ = fake_cli
    set_replies(
        {"sleep": 30, "payload": payload({"action": "final", "content": "late"})}
    )
    monkeypatch.setattr(claude_cli, "_TIMEOUT_FLOOR_S", 1)
    started = time.monotonic()
    with pytest.raises(RuntimeError, match="exceeded"):
        claude_cli.ClaudeCliLLM(timeout=1).invoke([{"role": "user", "content": "x"}])
    assert time.monotonic() - started < 10


def test_cancel_event_stops_the_child(fake_cli) -> None:
    set_replies, _ = fake_cli
    set_replies(
        {"sleep": 30, "payload": payload({"action": "final", "content": "late"})}
    )
    cancel = threading.Event()
    threading.Timer(1.0, cancel.set).start()
    started = time.monotonic()
    result = claude_cli._run_cli(
        system="S",
        user_input="x",
        allow_tools=False,
        model="m",
        timeout=60,
        cancel_event=cancel,
    )
    assert result == {"cancelled": True}
    assert time.monotonic() - started < 10


def test_slow_decision_outlives_the_stream_idle_timeout(fake_cli, monkeypatch) -> None:
    set_replies, _ = fake_cli
    set_replies(
        {"sleep": 3, "payload": payload({"action": "final", "content": "slow answer"})}
    )
    monkeypatch.setattr(claude_cli, "_HEARTBEAT_S", 0.2)
    use_provider(monkeypatch)

    waiting = []
    response = ChatLLM().stream_chat(
        [{"role": "user", "content": "x"}], idle_timeout_s=1,
        on_waiting_chunk=lambda: waiting.append(time.monotonic()),
    )

    assert response.content == "slow answer"
    assert waiting, "a real CLI child must emit waiting progress before its final response"


def test_stop_button_ends_a_running_decision(fake_cli, monkeypatch) -> None:
    set_replies, _ = fake_cli
    set_replies(
        {"sleep": 30, "payload": payload({"action": "final", "content": "late"})}
    )
    monkeypatch.setattr(claude_cli, "_HEARTBEAT_S", 0.2)
    use_provider(monkeypatch)
    stop_at = time.monotonic() + 1.0

    started = time.monotonic()
    response = ChatLLM().stream_chat(
        [{"role": "user", "content": "x"}],
        should_cancel=lambda: time.monotonic() > stop_at,
    )

    assert time.monotonic() - started < 10
    assert response.content == ""


def test_retries_share_one_deadline(fake_cli, monkeypatch) -> None:
    set_replies, calls = fake_cli
    set_replies({"sleep": 0.7, "payload": payload({"action": "final"})})
    monkeypatch.setattr(claude_cli, "_TIMEOUT_FLOOR_S", 1)

    with pytest.raises(RuntimeError, match="exceeded"):
        claude_cli.ClaudeCliLLM(timeout=1).invoke([{"role": "user", "content": "x"}])

    # Four attempts fit in 4 x 1s budgets but not in one shared 1s deadline.
    assert len(calls()) < 4


def test_image_input_is_refused_instead_of_dropped(fake_cli) -> None:
    set_replies, calls = fake_cli
    set_replies({"payload": payload({"action": "final", "content": "a cat"})})
    image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}

    with pytest.raises(RuntimeError, match="text input only"):
        claude_cli.ClaudeCliLLM().invoke(
            [{"role": "user", "content": [{"type": "text", "text": "what?"}, image]}]
        )

    assert calls() == []


def test_lone_langchain_human_message_is_passed_verbatim(fake_cli) -> None:
    from langchain_core.messages import HumanMessage, SystemMessage

    set_replies, calls = fake_cli
    set_replies({"payload": payload({"action": "final", "content": "ok"})})

    claude_cli.ClaudeCliLLM().invoke(
        [SystemMessage(content="SYS"), HumanMessage(content="exact prompt")]
    )

    assert calls()[0]["stdin"] == "exact prompt"


def test_unknown_bash_policy_fails_before_any_turn(fake_cli, monkeypatch) -> None:
    _, calls = fake_cli
    monkeypatch.setenv("CLAUDE_CLI_BASH_POLICY", "strict")

    with pytest.raises(RuntimeError, match="CLAUDE_CLI_BASH_POLICY"):
        claude_cli.ClaudeCliLLM()

    assert calls() == []


def test_auth_status_runs_without_the_anthropic_key(monkeypatch) -> None:
    seen: dict = {}

    def fake_run(cmd, **kwargs):
        seen.update(kwargs)
        return subprocess.CompletedProcess(
            cmd, 0, stdout='{"loggedIn": true, "authMethod": "claude.ai"}', stderr=""
        )

    monkeypatch.setenv("CLAUDE_CLI_PATH", "/bin/true")
    monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-not-used")
    # Replace the module's reference, not the shared ``subprocess`` module (#1123).
    monkeypatch.setattr(
        claude_cli,
        "subprocess",
        SimpleNamespace(run=fake_run, SubprocessError=subprocess.SubprocessError),
    )

    assert claude_cli.get_claude_cli_auth_status() == (True, "claude.ai")
    assert "ANTHROPIC_API_KEY" not in seen["env"]


def test_missing_cli_raises_with_setup_hint(monkeypatch) -> None:
    monkeypatch.delenv("CLAUDE_CLI_PATH", raising=False)
    monkeypatch.setattr(claude_cli.shutil, "which", lambda _: None)
    monkeypatch.setattr(claude_cli.Path, "home", lambda: Path("/nonexistent-home"))
    with pytest.raises(RuntimeError, match="claude auth login"):
        claude_cli.resolve_claude_cli()


# --- settings / preflight -----------------------------------------------------


def test_settings_report_cli_login(monkeypatch) -> None:
    monkeypatch.setattr(
        claude_cli, "get_claude_cli_auth_status", lambda: (True, "claude.ai")
    )

    response = settings_routes._build_llm_settings_response(
        {"LANGCHAIN_PROVIDER": "claude-cli"}
    )

    assert response.api_key_required is False
    assert response.api_key_configured is True
    assert response.api_key_hint == "claude.ai"


def test_preflight_uses_cli_login(monkeypatch) -> None:
    import src.providers.llm as llm
    from src import preflight

    monkeypatch.setenv("LANGCHAIN_PROVIDER", "claude-cli")
    monkeypatch.setenv("LANGCHAIN_MODEL_NAME", "claude-opus-5-5")
    monkeypatch.setattr(llm, "_ensure_dotenv", lambda: None)
    monkeypatch.setattr(llm, "_sync_provider_env", lambda: None)
    monkeypatch.setattr(
        llm,
        "provider_diagnostics",
        lambda: {"base_url": "", "timeout_seconds": 120, "max_retries": 2, "proxy": {}},
    )
    monkeypatch.setattr(
        claude_cli, "get_claude_cli_auth_status", lambda: (False, "not logged in")
    )

    result = preflight._check_llm_provider()

    assert result.status == "not_configured"
    assert "claude auth login" in result.impact


def test_usage_preserves_cache_split_as_input_token_details() -> None:
    result = claude_cli._usage({
        "usage": {
            "input_tokens": 10,
            "cache_creation_input_tokens": 5,
            "cache_read_input_tokens": 20,
            "output_tokens": 3,
        },
    })
    assert result == {
        "input_tokens": 35,
        "output_tokens": 3,
        "total_tokens": 38,
        "input_token_details": {"cache_read": 20, "cache_creation": 5},
    }


def test_usage_omits_details_when_no_cache() -> None:
    result = claude_cli._usage({
        "usage": {"input_tokens": 10, "output_tokens": 3},
    })
    assert result == {
        "input_tokens": 10,
        "output_tokens": 3,
        "total_tokens": 13,
    }


@pytest.mark.parametrize("streaming", [False, True])
def test_retry_details_reach_persisted_usage(fake_cli, monkeypatch, tmp_path, streaming):
    from src.agent.llm_usage import _new_llm_usage_summary, _record_llm_usage
    set_replies, _ = fake_cli
    replies = []
    for content, tokens in [("", 10), ("done", 20)]:
        reply = payload({"action": "final", "content": content})
        reply["usage"] = {"input_tokens": tokens, "output_tokens": 3,
                          "cache_creation_input_tokens": 5, "cache_read_input_tokens": 7}
        replies.append({"payload": reply})
    set_replies(*replies)
    use_provider(monkeypatch)
    client = ChatLLM()
    response = (client.stream_chat if streaming else client.chat)([{"role": "user", "content": "hi"}])
    usage = response.usage_metadata
    assert usage["input_tokens"] == 54
    assert usage["context_input_tokens"] == 32
    summary = _new_llm_usage_summary(client)
    _record_llm_usage(tmp_path, summary, usage, 1)
    saved = json.loads((tmp_path / "llm_usage.json").read_text())
    assert saved["totals"] == {"input_tokens": 54, "output_tokens": 6,
        "total_tokens": 60, "calls": 2, "cache_read_tokens": 14, "cache_creation_tokens": 10}


def test_cache_creation_total_is_not_added_to_its_breakdown():
    usage = claude_cli._usage({"usage": {"input_tokens": 10,
        "cache_creation_input_tokens": 5, "ephemeral_5m_input_tokens": 5,
        "cache_creation": {"ephemeral_5m_input_tokens": 5}}})
    assert usage["input_tokens"] == 15
    assert usage["input_token_details"]["cache_creation"] == 5


def test_exhausted_feedback_retries_keep_their_paid_usage(fake_cli, monkeypatch):
    set_replies, _ = fake_cli
    set_replies(*[{"payload": payload({"action": "final", "content": ""})}
                  for _ in range(claude_cli._MAX_RETRIES + 1)])
    use_provider(monkeypatch)
    with pytest.raises(Exception, match="without content") as failure:
        ChatLLM().stream_chat([{"role": "user", "content": "hi"}])
    usage = failure.value.usage_metadata
    assert usage["calls"] == 4
    assert usage["input_tokens"] == 40
    assert usage["output_tokens"] == 20