Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
"""Unit tests for ClaudeCliProvider.
All tests mock the Claude Code subprocess; no real ``claude`` calls are made.
"""
from __future__ import annotations
import asyncio
import json
from pathlib import Path
from typing import Any
import pytest
from repowise.core.providers.llm.base import GeneratedResponse, ProviderError
from repowise.core.providers.llm.claude_cli import (
ClaudeCliProvider,
_model_label,
_normalize_model,
_parse_result,
)
class FakeProcess:
def __init__(
self,
*,
returncode: int = 0,
stdout: str = "",
stderr: str = "",
on_communicate: Any | None = None,
transport: Any | None = None,
) -> None:
self.returncode = returncode
self._stdout = stdout
self._stderr = stderr
self._on_communicate = on_communicate
self._transport = transport
self.stdin_input: bytes | None = None
self.killed = False
async def communicate(self, input: bytes | None = None) -> tuple[bytes, bytes]:
self.stdin_input = input
if self._on_communicate is not None:
await self._on_communicate()
return self._stdout.encode("utf-8"), self._stderr.encode("utf-8")
def kill(self) -> None:
self.killed = True
async def wait(self) -> int:
return self.returncode
class FakeTransport:
def __init__(self) -> None:
self.closed = False
def close(self) -> None:
self.closed = True
def _success_json(text: str = "OK", **overrides: Any) -> str:
payload: dict[str, Any] = {
"type": "result",
"subtype": "success",
"is_error": False,
"stop_reason": "end_turn",
"num_turns": 1,
"total_cost_usd": 0.0179915,
"result": text,
"usage": {
"input_tokens": 120,
"output_tokens": 40,
"cache_read_input_tokens": 30,
"cache_creation_input_tokens": 7955,
},
}
payload.update(overrides)
return json.dumps(payload) + "\n"
@pytest.fixture
def claude_on_path(monkeypatch):
monkeypatch.setattr("shutil.which", lambda cmd: "/usr/bin/claude" if cmd == "claude" else None)
return "/usr/bin/claude"
# ---------------------------------------------------------------------------
# Model naming
# ---------------------------------------------------------------------------
def test_provider_name_and_default_model(claude_on_path):
provider = ClaudeCliProvider()
assert provider.provider_name == "claude_cli"
assert provider.model_name == "claude_cli/claude-haiku-5-5"
@pytest.mark.parametrize(
("given", "expected"),
[
(None, "claude-haiku-5-5"),
("claude-sonnet-4-6", "claude-sonnet-4-6"),
# A persisted label must round-trip rather than become "claude_cli/claude_cli/...".
("claude_cli/claude-sonnet-4-6", "claude-sonnet-4-6"),
("claude_cli/default", "claude-haiku-5-5"),
],
)
def test_model_normalization_round_trips(given, expected):
assert _normalize_model(given) == expected
def test_model_label_is_prefixed(claude_on_path):
assert _model_label("claude-opus-4-6") == "claude_cli/claude-opus-4-6"
assert ClaudeCliProvider(model="claude_cli/claude-opus-4-6").model_name == (
"claude_cli/claude-opus-4-6"
)
def test_invalid_model_name_is_rejected(claude_on_path):
"""Model names reach argv, so they are validated against a safe charset."""
with pytest.raises(ProviderError, match="Invalid model name"):
ClaudeCliProvider(model="claude; rm -rf /")
def test_missing_cli_raises(monkeypatch):
monkeypatch.setattr("shutil.which", lambda _cmd: None)
with pytest.raises(ProviderError, match="Claude Code CLI not found"):
ClaudeCliProvider()
def test_reasoning_modes_match_the_claude_cli_effort_flag(claude_on_path):
assert ClaudeCliProvider().supported_reasoning_modes() == (
"auto",
"low",
"medium",
"high",
"xhigh",
"max",
)
def test_available_model_options_are_labelled(claude_on_path):
options = ClaudeCliProvider().available_model_options()
assert [o.model for o in options] == [
"claude_cli/claude-haiku-5-5",
"claude_cli/claude-sonnet-5-5",
"claude_cli/claude-opus-5-5",
]
assert sum(1 for o in options if o.recommended) == 1
assert all(
o.reasoning_modes == ClaudeCliProvider().supported_reasoning_modes() for o in options
)
# ---------------------------------------------------------------------------
# generate()
# ---------------------------------------------------------------------------
async def test_generate_success(claude_on_path, monkeypatch):
proc = FakeProcess(stdout=_success_json("Hello from Claude"), transport=FakeTransport())
captured: dict[str, Any] = {}
async def fake_exec(*args, **kwargs):
captured["args"] = args
captured["kwargs"] = kwargs
return proc
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
result = await ClaudeCliProvider(model="claude-sonnet-4-6").generate(
"system rules", "user context"
)
assert isinstance(result, GeneratedResponse)
assert result.content == "Hello from Claude"
# The prompt total, not the uncached remainder: Claude Code splits a prompt
# across ``input_tokens``, ``cache_read_input_tokens`` and
# ``cache_creation_input_tokens``.
assert result.input_tokens == 120 + 30 + 7955
assert result.usage["uncached_input_tokens"] == 120
assert result.output_tokens == 40
# cached_tokens is the *read* half; creations are recorded separately.
assert result.cached_tokens == 30
assert result.usage["cache_creation_input_tokens"] == 7955
assert result.stop_reason == "end_turn"
assert result.usage["source"] == "claude_cli"
assert result.usage["model"] == "claude_cli/claude-sonnet-4-6"
# The user prompt goes on stdin, not argv (prompts are large).
assert proc.stdin_input == b"user context"
args = list(captured["args"])
assert args[0] == claude_on_path
assert "-p" in args
assert args[args.index("--model") + 1] == "claude-sonnet-4-6"
system_prompt = args[args.index("--system-prompt") + 1]
assert system_prompt.startswith("system rules\n\n")
assert "You have no tools available" in system_prompt
assert "single response" in system_prompt
assert args[args.index("--max-turns") + 1] == "1"
assert "--strict-mcp-config" in args
assert args[args.index("--tools") + 1] == ""
assert "--disallowed-tools" not in args
# --bare would force ANTHROPIC_API_KEY and never read the OAuth login,
# defeating the point of this provider.
assert "--bare" not in args
async def test_generate_runs_outside_the_repo(claude_on_path, monkeypatch, tmp_path):
"""cwd must not be a repo, or CLAUDE.md is auto-discovered into every prompt."""
captured: dict[str, Any] = {}
async def fake_exec(*args, **kwargs):
captured["kwargs"] = kwargs
return FakeProcess(stdout=_success_json())
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
await ClaudeCliProvider().generate("sys", "user")
cwd = captured["kwargs"]["cwd"]
assert cwd is not None
assert "repowise-claude-cli-" in str(cwd)
assert not Path(cwd).exists(), "the per-call scratch directory leaked"
@pytest.mark.parametrize("reasoning", ["low", "medium", "high", "xhigh", "max"])
async def test_supported_reasoning_is_passed_as_effort(claude_on_path, monkeypatch, reasoning):
captured: dict[str, Any] = {}
async def fake_exec(*args, **_kwargs):
captured["args"] = args
return FakeProcess(stdout=_success_json())
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
result = await ClaudeCliProvider().generate("sys", "user", reasoning=reasoning)
args = list(captured["args"])
assert result.content == "OK"
assert args[args.index("--effort") + 1] == reasoning
async def test_unsupported_reasoning_warns_but_does_not_raise(claude_on_path, monkeypatch):
"""Raising here would kill a whole docs run over a flag the CLI cannot express."""
captured: dict[str, Any] = {}
async def fake_exec(*args, **_kwargs):
captured["args"] = args
return FakeProcess(stdout=_success_json())
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
result = await ClaudeCliProvider().generate("sys", "user", reasoning="none")
assert result.content == "OK"
assert "--effort" not in captured["args"]
async def test_nonzero_exit_raises_with_stderr(claude_on_path, monkeypatch):
async def fake_exec(*_args, **_kwargs):
return FakeProcess(returncode=1, stderr="credit balance too low")
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
with pytest.raises(ProviderError, match="credit balance too low"):
await ClaudeCliProvider().generate("sys", "user")
async def test_nonzero_exit_surfaces_json_error_and_http_status(claude_on_path, monkeypatch):
"""Claude writes API failures to stdout as JSON and often leaves stderr empty."""
async def fake_exec(*_args, **_kwargs):
return FakeProcess(
returncode=1,
stdout=_success_json(
"There's an issue with the selected model.",
subtype="error_during_execution",
is_error=True,
api_error_status=404,
),
)
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
with pytest.raises(ProviderError, match="selected model") as caught:
await ClaudeCliProvider().generate("sys", "user")
assert caught.value.status_code == 404
async def test_error_subtype_raises(claude_on_path, monkeypatch):
async def fake_exec(*_args, **_kwargs):
return FakeProcess(
stdout=_success_json(
"", subtype="error_max_turns", is_error=True, api_error_status=None
)
)
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
with pytest.raises(ProviderError, match="error_max_turns"):
await ClaudeCliProvider().generate("sys", "user")
async def test_empty_result_raises(claude_on_path, monkeypatch):
async def fake_exec(*_args, **_kwargs):
return FakeProcess(stdout=_success_json(" "))
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
with pytest.raises(ProviderError, match="no result text"):
await ClaudeCliProvider().generate("sys", "user")
async def test_timeout_kills_process_and_raises(claude_on_path, monkeypatch):
monkeypatch.setattr(
"repowise.core.providers.llm.claude_cli._EXEC_TIMEOUT_SECONDS",
0.01,
)
async def on_communicate() -> None:
await asyncio.sleep(1)
proc = FakeProcess(stdout="", on_communicate=on_communicate)
async def fake_exec(*_args, **_kwargs):
return proc
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
with pytest.raises(ProviderError, match="timed out"):
await ClaudeCliProvider().generate("sys", "user")
assert proc.killed
async def test_cancellation_kills_process(claude_on_path, monkeypatch):
"""The interactive caller cancels at 180s; the 600s CLI must not keep running."""
started = asyncio.Event()
async def on_communicate() -> None:
started.set()
await asyncio.Event().wait()
proc = FakeProcess(on_communicate=on_communicate)
async def fake_exec(*_args, **_kwargs):
return proc
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
task = asyncio.create_task(ClaudeCliProvider().generate("sys", "user"))
await started.wait()
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert proc.killed
async def test_transport_is_closed(claude_on_path, monkeypatch):
transport = FakeTransport()
async def fake_exec(*_args, **_kwargs):
return FakeProcess(stdout=_success_json(), transport=transport)
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
await ClaudeCliProvider().generate("sys", "user")
assert transport.closed
# ---------------------------------------------------------------------------
# Output parsing
# ---------------------------------------------------------------------------
def test_parse_result_tolerates_leading_noise():
noisy = 'warning: something\n{"subtype": "success", "result": "hi"}\n'
assert _parse_result(noisy)["result"] == "hi"
def test_parse_result_rejects_unparseable_output():
with pytest.raises(ProviderError, match="could not parse"):
_parse_result("not json at all")
def test_parse_result_rejects_empty_output():
with pytest.raises(ProviderError, match="no output"):
_parse_result(" ")
def test_parse_result_reads_result_event_from_verbose_array():
"""With --verbose the CLI emits an event array, not a single object (#3161)."""
events = json.dumps(
[
{"type": "system", "subtype": "init", "session_id": "abc"},
{"type": "assistant", "message": {"content": [{"type": "text", "text": "hi"}]}},
{"type": "result", "subtype": "success", "is_error": False, "result": "hi"},
]
)
assert _parse_result(events)["result"] == "hi"
def test_parse_result_rejects_array_without_result_event():
"""An event array with no result event must keep the existing failure."""
events = json.dumps(
[
{"type": "system", "subtype": "init"},
{"type": "assistant", "message": {"content": [{"type": "text", "text": "hi"}]}},
]
)
with pytest.raises(ProviderError, match="could not parse"):
_parse_result(events)
async def test_verbose_array_nonzero_exit_surfaces_error_and_http_status(claude_on_path, monkeypatch):
"""A verbose run still reports the API failure carried by its result event."""
async def fake_exec(*_args, **_kwargs):
return FakeProcess(
returncode=1,
stdout=json.dumps(
[
{"type": "system", "subtype": "init"},
{
"type": "result",
"subtype": "error_during_execution",
"is_error": True,
"api_error_status": 404,
"result": "There's an issue with the selected model.",
},
]
),
)
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
with pytest.raises(ProviderError, match="selected model") as caught:
await ClaudeCliProvider().generate("sys", "user")
assert caught.value.status_code == 404
async def test_missing_usage_is_flagged_estimated(claude_on_path, monkeypatch):
async def fake_exec(*_args, **_kwargs):
return FakeProcess(stdout=json.dumps({"subtype": "success", "result": "hi"}) + "\n")
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
result = await ClaudeCliProvider().generate("sys", "user")
assert result.usage["estimated"] is True
assert result.input_tokens == 0
assert result.output_tokens == 0
# ---------------------------------------------------------------------------
# Token accounting and the cost ledger (#2267)
# ---------------------------------------------------------------------------
def _cached_page_json(*, uncached: int, read: int, created: int) -> str:
"""A ``claude -p`` payload shaped like a real page generation.
Claude Code splits the prompt across the three usage fields: a page prompt
carries a large stable prefix, so the bulk of it arrives as a cache write on
the first page of a type and a cache read on every later one, leaving a
couple of uncached tokens. Only reading ``input_tokens`` recorded 2 against
a ~20k-token page, and ``repowise status`` summed a whole wiki to 0.
"""
return _success_json(
"page prose",
usage={
"input_tokens": uncached,
"output_tokens": 2465,
"cache_read_input_tokens": read,
"cache_creation_input_tokens": created,
},
)
@pytest.mark.parametrize(
("uncached", "read", "created"),
[
# First page of a page type: the prompt is written to the cache.
(2, 0, 19311),
# Later pages of the same type: that prefix is read back.
(2, 19311, 0),
# A short prompt that fits entirely in the uncached remainder.
(120, 0, 0),
],
)
async def test_input_tokens_is_the_prompt_total(
claude_on_path, monkeypatch, uncached, read, created
):
async def fake_exec(*_args, **_kwargs):
return FakeProcess(stdout=_cached_page_json(uncached=uncached, read=read, created=created))
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
result = await ClaudeCliProvider().generate("sys", "user")
assert result.input_tokens == uncached + read + created
assert result.usage["uncached_input_tokens"] == uncached
# The read half stays its own number: cached_tokens means "served from
# cache", and the report and the UI both read it that way.
assert result.cached_tokens == read
assert result.usage["cache_creation_input_tokens"] == created
assert result.output_tokens == 2465
class _RecordingTracker:
"""Minimal stand-in that captures what the provider books."""
def __init__(self) -> None:
self.operation = "doc_generation"
self.calls: list[dict[str, Any]] = []
async def record(self, **kwargs: Any) -> float:
self.calls.append(kwargs)
return 0.0
async def test_cost_ledger_row_carries_the_prompt_total(claude_on_path, monkeypatch):
"""No row was written at all, so a claude_cli wiki produced an empty ledger.
``repowise costs`` printed "No cost records found" and the run report's
dollar figure stayed $0.000 because the tokens it multiplies by a rate were
never recorded.
"""
async def fake_exec(*_args, **_kwargs):
return FakeProcess(stdout=_cached_page_json(uncached=2, read=0, created=19311))
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
provider = ClaudeCliProvider(model="claude-opus-5")
tracker = _RecordingTracker()
provider._cost_tracker = tracker
await provider.generate("sys", "user")
assert len(tracker.calls) == 1
booked = tracker.calls[0]
# The prefixed label is what prices the call at zero; the bare slug would
# bill a subscription seat at API rates.
assert booked["model"] == "claude_cli/claude-opus-5"
assert booked["input_tokens"] == 19313
assert booked["output_tokens"] == 2465
assert booked["operation"] == "doc_generation"
async def test_cost_ledger_row_is_priced_at_zero_for_a_seat(claude_on_path, monkeypatch):
"""The ledger shows the run's token volume and $0.00 of API spend."""
async def fake_exec(*_args, **_kwargs):
return FakeProcess(stdout=_cached_page_json(uncached=2, read=0, created=19311))
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
from repowise.core.generation.cost_tracker import CostTracker, get_model_pricing
tracker = CostTracker()
provider = ClaudeCliProvider(model="claude-opus-5")
provider._cost_tracker = tracker
await provider.generate("sys", "user")
assert get_model_pricing(provider.model_name) == {"input": 0.0, "output": 0.0}
assert tracker.session_tokens == 19313 + 2465
assert tracker.session_cost == 0.0
async def test_a_failing_cost_tracker_does_not_break_generation(claude_on_path, monkeypatch):
"""Cost telemetry is non-critical: a tracker that raises must not lose the page."""
async def fake_exec(*_args, **_kwargs):
return FakeProcess(stdout=_success_json("still produced"))
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
class _ExplodingTracker:
operation = "doc_generation"
async def record(self, **_kwargs: Any) -> float:
raise RuntimeError("ledger unavailable")
provider = ClaudeCliProvider()
provider._cost_tracker = _ExplodingTracker()
result = await provider.generate("sys", "user")
assert result.content == "still produced"
async def test_no_cost_tracker_is_a_no_op(claude_on_path, monkeypatch):
"""The MCP and server paths construct the provider without a tracker."""
async def fake_exec(*_args, **_kwargs):
return FakeProcess(stdout=_success_json("no tracker"))
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
provider = ClaudeCliProvider()
assert not hasattr(provider, "_cost_tracker")
result = await provider.generate("sys", "user")
assert result.content == "no tracker"
assert result.input_tokens == 120 + 30 + 7955
# ---------------------------------------------------------------------------
# Registration
# ---------------------------------------------------------------------------
def test_registered_as_keyless_provider():
from repowise.core.providers.llm.registry import KEYLESS_PROVIDERS, provider_is_usable
assert "claude_cli" in KEYLESS_PROVIDERS
# Keyless: never rejected for a "missing" key.
assert provider_is_usable("claude_cli", lambda _name: None) is True
def test_subscription_usage_is_priced_at_zero():
"""A seat is not API spend; the bare `claude` prefixes would price it wrongly."""
from repowise.core.cost_estimator.pricing import lookup_cost
assert lookup_cost("claude_cli/claude-haiku-4-5") == (0.0, 0.0)
assert lookup_cost("claude_cli/claude-opus-4-6") == (0.0, 0.0)
# The keyed API path is unaffected.
assert lookup_cost("claude-haiku-4-5") != (0.0, 0.0)
def test_resolves_through_the_registry(claude_on_path):
from repowise.core.providers.llm.registry import get_provider
provider = get_provider("claude_cli")
assert provider.provider_name == "claude_cli"
assert provider.model_name == "claude_cli/claude-haiku-5-5"
def test_nested_session_markers_are_stripped(claude_on_path, monkeypatch):
"""Run from inside Claude Code, the child must not inherit the parent session."""
for name in ("CLAUDECODE", "CLAUDE_CODE_SESSION_ID", "CLAUDE_CODE_MESSAGING_TOKEN"):
monkeypatch.setenv(name, "parent")
monkeypatch.setenv("CLAUDE_CODE_USE_BEDROCK", "1")
monkeypatch.setenv("ANTHROPIC_MODEL", "keep")
env = ClaudeCliProvider().subprocess_env()
assert "CLAUDECODE" not in env
assert "CLAUDE_CODE_SESSION_ID" not in env
assert "CLAUDE_CODE_MESSAGING_TOKEN" not in env
assert env["CLAUDE_CODE_USE_BEDROCK"] == "1"
assert env["ANTHROPIC_MODEL"] == "keep"