Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
"""Contract every ``AgentCliProvider`` subclass must meet.
A new agent backend joins by adding one ``Backend`` entry to ``BACKENDS``.
The subprocess is faked; no real CLI runs.
"""
from __future__ import annotations
import asyncio
import json
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
from repowise.core.providers.llm.agent_cli import AgentCliProvider
from repowise.core.providers.llm.base import GeneratedResponse, ProviderError
from repowise.core.providers.llm.claude_cli import ClaudeCliProvider
from repowise.core.providers.llm.codex_cli import CodexCliProvider
from repowise.core.providers.llm.copilot_cli import CopilotCliProvider
from repowise.core.providers.llm.opencode import OpenCodeProvider
from repowise.core.providers.llm.registry import _BUILTIN_PROVIDERS
@dataclass(frozen=True)
class Backend:
cls: type[AgentCliProvider]
model: str # a native slug the CLI receives after ``--model``
# CLI output whose answer is the argument, with usage that parses to ``tokens``.
success_stdout: Callable[[str], str]
tokens: tuple[int, int, int] # parsed (input, output, cached)
books_cost: bool
# Flag that points the CLI at the repo; None means no flag is passed.
repo_flag: str | None
env: dict[str, str] | None # entries the subprocess env must carry
# True when the repo is passed as the subprocess cwd (opencode, after the
# OpenCode CLI v2 removed ``--dir``); False means a per-call scratch cwd.
repo_cwd: bool = False
def _jsonl(*events: dict[str, Any]) -> str:
return "".join(json.dumps(event) + "\n" for event in events)
def _claude_stdout(text: str) -> str:
usage = {
"input_tokens": 100,
"output_tokens": 40,
"cache_read_input_tokens": 30,
"cache_creation_input_tokens": 10,
}
return _jsonl({"subtype": "success", "is_error": False, "result": text, "usage": usage})
def _codex_stdout(text: str) -> str:
return _jsonl(
{"type": "item.completed", "item": {"type": "agent_message", "text": text}},
{
"type": "turn.completed",
"usage": {"input_tokens": 120, "cached_input_tokens": 30, "output_tokens": 40},
},
)
def _opencode_stdout(text: str) -> str:
return _jsonl(
{"type": "text", "part": {"text": text}},
{
"type": "step_finish",
"part": {"tokens": {"input": 80, "output": 10, "cache": {"read": 20, "write": 0}}},
},
)
def _copilot_stdout(text: str) -> str:
return _jsonl(
{
"type": "assistant.message",
"data": {"content": text, "toolRequests": []},
},
{
"type": "session.usage_checkpoint",
"data": {
"accountingSnapshot": {
"lastCallInputTokens": 120,
"lastCallOutputTokens": 40,
"totalPremiumRequestCost": 1,
"tokenDetails": {"cache_read": {"tokenCount": 30}},
}
},
},
)
BACKENDS = [
Backend(
ClaudeCliProvider,
"claude-sonnet-4-6",
_claude_stdout,
# Prompt total: uncached + cache read + cache write.
tokens=(140, 40, 30),
books_cost=True,
repo_flag=None,
# A filtered copy of the parent env (nested-session markers removed).
env={},
),
Backend(
CodexCliProvider,
"gpt-5.5",
_codex_stdout,
tokens=(120, 40, 30),
books_cost=False,
repo_flag="--cd",
env=None,
),
Backend(
OpenCodeProvider,
"deepseek/deepseek-v4-pro",
_opencode_stdout,
tokens=(80, 10, 20),
books_cost=False,
repo_flag=None,
env={"OPENCODE_DISABLE_PROJECT_CONFIG": "true"},
repo_cwd=True,
),
Backend(
CopilotCliProvider,
"gpt-5.5",
_copilot_stdout,
tokens=(120, 40, 30),
books_cost=False,
repo_flag="-C",
env=None,
),
]
class FakeProcess:
def __init__(
self,
*,
returncode: int = 0,
stdout: str = "",
stderr: str = "",
on_communicate: Any | None = None,
) -> None:
self.returncode = returncode
self._stdout = stdout
self._stderr = stderr
self._on_communicate = on_communicate
self._transport = FakeTransport()
self.stdin_input: bytes | None = None
self.killed = False
async def communicate(self, input: bytes | None = None) -> tuple[bytes, bytes]:
self.stdin_input = input
if self._on_communicate is not None:
await self._on_communicate()
return self._stdout.encode("utf-8"), self._stderr.encode("utf-8")
def kill(self) -> None:
self.killed = True
async def wait(self) -> int:
return self.returncode
class FakeTransport:
def __init__(self) -> None:
self.closed = False
def close(self) -> None:
self.closed = True
@pytest.fixture(params=BACKENDS, ids=lambda b: b.cls.provider_name)
def backend(request, monkeypatch, tmp_path) -> Backend:
exe = str(tmp_path / "bin" / request.param.cls.executable_name)
monkeypatch.setattr(
"shutil.which", lambda cmd: exe if cmd == request.param.cls.executable_name else None
)
return request.param
def _make(backend: Backend, tmp_path) -> AgentCliProvider:
# Every agent CLI provider is built with these kwargs by the registry.
return backend.cls(model=backend.model, repo_path=tmp_path)
def _fake_exec(monkeypatch, proc_factory: Callable[[], FakeProcess]) -> list[dict[str, Any]]:
calls: list[dict[str, Any]] = []
async def fake_exec(*args: str, **kwargs: Any) -> FakeProcess:
proc = proc_factory()
calls.append({"args": list(args), "kwargs": kwargs, "proc": proc})
return proc
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
return calls
def test_names_are_consistent(backend, tmp_path):
provider = _make(backend, tmp_path)
name = provider.provider_name
assert _BUILTIN_PROVIDERS[name][1] == backend.cls.__name__
assert provider.model_name == f"{name}/{backend.model}"
# A persisted label round-trips instead of gaining a second prefix.
assert backend.cls(model=provider.model_name, repo_path=tmp_path).model_name == (
provider.model_name
)
def test_missing_executable_raises(backend, monkeypatch, tmp_path):
monkeypatch.setattr("shutil.which", lambda _cmd: None)
with pytest.raises(ProviderError) as caught:
_make(backend, tmp_path)
assert caught.value.provider == backend.cls.provider_name
async def test_command_stdin_and_output(backend, monkeypatch, tmp_path):
calls = _fake_exec(monkeypatch, lambda: FakeProcess(stdout=backend.success_stdout("answer")))
result = await _make(backend, tmp_path).generate("system rules", "user context")
assert isinstance(result, GeneratedResponse)
assert result.content == "answer"
assert (result.input_tokens, result.output_tokens, result.cached_tokens) == backend.tokens
args = calls[0]["args"]
assert args[0].endswith(backend.cls.executable_name)
assert args[args.index("--model") + 1] == backend.model
# Prompts are large, so they travel on stdin, never argv.
assert calls[0]["kwargs"]["stdin"] == asyncio.subprocess.PIPE
assert b"user context" in calls[0]["proc"].stdin_input
assert not any("user context" in a for a in args)
assert calls[0]["proc"]._transport.closed
async def test_runs_against_the_repo_or_a_scratch_dir(backend, monkeypatch, tmp_path):
calls = _fake_exec(monkeypatch, lambda: FakeProcess(stdout=backend.success_stdout("ok")))
await _make(backend, tmp_path).generate("sys", "user")
args, cwd = calls[0]["args"], calls[0]["kwargs"]["cwd"]
if backend.repo_flag is not None:
assert args[args.index(backend.repo_flag) + 1] == str(tmp_path.resolve())
assert cwd is None
elif backend.repo_cwd:
# No flag: the repo travels as the subprocess cwd.
assert cwd == str(tmp_path.resolve())
else:
# A neutral per-call directory, removed afterwards.
assert cwd is not None and Path(cwd) != tmp_path.resolve()
assert not Path(cwd).exists()
async def test_subprocess_env(backend, monkeypatch, tmp_path):
calls = _fake_exec(monkeypatch, lambda: FakeProcess(stdout=backend.success_stdout("ok")))
await _make(backend, tmp_path).generate("sys", "user")
env = calls[0]["kwargs"]["env"]
if backend.env is None:
assert env is None # inherits the parent environment unchanged
else:
assert backend.env.items() <= env.items()
if backend.cls is OpenCodeProvider:
permissions = json.loads(env["OPENCODE_CONFIG_CONTENT"])["permission"]
assert permissions["edit"] == permissions["bash"] == "deny"
async def test_cost_booking(backend, monkeypatch, tmp_path):
_fake_exec(monkeypatch, lambda: FakeProcess(stdout=backend.success_stdout("ok")))
tracker = MagicMock(operation="doc_generation")
tracker.record = AsyncMock(return_value=0.0)
provider = _make(backend, tmp_path)
provider._cost_tracker = tracker
await provider.generate("sys", "user")
if backend.books_cost:
tracker.record.assert_awaited_once_with(
model=provider.model_name,
input_tokens=backend.tokens[0],
output_tokens=backend.tokens[1],
operation="doc_generation",
file_path=None,
)
else:
tracker.record.assert_not_awaited()
async def test_nonzero_exit_raises_with_stderr_tail(backend, monkeypatch, tmp_path):
stderr = "x" * 5_000 + " not logged in"
_fake_exec(monkeypatch, lambda: FakeProcess(returncode=1, stderr=stderr))
with pytest.raises(ProviderError) as caught:
await _make(backend, tmp_path).generate("sys", "user")
message = str(caught.value)
assert caught.value.provider == backend.cls.provider_name
assert message.endswith("not logged in")
assert len(message) < len(stderr)
async def test_timeout_kills_the_process(backend, monkeypatch, tmp_path):
monkeypatch.setattr(backend.cls, "exec_timeout_seconds", lambda _self: 0.01)
async def hang() -> None:
await asyncio.sleep(1)
calls = _fake_exec(monkeypatch, lambda: FakeProcess(on_communicate=hang))
with pytest.raises(ProviderError, match="timed out"):
await _make(backend, tmp_path).generate("sys", "user")
assert calls[0]["proc"].killed
assert calls[0]["proc"]._transport.closed
async def test_cancellation_kills_the_process(backend, monkeypatch, tmp_path):
started = asyncio.Event()
async def block() -> None:
started.set()
await asyncio.Event().wait()
calls = _fake_exec(monkeypatch, lambda: FakeProcess(on_communicate=block))
task = asyncio.create_task(_make(backend, tmp_path).generate("sys", "user"))
await started.wait()
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert calls[0]["proc"].killed
async def test_semaphore_bounds_concurrency(backend, monkeypatch, tmp_path):
monkeypatch.setenv(backend.cls.concurrency_env, "2")
active = 0
max_active = 0
async def track() -> None:
nonlocal active, max_active
active += 1
max_active = max(max_active, active)
await asyncio.sleep(0.01)
active -= 1
_fake_exec(
monkeypatch,
lambda: FakeProcess(stdout=backend.success_stdout("ok"), on_communicate=track),
)
provider = _make(backend, tmp_path)
await asyncio.gather(*[provider.generate("sys", f"user {i}") for i in range(5)])
assert max_active == 2