Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
"""Unit tests for CodexCliProvider.
All tests mock the Codex subprocess; no real Codex CLI calls are made.
"""
from __future__ import annotations
import asyncio
import contextlib
import json
from typing import Any
import pytest
from repowise.core.providers.llm.base import GeneratedResponse, ProviderError
from repowise.core.providers.llm.codex_cli import (
CodexCliProvider,
CodexModelReasoning,
_extract_codex_model_catalog,
_resolve_concurrency,
)
class FakeProcess:
def __init__(
self,
*,
returncode: int = 0,
stdout: str = "",
stderr: str = "",
on_communicate: Any | None = None,
transport: Any | None = None,
) -> None:
self.returncode = returncode
self._stdout = stdout
self._stderr = stderr
self._on_communicate = on_communicate
self._transport = transport
self.stdin_input: bytes | None = None
self.killed = False
async def communicate(self, input: bytes | None = None) -> tuple[bytes, bytes]:
self.stdin_input = input
if self._on_communicate is not None:
await self._on_communicate()
return self._stdout.encode("utf-8"), self._stderr.encode("utf-8")
def kill(self) -> None:
self.killed = True
async def wait(self) -> int:
return self.returncode
class FakeTransport:
def __init__(self) -> None:
self.closed = False
def close(self) -> None:
self.closed = True
def _jsonl(*events: dict[str, Any], noise: bool = False) -> str:
lines = [json.dumps(event) for event in events]
if noise:
lines.insert(1, "warning: this is not JSON")
return "\n".join(lines) + "\n"
def _success_jsonl(text: str = "OK") -> str:
return _jsonl(
{"type": "thread.started", "thread_id": "t1"},
{"type": "turn.started"},
{"type": "item.completed", "item": {"type": "agent_message", "text": text}},
{
"type": "turn.completed",
"usage": {
"input_tokens": 120,
"cached_input_tokens": 30,
"output_tokens": 40,
"reasoning_output_tokens": 10,
},
},
noise=True,
)
def _reasoning_catalog(
*efforts: str,
slug: str = "gpt-5.5",
default: str = "medium",
) -> dict[str, CodexModelReasoning]:
return {
slug: CodexModelReasoning(
slug=slug,
default_effort=default,
supported_efforts=efforts,
)
}
def test_extract_codex_model_catalog_keeps_only_reasoning_capabilities():
catalog = _extract_codex_model_catalog(
{
"models": [
{
"slug": "gpt-5.5",
"default_reasoning_level": "medium",
"supported_reasoning_levels": [
{"effort": "low", "description": "fast"},
{"effort": "high", "description": "deeper"},
],
"base_instructions": "large prompt metadata should stay ignored",
},
{"slug": "no-reasoning", "supported_reasoning_levels": []},
]
}
)
assert catalog == {
"gpt-5.5": CodexModelReasoning(
slug="gpt-5.5",
default_effort="medium",
supported_efforts=("low", "high"),
)
}
def test_provider_name_and_default_model(monkeypatch, tmp_path):
monkeypatch.setattr("shutil.which", lambda cmd: "codex" if cmd == "codex" else None)
provider = CodexCliProvider(repo_path=tmp_path)
assert provider.provider_name == "codex_cli"
assert provider.model_name == "codex_cli/default"
def test_custom_model_is_normalized_for_attribution(monkeypatch, tmp_path):
monkeypatch.setattr("shutil.which", lambda cmd: "codex" if cmd == "codex" else None)
assert CodexCliProvider(model="gpt-5.5", repo_path=tmp_path).model_name == "codex_cli/gpt-5.5"
assert (
CodexCliProvider(model="codex_cli/gpt-5.5", repo_path=tmp_path).model_name
== "codex_cli/gpt-5.5"
)
assert (
CodexCliProvider(model="codex_cli/default", repo_path=tmp_path).model_name
== "codex_cli/default"
)
def test_supported_reasoning_modes_reflect_codex_catalog(monkeypatch, tmp_path):
monkeypatch.setattr("shutil.which", lambda cmd: "codex" if cmd == "codex" else None)
monkeypatch.setattr(
"repowise.core.providers.llm.codex_cli._load_codex_model_catalog",
lambda _cmd: _reasoning_catalog("none", "low", "medium", "high", "xhigh"),
)
assert CodexCliProvider(
model="gpt-5.5",
repo_path=tmp_path,
).supported_reasoning_modes() == (
"auto",
"off",
"none",
"minimal",
"low",
"medium",
"high",
"xhigh",
)
def test_available_model_options_reflect_codex_catalog(monkeypatch, tmp_path):
monkeypatch.setattr("shutil.which", lambda cmd: "codex" if cmd == "codex" else None)
monkeypatch.setattr(
"repowise.core.providers.llm.codex_cli._load_codex_model_catalog",
lambda _cmd: _reasoning_catalog(
"none",
"low",
"medium",
"high",
slug="gpt-5.5",
default="medium",
),
)
options = CodexCliProvider(repo_path=tmp_path).available_model_options()
assert options[0].model == "codex_cli/default"
assert options[0].recommended is True
model_option = next(option for option in options if option.model == "codex_cli/gpt-5.5")
assert model_option.label == "gpt-5.5"
assert model_option.source == "local"
assert model_option.reasoning_modes == (
"auto",
"off",
"none",
"minimal",
"low",
"medium",
"high",
)
def test_missing_cli_raises(monkeypatch, tmp_path):
monkeypatch.setattr("shutil.which", lambda _cmd: None)
with pytest.raises(ProviderError, match="Codex CLI not found"):
CodexCliProvider(repo_path=tmp_path)
async def test_generate_invokes_codex_exec_with_stdin(monkeypatch, tmp_path):
codex_cmd = str(tmp_path / "bin" / "codex.CMD")
monkeypatch.setattr("shutil.which", lambda cmd: codex_cmd if cmd == "codex" else None)
monkeypatch.setattr(
"repowise.core.providers.llm.codex_cli._load_codex_model_catalog",
lambda _cmd: _reasoning_catalog("low", "medium", "high", "xhigh"),
)
captured: dict[str, Any] = {}
proc = FakeProcess(stdout=_success_jsonl("Hello from Codex"), stderr="plugin sync warning")
async def fake_exec(*args: str, **kwargs: Any) -> FakeProcess:
captured["args"] = args
captured["kwargs"] = kwargs
return proc
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
provider = CodexCliProvider(model="gpt-5.5", repo_path=tmp_path)
result = await provider.generate("system rules", "user context", reasoning="minimal")
assert isinstance(result, GeneratedResponse)
assert result.content == "Hello from Codex"
assert proc.stdin_input is not None
prompt = proc.stdin_input.decode("utf-8")
assert "system rules" in prompt
assert "user context" in prompt
args = list(captured["args"])
assert args[:6] == [codex_cmd, "exec", "--ephemeral", "--sandbox", "read-only", "--json"]
assert args[args.index("--cd") + 1] == str(tmp_path.resolve())
configs = [args[i + 1] for i, a in enumerate(args) if a == "--config"]
assert "project_doc_max_bytes=0" in configs
assert 'model_reasoning_effort="low"' in configs
assert args[args.index("--model") + 1] == "gpt-5.5"
assert args[-1] == "-"
assert captured["kwargs"]["stdin"] == asyncio.subprocess.PIPE
async def test_generate_passes_supported_codex_reasoning_effort(monkeypatch, tmp_path):
monkeypatch.setattr("shutil.which", lambda cmd: "codex" if cmd == "codex" else None)
monkeypatch.setattr(
"repowise.core.providers.llm.codex_cli._load_codex_model_catalog",
lambda _cmd: _reasoning_catalog("low", "medium", "high", "xhigh"),
)
captured: dict[str, Any] = {}
async def fake_exec(*args: str, **_kwargs: Any) -> FakeProcess:
captured["args"] = args
return FakeProcess(stdout=_success_jsonl("OK"))
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
await CodexCliProvider(model="gpt-5.5", repo_path=tmp_path).generate(
"sys",
"user",
reasoning="xhigh",
)
args = list(captured["args"])
configs = [args[i + 1] for i, a in enumerate(args) if a == "--config"]
assert "project_doc_max_bytes=0" in configs
assert 'model_reasoning_effort="xhigh"' in configs
async def test_generate_passes_unknown_model_effort_without_catalog_block(
monkeypatch,
tmp_path,
):
monkeypatch.setattr("shutil.which", lambda cmd: "codex" if cmd == "codex" else None)
monkeypatch.setattr(
"repowise.core.providers.llm.codex_cli._load_codex_model_catalog",
lambda _cmd: _reasoning_catalog("low", "medium", "high", "xhigh"),
)
captured: dict[str, Any] = {}
async def fake_exec(*args: str, **_kwargs: Any) -> FakeProcess:
captured["args"] = args
return FakeProcess(stdout=_success_jsonl("OK"))
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
await CodexCliProvider(model="future-codex", repo_path=tmp_path).generate(
"sys",
"user",
reasoning="max",
)
args = list(captured["args"])
configs = [args[i + 1] for i, a in enumerate(args) if a == "--config"]
assert "project_doc_max_bytes=0" in configs
assert 'model_reasoning_effort="max"' in configs
assert args[args.index("--model") + 1] == "future-codex"
async def test_generate_uses_jsonl_usage_and_ignores_noise(monkeypatch, tmp_path):
monkeypatch.setattr("shutil.which", lambda cmd: "codex" if cmd == "codex" else None)
async def fake_exec(*_args: str, **_kwargs: Any) -> FakeProcess:
return FakeProcess(stdout=_success_jsonl("OK"))
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
result = await CodexCliProvider(repo_path=tmp_path).generate("sys", "user")
assert result.input_tokens == 120
assert result.output_tokens == 40
assert result.cached_tokens == 30
assert result.usage["reasoning_output_tokens"] == 10
assert result.usage["source"] == "codex_exec"
assert "estimated" not in result.usage
async def test_generate_accepts_cache_hints(monkeypatch, tmp_path):
monkeypatch.setattr("shutil.which", lambda cmd: "codex" if cmd == "codex" else None)
async def fake_exec(*_args: str, **_kwargs: Any) -> FakeProcess:
return FakeProcess(stdout=_success_jsonl("OK"))
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
result = await CodexCliProvider(repo_path=tmp_path).generate(
"sys",
"user",
cache_hints=(),
)
assert result.content == "OK"
async def test_generate_closes_subprocess_transport(monkeypatch, tmp_path):
monkeypatch.setattr("shutil.which", lambda cmd: "codex" if cmd == "codex" else None)
transport = FakeTransport()
async def fake_exec(*_args: str, **_kwargs: Any) -> FakeProcess:
return FakeProcess(stdout=_success_jsonl("OK"), transport=transport)
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
await CodexCliProvider(repo_path=tmp_path).generate("sys", "user")
assert transport.closed is True
async def test_generate_marks_missing_usage_as_estimated(monkeypatch, tmp_path):
monkeypatch.setattr("shutil.which", lambda cmd: "codex" if cmd == "codex" else None)
async def fake_exec(*_args: str, **_kwargs: Any) -> FakeProcess:
return FakeProcess(
stdout=_jsonl(
{
"type": "item.completed",
"item": {"type": "agent_message", "text": "OK"},
}
)
)
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
result = await CodexCliProvider(repo_path=tmp_path).generate("sys", "user")
assert result.input_tokens == 0
assert result.output_tokens == 0
assert result.usage["estimated"] is True
async def test_generate_raises_on_nonzero_exit(monkeypatch, tmp_path):
monkeypatch.setattr("shutil.which", lambda cmd: "codex" if cmd == "codex" else None)
async def fake_exec(*_args: str, **_kwargs: Any) -> FakeProcess:
return FakeProcess(returncode=1, stdout="", stderr="not logged in")
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
with pytest.raises(ProviderError, match="not logged in"):
await CodexCliProvider(repo_path=tmp_path).generate("sys", "user")
async def test_generate_hides_structured_error_stdout(monkeypatch, tmp_path):
monkeypatch.setattr("shutil.which", lambda cmd: "codex" if cmd == "codex" else None)
async def fake_exec(*_args: str, **_kwargs: Any) -> FakeProcess:
return FakeProcess(returncode=2, stdout='{"error":{"message":"secret details"}}')
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
with pytest.raises(ProviderError, match="codex exec exited with 2") as exc_info:
await CodexCliProvider(repo_path=tmp_path).generate("sys", "user")
assert "secret details" not in str(exc_info.value)
async def test_generate_raises_when_jsonl_has_no_agent_message(monkeypatch, tmp_path):
monkeypatch.setattr("shutil.which", lambda cmd: "codex" if cmd == "codex" else None)
async def fake_exec(*_args: str, **_kwargs: Any) -> FakeProcess:
return FakeProcess(stdout=_jsonl({"type": "turn.completed", "usage": {}}))
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
with pytest.raises(ProviderError, match="no agent_message"):
await CodexCliProvider(repo_path=tmp_path).generate("sys", "user")
async def test_generate_respects_configured_concurrency(monkeypatch, tmp_path):
"""The semaphore is the fan-out knob: 2 in flight, never more."""
monkeypatch.setattr("shutil.which", lambda cmd: "codex" if cmd == "codex" else None)
monkeypatch.setenv("REPOWISE_CODEX_CLI_CONCURRENCY", "2")
active = 0
max_active = 0
# A barrier, not a sleep: on a loaded runner the first call could finish
# before the last one starts. The timeout keeps a serialised regression
# failing fast instead of hanging.
all_in = asyncio.Event()
async def on_communicate() -> None:
nonlocal active, max_active
active += 1
max_active = max(max_active, active)
if active == 2:
all_in.set()
with contextlib.suppress(TimeoutError):
await asyncio.wait_for(all_in.wait(), timeout=2)
active -= 1
async def fake_exec(*_args: str, **_kwargs: Any) -> FakeProcess:
return FakeProcess(stdout=_success_jsonl("OK"), on_communicate=on_communicate)
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
provider = CodexCliProvider(repo_path=tmp_path)
await asyncio.gather(
provider.generate("sys", "user 1"),
provider.generate("sys", "user 2"),
provider.generate("sys", "user 3"),
)
assert max_active == 2
async def test_generate_fans_out_to_the_default_concurrency(monkeypatch, tmp_path):
"""Serializing made a 104-page generate take about 95 minutes."""
monkeypatch.setattr("shutil.which", lambda cmd: "codex" if cmd == "codex" else None)
monkeypatch.delenv("REPOWISE_CODEX_CLI_CONCURRENCY", raising=False)
active = 0
max_active = 0
# A barrier, not a sleep: on a loaded runner the first call could finish
# before the last one starts. The timeout keeps a serialised regression
# failing fast instead of hanging.
all_in = asyncio.Event()
async def on_communicate() -> None:
nonlocal active, max_active
active += 1
max_active = max(max_active, active)
if active == 4:
all_in.set()
with contextlib.suppress(TimeoutError):
await asyncio.wait_for(all_in.wait(), timeout=2)
active -= 1
async def fake_exec(*_args: str, **_kwargs: Any) -> FakeProcess:
return FakeProcess(stdout=_success_jsonl("OK"), on_communicate=on_communicate)
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
provider = CodexCliProvider(repo_path=tmp_path)
await asyncio.gather(*[provider.generate("sys", f"user {i}") for i in range(4)])
assert max_active == 4
def test_concurrency_env_overrides_the_default(monkeypatch):
monkeypatch.setenv("REPOWISE_CODEX_CLI_CONCURRENCY", "6")
assert _resolve_concurrency() == 6
def test_concurrency_default_is_four(monkeypatch):
monkeypatch.delenv("REPOWISE_CODEX_CLI_CONCURRENCY", raising=False)
assert _resolve_concurrency() == 4
def test_concurrency_rejects_invalid_and_clamps_to_one(monkeypatch):
monkeypatch.setenv("REPOWISE_CODEX_CLI_CONCURRENCY", "not-a-number")
assert _resolve_concurrency() == 4
monkeypatch.setenv("REPOWISE_CODEX_CLI_CONCURRENCY", "0")
assert _resolve_concurrency() == 1
monkeypatch.setenv("REPOWISE_CODEX_CLI_CONCURRENCY", " 2 ")
assert _resolve_concurrency() == 2
async def test_generate_times_out_and_kills_codex_exec(monkeypatch, tmp_path):
monkeypatch.setattr("shutil.which", lambda cmd: "codex" if cmd == "codex" else None)
monkeypatch.setattr(
"repowise.core.providers.llm.codex_cli._EXEC_TIMEOUT_SECONDS",
0.01,
)
async def on_communicate() -> None:
await asyncio.sleep(1)
proc = FakeProcess(stdout="", on_communicate=on_communicate)
async def fake_exec(*_args: str, **_kwargs: Any) -> FakeProcess:
return proc
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
with pytest.raises(ProviderError, match="timed out"):
await CodexCliProvider(repo_path=tmp_path).generate("sys", "user")
assert proc.killed
async def test_generate_rejects_unsupported_reasoning_off(monkeypatch, tmp_path):
monkeypatch.setattr("shutil.which", lambda cmd: "codex" if cmd == "codex" else None)
monkeypatch.setattr(
"repowise.core.providers.llm.codex_cli._load_codex_model_catalog",
lambda _cmd: _reasoning_catalog("low", "medium", "high", "xhigh"),
)
async def fake_exec(*_args: str, **_kwargs: Any) -> FakeProcess:
raise AssertionError("subprocess should not be started")
monkeypatch.setattr("asyncio.create_subprocess_exec", fake_exec)
with pytest.raises(ProviderError, match="model_reasoning_effort='none'"):
await CodexCliProvider(repo_path=tmp_path).generate("sys", "user", reasoning="off")
def test_model_catalog_probe_never_inherits_stdin(monkeypatch):
"""Inside a stdio MCP server stdin is the JSON-RPC pipe."""
import subprocess
from repowise.core.providers.llm import codex_cli
seen: dict[str, Any] = {}
def fake_run(_args, **kwargs):
seen.update(kwargs)
return subprocess.CompletedProcess(_args, 1, "", "")
monkeypatch.setattr(codex_cli.subprocess, "run", fake_run)
codex_cli._load_codex_model_catalog.cache_clear()
assert codex_cli._load_codex_model_catalog("codex-stdin-probe") is None
assert seen["stdin"] is subprocess.DEVNULL