Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
"""Unit tests for `AnthropicClaudeProvider` and `OpenAICodexProvider`.
Covers:
- `interactive_argv` shape (every flag in the right place, defaults
applied, optional fields omitted when unset). Claude's argv no
longer carries ``-p``/``--output-format``/``--max-turns``/
``--max-budget-usd``/``--session-id``/``--include-hook-events`` —
interactive cutover ($docs-internal/claude_code_interactive_mode.md).
- `parse_event` round-trips JSON correctly, returns None for garbage
- `is_final_event` matches the right event types
- `event_to_session_result` reconstructs cost / session_id /
canonical_usage / response from vendored NDJSON
- `detect_auth_error` matches "not logged in" stderr patterns
- `supports()` flags align with `ai_cli_providers.json`
- Factory raises NotImplementedError for gemini
"""
from __future__ import annotations
import json
import pytest
from services.cli_agent import (
ClaudeTaskSpec,
CodexTaskSpec,
create_cli_provider,
)
from services.cli_agent.factory import is_supported
from services.cli_agent.protocol import AICliProvider, CanonicalUsage
# ---------------------------------------------------------------------------
# Factory contract
# ---------------------------------------------------------------------------
class TestFactory:
def test_claude_creates_provider(self):
p = create_cli_provider("claude")
assert p.name == "claude"
assert isinstance(p, AICliProvider)
def test_codex_creates_provider(self):
p = create_cli_provider("codex")
assert p.name == "codex"
assert isinstance(p, AICliProvider)
def test_gemini_raises_not_implemented(self):
with pytest.raises(NotImplementedError, match="deferred to v2"):
create_cli_provider("gemini")
def test_unknown_raises_value_error(self):
with pytest.raises(ValueError, match="Unknown CLI provider"):
create_cli_provider("openai")
def test_is_supported(self):
assert is_supported("claude") is True
assert is_supported("codex") is True
assert is_supported("gemini") is False
assert is_supported("nope") is False
# ---------------------------------------------------------------------------
# Claude provider
# ---------------------------------------------------------------------------
@pytest.fixture
def claude_provider():
return create_cli_provider("claude")
class TestClaudeArgv:
def test_minimum_required_flags(self, claude_provider):
task = ClaudeTaskSpec(prompt="hello world")
argv = claude_provider.interactive_argv(task, defaults={})
# VSCode-extension pattern (stream-json over stdio pipes, no PTY).
# ``-p`` / ``--print`` are NEVER emitted — we stay in interactive
# billing (entrypoint ``claude-vscode``, not ``sdk-cli``).
assert "-p" not in argv
assert "--print" not in argv
# Stream-json I/O flags ARE emitted now (replace the headless
# PTY pattern):
assert "--output-format" in argv
out_idx = argv.index("--output-format")
assert argv[out_idx + 1] == "stream-json"
assert "--input-format" in argv
in_idx = argv.index("--input-format")
assert argv[in_idx + 1] == "stream-json"
assert "--verbose" in argv # required for stream-json detail
assert "--ide" in argv # matches the VSCode extension argv; no lockfile is written
# The prompt is NEVER a positional in stream-json mode — it goes
# over ``proc.stdin``. ``--`` separator is gone.
assert "--" not in argv
assert "hello world" not in argv
# Defaults applied
assert "--model" in argv
assert "claude-sonnet-4-6" in argv # default_model from JSON
# ``-p``-only flags still not emitted (kept on the spec for back-compat)
assert "--max-turns" not in argv
assert "--max-budget-usd" not in argv
assert "--fallback-model" not in argv
# Tool + permission machinery preserved
assert "--allowedTools" in argv
assert "--permission-mode" in argv
# Default permission mode is read from ``config/ai_cli_providers.json``
# (currently ``dontAsk`` — "only pre-approved tools, no prompts",
# which gates ``--allowedTools`` strictly so claude's built-in
# Read/Edit/Bash/etc. can NOT fire unless explicitly listed).
# Callers wanting ``acceptEdits`` / ``bypassPermissions`` set it
# on ``task.permission_mode`` or via the per-call ``defaults`` arg.
perm_idx = argv.index("--permission-mode")
assert argv[perm_idx + 1] == "dontAsk"
def test_no_claude_builtins_in_default_allowlist(self, claude_provider):
"""Claude's built-in escape-hatch tools (Read, Edit, Bash, Glob,
Grep, Write, Skill, WebSearch, WebFetch) are intentionally NOT
in the default ``--allowedTools`` list when no skills are
wired. Combined with ``--permission-mode dontAsk`` (which
gates strictly on the allowlist) this means the agent can
ONLY invoke MCP tools that were explicitly wired through
``input-tools`` plus OpenCompany's own MCP infrastructure tools.
Disconnected workflow tools are also blocked by the per-handler
scope check in ``workflow_tools._build_handler`` reading the
rebound ``BatchContext.connected_tools``.
"""
task = ClaudeTaskSpec(prompt="x")
argv = claude_provider.interactive_argv(task, defaults={})
allowed_idx = argv.index("--allowedTools")
allowlist = argv[allowed_idx + 1]
entries = [e.strip() for e in allowlist.split(",")]
builtins = {
"Read",
"Edit",
"Bash",
"Glob",
"Grep",
"Write",
"Skill",
"WebSearch",
"WebFetch",
}
for entry in entries:
assert entry not in builtins, f"claude built-in {entry!r} leaked into default allowlist"
# OpenCompany MCP infrastructure must still be present so the
# agent can read the workspace + invoke connected skills.
assert "mcp__opencompany__getWorkspaceFiles" in entries
assert "mcp__opencompany__listSkills" in entries
assert "mcp__opencompany__getSkill" in entries
assert "mcp__opencompany__getCredential" in entries
assert "mcp__opencompany__broadcastLog" in entries
def test_skill_builtin_conditional_on_wired_skills(self, claude_provider):
"""``Skill`` is the one claude built-in we conditionally allow
— present in ``--allowedTools`` iff at least one skill is wired
through the agent's ``input-skill`` handle. Paired with
:func:`nodes.agent.claude_code_agent._skills.materialise_skills` which
writes the connected SKILL.md trees under
``<cwd>/.claude/skills/`` before spawn so the built-in skill
loader has something to discover.
When no skill is wired, ``Skill`` stays out of the allowlist
(the strict-allowlist invariant from
:meth:`test_no_claude_builtins_in_default_allowlist` still
holds for that path).
"""
task = ClaudeTaskSpec(prompt="x")
# No skills wired → Skill stays out.
argv_no_skills = claude_provider.interactive_argv(task, defaults={})
allowed_idx = argv_no_skills.index("--allowedTools")
entries_no_skills = [e.strip() for e in argv_no_skills[allowed_idx + 1].split(",")]
assert "Skill" not in entries_no_skills
# Skills wired → Skill enters the allowlist.
argv_with_skills = claude_provider.interactive_argv(
task,
defaults={},
connected_skill_names=["humanify-skill", "memory-skill"],
)
allowed_idx = argv_with_skills.index("--allowedTools")
entries_with_skills = [e.strip() for e in argv_with_skills[allowed_idx + 1].split(",")]
assert "Skill" in entries_with_skills
# Other built-ins still stay out — only Skill is conditionally
# enabled. The rest (Read, Edit, Bash, etc.) are never
# auto-added by the allowlist builder.
for forbidden in (
"Read",
"Edit",
"Bash",
"Glob",
"Grep",
"Write",
"WebSearch",
"WebFetch",
):
assert forbidden not in entries_with_skills
def test_session_id_flag_emitted_when_set(self, claude_provider):
"""A host-minted UUID (``ClaudeSessionPool._spawn`` sets it on a
cold spawn with no continuity flag) is passed as ``--session-id``
so the session UUID is known before the first stream event."""
task = ClaudeTaskSpec(prompt="x", session_id="sess-abc-123")
argv = claude_provider.interactive_argv(task, defaults={})
idx = argv.index("--session-id")
assert argv[idx + 1] == "sess-abc-123"
assert "--resume" not in argv
assert "--continue" not in argv
def test_resume_flag_emitted(self, claude_provider):
task = ClaudeTaskSpec(
prompt="x",
session_id="new-sess", # loses to an explicit resume
resume_session_id="prior-sess",
)
argv = claude_provider.interactive_argv(task, defaults={})
assert "--resume" in argv
assert "prior-sess" in argv
assert "--session-id" not in argv
# `--continue` is mutually exclusive with `--resume`; explicit
# UUID wins.
assert "--continue" not in argv
def test_continue_flag_emitted(self, claude_provider):
"""``continue_session=True`` produces ``--continue`` — the
cleaner default for memory-bound runs (claude auto-finds the
latest conversation under cwd, no UUID round-trip)."""
task = ClaudeTaskSpec(prompt="x", continue_session=True)
argv = claude_provider.interactive_argv(task, defaults={})
assert "--continue" in argv
assert "--resume" not in argv
assert "--session-id" not in argv
def test_resume_wins_over_continue(self, claude_provider):
"""If both are set (caller bug), explicit ``--resume`` wins —
matches the argv-builder's mutually-exclusive guard."""
task = ClaudeTaskSpec(
prompt="x",
resume_session_id="prior",
continue_session=True,
)
argv = claude_provider.interactive_argv(task, defaults={})
assert "--resume" in argv
assert "--continue" not in argv
def test_no_continuity_flag_by_default(self, claude_provider):
"""Fresh task with no continuity hint → no flag. Claude assigns
a new session UUID."""
task = ClaudeTaskSpec(prompt="x")
argv = claude_provider.interactive_argv(task, defaults={})
assert "--resume" not in argv
assert "--continue" not in argv
assert "--session-id" not in argv
def test_system_prompt_propagates(self, claude_provider):
task = ClaudeTaskSpec(prompt="x", system_prompt="be concise")
argv = claude_provider.interactive_argv(task, defaults={})
assert "--append-system-prompt" in argv
assert "be concise" in argv
def test_include_prompt_is_ignored_in_stream_json_mode(self, claude_provider):
"""``include_prompt`` is kept on the signature for back-compat
but is intentionally ignored in stream-json input mode — passing
``-- "<prompt>"`` would double-send the first turn (once via
argv, once via stdin). Both ``True`` and ``False`` produce the
same argv shape: no ``--`` separator, no positional prompt."""
task = ClaudeTaskSpec(prompt="will-be-piped-via-stdin")
argv_true = claude_provider.interactive_argv(
task,
defaults={},
include_prompt=True,
)
argv_false = claude_provider.interactive_argv(
task,
defaults={},
include_prompt=False,
)
for argv in (argv_true, argv_false):
assert "--" not in argv
assert "will-be-piped-via-stdin" not in argv
# Same shape regardless of the back-compat flag.
assert argv_true == argv_false
def test_wrong_task_type_raises(self, claude_provider):
codex_task = CodexTaskSpec(prompt="x")
with pytest.raises(TypeError, match="ClaudeTaskSpec"):
claude_provider.interactive_argv(codex_task, defaults={})
class TestClaudeParseEvent:
def test_valid_json_round_trips(self, claude_provider):
line = json.dumps({"type": "result", "result": "42", "session_id": "abc"})
event = claude_provider.parse_event(line)
assert event is not None
assert event["type"] == "result"
assert event["result"] == "42"
def test_garbage_returns_none(self, claude_provider):
assert claude_provider.parse_event("not json") is None
assert claude_provider.parse_event("") is None
assert claude_provider.parse_event(" ") is None
def test_is_final_event(self, claude_provider):
assert claude_provider.is_final_event({"type": "result"}) is True
assert claude_provider.is_final_event({"type": "assistant"}) is False
assert claude_provider.is_final_event({"type": "system"}) is False
class TestClaudeEventToSessionResult:
"""Vendored Claude stream-json fixture verified end-to-end."""
@pytest.fixture
def fixture_events(self):
return [
{"type": "system", "subtype": "init", "session_id": "sess-1"},
{
"type": "assistant",
"message": {
"content": [
{"type": "text", "text": "Sure, here's..."},
{"type": "tool_use", "id": "t1", "name": "Read", "input": {}},
],
},
"session_id": "sess-1",
},
{
"type": "tool_use",
"tool_name": "Edit",
"tool_input": {"file_path": "x.py"},
},
{
"type": "result",
"subtype": "success",
"result": "Done — refactored to async.",
"total_cost_usd": 0.4231,
"duration_ms": 18234,
"num_turns": 7,
"session_id": "sess-1",
"usage": {
"input_tokens": 12000,
"output_tokens": 3500,
"cache_creation_input_tokens": 500,
"cache_read_input_tokens": 8000,
},
},
]
def test_reconstructs_response(self, claude_provider, fixture_events):
result = claude_provider.event_to_session_result(fixture_events, "", 0)
assert result["response"] == "Done — refactored to async."
def test_reconstructs_cost(self, claude_provider, fixture_events):
result = claude_provider.event_to_session_result(fixture_events, "", 0)
assert result["cost_usd"] == pytest.approx(0.4231)
def test_reconstructs_session_id(self, claude_provider, fixture_events):
result = claude_provider.event_to_session_result(fixture_events, "", 0)
assert result["session_id"] == "sess-1"
def test_reconstructs_duration_and_turns(self, claude_provider, fixture_events):
result = claude_provider.event_to_session_result(fixture_events, "", 0)
assert result["duration_ms"] == 18234
assert result["num_turns"] == 7
def test_canonical_usage_normalises(self, claude_provider, fixture_events):
cu: CanonicalUsage = claude_provider.canonical_usage(fixture_events)
assert cu.input_tokens == 12000
assert cu.output_tokens == 3500
assert cu.cache_read == 8000 # remapped from cache_read_input_tokens
assert cu.cache_write == 500 # remapped from cache_creation_input_tokens
assert cu.request_count == 7 # from num_turns
def test_counts_tool_calls(self, claude_provider, fixture_events):
# Two tool_use events: one inside an assistant message, one standalone
result = claude_provider.event_to_session_result(fixture_events, "", 0)
assert result["tool_calls"] >= 2
def test_success_on_zero_exit_with_result_event(self, claude_provider, fixture_events):
result = claude_provider.event_to_session_result(fixture_events, "", 0)
assert result["success"] is True
assert result["error"] is None
def test_failure_on_non_zero_exit(self, claude_provider, fixture_events):
result = claude_provider.event_to_session_result(
fixture_events,
"exploded",
1,
)
assert result["success"] is False
assert "exploded" in (result["error"] or "")
def test_failure_on_missing_result_event(self, claude_provider):
events = [
{"type": "system", "subtype": "init", "session_id": "s"},
{"type": "assistant", "message": {}, "session_id": "s"},
]
result = claude_provider.event_to_session_result(events, "", 0)
assert result["success"] is False
assert "no result event" in (result["error"] or "")
class TestClaudeAuthDetection:
def test_logged_out_marker(self, claude_provider):
assert (
claude_provider.detect_auth_error(
"Please run 'claude login' first.",
1,
)
is True
)
def test_clean_run_not_auth_error(self, claude_provider):
assert claude_provider.detect_auth_error("", 0) is False
def test_unrelated_stderr_not_auth_error(self, claude_provider):
assert (
claude_provider.detect_auth_error(
"git: pathspec 'x' did not match any files\n",
1,
)
is False
)
class TestClaudeSupports:
def test_supports_full_feature_set(self, claude_provider):
for feature in (
"max_budget",
"max_turns",
"session_id",
"resume",
"mcp_runtime",
"json_cost",
"ide_lockfile",
):
assert claude_provider.supports(feature), feature
def test_does_not_support_sandbox(self, claude_provider):
assert claude_provider.supports("sandbox") is False
# ---------------------------------------------------------------------------
# Codex provider
# ---------------------------------------------------------------------------
@pytest.fixture
def codex_provider():
return create_cli_provider("codex")
class TestCodexArgv:
def test_no_session_no_budget_no_turns(self, codex_provider):
task = CodexTaskSpec(prompt="hello")
argv = codex_provider.interactive_argv(task, defaults={})
assert "--max-turns" not in argv
assert "--max-budget-usd" not in argv
assert "--session-id" not in argv
assert "--resume" not in argv
assert "--allowedTools" not in argv
def test_sandbox_flag(self, codex_provider):
task = CodexTaskSpec(prompt="x", sandbox="read-only")
argv = codex_provider.interactive_argv(task, defaults={})
assert "--sandbox" in argv
assert "read-only" in argv
def test_ask_for_approval_flag(self, codex_provider):
task = CodexTaskSpec(prompt="x", ask_for_approval="on-request")
argv = codex_provider.interactive_argv(task, defaults={})
assert "--ask-for-approval" in argv
assert "on-request" in argv
def test_default_sandbox_workspace_write(self, codex_provider):
task = CodexTaskSpec(prompt="x")
argv = codex_provider.interactive_argv(task, defaults={})
idx = argv.index("--sandbox")
assert argv[idx + 1] == "workspace-write"
def test_system_prompt_prepended(self, codex_provider):
task = CodexTaskSpec(prompt="user thing", system_prompt="be careful")
argv = codex_provider.interactive_argv(task, defaults={})
# Codex has no --system-prompt flag; we prepend with <system> tags
prompt_arg = argv[-1] # last arg is the prompt
assert "<system>" in prompt_arg
assert "be careful" in prompt_arg
assert "user thing" in prompt_arg
def test_wrong_task_type_raises(self, codex_provider):
claude_task = ClaudeTaskSpec(prompt="x")
with pytest.raises(TypeError, match="CodexTaskSpec"):
codex_provider.interactive_argv(claude_task, defaults={})
class TestCodexEventReconstruction:
@pytest.fixture
def fixture_events(self):
# Codex has no public stream-json schema; this is a plausible
# synthetic stream that exercises our best-effort matchers.
return [
{"type": "message", "text": "Working on it..."},
{"type": "assistant", "text": "Final answer is X."},
{"type": "complete", "duration_ms": 5000, "stats": {"turns": 3}},
]
def test_extracts_response_from_final_event(self, codex_provider, fixture_events):
result = codex_provider.event_to_session_result(fixture_events, "", 0)
# Response comes from final event's `text`/`response`/`result`/`content`
# if present; otherwise the last assistant message.
assert "Final answer is X." in result["response"]
def test_cost_always_none_for_codex(self, codex_provider, fixture_events):
result = codex_provider.event_to_session_result(fixture_events, "", 0)
assert result["cost_usd"] is None
def test_canonical_usage_zeros(self, codex_provider, fixture_events):
cu = codex_provider.canonical_usage(fixture_events)
assert cu.input_tokens == 0
assert cu.output_tokens == 0
class TestCodexAuthDetection:
def test_openai_api_key_missing(self, codex_provider):
assert (
codex_provider.detect_auth_error(
"Error: OPENAI_API_KEY not set.\n",
1,
)
is True
)
def test_401_marker(self, codex_provider):
assert codex_provider.detect_auth_error("HTTP 401 Unauthorized\n", 1) is True
class TestCodexSupports:
def test_supports_only_sandbox(self, codex_provider):
assert codex_provider.supports("sandbox") is True
for feature in (
"max_budget",
"max_turns",
"session_id",
"resume",
"json_cost",
"ide_lockfile",
):
assert codex_provider.supports(feature) is False, feature