Skip to content

armgit / ECC

Public Imported from GitHub
0

The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.

18.7 KB
Raw Blame
import asyncio
import json
import urllib.request
from collections.abc import Callable
from io import BytesIO
from types import SimpleNamespace
from typing import Any, cast

import pytest

from llm.core.interface import LLMProvider
from llm.core.types import LLMInput, Message, Role, ToolCall, ToolDefinition
from llm.providers.astraflow import AstraflowProvider
from llm.providers.atlas import AtlasProvider
from llm.providers.claude import ClaudeProvider
from llm.providers.constants import EMPTY_FILTERED_RESPONSE_ERROR
from llm.providers.ollama import OllamaProvider
from llm.providers.openai import OpenAIProvider
from llm.tools import ReActAgent, ToolExecutor, ToolRegistry


def _tool() -> ToolDefinition:
    return ToolDefinition(
        name="search",
        description="Search",
        parameters={"type": "object", "properties": {"query": {"type": "string"}}},
    )


class _OpenAICompletions:
    def __init__(self, response: SimpleNamespace | None = None) -> None:
        self.params = None
        self.response = response

    def create(self, **params):
        self.params = params
        if self.response:
            return self.response
        return _openai_response(model=params["model"])


class _OpenAIClient:
    def __init__(self, response: SimpleNamespace | None = None) -> None:
        self.completions = _OpenAICompletions(response=response)
        self.chat = SimpleNamespace(completions=self.completions)


class _AnthropicMessages:
    def __init__(self) -> None:
        self.params = None

    def create(self, **params):
        self.params = params
        return SimpleNamespace(
            content=[SimpleNamespace(text="ok", type="text")],
            model=params["model"],
            usage=SimpleNamespace(input_tokens=1, output_tokens=1),
            stop_reason="end_turn",
        )


class _AnthropicClient:
    def __init__(self) -> None:
        self.messages = _AnthropicMessages()
        self.api_key = "test"


def _openai_response(**overrides) -> SimpleNamespace:
    defaults = {
        "choices": [SimpleNamespace(message=SimpleNamespace(content="ok", tool_calls=None), finish_reason="stop")],
        "model": "gpt-4o-mini",
        "usage": SimpleNamespace(prompt_tokens=1, completion_tokens=1, total_tokens=2),
    }
    defaults.update(overrides)
    return SimpleNamespace(**defaults)


@pytest.mark.unit
@pytest.mark.parametrize(
    ("default_model", "request_model", "expected_model"),
    [
        (None, None, "gpt-4o-mini"),
        (None, "gpt-4o", "gpt-4o"),
        ("gpt-4o", None, "gpt-4o"),
        ("gpt-4o", "gpt-4-turbo", "gpt-4-turbo"),
    ],
)
def test_openai_generate_respects_configured_default_and_request_model(
    default_model: str | None, request_model: str | None, expected_model: str
) -> None:
    provider_options = {"api_key": "test"}
    if default_model is not None:
        provider_options["default_model"] = default_model
    provider = OpenAIProvider(**provider_options)
    client = _OpenAIClient()
    provider.client = client
    expected_default = default_model or "gpt-4o-mini"

    assert provider.get_default_model() == expected_default

    provider.generate(
        LLMInput(messages=[Message(role=Role.USER, content="hi")], model=request_model)
    )

    assert client.completions.params is not None
    assert client.completions.params["model"] == expected_model
    assert provider.get_default_model() == expected_default


def test_openai_provider_serializes_tools_for_chat_completions():
    provider = OpenAIProvider(api_key="test")
    client = _OpenAIClient()
    provider.client = client

    provider.generate(LLMInput(messages=[Message(role=Role.USER, content="hi")], tools=[_tool()]))

    assert client.completions.params["tools"] == [
        {
            "type": "function",
            "function": {
                "name": "search",
                "description": "Search",
                "parameters": {"type": "object", "properties": {"query": {"type": "string"}}},
                "strict": True,
            },
        }
    ]


@pytest.mark.unit
@pytest.mark.parametrize(
    "provider_type",
    [OpenAIProvider, AstraflowProvider, AtlasProvider],
)
def test_openai_compatible_provider_serializes_assistant_tool_calls(
    provider_type: Callable[..., LLMProvider],
) -> None:
    provider = cast(Any, provider_type(api_key="test"))
    client = _OpenAIClient()
    provider.client = client

    provider.generate(
        LLMInput(
            messages=[
                Message(role=Role.USER, content="Find ECC"),
                Message(
                    role=Role.ASSISTANT,
                    content="",
                    tool_calls=[
                        ToolCall(
                            id="call_1",
                            name="search",
                            arguments={"query": "ecc"},
                        )
                    ],
                ),
                Message(
                    role=Role.TOOL,
                    content="result",
                    tool_call_id="call_1",
                ),
            ]
        )
    )

    assert client.completions.params is not None
    assert client.completions.params["messages"] == [
        {"role": "user", "content": "Find ECC"},
        {
            "role": "assistant",
            "content": "",
            "tool_calls": [
                {
                    "id": "call_1",
                    "type": "function",
                    "function": {
                        "name": "search",
                        "arguments": '{"query": "ecc"}',
                    },
                },
            ],
        },
        {"role": "tool", "content": "result", "tool_call_id": "call_1"},
    ]

    submitted_params = client.completions.params
    with pytest.raises(ValueError, match="tool messages require a tool_call_id"):
        provider.generate(
            LLMInput(messages=[Message(role=Role.TOOL, content="unlinked result")])
        )
    assert client.completions.params is submitted_params

    for invalid_arguments in ({"score": float("nan")}, {"value": object()}):
        with pytest.raises(
            ValueError,
            match="tool call arguments must be valid JSON",
        ):
            provider.generate(
                LLMInput(
                    messages=[
                        Message(
                            role=Role.ASSISTANT,
                            content="",
                            tool_calls=[
                                ToolCall(
                                    id="call_invalid",
                                    name="score",
                                    arguments=invalid_arguments,
                                )
                            ],
                        )
                    ]
                )
            )
        assert client.completions.params is submitted_params


def test_openai_provider_can_be_constructed_without_credentials(monkeypatch):
    monkeypatch.delenv("OPENAI_API_KEY", raising=False)

    provider = OpenAIProvider()

    assert provider.validate_config() is False


def test_openai_provider_rejects_empty_or_filtered_responses():
    provider = OpenAIProvider(api_key="test")

    for response in [
        _openai_response(choices=[]),
        _openai_response(choices=[SimpleNamespace(message=None, finish_reason="content_filter")]),
    ]:
        provider.client = _OpenAIClient(response=response)
        with pytest.raises(ValueError, match=EMPTY_FILTERED_RESPONSE_ERROR):
            provider.generate(LLMInput(messages=[Message(role=Role.USER, content="hi")]))


def test_openai_provider_allows_missing_usage():
    provider = OpenAIProvider(api_key="test")
    provider.client = _OpenAIClient(response=_openai_response(usage=None))

    output = provider.generate(LLMInput(messages=[Message(role=Role.USER, content="hi")]))

    assert output.content == "ok"
    assert output.usage is None


@pytest.mark.parametrize(
    ("max_tokens", "temperature", "expected_options"),
    [
        (128, 1.0, {"num_predict": 128}),
        (128, 0.2, {"temperature": 0.2, "num_predict": 128}),
        (128, 0.0, {"temperature": 0.0, "num_predict": 128}),
        (0, 1.0, {"num_predict": 0}),
        (None, 1.0, {}),
        (None, 0.2, {"temperature": 0.2}),
    ],
)
def test_ollama_provider_serializes_generation_options(
    monkeypatch, max_tokens, temperature, expected_options
):
    requests = []

    def fake_urlopen(request, timeout):
        requests.append((request, timeout))
        return BytesIO(b'{"message": {"content": "ok"}, "done_reason": "stop"}')

    monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
    provider = OllamaProvider(base_url="http://localhost:11434", default_model="llama3.2")

    output = provider.generate(
        LLMInput(
            messages=[Message(role=Role.USER, content="hi")],
            max_tokens=max_tokens,
            temperature=temperature,
        )
    )

    assert len(requests) == 1
    request, timeout = requests[0]
    expected_payload = {
        "model": "llama3.2",
        "messages": [{"role": "user", "content": "hi"}],
        "stream": False,
    }
    if expected_options:
        expected_payload["options"] = expected_options
    assert json.loads(request.data) == expected_payload
    assert request.full_url == "http://localhost:11434/api/chat"
    assert request.get_method() == "POST"
    assert request.get_header("Content-type") == "application/json"
    assert timeout == 60
    assert output.content == "ok"
    assert output.model == "llama3.2"
    assert output.stop_reason == "stop"


class _FakeOllama:
    """Serves /api/show and replays queued /api/chat replies, recording requests."""

    def __init__(
        self,
        monkeypatch: pytest.MonkeyPatch,
        replies: list[dict[str, Any]],
        capabilities: list[str] | None = None,
    ) -> None:
        self.replies = list(replies)
        self.capabilities = capabilities
        self.chats: list[dict[str, Any]] = []
        self.shows: list[str] = []
        monkeypatch.setattr(urllib.request, "urlopen", self.urlopen)

    def urlopen(self, request: urllib.request.Request, timeout: float) -> BytesIO:
        body = json.loads(request.data)
        if request.full_url.endswith("/api/show"):
            self.shows.append(body["model"])
            if self.capabilities is None:
                raise OSError("connection refused")
            return BytesIO(json.dumps({"capabilities": self.capabilities}).encode())
        self.chats.append(body)
        return BytesIO(json.dumps(self.replies.pop(0)).encode())


def _ollama_tool_calls(*calls: tuple[str, dict[str, Any]]) -> dict[str, Any]:
    return {
        "message": {
            "content": "",
            "tool_calls": [{"function": {"name": name, "arguments": args}} for name, args in calls],
        },
        "done_reason": "stop",
    }


def _ollama_answer(text: str) -> dict[str, Any]:
    return {"message": {"content": text}, "done_reason": "stop"}


def _lookup_tool() -> ToolDefinition:
    return ToolDefinition(
        name="lookup",
        description="Look up a city",
        parameters={"type": "object", "properties": {"city": {"type": "string"}}},
    )


def test_ollama_provider_serializes_tools_and_parses_tool_calls(monkeypatch: pytest.MonkeyPatch) -> None:
    server = _FakeOllama(
        monkeypatch, [_ollama_tool_calls(("search", {"query": "ollama"}))], capabilities=["completion", "tools"]
    )
    provider = OllamaProvider(base_url="http://localhost:11434", default_model="qwen3")

    output = provider.generate(LLMInput(messages=[Message(role=Role.USER, content="hi")], tools=[_tool()]))

    assert server.shows == ["qwen3"]
    assert server.chats[0]["tools"] == [
        {
            "type": "function",
            "function": {
                "name": "search",
                "description": "Search",
                "parameters": {"type": "object", "properties": {"query": {"type": "string"}}},
                "strict": True,
            },
        }
    ]
    assert output.has_tool_calls
    assert output.tool_calls[0].name == "search"
    assert output.tool_calls[0].arguments == {"query": "ollama"}
    assert output.tool_calls[0].id


@pytest.mark.parametrize(
    ("model", "capabilities", "expect_tools", "expect_shows"),
    [
        ("llama3.2", None, True, []),
        ("codellama", None, False, []),
        ("qwen3", ["completion", "tools"], True, ["qwen3"]),
        ("gemma3", ["completion", "vision"], False, ["gemma3"]),
        ("unknown", None, False, ["unknown", "unknown"]),
    ],
)
def test_ollama_provider_sends_tools_only_to_models_that_declare_them(
    monkeypatch: pytest.MonkeyPatch,
    model: str,
    capabilities: list[str] | None,
    expect_tools: bool,
    expect_shows: list[str],
) -> None:
    server = _FakeOllama(monkeypatch, [_ollama_answer("ok"), _ollama_answer("ok")], capabilities=capabilities)
    provider = OllamaProvider(default_model=model)
    request = LLMInput(messages=[Message(role=Role.USER, content="hi")], tools=[_tool()])

    provider.generate(request)
    provider.generate(request)

    assert all(("tools" in chat) is expect_tools for chat in server.chats)
    # An answered capability lookup is cached per model; a failed one is retried.
    assert server.shows == expect_shows


def test_ollama_provider_retries_capability_lookup_after_a_failure(monkeypatch: pytest.MonkeyPatch) -> None:
    server = _FakeOllama(monkeypatch, [_ollama_answer("ok"), _ollama_answer("ok"), _ollama_answer("ok")])
    provider = OllamaProvider(default_model="qwen3")
    request = LLMInput(messages=[Message(role=Role.USER, content="hi")], tools=[_tool()])

    provider.generate(request)
    server.capabilities = ["completion", "tools"]
    provider.generate(request)
    provider.generate(request)

    assert ["tools" in chat for chat in server.chats] == [False, True, True]
    assert server.shows == ["qwen3", "qwen3"]


def test_ollama_react_agent_correlates_multiple_calls_without_ids(monkeypatch: pytest.MonkeyPatch) -> None:
    server = _FakeOllama(
        monkeypatch,
        [
            _ollama_tool_calls(("lookup", {"city": "Hefei"}), ("search", {"query": "weather"})),
            _ollama_answer("Hefei is sunny."),
        ],
        capabilities=["completion", "tools"],
    )
    registry = ToolRegistry()
    registry.register(_lookup_tool(), lambda city: f"city={city}")
    registry.register(_tool(), lambda query: f"query={query}")
    agent = ReActAgent(OllamaProvider(default_model="qwen3"), ToolExecutor(registry))

    output = asyncio.run(
        agent.run(
            LLMInput(
                messages=[Message(role=Role.USER, content="Weather in Hefei?")],
                tools=[_lookup_tool(), _tool()],
            )
        )
    )

    assert output.content == "Hefei is sunny."
    follow_up = server.chats[1]
    assert "tools" in follow_up
    assistant, first, second = follow_up["messages"][1:]
    ids = [call["id"] for call in assistant["tool_calls"]]
    assert all(ids) and len(set(ids)) == 2
    assert [call["function"]["name"] for call in assistant["tool_calls"]] == ["lookup", "search"]
    assert first == {"role": "tool", "content": "city=Hefei", "tool_call_id": ids[0], "tool_name": "lookup"}
    assert second == {"role": "tool", "content": "query=weather", "tool_call_id": ids[1], "tool_name": "search"}


def test_ollama_react_agent_reports_unknown_tool_under_its_own_name(monkeypatch: pytest.MonkeyPatch) -> None:
    server = _FakeOllama(
        monkeypatch,
        [_ollama_tool_calls(("missing", {}), ("search", {"query": "x"})), _ollama_answer("done")],
        capabilities=["completion", "tools"],
    )
    registry = ToolRegistry()
    registry.register(_tool(), lambda query: f"query={query}")
    agent = ReActAgent(OllamaProvider(default_model="qwen3"), ToolExecutor(registry))

    asyncio.run(agent.run(LLMInput(messages=[Message(role=Role.USER, content="go")], tools=[_tool()])))

    results = server.chats[1]["messages"][2:]
    assert [(r["tool_name"], r["content"]) for r in results] == [
        ("missing", "Error: Tool 'missing' not found"),
        ("search", "query=x"),
    ]


def test_ollama_react_agent_without_tool_support_answers_directly(monkeypatch: pytest.MonkeyPatch) -> None:
    server = _FakeOllama(monkeypatch, [_ollama_answer("I cannot call tools.")])
    agent = ReActAgent(OllamaProvider(default_model="codellama"), ToolExecutor())

    output = asyncio.run(agent.run(LLMInput(messages=[Message(role=Role.USER, content="hi")], tools=[_tool()])))

    assert output.content == "I cannot call tools."
    assert "tools" not in server.chats[0]
    assert server.shows == []


def test_claude_provider_serializes_tools_for_messages_api():
    provider = ClaudeProvider(api_key="test")
    client = _AnthropicClient()
    provider.client = client

    provider.generate(LLMInput(messages=[Message(role=Role.USER, content="hi")], tools=[_tool()]))

    assert client.messages.params["tools"] == [
        {
            "name": "search",
            "description": "Search",
            "input_schema": {"type": "object", "properties": {"query": {"type": "string"}}},
        }
    ]


@pytest.mark.parametrize('capabilities', ['tools', {'tools': True}, ['tools', 1]])
def test_ollama_malformed_capability_declaration_is_not_authority(monkeypatch, capabilities):
    server = _FakeOllama(monkeypatch, [_ollama_answer('ok'), _ollama_answer('ok')], capabilities=capabilities)
    provider = OllamaProvider(default_model='custom')
    request = LLMInput(messages=[Message(role=Role.USER, content='hi')], tools=[_tool()])
    provider.generate(request)
    server.capabilities = ['completion', 'tools']
    provider.generate(request)
    assert ['tools' in chat for chat in server.chats] == [False, True]
    assert server.shows == ['custom', 'custom']


def test_ollama_mixed_history_does_not_infer_tool_names_from_empty_ids(monkeypatch: pytest.MonkeyPatch) -> None:
    server = _FakeOllama(monkeypatch, [_ollama_answer("ok")])
    messages = [
        Message(role=Role.ASSISTANT, content="", tool_calls=[ToolCall(id="", name="first", arguments={}), ToolCall(id="", name="last", arguments={}), ToolCall(id="real-id", name="identified", arguments={})]),
        Message(role=Role.TOOL, content="unnamed missing"),
        Message(role=Role.TOOL, content="unnamed empty", tool_call_id=""),
        Message(role=Role.TOOL, content="explicit empty", tool_call_id="", name="explicit"),
        Message(role=Role.TOOL, content="identified result", tool_call_id="real-id"),
        Message(role=Role.TOOL, content="explicit override", tool_call_id="real-id", name="override"),
    ]
    OllamaProvider().generate(LLMInput(messages=messages))
    results = server.chats[0]["messages"][1:]
    assert "tool_name" not in results[0]
    assert "tool_name" not in results[1]
    assert [item["tool_name"] for item in results[2:]] == ["explicit", "identified", "override"]
    assert results[3]["tool_call_id"] == "real-id"