Skip to content

Open-source personal finance manager. Self-hosted, privacy-first.

15.8 KB
Raw Blame
"""Coverage for the streaming chat + embed paths in
app/agents/providers/{openai,ollama,anthropic}.py. These providers open
httpx.AsyncClient.stream(...) inline, so we replace AsyncClient with a
fake whose stream() yields a pre-baked SSE/JSONL response.
"""
from __future__ import annotations

import json
from contextlib import asynccontextmanager
from typing import Iterable

import pytest

import app.agents.providers.anthropic as anthropic_module
import app.agents.providers.ollama as ollama_module
from app.agents.providers.anthropic import AnthropicProvider
from app.agents.providers.base import (
    ChatMessage,
    LLMAuthError,
    LLMNotSupportedError,
    LLMRateLimitError,
    LLMUnavailableError,
    ToolDefinition,
)
from app.agents.providers.ollama import OllamaProvider
from app.agents.providers.openai import OpenAIProvider
from app.agents.providers.openai_compatible import OpenAICompatibleProvider


# --------------------------------------------------------------------- httpx fake

class _FakeStreamResponse:
    """Stand-in for the response object yielded from `client.stream(...)`."""

    def __init__(self, *, status_code: int, lines: Iterable[str], body: bytes = b""):
        self.status_code = status_code
        self._lines = list(lines)
        self._body = body

    async def aiter_lines(self):
        for ln in self._lines:
            yield ln

    async def aread(self) -> bytes:
        return self._body


class _FakeResponse:
    """Stand-in for a plain (non-streaming) response."""

    def __init__(self, *, status_code: int, json_body=None, text: str = ""):
        self.status_code = status_code
        self._body = json_body
        self.text = text

    def json(self):
        return self._body


class _FakeAsyncClient:
    queue_stream: list[_FakeStreamResponse] = []
    queue_post: list[_FakeResponse] = []
    posted: list[tuple[str, dict]] = []

    def __init__(self, *_, **__):  # accept (timeout=...) kw
        pass

    async def __aenter__(self):
        return self

    async def __aexit__(self, *exc):
        return False

    @asynccontextmanager
    async def stream(self, method, url, *, json=None, headers=None):  # noqa: A002
        type(self).posted.append((url, json or {}))
        yield type(self).queue_stream.pop(0)

    async def post(self, url, *, json=None, headers=None):  # noqa: A002
        type(self).posted.append((url, json or {}))
        return type(self).queue_post.pop(0)


@pytest.fixture(autouse=True)
def _fake_httpx(monkeypatch):
    _FakeAsyncClient.queue_stream = []
    _FakeAsyncClient.queue_post = []
    _FakeAsyncClient.posted = []
    monkeypatch.setattr(ollama_module.httpx, "AsyncClient", _FakeAsyncClient)
    monkeypatch.setattr(anthropic_module.httpx, "AsyncClient", _FakeAsyncClient)
    yield


# --------------------------------------------------------------------- OpenAI-compatible: chat_stream

@pytest.mark.asyncio
@pytest.mark.parametrize("provider_class", [OpenAICompatibleProvider, OpenAIProvider])
@pytest.mark.parametrize("signature_timing", ["first", "late", "absent"])
async def test_openai_tool_signatures_survive_multiple_rounds(provider_class, signature_timing):
    """Keep signatures on their own calls across parallel and sequential tools."""
    def response(deltas):
        return _FakeStreamResponse(status_code=200, lines=[
            "data: " + json.dumps({"choices": [{"delta": delta}]}) for delta in deltas
        ] + ['data: {"choices":[{"delta":{},"finish_reason":"tool_calls"}]}', "data: [DONE]"])

    signature_a = "opaque-A+/=="
    first = {"index": 0, "id": "a", "function": {"name": "lookup", "arguments": '{"city":'}}
    extra_a = {"google": {"thought_signature": signature_a}}
    if signature_timing == "first":
        first["extra_content"] = extra_a
    deltas = [
        {"tool_calls": [first]},
        {"tool_calls": [{"index": 1, "id": "b", "function": {"name": "lookup", "arguments": "{}"}}]},
        {"tool_calls": [{"index": 0, "function": {"arguments": '"Paris"}'}}]},
    ]
    if signature_timing == "late":
        deltas.append({"tool_calls": [{"index": 0, "extra_content": extra_a}]})
    second_call = {"index": 0, "id": "c", "function": {"name": "lookup", "arguments": "{}"}}
    if signature_timing != "absent":
        second_call["extra_content"] = {"google": {"thought_signature": "opaque-B=="}}
    _FakeAsyncClient.queue_stream.extend([
        response(deltas), response([{"tool_calls": [second_call]}]),
        response([{"content": "Done"}]),
    ])
    provider = provider_class(base_url="https://generativelanguage.googleapis.com/v1beta/openai")
    messages = [ChatMessage(role="user", content="Look up cities")]
    for round_index in range(2):
        result = await provider.chat(messages, model="gemini-test")
        if round_index == 0:
            assert [tc.id for tc in result.tool_calls] == ["a", "b"]
            assert result.tool_calls[0].arguments == {"city": "Paris"}
        messages.append(ChatMessage(role="assistant", tool_calls=result.tool_calls))
        messages.extend(ChatMessage(role="tool", tool_call_id=tc.id, content="ok") for tc in result.tool_calls)
    await provider.chat(messages, model="gemini-test")

    assert len(_FakeAsyncClient.posted) == 3
    for url, payload in _FakeAsyncClient.posted:
        assert url == "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions"
        for message in payload["messages"]:
            for call in message.get("tool_calls", []):
                if signature_timing == "absent" or call["id"] == "b":
                    assert set(call) == {"id", "type", "function"}
                else:
                    expected = signature_a if call["id"] == "a" else "opaque-B=="
                    assert call["extra_content"] == {"google": {"thought_signature": expected}}
    final_calls = [tc for m in _FakeAsyncClient.posted[-1][1]["messages"] for tc in m.get("tool_calls", [])]
    assert [tc["id"] for tc in final_calls] == ["a", "b", "c"]


# --------------------------------------------------------------------- Ollama: chat_stream

@pytest.mark.asyncio
async def test_ollama_chat_stream_yields_text_and_finish():
    """Two JSONL deltas + a final done frame should produce text deltas
    plus usage + finish chunks."""
    _FakeAsyncClient.queue_stream.append(_FakeStreamResponse(status_code=200, lines=[
        json.dumps({"message": {"content": "Hel"}}),
        json.dumps({"message": {"content": "lo"}}),
        json.dumps({
            "message": {"content": ""},
            "done": True, "done_reason": "stop",
            "prompt_eval_count": 7, "eval_count": 4,
        }),
        "",  # blank line — should be skipped
    ]))

    provider = OllamaProvider(base_url="http://ollama:11434")
    chunks = []
    async for chunk in provider.chat_stream(
        [ChatMessage(role="user", content="hi")], model="llama3", temperature=0.1, max_tokens=64,
    ):
        chunks.append(chunk)

    text = "".join(c.text or "" for c in chunks if c.type == "text_delta")
    assert text == "Hello"
    assert any(c.type == "usage" and c.usage and c.usage.input_tokens == 7 for c in chunks)
    assert any(c.type == "finish" and c.finish_reason == "stop" for c in chunks)

    # Verify the request payload picked up our temperature + num_predict.
    _, payload = _FakeAsyncClient.posted[-1]
    assert payload["options"]["temperature"] == 0.1
    assert payload["options"]["num_predict"] == 64


@pytest.mark.asyncio
async def test_ollama_chat_stream_emits_tool_call_events():
    """A streaming tool_calls payload should produce start/args/end chunks."""
    _FakeAsyncClient.queue_stream.append(_FakeStreamResponse(status_code=200, lines=[
        json.dumps({
            "message": {
                "content": "",
                "tool_calls": [
                    {"function": {"name": "list_accounts", "arguments": {"limit": 5}}},
                ],
            },
        }),
        json.dumps({"message": {"content": ""}, "done": True, "done_reason": "tool_calls"}),
    ]))

    provider = OllamaProvider()
    types = []
    async for c in provider.chat_stream([ChatMessage(role="user", content="x")], model="llama3"):
        types.append(c.type)
    assert "tool_call_start" in types
    assert "tool_call_args_delta" in types
    assert "tool_call_end" in types


@pytest.mark.asyncio
async def test_ollama_chat_stream_raises_on_http_error():
    _FakeAsyncClient.queue_stream.append(_FakeStreamResponse(
        status_code=503, lines=[], body=b"unavailable",
    ))
    provider = OllamaProvider()
    with pytest.raises(LLMUnavailableError):
        async for _ in provider.chat_stream(
            [ChatMessage(role="user", content="x")], model="llama3",
        ):
            pass


@pytest.mark.asyncio
async def test_ollama_chat_stream_includes_tools_in_payload():
    """When tools are passed, they must show up in the request body."""
    _FakeAsyncClient.queue_stream.append(_FakeStreamResponse(status_code=200, lines=[
        json.dumps({"message": {"content": ""}, "done": True, "done_reason": "stop"}),
    ]))
    provider = OllamaProvider()
    tools = [ToolDefinition(name="t", description="d", parameters={"type": "object"})]
    async for _ in provider.chat_stream(
        [ChatMessage(role="user", content="x")], model="llama3", tools=tools,
    ):
        pass
    _, payload = _FakeAsyncClient.posted[-1]
    assert payload["tools"][0]["function"]["name"] == "t"


# --------------------------------------------------------------------- Ollama: embed

@pytest.mark.asyncio
async def test_ollama_embed_returns_vectors_per_input():
    _FakeAsyncClient.queue_post.append(_FakeResponse(status_code=200, json_body={
        "embeddings": [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]],
    }))
    provider = OllamaProvider()
    out = await provider.embed(["a", "b"], model="nomic-embed-text")
    assert out == [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]


@pytest.mark.asyncio
async def test_ollama_embed_handles_missing_embeddings_array():
    """If the server returns an unexpected shape, we should yield [] rather than crash."""
    _FakeAsyncClient.queue_post.append(_FakeResponse(status_code=200, json_body={"unexpected": True}))
    provider = OllamaProvider()
    out = await provider.embed(["a"], model="nomic-embed-text")
    assert out == []


@pytest.mark.asyncio
async def test_ollama_embed_raises_on_http_error():
    _FakeAsyncClient.queue_post.append(_FakeResponse(status_code=500, text="boom"))
    provider = OllamaProvider()
    with pytest.raises(LLMUnavailableError):
        await provider.embed(["a"], model="nomic-embed-text")


# --------------------------------------------------------------------- Anthropic: chat_stream

@pytest.mark.asyncio
async def test_anthropic_chat_stream_parses_text_block_sequence():
    sse = [
        f"data: {json.dumps({'type': 'message_start', 'message': {'usage': {'input_tokens': 5, 'output_tokens': 0}}})}",
        f"data: {json.dumps({'type': 'content_block_start', 'index': 0, 'content_block': {'type': 'text'}})}",
        f"data: {json.dumps({'type': 'content_block_delta', 'index': 0, 'delta': {'type': 'text_delta', 'text': 'Hi'}})}",
        f"data: {json.dumps({'type': 'content_block_delta', 'index': 0, 'delta': {'type': 'text_delta', 'text': '!'}})}",
        f"data: {json.dumps({'type': 'content_block_stop', 'index': 0})}",
        f"data: {json.dumps({'type': 'message_delta', 'delta': {'stop_reason': 'end_turn'}, 'usage': {'output_tokens': 2}})}",
        "",  # blank — skipped
        "non-sse line — should be ignored",  # not 'data:' prefixed
    ]
    _FakeAsyncClient.queue_stream.append(_FakeStreamResponse(status_code=200, lines=sse))

    provider = AnthropicProvider(api_key="sk-test")
    chunks = []
    async for c in provider.chat_stream([ChatMessage(role="user", content="hi")], model="claude-x"):
        chunks.append(c)

    text = "".join(c.text or "" for c in chunks if c.type == "text_delta")
    assert text == "Hi!"
    assert any(c.type == "usage" and c.usage is not None and c.usage.output_tokens == 2 for c in chunks)
    assert any(c.type == "finish" and c.finish_reason == "end_turn" for c in chunks)


@pytest.mark.asyncio
async def test_anthropic_chat_stream_parses_tool_use_block():
    # Build SSE lines via explicit json.dumps payloads — avoids backslash
    # escapes inside f-strings, which Python 3.11 doesn't allow.
    events = [
        {"type": "content_block_start", "index": 0,
         "content_block": {"type": "tool_use", "id": "toolu_1", "name": "list_accounts"}},
        {"type": "content_block_delta", "index": 0,
         "delta": {"type": "input_json_delta", "partial_json": '{"limit":'}},
        {"type": "content_block_delta", "index": 0,
         "delta": {"type": "input_json_delta", "partial_json": "5}"}},
        {"type": "content_block_stop", "index": 0},
        {"type": "message_delta",
         "delta": {"stop_reason": "tool_use"}, "usage": {"output_tokens": 1}},
    ]
    sse = ["data: " + json.dumps(e) for e in events]
    _FakeAsyncClient.queue_stream.append(_FakeStreamResponse(status_code=200, lines=sse))

    provider = AnthropicProvider(api_key="sk-test")
    chunks = []
    async for c in provider.chat_stream([ChatMessage(role="user", content="x")], model="claude-x"):
        chunks.append(c)

    starts = [c for c in chunks if c.type == "tool_call_start"]
    args = [c for c in chunks if c.type == "tool_call_args_delta"]
    ends = [c for c in chunks if c.type == "tool_call_end"]
    assert starts and starts[0].tool_name == "list_accounts"
    assert "".join(c.args_delta or "" for c in args) == '{"limit":5}'
    assert ends and ends[0].tool_call_id == "toolu_1"


@pytest.mark.asyncio
async def test_anthropic_includes_system_and_tools_in_payload():
    sse = [f"data: {json.dumps({'type': 'message_delta', 'delta': {'stop_reason': 'end_turn'}, 'usage': {}})}"]
    _FakeAsyncClient.queue_stream.append(_FakeStreamResponse(status_code=200, lines=sse))
    provider = AnthropicProvider(api_key="sk-test")
    tools = [ToolDefinition(name="t", description="d", parameters={"type": "object"})]
    msgs = [
        ChatMessage(role="system", content="part1"),
        ChatMessage(role="system", content="part2"),
        ChatMessage(role="user", content="hi"),
    ]
    async for _ in provider.chat_stream(msgs, model="claude-x", tools=tools, max_tokens=200):
        pass
    _, payload = _FakeAsyncClient.posted[-1]
    assert payload["system"] == "part1\n\npart2"
    assert payload["max_tokens"] == 200
    assert payload["tools"][0]["name"] == "t"


@pytest.mark.asyncio
async def test_anthropic_payload_omits_temperature():
    sse = [f"data: {json.dumps({'type': 'message_delta', 'delta': {'stop_reason': 'end_turn'}, 'usage': {}})}"]
    _FakeAsyncClient.queue_stream.append(_FakeStreamResponse(status_code=200, lines=sse))
    provider = AnthropicProvider(api_key="sk-test")
    msgs = [ChatMessage(role="user", content="hi")]
    async for _ in provider.chat_stream(msgs, model="claude-x", temperature=0.2):
        pass
    _, payload = _FakeAsyncClient.posted[-1]
    assert "temperature" not in payload


# --------------------------------------------------------------------- Anthropic: HTTP errors → typed exceptions

@pytest.mark.asyncio
@pytest.mark.parametrize("status,exc_type", [
    (401, LLMAuthError),
    (403, LLMAuthError),
    (429, LLMRateLimitError),
    (500, LLMUnavailableError),
    (502, LLMUnavailableError),
    (400, LLMUnavailableError),
])
async def test_anthropic_http_errors_map_to_typed_exceptions(status, exc_type):
    _FakeAsyncClient.queue_stream.append(_FakeStreamResponse(
        status_code=status, lines=[], body=b'{"error":{"message":"x"}}',
    ))
    provider = AnthropicProvider(api_key="sk-test")
    with pytest.raises(exc_type):
        async for _ in provider.chat_stream([ChatMessage(role="user", content="x")], model="claude-x"):
            pass


@pytest.mark.asyncio
async def test_anthropic_embed_raises_not_supported():
    provider = AnthropicProvider(api_key="sk-test")
    with pytest.raises(LLMNotSupportedError):
        await provider.embed(["a"], model="x")