Skip to content

Personal AI, On Personal Devices

Python
18.2 KB
Raw Blame
"""Extended integration tests for new components (Phase 6+)."""

from __future__ import annotations

from unittest.mock import MagicMock

import pytest

from openjarvis.agents._stubs import AgentContext, AgentResult
from openjarvis.core.events import EventBus, EventType
from openjarvis.core.registry import AgentRegistry, ToolRegistry
from openjarvis.core.types import (
    Conversation,
    Message,
    Role,
)

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def _register_all():
    """Ensure agents and tools are registered."""
    from openjarvis.agents.native_openhands import NativeOpenHandsAgent
    from openjarvis.agents.native_react import NativeReActAgent
    from openjarvis.tools.calculator import CalculatorTool
    from openjarvis.tools.think import ThinkTool

    for key, cls in [
        ("native_react", NativeReActAgent),
        ("native_openhands", NativeOpenHandsAgent),
    ]:
        if not AgentRegistry.contains(key):
            AgentRegistry.register_value(key, cls)

    for key, cls in [
        ("calculator", CalculatorTool),
        ("think", ThinkTool),
    ]:
        if not ToolRegistry.contains(key):
            ToolRegistry.register_value(key, cls)


def _make_engine(responses):
    """Create a mock engine returning a sequence of responses."""
    engine = MagicMock()
    engine.engine_id = "mock"
    engine.health.return_value = True
    engine.list_models.return_value = ["test-model"]
    if isinstance(responses, list):
        engine.generate.side_effect = responses
    else:
        engine.generate.return_value = responses
    return engine


def _simple_response(content, model="test-model"):
    return {
        "content": content,
        "usage": {
            "prompt_tokens": 10,
            "completion_tokens": 5,
            "total_tokens": 15,
        },
        "model": model,
        "finish_reason": "stop",
    }


# ---------------------------------------------------------------------------
# ReAct pipeline integration
# ---------------------------------------------------------------------------


class TestReActPipeline:
    """End-to-end: ReAct agent with calculator tool."""

    def test_react_with_calculator_e2e(self):
        _register_all()
        from openjarvis.agents.native_react import NativeReActAgent
        from openjarvis.tools.calculator import CalculatorTool

        responses = [
            _simple_response(
                "Thought: I need to calculate 2+2.\n"
                "Action: calculator\n"
                'Action Input: {"expression":"2+2"}'
            ),
            _simple_response("Thought: The result is 4.\nFinal Answer: 2+2 equals 4."),
        ]
        engine = _make_engine(responses)
        bus = EventBus(record_history=True)
        agent = NativeReActAgent(
            engine,
            "test-model",
            tools=[CalculatorTool()],
            bus=bus,
        )
        result = agent.run("What is 2+2?")

        assert isinstance(result, AgentResult)
        assert "4" in result.content
        assert result.turns == 2
        assert len(result.tool_results) == 1
        assert result.tool_results[0].content == "4.0"

    def test_react_with_think_tool(self):
        _register_all()
        from openjarvis.agents.native_react import NativeReActAgent
        from openjarvis.tools.think import ThinkTool

        responses = [
            _simple_response(
                "Thought: Let me reason about this.\n"
                "Action: think\n"
                'Action Input: {"thought":"Step 1: analyze"}'
            ),
            _simple_response(
                "Thought: I have my analysis.\nFinal Answer: The answer is clear."
            ),
        ]
        engine = _make_engine(responses)
        agent = NativeReActAgent(
            engine,
            "test-model",
            tools=[ThinkTool()],
        )
        result = agent.run("Analyze this.")
        assert result.turns == 2
        assert result.tool_results[0].success is True

    def test_react_direct_answer(self):
        """ReAct returns immediately when no tool use is needed."""
        _register_all()
        from openjarvis.agents.native_react import NativeReActAgent

        engine = _make_engine(
            _simple_response("Thought: This is simple.\nFinal Answer: Hello!")
        )
        agent = NativeReActAgent(engine, "test-model")
        result = agent.run("Say hello")
        assert result.content == "Hello!"
        assert result.turns == 1

    def test_react_event_chain(self):
        """Verify agent-level event chain through ReAct run.

        INFERENCE_START/END are now published by InstrumentedEngine,
        not by agents directly.
        """
        _register_all()
        from openjarvis.agents.native_react import NativeReActAgent

        engine = _make_engine(_simple_response("Thought: done.\nFinal Answer: ok"))
        bus = EventBus(record_history=True)
        agent = NativeReActAgent(
            engine,
            "test-model",
            bus=bus,
        )
        agent.run("Test")

        types = [e.event_type for e in bus.history]
        assert EventType.AGENT_TURN_START in types
        assert EventType.AGENT_TURN_END in types


# ---------------------------------------------------------------------------
# OpenHands pipeline integration
# ---------------------------------------------------------------------------


class TestOpenHandsPipeline:
    """End-to-end: OpenHands agent with code execution."""

    def test_openhands_code_execution_e2e(self):
        _register_all()
        from openjarvis.agents.native_openhands import NativeOpenHandsAgent
        from openjarvis.tools.code_interpreter import (
            CodeInterpreterTool,
        )

        if not ToolRegistry.contains("code_interpreter"):
            ToolRegistry.register_value(
                "code_interpreter",
                CodeInterpreterTool,
            )

        responses = [
            _simple_response("I'll calculate this:\n```python\nprint(2 + 2)\n```"),
            _simple_response("The result is 4."),
        ]
        engine = _make_engine(responses)
        agent = NativeOpenHandsAgent(
            engine,
            "test-model",
            tools=[CodeInterpreterTool()],
        )
        result = agent.run("What is 2+2?")

        assert isinstance(result, AgentResult)
        assert result.turns == 2
        assert len(result.tool_results) == 1
        # The code_interpreter actually runs print(2+2)
        assert "4" in result.tool_results[0].content

    def test_openhands_direct_answer(self):
        """OpenHands returns directly when no code is needed."""
        _register_all()
        from openjarvis.agents.native_openhands import NativeOpenHandsAgent

        engine = _make_engine(_simple_response("Hello! How can I help?"))
        agent = NativeOpenHandsAgent(engine, "test-model")
        result = agent.run("Say hello")
        assert result.content == "Hello! How can I help?"
        assert result.turns == 1

    def test_openhands_event_chain(self):
        """Verify event chain through OpenHands run."""
        _register_all()
        from openjarvis.agents.native_openhands import NativeOpenHandsAgent

        engine = _make_engine(_simple_response("Direct answer."))
        bus = EventBus(record_history=True)
        agent = NativeOpenHandsAgent(
            engine,
            "test-model",
            bus=bus,
        )
        agent.run("Test")

        types = [e.event_type for e in bus.history]
        assert EventType.AGENT_TURN_START in types
        assert EventType.AGENT_TURN_END in types


# ---------------------------------------------------------------------------
# MCP integration
# ---------------------------------------------------------------------------


class TestMCPIntegration:
    """MCP server + client with real tools."""

    def test_mcp_server_with_all_tools(self):
        from openjarvis.mcp.client import MCPClient
        from openjarvis.mcp.server import MCPServer
        from openjarvis.mcp.transport import InProcessTransport
        from openjarvis.tools.calculator import CalculatorTool
        from openjarvis.tools.think import ThinkTool

        tools = [CalculatorTool(), ThinkTool()]
        server = MCPServer(tools)
        transport = InProcessTransport(server)
        client = MCPClient(transport)

        # Initialize
        caps = client.initialize()
        assert "serverInfo" in caps

        # List tools
        specs = client.list_tools()
        names = [s.name for s in specs]
        assert "calculator" in names
        assert "think" in names

        # Call calculator
        result = client.call_tool(
            "calculator",
            {"expression": "10*5"},
        )
        assert result["content"][0]["text"] == "50.0"
        assert result["isError"] is False

        # Call think
        result = client.call_tool(
            "think",
            {"thought": "reasoning step"},
        )
        assert result["content"][0]["text"] == "reasoning step"

        client.close()

    def test_mcp_unknown_tool_error(self):
        from openjarvis.mcp.client import MCPClient
        from openjarvis.mcp.protocol import MCPError
        from openjarvis.mcp.server import MCPServer
        from openjarvis.mcp.transport import InProcessTransport
        from openjarvis.tools.calculator import CalculatorTool

        server = MCPServer([CalculatorTool()])
        client = MCPClient(InProcessTransport(server))
        client.initialize()

        with pytest.raises(MCPError):
            client.call_tool("nonexistent", {})

        client.close()

    def test_mcp_roundtrip_lifecycle(self):
        """Full lifecycle: init -> list -> call -> result."""
        from openjarvis.mcp.client import MCPClient
        from openjarvis.mcp.server import MCPServer
        from openjarvis.mcp.transport import InProcessTransport
        from openjarvis.tools.calculator import CalculatorTool

        server = MCPServer([CalculatorTool()])
        client = MCPClient(InProcessTransport(server))

        # 1. Initialize
        caps = client.initialize()
        assert caps["protocolVersion"] == "2025-11-25"

        # 2. Discover tools
        tools = client.list_tools()
        assert len(tools) == 1
        assert tools[0].name == "calculator"

        # 3. Call tool
        result = client.call_tool(
            "calculator",
            {"expression": "7+3"},
        )
        assert result["content"][0]["text"] == "10.0"

        # 4. Close
        client.close()


# ---------------------------------------------------------------------------
# Cross-engine consistency
# ---------------------------------------------------------------------------


class TestCrossEngineConsistency:
    """Same query through different mock engine configs."""

    def test_same_query_same_format(self):
        """All engines return the same result dict shape."""
        _register_all()
        from openjarvis.agents.native_react import NativeReActAgent

        for engine_name in ["vllm", "ollama", "mock"]:
            engine = _make_engine(
                _simple_response(
                    "Thought: ok.\nFinal Answer: Result",
                    model="test-model",
                )
            )
            engine.engine_id = engine_name
            agent = NativeReActAgent(engine, "test-model")
            result = agent.run("Test query")
            assert isinstance(result, AgentResult)
            assert result.content == "Result"

    def test_tool_calls_across_engines(self):
        """Tool calling works regardless of engine mock."""
        _register_all()
        from openjarvis.agents.native_react import NativeReActAgent
        from openjarvis.tools.calculator import CalculatorTool

        for engine_name in ["vllm", "ollama"]:
            responses = [
                _simple_response(
                    "Thought: calc.\n"
                    "Action: calculator\n"
                    'Action Input: {"expression":"3*3"}'
                ),
                _simple_response("Thought: got 9.\nFinal Answer: 9"),
            ]
            engine = _make_engine(responses)
            engine.engine_id = engine_name
            agent = NativeReActAgent(
                engine,
                "test-model",
                tools=[CalculatorTool()],
            )
            result = agent.run("What is 3*3?")
            assert result.content == "9"
            tr = result.tool_results[0]
            assert tr.content == "9.0"
            assert tr.success is True


# ---------------------------------------------------------------------------
# Memory pipeline integration
# ---------------------------------------------------------------------------


class TestMemoryPipeline:
    """Index and retrieve across available backends."""

    def test_sqlite_index_and_retrieve(self, tmp_path):
        from openjarvis.tools.storage.sqlite import SQLiteMemory

        backend = SQLiteMemory(db_path=str(tmp_path / "mem.db"))
        backend.store(
            "Machine learning uses data to learn patterns",
            source="ml.md",
        )
        backend.store(
            "Python is a versatile programming language",
            source="py.md",
        )
        results = backend.retrieve("machine learning")
        assert len(results) >= 1
        assert "machine" in results[0].content.lower()

    def test_bm25_index_and_retrieve(self, tmp_path):
        try:
            from openjarvis.tools.storage.bm25 import BM25Memory
        except ImportError:
            pytest.skip("rank_bm25 not installed")

        backend = BM25Memory()
        backend.store("Neural networks for NLP", source="a.md")
        backend.store("Database indexing strategies", source="b.md")
        results = backend.retrieve("neural NLP")
        assert len(results) >= 1


# ---------------------------------------------------------------------------
# Model catalog integration
# ---------------------------------------------------------------------------


class TestModelCatalogIntegration:
    """All registered models have valid metadata."""

    def test_all_models_routable(self):
        """Every model in catalog has required fields."""
        from openjarvis.intelligence.model_catalog import (
            BUILTIN_MODELS,
        )

        for spec in BUILTIN_MODELS:
            assert spec.model_id, "model_id required"
            assert spec.context_length > 0
            if spec.requires_api_key:
                assert spec.provider

    def test_local_models_have_engine_compat(self):
        """Every local model has at least one engine."""
        from openjarvis.intelligence.model_catalog import (
            BUILTIN_MODELS,
        )

        local = [s for s in BUILTIN_MODELS if not s.requires_api_key]
        for spec in local:
            assert len(spec.supported_engines) >= 1, f"{spec.model_id} has no engines"

    def test_cloud_models_require_api_key(self):
        """All cloud models require an API key."""
        from openjarvis.intelligence.model_catalog import (
            BUILTIN_MODELS,
        )

        cloud_ids = [
            "gpt-5-mini",
            "claude-opus-4-6",
            "claude-sonnet-4-6",
            "claude-haiku-4-5",
            "gemini-2.5-pro",
            "gemini-2.5-flash",
            "gemini-3-pro",
            "gemini-3-flash",
        ]
        for mid in cloud_ids:
            matches = [s for s in BUILTIN_MODELS if s.model_id == mid]
            assert len(matches) == 1, f"Missing {mid}"
            assert matches[0].requires_api_key is True


# ---------------------------------------------------------------------------
# Agent routing matrix (lightweight integration)
# ---------------------------------------------------------------------------


class TestAgentRoutingMatrix:
    """Agents run consistently across different configurations."""

    @pytest.mark.parametrize(
        "agent_key",
        ["native_react", "native_openhands"],
    )
    def test_agent_returns_valid_result(self, agent_key):
        _register_all()

        engine = _make_engine(
            _simple_response(
                "Thought: done.\nFinal Answer: ok"
                if agent_key == "native_react"
                else "The answer is ok"
            )
        )
        agent_cls = AgentRegistry.get(agent_key)
        agent = agent_cls(engine, "test-model")
        result = agent.run("Test")

        assert isinstance(result, AgentResult)
        assert result.turns >= 1
        assert len(result.content) > 0

    @pytest.mark.parametrize(
        "agent_key",
        ["native_react", "native_openhands"],
    )
    def test_agent_emits_events(self, agent_key):
        _register_all()

        engine = _make_engine(
            _simple_response(
                "Thought: done.\nFinal Answer: ok"
                if agent_key == "native_react"
                else "Direct answer"
            )
        )
        bus = EventBus(record_history=True)
        agent_cls = AgentRegistry.get(agent_key)
        agent = agent_cls(engine, "test-model", bus=bus)
        agent.run("Test")

        types = [e.event_type for e in bus.history]
        assert EventType.AGENT_TURN_START in types
        assert EventType.AGENT_TURN_END in types

    def test_context_passing(self):
        """Agents accept and use AgentContext."""
        _register_all()
        from openjarvis.agents.native_react import NativeReActAgent

        engine = _make_engine(
            _simple_response(
                "Thought: I see the system message.\nFinal Answer: Got context."
            )
        )
        conv = Conversation()
        conv.add(
            Message(
                role=Role.SYSTEM,
                content="You are helpful.",
            )
        )
        ctx = AgentContext(conversation=conv)
        agent = NativeReActAgent(engine, "test-model")
        result = agent.run("Hello", context=ctx)
        assert result.content == "Got context."

        # The agent prompt and caller context are folded into one leading
        # system message so strict chat templates (for example Qwen) accept it.
        call_args = engine.generate.call_args
        msgs = call_args[0][0]
        system_messages = [m for m in msgs if m.role == Role.SYSTEM]
        assert system_messages == [msgs[0]]
        assert "You are a ReAct agent." in msgs[0].content
        assert "You are helpful." in msgs[0].content