Skip to content

Personal AI, On Personal Devices

Python
10.5 KB
Raw Blame
"""Security regressions for SkillOrchestra direct QA operations."""

from __future__ import annotations

from unittest.mock import MagicMock

from openjarvis.agents.hybrid.skillorchestra.agent import SkillOrchestraAgent
from openjarvis.agents.hybrid.skillorchestra.pool import ModelSpec
from openjarvis.agents.hybrid.skillorchestra.tools import run_code, run_search
from openjarvis.core.events import EventBus, EventType
from openjarvis.security.capabilities import CapabilityPolicy


class _Limiter:
    def __init__(self, denied_key: str = "") -> None:
        self.denied_key = denied_key
        self.keys: list[str] = []

    def check(self, key: str):
        self.keys.append(key)
        return key != self.denied_key, 3.0 if key == self.denied_key else 0.0


def _agent(policy, limiter, bus, agent_id="skill-runtime"):
    return SkillOrchestraAgent(
        MagicMock(),
        "cloud-model",
        bus=bus,
        capability_policy=policy,
        rate_limiter=limiter,
        agent_id=agent_id,
    )


def _grant_local_execution(policy, agent_id="skill-runtime"):
    for capability in ("code:execute", "file:read", "file:write"):
        policy.grant(agent_id, capability)


def test_enhance_reasoning_denial_prevents_python_subprocess(monkeypatch):
    from openjarvis.agents.hybrid.skillorchestra import tools as tools_module

    policy = CapabilityPolicy(default_deny=True)
    limiter = _Limiter()
    bus = EventBus(record_history=True)
    agent = _agent(policy, limiter, bus)
    call_alias = MagicMock()
    subprocess_run = MagicMock()
    monkeypatch.setattr(tools_module, "call_alias", call_alias)
    monkeypatch.setattr(tools_module.subprocess, "run", subprocess_run)

    result = run_code(
        agent,
        ModelSpec("reasoner-1", "model", "anthropic", "cloud"),
        context_str="context",
        problem="problem",
    )

    assert "code:execute" in result["exec_result"]
    call_alias.assert_not_called()
    subprocess_run.assert_not_called()
    assert limiter.keys == ["skill-runtime:skillorchestra_code"]
    assert any(
        event.event_type == EventType.CAPABILITY_DENIED
        and event.data
        == {
            "agent_id": "skill-runtime",
            "capability": "code:execute",
            "tool": "skillorchestra_code",
        }
        for event in bus.history
    )


def test_enhance_reasoning_requires_tempfile_capabilities(monkeypatch):
    from openjarvis.agents.hybrid.skillorchestra import tools as tools_module

    policy = CapabilityPolicy(default_deny=True)
    policy.grant("skill-runtime", "code:execute")
    limiter = _Limiter()
    bus = EventBus(record_history=True)
    agent = _agent(policy, limiter, bus)
    call_alias = MagicMock()
    subprocess_run = MagicMock()
    monkeypatch.setattr(tools_module, "call_alias", call_alias)
    monkeypatch.setattr(tools_module.subprocess, "run", subprocess_run)

    result = run_code(
        agent,
        ModelSpec("reasoner-1", "model", "anthropic", "cloud"),
        context_str="context",
        problem="problem",
    )

    assert "file:read" in result["exec_result"]
    call_alias.assert_not_called()
    subprocess_run.assert_not_called()
    assert any(
        event.event_type == EventType.CAPABILITY_DENIED
        and event.data["capability"] == "file:read"
        for event in bus.history
    )


def test_custom_retriever_rate_limit_prevents_http_post(monkeypatch):
    import requests

    from openjarvis.agents.hybrid.skillorchestra import tools as tools_module

    policy = CapabilityPolicy(default_deny=True)
    policy.grant("skill-runtime", "network:fetch")
    denied_key = "skill-runtime:skillorchestra_retriever"
    limiter = _Limiter(denied_key)
    bus = EventBus(record_history=True)
    agent = _agent(policy, limiter, bus)
    monkeypatch.setattr(
        tools_module,
        "call_alias",
        MagicMock(return_value=("<query>long enough query</query>", 1, 1, 0.0)),
    )
    post = MagicMock()
    monkeypatch.setattr(requests, "post", post)

    result = run_search(
        agent,
        ModelSpec("search-1", "model", "anthropic", "cloud"),
        context_str="context",
        problem="problem",
        retriever_url="https://retriever.example",
    )

    assert "Rate limit exceeded" in result["search_results_data"][0]
    post.assert_not_called()
    assert limiter.keys == [denied_key]
    assert any(
        event.event_type == EventType.RATE_LIMITED
        and event.data["agent_id"] == "skill-runtime"
        and event.data["tool"] == "skillorchestra_retriever"
        for event in bus.history
    )


def test_default_provider_search_denial_prevents_provider_call(monkeypatch):
    from openjarvis.agents.hybrid.skillorchestra import tools as tools_module

    policy = CapabilityPolicy(default_deny=True)
    _grant_local_execution(policy)
    limiter = _Limiter()
    bus = EventBus(record_history=True)
    agent = _agent(policy, limiter, bus)
    monkeypatch.setattr(
        tools_module,
        "call_alias",
        MagicMock(return_value=("<query>long enough query</query>", 1, 1, 0.0)),
    )
    provider_search = MagicMock()
    monkeypatch.setattr(agent, "_call_anthropic_agent", provider_search)

    result = run_search(
        agent,
        ModelSpec("search-1", "model", "anthropic", "cloud"),
        context_str="context",
        problem="problem",
    )

    assert "network:fetch" in result["search_results_data"][0]
    provider_search.assert_not_called()
    assert limiter.keys == ["skill-runtime:skillorchestra_provider_search"]
    assert any(
        event.event_type == EventType.CAPABILITY_DENIED
        and event.data
        == {
            "agent_id": "skill-runtime",
            "capability": "network:fetch",
            "tool": "skillorchestra_provider_search",
        }
        for event in bus.history
    )


def test_tavily_search_denial_prevents_fetch(monkeypatch):
    from openjarvis.agents.hybrid.skillorchestra import tools as tools_module

    policy = CapabilityPolicy(default_deny=True)
    _grant_local_execution(policy)
    limiter = _Limiter()
    bus = EventBus(record_history=True)
    agent = _agent(policy, limiter, bus)
    monkeypatch.setattr(
        tools_module,
        "call_alias",
        MagicMock(return_value=("<query>long enough query</query>", 1, 1, 0.0)),
    )
    tavily = MagicMock()
    monkeypatch.setattr(tools_module, "tavily_search_context", tavily)

    result = run_search(
        agent,
        ModelSpec("search-1", "model", "anthropic", "cloud"),
        context_str="context",
        problem="problem",
        search_backend="tavily",
    )

    assert "network:fetch" in result["search_results_data"][0]
    tavily.assert_not_called()
    assert limiter.keys == ["skill-runtime:skillorchestra_tavily"]
    assert any(
        event.event_type == EventType.CAPABILITY_DENIED
        and event.data["tool"] == "skillorchestra_tavily"
        for event in bus.history
    )


def test_provider_search_executes_with_positive_network_grant(monkeypatch):
    from openjarvis.agents.hybrid.skillorchestra import tools as tools_module

    policy = CapabilityPolicy(default_deny=True)
    policy.grant("skill-runtime", "network:fetch")
    limiter = _Limiter()
    bus = EventBus(record_history=True)
    agent = _agent(policy, limiter, bus)
    monkeypatch.setattr(
        tools_module,
        "call_alias",
        MagicMock(return_value=("<query>long enough query</query>", 1, 1, 0.0)),
    )
    provider_search = MagicMock(return_value=("provider findings", 2, 3, 1, 1))
    monkeypatch.setattr(agent, "_call_anthropic_agent", provider_search)

    result = run_search(
        agent,
        ModelSpec("search-1", "model", "anthropic", "cloud"),
        context_str="context",
        problem="problem",
    )

    provider_search.assert_called_once()
    assert result["search_results_data"] == ["provider findings"]
    assert result["web_search_uses"] == 1
    assert limiter.keys == ["skill-runtime:skillorchestra_provider_search"]
    assert any(
        event.event_type == EventType.TOOL_CALL_START
        and event.data["tool"] == "skillorchestra_provider_search"
        for event in bus.history
    )


def test_tavily_search_executes_with_positive_network_grant(monkeypatch):
    from openjarvis.agents.hybrid.skillorchestra import tools as tools_module

    policy = CapabilityPolicy(default_deny=True)
    policy.grant("skill-runtime", "network:fetch")
    limiter = _Limiter()
    bus = EventBus(record_history=True)
    agent = _agent(policy, limiter, bus)
    monkeypatch.setattr(
        tools_module,
        "call_alias",
        MagicMock(return_value=("<query>long enough query</query>", 1, 1, 0.0)),
    )
    tavily = MagicMock(
        return_value={"text": "tavily findings", "n_searches": 1, "cost_usd": 0.01}
    )
    monkeypatch.setattr(tools_module, "tavily_search_context", tavily)

    result = run_search(
        agent,
        ModelSpec("search-1", "model", "anthropic", "cloud"),
        context_str="context",
        problem="problem",
        search_backend="tavily",
    )

    tavily.assert_called_once_with("long enough query", max_results=5)
    assert result["search_results_data"] == ["tavily findings"]
    assert result["web_search_uses"] == 1
    assert limiter.keys == ["skill-runtime:skillorchestra_tavily"]


def test_retriever_configuration_is_derived_before_agent_run():
    policy = CapabilityPolicy(default_deny=True)
    _grant_local_execution(policy, "restricted-skill")
    limiter = _Limiter()
    agent = SkillOrchestraAgent(
        MagicMock(),
        "cloud-model",
        cfg={"retriever_url": "https://retriever.example"},
        capability_policy=policy,
        rate_limiter=limiter,
        agent_id="restricted-skill",
    )
    agent._run_paradigm = MagicMock(return_value=("should not run", {}))

    result = agent.run("research")

    assert "network:fetch" in result.content
    agent._run_paradigm.assert_not_called()
    assert limiter.keys == ["restricted-skill:hybrid_agent_run"]


def test_default_provider_search_is_derived_before_agent_run():
    policy = CapabilityPolicy(default_deny=True)
    _grant_local_execution(policy, "restricted-skill")
    limiter = _Limiter()
    agent = SkillOrchestraAgent(
        MagicMock(),
        "cloud-model",
        capability_policy=policy,
        rate_limiter=limiter,
        agent_id="restricted-skill",
    )
    agent._run_paradigm = MagicMock(return_value=("should not run", {}))

    result = agent.run("research")

    assert "network:fetch" in result.content
    agent._run_paradigm.assert_not_called()
    assert limiter.keys == ["restricted-skill:hybrid_agent_run"]