Skip to content

Self-hosted AI workspace.

Python
14.5 KB
Raw Blame
import asyncio
import json
from types import SimpleNamespace
import pytest

import src.tool_capabilities as tool_capabilities
import src.teacher_escalation as teacher_escalation


@pytest.fixture(autouse=True)
def _enable_approval_gate_for_legacy_gate_tests(monkeypatch):
    monkeypatch.setattr(tool_capabilities, "TOOL_APPROVAL_GATE_ENABLED", True)


@pytest.mark.asyncio
async def test_evaluate_turn_llm_ok(monkeypatch):
    seen = {}

    def fake_resolve_endpoint(prefix, fallback_url=None, owner=None):
        seen["prefix"] = prefix
        seen["owner"] = owner
        return "http://endpoint.local/v1", "utility-model", {}

    async def fake_llm_call_async(url, model, messages, **kwargs):
        seen["called"] = True
        return "ok"

    monkeypatch.setattr("src.endpoint_resolver.resolve_endpoint", fake_resolve_endpoint)
    monkeypatch.setattr("src.llm_core.llm_call_async", fake_llm_call_async)

    status, reason = await teacher_escalation.evaluate_turn_llm(
        user_request="test request",
        tool_results=[],
        agent_reply="test reply",
        student_endpoint_url="http://student.local/v1",
        owner="alice",
    )

    assert status == "ok"
    assert reason is None
    assert seen["prefix"] == "utility"
    assert seen["owner"] == "alice"
    assert seen["called"] is True


@pytest.mark.asyncio
async def test_evaluate_turn_llm_failure(monkeypatch):
    def fake_resolve_endpoint(prefix, fallback_url=None, owner=None):
        return "http://endpoint.local/v1", "utility-model", {}

    async def fake_llm_call_async(url, model, messages, **kwargs):
        return "  \"Failure\"  "

    monkeypatch.setattr("src.endpoint_resolver.resolve_endpoint", fake_resolve_endpoint)
    monkeypatch.setattr("src.llm_core.llm_call_async", fake_llm_call_async)

    status, reason = await teacher_escalation.evaluate_turn_llm(
        user_request="test request",
        tool_results=[],
        agent_reply="test reply",
        student_endpoint_url="http://student.local/v1",
        owner="alice",
    )

    assert status == "failure"
    assert "LLM evaluation flagged failure" in reason


@pytest.mark.asyncio
async def test_evaluate_turn_llm_contains_failure_but_not_exact_match(monkeypatch):
    def fake_resolve_endpoint(prefix, fallback_url=None, owner=None):
        return "http://endpoint.local/v1", "utility-model", {}

    async def fake_llm_call_async(url, model, messages, **kwargs):
        return "this agent execution is not a failure"

    monkeypatch.setattr("src.endpoint_resolver.resolve_endpoint", fake_resolve_endpoint)
    monkeypatch.setattr("src.llm_core.llm_call_async", fake_llm_call_async)

    status, reason = await teacher_escalation.evaluate_turn_llm(
        user_request="test request",
        tool_results=[],
        agent_reply="test reply",
        student_endpoint_url="http://student.local/v1",
        owner="alice",
    )

    assert status == "ok"
    assert reason is None


@pytest.mark.asyncio
async def test_evaluate_turn_llm_exception_handling(monkeypatch):
    def fake_resolve_endpoint(prefix, fallback_url=None, owner=None):
        return "http://endpoint.local/v1", "utility-model", {}

    async def fake_llm_call_async(url, model, messages, **kwargs):
        raise RuntimeError("model timeout")

    monkeypatch.setattr("src.endpoint_resolver.resolve_endpoint", fake_resolve_endpoint)
    monkeypatch.setattr("src.llm_core.llm_call_async", fake_llm_call_async)

    # Should degrade gracefully to "ok"
    status, reason = await teacher_escalation.evaluate_turn_llm(
        user_request="test request",
        tool_results=[],
        agent_reply="test reply",
        student_endpoint_url="http://student.local/v1",
        owner="alice",
    )

    assert status == "ok"
    assert reason is None


@pytest.mark.asyncio
async def test_maybe_escalate_triggers_tier2_background_task(monkeypatch):
    # Enable teacher settings
    monkeypatch.setattr("src.settings.get_setting", lambda key, default=None: {"teacher_enabled": True, "teacher_model": "teacher-model", "teacher_tier2_enabled": True}.get(key, default))

    # Regex check says OK
    monkeypatch.setattr("src.teacher_escalation.evaluate_turn_regex", lambda *args: ("ok", None))

    llm_eval_called = []
    async def fake_evaluate_turn_llm(*args, **kwargs):
        llm_eval_called.append(True)
        return "failure", "LLM flagged failure"

    monkeypatch.setattr("src.teacher_escalation.evaluate_turn_llm", fake_evaluate_turn_llm)

    escalate_called = []
    async def fake_escalate_and_learn(user_request, tool_results, agent_reply, failure_reason, owner):
        escalate_called.append(failure_reason)
        return "skill-slug"

    monkeypatch.setattr("src.teacher_escalation.escalate_and_learn", fake_escalate_and_learn)

    # Call maybe_escalate
    task = teacher_escalation.maybe_escalate(
        student_endpoint_url="http://student.local/v1",
        mode="agent",
        user_request="test request",
        tool_results=[],
        agent_reply="test reply",
        owner="alice",
    )

    assert task is not None
    assert task.get_name() == "teacher_escalation_tier2"

    # Await the background task execution
    await task

    assert llm_eval_called == [True]
    assert escalate_called == ["LLM flagged failure"]


@pytest.mark.asyncio
async def test_maybe_escalate_tier2_disabled_by_default(monkeypatch):
    # Enable teacher settings, but keep tier2 disabled
    monkeypatch.setattr("src.settings.get_setting", lambda key, default=None: {"teacher_enabled": True, "teacher_model": "teacher-model", "teacher_tier2_enabled": False}.get(key, default))

    # Regex check says OK
    monkeypatch.setattr("src.teacher_escalation.evaluate_turn_regex", lambda *args: ("ok", None))

    # Call maybe_escalate
    task = teacher_escalation.maybe_escalate(
        student_endpoint_url="http://student.local/v1",
        mode="agent",
        user_request="test request",
        tool_results=[],
        agent_reply="test reply",
        owner="alice",
    )

    # Should not start any background task since Tier 2 is disabled
    assert task is None


@pytest.mark.asyncio
async def test_background_teacher_learning_never_persists_without_approval(monkeypatch):
    monkeypatch.setattr(
        "src.settings.get_setting",
        lambda key, default=None: {
            "teacher_model": "teacher-model",
        }.get(key, default),
    )

    async def fail_teacher_call(*args, **kwargs):
        raise AssertionError("background learning spent a teacher call without approval UI")

    async def fail_direct_skill_save(*args, **kwargs):
        raise AssertionError("background teacher output was persisted directly")

    monkeypatch.setattr("src.teacher_escalation._call_teacher", fail_teacher_call)
    monkeypatch.setattr(
        "src.tool_implementations.do_manage_skills",
        fail_direct_skill_save,
    )

    saved = await teacher_escalation.escalate_and_learn(
        user_request="test request",
        tool_results=[],
        agent_reply="student failed",
        failure_reason="test failure",
        owner="alice",
    )

    assert saved is None


@pytest.mark.asyncio
async def test_run_teacher_inline_triggers_tier2_escalation(monkeypatch):
    from src.tool_approvals import tool_approval_store

    # Settings and gates
    monkeypatch.setattr("src.settings.get_setting", lambda key, default=None: {"teacher_enabled": True, "teacher_model": "teacher-model", "teacher_tier2_enabled": True}.get(key, default))
    monkeypatch.setattr("src.ai_interaction._resolve_model", lambda spec, owner=None: ("http://teacher.local/v1", "teacher-model", {}))

    # Regex evaluation says "ok"
    monkeypatch.setattr("src.teacher_escalation.evaluate_turn_regex", lambda *args: ("ok", None))

    # LLM evaluation flags "failure"
    async def fake_evaluate_turn_llm(*args, **kwargs):
        return "failure", "LLM flagged failure"
    monkeypatch.setattr("src.teacher_escalation.evaluate_turn_llm", fake_evaluate_turn_llm)

    # Mock stream_agent_loop recursively called by run_teacher_inline
    async def fake_stream_agent_loop(*args, **kwargs):
        yield "data: {\"type\": \"tool_output\", \"tool\": \"bash\"}\n\n"
        yield "data: {\"type\": \"text\", \"delta\": \"Teacher reply\"}\n\n"
        yield "data: " + json.dumps({
            "type": "metrics",
            "data": {
                "model": "teacher-model",
                "round_texts": ["Teacher reply"],
                "tool_events": [
                    {
                        "round": 1,
                        "tool": "bash",
                        "output": "done",
                        "exit_code": 0,
                    },
                ],
            },
        }) + "\n\n"
        yield "data: [DONE]\n\n"
    monkeypatch.setattr("src.agent_loop.stream_agent_loop", fake_stream_agent_loop)

    # Mock _call_teacher returning a skill definition
    async def fake_call_teacher(spec, prompt, owner=None):
        return '```json\n{"action": "add", "name": "test-skill"}\n```'
    monkeypatch.setattr("src.teacher_escalation._call_teacher", fake_call_teacher)

    async def fail_direct_skill_save(*args, **kwargs):
        raise AssertionError("teacher output was persisted without approval")

    monkeypatch.setattr(
        "src.tool_implementations.do_manage_skills",
        fail_direct_skill_save,
    )

    events = []
    async for evt in teacher_escalation.run_teacher_inline(
        student_endpoint_url="http://student.local/v1",
        student_messages=[{"role": "user", "content": "test request"}],
        student_tool_events=[],
        student_reply="student reply",
        owner="alice",
        session_id="teacher-approval-session",
    ):
        events.append(evt)

    # The teacher takeover runs, but its cross-model skill output is sealed for
    # an explicit approval instead of being written directly.
    assert any("teacher_takeover" in evt for evt in events)
    assert any("tool_output" in evt for evt in events)
    approval_event = next(
        json.loads(evt[6:])
        for evt in events
        if evt.startswith("data: ")
        and "\"kind\": \"tool_approval\"" in evt
        and "\"type\": \"tool_output\"" in evt
    )
    approval = approval_event["ask_user"]
    final_metrics = next(
        json.loads(evt[6:])
        for evt in reversed(events)
        if evt.startswith("data: ") and '"type": "metrics"' in evt
    )
    persisted_approval = final_metrics["data"]["tool_events"][-1]
    assert persisted_approval["ask_user"] == approval
    assert persisted_approval["round"] == 2
    pending = tool_approval_store.peek(approval["approval_id"])
    assert pending is not None
    assert pending.tool_name == "manage_skills"
    assert json.loads(pending.content)["name"] == "test-skill"
    assert pending.external_untrusted_context_seen is True
    tool_approval_store.consume(
        pending.approval_id,
        decision="deny",
        owner="alice",
        session_id="teacher-approval-session",
    )
    assert not any("skill_saved" in evt for evt in events)


@pytest.mark.asyncio
async def test_teacher_approval_keeps_parent_authority_and_skips_skill_save(
    monkeypatch,
):
    monkeypatch.setattr(
        "src.settings.get_setting",
        lambda key, default=None: {
            "teacher_enabled": True,
            "teacher_model": "teacher-model",
        }.get(key, default),
    )
    monkeypatch.setattr(
        "src.ai_interaction._resolve_model",
        lambda spec, owner=None: (
            "http://teacher.local/v1",
            "teacher-model",
            {},
        ),
    )
    monkeypatch.setattr(
        "src.teacher_escalation.evaluate_turn_regex",
        lambda *args: ("failure", "student failed"),
    )
    captured = {}
    approval = {
        "kind": "tool_approval",
        "approval_id": "opaque-id",
        "question": "Allow this exact action once?",
    }

    async def fake_stream_agent_loop(*args, **kwargs):
        captured.update(kwargs)
        yield "data: " + json.dumps({
            "type": "tool_output",
            "tool": "bash",
            "output": "Waiting for an exact user approval.",
            "ask_user": approval,
        }) + "\n\n"
        yield "data: [DONE]\n\n"

    async def fail_skill_distillation(*args, **kwargs):
        raise AssertionError("paused teacher trace was distilled into a skill")

    monkeypatch.setattr(
        "src.agent_loop.stream_agent_loop",
        fake_stream_agent_loop,
    )
    monkeypatch.setattr(
        "src.teacher_escalation._call_teacher",
        fail_skill_distillation,
    )
    active_document = object()
    active_email = {"uid": "email-1"}
    policy = object()

    events = []
    async for evt in teacher_escalation.run_teacher_inline(
        student_endpoint_url="http://student.local/v1",
        student_messages=[{"role": "user", "content": "test request"}],
        student_tool_events=[],
        student_reply="student reply",
        owner="alice",
        session_id="session-1",
        workspace="/workspace",
        disabled_tools={"web_fetch"},
        tool_policy=policy,
        active_document=active_document,
        active_email=active_email,
        external_untrusted_context_seen=True,
        delegated_credential=True,
    ):
        events.append(evt)

    assert captured["session_id"] == "session-1"
    assert captured["workspace"] == "/workspace"
    assert captured["disabled_tools"] == {"web_fetch"}
    assert captured["tool_policy"] is policy
    assert captured["active_document"] is active_document
    assert captured["active_email"] == active_email
    assert captured["external_untrusted_context_seen"] is True
    assert captured["delegated_credential"] is True
    assert any("opaque-id" in event for event in events)
    assert not any("skill_saved" in event for event in events)


@pytest.mark.asyncio
async def test_run_teacher_inline_tier2_disabled_by_default(monkeypatch):
    # Settings and gates (Tier 2 disabled)
    monkeypatch.setattr("src.settings.get_setting", lambda key, default=None: {"teacher_enabled": True, "teacher_model": "teacher-model", "teacher_tier2_enabled": False}.get(key, default))

    # Regex evaluation says "ok"
    monkeypatch.setattr("src.teacher_escalation.evaluate_turn_regex", lambda *args: ("ok", None))

    events = []
    async for evt in teacher_escalation.run_teacher_inline(
        student_endpoint_url="http://student.local/v1",
        student_messages=[{"role": "user", "content": "test request"}],
        student_tool_events=[],
        student_reply="student reply",
        owner="alice",
    ):
        events.append(evt)

    # Should exit early without any events (no takeover)
    assert len(events) == 0