Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

16.9 KB
Raw Blame
"""Tests for ChatOrchestrator routing and lifecycle."""

from __future__ import annotations

import asyncio
from unittest.mock import AsyncMock, MagicMock, patch

import pytest

from deeptutor.core.capability_protocol import CapabilityManifest, TurnCapability
from deeptutor.core.context import UnifiedContext
from deeptutor.core.stream import StreamEvent, StreamEventType
from deeptutor.runtime.orchestrator import ChatOrchestrator
from deeptutor.runtime.stream_bus import StreamBus, get_bus


@pytest.fixture(autouse=True)
def _patch_event_bus():
    """Prevent EventBus background processor from running during tests."""
    mock_bus = MagicMock()
    mock_bus.publish = AsyncMock()
    with patch("deeptutor.runtime.orchestrator.get_event_bus", return_value=mock_bus):
        yield
    from deeptutor.events.event_bus import EventBus

    EventBus.reset()


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


class _EchoCapability(TurnCapability):
    """Minimal capability that echoes the user message."""

    manifest = CapabilityManifest(
        name="echo",
        description="Echoes back user message.",
        stages=["responding"],
    )

    async def run(self, context: UnifiedContext, stream: StreamBus) -> None:
        await stream.content(context.user_message, source=self.name)


class _FailingCapability(TurnCapability):
    """Capability that raises."""

    manifest = CapabilityManifest(name="fail", description="Always fails.")

    async def run(self, context: UnifiedContext, stream: StreamBus) -> None:
        raise RuntimeError("intentional failure")


class _ParkedCapability(TurnCapability):
    """Capability that streams once, then waits until it is cancelled."""

    manifest = CapabilityManifest(name="parked", description="Waits after its first event.")

    def __init__(self) -> None:
        self.cancelled = asyncio.Event()

    async def run(self, context: UnifiedContext, stream: StreamBus) -> None:
        await stream.content("thinking", source=self.name)
        try:
            # Stands in for a long model round or an ``ask_user`` pause.
            await asyncio.Event().wait()
        except asyncio.CancelledError:
            self.cancelled.set()
            raise


def _make_orchestrator(
    capabilities: dict[str, TurnCapability] | None = None,
) -> ChatOrchestrator:
    """Build an orchestrator with fake registries."""
    cap_reg = MagicMock()
    cap_map = capabilities or {}
    cap_reg.get = lambda name: cap_map.get(name)
    cap_reg.list_capabilities = lambda: list(cap_map.keys())

    tool_reg = MagicMock()
    tool_reg.list_tools = MagicMock(return_value=[])
    tool_reg.build_openai_schemas = MagicMock(return_value=[])

    orch = ChatOrchestrator.__new__(ChatOrchestrator)
    orch._cap_registry = cap_reg
    orch._tool_registry = tool_reg
    return orch


# ---------------------------------------------------------------------------
# Routing
# ---------------------------------------------------------------------------


class TestOrchestratorRouting:
    @pytest.mark.asyncio
    async def test_routes_to_active_capability(self) -> None:
        echo = _EchoCapability()
        orch = _make_orchestrator({"echo": echo})

        ctx = UnifiedContext(
            user_message="ping",
            active_capability="echo",
        )
        events: list[StreamEvent] = []
        async for event in orch.handle(ctx):
            events.append(event)

        types = [e.type for e in events]
        assert StreamEventType.SESSION in types
        assert StreamEventType.CONTENT in types
        assert StreamEventType.DONE in types

        content_events = [e for e in events if e.type == StreamEventType.CONTENT]
        assert content_events[0].content == "ping"

    @pytest.mark.asyncio
    async def test_defaults_to_chat_capability(self) -> None:
        chat_cap = _EchoCapability()
        chat_cap.manifest = CapabilityManifest(
            name="chat", description="Default chat.", stages=["responding"]
        )
        orch = _make_orchestrator({"chat": chat_cap})

        ctx = UnifiedContext(user_message="hello")
        events: list[StreamEvent] = []
        async for event in orch.handle(ctx):
            events.append(event)

        content_events = [e for e in events if e.type == StreamEventType.CONTENT]
        assert len(content_events) == 1
        assert content_events[0].content == "hello"

    @pytest.mark.asyncio
    async def test_unknown_capability_yields_error(self) -> None:
        orch = _make_orchestrator({})

        ctx = UnifiedContext(
            user_message="hi",
            active_capability="nonexistent",
        )
        events: list[StreamEvent] = []
        async for event in orch.handle(ctx):
            events.append(event)

        error_events = [e for e in events if e.type == StreamEventType.ERROR]
        assert len(error_events) == 1
        assert "Unknown capability" in error_events[0].content
        assert error_events[0].metadata == {
            "turn_terminal": True,
            "status": "failed",
        }
        done_events = [e for e in events if e.type == StreamEventType.DONE]
        assert len(done_events) == 1
        assert done_events[0].metadata["status"] == "failed"


# ---------------------------------------------------------------------------
# Error handling
# ---------------------------------------------------------------------------


class TestOrchestratorErrorHandling:
    @pytest.mark.asyncio
    async def test_final_result_supersedes_intermediate_incomplete_result(self):
        class RecoveredCapability:
            async def run(self, _context, bus):
                await bus.result({"completed": False}, source="chat")
                await bus.tool_result("lookup", "retry succeeded", source="chat")
                await bus.result({"completed": True}, source="chat")

        orch = _make_orchestrator({"chat": RecoveredCapability()})
        events = [e async for e in orch.handle(UnifiedContext(user_message="practice"))]
        done = next(e for e in events if e.type == StreamEventType.DONE)
        assert done.metadata["status"] == "completed"

    @pytest.mark.asyncio
    @pytest.mark.parametrize("completed", [False, True, None])
    @pytest.mark.parametrize(
        "language, notice",
        [
            ("en", "The requested operation did not complete"),
            ("zh", "请求的操作未完成"),
        ],
    )
    async def test_explicit_incomplete_result_controls_terminal_status(
        self, completed, language, notice
    ):
        class ResultCapability:
            async def run(self, _context, bus):
                payload = {"response": "The requested operation was not delivered."}
                if completed is not None:
                    payload["completed"] = completed
                await bus.result(payload, source="chat")

        orch = _make_orchestrator({"chat": ResultCapability()})
        events = [
            e async for e in orch.handle(UnifiedContext(user_message="practice", language=language))
        ]
        done = next(e for e in events if e.type == StreamEventType.DONE)
        if completed is False:
            assert done.metadata["status"] == "failed"
            assert done.metadata["error_code"] == "capability_incomplete"
            assert done.metadata["retryable"] is True
            error = next(e for e in events if e.type == StreamEventType.ERROR)
            assert error.metadata["turn_terminal"] is True
            assert notice in error.content
            from deeptutor.services.session._turn_runtime_shared import (
                _resolve_turn_failure_metadata,
                _resolve_turn_outcome,
            )

            persisted = [
                {"type": e.type.value, "content": e.content, "metadata": e.metadata}
                for e in events
                if e is not done
            ]
            assert _resolve_turn_outcome(persisted, done)[0] == "failed"
            assert _resolve_turn_failure_metadata(persisted) == ("capability_incomplete", True)
        else:
            assert done.metadata["status"] == "completed"
            assert not any(e.type == StreamEventType.ERROR for e in events)

    @pytest.mark.asyncio
    async def test_capability_exception_yields_error_event(self) -> None:
        fail_cap = _FailingCapability()
        orch = _make_orchestrator({"fail": fail_cap})

        ctx = UnifiedContext(
            user_message="boom",
            active_capability="fail",
        )
        events: list[StreamEvent] = []
        async for event in orch.handle(ctx):
            events.append(event)

        error_events = [e for e in events if e.type == StreamEventType.ERROR]
        assert len(error_events) == 1
        assert "intentional failure" in error_events[0].content
        assert error_events[0].metadata == {
            "turn_terminal": True,
            "status": "failed",
        }

        done_events = [e for e in events if e.type == StreamEventType.DONE]
        assert len(done_events) == 1
        assert done_events[0].metadata["status"] == "failed"

    @pytest.mark.asyncio
    async def test_capability_exception_preserves_safe_error_metadata(self) -> None:
        class _StructuredError(RuntimeError):
            error_code = "provider_transport"
            retryable = True
            partial_response = False

        class _StructuredFailingCapability:
            async def run(self, _context, _bus) -> None:
                raise _StructuredError("Unable to reach the model provider. Please retry.")

        orch = _make_orchestrator({"fail": _StructuredFailingCapability()})
        events = [
            event
            async for event in orch.handle(
                UnifiedContext(user_message="boom", active_capability="fail")
            )
        ]

        error = next(event for event in events if event.type == StreamEventType.ERROR)
        assert error.metadata == {
            "turn_terminal": True,
            "status": "failed",
            "error_code": "provider_transport",
            "retryable": True,
            "partial_response": False,
        }
        done = next(event for event in events if event.type == StreamEventType.DONE)
        assert done.metadata == {
            "status": "failed",
            "error_code": "provider_transport",
            "retryable": True,
            "partial_response": False,
        }


# ---------------------------------------------------------------------------
# Cancellation
# ---------------------------------------------------------------------------


class TestOrchestratorCancellation:
    @pytest.mark.asyncio
    async def test_cancelling_the_consumer_cancels_the_capability(self) -> None:
        """Stopping a turn must stop the capability, not only the event stream."""
        parked = _ParkedCapability()
        orch = _make_orchestrator({"parked": parked})
        ctx = UnifiedContext(
            user_message="hi",
            active_capability="parked",
            metadata={"turn_id": "turn-cancelled"},
        )
        first_content = asyncio.Event()

        async def _consume() -> None:
            async for event in orch.handle(ctx):
                if event.type == StreamEventType.CONTENT:
                    first_content.set()

        consumer = asyncio.create_task(_consume())
        await asyncio.wait_for(first_content.wait(), timeout=1)
        consumer.cancel()
        with pytest.raises(asyncio.CancelledError):
            await consumer

        assert parked.cancelled.is_set()
        assert get_bus("turn-cancelled") is None

    @pytest.mark.asyncio
    async def test_closing_the_stream_early_cancels_the_capability(self) -> None:
        """A consumer that stops reading takes the capability down with it."""
        parked = _ParkedCapability()
        orch = _make_orchestrator({"parked": parked})
        ctx = UnifiedContext(
            user_message="hi",
            active_capability="parked",
            metadata={"turn_id": "turn-closed"},
        )

        stream = orch.handle(ctx)
        async for event in stream:
            if event.type == StreamEventType.CONTENT:
                break
        await stream.aclose()

        assert parked.cancelled.is_set()
        assert get_bus("turn-closed") is None


# ---------------------------------------------------------------------------
# Session ID management
# ---------------------------------------------------------------------------


class TestOrchestratorSessionId:
    @pytest.mark.asyncio
    async def test_assigns_session_id_if_missing(self) -> None:
        echo = _EchoCapability()
        orch = _make_orchestrator({"echo": echo})

        ctx = UnifiedContext(user_message="test", active_capability="echo")
        assert ctx.session_id == ""

        async for _ in orch.handle(ctx):
            pass

        assert ctx.session_id != ""

    @pytest.mark.asyncio
    async def test_preserves_existing_session_id(self) -> None:
        echo = _EchoCapability()
        orch = _make_orchestrator({"echo": echo})

        ctx = UnifiedContext(
            session_id="my-session",
            user_message="test",
            active_capability="echo",
        )
        async for _ in orch.handle(ctx):
            pass

        assert ctx.session_id == "my-session"


# ---------------------------------------------------------------------------
# List helpers
# ---------------------------------------------------------------------------


class TestOrchestratorHelpers:
    def test_list_tools(self) -> None:
        orch = _make_orchestrator()
        assert orch.list_tools() == []

    def test_list_capabilities(self) -> None:
        echo = _EchoCapability()
        orch = _make_orchestrator({"echo": echo})
        assert orch.list_capabilities() == ["echo"]

    def test_get_tool_schemas(self) -> None:
        orch = _make_orchestrator()
        schemas = orch.get_tool_schemas()
        assert isinstance(schemas, list)


class TestCompletionEventFields:
    def test_reads_agent_output_and_declared_event_metadata(self) -> None:
        from deeptutor.runtime.orchestrator import completion_event_fields

        ctx = UnifiedContext(
            user_message="hi",
            session_id="sess-1",
            metadata={
                "agent_output": "## Debrief\nNice work.",
                "turn_id": "turn-9",
                "event_metadata": {"practice_trace": "keep-me"},
            },
        )
        output, meta = completion_event_fields(ctx, "echo")
        assert output == "## Debrief\nNice work."
        assert meta["capability"] == "echo"
        assert meta["session_id"] == "sess-1"
        assert meta["turn_id"] == "turn-9"
        assert meta["practice_trace"] == "keep-me"

    def test_turn_scratchpad_is_not_published(self) -> None:
        """Only the declared sub-dict reaches the bus.

        Turn metadata holds live callables and the user's own answers; the
        EventBus fans out to the Partner channels, and a JSON-serialising
        subscriber cannot encode a function anyway.
        """
        from deeptutor.runtime.orchestrator import completion_event_fields

        ctx = UnifiedContext(
            user_message="hi",
            session_id="sess-1",
            metadata={
                "wait_for_user_reply": lambda: None,
                "ask_user_answers": [{"questionId": "q1", "text": "private"}],
                "event_metadata": {"room_id": "room-7"},
            },
        )
        _, meta = completion_event_fields(ctx, "whisper")
        assert meta["room_id"] == "room-7"
        assert "wait_for_user_reply" not in meta
        assert "ask_user_answers" not in meta

    def test_capability_and_ids_cannot_be_spoofed(self) -> None:
        from deeptutor.runtime.orchestrator import completion_event_fields

        ctx = UnifiedContext(
            user_message="hi",
            session_id="real-session",
            metadata={
                "turn_id": "t1",
                "event_metadata": {
                    "capability": "spoofed",
                    "session_id": "spoofed-session",
                    "turn_id": "spoofed-turn",
                },
            },
        )
        _, meta = completion_event_fields(ctx, "echo")
        assert meta["capability"] == "echo"
        assert meta["session_id"] == "real-session"
        assert meta["turn_id"] == "t1"

    def test_non_dict_event_metadata_is_ignored(self) -> None:
        from deeptutor.runtime.orchestrator import completion_event_fields

        ctx = UnifiedContext(
            user_message="hi",
            session_id="s",
            metadata={"event_metadata": "not-a-dict"},
        )
        _, meta = completion_event_fields(ctx, "chat")
        assert meta == {"capability": "chat", "session_id": "s", "turn_id": ""}

    def test_empty_agent_output_when_unset(self) -> None:
        from deeptutor.runtime.orchestrator import completion_event_fields

        ctx = UnifiedContext(user_message="hi", session_id="s", metadata={})
        output, meta = completion_event_fields(ctx, "chat")
        assert output == ""
        assert "agent_output" not in meta