Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

10.5 KB
Raw Blame
"""Research-worker progress delivered through the real Socket.IO transport.

This is the migration seam that the neighbouring suites intentionally stop
short of crossing:

* ``tests/test_end_to_end_journeys.py`` drives a real research worker but
  explicitly opens no socket;
* ``test_research_service_progress_integration.py`` drives the real progress
  callback but replaces ``_sio_emit`` with a mock; and
* ``test_realtime_channel_isolation.py`` drives real Engine.IO sessions but
  manually invokes ``emit_to_subscribers`` instead of producing an event from
  a research worker.

The Flask -> FastAPI port put a new boundary between those pieces. A research
thread calls ``asyncio.run_coroutine_threadsafe`` against the event loop saved
by the FastAPI lifespan, and that coroutine must reach the Socket.IO ASGI app
mounted at ``/ws``. Each piece can work alone while the hand-off is broken.

This test uses the existing journey suite's deterministic in-process LLM and
stubbed search egress. The LLM is gated with ``threading.Event`` so the real
worker cannot finish before the client has subscribed. No external socket or
model is used; the only wire involved is TestClient -> the real mounted
Engine.IO polling transport.
"""

from __future__ import annotations

import json
import threading
import time

import pytest
from fastapi.testclient import TestClient

from local_deep_research.llm import register_llm
from local_deep_research.web.research_state import get_research_field
from local_deep_research.web.services import socketio_asgi
from tests.security.test_realtime_channel_isolation import (
    _EIO_SEP,
    EioSession,
    _wait_until,
)
from tests.test_end_to_end_journeys import (
    PROVIDER,
    _StubChatModel,
    _new_client,
    _poll_until_terminal,
    _register_and_login,
    _start_research,
    stubs as journey_stubs,  # noqa: F401 -- imported pytest fixture
)


_MODEL_ENTERED = threading.Event()
_MODEL_RELEASE = threading.Event()


class _GatedChatModel(_StubChatModel):
    """Pause the first real model invocation until the socket subscribes."""

    def _generate(self, messages, stop=None, run_manager=None, **kwargs):
        _MODEL_ENTERED.set()
        if not _MODEL_RELEASE.wait(timeout=30):
            raise RuntimeError(
                "research realtime integration test never released the LLM"
            )
        return super()._generate(
            messages, stop=stop, run_manager=run_manager, **kwargs
        )


class _GatedLLMFactory:
    def __call__(
        self, model_name=None, temperature=None, settings_snapshot=None
    ):
        return _GatedChatModel()


@pytest.fixture(autouse=True)
def _isolated_socketio_state():
    """Keep the lifespan-bound loop and socket maps local to this test.

    ``socketio_asgi.init_lock`` only creates a lock when ``_lock`` is ``None``.
    Clearing it before entering TestClient's lifespan prevents a lock left by
    another test's now-closed portal from making this test order-dependent.
    """

    saved = (
        dict(socketio_asgi._sid_users),
        dict(socketio_asgi._sid_sessions),
        {
            key: set(value)
            for key, value in socketio_asgi._subscriptions.items()
        },
        socketio_asgi._lock,
        socketio_asgi._main_loop,
    )
    socketio_asgi._sid_users.clear()
    socketio_asgi._sid_sessions.clear()
    socketio_asgi._subscriptions.clear()
    socketio_asgi._lock = None
    yield
    users, sessions, subscriptions, lock, loop = saved
    socketio_asgi._sid_users.clear()
    socketio_asgi._sid_users.update(users)
    socketio_asgi._sid_sessions.clear()
    socketio_asgi._sid_sessions.update(sessions)
    socketio_asgi._subscriptions.clear()
    socketio_asgi._subscriptions.update(subscriptions)
    socketio_asgi._lock = lock
    socketio_asgi._main_loop = loop


@pytest.fixture
def gated_journey_llm(journey_stubs):  # noqa: F811 -- imported fixture name
    """Replace the journey fixture's immediate LLM with the gated variant."""

    _MODEL_ENTERED.clear()
    _MODEL_RELEASE.clear()
    register_llm(PROVIDER, _GatedLLMFactory())
    try:
        yield
    finally:
        # A failure between worker start and the explicit release must never
        # strand a daemon research thread in this fixture's temporary DB.
        _MODEL_RELEASE.set()


def _open_socket(client: TestClient, session_cookie: str) -> EioSession:
    session = EioSession(client, session_cookie)
    ack = session.open()
    assert ack.startswith("40"), f"Socket.IO handshake was refused: {ack!r}"
    assert session.sio_sid, f"handshake returned no Socket.IO sid: {ack!r}"
    return session


def _reply_to_engine_ping(session: EioSession, packet: str) -> None:
    """Keep a polling session alive if a bounded wait crosses pingInterval."""

    response = session.client.post(
        f"/ws/socket.io/?EIO=4&transport=polling&sid={session.eio_sid}",
        content="3" + packet[1:],
        headers={"Content-Type": "text/plain"},
    )
    assert response.status_code == 200, (
        f"Engine.IO PONG was rejected: {response.status_code} {response.text!r}"
    )


def _wait_for_worker_completion_event(
    session: EioSession, research_id: str, timeout: float = 30
) -> tuple[list, list[list]]:
    """Read real event frames until the worker's ``phase=complete`` emit.

    Returns the matching event and every event observed, so a failure names
    what did cross the wire instead of presenting only a timeout.
    """

    expected_name = f"research_progress_{research_id}"
    seen: list[list] = []
    deadline = time.monotonic() + timeout

    while time.monotonic() < deadline:
        raw = session.poll_raw()
        for packet in raw.split(_EIO_SEP):
            if packet.startswith("42"):
                event = json.loads(packet[2:])
                seen.append(event)
                if (
                    event[0] == expected_name
                    and isinstance(event[1], dict)
                    and event[1].get("phase") == "complete"
                ):
                    return event, seen
            elif packet.startswith("2"):
                _reply_to_engine_ping(session, packet)

    pytest.fail(
        "the research worker completed no real-time delivery within "
        f"{timeout}s; events observed: {seen!r}"
    )


def test_research_worker_delivers_completion_over_real_socketio(
    app, gated_journey_llm
):
    """POST -> worker thread -> lifespan loop -> mounted Socket.IO -> poll.

    The LLM-entry assertion is the positive control that the real background
    worker reached its model boundary. The subscription-map assertion proves
    the real ``subscribe_to_research`` handler accepted this user/research
    pair before the gate is released. The final assertion is on bytes decoded
    from Engine.IO, not on a mocked emitter or module-local call record.
    """

    client = _new_client(app)
    with client:
        username = _register_and_login(client)
        research_id = _start_research(
            client, "journey realtime worker delivery"
        )
        # Keep the actual thread while the model gate holds it in the registry.
        # Production cleanup removes that entry before the worker has exited.
        worker = get_research_field(research_id, "thread")

        try:
            assert isinstance(worker, threading.Thread), (
                "research start registered no background worker thread"
            )
            assert _MODEL_ENTERED.wait(timeout=30), (
                "POST /api/start_research returned an id, but its background "
                "worker never reached the deterministic LLM boundary"
            )

            session_cookie = client.cookies.get("session")
            assert session_cookie, "login established no signed session cookie"
            socket = _open_socket(client, session_cookie)

            socket.emit("subscribe_to_research", {"research_id": research_id})
            subscription_key = socketio_asgi._subscription_key(
                username, research_id
            )
            assert _wait_until(
                lambda: (
                    socket.sio_sid
                    in socketio_asgi._subscriptions.get(subscription_key, set())
                ),
                timeout=10,
            ), (
                "the real subscribe handler never registered the authenticated "
                f"socket; subscriptions={socketio_asgi._subscriptions!r}"
            )

            # Only now may the worker produce its remaining progress callbacks.
            _MODEL_RELEASE.set()
            completion, seen = _wait_for_worker_completion_event(
                socket, research_id
            )

            event_name, payload = completion
            assert event_name == f"research_progress_{research_id}"
            assert payload["message"] == "Research completed successfully", seen
            assert payload["progress"] == 100, seen
            progress_values = [
                observed["progress"]
                for name, observed in seen
                if name == event_name
                and isinstance(observed, dict)
                and observed.get("progress") is not None
            ]
            # Monotonic, and 100 only on the terminal frame: an earlier 100
            # would end the run in the UI (chat.js treats >= 100 as done).
            assert progress_values == sorted(progress_values), seen
            assert all(0 <= value < 100 for value in progress_values[:-1]), seen

            status = _poll_until_terminal(client, research_id)
            assert status["status"] == "completed", status
            # The HTTP status handler reads ResearchHistory from SQLCipher,
            # independently of the in-memory value delivered over the socket.
            assert status["progress"] == 100, status
        finally:
            _MODEL_RELEASE.set()
            if isinstance(worker, threading.Thread):
                # Completion is emitted before queue/database cleanup. Join
                # before closing the lifespan or restoring fixtures, including
                # on assertion failures. A room disappearing alone does not
                # prove the worker has finished using those resources.
                worker.join(timeout=60)
                assert not worker.is_alive(), (
                    "research worker did not finish after releasing the model"
                )

        # Subscription removal is scheduled onto the still-running lifespan
        # loop. Preserve this assertion separately from waiting for the worker.
        assert _wait_until(
            lambda: subscription_key not in socketio_asgi._subscriptions,
            timeout=15,
        ), "research worker did not finish subscription cleanup"