Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

6.7 KB
Raw Blame
"""A failed row-drop must leave the session usable for the notification.

Ported from ``tests/web/auth/test_cleanup_middleware.py`` on ``origin/main``
(``test_handles_operational_error``, ``test_handles_pending_rollback_error``,
``test_handles_timeout_error``, ``test_handles_generic_exception``,
``test_handles_rollback_failure``), deleted by the migration along with
``web/auth/cleanup_middleware.py`` itself.

The module is gone, but the WORK it did is not: the periodic sweep that
deleted finished ``UserActiveResearch`` rows was re-expressed as
``QueueProcessorV2._drop_active_research_row``, called from both terminal
notifications (see ``test_active_research_row_cleanup.py``). What did not
survive the move is main's error handling. Main had FIVE ``rollback()``
calls across its except branches — including an explicit
``except (OperationalError, PendingRollbackError, TimeoutError)`` arm — for
one reason: a failed statement leaves the SQLAlchemy session in a state
where every subsequent statement raises ``PendingRollbackError`` until
someone rolls back. The successor has a bare ``except Exception:
logger.exception(...)`` and no rollback at all.

Why that matters more here than it did in main. Main's sweep ran in a
``before_request`` hook — a poisoned session mostly damaged the request it
was already in. The successor is called on the line immediately before

    send_research_completed_notification_from_session(..., db_session=session)

on the SAME session object. So a swallowed failure in the bookkeeping
delete now poisons the very notification the swallow exists to protect: the
helper's own docstring says "Never let bookkeeping break the completion
notification the user is actually waiting on", and without the rollback
that is exactly what it does.

The existing successor test
``test_active_research_row_cleanup.py::test_bookkeeping_failure_never_breaks_the_notification``
does not catch this: it raises a bare ``RuntimeError``, which does not
poison a session, and it never drives the notification afterwards — it only
asserts the helper itself does not re-raise.

STATUS: the first two tests below FAIL on this branch. That is the point —
they are a faithful port of assertions main made, and the branch no longer
satisfies them. Do not weaken them.
"""

from unittest.mock import MagicMock

import pytest
from sqlalchemy.exc import OperationalError, PendingRollbackError

from local_deep_research.web.queue.processor_v2 import QueueProcessorV2


class _Session:
    """Fake session whose failed statement poisons it until rollback.

    This is the behaviour that makes the missing rollback consequential,
    so it is modelled rather than asserted about: once ``_broken`` is set,
    every ``query()`` raises ``PendingRollbackError`` — which is what a real
    SQLAlchemy session does — and only ``rollback()`` clears it.
    """

    def __init__(self, first_error):
        self._first_error = first_error
        self._broken = False
        self.rolled_back = False
        self.committed = False

    def query(self, _model):
        if self._first_error is not None:
            err, self._first_error = self._first_error, None
            self._broken = True
            raise err
        if self._broken:
            raise PendingRollbackError(
                "This Session's transaction has been rolled back due to a "
                "previous exception during flush."
            )
        return self

    def filter_by(self, **_kwargs):
        return self

    def delete(self, synchronize_session=False):  # noqa: ARG002
        return 1

    def commit(self):
        self.committed = True

    def rollback(self):
        self.rolled_back = True
        self._broken = False


@pytest.mark.parametrize(
    ("label", "error"),
    [
        ("OperationalError", OperationalError("test", {}, None)),
        ("PendingRollbackError", PendingRollbackError("pending rollback")),
        ("TimeoutError", TimeoutError("pool timed out")),
        ("generic Exception", RuntimeError("db gone")),
    ],
)
def test_failed_row_drop_rolls_the_session_back(label, error):
    """Main rolled back on every one of these. The successor rolls back on
    none, so the caller's next statement on the same session dies."""
    session = _Session(first_error=error)

    QueueProcessorV2._drop_active_research_row(session, "alice", "r-1")

    assert session.rolled_back, (
        f"a {label} during the row drop left the session un-rolled-back; "
        "the completion notification that runs next on this same session "
        "will fail with PendingRollbackError"
    )


def test_completion_notification_survives_a_failed_row_drop():
    """The consequence, end to end on one session.

    ``notify_research_completed`` drops the row and then sends the
    notification through the SAME session. If the drop poisons the session
    and nobody rolls back, the user never gets told their research finished.
    """
    from local_deep_research.web.queue import processor_v2 as mod

    session = _Session(first_error=PendingRollbackError("pending rollback"))
    ctx = MagicMock()
    ctx.__enter__.return_value = session
    ctx.__exit__.return_value = False

    notified = {}

    def _notify(username, research_id, db_session):
        # A real notification helper reads from the session it is handed.
        db_session.query(object)
        notified["ok"] = (username, research_id)

    with pytest.MonkeyPatch.context() as mp:
        mp.setattr(mod, "get_user_db_session", lambda *a, **k: ctx)
        mp.setattr(mod, "UserQueueService", MagicMock())
        mp.setattr(
            mod, "send_research_completed_notification_from_session", _notify
        )
        mp.setattr(
            "local_deep_research.research_library.search.services"
            ".research_history_indexer.auto_convert_research",
            lambda *a, **k: None,
        )
        QueueProcessorV2().notify_research_completed(
            username="alice", research_id="r-1"
        )

    assert notified.get("ok") == ("alice", "r-1"), (
        "the completion notification did not go out: the failed "
        "UserActiveResearch delete poisoned the shared session and "
        "_drop_active_research_row swallowed the error without rolling "
        "back, so the notification's own query raised too"
    )


def test_rollback_failure_is_itself_swallowed():
    """Main's ``test_handles_rollback_failure``: if the rollback ALSO fails
    (a genuinely exhausted pool), the helper still must not raise — the
    caller is on the user-visible completion path."""
    session = _Session(first_error=RuntimeError("db gone"))
    session.rollback = MagicMock(side_effect=Exception("rollback failed"))

    QueueProcessorV2._drop_active_research_row(session, "alice", "r-1")