Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

29.4 KB
Raw Blame
"""Contract tests for the research queue processor (``queue/processor_v2``).

These pin the properties the dispatcher must hold that the existing
``tests/web/queue/`` suite exercises only against ``unittest.mock``
sessions or a single in-memory SQLite connection:

* a queued research is claimed **exactly once** when two workers race,
  proven against a **real on-disk** SQLite file with two independent
  ``Session`` objects.  An in-memory SQLite database is private to its
  connection, so two sessions can never observe each other's committed
  claim and any "exactly once" assertion built on one passes vacuously.
* the per-user concurrency limit read at the dispatch site is the value
  returned by ``clamp_user_max_concurrent`` (not the raw, possibly
  inflated, stored setting).
* a run abandoned by a crash between the pre-spawn commit and the
  queue-row delete is reclaimed and re-dispatched rather than left stuck
  forever, while a genuinely in-flight claim is left alone.
* every dispatch-path DB access is scoped to the owning user.
* a logged-out user's queue tick never re-opens their decrypted
  database and never resumes their research.

Nothing here runs an LLM or a search: ``start_research_process`` (the
process/thread spawn boundary) is always stubbed.

Source under test: ``src/local_deep_research/web/queue/processor_v2.py``
"""

import threading
from contextlib import contextmanager
from unittest.mock import ANY, Mock, patch

from sqlalchemy import create_engine, event
from sqlalchemy.orm import sessionmaker

from local_deep_research.constants import ResearchStatus
from local_deep_research.database.models.active_research import (
    UserActiveResearch,
)
from local_deep_research.database.models.queue import QueueStatus, TaskMetadata
from local_deep_research.database.models.queued_research import QueuedResearch
from local_deep_research.database.models.research import ResearchHistory
from local_deep_research.database.session_passwords import (
    session_password_store,
)
from local_deep_research.web.queue.processor_v2 import QueueProcessorV2
from local_deep_research.web.routes.research_validation import MAX_QUERY_LENGTH

PROCESSOR_MODULE = "local_deep_research.web.queue.processor_v2"
RESEARCH_SERVICE = "local_deep_research.web.services.research_service"

# The dispatch SELECT in ``_start_queued_researches``; used to hook the
# exact instant between "which rows look claimable" and "claim one".
_DISPATCH_SELECT_MARKERS = ("FROM queued_researches", "ORDER BY", "LIMIT")

_TABLES = (
    QueuedResearch,
    QueueStatus,
    TaskMetadata,
    ResearchHistory,
    UserActiveResearch,
)


def _ondisk_engine(tmp_path, name="user.db"):
    """A real on-disk SQLite DB.

    On-disk (not ``:memory:``) is load-bearing: the concurrency tests
    need two connections that can see each other's committed writes.
    """
    engine = create_engine(
        f"sqlite:///{tmp_path / name}",
        connect_args={"timeout": 20},
    )
    for model in _TABLES:
        model.__table__.create(engine, checkfirst=True)
    return engine


def _seed_queued(
    session,
    username,
    research_id,
    *,
    position=1,
    query="what is a queue",
    status=ResearchStatus.QUEUED,
    is_processing=False,
    with_parent=True,
    with_metadata=True,
):
    session.add(
        QueuedResearch(
            username=username,
            research_id=research_id,
            query=query,
            mode="quick",
            position=position,
            is_processing=is_processing,
            # New-style snapshot with an empty submission => no runtime
            # overrides, so validate_search_overrides is a no-op and the
            # test exercises dispatch rather than override validation.
            settings_snapshot={"submission": {}, "settings_snapshot": {}},
        )
    )
    if with_parent:
        session.add(
            ResearchHistory(
                id=research_id,
                query=query,
                mode="quick",
                status=status,
                created_at="2026-01-01T00:00:00+00:00",
            )
        )
    if with_metadata:
        session.add(
            TaskMetadata(
                task_id=research_id, status="queued", task_type="research"
            )
        )


def _recording_start_research(calls, lock=None):
    """Stand in for ``_start_research`` and record every dispatch."""
    lock = lock or threading.Lock()

    def _start(db_session, username, password, queued_research):
        with lock:
            calls.append((username, queued_research.research_id))

    return _start


# ---------------------------------------------------------------------
# 1. Exactly-once claim under a real two-connection race
# ---------------------------------------------------------------------


def test_concurrent_dispatch_claims_a_queued_research_exactly_once(tmp_path):
    """Two workers that both SELECT the same claimable row must not both
    start it.

    The barrier holds each worker at the instant *after* its dispatch
    SELECT and *before* its claiming UPDATE, so both workers genuinely
    believe the row is unclaimed.  Only the conditional
    ``UPDATE ... WHERE is_processing = 0 AND EXISTS(parent QUEUED)``
    can break the tie; a plain read-then-assign cannot.
    """
    engine = _ondisk_engine(tmp_path)
    make_session = sessionmaker(bind=engine)

    seed = make_session()
    _seed_queued(seed, "alice", "race-1")
    seed.add(QueueStatus(active_tasks=0, queued_tasks=1))
    seed.commit()
    seed.close()

    barrier = threading.Barrier(2, timeout=30)
    already_synced = set()
    sync_lock = threading.Lock()
    barrier_breaks = []

    @event.listens_for(engine, "after_cursor_execute")
    def _pause_between_select_and_claim(
        conn, cursor, statement, parameters, context, executemany
    ):
        if not all(m in statement for m in _DISPATCH_SELECT_MARKERS):
            return
        ident = threading.get_ident()
        with sync_lock:
            if ident in already_synced:
                return
            already_synced.add(ident)
        try:
            barrier.wait()
        except threading.BrokenBarrierError as exc:  # pragma: no cover
            barrier_breaks.append(exc)

    started = []
    started_lock = threading.Lock()
    worker_errors = []

    def _worker():
        processor = QueueProcessorV2()
        processor._start_research = _recording_start_research(
            started, started_lock
        )
        session = make_session()
        try:
            processor._start_queued_researches(
                session, Mock(), "alice", "pw", 1
            )
        except BaseException as exc:  # pragma: no cover - reported below
            worker_errors.append(f"{type(exc).__name__}: {exc}")
        finally:
            session.close()

    threads = [threading.Thread(target=_worker) for _ in range(2)]
    for thread in threads:
        thread.start()
    for thread in threads:
        thread.join(60)

    event.remove(
        engine, "after_cursor_execute", _pause_between_select_and_claim
    )

    assert not [t for t in threads if t.is_alive()], "worker deadlocked"
    assert not worker_errors, worker_errors
    # Both workers really did reach the pre-claim window; without this the
    # test would "pass" simply because the second worker ran after the
    # first had already finished and deleted the row.
    assert len(already_synced) == 2, (
        "both workers must have raced through the dispatch SELECT; "
        f"only {len(already_synced)} did"
    )
    assert not barrier_breaks, barrier_breaks

    assert started == [("alice", "race-1")], (
        f"queued research must be started exactly once, got {started}"
    )

    check = make_session()
    try:
        assert check.query(QueuedResearch).count() == 0, (
            "the winning worker must consume the queue row"
        )
    finally:
        check.close()
        engine.dispose()


# ---------------------------------------------------------------------
# 2. Per-user concurrency limit is the clamped one
# ---------------------------------------------------------------------


@contextmanager
def _fake_user_db_session(session):
    yield session


def _run_user_queue_tick(processor, session, engine, *, username, session_id):
    """Drive ``_process_user_queue`` against a real session/engine.

    Returns ``(result, open_db_mock, get_session_mock)`` so callers can
    assert on whether the encrypted DB was opened at all.
    """
    open_db = Mock(return_value=engine)
    get_session = Mock(
        side_effect=lambda *a, **kw: _fake_user_db_session(session)
    )
    with (
        patch(f"{PROCESSOR_MODULE}.db_manager") as db_manager,
        patch(f"{PROCESSOR_MODULE}.get_user_db_session", get_session),
    ):
        db_manager.open_user_database = open_db
        result = processor._process_user_queue(username, session_id)
    return result, open_db, get_session


def test_dispatch_uses_the_clamped_per_user_max_concurrent(
    tmp_path, monkeypatch
):
    """A user whose stored ``app.max_concurrent_researches`` predates the
    schema cap must not be able to dispatch past the global ceiling.

    Five researches are queued, the stored per-user limit is 1000 and the
    server ceiling is 2, so exactly 2 may start this tick.
    """
    engine = _ondisk_engine(tmp_path)
    make_session = sessionmaker(bind=engine)
    session = make_session()
    for index in range(5):
        _seed_queued(session, "alice", f"clamp-{index}", position=index + 1)
    session.add(QueueStatus(active_tasks=0, queued_tasks=5))
    session.commit()

    monkeypatch.setattr(f"{RESEARCH_SERVICE}._MAX_GLOBAL_CONCURRENT", 2)

    settings_manager = Mock()
    settings_manager.get_setting.return_value = 1000
    monkeypatch.setattr(
        "local_deep_research.settings.manager.SettingsManager",
        Mock(return_value=settings_manager),
    )

    session_password_store.store_session_password("alice", "sess-clamp", "pw")
    processor = QueueProcessorV2()
    started = []
    processor._start_research = _recording_start_research(started)
    try:
        _run_user_queue_tick(
            processor,
            session,
            engine,
            username="alice",
            session_id="sess-clamp",
        )
    finally:
        session_password_store.clear_session("alice", "sess-clamp")

    settings_manager.get_setting.assert_any_call(
        "app.max_concurrent_researches", 3
    )
    assert len(started) == 2, (
        "dispatch must honour the clamped ceiling (2), not the raw stored "
        f"value (1000); started {len(started)}: {started}"
    )
    # Position order decides which two go first.
    assert [rid for _, rid in started] == ["clamp-0", "clamp-1"]

    remaining = {
        row.research_id: row.is_processing
        for row in session.query(QueuedResearch).all()
    }
    assert remaining == {
        "clamp-2": False,
        "clamp-3": False,
        "clamp-4": False,
    }, "unstarted rows must stay unclaimed and re-dispatchable"

    session.close()
    engine.dispose()


def test_dispatch_capacity_counts_live_direct_researches_not_queue_counter(
    tmp_path, monkeypatch
):
    """A direct run occupies a user slot even though QueueStatus missed it.

    ``QueueStatus.active_tasks`` is deliberately zero here.  The live
    ``UserActiveResearch`` row is the cross-entry-point source of truth used
    by the fresh-submit path, so the queued row must remain undispatched when
    the user's cap is one.
    """
    engine = _ondisk_engine(tmp_path)
    session = sessionmaker(bind=engine)()
    _seed_queued(session, "alice", "must-wait")
    session.add_all(
        [
            ResearchHistory(
                id="direct-holder",
                query="already running",
                mode="quick",
                status=ResearchStatus.IN_PROGRESS,
                created_at="2026-01-01T00:00:00+00:00",
            ),
            UserActiveResearch(
                username="alice",
                research_id="direct-holder",
                status=ResearchStatus.IN_PROGRESS,
                thread_id="1234",
            ),
            QueueStatus(active_tasks=0, queued_tasks=1),
        ]
    )
    session.commit()

    settings_manager = Mock()
    settings_manager.get_setting.return_value = 1
    monkeypatch.setattr(
        "local_deep_research.settings.manager.SettingsManager",
        Mock(return_value=settings_manager),
    )

    processor = QueueProcessorV2()
    started = []
    processor._start_research = _recording_start_research(started)
    reclaim = Mock(return_value=False)
    monkeypatch.setattr(
        "local_deep_research.web.routes.globals.reclaim_stale_user_active_research",
        reclaim,
    )
    session_password_store.store_session_password("alice", "sess-live", "pw")
    try:
        result, _, _ = _run_user_queue_tick(
            processor,
            session,
            engine,
            username="alice",
            session_id="sess-live",
        )
    finally:
        session_password_store.clear_session("alice", "sess-live")

    assert result is False, "the non-empty queue must remain scheduled"
    reclaim.assert_called_once_with(session, "alice", logger=ANY)
    assert started == []
    queued = (
        session.query(QueuedResearch).filter_by(research_id="must-wait").one()
    )
    assert queued.is_processing is False

    session.close()
    engine.dispose()


# ---------------------------------------------------------------------
# 3-4. Crash recovery vs. live in-flight claims
# ---------------------------------------------------------------------


def test_crashed_run_is_reclaimed_and_redispatched_not_left_stuck(tmp_path):
    """A row stranded ``is_processing=True`` with an ``IN_PROGRESS``
    parent and no live thread (server killed between the pre-spawn commit
    and the queue-row delete) must be recovered on the next tick.

    Without the reclaim pass the row is invisible to the
    ``is_processing=False`` dispatch filter and the research is stuck
    forever.
    """
    engine = _ondisk_engine(tmp_path)
    make_session = sessionmaker(bind=engine)
    session = make_session()
    _seed_queued(
        session,
        "alice",
        "crashed-1",
        is_processing=True,
        status=ResearchStatus.IN_PROGRESS,
    )
    session.commit()

    processor = QueueProcessorV2()
    started = []
    processor._start_research = _recording_start_research(started)

    processor._start_queued_researches(
        session, Mock(), "alice", "pw", available_slots=3
    )

    assert started == [("alice", "crashed-1")], (
        "the abandoned run must be re-dispatched, not left stranded; "
        f"got {started}"
    )
    assert session.query(QueuedResearch).count() == 0

    session.close()
    engine.dispose()


def test_reclaim_leaves_a_genuinely_live_claim_alone(tmp_path):
    """The same stranded-looking row must NOT be reclaimed while its
    research is registered as active — that would double-start a run
    whose worker is alive."""
    from local_deep_research.web import research_state

    engine = _ondisk_engine(tmp_path)
    make_session = sessionmaker(bind=engine)
    session = make_session()
    _seed_queued(
        session,
        "alice",
        "live-claim-1",
        is_processing=True,
        status=ResearchStatus.IN_PROGRESS,
    )
    session.commit()

    processor = QueueProcessorV2()
    started = []
    processor._start_research = _recording_start_research(started)

    research_state.set_active_research("live-claim-1", {"progress": 1})
    try:
        processor._start_queued_researches(
            session, Mock(), "alice", "pw", available_slots=3
        )
    finally:
        research_state.cleanup_research("live-claim-1")

    assert started == [], "a live in-flight claim must never be re-dispatched"
    row = session.query(QueuedResearch).one()
    assert row.is_processing is True, "the live claim must be preserved"
    parent = session.query(ResearchHistory).one()
    assert parent.status == ResearchStatus.IN_PROGRESS

    session.close()
    engine.dispose()


# ---------------------------------------------------------------------
# 5-6. Per-user scoping of dispatch-path DB access
# ---------------------------------------------------------------------


def test_dispatch_for_one_user_never_touches_another_users_queue_rows(
    tmp_path,
):
    """Dispatching for ``alice`` must neither start nor reclaim any row
    owned by ``bob``, even with slots to spare."""
    engine = _ondisk_engine(tmp_path)
    make_session = sessionmaker(bind=engine)
    session = make_session()
    _seed_queued(session, "alice", "alice-1", position=1)
    _seed_queued(session, "bob", "bob-1", position=1)
    _seed_queued(
        session,
        "bob",
        "bob-stranded",
        position=2,
        is_processing=True,
        status=ResearchStatus.IN_PROGRESS,
    )
    session.add(QueueStatus(active_tasks=0, queued_tasks=3))
    session.commit()

    processor = QueueProcessorV2()
    started = []
    processor._start_research = _recording_start_research(started)

    processor._start_queued_researches(
        session, Mock(), "alice", "pw", available_slots=10
    )

    assert started == [("alice", "alice-1")], (
        f"only alice's row may be dispatched, got {started}"
    )
    bob_rows = {
        row.research_id: row.is_processing
        for row in session.query(QueuedResearch).filter_by(username="bob").all()
    }
    assert bob_rows == {"bob-1": False, "bob-stranded": True}, (
        "bob's rows must be untouched by alice's tick (including his "
        "stranded claim, which only bob's own reclaim pass may revert)"
    )
    bob_parent = (
        session.query(ResearchHistory).filter_by(id="bob-stranded").one()
    )
    assert bob_parent.status == ResearchStatus.IN_PROGRESS

    session.close()
    engine.dispose()


def test_missing_parent_sweep_is_username_scoped(tmp_path):
    """Alice's sweep must not delete bob's orphaned queue row."""
    engine = _ondisk_engine(tmp_path)
    make_session = sessionmaker(bind=engine)
    session = make_session()
    _seed_queued(session, "alice", "alice-ok", position=1)
    # bob's parent ResearchHistory row is missing -> bob's rows are the
    # only orphans in the database.
    _seed_queued(session, "bob", "bob-orphan", position=1, with_parent=False)
    session.add(QueueStatus(active_tasks=0, queued_tasks=2))
    session.commit()

    processor = QueueProcessorV2()
    processor._start_research = _recording_start_research([])

    processor._start_queued_researches(
        session, Mock(), "alice", "pw", available_slots=10
    )

    surviving = {
        row.research_id
        for row in session.query(QueuedResearch).filter_by(username="bob").all()
    }
    try:
        assert surviving == {"bob-orphan"}, (
            "alice's sweep deleted bob's queue row"
        )
    finally:
        session.close()
        engine.dispose()


def _seed_claimed_orphan(session, username, research_id):
    """A claimed queue row whose parent ResearchHistory row is missing."""
    _seed_queued(
        session,
        username,
        research_id,
        position=1,
        is_processing=True,
        with_parent=False,
        with_metadata=False,
    )
    session.add(
        TaskMetadata(
            task_id=research_id, status="processing", task_type="research"
        )
    )


def test_missing_parent_sweep_keeps_another_users_claimed_orphan(tmp_path):
    """Alice's sweep must not delete bob's claimed orphan either.

    The sweep cleans up with ``include_claimed=True``, so a claim does not
    protect a row; only the sweep's username scoping keeps bob's row.
    """
    engine = _ondisk_engine(tmp_path)
    make_session = sessionmaker(bind=engine)
    session = make_session()
    _seed_queued(session, "alice", "alice-ok", position=1)
    _seed_claimed_orphan(session, "bob", "bob-claimed-orphan")
    session.add(QueueStatus(active_tasks=1, queued_tasks=1))
    session.commit()

    processor = QueueProcessorV2()
    processor._start_research = _recording_start_research([])

    processor._start_queued_researches(
        session, Mock(), "alice", "pw", available_slots=10
    )

    try:
        bob_rows = [
            (row.research_id, row.is_processing)
            for row in session.query(QueuedResearch)
            .filter_by(username="bob")
            .all()
        ]
        assert bob_rows == [("bob-claimed-orphan", True)], (
            "alice's sweep deleted bob's claimed queue row"
        )
        assert [
            (task.task_id, task.status)
            for task in session.query(TaskMetadata)
            .filter_by(task_id="bob-claimed-orphan")
            .all()
        ] == [("bob-claimed-orphan", "processing")]
    finally:
        session.close()
        engine.dispose()


def test_missing_parent_sweep_removes_the_users_own_claimed_orphan(tmp_path):
    """A missing parent leaves a claim nothing can dispatch, so alice's own
    claimed orphan is swept together with its metadata."""
    engine = _ondisk_engine(tmp_path)
    make_session = sessionmaker(bind=engine)
    session = make_session()
    _seed_claimed_orphan(session, "alice", "alice-claimed-orphan")
    session.add(QueueStatus(active_tasks=1, queued_tasks=0))
    session.commit()

    processor = QueueProcessorV2()
    processor._start_research = _recording_start_research([])

    processor._start_queued_researches(
        session, Mock(), "alice", "pw", available_slots=10
    )

    try:
        assert (
            session.query(QueuedResearch)
            .filter_by(research_id="alice-claimed-orphan")
            .count()
            == 0
        )
        assert (
            session.query(TaskMetadata)
            .filter_by(task_id="alice-claimed-orphan")
            .count()
            == 0
        )
    finally:
        session.close()
        engine.dispose()


# ---------------------------------------------------------------------
# 7-8. Logout must not resurrect the decrypted DB or resume research
# ---------------------------------------------------------------------


def test_queue_tick_for_a_logged_out_user_never_opens_their_database(
    tmp_path,
):
    """After logout (``clear_all_for_user``) the processor must abandon
    the tick before ``open_user_database`` — re-opening would resurrect
    the user's decrypted database and resume their research."""
    engine = _ondisk_engine(tmp_path)
    make_session = sessionmaker(bind=engine)
    session = make_session()
    _seed_queued(session, "alice", "logout-1")
    session.add(QueueStatus(active_tasks=0, queued_tasks=1))
    session.commit()

    processor = QueueProcessorV2()
    started = []
    processor._start_research = _recording_start_research(started)

    session_password_store.store_session_password("alice", "sess-out", "pw")
    session_password_store.clear_all_for_user("alice")  # <- logout

    result, open_db, get_session = _run_user_queue_tick(
        processor,
        session,
        engine,
        username="alice",
        session_id="sess-out",
    )

    assert open_db.call_count == 0, (
        "logged-out user's encrypted DB must not be re-opened"
    )
    assert get_session.call_count == 0, (
        "no session may be opened against a logged-out user's DB"
    )
    assert started == [], "a logged-out user's research must not resume"
    assert result is True, "the user must be dropped from the check set"
    assert session.query(QueuedResearch).one().is_processing is False

    session.close()
    engine.dispose()


def test_queue_tick_for_a_logged_in_user_does_open_their_database(
    tmp_path, monkeypatch
):
    """Positive control for the test above: with the session password
    still in the store the very same tick opens the DB and dispatches.

    Without this, the logout assertions could hold for any reason at all
    (e.g. the tick short-circuiting on some unrelated condition)."""
    engine = _ondisk_engine(tmp_path)
    make_session = sessionmaker(bind=engine)
    session = make_session()
    _seed_queued(session, "alice", "loggedin-1")
    session.add(QueueStatus(active_tasks=0, queued_tasks=1))
    session.commit()

    settings_manager = Mock()
    settings_manager.get_setting.return_value = 3
    monkeypatch.setattr(
        "local_deep_research.settings.manager.SettingsManager",
        Mock(return_value=settings_manager),
    )

    processor = QueueProcessorV2()
    started = []
    processor._start_research = _recording_start_research(started)

    session_password_store.store_session_password("alice", "sess-in", "pw")
    try:
        result, open_db, get_session = _run_user_queue_tick(
            processor,
            session,
            engine,
            username="alice",
            session_id="sess-in",
        )
    finally:
        session_password_store.clear_session("alice", "sess-in")

    assert open_db.call_count == 1
    assert open_db.call_args.args == ("alice", "pw")
    assert get_session.call_count == 1
    assert started == [("alice", "loggedin-1")]
    assert result is True

    session.close()
    engine.dispose()


# ---------------------------------------------------------------------
# 9-10. Query-length cap is shared by fresh requests and queue replay
# ---------------------------------------------------------------------


def _dispatch_one_with_spawn_stub(session):
    """Run the real ``_start_research`` with only the thread-spawn
    boundary stubbed.  Returns the stub so callers can inspect the
    arguments the worker would have been spawned with."""
    processor = QueueProcessorV2()
    spawn = Mock(return_value=Mock(ident=4242))
    failed = Mock()
    with (
        patch(f"{PROCESSOR_MODULE}.start_research_process", spawn),
        patch.object(processor, "notify_research_failed", failed),
    ):
        processor._start_queued_researches(
            session, Mock(), "alice", "pw", available_slots=1
        )
    return spawn, failed


def test_queue_replay_accepts_a_query_at_the_shared_length_cap(tmp_path):
    """The inclusive boundary remains dispatchable and is not truncated."""
    at_cap = "x" * MAX_QUERY_LENGTH
    engine = _ondisk_engine(tmp_path)
    session = sessionmaker(bind=engine)()
    _seed_queued(session, "alice", "long-1", query=at_cap)
    session.commit()

    spawn, failed = _dispatch_one_with_spawn_stub(session)

    assert spawn.call_count == 1
    spawned_query = spawn.call_args.args[1]
    assert spawned_query == at_cap
    assert len(spawned_query) == MAX_QUERY_LENGTH
    failed.assert_not_called()

    session.close()
    engine.dispose()


def test_queue_replay_enforces_the_same_query_length_cap(tmp_path):
    """An over-cap queued query must not be dispatched to a worker."""
    oversized = "x" * (MAX_QUERY_LENGTH + 1)
    engine = _ondisk_engine(tmp_path)
    session = sessionmaker(bind=engine)()
    _seed_queued(session, "alice", "long-2", query=oversized)
    session.commit()

    spawn, failed = _dispatch_one_with_spawn_stub(session)

    try:
        assert spawn.call_count == 0, (
            "the queue replay path spawned a worker for a query that "
            "/api/start_research would have rejected with HTTP 400"
        )
        failed.assert_called_once()
        assert "maximum length" in failed.call_args.kwargs["error_message"]
        assert (
            session.query(ResearchHistory).filter_by(id="long-2").one().status
            == ResearchStatus.FAILED
        )
        assert (
            session.query(QueuedResearch)
            .filter_by(research_id="long-2")
            .first()
            is None
        )
        assert (
            session.query(UserActiveResearch)
            .filter_by(research_id="long-2")
            .first()
            is None
        )
    finally:
        session.close()
        engine.dispose()


def test_direct_terminal_failure_consumes_all_persisted_queue_state(tmp_path):
    """A terminal direct-spawn failure cannot leave an immortal queue row.

    This drives the real ORM path: the task is first transitioned to
    processing, then spawning fails. History must become FAILED while the
    queued row, task metadata, active-cap row, and both counters are cleared in
    the same cleanup transaction.
    """
    engine = _ondisk_engine(tmp_path)
    session = sessionmaker(bind=engine)()
    _seed_queued(session, "alice", "terminal-direct")
    session.add(QueueStatus(active_tasks=0, queued_tasks=1))
    session.commit()

    processor = QueueProcessorV2()
    with (
        patch(
            f"{PROCESSOR_MODULE}.get_user_db_session",
            side_effect=lambda *_args, **_kwargs: _fake_user_db_session(
                session
            ),
        ),
        patch(
            f"{PROCESSOR_MODULE}.start_research_process",
            side_effect=RuntimeError("spawn unavailable"),
        ) as spawn,
    ):
        outcome = processor._start_research_directly(
            "alice",
            "terminal-direct",
            "pw",
            query="what is a queue",
            mode="quick",
            settings_snapshot={},
        )

    assert outcome.name == "TERMINAL_FAILURE"
    spawn.assert_called_once()
    assert (
        session.query(ResearchHistory)
        .filter_by(id="terminal-direct")
        .one()
        .status
        == ResearchStatus.FAILED
    )
    assert (
        session.query(QueuedResearch)
        .filter_by(research_id="terminal-direct")
        .first()
        is None
    )
    assert (
        session.query(TaskMetadata).filter_by(task_id="terminal-direct").first()
        is None
    )
    assert (
        session.query(UserActiveResearch)
        .filter_by(research_id="terminal-direct")
        .first()
        is None
    )
    queue_status = session.query(QueueStatus).one()
    assert (queue_status.active_tasks, queue_status.queued_tasks) == (0, 0)

    session.close()
    engine.dispose()