Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

12.6 KB
Raw Blame
"""Pins for a KNOWN, documented gap in the queue-drain path when
``web.queue_processor.enabled=false``.

Background (see ``QueueProcessorV2._drain_pending_operations`` and the
``Queue processor v2 disabled`` branch in
``local_deep_research.web.fastapi_app.lifespan``):

Research worker threads that cannot reach a user's encrypted database
directly (e.g. the password lookup failed) fall back to
``QueueProcessorV2.queue_progress_update`` / ``queue_error_update``, which
only append an entry to the in-memory ``pending_operations`` dict — they
never touch the database themselves. The ONLY code that ever drains that
dict is ``_drain_pending_operations``, and it only ever runs on the
background thread spawned by ``QueueProcessorV2.start()``. Under Flask, a
second always-on ``before_request`` hook also drained the queue on every
HTTP request; the FastAPI port has no equivalent.

So when ``web.queue_processor.enabled=false``, ``start()`` is never
called: nothing ever drains ``pending_operations``, and a terminal FAILED
status queued on that fallback path is silently dropped once
``_evict_stale_pending_operations`` reaps it past
``_PENDING_OPS_TTL_SECONDS`` (24h). This is a real, currently-accepted
trade-off (see the WARNING logged at startup) — these tests pin the
CURRENT behaviour precisely so it cannot regress further/silently, and so
a deliberate fix (e.g. adding a second drain path, or persisting
synchronously) has to consciously update this file rather than trip over
an unrelated test.
"""

import ast
import time
from pathlib import Path

import local_deep_research.web.fastapi_app as fastapi_app_module
import local_deep_research.web.queue.processor_v2 as processor_v2_module
from local_deep_research.web.queue.processor_v2 import QueueProcessorV2


# ---------------------------------------------------------------------------
# (a) _drain_pending_operations must be the ONLY caller of
#     process_pending_operations_for_user anywhere in the package.
# ---------------------------------------------------------------------------


def _find_calls_to(method_name: str, py_files) -> list[tuple[str, str, int]]:
    """Return ``(file, enclosing_function, lineno)`` for every call whose
    callee's final attribute/name is ``method_name``, across ``py_files``.

    AST-based (not a text grep) so it isn't fooled by the method name
    appearing in a comment or docstring, and isn't tripped up by
    formatting changes.
    """
    calls = []
    for path in py_files:
        source = path.read_text()
        if method_name not in source:
            continue  # cheap prefilter before paying for a parse
        tree = ast.parse(source, filename=str(path))

        func_stack: list[str] = []

        class _Visitor(ast.NodeVisitor):
            def visit_FunctionDef(self, node):  # noqa: N802
                func_stack.append(node.name)
                self.generic_visit(node)
                func_stack.pop()

            visit_AsyncFunctionDef = visit_FunctionDef

            def visit_Call(self, node):  # noqa: N802
                func = node.func
                name = (
                    func.attr
                    if isinstance(func, ast.Attribute)
                    else func.id
                    if isinstance(func, ast.Name)
                    else None
                )
                if name == method_name:
                    calls.append(
                        (
                            str(path),
                            func_stack[-1] if func_stack else "<module>",
                            node.lineno,
                        )
                    )
                self.generic_visit(node)

        _Visitor().visit(tree)
    return calls


def test_drain_pending_operations_is_the_sole_caller():
    """``process_pending_operations_for_user`` must have exactly one call
    site in the whole package: inside ``_drain_pending_operations``.

    If this fails because a NEW call site was added deliberately (e.g. a
    second drain path was reinstated, closing this gap), update this
    assertion as part of that change — don't just delete the test.
    """
    package_root = Path(processor_v2_module.__file__).resolve().parents[2]
    py_files = sorted(package_root.rglob("*.py"))
    assert py_files, "sanity: package scan must find source files"

    calls = _find_calls_to("process_pending_operations_for_user", py_files)

    # Exclude the `def process_pending_operations_for_user(...)` signature
    # itself, which visit_Call never matches (a def isn't a Call node), so
    # every entry here is a genuine call site.
    assert len(calls) == 1, (
        "Expected exactly one caller of process_pending_operations_for_user "
        f"(the drain gap assumes it's the sole consumer of pending_operations); "
        f"found {len(calls)}: {calls}"
    )
    (only_call,) = calls
    _file, enclosing_function, _lineno = only_call
    assert enclosing_function == "_drain_pending_operations", (
        "process_pending_operations_for_user's sole caller must be "
        f"_drain_pending_operations; found it inside {enclosing_function!r} "
        f"instead. If a second, deliberate drain path was added, update "
        f"this test to reflect the new invariant."
    )


# ---------------------------------------------------------------------------
# (b) With the processor NOT started, a queued error update is never
#     drained and is eventually dropped by TTL eviction.
# ---------------------------------------------------------------------------


def test_error_update_is_never_drained_when_processor_not_started():
    """KNOWN LIMITATION — not desired behaviour, just the documented
    status quo.

    ``queue_error_update`` only writes into the in-memory
    ``pending_operations`` dict; it never opens a database session
    itself. The only thing that ever moves an entry out of that dict is
    ``_drain_pending_operations``, running on the thread ``start()``
    spawns. This processor is deliberately never started (mirroring
    ``web.queue_processor.enabled=false``), so the queued FAILED status
    below has no path to persistence: it just sits in memory.
    """
    processor = QueueProcessorV2()
    assert processor.running is False
    assert processor.thread is None  # no drain thread was ever spawned

    processor.queue_error_update(
        username="orphaned-worker-user",
        research_id="r-gap-2",
        status="failed",
        error_message="db password unavailable to worker thread",
        metadata={},
        completed_at="2026-01-01T00:00:00",
    )

    # The write path is purely in-memory — pinning that no DB access
    # happened as a side effect of queuing (i.e. it did NOT get persisted
    # via some other route we're not aware of).
    assert len(processor.pending_operations) == 1
    (op,) = processor.pending_operations.values()
    assert op["operation_type"] == "error_update"
    assert op["status"] == "failed"
    assert op["research_id"] == "r-gap-2"

    # Still nothing running to drain it: the gap is that this state is
    # terminal, not transient.
    assert processor.running is False
    assert processor.thread is None
    assert len(processor.pending_operations) == 1


def test_undrained_error_update_is_silently_dropped_by_ttl_eviction():
    """Completes the pin above: because nothing ever drains it, the ONLY
    thing that ever removes the queued FAILED status is TTL eviction —
    i.e. the terminal status is not merely delayed, it is lost outright.

    Backdates the queued operation's timestamp past
    ``_PENDING_OPS_TTL_SECONDS`` and calls the eviction routine directly
    (the same routine ``queue_progress_update``/``queue_error_update``
    invoke on every write) to show the entry vanishes with no DB write
    ever having occurred.
    """
    processor = QueueProcessorV2()
    processor.queue_error_update(
        username="orphaned-worker-user",
        research_id="r-gap-2-ttl",
        status="failed",
        error_message="db password unavailable to worker thread",
        metadata={},
        completed_at="2026-01-01T00:00:00",
    )
    (op_id,) = processor.pending_operations.keys()

    # Backdate past the TTL, exactly like a real entry aging out because
    # no request/drain thread ever touched it.
    processor.pending_operations[op_id]["timestamp"] = (
        time.time() - processor._PENDING_OPS_TTL_SECONDS - 1
    )

    with processor._pending_operations_lock:
        processor._evict_stale_pending_operations()

    assert op_id not in processor.pending_operations, (
        "expected the stale, never-drained operation to be evicted by TTL"
    )
    assert processor.pending_operations == {}


# ---------------------------------------------------------------------------
# (c) The startup path must WARN (not merely INFO-log) when the processor
#     is disabled, and the message must name the consequence.
# ---------------------------------------------------------------------------


def _find_lifespan_if_else(test_name: str) -> ast.If:
    """Locate the ``if <test_name>: ... else: ...`` statement directly
    inside ``lifespan`` whose test is the bare name ``test_name``.
    """
    source = Path(fastapi_app_module.__file__).read_text()
    tree = ast.parse(source, filename=fastapi_app_module.__file__)

    lifespan_node = None
    for node in ast.walk(tree):
        if isinstance(node, ast.AsyncFunctionDef) and node.name == "lifespan":
            lifespan_node = node
            break
    assert lifespan_node is not None, "lifespan() not found in fastapi_app.py"

    for node in ast.walk(lifespan_node):
        if (
            isinstance(node, ast.If)
            and isinstance(node.test, ast.Name)
            and node.test.id == test_name
        ):
            return node
    raise AssertionError(
        f"No `if {test_name}: ... else: ...` found inside lifespan()"
    )


def _log_call_in(stmts: list[ast.stmt]) -> tuple[str, str]:
    """Given a branch's statement list, return (log_level, joined_message)
    for the single top-level ``logger.<level>(...)`` call in it (other
    top-level statements, e.g. ``queue_processor.start()``, are ignored).
    """
    logger_calls = [
        s.value
        for s in stmts
        if isinstance(s, ast.Expr)
        and isinstance(s.value, ast.Call)
        and isinstance(s.value.func, ast.Attribute)
        and isinstance(s.value.func.value, ast.Name)
        and s.value.func.value.id == "logger"
    ]
    assert len(logger_calls) == 1, (
        f"expected exactly one top-level logger.*(...) call in branch, "
        f"got {len(logger_calls)}: {stmts}"
    )
    call = logger_calls[0]
    level = call.func.attr
    # The message may be a single string literal or an implicitly
    # concatenated ast.Constant (Python folds adjacent literals at parse
    # time), or an f-string (JoinedStr) — handle the literal case, which
    # is what this branch actually uses.
    (arg,) = call.args
    assert isinstance(arg, ast.Constant) and isinstance(arg.value, str), (
        "expected a plain string literal log message"
    )
    return level, arg.value


def test_queue_processor_disabled_branch_logs_warning_naming_consequence():
    """Pins that the disabled-processor branch in ``lifespan`` uses
    ``logger.warning`` (elevated deliberately above the enabled branch's
    ``logger.info``) and that the message names the actual consequence
    (queued research won't dispatch; pending-operations updates won't
    persist). AST-based against the live source rather than a runtime
    boot, since fully exercising ``lifespan()`` would require starting/
    stopping unrelated real subsystems (news scheduler, connection-
    cleanup APScheduler) with no bearing on this gap.

    A change back to ``logger.info``, or a message that drops the
    consequence language, should fail this test.
    """
    if_node = _find_lifespan_if_else("queue_processor_enabled")

    enabled_level, _enabled_msg = _log_call_in(if_node.body)
    disabled_level, disabled_msg = _log_call_in(if_node.orelse)

    assert enabled_level == "info", (
        f"expected the enabled branch to log at info, found {enabled_level!r}"
    )
    assert disabled_level == "warning", (
        "expected the disabled-processor branch to log at WARNING "
        f"(not {enabled_level!r}'s level); found {disabled_level!r} instead"
    )

    lowered = disabled_msg.lower()
    assert "disabled" in lowered
    # Names BOTH halves of the consequence: queued research is orphaned,
    # AND the pending-operations fallback silently loses updates.
    assert "not be dispatched" in lowered, (
        "warning message should name that queued research won't dispatch"
    )
    assert "not be persisted" in lowered or "not persisted" in lowered, (
        "warning message should name that pending-operations updates "
        "won't be persisted"
    )