Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

33.2 KB
Raw Blame
"""Do settings saved through the real HTTP API actually change behaviour?

Complement to ``tests/test_advertised_but_dead_sweep.py``, which proved
*statically* that a set of settings keys has no reader anywhere in ``src/``.
This module asks the *behavioural* question for the settings that DO have
readers:

    save the setting through the real HTTP API, then exercise the feature,
    and assert the DOWNSTREAM CONSUMER saw the new value.

A GET-echoes-PUT round trip proves nothing; every test here records the call
arguments on a stub planted at the consumer boundary and asserts on those.
Every test carries a CONTROL: the setting is written twice, with two
different values, and the consumer must report both — a test that only pins
one value would pass against a hardcoded constant.

This is where a WSGI->ASGI port regresses silently: Flask's request-scoped
``g``/thread-locals became snapshots passed into anyio threadpool workers and
spawned threads, so a setting can be saved correctly, read correctly on the
request thread, and never reach the background research thread.
"""

from __future__ import annotations

import pytest


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def put_setting(client, key: str, value):
    """Save a setting through the real HTTP API. Fails loudly on rejection."""
    resp = client.put(f"/settings/api/{key}", json={"value": value})
    assert resp.status_code == 200, (
        f"PUT /settings/api/{key} = {value!r} -> {resp.status_code} "
        f"{resp.text[:400]}"
    )
    return resp


class ThreadBoundaryRecorder:
    """Stand-in for ``run_research_process`` planted in the router module.

    ``start_research_process`` is left REAL: it acquires the global
    concurrency semaphore, copies the request contextvars and spawns the
    background ``threading.Thread``. This recorder is what that thread
    actually executes, so everything it records was observed *on the
    research thread*, on the far side of the ASGI threadpool -> thread
    hand-off that the migration rewired.
    """

    def __init__(self, on_call=None):
        import threading

        self.calls = []
        self.errors = []
        self.on_call = on_call
        self.done = threading.Event()
        # When set, the recorder parks the research thread until released,
        # so the run counts as genuinely "active" for concurrency checks.
        self.hold = threading.Event()
        self.hold.set()

    def __call__(self, research_id, query, mode, **kwargs):
        import threading

        self.calls.append(
            {
                "research_id": research_id,
                "query": query,
                "mode": mode,
                "kwargs": kwargs,
                "thread_name": threading.current_thread().name,
                "is_main_thread": threading.current_thread()
                is threading.main_thread(),
            }
        )
        # Deliberately NOT wrapped in a bare ``except``: a failure inside
        # the production reader must surface, not be swallowed.
        try:
            if self.on_call is not None:
                self.on_call(self.calls[-1])
        except BaseException as exc:  # noqa: BLE001 - re-raised in the test
            self.errors.append(exc)
        finally:
            self.done.set()
        self.hold.wait(60)

    @property
    def last(self):
        assert self.calls, "run_research_process was never called"
        return self.calls[-1]

    def wait(self, timeout=20.0):
        assert self.done.wait(timeout), (
            "background research thread never invoked run_research_process"
        )
        self.done.clear()
        if self.errors:
            raise self.errors[-1]


def start_research(client, query="behavioural settings probe"):
    resp = client.post(
        "/api/start_research", json={"query": query, "mode": "quick"}
    )
    assert resp.status_code == 200, (
        f"POST /api/start_research -> {resp.status_code} {resp.text[:400]}"
    )
    return resp.json()


# ---------------------------------------------------------------------------
# 1. Request-thread reads: search.tool / llm.provider / llm.model /
#    search.iterations / search.questions_per_iteration /
#    search.search_strategy
#
# These six are resolved by ``_extract_research_params`` on the request
# thread and then handed to the background thread as explicit kwargs. The
# assertion is on what the THREAD received, not on what GET returns.
# ---------------------------------------------------------------------------

# (setting key, saved value A, saved value B, kwarg name seen by the thread)
REQUEST_THREAD_SETTINGS = [
    ("search.tool", "wikipedia", "arxiv", "search_engine"),
    ("search.iterations", 3, 7, "iterations"),
    ("search.questions_per_iteration", 2, 6, "questions_per_iteration"),
    (
        "search.search_strategy",
        "source-based",
        "topic-organization",
        "strategy",
    ),
    ("llm.model", "model-alpha", "model-beta", "model"),
    ("llm.provider", "anthropic", "openai", "model_provider"),
]


@pytest.mark.parametrize(
    "key,value_a,value_b,kwarg", REQUEST_THREAD_SETTINGS, ids=lambda v: str(v)
)
def test_setting_reaches_research_thread(
    authenticated_client, monkeypatch, key, value_a, value_b, kwarg
):
    """Saved setting -> background research thread kwarg.

    CONTROL: the setting is saved twice with two different values and the
    thread must report BOTH. A single-value check would pass against a
    hardcoded constant.
    """
    from local_deep_research.web.routers import research as research_router

    recorder = ThreadBoundaryRecorder()
    monkeypatch.setattr(
        research_router, "run_research_process", recorder, raising=True
    )

    # /api/start_research refuses to start at all without a configured
    # model, so seed one; the llm.model parametrisation overwrites it below.
    put_setting(authenticated_client, "llm.model", "seed-model")

    seen = []
    for value in (value_a, value_b):
        put_setting(authenticated_client, key, value)
        start_research(authenticated_client, query=f"probe {key} {value}")
        recorder.wait()
        call = recorder.last
        assert not call["is_main_thread"], (
            "run_research_process ran on the main thread; the background "
            "hand-off did not happen and this test would prove nothing"
        )
        seen.append(call["kwargs"].get(kwarg))

    assert seen == [value_a, value_b], (
        f"setting {key!r} did not reach the research thread as "
        f"kwarg {kwarg!r}: saved {[value_a, value_b]}, thread saw {seen}"
    )


# ---------------------------------------------------------------------------
# 2. Snapshot-carried settings: search.max_results / search.region /
#    search.snippets_only / search.search_language / search.safe_search /
#    search.max_filtered_results
#
# These are NOT passed as kwargs. They travel inside the settings snapshot
# captured on the request thread and are read on the RESEARCH THREAD by
# ``config.search_config.get_search`` via ``get_setting_from_snapshot`` --
# exactly the WSGI ``g`` -> ASGI snapshot rewrite. The consumer boundary is
# ``search_engine_factory.get_search``, which receives the resolved params
# as keyword arguments; the stub planted there records them.
# ---------------------------------------------------------------------------

# (setting key, value A, value B, factory kwarg name)
SNAPSHOT_SEARCH_SETTINGS = [
    ("search.max_results", 13, 27, "max_results"),
    ("search.region", "uk", "jp", "region"),
    ("search.snippets_only", False, True, "search_snippets_only"),
    ("search.search_language", "German", "French", "search_language"),
    ("search.safe_search", False, True, "safe_search"),
    ("search.max_filtered_results", 4, 9, "max_filtered_results"),
]


@pytest.mark.parametrize(
    "key,value_a,value_b,factory_kwarg",
    SNAPSHOT_SEARCH_SETTINGS,
    ids=lambda v: str(v),
)
def test_snapshot_setting_reaches_search_engine_factory(
    authenticated_client, monkeypatch, key, value_a, value_b, factory_kwarg
):
    """Saved setting -> settings snapshot -> research thread -> engine factory.

    The production reader (``config.search_config.get_search``) is executed
    ON the background research thread with the snapshot the route captured,
    and the assertion is on the kwargs the search-engine factory received.

    CONTROL: two different saved values, both must be observed downstream.
    """
    from local_deep_research.config import search_config
    from local_deep_research.web.routers import research as research_router

    factory_calls = []

    def _fake_factory(**kwargs):
        factory_calls.append(kwargs)

        class _Engine:
            pass

        return _Engine()

    monkeypatch.setattr(
        search_config, "factory_get_search", _fake_factory, raising=True
    )

    class _StubLLM:
        """Passed in so get_search does not build a real LLM."""

    def _drive_production_reader(call):
        # Runs on the background research thread.
        search_config.get_search(
            search_tool=call["kwargs"].get("search_engine"),
            llm_instance=_StubLLM(),
            username=call["kwargs"].get("username"),
            settings_snapshot=call["kwargs"].get("settings_snapshot"),
        )

    recorder = ThreadBoundaryRecorder(on_call=_drive_production_reader)
    monkeypatch.setattr(
        research_router, "run_research_process", recorder, raising=True
    )

    put_setting(authenticated_client, "llm.model", "seed-model")
    put_setting(authenticated_client, "search.tool", "wikipedia")

    seen = []
    for value in (value_a, value_b):
        put_setting(authenticated_client, key, value)
        start_research(authenticated_client, query=f"snapshot {key} {value}")
        recorder.wait()
        assert not recorder.last["is_main_thread"]
        assert factory_calls, (
            "search_engine_factory.get_search was never reached; the "
            "production reader did not run"
        )
        seen.append(factory_calls[-1].get(factory_kwarg))

    assert seen == [value_a, value_b], (
        f"setting {key!r} did not survive the snapshot -> thread -> factory "
        f"path: saved {[value_a, value_b]}, factory saw {seen}"
    )


# ---------------------------------------------------------------------------
# 3. LLM settings reaching the actual chat-model constructor
#
# ``llm.temperature`` is resolved by ``config.llm_config.get_llm`` from the
# snapshot; ``llm.max_tokens`` is resolved deeper still, inside the provider
# class's ``create_llm`` (via ``providers._helpers.compute_max_tokens``).
# The consumer boundary here is the LangChain client constructor itself
# (``ChatAnthropic``), stubbed so nothing talks to the network. Everything
# between the HTTP save and that constructor is production code, executed on
# the background research thread.
# ---------------------------------------------------------------------------


def _seed_anthropic(client):
    put_setting(client, "llm.provider", "anthropic")
    put_setting(client, "llm.model", "claude-3-5-sonnet-20241022")
    put_setting(client, "llm.anthropic.api_key", "sk-ant-test-key")


@pytest.mark.parametrize(
    "key,value_a,value_b,ctor_kwarg",
    [
        ("llm.temperature", 0.11, 0.83, "temperature"),
        ("llm.max_tokens", 1234, 4321, "max_tokens"),
    ],
    ids=lambda v: str(v),
)
def test_llm_setting_reaches_chat_model_constructor(
    authenticated_client, monkeypatch, key, value_a, value_b, ctor_kwarg
):
    """Saved LLM setting -> snapshot -> real worker -> ChatAnthropic(...).

    The real ``run_research_process`` resolves the LLM; the recorder sits on
    the LangChain client constructor, so nothing about the resolution is
    simulated.

    CONTROL: two different saved values, both must show up in the
    constructor kwargs.
    """
    probe = _install_real_worker_probe(
        authenticated_client, monkeypatch, lambda engine: None
    )

    # Unrestricted context window => compute_max_tokens applies no 80% cap,
    # so a saved llm.max_tokens must arrive verbatim.
    put_setting(authenticated_client, "llm.context_window_unrestricted", True)
    put_setting(authenticated_client, "llm.supports_max_tokens", True)

    seen = []
    for value in (value_a, value_b):
        put_setting(authenticated_client, key, value)
        start_research(authenticated_client, query=f"llm {key} {value}")
        probe.wait()
        assert probe.llm_ctor_calls, (
            "ChatAnthropic was never constructed; the real worker did not "
            "resolve an LLM"
        )
        seen.append(probe.llm_ctor_calls[-1].get(ctor_kwarg))

    assert seen == [value_a, value_b], (
        f"setting {key!r} did not reach the chat-model constructor as "
        f"{ctor_kwarg!r}: saved {[value_a, value_b]}, constructor saw {seen}"
    )


def test_context_window_settings_cap_max_tokens_in_the_real_worker(
    authenticated_client, monkeypatch
):
    """llm.context_window_unrestricted / llm.context_window_size take effect.

    ``compute_max_tokens`` caps ``llm.max_tokens`` at 80% of the effective
    context window. Unrestricted => the saved 5000 passes through; restricted
    to a 1000-token window => the constructor must see 800.

    CONTROL: the two rounds save the SAME llm.max_tokens, so the only thing
    that can change the constructor kwarg is the context-window settings.
    """
    probe = _install_real_worker_probe(
        authenticated_client, monkeypatch, lambda engine: None
    )
    put_setting(authenticated_client, "llm.supports_max_tokens", True)
    put_setting(authenticated_client, "llm.max_tokens", 5000)

    seen = []
    for unrestricted, window in ((True, 1000), (False, 1000)):
        put_setting(
            authenticated_client,
            "llm.context_window_unrestricted",
            unrestricted,
        )
        put_setting(authenticated_client, "llm.context_window_size", window)
        start_research(
            authenticated_client, query=f"ctx {unrestricted} {window}"
        )
        probe.wait()
        seen.append(probe.llm_ctor_calls[-1].get("max_tokens"))

    assert seen == [5000, 800], (
        "llm.context_window_unrestricted / llm.context_window_size did not "
        f"reach compute_max_tokens in the worker: {seen} (expected the saved "
        "5000 unrestricted, then 80% of a 1000-token window)"
    )


# ---------------------------------------------------------------------------
# 4. Concurrency cap: app.max_concurrent_researches
#
# Purely observable over HTTP: with the cap at 1, a second submission made
# while the first research thread is still alive must come back QUEUED; with
# the cap raised it must start. Nothing is stubbed except the thread body
# (parked so the first run really is in flight) and the queue-processor
# notification (so a queued run is not actually dispatched mid-test).
# ---------------------------------------------------------------------------


def _submit(client, query):
    resp = client.post(
        "/api/start_research", json={"query": query, "mode": "quick"}
    )
    assert resp.status_code == 200, (
        f"POST /api/start_research -> {resp.status_code} {resp.text[:300]}"
    )
    return resp.json()


def test_max_concurrent_researches_setting_gates_the_queue(
    authenticated_client, monkeypatch
):
    """app.max_concurrent_researches actually decides start-vs-queue.

    CONTROL: cap=1 must queue the second submission, cap=3 must start it.
    """
    from local_deep_research.web.queue import processor_v2
    from local_deep_research.web.routers import research as research_router

    notified = []
    monkeypatch.setattr(
        processor_v2.queue_processor,
        "notify_research_queued",
        lambda *a, **kw: notified.append(kw),
        raising=True,
    )

    recorder = ThreadBoundaryRecorder()
    recorder.hold.clear()  # park every research thread
    monkeypatch.setattr(
        research_router, "run_research_process", recorder, raising=True
    )

    put_setting(authenticated_client, "llm.model", "seed-model")

    statuses = {}
    try:
        put_setting(authenticated_client, "app.max_concurrent_researches", 1)
        first = _submit(authenticated_client, "cap-1 first")
        recorder.wait()
        second = _submit(authenticated_client, "cap-1 second")
        statuses["cap_1"] = (str(first["status"]), str(second["status"]))

        put_setting(authenticated_client, "app.max_concurrent_researches", 3)
        third = _submit(authenticated_client, "cap-3 third")
        recorder.wait()
        statuses["cap_3"] = str(third["status"])
    finally:
        recorder.hold.set()

    assert statuses["cap_1"][0] == "success", (
        f"first submission under cap=1 should start: {statuses['cap_1']}"
    )
    assert "queued" in statuses["cap_1"][1].lower(), (
        "second submission under app.max_concurrent_researches=1 was NOT "
        f"queued: {statuses['cap_1'][1]!r}"
    )
    assert notified, (
        "the queue processor was never notified, so nothing was queued"
    )
    # CONTROL: raising the cap must let the very same third submission run.
    assert statuses["cap_3"] == "success", (
        "after raising app.max_concurrent_researches to 3 the submission "
        f"was still queued: {statuses['cap_3']!r} — the cap is not being "
        "read from the saved setting"
    )


# ---------------------------------------------------------------------------
# 5. Egress policy: policy.egress_scope
#
# The saved scope is read back on the request thread by
# ``_precheck_engine_policy`` and decides whether a run using a non-primary
# engine is allowed at all. Observable entirely over HTTP.
# ---------------------------------------------------------------------------


def test_egress_scope_setting_gates_non_primary_engine(
    authenticated_client, monkeypatch
):
    """policy.egress_scope actually decides whether a run is refused.

    STRICT (only the primary engine) must refuse a run that names a
    different engine; PUBLIC_ONLY -- the CONTROL -- must allow the exact
    same request, because both engines are public.
    """
    from local_deep_research.web.routers import research as research_router

    recorder = ThreadBoundaryRecorder()
    monkeypatch.setattr(
        research_router, "run_research_process", recorder, raising=True
    )

    put_setting(authenticated_client, "llm.model", "seed-model")
    put_setting(authenticated_client, "search.tool", "wikipedia")

    body = {
        "query": "egress scope probe",
        "mode": "quick",
        "search_engine": "arxiv",
    }

    put_setting(authenticated_client, "policy.egress_scope", "strict")
    strict = authenticated_client.post("/api/start_research", json=body)

    put_setting(authenticated_client, "policy.egress_scope", "public_only")
    public = authenticated_client.post("/api/start_research", json=body)

    assert strict.status_code == 400, (
        "policy.egress_scope=strict did NOT refuse a non-primary engine: "
        f"{strict.status_code} {strict.text[:300]}"
    )
    strict_msg = strict.json()["message"].lower()
    assert "egress scope" in strict_msg and "strict" in strict_msg, (
        f"refusal was not the egress-scope policy: {strict.text[:300]}"
    )
    # CONTROL: the identical request under a permissive scope must run.
    assert public.status_code == 200, (
        "policy.egress_scope=public_only still refused the same request: "
        f"{public.status_code} {public.text[:300]}"
    )
    assert public.json()["status"] == "success"
    recorder.wait()
    assert recorder.last["kwargs"]["search_engine"] == "arxiv"


# ---------------------------------------------------------------------------
# 6. The OTHER save path: POST /settings/save_all_settings
#
# The settings UI posts the whole form to ``save_all_settings``, which is a
# completely separate handler from the single-key PUT exercised above (its
# own coercion, its own validation, its own commit). A setting can be
# honoured when saved one way and dropped when saved the other, so the bulk
# path gets its own end-to-end check.
# ---------------------------------------------------------------------------


def test_bulk_save_all_settings_reaches_search_engine_factory(
    authenticated_client, monkeypatch
):
    """POST /settings/save_all_settings -> snapshot -> thread -> factory.

    CONTROL: two bulk saves with different values, both observed downstream.
    """
    from local_deep_research.config import search_config
    from local_deep_research.web.routers import research as research_router

    factory_calls = []

    def _fake_factory(**kwargs):
        factory_calls.append(kwargs)
        return object()

    monkeypatch.setattr(
        search_config, "factory_get_search", _fake_factory, raising=True
    )

    class _StubLLM:
        pass

    def _drive_production_reader(call):
        search_config.get_search(
            search_tool=call["kwargs"].get("search_engine"),
            llm_instance=_StubLLM(),
            username=call["kwargs"].get("username"),
            settings_snapshot=call["kwargs"].get("settings_snapshot"),
        )

    recorder = ThreadBoundaryRecorder(on_call=_drive_production_reader)
    monkeypatch.setattr(
        research_router, "run_research_process", recorder, raising=True
    )

    seen_max_results = []
    seen_engine = []
    for max_results, tool in ((11, "wikipedia"), (22, "arxiv")):
        resp = authenticated_client.post(
            "/settings/save_all_settings",
            json={
                "llm.model": "seed-model",
                "search.max_results": max_results,
                "search.tool": tool,
            },
        )
        assert resp.status_code == 200, (
            f"save_all_settings -> {resp.status_code} {resp.text[:300]}"
        )
        start_research(authenticated_client, query=f"bulk {max_results}")
        recorder.wait()
        assert factory_calls
        seen_max_results.append(factory_calls[-1].get("max_results"))
        seen_engine.append(factory_calls[-1].get("search_tool"))

    assert seen_max_results == [11, 22], (
        "search.max_results saved through the bulk form did not reach the "
        f"search-engine factory: {seen_max_results}"
    )
    assert seen_engine == ["wikipedia", "arxiv"], (
        "search.tool saved through the bulk form did not reach the "
        f"search-engine factory: {seen_engine}"
    )


# ---------------------------------------------------------------------------
# 7. The REAL research worker
#
# Sections 1-6 stub ``run_research_process`` itself. This section does not:
# ``research_service.run_research_process`` runs for real -- it establishes
# the thread search context, rebuilds a ``SnapshotSettingsContext`` from the
# snapshot and installs it as the thread-local settings context, resolves the
# LLM -- and the only thing replaced is ``research_service.get_search``, the
# call right after that setup. The replacement invokes the REAL
# ``config.search_config.get_search`` (real engine factory, real engine
# object, real rate-limit tracker), takes its measurements, then raises to
# abort the run before any network work begins.
#
# This is the harness that can actually catch "the setting never reached the
# background thread": nothing about the snapshot -> thread-local hand-off is
# simulated here.
# ---------------------------------------------------------------------------


class RealWorkerProbe:
    """Replacement for ``research_service.get_search`` inside the real worker."""

    def __init__(self, observe):
        import threading

        self.observe = observe
        self.observations = []
        self.errors = []
        self.done = threading.Event()

    def __call__(self, **kwargs):
        import threading

        from local_deep_research.config.search_config import (
            get_search as real_get_search,
        )

        try:
            engine = real_get_search(**kwargs)
            self.observations.append(
                {
                    "engine": engine,
                    "kwargs": kwargs,
                    "observed": self.observe(engine),
                    "is_main_thread": threading.current_thread()
                    is threading.main_thread(),
                }
            )
        except BaseException as exc:  # re-raised on the test thread below
            self.errors.append(exc)
        finally:
            self.done.set()
        # Abort the run before the strategy does any network work. The
        # worker's own handler re-raises this, which is exactly what we want:
        # nothing further executes.
        raise RuntimeError("probe: aborting run after measurement")

    def wait(self, timeout=90.0):
        assert self.done.wait(timeout), (
            "the real research worker never reached get_search"
        )
        self.done.clear()
        if self.errors:
            raise self.errors[-1]
        last = self.observations[-1]
        assert not last["is_main_thread"], (
            "the worker ran on the main thread; the hand-off did not happen"
        )
        return last["observed"]


def _install_real_worker_probe(client, monkeypatch, observe):
    """Seed a stub-able LLM, plant the probe, return it."""
    from langchain_core.language_models.fake_chat_models import (
        FakeListChatModel,
    )
    from local_deep_research.llm.providers.implementations import (
        anthropic as anthropic_provider,
    )
    from local_deep_research.web.services import research_service

    llm_ctor_calls = []

    def _fake_chat_anthropic(**kwargs):
        llm_ctor_calls.append(kwargs)
        return FakeListChatModel(responses=["stub"])

    monkeypatch.setattr(
        anthropic_provider,
        "ChatAnthropic",
        _fake_chat_anthropic,
        raising=True,
    )
    _seed_anthropic(client)
    put_setting(client, "search.tool", "wikipedia")

    probe = RealWorkerProbe(observe)
    probe.llm_ctor_calls = llm_ctor_calls
    monkeypatch.setattr(research_service, "get_search", probe, raising=True)
    return probe


def test_search_and_rate_limit_settings_reach_the_real_engine(
    authenticated_client, monkeypatch
):
    """search.max_results + rate_limiting.* reach the real engine object.

    The assertions are on attributes of the actual ``WikipediaSearchEngine``
    and its ``AdaptiveRateLimitTracker`` built inside the real research
    worker -- not on anything this test constructed.

    CONTROL: every setting is written twice with different values.
    """

    def _observe(engine):
        tracker = engine.rate_tracker
        return {
            "engine_class": type(engine).__name__,
            "max_results": engine.max_results,
            "rate_limiting_enabled": tracker.enabled,
            "rate_limiting_memory_window": tracker.memory_window,
        }

    probe = _install_real_worker_probe(
        authenticated_client, monkeypatch, _observe
    )

    rounds = [
        {
            "search.max_results": 17,
            "rate_limiting.enabled": False,
            "rate_limiting.memory_window": 42,
        },
        {
            "search.max_results": 6,
            "rate_limiting.enabled": True,
            "rate_limiting.memory_window": 137,
        },
    ]

    seen = []
    for values in rounds:
        for key, value in values.items():
            put_setting(authenticated_client, key, value)
        start_research(authenticated_client, query=f"real worker {values}")
        seen.append(probe.wait())

    assert [o["engine_class"] for o in seen] == [
        "WikipediaSearchEngine",
        "WikipediaSearchEngine",
    ], f"the real engine factory was not exercised: {seen}"

    assert [o["max_results"] for o in seen] == [17, 6], (
        "search.max_results did not reach the real search engine built in "
        f"the research worker: {[o['max_results'] for o in seen]}"
    )
    assert [o["rate_limiting_enabled"] for o in seen] == [False, True], (
        "rate_limiting.enabled did not reach the engine's rate tracker: "
        f"{[o['rate_limiting_enabled'] for o in seen]}"
    )
    assert [o["rate_limiting_memory_window"] for o in seen] == [42, 137], (
        "rate_limiting.memory_window did not reach the engine's rate "
        f"tracker: {[o['rate_limiting_memory_window'] for o in seen]}"
    )


def test_report_citation_format_reaches_the_citation_formatter(
    authenticated_client, monkeypatch
):
    """report.citation_format reaches ``get_citation_formatter``.

    This one is worth its own check because ``get_citation_formatter`` reads
    the setting with NO snapshot argument at all -- it relies entirely on the
    thread-local settings context that the worker installs. If the ASGI port
    had failed to reinstate that context on the research thread, the reader
    would silently fall back to ``number_hyperlinks``.

    CONTROL: two different formats, and neither is the default, so a
    fallback to the default fails both halves.
    """
    from local_deep_research.text_optimization import CitationMode
    from local_deep_research.web.services.research_service import (
        get_citation_formatter,
    )

    def _observe(_engine):
        return get_citation_formatter().mode

    probe = _install_real_worker_probe(
        authenticated_client, monkeypatch, _observe
    )

    seen = []
    for value in ("no_hyperlinks", "domain_hyperlinks"):
        put_setting(authenticated_client, "report.citation_format", value)
        start_research(authenticated_client, query=f"citation {value}")
        seen.append(probe.wait())

    assert seen == [
        CitationMode.NO_HYPERLINKS,
        CitationMode.DOMAIN_HYPERLINKS,
    ], (
        "report.citation_format did not reach the citation formatter on the "
        f"research thread: {seen} (default is {CitationMode.NUMBER_HYPERLINKS})"
    )


def test_per_run_max_results_override_beats_the_saved_setting(
    authenticated_client, monkeypatch
):
    """A per-run ``max_results`` overrides the saved ``search.max_results``.

    The submission value is validated at the route and then written over the
    snapshot key inside the worker, so the real engine must be built with the
    submitted number -- and with the saved one when nothing is submitted.

    CONTROL: the saved setting is identical in both rounds, so the only thing
    that can move the engine's max_results is the per-run override.
    """
    probe = _install_real_worker_probe(
        authenticated_client, monkeypatch, lambda engine: engine.max_results
    )
    put_setting(authenticated_client, "search.max_results", 17)

    seen = []
    for body in (
        {"query": "override on", "mode": "quick", "max_results": 5},
        {"query": "override off", "mode": "quick"},
    ):
        resp = authenticated_client.post("/api/start_research", json=body)
        assert resp.status_code == 200, resp.text[:300]
        seen.append(probe.wait())

    assert seen == [5, 17], (
        "per-run max_results / saved search.max_results precedence is wrong: "
        f"{seen} (expected the submitted 5, then the saved 17)"
    )


# ---------------------------------------------------------------------------
# 8. Positive control for the whole harness
#
# Everything above asserts "the value arrived". That is only meaningful if
# the harness can tell the difference — i.e. if it would FAIL when the value
# does not arrive. This test injects exactly the regression the ASGI port
# risks (the key is saved, read correctly on the request thread, and then
# dropped on the way into the background thread) and shows the same
# measurement point reports the library default instead.
# ---------------------------------------------------------------------------


def test_harness_detects_a_setting_lost_at_the_thread_boundary(
    authenticated_client, monkeypatch
):
    """Drop one key from the snapshot as it crosses into the worker.

    The setting is still saved, and ``GET /settings/api/<key>`` still echoes
    it — the round trip is intact. The engine built inside the worker must
    nonetheless fall back to the built-in default, proving the section-7
    assertions are not tautological.
    """
    from local_deep_research.web.routers import research as research_router

    real_spawn = research_router.start_research_process

    def _spawn_with_key_dropped(*args, **kwargs):
        snapshot = kwargs.get("settings_snapshot")
        assert isinstance(snapshot, dict)
        assert "search.max_results" in snapshot, (
            "the route did not put search.max_results in the snapshot at "
            "all; this control cannot distinguish anything"
        )
        snapshot.pop("search.max_results")
        return real_spawn(*args, **kwargs)

    monkeypatch.setattr(
        research_router,
        "start_research_process",
        _spawn_with_key_dropped,
        raising=True,
    )

    probe = _install_real_worker_probe(
        authenticated_client, monkeypatch, lambda engine: engine.max_results
    )
    put_setting(authenticated_client, "search.max_results", 17)

    # The save itself is fine and the API echoes it back...
    echoed = authenticated_client.get("/settings/api/search.max_results")
    assert echoed.status_code == 200
    assert echoed.json()["value"] == 17

    # ...but the worker never sees it.
    start_research(authenticated_client, query="lost at the boundary")
    observed = probe.wait()

    assert observed == 10, (
        "expected the get_search fallback default (10) once the key is "
        f"dropped at the thread boundary, got {observed}"
    )
    assert observed != 17, (
        "the harness cannot distinguish a delivered setting from a lost one"
    )