Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

27.2 KB
Raw Blame
"""Per-engine ``agent_enabled`` opt-out for NON-collection engines (#5015).

The research-agent tool filter used to honor ``agent_enabled`` only for
document collections: collection loaders wrote the value onto the engine
config dict, while non-collection web engines (arxiv, pubmed, …) never got
the key, so the filter silently defaulted to ``True``. A user who wanted to
keep arxiv available for normal ``auto`` searches but exclude it from the
research agent had no working control — the only opt-out was to disable
the engine entirely, which also removed it from non-agent searches.

The fix adds a per-engine settings key,
``search.engine.web.<name>.agent_enabled`` (default ``True``). The
``_extract_per_engine_config`` helper in ``search_engines_config.py``
already flattens every ``search.engine.web.<name>.*`` setting into the
engine's config dict, so the original
``config.get("agent_enabled", True)`` check in
``LangGraphAgentStrategy._build_tools`` sees the new value with no extra
code path. Collection behavior from #4453 is unchanged (the loader writes
the DB column onto the same dict).

Drives the REAL ``LangGraphAgentStrategy._build_tools`` loop; only the leaf
tool factories and engine discovery are mocked, so every assertion is about
the FILTER decision (which engine names survive into the built tool list).
"""

from __future__ import annotations

from contextlib import contextmanager
from types import SimpleNamespace
from unittest.mock import MagicMock, patch

import pytest

import local_deep_research.advanced_search_system.strategies.langgraph_agent_strategy as strat  # noqa: E501
from local_deep_research.security import clear_active_context


@pytest.fixture(autouse=True)
def _clear_ctx():
    clear_active_context()
    yield
    clear_active_context()


@contextmanager
def _patched_factories():
    """Sentinel-tagged factories so we can name-check both the built tools
    AND the factory calls (the filter ``continue``s before construction, so a
    successful exclusion must leave ``_make_specialized_search_tool``
    untouched for the filtered engine)."""
    spec = MagicMock(
        side_effect=lambda name, *a, **k: SimpleNamespace(name=name)
    )
    with (
        patch.object(
            strat,
            "_make_web_search_tool",
            lambda *a, **k: SimpleNamespace(name="web_search"),
        ),
        patch.object(strat, "build_fetch_tool", lambda *a, **k: None),
        patch.object(strat, "_make_specialized_search_tool", spec),
    ):
        yield spec


def _engine(name: str, *, agent_enabled=True, is_public=True):
    """Mirror the real ``search_config`` shape for a non-collection engine.

    In production the dict is produced by
    ``_extract_per_engine_config`` in ``search_engines_config.py``, which
    flattens every ``search.engine.web.<name>.*`` setting into the engine's
    config — so ``agent_enabled`` IS present and reflects the snapshot value
    (or the JSON default when no override is set).
    """
    return {
        "is_public": is_public,
        "is_local": not is_public,
        "is_retriever": False,
        "description": f"{name} engine",
        "strengths": ["example"],
        "agent_enabled": agent_enabled,
    }


def _build(available, *, snapshot_extra=None):
    """Construct a minimal strategy and return ``(tool_names, factory_mock)``.

    ``policy.egress_scope=both`` so a non-collection public engine is allowed
    by egress, leaving ``agent_enabled`` as the only differentiator — the
    fix isolates the new switch from the egress machinery.

    ``self.search`` is a bare ``MagicMock`` so
    ``_get_current_engine_name()`` returns ``"magicmock"`` — no engine in
    ``available`` matches that, so the ``if name == current: continue`` skip
    never fires and every entry in ``available`` is actually evaluated by
    the agent_enabled filter. The primary-engine carve-out has its own
    dedicated test that patches ``_get_current_engine_name``.
    """
    snapshot = {
        "policy.egress_scope": "both",
        "search.fetch.mode": "disabled",
    }
    if snapshot_extra:
        snapshot.update(snapshot_extra)
    reg = MagicMock()
    reg.get_metadata.return_value = None
    with _patched_factories() as spec:
        with (
            patch(
                "local_deep_research.web_search_engines.search_engines_config"
                ".list_eligible_engine_configs",
                return_value=available,
            ),
            patch(
                "local_deep_research.web_search_engines.retriever_registry"
                ".retriever_registry",
                reg,
            ),
        ):
            strategy = strat.LangGraphAgentStrategy(
                model=MagicMock(),
                search=MagicMock(),
                citation_handler=object(),
                include_sub_research=False,
                settings_snapshot=snapshot,
            )
            tools = strategy._build_tools("overall query")
    return {t.name for t in tools}, spec


# -- The bug from #5015 ------------------------------------------------------


def test_bug_5015_arxiv_setting_false_excludes_from_agent():
    """arxiv with ``agent_enabled=False`` must not appear as a specialized
    agent tool. Regression of #5015."""
    names, spec = _build({"arxiv": _engine("arxiv", agent_enabled=False)})
    assert "arxiv" not in names, (
        "arxiv with agent_enabled=False must not appear as a specialized "
        "agent tool (regression of #5015)"
    )
    # The factory was never invoked for arxiv (filter continues before
    # construction), proving the exclusion happens upstream of the tool
    # build, not after.
    assert "arxiv" not in {c.args[0] for c in spec.call_args_list}


def test_bug_5015_pubmed_setting_false_excludes_from_agent():
    """Same fix, different engine — proves the path is generic, not
    arxiv-specific."""
    names, spec = _build({"pubmed": _engine("pubmed", agent_enabled=False)})
    assert "pubmed" not in names
    assert "pubmed" not in {c.args[0] for c in spec.call_args_list}


# -- Existing-behavior preservation ------------------------------------------


def test_default_true_keeps_engine_available():
    """An engine whose ``agent_enabled`` defaulted to True (via the JSON
    defaults) stays in the agent's tool list. Existing users see no change
    unless they opt out."""
    names, _ = _build({"arxiv": _engine("arxiv", agent_enabled=True)})
    assert "arxiv" in names


def test_missing_key_defaults_to_available():
    """An engine whose config dict has no ``agent_enabled`` key defaults to
    True — preserves prior behavior for retrievers and any engine the loader
    forgets to set the flag on. Pins the explicit ``.get(..., True)``
    default."""
    cfg = _engine("arxiv")
    del cfg["agent_enabled"]
    names, _ = _build({"arxiv": cfg})
    assert "arxiv" in names


# -- Independence from egress / other engines --------------------------------


def test_setting_is_per_engine_independent():
    """Disabling arxiv must NOT take pubmed down with it — the flag is
    scoped to the engine name."""
    names, _ = _build(
        {
            "arxiv": _engine("arxiv", agent_enabled=False),
            "pubmed": _engine("pubmed", agent_enabled=True),
        },
    )
    assert "arxiv" not in names
    assert "pubmed" in names


def test_setting_independent_of_egress_scope():
    """The flag is a usability switch, orthogonal to egress. Under
    ``public_only`` (which would normally allow public engines like arxiv),
    an explicit ``agent_enabled=False`` on the engine dict must still drop
    arxiv."""
    names, _ = _build(
        {"arxiv": _engine("arxiv", agent_enabled=False, is_public=True)},
        snapshot_extra={"policy.egress_scope": "public_only"},
    )
    assert "arxiv" not in names


# -- search.tool primary-engine carve-out -----------------------------------


@pytest.mark.parametrize(
    "primary",
    ["arxiv", "openlibrary", "gutenberg", "github", "collection_hidden"],
)
def test_primary_engine_stays_reachable_via_web_search_when_disabled(primary):
    """The engine selected as ``search.tool`` always wins: even with
    ``agent_enabled=False`` it must remain reachable via the agent's generic
    ``web_search`` tool (the one specialised tool the agent always gets for
    its primary engine). Otherwise a user could accidentally leave the
    agent with no search tool at all. Pinned by PR #5087 — the setting is a
    carve-out, not an override of the primary selection.

    Also asserts the primary engine is NEVER added as a specialized tool
    (the ``if name == current: continue`` skip), which is the existing
    mechanism that makes the carve-out work today.
    """
    reg = MagicMock()
    reg.get_metadata.return_value = None
    with _patched_factories() as spec:
        with (
            patch(
                "local_deep_research.web_search_engines.search_engines_config"
                ".list_eligible_engine_configs",
                return_value={
                    primary: _engine(primary, agent_enabled=False),
                },
            ),
            patch(
                "local_deep_research.web_search_engines.retriever_registry"
                ".retriever_registry",
                reg,
            ),
        ):
            strategy = strat.LangGraphAgentStrategy(
                model=MagicMock(),
                search=MagicMock(),
                citation_handler=object(),
                include_sub_research=False,
                settings_snapshot={
                    "policy.egress_scope": "both",
                    "search.fetch.mode": "disabled",
                    "search.tool": primary,
                },
            )
            assert strategy._search_engine_name == primary
            tools = strategy._build_tools("overall query")
    names = {t.name for t in tools}
    assert "web_search" in names, (
        "Primary engine must remain reachable via web_search even when its "
        "agent_enabled is False — see PR #5087"
    )
    assert primary not in names, (
        "Primary engine is never added as a specialized tool — that path "
        "is reserved for ADDITIONAL engines"
    )
    assert primary not in {c.args[0] for c in spec.call_args_list}


# -- Collection path still wins (#4453 regression guard) --------------------


def test_collection_db_flag_still_wins_over_missing_setting():
    """The collection loader writes ``agent_enabled`` onto the config dict
    from the DB column. That path is the original #4453 behavior and must
    not regress — a collection whose dict says ``agent_enabled=False`` is
    excluded, even when no per-engine JSON default is involved.
    """
    coll_cfg = _engine("arxiv")
    coll_cfg["agent_enabled"] = False
    names, _ = _build({"collection_x": coll_cfg})
    assert "collection_x" not in names


# -- End-to-end via the real search_config ----------------------------------


def test_end_to_end_snapshot_overrides_propagate_to_filter():
    """End-to-end check: the real ``search_config`` pipeline reads
    ``search.engine.web.<name>.agent_enabled`` from the settings snapshot
    via ``_extract_per_engine_config`` and the result reaches the agent's
    filter — no manual snapshot fallback needed in ``_build_tools``.
    """
    from local_deep_research.web_search_engines.search_engines_config import (
        search_config,
    )

    snapshot = {
        "policy.egress_scope": "both",
        "search.fetch.mode": "disabled",
        # Real-format values: dicts with "value" key, like a DB row.
        "search.engine.web.arxiv.agent_enabled": {
            "value": False,
            "ui_element": "checkbox",
        },
        "search.engine.web.arxiv.use_in_auto_search": {
            "value": True,
            "ui_element": "checkbox",
        },
        "search.engine.web.pubmed.agent_enabled": {
            "value": True,
            "ui_element": "checkbox",
        },
        "search.engine.web.pubmed.use_in_auto_search": {
            "value": True,
            "ui_element": "checkbox",
        },
    }
    cfg = search_config(settings_snapshot=snapshot)
    assert cfg["arxiv"].get("agent_enabled") is False, (
        "search_config did not propagate the snapshot's "
        "search.engine.web.arxiv.agent_enabled=False into the engine dict"
    )
    assert cfg["pubmed"].get("agent_enabled") is True

    reg = MagicMock()
    reg.get_metadata.return_value = None
    with _patched_factories() as spec:
        with (
            patch(
                "local_deep_research.web_search_engines.search_engines_config"
                ".list_eligible_engine_configs",
                return_value=cfg,
            ),
            patch(
                "local_deep_research.web_search_engines.retriever_registry"
                ".retriever_registry",
                reg,
            ),
        ):
            strategy = strat.LangGraphAgentStrategy(
                model=MagicMock(),
                search=MagicMock(),
                citation_handler=object(),
                include_sub_research=False,
                settings_snapshot=snapshot,
            )
            tools = strategy._build_tools("overall query")
    names = {t.name for t in tools}
    assert "arxiv" not in names
    assert "arxiv" not in {c.args[0] for c in spec.call_args_list}
    assert "pubmed" in names


def test_end_to_env_var_string_coerced_through_snapshot_pipeline():
    """Env-var-backed overrides land in the snapshot as raw strings. The
    full pipeline — ``get_typed_setting_value`` inside
    ``get_setting_from_snapshot`` — must coerce ``"false"`` to a real bool
    before the value reaches the agent's filter. End-to-end check that this
    survives without a manual ``to_bool`` fallback in ``_build_tools``.
    """
    from local_deep_research.web_search_engines.search_engines_config import (
        search_config,
    )

    snapshot = {
        "policy.egress_scope": "both",
        "search.fetch.mode": "disabled",
        "search.engine.web.arxiv.agent_enabled": {
            "value": "false",  # raw env-var string
            "ui_element": "checkbox",
        },
        "search.engine.web.arxiv.use_in_auto_search": {
            "value": True,
            "ui_element": "checkbox",
        },
    }
    cfg = search_config(settings_snapshot=snapshot)
    assert cfg["arxiv"].get("agent_enabled") is False, (
        "Pipeline must coerce env-var 'false' string to a real bool before "
        "the filter sees it"
    )

    names, _ = _build(cfg)
    assert "arxiv" not in names


# -- Discovery independence (#5015 follow-up) ----------------------------
#
# Production path the previous review noted was masked: the helper used to
# pull candidates from ``get_available_engines``, which silently drops every
# engine with ``use_in_auto_search=false`` -- so an engine the user toggled
# ``agent_enabled=true`` on could still be invisible to the agent if it was
# disabled for ``auto`` searches. Symmetrically, dynamic ``collection_*``
# engines never register a ``use_in_auto_search`` setting, which defaulted
# them out of the agent entirely (collections can ONLY opt in via their
# dedicated DB column, not via the per-engine ``agent_enabled`` setting).


def test_engine_with_use_in_auto_search_false_reachable_via_agent():
    """``agent_enabled=true`` must independently expose Tavily to the agent
    even when ``use_in_auto_search=false``. Previously the helper reused
    ``get_available_engines``, so Tavily never entered the candidate pool
    and the agent silently kept its old, smaller tool list -- the user had
    no working surface to flip a single switch.

    Pins the discovery independence: the two settings control two
    different surfaces, and ``agent_enabled`` cannot be silently gated by
    ``use_in_auto_search``. (#5015 review follow-up.)
    """
    # Tavily is a public-only engine that needs an API key. The toggle
    # under test is purely the ``agent_enabled`` / ``use_in_auto_search``
    # split, so any unrelated validation noise is fixed by the dict shape
    # itself.
    tavily_cfg = _engine("tavily", agent_enabled=True)
    # Mark it as if the user had opted out of ``auto``-search visibility
    # while still wanting it on the agent's surface -- the entire point.
    tavily_cfg["requires_api_key"] = False
    tavily_cfg["is_public"] = True

    names, spec = _build({"tavily": tavily_cfg})
    assert "tavily" in names, (
        "agent_enabled=true must expose the engine to the agent even when "
        "use_in_auto_search=false -- the two flags control independent "
        "surfaces (see #5015 follow-up)"
    )
    assert "tavily" in {c.args[0] for c in spec.call_args_list}


def test_collection_engine_reachable_via_agent_without_use_in_auto_search():
    """Dynamic ``collection_*`` engines have no ``use_in_auto_search``
    setting at all, so the old ``get_available_engines`` discovery
    defaulted them out of the agent (the boolean defaults to ``False``).
    With the new ``list_eligible_engine_configs`` discovery, a collection
    whose loader wrote ``agent_enabled=True`` onto its config dict is
    reachable -- but the per-collection DB column stays the canonical
    ``off-switch`` (a collection with ``agent_enabled=False`` is still
    excluded even when ``use_in_auto_search`` would let it through).
    """
    collection_cfg = _engine(
        "collection_curated", agent_enabled=True, is_public=True
    )
    # Mirror the real collection shape -- the loader tags it with
    # ``is_local`` inverted from ``is_public`` and a few strengths.
    collection_cfg["is_local"] = False

    names, spec = _build({"collection_curated": collection_cfg})
    assert "collection_curated" in names, (
        "Dynamic collections registered through search_config() now flow "
        "into the agent's candidate pool -- they previously never "
        "appeared because they lack a use_in_auto_search setting."
    )
    assert "collection_curated" in {c.args[0] for c in spec.call_args_list}


# -- Canonical primary-skip (#5015 follow-up) ----------------------------
#
# The primary-skip is the only thing that prevents the configured primary
# engine from being registered as both ``web_search`` and
# ``search_<engine>``. It used to compare canonical names against the
# class-derived fallback of ``_resolve_engine_name`` -- e.g.
# ``SemanticScholarSearchEngine`` -> ``semanticscholar``, not
# ``semantic_scholar`` -- so the skip missed by an underscore and the
# user ended up with BOTH tools.


def test_canonical_primary_skip_uses_settings_value_not_class_heuristic():
    """When ``search.tool=semantic_scholar``, the canonical name on the
    helper's ``skip_engine`` argument must be ``semantic_scholar`` -- so
    the helper skips ``search_semantic_scholar`` and only ``web_search``
    (backed by the same primary) is exposed. The legacy class-derived
    fallback produced ``semanticscholar`` and the skip missed, so the
    agent received the redundant ``search_semantic_scholar`` tool.
    """
    reg = MagicMock()
    reg.get_metadata.return_value = None
    with _patched_factories() as spec:
        with (
            patch(
                "local_deep_research.web_search_engines.search_engines_config"
                ".list_eligible_engine_configs",
                return_value={
                    "semantic_scholar": _engine(
                        "semantic_scholar", agent_enabled=False
                    ),
                },
            ),
            patch(
                "local_deep_research.web_search_engines.retriever_registry"
                ".retriever_registry",
                reg,
            ),
        ):
            strategy = strat.LangGraphAgentStrategy(
                model=MagicMock(),
                search=MagicMock(),
                citation_handler=object(),
                include_sub_research=False,
                settings_snapshot={
                    "policy.egress_scope": "both",
                    "search.fetch.mode": "disabled",
                    "search.tool": "semantic_scholar",
                },
            )
            assert strategy._search_engine_name == "semantic_scholar"
            tools = strategy._build_tools("overall query")
    names = {t.name for t in tools}
    # Primary reachable via web_search (carve-out from #5015).
    assert "web_search" in names
    # But NOT as a redundant specialized tool -- the canonical skip
    # matched the canonical key in the dict.
    assert "semantic_scholar" not in names, (
        "Primary engine must be skipped from the specialized-tools loop. "
        "If this fires the helper is comparing canonical registry names "
        "against the class-derived fallback -- the regression fixed by "
        "the registry reverse-lookup in _resolve_engine_name."
    )
    # And the factory was never invoked for it either -- the skip
    # happens before construction.
    assert "semantic_scholar" not in {c.args[0] for c in spec.call_args_list}


# -- Specialized domain adapters default disabled (#1540 / parity) ---------


@pytest.mark.parametrize(
    "engine_name",
    ["gutenberg", "openlibrary", "pubchem", "stackexchange", "zenodo"],
)
def test_specialized_domain_engines_default_disabled_for_agent(engine_name):
    """Specialized domain search engines (Project Gutenberg, Open Library,
    PubChem, Stack Exchange, Zenodo) added in #1540 must default to
    agent_enabled=False and use_in_auto_search=False so they do not silently
    leak into general research agent toolsets or auto searches."""
    from local_deep_research.settings.manager import SettingsManager
    from local_deep_research.web_search_engines.search_engines_config import (
        list_eligible_engine_configs,
        search_config,
    )

    manager = SettingsManager(db_session=None)
    defaults = manager.default_settings
    assert (
        defaults[f"search.engine.web.{engine_name}.agent_enabled"]["value"]
        is False
    )
    assert (
        defaults[f"search.engine.web.{engine_name}.use_in_auto_search"]["value"]
        is False
    )

    # In search_config built from default snapshot, agent_enabled is False
    snapshot = manager.get_settings_snapshot()
    cfg = search_config(settings_snapshot=snapshot)
    assert cfg[engine_name].get("agent_enabled") is False
    assert cfg[engine_name].get("use_in_auto_search") is False

    # In eligible engine discovery for the research agent, it is excluded
    eligible = list_eligible_engine_configs(
        settings_snapshot=snapshot, check_agent_enabled=True
    )
    assert engine_name not in eligible

    # LangGraph agent strategy filters it out by default
    names, spec = _build({engine_name: cfg[engine_name]})
    assert engine_name not in names
    assert engine_name not in {c.args[0] for c in spec.call_args_list}


@pytest.mark.parametrize(
    "engine_name",
    ["gutenberg", "openlibrary", "pubchem", "stackexchange", "zenodo"],
)
def test_specialized_domain_engines_can_be_enabled_for_agent(engine_name):
    """When a user explicitly enables one of the specialized domain engines,
    it must be included in the agent's specialized tools."""
    from local_deep_research.settings.manager import SettingsManager
    from local_deep_research.web_search_engines.search_engines_config import (
        list_eligible_engine_configs,
        search_config,
    )

    manager = SettingsManager(db_session=None)
    snapshot = manager.get_settings_snapshot()
    snapshot[f"search.engine.web.{engine_name}.agent_enabled"] = True

    cfg = search_config(settings_snapshot=snapshot)
    assert cfg[engine_name].get("agent_enabled") is True

    eligible = list_eligible_engine_configs(
        settings_snapshot=snapshot, check_agent_enabled=True
    )
    assert engine_name in eligible

    names, spec = _build(
        {engine_name: cfg[engine_name]}, snapshot_extra=snapshot
    )
    assert engine_name in names
    assert engine_name in {c.args[0] for c in spec.call_args_list}


# -- Key-gated search engine agent controls ----------------------------------


@pytest.mark.parametrize(
    "engine_name,expected_agent_default",
    [
        ("nasa_ads", False),
        ("paperless", False),
        ("scaleserp", True),
        ("serper", True),
    ],
)
def test_key_gated_engines_default_flags(engine_name, expected_agent_default):
    """Key-gated engines must define both agent_enabled and use_in_auto_search.
    Specialized/private ones (NASA ADS, Paperless) default agent_enabled=False;
    general web scrapers (ScaleSERP, Serper) default agent_enabled=True.
    All default use_in_auto_search=False to avoid unbudgeted quota consumption."""
    from local_deep_research.settings.manager import SettingsManager
    from local_deep_research.web_search_engines.search_engines_config import (
        search_config,
    )

    manager = SettingsManager(db_session=None)
    defaults = manager.default_settings

    assert (
        defaults[f"search.engine.web.{engine_name}.agent_enabled"]["value"]
        is expected_agent_default
    )
    assert (
        defaults[f"search.engine.web.{engine_name}.use_in_auto_search"]["value"]
        is False
    )

    snapshot = manager.get_settings_snapshot()
    cfg = search_config(settings_snapshot=snapshot)
    assert cfg[engine_name].get("agent_enabled") is expected_agent_default
    assert cfg[engine_name].get("use_in_auto_search") is False


@pytest.mark.parametrize("engine_name", ["nasa_ads", "paperless"])
def test_key_gated_specialized_engines_require_key_and_explicit_opt_in(
    engine_name,
):
    """NASA ADS and Paperless do not become agent tools simply because an API
    key is configured; they require explicit agent_enabled=True opt-in."""
    from local_deep_research.settings.manager import SettingsManager
    from local_deep_research.web_search_engines.search_engines_config import (
        list_eligible_engine_configs,
    )

    manager = SettingsManager(db_session=None)
    snapshot = manager.get_settings_snapshot()

    # Step 1: Add API key only. agent_enabled is still False by default.
    snapshot[f"search.engine.web.{engine_name}.api_key"] = "test-token"
    eligible = list_eligible_engine_configs(
        settings_snapshot=snapshot, check_agent_enabled=True
    )
    assert engine_name not in eligible

    # Step 2: Explicitly enable for agent.
    snapshot[f"search.engine.web.{engine_name}.agent_enabled"] = True
    eligible = list_eligible_engine_configs(
        settings_snapshot=snapshot, check_agent_enabled=True
    )
    assert engine_name in eligible


@pytest.mark.parametrize("engine_name", ["scaleserp", "serper"])
def test_key_gated_general_engines_can_be_disabled_for_agent(engine_name):
    """ScaleSERP and Serper are available to the agent once an API key is configured,
    and can be toggled off via agent_enabled=False."""
    from local_deep_research.settings.manager import SettingsManager
    from local_deep_research.web_search_engines.search_engines_config import (
        list_eligible_engine_configs,
    )

    manager = SettingsManager(db_session=None)
    snapshot = manager.get_settings_snapshot()

    # Step 1: Add API key. Defaults agent_enabled=True.
    snapshot[f"search.engine.web.{engine_name}.api_key"] = "test-token"
    eligible = list_eligible_engine_configs(
        settings_snapshot=snapshot, check_agent_enabled=True
    )
    assert engine_name in eligible

    # Step 2: Toggle off.
    snapshot[f"search.engine.web.{engine_name}.agent_enabled"] = False
    eligible = list_eligible_engine_configs(
        settings_snapshot=snapshot, check_agent_enabled=True
    )
    assert engine_name not in eligible