~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(egress): follow-up nits from searxng egress guard review (#5574)
LearningCircuitfb8e03fd2cf7
diff --git a/src/local_deep_research/security/egress/validators.py b/src/local_deep_research/security/egress/validators.pyindex 7d3bdbe939..2c04022da4 100644--- a/src/local_deep_research/security/egress/validators.py+++ b/src/local_deep_research/security/egress/validators.py@@ -363,8 +363,9 @@ def _normalize_engine_url_for_comparison(value):Collapses only variation that denotes the exact SAME url:- scheme and host case (``HTTP://LOCALHOST`` == ``http://localhost``),- - a single trailing slash on the path (``rstrip("/")`` also makes an- empty path and ``"/"`` compare equal).+ - any trailing slash(es) on the path (``rstrip("/")`` strips all of+ them, so ``"/path//"`` and ``"/path"`` compare equal, and an+ empty path and ``"/"`` compare equal too).Everything that could denote a DIFFERENT destination is preservedverbatim and NOT normalized away: the host itself, the port, any pathdiff --git a/tests/security/test_egress_settings_save_validators.py b/tests/security/test_egress_settings_save_validators.pyindex 99e964aef8..1cc14ce617 100644--- a/tests/security/test_egress_settings_save_validators.py+++ b/tests/security/test_egress_settings_save_validators.py@@ -659,9 +659,12 @@ def test_unparseable_stored_or_submitted_value_falls_back_to_exact_match():}# Different value than stored, and unparseable -> falls back to exact# string compare, which does not match -> a genuine change -> validated.- # The submitted value is also unparseable-as-strict-URL, so the SSRF- # validator itself will refuse it as not-a-valid-URL rather than skip it- # as "unchanged".+ # ``urlsplit`` itself does not raise on a non-numeric port (that only+ # happens when ``.port`` is accessed, which the SSRF check never does),+ # so the submitted value parses fine as far as ``_engine_url_ssrf_error``+ # is concerned: its hostname is "localhost", which resolves to loopback+ # and is refused as a private/loopback address — not because+ # "http://localhost:xyz" fails to parse as a URL.err = validate_engine_instance_urls({SEARXNG_URL_KEY: "http://localhost:xyz"}, all_db_settings)@@ -715,5 +718,103 @@ def test_hex_loopback_literal_instance_url_is_rejected_as_a_change():assert err["key"] == SEARXNG_URL_KEY+# ---------------------------------------------------------------------------+# Registry contract — a future public engine must not be able to silently+# skip the save-time SSRF guard.+#+# ``_public_engine_url_settings()`` only guards an engine's URL when its+# registry class exposes BOTH ``is_public is True`` and a non-empty *string*+# ``url_setting``; it is seeded with the SearXNG key as a hardcoded fallback,+# but otherwise derives the guarded set by walking ``ENGINE_REGISTRY`` and+# reading those two class attributes (see ``validators.py``). If a future+# public engine declares a non-string or empty-string ``url_setting``, it+# would silently fall outside the guard (a genuinely absent/``None``+# ``url_setting`` has no configurable URL and is intentionally out of+# scope). These tests pin the contract so that regression is caught.+# ---------------------------------------------------------------------------+++def test_public_registry_engines_with_a_url_setting_are_all_guarded():+ """Walk the REAL engine registry: every class that is ``is_public is+ True`` and declares a non-``None`` ``url_setting`` must (a) declare it+ as a non-empty string, and (b) have that key present in+ ``_public_engine_url_settings()``. This is the exact contract+ ``_public_engine_url_settings()`` relies on to auto-discover public+ engines -- if a future engine declares a non-string or empty-string+ ``url_setting``, this test fails instead of the engine silently+ shipping without the SSRF guard. A public engine whose ``url_setting``+ is genuinely absent/``None`` has no configurable URL (no SSRF surface)+ and is intentionally skipped.++ Investigated registry structure: ``ENGINE_REGISTRY``+ (``web_search_engines/engine_registry.py``) maps every engine name to+ its module/class; ``_get_engine_class`` lazily imports each. As of this+ writing, ``elasticsearch`` and ``paperless`` also declare a+ ``url_setting`` but are LOCAL-nature engines (``is_local = True``,+ ``is_public`` unset) and are correctly excluded from this guard --+ only ``searxng`` combines ``is_public = True`` with a ``url_setting``.+ """+ from local_deep_research.security.egress.policy import _get_engine_class+ from local_deep_research.security.egress.validators import (+ _public_engine_url_settings,+ )+ from local_deep_research.web_search_engines.engine_registry import (+ ENGINE_REGISTRY,+ )++ guarded = _public_engine_url_settings()+ public_engines_with_url_setting = []+ for name in ENGINE_REGISTRY:+ cls = _get_engine_class(name)+ if cls is None:+ # Not expected to happen for any registry entry in a working+ # checkout, but importability isn't this test's concern.+ continue+ if getattr(cls, "is_public", None) is not True:+ continue+ url_setting = getattr(cls, "url_setting", None)+ if url_setting is None:+ # Genuinely absent url_setting -> no configurable URL -> no+ # SSRF surface for this engine. Intentionally out of scope.+ continue+ assert isinstance(url_setting, str) and url_setting, (+ f"{name}.url_setting must be a non-empty string to be picked "+ "up by _public_engine_url_settings() -- a non-string or "+ "empty-string value silently opts this public engine out of "+ "the save-time SSRF guard"+ )+ assert url_setting in guarded, (+ f"{name} is is_public=True with url_setting={url_setting!r} "+ "but that key is not covered by _public_engine_url_settings() "+ "-- a future public engine's configurable URL must be "+ "reachable through that function or it silently skips the "+ "save-time SSRF guard"+ )+ public_engines_with_url_setting.append(name)++ # Known case today. Fail loudly (rather than vacuously passing) if+ # registry wiring ever changes so this loop stops covering anything.+ assert "searxng" in public_engines_with_url_setting+++def test_searxng_is_public_url_setting_contract_matches_guard():+ """Minimum-viable version of the contract above, independent of+ registry enumeration: SearXNG itself is ``is_public = True``, declares+ a non-empty string ``url_setting``, and that key is in the guarded+ set consumed by ``validate_engine_instance_urls``."""+ from local_deep_research.security.egress.validators import (+ _public_engine_url_settings,+ )+ from local_deep_research.web_search_engines.engines.search_engine_searxng import (+ SearXNGSearchEngine,+ )++ assert SearXNGSearchEngine.is_public is True+ assert isinstance(SearXNGSearchEngine.url_setting, str)+ assert SearXNGSearchEngine.url_setting+ assert SearXNGSearchEngine.url_setting in _public_engine_url_settings()+ assert SearXNGSearchEngine.url_setting == SEARXNG_URL_KEY++if __name__ == "__main__": # pragma: no coverraise SystemExit(pytest.main([__file__, "-q"]))