Personal AI, On Personal Devices
Python
fix(security): validate web search redirect targets (#966)
Haotian Zheng7252d50143d3
diff --git a/src/openjarvis/tools/web_search.py b/src/openjarvis/tools/web_search.pyindex 61b29fd2..e50d7619 100644--- a/src/openjarvis/tools/web_search.py+++ b/src/openjarvis/tools/web_search.py@@ -20,6 +20,7 @@ from __future__ import annotationsimport loggingimport osfrom typing import Any+from urllib.parse import urljoinfrom openjarvis import __version__from openjarvis.core.registry import ToolRegistry@@ -44,6 +45,9 @@ YOUCOM_USER_AGENT = (f"openjarvis/{__version__} youdotcom-integration/open-jarvis-openjarvis")+_MAX_FETCH_REDIRECTS = 5+_REDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})+# Keyless tier exhaustion (402) and per-IP throttling (429) both mean "get a# key", which is a different remedy from a generic HTTP failure._KEYLESS_LIMIT_STATUSES = (402, 429)@@ -173,24 +177,39 @@ class WebSearchTool(BaseTool):@staticmethoddef _fetch_url(url: str, max_chars: int = 6000) -> str:- """Fetch a URL and return extracted text content."""+ """Fetch a URL and return extracted text after checking each redirect."""import re as _reimport httpxurl = WebSearchTool._normalize_url(url)- ssrf_error = check_ssrf(url)- if ssrf_error:- raise ValueError(ssrf_error)- resp = httpx.get(- url.strip(),- follow_redirects=True,+ current_url = url.strip()+ # One client preserves cookie scope across redirect hops. Its lifetime+ # is limited to this fetch so unrelated requests do not share cookies.+ with httpx.Client(+ follow_redirects=False,timeout=30.0,headers={"User-Agent": "Mozilla/5.0 (compatible; OpenJarvis/1.0; +https://github.com/openjarvis)"},- )- resp.raise_for_status()+ ) as client:+ for _ in range(_MAX_FETCH_REDIRECTS + 1):+ ssrf_error = check_ssrf(current_url)+ if ssrf_error:+ raise ValueError(ssrf_error)+ resp = client.get(current_url)+ if resp.status_code not in _REDIRECT_STATUS_CODES:+ resp.raise_for_status()+ break+ location = resp.headers.get("location", "")+ if not location:+ resp.raise_for_status()+ break+ current_url = urljoin(str(resp.url), location)+ else:+ raise ValueError(+ f"URL exceeded the maximum of {_MAX_FETCH_REDIRECTS} redirects"+ )content_type = resp.headers.get("content-type", "")if "application/pdf" in content_type:return (diff --git a/tests/agents/test_native_openhands.py b/tests/agents/test_native_openhands.pyindex bf370ade..6fe49581 100644--- a/tests/agents/test_native_openhands.py+++ b/tests/agents/test_native_openhands.py@@ -554,7 +554,7 @@ class TestUrlExpansion:mock_resp.text = "<html><body>Page content</body></html>"mock_resp.headers = {"content-type": "text/html"}mock_resp.raise_for_status = MagicMock()- monkeypatch.setattr(httpx, "get", MagicMock(return_value=mock_resp))+ monkeypatch.setattr(httpx.Client, "get", MagicMock(return_value=mock_resp))agent = NativeOpenHandsAgent(MagicMock(), "test-model")text, expanded = agent._expand_urls("Summarize: https://example.com/article")@@ -566,7 +566,7 @@ class TestUrlExpansion:import httpxmonkeypatch.setattr(- httpx,+ httpx.Client,"get",MagicMock(side_effect=Exception("Connection error")),)@@ -591,7 +591,7 @@ class TestUrlExpansion:return True, 0.0fetch = MagicMock()- monkeypatch.setattr(httpx, "get", fetch)+ monkeypatch.setattr(httpx.Client, "get", fetch)policy = CapabilityPolicy(default_deny=True)policy.grant("_default", "network:fetch")policy.deny("restricted-native", "network:fetch")@@ -627,7 +627,7 @@ class TestUrlExpansion:mock_resp.text = "<html><body>Article text here</body></html>"mock_resp.headers = {"content-type": "text/html"}mock_resp.raise_for_status = MagicMock()- monkeypatch.setattr(httpx, "get", MagicMock(return_value=mock_resp))+ monkeypatch.setattr(httpx.Client, "get", MagicMock(return_value=mock_resp))engine = MagicMock()engine.engine_id = "mock"diff --git a/tests/tools/test_web_search.py b/tests/tools/test_web_search.pyindex b8584977..7c959e9a 100644--- a/tests/tools/test_web_search.py+++ b/tests/tools/test_web_search.py@@ -5,6 +5,8 @@ from __future__ import annotationsimport sysfrom unittest.mock import MagicMock, patch+import pytest+from openjarvis.core.registry import ToolRegistryfrom openjarvis.tools.web_search import WebSearchTool@@ -413,7 +415,7 @@ class TestUrlFetching:mock_resp.text = "<html><body><p>Hello world</p></body></html>"mock_resp.headers = {"content-type": "text/html"}mock_resp.raise_for_status = MagicMock()- monkeypatch.setattr(httpx, "get", MagicMock(return_value=mock_resp))+ monkeypatch.setattr(httpx.Client, "get", MagicMock(return_value=mock_resp))content = WebSearchTool._fetch_url("https://example.com")assert "Hello world" in content@@ -426,7 +428,7 @@ class TestUrlFetching:mock_resp.text = "<html><script>var x=1;</script><body>Content</body></html>"mock_resp.headers = {"content-type": "text/html"}mock_resp.raise_for_status = MagicMock()- monkeypatch.setattr(httpx, "get", MagicMock(return_value=mock_resp))+ monkeypatch.setattr(httpx.Client, "get", MagicMock(return_value=mock_resp))content = WebSearchTool._fetch_url("https://example.com")assert "var x" not in content@@ -440,7 +442,7 @@ class TestUrlFetching:mock_resp.text = "<p>" + "x" * 10000 + "</p>"mock_resp.headers = {"content-type": "text/html"}mock_resp.raise_for_status = MagicMock()- monkeypatch.setattr(httpx, "get", MagicMock(return_value=mock_resp))+ monkeypatch.setattr(httpx.Client, "get", MagicMock(return_value=mock_resp))content = WebSearchTool._fetch_url("https://example.com", max_chars=100)assert len(content) < 200@@ -454,13 +456,139 @@ class TestUrlFetching:mock_resp.text = "%PDF-1.4 binary data"mock_resp.headers = {"content-type": "application/pdf"}mock_resp.raise_for_status = MagicMock()- monkeypatch.setattr(httpx, "get", MagicMock(return_value=mock_resp))+ monkeypatch.setattr(httpx.Client, "get", MagicMock(return_value=mock_resp))content = WebSearchTool._fetch_url("https://example.com/file.pdf")assert "PDF" in contentassert "cannot be read" in content+class TestFetchRedirectSession:+ @pytest.fixture(autouse=True)+ def _guard(self, monkeypatch):+ import openjarvis.tools.web_search as web_search++ guard = MagicMock(return_value=None)+ monkeypatch.setattr(web_search, "check_ssrf", guard)+ return guard++ @pytest.mark.parametrize("status", [301, 302, 303, 307, 308])+ def test_redirect_keeps_cookie_and_closes_client(self, monkeypatch, status):+ """Keep the actual client lifecycle so a fresh client per hop fails."""+ import httpx++ requests = []+ closed = []+ exit_transport = httpx.HTTPTransport.__exit__++ def close_transport(transport, *args):+ closed.append(transport)+ return exit_transport(transport, *args)++ def handler(transport, request):+ requests.append(request)+ if request.url.path == "/start":+ return httpx.Response(+ status,+ headers={+ "location": "/article",+ "set-cookie": "session=redirect; Path=/; Secure",+ },+ )+ if request.headers.get("cookie") != "session=redirect":+ return httpx.Response(403)+ return httpx.Response(200, text="article with redirect cookie")++ monkeypatch.setattr(httpx.HTTPTransport, "handle_request", handler)+ monkeypatch.setattr(httpx.HTTPTransport, "__exit__", close_transport)++ assert WebSearchTool._fetch_url("https://public.example/start") == (+ "article with redirect cookie"+ )+ assert len(requests) == 2+ assert closed++ @pytest.mark.parametrize(+ "destination",+ ["https://other.example/article", "http://public.example/article"],+ )+ def test_cookie_is_scoped_to_host_and_fetch(self, monkeypatch, destination):+ import httpx++ requests = []++ def handler(transport, request):+ requests.append(request)+ if request.url.path == "/start":+ return httpx.Response(+ 302,+ headers={+ "location": destination,+ "set-cookie": "session=private; Path=/; Secure",+ },+ )+ return httpx.Response(200, text="article")++ monkeypatch.setattr(httpx.HTTPTransport, "handle_request", handler)++ assert WebSearchTool._fetch_url("https://public.example/start") == "article"+ assert WebSearchTool._fetch_url("https://public.example/article") == "article"+ assert [request.headers.get("cookie") for request in requests] == [+ None,+ None,+ None,+ ]++ def test_private_redirect_with_cookie_is_never_requested(self, monkeypatch, _guard):+ import httpx++ requests = []++ def handler(transport, request):+ requests.append(request)+ return httpx.Response(+ 302,+ headers={+ "location": "http://127.0.0.1/admin",+ "set-cookie": "session=private; Path=/",+ },+ )++ _guard.side_effect = [None, "private IP blocked"]+ monkeypatch.setattr(httpx.HTTPTransport, "handle_request", handler)++ with pytest.raises(ValueError, match="private IP blocked"):+ WebSearchTool._fetch_url("https://public.example/start")+ assert len(requests) == 1+ assert [call.args[0] for call in _guard.call_args_list] == [+ "https://public.example/start",+ "http://127.0.0.1/admin",+ ]++ @pytest.mark.parametrize("redirects", [5, 6])+ def test_redirect_limit(self, monkeypatch, _guard, redirects):+ import httpx++ requests = []++ def handler(transport, request):+ requests.append(request)+ hop = int(request.url.path.removeprefix("/"))+ if hop < redirects:+ return httpx.Response(302, headers={"location": f"/{hop + 1}"})+ return httpx.Response(200, text="article")++ monkeypatch.setattr(httpx.HTTPTransport, "handle_request", handler)++ if redirects == 5:+ assert WebSearchTool._fetch_url("https://public.example/0") == "article"+ else:+ with pytest.raises(ValueError, match="maximum of 5 redirects"):+ WebSearchTool._fetch_url("https://public.example/0")+ assert len(requests) == 6+ assert _guard.call_count == 6++class TestExecuteWithUrl:def _mock_ssrf(self, monkeypatch):"""Stub out the SSRF check (requires Rust backend)."""@@ -477,7 +605,7 @@ class TestExecuteWithUrl:mock_resp.text = "<html><body>Page content here</body></html>"mock_resp.headers = {"content-type": "text/html"}mock_resp.raise_for_status = MagicMock()- monkeypatch.setattr(httpx, "get", MagicMock(return_value=mock_resp))+ monkeypatch.setattr(httpx.Client, "get", MagicMock(return_value=mock_resp))tool = WebSearchTool(api_key="test-key")result = tool.execute(query="https://example.com/article")@@ -494,7 +622,7 @@ class TestExecuteWithUrl:mock_resp.text = "<html><body>Article text</body></html>"mock_resp.headers = {"content-type": "text/html"}mock_resp.raise_for_status = MagicMock()- monkeypatch.setattr(httpx, "get", MagicMock(return_value=mock_resp))+ monkeypatch.setattr(httpx.Client, "get", MagicMock(return_value=mock_resp))tool = WebSearchTool(api_key="test-key")result = tool.execute(query="Summarize https://example.com/article please")@@ -522,7 +650,7 @@ class TestExecuteWithUrl:self._mock_ssrf(monkeypatch)monkeypatch.setattr(- httpx,+ httpx.Client,"get",MagicMock(side_effect=httpx.HTTPError("Connection failed")),)@@ -532,6 +660,97 @@ class TestExecuteWithUrl:assert result.success is Falseassert "Failed to fetch URL" in result.content+ def test_url_redirect_to_private_ip_is_blocked(self, monkeypatch):+ """Every URL redirect hop must be checked for SSRF."""+ import httpx++ import openjarvis.tools.web_search as _ws++ requests = []++ def handler(request):+ requests.append(str(request.url))+ if request.url.host == "public.example.com":+ return httpx.Response(+ 302,+ headers={"location": "http://127.0.0.1/admin"},+ request=request,+ )+ return httpx.Response(+ 200,+ text="internal response",+ headers={"content-type": "text/html"},+ request=request,+ )++ guard = MagicMock(+ side_effect=[None, None, "URL resolves to private IP: 127.0.0.1"]+ )+ monkeypatch.setattr(+ httpx.HTTPTransport,+ "handle_request",+ lambda self, request: handler(request),+ )+ monkeypatch.setattr(_ws, "check_ssrf", guard)+ result = WebSearchTool(engine="youcom").execute(+ query="https://public.example.com/start"+ )++ assert result.success is False+ assert "127.0.0.1" in result.content+ assert requests == ["https://public.example.com/start"]+ assert [call.args[0] for call in guard.call_args_list] == [+ "https://public.example.com/start",+ "https://public.example.com/start",+ "http://127.0.0.1/admin",+ ]++ def test_url_public_redirect_is_followed_after_ssrf_check(self, monkeypatch):+ """Public relative redirects still work when checked hop by hop."""+ import httpx++ import openjarvis.tools.web_search as _ws++ requests = []++ def handler(request):+ requests.append(str(request.url))+ if request.url.path == "/start":+ return httpx.Response(+ 302,+ headers={"location": "/article"},+ request=request,+ )+ return httpx.Response(+ 200,+ text="public article",+ headers={"content-type": "text/html"},+ request=request,+ )++ guard = MagicMock(return_value=None)+ monkeypatch.setattr(+ httpx.HTTPTransport,+ "handle_request",+ lambda self, request: handler(request),+ )+ monkeypatch.setattr(_ws, "check_ssrf", guard)+ result = WebSearchTool(engine="youcom").execute(+ query="https://public.example.com/start"+ )++ assert result.success is True+ assert result.content == "public article"+ assert requests == [+ "https://public.example.com/start",+ "https://public.example.com/article",+ ]+ assert [call.args[0] for call in guard.call_args_list] == [+ "https://public.example.com/start",+ "https://public.example.com/start",+ "https://public.example.com/article",+ ]+# ---------------------------------------------------------------------------# Engine selection and the You.com engine (#923)@@ -841,7 +1060,7 @@ class TestYouComContentsExtraction:resp.text = "<html><body>Local text</body></html>"resp.headers = {"content-type": "text/html"}resp.raise_for_status = MagicMock()- monkeypatch.setattr(httpx, "get", MagicMock(return_value=resp))+ monkeypatch.setattr(httpx.Client, "get", MagicMock(return_value=resp))monkeypatch.setattr(httpx, "post", MagicMock(side_effect=AssertionError("must not be called")))@@ -863,7 +1082,7 @@ class TestYouComContentsExtraction:resp.text = "<html><body>Local text</body></html>"resp.headers = {"content-type": "text/html"}resp.raise_for_status = MagicMock()- monkeypatch.setattr(httpx, "get", MagicMock(return_value=resp))+ monkeypatch.setattr(httpx.Client, "get", MagicMock(return_value=resp))result = WebSearchTool(engine="youcom").execute(query="https://example.com/a")