Skip to content

Personal AI, On Personal Devices

Python

fix(security): validate web search redirect targets (#966)

Haotian Zheng7252d50143d3
diff --git a/src/openjarvis/tools/web_search.py b/src/openjarvis/tools/web_search.py
index 61b29fd2..e50d7619 100644
--- a/src/openjarvis/tools/web_search.py
+++ b/src/openjarvis/tools/web_search.py
@@ -20,6 +20,7 @@ from __future__ import annotations
import logging
import os
from typing import Any
+from urllib.parse import urljoin
from openjarvis import __version__
from openjarvis.core.registry import ToolRegistry
@@ -44,6 +45,9 @@ YOUCOM_USER_AGENT = (
f"openjarvis/{__version__} youdotcom-integration/open-jarvis-openjarvis"
)
+_MAX_FETCH_REDIRECTS = 5
+_REDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})
+
# Keyless tier exhaustion (402) and per-IP throttling (429) both mean "get a
# key", which is a different remedy from a generic HTTP failure.
_KEYLESS_LIMIT_STATUSES = (402, 429)
@@ -173,24 +177,39 @@ class WebSearchTool(BaseTool):
@staticmethod
def _fetch_url(url: str, max_chars: int = 6000) -> str:
- """Fetch a URL and return extracted text content."""
+ """Fetch a URL and return extracted text after checking each redirect."""
import re as _re
import httpx
url = WebSearchTool._normalize_url(url)
- ssrf_error = check_ssrf(url)
- if ssrf_error:
- raise ValueError(ssrf_error)
- resp = httpx.get(
- url.strip(),
- follow_redirects=True,
+ current_url = url.strip()
+ # One client preserves cookie scope across redirect hops. Its lifetime
+ # is limited to this fetch so unrelated requests do not share cookies.
+ with httpx.Client(
+ follow_redirects=False,
timeout=30.0,
headers={
"User-Agent": "Mozilla/5.0 (compatible; OpenJarvis/1.0; +https://github.com/openjarvis)"
},
- )
- resp.raise_for_status()
+ ) as client:
+ for _ in range(_MAX_FETCH_REDIRECTS + 1):
+ ssrf_error = check_ssrf(current_url)
+ if ssrf_error:
+ raise ValueError(ssrf_error)
+ resp = client.get(current_url)
+ if resp.status_code not in _REDIRECT_STATUS_CODES:
+ resp.raise_for_status()
+ break
+ location = resp.headers.get("location", "")
+ if not location:
+ resp.raise_for_status()
+ break
+ current_url = urljoin(str(resp.url), location)
+ else:
+ raise ValueError(
+ f"URL exceeded the maximum of {_MAX_FETCH_REDIRECTS} redirects"
+ )
content_type = resp.headers.get("content-type", "")
if "application/pdf" in content_type:
return (
diff --git a/tests/agents/test_native_openhands.py b/tests/agents/test_native_openhands.py
index bf370ade..6fe49581 100644
--- a/tests/agents/test_native_openhands.py
+++ b/tests/agents/test_native_openhands.py
@@ -554,7 +554,7 @@ class TestUrlExpansion:
mock_resp.text = "<html><body>Page content</body></html>"
mock_resp.headers = {"content-type": "text/html"}
mock_resp.raise_for_status = MagicMock()
- monkeypatch.setattr(httpx, "get", MagicMock(return_value=mock_resp))
+ monkeypatch.setattr(httpx.Client, "get", MagicMock(return_value=mock_resp))
agent = NativeOpenHandsAgent(MagicMock(), "test-model")
text, expanded = agent._expand_urls("Summarize: https://example.com/article")
@@ -566,7 +566,7 @@ class TestUrlExpansion:
import httpx
monkeypatch.setattr(
- httpx,
+ httpx.Client,
"get",
MagicMock(side_effect=Exception("Connection error")),
)
@@ -591,7 +591,7 @@ class TestUrlExpansion:
return True, 0.0
fetch = MagicMock()
- monkeypatch.setattr(httpx, "get", fetch)
+ monkeypatch.setattr(httpx.Client, "get", fetch)
policy = CapabilityPolicy(default_deny=True)
policy.grant("_default", "network:fetch")
policy.deny("restricted-native", "network:fetch")
@@ -627,7 +627,7 @@ class TestUrlExpansion:
mock_resp.text = "<html><body>Article text here</body></html>"
mock_resp.headers = {"content-type": "text/html"}
mock_resp.raise_for_status = MagicMock()
- monkeypatch.setattr(httpx, "get", MagicMock(return_value=mock_resp))
+ monkeypatch.setattr(httpx.Client, "get", MagicMock(return_value=mock_resp))
engine = MagicMock()
engine.engine_id = "mock"
diff --git a/tests/tools/test_web_search.py b/tests/tools/test_web_search.py
index b8584977..7c959e9a 100644
--- a/tests/tools/test_web_search.py
+++ b/tests/tools/test_web_search.py
@@ -5,6 +5,8 @@ from __future__ import annotations
import sys
from unittest.mock import MagicMock, patch
+import pytest
+
from openjarvis.core.registry import ToolRegistry
from openjarvis.tools.web_search import WebSearchTool
@@ -413,7 +415,7 @@ class TestUrlFetching:
mock_resp.text = "<html><body><p>Hello world</p></body></html>"
mock_resp.headers = {"content-type": "text/html"}
mock_resp.raise_for_status = MagicMock()
- monkeypatch.setattr(httpx, "get", MagicMock(return_value=mock_resp))
+ monkeypatch.setattr(httpx.Client, "get", MagicMock(return_value=mock_resp))
content = WebSearchTool._fetch_url("https://example.com")
assert "Hello world" in content
@@ -426,7 +428,7 @@ class TestUrlFetching:
mock_resp.text = "<html><script>var x=1;</script><body>Content</body></html>"
mock_resp.headers = {"content-type": "text/html"}
mock_resp.raise_for_status = MagicMock()
- monkeypatch.setattr(httpx, "get", MagicMock(return_value=mock_resp))
+ monkeypatch.setattr(httpx.Client, "get", MagicMock(return_value=mock_resp))
content = WebSearchTool._fetch_url("https://example.com")
assert "var x" not in content
@@ -440,7 +442,7 @@ class TestUrlFetching:
mock_resp.text = "<p>" + "x" * 10000 + "</p>"
mock_resp.headers = {"content-type": "text/html"}
mock_resp.raise_for_status = MagicMock()
- monkeypatch.setattr(httpx, "get", MagicMock(return_value=mock_resp))
+ monkeypatch.setattr(httpx.Client, "get", MagicMock(return_value=mock_resp))
content = WebSearchTool._fetch_url("https://example.com", max_chars=100)
assert len(content) < 200
@@ -454,13 +456,139 @@ class TestUrlFetching:
mock_resp.text = "%PDF-1.4 binary data"
mock_resp.headers = {"content-type": "application/pdf"}
mock_resp.raise_for_status = MagicMock()
- monkeypatch.setattr(httpx, "get", MagicMock(return_value=mock_resp))
+ monkeypatch.setattr(httpx.Client, "get", MagicMock(return_value=mock_resp))
content = WebSearchTool._fetch_url("https://example.com/file.pdf")
assert "PDF" in content
assert "cannot be read" in content
+class TestFetchRedirectSession:
+ @pytest.fixture(autouse=True)
+ def _guard(self, monkeypatch):
+ import openjarvis.tools.web_search as web_search
+
+ guard = MagicMock(return_value=None)
+ monkeypatch.setattr(web_search, "check_ssrf", guard)
+ return guard
+
+ @pytest.mark.parametrize("status", [301, 302, 303, 307, 308])
+ def test_redirect_keeps_cookie_and_closes_client(self, monkeypatch, status):
+ """Keep the actual client lifecycle so a fresh client per hop fails."""
+ import httpx
+
+ requests = []
+ closed = []
+ exit_transport = httpx.HTTPTransport.__exit__
+
+ def close_transport(transport, *args):
+ closed.append(transport)
+ return exit_transport(transport, *args)
+
+ def handler(transport, request):
+ requests.append(request)
+ if request.url.path == "/start":
+ return httpx.Response(
+ status,
+ headers={
+ "location": "/article",
+ "set-cookie": "session=redirect; Path=/; Secure",
+ },
+ )
+ if request.headers.get("cookie") != "session=redirect":
+ return httpx.Response(403)
+ return httpx.Response(200, text="article with redirect cookie")
+
+ monkeypatch.setattr(httpx.HTTPTransport, "handle_request", handler)
+ monkeypatch.setattr(httpx.HTTPTransport, "__exit__", close_transport)
+
+ assert WebSearchTool._fetch_url("https://public.example/start") == (
+ "article with redirect cookie"
+ )
+ assert len(requests) == 2
+ assert closed
+
+ @pytest.mark.parametrize(
+ "destination",
+ ["https://other.example/article", "http://public.example/article"],
+ )
+ def test_cookie_is_scoped_to_host_and_fetch(self, monkeypatch, destination):
+ import httpx
+
+ requests = []
+
+ def handler(transport, request):
+ requests.append(request)
+ if request.url.path == "/start":
+ return httpx.Response(
+ 302,
+ headers={
+ "location": destination,
+ "set-cookie": "session=private; Path=/; Secure",
+ },
+ )
+ return httpx.Response(200, text="article")
+
+ monkeypatch.setattr(httpx.HTTPTransport, "handle_request", handler)
+
+ assert WebSearchTool._fetch_url("https://public.example/start") == "article"
+ assert WebSearchTool._fetch_url("https://public.example/article") == "article"
+ assert [request.headers.get("cookie") for request in requests] == [
+ None,
+ None,
+ None,
+ ]
+
+ def test_private_redirect_with_cookie_is_never_requested(self, monkeypatch, _guard):
+ import httpx
+
+ requests = []
+
+ def handler(transport, request):
+ requests.append(request)
+ return httpx.Response(
+ 302,
+ headers={
+ "location": "http://127.0.0.1/admin",
+ "set-cookie": "session=private; Path=/",
+ },
+ )
+
+ _guard.side_effect = [None, "private IP blocked"]
+ monkeypatch.setattr(httpx.HTTPTransport, "handle_request", handler)
+
+ with pytest.raises(ValueError, match="private IP blocked"):
+ WebSearchTool._fetch_url("https://public.example/start")
+ assert len(requests) == 1
+ assert [call.args[0] for call in _guard.call_args_list] == [
+ "https://public.example/start",
+ "http://127.0.0.1/admin",
+ ]
+
+ @pytest.mark.parametrize("redirects", [5, 6])
+ def test_redirect_limit(self, monkeypatch, _guard, redirects):
+ import httpx
+
+ requests = []
+
+ def handler(transport, request):
+ requests.append(request)
+ hop = int(request.url.path.removeprefix("/"))
+ if hop < redirects:
+ return httpx.Response(302, headers={"location": f"/{hop + 1}"})
+ return httpx.Response(200, text="article")
+
+ monkeypatch.setattr(httpx.HTTPTransport, "handle_request", handler)
+
+ if redirects == 5:
+ assert WebSearchTool._fetch_url("https://public.example/0") == "article"
+ else:
+ with pytest.raises(ValueError, match="maximum of 5 redirects"):
+ WebSearchTool._fetch_url("https://public.example/0")
+ assert len(requests) == 6
+ assert _guard.call_count == 6
+
+
class TestExecuteWithUrl:
def _mock_ssrf(self, monkeypatch):
"""Stub out the SSRF check (requires Rust backend)."""
@@ -477,7 +605,7 @@ class TestExecuteWithUrl:
mock_resp.text = "<html><body>Page content here</body></html>"
mock_resp.headers = {"content-type": "text/html"}
mock_resp.raise_for_status = MagicMock()
- monkeypatch.setattr(httpx, "get", MagicMock(return_value=mock_resp))
+ monkeypatch.setattr(httpx.Client, "get", MagicMock(return_value=mock_resp))
tool = WebSearchTool(api_key="test-key")
result = tool.execute(query="https://example.com/article")
@@ -494,7 +622,7 @@ class TestExecuteWithUrl:
mock_resp.text = "<html><body>Article text</body></html>"
mock_resp.headers = {"content-type": "text/html"}
mock_resp.raise_for_status = MagicMock()
- monkeypatch.setattr(httpx, "get", MagicMock(return_value=mock_resp))
+ monkeypatch.setattr(httpx.Client, "get", MagicMock(return_value=mock_resp))
tool = WebSearchTool(api_key="test-key")
result = tool.execute(query="Summarize https://example.com/article please")
@@ -522,7 +650,7 @@ class TestExecuteWithUrl:
self._mock_ssrf(monkeypatch)
monkeypatch.setattr(
- httpx,
+ httpx.Client,
"get",
MagicMock(side_effect=httpx.HTTPError("Connection failed")),
)
@@ -532,6 +660,97 @@ class TestExecuteWithUrl:
assert result.success is False
assert "Failed to fetch URL" in result.content
+ def test_url_redirect_to_private_ip_is_blocked(self, monkeypatch):
+ """Every URL redirect hop must be checked for SSRF."""
+ import httpx
+
+ import openjarvis.tools.web_search as _ws
+
+ requests = []
+
+ def handler(request):
+ requests.append(str(request.url))
+ if request.url.host == "public.example.com":
+ return httpx.Response(
+ 302,
+ headers={"location": "http://127.0.0.1/admin"},
+ request=request,
+ )
+ return httpx.Response(
+ 200,
+ text="internal response",
+ headers={"content-type": "text/html"},
+ request=request,
+ )
+
+ guard = MagicMock(
+ side_effect=[None, None, "URL resolves to private IP: 127.0.0.1"]
+ )
+ monkeypatch.setattr(
+ httpx.HTTPTransport,
+ "handle_request",
+ lambda self, request: handler(request),
+ )
+ monkeypatch.setattr(_ws, "check_ssrf", guard)
+ result = WebSearchTool(engine="youcom").execute(
+ query="https://public.example.com/start"
+ )
+
+ assert result.success is False
+ assert "127.0.0.1" in result.content
+ assert requests == ["https://public.example.com/start"]
+ assert [call.args[0] for call in guard.call_args_list] == [
+ "https://public.example.com/start",
+ "https://public.example.com/start",
+ "http://127.0.0.1/admin",
+ ]
+
+ def test_url_public_redirect_is_followed_after_ssrf_check(self, monkeypatch):
+ """Public relative redirects still work when checked hop by hop."""
+ import httpx
+
+ import openjarvis.tools.web_search as _ws
+
+ requests = []
+
+ def handler(request):
+ requests.append(str(request.url))
+ if request.url.path == "/start":
+ return httpx.Response(
+ 302,
+ headers={"location": "/article"},
+ request=request,
+ )
+ return httpx.Response(
+ 200,
+ text="public article",
+ headers={"content-type": "text/html"},
+ request=request,
+ )
+
+ guard = MagicMock(return_value=None)
+ monkeypatch.setattr(
+ httpx.HTTPTransport,
+ "handle_request",
+ lambda self, request: handler(request),
+ )
+ monkeypatch.setattr(_ws, "check_ssrf", guard)
+ result = WebSearchTool(engine="youcom").execute(
+ query="https://public.example.com/start"
+ )
+
+ assert result.success is True
+ assert result.content == "public article"
+ assert requests == [
+ "https://public.example.com/start",
+ "https://public.example.com/article",
+ ]
+ assert [call.args[0] for call in guard.call_args_list] == [
+ "https://public.example.com/start",
+ "https://public.example.com/start",
+ "https://public.example.com/article",
+ ]
+
# ---------------------------------------------------------------------------
# Engine selection and the You.com engine (#923)
@@ -841,7 +1060,7 @@ class TestYouComContentsExtraction:
resp.text = "<html><body>Local text</body></html>"
resp.headers = {"content-type": "text/html"}
resp.raise_for_status = MagicMock()
- monkeypatch.setattr(httpx, "get", MagicMock(return_value=resp))
+ monkeypatch.setattr(httpx.Client, "get", MagicMock(return_value=resp))
monkeypatch.setattr(
httpx, "post", MagicMock(side_effect=AssertionError("must not be called"))
)
@@ -863,7 +1082,7 @@ class TestYouComContentsExtraction:
resp.text = "<html><body>Local text</body></html>"
resp.headers = {"content-type": "text/html"}
resp.raise_for_status = MagicMock()
- monkeypatch.setattr(httpx, "get", MagicMock(return_value=resp))
+ monkeypatch.setattr(httpx.Client, "get", MagicMock(return_value=resp))
result = WebSearchTool(engine="youcom").execute(query="https://example.com/a")