Your lightweight open-source personal agent.
Python agent-frameworkai-agentai-agentschatbotchatopsdiscord-botllm-agentsllmslocal-llmmcpmodel-context-protocolmulti-agentopenai-compatibleopenclawpersonal-ai-assistantpythonself-hostedtelegram-bot-ai-assistantwebuiworkflow-automation
fix(web): validate redirect targets before fetching
hinotoi-agentff173045fe39
diff --git a/nanobot/agent/tools/web.py b/nanobot/agent/tools/web.pyindex 7859b45dc..112167b48 100644--- a/nanobot/agent/tools/web.py+++ b/nanobot/agent/tools/web.py@@ -8,7 +8,7 @@ import jsonimport osimport refrom typing import Any, Callable-from urllib.parse import quote, urlparse+from urllib.parse import quote, urljoin, urlparseimport httpxfrom loguru import logger@@ -78,9 +78,41 @@ def _validate_url(url: str) -> tuple[bool, str]:def _validate_url_safe(url: str) -> tuple[bool, str]:"""Validate URL with SSRF protection: scheme, domain, and resolved IP check."""from nanobot.security.network import validate_url_target+return validate_url_target(url)+async def _get_with_safe_redirects(+ client: httpx.AsyncClient,+ url: str,+ headers: dict[str, str] | None = None,+) -> tuple[httpx.Response | None, str | None]:+ """GET a URL while validating every redirect target before requesting it."""+ current_url = url+ for _ in range(MAX_REDIRECTS + 1):+ is_valid, error_msg = _validate_url_safe(current_url)+ if not is_valid:+ return None, f"Redirect blocked: {error_msg}"++ response = await client.get(current_url, headers=headers, follow_redirects=False)+ if not response.is_redirect:+ return response, None++ location = response.headers.get("location")+ if not location:+ return response, None++ next_url = urljoin(str(response.url), location)+ is_valid, error_msg = _validate_url_safe(next_url)+ if not is_valid:+ await response.aclose()+ return None, f"Redirect blocked: {error_msg}"++ await response.aclose()+ current_url = next_url++ return None, f"Too many redirects: exceeded limit of {MAX_REDIRECTS}"+def _format_results(query: str, items: list[dict[str, Any]], n: int) -> str:"""Format provider results into shared plaintext output."""if not items:@@ -488,19 +520,22 @@ class WebFetchTool(Tool):# Detect and fetch images directly to avoid Jina's textual image captioningtry:- async with httpx.AsyncClient(proxy=self.proxy, follow_redirects=True, max_redirects=MAX_REDIRECTS, timeout=15.0) as client:- async with client.stream("GET", url, headers={"User-Agent": self.user_agent}) as r:- from nanobot.security.network import validate_resolved_url-- redir_ok, redir_err = validate_resolved_url(str(r.url))- if not redir_ok:- return json.dumps({"error": f"Redirect blocked: {redir_err}", "url": url}, ensure_ascii=False)-- ctype = r.headers.get("content-type", "")- if ctype.startswith("image/"):- r.raise_for_status()- raw = await r.aread()- return build_image_content_blocks(raw, ctype, url, f"(Image fetched from: {url})")+ async with httpx.AsyncClient(proxy=self.proxy, timeout=15.0) as client:+ r, redirect_error = await _get_with_safe_redirects(+ client,+ url,+ headers={"User-Agent": self.user_agent},+ )+ if redirect_error:+ return json.dumps({"error": redirect_error, "url": url}, ensure_ascii=False)+ if r is None:+ return json.dumps({"error": "Fetch failed", "url": url}, ensure_ascii=False)++ ctype = r.headers.get("content-type", "")+ if ctype.startswith("image/"):+ r.raise_for_status()+ raw = r.content+ return build_image_content_blocks(raw, ctype, url, f"(Image fetched from: {url})")except Exception as e:logger.debug("Pre-fetch image detection failed for {}: {}", url, e)@@ -553,19 +588,20 @@ class WebFetchTool(Tool):try:async with httpx.AsyncClient(- follow_redirects=True,- max_redirects=MAX_REDIRECTS,timeout=30.0,proxy=self.proxy,) as client:- r = await client.get(url, headers={"User-Agent": self.user_agent})+ r, redirect_error = await _get_with_safe_redirects(+ client,+ url,+ headers={"User-Agent": self.user_agent},+ )+ if redirect_error:+ return json.dumps({"error": redirect_error, "url": url}, ensure_ascii=False)+ if r is None:+ return json.dumps({"error": "Fetch failed", "url": url}, ensure_ascii=False)r.raise_for_status()- from nanobot.security.network import validate_resolved_url- redir_ok, redir_err = validate_resolved_url(str(r.url))- if not redir_ok:- return json.dumps({"error": f"Redirect blocked: {redir_err}", "url": url}, ensure_ascii=False)-ctype = r.headers.get("content-type", "")if ctype.startswith("image/"):return build_image_content_blocks(r.content, ctype, url, f"(Image fetched from: {url})")diff --git a/tests/tools/test_web_fetch_security.py b/tests/tools/test_web_fetch_security.pyindex 58664cf33..25286047d 100644--- a/tests/tools/test_web_fetch_security.py+++ b/tests/tools/test_web_fetch_security.py@@ -6,11 +6,15 @@ import jsonimport socketfrom unittest.mock import patch+import httpximport pytest+from nanobot.agent.tools import web as web_modulefrom nanobot.agent.tools.web import WebFetchToolfrom nanobot.config.schema import WebFetchConfig+_REAL_GETADDRINFO = socket.getaddrinfo+def _fake_resolve_private(hostname, port, family=0, type_=0):return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("169.254.169.254", 0))]@@ -54,6 +58,7 @@ async def test_web_fetch_result_contains_untrusted_flag():url = "https://example.com/page"text = fake_htmlheaders = {"content-type": "text/html"}+ is_redirect = Falsedef raise_for_status(self): passdef json(self): return {}@@ -95,6 +100,7 @@ async def test_web_fetch_can_skip_jina_and_use_custom_user_agent(monkeypatch):url = "https://example.com/page"text = "<html><head><title>Test</title></head><body><p>Hello world</p></body></html>"headers = {"content-type": "text/html"}+ is_redirect = Falsedef raise_for_status(self):return None@@ -113,7 +119,7 @@ async def test_web_fetch_can_skip_jina_and_use_custom_user_agent(monkeypatch):seen_headers.append(headers or {})return FakeStreamResponse()- async def get(self, url, headers=None):+ async def get(self, url, headers=None, **kwargs):seen_headers.append(headers or {})return FakeResponse()@@ -133,43 +139,83 @@ async def test_web_fetch_can_skip_jina_and_use_custom_user_agent(monkeypatch):@pytest.mark.asyncioasync def test_web_fetch_blocks_private_redirect_before_returning_image(monkeypatch):- tool = WebFetchTool()+ tool = WebFetchTool(config=WebFetchConfig(use_jina_reader=False))++ def handler(request: httpx.Request) -> httpx.Response:+ if str(request.url) == "https://example.com/image.png":+ return httpx.Response(+ 302,+ headers={"Location": "http://127.0.0.1/secret.png"},+ request=request,+ )+ if str(request.url) == "http://127.0.0.1/secret.png":+ return httpx.Response(+ 200,+ headers={"content-type": "image/png"},+ content=b"\x89PNG\r\n\x1a\n",+ request=request,+ )+ return httpx.Response(404, request=request)++ transport = httpx.MockTransport(handler)+ real_async_client = httpx.AsyncClient++ class TransportAsyncClient(real_async_client):+ def __init__(self, *args, **kwargs):+ kwargs.pop("proxy", None)+ super().__init__(*args, transport=transport, **kwargs)- class FakeStreamResponse:- headers = {"content-type": "image/png"}- url = "http://127.0.0.1/secret.png"- content = b"\x89PNG\r\n\x1a\n"+ monkeypatch.setattr("nanobot.agent.tools.web.httpx.AsyncClient", TransportAsyncClient)- async def __aenter__(self):- return self+ def resolve_public_start_only(hostname, port, family=0, type_=0):+ if hostname == "example.com":+ return _fake_resolve_public(hostname, port, family, type_)+ return _REAL_GETADDRINFO(hostname, port, family, type_)- async def __aexit__(self, exc_type, exc, tb):- return False+ with patch("nanobot.security.network.socket.getaddrinfo", resolve_public_start_only):+ result = await tool.execute(url="https://example.com/image.png")- async def aread(self):- return self.content+ data = json.loads(result)+ assert "error" in data+ assert "redirect blocked" in data["error"].lower()- def raise_for_status(self):- return None- class FakeClient:+async def test_web_fetch_does_not_request_private_redirect_target(monkeypatch):+ tool = WebFetchTool(config=WebFetchConfig(use_jina_reader=False))+ requested: list[str] = []++ def handler(request: httpx.Request) -> httpx.Response:+ requested.append(str(request.url))+ if str(request.url) == "https://attacker.example/start":+ return httpx.Response(+ 302,+ headers={"Location": "http://127.0.0.1:8765/metadata"},+ request=request,+ )+ if str(request.url) == "http://127.0.0.1:8765/metadata":+ return httpx.Response(200, content=b"internal secret", request=request)+ return httpx.Response(404, request=request)++ transport = httpx.MockTransport(handler)+ real_async_client = httpx.AsyncClient++ class TransportAsyncClient(real_async_client):def __init__(self, *args, **kwargs):- pass-- async def __aenter__(self):- return self-- async def __aexit__(self, exc_type, exc, tb):- return False+ kwargs["transport"] = transport+ super().__init__(*args, **kwargs)- def stream(self, method, url, headers=None):- return FakeStreamResponse()+ monkeypatch.setattr(web_module.httpx, "AsyncClient", TransportAsyncClient)- monkeypatch.setattr("nanobot.agent.tools.web.httpx.AsyncClient", FakeClient)+ def resolve_public_start_only(hostname, port, family=0, type_=0):+ if hostname == "attacker.example":+ return _fake_resolve_public(hostname, port, family, type_)+ return _REAL_GETADDRINFO(hostname, port, family, type_)- with patch("nanobot.security.network.socket.getaddrinfo", _fake_resolve_public):- result = await tool.execute(url="https://example.com/image.png")+ with patch("nanobot.security.network.socket.getaddrinfo", resolve_public_start_only):+ result = await tool.execute(url="https://attacker.example/start")data = json.loads(result)assert "error" in dataassert "redirect blocked" in data["error"].lower()+ assert requested == ["https://attacker.example/start"]