AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(classic): harden outbound request handling (SSRF + URL validation) (#13422)
Krzysztof Czerwinskic807c3690060
diff --git a/classic/forge/forge/components/git_operations/git_operations.py b/classic/forge/forge/components/git_operations/git_operations.pyindex 48567edd20..8ad9aa05ce 100644--- a/classic/forge/forge/components/git_operations/git_operations.py+++ b/classic/forge/forge/components/git_operations/git_operations.py@@ -1,5 +1,6 @@from pathlib import Pathfrom typing import Iterator, Optional+from urllib.parse import urlparsefrom git.exc import GitCommandError, InvalidGitRepositoryErrorfrom git.repo import Repo@@ -93,6 +94,16 @@ class GitOperationsComponent(Returns:str: The result of the clone operation."""+ # Only inject credentials when cloning from GitHub. The URL is+ # attacker-controllable, so without this check the configured+ # credentials would be sent to any host the URL points to.+ hostname = (urlparse(url).hostname or "").lower()+ if hostname not in ("github.com", "www.github.com"):+ raise CommandExecutionError(+ f"Refusing to clone from non-GitHub host '{hostname}': "+ "credential injection is only allowed for github.com"+ )+split_url = url.split("//")api_key = (self.config.github_api_key.get_secret_value()diff --git a/classic/forge/forge/components/git_operations/test_git_operations.py b/classic/forge/forge/components/git_operations/test_git_operations.pyindex 8d6cb4f86d..d833ab070e 100644--- a/classic/forge/forge/components/git_operations/test_git_operations.py+++ b/classic/forge/forge/components/git_operations/test_git_operations.py@@ -55,3 +55,38 @@ def test_clone_repository_error(with pytest.raises(CommandExecutionError):git_ops_component.clone_repository(url, clone_path)+++def test_clone_github_host_injects_credentials(+ git_ops_component: GitOperationsComponent,+ storage: FileStorage,+ mock_clone_from,+):+ mock_clone_from.return_value = None++ url = "https://github.com/owner/repo.git"+ clone_path = storage.get_path("github-repo")++ git_ops_component.clone_repository(url, clone_path)++ mock_clone_from.assert_called_once()+ called_url = mock_clone_from.call_args.kwargs["url"]+ assert "github.com/owner/repo.git" in called_url+ assert called_url.startswith(+ f"https://{git_ops_component.config.github_username}:"+ f"{git_ops_component.config.github_api_key}@github.com"+ )+++def test_clone_non_github_host_is_refused(+ git_ops_component: GitOperationsComponent,+ storage: FileStorage,+ mock_clone_from,+):+ url = "https://evil.example.com/repo.git"+ clone_path = storage.get_path("evil-repo")++ with pytest.raises(CommandExecutionError):+ git_ops_component.clone_repository(url, clone_path)++ mock_clone_from.assert_not_called()diff --git a/classic/forge/forge/components/http_client/http_client.py b/classic/forge/forge/components/http_client/http_client.pyindex 132348ec79..2ded9dbb41 100644--- a/classic/forge/forge/components/http_client/http_client.py+++ b/classic/forge/forge/components/http_client/http_client.py@@ -1,6 +1,7 @@import jsonimport loggingfrom typing import Any, Iterator, Optional+from urllib.parse import urljoin, urlparseimport requestsfrom pydantic import BaseModel, Field@@ -10,6 +11,7 @@ from forge.agent.protocols import CommandProvider, DirectiveProviderfrom forge.command import Command, commandfrom forge.models.json_schema import JSONSchemafrom forge.utils.exceptions import HTTPError+from forge.utils.url_validator import check_public_addresslogger = logging.getLogger(__name__)@@ -41,6 +43,7 @@ class HTTPClientComponent("""Provides commands to make HTTP requests."""config_class = HTTPClientConfiguration+ MAX_REDIRECTS = 5def __init__(self, config: Optional[HTTPClientConfiguration] = None):ConfigurableComponent.__init__(self, config)@@ -57,17 +60,26 @@ class HTTPClientComponent(yield self.http_deletedef _is_domain_allowed(self, url: str) -> bool:- """Check if the URL's domain is in the allowed list."""+ """Check if the URL's host is in the allowed list."""+ # Backslashes are normalized to forward slashes by requests/browsers,+ # which lets userinfo tricks like "http://evil\@allowed.com" reach an+ # unintended host. Reject them up front, before the empty-allowlist+ # shortcut, so a malformed URL can't slip through the default config.+ if "\\" in url:+ return False+if not self.config.allowed_domains:return True- from urllib.parse import urlparse-parsed = urlparse(url)- domain = parsed.netloc.lower()+ # .hostname strips userinfo and port and lowercases the host.+ host = parsed.hostname+ if not host:+ return Falsefor allowed in self.config.allowed_domains:- if domain == allowed.lower() or domain.endswith("." + allowed.lower()):+ allowed_host = allowed.lower()+ if host == allowed_host or host.endswith("." + allowed_host):return Truereturn False@@ -93,44 +105,74 @@ class HTTPClientComponent(Returns:dict: Structured response with status, headers, and body"""- if not self._is_domain_allowed(url):- raise HTTPError(- f"Domain not in allowed list. Allowed: {self.config.allowed_domains}",- url=url,- )-request_timeout = timeout or self.config.default_timeoutrequest_headers = headers or {}+ method = method.upper()+ if method not in ("GET", "POST", "PUT", "DELETE"):+ raise HTTPError(f"Unsupported HTTP method: {method}", url=url)++ # Follow redirects manually so every hop is re-validated. `requests`+ # follows redirects by default, which would otherwise let a public URL+ # redirect to 127.0.0.1 / metadata IPs and bypass the checks below.+ current_url = url+ current_method = method+ current_body = body+ current_params = params+try:- if method == "GET":- response = self.session.get(- url, headers=request_headers, params=params, timeout=request_timeout- )- elif method == "POST":- response = self.session.post(- url,- headers=request_headers,- params=params,- json=body if isinstance(body, dict) else None,- data=body if isinstance(body, str) else None,- timeout=request_timeout,- )- elif method == "PUT":- response = self.session.put(- url,+ for _ in range(self.MAX_REDIRECTS + 1):+ # Block internal/private targets regardless of the allowlist.+ # With the default empty allowlist `_is_domain_allowed` returns+ # True, so without this the client could still reach 127.0.0.1,+ # metadata IPs, etc. — checked on the initial URL and every hop.+ if not check_public_address(current_url):+ raise HTTPError(+ "Access to internal/private addresses is restricted",+ url=current_url,+ )+ if not self._is_domain_allowed(current_url):+ raise HTTPError(+ "Domain not in allowed list. "+ f"Allowed: {self.config.allowed_domains}",+ url=current_url,+ )++ response = self.session.request(+ current_method,+ current_url,headers=request_headers,- params=params,- json=body if isinstance(body, dict) else None,- data=body if isinstance(body, str) else None,+ params=current_params,+ json=current_body if isinstance(current_body, dict) else None,+ data=current_body if isinstance(current_body, str) else None,timeout=request_timeout,+ allow_redirects=False,)- elif method == "DELETE":- response = self.session.delete(- url, headers=request_headers, params=params, timeout=request_timeout- )++ location = response.headers.get("Location")+ if not (response.is_redirect and location):+ break++ # Resolve the next hop; it gets re-validated next iteration.+ next_url = urljoin(current_url, location)+ # requests strips Authorization when a redirect crosses hosts+ # (Session.rebuild_auth); mirror that here so caller-supplied+ # credentials don't follow a redirect to a different host.+ if urlparse(next_url).hostname != urlparse(current_url).hostname:+ request_headers = {+ key: value+ for key, value in request_headers.items()+ if key.lower() != "authorization"+ }+ current_url = next_url+ # Per HTTP semantics, 301/302/303 downgrade to GET without a body.+ if response.status_code in (301, 302, 303):+ current_method = "GET"+ current_body = None+ # The redirect target carries its own query string.+ current_params = Noneelse:- raise HTTPError(f"Unsupported HTTP method: {method}", url=url)+ raise HTTPError("Too many redirects", url=url)# Check response sizecontent_length = len(response.content)diff --git a/classic/forge/forge/components/http_client/test_http_client.py b/classic/forge/forge/components/http_client/test_http_client.pynew file mode 100644index 0000000000..62ac69ff0d--- /dev/null+++ b/classic/forge/forge/components/http_client/test_http_client.py@@ -0,0 +1,124 @@+"""Tests for the HTTP client component."""++from unittest.mock import MagicMock++import pytest++from forge.utils.exceptions import HTTPError++from .http_client import HTTPClientComponent, HTTPClientConfiguration+++def allowlisted_component():+ """Create an HTTPClientComponent restricted to whitelist.com."""+ config = HTTPClientConfiguration(allowed_domains=["whitelist.com"])+ return HTTPClientComponent(config)+++def test_allowed_domain_passes(allowlisted_component):+ assert allowlisted_component._is_domain_allowed("https://whitelist.com/path")+++def test_subdomain_of_allowed_domain_passes(allowlisted_component):+ assert allowlisted_component._is_domain_allowed("https://api.whitelist.com/path")+++def test_no_allowlist_allows_everything():+ component = HTTPClientComponent(HTTPClientConfiguration())+ assert component._is_domain_allowed("http://127.0.0.1/")+++def test_disallowed_domain_rejected(allowlisted_component):+ assert not allowlisted_component._is_domain_allowed("http://127.0.0.1/")+++def test_userinfo_backslash_bypass_rejected(allowlisted_component):+ # Regression: netloc-based check passed but request hit 127.0.0.1.+ assert not allowlisted_component._is_domain_allowed(+ "http://127.0.0.1:6666\\@.whitelist.com"+ )+++def test_userinfo_bypass_rejected(allowlisted_component):+ assert not allowlisted_component._is_domain_allowed(+ "http://[email protected]/"+ )+++def test_make_request_blocks_internal_address_without_allowlist():+ # Even with the default empty allowlist, _make_request must reject+ # internal/private targets before issuing the request.+ component = HTTPClientComponent(HTTPClientConfiguration())+ with pytest.raises(HTTPError):+ component._make_request("GET", "http://127.0.0.1/")+++def test_backslash_url_rejected_without_allowlist():+ # Backslash authority trick must be rejected even with the default empty+ # allowlist (requests normalizes "\" and would reach an unintended host).+ component = HTTPClientComponent(HTTPClientConfiguration())+ assert not component._is_domain_allowed("http://127.0.0.1:6666\\@example.com/")+ with pytest.raises(HTTPError):+ component._make_request("GET", "http://127.0.0.1:6666\\@example.com/")+++def _redirect_response(location: str) -> MagicMock:+ response = MagicMock()+ response.is_redirect = True+ response.status_code = 302+ response.headers = {"Location": location}+ return response+++def _final_response() -> MagicMock:+ response = MagicMock()+ response.is_redirect = False+ response.status_code = 200+ response.headers = {}+ response.content = b"ok"+ return response+++def test_cross_host_redirect_strips_authorization_header():+ # requests strips Authorization when following a cross-host redirect+ # (Session.rebuild_auth); the manual redirect loop must do the same.+ component = HTTPClientComponent(HTTPClientConfiguration())+ component.session.request = MagicMock(+ side_effect=[_redirect_response("https://example.org/"), _final_response()]+ )++ component._make_request(+ "GET", "https://example.com/", headers={"Authorization": "Bearer token"}+ )++ calls = component.session.request.call_args_list+ assert calls[0].kwargs["headers"] == {"Authorization": "Bearer token"}+ assert calls[1].kwargs["headers"] == {}+++def test_same_host_redirect_keeps_authorization_header():+ component = HTTPClientComponent(HTTPClientConfiguration())+ component.session.request = MagicMock(+ side_effect=[_redirect_response("/moved"), _final_response()]+ )++ component._make_request(+ "GET", "https://example.com/", headers={"Authorization": "Bearer token"}+ )++ calls = component.session.request.call_args_list+ assert calls[1].kwargs["headers"] == {"Authorization": "Bearer token"}+++def test_make_request_blocks_redirect_to_internal():+ # A public URL that redirects to an internal host must be rejected: redirects+ # are followed manually and each hop is re-validated.+ component = HTTPClientComponent(HTTPClientConfiguration())+ redirect = MagicMock()+ redirect.is_redirect = True+ redirect.status_code = 302+ redirect.headers = {"Location": "http://127.0.0.1/"}+ component.session.request = MagicMock(return_value=redirect)+ with pytest.raises(HTTPError):+ component._make_request("GET", "https://example.com/")diff --git a/classic/forge/forge/utils/test_url_validator.py b/classic/forge/forge/utils/test_url_validator.pyindex dc2cce3008..31bcc6a629 100644--- a/classic/forge/forge/utils/test_url_validator.py+++ b/classic/forge/forge/utils/test_url_validator.py@@ -155,3 +155,22 @@ def test_internationalized_url():"""url = "http://例子.测试"assert dummy_method(url) == url+++ssrf_internal_urls = (+ ("http://127.0.0.1/"),+ ("http://169.254.169.254/"),+ ("http://10.0.0.1/"),+ ("http://localhost/"),+ ("http://2130706433/"),+)++[email protected]("url", ssrf_internal_urls)+def test_url_validation_blocks_internal_addresses(url):+ """+ Tests that the validator rejects URLs resolving to internal/private addresses+ (SSRF protection), including the cloud metadata IP and alternate encodings.+ """+ with raises(ValueError):+ dummy_method(url)diff --git a/classic/forge/forge/utils/url_validator.py b/classic/forge/forge/utils/url_validator.pyindex 16536ef65b..0680d566cf 100644--- a/classic/forge/forge/utils/url_validator.py+++ b/classic/forge/forge/utils/url_validator.py@@ -1,5 +1,7 @@import functools+import ipaddressimport re+import socketfrom inspect import signaturefrom typing import Callable, ParamSpec, TypeVarfrom urllib.parse import urljoin, urlparse@@ -34,6 +36,8 @@ def validate_url(func: Callable[P, T]) -> Callable[P, T]:raise ValueError("Access to local files is restricted")if len(url) > 2000:raise ValueError("URL is too long")+ if not check_public_address(url):+ raise ValueError("Access to internal/private addresses is restricted")bound_args.arguments["url"] = sanitize_url(url)@@ -88,3 +92,50 @@ def check_local_file_access(url: str) -> bool:]return any(url.startswith(prefix) for prefix in local_file_prefixes)+++# Cloud instance metadata endpoint commonly abused via SSRF.+METADATA_IP = "169.254.169.254"+++def check_public_address(url: str) -> bool:+ """Check that the URL's hostname resolves only to public addresses (SSRF guard).++ Resolves the hostname and rejects the URL if any resolved IP is private,+ loopback, link-local, reserved, multicast, unspecified, or the cloud metadata+ IP. ``ipaddress.ip_address`` canonicalizes numeric/alternate encodings (decimal,+ hex, IPv4-mapped IPv6), so those are caught too.++ Note: this is resolve-time validation. DNS rebinding (TOCTOU between this check+ and the actual request) is a known residual limitation not addressed here.++ Args:+ url (str): The URL to check++ Returns:+ bool: True if every resolved address is public, False otherwise+ """+ hostname = urlparse(url).hostname+ if not hostname:+ return False++ try:+ addr_infos = socket.getaddrinfo(hostname, None)+ except socket.gaierror:+ # Hostname did not resolve; nothing private to reach, leave it to the caller.+ return True++ for addr_info in addr_infos:+ ip = ipaddress.ip_address(addr_info[4][0])+ if (+ ip.is_private+ or ip.is_loopback+ or ip.is_link_local+ or ip.is_reserved+ or ip.is_multicast+ or ip.is_unspecified+ or str(ip) == METADATA_IP+ ):+ return False++ return Truediff --git a/classic/original_autogpt/scripts/llamafile/serve.py b/classic/original_autogpt/scripts/llamafile/serve.pyindex f34c6268d2..e391c5849d 100755--- a/classic/original_autogpt/scripts/llamafile/serve.py+++ b/classic/original_autogpt/scripts/llamafile/serve.py@@ -10,8 +10,10 @@ Usage:import osimport platformimport subprocess+import urllib.requestfrom pathlib import Pathfrom typing import Optional+from urllib.parse import urlparseimport click@@ -20,6 +22,15 @@ LLAMAFILE_URL = f"https://huggingface.co/jartine/Mistral-7B-Instruct-v0.2-llamafLLAMAFILE_EXE = Path("llamafile.exe")LLAMAFILE_EXE_URL = "https://github.com/Mozilla-Ocho/llamafile/releases/download/0.8.6/llamafile-0.8.6" # noqa+# Hosts that are allowed to serve llamafile/model download artifacts. Restricting+# downloads to these hosts prevents SSRF / fetching from arbitrary, attacker-controlled+# URLs.+ALLOWED_DOWNLOAD_HOSTS = (+ "huggingface.co",+ "github.com",+ "objects.githubusercontent.com",+)+@click.command()@click.option(@@ -141,10 +152,51 @@ def main(# a lot of memory.+def _assert_download_url_allowed(url: str) -> None:+ """Validate a download URL before fetching it.++ Requires an ``https`` scheme and a host within ``ALLOWED_DOWNLOAD_HOSTS``+ (exact match or a proper subdomain). Raises ``ValueError`` otherwise to+ prevent SSRF / downloads from arbitrary hosts.+ """+ parsed = urlparse(url)+ if parsed.scheme != "https":+ raise ValueError(+ f"Refusing to download from non-https URL: {url!r}. "+ "Only https download URLs are allowed."+ )++ host = (parsed.hostname or "").lower()+ is_allowed = any(+ host == allowed or host.endswith(f".{allowed}")+ for allowed in ALLOWED_DOWNLOAD_HOSTS+ )+ if not is_allowed:+ raise ValueError(+ f"Refusing to download from disallowed host {host!r}. "+ f"Allowed hosts: {', '.join(ALLOWED_DOWNLOAD_HOSTS)}."+ )+++class _AllowlistRedirectHandler(urllib.request.HTTPRedirectHandler):+ """Re-validate every redirect target against the download allowlist.++ `urlretrieve` follows redirects, so validating only the initial URL would+ let a redirect bounce the download to an internal/arbitrary host. This+ re-runs `_assert_download_url_allowed` on each hop.+ """++ def redirect_request(self, req, fp, code, msg, headers, newurl):+ _assert_download_url_allowed(newurl)+ return super().redirect_request(req, fp, code, msg, headers, newurl)++def download_file(url: str, to_file: Path) -> None:+ _assert_download_url_allowed(url)print(f"Downloading {to_file.name}...")- import urllib.request+ opener = urllib.request.build_opener(_AllowlistRedirectHandler())+ urllib.request.install_opener(opener)urllib.request.urlretrieve(url, to_file, reporthook=report_download_progress)print()diff --git a/classic/original_autogpt/tests/unit/test_llamafile_serve.py b/classic/original_autogpt/tests/unit/test_llamafile_serve.pynew file mode 100644index 0000000000..7196789f91--- /dev/null+++ b/classic/original_autogpt/tests/unit/test_llamafile_serve.py@@ -0,0 +1,57 @@+import importlib.util+from pathlib import Path++import pytest++SERVE_PATH = Path(__file__).resolve().parents[2] / "scripts" / "llamafile" / "serve.py"+++def _load_serve_module():+ spec = importlib.util.spec_from_file_location("llamafile_serve", SERVE_PATH)+ assert spec and spec.loader+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++serve = _load_serve_module()+++def test_default_llamafile_urls_are_allowed():+ serve._assert_download_url_allowed(serve.LLAMAFILE_URL)+ serve._assert_download_url_allowed(serve.LLAMAFILE_EXE_URL)+++def test_allowed_huggingface_url_passes():+ serve._assert_download_url_allowed(+ "https://huggingface.co/some-org/some-model/resolve/main/model.llamafile"+ )+++ "url",+ [+ "http://huggingface.co/some-org/some-model/resolve/main/model.llamafile",+ "https://169.254.169.254/latest/meta-data/",+ "https://evil.example.com/model.llamafile",+ "https://huggingface.co.evil.example.com/model.llamafile",+ ],+)+def test_disallowed_urls_are_rejected(url: str):+ with pytest.raises(ValueError):+ serve._assert_download_url_allowed(url)+++def test_redirect_to_disallowed_host_is_rejected():+ # A redirect that bounces the download to a disallowed host must be rejected+ # (our check runs before delegating to the base redirect handler).+ handler = serve._AllowlistRedirectHandler()+ with pytest.raises(ValueError):+ handler.redirect_request(+ None,+ None,+ 302,+ "Found",+ {},+ "https://evil.example.com/model.llamafile",+ )