Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(classic): harden outbound request handling (SSRF + URL validation) (#13422)

Krzysztof Czerwinskic807c3690060
diff --git a/classic/forge/forge/components/git_operations/git_operations.py b/classic/forge/forge/components/git_operations/git_operations.py
index 48567edd20..8ad9aa05ce 100644
--- a/classic/forge/forge/components/git_operations/git_operations.py
+++ b/classic/forge/forge/components/git_operations/git_operations.py
@@ -1,5 +1,6 @@
from pathlib import Path
from typing import Iterator, Optional
+from urllib.parse import urlparse
from git.exc import GitCommandError, InvalidGitRepositoryError
from git.repo import Repo
@@ -93,6 +94,16 @@ class GitOperationsComponent(
Returns:
str: The result of the clone operation.
"""
+ # Only inject credentials when cloning from GitHub. The URL is
+ # attacker-controllable, so without this check the configured
+ # credentials would be sent to any host the URL points to.
+ hostname = (urlparse(url).hostname or "").lower()
+ if hostname not in ("github.com", "www.github.com"):
+ raise CommandExecutionError(
+ f"Refusing to clone from non-GitHub host '{hostname}': "
+ "credential injection is only allowed for github.com"
+ )
+
split_url = url.split("//")
api_key = (
self.config.github_api_key.get_secret_value()
diff --git a/classic/forge/forge/components/git_operations/test_git_operations.py b/classic/forge/forge/components/git_operations/test_git_operations.py
index 8d6cb4f86d..d833ab070e 100644
--- a/classic/forge/forge/components/git_operations/test_git_operations.py
+++ b/classic/forge/forge/components/git_operations/test_git_operations.py
@@ -55,3 +55,38 @@ def test_clone_repository_error(
with pytest.raises(CommandExecutionError):
git_ops_component.clone_repository(url, clone_path)
+
+
+def test_clone_github_host_injects_credentials(
+ git_ops_component: GitOperationsComponent,
+ storage: FileStorage,
+ mock_clone_from,
+):
+ mock_clone_from.return_value = None
+
+ url = "https://github.com/owner/repo.git"
+ clone_path = storage.get_path("github-repo")
+
+ git_ops_component.clone_repository(url, clone_path)
+
+ mock_clone_from.assert_called_once()
+ called_url = mock_clone_from.call_args.kwargs["url"]
+ assert "github.com/owner/repo.git" in called_url
+ assert called_url.startswith(
+ f"https://{git_ops_component.config.github_username}:"
+ f"{git_ops_component.config.github_api_key}@github.com"
+ )
+
+
+def test_clone_non_github_host_is_refused(
+ git_ops_component: GitOperationsComponent,
+ storage: FileStorage,
+ mock_clone_from,
+):
+ url = "https://evil.example.com/repo.git"
+ clone_path = storage.get_path("evil-repo")
+
+ with pytest.raises(CommandExecutionError):
+ git_ops_component.clone_repository(url, clone_path)
+
+ mock_clone_from.assert_not_called()
diff --git a/classic/forge/forge/components/http_client/http_client.py b/classic/forge/forge/components/http_client/http_client.py
index 132348ec79..2ded9dbb41 100644
--- a/classic/forge/forge/components/http_client/http_client.py
+++ b/classic/forge/forge/components/http_client/http_client.py
@@ -1,6 +1,7 @@
import json
import logging
from typing import Any, Iterator, Optional
+from urllib.parse import urljoin, urlparse
import requests
from pydantic import BaseModel, Field
@@ -10,6 +11,7 @@ from forge.agent.protocols import CommandProvider, DirectiveProvider
from forge.command import Command, command
from forge.models.json_schema import JSONSchema
from forge.utils.exceptions import HTTPError
+from forge.utils.url_validator import check_public_address
logger = logging.getLogger(__name__)
@@ -41,6 +43,7 @@ class HTTPClientComponent(
"""Provides commands to make HTTP requests."""
config_class = HTTPClientConfiguration
+ MAX_REDIRECTS = 5
def __init__(self, config: Optional[HTTPClientConfiguration] = None):
ConfigurableComponent.__init__(self, config)
@@ -57,17 +60,26 @@ class HTTPClientComponent(
yield self.http_delete
def _is_domain_allowed(self, url: str) -> bool:
- """Check if the URL's domain is in the allowed list."""
+ """Check if the URL's host is in the allowed list."""
+ # Backslashes are normalized to forward slashes by requests/browsers,
+ # which lets userinfo tricks like "http://evil\@allowed.com" reach an
+ # unintended host. Reject them up front, before the empty-allowlist
+ # shortcut, so a malformed URL can't slip through the default config.
+ if "\\" in url:
+ return False
+
if not self.config.allowed_domains:
return True
- from urllib.parse import urlparse
-
parsed = urlparse(url)
- domain = parsed.netloc.lower()
+ # .hostname strips userinfo and port and lowercases the host.
+ host = parsed.hostname
+ if not host:
+ return False
for allowed in self.config.allowed_domains:
- if domain == allowed.lower() or domain.endswith("." + allowed.lower()):
+ allowed_host = allowed.lower()
+ if host == allowed_host or host.endswith("." + allowed_host):
return True
return False
@@ -93,44 +105,74 @@ class HTTPClientComponent(
Returns:
dict: Structured response with status, headers, and body
"""
- if not self._is_domain_allowed(url):
- raise HTTPError(
- f"Domain not in allowed list. Allowed: {self.config.allowed_domains}",
- url=url,
- )
-
request_timeout = timeout or self.config.default_timeout
request_headers = headers or {}
+ method = method.upper()
+ if method not in ("GET", "POST", "PUT", "DELETE"):
+ raise HTTPError(f"Unsupported HTTP method: {method}", url=url)
+
+ # Follow redirects manually so every hop is re-validated. `requests`
+ # follows redirects by default, which would otherwise let a public URL
+ # redirect to 127.0.0.1 / metadata IPs and bypass the checks below.
+ current_url = url
+ current_method = method
+ current_body = body
+ current_params = params
+
try:
- if method == "GET":
- response = self.session.get(
- url, headers=request_headers, params=params, timeout=request_timeout
- )
- elif method == "POST":
- response = self.session.post(
- url,
- headers=request_headers,
- params=params,
- json=body if isinstance(body, dict) else None,
- data=body if isinstance(body, str) else None,
- timeout=request_timeout,
- )
- elif method == "PUT":
- response = self.session.put(
- url,
+ for _ in range(self.MAX_REDIRECTS + 1):
+ # Block internal/private targets regardless of the allowlist.
+ # With the default empty allowlist `_is_domain_allowed` returns
+ # True, so without this the client could still reach 127.0.0.1,
+ # metadata IPs, etc. — checked on the initial URL and every hop.
+ if not check_public_address(current_url):
+ raise HTTPError(
+ "Access to internal/private addresses is restricted",
+ url=current_url,
+ )
+ if not self._is_domain_allowed(current_url):
+ raise HTTPError(
+ "Domain not in allowed list. "
+ f"Allowed: {self.config.allowed_domains}",
+ url=current_url,
+ )
+
+ response = self.session.request(
+ current_method,
+ current_url,
headers=request_headers,
- params=params,
- json=body if isinstance(body, dict) else None,
- data=body if isinstance(body, str) else None,
+ params=current_params,
+ json=current_body if isinstance(current_body, dict) else None,
+ data=current_body if isinstance(current_body, str) else None,
timeout=request_timeout,
+ allow_redirects=False,
)
- elif method == "DELETE":
- response = self.session.delete(
- url, headers=request_headers, params=params, timeout=request_timeout
- )
+
+ location = response.headers.get("Location")
+ if not (response.is_redirect and location):
+ break
+
+ # Resolve the next hop; it gets re-validated next iteration.
+ next_url = urljoin(current_url, location)
+ # requests strips Authorization when a redirect crosses hosts
+ # (Session.rebuild_auth); mirror that here so caller-supplied
+ # credentials don't follow a redirect to a different host.
+ if urlparse(next_url).hostname != urlparse(current_url).hostname:
+ request_headers = {
+ key: value
+ for key, value in request_headers.items()
+ if key.lower() != "authorization"
+ }
+ current_url = next_url
+ # Per HTTP semantics, 301/302/303 downgrade to GET without a body.
+ if response.status_code in (301, 302, 303):
+ current_method = "GET"
+ current_body = None
+ # The redirect target carries its own query string.
+ current_params = None
else:
- raise HTTPError(f"Unsupported HTTP method: {method}", url=url)
+ raise HTTPError("Too many redirects", url=url)
# Check response size
content_length = len(response.content)
diff --git a/classic/forge/forge/components/http_client/test_http_client.py b/classic/forge/forge/components/http_client/test_http_client.py
new file mode 100644
index 0000000000..62ac69ff0d
--- /dev/null
+++ b/classic/forge/forge/components/http_client/test_http_client.py
@@ -0,0 +1,124 @@
+"""Tests for the HTTP client component."""
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from forge.utils.exceptions import HTTPError
+
+from .http_client import HTTPClientComponent, HTTPClientConfiguration
+
+
+def allowlisted_component():
+ """Create an HTTPClientComponent restricted to whitelist.com."""
+ config = HTTPClientConfiguration(allowed_domains=["whitelist.com"])
+ return HTTPClientComponent(config)
+
+
+def test_allowed_domain_passes(allowlisted_component):
+ assert allowlisted_component._is_domain_allowed("https://whitelist.com/path")
+
+
+def test_subdomain_of_allowed_domain_passes(allowlisted_component):
+ assert allowlisted_component._is_domain_allowed("https://api.whitelist.com/path")
+
+
+def test_no_allowlist_allows_everything():
+ component = HTTPClientComponent(HTTPClientConfiguration())
+ assert component._is_domain_allowed("http://127.0.0.1/")
+
+
+def test_disallowed_domain_rejected(allowlisted_component):
+ assert not allowlisted_component._is_domain_allowed("http://127.0.0.1/")
+
+
+def test_userinfo_backslash_bypass_rejected(allowlisted_component):
+ # Regression: netloc-based check passed but request hit 127.0.0.1.
+ assert not allowlisted_component._is_domain_allowed(
+ "http://127.0.0.1:6666\\@.whitelist.com"
+ )
+
+
+def test_userinfo_bypass_rejected(allowlisted_component):
+ assert not allowlisted_component._is_domain_allowed(
+ "http://[email protected]/"
+ )
+
+
+def test_make_request_blocks_internal_address_without_allowlist():
+ # Even with the default empty allowlist, _make_request must reject
+ # internal/private targets before issuing the request.
+ component = HTTPClientComponent(HTTPClientConfiguration())
+ with pytest.raises(HTTPError):
+ component._make_request("GET", "http://127.0.0.1/")
+
+
+def test_backslash_url_rejected_without_allowlist():
+ # Backslash authority trick must be rejected even with the default empty
+ # allowlist (requests normalizes "\" and would reach an unintended host).
+ component = HTTPClientComponent(HTTPClientConfiguration())
+ assert not component._is_domain_allowed("http://127.0.0.1:6666\\@example.com/")
+ with pytest.raises(HTTPError):
+ component._make_request("GET", "http://127.0.0.1:6666\\@example.com/")
+
+
+def _redirect_response(location: str) -> MagicMock:
+ response = MagicMock()
+ response.is_redirect = True
+ response.status_code = 302
+ response.headers = {"Location": location}
+ return response
+
+
+def _final_response() -> MagicMock:
+ response = MagicMock()
+ response.is_redirect = False
+ response.status_code = 200
+ response.headers = {}
+ response.content = b"ok"
+ return response
+
+
+def test_cross_host_redirect_strips_authorization_header():
+ # requests strips Authorization when following a cross-host redirect
+ # (Session.rebuild_auth); the manual redirect loop must do the same.
+ component = HTTPClientComponent(HTTPClientConfiguration())
+ component.session.request = MagicMock(
+ side_effect=[_redirect_response("https://example.org/"), _final_response()]
+ )
+
+ component._make_request(
+ "GET", "https://example.com/", headers={"Authorization": "Bearer token"}
+ )
+
+ calls = component.session.request.call_args_list
+ assert calls[0].kwargs["headers"] == {"Authorization": "Bearer token"}
+ assert calls[1].kwargs["headers"] == {}
+
+
+def test_same_host_redirect_keeps_authorization_header():
+ component = HTTPClientComponent(HTTPClientConfiguration())
+ component.session.request = MagicMock(
+ side_effect=[_redirect_response("/moved"), _final_response()]
+ )
+
+ component._make_request(
+ "GET", "https://example.com/", headers={"Authorization": "Bearer token"}
+ )
+
+ calls = component.session.request.call_args_list
+ assert calls[1].kwargs["headers"] == {"Authorization": "Bearer token"}
+
+
+def test_make_request_blocks_redirect_to_internal():
+ # A public URL that redirects to an internal host must be rejected: redirects
+ # are followed manually and each hop is re-validated.
+ component = HTTPClientComponent(HTTPClientConfiguration())
+ redirect = MagicMock()
+ redirect.is_redirect = True
+ redirect.status_code = 302
+ redirect.headers = {"Location": "http://127.0.0.1/"}
+ component.session.request = MagicMock(return_value=redirect)
+ with pytest.raises(HTTPError):
+ component._make_request("GET", "https://example.com/")
diff --git a/classic/forge/forge/utils/test_url_validator.py b/classic/forge/forge/utils/test_url_validator.py
index dc2cce3008..31bcc6a629 100644
--- a/classic/forge/forge/utils/test_url_validator.py
+++ b/classic/forge/forge/utils/test_url_validator.py
@@ -155,3 +155,22 @@ def test_internationalized_url():
"""
url = "http://例子.测试"
assert dummy_method(url) == url
+
+
+ssrf_internal_urls = (
+ ("http://127.0.0.1/"),
+ ("http://169.254.169.254/"),
+ ("http://10.0.0.1/"),
+ ("http://localhost/"),
+ ("http://2130706433/"),
+)
+
+
[email protected]("url", ssrf_internal_urls)
+def test_url_validation_blocks_internal_addresses(url):
+ """
+ Tests that the validator rejects URLs resolving to internal/private addresses
+ (SSRF protection), including the cloud metadata IP and alternate encodings.
+ """
+ with raises(ValueError):
+ dummy_method(url)
diff --git a/classic/forge/forge/utils/url_validator.py b/classic/forge/forge/utils/url_validator.py
index 16536ef65b..0680d566cf 100644
--- a/classic/forge/forge/utils/url_validator.py
+++ b/classic/forge/forge/utils/url_validator.py
@@ -1,5 +1,7 @@
import functools
+import ipaddress
import re
+import socket
from inspect import signature
from typing import Callable, ParamSpec, TypeVar
from urllib.parse import urljoin, urlparse
@@ -34,6 +36,8 @@ def validate_url(func: Callable[P, T]) -> Callable[P, T]:
raise ValueError("Access to local files is restricted")
if len(url) > 2000:
raise ValueError("URL is too long")
+ if not check_public_address(url):
+ raise ValueError("Access to internal/private addresses is restricted")
bound_args.arguments["url"] = sanitize_url(url)
@@ -88,3 +92,50 @@ def check_local_file_access(url: str) -> bool:
]
return any(url.startswith(prefix) for prefix in local_file_prefixes)
+
+
+# Cloud instance metadata endpoint commonly abused via SSRF.
+METADATA_IP = "169.254.169.254"
+
+
+def check_public_address(url: str) -> bool:
+ """Check that the URL's hostname resolves only to public addresses (SSRF guard).
+
+ Resolves the hostname and rejects the URL if any resolved IP is private,
+ loopback, link-local, reserved, multicast, unspecified, or the cloud metadata
+ IP. ``ipaddress.ip_address`` canonicalizes numeric/alternate encodings (decimal,
+ hex, IPv4-mapped IPv6), so those are caught too.
+
+ Note: this is resolve-time validation. DNS rebinding (TOCTOU between this check
+ and the actual request) is a known residual limitation not addressed here.
+
+ Args:
+ url (str): The URL to check
+
+ Returns:
+ bool: True if every resolved address is public, False otherwise
+ """
+ hostname = urlparse(url).hostname
+ if not hostname:
+ return False
+
+ try:
+ addr_infos = socket.getaddrinfo(hostname, None)
+ except socket.gaierror:
+ # Hostname did not resolve; nothing private to reach, leave it to the caller.
+ return True
+
+ for addr_info in addr_infos:
+ ip = ipaddress.ip_address(addr_info[4][0])
+ if (
+ ip.is_private
+ or ip.is_loopback
+ or ip.is_link_local
+ or ip.is_reserved
+ or ip.is_multicast
+ or ip.is_unspecified
+ or str(ip) == METADATA_IP
+ ):
+ return False
+
+ return True
diff --git a/classic/original_autogpt/scripts/llamafile/serve.py b/classic/original_autogpt/scripts/llamafile/serve.py
index f34c6268d2..e391c5849d 100755
--- a/classic/original_autogpt/scripts/llamafile/serve.py
+++ b/classic/original_autogpt/scripts/llamafile/serve.py
@@ -10,8 +10,10 @@ Usage:
import os
import platform
import subprocess
+import urllib.request
from pathlib import Path
from typing import Optional
+from urllib.parse import urlparse
import click
@@ -20,6 +22,15 @@ LLAMAFILE_URL = f"https://huggingface.co/jartine/Mistral-7B-Instruct-v0.2-llamaf
LLAMAFILE_EXE = Path("llamafile.exe")
LLAMAFILE_EXE_URL = "https://github.com/Mozilla-Ocho/llamafile/releases/download/0.8.6/llamafile-0.8.6" # noqa
+# Hosts that are allowed to serve llamafile/model download artifacts. Restricting
+# downloads to these hosts prevents SSRF / fetching from arbitrary, attacker-controlled
+# URLs.
+ALLOWED_DOWNLOAD_HOSTS = (
+ "huggingface.co",
+ "github.com",
+ "objects.githubusercontent.com",
+)
+
@click.command()
@click.option(
@@ -141,10 +152,51 @@ def main(
# a lot of memory.
+def _assert_download_url_allowed(url: str) -> None:
+ """Validate a download URL before fetching it.
+
+ Requires an ``https`` scheme and a host within ``ALLOWED_DOWNLOAD_HOSTS``
+ (exact match or a proper subdomain). Raises ``ValueError`` otherwise to
+ prevent SSRF / downloads from arbitrary hosts.
+ """
+ parsed = urlparse(url)
+ if parsed.scheme != "https":
+ raise ValueError(
+ f"Refusing to download from non-https URL: {url!r}. "
+ "Only https download URLs are allowed."
+ )
+
+ host = (parsed.hostname or "").lower()
+ is_allowed = any(
+ host == allowed or host.endswith(f".{allowed}")
+ for allowed in ALLOWED_DOWNLOAD_HOSTS
+ )
+ if not is_allowed:
+ raise ValueError(
+ f"Refusing to download from disallowed host {host!r}. "
+ f"Allowed hosts: {', '.join(ALLOWED_DOWNLOAD_HOSTS)}."
+ )
+
+
+class _AllowlistRedirectHandler(urllib.request.HTTPRedirectHandler):
+ """Re-validate every redirect target against the download allowlist.
+
+ `urlretrieve` follows redirects, so validating only the initial URL would
+ let a redirect bounce the download to an internal/arbitrary host. This
+ re-runs `_assert_download_url_allowed` on each hop.
+ """
+
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
+ _assert_download_url_allowed(newurl)
+ return super().redirect_request(req, fp, code, msg, headers, newurl)
+
+
def download_file(url: str, to_file: Path) -> None:
+ _assert_download_url_allowed(url)
print(f"Downloading {to_file.name}...")
- import urllib.request
+ opener = urllib.request.build_opener(_AllowlistRedirectHandler())
+ urllib.request.install_opener(opener)
urllib.request.urlretrieve(url, to_file, reporthook=report_download_progress)
print()
diff --git a/classic/original_autogpt/tests/unit/test_llamafile_serve.py b/classic/original_autogpt/tests/unit/test_llamafile_serve.py
new file mode 100644
index 0000000000..7196789f91
--- /dev/null
+++ b/classic/original_autogpt/tests/unit/test_llamafile_serve.py
@@ -0,0 +1,57 @@
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+SERVE_PATH = Path(__file__).resolve().parents[2] / "scripts" / "llamafile" / "serve.py"
+
+
+def _load_serve_module():
+ spec = importlib.util.spec_from_file_location("llamafile_serve", SERVE_PATH)
+ assert spec and spec.loader
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+serve = _load_serve_module()
+
+
+def test_default_llamafile_urls_are_allowed():
+ serve._assert_download_url_allowed(serve.LLAMAFILE_URL)
+ serve._assert_download_url_allowed(serve.LLAMAFILE_EXE_URL)
+
+
+def test_allowed_huggingface_url_passes():
+ serve._assert_download_url_allowed(
+ "https://huggingface.co/some-org/some-model/resolve/main/model.llamafile"
+ )
+
+
+ "url",
+ [
+ "http://huggingface.co/some-org/some-model/resolve/main/model.llamafile",
+ "https://169.254.169.254/latest/meta-data/",
+ "https://evil.example.com/model.llamafile",
+ "https://huggingface.co.evil.example.com/model.llamafile",
+ ],
+)
+def test_disallowed_urls_are_rejected(url: str):
+ with pytest.raises(ValueError):
+ serve._assert_download_url_allowed(url)
+
+
+def test_redirect_to_disallowed_host_is_rejected():
+ # A redirect that bounces the download to a disallowed host must be rejected
+ # (our check runs before delegating to the base redirect handler).
+ handler = serve._AllowlistRedirectHandler()
+ with pytest.raises(ValueError):
+ handler.redirect_request(
+ None,
+ None,
+ 302,
+ "Found",
+ {},
+ "https://evil.example.com/model.llamafile",
+ )