AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): Unbreak URL handling for GitHub blocks (#9989)
Reinier van der Leerf466b010e452
diff --git a/autogpt_platform/backend/backend/blocks/github/_api.py b/autogpt_platform/backend/backend/blocks/github/_api.pyindex 17415b0af3..03e4926673 100644--- a/autogpt_platform/backend/backend/blocks/github/_api.py+++ b/autogpt_platform/backend/backend/blocks/github/_api.py@@ -1,19 +1,30 @@+from typing import overloadfrom urllib.parse import urlparsefrom backend.blocks.github._auth import (GithubCredentials,GithubFineGrainedAPICredentials,)-from backend.util.request import Requests+from backend.util.request import URL, Requests-def _convert_to_api_url(url: str) -> str:+@overload+def _convert_to_api_url(url: str) -> str: ...+++@overload+def _convert_to_api_url(url: URL) -> URL: ...+++def _convert_to_api_url(url: str | URL) -> str | URL:"""Converts a standard GitHub URL to the corresponding GitHub API URL.Handles repository URLs, issue URLs, pull request URLs, and more."""- parsed_url = urlparse(url)- path_parts = parsed_url.path.strip("/").split("/")+ if url_as_str := isinstance(url, str):+ url = urlparse(url)++ path_parts = url.path.strip("/").split("/")if len(path_parts) >= 2:owner, repo = path_parts[0], path_parts[1]@@ -28,7 +39,7 @@ def _convert_to_api_url(url: str) -> str:else:raise ValueError("Invalid GitHub URL format.")- return api_url+ return api_url if url_as_str else urlparse(api_url)def _get_headers(credentials: GithubCredentials) -> dict[str, str]:diff --git a/autogpt_platform/backend/backend/util/request.py b/autogpt_platform/backend/backend/util/request.pyindex e8ffae1ace..9086e779c6 100644--- a/autogpt_platform/backend/backend/util/request.py+++ b/autogpt_platform/backend/backend/util/request.py@@ -2,8 +2,9 @@ import ipaddressimport reimport socketimport ssl-from typing import Callable-from urllib.parse import quote, urljoin, urlparse, urlunparse+from typing import Callable, Optional+from urllib.parse import ParseResult as URL+from urllib.parse import quote, urljoin, urlparseimport idnaimport requests as req@@ -44,17 +45,15 @@ def _is_ip_blocked(ip: str) -> bool:return any(ip_addr in network for network in BLOCKED_IP_NETWORKS)-def _remove_insecure_headers(headers: dict, old_url: str, new_url: str) -> dict:+def _remove_insecure_headers(headers: dict, old_url: URL, new_url: URL) -> dict:"""Removes sensitive headers (Authorization, Proxy-Authorization, Cookie)if the scheme/host/port of new_url differ from old_url."""- old_parsed = urlparse(old_url)- new_parsed = urlparse(new_url)if (- (old_parsed.scheme != new_parsed.scheme)- or (old_parsed.hostname != new_parsed.hostname)- or (old_parsed.port != new_parsed.port)+ (old_url.scheme != new_url.scheme)+ or (old_url.hostname != new_url.hostname)+ or (old_url.port != new_url.port)):headers.pop("Authorization", None)headers.pop("Proxy-Authorization", None)@@ -81,19 +80,16 @@ class HostSSLAdapter(HTTPAdapter):)-def validate_url(- url: str,- trusted_origins: list[str],- enable_dns_rebinding: bool = True,-) -> tuple[str, str]:+def validate_url(url: str, trusted_origins: list[str]) -> tuple[URL, bool, list[str]]:"""Validates the URL to prevent SSRF attacks by ensuring it does not pointto a private, link-local, or otherwise blocked IP address — unlessthe hostname is explicitly trusted.- Returns a tuple of:- - pinned_url: a URL that has the netloc replaced with the validated IP- - ascii_hostname: the original ASCII hostname (IDNA-decoded) for use in the Host header+ Returns:+ str: The validated, canonicalized, parsed URL+ is_trusted: Boolean indicating if the hostname is in trusted_origins+ ip_addresses: List of IP addresses for the host; empty if the host is trusted"""# Canonicalize URLurl = url.strip("/ ").replace("\\", "/")@@ -122,45 +118,56 @@ def validate_url(if not HOSTNAME_REGEX.match(ascii_hostname):raise ValueError("Hostname contains invalid characters.")- # If hostname is trusted, skip IP-based checks but still return pinned URL- if ascii_hostname in trusted_origins:- pinned_netloc = ascii_hostname- if parsed.port:- pinned_netloc += f":{parsed.port}"-- pinned_url = urlunparse(- (- parsed.scheme,- pinned_netloc,- quote(parsed.path, safe="/%:@"),- parsed.params,- parsed.query,- parsed.fragment,- )- )- return pinned_url, ascii_hostname+ # Check if hostname is trusted+ is_trusted = ascii_hostname in trusted_origins++ # If not trusted, validate IP addresses+ ip_addresses: list[str] = []+ if not is_trusted:+ # Resolve all IP addresses for the hostname+ ip_addresses = _resolve_host(ascii_hostname)++ # Block any IP address that belongs to a blocked range+ for ip_str in ip_addresses:+ if _is_ip_blocked(ip_str):+ raise ValueError(+ f"Access to blocked or private IP address {ip_str} "+ f"for hostname {ascii_hostname} is not allowed."+ )++ return (+ URL(+ parsed.scheme,+ ascii_hostname,+ quote(parsed.path, safe="/%:@"),+ parsed.params,+ parsed.query,+ parsed.fragment,+ ),+ is_trusted,+ ip_addresses,+ )- # Resolve all IP addresses for the hostname- try:- ip_list = [str(res[4][0]) for res in socket.getaddrinfo(ascii_hostname, None)]- ipv4 = [ip for ip in ip_list if ":" not in ip]- ipv6 = [ip for ip in ip_list if ":" in ip]- ip_addresses = ipv4 + ipv6 # Prefer IPv4 over IPv6- except socket.gaierror:- raise ValueError(f"Unable to resolve IP address for hostname {ascii_hostname}")++def pin_url(url: URL, ip_addresses: Optional[list[str]] = None) -> URL:+ """+ Pins a URL to a specific IP address to prevent DNS rebinding attacks.++ Args:+ url: The original URL+ ip_addresses: List of IP addresses corresponding to the URL's host++ Returns:+ pinned_url: The URL with hostname replaced with IP address+ """+ if not url.hostname:+ raise ValueError(f"URL has no hostname: {url}")if not ip_addresses:- raise ValueError(f"No IP addresses found for {ascii_hostname}")-- # Block any IP address that belongs to a blocked range- for ip_str in ip_addresses:- if _is_ip_blocked(ip_str):- raise ValueError(- f"Access to blocked or private IP address {ip_str} "- f"for hostname {ascii_hostname} is not allowed."- )+ # Resolve all IP addresses for the hostname+ ip_addresses = _resolve_host(url.hostname)- # Pin to the first valid IP (for SSRF defense).+ # Pin to the first valid IP (for SSRF defense)pinned_ip = ip_addresses[0]# If it's IPv6, bracket it@@ -169,24 +176,31 @@ def validate_url(else:pinned_netloc = pinned_ip- if parsed.port:- pinned_netloc += f":{parsed.port}"-- if not enable_dns_rebinding:- pinned_netloc = ascii_hostname+ if url.port:+ pinned_netloc += f":{url.port}"- pinned_url = urlunparse(- (- parsed.scheme,- pinned_netloc,- quote(parsed.path, safe="/%:@"),- parsed.params,- parsed.query,- parsed.fragment,- )+ return URL(+ url.scheme,+ pinned_netloc,+ url.path,+ url.params,+ url.query,+ url.fragment,)- return pinned_url, ascii_hostname # (pinned_url, original_hostname)++def _resolve_host(hostname: str) -> list[str]:+ try:+ ip_list = [str(res[4][0]) for res in socket.getaddrinfo(hostname, None)]+ ipv4 = [ip for ip in ip_list if ":" not in ip]+ ipv6 = [ip for ip in ip_list if ":" in ip]+ ip_addresses = ipv4 + ipv6 # Prefer IPv4 over IPv6+ except socket.gaierror:+ raise ValueError(f"Unable to resolve IP address for hostname {hostname}")++ if not ip_addresses:+ raise ValueError(f"No IP addresses found for {hostname}")+ return ip_addressesclass Requests:@@ -200,7 +214,7 @@ class Requests:self,trusted_origins: list[str] | None = None,raise_for_status: bool = True,- extra_url_validator: Callable[[str], str] | None = None,+ extra_url_validator: Callable[[URL], URL] | None = None,extra_headers: dict[str, str] | None = None,):self.trusted_origins = []@@ -224,12 +238,17 @@ class Requests:*args,**kwargs,) -> req.Response:+ # Validate URL and get trust status+ url, is_trusted, ip_addresses = validate_url(url, self.trusted_origins)+# Apply any extra user-defined validation/transformationif self.extra_url_validator is not None:url = self.extra_url_validator(url)- # Validate URL and get pinned URL + hostname- pinned_url, hostname = validate_url(url, self.trusted_origins)+ # Pin the URL if untrusted+ hostname = url.hostname+ if not is_trusted:+ url = pin_url(url, ip_addresses)# Merge any extra headersheaders = dict(headers) if headers else {}@@ -240,21 +259,18 @@ class Requests:# If untrusted, the hostname in the URL is replaced with the corresponding# IP address, and we need to override the Host header with the actual hostname.- if (pinned := urlparse(pinned_url)).hostname != hostname:+ if url.hostname != hostname:headers["Host"] = hostname# If hostname was untrusted and we replaced it by (pinned it to) its IP,# we also need to attach a custom SNI adapter to make SSL work:- mount_prefix = f"{pinned.scheme}://{pinned.hostname}"- if pinned.port:- mount_prefix += f":{pinned.port}"adapter = HostSSLAdapter(ssl_hostname=hostname)session.mount("https://", adapter)# Perform the request with redirects disabled for manual handlingresponse = session.request(method,- pinned_url,+ url.geturl(),headers=headers,allow_redirects=False,*args,@@ -275,13 +291,13 @@ class Requests:# The base URL is the pinned_url we just used# so that relative redirects resolve correctly.- new_url = urljoin(pinned_url, location)+ redirect_url = urlparse(urljoin(url.geturl(), location))# Carry forward the same headers but update Host- new_headers = _remove_insecure_headers(dict(headers), url, new_url)+ new_headers = _remove_insecure_headers(headers, url, redirect_url)return self.request(method,- new_url,+ redirect_url.geturl(),headers=new_headers,allow_redirects=allow_redirects,max_redirects=max_redirects - 1,diff --git a/autogpt_platform/backend/test/util/test_request.py b/autogpt_platform/backend/test/util/test_request.pyindex 30eb79a220..e973e7dd6e 100644--- a/autogpt_platform/backend/test/util/test_request.py+++ b/autogpt_platform/backend/test/util/test_request.py@@ -1,10 +1,10 @@import pytest-from backend.util.request import validate_url+from backend.util.request import pin_url, validate_url@pytest.mark.parametrize(- "url, trusted_origins, expected_value, should_raise",+ "raw_url, trusted_origins, expected_value, should_raise",[# Rejected IP ranges("localhost", [], None, True),@@ -55,14 +55,14 @@ from backend.util.request import validate_url],)def test_validate_url_no_dns_rebinding(- url, trusted_origins, expected_value, should_raise+ raw_url: str, trusted_origins: list[str], expected_value: str, should_raise: bool):if should_raise:with pytest.raises(ValueError):- validate_url(url, trusted_origins, enable_dns_rebinding=False)+ validate_url(raw_url, trusted_origins)else:- url, host = validate_url(url, trusted_origins, enable_dns_rebinding=False)- assert url == expected_value+ validated_url, _, _ = validate_url(raw_url, trusted_origins)+ assert validated_url.geturl() == expected_value@pytest.mark.parametrize(@@ -79,7 +79,11 @@ def test_validate_url_no_dns_rebinding(],)def test_dns_rebinding_fix(- monkeypatch, hostname, resolved_ips, expect_error, expected_ip+ monkeypatch,+ hostname: str,+ resolved_ips: list[str],+ expect_error: bool,+ expected_ip: str,):"""Tests that validate_url pins the first valid public IP address, and rejects@@ -96,11 +100,13 @@ def test_dns_rebinding_fix(if expect_error:# If any IP is blocked, we expect a ValueErrorwith pytest.raises(ValueError):- validate_url(hostname, [])+ url, _, ip_addresses = validate_url(hostname, [])+ pin_url(url, ip_addresses)else:- pinned_url, ascii_hostname = validate_url(hostname, [])+ url, _, ip_addresses = validate_url(hostname, [])+ pinned_url = pin_url(url, ip_addresses).geturl()# The pinned_url should contain the first valid IPassert pinned_url.startswith("http://") or pinned_url.startswith("https://")assert expected_ip in pinned_url- # The ascii_hostname should match our original hostname after IDNA encoding- assert ascii_hostname == hostname+ # The unpinned URL's hostname should match our original IDNA encoded hostname+ assert url.hostname == hostname