AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): Make URL pinning work with `extra_url_validator` (#9940)
Nicholas Tindle9bef383df2fc
diff --git a/.vscode/launch.json b/.vscode/launch.jsonindex 0c6b896b8e..6740265f33 100644--- a/.vscode/launch.json+++ b/.vscode/launch.json@@ -32,9 +32,9 @@"type": "debugpy","request": "launch","module": "backend.app",- // "env": {- // "ENV": "dev"- // },+ "env": {+ "OBJC_DISABLE_INITIALIZE_FORK_SAFETY": "YES"+ },"envFile": "${workspaceFolder}/backend/.env","justMyCode": false,"cwd": "${workspaceFolder}/autogpt_platform/backend"diff --git a/autogpt_platform/backend/backend/util/request.py b/autogpt_platform/backend/backend/util/request.pyindex 070758a56d..e8ffae1ace 100644--- a/autogpt_platform/backend/backend/util/request.py+++ b/autogpt_platform/backend/backend/util/request.py@@ -224,33 +224,31 @@ class Requests:*args,**kwargs,) -> req.Response:- # Validate URL and get pinned URL + original hostname- pinned_url, original_hostname = validate_url(url, self.trusted_origins)-# Apply any extra user-defined validation/transformationif self.extra_url_validator is not None:- pinned_url = self.extra_url_validator(pinned_url)+ url = self.extra_url_validator(url)++ # Validate URL and get pinned URL + hostname+ pinned_url, hostname = validate_url(url, self.trusted_origins)# Merge any extra headersheaders = dict(headers) if headers else {}if self.extra_headers is not None:headers.update(self.extra_headers)- # Force the Host header to the original hostname- headers["Host"] = original_hostname-- # Create a fresh session & mount our HostSSLAdapter if pinned to IPsession = req.Session()- pinned_parsed = urlparse(pinned_url)-- # If pinned_url netloc is an IP (not in trusted_origins),- # then we attach the custom SNI adapter:- if pinned_parsed.hostname and pinned_parsed.hostname != original_hostname:- # That means we definitely pinned to an IP- mount_prefix = f"{pinned_parsed.scheme}://{pinned_parsed.hostname}"- if pinned_parsed.port:- mount_prefix += f":{pinned_parsed.port}"- adapter = HostSSLAdapter(ssl_hostname=original_hostname)++ # If untrusted, the hostname in the URL is replaced with the corresponding+ # IP address, and we need to override the Host header with the actual hostname.+ if (pinned := urlparse(pinned_url)).hostname != hostname:+ headers["Host"] = hostname++ # If hostname was untrusted and we replaced it by (pinned it to) its IP,+ # we also need to attach a custom SNI adapter to make SSL work:+ mount_prefix = f"{pinned.scheme}://{pinned.hostname}"+ if pinned.port:+ mount_prefix += f":{pinned.port}"+ adapter = HostSSLAdapter(ssl_hostname=hostname)session.mount("https://", adapter)# Perform the request with redirects disabled for manual handling