Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): Make URL pinning work with `extra_url_validator` (#9940)

Nicholas Tindle9bef383df2fc
diff --git a/.vscode/launch.json b/.vscode/launch.json
index 0c6b896b8e..6740265f33 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -32,9 +32,9 @@
"type": "debugpy",
"request": "launch",
"module": "backend.app",
- // "env": {
- // "ENV": "dev"
- // },
+ "env": {
+ "OBJC_DISABLE_INITIALIZE_FORK_SAFETY": "YES"
+ },
"envFile": "${workspaceFolder}/backend/.env",
"justMyCode": false,
"cwd": "${workspaceFolder}/autogpt_platform/backend"
diff --git a/autogpt_platform/backend/backend/util/request.py b/autogpt_platform/backend/backend/util/request.py
index 070758a56d..e8ffae1ace 100644
--- a/autogpt_platform/backend/backend/util/request.py
+++ b/autogpt_platform/backend/backend/util/request.py
@@ -224,33 +224,31 @@ class Requests:
*args,
**kwargs,
) -> req.Response:
- # Validate URL and get pinned URL + original hostname
- pinned_url, original_hostname = validate_url(url, self.trusted_origins)
-
# Apply any extra user-defined validation/transformation
if self.extra_url_validator is not None:
- pinned_url = self.extra_url_validator(pinned_url)
+ url = self.extra_url_validator(url)
+
+ # Validate URL and get pinned URL + hostname
+ pinned_url, hostname = validate_url(url, self.trusted_origins)
# Merge any extra headers
headers = dict(headers) if headers else {}
if self.extra_headers is not None:
headers.update(self.extra_headers)
- # Force the Host header to the original hostname
- headers["Host"] = original_hostname
-
- # Create a fresh session & mount our HostSSLAdapter if pinned to IP
session = req.Session()
- pinned_parsed = urlparse(pinned_url)
-
- # If pinned_url netloc is an IP (not in trusted_origins),
- # then we attach the custom SNI adapter:
- if pinned_parsed.hostname and pinned_parsed.hostname != original_hostname:
- # That means we definitely pinned to an IP
- mount_prefix = f"{pinned_parsed.scheme}://{pinned_parsed.hostname}"
- if pinned_parsed.port:
- mount_prefix += f":{pinned_parsed.port}"
- adapter = HostSSLAdapter(ssl_hostname=original_hostname)
+
+ # If untrusted, the hostname in the URL is replaced with the corresponding
+ # IP address, and we need to override the Host header with the actual hostname.
+ if (pinned := urlparse(pinned_url)).hostname != hostname:
+ headers["Host"] = hostname
+
+ # If hostname was untrusted and we replaced it by (pinned it to) its IP,
+ # we also need to attach a custom SNI adapter to make SSL work:
+ mount_prefix = f"{pinned.scheme}://{pinned.hostname}"
+ if pinned.port:
+ mount_prefix += f":{pinned.port}"
+ adapter = HostSSLAdapter(ssl_hostname=hostname)
session.mount("https://", adapter)
# Perform the request with redirects disabled for manual handling