Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): reject ambiguous url hosts in custom source manifests (#125422)

Tom Owers5f96098b9f89
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py
index 127a6fa885ab..00f83a91846a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py
@@ -13,7 +13,9 @@ from django.db import IntegrityError
import structlog
from jsonpath_ng.exceptions import JSONPathError
from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator
-from requests import PreparedRequest, Response, Timeout
+from requests import PreparedRequest, Request, RequestException, Response, Timeout
+from urllib3.exceptions import LocationParseError
+from urllib3.util import parse_url
from urllib3.util.retry import Retry
from posthog.cloud_utils import is_cloud
@@ -619,8 +621,13 @@ def _has_wrapping_quote(url: str) -> bool:
def _check_url(url: str, team_id: int) -> tuple[bool, str | None]:
- # `_url_hostname` mirrors the real connect host (backslash/whitespace-normalized) so the
- # validator can't be fooled into vetting a different host than the request reaches.
+ # `_url_hostname` returns the host the HTTP client connects to, so the validator can't be
+ # fooled into vetting a different host than the request reaches.
+ if _has_ambiguous_authority(url):
+ return (
+ False,
+ "Remove the backslash (\\ or %5C) from the URL's host and enter just the address (for example, https://api.example.com).",
+ )
hostname = _url_hostname(url)
if not hostname:
if _has_leading_http_method(url):
@@ -745,24 +752,69 @@ def manifest_request_hosts(manifest_json: Any) -> frozenset[str]:
if token_url is not None:
urls.append(token_url)
- # `_url_hostname` returns an already-lowercased host.
- hosts = {host for url in urls if (host := _url_hostname(url))}
+ hosts: set[str] = set()
+ for url in urls:
+ if _has_ambiguous_authority(url):
+ # Fail closed: a URL whose host depends on the parser counts as a new destination.
+ hosts.add(f"{_AMBIGUOUS_HOST_MARKER}{url}")
+ elif host := _url_hostname(url):
+ hosts.add(host)
return frozenset(hosts)
+# Prefix for a `manifest_request_hosts` entry that stands in for a URL with no single host.
+# It can't match a real hostname, so the re-entry gate always sees that URL as a new destination.
+_AMBIGUOUS_HOST_MARKER = "ambiguous-url:"
+
+
+def _transport_hostname(url: str) -> str | None:
+ """The host urllib3 reads from ``url``. requests prepares and connects with this parser."""
+ try:
+ # requests strips leading whitespace before it parses the URL.
+ host = parse_url(url.lstrip()).host
+ except LocationParseError:
+ return None
+ return host.lower().strip("[]") if host else None
+
+
+def _has_ambiguous_authority(url: str) -> bool:
+ """True when ``urlparse`` and the HTTP client could read a different host from ``url``.
+
+ ``urlparse`` reads a backslash in the authority as userinfo, but urllib3 ends the authority
+ there: ``https://evil.example\\@trusted.example/`` connects to ``evil.example``. A
+ percent-encoded backslash stays userinfo for both parsers, but a client that decodes first
+ splits on it. No rewrite of these forms matches every parser, so such a URL is rejected.
+ Any other host disagreement between the two parsers is rejected the same way.
+ """
+ try:
+ parsed = urlparse(url)
+ netloc = parsed.netloc.lower()
+ hostname = parsed.hostname
+ except ValueError:
+ return True
+ if "\\" in netloc or "%5c" in netloc:
+ return True
+ if hostname is None:
+ # No host for urlparse means no host for the validator either; `_check_url` rejects it.
+ return False
+ if not hostname.isascii():
+ # urllib3 sends the IDNA (punycode) form of an internationalized host.
+ try:
+ hostname = hostname.encode("idna").decode("ascii")
+ except UnicodeError:
+ return True
+ return hostname != _transport_hostname(url)
+
+
def _url_hostname(url: str) -> str | None:
- """The host the HTTP client will actually connect to.
-
- `urlparse` treats a backslash — and its ``%5c`` encoding — as ordinary
- userinfo, so ``https://evil.example\\@trusted.example/`` parses as host
- ``trusted.example`` here, while requests/urllib3 (per the WHATWG URL rules)
- treat ``\\`` as a path separator and connect to ``evil.example``. Normalizing
- those to ``/`` before parsing keeps the retarget guard aligned with the real
- destination, so an ambiguous-authority URL can't smuggle the credential to a
- new host while appearing unchanged.
+ """The lowercased host the HTTP client will actually connect to.
+
+ ``None`` when the URL has no host, or when its host depends on which parser reads it
+ (see :func:`_has_ambiguous_authority`).
"""
- normalized = url.replace("\\", "/").replace("%5c", "/").replace("%5C", "/")
- return urlparse(normalized).hostname
+ if _has_ambiguous_authority(url):
+ return None
+ return _transport_hostname(url) if urlparse(url).hostname else None
# Probe values substituted for a fan-out child's resolve placeholder to test
@@ -1198,6 +1250,7 @@ class CustomSource(SimpleSource[CustomSourceConfig]):
# dependency map (not a local copy of its resolve-param detection) tells
# us which resources are children.
probeable = [resource for resource in manifest["resources"] if resolved.get(resource.get("name")) is None]
+ vetted_hosts = manifest_request_hosts(manifest)
for resource in probeable[:PROBE_MAX_RESOURCES]:
endpoint = resource.get("endpoint", {})
method = (endpoint.get("method") or "GET").upper()
@@ -1210,6 +1263,17 @@ class CustomSource(SimpleSource[CustomSourceConfig]):
# answer differently at probe vs sync time.
probe_params = _static_probe_params(endpoint.get("params"))
probe_json = endpoint.get("json")
+ # Check the host of the URL requests will actually send, before the credential goes out.
+ try:
+ prepared_url = Request(method, url, params=probe_params or None).prepare().url
+ except (RequestException, ValueError):
+ prepared_url = None
+ prepared_host = _url_hostname(prepared_url) if prepared_url else None
+ if prepared_host is None or prepared_host not in vetted_hosts:
+ return False, (
+ f"Resource {resource['name']!r}: {display_url} does not point to a host in this manifest. "
+ "Check the URL and try again."
+ )
try:
# stream=True so the body isn't buffered into memory; only a 401/403
# snippet is read below, and the response is always closed.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py
index cd63f6aa02af..fdedf7808ecb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py
@@ -388,6 +388,42 @@ class TestValidateManifestUrls(SimpleTestCase):
assert not ok
assert "users" in (err or "")
+ @parameterized.expand(
+ [
+ ("base_url_encoded_lower", "base_url", "https://api.example.com%[email protected]/"),
+ ("base_url_encoded_upper", "base_url", "https://api.example.com%[email protected]/"),
+ ("base_url_raw", "base_url", "https://attacker.example.net\\@api.example.com/"),
+ ("token_url_encoded", "token_url", "https://auth.example.com%[email protected]/token"),
+ ("resource_path_encoded", "path", "https://api.example.com%[email protected]/users"),
+ ]
+ )
+ @override_settings(CLOUD_DEPLOYMENT="")
+ def test_rejects_ambiguous_authority(self, _name: str, field: str, url: str):
+ manifest = _minimal_manifest()
+ if field == "base_url":
+ manifest["client"]["base_url"] = url
+ elif field == "token_url":
+ manifest["client"]["auth"] = {"type": "oauth2", "client_id": "cid", "token_url": url}
+ else:
+ manifest["resources"][0]["endpoint"]["path"] = url
+ ok, err = validate_manifest_urls(manifest, team_id=999)
+ assert not ok
+ assert "backslash" in (err or "")
+
+ @parameterized.expand(
+ [
+ ("encoded_backslash_in_path", "https://api.example.com/v1", "/files/a%5Cb"),
+ ("encoded_backslash_in_query", "https://api.example.com/v1", "/users?filter=a%5cb"),
+ ("internationalized_host", "https://bücher.example/v1", "/users"),
+ ]
+ )
+ @override_settings(CLOUD_DEPLOYMENT="")
+ def test_accepts_unambiguous_authority(self, _name: str, base_url: str, path: str):
+ manifest = _minimal_manifest(base_url)
+ manifest["resources"][0]["endpoint"]["path"] = path
+ ok, err = validate_manifest_urls(manifest, team_id=999)
+ assert ok, err
+
@parameterized.expand(
[
("http_public", "http://api.example.com"),
@@ -1218,6 +1254,27 @@ class TestCustomSourceValidateCredentials(SimpleTestCase):
assert "from https://api.example.com/users " in (err or "")
assert "inline-secret-123" not in (err or "")
+ @parameterized.expand(
+ [
+ ("base_url_encoded_lower", "base_url", "https://api.example.com%[email protected]/"),
+ ("base_url_encoded_upper", "base_url", "https://api.example.com%[email protected]/"),
+ ("token_url_encoded", "token_url", "https://auth.example.com%[email protected]/oauth2/token"),
+ ]
+ )
+ def test_ambiguous_authority_sends_no_request(self, _name, field, url):
+ if field == "base_url":
+ manifest = _minimal_manifest(url)
+ config = CustomSourceConfig(manifest_json=json.dumps(manifest), auth_token="abc")
+ else:
+ manifest = _oauth2_manifest()
+ manifest["client"]["auth"]["token_url"] = url
+ config = CustomSourceConfig(manifest_json=json.dumps(manifest), auth_oauth2_client_secret="cs")
+ with scripted_network([]) as network:
+ ok, err = CustomSource().validate_credentials(config, team_id=999)
+ assert not ok
+ assert "backslash" in (err or "")
+ assert network.requests_log == []
+
@parameterized.expand([401, 403])
def test_returns_false_on_auth_rejection(self, status_code):
source = CustomSource()
@@ -1597,16 +1654,17 @@ class TestManifestRequestHosts(SimpleTestCase):
)
assert manifest_request_hosts(manifest) == frozenset({"api.example.com"})
- def test_backslash_authority_resolves_to_real_host(self):
- # `https://evil\@trusted/` connects to `evil` (urllib3/WHATWG) even though urlparse
- # alone reads `trusted` — the guard must see the real destination, not be fooled.
- manifest = json.dumps(
- {
- "client": {"base_url": "https://attacker.example.net\\@api.example.com/"},
- "resources": [{"name": "r", "endpoint": {"path": "/users"}}],
- }
- )
- assert manifest_request_hosts(manifest) == frozenset({"attacker.example.net"})
+ @parameterized.expand(
+ [
+ ("raw_backslash", "https://attacker.example.net\\@api.example.com/"),
+ ("encoded_lower", "https://api.example.com%[email protected]/"),
+ ("encoded_upper", "https://api.example.com%[email protected]/"),
+ ]
+ )
+ def test_ambiguous_authority_counts_as_new_host(self, _name, base_url):
+ existing = {"manifest_json": self._manifest("https://api.example.com/", ["/users"])}
+ incoming = {"manifest_json": self._manifest(base_url, ["/users"])}
+ assert CustomSource().job_inputs_add_connection_host(incoming, existing)
def test_host_is_lowercased(self):
assert manifest_request_hosts(self._manifest("https://API.Example.COM", [])) == frozenset({"api.example.com"})