:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse-sources): reject ambiguous url hosts in custom source manifests (#125422)
Tom Owers5f96098b9f89
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.pyindex 127a6fa885ab..00f83a91846a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py@@ -13,7 +13,9 @@ from django.db import IntegrityErrorimport structlogfrom jsonpath_ng.exceptions import JSONPathErrorfrom pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator-from requests import PreparedRequest, Response, Timeout+from requests import PreparedRequest, Request, RequestException, Response, Timeout+from urllib3.exceptions import LocationParseError+from urllib3.util import parse_urlfrom urllib3.util.retry import Retryfrom posthog.cloud_utils import is_cloud@@ -619,8 +621,13 @@ def _has_wrapping_quote(url: str) -> bool:def _check_url(url: str, team_id: int) -> tuple[bool, str | None]:- # `_url_hostname` mirrors the real connect host (backslash/whitespace-normalized) so the- # validator can't be fooled into vetting a different host than the request reaches.+ # `_url_hostname` returns the host the HTTP client connects to, so the validator can't be+ # fooled into vetting a different host than the request reaches.+ if _has_ambiguous_authority(url):+ return (+ False,+ "Remove the backslash (\\ or %5C) from the URL's host and enter just the address (for example, https://api.example.com).",+ )hostname = _url_hostname(url)if not hostname:if _has_leading_http_method(url):@@ -745,24 +752,69 @@ def manifest_request_hosts(manifest_json: Any) -> frozenset[str]:if token_url is not None:urls.append(token_url)- # `_url_hostname` returns an already-lowercased host.- hosts = {host for url in urls if (host := _url_hostname(url))}+ hosts: set[str] = set()+ for url in urls:+ if _has_ambiguous_authority(url):+ # Fail closed: a URL whose host depends on the parser counts as a new destination.+ hosts.add(f"{_AMBIGUOUS_HOST_MARKER}{url}")+ elif host := _url_hostname(url):+ hosts.add(host)return frozenset(hosts)+# Prefix for a `manifest_request_hosts` entry that stands in for a URL with no single host.+# It can't match a real hostname, so the re-entry gate always sees that URL as a new destination.+_AMBIGUOUS_HOST_MARKER = "ambiguous-url:"+++def _transport_hostname(url: str) -> str | None:+ """The host urllib3 reads from ``url``. requests prepares and connects with this parser."""+ try:+ # requests strips leading whitespace before it parses the URL.+ host = parse_url(url.lstrip()).host+ except LocationParseError:+ return None+ return host.lower().strip("[]") if host else None+++def _has_ambiguous_authority(url: str) -> bool:+ """True when ``urlparse`` and the HTTP client could read a different host from ``url``.++ ``urlparse`` reads a backslash in the authority as userinfo, but urllib3 ends the authority+ there: ``https://evil.example\\@trusted.example/`` connects to ``evil.example``. A+ percent-encoded backslash stays userinfo for both parsers, but a client that decodes first+ splits on it. No rewrite of these forms matches every parser, so such a URL is rejected.+ Any other host disagreement between the two parsers is rejected the same way.+ """+ try:+ parsed = urlparse(url)+ netloc = parsed.netloc.lower()+ hostname = parsed.hostname+ except ValueError:+ return True+ if "\\" in netloc or "%5c" in netloc:+ return True+ if hostname is None:+ # No host for urlparse means no host for the validator either; `_check_url` rejects it.+ return False+ if not hostname.isascii():+ # urllib3 sends the IDNA (punycode) form of an internationalized host.+ try:+ hostname = hostname.encode("idna").decode("ascii")+ except UnicodeError:+ return True+ return hostname != _transport_hostname(url)++def _url_hostname(url: str) -> str | None:- """The host the HTTP client will actually connect to.-- `urlparse` treats a backslash — and its ``%5c`` encoding — as ordinary- userinfo, so ``https://evil.example\\@trusted.example/`` parses as host- ``trusted.example`` here, while requests/urllib3 (per the WHATWG URL rules)- treat ``\\`` as a path separator and connect to ``evil.example``. Normalizing- those to ``/`` before parsing keeps the retarget guard aligned with the real- destination, so an ambiguous-authority URL can't smuggle the credential to a- new host while appearing unchanged.+ """The lowercased host the HTTP client will actually connect to.++ ``None`` when the URL has no host, or when its host depends on which parser reads it+ (see :func:`_has_ambiguous_authority`)."""- normalized = url.replace("\\", "/").replace("%5c", "/").replace("%5C", "/")- return urlparse(normalized).hostname+ if _has_ambiguous_authority(url):+ return None+ return _transport_hostname(url) if urlparse(url).hostname else None# Probe values substituted for a fan-out child's resolve placeholder to test@@ -1198,6 +1250,7 @@ class CustomSource(SimpleSource[CustomSourceConfig]):# dependency map (not a local copy of its resolve-param detection) tells# us which resources are children.probeable = [resource for resource in manifest["resources"] if resolved.get(resource.get("name")) is None]+ vetted_hosts = manifest_request_hosts(manifest)for resource in probeable[:PROBE_MAX_RESOURCES]:endpoint = resource.get("endpoint", {})method = (endpoint.get("method") or "GET").upper()@@ -1210,6 +1263,17 @@ class CustomSource(SimpleSource[CustomSourceConfig]):# answer differently at probe vs sync time.probe_params = _static_probe_params(endpoint.get("params"))probe_json = endpoint.get("json")+ # Check the host of the URL requests will actually send, before the credential goes out.+ try:+ prepared_url = Request(method, url, params=probe_params or None).prepare().url+ except (RequestException, ValueError):+ prepared_url = None+ prepared_host = _url_hostname(prepared_url) if prepared_url else None+ if prepared_host is None or prepared_host not in vetted_hosts:+ return False, (+ f"Resource {resource['name']!r}: {display_url} does not point to a host in this manifest. "+ "Check the URL and try again."+ )try:# stream=True so the body isn't buffered into memory; only a 401/403# snippet is read below, and the response is always closed.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.pyindex cd63f6aa02af..fdedf7808ecb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py@@ -388,6 +388,42 @@ class TestValidateManifestUrls(SimpleTestCase):assert not okassert "users" in (err or "")+ @parameterized.expand(+ [+ ("base_url_encoded_lower", "base_url", "https://api.example.com%[email protected]/"),+ ("base_url_encoded_upper", "base_url", "https://api.example.com%[email protected]/"),+ ("base_url_raw", "base_url", "https://attacker.example.net\\@api.example.com/"),+ ("token_url_encoded", "token_url", "https://auth.example.com%[email protected]/token"),+ ("resource_path_encoded", "path", "https://api.example.com%[email protected]/users"),+ ]+ )+ @override_settings(CLOUD_DEPLOYMENT="")+ def test_rejects_ambiguous_authority(self, _name: str, field: str, url: str):+ manifest = _minimal_manifest()+ if field == "base_url":+ manifest["client"]["base_url"] = url+ elif field == "token_url":+ manifest["client"]["auth"] = {"type": "oauth2", "client_id": "cid", "token_url": url}+ else:+ manifest["resources"][0]["endpoint"]["path"] = url+ ok, err = validate_manifest_urls(manifest, team_id=999)+ assert not ok+ assert "backslash" in (err or "")++ @parameterized.expand(+ [+ ("encoded_backslash_in_path", "https://api.example.com/v1", "/files/a%5Cb"),+ ("encoded_backslash_in_query", "https://api.example.com/v1", "/users?filter=a%5cb"),+ ("internationalized_host", "https://bücher.example/v1", "/users"),+ ]+ )+ @override_settings(CLOUD_DEPLOYMENT="")+ def test_accepts_unambiguous_authority(self, _name: str, base_url: str, path: str):+ manifest = _minimal_manifest(base_url)+ manifest["resources"][0]["endpoint"]["path"] = path+ ok, err = validate_manifest_urls(manifest, team_id=999)+ assert ok, err+@parameterized.expand([("http_public", "http://api.example.com"),@@ -1218,6 +1254,27 @@ class TestCustomSourceValidateCredentials(SimpleTestCase):assert "from https://api.example.com/users " in (err or "")assert "inline-secret-123" not in (err or "")+ @parameterized.expand(+ [+ ("base_url_encoded_lower", "base_url", "https://api.example.com%[email protected]/"),+ ("base_url_encoded_upper", "base_url", "https://api.example.com%[email protected]/"),+ ("token_url_encoded", "token_url", "https://auth.example.com%[email protected]/oauth2/token"),+ ]+ )+ def test_ambiguous_authority_sends_no_request(self, _name, field, url):+ if field == "base_url":+ manifest = _minimal_manifest(url)+ config = CustomSourceConfig(manifest_json=json.dumps(manifest), auth_token="abc")+ else:+ manifest = _oauth2_manifest()+ manifest["client"]["auth"]["token_url"] = url+ config = CustomSourceConfig(manifest_json=json.dumps(manifest), auth_oauth2_client_secret="cs")+ with scripted_network([]) as network:+ ok, err = CustomSource().validate_credentials(config, team_id=999)+ assert not ok+ assert "backslash" in (err or "")+ assert network.requests_log == []+@parameterized.expand([401, 403])def test_returns_false_on_auth_rejection(self, status_code):source = CustomSource()@@ -1597,16 +1654,17 @@ class TestManifestRequestHosts(SimpleTestCase):)assert manifest_request_hosts(manifest) == frozenset({"api.example.com"})- def test_backslash_authority_resolves_to_real_host(self):- # `https://evil\@trusted/` connects to `evil` (urllib3/WHATWG) even though urlparse- # alone reads `trusted` — the guard must see the real destination, not be fooled.- manifest = json.dumps(- {- "client": {"base_url": "https://attacker.example.net\\@api.example.com/"},- "resources": [{"name": "r", "endpoint": {"path": "/users"}}],- }- )- assert manifest_request_hosts(manifest) == frozenset({"attacker.example.net"})+ @parameterized.expand(+ [+ ("raw_backslash", "https://attacker.example.net\\@api.example.com/"),+ ("encoded_lower", "https://api.example.com%[email protected]/"),+ ("encoded_upper", "https://api.example.com%[email protected]/"),+ ]+ )+ def test_ambiguous_authority_counts_as_new_host(self, _name, base_url):+ existing = {"manifest_json": self._manifest("https://api.example.com/", ["/users"])}+ incoming = {"manifest_json": self._manifest(base_url, ["/users"])}+ assert CustomSource().job_inputs_add_connection_host(incoming, existing)def test_host_is_lowercased(self):assert manifest_request_hosts(self._manifest("https://API.Example.COM", [])) == frozenset({"api.example.com"})