:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(custom): drop the query string from urls in probe error messages (#120917)
Tom Owers87ef0dc3a121
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.pyindex aa3cd535c404..4ca8edbe214d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py@@ -6,7 +6,7 @@ import graphlibfrom collections.abc import Callablefrom datetime import datefrom typing import Any, Literal, NamedTuple, Optional, cast-from urllib.parse import quote, quote_plus, urlparse+from urllib.parse import parse_qsl, quote, quote_plus, urlparse, urlsplit, urlunsplitfrom django.db import IntegrityError@@ -1203,6 +1203,7 @@ class CustomSource(SimpleSource[CustomSourceConfig]):path = endpoint.get("path", "")# Resolve via the shared helper so the probe hits the same host the sync will.url = resolve_request_url(base_url, path)+ display_url = _url_without_query(url)# Replay the configured query params and request body so the probe# matches what the sync sends — an endpoint that needs them shouldn't# answer differently at probe vs sync time.@@ -1224,13 +1225,13 @@ class CustomSource(SimpleSource[CustomSourceConfig]):# str(exc) here is the raw urllib3 "HTTPSConnectionPool(...): Read timed out" dump,# which isn't actionable — surface the configured timeouts instead.return False, (- f"Resource {resource['name']!r}: timed out reaching {url} "+ f"Resource {resource['name']!r}: timed out reaching {display_url} "f"(connect timeout {PROBE_CONNECT_TIMEOUT}s, read timeout {PROBE_READ_TIMEOUT}s). ""The endpoint may be slow or temporarily unreachable — check the URL and try again.")except Exception:return False, (- f"Resource {resource['name']!r}: could not reach {url}. "+ f"Resource {resource['name']!r}: could not reach {display_url}. ""Check that the URL is correct and the endpoint is reachable.")@@ -1243,12 +1244,15 @@ class CustomSource(SimpleSource[CustomSourceConfig]):if response.status_code in (401, 403):message = (f"Resource {resource['name']!r}: the upstream API rejected the request with "- f"HTTP {response.status_code} from {url} — check the configured auth credentials."+ f"HTTP {response.status_code} from {display_url} — check the configured auth credentials.")# Redact the credential from the echoed body before surfacing it: an upstream# could reflect the sent credential (e.g. an OAuth2 bearer token minted above) in# its 401/403 error, and the snippet would otherwise leak it into a user message.- snippet = _redact_secrets(_read_capped_text(response), auth_secret_values(probe_auth))+ snippet = _redact_secrets(+ _read_capped_text(response),+ (*auth_secret_values(probe_auth), *_url_embedded_secrets(url, probe_params)),+ )if snippet:message += f" The upstream responded: {snippet}"return False, message@@ -1700,6 +1704,34 @@ def _infer_columns(rows: list[dict[str, Any]]) -> list[dict[str, str]]:return [{"name": name, "type": type_label} for name, type_label in columns.items()]+def _url_without_query(url: str) -> str:+ # A URL can carry a credential in its userinfo or query string (`?access_token=...`). Those values+ # are not auth secrets that `_redact_secrets` knows about, so drop both from echoed URLs.+ parts = urlsplit(url)+ return urlunsplit(parts._replace(netloc=parts.netloc.rpartition("@")[2], query="", fragment=""))+++_URL_SECRET_PARAM_MARKERS = ("token", "key", "secret", "password", "auth", "sig")+++def _url_embedded_secrets(url: str, params: dict[str, Any] | None = None) -> tuple[str, ...]:+ # Only credential-looking params: redacting every query value would mangle the snippet wherever a+ # short value such as `1` appears. Static endpoint params are merged into the sent URL, so they+ # can be echoed back the same way as a query string typed into the path.+ parts = urlsplit(url)+ pairs = list(parse_qsl(parts.query))+ for key, value in (params or {}).items():+ for item in value if isinstance(value, list | tuple) else (value,):+ if item is not None:+ pairs.append((str(key), str(item)))+ secrets = [+ value for key, value in pairs if value and any(marker in key.lower() for marker in _URL_SECRET_PARAM_MARKERS)+ ]+ # Either half of the userinfo can be the credential (`https://<api_key>@host` is common).+ secrets.extend(part for part in (parts.username, parts.password) if part)+ return tuple(secrets)++def _redact_secrets(text: str, secrets: tuple[str, ...]) -> str:# Redact the raw secret and its URL-encoded forms: a key in a query param is# percent-encoded in `response.url`, which an HTTPError carries verbatim.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.pyindex c5e33d601895..cd63f6aa02af 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py@@ -1196,6 +1196,28 @@ class TestCustomSourceValidateCredentials(SimpleTestCase):assert "sk_secret_value" not in (err or "")assert "***" in (err or "")+ @parameterized.expand(+ [+ ("query", "https://api.example.com", "/users?status=active&access_token=inline-secret-123"),+ ("userinfo", "https://reader:[email protected]", "/users"),+ ("username_only", "https://[email protected]", "/users"),+ ("static_param", "https://api.example.com", "/users", {"api_key": "inline-secret-123"}),+ ]+ )+ def test_auth_rejection_omits_url_credentials(self, _name, base_url, path, params=None):+ manifest = _minimal_manifest(base_url)+ manifest["resources"][0]["endpoint"]["path"] = path+ if params:+ manifest["resources"][0]["endpoint"]["params"] = params+ source = CustomSource()+ config = CustomSourceConfig(manifest_json=json.dumps(manifest), auth_token="abc")+ body = b"forbidden: https://api.example.com/users?access_token=inline-secret-123 reader:inline-secret-123"+ with scripted_network([ScriptedResponse(status=403, body=body)]):+ ok, err = source.validate_credentials(config, team_id=999)+ assert not ok+ assert "from https://api.example.com/users " in (err or "")+ assert "inline-secret-123" not in (err or "")+@parameterized.expand([401, 403])def test_returns_false_on_auth_rejection(self, status_code):source = CustomSource()