:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): follow presigned storage redirects for large rest responses (#125057)
Tom Owers4f9da113a0a4
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/braintrust/braintrust.py b/products/warehouse_sources/backend/temporal/data_imports/sources/braintrust/braintrust.pyindex 64f177cff8e8..7c39ffe46360 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/braintrust/braintrust.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/braintrust/braintrust.py@@ -11,6 +11,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.braintrustAUTH_ERRORS,ENDPOINTS,PAGE_SIZE,+ S3_HOST_PATTERN,BraintrustEndpoint,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (@@ -155,6 +156,9 @@ def _client_config(config: BraintrustSourceConfig, team_id: int) -> ClientConfig"auth": {"type": "bearer", "token": config.api_key},"allowed_hosts": [],"allow_redirects": False,+ # A response over the API's size limit comes back as a 303 to a presigned S3 object. Only S3+ # hosts match: other `amazonaws.com` names, like EC2 ones, can resolve to private addresses.+ "presigned_redirect_host_pattern": S3_HOST_PATTERN,"request_timeout": (10, 60),"paginator": {"type": "cursor",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/braintrust/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/braintrust/settings.pyindex 66667cf40210..5256d576b7ec 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/braintrust/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/braintrust/settings.py@@ -10,6 +10,8 @@ from products.warehouse_sources.backend.types import IncrementalFieldPAGE_SIZE = 100# Fetch endpoints count `limit` in traces, so one page can hold many more rows than this.EVENTS_PAGE_SIZE = 100+# Virtual-hosted S3 bucket hosts, global or regional (`<bucket>.s3.<region>.amazonaws.com`).+S3_HOST_PATTERN = r"[a-z0-9][a-z0-9.-]*\.s3(?:[.-][a-z0-9-]+)?\.amazonaws\.com"@frozendiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.pyindex e11fe8cf0e26..7af0bc712c1e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py@@ -358,6 +358,7 @@ def create_resources(retry_backoff_max_seconds=client_config.get("retry_backoff_max_seconds", DEFAULT_RETRY_BACKOFF_MAX_SECONDS),allowed_hosts=client_config.get("allowed_hosts"),allow_redirects=client_config.get("allow_redirects", True),+ presigned_redirect_host_pattern=client_config.get("presigned_redirect_host_pattern"),request_timeout=client_config.get("request_timeout"),capture=client_config.get("capture", True),**_retry_limit_overrides(client_config),diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.pyindex 83d3377f27c5..dfcb53919bf0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py@@ -1,6 +1,7 @@import reimport copyimport time+import zlibimport loggingfrom collections.abc import Callable, Iteratorfrom datetime import UTC, datetime@@ -8,6 +9,7 @@ from email.utils import parsedate_to_datetimefrom typing import Any, Optionalfrom urllib.parse import urlsplit+from opentelemetry.instrumentation.utils import suppress_http_instrumentationfrom requests import PreparedRequest, Request, Response, Sessionfrom requests.auth import AuthBasefrom requests.exceptions import (@@ -15,10 +17,12 @@ from requests.exceptions import (ConnectionError as RequestsConnectionError,HTTPError,JSONDecodeError as RequestsJSONDecodeError,+ RequestException,Timeout as RequestsTimeout,TooManyRedirects,)from tenacity import RetryCallState, retry, retry_if_exception_type+from urllib3.exceptions import HTTPError as Urllib3HTTPErrorfrom posthog.temporal.common.errors import NonReportableError@@ -29,6 +33,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.httrequest_deadline,suspend_adapter_retries,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.observer import redact_request_urlsfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.interruptible_wait import (interruptible_wait,)@@ -169,11 +174,12 @@ def _safe_url(url: str) -> str:``api_key`` auth with ``location: "query"`` carries the secret in the querystring, so keep only the parts that are safe to surface."""parts = urlsplit(url)- if not parts.scheme:- return urlhost = parts.hostname or ""if parts.port:host = f"{host}:{parts.port}"+ if not parts.scheme:+ # A redirect `Location` can be relative or scheme-relative and still carry a signed query.+ return f"//{host}{parts.path}" if parts.netloc else parts.pathreturn f"{parts.scheme}://{host}{parts.path}"@@ -191,6 +197,11 @@ DEFAULT_RETRY_ATTEMPTS = 5# budget outlasts the window.DEFAULT_RETRY_BACKOFF_MAX_SECONDS = 60.0+# Ceiling on an offloaded response body, compressed or not, so a hostile redirect target can't+# exhaust worker memory with an oversized object or a gzip bomb.+MAX_PRESIGNED_RESPONSE_BYTES = 512 * 1024 * 1024+_PRESIGNED_READ_CHUNK_BYTES = 1024 * 1024+# Default network ports per scheme, used to compare a request URL's effective port against the# base origin's when host-pinning is enabled._DEFAULT_PORTS = {"http": 80, "https": 443}@@ -394,6 +405,7 @@ class RESTClient:retry_backoff_max_seconds: float = DEFAULT_RETRY_BACKOFF_MAX_SECONDS,allowed_hosts: Optional[list[str]] = None,allow_redirects: bool = True,+ presigned_redirect_host_pattern: Optional[str] = None,request_timeout: Optional[RequestTimeout] = None,capture: bool = True,retry_budget_seconds: Optional[float] = None,@@ -417,6 +429,9 @@ class RESTClient:# a tighter value. ``NO_REQUEST_TIMEOUT`` is the only way to send a request with no deadline.self._request_timeout = request_timeoutself._allow_redirects = allow_redirects+ self._presigned_redirect_host = (+ re.compile(presigned_redirect_host_pattern, re.IGNORECASE) if presigned_redirect_host_pattern else None+ )# When set (even to an empty list), every outgoing request URL — including# paginator next-page links and seeded resume URLs — must resolve to one of# these hosts (the base_url host is always implicitly allowed). This pins@@ -503,6 +518,73 @@ class RESTClient:))+ def _presigned_redirect_url(self, response: Response) -> Optional[str]:+ if self._presigned_redirect_host is None or response.status_code not in (302, 303):+ return None+ location = response.headers.get("Location")+ if not location:+ return None+ split = urlsplit(location)+ if (+ split.scheme.lower() != "https"+ or split.username is not None+ or split.password is not None+ or split.port not in (None, 443)+ or not self._presigned_redirect_host.fullmatch(split.hostname or "")+ ):+ return None+ return location++ def _fetch_presigned_redirect(self, url: str) -> Response:+ # The signed URL is its own credential, so the request is built outside the session to keep+ # the API auth and session headers off it, and its URL stays out of logs, samples and traces.+ # Errors name only `_safe_url(url)` and drop their cause, which embeds the signed URL.+ safe_url = _safe_url(url)+ deadline = current_request_deadline()+ prepared = PreparedRequest()+ prepared.prepare(method="GET", url=url)+ try:+ with redact_request_urls(), suppress_http_instrumentation():+ response = self._send(prepared, stream=True)+ try:+ if not response.ok:+ # An expired or missing object is fixed by asking the API again for a fresh URL.+ raise RESTClientRetryableError(+ f"HTTP {response.status_code} fetching an offloaded response from {safe_url}"+ )+ chunks: list[bytes] = []+ size = 0+ # Raw bytes, so urllib3 can't inflate a `Content-Encoding: gzip` body past the cap.+ for chunk in response.raw.stream(_PRESIGNED_READ_CHUNK_BYTES, decode_content=False):+ chunks.append(chunk)+ size += len(chunk)+ if size > MAX_PRESIGNED_RESPONSE_BYTES:+ break+ # Socket timeouts bound each read, not a body that keeps trickling in.+ if deadline is not None and _monotonic() > deadline:+ raise RESTClientRetryableError(f"Timed out reading an offloaded response from {safe_url}")+ body = b"".join(chunks)+ finally:+ response.close()+ except (RequestException, Urllib3HTTPError) as e:+ raise RESTClientRetryableError(+ f"Connection error ({type(e).__name__}) fetching an offloaded response from {safe_url}"+ ) from None+ if body[:2] == b"\x1f\x8b":+ decompressor = zlib.decompressobj(wbits=zlib.MAX_WBITS | 16)+ try:+ body = decompressor.decompress(body, MAX_PRESIGNED_RESPONSE_BYTES + 1)+ except zlib.error:+ raise RESTClientRetryableError(f"Corrupt offloaded response from {safe_url}") from None+ if len(body) <= MAX_PRESIGNED_RESPONSE_BYTES and not decompressor.eof:+ raise RESTClientRetryableError(f"Truncated offloaded response from {safe_url}")+ if len(body) > MAX_PRESIGNED_RESPONSE_BYTES:+ raise RESTClientNonRetryableError(+ f"Offloaded response from {safe_url} exceeds {MAX_PRESIGNED_RESPONSE_BYTES} bytes"+ )+ response._content = body+ return response+def paginate(self,path: str = "",@@ -601,14 +683,17 @@ class RESTClient:outer = current_request_deadline()return deadline if outer is None else min(deadline, outer)- def _send(self, prepared: PreparedRequest) -> Response:+ def _send(self, prepared: PreparedRequest, stream: bool = False) -> Response:+ kwargs: dict[str, Any] = {"allow_redirects": self._allow_redirects, "timeout": self._request_timeout}+ if stream:+ kwargs["stream"] = Trueif self._max_retry_attempts <= 1:# This client does not retry, so the adapter keeps its own retries.- return self.session.send(prepared, allow_redirects=self._allow_redirects, timeout=self._request_timeout)+ return self.session.send(prepared, **kwargs)# This client owns the retries. With the adapter's retries on too, the tries and the waits# of the two layers multiply, and the adapter's waits have no safe point.with suspend_adapter_retries():- return self.session.send(prepared, allow_redirects=self._allow_redirects, timeout=self._request_timeout)+ return self.session.send(prepared, **kwargs)@retry(retry=retry_if_exception_type(RESTClientRetryableError),@@ -644,6 +729,9 @@ class RESTClient:try:with request_deadline(deadline):response = self._send(prepared)+ presigned_url = self._presigned_redirect_url(response)+ if presigned_url is not None:+ response = self._fetch_presigned_redirect(presigned_url)except ChunkedEncodingError as e:raise RESTClientRetryableError(self._redact(f"Connection broken while reading response: {e}")) from eexcept RequestsConnectionError as e:@@ -682,7 +770,7 @@ class RESTClient:raise ValueError(self._redact(f"Unexpected redirect ({response.status_code}) to "- f"{response.headers.get('Location')!r} from {prepared.url}; refusing to follow."+ f"{_safe_url(response.headers.get('Location') or '')!r} from {prepared.url}; refusing to follow."))diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_ssrf_host_pinning.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_ssrf_host_pinning.pyindex 2faf9eab26de..e17cd3d383c3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_ssrf_host_pinning.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_ssrf_host_pinning.py@@ -1,3 +1,5 @@+import io+import gzipimport jsonfrom typing import Any, Optional@@ -5,12 +7,19 @@ import pytestfrom unittest.mock import MagicMock, patchfrom parameterized import parameterized-from requests import Response+from requests import Response, Session+from requests.exceptions import ConnectionError as RequestsConnectionError+from urllib3 import HTTPResponse+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import BearerTokenAuthfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (JSONResponsePaginator,)-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClient+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (+ RESTClient,+ RESTClientNonRetryableError,+ RESTClientRetryableError,+)MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client"@@ -41,6 +50,35 @@ def _session_echoing_url(MockSession) -> MagicMock:return mock_session+S3_PATTERN = r"[a-z0-9.-]+\.s3\.[a-z0-9-]+\.amazonaws\.com"+STORAGE_URL = "https://bucket.s3.us-east-1.amazonaws.com/responses/page.json.gz?X-Amz-Signature=sig"+++def _storage_response(body: bytes, status: int = 200) -> Response:+ resp = Response()+ resp.status_code = status+ resp.raw = HTTPResponse(body=io.BytesIO(body), status=status, preload_content=False)+ return resp+++def _redirect_to_storage() -> Response:+ return _make_response({}, status_code=303, headers={"Location": STORAGE_URL})+++def _presigned_client() -> tuple[Session, RESTClient]:+ session = Session()+ client = RESTClient(+ base_url="https://api.example.com",+ auth=BearerTokenAuth("api-secret"),+ headers={"X-Api-Key": "header-secret"},+ session=session,+ max_retry_attempts=1,+ allow_redirects=False,+ presigned_redirect_host_pattern=S3_PATTERN,+ )+ return session, client++class TestSSRFHostPinning:@parameterized.expand([@@ -110,16 +148,35 @@ class TestSSRFHostPinning:pages = list(pager)assert [row["id"] for page in pages for row in page] == [1, 2]+ @parameterized.expand(+ [+ ("redirects_disabled", None, "https://evil.com/steal"),+ (+ "host_outside_pattern",+ S3_PATTERN,+ "https://evil.com/a.s3.us-east-1.amazonaws.com?X-Amz-Signature=sekret",+ ),+ ("non_s3_aws_host", S3_PATTERN, "https://ec2-1-2-3-4.compute-1.amazonaws.com/o?X-Amz-Signature=sekret"),+ ("presigned_host_over_http", S3_PATTERN, "http://b.s3.us-east-1.amazonaws.com/o?X-Amz-Signature=sekret"),+ ("scheme_relative_location", S3_PATTERN, "//b.s3.us-east-1.amazonaws.com/o?X-Amz-Signature=sekret"),+ ]+ )@patch(f"{MODULE}.make_tracked_session")- def test_redirect_rejected_when_disabled(self, MockSession) -> None:+ def test_redirect_rejected_when_disabled(+ self, _name: str, presigned_pattern: Optional[str], location: str, MockSession+ ) -> None:mock_session = _session_echoing_url(MockSession)- mock_session.send.return_value = _make_response(- {}, status_code=302, headers={"Location": "https://evil.com/steal"}- )+ mock_session.send.return_value = _make_response({}, status_code=303, headers={"Location": location})- client = RESTClient(base_url="https://api.example.com", allow_redirects=False)- with pytest.raises(ValueError, match="Unexpected redirect"):+ client = RESTClient(+ base_url="https://api.example.com",+ allow_redirects=False,+ presigned_redirect_host_pattern=presigned_pattern,+ )+ with pytest.raises(ValueError, match="Unexpected redirect") as exc_info:list(client.paginate(path="/items", data_selector="results"))+ assert "sekret" not in str(exc_info.value)+ assert mock_session.send.call_count == 1# The redirect target was never followed: send ran once and got no allow_redirects._, kwargs = mock_session.send.call_args@@ -135,3 +192,52 @@ class TestSSRFHostPinning:_, kwargs = mock_session.send.call_argsassert kwargs.get("allow_redirects") is True++ @parameterized.expand([("plain_body", False), ("gzipped_body", True)])+ def test_presigned_redirect_fetched_without_credentials(self, _name: str, gzipped: bool) -> None:+ body = json.dumps({"results": [{"id": 1}]}).encode()+ session, client = _presigned_client()++ with patch.object(+ session,+ "send",+ side_effect=[_redirect_to_storage(), _storage_response(gzip.compress(body) if gzipped else body)],+ ) as send:+ pages = list(client.paginate(path="/items", method="post", data_selector="results"))++ assert pages == [[{"id": 1}]]+ followed = send.call_args_list[1].args[0]+ assert (followed.method, followed.url) == ("GET", STORAGE_URL)+ assert "Authorization" not in followed.headers+ assert "X-Api-Key" not in followed.headers++ @parameterized.expand(+ [+ ("expired_url", _storage_response(b"", status=403), RESTClientRetryableError),+ (+ "connection_error",+ RequestsConnectionError(f"Max retries exceeded with url: {STORAGE_URL}"),+ RESTClientRetryableError,+ ),+ ("corrupt_gzip", _storage_response(b"\x1f\x8b" + b"\x00" * 20), RESTClientRetryableError),+ (+ "truncated_gzip",+ _storage_response(gzip.compress(b'{"results": []}' * 100)[:30]),+ RESTClientRetryableError,+ ),+ ("gzip_bomb", _storage_response(gzip.compress(b"\x00" * 4096)), RESTClientNonRetryableError),+ ("oversized_body", _storage_response(b'{"results": []}' + b" " * 4096), RESTClientNonRetryableError),+ ]+ )+ @patch(f"{MODULE}.MAX_PRESIGNED_RESPONSE_BYTES", 1024)+ def test_presigned_fetch_failure_hides_signed_url(+ self, _name: str, storage_outcome: Any, expected: type[Exception]+ ) -> None:+ session, client = _presigned_client()++ with patch.object(session, "send", side_effect=[_redirect_to_storage(), storage_outcome]):+ with pytest.raises(expected) as exc_info:+ list(client.paginate(path="/items", method="post", data_selector="results"))++ assert "X-Amz-Signature" not in str(exc_info.value)+ assert exc_info.value.__cause__ is Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.pyindex df9983c64c31..ab8d6d0603c0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py@@ -229,6 +229,10 @@ class ClientConfig(TypedDict, total=False):# When False, redirects are not followed and any 3xx is rejected — closes the redirect-based# off-host escape that host-pinning alone would miss. Defaults to True (follow redirects).allow_redirects: bool+ # Regex the host of a vendor's presigned storage must fully match. With redirects disabled, a+ # 302/303 to an HTTPS host matching it is fetched with a bare GET that carries no credentials,+ # and its body (gunzipped when needed, capped in size) is read as the page.+ presigned_redirect_host_pattern: Optional[str]# Per-request (connect, read) timeout in seconds. Left unset, the tracked session applies# `default_request_timeout()`. A source pointed at a customer-controlled host should set a# tighter value. A single float applies to both connect and read; a tuple sets them separately.