Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): follow presigned storage redirects for large rest responses (#125057)

Tom Owers4f9da113a0a4
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/braintrust/braintrust.py b/products/warehouse_sources/backend/temporal/data_imports/sources/braintrust/braintrust.py
index 64f177cff8e8..7c39ffe46360 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/braintrust/braintrust.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/braintrust/braintrust.py
@@ -11,6 +11,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.braintrust
AUTH_ERRORS,
ENDPOINTS,
PAGE_SIZE,
+ S3_HOST_PATTERN,
BraintrustEndpoint,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
@@ -155,6 +156,9 @@ def _client_config(config: BraintrustSourceConfig, team_id: int) -> ClientConfig
"auth": {"type": "bearer", "token": config.api_key},
"allowed_hosts": [],
"allow_redirects": False,
+ # A response over the API's size limit comes back as a 303 to a presigned S3 object. Only S3
+ # hosts match: other `amazonaws.com` names, like EC2 ones, can resolve to private addresses.
+ "presigned_redirect_host_pattern": S3_HOST_PATTERN,
"request_timeout": (10, 60),
"paginator": {
"type": "cursor",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/braintrust/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/braintrust/settings.py
index 66667cf40210..5256d576b7ec 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/braintrust/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/braintrust/settings.py
@@ -10,6 +10,8 @@ from products.warehouse_sources.backend.types import IncrementalField
PAGE_SIZE = 100
# Fetch endpoints count `limit` in traces, so one page can hold many more rows than this.
EVENTS_PAGE_SIZE = 100
+# Virtual-hosted S3 bucket hosts, global or regional (`<bucket>.s3.<region>.amazonaws.com`).
+S3_HOST_PATTERN = r"[a-z0-9][a-z0-9.-]*\.s3(?:[.-][a-z0-9-]+)?\.amazonaws\.com"
@frozen
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py
index e11fe8cf0e26..7af0bc712c1e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py
@@ -358,6 +358,7 @@ def create_resources(
retry_backoff_max_seconds=client_config.get("retry_backoff_max_seconds", DEFAULT_RETRY_BACKOFF_MAX_SECONDS),
allowed_hosts=client_config.get("allowed_hosts"),
allow_redirects=client_config.get("allow_redirects", True),
+ presigned_redirect_host_pattern=client_config.get("presigned_redirect_host_pattern"),
request_timeout=client_config.get("request_timeout"),
capture=client_config.get("capture", True),
**_retry_limit_overrides(client_config),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py
index 83d3377f27c5..dfcb53919bf0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py
@@ -1,6 +1,7 @@
import re
import copy
import time
+import zlib
import logging
from collections.abc import Callable, Iterator
from datetime import UTC, datetime
@@ -8,6 +9,7 @@ from email.utils import parsedate_to_datetime
from typing import Any, Optional
from urllib.parse import urlsplit
+from opentelemetry.instrumentation.utils import suppress_http_instrumentation
from requests import PreparedRequest, Request, Response, Session
from requests.auth import AuthBase
from requests.exceptions import (
@@ -15,10 +17,12 @@ from requests.exceptions import (
ConnectionError as RequestsConnectionError,
HTTPError,
JSONDecodeError as RequestsJSONDecodeError,
+ RequestException,
Timeout as RequestsTimeout,
TooManyRedirects,
)
from tenacity import RetryCallState, retry, retry_if_exception_type
+from urllib3.exceptions import HTTPError as Urllib3HTTPError
from posthog.temporal.common.errors import NonReportableError
@@ -29,6 +33,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.htt
request_deadline,
suspend_adapter_retries,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.observer import redact_request_urls
from products.warehouse_sources.backend.temporal.data_imports.sources.common.interruptible_wait import (
interruptible_wait,
)
@@ -169,11 +174,12 @@ def _safe_url(url: str) -> str:
``api_key`` auth with ``location: "query"`` carries the secret in the query
string, so keep only the parts that are safe to surface."""
parts = urlsplit(url)
- if not parts.scheme:
- return url
host = parts.hostname or ""
if parts.port:
host = f"{host}:{parts.port}"
+ if not parts.scheme:
+ # A redirect `Location` can be relative or scheme-relative and still carry a signed query.
+ return f"//{host}{parts.path}" if parts.netloc else parts.path
return f"{parts.scheme}://{host}{parts.path}"
@@ -191,6 +197,11 @@ DEFAULT_RETRY_ATTEMPTS = 5
# budget outlasts the window.
DEFAULT_RETRY_BACKOFF_MAX_SECONDS = 60.0
+# Ceiling on an offloaded response body, compressed or not, so a hostile redirect target can't
+# exhaust worker memory with an oversized object or a gzip bomb.
+MAX_PRESIGNED_RESPONSE_BYTES = 512 * 1024 * 1024
+_PRESIGNED_READ_CHUNK_BYTES = 1024 * 1024
+
# Default network ports per scheme, used to compare a request URL's effective port against the
# base origin's when host-pinning is enabled.
_DEFAULT_PORTS = {"http": 80, "https": 443}
@@ -394,6 +405,7 @@ class RESTClient:
retry_backoff_max_seconds: float = DEFAULT_RETRY_BACKOFF_MAX_SECONDS,
allowed_hosts: Optional[list[str]] = None,
allow_redirects: bool = True,
+ presigned_redirect_host_pattern: Optional[str] = None,
request_timeout: Optional[RequestTimeout] = None,
capture: bool = True,
retry_budget_seconds: Optional[float] = None,
@@ -417,6 +429,9 @@ class RESTClient:
# a tighter value. ``NO_REQUEST_TIMEOUT`` is the only way to send a request with no deadline.
self._request_timeout = request_timeout
self._allow_redirects = allow_redirects
+ self._presigned_redirect_host = (
+ re.compile(presigned_redirect_host_pattern, re.IGNORECASE) if presigned_redirect_host_pattern else None
+ )
# When set (even to an empty list), every outgoing request URL — including
# paginator next-page links and seeded resume URLs — must resolve to one of
# these hosts (the base_url host is always implicitly allowed). This pins
@@ -503,6 +518,73 @@ class RESTClient:
)
)
+ def _presigned_redirect_url(self, response: Response) -> Optional[str]:
+ if self._presigned_redirect_host is None or response.status_code not in (302, 303):
+ return None
+ location = response.headers.get("Location")
+ if not location:
+ return None
+ split = urlsplit(location)
+ if (
+ split.scheme.lower() != "https"
+ or split.username is not None
+ or split.password is not None
+ or split.port not in (None, 443)
+ or not self._presigned_redirect_host.fullmatch(split.hostname or "")
+ ):
+ return None
+ return location
+
+ def _fetch_presigned_redirect(self, url: str) -> Response:
+ # The signed URL is its own credential, so the request is built outside the session to keep
+ # the API auth and session headers off it, and its URL stays out of logs, samples and traces.
+ # Errors name only `_safe_url(url)` and drop their cause, which embeds the signed URL.
+ safe_url = _safe_url(url)
+ deadline = current_request_deadline()
+ prepared = PreparedRequest()
+ prepared.prepare(method="GET", url=url)
+ try:
+ with redact_request_urls(), suppress_http_instrumentation():
+ response = self._send(prepared, stream=True)
+ try:
+ if not response.ok:
+ # An expired or missing object is fixed by asking the API again for a fresh URL.
+ raise RESTClientRetryableError(
+ f"HTTP {response.status_code} fetching an offloaded response from {safe_url}"
+ )
+ chunks: list[bytes] = []
+ size = 0
+ # Raw bytes, so urllib3 can't inflate a `Content-Encoding: gzip` body past the cap.
+ for chunk in response.raw.stream(_PRESIGNED_READ_CHUNK_BYTES, decode_content=False):
+ chunks.append(chunk)
+ size += len(chunk)
+ if size > MAX_PRESIGNED_RESPONSE_BYTES:
+ break
+ # Socket timeouts bound each read, not a body that keeps trickling in.
+ if deadline is not None and _monotonic() > deadline:
+ raise RESTClientRetryableError(f"Timed out reading an offloaded response from {safe_url}")
+ body = b"".join(chunks)
+ finally:
+ response.close()
+ except (RequestException, Urllib3HTTPError) as e:
+ raise RESTClientRetryableError(
+ f"Connection error ({type(e).__name__}) fetching an offloaded response from {safe_url}"
+ ) from None
+ if body[:2] == b"\x1f\x8b":
+ decompressor = zlib.decompressobj(wbits=zlib.MAX_WBITS | 16)
+ try:
+ body = decompressor.decompress(body, MAX_PRESIGNED_RESPONSE_BYTES + 1)
+ except zlib.error:
+ raise RESTClientRetryableError(f"Corrupt offloaded response from {safe_url}") from None
+ if len(body) <= MAX_PRESIGNED_RESPONSE_BYTES and not decompressor.eof:
+ raise RESTClientRetryableError(f"Truncated offloaded response from {safe_url}")
+ if len(body) > MAX_PRESIGNED_RESPONSE_BYTES:
+ raise RESTClientNonRetryableError(
+ f"Offloaded response from {safe_url} exceeds {MAX_PRESIGNED_RESPONSE_BYTES} bytes"
+ )
+ response._content = body
+ return response
+
def paginate(
self,
path: str = "",
@@ -601,14 +683,17 @@ class RESTClient:
outer = current_request_deadline()
return deadline if outer is None else min(deadline, outer)
- def _send(self, prepared: PreparedRequest) -> Response:
+ def _send(self, prepared: PreparedRequest, stream: bool = False) -> Response:
+ kwargs: dict[str, Any] = {"allow_redirects": self._allow_redirects, "timeout": self._request_timeout}
+ if stream:
+ kwargs["stream"] = True
if self._max_retry_attempts <= 1:
# This client does not retry, so the adapter keeps its own retries.
- return self.session.send(prepared, allow_redirects=self._allow_redirects, timeout=self._request_timeout)
+ return self.session.send(prepared, **kwargs)
# This client owns the retries. With the adapter's retries on too, the tries and the waits
# of the two layers multiply, and the adapter's waits have no safe point.
with suspend_adapter_retries():
- return self.session.send(prepared, allow_redirects=self._allow_redirects, timeout=self._request_timeout)
+ return self.session.send(prepared, **kwargs)
@retry(
retry=retry_if_exception_type(RESTClientRetryableError),
@@ -644,6 +729,9 @@ class RESTClient:
try:
with request_deadline(deadline):
response = self._send(prepared)
+ presigned_url = self._presigned_redirect_url(response)
+ if presigned_url is not None:
+ response = self._fetch_presigned_redirect(presigned_url)
except ChunkedEncodingError as e:
raise RESTClientRetryableError(self._redact(f"Connection broken while reading response: {e}")) from e
except RequestsConnectionError as e:
@@ -682,7 +770,7 @@ class RESTClient:
raise ValueError(
self._redact(
f"Unexpected redirect ({response.status_code}) to "
- f"{response.headers.get('Location')!r} from {prepared.url}; refusing to follow."
+ f"{_safe_url(response.headers.get('Location') or '')!r} from {prepared.url}; refusing to follow."
)
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_ssrf_host_pinning.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_ssrf_host_pinning.py
index 2faf9eab26de..e17cd3d383c3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_ssrf_host_pinning.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_ssrf_host_pinning.py
@@ -1,3 +1,5 @@
+import io
+import gzip
import json
from typing import Any, Optional
@@ -5,12 +7,19 @@ import pytest
from unittest.mock import MagicMock, patch
from parameterized import parameterized
-from requests import Response
+from requests import Response, Session
+from requests.exceptions import ConnectionError as RequestsConnectionError
+from urllib3 import HTTPResponse
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import BearerTokenAuth
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
JSONResponsePaginator,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClient
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (
+ RESTClient,
+ RESTClientNonRetryableError,
+ RESTClientRetryableError,
+)
MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client"
@@ -41,6 +50,35 @@ def _session_echoing_url(MockSession) -> MagicMock:
return mock_session
+S3_PATTERN = r"[a-z0-9.-]+\.s3\.[a-z0-9-]+\.amazonaws\.com"
+STORAGE_URL = "https://bucket.s3.us-east-1.amazonaws.com/responses/page.json.gz?X-Amz-Signature=sig"
+
+
+def _storage_response(body: bytes, status: int = 200) -> Response:
+ resp = Response()
+ resp.status_code = status
+ resp.raw = HTTPResponse(body=io.BytesIO(body), status=status, preload_content=False)
+ return resp
+
+
+def _redirect_to_storage() -> Response:
+ return _make_response({}, status_code=303, headers={"Location": STORAGE_URL})
+
+
+def _presigned_client() -> tuple[Session, RESTClient]:
+ session = Session()
+ client = RESTClient(
+ base_url="https://api.example.com",
+ auth=BearerTokenAuth("api-secret"),
+ headers={"X-Api-Key": "header-secret"},
+ session=session,
+ max_retry_attempts=1,
+ allow_redirects=False,
+ presigned_redirect_host_pattern=S3_PATTERN,
+ )
+ return session, client
+
+
class TestSSRFHostPinning:
@parameterized.expand(
[
@@ -110,16 +148,35 @@ class TestSSRFHostPinning:
pages = list(pager)
assert [row["id"] for page in pages for row in page] == [1, 2]
+ @parameterized.expand(
+ [
+ ("redirects_disabled", None, "https://evil.com/steal"),
+ (
+ "host_outside_pattern",
+ S3_PATTERN,
+ "https://evil.com/a.s3.us-east-1.amazonaws.com?X-Amz-Signature=sekret",
+ ),
+ ("non_s3_aws_host", S3_PATTERN, "https://ec2-1-2-3-4.compute-1.amazonaws.com/o?X-Amz-Signature=sekret"),
+ ("presigned_host_over_http", S3_PATTERN, "http://b.s3.us-east-1.amazonaws.com/o?X-Amz-Signature=sekret"),
+ ("scheme_relative_location", S3_PATTERN, "//b.s3.us-east-1.amazonaws.com/o?X-Amz-Signature=sekret"),
+ ]
+ )
@patch(f"{MODULE}.make_tracked_session")
- def test_redirect_rejected_when_disabled(self, MockSession) -> None:
+ def test_redirect_rejected_when_disabled(
+ self, _name: str, presigned_pattern: Optional[str], location: str, MockSession
+ ) -> None:
mock_session = _session_echoing_url(MockSession)
- mock_session.send.return_value = _make_response(
- {}, status_code=302, headers={"Location": "https://evil.com/steal"}
- )
+ mock_session.send.return_value = _make_response({}, status_code=303, headers={"Location": location})
- client = RESTClient(base_url="https://api.example.com", allow_redirects=False)
- with pytest.raises(ValueError, match="Unexpected redirect"):
+ client = RESTClient(
+ base_url="https://api.example.com",
+ allow_redirects=False,
+ presigned_redirect_host_pattern=presigned_pattern,
+ )
+ with pytest.raises(ValueError, match="Unexpected redirect") as exc_info:
list(client.paginate(path="/items", data_selector="results"))
+ assert "sekret" not in str(exc_info.value)
+ assert mock_session.send.call_count == 1
# The redirect target was never followed: send ran once and got no allow_redirects.
_, kwargs = mock_session.send.call_args
@@ -135,3 +192,52 @@ class TestSSRFHostPinning:
_, kwargs = mock_session.send.call_args
assert kwargs.get("allow_redirects") is True
+
+ @parameterized.expand([("plain_body", False), ("gzipped_body", True)])
+ def test_presigned_redirect_fetched_without_credentials(self, _name: str, gzipped: bool) -> None:
+ body = json.dumps({"results": [{"id": 1}]}).encode()
+ session, client = _presigned_client()
+
+ with patch.object(
+ session,
+ "send",
+ side_effect=[_redirect_to_storage(), _storage_response(gzip.compress(body) if gzipped else body)],
+ ) as send:
+ pages = list(client.paginate(path="/items", method="post", data_selector="results"))
+
+ assert pages == [[{"id": 1}]]
+ followed = send.call_args_list[1].args[0]
+ assert (followed.method, followed.url) == ("GET", STORAGE_URL)
+ assert "Authorization" not in followed.headers
+ assert "X-Api-Key" not in followed.headers
+
+ @parameterized.expand(
+ [
+ ("expired_url", _storage_response(b"", status=403), RESTClientRetryableError),
+ (
+ "connection_error",
+ RequestsConnectionError(f"Max retries exceeded with url: {STORAGE_URL}"),
+ RESTClientRetryableError,
+ ),
+ ("corrupt_gzip", _storage_response(b"\x1f\x8b" + b"\x00" * 20), RESTClientRetryableError),
+ (
+ "truncated_gzip",
+ _storage_response(gzip.compress(b'{"results": []}' * 100)[:30]),
+ RESTClientRetryableError,
+ ),
+ ("gzip_bomb", _storage_response(gzip.compress(b"\x00" * 4096)), RESTClientNonRetryableError),
+ ("oversized_body", _storage_response(b'{"results": []}' + b" " * 4096), RESTClientNonRetryableError),
+ ]
+ )
+ @patch(f"{MODULE}.MAX_PRESIGNED_RESPONSE_BYTES", 1024)
+ def test_presigned_fetch_failure_hides_signed_url(
+ self, _name: str, storage_outcome: Any, expected: type[Exception]
+ ) -> None:
+ session, client = _presigned_client()
+
+ with patch.object(session, "send", side_effect=[_redirect_to_storage(), storage_outcome]):
+ with pytest.raises(expected) as exc_info:
+ list(client.paginate(path="/items", method="post", data_selector="results"))
+
+ assert "X-Amz-Signature" not in str(exc_info.value)
+ assert exc_info.value.__cause__ is None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py
index df9983c64c31..ab8d6d0603c0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py
@@ -229,6 +229,10 @@ class ClientConfig(TypedDict, total=False):
# When False, redirects are not followed and any 3xx is rejected — closes the redirect-based
# off-host escape that host-pinning alone would miss. Defaults to True (follow redirects).
allow_redirects: bool
+ # Regex the host of a vendor's presigned storage must fully match. With redirects disabled, a
+ # 302/303 to an HTTPS host matching it is fetched with a bare GET that carries no credentials,
+ # and its body (gunzipped when needed, capped in size) is read as the page.
+ presigned_redirect_host_pattern: Optional[str]
# Per-request (connect, read) timeout in seconds. Left unset, the tracked session applies
# `default_request_timeout()`. A source pointed at a customer-controlled host should set a
# tighter value. A single float applies to both connect and read; a tuple sets them separately.