Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): treat an empty 2xx body as zero rows for required list endpoints (#127155)

Tom Owersb2f28a3d90cf
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py
index dfcb53919bf0..f5cf7fe62e5c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py
@@ -86,6 +86,10 @@ class RESTClientNonRetryableError(NonReportableError):
_JSON_START_BYTES = frozenset(b'{["-tfn0123456789')
+def _is_empty_body(response: Response) -> bool:
+ return not response.content or not response.content.strip()
+
+
def _looks_like_json(content: bytes) -> bool:
stripped = content.lstrip()
return bool(stripped) and stripped[0] in _JSON_START_BYTES
@@ -653,9 +657,14 @@ class RESTClient:
except IgnoreResponseException:
break
- data = self._extract_response(
- body, data_selector, required=data_selector_required, empty_body_ok=data_selector_empty_ok
- )
+ # An empty 2xx body is a complete "no data" answer, not a changed response shape, so
+ # ``required`` must not reject it. A literal JSON ``null`` body still fails loud.
+ if body is None and _is_empty_body(response):
+ data: list[Any] = []
+ else:
+ data = self._extract_response(
+ body, data_selector, required=data_selector_required, empty_body_ok=data_selector_empty_ok
+ )
if paginator is not None:
paginator.update_state(response, data)
@@ -821,7 +830,7 @@ class RESTClient:
# answer (e.g. an endpoint with nothing to return), not a truncated page —
# retrying can't conjure rows, so treat it as an empty page and let the
# paginator stop.
- if not response.content or not response.content.strip():
+ if _is_empty_body(response):
return response, None
# A body that doesn't even begin with a JSON value is not a truncated page:
# the endpoint returned non-JSON content (an HTML or plain-text error page, a
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_required_selector.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_required_selector.py
index bbf0d4252ad1..de2e1f80efb7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_required_selector.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_required_selector.py
@@ -1,10 +1,16 @@
import pytest
-from unittest.mock import Mock
+from unittest.mock import MagicMock, Mock, patch
from parameterized import parameterized
+from requests import Response
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
+ SinglePagePaginator,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClient
+MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client"
+
def _client() -> RESTClient:
return RESTClient(base_url="https://api.example.com", session=Mock())
@@ -52,3 +58,37 @@ class TestRequiredListBody:
def test_list_body_ok_when_required_and_no_selector(self) -> None:
assert _client()._extract_response([{"id": 1}], None, required=True) == [{"id": 1}]
+
+
+class TestRequiredEmptyHttpBody:
+ @parameterized.expand(
+ [
+ ("200_no_selector", 200, None),
+ ("200_selector", 200, "data"),
+ ("204_no_selector", 204, None),
+ ("204_selector", 204, "data"),
+ ]
+ )
+ @patch(f"{MODULE}.make_tracked_session")
+ def test_empty_body_is_zero_rows_when_required(
+ self, _name: str, status: int, data_selector: str | None, MockSession: MagicMock
+ ) -> None:
+ response = Response()
+ response.status_code = status
+ response._content = b""
+ response.url = "https://api.example.com/items"
+ session = MockSession.return_value
+ session.headers = {}
+ session.prepare_request.return_value = MagicMock()
+ session.send.return_value = response
+
+ pages = list(
+ RESTClient(base_url="https://api.example.com").paginate(
+ path="/items",
+ data_selector=data_selector,
+ paginator=SinglePagePaginator(),
+ data_selector_required=True,
+ )
+ )
+
+ assert pages == [[]]