Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): reject null or missing required paginator keys (#128007)

Tom Owers4cbfa9df4cdf
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py
index 54b3f35989f0..e4648f1e6f24 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py
@@ -476,12 +476,17 @@ def _validate_paginator_configs(manifest: dict[str, Any]) -> None:
if isinstance(client, dict):
_check_paginator_config(client.get("paginator"), "client.paginator")
+ defaults = manifest.get("resource_defaults")
+ default_endpoint = defaults.get("endpoint") if isinstance(defaults, dict) else None
+ default_paginator = default_endpoint.get("paginator") if isinstance(default_endpoint, dict) else None
+
for resource in manifest.get("resources") or []:
- if not isinstance(resource, dict):
- continue
- endpoint = resource.get("endpoint")
- paginator = endpoint.get("paginator") if isinstance(endpoint, dict) else None
- _check_paginator_config(paginator, f"Resource {resource.get('name')!r}: endpoint.paginator")
+ endpoint = resource.get("endpoint") if isinstance(resource, dict) else None
+ # The engine replaces the default paginator wholesale when a resource declares its own.
+ if isinstance(endpoint, dict) and "paginator" in endpoint:
+ _check_paginator_config(endpoint["paginator"], f"Resource {resource.get('name')!r}: endpoint.paginator")
+ else:
+ _check_paginator_config(default_paginator, "resource_defaults.endpoint.paginator")
def _check_paginator_config(paginator: Any, location: str) -> None:
@@ -496,16 +501,25 @@ def _check_paginator_config(paginator: Any, location: str) -> None:
# "auto" (and any type mapped to None) applies no paginator, so extra keys are ignored.
if paginator_class is None:
return
- supported = {
- name
+ params = {
+ name: param
for name, param in inspect.signature(paginator_class).parameters.items()
if param.kind in (param.POSITIONAL_OR_KEYWORD, param.KEYWORD_ONLY)
}
- unsupported = sorted(set(paginator) - {"type"} - supported)
+ unsupported = sorted(set(paginator) - {"type"} - set(params))
if unsupported:
raise ManifestValidationError(
f"{location} has unsupported {'keys' if len(unsupported) > 1 else 'key'} "
- f"{', '.join(unsupported)}. Allowed keys: {', '.join(sorted(supported))}"
+ f"{', '.join(unsupported)}. Allowed keys: {', '.join(sorted(params))}"
+ )
+ # A missing required key fails the constructor with a retryable TypeError, and a null one
+ # (e.g. offset `limit`) only fails once pagination does arithmetic on it.
+ missing = sorted(
+ name for name, param in params.items() if param.default is param.empty and paginator.get(name) is None
+ )
+ if missing:
+ raise ManifestValidationError(
+ f"{location} requires a non-null value for {', '.join(missing)} when type is {paginator.get('type')!r}"
)
@@ -1409,7 +1423,13 @@ class CustomSource(SimpleSource[CustomSourceConfig]):
# endpoint.incremental block missing start_param crashes the engine with a bare,
# retryable KeyError. Reject it as a ValueError so it fails fast and non-retryably.
_validate_incremental_configs({"resources": engine_resources})
- _validate_paginator_configs({"client": manifest.get("client"), "resources": engine_resources})
+ _validate_paginator_configs(
+ {
+ "client": manifest.get("client"),
+ "resource_defaults": manifest.get("resource_defaults"),
+ "resources": engine_resources,
+ }
+ )
# The engine serializes a datetime watermark via str() (space-separated),
# which strict APIs reject — format it to the declared wire format first.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py
index 10f3b0efbcc9..78874e690fa4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py
@@ -2769,21 +2769,29 @@ class TestCustomSourceIncrementalUnsupportedKeys(SimpleTestCase):
assert err is not None and "resource_defaults" in err
-class TestCustomSourcePaginatorUnsupportedKeys(SimpleTestCase):
- def _manifest(self) -> dict:
+_INVALID_PAGINATOR_CASES = [
+ ("unsupported_key", {"type": "offset", "limit": 100, "limit_body_path": "meta.limit"}, False, "limit_body_path"),
+ ("null_required_key", {"type": "offset", "limit": None}, False, "non-null value for limit"),
+ ("missing_required_key", {"type": "offset"}, False, "non-null value for limit"),
+ ("inherited_null_required_key", {"type": "offset", "limit": None}, True, "non-null value for limit"),
+]
+
+
+class TestCustomSourcePaginatorInvalidConfig(SimpleTestCase):
+ def _manifest(self, paginator: dict, in_defaults: bool) -> dict:
manifest = _minimal_manifest()
- manifest["resources"][0]["endpoint"]["paginator"] = {
- "type": "offset",
- "limit": 100,
- "limit_body_path": "meta.limit",
- }
+ if in_defaults:
+ manifest["resource_defaults"] = {"endpoint": {"paginator": paginator}}
+ else:
+ manifest["resources"][0]["endpoint"]["paginator"] = paginator
return manifest
+ @parameterized.expand(_INVALID_PAGINATOR_CASES)
@patch("products.warehouse_sources.backend.temporal.data_imports.sources.custom.source.rest_api_resources")
- def test_unsupported_key_raises_non_retryable_before_engine(self, mock_resources):
+ def test_raises_non_retryable_before_engine(self, _name, paginator, in_defaults, expected, mock_resources):
mock_resources.return_value = [_fake_resource("users")]
source = CustomSource()
- config = CustomSourceConfig(manifest_json=json.dumps(self._manifest()))
+ config = CustomSourceConfig(manifest_json=json.dumps(self._manifest(paginator, in_defaults)))
inputs = MagicMock(
team_id=1,
schema_name="users",
@@ -2793,15 +2801,34 @@ class TestCustomSourcePaginatorUnsupportedKeys(SimpleTestCase):
)
with self.assertRaises(NonRetryableException) as ctx:
source.source_for_pipeline(config, inputs)
- assert "limit_body_path" in str(ctx.exception)
+ assert expected in str(ctx.exception)
mock_resources.assert_not_called()
- def test_unsupported_key_rejected_at_validation(self):
+ @parameterized.expand(_INVALID_PAGINATOR_CASES)
+ def test_rejected_at_validation(self, _name, paginator, in_defaults, expected):
source = CustomSource()
- config = CustomSourceConfig(manifest_json=json.dumps(self._manifest()), auth_token="abc")
+ config = CustomSourceConfig(manifest_json=json.dumps(self._manifest(paginator, in_defaults)), auth_token="abc")
ok, err = source.validate_credentials(config, team_id=999)
assert ok is False
- assert err is not None and "limit_body_path" in err and "'users'" in err
+ location = "resource_defaults.endpoint.paginator" if in_defaults else "'users'"
+ assert err is not None and expected in err and location in err
+
+ @patch("products.warehouse_sources.backend.temporal.data_imports.sources.custom.source.rest_api_resources")
+ def test_overridden_default_paginator_is_not_checked(self, mock_resources):
+ mock_resources.return_value = [_fake_resource("users")]
+ manifest = self._manifest({"type": "offset", "limit": None}, in_defaults=True)
+ manifest["resources"][0]["endpoint"]["paginator"] = {"type": "offset", "limit": 100}
+ source = CustomSource()
+ config = CustomSourceConfig(manifest_json=json.dumps(manifest))
+ inputs = MagicMock(
+ team_id=1,
+ schema_name="users",
+ job_id="job-1",
+ should_use_incremental_field=False,
+ db_incremental_field_last_value=None,
+ )
+ source.source_for_pipeline(config, inputs)
+ mock_resources.assert_called_once()
def _apikey_manifest() -> dict: