:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): reject null or missing required paginator keys (#128007)
Tom Owers4cbfa9df4cdf
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.pyindex 54b3f35989f0..e4648f1e6f24 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/source.py@@ -476,12 +476,17 @@ def _validate_paginator_configs(manifest: dict[str, Any]) -> None:if isinstance(client, dict):_check_paginator_config(client.get("paginator"), "client.paginator")+ defaults = manifest.get("resource_defaults")+ default_endpoint = defaults.get("endpoint") if isinstance(defaults, dict) else None+ default_paginator = default_endpoint.get("paginator") if isinstance(default_endpoint, dict) else None+for resource in manifest.get("resources") or []:- if not isinstance(resource, dict):- continue- endpoint = resource.get("endpoint")- paginator = endpoint.get("paginator") if isinstance(endpoint, dict) else None- _check_paginator_config(paginator, f"Resource {resource.get('name')!r}: endpoint.paginator")+ endpoint = resource.get("endpoint") if isinstance(resource, dict) else None+ # The engine replaces the default paginator wholesale when a resource declares its own.+ if isinstance(endpoint, dict) and "paginator" in endpoint:+ _check_paginator_config(endpoint["paginator"], f"Resource {resource.get('name')!r}: endpoint.paginator")+ else:+ _check_paginator_config(default_paginator, "resource_defaults.endpoint.paginator")def _check_paginator_config(paginator: Any, location: str) -> None:@@ -496,16 +501,25 @@ def _check_paginator_config(paginator: Any, location: str) -> None:# "auto" (and any type mapped to None) applies no paginator, so extra keys are ignored.if paginator_class is None:return- supported = {- name+ params = {+ name: paramfor name, param in inspect.signature(paginator_class).parameters.items()if param.kind in (param.POSITIONAL_OR_KEYWORD, param.KEYWORD_ONLY)}- unsupported = sorted(set(paginator) - {"type"} - supported)+ unsupported = sorted(set(paginator) - {"type"} - set(params))if unsupported:raise ManifestValidationError(f"{location} has unsupported {'keys' if len(unsupported) > 1 else 'key'} "- f"{', '.join(unsupported)}. Allowed keys: {', '.join(sorted(supported))}"+ f"{', '.join(unsupported)}. Allowed keys: {', '.join(sorted(params))}"+ )+ # A missing required key fails the constructor with a retryable TypeError, and a null one+ # (e.g. offset `limit`) only fails once pagination does arithmetic on it.+ missing = sorted(+ name for name, param in params.items() if param.default is param.empty and paginator.get(name) is None+ )+ if missing:+ raise ManifestValidationError(+ f"{location} requires a non-null value for {', '.join(missing)} when type is {paginator.get('type')!r}")@@ -1409,7 +1423,13 @@ class CustomSource(SimpleSource[CustomSourceConfig]):# endpoint.incremental block missing start_param crashes the engine with a bare,# retryable KeyError. Reject it as a ValueError so it fails fast and non-retryably._validate_incremental_configs({"resources": engine_resources})- _validate_paginator_configs({"client": manifest.get("client"), "resources": engine_resources})+ _validate_paginator_configs(+ {+ "client": manifest.get("client"),+ "resource_defaults": manifest.get("resource_defaults"),+ "resources": engine_resources,+ }+ )# The engine serializes a datetime watermark via str() (space-separated),# which strict APIs reject — format it to the declared wire format first.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.pyindex 10f3b0efbcc9..78874e690fa4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py@@ -2769,21 +2769,29 @@ class TestCustomSourceIncrementalUnsupportedKeys(SimpleTestCase):assert err is not None and "resource_defaults" in err-class TestCustomSourcePaginatorUnsupportedKeys(SimpleTestCase):- def _manifest(self) -> dict:+_INVALID_PAGINATOR_CASES = [+ ("unsupported_key", {"type": "offset", "limit": 100, "limit_body_path": "meta.limit"}, False, "limit_body_path"),+ ("null_required_key", {"type": "offset", "limit": None}, False, "non-null value for limit"),+ ("missing_required_key", {"type": "offset"}, False, "non-null value for limit"),+ ("inherited_null_required_key", {"type": "offset", "limit": None}, True, "non-null value for limit"),+]+++class TestCustomSourcePaginatorInvalidConfig(SimpleTestCase):+ def _manifest(self, paginator: dict, in_defaults: bool) -> dict:manifest = _minimal_manifest()- manifest["resources"][0]["endpoint"]["paginator"] = {- "type": "offset",- "limit": 100,- "limit_body_path": "meta.limit",- }+ if in_defaults:+ manifest["resource_defaults"] = {"endpoint": {"paginator": paginator}}+ else:+ manifest["resources"][0]["endpoint"]["paginator"] = paginatorreturn manifest+ @parameterized.expand(_INVALID_PAGINATOR_CASES)@patch("products.warehouse_sources.backend.temporal.data_imports.sources.custom.source.rest_api_resources")- def test_unsupported_key_raises_non_retryable_before_engine(self, mock_resources):+ def test_raises_non_retryable_before_engine(self, _name, paginator, in_defaults, expected, mock_resources):mock_resources.return_value = [_fake_resource("users")]source = CustomSource()- config = CustomSourceConfig(manifest_json=json.dumps(self._manifest()))+ config = CustomSourceConfig(manifest_json=json.dumps(self._manifest(paginator, in_defaults)))inputs = MagicMock(team_id=1,schema_name="users",@@ -2793,15 +2801,34 @@ class TestCustomSourcePaginatorUnsupportedKeys(SimpleTestCase):)with self.assertRaises(NonRetryableException) as ctx:source.source_for_pipeline(config, inputs)- assert "limit_body_path" in str(ctx.exception)+ assert expected in str(ctx.exception)mock_resources.assert_not_called()- def test_unsupported_key_rejected_at_validation(self):+ @parameterized.expand(_INVALID_PAGINATOR_CASES)+ def test_rejected_at_validation(self, _name, paginator, in_defaults, expected):source = CustomSource()- config = CustomSourceConfig(manifest_json=json.dumps(self._manifest()), auth_token="abc")+ config = CustomSourceConfig(manifest_json=json.dumps(self._manifest(paginator, in_defaults)), auth_token="abc")ok, err = source.validate_credentials(config, team_id=999)assert ok is False- assert err is not None and "limit_body_path" in err and "'users'" in err+ location = "resource_defaults.endpoint.paginator" if in_defaults else "'users'"+ assert err is not None and expected in err and location in err++ @patch("products.warehouse_sources.backend.temporal.data_imports.sources.custom.source.rest_api_resources")+ def test_overridden_default_paginator_is_not_checked(self, mock_resources):+ mock_resources.return_value = [_fake_resource("users")]+ manifest = self._manifest({"type": "offset", "limit": None}, in_defaults=True)+ manifest["resources"][0]["endpoint"]["paginator"] = {"type": "offset", "limit": 100}+ source = CustomSource()+ config = CustomSourceConfig(manifest_json=json.dumps(manifest))+ inputs = MagicMock(+ team_id=1,+ schema_name="users",+ job_id="job-1",+ should_use_incremental_field=False,+ db_incremental_field_last_value=None,+ )+ source.source_for_pipeline(config, inputs)+ mock_resources.assert_called_once()def _apikey_manifest() -> dict: