:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): retry cloudinary's non-standard 420 rate limit (#126859)
Tom Owersbbf3d3674248
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/cloudinary.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/cloudinary.pyindex 2f2dd1646d48..028d1ea4672f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/cloudinary.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/cloudinary.py@@ -15,6 +15,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.cloudinaryfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_sessionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (RESTAPIConfig,+ create_response_hooks,rest_api_resource,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import HttpBasicAuth@@ -23,7 +24,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resSinglePagePaginator,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClient-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import EndpointResource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (+ EndpointResource,+ ResponseAction,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse@@ -34,6 +38,12 @@ CLOUDINARY_API_HOSTS = {}REQUEST_TIMEOUT_SECONDS = 30+# Cloudinary rate-limits the Admin API and returns a 420 ("Enhance Your Calm") rather than the+# standard 429 — the same non-standard code `validate_credentials` below already recognizes. The+# shared client's own retry check only fires on 429/5xx, so 420 is promoted to retryable here.+_RETRY_ACTIONS: list[ResponseAction] = [{"status_code": 420, "action": "retry"}]+_RETRY_HOOKS = create_response_hooks(_RETRY_ACTIONS)+@dataclasses.dataclass(frozen=True)class CloudinaryResumeConfig:@@ -64,6 +74,7 @@ def get_resource(endpoint: str) -> EndpointResource:"data_selector": f"{config.data_key}[*]","path": config.path,"params": dict(config.params),+ "response_actions": _RETRY_ACTIONS,},"table_format": "delta",}@@ -104,6 +115,7 @@ def iter_usage(path=CLOUDINARY_ENDPOINTS["usage"].path.format(date=day.isoformat()),paginator=SinglePagePaginator(),data_selector="$",+ hooks=_RETRY_HOOKS,):rows = [{**row, "date": day.isoformat()} for row in page]manager.save_state(CloudinaryResumeConfig(cursor=(day + timedelta(days=1)).isoformat()))@@ -123,7 +135,10 @@ def iter_subfolders(cursor = resume.cursor or Noneelse:for page in client.paginate(- path=CLOUDINARY_ENDPOINTS["folders"].path, params={"max_results": MAX_RESULTS}, data_selector="folders"+ path=CLOUDINARY_ENDPOINTS["folders"].path,+ params={"max_results": MAX_RESULTS},+ data_selector="folders",+ hooks=_RETRY_HOOKS,):pending.extend(folder["path"] for folder in page)manager.safe_point()@@ -143,6 +158,7 @@ def iter_subfolders(data_selector="folders",initial_paginator_state={"cursor": cursor} if cursor else None,page_state_hook=remember_cursor,+ hooks=_RETRY_HOOKS,):pending.extend(folder["path"] for folder in page)if next_cursor["value"]:@@ -175,7 +191,10 @@ def iter_last_access_report_resources() -> Iterator[list[dict[str, Any]]]:report_ids: list[str] = []for page in client.paginate(- path=LAST_ACCESS_REPORTS_PATH, params={"max_results": MAX_RESULTS}, data_selector="reports"+ path=LAST_ACCESS_REPORTS_PATH,+ params={"max_results": MAX_RESULTS},+ data_selector="reports",+ hooks=_RETRY_HOOKS,):# A report still being generated has no resources to list yet.report_ids.extend(report["id"] for report in page if report.get("status") == "done")@@ -197,6 +216,7 @@ def iter_last_access_report_resources(data_selector="resources",initial_paginator_state={"cursor": cursor} if cursor else None,page_state_hook=_report_state_hook(manager, report_id, next_report_id),+ hooks=_RETRY_HOOKS,):if page:yield [{**row, "report_id": report_id} for row in page]diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/tests/test_cloudinary.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/tests/test_cloudinary.pyindex d78dbe27fce0..fffa171b6fbd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/tests/test_cloudinary.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/tests/test_cloudinary.py@@ -46,6 +46,17 @@ class TestCloudinarySource:assert result.raised is Noneassert result.params("next_cursor") == ["page-2"]+ def test_a_rate_limit_is_retried_rather_than_failing_the_sync(self) -> None:+ # Cloudinary rate-limits with a 420 ("Enhance Your Calm") instead of the standard 429.+ result = _driver().run(+ "images",+ [ScriptedResponse(status=420), ScriptedResponse(json={"resources": [{"asset_id": "one"}]})],+ )++ assert result.raised is None+ assert [row["asset_id"] for row in result.rows] == ["one"]+ assert len(result.requests) == 2+def test_only_a_real_cursor_is_checkpointed(self) -> None:without_cursor = _driver().run("images", [ScriptedResponse(json={"resources": [{"asset_id": "one"}]})])with_cursor = _driver().run(@@ -84,6 +95,21 @@ class TestCloudinarySubfolders:CloudinaryResumeConfig(cursor="", pending_folders=["shop/red shoes", "shop/hats"]),]+ def test_a_rate_limit_is_retried_rather_than_failing_the_sync(self) -> None:+ # The custom folder-walk endpoints pass their own response hooks to `client.paginate`,+ # a separate wiring point from the declarative `response_actions` the REST resource path uses.+ result = _driver().run(+ "subfolders",+ [+ ScriptedResponse(status=420),+ ScriptedResponse(json={"folders": [{"path": "shop"}]}),+ ScriptedResponse(json={"folders": []}),+ ],+ )++ assert result.raised is None+ assert len(result.requests) == 3+def test_a_resume_continues_the_saved_folder_without_relisting_the_root(self) -> None:def answer(request: RecordedRequest) -> ScriptedResponse:pages: dict[tuple[str, str | None], dict[str, Any]] = {