Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): retry cloudinary's non-standard 420 rate limit (#126859)

Tom Owersbbf3d3674248
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/cloudinary.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/cloudinary.py
index 2f2dd1646d48..028d1ea4672f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/cloudinary.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/cloudinary.py
@@ -15,6 +15,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.cloudinary
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
+ create_response_hooks,
rest_api_resource,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import HttpBasicAuth
@@ -23,7 +24,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
SinglePagePaginator,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClient
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import EndpointResource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
+ EndpointResource,
+ ResponseAction,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -34,6 +38,12 @@ CLOUDINARY_API_HOSTS = {
}
REQUEST_TIMEOUT_SECONDS = 30
+# Cloudinary rate-limits the Admin API and returns a 420 ("Enhance Your Calm") rather than the
+# standard 429 — the same non-standard code `validate_credentials` below already recognizes. The
+# shared client's own retry check only fires on 429/5xx, so 420 is promoted to retryable here.
+_RETRY_ACTIONS: list[ResponseAction] = [{"status_code": 420, "action": "retry"}]
+_RETRY_HOOKS = create_response_hooks(_RETRY_ACTIONS)
+
@dataclasses.dataclass(frozen=True)
class CloudinaryResumeConfig:
@@ -64,6 +74,7 @@ def get_resource(endpoint: str) -> EndpointResource:
"data_selector": f"{config.data_key}[*]",
"path": config.path,
"params": dict(config.params),
+ "response_actions": _RETRY_ACTIONS,
},
"table_format": "delta",
}
@@ -104,6 +115,7 @@ def iter_usage(
path=CLOUDINARY_ENDPOINTS["usage"].path.format(date=day.isoformat()),
paginator=SinglePagePaginator(),
data_selector="$",
+ hooks=_RETRY_HOOKS,
):
rows = [{**row, "date": day.isoformat()} for row in page]
manager.save_state(CloudinaryResumeConfig(cursor=(day + timedelta(days=1)).isoformat()))
@@ -123,7 +135,10 @@ def iter_subfolders(
cursor = resume.cursor or None
else:
for page in client.paginate(
- path=CLOUDINARY_ENDPOINTS["folders"].path, params={"max_results": MAX_RESULTS}, data_selector="folders"
+ path=CLOUDINARY_ENDPOINTS["folders"].path,
+ params={"max_results": MAX_RESULTS},
+ data_selector="folders",
+ hooks=_RETRY_HOOKS,
):
pending.extend(folder["path"] for folder in page)
manager.safe_point()
@@ -143,6 +158,7 @@ def iter_subfolders(
data_selector="folders",
initial_paginator_state={"cursor": cursor} if cursor else None,
page_state_hook=remember_cursor,
+ hooks=_RETRY_HOOKS,
):
pending.extend(folder["path"] for folder in page)
if next_cursor["value"]:
@@ -175,7 +191,10 @@ def iter_last_access_report_resources(
) -> Iterator[list[dict[str, Any]]]:
report_ids: list[str] = []
for page in client.paginate(
- path=LAST_ACCESS_REPORTS_PATH, params={"max_results": MAX_RESULTS}, data_selector="reports"
+ path=LAST_ACCESS_REPORTS_PATH,
+ params={"max_results": MAX_RESULTS},
+ data_selector="reports",
+ hooks=_RETRY_HOOKS,
):
# A report still being generated has no resources to list yet.
report_ids.extend(report["id"] for report in page if report.get("status") == "done")
@@ -197,6 +216,7 @@ def iter_last_access_report_resources(
data_selector="resources",
initial_paginator_state={"cursor": cursor} if cursor else None,
page_state_hook=_report_state_hook(manager, report_id, next_report_id),
+ hooks=_RETRY_HOOKS,
):
if page:
yield [{**row, "report_id": report_id} for row in page]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/tests/test_cloudinary.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/tests/test_cloudinary.py
index d78dbe27fce0..fffa171b6fbd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/tests/test_cloudinary.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/tests/test_cloudinary.py
@@ -46,6 +46,17 @@ class TestCloudinarySource:
assert result.raised is None
assert result.params("next_cursor") == ["page-2"]
+ def test_a_rate_limit_is_retried_rather_than_failing_the_sync(self) -> None:
+ # Cloudinary rate-limits with a 420 ("Enhance Your Calm") instead of the standard 429.
+ result = _driver().run(
+ "images",
+ [ScriptedResponse(status=420), ScriptedResponse(json={"resources": [{"asset_id": "one"}]})],
+ )
+
+ assert result.raised is None
+ assert [row["asset_id"] for row in result.rows] == ["one"]
+ assert len(result.requests) == 2
+
def test_only_a_real_cursor_is_checkpointed(self) -> None:
without_cursor = _driver().run("images", [ScriptedResponse(json={"resources": [{"asset_id": "one"}]})])
with_cursor = _driver().run(
@@ -84,6 +95,21 @@ class TestCloudinarySubfolders:
CloudinaryResumeConfig(cursor="", pending_folders=["shop/red shoes", "shop/hats"]),
]
+ def test_a_rate_limit_is_retried_rather_than_failing_the_sync(self) -> None:
+ # The custom folder-walk endpoints pass their own response hooks to `client.paginate`,
+ # a separate wiring point from the declarative `response_actions` the REST resource path uses.
+ result = _driver().run(
+ "subfolders",
+ [
+ ScriptedResponse(status=420),
+ ScriptedResponse(json={"folders": [{"path": "shop"}]}),
+ ScriptedResponse(json={"folders": []}),
+ ],
+ )
+
+ assert result.raised is None
+ assert len(result.requests) == 3
+
def test_a_resume_continues_the_saved_folder_without_relisting_the_root(self) -> None:
def answer(request: RecordedRequest) -> ScriptedResponse:
pages: dict[tuple[str, str | None], dict[str, Any]] = {