Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add cloudinary usage, subfolders, and last access tables (#119252)

Tom Owersb031b5d8c6d3
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index adac34f8bbea..37ead4d7ae38 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -2823,10 +2823,10 @@ Today (6): `folders`, `images`, `raw_files`, `transformations`, `upload_presets`
Diffed against: <https://cloudinary.com/documentation/admin_api.md>
-- [ ] `Usage (GET /v1_1/{cloud_name}/usage)` — measure bandwidth, storage, transformations, requests, and credits against account limits (high)
-- [ ] `Derived assets (GET /v1_1/{cloud_name}/resources/{asset_id})` — measure storage used by transformed variants and connect each variant to its original asset (medium)
-- [ ] `Last access report resources (GET /v1_1/{cloud_name}/resources/last_access_report/{report_id})` — identify unused media and estimate storage savings from asset retention policies (medium)
-- [ ] `Subfolders (GET /v1_1/{cloud_name}/folders/{folder_path})` — complete the folder hierarchy for asset organization and reporting below the root (medium)
+- [x] `Usage (GET /v1_1/{cloud_name}/usage)` — measure bandwidth, storage, transformations, requests, and credits against account limits (high). Added as `usage`, one row per day for the last 89 days, incremental on `date`.
+- ~~`Derived assets (GET /v1_1/{cloud_name}/resources/{asset_id})`~~ — skipped: derived assets only come from the per-asset detail call, so a sync would cost one Admin API request per asset against an hourly limit of 500 to 2,000 requests
+- [x] `Last access report resources (GET /v1_1/{cloud_name}/resources/last_access_report/{report_id})` — identify unused media and estimate storage savings from asset retention policies (medium). Added as `last_access_report_resources` (fan-out over completed reports from `GET /resources_last_access_reports`).
+- [x] `Subfolders (GET /v1_1/{cloud_name}/folders/{folder_path})` — complete the folder hierarchy for asset organization and reporting below the root (medium). Added as `subfolders` (recursive walk from the root folders).
Note: The official Admin API index, linked reference pages, and Python SDK GET methods were compared with the source settings. Derived assets require requests per asset; subfolders require recursive enumeration. Last access reports need existing report IDs and an enabled Enterprise feature. Configuration and filtered views of existing asset lists were excluded.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/canonical_descriptions.py
index b00fd41e46ea..9c4f90bf2c23 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/canonical_descriptions.py
@@ -48,6 +48,45 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"external_id": "Cloudinary's identifier for the folder.",
},
},
+ "subfolders": {
+ "description": "A folder below the root of the Cloudinary media library, at any depth.",
+ "docs_url": "https://cloudinary.com/documentation/admin_api#get_subfolders",
+ "columns": {
+ "name": "Folder name.",
+ "path": "Full path of the folder, including its parents.",
+ "external_id": "Cloudinary's identifier for the folder.",
+ },
+ },
+ "last_access_report_resources": {
+ "description": "An asset listed in a completed last access report, with the last time it was delivered.",
+ "docs_url": "https://cloudinary.com/documentation/admin_api#get_resources_in_a_last_access_report",
+ "columns": {
+ **_ASSET_COLUMNS,
+ "report_id": "ID of the last access report that lists the asset.",
+ "asset_folder": "Folder the asset sits in.",
+ "display_name": "Name shown for the asset in the Media Library.",
+ "last_access": "Date and time the asset was last delivered. Empty when it was never delivered.",
+ },
+ },
+ "usage": {
+ "description": "Account usage for one day: storage, bandwidth, transformations, requests, credits, and add-ons against plan limits.",
+ "docs_url": "https://cloudinary.com/documentation/admin_api#get_product_environment_usage_details",
+ "columns": {
+ "date": "Day the usage figures were requested for.",
+ "plan": "Cloudinary plan the account was on.",
+ "last_updated": "Date Cloudinary last refreshed the usage figures.",
+ "date_requested": "Date and time the usage report covers.",
+ "transformations": "Transformations used, the plan limit, the percentage used, and a breakdown by type.",
+ "objects": "Number of stored objects.",
+ "bandwidth": "Bytes delivered, the plan limit, and the percentage used.",
+ "storage": "Bytes stored, the plan limit, and the percentage used.",
+ "requests": "Number of delivery requests.",
+ "resources": "Number of uploaded assets.",
+ "derived_resources": "Number of derived assets generated from transformations.",
+ "credits": "On credit-based plans, credits used, the plan limit, and the percentage used.",
+ "media_limits": "Maximum file sizes and dimensions the plan allows.",
+ },
+ },
"transformations": {
"description": "A stored transformation, meaning a named recipe Cloudinary applies when delivering an asset.",
"docs_url": "https://cloudinary.com/documentation/admin_api#get_transformations",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/cloudinary.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/cloudinary.py
index faa883e01974..2f2dd1646d48 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/cloudinary.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/cloudinary.py
@@ -1,15 +1,28 @@
import dataclasses
+from collections import deque
+from collections.abc import Callable, Iterator
+from datetime import UTC, date, datetime, timedelta
from typing import Any, Optional
+from urllib.parse import quote
-from products.warehouse_sources.backend.temporal.data_imports.sources.cloudinary.settings import CLOUDINARY_ENDPOINTS
+from products.warehouse_sources.backend.temporal.data_imports.sources.cloudinary.settings import (
+ CLOUDINARY_ENDPOINTS,
+ LAST_ACCESS_REPORTS_PATH,
+ MAX_RESULTS,
+ USAGE_HISTORY_DAYS,
+ USAGE_RESTATEMENT_DAYS,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import HttpBasicAuth
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
JSONResponseCursorPaginator,
+ SinglePagePaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClient
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import EndpointResource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -25,6 +38,10 @@ REQUEST_TIMEOUT_SECONDS = 30
@dataclasses.dataclass(frozen=True)
class CloudinaryResumeConfig:
cursor: str
+ # Set for last access report resources, where a cursor only means something within one report.
+ report_id: Optional[str] = None
+ # Set for subfolders: the folders still to list, where `cursor` continues the first one.
+ pending_folders: Optional[list[str]] = None
def base_url(cloud_name: str, region: str) -> str:
@@ -32,12 +49,16 @@ def base_url(cloud_name: str, region: str) -> str:
return f"{host}/v1_1/{cloud_name}"
+def cursor_paginator() -> JSONResponseCursorPaginator:
+ return JSONResponseCursorPaginator(cursor_path="next_cursor", cursor_param="next_cursor")
+
+
def get_resource(endpoint: str) -> EndpointResource:
config = CLOUDINARY_ENDPOINTS[endpoint]
return {
"name": config.name,
"table_name": config.name,
- "primary_key": config.primary_key,
+ "primary_key": config.primary_keys,
"write_disposition": "replace",
"endpoint": {
"data_selector": f"{config.data_key}[*]",
@@ -48,6 +69,171 @@ def get_resource(endpoint: str) -> EndpointResource:
}
+def _as_date(value: Any) -> Optional[date]:
+ if isinstance(value, datetime):
+ return value.date()
+ if isinstance(value, date):
+ return value
+ if isinstance(value, str):
+ try:
+ return date.fromisoformat(value[:10])
+ except ValueError:
+ return None
+ return None
+
+
+def usage_start_date(should_use_incremental_field: bool, db_incremental_field_last_value: Any, today: date) -> date:
+ earliest = today - timedelta(days=USAGE_HISTORY_DAYS)
+ watermark = _as_date(db_incremental_field_last_value) if should_use_incremental_field else None
+ if watermark is None:
+ return earliest
+ return min(max(watermark - timedelta(days=USAGE_RESTATEMENT_DAYS), earliest), today)
+
+
+def iter_usage(
+ client: RESTClient,
+ start: date,
+ end: date,
+ manager: ResumableSourceManager[CloudinaryResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ resume = manager.load_state() if manager.can_resume() else None
+ resumed_day = _as_date(resume.cursor) if resume else None
+ day = max(resumed_day, start) if resumed_day else start
+ while day <= end:
+ for page in client.paginate(
+ path=CLOUDINARY_ENDPOINTS["usage"].path.format(date=day.isoformat()),
+ paginator=SinglePagePaginator(),
+ data_selector="$",
+ ):
+ rows = [{**row, "date": day.isoformat()} for row in page]
+ manager.save_state(CloudinaryResumeConfig(cursor=(day + timedelta(days=1)).isoformat()))
+ yield rows
+ day += timedelta(days=1)
+
+
+def iter_subfolders(
+ client: RESTClient, manager: ResumableSourceManager[CloudinaryResumeConfig]
+) -> Iterator[list[dict[str, Any]]]:
+ # The folder tree has no listing below the root, so walk it one folder at a time.
+ resume = manager.load_state() if manager.can_resume() else None
+ cursor: Optional[str] = None
+ pending: deque[str] = deque()
+ if resume is not None and resume.pending_folders is not None:
+ pending.extend(resume.pending_folders)
+ cursor = resume.cursor or None
+ else:
+ for page in client.paginate(
+ path=CLOUDINARY_ENDPOINTS["folders"].path, params={"max_results": MAX_RESULTS}, data_selector="folders"
+ ):
+ pending.extend(folder["path"] for folder in page)
+ manager.safe_point()
+
+ next_cursor: dict[str, Optional[str]] = {"value": None}
+
+ def remember_cursor(state: Optional[dict[str, Any]], has_next_page: bool) -> None:
+ next_cursor["value"] = str(state["cursor"]) if has_next_page and state and state.get("cursor") else None
+
+ endpoint_config = CLOUDINARY_ENDPOINTS["subfolders"]
+ while pending:
+ parent = pending.popleft()
+ manager.safe_point()
+ for page in client.paginate(
+ path=endpoint_config.path.format(folder=quote(parent, safe="/")),
+ params=dict(endpoint_config.params),
+ data_selector="folders",
+ initial_paginator_state={"cursor": cursor} if cursor else None,
+ page_state_hook=remember_cursor,
+ ):
+ pending.extend(folder["path"] for folder in page)
+ if next_cursor["value"]:
+ manager.save_state(
+ CloudinaryResumeConfig(cursor=next_cursor["value"], pending_folders=[parent, *pending])
+ )
+ else:
+ manager.save_state(CloudinaryResumeConfig(cursor="", pending_folders=list(pending)))
+ if page:
+ yield page
+ else:
+ manager.safe_point()
+ cursor = None
+
+
+def _report_state_hook(
+ manager: ResumableSourceManager[CloudinaryResumeConfig], report_id: str, next_report_id: Optional[str]
+) -> Callable[[Optional[dict[str, Any]], bool], None]:
+ def stage(state: Optional[dict[str, Any]], has_next_page: bool) -> None:
+ if has_next_page and state and state.get("cursor"):
+ manager.save_state(CloudinaryResumeConfig(cursor=str(state["cursor"]), report_id=report_id))
+ elif not has_next_page and next_report_id is not None:
+ manager.save_state(CloudinaryResumeConfig(cursor="", report_id=next_report_id))
+
+ return stage
+
+
+def iter_last_access_report_resources(
+ client: RESTClient, manager: ResumableSourceManager[CloudinaryResumeConfig]
+) -> Iterator[list[dict[str, Any]]]:
+ report_ids: list[str] = []
+ for page in client.paginate(
+ path=LAST_ACCESS_REPORTS_PATH, params={"max_results": MAX_RESULTS}, data_selector="reports"
+ ):
+ # A report still being generated has no resources to list yet.
+ report_ids.extend(report["id"] for report in page if report.get("status") == "done")
+ manager.safe_point()
+
+ resume = manager.load_state() if manager.can_resume() else None
+ cursor: Optional[str] = None
+ if resume is not None and resume.report_id in report_ids:
+ report_ids = report_ids[report_ids.index(resume.report_id) :]
+ cursor = resume.cursor or None
+
+ endpoint_config = CLOUDINARY_ENDPOINTS["last_access_report_resources"]
+ for index, report_id in enumerate(report_ids):
+ next_report_id = report_ids[index + 1] if index + 1 < len(report_ids) else None
+ manager.safe_point()
+ for page in client.paginate(
+ path=endpoint_config.path.format(report_id=quote(report_id, safe="")),
+ params=dict(endpoint_config.params),
+ data_selector="resources",
+ initial_paginator_state={"cursor": cursor} if cursor else None,
+ page_state_hook=_report_state_hook(manager, report_id, next_report_id),
+ ):
+ if page:
+ yield [{**row, "report_id": report_id} for row in page]
+ cursor = None
+
+
+def _client(cloud_name: str, api_key: str, api_secret: str, region: str) -> RESTClient:
+ return RESTClient(
+ base_url=base_url(cloud_name, region),
+ auth=HttpBasicAuth(username=api_key, password=api_secret),
+ paginator=cursor_paginator(),
+ )
+
+
+def _custom_items(
+ endpoint: str,
+ client: RESTClient,
+ manager: ResumableSourceManager[CloudinaryResumeConfig],
+ should_use_incremental_field: bool,
+ db_incremental_field_last_value: Any,
+) -> Iterator[list[dict[str, Any]]]:
+ try:
+ if endpoint == "usage":
+ today = datetime.now(UTC).date()
+ start = usage_start_date(should_use_incremental_field, db_incremental_field_last_value, today)
+ yield from iter_usage(client, start, today, manager)
+ elif endpoint == "subfolders":
+ yield from iter_subfolders(client, manager)
+ else:
+ yield from iter_last_access_report_resources(client, manager)
+ finally:
+ client.session.close()
+
+
+CUSTOM_ENDPOINTS = ("usage", "subfolders", "last_access_report_resources")
+
+
def cloudinary_source(
cloud_name: str,
api_key: str,
@@ -57,13 +243,39 @@ def cloudinary_source(
team_id: int,
job_id: str,
resumable_source_manager: ResumableSourceManager[CloudinaryResumeConfig],
+ should_use_incremental_field: bool = False,
+ db_incremental_field_last_value: Any = None,
) -> SourceResponse:
endpoint_config = CLOUDINARY_ENDPOINTS[endpoint]
+ partition_key = endpoint_config.partition_key
+ partitioning: dict[str, Any] = {
+ "partition_count": 1 if partition_key else None,
+ "partition_size": 1 if partition_key else None,
+ "partition_mode": "datetime" if partition_key else None,
+ "partition_format": endpoint_config.partition_format if partition_key else None,
+ "partition_keys": [partition_key] if partition_key else None,
+ }
+
+ if endpoint in CUSTOM_ENDPOINTS:
+ return SourceResponse(
+ name=endpoint,
+ items=lambda: _custom_items(
+ endpoint,
+ _client(cloud_name, api_key, api_secret, region),
+ resumable_source_manager,
+ should_use_incremental_field,
+ db_incremental_field_last_value,
+ ),
+ primary_keys=endpoint_config.primary_keys,
+ sort_mode="asc",
+ **partitioning,
+ )
+
config: RESTAPIConfig = {
"client": {
"base_url": base_url(cloud_name, region),
"auth": {"type": "http_basic", "username": api_key, "password": api_secret},
- "paginator": JSONResponseCursorPaginator(cursor_path="next_cursor", cursor_param="next_cursor"),
+ "paginator": cursor_paginator(),
},
"resource_defaults": {"write_disposition": "replace"},
"resources": [get_resource(endpoint)],
@@ -88,17 +300,12 @@ def cloudinary_source(
initial_paginator_state=initial_paginator_state,
)
- partition_key = endpoint_config.partition_key
return SourceResponse(
name=endpoint,
items=lambda: resource,
- primary_keys=[endpoint_config.primary_key],
+ primary_keys=endpoint_config.primary_keys,
column_hints=resource.column_hints,
- partition_count=1 if partition_key else None,
- partition_size=1 if partition_key else None,
- partition_mode="datetime" if partition_key else None,
- partition_format="week" if partition_key else None,
- partition_keys=[partition_key] if partition_key else None,
+ **partitioning,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/settings.py
index 9bb5b453db5c..643d5f6f821b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/settings.py
@@ -3,18 +3,20 @@ from typing import Any, Optional
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.types import IncrementalField
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import PartitionFormat
+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
@frozen
class CloudinaryEndpointConfig:
name: str
path: str
- # Key the rows sit under in the response envelope.
- data_key: str
- primary_key: str
+ primary_keys: list[str]
+ # Key the rows sit under in the response envelope. None when the whole body is one row.
+ data_key: Optional[str] = None
params: dict[str, Any] = field(default_factory=dict)
partition_key: Optional[str] = None
+ partition_format: PartitionFormat = "week"
# Cloudinary caps a page at 500 rows and counts every call against an hourly quota, so ask for the
@@ -26,7 +28,7 @@ CLOUDINARY_ENDPOINTS: dict[str, CloudinaryEndpointConfig] = {
name="images",
path="/resources/image",
data_key="resources",
- primary_key="asset_id",
+ primary_keys=["asset_id"],
params={"max_results": MAX_RESULTS, "context": "true", "tags": "true"},
partition_key="created_at",
),
@@ -34,7 +36,7 @@ CLOUDINARY_ENDPOINTS: dict[str, CloudinaryEndpointConfig] = {
name="videos",
path="/resources/video",
data_key="resources",
- primary_key="asset_id",
+ primary_keys=["asset_id"],
params={"max_results": MAX_RESULTS, "context": "true", "tags": "true"},
partition_key="created_at",
),
@@ -42,7 +44,7 @@ CLOUDINARY_ENDPOINTS: dict[str, CloudinaryEndpointConfig] = {
name="raw_files",
path="/resources/raw",
data_key="resources",
- primary_key="asset_id",
+ primary_keys=["asset_id"],
params={"max_results": MAX_RESULTS, "context": "true", "tags": "true"},
partition_key="created_at",
),
@@ -50,28 +52,67 @@ CLOUDINARY_ENDPOINTS: dict[str, CloudinaryEndpointConfig] = {
name="folders",
path="/folders",
data_key="folders",
- primary_key="path",
+ primary_keys=["path"],
params={"max_results": MAX_RESULTS},
),
"transformations": CloudinaryEndpointConfig(
name="transformations",
path="/transformations",
data_key="transformations",
- primary_key="name",
+ primary_keys=["name"],
params={"max_results": MAX_RESULTS},
),
"upload_presets": CloudinaryEndpointConfig(
name="upload_presets",
path="/upload_presets",
data_key="presets",
- primary_key="name",
+ primary_keys=["name"],
params={"max_results": MAX_RESULTS},
),
+ "subfolders": CloudinaryEndpointConfig(
+ name="subfolders",
+ path="/folders/{folder}",
+ data_key="folders",
+ primary_keys=["path"],
+ params={"max_results": MAX_RESULTS},
+ ),
+ "last_access_report_resources": CloudinaryEndpointConfig(
+ name="last_access_report_resources",
+ path="/resources/last_access_report/{report_id}",
+ data_key="resources",
+ primary_keys=["report_id", "asset_id"],
+ params={"max_results": MAX_RESULTS},
+ partition_key="created_at",
+ ),
+ "usage": CloudinaryEndpointConfig(
+ name="usage",
+ path="/usage/{date}",
+ primary_keys=["date"],
+ partition_key="date",
+ partition_format="month",
+ ),
}
ENDPOINTS = tuple(CLOUDINARY_ENDPOINTS.keys())
+LAST_ACCESS_REPORTS_PATH = "/resources_last_access_reports"
+
+# The usage endpoint only answers for dates in the last 3 months, and the shortest 3-month span is 89 days.
+USAGE_HISTORY_DAYS = 89
+# Cloudinary refreshes usage figures periodically, so an incremental sync re-reads the last few days.
+USAGE_RESTATEMENT_DAYS = 2
+
# Cloudinary's only time filter on the asset lists is `start_at`, which selects on update time while
-# the list itself is ordered by creation time. A watermark built on it would skip rows, so every
-# table is a full refresh until that ordering can be verified against a live account.
-INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}
+# the list itself is ordered by creation time. A watermark built on it would skip rows, so the list
+# tables are a full refresh until that ordering can be verified against a live account. Usage is
+# requested one date at a time, so its date is a real server-side filter.
+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
+ "usage": [
+ {
+ "label": "date",
+ "type": IncrementalFieldType.Date,
+ "field": "date",
+ "field_type": IncrementalFieldType.Date,
+ }
+ ],
+}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/source.py
index 5de61be46f35..975bcb1f6d42 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/source.py
@@ -111,11 +111,11 @@ class CloudinarySource(ResumableSource[CloudinarySourceConfig, CloudinaryResumeC
name=ExternalDataSourceType.CLOUDINARY,
category=DataWarehouseSourceCategory.FILE_STORAGE,
label="Cloudinary",
- caption="""Enter your Cloudinary cloud name, API key, and API secret to sync your media library metadata into the PostHog Data warehouse: images, videos, raw files, folders, transformations, and upload presets.
+ caption="""Enter your Cloudinary cloud name, API key, and API secret to sync your media library metadata into the PostHog Data warehouse: images, videos, raw files, folders, subfolders, transformations, upload presets, daily usage, and last access reports.
Find all three in your Cloudinary console under Settings, then API Keys. The source only reads metadata, so a key with read access is enough. Media files themselves are never downloaded.
-Cloudinary limits Admin API calls per hour, 500 on free plans, so a first sync of a large media library can take a while.""",
+Cloudinary limits Admin API calls per hour, 500 on free plans, so a first sync of a large media library can take a while. Subfolders take one call per folder, and usage takes one call per day. Last access reports are an Enterprise feature that Cloudinary turns on by request.""",
iconPath="/static/services/cloudinary.png",
docsUrl="https://posthog.com/docs/cdp/sources/cloudinary",
keywords=["cloudinary.com", "media", "images", "cdn"],
@@ -179,7 +179,7 @@ Cloudinary limits Admin API calls per hour, 500 on free plans, so a first sync o
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)
+ return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, merge_only=("usage",))
def validate_credentials(
self,
@@ -211,4 +211,6 @@ Cloudinary limits Admin API calls per hour, 500 on free plans, so a first sync o
team_id=inputs.team_id,
job_id=inputs.job_id,
resumable_source_manager=resumable_source_manager,
+ should_use_incremental_field=inputs.should_use_incremental_field,
+ db_incremental_field_last_value=inputs.db_incremental_field_last_value,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/tests/test_cloudinary.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/tests/test_cloudinary.py
index 7361edaf1a55..5a188ec69e33 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/tests/test_cloudinary.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudinary/tests/test_cloudinary.py
@@ -1,11 +1,17 @@
+from datetime import date
+from typing import Any
+
import pytest
+import time_machine
from products.warehouse_sources.backend.temporal.data_imports.sources.cloudinary.cloudinary import (
CloudinaryResumeConfig,
+ usage_start_date,
validate_credentials,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.cloudinary.source import CloudinarySource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+ RecordedRequest,
ScriptedResponse,
SourceDriver,
always,
@@ -57,6 +63,125 @@ class TestCloudinarySource:
assert with_cursor.saved_states == [CloudinaryResumeConfig(cursor="next")]
+class TestCloudinarySubfolders:
+ def test_the_walk_reaches_every_depth_and_skips_the_root_folders(self) -> None:
+ def answer(request: RecordedRequest) -> ScriptedResponse:
+ pages = {
+ ("/v1_1/my-cloud/folders", None): {"folders": [{"path": "shop"}]},
+ ("/v1_1/my-cloud/folders/shop", None): {"folders": [{"path": "shop/red shoes"}], "next_cursor": "p2"},
+ ("/v1_1/my-cloud/folders/shop", "p2"): {"folders": [{"path": "shop/hats"}]},
+ ("/v1_1/my-cloud/folders/shop/red%20shoes", None): {"folders": [{"path": "shop/red shoes/2026"}]},
+ }
+ return ScriptedResponse(json=pages.get((request.path, request.param("next_cursor")), {"folders": []}))
+
+ result = _driver().run("subfolders", answer)
+
+ assert result.raised is None
+ assert [row["path"] for row in result.rows] == ["shop/red shoes", "shop/hats", "shop/red shoes/2026"]
+ assert len(result.requests) == 6
+ assert result.saved_states[:2] == [
+ CloudinaryResumeConfig(cursor="p2", pending_folders=["shop", "shop/red shoes"]),
+ CloudinaryResumeConfig(cursor="", pending_folders=["shop/red shoes", "shop/hats"]),
+ ]
+
+ def test_a_resume_continues_the_saved_folder_without_relisting_the_root(self) -> None:
+ def answer(request: RecordedRequest) -> ScriptedResponse:
+ pages: dict[tuple[str, str | None], dict[str, Any]] = {
+ ("/v1_1/my-cloud/folders/shop", "p2"): {"folders": [{"path": "shop/hats"}]},
+ }
+ return ScriptedResponse(json=pages.get((request.path, request.param("next_cursor")), {"folders": []}))
+
+ result = _driver().run(
+ "subfolders",
+ answer,
+ resume_state=CloudinaryResumeConfig(cursor="p2", pending_folders=["shop", "shop/red shoes"]),
+ )
+
+ assert result.raised is None
+ assert [row["path"] for row in result.rows] == ["shop/hats"]
+ assert result.paths == [
+ "/v1_1/my-cloud/folders/shop",
+ "/v1_1/my-cloud/folders/shop/red%20shoes",
+ "/v1_1/my-cloud/folders/shop/hats",
+ ]
+
+
+class TestCloudinaryLastAccessReports:
+ @staticmethod
+ def _answer(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == "/v1_1/my-cloud/resources_last_access_reports":
+ return ScriptedResponse(
+ json={
+ "reports": [
+ {"id": "r1", "status": "done"},
+ {"id": "r2", "status": "processing"},
+ {"id": "r3", "status": "done"},
+ ]
+ }
+ )
+ pages = {
+ ("r1", None): {"resources": [{"asset_id": "a"}], "next_cursor": "c2"},
+ ("r1", "c2"): {"resources": [{"asset_id": "b"}]},
+ ("r3", None): {"resources": [{"asset_id": "a"}]},
+ }
+ report_id = request.path.rsplit("/", 1)[-1]
+ return ScriptedResponse(json=pages[(report_id, request.param("next_cursor"))])
+
+ def test_each_completed_report_is_read_and_tagged_with_its_id(self) -> None:
+ result = _driver().run("last_access_report_resources", self._answer)
+
+ assert result.raised is None
+ assert [(row["report_id"], row["asset_id"]) for row in result.rows] == [("r1", "a"), ("r1", "b"), ("r3", "a")]
+ assert result.saved_states == [
+ CloudinaryResumeConfig(cursor="c2", report_id="r1"),
+ CloudinaryResumeConfig(cursor="", report_id="r3"),
+ ]
+
+ def test_a_resume_continues_inside_the_saved_report(self) -> None:
+ result = _driver().run(
+ "last_access_report_resources",
+ self._answer,
+ resume_state=CloudinaryResumeConfig(cursor="c2", report_id="r1"),
+ )
+
+ assert result.raised is None
+ assert [(row["report_id"], row["asset_id"]) for row in result.rows] == [("r1", "b"), ("r3", "a")]
+
+
+class TestCloudinaryUsage:
+ @pytest.mark.parametrize(
+ "incremental,watermark,expected",
+ [
+ (False, None, date(2026, 7, 21)),
+ (True, None, date(2026, 7, 21)),
+ (True, "2026-10-10", date(2026, 10, 8)),
+ (True, date(2026, 10, 10), date(2026, 10, 8)),
+ (True, "2025-01-01", date(2026, 7, 21)),
+ (False, "2026-10-10", date(2026, 7, 21)),
+ ],
+ )
+ def test_start_date_stays_inside_the_window_cloudinary_answers_for(
+ self, incremental: bool, watermark: object, expected: date
+ ) -> None:
+ assert usage_start_date(incremental, watermark, date(2026, 10, 18)) == expected
+
+ @time_machine.travel("2026-10-18T12:00:00Z", tick=False)
+ def test_each_day_is_requested_once_and_keyed_by_its_date(self) -> None:
+ result = _driver().run(
+ "usage",
+ always(ScriptedResponse(json={"plan": "Plus", "storage": {"usage": 1}})),
+ resume_state=CloudinaryResumeConfig(cursor="2026-10-17"),
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/v1_1/my-cloud/usage/2026-10-17", "/v1_1/my-cloud/usage/2026-10-18"]
+ assert [(row["date"], row["plan"]) for row in result.rows] == [("2026-10-17", "Plus"), ("2026-10-18", "Plus")]
+ assert result.saved_states == [
+ CloudinaryResumeConfig(cursor="2026-10-18"),
+ CloudinaryResumeConfig(cursor="2026-10-19"),
+ ]
+
+
class TestCloudinaryCredentials:
@pytest.mark.parametrize(
"status_code,expected_valid",