Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 1 missing table to the Pexels source (#121057)

Tom Owers92e52fb8f0fe
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 984f45fe0231..19b89527d0f9 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -881,8 +881,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/persona/persona.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/persona/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/personio/personio.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/pexels/pexels.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/pexels/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pganalyze/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/phyllo.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/settings.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 24c431085000..68a0189619f3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9357,7 +9357,7 @@ Today (4): `curated_photos`, `featured_collections`, `my_collections`, `popular_
Diffed against: <https://www.pexels.com/api/documentation/>
-- [ ] `GET /v1/collections/{id} (collection media)` — the only way to resolve which photos/videos belong to the collections we already sync - without it featured_collections and my_collections are unjoinable ID rows (high)
+- [x] `GET /v1/collections/{id} (collection media)` — the only way to resolve which photos/videos belong to the collections we already sync - without it featured_collections and my_collections are unjoinable ID rows (high)
Note: Pexels' whole documented surface is 9 endpoints: /v1/search, /v1/curated, /v1/photos/{id}, /videos/search, /videos/popular, /videos/videos/{id}, /v1/collections/featured, /v1/collections, /v1/collections/{id}. The source already implements search_photos and search_videos, but they are gated on a configured search query so they never appear in get_schemas - that is not a gap. www.pexels.com blocks curl with a Cloudflare 403; the page was read via the fetch tool instead.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/canonical_descriptions.py
index 87ad9a726c3f..fb5c1995791a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/canonical_descriptions.py
@@ -48,6 +48,18 @@ _COLLECTION_COLUMNS = {
"videos_count": "Number of videos in the collection.",
}
+_COLLECTION_MEDIA_COLUMNS = {
+ **_PHOTO_COLUMNS,
+ **_VIDEO_COLUMNS,
+ "id": "Unique identifier for the photo or video. Photo and video ids are separate sequences.",
+ "width": "Real width of the photo or video in pixels.",
+ "height": "Real height of the photo or video in pixels.",
+ "url": "Pexels URL where the photo or video is located.",
+ "avg_color": "Average color of the photo or video, when available.",
+ "type": "Kind of media item: `Photo` or `Video`.",
+ "collection_id": "Identifier of the collection that contains the media item.",
+}
+
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"curated_photos": {
"description": "A real-time selection of photos curated by the Pexels team.",
@@ -79,4 +91,9 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"docs_url": "https://www.pexels.com/api/documentation/#collections-all",
"columns": _COLLECTION_COLUMNS,
},
+ "collection_media": {
+ "description": "Photos and videos in each collection owned by the authenticated Pexels account.",
+ "docs_url": "https://www.pexels.com/api/documentation/#collections-media",
+ "columns": _COLLECTION_MEDIA_COLUMNS,
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/pexels.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/pexels.py
index f3cc5d594512..30f6c0524fcd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/pexels.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/pexels.py
@@ -1,33 +1,40 @@
-import dataclasses
+from collections.abc import Iterable
from typing import Any, Optional
from urllib.parse import urlencode
+from posthog.dataclasses import frozen
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
PageNumberPaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.pexels.settings import (
+ PER_PAGE,
PEXELS_ENDPOINTS,
PexelsEndpointConfig,
)
PEXELS_BASE_URL = "https://api.pexels.com"
-# Pexels caps `per_page` at 80; request the max to minimise round trips.
-PER_PAGE = 80
REQUEST_TIMEOUT = 30
+@frozen
class PexelsResumeConfig:
# 1-based page number to (re-)start from. Pexels uses page-number pagination.
page: int = 1
+ # Fan-out progress of a child endpoint: the finished parent paths and the in-progress one.
+ fanout_state: Optional[dict[str, Any]] = None
def _get_headers(api_key: str) -> dict[str, str]:
@@ -52,6 +59,62 @@ def validate_credentials(api_key: str) -> bool:
return ok
+def _paginator() -> PageNumberPaginator:
+ # Pexels exposes no total-pages field; termination is the first empty page (the paginator
+ # requests one page past the last populated one, whose rows dedupe on `id` if re-run).
+ return PageNumberPaginator(base_page=1, page=1, total_path=None)
+
+
+def _client_config(api_key: str) -> ClientConfig:
+ return {
+ "base_url": PEXELS_BASE_URL,
+ # Only the non-secret Accept header goes here; the raw-key Authorization is supplied via
+ # the framework auth config (api_key in the Authorization header, no "Bearer " prefix) so
+ # its value is redacted from logs and captured samples automatically.
+ "headers": {"Accept": "application/json"},
+ "auth": {"type": "api_key", "api_key": api_key, "name": "Authorization", "location": "header"},
+ "paginator": _paginator(),
+ }
+
+
+def _fanout_resource(
+ api_key: str,
+ config: PexelsEndpointConfig,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[PexelsResumeConfig],
+) -> Iterable[Any]:
+ assert config.fanout is not None
+ parent = PEXELS_ENDPOINTS[config.fanout.parent_name]
+
+ initial_state: Optional[dict[str, Any]] = None
+ if resumable_source_manager.can_resume():
+ resume = resumable_source_manager.load_state()
+ if resume is not None:
+ initial_state = resume.fanout_state
+
+ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if state:
+ resumable_source_manager.save_state(PexelsResumeConfig(fanout_state=state))
+
+ return build_dependent_resource(
+ endpoint_configs=PEXELS_ENDPOINTS,
+ child_endpoint=config.name,
+ fanout=config.fanout,
+ client_config=_client_config(api_key),
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ parent_endpoint_extra={"data_selector": parent.data_key},
+ # A path that ends in `{id}` reads as a single object and defaults to one page, so name the paginator.
+ child_endpoint_extra={"data_selector": config.data_key, "paginator": _paginator()},
+ page_size_param="per_page",
+ resume_hook=save_checkpoint,
+ initial_paginator_state=initial_state,
+ )
+
+
def pexels_source(
api_key: str,
endpoint: str,
@@ -66,22 +129,22 @@ def pexels_source(
if config.requires_query and not search_query:
raise ValueError(f"Endpoint '{endpoint}' requires a search query but none was provided.")
+ if config.fanout is not None:
+ fanout_resource = _fanout_resource(api_key, config, team_id, job_id, resumable_source_manager)
+ return SourceResponse(
+ name=endpoint,
+ items=lambda: fanout_resource,
+ primary_keys=config.primary_keys,
+ partition_count=1,
+ partition_size=1,
+ )
+
params: dict[str, Any] = {"per_page": PER_PAGE}
if config.requires_query:
params["query"] = search_query
rest_config: RESTAPIConfig = {
- "client": {
- "base_url": PEXELS_BASE_URL,
- # Only the non-secret Accept header goes here; the raw-key Authorization is supplied via
- # the framework auth config (api_key in the Authorization header, no "Bearer " prefix) so
- # its value is redacted from logs and captured samples automatically.
- "headers": {"Accept": "application/json"},
- "auth": {"type": "api_key", "api_key": api_key, "name": "Authorization", "location": "header"},
- # Pexels exposes no total-pages field; termination is the first empty page (the paginator
- # requests one page past the last populated one, whose rows dedupe on `id` if re-run).
- "paginator": PageNumberPaginator(base_page=1, page=1, total_path=None),
- },
+ "client": _client_config(api_key),
# Per-resource settings are fully specified below, so no shared defaults are needed.
"resource_defaults": {},
"resources": [
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/settings.py
index ed3fc177fd75..e9f5c567a193 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/settings.py
@@ -1,9 +1,17 @@
-from dataclasses import dataclass, field
+from dataclasses import field
+from posthog.dataclasses import frozen
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
from products.warehouse_sources.backend.types import IncrementalField
+# Pexels caps `per_page` at 80; request the max to minimise round trips.
+PER_PAGE = 80
+
-@dataclass
+@frozen
class PexelsEndpointConfig:
name: str
path: str # relative to PEXELS_BASE_URL
@@ -15,6 +23,11 @@ class PexelsEndpointConfig:
primary_keys: list[str] = field(default_factory=lambda: ["id"])
should_sync_default: bool = True
description: str | None = None
+ # Set for child endpoints that are fetched once for each row of a parent endpoint.
+ fanout: DependentEndpointConfig | None = None
+ page_size: int = PER_PAGE
+ incremental_fields: list[IncrementalField] = field(default_factory=list)
+ default_incremental_field: str | None = None
PEXELS_ENDPOINTS: dict[str, PexelsEndpointConfig] = {
@@ -42,6 +55,24 @@ PEXELS_ENDPOINTS: dict[str, PexelsEndpointConfig] = {
data_key="collections",
description="Collections owned by the authenticated Pexels account.",
),
+ "collection_media": PexelsEndpointConfig(
+ name="collection_media",
+ path="/v1/collections/{collection_id}",
+ data_key="media",
+ # Photo and video ids are separate sequences, and one media item can sit in several
+ # collections, so only the full triple is unique across the table.
+ primary_keys=["collection_id", "type", "id"],
+ description="Photos and videos in each collection owned by the authenticated Pexels account.",
+ fanout=DependentEndpointConfig(
+ parent_name="my_collections",
+ resolve_param="collection_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "collection_id"},
+ # A collection deleted after the parent listing returns 404; skip it, not the whole sync.
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
"search_photos": PexelsEndpointConfig(
name="search_photos",
path="/v1/search",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/tests/test_pexels.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/tests/test_pexels.py
index 97f53c8f135d..047564b69e0a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/tests/test_pexels.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/tests/test_pexels.py
@@ -6,6 +6,7 @@ from parameterized import parameterized
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
ScriptedResponse,
SourceDriver,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.pexels import PexelsSourceConfig
@@ -25,6 +26,10 @@ def _page(items: list[dict[str, Any]], data_key: str = "photos") -> ScriptedResp
return ScriptedResponse(json={data_key: items})
+_PHOTO = {"id": 7, "type": "Photo", "photographer": "A. Example"}
+_VIDEO = {"id": 7, "type": "Video", "duration": 12}
+
+
class TestBuildUrl:
@parameterized.expand(
[
@@ -69,6 +74,42 @@ class TestPagination:
assert result.requests == []
+class TestCollectionMedia:
+ @parameterized.expand(
+ [
+ ("empty_collection", ScriptedResponse(json={"media": []})),
+ ("collection_deleted_after_listing", ScriptedResponse(status=404, json={"error": "Not Found"})),
+ ]
+ )
+ def test_walks_every_owned_collection(self, _name: str, second_collection: ScriptedResponse) -> None:
+ result = _driver().run(
+ "collection_media",
+ route(
+ {
+ "/v1/collections": [
+ _page([{"id": "abc1"}, {"id": "def2"}], "collections"),
+ _page([], "collections"),
+ ],
+ "/v1/collections/abc1": [_page([_PHOTO, _VIDEO], "media"), _page([], "media")],
+ "/v1/collections/def2": [second_collection],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert result.rows == [
+ {**_PHOTO, "collection_id": "abc1"},
+ {**_VIDEO, "collection_id": "abc1"},
+ ]
+ assert [(r.path, r.param("page"), r.param("per_page")) for r in result.requests] == [
+ ("/v1/collections", "1", "80"),
+ ("/v1/collections/abc1", "1", "80"),
+ ("/v1/collections/abc1", "2", "80"),
+ ("/v1/collections/def2", "1", "80"),
+ ("/v1/collections", "2", "80"),
+ ]
+
+
class TestResume:
def test_resumes_from_saved_page(self) -> None:
# A resumed job must start at the saved page, not page 1.
@@ -80,6 +121,42 @@ class TestResume:
assert result.rows == [{"id": 2}]
assert result.params("page")[0] == "2"
+ @parameterized.expand(
+ [
+ (
+ "skips_finished_collection",
+ {"completed": ["/v1/collections/abc1"], "current": None, "child_state": None},
+ [("/v1/collections/def2", "1")],
+ ),
+ (
+ "continues_collection_in_progress",
+ {"completed": [], "current": "/v1/collections/abc1", "child_state": {"page": 3}},
+ [("/v1/collections/abc1", "3"), ("/v1/collections/def2", "1")],
+ ),
+ ]
+ )
+ def test_collection_media_resumes_from_saved_fanout_state(
+ self, _name: str, fanout_state: dict[str, Any], expected_child_requests: list[tuple[str, str]]
+ ) -> None:
+ result = _driver().run(
+ "collection_media",
+ route(
+ {
+ "/v1/collections": [
+ _page([{"id": "abc1"}, {"id": "def2"}], "collections"),
+ _page([], "collections"),
+ ],
+ "/v1/collections/abc1": [_page([], "media")],
+ "/v1/collections/def2": [_page([], "media")],
+ }
+ ),
+ resume_state=PexelsResumeConfig(fanout_state=fanout_state),
+ )
+
+ assert result.raised is None
+ child_requests = [(r.path, r.param("page")) for r in result.requests if r.path != "/v1/collections"]
+ assert child_requests == expected_child_requests
+
class TestRetryClassification:
def test_unauthorized_raises_and_is_not_retried(self) -> None: