:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 1 missing table to the Pexels source (#121057)
Tom Owers92e52fb8f0fe
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 984f45fe0231..19b89527d0f9 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -881,8 +881,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/persona/persona.py1 products/warehouse_sources/backend/temporal/data_imports/sources/persona/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/personio/personio.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/pexels/pexels.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/pexels/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/pganalyze/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/phyllo.py1 products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/settings.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 24c431085000..68a0189619f3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -9357,7 +9357,7 @@ Today (4): `curated_photos`, `featured_collections`, `my_collections`, `popular_Diffed against: <https://www.pexels.com/api/documentation/>-- [ ] `GET /v1/collections/{id} (collection media)` — the only way to resolve which photos/videos belong to the collections we already sync - without it featured_collections and my_collections are unjoinable ID rows (high)+- [x] `GET /v1/collections/{id} (collection media)` — the only way to resolve which photos/videos belong to the collections we already sync - without it featured_collections and my_collections are unjoinable ID rows (high)Note: Pexels' whole documented surface is 9 endpoints: /v1/search, /v1/curated, /v1/photos/{id}, /videos/search, /videos/popular, /videos/videos/{id}, /v1/collections/featured, /v1/collections, /v1/collections/{id}. The source already implements search_photos and search_videos, but they are gated on a configured search query so they never appear in get_schemas - that is not a gap. www.pexels.com blocks curl with a Cloudflare 403; the page was read via the fetch tool instead.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/canonical_descriptions.pyindex 87ad9a726c3f..fb5c1995791a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/canonical_descriptions.py@@ -48,6 +48,18 @@ _COLLECTION_COLUMNS = {"videos_count": "Number of videos in the collection.",}+_COLLECTION_MEDIA_COLUMNS = {+ **_PHOTO_COLUMNS,+ **_VIDEO_COLUMNS,+ "id": "Unique identifier for the photo or video. Photo and video ids are separate sequences.",+ "width": "Real width of the photo or video in pixels.",+ "height": "Real height of the photo or video in pixels.",+ "url": "Pexels URL where the photo or video is located.",+ "avg_color": "Average color of the photo or video, when available.",+ "type": "Kind of media item: `Photo` or `Video`.",+ "collection_id": "Identifier of the collection that contains the media item.",+}+CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"curated_photos": {"description": "A real-time selection of photos curated by the Pexels team.",@@ -79,4 +91,9 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"docs_url": "https://www.pexels.com/api/documentation/#collections-all","columns": _COLLECTION_COLUMNS,},+ "collection_media": {+ "description": "Photos and videos in each collection owned by the authenticated Pexels account.",+ "docs_url": "https://www.pexels.com/api/documentation/#collections-media",+ "columns": _COLLECTION_MEDIA_COLUMNS,+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/pexels.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/pexels.pyindex f3cc5d594512..30f6c0524fcd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/pexels.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/pexels.py@@ -1,33 +1,40 @@-import dataclasses+from collections.abc import Iterablefrom typing import Any, Optionalfrom urllib.parse import urlencode+from posthog.dataclasses import frozen+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_sessionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (RESTAPIConfig,rest_api_resource,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ build_dependent_resource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (PageNumberPaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.pexels.settings import (+ PER_PAGE,PEXELS_ENDPOINTS,PexelsEndpointConfig,)PEXELS_BASE_URL = "https://api.pexels.com"-# Pexels caps `per_page` at 80; request the max to minimise round trips.-PER_PAGE = 80REQUEST_TIMEOUT = 30+@frozenclass PexelsResumeConfig:# 1-based page number to (re-)start from. Pexels uses page-number pagination.page: int = 1+ # Fan-out progress of a child endpoint: the finished parent paths and the in-progress one.+ fanout_state: Optional[dict[str, Any]] = Nonedef _get_headers(api_key: str) -> dict[str, str]:@@ -52,6 +59,62 @@ def validate_credentials(api_key: str) -> bool:return ok+def _paginator() -> PageNumberPaginator:+ # Pexels exposes no total-pages field; termination is the first empty page (the paginator+ # requests one page past the last populated one, whose rows dedupe on `id` if re-run).+ return PageNumberPaginator(base_page=1, page=1, total_path=None)+++def _client_config(api_key: str) -> ClientConfig:+ return {+ "base_url": PEXELS_BASE_URL,+ # Only the non-secret Accept header goes here; the raw-key Authorization is supplied via+ # the framework auth config (api_key in the Authorization header, no "Bearer " prefix) so+ # its value is redacted from logs and captured samples automatically.+ "headers": {"Accept": "application/json"},+ "auth": {"type": "api_key", "api_key": api_key, "name": "Authorization", "location": "header"},+ "paginator": _paginator(),+ }+++def _fanout_resource(+ api_key: str,+ config: PexelsEndpointConfig,+ team_id: int,+ job_id: str,+ resumable_source_manager: ResumableSourceManager[PexelsResumeConfig],+) -> Iterable[Any]:+ assert config.fanout is not None+ parent = PEXELS_ENDPOINTS[config.fanout.parent_name]++ initial_state: Optional[dict[str, Any]] = None+ if resumable_source_manager.can_resume():+ resume = resumable_source_manager.load_state()+ if resume is not None:+ initial_state = resume.fanout_state++ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:+ if state:+ resumable_source_manager.save_state(PexelsResumeConfig(fanout_state=state))++ return build_dependent_resource(+ endpoint_configs=PEXELS_ENDPOINTS,+ child_endpoint=config.name,+ fanout=config.fanout,+ client_config=_client_config(api_key),+ path_format_values={},+ team_id=team_id,+ job_id=job_id,+ db_incremental_field_last_value=None,+ parent_endpoint_extra={"data_selector": parent.data_key},+ # A path that ends in `{id}` reads as a single object and defaults to one page, so name the paginator.+ child_endpoint_extra={"data_selector": config.data_key, "paginator": _paginator()},+ page_size_param="per_page",+ resume_hook=save_checkpoint,+ initial_paginator_state=initial_state,+ )++def pexels_source(api_key: str,endpoint: str,@@ -66,22 +129,22 @@ def pexels_source(if config.requires_query and not search_query:raise ValueError(f"Endpoint '{endpoint}' requires a search query but none was provided.")+ if config.fanout is not None:+ fanout_resource = _fanout_resource(api_key, config, team_id, job_id, resumable_source_manager)+ return SourceResponse(+ name=endpoint,+ items=lambda: fanout_resource,+ primary_keys=config.primary_keys,+ partition_count=1,+ partition_size=1,+ )+params: dict[str, Any] = {"per_page": PER_PAGE}if config.requires_query:params["query"] = search_queryrest_config: RESTAPIConfig = {- "client": {- "base_url": PEXELS_BASE_URL,- # Only the non-secret Accept header goes here; the raw-key Authorization is supplied via- # the framework auth config (api_key in the Authorization header, no "Bearer " prefix) so- # its value is redacted from logs and captured samples automatically.- "headers": {"Accept": "application/json"},- "auth": {"type": "api_key", "api_key": api_key, "name": "Authorization", "location": "header"},- # Pexels exposes no total-pages field; termination is the first empty page (the paginator- # requests one page past the last populated one, whose rows dedupe on `id` if re-run).- "paginator": PageNumberPaginator(base_page=1, page=1, total_path=None),- },+ "client": _client_config(api_key),# Per-resource settings are fully specified below, so no shared defaults are needed."resource_defaults": {},"resources": [diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/settings.pyindex ed3fc177fd75..e9f5c567a193 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/settings.py@@ -1,9 +1,17 @@-from dataclasses import dataclass, field+from dataclasses import field+from posthog.dataclasses import frozen++from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ DependentEndpointConfig,+)from products.warehouse_sources.backend.types import IncrementalField+# Pexels caps `per_page` at 80; request the max to minimise round trips.+PER_PAGE = 80+-@dataclass+@frozenclass PexelsEndpointConfig:name: strpath: str # relative to PEXELS_BASE_URL@@ -15,6 +23,11 @@ class PexelsEndpointConfig:primary_keys: list[str] = field(default_factory=lambda: ["id"])should_sync_default: bool = Truedescription: str | None = None+ # Set for child endpoints that are fetched once for each row of a parent endpoint.+ fanout: DependentEndpointConfig | None = None+ page_size: int = PER_PAGE+ incremental_fields: list[IncrementalField] = field(default_factory=list)+ default_incremental_field: str | None = NonePEXELS_ENDPOINTS: dict[str, PexelsEndpointConfig] = {@@ -42,6 +55,24 @@ PEXELS_ENDPOINTS: dict[str, PexelsEndpointConfig] = {data_key="collections",description="Collections owned by the authenticated Pexels account.",),+ "collection_media": PexelsEndpointConfig(+ name="collection_media",+ path="/v1/collections/{collection_id}",+ data_key="media",+ # Photo and video ids are separate sequences, and one media item can sit in several+ # collections, so only the full triple is unique across the table.+ primary_keys=["collection_id", "type", "id"],+ description="Photos and videos in each collection owned by the authenticated Pexels account.",+ fanout=DependentEndpointConfig(+ parent_name="my_collections",+ resolve_param="collection_id",+ resolve_field="id",+ include_from_parent=["id"],+ parent_field_renames={"id": "collection_id"},+ # A collection deleted after the parent listing returns 404; skip it, not the whole sync.+ child_response_actions=[{"status_code": 404, "action": "ignore"}],+ ),+ ),"search_photos": PexelsEndpointConfig(name="search_photos",path="/v1/search",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/tests/test_pexels.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/tests/test_pexels.pyindex 97f53c8f135d..047564b69e0a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/tests/test_pexels.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pexels/tests/test_pexels.py@@ -6,6 +6,7 @@ from parameterized import parameterizedfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (ScriptedResponse,SourceDriver,+ route,scripted_network,)from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.pexels import PexelsSourceConfig@@ -25,6 +26,10 @@ def _page(items: list[dict[str, Any]], data_key: str = "photos") -> ScriptedRespreturn ScriptedResponse(json={data_key: items})+_PHOTO = {"id": 7, "type": "Photo", "photographer": "A. Example"}+_VIDEO = {"id": 7, "type": "Video", "duration": 12}++class TestBuildUrl:@parameterized.expand([@@ -69,6 +74,42 @@ class TestPagination:assert result.requests == []+class TestCollectionMedia:+ @parameterized.expand(+ [+ ("empty_collection", ScriptedResponse(json={"media": []})),+ ("collection_deleted_after_listing", ScriptedResponse(status=404, json={"error": "Not Found"})),+ ]+ )+ def test_walks_every_owned_collection(self, _name: str, second_collection: ScriptedResponse) -> None:+ result = _driver().run(+ "collection_media",+ route(+ {+ "/v1/collections": [+ _page([{"id": "abc1"}, {"id": "def2"}], "collections"),+ _page([], "collections"),+ ],+ "/v1/collections/abc1": [_page([_PHOTO, _VIDEO], "media"), _page([], "media")],+ "/v1/collections/def2": [second_collection],+ }+ ),+ )++ assert result.raised is None+ assert result.rows == [+ {**_PHOTO, "collection_id": "abc1"},+ {**_VIDEO, "collection_id": "abc1"},+ ]+ assert [(r.path, r.param("page"), r.param("per_page")) for r in result.requests] == [+ ("/v1/collections", "1", "80"),+ ("/v1/collections/abc1", "1", "80"),+ ("/v1/collections/abc1", "2", "80"),+ ("/v1/collections/def2", "1", "80"),+ ("/v1/collections", "2", "80"),+ ]++class TestResume:def test_resumes_from_saved_page(self) -> None:# A resumed job must start at the saved page, not page 1.@@ -80,6 +121,42 @@ class TestResume:assert result.rows == [{"id": 2}]assert result.params("page")[0] == "2"+ @parameterized.expand(+ [+ (+ "skips_finished_collection",+ {"completed": ["/v1/collections/abc1"], "current": None, "child_state": None},+ [("/v1/collections/def2", "1")],+ ),+ (+ "continues_collection_in_progress",+ {"completed": [], "current": "/v1/collections/abc1", "child_state": {"page": 3}},+ [("/v1/collections/abc1", "3"), ("/v1/collections/def2", "1")],+ ),+ ]+ )+ def test_collection_media_resumes_from_saved_fanout_state(+ self, _name: str, fanout_state: dict[str, Any], expected_child_requests: list[tuple[str, str]]+ ) -> None:+ result = _driver().run(+ "collection_media",+ route(+ {+ "/v1/collections": [+ _page([{"id": "abc1"}, {"id": "def2"}], "collections"),+ _page([], "collections"),+ ],+ "/v1/collections/abc1": [_page([], "media")],+ "/v1/collections/def2": [_page([], "media")],+ }+ ),+ resume_state=PexelsResumeConfig(fanout_state=fanout_state),+ )++ assert result.raised is None+ child_requests = [(r.path, r.param("page")) for r in result.requests if r.path != "/v1/collections"]+ assert child_requests == expected_child_requests+class TestRetryClassification:def test_unauthorized_raises_and_is_not_retried(self) -> None: