:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(rss): add collaborators, keywords and midrolls tables (#128460)
Tom Owers495d34ebf80a
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 11dca2133093..0a976bea12f4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -10454,12 +10454,12 @@ Today (3): `categories`, `episodes`, `podcasts`Diffed against: <https://api.rss.com/v4/openapi.json>-- [ ] `locations (GET /v4/locations)` — lookup table resolving the location IDs carried on podcasts we already sync (high)-- [ ] `podcasts/{id}/collaborators` — membership table of who works on each podcast (high)+- [ ] `locations (GET /v4/locations)` — lookup table resolving the location IDs carried on podcasts we already sync (high); skipped: a search endpoint that requires a free-text `filter` string, not a listable collection+- [x] `podcasts/{id}/collaborators` — membership table of who works on each podcast (high) → `collaborators` (full refresh, fan-out over `podcasts`)- [ ] `roles (GET /v4/roles)` — lookup resolving the role IDs on collaborators (medium)- [ ] `podcasts/{id}/episodes/{id}/chapters` — per-episode chapter structure, a real sub-resource of data we already sync (medium)-- [ ] `podcasts/{id}/episodes/{id}/midrolls` — ad-insertion markers, the monetization dimension of each episode (medium)-- [ ] `podcasts/{id}/keywords` — discovery/SEO dimensions per podcast (medium)+- [x] `podcasts/{id}/episodes/{id}/midrolls` — ad-insertion markers, the monetization dimension of each episode (medium) → `midrolls` (full refresh, fan-out over `podcasts` then `episodes`)+- [x] `podcasts/{id}/keywords` — discovery/SEO dimensions per podcast (medium) → `keywords` (full refresh, fan-out over `podcasts`)- [ ] `podcasts/{id}/episodes/{id}/soundbites` — promo clip metadata per episode (low)- [ ] `podcasts/{id}/episodes/{id}/transcript` — episode transcript text for content analysis (low)diff --git a/products/warehouse_sources/sources/rss/canonical_descriptions.py b/products/warehouse_sources/sources/rss/canonical_descriptions.pyindex 721eab3738e6..ad6cddc8268d 100644--- a/products/warehouse_sources/sources/rss/canonical_descriptions.py+++ b/products/warehouse_sources/sources/rss/canonical_descriptions.py@@ -64,4 +64,35 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"localized_label": "The category label localized to your account language.",},},+ "collaborators": {+ "description": "A person invited to work on a podcast hosted on your RSS.com account.",+ "docs_url": "https://api.rss.com/v4/docs",+ "columns": {+ "id": "The unique ID of the collaborator.",+ "podcast_id": "The ID of the podcast the collaborator works on.",+ "email": "The email address of the collaborator.",+ "name": "The name of the collaborator.",+ "status": "The invitation status of the collaborator: INVITED, ACCEPTED, or REJECTED.",+ "role_id": "The ID of the role the collaborator has on the podcast.",+ },+ },+ "keywords": {+ "description": "A keyword assigned to a podcast hosted on your RSS.com account.",+ "docs_url": "https://api.rss.com/v4/docs",+ "columns": {+ "id": "The unique ID of the keyword.",+ "podcast_id": "The ID of the podcast the keyword is assigned to.",+ "label": "The keyword label.",+ },+ },+ "midrolls": {+ "description": "A midroll ad insertion point in a podcast episode.",+ "docs_url": "https://api.rss.com/v4/docs",+ "columns": {+ "id": "The unique ID of the midroll.",+ "podcast_id": "The ID of the podcast the episode belongs to.",+ "episode_id": "The ID of the episode the midroll belongs to.",+ "start_time_ms": "The time (in milliseconds) when the midroll starts.",+ },+ },}diff --git a/products/warehouse_sources/sources/rss/rss.py b/products/warehouse_sources/sources/rss/rss.pyindex c69c8047a82b..6bb7783accb4 100644--- a/products/warehouse_sources/sources/rss/rss.py+++ b/products/warehouse_sources/sources/rss/rss.py@@ -1,4 +1,5 @@import dataclasses+from collections.abc import Iterablefrom typing import Any, Optionalfrom requests import Response@@ -6,6 +7,8 @@ from requests import Responsefrom products.warehouse_sources.sources.rss.settings import RSS_ENDPOINTSfrom products.warehouse_sources.sources.sdk import (AuthConfig,+ BasePaginator,+ ClientConfig,EndpointResource,PageNumberPaginator,Resource,@@ -13,6 +16,7 @@ from products.warehouse_sources.sources.sdk import (ResumableSourceManager,SinglePagePaginator,SourceResponse,+ build_chained_resource,make_tracked_session,rest_api_resource,rest_api_resources,@@ -31,10 +35,10 @@ DEFAULT_PROBE_PATH = "/podcasts"@dataclasses.dataclass(frozen=False)class RssResumeConfig:- """Resume state for the per-podcast episodes fan-out.+ """Resume state for the per-podcast fan-outs (`episodes`, `collaborators`, `keywords`).- `podcasts` and `categories` are single unpaginated requests, so only the episodes endpoint- persists state. The framework fan-out now checkpoints into `fanout_state`; the legacy fields are+ `podcasts` and `categories` are single unpaginated requests, and the chained `midrolls` fan-out+ cannot checkpoint, so only the per-podcast fan-outs persist state. The framework fan-out now checkpoints into `fanout_state`; the legacy fields arekept (with defaults) only so state saved before the migration still parses(`ResumableSourceManager._load_json` does `dataclass(**saved)`). An old-shape bookmark restartsthe fan-out from scratch — merge dedupes the re-pulled rows on the primary key.@@ -72,6 +76,14 @@ def _auth_config(api_key: str) -> AuthConfig:return {"type": "api_key", "api_key": api_key, "name": "X-Api-Key", "location": "header"}+def _client_config(api_key: str) -> ClientConfig:+ return {+ "base_url": RSS_BASE_URL,+ "headers": {"Accept": "application/json"},+ "auth": _auth_config(api_key),+ }++def _top_level_resource(endpoint: str, api_key: str, team_id: int, job_id: str) -> Resource:config = RSS_ENDPOINTS[endpoint]rest_config: RESTAPIConfig = {@@ -98,59 +110,65 @@ def _top_level_resource(endpoint: str, api_key: str, team_id: int, job_id: str)return rest_api_resource(rest_config, team_id, job_id, None)-def _fan_out_episodes_resource(- api_key: str,- team_id: int,- job_id: str,- manager: ResumableSourceManager[RssResumeConfig],-) -> Resource:- config = RSS_ENDPOINTS["episodes"]- parent_config = RSS_ENDPOINTS["podcasts"]+def _podcasts_parent_resource() -> EndpointResource:+ return {+ "name": RSS_ENDPOINTS["podcasts"].name,+ "endpoint": {+ "path": RSS_ENDPOINTS["podcasts"].path,+ "paginator": SinglePagePaginator(),+ "data_selector_malformed_retryable": True,+ },+ }+++def _podcast_child_resource(endpoint: str) -> EndpointResource:+ config = RSS_ENDPOINTS[endpoint]+ parent_name = RSS_ENDPOINTS["podcasts"].name# include_from_parent injects the parent's id under `_podcasts_id`; rename it to `podcast_id`# (kept as-is, no cast) so child rows keep the exact shape the hand-rolled source produced# (`{**item, "podcast_id": podcast_id}`). It's part of the composite primary key.- prefixed_parent_id = f"_{parent_config.name}_id"+ prefixed_parent_id = f"_{parent_name}_id"def _inject_podcast_id(row: dict[str, Any]) -> dict[str, Any]:row["podcast_id"] = row.pop(prefixed_parent_id)return row- parent_resource: EndpointResource = {- "name": parent_config.name,- "endpoint": {- "path": parent_config.path,- "paginator": SinglePagePaginator(),- "data_selector_malformed_retryable": True,- },- }- child_resource: EndpointResource = {+ params: dict[str, Any] = {"podcast_id": {"type": "resolve", "resource": parent_name, "field": "id"}}+ paginator: BasePaginator+ if config.paginated:+ # `order=oldest` gives stable append-only ordering: episodes published mid-sync land on the+ # final pages instead of shifting every earlier page boundary by one.+ params.update({"order": "oldest", "limit": PAGE_SIZE})+ paginator = RssPageNumberPaginator(limit=PAGE_SIZE)+ else:+ paginator = SinglePagePaginator()++ return {"name": config.name,"include_from_parent": ["id"],"data_map": _inject_podcast_id,"endpoint": {"path": config.path,- "params": {- "podcast_id": {"type": "resolve", "resource": parent_config.name, "field": "id"},- # `order=oldest` gives stable append-only ordering: episodes published mid-sync land- # on the final pages instead of shifting every earlier page boundary by one.- "order": "oldest",- "limit": PAGE_SIZE,- },- "paginator": RssPageNumberPaginator(limit=PAGE_SIZE),+ "params": params,+ "paginator": paginator,# A bare JSON array is expected; a non-list body means the response shape changed —# fail loud instead of silently syncing 0 rows."data_selector_required": True,},}++def _fan_out_podcasts_resource(+ endpoint: str,+ api_key: str,+ team_id: int,+ job_id: str,+ manager: ResumableSourceManager[RssResumeConfig],+) -> Resource:rest_config: RESTAPIConfig = {- "client": {- "base_url": RSS_BASE_URL,- "headers": {"Accept": "application/json"},- "auth": _auth_config(api_key),- },- "resources": [parent_resource, child_resource],+ "client": _client_config(api_key),+ "resources": [_podcasts_parent_resource(), _podcast_child_resource(endpoint)],}initial_paginator_state: Optional[dict[str, Any]] = None@@ -173,7 +191,38 @@ def _fan_out_episodes_resource(resume_hook=save_checkpoint,initial_paginator_state=initial_paginator_state,)- return next(resource for resource in resources if resource.name == config.name)+ return next(resource for resource in resources if resource.name == endpoint)+++def _fan_out_episodes_resource(endpoint: str, api_key: str, team_id: int, job_id: str) -> Iterable[Any]:+ config = RSS_ENDPOINTS[endpoint]+ episodes_name = RSS_ENDPOINTS["episodes"].name++ child_resource: EndpointResource = {+ "name": config.name,+ "include_from_parent": ["id", "podcast_id"],+ "endpoint": {+ "path": config.path,+ "params": {+ "podcast_id": {"type": "resolve", "resource": episodes_name, "field": "podcast_id"},+ "episode_id": {"type": "resolve", "resource": episodes_name, "field": "id"},+ },+ "paginator": SinglePagePaginator(),+ "data_selector_required": True,+ # An episode deleted between its episodes page and this fetch must not sink the run.+ "response_actions": [{"status_code": 404, "action": "ignore"}],+ },+ }++ return build_chained_resource(+ resources=[_podcasts_parent_resource(), _podcast_child_resource(episodes_name), child_resource],+ child_name=config.name,+ parent_name=episodes_name,+ parent_field_renames={"id": "episode_id", "podcast_id": "podcast_id"},+ client_config=_client_config(api_key),+ team_id=team_id,+ job_id=job_id,+ )def rss_source(@@ -185,8 +234,11 @@ def rss_source() -> SourceResponse:config = RSS_ENDPOINTS[endpoint]- if config.fan_out_podcasts:- resource = _fan_out_episodes_resource(api_key, team_id, job_id, resumable_source_manager)+ resource: Iterable[Any]+ if config.fan_out_episodes:+ resource = _fan_out_episodes_resource(endpoint, api_key, team_id, job_id)+ elif config.fan_out_podcasts:+ resource = _fan_out_podcasts_resource(endpoint, api_key, team_id, job_id, resumable_source_manager)else:resource = _top_level_resource(endpoint, api_key, team_id, job_id)diff --git a/products/warehouse_sources/sources/rss/settings.py b/products/warehouse_sources/sources/rss/settings.pyindex d4736643f32f..b864ca8ca24a 100644--- a/products/warehouse_sources/sources/rss/settings.py+++ b/products/warehouse_sources/sources/rss/settings.py@@ -14,6 +14,10 @@ class RssEndpointConfig:# True for per-podcast endpoints (`{podcast_id}` in the path): the source iterates# `/podcasts` and queries the endpoint once per podcast, injecting `podcast_id` into every row.fan_out_podcasts: bool = False+ # True for per-episode endpoints (`{episode_id}` in the path): the source iterates every+ # podcast's episodes and queries the endpoint once per episode, injecting `podcast_id` and+ # `episode_id` into every row.+ fan_out_episodes: bool = False# RSS.com Core API v4 list endpoints (https://api.rss.com/v4/docs). All are full-refresh only: the@@ -32,6 +36,26 @@ RSS_ENDPOINTS: dict[str, RssEndpointConfig] = {fan_out_podcasts=True,),"categories": RssEndpointConfig(name="categories", path="/categories"),+ # Collaborator ids and keyword ids are not documented as globally unique either, so the parent+ # id is part of each per-podcast key.+ "collaborators": RssEndpointConfig(+ name="collaborators",+ path="/podcasts/{podcast_id}/collaborators",+ primary_keys=["podcast_id", "id"],+ fan_out_podcasts=True,+ ),+ "keywords": RssEndpointConfig(+ name="keywords",+ path="/podcasts/{podcast_id}/keywords",+ primary_keys=["podcast_id", "id"],+ fan_out_podcasts=True,+ ),+ "midrolls": RssEndpointConfig(+ name="midrolls",+ path="/podcasts/{podcast_id}/episodes/{episode_id}/midrolls",+ primary_keys=["podcast_id", "episode_id", "id"],+ fan_out_episodes=True,+ ),}ENDPOINTS = tuple(RSS_ENDPOINTS.keys())diff --git a/products/warehouse_sources/sources/rss/tests/test_rss.py b/products/warehouse_sources/sources/rss/tests/test_rss.pyindex 5ba78efcbe79..5d7ae361c3bb 100644--- a/products/warehouse_sources/sources/rss/tests/test_rss.py+++ b/products/warehouse_sources/sources/rss/tests/test_rss.py@@ -4,7 +4,7 @@ import requestsfrom parameterized import parameterizedfrom products.warehouse_sources.sources.rss._config import RssSourceConfig-from products.warehouse_sources.sources.rss.rss import RSS_BASE_URL, RssResumeConfig, validate_credentials+from products.warehouse_sources.sources.rss.rss import PAGE_SIZE, RSS_BASE_URL, RssResumeConfig, validate_credentialsfrom products.warehouse_sources.sources.rss.settings import ENDPOINTS, RSS_ENDPOINTSfrom products.warehouse_sources.sources.rss.source import RssSourcefrom products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, always, scripted_network@@ -121,6 +121,139 @@ class TestEpisodesFanOut:assert state.fanout_state is None+class TestPerPodcastFanOut:+ @parameterized.expand([("collaborators",), ("keywords",)])+ def test_unpaginated_child_is_one_request_per_podcast(self, endpoint: str) -> None:+ result = _driver().run(+ endpoint,+ [+ _response([{"id": 1}, {"id": 2}]),+ _response([{"id": "a"}, {"id": "b"}]),+ _response([{"id": "a"}]),+ ],+ )++ assert [(r["podcast_id"], r["id"]) for r in result.rows] == [(1, "a"), (1, "b"), (2, "a")]+ assert all("_podcasts_id" not in r for r in result.rows)+ assert [url.split("?")[0] for url in result.urls] == [+ f"{RSS_BASE_URL}/podcasts",+ f"{RSS_BASE_URL}/podcasts/1/{endpoint}",+ f"{RSS_BASE_URL}/podcasts/2/{endpoint}",+ ]+ # These endpoints take no pagination params, so none are sent.+ assert result.queries[1:] == [{}, {}]++ @parameterized.expand([("collaborators",), ("keywords",)])+ def test_empty_podcast_list_requests_no_child(self, endpoint: str) -> None:+ result = _driver().run(endpoint, [_response([])])++ assert result.rows == []+ assert result.urls == [f"{RSS_BASE_URL}/podcasts"]++ @parameterized.expand([("collaborators",), ("keywords",)])+ def test_resume_skips_completed_podcasts(self, endpoint: str) -> None:+ result = _driver().run(+ endpoint,+ [_response([{"id": 1}, {"id": 2}]), _response([{"id": "b"}])],+ resume_state=RssResumeConfig(+ fanout_state={"completed": [f"/podcasts/1/{endpoint}"], "current": None, "child_state": None}+ ),+ )++ assert [(r["podcast_id"], r["id"]) for r in result.rows] == [(2, "b")]+ assert [url.split("?")[0] for url in result.urls] == [+ f"{RSS_BASE_URL}/podcasts",+ f"{RSS_BASE_URL}/podcasts/2/{endpoint}",+ ]++ @parameterized.expand([("collaborators",), ("keywords",)])+ def test_non_list_child_body_raises_loudly(self, endpoint: str) -> None:+ result = _driver().run(endpoint, [_response([{"id": 1}]), _response({"error": "unexpected"})])++ assert isinstance(result.raised, ValueError)+ assert "list response body" in str(result.raised)+++class TestMidrollsChainedFanOut:+ def test_injects_podcast_and_episode_ids(self) -> None:+ result = _driver().run(+ "midrolls",+ [+ _response([{"id": 1}, {"id": 2}]),+ _response(_episodes(10, 2)),+ _response([{"id": 100, "start_time_ms": 5000}]),+ _response([{"id": 101, "start_time_ms": 0}, {"id": 102, "start_time_ms": 9000}]),+ _response(_episodes(20, 1)),+ _response([{"id": 100, "start_time_ms": 1000}]),+ ],+ )++ # Midroll ids are only unique within an episode, so both parent ids stay on every row.+ assert [(r["podcast_id"], r["episode_id"], r["id"]) for r in result.rows] == [+ (1, 10, 100),+ (1, 11, 101),+ (1, 11, 102),+ (2, 20, 100),+ ]+ assert all(not key.startswith("_") for r in result.rows for key in r)+ assert [url.split("?")[0] for url in result.urls] == [+ f"{RSS_BASE_URL}/podcasts",+ f"{RSS_BASE_URL}/podcasts/1/episodes",+ f"{RSS_BASE_URL}/podcasts/1/episodes/10/midrolls",+ f"{RSS_BASE_URL}/podcasts/1/episodes/11/midrolls",+ f"{RSS_BASE_URL}/podcasts/2/episodes",+ f"{RSS_BASE_URL}/podcasts/2/episodes/20/midrolls",+ ]++ def test_paginates_episodes_before_fetching_midrolls(self) -> None:+ result = _driver().run(+ "midrolls",+ [+ _response([{"id": 1}]),+ _response(_episodes(0, PAGE_SIZE)),+ *[_response([]) for _ in range(PAGE_SIZE)],+ _response(_episodes(PAGE_SIZE, 1)),+ _response([{"id": 7, "start_time_ms": 0}]),+ ],+ )++ assert [(r["podcast_id"], r["episode_id"], r["id"]) for r in result.rows] == [(1, PAGE_SIZE, 7)]+ episode_pages = [request.param("page") for request in result.requests if request.path.endswith("/episodes")]+ assert episode_pages == ["1", "2"]+ assert len([r for r in result.requests if r.path.endswith("/midrolls")]) == PAGE_SIZE + 1++ def test_deleted_episode_404_is_skipped(self) -> None:+ result = _driver().run(+ "midrolls",+ [+ _response([{"id": 1}]),+ _response(_episodes(10, 2)),+ _response({"error": "not found"}, status=404),+ _response([{"id": 100, "start_time_ms": 0}]),+ ],+ )++ assert [(r["podcast_id"], r["episode_id"], r["id"]) for r in result.rows] == [(1, 11, 100)]+ assert result.raised is None++ def test_client_error_propagates(self) -> None:+ result = _driver().run(+ "midrolls",+ [_response([{"id": 1}]), _response(_episodes(10, 1)), _response({"error": "forbidden"}, status=403)],+ )++ assert isinstance(result.raised, requests.HTTPError)++ def test_chain_saves_no_resume_state(self) -> None:+ result = _driver().run(+ "midrolls",+ [_response([{"id": 1}]), _response(_episodes(10, 1)), _response([{"id": 100, "start_time_ms": 0}])],+ )++ # A multi-level fan-out cannot checkpoint; a retry re-fetches and merge dedupes.+ assert result.saved_states == []++class TestRssSourceResponse:@parameterized.expand([(e,) for e in ENDPOINTS])def test_source_response_shape(self, endpoint: str) -> None:@@ -133,7 +266,15 @@ class TestRssSourceResponse:# Episodes carry no stable creation timestamp, so no endpoint is partitioned.assert response.partition_mode is None- def test_episodes_key_includes_parent_id(self) -> None:- # Fan-out child rows aggregate every podcast's episodes into one table; the spec doesn't- # document episode ids as globally unique, so the key must include the parent id.- assert RSS_ENDPOINTS["episodes"].primary_keys == ["podcast_id", "id"]+ @parameterized.expand(+ [+ ("episodes", ["podcast_id", "id"]),+ ("collaborators", ["podcast_id", "id"]),+ ("keywords", ["podcast_id", "id"]),+ ("midrolls", ["podcast_id", "episode_id", "id"]),+ ]+ )+ def test_fan_out_keys_include_parent_ids(self, endpoint: str, expected: list[str]) -> None:+ # Fan-out child rows aggregate every parent's rows into one table; the spec doesn't document+ # child ids as globally unique, so the key must include the parent ids.+ assert RSS_ENDPOINTS[endpoint].primary_keys == expected