Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(rss): add collaborators, keywords and midrolls tables (#128460)

Tom Owers495d34ebf80a
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 11dca2133093..0a976bea12f4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10454,12 +10454,12 @@ Today (3): `categories`, `episodes`, `podcasts`
Diffed against: <https://api.rss.com/v4/openapi.json>
-- [ ] `locations (GET /v4/locations)` — lookup table resolving the location IDs carried on podcasts we already sync (high)
-- [ ] `podcasts/{id}/collaborators` — membership table of who works on each podcast (high)
+- [ ] `locations (GET /v4/locations)` — lookup table resolving the location IDs carried on podcasts we already sync (high); skipped: a search endpoint that requires a free-text `filter` string, not a listable collection
+- [x] `podcasts/{id}/collaborators` — membership table of who works on each podcast (high) → `collaborators` (full refresh, fan-out over `podcasts`)
- [ ] `roles (GET /v4/roles)` — lookup resolving the role IDs on collaborators (medium)
- [ ] `podcasts/{id}/episodes/{id}/chapters` — per-episode chapter structure, a real sub-resource of data we already sync (medium)
-- [ ] `podcasts/{id}/episodes/{id}/midrolls` — ad-insertion markers, the monetization dimension of each episode (medium)
-- [ ] `podcasts/{id}/keywords` — discovery/SEO dimensions per podcast (medium)
+- [x] `podcasts/{id}/episodes/{id}/midrolls` — ad-insertion markers, the monetization dimension of each episode (medium) → `midrolls` (full refresh, fan-out over `podcasts` then `episodes`)
+- [x] `podcasts/{id}/keywords` — discovery/SEO dimensions per podcast (medium) → `keywords` (full refresh, fan-out over `podcasts`)
- [ ] `podcasts/{id}/episodes/{id}/soundbites` — promo clip metadata per episode (low)
- [ ] `podcasts/{id}/episodes/{id}/transcript` — episode transcript text for content analysis (low)
diff --git a/products/warehouse_sources/sources/rss/canonical_descriptions.py b/products/warehouse_sources/sources/rss/canonical_descriptions.py
index 721eab3738e6..ad6cddc8268d 100644
--- a/products/warehouse_sources/sources/rss/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/rss/canonical_descriptions.py
@@ -64,4 +64,35 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"localized_label": "The category label localized to your account language.",
},
},
+ "collaborators": {
+ "description": "A person invited to work on a podcast hosted on your RSS.com account.",
+ "docs_url": "https://api.rss.com/v4/docs",
+ "columns": {
+ "id": "The unique ID of the collaborator.",
+ "podcast_id": "The ID of the podcast the collaborator works on.",
+ "email": "The email address of the collaborator.",
+ "name": "The name of the collaborator.",
+ "status": "The invitation status of the collaborator: INVITED, ACCEPTED, or REJECTED.",
+ "role_id": "The ID of the role the collaborator has on the podcast.",
+ },
+ },
+ "keywords": {
+ "description": "A keyword assigned to a podcast hosted on your RSS.com account.",
+ "docs_url": "https://api.rss.com/v4/docs",
+ "columns": {
+ "id": "The unique ID of the keyword.",
+ "podcast_id": "The ID of the podcast the keyword is assigned to.",
+ "label": "The keyword label.",
+ },
+ },
+ "midrolls": {
+ "description": "A midroll ad insertion point in a podcast episode.",
+ "docs_url": "https://api.rss.com/v4/docs",
+ "columns": {
+ "id": "The unique ID of the midroll.",
+ "podcast_id": "The ID of the podcast the episode belongs to.",
+ "episode_id": "The ID of the episode the midroll belongs to.",
+ "start_time_ms": "The time (in milliseconds) when the midroll starts.",
+ },
+ },
}
diff --git a/products/warehouse_sources/sources/rss/rss.py b/products/warehouse_sources/sources/rss/rss.py
index c69c8047a82b..6bb7783accb4 100644
--- a/products/warehouse_sources/sources/rss/rss.py
+++ b/products/warehouse_sources/sources/rss/rss.py
@@ -1,4 +1,5 @@
import dataclasses
+from collections.abc import Iterable
from typing import Any, Optional
from requests import Response
@@ -6,6 +7,8 @@ from requests import Response
from products.warehouse_sources.sources.rss.settings import RSS_ENDPOINTS
from products.warehouse_sources.sources.sdk import (
AuthConfig,
+ BasePaginator,
+ ClientConfig,
EndpointResource,
PageNumberPaginator,
Resource,
@@ -13,6 +16,7 @@ from products.warehouse_sources.sources.sdk import (
ResumableSourceManager,
SinglePagePaginator,
SourceResponse,
+ build_chained_resource,
make_tracked_session,
rest_api_resource,
rest_api_resources,
@@ -31,10 +35,10 @@ DEFAULT_PROBE_PATH = "/podcasts"
@dataclasses.dataclass(frozen=False)
class RssResumeConfig:
- """Resume state for the per-podcast episodes fan-out.
+ """Resume state for the per-podcast fan-outs (`episodes`, `collaborators`, `keywords`).
- `podcasts` and `categories` are single unpaginated requests, so only the episodes endpoint
- persists state. The framework fan-out now checkpoints into `fanout_state`; the legacy fields are
+ `podcasts` and `categories` are single unpaginated requests, and the chained `midrolls` fan-out
+ cannot checkpoint, so only the per-podcast fan-outs persist state. The framework fan-out now checkpoints into `fanout_state`; the legacy fields are
kept (with defaults) only so state saved before the migration still parses
(`ResumableSourceManager._load_json` does `dataclass(**saved)`). An old-shape bookmark restarts
the fan-out from scratch — merge dedupes the re-pulled rows on the primary key.
@@ -72,6 +76,14 @@ def _auth_config(api_key: str) -> AuthConfig:
return {"type": "api_key", "api_key": api_key, "name": "X-Api-Key", "location": "header"}
+def _client_config(api_key: str) -> ClientConfig:
+ return {
+ "base_url": RSS_BASE_URL,
+ "headers": {"Accept": "application/json"},
+ "auth": _auth_config(api_key),
+ }
+
+
def _top_level_resource(endpoint: str, api_key: str, team_id: int, job_id: str) -> Resource:
config = RSS_ENDPOINTS[endpoint]
rest_config: RESTAPIConfig = {
@@ -98,59 +110,65 @@ def _top_level_resource(endpoint: str, api_key: str, team_id: int, job_id: str)
return rest_api_resource(rest_config, team_id, job_id, None)
-def _fan_out_episodes_resource(
- api_key: str,
- team_id: int,
- job_id: str,
- manager: ResumableSourceManager[RssResumeConfig],
-) -> Resource:
- config = RSS_ENDPOINTS["episodes"]
- parent_config = RSS_ENDPOINTS["podcasts"]
+def _podcasts_parent_resource() -> EndpointResource:
+ return {
+ "name": RSS_ENDPOINTS["podcasts"].name,
+ "endpoint": {
+ "path": RSS_ENDPOINTS["podcasts"].path,
+ "paginator": SinglePagePaginator(),
+ "data_selector_malformed_retryable": True,
+ },
+ }
+
+
+def _podcast_child_resource(endpoint: str) -> EndpointResource:
+ config = RSS_ENDPOINTS[endpoint]
+ parent_name = RSS_ENDPOINTS["podcasts"].name
# include_from_parent injects the parent's id under `_podcasts_id`; rename it to `podcast_id`
# (kept as-is, no cast) so child rows keep the exact shape the hand-rolled source produced
# (`{**item, "podcast_id": podcast_id}`). It's part of the composite primary key.
- prefixed_parent_id = f"_{parent_config.name}_id"
+ prefixed_parent_id = f"_{parent_name}_id"
def _inject_podcast_id(row: dict[str, Any]) -> dict[str, Any]:
row["podcast_id"] = row.pop(prefixed_parent_id)
return row
- parent_resource: EndpointResource = {
- "name": parent_config.name,
- "endpoint": {
- "path": parent_config.path,
- "paginator": SinglePagePaginator(),
- "data_selector_malformed_retryable": True,
- },
- }
- child_resource: EndpointResource = {
+ params: dict[str, Any] = {"podcast_id": {"type": "resolve", "resource": parent_name, "field": "id"}}
+ paginator: BasePaginator
+ if config.paginated:
+ # `order=oldest` gives stable append-only ordering: episodes published mid-sync land on the
+ # final pages instead of shifting every earlier page boundary by one.
+ params.update({"order": "oldest", "limit": PAGE_SIZE})
+ paginator = RssPageNumberPaginator(limit=PAGE_SIZE)
+ else:
+ paginator = SinglePagePaginator()
+
+ return {
"name": config.name,
"include_from_parent": ["id"],
"data_map": _inject_podcast_id,
"endpoint": {
"path": config.path,
- "params": {
- "podcast_id": {"type": "resolve", "resource": parent_config.name, "field": "id"},
- # `order=oldest` gives stable append-only ordering: episodes published mid-sync land
- # on the final pages instead of shifting every earlier page boundary by one.
- "order": "oldest",
- "limit": PAGE_SIZE,
- },
- "paginator": RssPageNumberPaginator(limit=PAGE_SIZE),
+ "params": params,
+ "paginator": paginator,
# A bare JSON array is expected; a non-list body means the response shape changed —
# fail loud instead of silently syncing 0 rows.
"data_selector_required": True,
},
}
+
+def _fan_out_podcasts_resource(
+ endpoint: str,
+ api_key: str,
+ team_id: int,
+ job_id: str,
+ manager: ResumableSourceManager[RssResumeConfig],
+) -> Resource:
rest_config: RESTAPIConfig = {
- "client": {
- "base_url": RSS_BASE_URL,
- "headers": {"Accept": "application/json"},
- "auth": _auth_config(api_key),
- },
- "resources": [parent_resource, child_resource],
+ "client": _client_config(api_key),
+ "resources": [_podcasts_parent_resource(), _podcast_child_resource(endpoint)],
}
initial_paginator_state: Optional[dict[str, Any]] = None
@@ -173,7 +191,38 @@ def _fan_out_episodes_resource(
resume_hook=save_checkpoint,
initial_paginator_state=initial_paginator_state,
)
- return next(resource for resource in resources if resource.name == config.name)
+ return next(resource for resource in resources if resource.name == endpoint)
+
+
+def _fan_out_episodes_resource(endpoint: str, api_key: str, team_id: int, job_id: str) -> Iterable[Any]:
+ config = RSS_ENDPOINTS[endpoint]
+ episodes_name = RSS_ENDPOINTS["episodes"].name
+
+ child_resource: EndpointResource = {
+ "name": config.name,
+ "include_from_parent": ["id", "podcast_id"],
+ "endpoint": {
+ "path": config.path,
+ "params": {
+ "podcast_id": {"type": "resolve", "resource": episodes_name, "field": "podcast_id"},
+ "episode_id": {"type": "resolve", "resource": episodes_name, "field": "id"},
+ },
+ "paginator": SinglePagePaginator(),
+ "data_selector_required": True,
+ # An episode deleted between its episodes page and this fetch must not sink the run.
+ "response_actions": [{"status_code": 404, "action": "ignore"}],
+ },
+ }
+
+ return build_chained_resource(
+ resources=[_podcasts_parent_resource(), _podcast_child_resource(episodes_name), child_resource],
+ child_name=config.name,
+ parent_name=episodes_name,
+ parent_field_renames={"id": "episode_id", "podcast_id": "podcast_id"},
+ client_config=_client_config(api_key),
+ team_id=team_id,
+ job_id=job_id,
+ )
def rss_source(
@@ -185,8 +234,11 @@ def rss_source(
) -> SourceResponse:
config = RSS_ENDPOINTS[endpoint]
- if config.fan_out_podcasts:
- resource = _fan_out_episodes_resource(api_key, team_id, job_id, resumable_source_manager)
+ resource: Iterable[Any]
+ if config.fan_out_episodes:
+ resource = _fan_out_episodes_resource(endpoint, api_key, team_id, job_id)
+ elif config.fan_out_podcasts:
+ resource = _fan_out_podcasts_resource(endpoint, api_key, team_id, job_id, resumable_source_manager)
else:
resource = _top_level_resource(endpoint, api_key, team_id, job_id)
diff --git a/products/warehouse_sources/sources/rss/settings.py b/products/warehouse_sources/sources/rss/settings.py
index d4736643f32f..b864ca8ca24a 100644
--- a/products/warehouse_sources/sources/rss/settings.py
+++ b/products/warehouse_sources/sources/rss/settings.py
@@ -14,6 +14,10 @@ class RssEndpointConfig:
# True for per-podcast endpoints (`{podcast_id}` in the path): the source iterates
# `/podcasts` and queries the endpoint once per podcast, injecting `podcast_id` into every row.
fan_out_podcasts: bool = False
+ # True for per-episode endpoints (`{episode_id}` in the path): the source iterates every
+ # podcast's episodes and queries the endpoint once per episode, injecting `podcast_id` and
+ # `episode_id` into every row.
+ fan_out_episodes: bool = False
# RSS.com Core API v4 list endpoints (https://api.rss.com/v4/docs). All are full-refresh only: the
@@ -32,6 +36,26 @@ RSS_ENDPOINTS: dict[str, RssEndpointConfig] = {
fan_out_podcasts=True,
),
"categories": RssEndpointConfig(name="categories", path="/categories"),
+ # Collaborator ids and keyword ids are not documented as globally unique either, so the parent
+ # id is part of each per-podcast key.
+ "collaborators": RssEndpointConfig(
+ name="collaborators",
+ path="/podcasts/{podcast_id}/collaborators",
+ primary_keys=["podcast_id", "id"],
+ fan_out_podcasts=True,
+ ),
+ "keywords": RssEndpointConfig(
+ name="keywords",
+ path="/podcasts/{podcast_id}/keywords",
+ primary_keys=["podcast_id", "id"],
+ fan_out_podcasts=True,
+ ),
+ "midrolls": RssEndpointConfig(
+ name="midrolls",
+ path="/podcasts/{podcast_id}/episodes/{episode_id}/midrolls",
+ primary_keys=["podcast_id", "episode_id", "id"],
+ fan_out_episodes=True,
+ ),
}
ENDPOINTS = tuple(RSS_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/sources/rss/tests/test_rss.py b/products/warehouse_sources/sources/rss/tests/test_rss.py
index 5ba78efcbe79..5d7ae361c3bb 100644
--- a/products/warehouse_sources/sources/rss/tests/test_rss.py
+++ b/products/warehouse_sources/sources/rss/tests/test_rss.py
@@ -4,7 +4,7 @@ import requests
from parameterized import parameterized
from products.warehouse_sources.sources.rss._config import RssSourceConfig
-from products.warehouse_sources.sources.rss.rss import RSS_BASE_URL, RssResumeConfig, validate_credentials
+from products.warehouse_sources.sources.rss.rss import PAGE_SIZE, RSS_BASE_URL, RssResumeConfig, validate_credentials
from products.warehouse_sources.sources.rss.settings import ENDPOINTS, RSS_ENDPOINTS
from products.warehouse_sources.sources.rss.source import RssSource
from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, always, scripted_network
@@ -121,6 +121,139 @@ class TestEpisodesFanOut:
assert state.fanout_state is None
+class TestPerPodcastFanOut:
+ @parameterized.expand([("collaborators",), ("keywords",)])
+ def test_unpaginated_child_is_one_request_per_podcast(self, endpoint: str) -> None:
+ result = _driver().run(
+ endpoint,
+ [
+ _response([{"id": 1}, {"id": 2}]),
+ _response([{"id": "a"}, {"id": "b"}]),
+ _response([{"id": "a"}]),
+ ],
+ )
+
+ assert [(r["podcast_id"], r["id"]) for r in result.rows] == [(1, "a"), (1, "b"), (2, "a")]
+ assert all("_podcasts_id" not in r for r in result.rows)
+ assert [url.split("?")[0] for url in result.urls] == [
+ f"{RSS_BASE_URL}/podcasts",
+ f"{RSS_BASE_URL}/podcasts/1/{endpoint}",
+ f"{RSS_BASE_URL}/podcasts/2/{endpoint}",
+ ]
+ # These endpoints take no pagination params, so none are sent.
+ assert result.queries[1:] == [{}, {}]
+
+ @parameterized.expand([("collaborators",), ("keywords",)])
+ def test_empty_podcast_list_requests_no_child(self, endpoint: str) -> None:
+ result = _driver().run(endpoint, [_response([])])
+
+ assert result.rows == []
+ assert result.urls == [f"{RSS_BASE_URL}/podcasts"]
+
+ @parameterized.expand([("collaborators",), ("keywords",)])
+ def test_resume_skips_completed_podcasts(self, endpoint: str) -> None:
+ result = _driver().run(
+ endpoint,
+ [_response([{"id": 1}, {"id": 2}]), _response([{"id": "b"}])],
+ resume_state=RssResumeConfig(
+ fanout_state={"completed": [f"/podcasts/1/{endpoint}"], "current": None, "child_state": None}
+ ),
+ )
+
+ assert [(r["podcast_id"], r["id"]) for r in result.rows] == [(2, "b")]
+ assert [url.split("?")[0] for url in result.urls] == [
+ f"{RSS_BASE_URL}/podcasts",
+ f"{RSS_BASE_URL}/podcasts/2/{endpoint}",
+ ]
+
+ @parameterized.expand([("collaborators",), ("keywords",)])
+ def test_non_list_child_body_raises_loudly(self, endpoint: str) -> None:
+ result = _driver().run(endpoint, [_response([{"id": 1}]), _response({"error": "unexpected"})])
+
+ assert isinstance(result.raised, ValueError)
+ assert "list response body" in str(result.raised)
+
+
+class TestMidrollsChainedFanOut:
+ def test_injects_podcast_and_episode_ids(self) -> None:
+ result = _driver().run(
+ "midrolls",
+ [
+ _response([{"id": 1}, {"id": 2}]),
+ _response(_episodes(10, 2)),
+ _response([{"id": 100, "start_time_ms": 5000}]),
+ _response([{"id": 101, "start_time_ms": 0}, {"id": 102, "start_time_ms": 9000}]),
+ _response(_episodes(20, 1)),
+ _response([{"id": 100, "start_time_ms": 1000}]),
+ ],
+ )
+
+ # Midroll ids are only unique within an episode, so both parent ids stay on every row.
+ assert [(r["podcast_id"], r["episode_id"], r["id"]) for r in result.rows] == [
+ (1, 10, 100),
+ (1, 11, 101),
+ (1, 11, 102),
+ (2, 20, 100),
+ ]
+ assert all(not key.startswith("_") for r in result.rows for key in r)
+ assert [url.split("?")[0] for url in result.urls] == [
+ f"{RSS_BASE_URL}/podcasts",
+ f"{RSS_BASE_URL}/podcasts/1/episodes",
+ f"{RSS_BASE_URL}/podcasts/1/episodes/10/midrolls",
+ f"{RSS_BASE_URL}/podcasts/1/episodes/11/midrolls",
+ f"{RSS_BASE_URL}/podcasts/2/episodes",
+ f"{RSS_BASE_URL}/podcasts/2/episodes/20/midrolls",
+ ]
+
+ def test_paginates_episodes_before_fetching_midrolls(self) -> None:
+ result = _driver().run(
+ "midrolls",
+ [
+ _response([{"id": 1}]),
+ _response(_episodes(0, PAGE_SIZE)),
+ *[_response([]) for _ in range(PAGE_SIZE)],
+ _response(_episodes(PAGE_SIZE, 1)),
+ _response([{"id": 7, "start_time_ms": 0}]),
+ ],
+ )
+
+ assert [(r["podcast_id"], r["episode_id"], r["id"]) for r in result.rows] == [(1, PAGE_SIZE, 7)]
+ episode_pages = [request.param("page") for request in result.requests if request.path.endswith("/episodes")]
+ assert episode_pages == ["1", "2"]
+ assert len([r for r in result.requests if r.path.endswith("/midrolls")]) == PAGE_SIZE + 1
+
+ def test_deleted_episode_404_is_skipped(self) -> None:
+ result = _driver().run(
+ "midrolls",
+ [
+ _response([{"id": 1}]),
+ _response(_episodes(10, 2)),
+ _response({"error": "not found"}, status=404),
+ _response([{"id": 100, "start_time_ms": 0}]),
+ ],
+ )
+
+ assert [(r["podcast_id"], r["episode_id"], r["id"]) for r in result.rows] == [(1, 11, 100)]
+ assert result.raised is None
+
+ def test_client_error_propagates(self) -> None:
+ result = _driver().run(
+ "midrolls",
+ [_response([{"id": 1}]), _response(_episodes(10, 1)), _response({"error": "forbidden"}, status=403)],
+ )
+
+ assert isinstance(result.raised, requests.HTTPError)
+
+ def test_chain_saves_no_resume_state(self) -> None:
+ result = _driver().run(
+ "midrolls",
+ [_response([{"id": 1}]), _response(_episodes(10, 1)), _response([{"id": 100, "start_time_ms": 0}])],
+ )
+
+ # A multi-level fan-out cannot checkpoint; a retry re-fetches and merge dedupes.
+ assert result.saved_states == []
+
+
class TestRssSourceResponse:
@parameterized.expand([(e,) for e in ENDPOINTS])
def test_source_response_shape(self, endpoint: str) -> None:
@@ -133,7 +266,15 @@ class TestRssSourceResponse:
# Episodes carry no stable creation timestamp, so no endpoint is partitioned.
assert response.partition_mode is None
- def test_episodes_key_includes_parent_id(self) -> None:
- # Fan-out child rows aggregate every podcast's episodes into one table; the spec doesn't
- # document episode ids as globally unique, so the key must include the parent id.
- assert RSS_ENDPOINTS["episodes"].primary_keys == ["podcast_id", "id"]
+ @parameterized.expand(
+ [
+ ("episodes", ["podcast_id", "id"]),
+ ("collaborators", ["podcast_id", "id"]),
+ ("keywords", ["podcast_id", "id"]),
+ ("midrolls", ["podcast_id", "episode_id", "id"]),
+ ]
+ )
+ def test_fan_out_keys_include_parent_ids(self, endpoint: str, expected: list[str]) -> None:
+ # Fan-out child rows aggregate every parent's rows into one table; the spec doesn't document
+ # child ids as globally unique, so the key must include the parent ids.
+ assert RSS_ENDPOINTS[endpoint].primary_keys == expected