Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add buzzsprout downloads, contributors, fan mail and listings tables (#119204)

Tom Owersfe81c8f6196a
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index a1ffb3423b8b..c8bed2d451b7 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -334,7 +334,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/bunny/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/buttondown/buttondown.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/buttondown/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/cal_com/cal_com.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/cal_com/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/calendly/calendly.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 5bbf008159b8..2d502c6d081d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -2077,13 +2077,16 @@ Note: Fetched the official Markdown reference and checked all six resource opera
## Buzzsprout — adequate
-Today (2): `episodes`, `podcasts`
+Today (6): `contributors`, `downloads`, `episodes`, `fan_mail`, `listings`, `podcasts`
Diffed against: <https://github.com/Buzzsprout/buzzsprout-api/blob/master/README.md>
-No material gaps found.
+- [x] `/api/:podcast_id/downloads` — daily download totals per episode, requested one podcast-local date at a time (high)
+- [x] `/api/:podcast_id/contributors` — on-show credits (hosts, guests, producers) for the podcast (medium)
+- [x] `/api/:podcast_id/fan_mail` — inbound listener messages (medium)
+- [x] `/api/:podcast_id/listings` — directory listing status per podcast directory (low)
-Note: The vendor's official API repo (github.com/Buzzsprout/buzzsprout-api) contains exactly two resource sections — sections/podcasts.md and sections/episodes.md, confirmed via the GitHub contents API. Both are already exposed. There is no publicly documented downloads/analytics endpoint; I found no spec for one and did not want to invent it.
+Note: The vendor's API repo later added more resource sections (stats, contributors, fan mail, listings and others). The per-episode `/api/:podcast_id/:episode_id/downloads` detail route is not synced: it needs a request per episode per day.
## CalCom — gaps
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/buzzsprout.py b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/buzzsprout.py
index 1167aa7a56b8..4c83eeae103f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/buzzsprout.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/buzzsprout.py
@@ -1,15 +1,30 @@
+from collections.abc import Callable
+from datetime import UTC, date, datetime, timedelta
+from typing import Any, ClassVar, Optional
+
+from requests import Request, Response
+
+from posthog.dataclasses import frozen
+
from products.warehouse_sources.backend.temporal.data_imports.sources.buzzsprout.settings import (
BUZZSPROUT_ENDPOINTS,
BuzzsproutEndpointConfig,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.cursor import SourceCursorManager
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ default_request_timeout,
+ make_tracked_session,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
rest_api_resource,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
+ BasePaginator,
SinglePagePaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import Endpoint
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -18,6 +33,71 @@ BUZZSPROUT_BASE_URL = "https://www.buzzsprout.com/api"
# Buzzsprout blocks requests sent with a default/bot User-Agent, so we identify ourselves explicitly.
USER_AGENT = "PostHog Data Warehouse (https://posthog.com)"
+# Buzzsprout allows 60 requests a minute per token. The downloads table makes one request per day of
+# history and outruns that window, so the extra attempts let the backoff wait the window out.
+MAX_RETRIES = 8
+
+
+@frozen
+class BuzzsproutResumeConfig:
+ # Day paginator checkpoint for the downloads table: the next podcast-local day to request.
+ day_state: dict[str, Any] | None = None
+
+
+@frozen
+class BuzzsproutDownloadsCursor:
+ """The first podcast-local day (ISO date) the next incremental `downloads` sync reads.
+
+ Days with no downloads yield no rows, so the row watermark cannot record that they were read.
+ Without this cursor every sync of a quiet podcast would read every day since its last download.
+ """
+
+ cursor_kind: ClassVar[str] = "buzzsprout_downloads_day"
+ next_date: str
+
+
+class BuzzsproutDayPaginator(BasePaginator):
+ """Walks an endpoint that takes one required `date` per request, from `start_day` to `end_day`.
+
+ The rows do not name the day they count, so the paginator stamps it on each row before the page
+ is handed on.
+ """
+
+ def __init__(self, start_day: date, end_day: date) -> None:
+ super().__init__()
+ self._current_day = min(start_day, end_day)
+ self._end_day = end_day
+
+ def _apply(self, request: Request) -> None:
+ if request.params is None:
+ request.params = {}
+ request.params["date"] = self._current_day.isoformat()
+
+ def init_request(self, request: Request) -> None:
+ self._apply(request)
+
+ def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:
+ for row in data or []:
+ if isinstance(row, dict):
+ row["date"] = self._current_day.isoformat()
+ self._current_day += timedelta(days=1)
+ self._has_next_page = self._current_day <= self._end_day
+
+ def update_request(self, request: Request) -> None:
+ self._apply(request)
+
+ def get_resume_state(self) -> Optional[dict[str, Any]]:
+ return {"date": self._current_day.isoformat()} if self._has_next_page else None
+
+ def set_resume_state(self, state: dict[str, Any]) -> None:
+ saved_day = state.get("date")
+ if saved_day:
+ self._current_day = date.fromisoformat(str(saved_day))
+ self._has_next_page = True
+
+ def __str__(self) -> str:
+ return f"BuzzsproutDayPaginator(current_day={self._current_day}, end_day={self._end_day})"
+
def _auth_header_value(api_token: str) -> str:
# Buzzsprout's documented auth scheme is a static account token in the Authorization header,
@@ -31,6 +111,44 @@ def _build_path(podcast_id: str, config: BuzzsproutEndpointConfig) -> str:
return f"{podcast_id.strip()}/{config.path}"
+def _headers(api_token: str) -> dict[str, str]:
+ return {
+ "Authorization": _auth_header_value(api_token),
+ "User-Agent": USER_AGENT,
+ "Accept": "application/json",
+ }
+
+
+def _as_date(value: Any) -> date:
+ # datetime subclasses date, so it has to be tested first.
+ if isinstance(value, datetime):
+ return value.date()
+ if isinstance(value, date):
+ return value
+ return datetime.fromisoformat(str(value)).date()
+
+
+def _first_episode_day(api_token: str, podcast_id: str) -> Optional[date]:
+ """The earliest episode publish day, as a podcast cannot have downloads before it."""
+ session = make_tracked_session(redact_values=(api_token,))
+ response = session.get(
+ f"{BUZZSPROUT_BASE_URL}/{podcast_id.strip()}/episodes.json",
+ headers=_headers(api_token),
+ timeout=default_request_timeout(),
+ )
+ response.raise_for_status()
+ body = response.json()
+ # A wrong shape here would silently move the backfill start to yesterday, so fail loud instead.
+ if not isinstance(body, list):
+ raise ValueError("Buzzsprout returned an unexpected episodes response: expected a list response body")
+ days = [
+ _as_date(episode["published_at"])
+ for episode in body
+ if isinstance(episode, dict) and episode.get("published_at")
+ ]
+ return min(days) if days else None
+
+
def validate_credentials(api_token: str, podcast_id: str) -> tuple[bool, str | None]:
podcast_id = podcast_id.strip()
if not podcast_id:
@@ -41,11 +159,7 @@ def validate_credentials(api_token: str, podcast_id: str) -> tuple[bool, str | N
ok, status = validate_via_probe(
lambda: make_tracked_session(redact_values=(api_token,)),
f"{BUZZSPROUT_BASE_URL}/{podcast_id}/episodes.json",
- headers={
- "Authorization": _auth_header_value(api_token),
- "User-Agent": USER_AGENT,
- "Accept": "application/json",
- },
+ headers=_headers(api_token),
)
if ok:
@@ -64,9 +178,71 @@ def validate_credentials(api_token: str, podcast_id: str) -> tuple[bool, str | N
return False, f"Buzzsprout API returned an unexpected status code: {status}"
-def buzzsprout_source(api_token: str, podcast_id: str, endpoint: str, team_id: int, job_id: str) -> SourceResponse:
+def buzzsprout_source(
+ api_token: str,
+ podcast_id: str,
+ endpoint: str,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[BuzzsproutResumeConfig],
+ db_incremental_field_last_value: Any = None,
+ db_incremental_field_last_value_before_lookback: Any = None,
+ cursor_manager: Optional[SourceCursorManager[BuzzsproutDownloadsCursor]] = None,
+) -> SourceResponse:
config = BUZZSPROUT_ENDPOINTS[endpoint]
+ endpoint_config: Endpoint = {
+ "path": _build_path(podcast_id, config),
+ # Every documented endpoint returns a bare JSON array; require a list so an
+ # unexpected object body fails loud instead of syncing it as a row.
+ "data_selector_required": True,
+ }
+ resume_hook: Optional[Callable[[Optional[dict[str, Any]]], None]] = None
+ initial_paginator_state: Optional[dict[str, Any]] = None
+ if config.fan_out_over_days:
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ initial_paginator_state = resume.day_state if resume is not None else None
+ # Stop at yesterday in UTC: no time zone is more than 12 hours behind UTC, so that day has
+ # started for every podcast, and the endpoint never gets a future podcast-local date.
+ end_day = datetime.now(UTC).date() - timedelta(days=1)
+ stored = cursor_manager.load() if cursor_manager is not None else None
+ known_starts: list[date] = []
+ if db_incremental_field_last_value is not None:
+ known_starts.append(_as_date(db_incremental_field_last_value))
+ if stored is not None:
+ # The pipeline moves the watermark back by the schema's lookback. The cursor moves back by
+ # the same amount, so a lookback still reads earlier days again.
+ lookback = timedelta(0)
+ if (
+ db_incremental_field_last_value is not None
+ and db_incremental_field_last_value_before_lookback is not None
+ ):
+ lookback = max(
+ _as_date(db_incremental_field_last_value_before_lookback)
+ - _as_date(db_incremental_field_last_value),
+ timedelta(0),
+ )
+ known_starts.append(date.fromisoformat(stored.next_date) - lookback)
+
+ if known_starts:
+ start_day = max(known_starts)
+ elif initial_paginator_state is not None:
+ # The saved day replaces the start day, so skip the episodes lookup.
+ start_day = end_day
+ else:
+ start_day = _first_episode_day(api_token, podcast_id) or end_day
+ endpoint_config["paginator"] = BuzzsproutDayPaginator(start_day, end_day)
+ if cursor_manager is not None:
+ # The pipeline stores a staged cursor only after the extraction finishes, so staging it
+ # before the walk is safe. The last day can still be in progress, so the next sync reads it again.
+ cursor_manager.stage(BuzzsproutDownloadsCursor(next_date=end_day.isoformat()))
+
+ def save_day_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if state:
+ resumable_source_manager.save_state(BuzzsproutResumeConfig(day_state=state))
+
+ resume_hook = save_day_checkpoint
+
rest_config: RESTAPIConfig = {
"client": {
"base_url": BUZZSPROUT_BASE_URL,
@@ -82,21 +258,24 @@ def buzzsprout_source(api_token: str, podcast_id: str, endpoint: str, team_id: i
# Buzzsprout has no pagination: each endpoint returns its full array in one response,
# so a single fetch is the whole table.
"paginator": SinglePagePaginator(),
+ "max_retries": MAX_RETRIES,
},
"resources": [
{
"name": endpoint,
- "endpoint": {
- "path": _build_path(podcast_id, config),
- # Every documented endpoint returns a bare JSON array; require a list so an
- # unexpected object body fails loud instead of syncing it as a row.
- "data_selector_required": True,
- },
+ "endpoint": endpoint_config,
}
],
}
- resource = rest_api_resource(rest_config, team_id, job_id, None)
+ resource = rest_api_resource(
+ rest_config,
+ team_id,
+ job_id,
+ None,
+ resume_hook=resume_hook,
+ initial_paginator_state=initial_paginator_state,
+ )
return SourceResponse(
name=endpoint,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/canonical_descriptions.py
index b27f4bcdc4b6..2b0cd60c822d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/canonical_descriptions.py
@@ -53,4 +53,53 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"background_url": "URL of the podcast background image.",
},
},
+ "downloads": {
+ "description": "Daily download totals per episode for the configured podcast. Only episodes with at least one download that day have a row.",
+ "docs_url": "https://github.com/Buzzsprout/buzzsprout-api/blob/master/sections/stats.md",
+ "columns": {
+ "date": "Podcast-local calendar date (YYYY-MM-DD) the downloads are counted for.",
+ "episode_id": "Identifier of the episode that was downloaded.",
+ "total": "Number of downloads of the episode on that date.",
+ },
+ },
+ "contributors": {
+ "description": "On-show credits for the configured podcast. Contributors appear on episodes but have no Buzzsprout login.",
+ "docs_url": "https://github.com/Buzzsprout/buzzsprout-api/blob/master/sections/contributors.md",
+ "columns": {
+ "id": "Unique identifier for the contributor.",
+ "name": "Contributor name.",
+ "role": "Contributor role: Host, Co-Host, Producer, Editor or Guest.",
+ "bio": "Contributor bio as an HTML string.",
+ "url": "Contributor website URL.",
+ },
+ },
+ "fan_mail": {
+ "description": "Inbound listener messages (text or voice) for the configured podcast. Blocked messages are not included.",
+ "docs_url": "https://github.com/Buzzsprout/buzzsprout-api/blob/master/sections/fan_mail.md",
+ "columns": {
+ "id": "Unique identifier for the message.",
+ "status": "Read status: unread or read.",
+ "published": "Whether the message is published and not blocked.",
+ "blocked": "Whether the sender is blocked.",
+ "text": "Message body, or the transcript of a voice message.",
+ "sender_info": "Short sender label, for example the last four digits of a phone number.",
+ "city": "Sender city, if known.",
+ "state": "Sender state or region, if known.",
+ "country": "Sender country, if known.",
+ "created_at": "ISO 8601 timestamp the message was received.",
+ "published_at": "ISO 8601 timestamp the message was published, or null.",
+ "message_type": "Message type: TextMessage or VoiceMessage.",
+ },
+ },
+ "listings": {
+ "description": "Podcast directory listings for the configured podcast, one row per directory.",
+ "docs_url": "https://github.com/Buzzsprout/buzzsprout-api/blob/master/sections/listings.md",
+ "columns": {
+ "id": "Listing identifier, or null for a directory the podcast was never submitted to.",
+ "directory": "Directory key, for example Apple or Spotify.",
+ "name": "Directory display name.",
+ "status": "Listing status: listed, submitted, unlisted or failed.",
+ "url": "URL of the podcast in the directory, or null.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/settings.py
index 0fb3d567e23b..81d2249197e0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/settings.py
@@ -1,10 +1,10 @@
from dataclasses import dataclass, field
from typing import Optional
-from products.warehouse_sources.backend.types import IncrementalField
+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
-@dataclass
+@dataclass(frozen=True)
class BuzzsproutEndpointConfig:
name: str
# Path suffix after the (optional) podcast_id segment, e.g. "episodes.json". Buzzsprout requires
@@ -17,11 +17,15 @@ class BuzzsproutEndpointConfig:
should_sync_default: bool = True
# Account-scoped endpoints are called without the podcast_id path segment (e.g. /api/podcasts.json).
account_scoped: bool = False
+ # Endpoints that take one required `date` per request are walked day by day.
+ fan_out_over_days: bool = False
+ default_incremental_lookback_seconds: Optional[int] = None
description: Optional[str] = None
-# Buzzsprout's v1 API exposes two list resources, both returning the full unpaginated array on every
-# request with no server-side timestamp filter — so both are full refresh only.
+# Buzzsprout's list resources return the full unpaginated array on every request with no server-side
+# timestamp filter, so they are full refresh only. `downloads` is the exception: it is queried one day
+# at a time, so the day itself is the server-side filter.
BUZZSPROUT_ENDPOINTS: dict[str, BuzzsproutEndpointConfig] = {
"episodes": BuzzsproutEndpointConfig(
name="episodes",
@@ -40,6 +44,46 @@ BUZZSPROUT_ENDPOINTS: dict[str, BuzzsproutEndpointConfig] = {
account_scoped=True,
description="Podcasts the API token can access, with their metadata. Full refresh.",
),
+ "downloads": BuzzsproutEndpointConfig(
+ name="downloads",
+ path="downloads.json",
+ # The endpoint returns one row per episode with downloads that day, and the row does not name
+ # the day, so the day is stamped on and forms part of the key.
+ primary_keys=["date", "episode_id"],
+ partition_key="date",
+ incremental_fields=[
+ {
+ "label": "date",
+ "type": IncrementalFieldType.Date,
+ "field": "date",
+ "field_type": IncrementalFieldType.Date,
+ }
+ ],
+ fan_out_over_days=True,
+ # The last synced day can still be in progress in the podcast's time zone, so re-read it.
+ default_incremental_lookback_seconds=2 * 24 * 60 * 60,
+ description="Daily download totals per episode, by podcast-local date. A full refresh reads every "
+ "day since the first published episode, one request per day.",
+ ),
+ "contributors": BuzzsproutEndpointConfig(
+ name="contributors",
+ path="contributors.json",
+ description="On-show credits (hosts, co-hosts, producers, editors and guests) for the configured "
+ "podcast. Full refresh.",
+ ),
+ "fan_mail": BuzzsproutEndpointConfig(
+ name="fan_mail",
+ path="fan_mail.json",
+ partition_key="created_at",
+ description="Unblocked listener messages sent to the configured podcast. Full refresh.",
+ ),
+ "listings": BuzzsproutEndpointConfig(
+ name="listings",
+ path="listings.json",
+ # Directories the podcast was never submitted to have a null `id`, so key on the directory.
+ primary_keys=["directory"],
+ description="Podcast directory listings (Apple Podcasts, Spotify and others) and their status. Full refresh.",
+ ),
}
ENDPOINTS = tuple(BUZZSPROUT_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/source.py
index 20cb885e7358..ad778eec9102 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/source.py
@@ -9,6 +9,8 @@ from products.warehouse_sources.backend.facade.source_config import (
SourceFieldInputConfigType,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.buzzsprout.buzzsprout import (
+ BuzzsproutDownloadsCursor,
+ BuzzsproutResumeConfig,
buzzsprout_source,
validate_credentials as validate_buzzsprout_credentials,
)
@@ -17,11 +19,13 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.buzzsprout
ENDPOINTS,
INCREMENTAL_FIELDS,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
CanonicalDescriptions,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.cursor import CursorSource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (
SourceSchema,
build_endpoint_schemas,
@@ -34,7 +38,10 @@ from products.warehouse_sources.backend.types import ExternalDataSourceType
@SourceRegistry.register
-class BuzzsproutSource(SimpleSource[BuzzsproutSourceConfig]):
+class BuzzsproutSource(
+ ResumableSource[BuzzsproutSourceConfig, BuzzsproutResumeConfig],
+ CursorSource[BuzzsproutDownloadsCursor],
+):
codebase_detection = detect(env=("^BUZZSPROUT_",))
api_docs_url = "https://github.com/Buzzsprout/buzzsprout-api"
@@ -105,14 +112,15 @@ You can find both your API token and podcast ID in your [Buzzsprout API settings
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- # Buzzsprout returns the full array on every request with no server-side timestamp filter, so
- # an "incremental" sync would cost the same as a full refresh. Every endpoint has no
- # incremental fields, so this ships full refresh only — merge on the primary key keeps
- # mutable fields (e.g. total_plays) fresh.
+ # Buzzsprout's list endpoints return the full array on every request with no server-side
+ # timestamp filter, so an "incremental" sync would cost the same as a full refresh. Only
+ # `downloads`, which is requested one day at a time, has incremental fields. It re-reads its
+ # last days, so append would duplicate rows and it supports merge only.
return build_endpoint_schemas(
ENDPOINTS,
INCREMENTAL_FIELDS,
names,
+ merge_only=[endpoint for endpoint, config in BUZZSPROUT_ENDPOINTS.items() if config.incremental_fields],
descriptions={
endpoint: config.description
for endpoint, config in BUZZSPROUT_ENDPOINTS.items()
@@ -121,6 +129,11 @@ You can find both your API token and podcast ID in your [Buzzsprout API settings
should_sync_default={
endpoint: config.should_sync_default for endpoint, config in BUZZSPROUT_ENDPOINTS.items()
},
+ default_incremental_lookback_seconds={
+ endpoint: config.default_incremental_lookback_seconds
+ for endpoint, config in BUZZSPROUT_ENDPOINTS.items()
+ if config.default_incremental_lookback_seconds is not None
+ },
)
def validate_credentials(
@@ -132,11 +145,38 @@ You can find both your API token and podcast ID in your [Buzzsprout API settings
) -> tuple[bool, str | None]:
return validate_buzzsprout_credentials(config.api_token, config.podcast_id)
- def source_for_pipeline(self, config: BuzzsproutSourceConfig, inputs: SourceInputs) -> SourceResponse:
+ def cursor_class(self) -> type[BuzzsproutDownloadsCursor]:
+ return BuzzsproutDownloadsCursor
+
+ def merge_cursors(
+ self, current: BuzzsproutDownloadsCursor, candidate: BuzzsproutDownloadsCursor
+ ) -> BuzzsproutDownloadsCursor:
+ return max(current, candidate, key=lambda cursor: cursor.next_date)
+
+ def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[BuzzsproutResumeConfig]:
+ return ResumableSourceManager[BuzzsproutResumeConfig](inputs, BuzzsproutResumeConfig)
+
+ def source_for_pipeline(
+ self,
+ config: BuzzsproutSourceConfig,
+ resumable_source_manager: ResumableSourceManager[BuzzsproutResumeConfig],
+ inputs: SourceInputs,
+ ) -> SourceResponse:
return buzzsprout_source(
api_token=config.api_token,
podcast_id=config.podcast_id,
endpoint=inputs.schema_name,
team_id=inputs.team_id,
job_id=inputs.job_id,
+ resumable_source_manager=resumable_source_manager,
+ db_incremental_field_last_value=inputs.db_incremental_field_last_value
+ if inputs.should_use_incremental_field
+ else None,
+ db_incremental_field_last_value_before_lookback=inputs.db_incremental_field_last_value_before_lookback
+ if inputs.should_use_incremental_field
+ else None,
+ # A full refresh rebuilds the table, so it starts from the first episode and keeps no cursor.
+ cursor_manager=self.get_cursor_manager(inputs)
+ if BUZZSPROUT_ENDPOINTS[inputs.schema_name].fan_out_over_days and inputs.should_use_incremental_field
+ else None,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/tests/test_buzzsprout.py b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/tests/test_buzzsprout.py
index 1af3052d48ef..4acc5a508827 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/tests/test_buzzsprout.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/tests/test_buzzsprout.py
@@ -1,4 +1,6 @@
import re
+from collections.abc import Callable
+from datetime import UTC, date, datetime, timedelta
import pytest
@@ -7,11 +9,15 @@ import requests
from products.warehouse_sources.backend.temporal.data_imports.sources.buzzsprout.buzzsprout import (
BUZZSPROUT_BASE_URL,
USER_AGENT,
+ BuzzsproutDownloadsCursor,
+ BuzzsproutResumeConfig,
validate_credentials,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.buzzsprout.settings import ENDPOINTS
from products.warehouse_sources.backend.temporal.data_imports.sources.buzzsprout.source import BuzzsproutSource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.cursor import SourceCursorManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+ RecordedRequest,
ScriptedResponse,
SourceDriver,
always,
@@ -72,10 +78,95 @@ class TestBuzzsproutRows:
assert re.search(rf"{status} Client Error: .* for url: https://www\.buzzsprout\.com", str(result.raised))
+def _yesterday() -> date:
+ return datetime.now(UTC).date() - timedelta(days=1)
+
+
+def _downloads_script(first_published_at: str) -> Callable[[RecordedRequest], ScriptedResponse]:
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path.endswith("/episodes.json"):
+ return ScriptedResponse(
+ json=[{"id": 1, "published_at": first_published_at}, {"id": 2, "published_at": None}]
+ )
+ return ScriptedResponse(json=[{"episode_id": 1, "total": int((request.param("date") or "")[-2:])}])
+
+ return respond
+
+
+class TestBuzzsproutDownloads:
+ def test_full_refresh_walks_each_day_from_first_episode_to_yesterday(self) -> None:
+ yesterday = _yesterday()
+ first_day = yesterday - timedelta(days=2)
+ # The offset keeps the podcast-local publish day, rather than shifting it to UTC.
+ result = _driver().run("downloads", _downloads_script(f"{first_day.isoformat()}T23:30:00.000-04:00"))
+
+ days = [(first_day + timedelta(days=offset)).isoformat() for offset in range(3)]
+ assert result.raised is None
+ assert result.paths == ["/api/123456/episodes.json"] + ["/api/123456/downloads.json"] * 3
+ assert result.params("date") == [None, *days]
+ assert result.rows == [{"episode_id": 1, "total": int(day[-2:]), "date": day} for day in days]
+
+ @pytest.mark.parametrize(
+ "watermark_days_ago, cursor_days_ago, lookback_days, first_read_days_ago",
+ [
+ pytest.param(2, None, 0, 2, id="watermark_only"),
+ # A quiet podcast has an old watermark, so the stored cursor decides where to start.
+ pytest.param(30, 1, 0, 1, id="cursor_ahead_of_watermark"),
+ pytest.param(30, 1, 1, 2, id="lookback_moves_the_cursor_back"),
+ ],
+ )
+ def test_incremental_start_day(
+ self,
+ watermark_days_ago: int,
+ cursor_days_ago: int | None,
+ lookback_days: int,
+ first_read_days_ago: int,
+ ) -> None:
+ yesterday = _yesterday()
+ stored = (
+ BuzzsproutDownloadsCursor(next_date=(yesterday - timedelta(days=cursor_days_ago)).isoformat())
+ if cursor_days_ago is not None
+ else None
+ )
+ cursor = SourceCursorManager(BuzzsproutDownloadsCursor, stored, BuzzsproutSource())
+ watermark = yesterday - timedelta(days=watermark_days_ago)
+ result = _driver().run(
+ "downloads",
+ _downloads_script("2019-01-01T00:00:00Z"),
+ incremental_field="date",
+ db_incremental_field_last_value=watermark.isoformat(),
+ db_incremental_field_last_value_before_lookback=(watermark + timedelta(days=lookback_days)).isoformat(),
+ source_cursor=cursor,
+ )
+
+ assert result.raised is None
+ assert result.params("date") == [
+ (yesterday - timedelta(days=offset)).isoformat() for offset in range(first_read_days_ago, -1, -1)
+ ]
+ assert cursor.staged == BuzzsproutDownloadsCursor(next_date=yesterday.isoformat())
+
+ def test_non_list_episodes_body_fails_before_choosing_the_start_day(self) -> None:
+ result = _driver().run("downloads", [ScriptedResponse(json={"unexpected": "object"})])
+
+ assert isinstance(result.raised, ValueError)
+ assert result.paths == ["/api/123456/episodes.json"]
+
+ def test_resumes_from_saved_day(self) -> None:
+ yesterday = _yesterday()
+ result = _driver().run(
+ "downloads",
+ _downloads_script("2019-01-01T00:00:00Z"),
+ resume_state=BuzzsproutResumeConfig(day_state={"date": yesterday.isoformat()}),
+ )
+
+ assert result.raised is None
+ assert result.params("date") == [yesterday.isoformat()]
+
+
class TestBuzzsproutSourceResponse:
@pytest.mark.parametrize("endpoint", list(ENDPOINTS))
def test_source_response_names_match_endpoints(self, endpoint: str) -> None:
- result = _driver().run(endpoint, [ScriptedResponse(json=[{"id": 1}])])
+ result = _driver().run(endpoint, always(ScriptedResponse(json=[{"id": 1}])))
assert result.raised is None
assert result.response is not None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/tests/test_buzzsprout_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/tests/test_buzzsprout_source.py
index 9caef60754d7..13ff7308c361 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/tests/test_buzzsprout_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/tests/test_buzzsprout_source.py
@@ -32,11 +32,12 @@ class TestBuzzsproutSource:
assert podcast_field.secret is False
@pytest.mark.parametrize("endpoint", list(ENDPOINTS))
- def test_get_schemas_full_refresh_only(self, endpoint):
- # Buzzsprout has no server-side timestamp filter, so every endpoint is full refresh only.
+ def test_get_schemas_sync_modes(self, endpoint):
+ # Only downloads has a server-side filter (its per-request date). It re-reads its last days,
+ # so append would duplicate rows.
schemas = {schema.name: schema for schema in self.source.get_schemas(self.config, self.team_id)}
- assert schemas[endpoint].supports_incremental is False
+ assert schemas[endpoint].supports_incremental is (endpoint == "downloads")
assert schemas[endpoint].supports_append is False
def test_non_retryable_errors_include_auth_failures(self):