:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add buzzsprout downloads, contributors, fan mail and listings tables (#119204)
Tom Owersfe81c8f6196a
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex a1ffb3423b8b..c8bed2d451b7 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -334,7 +334,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/bunny/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/buttondown/buttondown.py1 products/warehouse_sources/backend/temporal/data_imports/sources/buttondown/settings.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/cal_com/cal_com.py1 products/warehouse_sources/backend/temporal/data_imports/sources/cal_com/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/calendly/calendly.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 5bbf008159b8..2d502c6d081d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -2077,13 +2077,16 @@ Note: Fetched the official Markdown reference and checked all six resource opera## Buzzsprout — adequate-Today (2): `episodes`, `podcasts`+Today (6): `contributors`, `downloads`, `episodes`, `fan_mail`, `listings`, `podcasts`Diffed against: <https://github.com/Buzzsprout/buzzsprout-api/blob/master/README.md>-No material gaps found.+- [x] `/api/:podcast_id/downloads` — daily download totals per episode, requested one podcast-local date at a time (high)+- [x] `/api/:podcast_id/contributors` — on-show credits (hosts, guests, producers) for the podcast (medium)+- [x] `/api/:podcast_id/fan_mail` — inbound listener messages (medium)+- [x] `/api/:podcast_id/listings` — directory listing status per podcast directory (low)-Note: The vendor's official API repo (github.com/Buzzsprout/buzzsprout-api) contains exactly two resource sections — sections/podcasts.md and sections/episodes.md, confirmed via the GitHub contents API. Both are already exposed. There is no publicly documented downloads/analytics endpoint; I found no spec for one and did not want to invent it.+Note: The vendor's API repo later added more resource sections (stats, contributors, fan mail, listings and others). The per-episode `/api/:podcast_id/:episode_id/downloads` detail route is not synced: it needs a request per episode per day.## CalCom — gapsdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/buzzsprout.py b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/buzzsprout.pyindex 1167aa7a56b8..4c83eeae103f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/buzzsprout.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/buzzsprout.py@@ -1,15 +1,30 @@+from collections.abc import Callable+from datetime import UTC, date, datetime, timedelta+from typing import Any, ClassVar, Optional++from requests import Request, Response++from posthog.dataclasses import frozen+from products.warehouse_sources.backend.temporal.data_imports.sources.buzzsprout.settings import (BUZZSPROUT_ENDPOINTS,BuzzsproutEndpointConfig,)-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session+from products.warehouse_sources.backend.temporal.data_imports.sources.common.cursor import SourceCursorManager+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (+ default_request_timeout,+ make_tracked_session,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (RESTAPIConfig,rest_api_resource,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (+ BasePaginator,SinglePagePaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import Endpoint+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse@@ -18,6 +33,71 @@ BUZZSPROUT_BASE_URL = "https://www.buzzsprout.com/api"# Buzzsprout blocks requests sent with a default/bot User-Agent, so we identify ourselves explicitly.USER_AGENT = "PostHog Data Warehouse (https://posthog.com)"+# Buzzsprout allows 60 requests a minute per token. The downloads table makes one request per day of+# history and outruns that window, so the extra attempts let the backoff wait the window out.+MAX_RETRIES = 8+++@frozen+class BuzzsproutResumeConfig:+ # Day paginator checkpoint for the downloads table: the next podcast-local day to request.+ day_state: dict[str, Any] | None = None+++@frozen+class BuzzsproutDownloadsCursor:+ """The first podcast-local day (ISO date) the next incremental `downloads` sync reads.++ Days with no downloads yield no rows, so the row watermark cannot record that they were read.+ Without this cursor every sync of a quiet podcast would read every day since its last download.+ """++ cursor_kind: ClassVar[str] = "buzzsprout_downloads_day"+ next_date: str+++class BuzzsproutDayPaginator(BasePaginator):+ """Walks an endpoint that takes one required `date` per request, from `start_day` to `end_day`.++ The rows do not name the day they count, so the paginator stamps it on each row before the page+ is handed on.+ """++ def __init__(self, start_day: date, end_day: date) -> None:+ super().__init__()+ self._current_day = min(start_day, end_day)+ self._end_day = end_day++ def _apply(self, request: Request) -> None:+ if request.params is None:+ request.params = {}+ request.params["date"] = self._current_day.isoformat()++ def init_request(self, request: Request) -> None:+ self._apply(request)++ def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:+ for row in data or []:+ if isinstance(row, dict):+ row["date"] = self._current_day.isoformat()+ self._current_day += timedelta(days=1)+ self._has_next_page = self._current_day <= self._end_day++ def update_request(self, request: Request) -> None:+ self._apply(request)++ def get_resume_state(self) -> Optional[dict[str, Any]]:+ return {"date": self._current_day.isoformat()} if self._has_next_page else None++ def set_resume_state(self, state: dict[str, Any]) -> None:+ saved_day = state.get("date")+ if saved_day:+ self._current_day = date.fromisoformat(str(saved_day))+ self._has_next_page = True++ def __str__(self) -> str:+ return f"BuzzsproutDayPaginator(current_day={self._current_day}, end_day={self._end_day})"+def _auth_header_value(api_token: str) -> str:# Buzzsprout's documented auth scheme is a static account token in the Authorization header,@@ -31,6 +111,44 @@ def _build_path(podcast_id: str, config: BuzzsproutEndpointConfig) -> str:return f"{podcast_id.strip()}/{config.path}"+def _headers(api_token: str) -> dict[str, str]:+ return {+ "Authorization": _auth_header_value(api_token),+ "User-Agent": USER_AGENT,+ "Accept": "application/json",+ }+++def _as_date(value: Any) -> date:+ # datetime subclasses date, so it has to be tested first.+ if isinstance(value, datetime):+ return value.date()+ if isinstance(value, date):+ return value+ return datetime.fromisoformat(str(value)).date()+++def _first_episode_day(api_token: str, podcast_id: str) -> Optional[date]:+ """The earliest episode publish day, as a podcast cannot have downloads before it."""+ session = make_tracked_session(redact_values=(api_token,))+ response = session.get(+ f"{BUZZSPROUT_BASE_URL}/{podcast_id.strip()}/episodes.json",+ headers=_headers(api_token),+ timeout=default_request_timeout(),+ )+ response.raise_for_status()+ body = response.json()+ # A wrong shape here would silently move the backfill start to yesterday, so fail loud instead.+ if not isinstance(body, list):+ raise ValueError("Buzzsprout returned an unexpected episodes response: expected a list response body")+ days = [+ _as_date(episode["published_at"])+ for episode in body+ if isinstance(episode, dict) and episode.get("published_at")+ ]+ return min(days) if days else None++def validate_credentials(api_token: str, podcast_id: str) -> tuple[bool, str | None]:podcast_id = podcast_id.strip()if not podcast_id:@@ -41,11 +159,7 @@ def validate_credentials(api_token: str, podcast_id: str) -> tuple[bool, str | Nok, status = validate_via_probe(lambda: make_tracked_session(redact_values=(api_token,)),f"{BUZZSPROUT_BASE_URL}/{podcast_id}/episodes.json",- headers={- "Authorization": _auth_header_value(api_token),- "User-Agent": USER_AGENT,- "Accept": "application/json",- },+ headers=_headers(api_token),)if ok:@@ -64,9 +178,71 @@ def validate_credentials(api_token: str, podcast_id: str) -> tuple[bool, str | Nreturn False, f"Buzzsprout API returned an unexpected status code: {status}"-def buzzsprout_source(api_token: str, podcast_id: str, endpoint: str, team_id: int, job_id: str) -> SourceResponse:+def buzzsprout_source(+ api_token: str,+ podcast_id: str,+ endpoint: str,+ team_id: int,+ job_id: str,+ resumable_source_manager: ResumableSourceManager[BuzzsproutResumeConfig],+ db_incremental_field_last_value: Any = None,+ db_incremental_field_last_value_before_lookback: Any = None,+ cursor_manager: Optional[SourceCursorManager[BuzzsproutDownloadsCursor]] = None,+) -> SourceResponse:config = BUZZSPROUT_ENDPOINTS[endpoint]+ endpoint_config: Endpoint = {+ "path": _build_path(podcast_id, config),+ # Every documented endpoint returns a bare JSON array; require a list so an+ # unexpected object body fails loud instead of syncing it as a row.+ "data_selector_required": True,+ }+ resume_hook: Optional[Callable[[Optional[dict[str, Any]]], None]] = None+ initial_paginator_state: Optional[dict[str, Any]] = None+ if config.fan_out_over_days:+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ initial_paginator_state = resume.day_state if resume is not None else None+ # Stop at yesterday in UTC: no time zone is more than 12 hours behind UTC, so that day has+ # started for every podcast, and the endpoint never gets a future podcast-local date.+ end_day = datetime.now(UTC).date() - timedelta(days=1)+ stored = cursor_manager.load() if cursor_manager is not None else None+ known_starts: list[date] = []+ if db_incremental_field_last_value is not None:+ known_starts.append(_as_date(db_incremental_field_last_value))+ if stored is not None:+ # The pipeline moves the watermark back by the schema's lookback. The cursor moves back by+ # the same amount, so a lookback still reads earlier days again.+ lookback = timedelta(0)+ if (+ db_incremental_field_last_value is not None+ and db_incremental_field_last_value_before_lookback is not None+ ):+ lookback = max(+ _as_date(db_incremental_field_last_value_before_lookback)+ - _as_date(db_incremental_field_last_value),+ timedelta(0),+ )+ known_starts.append(date.fromisoformat(stored.next_date) - lookback)++ if known_starts:+ start_day = max(known_starts)+ elif initial_paginator_state is not None:+ # The saved day replaces the start day, so skip the episodes lookup.+ start_day = end_day+ else:+ start_day = _first_episode_day(api_token, podcast_id) or end_day+ endpoint_config["paginator"] = BuzzsproutDayPaginator(start_day, end_day)+ if cursor_manager is not None:+ # The pipeline stores a staged cursor only after the extraction finishes, so staging it+ # before the walk is safe. The last day can still be in progress, so the next sync reads it again.+ cursor_manager.stage(BuzzsproutDownloadsCursor(next_date=end_day.isoformat()))++ def save_day_checkpoint(state: Optional[dict[str, Any]]) -> None:+ if state:+ resumable_source_manager.save_state(BuzzsproutResumeConfig(day_state=state))++ resume_hook = save_day_checkpoint+rest_config: RESTAPIConfig = {"client": {"base_url": BUZZSPROUT_BASE_URL,@@ -82,21 +258,24 @@ def buzzsprout_source(api_token: str, podcast_id: str, endpoint: str, team_id: i# Buzzsprout has no pagination: each endpoint returns its full array in one response,# so a single fetch is the whole table."paginator": SinglePagePaginator(),+ "max_retries": MAX_RETRIES,},"resources": [{"name": endpoint,- "endpoint": {- "path": _build_path(podcast_id, config),- # Every documented endpoint returns a bare JSON array; require a list so an- # unexpected object body fails loud instead of syncing it as a row.- "data_selector_required": True,- },+ "endpoint": endpoint_config,}],}- resource = rest_api_resource(rest_config, team_id, job_id, None)+ resource = rest_api_resource(+ rest_config,+ team_id,+ job_id,+ None,+ resume_hook=resume_hook,+ initial_paginator_state=initial_paginator_state,+ )return SourceResponse(name=endpoint,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/canonical_descriptions.pyindex b27f4bcdc4b6..2b0cd60c822d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/canonical_descriptions.py@@ -53,4 +53,53 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"background_url": "URL of the podcast background image.",},},+ "downloads": {+ "description": "Daily download totals per episode for the configured podcast. Only episodes with at least one download that day have a row.",+ "docs_url": "https://github.com/Buzzsprout/buzzsprout-api/blob/master/sections/stats.md",+ "columns": {+ "date": "Podcast-local calendar date (YYYY-MM-DD) the downloads are counted for.",+ "episode_id": "Identifier of the episode that was downloaded.",+ "total": "Number of downloads of the episode on that date.",+ },+ },+ "contributors": {+ "description": "On-show credits for the configured podcast. Contributors appear on episodes but have no Buzzsprout login.",+ "docs_url": "https://github.com/Buzzsprout/buzzsprout-api/blob/master/sections/contributors.md",+ "columns": {+ "id": "Unique identifier for the contributor.",+ "name": "Contributor name.",+ "role": "Contributor role: Host, Co-Host, Producer, Editor or Guest.",+ "bio": "Contributor bio as an HTML string.",+ "url": "Contributor website URL.",+ },+ },+ "fan_mail": {+ "description": "Inbound listener messages (text or voice) for the configured podcast. Blocked messages are not included.",+ "docs_url": "https://github.com/Buzzsprout/buzzsprout-api/blob/master/sections/fan_mail.md",+ "columns": {+ "id": "Unique identifier for the message.",+ "status": "Read status: unread or read.",+ "published": "Whether the message is published and not blocked.",+ "blocked": "Whether the sender is blocked.",+ "text": "Message body, or the transcript of a voice message.",+ "sender_info": "Short sender label, for example the last four digits of a phone number.",+ "city": "Sender city, if known.",+ "state": "Sender state or region, if known.",+ "country": "Sender country, if known.",+ "created_at": "ISO 8601 timestamp the message was received.",+ "published_at": "ISO 8601 timestamp the message was published, or null.",+ "message_type": "Message type: TextMessage or VoiceMessage.",+ },+ },+ "listings": {+ "description": "Podcast directory listings for the configured podcast, one row per directory.",+ "docs_url": "https://github.com/Buzzsprout/buzzsprout-api/blob/master/sections/listings.md",+ "columns": {+ "id": "Listing identifier, or null for a directory the podcast was never submitted to.",+ "directory": "Directory key, for example Apple or Spotify.",+ "name": "Directory display name.",+ "status": "Listing status: listed, submitted, unlisted or failed.",+ "url": "URL of the podcast in the directory, or null.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/settings.pyindex 0fb3d567e23b..81d2249197e0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/settings.py@@ -1,10 +1,10 @@from dataclasses import dataclass, fieldfrom typing import Optional-from products.warehouse_sources.backend.types import IncrementalField+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType-@dataclass+@dataclass(frozen=True)class BuzzsproutEndpointConfig:name: str# Path suffix after the (optional) podcast_id segment, e.g. "episodes.json". Buzzsprout requires@@ -17,11 +17,15 @@ class BuzzsproutEndpointConfig:should_sync_default: bool = True# Account-scoped endpoints are called without the podcast_id path segment (e.g. /api/podcasts.json).account_scoped: bool = False+ # Endpoints that take one required `date` per request are walked day by day.+ fan_out_over_days: bool = False+ default_incremental_lookback_seconds: Optional[int] = Nonedescription: Optional[str] = None-# Buzzsprout's v1 API exposes two list resources, both returning the full unpaginated array on every-# request with no server-side timestamp filter — so both are full refresh only.+# Buzzsprout's list resources return the full unpaginated array on every request with no server-side+# timestamp filter, so they are full refresh only. `downloads` is the exception: it is queried one day+# at a time, so the day itself is the server-side filter.BUZZSPROUT_ENDPOINTS: dict[str, BuzzsproutEndpointConfig] = {"episodes": BuzzsproutEndpointConfig(name="episodes",@@ -40,6 +44,46 @@ BUZZSPROUT_ENDPOINTS: dict[str, BuzzsproutEndpointConfig] = {account_scoped=True,description="Podcasts the API token can access, with their metadata. Full refresh.",),+ "downloads": BuzzsproutEndpointConfig(+ name="downloads",+ path="downloads.json",+ # The endpoint returns one row per episode with downloads that day, and the row does not name+ # the day, so the day is stamped on and forms part of the key.+ primary_keys=["date", "episode_id"],+ partition_key="date",+ incremental_fields=[+ {+ "label": "date",+ "type": IncrementalFieldType.Date,+ "field": "date",+ "field_type": IncrementalFieldType.Date,+ }+ ],+ fan_out_over_days=True,+ # The last synced day can still be in progress in the podcast's time zone, so re-read it.+ default_incremental_lookback_seconds=2 * 24 * 60 * 60,+ description="Daily download totals per episode, by podcast-local date. A full refresh reads every "+ "day since the first published episode, one request per day.",+ ),+ "contributors": BuzzsproutEndpointConfig(+ name="contributors",+ path="contributors.json",+ description="On-show credits (hosts, co-hosts, producers, editors and guests) for the configured "+ "podcast. Full refresh.",+ ),+ "fan_mail": BuzzsproutEndpointConfig(+ name="fan_mail",+ path="fan_mail.json",+ partition_key="created_at",+ description="Unblocked listener messages sent to the configured podcast. Full refresh.",+ ),+ "listings": BuzzsproutEndpointConfig(+ name="listings",+ path="listings.json",+ # Directories the podcast was never submitted to have a null `id`, so key on the directory.+ primary_keys=["directory"],+ description="Podcast directory listings (Apple Podcasts, Spotify and others) and their status. Full refresh.",+ ),}ENDPOINTS = tuple(BUZZSPROUT_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/source.pyindex 20cb885e7358..ad778eec9102 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/source.py@@ -9,6 +9,8 @@ from products.warehouse_sources.backend.facade.source_config import (SourceFieldInputConfigType,)from products.warehouse_sources.backend.temporal.data_imports.sources.buzzsprout.buzzsprout import (+ BuzzsproutDownloadsCursor,+ BuzzsproutResumeConfig,buzzsprout_source,validate_credentials as validate_buzzsprout_credentials,)@@ -17,11 +19,13 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.buzzsproutENDPOINTS,INCREMENTAL_FIELDS,)-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (CanonicalDescriptions,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.cursor import CursorSourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (SourceSchema,build_endpoint_schemas,@@ -34,7 +38,10 @@ from products.warehouse_sources.backend.types import ExternalDataSourceType@SourceRegistry.register-class BuzzsproutSource(SimpleSource[BuzzsproutSourceConfig]):+class BuzzsproutSource(+ ResumableSource[BuzzsproutSourceConfig, BuzzsproutResumeConfig],+ CursorSource[BuzzsproutDownloadsCursor],+):codebase_detection = detect(env=("^BUZZSPROUT_",))api_docs_url = "https://github.com/Buzzsprout/buzzsprout-api"@@ -105,14 +112,15 @@ You can find both your API token and podcast ID in your [Buzzsprout API settingsforce_refresh: bool = False,api_version: str | None = None,) -> list[SourceSchema]:- # Buzzsprout returns the full array on every request with no server-side timestamp filter, so- # an "incremental" sync would cost the same as a full refresh. Every endpoint has no- # incremental fields, so this ships full refresh only — merge on the primary key keeps- # mutable fields (e.g. total_plays) fresh.+ # Buzzsprout's list endpoints return the full array on every request with no server-side+ # timestamp filter, so an "incremental" sync would cost the same as a full refresh. Only+ # `downloads`, which is requested one day at a time, has incremental fields. It re-reads its+ # last days, so append would duplicate rows and it supports merge only.return build_endpoint_schemas(ENDPOINTS,INCREMENTAL_FIELDS,names,+ merge_only=[endpoint for endpoint, config in BUZZSPROUT_ENDPOINTS.items() if config.incremental_fields],descriptions={endpoint: config.descriptionfor endpoint, config in BUZZSPROUT_ENDPOINTS.items()@@ -121,6 +129,11 @@ You can find both your API token and podcast ID in your [Buzzsprout API settingsshould_sync_default={endpoint: config.should_sync_default for endpoint, config in BUZZSPROUT_ENDPOINTS.items()},+ default_incremental_lookback_seconds={+ endpoint: config.default_incremental_lookback_seconds+ for endpoint, config in BUZZSPROUT_ENDPOINTS.items()+ if config.default_incremental_lookback_seconds is not None+ },)def validate_credentials(@@ -132,11 +145,38 @@ You can find both your API token and podcast ID in your [Buzzsprout API settings) -> tuple[bool, str | None]:return validate_buzzsprout_credentials(config.api_token, config.podcast_id)- def source_for_pipeline(self, config: BuzzsproutSourceConfig, inputs: SourceInputs) -> SourceResponse:+ def cursor_class(self) -> type[BuzzsproutDownloadsCursor]:+ return BuzzsproutDownloadsCursor++ def merge_cursors(+ self, current: BuzzsproutDownloadsCursor, candidate: BuzzsproutDownloadsCursor+ ) -> BuzzsproutDownloadsCursor:+ return max(current, candidate, key=lambda cursor: cursor.next_date)++ def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[BuzzsproutResumeConfig]:+ return ResumableSourceManager[BuzzsproutResumeConfig](inputs, BuzzsproutResumeConfig)++ def source_for_pipeline(+ self,+ config: BuzzsproutSourceConfig,+ resumable_source_manager: ResumableSourceManager[BuzzsproutResumeConfig],+ inputs: SourceInputs,+ ) -> SourceResponse:return buzzsprout_source(api_token=config.api_token,podcast_id=config.podcast_id,endpoint=inputs.schema_name,team_id=inputs.team_id,job_id=inputs.job_id,+ resumable_source_manager=resumable_source_manager,+ db_incremental_field_last_value=inputs.db_incremental_field_last_value+ if inputs.should_use_incremental_field+ else None,+ db_incremental_field_last_value_before_lookback=inputs.db_incremental_field_last_value_before_lookback+ if inputs.should_use_incremental_field+ else None,+ # A full refresh rebuilds the table, so it starts from the first episode and keeps no cursor.+ cursor_manager=self.get_cursor_manager(inputs)+ if BUZZSPROUT_ENDPOINTS[inputs.schema_name].fan_out_over_days and inputs.should_use_incremental_field+ else None,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/tests/test_buzzsprout.py b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/tests/test_buzzsprout.pyindex 1af3052d48ef..4acc5a508827 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/tests/test_buzzsprout.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/tests/test_buzzsprout.py@@ -1,4 +1,6 @@import re+from collections.abc import Callable+from datetime import UTC, date, datetime, timedeltaimport pytest@@ -7,11 +9,15 @@ import requestsfrom products.warehouse_sources.backend.temporal.data_imports.sources.buzzsprout.buzzsprout import (BUZZSPROUT_BASE_URL,USER_AGENT,+ BuzzsproutDownloadsCursor,+ BuzzsproutResumeConfig,validate_credentials,)from products.warehouse_sources.backend.temporal.data_imports.sources.buzzsprout.settings import ENDPOINTSfrom products.warehouse_sources.backend.temporal.data_imports.sources.buzzsprout.source import BuzzsproutSource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.cursor import SourceCursorManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (+ RecordedRequest,ScriptedResponse,SourceDriver,always,@@ -72,10 +78,95 @@ class TestBuzzsproutRows:assert re.search(rf"{status} Client Error: .* for url: https://www\.buzzsprout\.com", str(result.raised))+def _yesterday() -> date:+ return datetime.now(UTC).date() - timedelta(days=1)+++def _downloads_script(first_published_at: str) -> Callable[[RecordedRequest], ScriptedResponse]:+ def respond(request: RecordedRequest) -> ScriptedResponse:+ if request.path.endswith("/episodes.json"):+ return ScriptedResponse(+ json=[{"id": 1, "published_at": first_published_at}, {"id": 2, "published_at": None}]+ )+ return ScriptedResponse(json=[{"episode_id": 1, "total": int((request.param("date") or "")[-2:])}])++ return respond+++class TestBuzzsproutDownloads:+ def test_full_refresh_walks_each_day_from_first_episode_to_yesterday(self) -> None:+ yesterday = _yesterday()+ first_day = yesterday - timedelta(days=2)+ # The offset keeps the podcast-local publish day, rather than shifting it to UTC.+ result = _driver().run("downloads", _downloads_script(f"{first_day.isoformat()}T23:30:00.000-04:00"))++ days = [(first_day + timedelta(days=offset)).isoformat() for offset in range(3)]+ assert result.raised is None+ assert result.paths == ["/api/123456/episodes.json"] + ["/api/123456/downloads.json"] * 3+ assert result.params("date") == [None, *days]+ assert result.rows == [{"episode_id": 1, "total": int(day[-2:]), "date": day} for day in days]++ @pytest.mark.parametrize(+ "watermark_days_ago, cursor_days_ago, lookback_days, first_read_days_ago",+ [+ pytest.param(2, None, 0, 2, id="watermark_only"),+ # A quiet podcast has an old watermark, so the stored cursor decides where to start.+ pytest.param(30, 1, 0, 1, id="cursor_ahead_of_watermark"),+ pytest.param(30, 1, 1, 2, id="lookback_moves_the_cursor_back"),+ ],+ )+ def test_incremental_start_day(+ self,+ watermark_days_ago: int,+ cursor_days_ago: int | None,+ lookback_days: int,+ first_read_days_ago: int,+ ) -> None:+ yesterday = _yesterday()+ stored = (+ BuzzsproutDownloadsCursor(next_date=(yesterday - timedelta(days=cursor_days_ago)).isoformat())+ if cursor_days_ago is not None+ else None+ )+ cursor = SourceCursorManager(BuzzsproutDownloadsCursor, stored, BuzzsproutSource())+ watermark = yesterday - timedelta(days=watermark_days_ago)+ result = _driver().run(+ "downloads",+ _downloads_script("2019-01-01T00:00:00Z"),+ incremental_field="date",+ db_incremental_field_last_value=watermark.isoformat(),+ db_incremental_field_last_value_before_lookback=(watermark + timedelta(days=lookback_days)).isoformat(),+ source_cursor=cursor,+ )++ assert result.raised is None+ assert result.params("date") == [+ (yesterday - timedelta(days=offset)).isoformat() for offset in range(first_read_days_ago, -1, -1)+ ]+ assert cursor.staged == BuzzsproutDownloadsCursor(next_date=yesterday.isoformat())++ def test_non_list_episodes_body_fails_before_choosing_the_start_day(self) -> None:+ result = _driver().run("downloads", [ScriptedResponse(json={"unexpected": "object"})])++ assert isinstance(result.raised, ValueError)+ assert result.paths == ["/api/123456/episodes.json"]++ def test_resumes_from_saved_day(self) -> None:+ yesterday = _yesterday()+ result = _driver().run(+ "downloads",+ _downloads_script("2019-01-01T00:00:00Z"),+ resume_state=BuzzsproutResumeConfig(day_state={"date": yesterday.isoformat()}),+ )++ assert result.raised is None+ assert result.params("date") == [yesterday.isoformat()]++class TestBuzzsproutSourceResponse:@pytest.mark.parametrize("endpoint", list(ENDPOINTS))def test_source_response_names_match_endpoints(self, endpoint: str) -> None:- result = _driver().run(endpoint, [ScriptedResponse(json=[{"id": 1}])])+ result = _driver().run(endpoint, always(ScriptedResponse(json=[{"id": 1}])))assert result.raised is Noneassert result.response is not Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/tests/test_buzzsprout_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/tests/test_buzzsprout_source.pyindex 9caef60754d7..13ff7308c361 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/tests/test_buzzsprout_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/buzzsprout/tests/test_buzzsprout_source.py@@ -32,11 +32,12 @@ class TestBuzzsproutSource:assert podcast_field.secret is False@pytest.mark.parametrize("endpoint", list(ENDPOINTS))- def test_get_schemas_full_refresh_only(self, endpoint):- # Buzzsprout has no server-side timestamp filter, so every endpoint is full refresh only.+ def test_get_schemas_sync_modes(self, endpoint):+ # Only downloads has a server-side filter (its per-request date). It re-reads its last days,+ # so append would duplicate rows.schemas = {schema.name: schema for schema in self.source.get_schemas(self.config, self.team_id)}- assert schemas[endpoint].supports_incremental is False+ assert schemas[endpoint].supports_incremental is (endpoint == "downloads")assert schemas[endpoint].supports_append is Falsedef test_non_retryable_errors_include_auth_failures(self):