Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 2 missing tables to the YouTube Analytics source (#125295)

Tom Owers52db8b2c19de
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 66583ba07b2d..f2af31fac015 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -13510,9 +13510,9 @@ Diffed against: <https://developers.google.com/youtube/analytics/channel_reports
- [ ] `Estimated revenue reports (GET /v2/reports, metrics=estimatedRevenue,estimatedAdRevenue,estimatedRedPartnerRevenue)` — measure channel earnings and revenue trends that the existing viewing reports cannot produce (high)
- [x] `Audience retention (GET /v2/reports, dimensions=elapsedVideoTimeRatio)` — identify where viewers leave or replay each video, beyond its average viewing duration (high)
- [x] `Traffic source detail (GET /v2/reports, dimensions=insightTrafficSourceDetail)` — identify search terms, referring sites, and related videos behind broad traffic source totals (medium)
-- [ ] `Playlist reports (GET /v2/reports, filters=playlist=={playlist_id})` — measure playlist starts, viewing time, and views per start to compare playlist performance (medium)
+- [x] `Playlist reports (GET /v2/reports, filters=playlist=={playlist_id})` — measure playlist starts, viewing time, and views per start to compare playlist performance (medium)
- [ ] `Ad performance by ad type (GET /v2/reports, dimensions=adType)` — compare advertising impressions, gross revenue, and CPM across ad formats (medium)
-- [ ] `Concurrent viewers (GET /v2/reports, dimensions=livestreamPosition)` — analyze live audience peaks and viewing patterns during each broadcast (medium)
+- [x] `Concurrent viewers (GET /v2/reports, dimensions=livestreamPosition)` — analyze live audience peaks and viewing patterns during each broadcast (medium)
- [ ] `Membership cancellations (GET /v2/reports, dimensions=membershipsCancellationSurveyReason)` — explain channel membership churn using cancellation reasons and response counts (medium)
Note: Diffed official channel report definitions and https://youtubeanalytics.googleapis.com/$discovery/rest?version=v2 against the ten fixed report configurations. Video metadata was verified at https://developers.google.com/youtube/v3/docs/videos/list and can use video IDs already returned by top_videos. Retention, playlist, and live reports need parent IDs; revenue and advertising reports require the monetary read scope. These report gaps need new query configurations; excluded group management and field omissions within existing reports.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/youtube_analytics/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/youtube_analytics/canonical_descriptions.py
index 9c115648682d..e4e182e84c9a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/youtube_analytics/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/youtube_analytics/canonical_descriptions.py
@@ -4,11 +4,13 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.can
from products.warehouse_sources.backend.temporal.data_imports.sources.youtube_analytics.settings import (
AUDIENCE_RETENTION,
CHANNEL_DAILY,
+ CONCURRENT_VIEWERS,
DEMOGRAPHICS,
DEVICE_TYPES,
GEOGRAPHY,
OPERATING_SYSTEMS,
PLAYBACK_LOCATIONS,
+ PLAYLIST_DAILY,
SHARING_SERVICES,
SUBSCRIBED_STATUS,
TOP_VIDEOS,
@@ -203,4 +205,47 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"totalSegmentImpressions": "The number of times this segment of the video was viewed.",
},
},
+ PLAYLIST_DAILY: {
+ "description": (
+ "Daily activity for each playlist the channel owns: playlist views, starts, saves and viewing time, "
+ "plus views of the channel's own videos in the playlist."
+ ),
+ "docs_url": _METRICS_DOCS_URL,
+ "columns": {
+ "day": _DAY,
+ "playlist": "The ID of the playlist the metrics describe.",
+ "views": "The number of views of the channel's own videos in the playlist, wherever they were watched.",
+ "estimatedMinutesWatched": (
+ "The number of minutes users watched the channel's own videos in the playlist, wherever they were watched."
+ ),
+ "averageViewDuration": "The average length, in seconds, of a playback of the channel's own videos in the playlist.",
+ "playlistViews": "The number of times videos were viewed in the context of the playlist.",
+ "playlistStarts": "The number of times users started playback of the playlist on the web.",
+ "playlistSaves": "The net number of times users saved the playlist.",
+ "viewsPerPlaylistStart": (
+ "The average number of video views each time the playlist was started. Counts only web playback."
+ ),
+ "playlistEstimatedMinutesWatched": "The number of minutes users watched videos in the context of the playlist.",
+ "playlistAverageViewDuration": (
+ "The average length, in seconds, of a video playback in the context of the playlist."
+ ),
+ "averageTimeInPlaylist": (
+ "The estimated average time, in minutes, that a user spent watching videos in the playlist after "
+ "starting it. Counts only web playback."
+ ),
+ },
+ },
+ CONCURRENT_VIEWERS: {
+ "description": (
+ "Concurrent viewer counts through each live stream broadcast in the sync window, with one point per "
+ "minute of the stream."
+ ),
+ "docs_url": _METRICS_DOCS_URL,
+ "columns": {
+ "video": "The ID of the live-streamed video the counts describe.",
+ "livestreamPosition": "The minute of the live stream that the counts describe.",
+ "averageConcurrentViewers": "The average number of viewers watching the live stream at this position.",
+ "peakConcurrentViewers": "The largest number of viewers watching the live stream at the same time at this position.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/youtube_analytics/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/youtube_analytics/settings.py
index 0cd7881793a2..8d97bd88822f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/youtube_analytics/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/youtube_analytics/settings.py
@@ -23,11 +23,14 @@ MAX_CHANNELS_PER_PAGE = 50
# `playlistItems.list` pages at most 50 items, and `videos.list` takes at most 50 IDs per request.
MAX_VIDEOS_PER_PAGE = 50
+
+# `playlists.list` pages at most 50 playlists.
+MAX_PLAYLISTS_PER_PAGE = 50
VIDEO_PARTS = "snippet,contentDetails,statistics,status"
-# Audience retention needs one request per video, so it covers only the channel's top videos by views
+# Per-video reports need one request per video, so they cover only the channel's top videos by views
# in the sync window. 200 is the most the top-videos report returns in one request.
-MAX_RETENTION_VIDEOS = 200
+MAX_PER_VIDEO_REPORT_VIDEOS = 200
# YouTube keeps revising the most recent days for a while, and the current day is always
# partial, so the sync stops short of today and re-reads a trailing window every run.
@@ -59,6 +62,10 @@ SUBSCRIBED_STATUS = "subscribed_status"
TRAFFIC_SOURCE_DETAILS = "traffic_source_details"
VIDEOS = "videos"
AUDIENCE_RETENTION = "audience_retention"
+PLAYLIST_DAILY = "playlist_daily"
+CONCURRENT_VIEWERS = "concurrent_viewers"
+
+PLAYLIST_DIMENSION = "playlist"
# Every row carries the day it describes, either from the report's own `day` dimension or
# stamped from the one-day query window.
@@ -83,6 +90,8 @@ class YouTubeAnalyticsReportConfig:
# stamped with the value it was filtered on.
filter_dimension: str | None = None
filter_values: tuple[str, ...] = ()
+ # Take the filter values from the playlists the channel owns instead of `filter_values`.
+ filter_values_from_playlists: bool = False
should_sync_default: bool = True
@property
@@ -90,6 +99,11 @@ class YouTubeAnalyticsReportConfig:
filter_keys = [self.filter_dimension] if self.filter_dimension else []
return [DAY_FIELD, *filter_keys, *self.dimensions]
+ @property
+ def rows_ascend_by_day(self) -> bool:
+ # A multi-day window queried once per filter value restarts at the window's first day for each value.
+ return not (self.day_is_dimension and self.filter_dimension)
+
YOUTUBE_ANALYTICS_REPORTS: dict[str, YouTubeAnalyticsReportConfig] = {
CHANNEL_DAILY: YouTubeAnalyticsReportConfig(
@@ -205,21 +219,72 @@ YOUTUBE_ANALYTICS_REPORTS: dict[str, YouTubeAnalyticsReportConfig] = {
"YT_SEARCH",
),
),
+ PLAYLIST_DAILY: YouTubeAnalyticsReportConfig(
+ name=PLAYLIST_DAILY,
+ dimensions=(),
+ metrics=(
+ "views",
+ "estimatedMinutesWatched",
+ "averageViewDuration",
+ "playlistViews",
+ "playlistStarts",
+ "playlistSaves",
+ "viewsPerPlaylistStart",
+ "playlistEstimatedMinutesWatched",
+ "playlistAverageViewDuration",
+ "averageTimeInPlaylist",
+ ),
+ day_is_dimension=True,
+ sort="day",
+ # The report requires exactly one playlist, so it runs once per playlist per window.
+ filter_dimension=PLAYLIST_DIMENSION,
+ filter_values_from_playlists=True,
+ # A channel with many playlists makes the backfill heavy, so it is opt-in.
+ should_sync_default=False,
+ ),
}
-RETENTION_DIMENSION = "elapsedVideoTimeRatio"
-RETENTION_METRICS = (
- "audienceWatchRatio",
- "relativeRetentionPerformance",
- "startedWatching",
- "stoppedWatching",
- "totalSegmentImpressions",
-)
+
+@dataclass(frozen=True)
+class YouTubeAnalyticsVideoReportConfig:
+ """A report that accepts exactly one video per request, run for the channel's top videos."""
+
+ name: str
+ dimension: str
+ metrics: tuple[str, ...]
+ # Narrows the top-videos query that picks which videos to report on.
+ video_filter: str | None = None
+
+ @property
+ def primary_keys(self) -> list[str]:
+ return ["video", self.dimension]
+
+
+PER_VIDEO_REPORTS: dict[str, YouTubeAnalyticsVideoReportConfig] = {
+ AUDIENCE_RETENTION: YouTubeAnalyticsVideoReportConfig(
+ name=AUDIENCE_RETENTION,
+ dimension="elapsedVideoTimeRatio",
+ metrics=(
+ "audienceWatchRatio",
+ "relativeRetentionPerformance",
+ "startedWatching",
+ "stoppedWatching",
+ "totalSegmentImpressions",
+ ),
+ ),
+ CONCURRENT_VIEWERS: YouTubeAnalyticsVideoReportConfig(
+ name=CONCURRENT_VIEWERS,
+ dimension="livestreamPosition",
+ metrics=("averageConcurrentViewers", "peakConcurrentViewers"),
+ # Only a live stream has live views, so this picks the streams broadcast in the sync window.
+ video_filter="liveOrOnDemand==LIVE",
+ ),
+}
# Tables with no day grain. They have no server-side change filter, so they only full refresh.
FULL_REFRESH_PRIMARY_KEYS: dict[str, list[str]] = {
VIDEOS: ["id"],
- AUDIENCE_RETENTION: ["video", RETENTION_DIMENSION],
+ **{name: report.primary_keys for name, report in PER_VIDEO_REPORTS.items()},
}
ENDPOINTS = (*YOUTUBE_ANALYTICS_REPORTS.keys(), *FULL_REFRESH_PRIMARY_KEYS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/youtube_analytics/tests/test_youtube_analytics.py b/products/warehouse_sources/backend/temporal/data_imports/sources/youtube_analytics/tests/test_youtube_analytics.py
index a56061c3227f..34f229a89957 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/youtube_analytics/tests/test_youtube_analytics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/youtube_analytics/tests/test_youtube_analytics.py
@@ -23,9 +23,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
from products.warehouse_sources.backend.temporal.data_imports.sources.youtube_analytics.settings import (
AUDIENCE_RETENTION,
CHANNEL_DAILY,
+ CONCURRENT_VIEWERS,
DEMOGRAPHICS,
GEOGRAPHY,
MAX_RESULTS_PER_PAGE,
+ PLAYLIST_DAILY,
TOP_VIDEOS,
TRAFFIC_SOURCE_DETAILS,
VIDEOS,
@@ -87,6 +89,13 @@ def _playlist_page(video_ids: list[str], next_page_token: str | None = None) ->
return ScriptedResponse(json=body)
+def _playlists(playlist_ids: list[str], next_page_token: str | None = None) -> ScriptedResponse:
+ body: dict[str, Any] = {"items": [{"id": playlist_id} for playlist_id in playlist_ids]}
+ if next_page_token:
+ body["nextPageToken"] = next_page_token
+ return ScriptedResponse(json=body)
+
+
def _videos(video_ids: list[str]) -> ScriptedResponse:
return ScriptedResponse(
json={"items": [{"id": video_id, "snippet": {"title": f"Video {video_id}"}} for video_id in video_ids]}
@@ -341,28 +350,78 @@ class TestGetRows:
assert isinstance(result.raised, requests.HTTPError)
- def test_audience_retention_fetches_one_curve_per_top_video(self) -> None:
+ @parameterized.expand(
+ [
+ (AUDIENCE_RETENTION, "elapsedVideoTimeRatio", "audienceWatchRatio", None),
+ # Without the live filter the top videos would be on-demand uploads that have no live curve.
+ (CONCURRENT_VIEWERS, "livestreamPosition", "peakConcurrentViewers", "liveOrOnDemand==LIVE"),
+ ]
+ )
+ def test_per_video_report_fetches_one_report_per_top_video(
+ self, endpoint: str, dimension: str, metric: str, top_videos_filter: str | None
+ ) -> None:
script = [
ScriptedResponse(json=_result(["video", "views"], [["vid-a", 10], ["vid-b", 5]])),
- ScriptedResponse(json=_result(["elapsedVideoTimeRatio", "audienceWatchRatio"], [[0.01, 1.2], [0.02, 0.9]])),
- ScriptedResponse(json=_result(["elapsedVideoTimeRatio", "audienceWatchRatio"], [])),
+ ScriptedResponse(json=_result([dimension, metric], [[1, 120], [2, 90]])),
+ ScriptedResponse(json=_result([dimension, metric], [])),
]
result = _driver(start_date="2026-07-01").run(
- AUDIENCE_RETENTION, script, should_use_incremental_field=True, db_incremental_field_last_value="2026-07-20"
+ endpoint, script, should_use_incremental_field=True, db_incremental_field_last_value="2026-07-20"
)
assert result.raised is None
- assert result.params("filters") == [None, "video==vid-a", "video==vid-b"]
- assert result.params("dimensions") == ["video", "elapsedVideoTimeRatio", "elapsedVideoTimeRatio"]
- # Retention covers the whole sync window, never a watermark slice of it.
+ assert result.params("filters") == [top_videos_filter, "video==vid-a", "video==vid-b"]
+ assert result.params("dimensions") == ["video", dimension, dimension]
+ # A per-video report covers the whole sync window, never a watermark slice of it.
assert set(result.params("startDate")) == {"2026-07-01"}
assert set(result.params("endDate")) == {"2026-07-25"}
assert result.rows == [
- {"elapsedVideoTimeRatio": 0.01, "audienceWatchRatio": 1.2, "video": "vid-a"},
- {"elapsedVideoTimeRatio": 0.02, "audienceWatchRatio": 0.9, "video": "vid-a"},
+ {dimension: 1, metric: 120, "video": "vid-a"},
+ {dimension: 2, metric: 90, "video": "vid-a"},
+ ]
+
+ @parameterized.expand([("pinned_channel", "UC123", "channelId", "UC123"), ("blank_channel", "", "mine", "true")])
+ def test_playlist_daily_queries_each_owned_playlist_per_window(
+ self, _name: str, channel_id: str, channel_param: str, channel_value: str
+ ) -> None:
+ script = route(
+ {
+ "/playlists": [
+ _playlists(["PL-a", "PL-b"], next_page_token="page-2"),
+ # A playlist created mid-walk pushes PL-b onto the next page as well.
+ _playlists(["PL-b"]),
+ ],
+ "/reports": [
+ ScriptedResponse(json=_result(["day", "playlistStarts"], [["2026-07-24", 3], ["2026-07-25", 4]])),
+ ScriptedResponse(json=_result(["day", "playlistStarts"], [])),
+ ],
+ }
+ )
+
+ result = _driver(start_date="2026-07-24", channel_id=channel_id).run(PLAYLIST_DAILY, script)
+
+ assert result.raised is None
+ playlist_requests = [r for r in result.requests if r.path.endswith("/playlists")]
+ assert [(r.param(channel_param), r.param("pageToken")) for r in playlist_requests] == [
+ (channel_value, None),
+ (channel_value, "page-2"),
+ ]
+ # The report rejects a request without exactly one playlist.
+ assert result.params("filters") == [None, None, "playlist==PL-a", "playlist==PL-b"]
+ assert set(result.params("dimensions")) == {None, "day"}
+ assert result.rows == [
+ {"day": datetime(2026, 7, 24, tzinfo=UTC), "playlistStarts": 3, "playlist": "PL-a"},
+ {"day": datetime(2026, 7, 25, tzinfo=UTC), "playlistStarts": 4, "playlist": "PL-a"},
]
+ def test_playlist_daily_sends_no_report_request_for_a_channel_without_playlists(self) -> None:
+ result = _driver(start_date="2026-01-01").run(PLAYLIST_DAILY, route({"/playlists": [_playlists([])]}))
+
+ assert result.raised is None
+ assert result.rows == []
+ assert [r.path.rsplit("/", 1)[-1] for r in result.requests] == ["playlists"]
+
def test_resumes_from_saved_state_instead_of_the_configured_start(self) -> None:
result = _driver(start_date="2026-01-01").run(
GEOGRAPHY,
@@ -426,21 +485,27 @@ class TestValidateCredentials:
class TestYouTubeAnalyticsSourceResponse:
@parameterized.expand(
[
- (CHANNEL_DAILY, ["day"]),
- (TOP_VIDEOS, ["day", "video"]),
- (DEMOGRAPHICS, ["day", "ageGroup", "gender"]),
- (TRAFFIC_SOURCE_DETAILS, ["day", "insightTrafficSourceType", "insightTrafficSourceDetail"]),
- (VIDEOS, ["id"]),
- (AUDIENCE_RETENTION, ["video", "elapsedVideoTimeRatio"]),
+ (CHANNEL_DAILY, ["day"], "asc"),
+ (TOP_VIDEOS, ["day", "video"], "asc"),
+ (DEMOGRAPHICS, ["day", "ageGroup", "gender"], "asc"),
+ (TRAFFIC_SOURCE_DETAILS, ["day", "insightTrafficSourceType", "insightTrafficSourceDetail"], "asc"),
+ # Each playlist restarts at the first day of the 90-day window, so the rows are not in day order.
+ (PLAYLIST_DAILY, ["day", "playlist"], None),
+ (VIDEOS, ["id"], None),
+ (AUDIENCE_RETENTION, ["video", "elapsedVideoTimeRatio"], None),
+ (CONCURRENT_VIEWERS, ["video", "livestreamPosition"], None),
]
)
- def test_primary_keys_are_unique_table_wide(self, endpoint: str, expected: list[str]) -> None:
+ def test_primary_keys_are_unique_table_wide(
+ self, endpoint: str, expected: list[str], sort_mode: str | None
+ ) -> None:
with _connected_account():
response = _driver().run(endpoint, []).response
assert response is not None
assert response.name == endpoint
assert response.primary_keys == expected
+ assert response.sort_mode == sort_mode
@time_machine.travel(FROZEN_NOW, tick=False)
def test_items_is_lazy_until_iterated(self) -> None:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/youtube_analytics/youtube_analytics.py b/products/warehouse_sources/backend/temporal/data_imports/sources/youtube_analytics/youtube_analytics.py
index 614a1ee89cd1..0fdbd31c4a7d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/youtube_analytics/youtube_analytics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/youtube_analytics/youtube_analytics.py
@@ -12,24 +12,24 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.htt
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.youtube_analytics.settings import (
- AUDIENCE_RETENTION,
DATA_LATENCY_DAYS,
DAY_DIMENSION_WINDOW_DAYS,
DAY_FIELD,
DEFAULT_LOOKBACK_DAYS,
FULL_REFRESH_PRIMARY_KEYS,
MAX_CHANNELS_PER_PAGE,
- MAX_RETENTION_VIDEOS,
+ MAX_PER_VIDEO_REPORT_VIDEOS,
+ MAX_PLAYLISTS_PER_PAGE,
MAX_VIDEOS_PER_PAGE,
MIN_START_DATE,
- RETENTION_DIMENSION,
- RETENTION_METRICS,
+ PER_VIDEO_REPORTS,
VIDEO_PARTS,
VIDEOS,
YOUTUBE_ANALYTICS_HOST,
YOUTUBE_ANALYTICS_REPORTS,
YOUTUBE_DATA_HOST,
YouTubeAnalyticsReportConfig,
+ YouTubeAnalyticsVideoReportConfig,
)
REQUEST_TIMEOUT_SECONDS = 120
@@ -267,6 +267,33 @@ def _fetch_window(
return collected
+def _playlist_ids(
+ client: YouTubeAnalyticsClient,
+ channel_id: str | None,
+ resumable_source_manager: ResumableSourceManager[YouTubeAnalyticsResumeConfig],
+) -> list[str]:
+ """IDs of the playlists the channel owns. Playlist reports only cover the owner's playlists."""
+ params = {"part": "id", "maxResults": str(MAX_PLAYLISTS_PER_PAGE)}
+ cleaned = (channel_id or "").strip()
+ if cleaned:
+ params["channelId"] = cleaned
+ else:
+ params["mine"] = "true"
+ # A playlist created mid-walk shifts the list, which can repeat a playlist on the next page.
+ playlist_ids: dict[str, None] = {}
+ while True:
+ body = client.get_json(f"{YOUTUBE_DATA_HOST}/playlists?{urlencode(params)}")
+ for item in body.get("items") or []:
+ if item.get("id"):
+ playlist_ids[item["id"]] = None
+ page_token = body.get("nextPageToken")
+ if not page_token:
+ return list(playlist_ids)
+ params["pageToken"] = page_token
+ # No report row has been read yet, so a hand-off here loses nothing.
+ resumable_source_manager.safe_point()
+
+
def get_rows(
access_token: str,
refresh_access_token: AccessTokenRefresher | None,
@@ -298,13 +325,20 @@ def get_rows(
cursor = resumed.date()
logger.debug(f"YouTube Analytics: resuming {endpoint} from {cursor.isoformat()}")
+ filter_values: tuple[str | None, ...] = report.filter_values or (None,)
+ if report.filter_values_from_playlists:
+ filter_values = tuple(_playlist_ids(client, channel_id, resumable_source_manager))
+ if not filter_values:
+ logger.debug(f"YouTube Analytics: the channel owns no playlists, so {endpoint} has nothing to sync")
+ resumable_source_manager.clear_state()
+ return
+
while cursor <= end_day:
if report.day_is_dimension:
window_end = min(cursor + timedelta(days=DAY_DIMENSION_WINDOW_DAYS - 1), end_day)
else:
window_end = cursor
- filter_values: tuple[str | None, ...] = report.filter_values or (None,)
for filter_value in filter_values:
rows = _fetch_window(client, report, ids, cursor, window_end, filter_value)
if rows:
@@ -387,7 +421,8 @@ def get_video_rows(
break
-def get_audience_retention_rows(
+def get_per_video_rows(
+ report: YouTubeAnalyticsVideoReportConfig,
access_token: str,
refresh_access_token: AccessTokenRefresher | None,
channel_id: str | None,
@@ -396,7 +431,7 @@ def get_audience_retention_rows(
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[YouTubeAnalyticsResumeConfig],
) -> Iterator[list[dict[str, Any]]]:
- """Retention curves over the sync window for the channel's top videos by views in that window."""
+ """One report per video over the sync window, for the channel's top videos by views in that window."""
client = YouTubeAnalyticsClient(
access_token, refresh_access_token=refresh_access_token, api_version=api_version, logger=logger
)
@@ -407,17 +442,16 @@ def get_audience_retention_rows(
return
window = {"ids": channel_ids_param(channel_id), "startDate": start_day.isoformat(), "endDate": end_day.isoformat()}
- top_videos = rows_from_result(
- client.query(
- {
- **window,
- "dimensions": "video",
- "metrics": "views",
- "sort": "-views",
- "maxResults": str(MAX_RETENTION_VIDEOS),
- }
- )
- )
+ top_videos_params = {
+ **window,
+ "dimensions": "video",
+ "metrics": "views",
+ "sort": "-views",
+ "maxResults": str(MAX_PER_VIDEO_REPORT_VIDEOS),
+ }
+ if report.video_filter:
+ top_videos_params["filters"] = report.video_filter
+ top_videos = rows_from_result(client.query(top_videos_params))
for video_id in [row.get("video") for row in top_videos]:
if not video_id:
@@ -427,8 +461,8 @@ def get_audience_retention_rows(
client.query(
{
**window,
- "dimensions": RETENTION_DIMENSION,
- "metrics": ",".join(RETENTION_METRICS),
+ "dimensions": report.dimension,
+ "metrics": ",".join(report.metrics),
"filters": f"video=={video_id}",
}
)
@@ -466,10 +500,12 @@ def youtube_analytics_source(
primary_keys=FULL_REFRESH_PRIMARY_KEYS[VIDEOS],
)
- if endpoint == AUDIENCE_RETENTION:
+ if endpoint in PER_VIDEO_REPORTS:
+ per_video_report = PER_VIDEO_REPORTS[endpoint]
return SourceResponse(
name=endpoint,
- items=lambda: get_audience_retention_rows(
+ items=lambda: get_per_video_rows(
+ report=per_video_report,
access_token=access_token,
refresh_access_token=refresh_access_token,
channel_id=channel_id,
@@ -478,7 +514,7 @@ def youtube_analytics_source(
logger=logger,
resumable_source_manager=resumable_source_manager,
),
- primary_keys=FULL_REFRESH_PRIMARY_KEYS[AUDIENCE_RETENTION],
+ primary_keys=per_video_report.primary_keys,
)
report = YOUTUBE_ANALYTICS_REPORTS[endpoint]
@@ -503,5 +539,5 @@ def youtube_analytics_source(
partition_mode="datetime",
partition_format="month",
partition_keys=[DAY_FIELD],
- sort_mode="asc",
+ sort_mode="asc" if report.rows_ascend_by_day else None,
)