Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add requested fields to four ad and seo sources (#124228)

Tom Owerse4d1d787acd8
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/ahrefs.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/ahrefs.py
index 3520d0d60585..913651f5098f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/ahrefs.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/ahrefs.py
@@ -11,8 +11,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.ahrefs.set
ENDPOINTS,
HISTORY_DAYS,
MAX_COMPETITORS,
+ SITE_METRICS_FIRST_SYNC_DAYS,
+ SITE_METRICS_MAX_SYNC_DAYS,
TOP_DOMAINS_COUNT,
TOP_DOMAINS_PAGE_ROWS,
+ DailyReport,
EndpointSettings,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
@@ -32,6 +35,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
Endpoint,
ResponseAction,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.safe_point import reach_safe_point
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import schema_for_resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -122,10 +126,7 @@ def _site_audit_target(api_key: str, project_id: str) -> dict[str, Any]:
raise ValueError("ahrefs_project_not_found")
-def _target_params(api_key: str, project_id: str, names: list[str]) -> dict[str, str]:
- if not names:
- return {}
- project = _site_audit_target(api_key, project_id)
+def _project_target_params(project: dict[str, Any], names: list[str]) -> dict[str, str]:
values = {
"target": project["target_url"],
"mode": project.get("target_mode"),
@@ -134,6 +135,12 @@ def _target_params(api_key: str, project_id: str, names: list[str]) -> dict[str,
return {name: value for name, value in values.items() if name in names and value}
+def _target_params(api_key: str, project_id: str, names: list[str]) -> dict[str, str]:
+ if not names:
+ return {}
+ return _project_target_params(_site_audit_target(api_key, project_id), names)
+
+
def _post(settings: EndpointSettings, body: dict[str, Any]) -> Endpoint:
return {"path": settings["path"], "method": "POST", "json": {"output": "json", **settings.get("body", {}), **body}}
@@ -210,16 +217,52 @@ def _endpoint_requests(
yield {"path": settings["path"], "params": params}, {}
+def _as_date(value: Any) -> date:
+ if isinstance(value, datetime):
+ return value.date()
+ if isinstance(value, date):
+ return value
+ return date.fromisoformat(str(value)[:10])
+
+
def _history_start(should_use_incremental_field: bool, last_value: Any, today: date) -> str:
if should_use_incremental_field and last_value:
- if isinstance(last_value, datetime):
- return last_value.date().isoformat()
- if isinstance(last_value, date):
- return last_value.isoformat()
- return date.fromisoformat(str(last_value)[:10]).isoformat()
+ return _as_date(last_value).isoformat()
return (today - timedelta(days=HISTORY_DAYS)).isoformat()
+def _site_metrics_dates(should_use_incremental_field: bool, last_value: Any, today: date) -> list[date]:
+ # Each day costs API units, so an incremental sync starts after the last synced day.
+ if should_use_incremental_field and last_value:
+ start = max(_as_date(last_value) + timedelta(days=1), today - timedelta(days=SITE_METRICS_MAX_SYNC_DAYS - 1))
+ else:
+ start = today - timedelta(days=SITE_METRICS_FIRST_SYNC_DAYS - 1)
+ return [start + timedelta(days=offset) for offset in range((today - start).days + 1)]
+
+
+def _site_metrics_rows(
+ api_key: str,
+ project_id: str,
+ reports: list[DailyReport],
+ should_use_incremental_field: bool,
+ db_incremental_field_last_value: Any,
+) -> Iterator[list[dict[str, object]]]:
+ dates = _site_metrics_dates(should_use_incremental_field, db_incremental_field_last_value, datetime.now(UTC).date())
+ if not dates:
+ return
+ project = _site_audit_target(api_key, project_id)
+ # No row is fetched yet, so the run can stop here before the reports of the first day.
+ reach_safe_point()
+ for day in dates:
+ row: dict[str, object] = {"project_id": project_id, "target": project["target_url"], "date": day.isoformat()}
+ for report in reports:
+ params = {**_project_target_params(project, report["target_params"]), "date": day.isoformat()}
+ values = list(_lookup(api_key, report["path"], params, report["data_selector"]))
+ found = values[0] if values else {}
+ row.update({column: found.get(field) for field, column in report["columns"].items()})
+ yield [row]
+
+
def _unique_rows(page: list[dict[str, object]], primary_keys: list[str]) -> list[dict[str, object]]:
seen: set[tuple[object, ...]] = set()
rows = []
@@ -242,6 +285,22 @@ def ahrefs_source(
) -> SourceResponse:
settings = schema_for_resource(ENDPOINTS, endpoint)
normalized_project_id = str(int(project_id))
+ daily_reports = settings.get("daily_reports")
+ if daily_reports:
+ return SourceResponse(
+ name=endpoint,
+ items=lambda: _site_metrics_rows(
+ api_key,
+ normalized_project_id,
+ daily_reports,
+ should_use_incremental_field,
+ db_incremental_field_last_value,
+ ),
+ primary_keys=settings["primary_keys"],
+ # Days are read from the oldest to today.
+ sort_mode="asc",
+ supports_resume=False,
+ )
project_scoped = settings.get("project_scoped", True)
history = settings.get("history", False)
row_limit = settings["params"].get("limit") or settings.get("body", {}).get("limit")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/canonical_descriptions.py
index 1135f67425db..67ae1cc33add 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/canonical_descriptions.py
@@ -83,6 +83,26 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"paid_cost": "Estimated cost of the monthly paid search traffic, in USD cents.",
},
},
+ "site_metrics_daily": {
+ "description": (
+ "Daily backlink, Domain Rating, and organic search totals for the selected Site Audit project's target, "
+ "one row per day."
+ ),
+ "docs_url": "https://docs.ahrefs.com/en/api/reference/site-explorer/get-metrics",
+ "columns": {
+ "project_id": "Identifier of the selected Site Audit project.",
+ "target": "Target URL of the Site Audit project that the metrics report on.",
+ "date": "Date the metrics report on.",
+ "domain_rating": "Strength of the target's backlink profile compared to other websites, from 0 to 100.",
+ "ahrefs_rank": "Rank of the target's backlink profile among all websites in Ahrefs, where 1 is the strongest.",
+ "live_backlinks": "Total number of live links from other websites to the target.",
+ "live_refdomains": "Total number of unique domains with live links to the target.",
+ "org_keywords": "Number of keywords the target ranks for in the top 100 organic search results.",
+ "org_keywords_1_3": "Number of keywords the target ranks for in the top 3 organic search results.",
+ "org_traffic": "Estimated monthly visitors to the target from organic search.",
+ "org_cost": "Estimated value of the target's monthly organic search traffic, in USD cents.",
+ },
+ },
"backlinks": {
"description": "Live backlinks to the selected Site Audit project's target, strongest referring domains first.",
"docs_url": "https://docs.ahrefs.com/en/api/reference/site-explorer/get-all-backlinks",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/settings.py
index 96dbcc34be67..a44ee20275db 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/settings.py
@@ -17,6 +17,10 @@ MAX_COMPETITORS = 20
MAX_WEB_PAGE_ROWS = 1000
TOP_DOMAINS_COUNT = 1_000_000
TOP_DOMAINS_PAGE_ROWS = 100_000
+# Each day costs 3 requests and at least 150 API units, so a first sync reads only the last week.
+SITE_METRICS_FIRST_SYNC_DAYS = 7
+# Longer than the longest sync frequency, so a scheduled sync leaves no day out.
+SITE_METRICS_MAX_SYNC_DAYS = 31
BACKLINK_FIELDS = (
"url_from",
@@ -93,9 +97,45 @@ RANK_TRACKER_FIELDS = (
)
-class EndpointSettings(TypedDict):
+class DailyReport(TypedDict):
path: str
data_selector: str
+ target_params: list[str]
+ # Response field to row column.
+ columns: dict[str, str]
+
+
+SITE_METRICS_REPORTS: list[DailyReport] = [
+ {
+ "path": "site-explorer/domain-rating",
+ "data_selector": "domain_rating",
+ "target_params": ["target"],
+ "columns": {"domain_rating": "domain_rating", "ahrefs_rank": "ahrefs_rank"},
+ },
+ {
+ "path": "site-explorer/backlinks-stats",
+ "data_selector": "metrics",
+ "target_params": ["target", "mode", "protocol"],
+ "columns": {"live": "live_backlinks", "live_refdomains": "live_refdomains"},
+ },
+ {
+ "path": "site-explorer/metrics",
+ "data_selector": "metrics",
+ "target_params": ["target", "mode", "protocol"],
+ "columns": {
+ "org_keywords": "org_keywords",
+ "org_keywords_1_3": "org_keywords_1_3",
+ "org_traffic": "org_traffic",
+ "org_cost": "org_cost",
+ },
+ },
+]
+
+
+class EndpointSettings(TypedDict):
+ # Tables built from more than one report have no single path or data selector.
+ path: NotRequired[str]
+ data_selector: NotRequired[str]
primary_keys: list[str]
params: dict[str, str | int]
description: str
@@ -112,6 +152,8 @@ class EndpointSettings(TypedDict):
snapshot_date: NotRequired[bool]
# Rank Tracker reports one device per request, so the table reads each and tags rows with it.
devices: NotRequired[list[str]]
+ # Each sync adds one row per day, merged from these reports for that date.
+ daily_reports: NotRequired[list[DailyReport]]
ENDPOINTS: dict[str, EndpointSettings] = {
@@ -169,6 +211,15 @@ ENDPOINTS: dict[str, EndpointSettings] = {
"target_params": ["target", "mode", "protocol"],
"history": True,
},
+ "site_metrics_daily": {
+ "primary_keys": ["project_id", "date"],
+ "params": {},
+ "description": (
+ "Daily Domain Rating, Ahrefs Rank, live backlink and referring domain totals, and organic "
+ "keyword and traffic totals of the selected project's target."
+ ),
+ "daily_reports": SITE_METRICS_REPORTS,
+ },
"backlinks": {
"path": "site-explorer/all-backlinks",
"data_selector": "backlinks",
@@ -293,7 +344,7 @@ ENDPOINTS: dict[str, EndpointSettings] = {
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
name: [incremental_field("date", IncrementalFieldType.Date)]
for name, endpoint in ENDPOINTS.items()
- if endpoint.get("history")
+ if endpoint.get("history") or endpoint.get("daily_reports")
}
AUTH_ERROR = "Ahrefs authentication failed. Check your API key in Account settings > API keys."
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/source.py
index 410fec26b696..62d1cc1c37fe 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/source.py
@@ -26,6 +26,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.ahrefs.set
NON_RETRYABLE_ERRORS,
QUOTA_RETRY_ERROR,
RETRYABLE_QUOTA_ERRORS,
+ SITE_METRICS_FIRST_SYNC_DAYS,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
@@ -121,9 +122,12 @@ class AhrefsSource(SimpleSource[AhrefsSourceConfig]):
"Metrics history costs 10 units per metric for each month returned. "
f"Each sync imports the latest crawl, with a sample of up to {MAX_PAGE_ROWS} pages. "
"Page data requires verified project ownership. "
- "Domain Rating, metrics history, backlinks, organic keywords, and referring domains report on the "
+ "Domain Rating, metrics history, daily site metrics, backlinks, organic keywords, and referring domains report on the "
"Site Audit project's target. Rank Tracker rankings come from the same project. "
- f"History tables start {HISTORY_DAYS} days back. Backlinks import up to {MAX_BACKLINK_ROWS} live links, "
+ f"History tables start {HISTORY_DAYS} days back. "
+ "Daily site metrics cost 150 units for each day. "
+ f"The first sync reads the last {SITE_METRICS_FIRST_SYNC_DAYS} days, and later syncs add each new day. "
+ f"Backlinks import up to {MAX_BACKLINK_ROWS} live links, "
f"organic keywords up to {MAX_KEYWORD_ROWS} keywords, and referring domains up to {MAX_REFDOMAIN_ROWS} domains. "
"Link intersect tables compare the target with the project's competitors in Ahrefs. "
"Brand Radar web pages come from the Brand Radar reports linked to the project. "
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/tests/test_ahrefs.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/tests/test_ahrefs.py
index 10cdd34bd706..9a185ccc2c2f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/tests/test_ahrefs.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/tests/test_ahrefs.py
@@ -1,5 +1,6 @@
import json
from collections.abc import Iterable, Iterator
+from datetime import date, timedelta
from typing import Any, cast
from urllib.parse import parse_qs, urlparse
@@ -24,6 +25,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
RESTClientRetryableError,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+ RecordedRequest,
ScriptedResponse,
SourceDriver,
always,
@@ -273,6 +275,112 @@ def test_rank_tracker_reads_each_device() -> None:
}
+SITE_METRICS_PATHS = [
+ "/v3/site-explorer/domain-rating",
+ "/v3/site-explorer/backlinks-stats",
+ "/v3/site-explorer/metrics",
+]
+
+
+def site_metrics_answer(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == "/v3/site-audit/projects":
+ return ScriptedResponse(json={"healthscores": [TARGET]})
+ day = int((request.param("date") or "")[-2:])
+ if request.path == "/v3/site-explorer/domain-rating":
+ return ScriptedResponse(json={"domain_rating": {"domain_rating": 50.0 + day, "ahrefs_rank": None}})
+ if request.path == "/v3/site-explorer/backlinks-stats":
+ return ScriptedResponse(
+ json={"metrics": {"all_time": 900, "all_time_refdomains": 90, "live": 500 + day, "live_refdomains": 40}}
+ )
+ return ScriptedResponse(
+ json={
+ "metrics": {
+ "org_keywords": 300 + day,
+ "org_keywords_1_3": 20,
+ "org_traffic": 1000,
+ "org_cost": None,
+ "paid_traffic": 7,
+ }
+ }
+ )
+
+
+def iso_days(start: str, end: str) -> list[str]:
+ first, last = date.fromisoformat(start), date.fromisoformat(end)
+ return [(first + timedelta(days=offset)).isoformat() for offset in range((last - first).days + 1)]
+
+
+ ("last_value", "incremental", "expected_days"),
+ [
+ (None, True, iso_days("2026-10-07", "2026-10-13")),
+ ("2026-10-11", False, iso_days("2026-10-07", "2026-10-13")),
+ ("2026-10-11", True, ["2026-10-12", "2026-10-13"]),
+ (date(2026, 10, 12), True, ["2026-10-13"]),
+ ("2026-10-13", True, []),
+ ("2026-05-01", True, iso_days("2026-09-13", "2026-10-13")),
+ ],
+)
+@time_machine.travel("2026-10-13T12:00:00Z", tick=False)
+def test_site_metrics_daily_reads_each_day_once(
+ last_value: object, incremental: bool, expected_days: list[str]
+) -> None:
+ result = SourceDriver(AhrefsSource(), AhrefsSourceConfig(api_key="test-key", project_id="123")).run(
+ "site_metrics_daily",
+ site_metrics_answer,
+ incremental_field="date",
+ should_use_incremental_field=incremental,
+ db_incremental_field_last_value=last_value,
+ )
+
+ assert result.raised is None
+ assert result.response is not None and result.response.sort_mode == "asc"
+ assert [row["date"] for row in result.rows] == expected_days
+ expected_requests: list[tuple[str, str | None]] = [("/v3/site-audit/projects", None)] if expected_days else []
+ expected_requests += [(path, day) for day in expected_days for path in SITE_METRICS_PATHS]
+ assert [(request.path, request.param("date")) for request in result.requests] == expected_requests
+
+
+@time_machine.travel("2026-10-13T12:00:00Z", tick=False)
+def test_site_metrics_daily_merges_reports_into_one_row_per_day() -> None:
+ result = SourceDriver(AhrefsSource(), AhrefsSourceConfig(api_key="test-key", project_id="00123")).run(
+ "site_metrics_daily",
+ site_metrics_answer,
+ incremental_field="date",
+ db_incremental_field_last_value="2026-10-11",
+ )
+
+ assert result.raised is None
+ assert result.rows == [
+ {
+ "project_id": "123",
+ "target": "https://example.com/",
+ "date": day,
+ "domain_rating": 50.0 + int(day[-2:]),
+ "ahrefs_rank": None,
+ "live_backlinks": 500 + int(day[-2:]),
+ "live_refdomains": 40,
+ "org_keywords": 300 + int(day[-2:]),
+ "org_keywords_1_3": 20,
+ "org_traffic": 1000,
+ "org_cost": None,
+ }
+ for day in ["2026-10-12", "2026-10-13"]
+ ]
+ target_query = {"target": ["https://example.com/"], "mode": ["domain"], "protocol": ["https"]}
+ expected_queries = {
+ "/v3/site-explorer/domain-rating": {"target": ["https://example.com/"]},
+ "/v3/site-explorer/backlinks-stats": target_query,
+ "/v3/site-explorer/metrics": target_query,
+ }
+ for request in result.requests[1:]:
+ assert {name: list(values) for name, values in request.query.items()} == {
+ "output": ["json"],
+ "date": [request.param("date")],
+ **expected_queries[request.path],
+ }
+
+
@pytest.mark.parametrize("healthscores", [[], [{"project_id": "123", "target_url": None}]])
def test_target_report_without_site_audit_target_fails_permanently(healthscores: list[dict[str, object]]) -> None:
source = AhrefsSource()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/canonical_descriptions.py
index e2c826a49cd9..f4d6ff6c1e2a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/canonical_descriptions.py
@@ -137,6 +137,17 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
**_DAILY_STATS_COLUMNS,
},
},
+ "campaign_stats_by_sender": {
+ "description": "Daily outreach totals for each campaign, split by the LinkedIn sender account that did the outreach.",
+ "docs_url": _API_DOCS_URL,
+ "columns": {
+ "linkedInAccountId": "Identifier of the LinkedIn sender account the stats cover. Matches `id` in linkedin_accounts.",
+ "campaignId": "Identifier of the campaign the stats cover.",
+ "campaignName": "Name of the campaign.",
+ "isCampaignDeleted": "Whether the campaign has been deleted.",
+ **_DAILY_STATS_COLUMNS,
+ },
+ },
"list_companies": {
"description": "A company belonging to a company list, with its LinkedIn company page details.",
"docs_url": _API_DOCS_URL,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/heyreach.py b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/heyreach.py
index c4a0b3646ff7..2e5a2185e9ff 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/heyreach.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/heyreach.py
@@ -28,6 +28,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.heyreach.s
PAGE_SIZE,
REQUEST_TIMEOUT_SECONDS,
HeyReachEndpointConfig,
+ HeyReachFanoutConfig,
)
# HeyReach launched in 2021, so this start date covers every account's full stats history.
@@ -150,12 +151,8 @@ def _top_level_rows(
)
-def _fanout_rows(api_key: str, endpoint_config: HeyReachEndpointConfig) -> Iterator[list[dict[str, Any]]]:
- fanout = endpoint_config.fanout
- assert fanout is not None
+def _parent_ids(client: RESTClient, fanout: HeyReachFanoutConfig) -> Iterator[Any]:
parent_config = ENDPOINTS[fanout.parent]
- client = _make_client(api_key)
-
for parent_page in client.paginate(
path=parent_config.path,
method="post",
@@ -165,19 +162,27 @@ def _fanout_rows(api_key: str, endpoint_config: HeyReachEndpointConfig) -> Itera
):
for parent_row in parent_page:
parent_id = parent_row.get("id")
- if parent_id is None:
- continue
- for child_page in client.paginate(
- path=endpoint_config.path,
- method="post",
- json={**endpoint_config.request_body, fanout.body_param: parent_id},
- paginator=_child_paginator(endpoint_config),
- data_selector="items",
- ):
- # An API-provided field of the same name wins over the injected parent id.
- rows = [{fanout.body_param: parent_id, **row} for row in child_page]
- if rows:
- yield rows
+ if parent_id is not None:
+ yield parent_id
+
+
+def _fanout_rows(api_key: str, endpoint_config: HeyReachEndpointConfig) -> Iterator[list[dict[str, Any]]]:
+ fanout = endpoint_config.fanout
+ assert fanout is not None
+ client = _make_client(api_key)
+
+ for parent_id in _parent_ids(client, fanout):
+ for child_page in client.paginate(
+ path=endpoint_config.path,
+ method="post",
+ json={**endpoint_config.request_body, fanout.body_param: parent_id},
+ paginator=_child_paginator(endpoint_config),
+ data_selector="items",
+ ):
+ # An API-provided field of the same name wins over the injected parent id.
+ rows = [{fanout.body_param: parent_id, **row} for row in child_page]
+ if rows:
+ yield rows
def _stats_windows(window_days: int | None) -> Iterator[StatsWindow]:
@@ -197,17 +202,19 @@ def _stats_windows(window_days: int | None) -> Iterator[StatsWindow]:
start += step
-def _daily_stats_rows(
- api_key: str,
+def _stats_window_rows(
+ client: RESTClient,
endpoint_config: HeyReachEndpointConfig,
resumable_source_manager: ResumableSourceManager[HeyReachResumeConfig],
+ filters: dict[str, list[Any]],
+ injected: dict[str, Any],
) -> Iterator[list[dict[str, Any]]]:
- client = _make_client(api_key)
for window in _stats_windows(endpoint_config.stats_window_days):
body = {
# Empty arrays mean "all senders" / "all campaigns".
"accountIds": [],
"campaignIds": [],
+ **filters,
"startDate": window.start_date,
"endDate": window.end_date,
}
@@ -225,16 +232,41 @@ def _daily_stats_rows(
date = parser.parse(day)
# The by-campaign breakdown maps each day to a list of per-campaign stats.
for entry in stats if isinstance(stats, list) else [stats]:
- rows.append({"date": date, **(entry or {})})
+ # The request filter is the source of truth for the injected ids, so they
+ # win over a field of the same name in the response.
+ rows.append({"date": date, **(entry or {}), **injected})
if len(rows) >= STATS_BATCH_SIZE:
yield rows
rows = []
if rows:
yield rows
- # Windows before the account existed return no rows; nothing is buffered or staged here.
+ # Windows before the account or sender existed return no rows; nothing is buffered or staged here.
resumable_source_manager.safe_point()
+def _daily_stats_rows(
+ api_key: str,
+ endpoint_config: HeyReachEndpointConfig,
+ resumable_source_manager: ResumableSourceManager[HeyReachResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ client = _make_client(api_key)
+ fanout = endpoint_config.fanout
+ if fanout is None:
+ yield from _stats_window_rows(client, endpoint_config, resumable_source_manager, {}, {})
+ return
+
+ # The response does not echo the filter back, so each parent id is injected into its rows.
+ column = fanout.row_column or fanout.body_param
+ for parent_id in _parent_ids(client, fanout):
+ yield from _stats_window_rows(
+ client,
+ endpoint_config,
+ resumable_source_manager,
+ {fanout.body_param: [parent_id]},
+ {column: parent_id},
+ )
+
+
def _build_response(endpoint_config: HeyReachEndpointConfig, items: Callable[[], Iterable[Any]]) -> SourceResponse:
return SourceResponse(
name=endpoint_config.name,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/settings.py
index 0dd26e9426ef..336b364d0011 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/settings.py
@@ -25,6 +25,8 @@ class HeyReachFanoutConfig:
# since the API does not echo the parent id back.
body_param: str
parent_body: dict[str, Any] = field(default_factory=dict)
+ # Column for the injected parent id, when it is not `body_param`.
+ row_column: str | None = None
@frozen
@@ -42,6 +44,7 @@ class HeyReachEndpointConfig:
stats_window_days: int | None = None
# The endpoint pages with `pageNumber`/`pageSize` in the body instead of `offset`/`limit`.
page_number_pagination: bool = False
+ should_sync_default: bool = True
ENDPOINTS: dict[str, HeyReachEndpointConfig] = {
@@ -117,6 +120,22 @@ ENDPOINTS: dict[str, HeyReachEndpointConfig] = {
# Rows grow with campaigns x days, so the full history doesn't fit one response.
stats_window_days=30,
),
+ "campaign_stats_by_sender": HeyReachEndpointConfig(
+ name="campaign_stats_by_sender",
+ path="/stats/GetOverallStatsByCampaign",
+ primary_keys=["date", "campaignId", "linkedInAccountId"],
+ partition_key="date",
+ daily_stats=True,
+ stats_window_days=30,
+ # The stats filters take lists of sender ids, so each sender gets its own `accountIds: [id]` walk.
+ fanout=HeyReachFanoutConfig(
+ parent="linkedin_accounts",
+ body_param="accountIds",
+ row_column="linkedInAccountId",
+ ),
+ # One full window walk per sender account multiplies the requests of `campaign_stats`, so it is opt-in.
+ should_sync_default=False,
+ ),
"sender_network": HeyReachEndpointConfig(
name="sender_network",
path="/MyNetwork/GetMyNetworkForSender",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/source.py
index 2d7e9f708c39..101c59621a07 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/source.py
@@ -73,7 +73,12 @@ class HeyReachSource(ResumableSource[HeyReachSourceConfig, HeyReachResumeConfig]
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)
+ return build_endpoint_schemas(
+ ENDPOINTS,
+ INCREMENTAL_FIELDS,
+ names,
+ should_sync_default={name: config.should_sync_default for name, config in ENDPOINTS.items()},
+ )
def validate_credentials(
self,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/tests/test_heyreach.py b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/tests/test_heyreach.py
index 9a7966d9e834..0583febe9833 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/tests/test_heyreach.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/tests/test_heyreach.py
@@ -169,7 +169,21 @@ class TestDailyStats:
assert body["accountIds"] == []
assert body["campaignIds"] == []
- def test_campaign_stats_requests_contiguous_windows_and_yields_one_row_per_day_and_campaign(self) -> None:
+ @pytest.mark.parametrize(
+ ("endpoint", "sender_ids", "expected_account_filters", "expected_sender_id"),
+ [
+ ("campaign_stats", None, [[]], None),
+ # Sender 8 has no stats in any window, so it yields no rows but still walks every window.
+ ("campaign_stats_by_sender", [7, 8], [[7], [8]], 7),
+ ],
+ )
+ def test_campaign_stats_walk_contiguous_windows_per_sender_filter(
+ self,
+ endpoint: str,
+ sender_ids: list[int] | None,
+ expected_account_filters: list[list[int]],
+ expected_sender_id: int | None,
+ ) -> None:
payload = {
"byDayStats": {
"2024-12-17T00:00:00Z": [
@@ -182,24 +196,42 @@ class TestDailyStats:
}
def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == "/api/public/li_account/GetAll":
+ assert sender_ids is not None
+ return _page([{"id": sender_id} for sender_id in sender_ids], len(sender_ids))
body = request.json()
- if body["startDate"] <= "2024-12-17" <= body["endDate"]:
+ if body["accountIds"] in ([], [7]) and body["startDate"] <= "2024-12-17" <= body["endDate"]:
return ScriptedResponse(json=payload)
return ScriptedResponse(json={"byDayStats": {}, "overallStats": []})
- result = _driver().run("campaign_stats", respond)
+ result = _driver().run(endpoint, respond)
assert result.raised is None
- assert set(result.paths) == {"/api/public/stats/GetOverallStatsByCampaign"}
- windows = [(parser.parse(r.json()["startDate"]), parser.parse(r.json()["endDate"])) for r in result.requests]
- assert windows[0][0] == parser.parse(STATS_START_DATE)
- assert windows[-1][1] > datetime.now(tz=UTC) - timedelta(minutes=5)
- for (start, end), (next_start, _) in itertools.pairwise(windows):
- assert end - start < timedelta(days=30)
- assert next_start - end == timedelta(milliseconds=1)
- assert [(row["date"].isoformat(), row["campaignId"], row["messagesSent"]) for row in result.rows] == [
- ("2024-12-17T00:00:00+00:00", 1, 3),
- ("2024-12-17T00:00:00+00:00", 2, 4),
+ stats_requests = [r for r in result.requests if r.path == "/api/public/stats/GetOverallStatsByCampaign"]
+ assert len(result.requests) - len(stats_requests) == (0 if sender_ids is None else 1)
+ assert all(r.json()["campaignIds"] == [] for r in stats_requests)
+
+ # Each sender filter walks its own full window series, one filter after the other.
+ account_filters = [r.json()["accountIds"] for r in stats_requests]
+ assert [key for key, _ in itertools.groupby(account_filters)] == expected_account_filters
+ for account_filter in expected_account_filters:
+ windows = [
+ (parser.parse(r.json()["startDate"]), parser.parse(r.json()["endDate"]))
+ for r in stats_requests
+ if r.json()["accountIds"] == account_filter
+ ]
+ assert windows[0][0] == parser.parse(STATS_START_DATE)
+ assert windows[-1][1] > datetime.now(tz=UTC) - timedelta(minutes=5)
+ for (start, end), (next_start, _) in itertools.pairwise(windows):
+ assert end - start < timedelta(days=30)
+ assert next_start - end == timedelta(milliseconds=1)
+
+ assert [
+ (row["date"].isoformat(), row["campaignId"], row["messagesSent"], row.get("linkedInAccountId"))
+ for row in result.rows
+ ] == [
+ ("2024-12-17T00:00:00+00:00", 1, 3, expected_sender_id),
+ ("2024-12-17T00:00:00+00:00", 2, 4, expected_sender_id),
]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/canonical_descriptions.py
index b18c59b463fe..725543d90df7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/canonical_descriptions.py
@@ -30,6 +30,13 @@ _STATS_COLUMNS = {
"follows": "Number of new followers attributed to the ad.",
}
+# Requested from LinkedIn for the campaign and creative performance tables only.
+_PERFORMANCE_ONLY_COLUMNS = {
+ "external_website_post_click_conversions": "Conversions tracked on an external website after a member clicked the ad.",
+ "external_website_post_view_conversions": "Conversions tracked on an external website after a member saw the ad without clicking it.",
+ "approximate_member_reach": "Estimated number of unique members who saw the ad at least once on this day. Do not add it up across days, because one member can count on more than one day.",
+}
+
# Professional demographic breakdowns carry the same metrics minus conversion value, which LinkedIn
# only reports for non-demographic pivots.
_DEMOGRAPHIC_STATS_COLUMNS = {
@@ -105,6 +112,11 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"intended_status": "The creative's intended status (e.g. ACTIVE, PAUSED, DRAFT).",
"is_serving": "Whether the creative is currently being served.",
"review": "The creative's ad review status.",
+ "content": "The content the creative sponsors, as LinkedIn returns it: a post reference, or inline text, spotlight or other dynamic ad content.",
+ "content_reference": "URN of the sponsored post (urn:li:share or urn:li:ugcPost) or InMail content. Empty for ads with inline content, such as text ads.",
+ "post_url": "LinkedIn URL of the sponsored post, built from content_reference. Empty when the creative does not sponsor a post.",
+ "landing_page_url": "URL a member goes to when they click the ad, including any UTM parameters. Read from text and spotlight ad content, otherwise from the creative metadata, which needs LinkedIn API version 202610 or later.",
+ "leadgen_call_to_action": "Lead gen form call to action: the form URN (destination) and the button label. Only set for lead generation campaigns.",
"created_at": "Time the creative was created.",
"last_modified_at": "Time the creative was last modified.",
"created_time": "Time the creative was created.",
@@ -115,6 +127,7 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"docs_url": "https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting",
"columns": {
**_STATS_COLUMNS,
+ **_PERFORMANCE_ONLY_COLUMNS,
"campaign_id": "ID of the campaign these metrics are for.",
},
},
@@ -131,6 +144,7 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"docs_url": "https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting",
"columns": {
**_STATS_COLUMNS,
+ **_PERFORMANCE_ONLY_COLUMNS,
"creative_id": "ID of the creative these metrics are for.",
"date_range": "The reporting date range these metrics cover.",
},
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/client.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/client.py
index ab21916d9321..d9bba84bf275 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/client.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/client.py
@@ -1,7 +1,7 @@
import re
import json
import datetime as dt
-from collections.abc import Generator
+from collections.abc import Callable, Generator
from typing import Any, Optional
import requests
@@ -33,6 +33,9 @@ MAX_INDEX_PAGES = 100
# Default LinkedIn Marketing API version header. Callers pass a resolved version through the
# source's version dispatch; this default backs the legacy `v1` pin and credential-probe paths.
API_VERSION = "202508"
+# Creative `metadata` (which holds the landing page URL) exists from 202610, and LinkedIn returns it
+# only on the single-creative GET. Batch get and the finders ignore it.
+CREATIVE_METADATA_MIN_API_VERSION = "202610"
# `q=analytics` silently truncates a response at 15k elements. With DAILY granularity an element
# is one (day × active pivot value), so a window of N days over an account with P active pivot
@@ -185,6 +188,41 @@ class LinkedinAdsClient:
page_size=CREATIVES_PAGE_SIZE,
)
+ def supports_creative_metadata(self) -> bool:
+ return self.api_version >= CREATIVE_METADATA_MIN_API_VERSION
+
+ def get_creative_landing_page(self, account_id: str, creative_urn: str) -> str | None:
+ """The landing page URL of one creative, from its `metadata`. Needs a 202610+ API version.
+
+ Returns None when LinkedIn has no landing page for the creative, or rejects the lookup with a
+ 4xx (for example a removed creative). One creative must not fail the whole table, and a 404
+ here must not surface as the account-level "not found" error.
+ """
+ account_endpoint = LINKEDIN_ADS_ENDPOINTS[LinkedinAdsResource.Accounts]
+ creatives_endpoint = LINKEDIN_ADS_ENDPOINTS[LinkedinAdsResource.Creatives]
+ try:
+ response = self._send(
+ lambda: self.client.get(
+ resource_path=f"/{account_endpoint}/{{account_id}}/{creatives_endpoint}/{{creative_urn}}",
+ path_keys={"account_id": account_id, "creative_urn": creative_urn},
+ query_params={"fields": "metadata"},
+ access_token=self.access_token,
+ version_string=self.api_version,
+ )
+ )
+ except LinkedinAdsApiError as e:
+ logger.warning(
+ "linkedin_ads.creative_metadata_lookup_failed",
+ creative=creative_urn,
+ api_status_code=e.api_status_code,
+ )
+ return None
+
+ entity = response.entity if isinstance(response.entity, dict) else {}
+ metadata = entity.get("metadata")
+ landing_page = metadata.get("landingPage") if isinstance(metadata, dict) else None
+ return landing_page if isinstance(landing_page, str) and landing_page else None
+
def get_conversions(self, account_id: str) -> Generator[tuple[list[dict[str, Any]], None]]:
"""Conversion rules defined on the ad account. `q=account` paginates with Rest.li index
parameters (`start`/`count`) rather than the pageToken envelope the ad entity finders use,
@@ -233,7 +271,8 @@ class LinkedinAdsClient:
)
return
- chunk_days = ANALYTICS_INITIAL_CHUNK_DAYS
+ max_chunk_days = RESOURCE_SCHEMAS[resource].get("max_window_days", ANALYTICS_INITIAL_CHUNK_DAYS)
+ chunk_days = max_chunk_days
chunk_start = dt.date.fromisoformat(date_start)
end_date = dt.date.fromisoformat(date_end)
while chunk_start <= end_date:
@@ -266,8 +305,8 @@ class LinkedinAdsClient:
# Plenty of headroom left: widen the next window so a sparse range doesn't keep paying
# for a window we shrank to fit an earlier dense spike.
- if not capped and len(elements) < ANALYTICS_GROW_THRESHOLD and chunk_days < ANALYTICS_INITIAL_CHUNK_DAYS:
- chunk_days = min(ANALYTICS_INITIAL_CHUNK_DAYS, chunk_days * 2)
+ if not capped and len(elements) < ANALYTICS_GROW_THRESHOLD and chunk_days < max_chunk_days:
+ chunk_days = min(max_chunk_days, chunk_days * 2)
def get_data_by_resource(
self,
@@ -317,8 +356,8 @@ class LinkedinAdsClient:
wait=_retry_wait,
reraise=True,
)
- def _call_finder(self, resource_path: str, finder: str, params: dict[str, Any]) -> Any:
- """Call the Restli finder with bounded retries on transient transport and 5xx/429 errors.
+ def _send(self, request: Callable[[], Any]) -> Any:
+ """Send a Restli request with bounded retries on transient transport and 5xx/429 errors.
LinkedIn's edge occasionally drops TLS connections mid-handshake (SSLEOFError) or
returns 504s; those surface here as `requests.ConnectionError` / `requests.Timeout`
@@ -334,13 +373,7 @@ class LinkedinAdsClient:
429s stay retryable and honour the Retry-After header when present.
"""
try:
- response = self.client.finder(
- resource_path=resource_path,
- finder_name=finder,
- access_token=self.access_token,
- query_params=params,
- version_string=self.api_version,
- )
+ response = request()
except ResponseFormattingError as e:
# The Restli client parses the body as JSON before returning, so a non-JSON response
# (an empty payload or an HTML error page from a 5xx gateway/proxy) raises here as a
@@ -367,6 +400,17 @@ class LinkedinAdsClient:
return response
+ def _call_finder(self, resource_path: str, finder: str, params: dict[str, Any]) -> Any:
+ return self._send(
+ lambda: self.client.finder(
+ resource_path=resource_path,
+ finder_name=finder,
+ access_token=self.access_token,
+ query_params=params,
+ version_string=self.api_version,
+ )
+ )
+
def _make_request(
self, endpoint: LinkedinAdsResource, finder: str, extra_params: dict | None = None, path: str | None = None
) -> list[dict[str, Any]]:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/linkedin_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/linkedin_ads.py
index 023ba0069e75..a66b91ff947d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/linkedin_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/linkedin_ads.py
@@ -28,9 +28,12 @@ from products.warehouse_sources.backend.types import IncrementalFieldType
from .client import API_VERSION, LinkedinAdsClient, LinkedinAdsDailyRateLimitError, LinkedinAdsResource
from .schemas import (
+ CREATIVE_INLINE_LANDING_PAGE_TYPES,
+ CREATIVE_REFERENCE_CONTENT_TYPES,
EPOCH_MS_VIRTUAL_COLUMNS,
FLOAT_FIELDS,
INT_FIELDS,
+ POST_URN_PREFIXES,
RESOURCE_SCHEMAS,
URN_COLUMNS,
VIRTUAL_COLUMN_URN_MAPPING,
@@ -275,6 +278,10 @@ def linkedin_ads_source(
try:
for page, next_page_token in data_pages:
+ if resource == LinkedinAdsResource.Creatives and client.supports_creative_metadata():
+ _add_creative_landing_pages(
+ page, client, config.account_id, safe_point=resumable_source_manager.safe_point
+ )
# None signals an unflattenable PK (malformed creative URN) — drop the row.
flattened_records = [
flat for record in page if (flat := _flatten_linkedin_record(record, schema)) is not None
@@ -313,6 +320,70 @@ def linkedin_ads_source(
)
+def _inline_landing_page(content: dict[str, typing.Any]) -> str | None:
+ """Text and spotlight ads carry their landing page in the creative itself."""
+ for ad_type in CREATIVE_INLINE_LANDING_PAGE_TYPES:
+ ad_content = content.get(ad_type)
+ landing_page = ad_content.get("landingPage") if isinstance(ad_content, dict) else None
+ if isinstance(landing_page, str) and landing_page:
+ return landing_page
+ return None
+
+
+def _add_creative_landing_pages(
+ page: list[dict[str, typing.Any]],
+ client: LinkedinAdsClient,
+ account_id: str,
+ safe_point: collections.abc.Callable[[], None],
+) -> None:
+ """Look up the landing page of each creative that does not carry one inline, at most one request
+ per creative. The result goes into the record's `metadata`, the shape LinkedIn uses for it."""
+ for record in page:
+ creative_urn = record.get("id")
+ content = record.get("content")
+ if (
+ not isinstance(creative_urn, str)
+ # A draft never served, so no visit can come from it.
+ or record.get("intendedStatus") == "DRAFT"
+ or (isinstance(content, dict) and _inline_landing_page(content) is not None)
+ ):
+ continue
+ # Rows of this page sit after the staged cursor, so a resume reads them again.
+ safe_point()
+ record["metadata"] = {"landingPage": client.get_creative_landing_page(account_id, creative_urn)}
+
+
+def _creative_destination_columns(record: dict[str, typing.Any]) -> dict[str, str | None]:
+ """Hoist the sponsored post and the landing page out of a creative's nested `content`."""
+ content = record.get("content")
+ content = content if isinstance(content, dict) else {}
+
+ reference = content.get("reference")
+ if not isinstance(reference, str):
+ # From 202610 LinkedIn can return the reference inside a format-specific member.
+ reference = next(
+ (
+ nested["reference"]
+ for ad_type in CREATIVE_REFERENCE_CONTENT_TYPES
+ if isinstance(nested := content.get(ad_type), dict) and isinstance(nested.get("reference"), str)
+ ),
+ None,
+ )
+ reference = reference or None
+
+ post_url = None
+ if reference and reference.startswith(POST_URN_PREFIXES):
+ post_url = f"https://www.linkedin.com/feed/update/{reference}/"
+
+ landing_page = _inline_landing_page(content)
+ if landing_page is None:
+ metadata = record.get("metadata")
+ looked_up = metadata.get("landingPage") if isinstance(metadata, dict) else None
+ landing_page = looked_up if isinstance(looked_up, str) and looked_up else None
+
+ return {"content_reference": reference, "post_url": post_url, "landing_page_url": landing_page}
+
+
def _convert_date_object_to_date(date_obj: dict[str, int] | None) -> dt.date | None:
"""Convert LinkedIn date object to Python date."""
if isinstance(date_obj, dict) and all(k in date_obj for k in ["year", "month", "day"]):
@@ -418,6 +489,9 @@ def _flatten_linkedin_record(
flattened["value_amount"] = None
flattened["value_currency_code"] = None
+ elif field_name == "content":
+ flattened.update(_creative_destination_columns(record))
+
elif field_name in URN_COLUMNS:
urn_value = record.get(field_name)
extracted_id: int | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/schemas.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/schemas.py
index be0317bfe30e..b59ffae419a2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/schemas.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/schemas.py
@@ -15,6 +15,9 @@ INT_FIELDS = {
"impressions",
"clicks",
"externalWebsiteConversions",
+ "externalWebsitePostClickConversions",
+ "externalWebsitePostViewConversions",
+ "approximateMemberReach",
"landingPageClicks",
"totalEngagements",
"videoViews",
@@ -36,6 +39,13 @@ EPOCH_MS_VIRTUAL_COLUMNS = {
"lastModified": "last_modified_time",
}
+# Creative `content` members that hold the landing page URL in the creative itself.
+CREATIVE_INLINE_LANDING_PAGE_TYPES = ("textAd", "spotlight")
+# Creative `content` members that hold the sponsored post URN, besides `content.reference`.
+CREATIVE_REFERENCE_CONTENT_TYPES = ("standardUpdateAd", "videoAd")
+# Sponsored post URNs that open at https://www.linkedin.com/feed/update/<urn>/ (Posts API docs).
+POST_URN_PREFIXES = ("urn:li:share:", "urn:li:ugcPost:")
+
# This maps the the URN type to the virtual column name. LinkedIn Ads API uses URNs to identify resources.
# URNs are like "urn:li:sponsoredCampaign:12345678"
VIRTUAL_COLUMN_URN_MAPPING = {
@@ -126,6 +136,12 @@ class ResourceSchema(TypedDict):
initial_lookback_days: NotRequired[int]
# False leaves the table unticked in the schema picker.
should_sync_default: NotRequired[bool]
+ # Upper bound on the days in one analytics request, for metrics LinkedIn returns only on short ranges.
+ max_window_days: NotRequired[int]
+
+
+# LinkedIn returns `approximateMemberReach` only when the date range of the request is 92 days or less.
+REACH_MAX_WINDOW_DAYS = 92
# LinkedIn retains professional demographic reporting for two years, versus ten for performance
@@ -235,7 +251,8 @@ RESOURCE_SCHEMAS: dict[LinkedinAdsResource, ResourceSchema] = {
},
LinkedinAdsResource.Creatives: {
"resource_name": "creatives",
- # CreativeV11 rejects `type` and `changeAuditStamps` — don't add them here.
+ # CreativeV11 rejects `type` and `changeAuditStamps` — don't add them here. Never add
+ # `metadata` either: LinkedIn returns a 400 for that projection on versions before 202610.
"field_names": [
"id",
"account",
@@ -244,6 +261,8 @@ RESOURCE_SCHEMAS: dict[LinkedinAdsResource, ResourceSchema] = {
"intendedStatus",
"isServing",
"review",
+ "content",
+ "leadgenCallToAction",
"createdAt",
"lastModifiedAt",
],
@@ -288,6 +307,9 @@ RESOURCE_SCHEMAS: dict[LinkedinAdsResource, ResourceSchema] = {
"costInUsd",
"costInLocalCurrency",
"externalWebsiteConversions",
+ "externalWebsitePostClickConversions",
+ "externalWebsitePostViewConversions",
+ "approximateMemberReach",
"conversionValueInLocalCurrency",
"landingPageClicks",
"totalEngagements",
@@ -305,6 +327,7 @@ RESOURCE_SCHEMAS: dict[LinkedinAdsResource, ResourceSchema] = {
"partition_format": "week",
"is_stats": True,
"partition_size": 1,
+ "max_window_days": REACH_MAX_WINDOW_DAYS,
},
LinkedinAdsResource.CampaignGroupStats: {
"resource_name": "campaign_group_stats",
@@ -344,6 +367,9 @@ RESOURCE_SCHEMAS: dict[LinkedinAdsResource, ResourceSchema] = {
"costInUsd",
"costInLocalCurrency",
"externalWebsiteConversions",
+ "externalWebsitePostClickConversions",
+ "externalWebsitePostViewConversions",
+ "approximateMemberReach",
"conversionValueInLocalCurrency",
"landingPageClicks",
"totalEngagements",
@@ -361,6 +387,7 @@ RESOURCE_SCHEMAS: dict[LinkedinAdsResource, ResourceSchema] = {
"partition_format": "week",
"is_stats": True,
"partition_size": 1,
+ "max_window_days": REACH_MAX_WINDOW_DAYS,
},
LinkedinAdsResource.MemberCompanyStats: _demographic_stats_schema("member_company_stats"),
LinkedinAdsResource.MemberCompanySizeStats: _demographic_stats_schema("member_company_size_stats"),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/tests/test_linkedin_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/tests/test_linkedin_ads.py
index 7511938ebb0d..d797f6d065c0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/tests/test_linkedin_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/tests/test_linkedin_ads.py
@@ -22,6 +22,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
LinkedinAdsSourceConfig,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_ads.client import (
+ LinkedinAdsClient,
LinkedinAdsDailyRateLimitError,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_ads.linkedin_ads import (
@@ -1177,3 +1178,138 @@ class TestConversionsFlattening:
assert flattened is not None
assert flattened["value_amount"] is None
assert flattened["value_currency_code"] is None
+
+
+class TestCreativeDestinationColumns:
+ @pytest.mark.parametrize(
+ "record, expected",
+ [
+ pytest.param(
+ {
+ "content": {"reference": "urn:li:share:6778045555198214144"},
+ "metadata": {"landingPage": "https://example.com/offer?utm_source=linkedin&utm_content=c1"},
+ },
+ ("urn:li:share:6778045555198214144", "https://example.com/offer?utm_source=linkedin&utm_content=c1"),
+ id="sponsored_post_with_looked_up_landing_page",
+ ),
+ pytest.param(
+ {"content": {"standardUpdateAd": {"reference": "urn:li:ugcPost:6856810298419044352"}}},
+ ("urn:li:ugcPost:6856810298419044352", None),
+ id="reference_inside_a_format_member",
+ ),
+ pytest.param(
+ {
+ "content": {"textAd": {"headline": "Try it", "landingPage": "https://example.com/?utm_id=7"}},
+ "metadata": {"landingPage": "https://example.com/ignored"},
+ },
+ (None, "https://example.com/?utm_id=7"),
+ id="text_ad_without_a_reference",
+ ),
+ pytest.param(
+ {"content": {"reference": "urn:li:adInMailContent:1234"}, "metadata": {"landingPage": ""}},
+ ("urn:li:adInMailContent:1234", None),
+ id="inmail_reference_is_not_a_post",
+ ),
+ pytest.param({"content": None, "metadata": {"landingPage": None}}, (None, None), id="no_content"),
+ ],
+ )
+ def test_creative_row_hoists_post_and_landing_page(self, record: dict, expected: tuple):
+ flattened = _flatten_linkedin_record({"id": "urn:li:sponsoredCreative:1", **record}, get_schemas()["creatives"])
+
+ assert flattened is not None
+ reference, landing_page = expected
+ assert flattened["content_reference"] == reference
+ assert flattened["landing_page_url"] == landing_page
+ is_post = reference is not None and reference.startswith(("urn:li:share:", "urn:li:ugcPost:"))
+ assert flattened["post_url"] == (f"https://www.linkedin.com/feed/update/{reference}/" if is_post else None)
+ assert "metadata" not in flattened
+
+
[email protected]("products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_ads.client.RestliClient")
+ "products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_ads.linkedin_ads.linkedin_ads_client"
+)
+class TestCreativeLandingPageLookup:
+ _CREATIVES = [
+ {"id": "urn:li:sponsoredCreative:1", "intendedStatus": "ACTIVE", "content": {"reference": "urn:li:share:11"}},
+ {
+ "id": "urn:li:sponsoredCreative:2",
+ "intendedStatus": "ACTIVE",
+ "content": {"textAd": {"landingPage": "https://example.com/text?utm_source=li"}},
+ },
+ {"id": "urn:li:sponsoredCreative:3", "intendedStatus": "DRAFT", "content": {"reference": "urn:li:ugcPost:33"}},
+ {
+ "id": "urn:li:sponsoredCreative:4",
+ "intendedStatus": "ARCHIVED",
+ "content": {"reference": "urn:li:ugcPost:44"},
+ },
+ ]
+
+ @staticmethod
+ def _get_creative(**kwargs: typing.Any) -> mock.MagicMock:
+ creative_urn = kwargs["path_keys"]["creative_urn"]
+ if creative_urn == "urn:li:sponsoredCreative:1":
+ return mock.MagicMock(
+ status_code=200, entity={"metadata": {"landingPage": "https://example.com/post?utm_content=1"}}
+ )
+ not_found = mock.MagicMock(status_code=404)
+ not_found.response.text = '{"status":404,"code":"RESOURCE_NOT_FOUND","message":"Creative not found"}'
+ return not_found
+
+ @pytest.mark.parametrize(
+ "api_version, expected_lookups, expected_landing_pages",
+ [
+ pytest.param(
+ "202610",
+ ["urn:li:sponsoredCreative:1", "urn:li:sponsoredCreative:4"],
+ {
+ 1: "https://example.com/post?utm_content=1",
+ 2: "https://example.com/text?utm_source=li",
+ 3: None,
+ 4: None,
+ },
+ id="metadata_supported",
+ ),
+ pytest.param(
+ "202609",
+ [],
+ {1: None, 2: "https://example.com/text?utm_source=li", 3: None, 4: None},
+ id="metadata_not_in_this_version",
+ ),
+ ],
+ )
+ def test_landing_pages_are_looked_up_only_where_needed(
+ self,
+ mock_client_func,
+ mock_restli_client,
+ api_version: str,
+ expected_lookups: list[str],
+ expected_landing_pages: dict[int, str | None],
+ ):
+ finder_response = mock.MagicMock(status_code=200, elements=[dict(c) for c in self._CREATIVES])
+ finder_response.response.text = '{"metadata": {}}'
+ mock_restli_client.return_value.finder.return_value = finder_response
+ mock_restli_client.return_value.get.side_effect = self._get_creative
+ mock_client_func.return_value = LinkedinAdsClient("token", api_version=api_version)
+ manager = _make_resume_manager()
+
+ result = linkedin_ads_source(
+ config=LinkedinAdsSourceConfig(linkedin_ads_integration_id=123, account_id="456"),
+ resource_name="creatives",
+ team_id=789,
+ resumable_source_manager=manager,
+ logger=mock.MagicMock(),
+ )
+ items = result.items()
+ assert isinstance(items, Iterable)
+ rows = [row for table in items for row in table.to_pylist()]
+
+ get_calls = mock_restli_client.return_value.get.call_args_list
+ assert [call.kwargs["path_keys"]["creative_urn"] for call in get_calls] == expected_lookups
+ for call in get_calls:
+ assert call.kwargs["path_keys"]["account_id"] == "456"
+ assert call.kwargs["query_params"] == {"fields": "metadata"}
+ assert call.kwargs["version_string"] == api_version
+ # Each lookup is preceded by a safe point, so a long page of lookups can still hand off.
+ assert manager.safe_point.call_count == len(expected_lookups)
+ assert {row["id"]: row["landing_page_url"] for row in rows} == expected_landing_pages
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/tests/test_linkedin_client.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/tests/test_linkedin_client.py
index 842b558650f9..b30e896fc27e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/tests/test_linkedin_client.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/tests/test_linkedin_client.py
@@ -17,6 +17,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_a
LinkedinAdsResource,
LinkedinAdsRetryableError,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_ads.schemas import (
+ FLOAT_FIELDS,
+ INT_FIELDS,
+)
class TestLinkedinAdsClient:
@@ -205,10 +209,24 @@ class TestLinkedinAdsClient:
assert elements[0]["costInUsd"] == 25.50
assert mock_client_instance.finder.call_count == 1
+ @pytest.mark.parametrize(
+ "pivot,expected_windows",
+ [
+ # Uncapped responses keep one wide window, so a small account does not pay one call per week.
+ (LinkedinAdsPivot.CAMPAIGN_GROUP, [("2024-01-01", "2024-07-19")]),
+ # Tables that request reach stay at 92 days or less, or LinkedIn leaves reach out.
+ (
+ LinkedinAdsPivot.CAMPAIGN,
+ [("2024-01-01", "2024-04-01"), ("2024-04-02", "2024-07-02"), ("2024-07-03", "2024-07-19")],
+ ),
+ (
+ LinkedinAdsPivot.CREATIVE,
+ [("2024-01-01", "2024-04-01"), ("2024-04-02", "2024-07-02"), ("2024-07-03", "2024-07-19")],
+ ),
+ ],
+ )
@mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_ads.client.RestliClient")
- def test_get_analytics_uncapped_range_fits_in_single_wide_window(self, mock_restli_client):
- """When responses never cap, even a long range is fetched in a single wide window —
- a small account no longer pays one call per week."""
+ def test_get_analytics_uncapped_range_windows(self, mock_restli_client, pivot, expected_windows):
mock_response = mock.MagicMock()
mock_response.status_code = 200
mock_response.elements = [{"impressions": 100}]
@@ -217,22 +235,23 @@ class TestLinkedinAdsClient:
mock_client_instance.finder.return_value = mock_response
client = LinkedinAdsClient(self.access_token)
- # 200-day range, well under the 365-day initial window → one call.
- pages = list(
+ list(
client.get_analytics(
account_id=self.account_id,
- pivot=LinkedinAdsPivot.CREATIVE,
+ pivot=pivot,
date_start="2024-01-01",
date_end="2024-07-19",
)
)
- assert len(pages) == 1
- assert mock_client_instance.finder.call_count == 1
- assert mock_client_instance.finder.call_args[1]["query_params"]["dateRange"] == {
- "start": {"year": 2024, "month": 1, "day": 1},
- "end": {"year": 2024, "month": 7, "day": 19},
- }
+ def as_date(value: dict[str, int]) -> str:
+ return dt.date(value["year"], value["month"], value["day"]).isoformat()
+
+ windows = [
+ (as_date(c[1]["query_params"]["dateRange"]["start"]), as_date(c[1]["query_params"]["dateRange"]["end"]))
+ for c in mock_client_instance.finder.call_args_list
+ ]
+ assert windows == expected_windows
@mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.linkedin_ads.client.RestliClient")
def test_get_analytics_shrinks_window_on_capped_response(self, mock_restli_client):
@@ -264,7 +283,7 @@ class TestLinkedinAdsClient:
pages = list(
client.get_analytics(
account_id=self.account_id,
- pivot=LinkedinAdsPivot.CAMPAIGN,
+ pivot=LinkedinAdsPivot.CAMPAIGN_GROUP,
date_start="2024-01-01",
date_end="2025-12-31",
)
@@ -596,7 +615,14 @@ class TestLinkedinAdsClient:
)
)
- assert mock_client_instance.finder.call_args[1]["query_params"]["pivot"] == expected_pivot
+ query_params = mock_client_instance.finder.call_args[1]["query_params"]
+ assert query_params["pivot"] == expected_pivot
+ # LinkedIn accepts at most 20 fields per adAnalytics request, and every requested metric
+ # must be coerced, else a batch where LinkedIn omitted it breaks the schema merge.
+ requested_fields = query_params["fields"].split(",")
+ assert len(requested_fields) <= 20
+ metrics = set(requested_fields) - {"dateRange", "pivotValues"}
+ assert metrics <= INT_FIELDS | FLOAT_FIELDS, metrics - (INT_FIELDS | FLOAT_FIELDS)
@pytest.mark.parametrize(
"resource",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/canonical_descriptions.py
index fc3cf97d8a95..54d20f8715c9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/canonical_descriptions.py
@@ -42,6 +42,30 @@ _COMMON_STATS_COLUMNS = {
}
+# Video watch-depth columns. The plain stats tables and the country and platform breakdown tables
+# carry them.
+_VIDEO_WATCH_DEPTH_COLUMNS = {
+ "video_30_sec_watched_actions": "Number of times the video played for at least 30 seconds, or nearly to the end for shorter videos, reported by action type.",
+ "video_p25_watched_actions": "Number of times the video played to 25% of its length, including plays that skipped to that point, reported by action type.",
+ "video_p50_watched_actions": "Number of times the video played to 50% of its length, including plays that skipped to that point, reported by action type.",
+ "video_p75_watched_actions": "Number of times the video played to 75% of its length, including plays that skipped to that point, reported by action type.",
+ "video_p95_watched_actions": "Number of times the video played to 95% of its length, including plays that skipped to that point, reported by action type.",
+ "video_p100_watched_actions": "Number of times the video played to its end, including plays that skipped to that point, reported by action type.",
+ "video_avg_time_watched_actions": "Average time the video played per impression, including replays, reported by action type.",
+}
+
+# The video columns of the plain stats tables, beyond those in `_COMMON_STATS_COLUMNS`.
+_PLAIN_STATS_VIDEO_COLUMNS = {
+ **_VIDEO_WATCH_DEPTH_COLUMNS,
+ "video_continuous_2_sec_watched_actions": "Number of times the video played for at least 2 continuous seconds, reported by action type.",
+ "video_play_curve_actions": (
+ "Video retention curve: the percentage of plays still watching at each point, as an array by action type. "
+ "Entries 0 to 14 are seconds 0 to 14, entries 15 to 17 are 5-second ranges up to 30 seconds, "
+ "entries 18 to 20 are 10-second ranges up to 60 seconds, and entry 21 is past 60 seconds."
+ ),
+}
+
+
def _stats_columns(**overrides: str) -> dict[str, str]:
return {**_COMMON_STATS_COLUMNS, **overrides}
@@ -148,6 +172,7 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"docs_url": "https://developers.facebook.com/docs/marketing-api/insights/",
"columns": _stats_columns(
campaign_id="The ID of the campaign the metrics belong to.",
+ **_PLAIN_STATS_VIDEO_COLUMNS,
),
},
MetaAdsResource.AdsetStats: {
@@ -156,6 +181,7 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"columns": _stats_columns(
adset_id="The ID of the ad set the metrics belong to.",
campaign_id="The ID of the campaign the ad set belongs to.",
+ **_PLAIN_STATS_VIDEO_COLUMNS,
),
},
MetaAdsResource.AdStats: {
@@ -165,12 +191,7 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
ad_id="The ID of the ad the metrics belong to.",
adset_id="The ID of the ad set the ad belongs to.",
campaign_id="The ID of the campaign the ad belongs to.",
- video_30_sec_watched_actions="The number of times the video was watched for at least 30 seconds.",
- video_p25_watched_actions="The number of times the video was watched to 25% of its length.",
- video_p50_watched_actions="The number of times the video was watched to 50% of its length.",
- video_p75_watched_actions="The number of times the video was watched to 75% of its length.",
- video_p95_watched_actions="The number of times the video was watched to 95% of its length.",
- video_p100_watched_actions="The number of times the video was watched to 100% of its length.",
+ **_PLAIN_STATS_VIDEO_COLUMNS,
),
},
MetaAdsResource.AdAccount: {
@@ -305,6 +326,7 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"docs_url": _INSIGHTS_BREAKDOWNS_DOCS_URL,
"columns": _campaign_breakdown_columns(
country="Two-letter country code the metrics were delivered in.",
+ **_VIDEO_WATCH_DEPTH_COLUMNS,
),
},
MetaAdsResource.CampaignStatsByRegion: {
@@ -321,6 +343,7 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
publisher_platform="The platform the ads ran on (facebook, instagram, audience_network, messenger).",
platform_position="The placement within the platform (e.g. feed, story, reels).",
impression_device="The device the impression was served on (e.g. iphone, android_smartphone, desktop).",
+ **_VIDEO_WATCH_DEPTH_COLUMNS,
),
},
MetaAdsResource.CampaignStatsHourly: {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/schemas.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/schemas.py
index 20eb37f46b44..3845987b8c60 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/schemas.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/schemas.py
@@ -144,6 +144,25 @@ CAMPAIGN_BREAKDOWN_STATS_FIELDS = ["campaign_id", *_BREAKDOWN_METRIC_FIELDS]
ADSET_BREAKDOWN_STATS_FIELDS = ["adset_id", "campaign_id", *_BREAKDOWN_METRIC_FIELDS]
AD_BREAKDOWN_STATS_FIELDS = ["ad_id", "adset_id", "campaign_id", *_BREAKDOWN_METRIC_FIELDS]
+# How far into a video people watched. Only the country and platform breakdown tables request
+# these. Meta rejects the whole request when the `region` breakdown asks for some of them:
+# "video_avg_time_watched_actions field cannot be requested with the region breakdown", and the
+# p25 to p100 fields "do not support region breakdown".
+_VIDEO_WATCH_DEPTH_FIELDS = [
+ "video_30_sec_watched_actions",
+ "video_p25_watched_actions",
+ "video_p50_watched_actions",
+ "video_p75_watched_actions",
+ "video_p95_watched_actions",
+ "video_p100_watched_actions",
+ "video_avg_time_watched_actions",
+]
+
+
+def _with_video_watch_depth(field_names: list[str]) -> list[str]:
+ return [*field_names, *_VIDEO_WATCH_DEPTH_FIELDS]
+
+
# Meta: "Hourly breakdowns do not support unique fields, which are any fields prepended with
# `unique_*`, `reach` or `frequency`." `cpp` and `cost_per_unique_click` are derived from those.
# A person who saw an ad cannot be counted once across a split Meta does not deduplicate over, so
@@ -161,7 +180,7 @@ _UNIQUE_METRIC_FIELDS = {
# Hourly and creative-asset breakdowns do not support these video metrics.
-_VIDEO_METRIC_FIELDS = {"video_play_actions", "video_thruplay_watched_actions"}
+_VIDEO_METRIC_FIELDS = {"video_play_actions", "video_thruplay_watched_actions", *_VIDEO_WATCH_DEPTH_FIELDS}
def _without_metrics(field_names: list[str], excluded_fields: set[str]) -> list[str]:
@@ -311,6 +330,9 @@ RESOURCE_SCHEMAS: dict[MetaAdsResource, dict[str, Any]] = {
"video_p75_watched_actions",
"video_p95_watched_actions",
"video_p100_watched_actions",
+ "video_avg_time_watched_actions",
+ "video_continuous_2_sec_watched_actions",
+ "video_play_curve_actions",
],
"partition_mode": "datetime",
"partition_format": "week",
@@ -383,8 +405,17 @@ RESOURCE_SCHEMAS: dict[MetaAdsResource, dict[str, Any]] = {
"purchase_roas",
"website_purchase_roas",
"attribution_setting",
+ "video_30_sec_watched_actions",
"video_play_actions",
"video_thruplay_watched_actions",
+ "video_p25_watched_actions",
+ "video_p50_watched_actions",
+ "video_p75_watched_actions",
+ "video_p95_watched_actions",
+ "video_p100_watched_actions",
+ "video_avg_time_watched_actions",
+ "video_continuous_2_sec_watched_actions",
+ "video_play_curve_actions",
],
"partition_mode": "datetime",
"partition_format": "week",
@@ -453,8 +484,17 @@ RESOURCE_SCHEMAS: dict[MetaAdsResource, dict[str, Any]] = {
"purchase_roas",
"website_purchase_roas",
"attribution_setting",
+ "video_30_sec_watched_actions",
"video_play_actions",
"video_thruplay_watched_actions",
+ "video_p25_watched_actions",
+ "video_p50_watched_actions",
+ "video_p75_watched_actions",
+ "video_p95_watched_actions",
+ "video_p100_watched_actions",
+ "video_avg_time_watched_actions",
+ "video_continuous_2_sec_watched_actions",
+ "video_play_curve_actions",
],
"partition_mode": "datetime",
"partition_format": "week",
@@ -623,19 +663,27 @@ RESOURCE_SCHEMAS: dict[MetaAdsResource, dict[str, Any]] = {
MetaAdsResource.CampaignStatsByAgeGender: _breakdown_stats(
"campaign", ["age", "gender"], CAMPAIGN_BREAKDOWN_STATS_FIELDS
),
- MetaAdsResource.CampaignStatsByCountry: _breakdown_stats("campaign", ["country"], CAMPAIGN_BREAKDOWN_STATS_FIELDS),
+ MetaAdsResource.CampaignStatsByCountry: _breakdown_stats(
+ "campaign", ["country"], _with_video_watch_depth(CAMPAIGN_BREAKDOWN_STATS_FIELDS)
+ ),
MetaAdsResource.CampaignStatsByRegion: _breakdown_stats("campaign", ["region"], CAMPAIGN_BREAKDOWN_STATS_FIELDS),
MetaAdsResource.CampaignStatsByPlatform: _breakdown_stats(
- "campaign", ["publisher_platform", "platform_position", "impression_device"], CAMPAIGN_BREAKDOWN_STATS_FIELDS
+ "campaign",
+ ["publisher_platform", "platform_position", "impression_device"],
+ _with_video_watch_depth(CAMPAIGN_BREAKDOWN_STATS_FIELDS),
),
MetaAdsResource.CampaignStatsHourly: _breakdown_stats(
"campaign", ["hourly_stats_aggregated_by_advertiser_time_zone"], HOURLY_BREAKDOWN_STATS_FIELDS
),
MetaAdsResource.AdsetStatsByAgeGender: _breakdown_stats("adset", ["age", "gender"], ADSET_BREAKDOWN_STATS_FIELDS),
- MetaAdsResource.AdsetStatsByCountry: _breakdown_stats("adset", ["country"], ADSET_BREAKDOWN_STATS_FIELDS),
+ MetaAdsResource.AdsetStatsByCountry: _breakdown_stats(
+ "adset", ["country"], _with_video_watch_depth(ADSET_BREAKDOWN_STATS_FIELDS)
+ ),
MetaAdsResource.AdsetStatsByRegion: _breakdown_stats("adset", ["region"], ADSET_BREAKDOWN_STATS_FIELDS),
MetaAdsResource.AdsetStatsByPlatform: _breakdown_stats(
- "adset", ["publisher_platform", "platform_position", "impression_device"], ADSET_BREAKDOWN_STATS_FIELDS
+ "adset",
+ ["publisher_platform", "platform_position", "impression_device"],
+ _with_video_watch_depth(ADSET_BREAKDOWN_STATS_FIELDS),
),
MetaAdsResource.AdsetStatsHourly: _breakdown_stats(
"adset",
@@ -643,10 +691,14 @@ RESOURCE_SCHEMAS: dict[MetaAdsResource, dict[str, Any]] = {
_without_metrics(ADSET_BREAKDOWN_STATS_FIELDS, _UNIQUE_METRIC_FIELDS | _VIDEO_METRIC_FIELDS),
),
MetaAdsResource.AdStatsByAgeGender: _breakdown_stats("ad", ["age", "gender"], AD_BREAKDOWN_STATS_FIELDS),
- MetaAdsResource.AdStatsByCountry: _breakdown_stats("ad", ["country"], AD_BREAKDOWN_STATS_FIELDS),
+ MetaAdsResource.AdStatsByCountry: _breakdown_stats(
+ "ad", ["country"], _with_video_watch_depth(AD_BREAKDOWN_STATS_FIELDS)
+ ),
MetaAdsResource.AdStatsByRegion: _breakdown_stats("ad", ["region"], AD_BREAKDOWN_STATS_FIELDS),
MetaAdsResource.AdStatsByPlatform: _breakdown_stats(
- "ad", ["publisher_platform", "platform_position", "impression_device"], AD_BREAKDOWN_STATS_FIELDS
+ "ad",
+ ["publisher_platform", "platform_position", "impression_device"],
+ _with_video_watch_depth(AD_BREAKDOWN_STATS_FIELDS),
),
MetaAdsResource.AdStatsHourly: _breakdown_stats(
"ad",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py
index 6f2c823ce6b5..e46ef313719b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py
@@ -2427,13 +2427,29 @@ class TestBreakdownStatsRequests:
assert fields_param is not None
requested_fields = set(fields_param.split(","))
video_fields = {"video_play_actions", "video_thruplay_watched_actions"}
+ # Meta rejects the whole request when `region` asks for these, so they must not leak into
+ # the field list that every breakdown shares.
+ watch_depth_fields = {
+ "video_30_sec_watched_actions",
+ "video_p25_watched_actions",
+ "video_p50_watched_actions",
+ "video_p75_watched_actions",
+ "video_p95_watched_actions",
+ "video_p100_watched_actions",
+ "video_avg_time_watched_actions",
+ }
+ breakdowns = schema.extra_params["breakdowns"]
if (
- "hourly_stats_aggregated_by_advertiser_time_zone" in schema.extra_params["breakdowns"]
+ "hourly_stats_aggregated_by_advertiser_time_zone" in breakdowns
or endpoint == MetaAdsResource.AdStatsByLinkUrl
):
- assert video_fields.isdisjoint(requested_fields)
+ assert (video_fields | watch_depth_fields).isdisjoint(requested_fields)
+ elif breakdowns in ("country", "publisher_platform,platform_position,impression_device"):
+ assert video_fields | watch_depth_fields <= requested_fields
else:
assert video_fields <= requested_fields
+ if breakdowns == "region":
+ assert watch_depth_fields.isdisjoint(requested_fields)
def test_non_stats_endpoint_is_not_windowed(self, monkeypatch) -> None:
result = self._capture_request(monkeypatch, MetaAdsResource.AdCreatives)