Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(meta-ads): sync the ads table past an ad with an invalid advantage+ creative (#119477)

Tom Owers8290d6afc9d8
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py
index 1a2bd2d47b8b..691b36057473 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py
@@ -211,6 +211,8 @@ class MetaAdsSchema:
single_object: bool = False
# Scalar columns to copy out of the nested objects in each row (see `HoistedColumn`).
hoisted_columns: tuple[HoistedColumn, ...] = ()
+ # Fields Meta can refuse to serve for a single row (see `META_INVALID_DOF_CREATIVE_ERROR_SUBCODE`).
+ refusable_fields: tuple[str, ...] = ()
# Note: can make this static but keeping schemas.py to match other schema files for now
@@ -231,6 +233,7 @@ def get_schemas() -> dict[str, MetaAdsSchema]:
is_stats=schema_def.get("is_stats", False),
single_object=schema_def.get("single_object", False),
hoisted_columns=schema_def.get("hoisted_columns", ()),
+ refusable_fields=schema_def.get("refusable_fields", ()),
)
schemas[resource_name] = schema
@@ -275,16 +278,20 @@ PAGE_LIMIT_FALLBACK_SIZES = [500, 100, 50]
MALFORMED_JSON_MAX_ATTEMPTS = 3
+def _override_query_param(url: str, key: str, value: str) -> str:
+ parts = urlsplit(url)
+ pairs = [(k, v) for k, v in parse_qsl(parts.query, keep_blank_values=True) if k != key]
+ pairs.append((key, value))
+ return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(pairs), parts.fragment))
+
+
def _override_limit(url: str, limit: int) -> str:
"""Return ``url`` with its ``limit`` query parameter overridden.
Meta's ``paging.next`` URLs encode the limit that produced the cursor; if
we want a smaller batch on the retry, we have to rewrite the URL.
"""
- parts = urlsplit(url)
- pairs = [(k, v) for k, v in parse_qsl(parts.query, keep_blank_values=True) if k != "limit"]
- pairs.append(("limit", str(limit)))
- return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(pairs), parts.fragment))
+ return _override_query_param(url, "limit", str(limit))
def _next_smaller_limit(current: int) -> int | None:
@@ -459,6 +466,16 @@ META_RATE_LIMIT_ERROR_CODES = {4, 17, 32, 613}
# scratch with a new cursor.
META_INVALID_CURSOR_ERROR_CODE = 2642
+# Meta error code 100, subcode 2446211: Meta validates an Advantage+ ("degrees of freedom") creative
+# again when it serves the creative-backed fields of an ad, and refuses when the creative's asset feed
+# spec no longer matches its object story spec. Meta refuses the whole page for one such ad, so
+# without a fallback every sync of the table stops at that ad. The schema's `refusable_fields` name
+# the fields to drop from the page request and then read again per ad (see `_restore_refusable_fields`).
+META_INVALID_DOF_CREATIVE_ERROR_SUBCODE = 2446211
+
+# Meta's multi-id lookup (`GET /?ids=...`) accepts at most 50 ids in one request.
+META_NODE_LOOKUP_MAX_IDS = 50
+
META_AUTH_ERROR_MESSAGE = (
"Meta Ads access token is invalid, expired, or lacks the required permissions. Please re-authorize the integration."
)
@@ -555,6 +572,10 @@ def _is_invalid_cursor_error(response: Response) -> bool:
return _meta_error_code(response) == META_INVALID_CURSOR_ERROR_CODE
+def _is_invalid_dof_creative_error(response: Response) -> bool:
+ return _meta_error_subcode(response) == META_INVALID_DOF_CREATIVE_ERROR_SUBCODE
+
+
def _raise_meta_api_error(response: Response) -> typing.NoReturn:
"""Raise a descriptive exception for a non-200 Meta API response.
@@ -683,11 +704,77 @@ def list_ad_accounts(integration: Integration) -> list[dict]:
raise IntegrationAccountListingError("Meta returned too many pages while listing ad accounts. Please try again.")
+def _is_refused_node(node_id: str, node: typing.Any, fields: tuple[str, ...]) -> bool:
+ """Return True for a node in a successful multi-id lookup that carries Meta's creative refusal.
+
+ Meta can report an error per node inside an HTTP 200 lookup. Any error other than the refusal
+ raises, so a transient or permission failure is not stored as empty fields.
+ """
+ error = node.get("error") if isinstance(node, dict) else None
+ if not isinstance(error, dict):
+ return False
+ if error.get("error_subcode") != META_INVALID_DOF_CREATIVE_ERROR_SUBCODE:
+ raise Exception(f"Meta API request failed: node lookup error - {json.dumps(error)}")
+ logger.warning("meta_ads_refused_fields_for_node", node_id=node_id, fields=fields)
+ return True
+
+
+def _fetch_refusable_fields(
+ node_lookup_url: str, ids: list[str], fields: tuple[str, ...], access_token: str
+) -> dict[str, dict]:
+ """Read ``fields`` for each node in ``ids``, keyed by id, without the nodes Meta refuses to serve.
+
+ A multi-id lookup fails as a whole when Meta refuses one node, so a refused batch is split in
+ half until the refused node is alone. Only that node then misses the fields.
+ """
+ params = {"ids": ",".join(ids), "fields": ",".join(fields), "access_token": access_token}
+ malformed_json_attempts = 0
+ while True:
+ response = _get_initial_request(node_lookup_url, params)
+ if response.status_code != 200:
+ break
+ try:
+ nodes = response.json()
+ except RequestsJSONDecodeError:
+ malformed_json_attempts += 1
+ if malformed_json_attempts >= MALFORMED_JSON_MAX_ATTEMPTS:
+ raise
+ continue
+ return {node_id: node for node_id, node in nodes.items() if not _is_refused_node(node_id, node, fields)}
+ if not _is_invalid_dof_creative_error(response):
+ _raise_meta_api_error(response)
+ if len(ids) == 1:
+ logger.warning("meta_ads_refused_fields_for_node", node_id=ids[0], fields=fields)
+ return {}
+ middle = len(ids) // 2
+ return {
+ **_fetch_refusable_fields(node_lookup_url, ids[:middle], fields, access_token),
+ **_fetch_refusable_fields(node_lookup_url, ids[middle:], fields, access_token),
+ }
+
+
+def _restore_refusable_fields(
+ rows: list[dict], node_lookup_url: str, fields: tuple[str, ...], access_token: str
+) -> list[dict]:
+ """Add ``fields`` back to rows fetched without them, as ``None`` for a row Meta refuses them for."""
+ ids = [str(row["id"]) for row in rows if row.get("id")]
+ nodes: dict[str, dict] = {}
+ for start in range(0, len(ids), META_NODE_LOOKUP_MAX_IDS):
+ nodes.update(
+ _fetch_refusable_fields(
+ node_lookup_url, ids[start : start + META_NODE_LOOKUP_MAX_IDS], fields, access_token
+ )
+ )
+ return [{**row, **{field: nodes.get(str(row.get("id")), {}).get(field) for field in fields}} for row in rows]
+
+
def _iter_simple_pagination(
initial_url: str,
params: dict,
resume_config: MetaAdsResumeConfig | None,
resumable_source_manager: ResumableSourceManager[MetaAdsResumeConfig],
+ refusable_fields: tuple[str, ...] = (),
+ node_lookup_url: str | None = None,
) -> collections.abc.Generator[list[dict]]:
"""Iterate a non-time-range Graph API request via ``paging.next`` URLs.
@@ -704,9 +791,17 @@ def _iter_simple_pagination(
yet, and a cursor points at the start of the next (not-yet-yielded) page.
If the smallest limit still fails, the page is retried with backoff and
then raised as retryable (see ``ENTITY_PAGE_REFUSED_ERROR_MESSAGE``).
+
+ When Meta refuses a page for one row's ``refusable_fields`` (see
+ ``META_INVALID_DOF_CREATIVE_ERROR_SUBCODE``), the same page is fetched
+ again without those fields, and the fields are read back per row from
+ ``node_lookup_url``. The next page asks for every field again.
"""
access_token = params["access_token"]
current_limit = PAGE_LIMIT_FALLBACK_SIZES[0]
+ all_fields = params.get("fields", "")
+ reduced_fields = ",".join(field for field in all_fields.split(",") if field not in refusable_fields)
+ without_refusable_fields = False
# None while on the initial request; set to the active ``paging.next``
# cursor once we start following pages. Used to retry-at-smaller-limit.
@@ -719,15 +814,18 @@ def _iter_simple_pagination(
# healthy syncs keep their original request shape (saved cursor URLs and
# the caller's params already encode the default limit).
if cursor_url is not None:
- url = (
- _override_limit(cursor_url, current_limit)
- if current_limit != PAGE_LIMIT_FALLBACK_SIZES[0]
- else cursor_url
- )
+ url = cursor_url
+ if current_limit != PAGE_LIMIT_FALLBACK_SIZES[0]:
+ url = _override_limit(url, current_limit)
+ if without_refusable_fields:
+ url = _override_query_param(url, "fields", reduced_fields)
return _fetch_paging_url(url, access_token)
+ request_params = params
if current_limit != PAGE_LIMIT_FALLBACK_SIZES[0]:
- return _get_initial_request(initial_url, {**params, "limit": current_limit})
- return _get_initial_request(initial_url, params)
+ request_params = {**request_params, "limit": current_limit}
+ if without_refusable_fields:
+ request_params = {**request_params, "fields": reduced_fields}
+ return _get_initial_request(initial_url, request_params)
response = _issue()
malformed_json_attempts = 0
@@ -735,6 +833,15 @@ def _iter_simple_pagination(
while True:
if response.status_code != 200:
+ if (
+ refusable_fields
+ and node_lookup_url is not None
+ and not without_refusable_fields
+ and _is_invalid_dof_creative_error(response)
+ ):
+ without_refusable_fields = True
+ response = _issue()
+ continue
# Too-much-data: shrink the page limit and retry the same request.
# Re-issuing the same URL/cursor at a smaller limit is safe — no
# already-yielded rows are re-emitted.
@@ -769,7 +876,10 @@ def _iter_simple_pagination(
malformed_json_attempts = 0
smallest_limit_retries = 0
- yield response_payload.get("data", [])
+ rows = response_payload.get("data", [])
+ if without_refusable_fields and node_lookup_url is not None:
+ rows = _restore_refusable_fields(rows, node_lookup_url, refusable_fields, access_token)
+ yield rows
next_url = response_payload.get("paging", {}).get("next")
if not next_url:
@@ -780,6 +890,10 @@ def _iter_simple_pagination(
# Strip access_token from the URL before using it so we don't end up with a
# duplicated `access_token` query param (requests merges `params=...` into the URL).
cursor_url = _strip_access_token(next_url)
+ if without_refusable_fields:
+ # The cursor of a page fetched without the refusable fields carries the reduced field list.
+ cursor_url = _override_query_param(cursor_url, "fields", all_fields)
+ without_refusable_fields = False
resumable_source_manager.save_state(MetaAdsResumeConfig(next_url=cursor_url))
response = _issue()
@@ -1006,6 +1120,8 @@ def _make_paginated_api_request(
access_token: str,
time_range: dict | None,
resumable_source_manager: ResumableSourceManager[MetaAdsResumeConfig],
+ refusable_fields: tuple[str, ...] = (),
+ node_lookup_url: str | None = None,
) -> collections.abc.Generator[list[dict]]:
"""Make paginated requests to the Meta Graph API.
This function handles two types of pagination:
@@ -1017,7 +1133,9 @@ def _make_paginated_api_request(
resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
if time_range is None:
- yield from _iter_simple_pagination(url, params, resume_config, resumable_source_manager)
+ yield from _iter_simple_pagination(
+ url, params, resume_config, resumable_source_manager, refusable_fields, node_lookup_url
+ )
else:
yield from _iter_time_range_pagination(url, params, time_range, resume_config, resumable_source_manager)
@@ -1127,7 +1245,15 @@ def meta_ads_source(
params.update(_attribution_params(config))
yield from _hoist_columns(
- _make_paginated_api_request(formatted_url, params, access_token, time_range, resumable_source_manager),
+ _make_paginated_api_request(
+ formatted_url,
+ params,
+ access_token,
+ time_range,
+ resumable_source_manager,
+ schema.refusable_fields,
+ f"https://graph.facebook.com/{api_version}/",
+ ),
schema.hoisted_columns,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/schemas.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/schemas.py
index 7e34343155c0..49f024f48414 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/schemas.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/schemas.py
@@ -251,6 +251,9 @@ RESOURCE_SCHEMAS: dict[MetaAdsResource, dict[str, Any]] = {
# The Ad node returns its creative as a nested object. Without a scalar id there is no join
# from an ad to `ad_creatives`, where the destination URL and the URL tags live.
"hoisted_columns": (HoistedColumn(source_field="creative", key="id", column="creative_id"),),
+ # These fields come from the ad's creative. Meta can refuse them for one ad with an invalid
+ # Advantage+ creative, and that ad then syncs without them instead of failing the table.
+ "refusable_fields": ("creative", "tracking_specs", "conversion_specs"),
},
MetaAdsResource.AdStats: {
"primary_keys": ["ad_id", "account_id", "date_start"],
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py
index 0b40ac02241a..83ec75acdae1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py
@@ -348,6 +348,90 @@ class TestSimplePaginationLimitFallback:
assert mock_get.return_value.get.call_count == 1
+class TestSimplePaginationRefusedFields:
+ INITIAL_URL = "https://graph.facebook.com/v20/act_123/ads"
+ NODE_LOOKUP_URL = "https://graph.facebook.com/v20/"
+ PARAMS: dict[str, Any] = {"fields": "id,name,creative", "limit": 500, "access_token": "tok"}
+ REFUSED_BODY: dict[str, Any] = {
+ "error": {"message": "Invalid parameter", "code": 100, "error_subcode": 2446211, "is_transient": False}
+ }
+
+ def test_refused_row_syncs_without_refusable_fields(self) -> None:
+ manager = _build_manager()
+ responses = [
+ _mock_response(400, self.REFUSED_BODY),
+ _mock_response(
+ 200,
+ {
+ "data": [{"id": "1", "name": "a"}, {"id": "2", "name": "b"}],
+ "paging": {"next": "https://graph.facebook.com/v20/act_123/ads?fields=id%2Cname&after=p1"},
+ },
+ ),
+ # The lookup for both ads is refused, so it splits until the refused ad is alone.
+ _mock_response(400, self.REFUSED_BODY),
+ _mock_response(200, {"1": {"id": "1", "creative": {"id": "c1"}}}),
+ _mock_response(400, self.REFUSED_BODY),
+ _mock_response(200, {"data": [{"id": "3", "name": "c", "creative": {"id": "c3"}}], "paging": {}}),
+ ]
+
+ with mock.patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.meta_ads.meta_ads.make_tracked_session"
+ ) as mock_get:
+ mock_get.return_value.get.side_effect = responses
+ batches = list(
+ _iter_simple_pagination(
+ self.INITIAL_URL, self.PARAMS, None, manager, ("creative",), self.NODE_LOOKUP_URL
+ )
+ )
+
+ assert batches == [
+ [{"id": "1", "name": "a", "creative": {"id": "c1"}}, {"id": "2", "name": "b", "creative": None}],
+ [{"id": "3", "name": "c", "creative": {"id": "c3"}}],
+ ]
+ calls = mock_get.return_value.get.call_args_list
+ assert calls[1].kwargs["params"]["fields"] == "id,name"
+ assert calls[2].kwargs["params"]["ids"] == "1,2"
+ assert "fields=id%2Cname%2Ccreative" in calls[5].args[0]
+ saved = manager.save_state.call_args.args[0]
+ assert saved.next_url is not None and "fields=id%2Cname%2Ccreative" in saved.next_url
+
+ def test_lookup_node_error_other_than_refusal_raises(self) -> None:
+ manager = _build_manager()
+ responses = [
+ _mock_response(400, self.REFUSED_BODY),
+ _mock_response(200, {"data": [{"id": "1", "name": "a"}], "paging": {}}),
+ _mock_response(200, {"1": {"error": {"message": "Service temporarily unavailable", "code": 2}}}),
+ ]
+
+ with mock.patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.meta_ads.meta_ads.make_tracked_session"
+ ) as mock_get:
+ mock_get.return_value.get.side_effect = responses
+ with pytest.raises(Exception, match="node lookup error"):
+ list(
+ _iter_simple_pagination(
+ self.INITIAL_URL, self.PARAMS, None, manager, ("creative",), self.NODE_LOOKUP_URL
+ )
+ )
+
+ def test_refusal_without_refusable_fields_raises(self) -> None:
+ manager = _build_manager()
+ responses = [_mock_response(400, self.REFUSED_BODY), _mock_response(400, self.REFUSED_BODY)]
+
+ with mock.patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.meta_ads.meta_ads.make_tracked_session"
+ ) as mock_get:
+ mock_get.return_value.get.side_effect = responses
+ with pytest.raises(Exception, match="Meta API request failed: 400"):
+ list(
+ _iter_simple_pagination(
+ self.INITIAL_URL, self.PARAMS, None, manager, ("creative",), self.NODE_LOOKUP_URL
+ )
+ )
+
+ assert mock_get.return_value.get.call_count == 2
+
+
class TestSimplePaginationMalformedJson:
INITIAL_URL = "https://graph.facebook.com/v20/act_123/campaigns"
PARAMS: dict[str, Any] = {"fields": "id,name", "limit": 500, "access_token": "tok"}