:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(meta-ads): sync the ads table past an ad with an invalid advantage+ creative (#119477)
Tom Owers8290d6afc9d8
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.pyindex 1a2bd2d47b8b..691b36057473 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py@@ -211,6 +211,8 @@ class MetaAdsSchema:single_object: bool = False# Scalar columns to copy out of the nested objects in each row (see `HoistedColumn`).hoisted_columns: tuple[HoistedColumn, ...] = ()+ # Fields Meta can refuse to serve for a single row (see `META_INVALID_DOF_CREATIVE_ERROR_SUBCODE`).+ refusable_fields: tuple[str, ...] = ()# Note: can make this static but keeping schemas.py to match other schema files for now@@ -231,6 +233,7 @@ def get_schemas() -> dict[str, MetaAdsSchema]:is_stats=schema_def.get("is_stats", False),single_object=schema_def.get("single_object", False),hoisted_columns=schema_def.get("hoisted_columns", ()),+ refusable_fields=schema_def.get("refusable_fields", ()),)schemas[resource_name] = schema@@ -275,16 +278,20 @@ PAGE_LIMIT_FALLBACK_SIZES = [500, 100, 50]MALFORMED_JSON_MAX_ATTEMPTS = 3+def _override_query_param(url: str, key: str, value: str) -> str:+ parts = urlsplit(url)+ pairs = [(k, v) for k, v in parse_qsl(parts.query, keep_blank_values=True) if k != key]+ pairs.append((key, value))+ return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(pairs), parts.fragment))++def _override_limit(url: str, limit: int) -> str:"""Return ``url`` with its ``limit`` query parameter overridden.Meta's ``paging.next`` URLs encode the limit that produced the cursor; ifwe want a smaller batch on the retry, we have to rewrite the URL."""- parts = urlsplit(url)- pairs = [(k, v) for k, v in parse_qsl(parts.query, keep_blank_values=True) if k != "limit"]- pairs.append(("limit", str(limit)))- return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(pairs), parts.fragment))+ return _override_query_param(url, "limit", str(limit))def _next_smaller_limit(current: int) -> int | None:@@ -459,6 +466,16 @@ META_RATE_LIMIT_ERROR_CODES = {4, 17, 32, 613}# scratch with a new cursor.META_INVALID_CURSOR_ERROR_CODE = 2642+# Meta error code 100, subcode 2446211: Meta validates an Advantage+ ("degrees of freedom") creative+# again when it serves the creative-backed fields of an ad, and refuses when the creative's asset feed+# spec no longer matches its object story spec. Meta refuses the whole page for one such ad, so+# without a fallback every sync of the table stops at that ad. The schema's `refusable_fields` name+# the fields to drop from the page request and then read again per ad (see `_restore_refusable_fields`).+META_INVALID_DOF_CREATIVE_ERROR_SUBCODE = 2446211++# Meta's multi-id lookup (`GET /?ids=...`) accepts at most 50 ids in one request.+META_NODE_LOOKUP_MAX_IDS = 50+META_AUTH_ERROR_MESSAGE = ("Meta Ads access token is invalid, expired, or lacks the required permissions. Please re-authorize the integration.")@@ -555,6 +572,10 @@ def _is_invalid_cursor_error(response: Response) -> bool:return _meta_error_code(response) == META_INVALID_CURSOR_ERROR_CODE+def _is_invalid_dof_creative_error(response: Response) -> bool:+ return _meta_error_subcode(response) == META_INVALID_DOF_CREATIVE_ERROR_SUBCODE++def _raise_meta_api_error(response: Response) -> typing.NoReturn:"""Raise a descriptive exception for a non-200 Meta API response.@@ -683,11 +704,77 @@ def list_ad_accounts(integration: Integration) -> list[dict]:raise IntegrationAccountListingError("Meta returned too many pages while listing ad accounts. Please try again.")+def _is_refused_node(node_id: str, node: typing.Any, fields: tuple[str, ...]) -> bool:+ """Return True for a node in a successful multi-id lookup that carries Meta's creative refusal.++ Meta can report an error per node inside an HTTP 200 lookup. Any error other than the refusal+ raises, so a transient or permission failure is not stored as empty fields.+ """+ error = node.get("error") if isinstance(node, dict) else None+ if not isinstance(error, dict):+ return False+ if error.get("error_subcode") != META_INVALID_DOF_CREATIVE_ERROR_SUBCODE:+ raise Exception(f"Meta API request failed: node lookup error - {json.dumps(error)}")+ logger.warning("meta_ads_refused_fields_for_node", node_id=node_id, fields=fields)+ return True+++def _fetch_refusable_fields(+ node_lookup_url: str, ids: list[str], fields: tuple[str, ...], access_token: str+) -> dict[str, dict]:+ """Read ``fields`` for each node in ``ids``, keyed by id, without the nodes Meta refuses to serve.++ A multi-id lookup fails as a whole when Meta refuses one node, so a refused batch is split in+ half until the refused node is alone. Only that node then misses the fields.+ """+ params = {"ids": ",".join(ids), "fields": ",".join(fields), "access_token": access_token}+ malformed_json_attempts = 0+ while True:+ response = _get_initial_request(node_lookup_url, params)+ if response.status_code != 200:+ break+ try:+ nodes = response.json()+ except RequestsJSONDecodeError:+ malformed_json_attempts += 1+ if malformed_json_attempts >= MALFORMED_JSON_MAX_ATTEMPTS:+ raise+ continue+ return {node_id: node for node_id, node in nodes.items() if not _is_refused_node(node_id, node, fields)}+ if not _is_invalid_dof_creative_error(response):+ _raise_meta_api_error(response)+ if len(ids) == 1:+ logger.warning("meta_ads_refused_fields_for_node", node_id=ids[0], fields=fields)+ return {}+ middle = len(ids) // 2+ return {+ **_fetch_refusable_fields(node_lookup_url, ids[:middle], fields, access_token),+ **_fetch_refusable_fields(node_lookup_url, ids[middle:], fields, access_token),+ }+++def _restore_refusable_fields(+ rows: list[dict], node_lookup_url: str, fields: tuple[str, ...], access_token: str+) -> list[dict]:+ """Add ``fields`` back to rows fetched without them, as ``None`` for a row Meta refuses them for."""+ ids = [str(row["id"]) for row in rows if row.get("id")]+ nodes: dict[str, dict] = {}+ for start in range(0, len(ids), META_NODE_LOOKUP_MAX_IDS):+ nodes.update(+ _fetch_refusable_fields(+ node_lookup_url, ids[start : start + META_NODE_LOOKUP_MAX_IDS], fields, access_token+ )+ )+ return [{**row, **{field: nodes.get(str(row.get("id")), {}).get(field) for field in fields}} for row in rows]++def _iter_simple_pagination(initial_url: str,params: dict,resume_config: MetaAdsResumeConfig | None,resumable_source_manager: ResumableSourceManager[MetaAdsResumeConfig],+ refusable_fields: tuple[str, ...] = (),+ node_lookup_url: str | None = None,) -> collections.abc.Generator[list[dict]]:"""Iterate a non-time-range Graph API request via ``paging.next`` URLs.@@ -704,9 +791,17 @@ def _iter_simple_pagination(yet, and a cursor points at the start of the next (not-yet-yielded) page.If the smallest limit still fails, the page is retried with backoff andthen raised as retryable (see ``ENTITY_PAGE_REFUSED_ERROR_MESSAGE``).++ When Meta refuses a page for one row's ``refusable_fields`` (see+ ``META_INVALID_DOF_CREATIVE_ERROR_SUBCODE``), the same page is fetched+ again without those fields, and the fields are read back per row from+ ``node_lookup_url``. The next page asks for every field again."""access_token = params["access_token"]current_limit = PAGE_LIMIT_FALLBACK_SIZES[0]+ all_fields = params.get("fields", "")+ reduced_fields = ",".join(field for field in all_fields.split(",") if field not in refusable_fields)+ without_refusable_fields = False# None while on the initial request; set to the active ``paging.next``# cursor once we start following pages. Used to retry-at-smaller-limit.@@ -719,15 +814,18 @@ def _iter_simple_pagination(# healthy syncs keep their original request shape (saved cursor URLs and# the caller's params already encode the default limit).if cursor_url is not None:- url = (- _override_limit(cursor_url, current_limit)- if current_limit != PAGE_LIMIT_FALLBACK_SIZES[0]- else cursor_url- )+ url = cursor_url+ if current_limit != PAGE_LIMIT_FALLBACK_SIZES[0]:+ url = _override_limit(url, current_limit)+ if without_refusable_fields:+ url = _override_query_param(url, "fields", reduced_fields)return _fetch_paging_url(url, access_token)+ request_params = paramsif current_limit != PAGE_LIMIT_FALLBACK_SIZES[0]:- return _get_initial_request(initial_url, {**params, "limit": current_limit})- return _get_initial_request(initial_url, params)+ request_params = {**request_params, "limit": current_limit}+ if without_refusable_fields:+ request_params = {**request_params, "fields": reduced_fields}+ return _get_initial_request(initial_url, request_params)response = _issue()malformed_json_attempts = 0@@ -735,6 +833,15 @@ def _iter_simple_pagination(while True:if response.status_code != 200:+ if (+ refusable_fields+ and node_lookup_url is not None+ and not without_refusable_fields+ and _is_invalid_dof_creative_error(response)+ ):+ without_refusable_fields = True+ response = _issue()+ continue# Too-much-data: shrink the page limit and retry the same request.# Re-issuing the same URL/cursor at a smaller limit is safe — no# already-yielded rows are re-emitted.@@ -769,7 +876,10 @@ def _iter_simple_pagination(malformed_json_attempts = 0smallest_limit_retries = 0- yield response_payload.get("data", [])+ rows = response_payload.get("data", [])+ if without_refusable_fields and node_lookup_url is not None:+ rows = _restore_refusable_fields(rows, node_lookup_url, refusable_fields, access_token)+ yield rowsnext_url = response_payload.get("paging", {}).get("next")if not next_url:@@ -780,6 +890,10 @@ def _iter_simple_pagination(# Strip access_token from the URL before using it so we don't end up with a# duplicated `access_token` query param (requests merges `params=...` into the URL).cursor_url = _strip_access_token(next_url)+ if without_refusable_fields:+ # The cursor of a page fetched without the refusable fields carries the reduced field list.+ cursor_url = _override_query_param(cursor_url, "fields", all_fields)+ without_refusable_fields = Falseresumable_source_manager.save_state(MetaAdsResumeConfig(next_url=cursor_url))response = _issue()@@ -1006,6 +1120,8 @@ def _make_paginated_api_request(access_token: str,time_range: dict | None,resumable_source_manager: ResumableSourceManager[MetaAdsResumeConfig],+ refusable_fields: tuple[str, ...] = (),+ node_lookup_url: str | None = None,) -> collections.abc.Generator[list[dict]]:"""Make paginated requests to the Meta Graph API.This function handles two types of pagination:@@ -1017,7 +1133,9 @@ def _make_paginated_api_request(resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else Noneif time_range is None:- yield from _iter_simple_pagination(url, params, resume_config, resumable_source_manager)+ yield from _iter_simple_pagination(+ url, params, resume_config, resumable_source_manager, refusable_fields, node_lookup_url+ )else:yield from _iter_time_range_pagination(url, params, time_range, resume_config, resumable_source_manager)@@ -1127,7 +1245,15 @@ def meta_ads_source(params.update(_attribution_params(config))yield from _hoist_columns(- _make_paginated_api_request(formatted_url, params, access_token, time_range, resumable_source_manager),+ _make_paginated_api_request(+ formatted_url,+ params,+ access_token,+ time_range,+ resumable_source_manager,+ schema.refusable_fields,+ f"https://graph.facebook.com/{api_version}/",+ ),schema.hoisted_columns,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/schemas.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/schemas.pyindex 7e34343155c0..49f024f48414 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/schemas.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/schemas.py@@ -251,6 +251,9 @@ RESOURCE_SCHEMAS: dict[MetaAdsResource, dict[str, Any]] = {# The Ad node returns its creative as a nested object. Without a scalar id there is no join# from an ad to `ad_creatives`, where the destination URL and the URL tags live."hoisted_columns": (HoistedColumn(source_field="creative", key="id", column="creative_id"),),+ # These fields come from the ad's creative. Meta can refuse them for one ad with an invalid+ # Advantage+ creative, and that ad then syncs without them instead of failing the table.+ "refusable_fields": ("creative", "tracking_specs", "conversion_specs"),},MetaAdsResource.AdStats: {"primary_keys": ["ad_id", "account_id", "date_start"],diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.pyindex 0b40ac02241a..83ec75acdae1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py@@ -348,6 +348,90 @@ class TestSimplePaginationLimitFallback:assert mock_get.return_value.get.call_count == 1+class TestSimplePaginationRefusedFields:+ INITIAL_URL = "https://graph.facebook.com/v20/act_123/ads"+ NODE_LOOKUP_URL = "https://graph.facebook.com/v20/"+ PARAMS: dict[str, Any] = {"fields": "id,name,creative", "limit": 500, "access_token": "tok"}+ REFUSED_BODY: dict[str, Any] = {+ "error": {"message": "Invalid parameter", "code": 100, "error_subcode": 2446211, "is_transient": False}+ }++ def test_refused_row_syncs_without_refusable_fields(self) -> None:+ manager = _build_manager()+ responses = [+ _mock_response(400, self.REFUSED_BODY),+ _mock_response(+ 200,+ {+ "data": [{"id": "1", "name": "a"}, {"id": "2", "name": "b"}],+ "paging": {"next": "https://graph.facebook.com/v20/act_123/ads?fields=id%2Cname&after=p1"},+ },+ ),+ # The lookup for both ads is refused, so it splits until the refused ad is alone.+ _mock_response(400, self.REFUSED_BODY),+ _mock_response(200, {"1": {"id": "1", "creative": {"id": "c1"}}}),+ _mock_response(400, self.REFUSED_BODY),+ _mock_response(200, {"data": [{"id": "3", "name": "c", "creative": {"id": "c3"}}], "paging": {}}),+ ]++ with mock.patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.meta_ads.meta_ads.make_tracked_session"+ ) as mock_get:+ mock_get.return_value.get.side_effect = responses+ batches = list(+ _iter_simple_pagination(+ self.INITIAL_URL, self.PARAMS, None, manager, ("creative",), self.NODE_LOOKUP_URL+ )+ )++ assert batches == [+ [{"id": "1", "name": "a", "creative": {"id": "c1"}}, {"id": "2", "name": "b", "creative": None}],+ [{"id": "3", "name": "c", "creative": {"id": "c3"}}],+ ]+ calls = mock_get.return_value.get.call_args_list+ assert calls[1].kwargs["params"]["fields"] == "id,name"+ assert calls[2].kwargs["params"]["ids"] == "1,2"+ assert "fields=id%2Cname%2Ccreative" in calls[5].args[0]+ saved = manager.save_state.call_args.args[0]+ assert saved.next_url is not None and "fields=id%2Cname%2Ccreative" in saved.next_url++ def test_lookup_node_error_other_than_refusal_raises(self) -> None:+ manager = _build_manager()+ responses = [+ _mock_response(400, self.REFUSED_BODY),+ _mock_response(200, {"data": [{"id": "1", "name": "a"}], "paging": {}}),+ _mock_response(200, {"1": {"error": {"message": "Service temporarily unavailable", "code": 2}}}),+ ]++ with mock.patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.meta_ads.meta_ads.make_tracked_session"+ ) as mock_get:+ mock_get.return_value.get.side_effect = responses+ with pytest.raises(Exception, match="node lookup error"):+ list(+ _iter_simple_pagination(+ self.INITIAL_URL, self.PARAMS, None, manager, ("creative",), self.NODE_LOOKUP_URL+ )+ )++ def test_refusal_without_refusable_fields_raises(self) -> None:+ manager = _build_manager()+ responses = [_mock_response(400, self.REFUSED_BODY), _mock_response(400, self.REFUSED_BODY)]++ with mock.patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.meta_ads.meta_ads.make_tracked_session"+ ) as mock_get:+ mock_get.return_value.get.side_effect = responses+ with pytest.raises(Exception, match="Meta API request failed: 400"):+ list(+ _iter_simple_pagination(+ self.INITIAL_URL, self.PARAMS, None, manager, ("creative",), self.NODE_LOOKUP_URL+ )+ )++ assert mock_get.return_value.get.call_count == 2++class TestSimplePaginationMalformedJson:INITIAL_URL = "https://graph.facebook.com/v20/act_123/campaigns"PARAMS: dict[str, Any] = {"fields": "id,name", "limit": 500, "access_token": "tok"}