:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(meta_ads): recover when an adset's budget is refused (#127521)
Tom Owers01a148df09af
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.pyindex 874f098c1de7..374151a9ad20 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py@@ -472,15 +472,17 @@ META_RATE_LIMIT_ERROR_CODES = {4, 17, 32, 613}# scratch with a new cursor.META_INVALID_CURSOR_ERROR_CODE = 2642-# Meta error code 100 with one of these subcodes: Meta validates a creative again when it serves the-# creative-backed fields of an ad or an ad creative, and refuses one it can no longer render.+# Meta error code 100 with one of these subcodes: Meta refuses to serve a field it cannot compute+# for one row of a page, rejecting the whole page unless we ask again without it.# 2446211: an Advantage+ ("degrees of freedom") creative whose asset feed spec no longer matches# its object story spec.# 1443048: a creative whose object story spec is malformed, for example one with no Page.+# 1885649: an adset's budget fields, when the adset belongs to a Campaign Budget Optimization+# campaign and so carries no budget of its own — the campaign does instead.# Meta refuses the whole page for one such row, so without a fallback every sync of the table stops# at that row. The schema's `refusable_fields` name the fields to drop from the page request and then# read again per row (see `_restore_refusable_fields`).-META_REFUSED_CREATIVE_ERROR_SUBCODES = frozenset({2446211, 1443048})+META_REFUSED_CREATIVE_ERROR_SUBCODES = frozenset({2446211, 1443048, 1885649})# Meta's batch endpoint (see `_fetch_refusable_fields`) accepts at most 50 subrequests per call.# https://developers.facebook.com/docs/graph-api/batch-requests@@ -813,19 +815,23 @@ def _is_transient_batch_subresponse_error(sub_response: typing.Any, error: typin)-def _fetch_refusable_fields(+def _fetch_fields_for_nodes(node_lookup_url: str, ids: list[str], fields: tuple[str, ...], access_token: str-) -> dict[str, dict]:- """Read ``fields`` for each node in ``ids``, keyed by id, skipping a node Meta refuses to serve.+) -> tuple[dict[str, dict], list[str]]:+ """Read ``fields`` together for each node in ``ids``, keyed by id.Uses Meta's batch endpoint (``POST`` with a ``batch`` parameter,https://developers.facebook.com/docs/graph-api/batch-requests) rather than the root multi-idlookup (``GET /?ids=...``), which Meta deprecated in v26.0+. Each subrequest carries its ownstatus, so one refused node doesn't need the rest of its batch retried or bisected, and up to``META_BATCH_MAX_SUBREQUESTS`` ids are read per request rather than one request per id.++ Returns the successful nodes plus the ids Meta refused the combined ``fields`` request for, so+ the caller can retry those narrower (see ``_fetch_refusable_fields``)."""fields_param = ",".join(fields)nodes: dict[str, dict] = {}+ refused_ids: list[str] = []for start in range(0, len(ids), META_BATCH_MAX_SUBREQUESTS):chunk = ids[start : start + META_BATCH_MAX_SUBREQUESTS]data = {@@ -883,16 +889,41 @@ def _fetch_refusable_fields(if sub_response.get("code") == 200 and isinstance(body, dict):nodes[node_id] = bodyelif _is_refused_batch_subresponse(error):- logger.warning("meta_ads_refused_fields_for_node", node_id=node_id, fields=fields)+ refused_ids.append(node_id)else:raise Exception(f"Meta API request failed: node lookup error - {json.dumps(error or body)}")+ return nodes, refused_ids+++def _fetch_refusable_fields(+ node_lookup_url: str, ids: list[str], fields: tuple[str, ...], access_token: str+) -> dict[str, dict]:+ """Read ``fields`` for each node in ``ids``, keyed by id, skipping a field Meta refuses to serve.++ A node refused for the combined ``fields`` request is retried with each field requested on its+ own. Meta can refuse one field of several for reasons specific to that field alone (e.g. an+ adset's budget under Campaign Budget Optimization) while happily serving the rest, so asking only+ once for all of them together would otherwise blank sibling fields Meta never actually refused.+ """+ nodes, refused_ids = _fetch_fields_for_nodes(node_lookup_url, ids, fields, access_token)+ if not refused_ids:+ return nodes+ if len(fields) == 1:+ logger.warning("meta_ads_refused_fields_for_node", node_ids=refused_ids, fields=fields)+ return nodes+ for field in fields:+ field_nodes, still_refused = _fetch_fields_for_nodes(node_lookup_url, refused_ids, (field,), access_token)+ for node_id, body in field_nodes.items():+ nodes.setdefault(node_id, {}).update(body)+ if still_refused:+ logger.warning("meta_ads_refused_fields_for_node", node_ids=still_refused, fields=(field,))return nodesdef _restore_refusable_fields(rows: list[dict], node_lookup_url: str, fields: tuple[str, ...], access_token: str) -> list[dict]:- """Add ``fields`` back to rows fetched without them, as ``None`` for a row Meta refuses them for."""+ """Add ``fields`` back to rows fetched without them, as ``None`` for a field Meta refuses for that row."""ids = [str(row["id"]) for row in rows if row.get("id")]nodes = _fetch_refusable_fields(node_lookup_url, ids, fields, access_token)return [{**row, **{field: nodes.get(str(row.get("id")), {}).get(field) for field in fields}} for row in rows]diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/schemas.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/schemas.pyindex b87fcc682e7d..d5f3eda6eba6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/schemas.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/schemas.py@@ -368,7 +368,9 @@ RESOURCE_SCHEMAS: dict[MetaAdsResource, dict[str, Any]] = {"partition_format": "week","partition_keys": ["created_time"],# An adset's targeting spec can be large enough that Meta refuses even the smallest page.- "refusable_fields": ("targeting", "promoted_object"),+ # Its budget fields can also be refused outright for an adset under Campaign Budget+ # Optimization, whose budget lives on the campaign instead (error_subcode 1885649).+ "refusable_fields": ("targeting", "promoted_object", "budget_remaining", "daily_budget", "lifetime_budget"),},MetaAdsResource.AdsetStats: {"primary_keys": ["adset_id", "account_id", "date_start"],diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/source.pyindex 2d4967d86c31..1ebf2a489905 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/source.py@@ -271,6 +271,13 @@ class MetaAdsSource(ResumableSource[MetaAdsSourceConfig, MetaAdsResumeConfig], O"Meta refused to return an ad creative whose story is incomplete, such as one with no ""Page. Fix or delete that creative in Ads Manager; the next sync runs on schedule."),+ # Graph API code 100, subcode 1885649: Meta refuses to return budget fields for an ad set+ # under Campaign Budget Optimization, whose budget lives on the campaign instead. Keyed on+ # the subcode for the same reason as above — Meta localizes the message.+ '"error_subcode":1885649': (+ "Meta refused to return the budget for an ad set that uses Campaign Budget "+ "Optimization; its budget lives on the campaign instead. The next sync runs on schedule."+ ),}def get_schemas(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.pyindex c86f6bb8653a..70c76bec4f3d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py@@ -438,7 +438,7 @@ class TestSimplePaginationRefusedFields:response.text = ""return response- @pytest.mark.parametrize("subcode", [2446211, 1443048])+ @pytest.mark.parametrize("subcode", [2446211, 1443048, 1885649])def test_refused_row_syncs_without_refusable_fields(self, subcode: int) -> None:manager = _build_manager()refused_body = {"error": {**self.REFUSED_BODY["error"], "error_subcode": subcode}}@@ -485,6 +485,53 @@ class TestSimplePaginationRefusedFields:saved = manager.save_state.call_args.args[0]assert saved.next_url is not None and "fields=id%2Cname%2Ccreative" in saved.next_url+ def test_refused_node_recovers_other_fields_individually(self) -> None:+ manager = _build_manager()+ refused_body = {"error": {**self.REFUSED_BODY["error"], "error_subcode": 1885649}}+ params = {**self.PARAMS, "fields": "id,name,targeting,promoted_object,daily_budget"}+ get_responses = [+ _mock_response(400, refused_body),+ _mock_response(200, {"data": [{"id": "1", "name": "a"}], "paging": {}}),+ ]+ # The combined lookup is refused for id 1 because one of its three fields (daily_budget) is+ # missing. Retrying each field on its own recovers targeting and promoted_object; only+ # daily_budget, the field Meta actually refuses, stays blank.+ batch_responses = [+ self._batch_response([(400, refused_body)]),+ self._batch_response([(200, {"id": "1", "targeting": {"age_min": 18}})]),+ self._batch_response([(200, {"id": "1", "promoted_object": {"pixel_id": "px1"}})]),+ self._batch_response([(400, refused_body)]),+ ]++ with mock.patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.meta_ads.meta_ads.make_tracked_session"+ ) as mock_get:+ mock_get.return_value.get.side_effect = get_responses+ mock_get.return_value.post.side_effect = batch_responses+ batches = list(+ _iter_simple_pagination(+ self.INITIAL_URL,+ params,+ None,+ manager,+ ("targeting", "promoted_object", "daily_budget"),+ self.NODE_LOOKUP_URL,+ )+ )++ assert batches == [+ [+ {+ "id": "1",+ "name": "a",+ "targeting": {"age_min": 18},+ "promoted_object": {"pixel_id": "px1"},+ "daily_budget": None,+ }+ ]+ ]+ assert mock_get.return_value.post.call_count == 4+def test_batch_lookup_error_other_than_refusal_raises(self) -> None:manager = _build_manager()other_error_body = {"error": {"message": "Something else went wrong", "code": 100}}