Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(meta_ads): recover when an adset's budget is refused (#127521)

Tom Owers01a148df09af
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py
index 874f098c1de7..374151a9ad20 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py
@@ -472,15 +472,17 @@ META_RATE_LIMIT_ERROR_CODES = {4, 17, 32, 613}
# scratch with a new cursor.
META_INVALID_CURSOR_ERROR_CODE = 2642
-# Meta error code 100 with one of these subcodes: Meta validates a creative again when it serves the
-# creative-backed fields of an ad or an ad creative, and refuses one it can no longer render.
+# Meta error code 100 with one of these subcodes: Meta refuses to serve a field it cannot compute
+# for one row of a page, rejecting the whole page unless we ask again without it.
# 2446211: an Advantage+ ("degrees of freedom") creative whose asset feed spec no longer matches
# its object story spec.
# 1443048: a creative whose object story spec is malformed, for example one with no Page.
+# 1885649: an adset's budget fields, when the adset belongs to a Campaign Budget Optimization
+# campaign and so carries no budget of its own — the campaign does instead.
# Meta refuses the whole page for one such row, so without a fallback every sync of the table stops
# at that row. The schema's `refusable_fields` name the fields to drop from the page request and then
# read again per row (see `_restore_refusable_fields`).
-META_REFUSED_CREATIVE_ERROR_SUBCODES = frozenset({2446211, 1443048})
+META_REFUSED_CREATIVE_ERROR_SUBCODES = frozenset({2446211, 1443048, 1885649})
# Meta's batch endpoint (see `_fetch_refusable_fields`) accepts at most 50 subrequests per call.
# https://developers.facebook.com/docs/graph-api/batch-requests
@@ -813,19 +815,23 @@ def _is_transient_batch_subresponse_error(sub_response: typing.Any, error: typin
)
-def _fetch_refusable_fields(
+def _fetch_fields_for_nodes(
node_lookup_url: str, ids: list[str], fields: tuple[str, ...], access_token: str
-) -> dict[str, dict]:
- """Read ``fields`` for each node in ``ids``, keyed by id, skipping a node Meta refuses to serve.
+) -> tuple[dict[str, dict], list[str]]:
+ """Read ``fields`` together for each node in ``ids``, keyed by id.
Uses Meta's batch endpoint (``POST`` with a ``batch`` parameter,
https://developers.facebook.com/docs/graph-api/batch-requests) rather than the root multi-id
lookup (``GET /?ids=...``), which Meta deprecated in v26.0+. Each subrequest carries its own
status, so one refused node doesn't need the rest of its batch retried or bisected, and up to
``META_BATCH_MAX_SUBREQUESTS`` ids are read per request rather than one request per id.
+
+ Returns the successful nodes plus the ids Meta refused the combined ``fields`` request for, so
+ the caller can retry those narrower (see ``_fetch_refusable_fields``).
"""
fields_param = ",".join(fields)
nodes: dict[str, dict] = {}
+ refused_ids: list[str] = []
for start in range(0, len(ids), META_BATCH_MAX_SUBREQUESTS):
chunk = ids[start : start + META_BATCH_MAX_SUBREQUESTS]
data = {
@@ -883,16 +889,41 @@ def _fetch_refusable_fields(
if sub_response.get("code") == 200 and isinstance(body, dict):
nodes[node_id] = body
elif _is_refused_batch_subresponse(error):
- logger.warning("meta_ads_refused_fields_for_node", node_id=node_id, fields=fields)
+ refused_ids.append(node_id)
else:
raise Exception(f"Meta API request failed: node lookup error - {json.dumps(error or body)}")
+ return nodes, refused_ids
+
+
+def _fetch_refusable_fields(
+ node_lookup_url: str, ids: list[str], fields: tuple[str, ...], access_token: str
+) -> dict[str, dict]:
+ """Read ``fields`` for each node in ``ids``, keyed by id, skipping a field Meta refuses to serve.
+
+ A node refused for the combined ``fields`` request is retried with each field requested on its
+ own. Meta can refuse one field of several for reasons specific to that field alone (e.g. an
+ adset's budget under Campaign Budget Optimization) while happily serving the rest, so asking only
+ once for all of them together would otherwise blank sibling fields Meta never actually refused.
+ """
+ nodes, refused_ids = _fetch_fields_for_nodes(node_lookup_url, ids, fields, access_token)
+ if not refused_ids:
+ return nodes
+ if len(fields) == 1:
+ logger.warning("meta_ads_refused_fields_for_node", node_ids=refused_ids, fields=fields)
+ return nodes
+ for field in fields:
+ field_nodes, still_refused = _fetch_fields_for_nodes(node_lookup_url, refused_ids, (field,), access_token)
+ for node_id, body in field_nodes.items():
+ nodes.setdefault(node_id, {}).update(body)
+ if still_refused:
+ logger.warning("meta_ads_refused_fields_for_node", node_ids=still_refused, fields=(field,))
return nodes
def _restore_refusable_fields(
rows: list[dict], node_lookup_url: str, fields: tuple[str, ...], access_token: str
) -> list[dict]:
- """Add ``fields`` back to rows fetched without them, as ``None`` for a row Meta refuses them for."""
+ """Add ``fields`` back to rows fetched without them, as ``None`` for a field Meta refuses for that row."""
ids = [str(row["id"]) for row in rows if row.get("id")]
nodes = _fetch_refusable_fields(node_lookup_url, ids, fields, access_token)
return [{**row, **{field: nodes.get(str(row.get("id")), {}).get(field) for field in fields}} for row in rows]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/schemas.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/schemas.py
index b87fcc682e7d..d5f3eda6eba6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/schemas.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/schemas.py
@@ -368,7 +368,9 @@ RESOURCE_SCHEMAS: dict[MetaAdsResource, dict[str, Any]] = {
"partition_format": "week",
"partition_keys": ["created_time"],
# An adset's targeting spec can be large enough that Meta refuses even the smallest page.
- "refusable_fields": ("targeting", "promoted_object"),
+ # Its budget fields can also be refused outright for an adset under Campaign Budget
+ # Optimization, whose budget lives on the campaign instead (error_subcode 1885649).
+ "refusable_fields": ("targeting", "promoted_object", "budget_remaining", "daily_budget", "lifetime_budget"),
},
MetaAdsResource.AdsetStats: {
"primary_keys": ["adset_id", "account_id", "date_start"],
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/source.py
index 2d4967d86c31..1ebf2a489905 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/source.py
@@ -271,6 +271,13 @@ class MetaAdsSource(ResumableSource[MetaAdsSourceConfig, MetaAdsResumeConfig], O
"Meta refused to return an ad creative whose story is incomplete, such as one with no "
"Page. Fix or delete that creative in Ads Manager; the next sync runs on schedule."
),
+ # Graph API code 100, subcode 1885649: Meta refuses to return budget fields for an ad set
+ # under Campaign Budget Optimization, whose budget lives on the campaign instead. Keyed on
+ # the subcode for the same reason as above — Meta localizes the message.
+ '"error_subcode":1885649': (
+ "Meta refused to return the budget for an ad set that uses Campaign Budget "
+ "Optimization; its budget lives on the campaign instead. The next sync runs on schedule."
+ ),
}
def get_schemas(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py
index c86f6bb8653a..70c76bec4f3d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py
@@ -438,7 +438,7 @@ class TestSimplePaginationRefusedFields:
response.text = ""
return response
- @pytest.mark.parametrize("subcode", [2446211, 1443048])
+ @pytest.mark.parametrize("subcode", [2446211, 1443048, 1885649])
def test_refused_row_syncs_without_refusable_fields(self, subcode: int) -> None:
manager = _build_manager()
refused_body = {"error": {**self.REFUSED_BODY["error"], "error_subcode": subcode}}
@@ -485,6 +485,53 @@ class TestSimplePaginationRefusedFields:
saved = manager.save_state.call_args.args[0]
assert saved.next_url is not None and "fields=id%2Cname%2Ccreative" in saved.next_url
+ def test_refused_node_recovers_other_fields_individually(self) -> None:
+ manager = _build_manager()
+ refused_body = {"error": {**self.REFUSED_BODY["error"], "error_subcode": 1885649}}
+ params = {**self.PARAMS, "fields": "id,name,targeting,promoted_object,daily_budget"}
+ get_responses = [
+ _mock_response(400, refused_body),
+ _mock_response(200, {"data": [{"id": "1", "name": "a"}], "paging": {}}),
+ ]
+ # The combined lookup is refused for id 1 because one of its three fields (daily_budget) is
+ # missing. Retrying each field on its own recovers targeting and promoted_object; only
+ # daily_budget, the field Meta actually refuses, stays blank.
+ batch_responses = [
+ self._batch_response([(400, refused_body)]),
+ self._batch_response([(200, {"id": "1", "targeting": {"age_min": 18}})]),
+ self._batch_response([(200, {"id": "1", "promoted_object": {"pixel_id": "px1"}})]),
+ self._batch_response([(400, refused_body)]),
+ ]
+
+ with mock.patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.meta_ads.meta_ads.make_tracked_session"
+ ) as mock_get:
+ mock_get.return_value.get.side_effect = get_responses
+ mock_get.return_value.post.side_effect = batch_responses
+ batches = list(
+ _iter_simple_pagination(
+ self.INITIAL_URL,
+ params,
+ None,
+ manager,
+ ("targeting", "promoted_object", "daily_budget"),
+ self.NODE_LOOKUP_URL,
+ )
+ )
+
+ assert batches == [
+ [
+ {
+ "id": "1",
+ "name": "a",
+ "targeting": {"age_min": 18},
+ "promoted_object": {"pixel_id": "px1"},
+ "daily_budget": None,
+ }
+ ]
+ ]
+ assert mock_get.return_value.post.call_count == 4
+
def test_batch_lookup_error_other_than_refusal_raises(self) -> None:
manager = _build_manager()
other_error_body = {"error": {"message": "Something else went wrong", "code": 100}}