Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(instagram): sync the media table past a video meta can no longer find (#121975)

Tom Owers25dcfcc44139
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py
index 550a13d870e2..083e50794cbd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py
@@ -1,6 +1,6 @@
import re
import dataclasses
-from collections.abc import Iterator, Sequence
+from collections.abc import Callable, Iterator, Sequence
from datetime import UTC, date, datetime, timedelta
from typing import Any, Optional
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
@@ -23,9 +23,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.instagram.
GRAPH_API_HOST,
INSTAGRAM_ENDPOINTS,
MAX_INSIGHTS_LOOKBACK_DAYS,
+ MEDIA_FIELDS,
MEDIA_INSIGHT_BREAKDOWNS,
MEDIA_INSIGHT_METRICS,
+ MEDIA_PAGE_SAFE_FIELDS,
MEDIA_PARENT_FIELDS,
+ MEDIA_REFUSABLE_FIELDS,
PAGE_FIELDS,
PAGE_SIZE,
TOP_LEVEL_COMMENT_FIELDS,
@@ -74,6 +77,9 @@ META_AUTH_ERROR_CODES = frozenset({102, 190, 458, 459, 460, 463, 464, 467})
META_THROTTLE_ERROR_CODES = frozenset({4, 17, 32, 613})
META_TRANSIENT_ERROR_CODES = frozenset({1, 2})
META_PERMISSION_ERROR_CODE = 10
+# Meta error code 9005, "Video content was not found": a media page asks a video post that Meta can
+# no longer find for one of `MEDIA_REFUSABLE_FIELDS`.
+META_VIDEO_NOT_FOUND_ERROR_CODE = 9005
class InstagramAPIError(Exception):
@@ -93,7 +99,9 @@ class InstagramPermissionError(InstagramAPIError):
class InstagramBadRequestError(InstagramAPIError):
- pass
+ def __init__(self, message: str, code: Any = None) -> None:
+ super().__init__(message)
+ self.code = code
class InstagramRequestBudgetError(InstagramAPIError):
@@ -317,13 +325,14 @@ class InstagramClient:
raise InstagramPermissionError(f"{PERMISSION_ERROR_PREFIX}: {detail}")
self._logger.error(f"Instagram API error: {detail}")
- raise InstagramBadRequestError(f"Instagram API error: {detail}")
+ raise InstagramBadRequestError(f"Instagram API error: {detail}", code=code)
def _iter_pages(
client: InstagramClient,
url: str,
max_pages: int,
+ get: Optional[Callable[[str], dict[str, Any]]] = None,
) -> Iterator[tuple[list[dict[str, Any]], Optional[str]]]:
"""Yield `(rows, next_url)` per page of a cursor-paginated edge.
@@ -334,7 +343,7 @@ def _iter_pages(
pages = 0
current: Optional[str] = url
while current is not None and pages < max_pages and client.has_request_budget():
- body = client.get(current)
+ body = (get or client.get)(current)
rows = body.get("data")
rows = rows if isinstance(rows, list) else []
paging = body.get("paging") or {}
@@ -372,6 +381,40 @@ def _insight_points(entry: dict[str, Any]) -> list[dict[str, Any]]:
return []
+def _media_page(client: InstagramClient, url: str, logger: FilteringBoundLogger) -> dict[str, Any]:
+ """Read one page of the media edge, reading it again without the refusable fields if Meta refuses it.
+
+ Each post on a refused page then reads the refusable fields on its own, so only the post Meta
+ cannot serve keeps them empty. The next page asks for every field again.
+ """
+ try:
+ return client.get(url)
+ except InstagramBadRequestError as error:
+ if error.code != META_VIDEO_NOT_FOUND_ERROR_CODE:
+ raise
+ logger.warning(f"Instagram: re-reading a media page without its refusable fields: {error}")
+
+ body = client.get(_with_query_param(url, "fields", MEDIA_PAGE_SAFE_FIELDS))
+ rows = body.get("data")
+ for row in rows if isinstance(rows, list) else []:
+ if not isinstance(row, dict) or not row.get("id"):
+ continue
+ if not client.has_request_budget():
+ # Without a cursor the checkpoint stays on this page, so a resumed attempt reads it again.
+ body.pop("paging", None)
+ break
+ media_id = str(row["id"])
+ try:
+ extra = client.get(client.build_url(media_id, {"fields": MEDIA_REFUSABLE_FIELDS}))
+ except InstagramBadRequestError as error:
+ if error.code != META_VIDEO_NOT_FOUND_ERROR_CODE:
+ raise
+ logger.warning(f"Instagram: skipping refusable fields for media {media_id}: {error}")
+ continue
+ row.update({name: value for name, value in extra.items() if name != "id"})
+ return body
+
+
def _media_rows(
client: InstagramClient,
instagram_account_id: str,
@@ -389,7 +432,8 @@ def _media_rows(
_account_path(instagram_account_id, config.edge), params
)
- for rows, next_url in _iter_pages(client, url, MAX_PAGES_PER_STREAM):
+ get_page = (lambda page_url: _media_page(client, page_url, logger)) if config.fields == MEDIA_FIELDS else None
+ for rows, next_url in _iter_pages(client, url, MAX_PAGES_PER_STREAM, get=get_page):
if rows:
yield [_normalize_row(row, config.timestamp_fields) for row in rows]
# Checkpoint after the yield: a crash then re-yields the last page instead of
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py
index 2abb8b34e882..c19a222202c8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py
@@ -15,11 +15,18 @@ GRAPH_API_HOST = "https://graph.facebook.com"
# Node/edge field selections. Meta returns nothing but `id` unless `fields` is passed,
# so every request names its columns explicitly.
ACCOUNT_FIELDS = "id,username,name,biography,website,profile_picture_url,followers_count,follows_count,media_count"
-MEDIA_FIELDS = (
+MEDIA_PAGE_SAFE_FIELDS = (
"id,caption,media_type,media_product_type,media_url,permalink,thumbnail_url,"
- "timestamp,username,shortcode,like_count,comments_count,is_comment_enabled,is_shared_to_feed,owner,"
- "boost_ads_list,boost_eligibility_info,copyright_check_information,legacy_instagram_media_id"
+ "timestamp,username,shortcode,like_count,comments_count,is_comment_enabled"
)
+# Meta refuses a whole media page when one post on it is a video Meta can no longer find and the
+# page asks for one of these fields. A refused page is read again with the safe fields only, and
+# each of its posts then reads these fields on its own (see `_media_page`).
+MEDIA_REFUSABLE_FIELDS = (
+ "is_shared_to_feed,owner,boost_ads_list,boost_eligibility_info,copyright_check_information,"
+ "legacy_instagram_media_id"
+)
+MEDIA_FIELDS = f"{MEDIA_PAGE_SAFE_FIELDS},{MEDIA_REFUSABLE_FIELDS}"
STORY_FIELDS = "id,caption,media_type,media_product_type,media_url,permalink,thumbnail_url,timestamp,username"
COMMENT_FIELDS = "id,text,timestamp,username,like_count,hidden,parent_id,from"
# The media comments edge lists top-level comments only. Expanding `replies` inline brings
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py
index 34235a43e537..6c665da7adb1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py
@@ -53,6 +53,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.instagram.
INSTAGRAM_ENDPOINTS,
MAX_INSIGHTS_LOOKBACK_DAYS,
MEDIA_INSIGHT_METRICS,
+ MEDIA_PAGE_SAFE_FIELDS,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.instagram.source import InstagramSource
@@ -353,6 +354,29 @@ class TestInstagramEdges:
assert result.rows[0]["timestamp"] == "2024-03-01T18:10:00+00:00"
assert result.params("after") == [None, "CUR"]
+ @pytest.mark.parametrize("second_post_code,fails", [(9005, False), (100, True)])
+ def test_a_media_page_meta_refuses_for_one_video_is_read_again_without_the_refusable_fields(
+ self, second_post_code: int, fails: bool
+ ) -> None:
+ video_not_found = ScriptedResponse(status=400, json={"error": {"code": 9005, "message": "not found"}})
+ result = _drive(
+ "media",
+ [
+ video_not_found,
+ ScriptedResponse(json=_page([{"id": "1"}, {"id": "2"}])),
+ ScriptedResponse(json={"id": "1", "owner": {"id": "o1"}}),
+ ScriptedResponse(status=400, json={"error": {"code": second_post_code, "message": "no"}}),
+ ],
+ )
+
+ assert result.params("fields")[1] == MEDIA_PAGE_SAFE_FIELDS
+ assert result.paths[2:] == ["/v23.0/1", "/v23.0/2"]
+ if fails:
+ assert isinstance(result.raised, InstagramBadRequestError)
+ else:
+ assert result.raised is None
+ assert result.rows == [{"id": "1", "owner": {"id": "o1"}}, {"id": "2"}]
+
def test_media_stops_when_the_cursor_is_missing_even_though_next_is_present(self) -> None:
result = _drive("media", [ScriptedResponse(json={"data": [{"id": "1"}], "paging": {"next": "https://x/next"}})])