:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(instagram): sync the media table past a video meta can no longer find (#121975)
Tom Owers25dcfcc44139
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.pyindex 550a13d870e2..083e50794cbd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py@@ -1,6 +1,6 @@import reimport dataclasses-from collections.abc import Iterator, Sequence+from collections.abc import Callable, Iterator, Sequencefrom datetime import UTC, date, datetime, timedeltafrom typing import Any, Optionalfrom urllib.parse import parse_qsl, urlencode, urlparse, urlunparse@@ -23,9 +23,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.instagram.GRAPH_API_HOST,INSTAGRAM_ENDPOINTS,MAX_INSIGHTS_LOOKBACK_DAYS,+ MEDIA_FIELDS,MEDIA_INSIGHT_BREAKDOWNS,MEDIA_INSIGHT_METRICS,+ MEDIA_PAGE_SAFE_FIELDS,MEDIA_PARENT_FIELDS,+ MEDIA_REFUSABLE_FIELDS,PAGE_FIELDS,PAGE_SIZE,TOP_LEVEL_COMMENT_FIELDS,@@ -74,6 +77,9 @@ META_AUTH_ERROR_CODES = frozenset({102, 190, 458, 459, 460, 463, 464, 467})META_THROTTLE_ERROR_CODES = frozenset({4, 17, 32, 613})META_TRANSIENT_ERROR_CODES = frozenset({1, 2})META_PERMISSION_ERROR_CODE = 10+# Meta error code 9005, "Video content was not found": a media page asks a video post that Meta can+# no longer find for one of `MEDIA_REFUSABLE_FIELDS`.+META_VIDEO_NOT_FOUND_ERROR_CODE = 9005class InstagramAPIError(Exception):@@ -93,7 +99,9 @@ class InstagramPermissionError(InstagramAPIError):class InstagramBadRequestError(InstagramAPIError):- pass+ def __init__(self, message: str, code: Any = None) -> None:+ super().__init__(message)+ self.code = codeclass InstagramRequestBudgetError(InstagramAPIError):@@ -317,13 +325,14 @@ class InstagramClient:raise InstagramPermissionError(f"{PERMISSION_ERROR_PREFIX}: {detail}")self._logger.error(f"Instagram API error: {detail}")- raise InstagramBadRequestError(f"Instagram API error: {detail}")+ raise InstagramBadRequestError(f"Instagram API error: {detail}", code=code)def _iter_pages(client: InstagramClient,url: str,max_pages: int,+ get: Optional[Callable[[str], dict[str, Any]]] = None,) -> Iterator[tuple[list[dict[str, Any]], Optional[str]]]:"""Yield `(rows, next_url)` per page of a cursor-paginated edge.@@ -334,7 +343,7 @@ def _iter_pages(pages = 0current: Optional[str] = urlwhile current is not None and pages < max_pages and client.has_request_budget():- body = client.get(current)+ body = (get or client.get)(current)rows = body.get("data")rows = rows if isinstance(rows, list) else []paging = body.get("paging") or {}@@ -372,6 +381,40 @@ def _insight_points(entry: dict[str, Any]) -> list[dict[str, Any]]:return []+def _media_page(client: InstagramClient, url: str, logger: FilteringBoundLogger) -> dict[str, Any]:+ """Read one page of the media edge, reading it again without the refusable fields if Meta refuses it.++ Each post on a refused page then reads the refusable fields on its own, so only the post Meta+ cannot serve keeps them empty. The next page asks for every field again.+ """+ try:+ return client.get(url)+ except InstagramBadRequestError as error:+ if error.code != META_VIDEO_NOT_FOUND_ERROR_CODE:+ raise+ logger.warning(f"Instagram: re-reading a media page without its refusable fields: {error}")++ body = client.get(_with_query_param(url, "fields", MEDIA_PAGE_SAFE_FIELDS))+ rows = body.get("data")+ for row in rows if isinstance(rows, list) else []:+ if not isinstance(row, dict) or not row.get("id"):+ continue+ if not client.has_request_budget():+ # Without a cursor the checkpoint stays on this page, so a resumed attempt reads it again.+ body.pop("paging", None)+ break+ media_id = str(row["id"])+ try:+ extra = client.get(client.build_url(media_id, {"fields": MEDIA_REFUSABLE_FIELDS}))+ except InstagramBadRequestError as error:+ if error.code != META_VIDEO_NOT_FOUND_ERROR_CODE:+ raise+ logger.warning(f"Instagram: skipping refusable fields for media {media_id}: {error}")+ continue+ row.update({name: value for name, value in extra.items() if name != "id"})+ return body++def _media_rows(client: InstagramClient,instagram_account_id: str,@@ -389,7 +432,8 @@ def _media_rows(_account_path(instagram_account_id, config.edge), params)- for rows, next_url in _iter_pages(client, url, MAX_PAGES_PER_STREAM):+ get_page = (lambda page_url: _media_page(client, page_url, logger)) if config.fields == MEDIA_FIELDS else None+ for rows, next_url in _iter_pages(client, url, MAX_PAGES_PER_STREAM, get=get_page):if rows:yield [_normalize_row(row, config.timestamp_fields) for row in rows]# Checkpoint after the yield: a crash then re-yields the last page instead ofdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.pyindex 2abb8b34e882..c19a222202c8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py@@ -15,11 +15,18 @@ GRAPH_API_HOST = "https://graph.facebook.com"# Node/edge field selections. Meta returns nothing but `id` unless `fields` is passed,# so every request names its columns explicitly.ACCOUNT_FIELDS = "id,username,name,biography,website,profile_picture_url,followers_count,follows_count,media_count"-MEDIA_FIELDS = (+MEDIA_PAGE_SAFE_FIELDS = ("id,caption,media_type,media_product_type,media_url,permalink,thumbnail_url,"- "timestamp,username,shortcode,like_count,comments_count,is_comment_enabled,is_shared_to_feed,owner,"- "boost_ads_list,boost_eligibility_info,copyright_check_information,legacy_instagram_media_id"+ "timestamp,username,shortcode,like_count,comments_count,is_comment_enabled")+# Meta refuses a whole media page when one post on it is a video Meta can no longer find and the+# page asks for one of these fields. A refused page is read again with the safe fields only, and+# each of its posts then reads these fields on its own (see `_media_page`).+MEDIA_REFUSABLE_FIELDS = (+ "is_shared_to_feed,owner,boost_ads_list,boost_eligibility_info,copyright_check_information,"+ "legacy_instagram_media_id"+)+MEDIA_FIELDS = f"{MEDIA_PAGE_SAFE_FIELDS},{MEDIA_REFUSABLE_FIELDS}"STORY_FIELDS = "id,caption,media_type,media_product_type,media_url,permalink,thumbnail_url,timestamp,username"COMMENT_FIELDS = "id,text,timestamp,username,like_count,hidden,parent_id,from"# The media comments edge lists top-level comments only. Expanding `replies` inline bringsdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.pyindex 34235a43e537..6c665da7adb1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py@@ -53,6 +53,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.instagram.INSTAGRAM_ENDPOINTS,MAX_INSIGHTS_LOOKBACK_DAYS,MEDIA_INSIGHT_METRICS,+ MEDIA_PAGE_SAFE_FIELDS,)from products.warehouse_sources.backend.temporal.data_imports.sources.instagram.source import InstagramSource@@ -353,6 +354,29 @@ class TestInstagramEdges:assert result.rows[0]["timestamp"] == "2024-03-01T18:10:00+00:00"assert result.params("after") == [None, "CUR"]+ @pytest.mark.parametrize("second_post_code,fails", [(9005, False), (100, True)])+ def test_a_media_page_meta_refuses_for_one_video_is_read_again_without_the_refusable_fields(+ self, second_post_code: int, fails: bool+ ) -> None:+ video_not_found = ScriptedResponse(status=400, json={"error": {"code": 9005, "message": "not found"}})+ result = _drive(+ "media",+ [+ video_not_found,+ ScriptedResponse(json=_page([{"id": "1"}, {"id": "2"}])),+ ScriptedResponse(json={"id": "1", "owner": {"id": "o1"}}),+ ScriptedResponse(status=400, json={"error": {"code": second_post_code, "message": "no"}}),+ ],+ )++ assert result.params("fields")[1] == MEDIA_PAGE_SAFE_FIELDS+ assert result.paths[2:] == ["/v23.0/1", "/v23.0/2"]+ if fails:+ assert isinstance(result.raised, InstagramBadRequestError)+ else:+ assert result.raised is None+ assert result.rows == [{"id": "1", "owner": {"id": "o1"}}, {"id": "2"}]+def test_media_stops_when_the_cursor_is_missing_even_though_next_is_present(self) -> None:result = _drive("media", [ScriptedResponse(json={"data": [{"id": "1"}], "paging": {"next": "https://x/next"}})])