:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(instagram): sync comment replies into media_comments (#118816)
posthog[bot]fa25814d0ce2
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 119410d83bdb..6679f0791744 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -6363,7 +6363,7 @@ Today (6): `account`, `account_insights`, `media`, `media_comments`, `media_insiDiffed against: <https://raw.githubusercontent.com/facebook/facebook-python-business-sdk/main/facebook_business/adobjects/igmedia.py>-- [ ] `Replies (GET /{ig-comment-id}/replies)` — include conversations below comments when measuring engagement and response activity (medium)+- [x] `Replies (GET /{ig-comment-id}/replies)` — include conversations below comments when measuring engagement and response activity (medium). Added to `media_comments` with `parent_id` set to the top-level comment. The first page of replies comes back inline on each comment, and `/{ig-comment-id}/replies` is called only when a thread has more.- [ ] `Tagged media (GET /{ig-user-id}/tags)` — measure customer mentions and content that tags the account but belongs to other creators (medium)- [ ] `Children (GET /{ig-media-id}/children)` — identify individual images and videos within carousel posts (medium)- [ ] `Collaborators (GET /{ig-media-id}/collaborators)` — attribute collaborative posts to participating accounts (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.pyindex 487ac4212fc9..60fc93718552 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py@@ -64,7 +64,7 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {},},"media_comments": {- "description": "Comments on the account's media, one row per comment. Keyed by media and comment, because the table aggregates comments across every post.",+ "description": "Comments and comment replies on the account's media, one row per comment. Keyed by media and comment, because the table aggregates comments across every post. Replies have parent_id set, so the row count per media matches comments_count on media.","docs_url": _COMMENT_DOCS,"columns": {"id": "Comment identifier.",@@ -74,6 +74,8 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"username": "Username of the account that left the comment.","like_count": "Number of likes on the comment.","hidden": "Whether the comment is hidden on the media.",+ "parent_id": "Identifier of the top-level comment this row replies to. Null for top-level comments.",+ "from": "The account that left the comment, as an object with id and username.",},},"media_insights": {diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.pyindex 4e982ec66d31..a40815c2b77a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py@@ -29,6 +29,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.instagram.MEDIA_PARENT_FIELDS,PAGE_FIELDS,PAGE_SIZE,+ TOP_LEVEL_COMMENT_FIELDS,InstagramEndpointConfig,)@@ -454,12 +455,22 @@ def _media_comment_rows(logger: FilteringBoundLogger,) -> list[dict[str, Any]]:media_id = str(media["id"])- url = client.build_url(f"{media_id}/comments", {"fields": config.fields, "limit": str(PAGE_SIZE)})+ url = client.build_url(f"{media_id}/comments", {"fields": TOP_LEVEL_COMMENT_FIELDS, "limit": str(PAGE_SIZE)})rows: list[dict[str, Any]] = []try:for page, _ in _iter_pages(client, url, MAX_PAGES_PER_PARENT):- rows.extend(_normalize_row({**comment, "media_id": media_id}, config.timestamp_fields) for comment in page)+ for comment in page:+ replies = comment.pop("replies", None)+ rows.append(_normalize_row({**comment, "media_id": media_id}, config.timestamp_fields))+ comment_id = comment.get("id")+ if comment_id and isinstance(replies, dict):+ rows.extend(+ _normalize_row(+ {**reply, "media_id": media_id, "parent_id": str(comment_id)}, config.timestamp_fields+ )+ for reply in _comment_replies(client, str(comment_id), replies, config)+ )except InstagramBadRequestError as error:# Comments are disabled or the post was deleted mid-sync. Neither is fixable by# retrying and neither should sink the whole table.@@ -469,6 +480,28 @@ def _media_comment_rows(return rows+def _comment_replies(+ client: InstagramClient,+ comment_id: str,+ first_page: dict[str, Any],+ config: InstagramEndpointConfig,+) -> Iterator[dict[str, Any]]:+ """Yield every reply in a thread, starting from the page expanded inline on the comment."""+ replies = first_page.get("data")+ yield from (reply for reply in (replies if isinstance(replies, list) else []) if isinstance(reply, dict))++ paging = first_page.get("paging") or {}+ after = (paging.get("cursors") or {}).get("after")+ if not (paging.get("next") and after):+ return++ url = client.build_url(+ f"{comment_id}/replies", {"fields": config.fields, "limit": str(PAGE_SIZE), "after": str(after)}+ )+ for page, _ in _iter_pages(client, url, MAX_PAGES_PER_PARENT):+ yield from page++def _media_insight_rows(client: InstagramClient,media: dict[str, Any],diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.pyindex 0d40eb19cc88..851607637242 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py@@ -19,7 +19,11 @@ MEDIA_FIELDS = ("boost_ads_list,boost_eligibility_info,copyright_check_information,legacy_instagram_media_id")STORY_FIELDS = "id,caption,media_type,media_product_type,media_url,permalink,thumbnail_url,timestamp,username"-COMMENT_FIELDS = "id,text,timestamp,username,like_count,hidden"+COMMENT_FIELDS = "id,text,timestamp,username,like_count,hidden,parent_id,from"+# The media comments edge lists top-level comments only. Expanding `replies` inline brings+# the first page of each thread back on the same request, so a thread costs an extra call+# only when it overflows that page.+TOP_LEVEL_COMMENT_FIELDS = f"{COMMENT_FIELDS},replies.limit(100){{{COMMENT_FIELDS}}}"# Parent listing for the fan-out endpoints — only what the child rows need to be keyed# and partitioned, so the fan-out doesn't pay for the full media projection.MEDIA_PARENT_FIELDS = "id,timestamp,media_product_type"@@ -138,7 +142,8 @@ INSTAGRAM_ENDPOINTS: dict[str, InstagramEndpointConfig] = {timestamp_fields=("timestamp",),),# Comments have no server-side time filter, so the fan-out is a full refresh. The- # media id is part of the key: the table aggregates comments from every post.+ # media id is part of the key: the table aggregates comments from every post. Replies+ # land in the same table with `parent_id` set to the top-level comment."media_comments": InstagramEndpointConfig(name="media_comments",edge="media",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.pyindex 5ca069260f25..b2ed3a438795 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py@@ -433,6 +433,34 @@ class TestInstagramFanOut:assert [(row["media_id"], row["id"]) for row in result.rows] == [("m1", "c1"), ("m2", "c2")]assert result.rows[0]["timestamp"] == "2024-03-01T18:10:00+00:00"+ def test_replies_are_written_under_their_parent_comment_including_overflow_pages(self) -> None:+ result = _drive(+ "media_comments",+ [+ ScriptedResponse(json=_page([{"id": "m1"}])),+ ScriptedResponse(+ json=_page(+ [+ {"id": "c1", "replies": _page([{"id": "r1"}], "RCUR")},+ {"id": "c2", "replies": _page([])},+ ]+ )+ ),+ ScriptedResponse(json=_page([{"id": "r2"}])),+ ],+ )++ assert result.raised is None+ assert [(row["id"], row.get("parent_id")) for row in result.rows] == [+ ("c1", None),+ ("r1", "c1"),+ ("r2", "c1"),+ ("c2", None),+ ]+ assert all("replies" not in row and row["media_id"] == "m1" for row in result.rows)+ assert result.paths[2] == "/v23.0/c1/replies"+ assert result.requests[2].param("after") == "RCUR"+def test_a_post_with_comments_disabled_is_skipped_rather_than_failing_the_table(self) -> None:result = _drive("media_comments",