Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(instagram): sync comment replies into media_comments (#118816)

posthog[bot]fa25814d0ce2
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 119410d83bdb..6679f0791744 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -6363,7 +6363,7 @@ Today (6): `account`, `account_insights`, `media`, `media_comments`, `media_insi
Diffed against: <https://raw.githubusercontent.com/facebook/facebook-python-business-sdk/main/facebook_business/adobjects/igmedia.py>
-- [ ] `Replies (GET /{ig-comment-id}/replies)` — include conversations below comments when measuring engagement and response activity (medium)
+- [x] `Replies (GET /{ig-comment-id}/replies)` — include conversations below comments when measuring engagement and response activity (medium). Added to `media_comments` with `parent_id` set to the top-level comment. The first page of replies comes back inline on each comment, and `/{ig-comment-id}/replies` is called only when a thread has more.
- [ ] `Tagged media (GET /{ig-user-id}/tags)` — measure customer mentions and content that tags the account but belongs to other creators (medium)
- [ ] `Children (GET /{ig-media-id}/children)` — identify individual images and videos within carousel posts (medium)
- [ ] `Collaborators (GET /{ig-media-id}/collaborators)` — attribute collaborative posts to participating accounts (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py
index 487ac4212fc9..60fc93718552 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py
@@ -64,7 +64,7 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
},
},
"media_comments": {
- "description": "Comments on the account's media, one row per comment. Keyed by media and comment, because the table aggregates comments across every post.",
+ "description": "Comments and comment replies on the account's media, one row per comment. Keyed by media and comment, because the table aggregates comments across every post. Replies have parent_id set, so the row count per media matches comments_count on media.",
"docs_url": _COMMENT_DOCS,
"columns": {
"id": "Comment identifier.",
@@ -74,6 +74,8 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"username": "Username of the account that left the comment.",
"like_count": "Number of likes on the comment.",
"hidden": "Whether the comment is hidden on the media.",
+ "parent_id": "Identifier of the top-level comment this row replies to. Null for top-level comments.",
+ "from": "The account that left the comment, as an object with id and username.",
},
},
"media_insights": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py
index 4e982ec66d31..a40815c2b77a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py
@@ -29,6 +29,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.instagram.
MEDIA_PARENT_FIELDS,
PAGE_FIELDS,
PAGE_SIZE,
+ TOP_LEVEL_COMMENT_FIELDS,
InstagramEndpointConfig,
)
@@ -454,12 +455,22 @@ def _media_comment_rows(
logger: FilteringBoundLogger,
) -> list[dict[str, Any]]:
media_id = str(media["id"])
- url = client.build_url(f"{media_id}/comments", {"fields": config.fields, "limit": str(PAGE_SIZE)})
+ url = client.build_url(f"{media_id}/comments", {"fields": TOP_LEVEL_COMMENT_FIELDS, "limit": str(PAGE_SIZE)})
rows: list[dict[str, Any]] = []
try:
for page, _ in _iter_pages(client, url, MAX_PAGES_PER_PARENT):
- rows.extend(_normalize_row({**comment, "media_id": media_id}, config.timestamp_fields) for comment in page)
+ for comment in page:
+ replies = comment.pop("replies", None)
+ rows.append(_normalize_row({**comment, "media_id": media_id}, config.timestamp_fields))
+ comment_id = comment.get("id")
+ if comment_id and isinstance(replies, dict):
+ rows.extend(
+ _normalize_row(
+ {**reply, "media_id": media_id, "parent_id": str(comment_id)}, config.timestamp_fields
+ )
+ for reply in _comment_replies(client, str(comment_id), replies, config)
+ )
except InstagramBadRequestError as error:
# Comments are disabled or the post was deleted mid-sync. Neither is fixable by
# retrying and neither should sink the whole table.
@@ -469,6 +480,28 @@ def _media_comment_rows(
return rows
+def _comment_replies(
+ client: InstagramClient,
+ comment_id: str,
+ first_page: dict[str, Any],
+ config: InstagramEndpointConfig,
+) -> Iterator[dict[str, Any]]:
+ """Yield every reply in a thread, starting from the page expanded inline on the comment."""
+ replies = first_page.get("data")
+ yield from (reply for reply in (replies if isinstance(replies, list) else []) if isinstance(reply, dict))
+
+ paging = first_page.get("paging") or {}
+ after = (paging.get("cursors") or {}).get("after")
+ if not (paging.get("next") and after):
+ return
+
+ url = client.build_url(
+ f"{comment_id}/replies", {"fields": config.fields, "limit": str(PAGE_SIZE), "after": str(after)}
+ )
+ for page, _ in _iter_pages(client, url, MAX_PAGES_PER_PARENT):
+ yield from page
+
+
def _media_insight_rows(
client: InstagramClient,
media: dict[str, Any],
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py
index 0d40eb19cc88..851607637242 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py
@@ -19,7 +19,11 @@ MEDIA_FIELDS = (
"boost_ads_list,boost_eligibility_info,copyright_check_information,legacy_instagram_media_id"
)
STORY_FIELDS = "id,caption,media_type,media_product_type,media_url,permalink,thumbnail_url,timestamp,username"
-COMMENT_FIELDS = "id,text,timestamp,username,like_count,hidden"
+COMMENT_FIELDS = "id,text,timestamp,username,like_count,hidden,parent_id,from"
+# The media comments edge lists top-level comments only. Expanding `replies` inline brings
+# the first page of each thread back on the same request, so a thread costs an extra call
+# only when it overflows that page.
+TOP_LEVEL_COMMENT_FIELDS = f"{COMMENT_FIELDS},replies.limit(100){{{COMMENT_FIELDS}}}"
# Parent listing for the fan-out endpoints — only what the child rows need to be keyed
# and partitioned, so the fan-out doesn't pay for the full media projection.
MEDIA_PARENT_FIELDS = "id,timestamp,media_product_type"
@@ -138,7 +142,8 @@ INSTAGRAM_ENDPOINTS: dict[str, InstagramEndpointConfig] = {
timestamp_fields=("timestamp",),
),
# Comments have no server-side time filter, so the fan-out is a full refresh. The
- # media id is part of the key: the table aggregates comments from every post.
+ # media id is part of the key: the table aggregates comments from every post. Replies
+ # land in the same table with `parent_id` set to the top-level comment.
"media_comments": InstagramEndpointConfig(
name="media_comments",
edge="media",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py
index 5ca069260f25..b2ed3a438795 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py
@@ -433,6 +433,34 @@ class TestInstagramFanOut:
assert [(row["media_id"], row["id"]) for row in result.rows] == [("m1", "c1"), ("m2", "c2")]
assert result.rows[0]["timestamp"] == "2024-03-01T18:10:00+00:00"
+ def test_replies_are_written_under_their_parent_comment_including_overflow_pages(self) -> None:
+ result = _drive(
+ "media_comments",
+ [
+ ScriptedResponse(json=_page([{"id": "m1"}])),
+ ScriptedResponse(
+ json=_page(
+ [
+ {"id": "c1", "replies": _page([{"id": "r1"}], "RCUR")},
+ {"id": "c2", "replies": _page([])},
+ ]
+ )
+ ),
+ ScriptedResponse(json=_page([{"id": "r2"}])),
+ ],
+ )
+
+ assert result.raised is None
+ assert [(row["id"], row.get("parent_id")) for row in result.rows] == [
+ ("c1", None),
+ ("r1", "c1"),
+ ("r2", "c1"),
+ ("c2", None),
+ ]
+ assert all("replies" not in row and row["media_id"] == "m1" for row in result.rows)
+ assert result.paths[2] == "/v23.0/c1/replies"
+ assert result.requests[2].param("after") == "RCUR"
+
def test_a_post_with_comments_disabled_is_skipped_rather_than_failing_the_table(self) -> None:
result = _drive(
"media_comments",