Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): re-read all instagram media and add media fields (#118815)

posthog[bot]a3fc003d6a4f
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py
index 85b738500b51..99eabf27ff43 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py
@@ -24,7 +24,7 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
},
},
"media": {
- "description": "Posts, reels and carousels published by the account, newest first. Limited to the 10,000 most recently created media.",
+ "description": "Posts, reels and carousels published by the account, newest first. Limited to the 10,000 most recently created media. Every sync re-reads each post from the start date on, so like and comment counts stay current.",
"docs_url": _MEDIA_DOCS,
"columns": {
"id": "Media identifier.",
@@ -40,6 +40,12 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"like_count": "Number of likes. Omitted if the owner has hidden like counts.",
"comments_count": "Number of comments on the media.",
"is_comment_enabled": "Whether comments are enabled on the media.",
+ "is_shared_to_feed": "Whether the reel also appears in the Feed and Reels tabs. Reels only.",
+ "owner": "The account that created the media, as an object with its ID.",
+ "boost_ads_list": "Ads that boost this media, with each ad's ID and delivery status.",
+ "boost_eligibility_info": "Whether the media can be boosted as an ad, and the reason if it cannot.",
+ "copyright_check_information": "Status of Instagram's copyright check on the media, with any matches found.",
+ "legacy_instagram_media_id": "Media ID that the Marketing API used in Graph API v21.0 and older.",
},
},
"stories": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py
index 0c0ca24c8718..b70799042e39 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py
@@ -20,6 +20,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.instagram.
ACCOUNT_INSIGHTS_WINDOW_DAYS,
DEFAULT_INSIGHTS_LOOKBACK_DAYS,
DEFAULT_MEDIA_INSIGHT_METRICS,
+ ENDPOINTS_REREAD_EVERY_SYNC,
GRAPH_API_HOST,
INSTAGRAM_ENDPOINTS,
MAX_INSIGHTS_LOOKBACK_DAYS,
@@ -746,7 +747,11 @@ def get_rows(
since: Optional[int] = None
if config.supports_time_window:
- if should_use_incremental_field and db_incremental_field_last_value is not None:
+ if (
+ should_use_incremental_field
+ and db_incremental_field_last_value is not None
+ and endpoint not in ENDPOINTS_REREAD_EVERY_SYNC
+ ):
since = _to_unix_seconds(db_incremental_field_last_value)
if since is None:
since = _to_unix_seconds(start_date)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py
index 9bdad01acf74..89c2a00dcb97 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py
@@ -15,7 +15,8 @@ GRAPH_API_HOST = "https://graph.facebook.com"
ACCOUNT_FIELDS = "id,username,name,biography,website,profile_picture_url,followers_count,follows_count,media_count"
MEDIA_FIELDS = (
"id,caption,media_type,media_product_type,media_url,permalink,thumbnail_url,"
- "timestamp,username,shortcode,like_count,comments_count,is_comment_enabled"
+ "timestamp,username,shortcode,like_count,comments_count,is_comment_enabled,is_shared_to_feed,owner,"
+ "boost_ads_list,boost_eligibility_info,copyright_check_information,legacy_instagram_media_id"
)
STORY_FIELDS = "id,caption,media_type,media_product_type,media_url,permalink,thumbnail_url,timestamp,username"
COMMENT_FIELDS = "id,text,timestamp,username,like_count,hidden"
@@ -161,6 +162,7 @@ INSTAGRAM_ENDPOINTS: dict[str, InstagramEndpointConfig] = {
}
ENDPOINTS = tuple(INSTAGRAM_ENDPOINTS.keys())
+ENDPOINTS_REREAD_EVERY_SYNC: frozenset[str] = frozenset({"media"})
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
name: config.incremental_fields for name, config in INSTAGRAM_ENDPOINTS.items()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py
index e216fd13f32c..4b04abd23287 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py
@@ -350,7 +350,7 @@ class TestInstagramEdges:
assert [row["id"] for row in result.rows] == ["1"]
assert len(result.requests) == 1
- def test_incremental_media_sync_asks_meta_for_the_window_after_the_watermark(self) -> None:
+ def test_an_incremental_media_sync_re_reads_every_post_instead_of_resuming_at_the_watermark(self) -> None:
result = _drive(
"media",
[ScriptedResponse(json={"data": []})],
@@ -359,7 +359,7 @@ class TestInstagramEdges:
)
assert result.raised is None
- assert result.params("since") == ["1704067200"]
+ assert result.params("since") == [None]
def test_a_full_refresh_media_sync_sends_no_time_window(self) -> None:
result = _drive(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram_source.py
index 4e5364798569..73b33afee9ab 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram_source.py
@@ -211,7 +211,7 @@ class TestInstagramSource:
assert result.response is not None and result.response.name == "media"
assert result.paths == [f"/{pinned_version}/{ACCOUNT_ID}/media"]
assert result.requests[0].headers["authorization"] == "Bearer refreshed-token"
- assert result.params("since") == ["1717200000"]
+ assert result.params("since") == ["1704067200"]
def test_a_full_refresh_run_never_forwards_a_watermark(self) -> None:
with mock.patch.object(InstagramSource, "_access_token", return_value="tok"):