:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): re-read all instagram media and add media fields (#118815)
posthog[bot]a3fc003d6a4f
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.pyindex 85b738500b51..99eabf27ff43 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py@@ -24,7 +24,7 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {},},"media": {- "description": "Posts, reels and carousels published by the account, newest first. Limited to the 10,000 most recently created media.",+ "description": "Posts, reels and carousels published by the account, newest first. Limited to the 10,000 most recently created media. Every sync re-reads each post from the start date on, so like and comment counts stay current.","docs_url": _MEDIA_DOCS,"columns": {"id": "Media identifier.",@@ -40,6 +40,12 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"like_count": "Number of likes. Omitted if the owner has hidden like counts.","comments_count": "Number of comments on the media.","is_comment_enabled": "Whether comments are enabled on the media.",+ "is_shared_to_feed": "Whether the reel also appears in the Feed and Reels tabs. Reels only.",+ "owner": "The account that created the media, as an object with its ID.",+ "boost_ads_list": "Ads that boost this media, with each ad's ID and delivery status.",+ "boost_eligibility_info": "Whether the media can be boosted as an ad, and the reason if it cannot.",+ "copyright_check_information": "Status of Instagram's copyright check on the media, with any matches found.",+ "legacy_instagram_media_id": "Media ID that the Marketing API used in Graph API v21.0 and older.",},},"stories": {diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.pyindex 0c0ca24c8718..b70799042e39 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py@@ -20,6 +20,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.instagram.ACCOUNT_INSIGHTS_WINDOW_DAYS,DEFAULT_INSIGHTS_LOOKBACK_DAYS,DEFAULT_MEDIA_INSIGHT_METRICS,+ ENDPOINTS_REREAD_EVERY_SYNC,GRAPH_API_HOST,INSTAGRAM_ENDPOINTS,MAX_INSIGHTS_LOOKBACK_DAYS,@@ -746,7 +747,11 @@ def get_rows(since: Optional[int] = Noneif config.supports_time_window:- if should_use_incremental_field and db_incremental_field_last_value is not None:+ if (+ should_use_incremental_field+ and db_incremental_field_last_value is not None+ and endpoint not in ENDPOINTS_REREAD_EVERY_SYNC+ ):since = _to_unix_seconds(db_incremental_field_last_value)if since is None:since = _to_unix_seconds(start_date)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.pyindex 9bdad01acf74..89c2a00dcb97 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py@@ -15,7 +15,8 @@ GRAPH_API_HOST = "https://graph.facebook.com"ACCOUNT_FIELDS = "id,username,name,biography,website,profile_picture_url,followers_count,follows_count,media_count"MEDIA_FIELDS = ("id,caption,media_type,media_product_type,media_url,permalink,thumbnail_url,"- "timestamp,username,shortcode,like_count,comments_count,is_comment_enabled"+ "timestamp,username,shortcode,like_count,comments_count,is_comment_enabled,is_shared_to_feed,owner,"+ "boost_ads_list,boost_eligibility_info,copyright_check_information,legacy_instagram_media_id")STORY_FIELDS = "id,caption,media_type,media_product_type,media_url,permalink,thumbnail_url,timestamp,username"COMMENT_FIELDS = "id,text,timestamp,username,like_count,hidden"@@ -161,6 +162,7 @@ INSTAGRAM_ENDPOINTS: dict[str, InstagramEndpointConfig] = {}ENDPOINTS = tuple(INSTAGRAM_ENDPOINTS.keys())+ENDPOINTS_REREAD_EVERY_SYNC: frozenset[str] = frozenset({"media"})INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {name: config.incremental_fields for name, config in INSTAGRAM_ENDPOINTS.items()diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.pyindex e216fd13f32c..4b04abd23287 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py@@ -350,7 +350,7 @@ class TestInstagramEdges:assert [row["id"] for row in result.rows] == ["1"]assert len(result.requests) == 1- def test_incremental_media_sync_asks_meta_for_the_window_after_the_watermark(self) -> None:+ def test_an_incremental_media_sync_re_reads_every_post_instead_of_resuming_at_the_watermark(self) -> None:result = _drive("media",[ScriptedResponse(json={"data": []})],@@ -359,7 +359,7 @@ class TestInstagramEdges:)assert result.raised is None- assert result.params("since") == ["1704067200"]+ assert result.params("since") == [None]def test_a_full_refresh_media_sync_sends_no_time_window(self) -> None:result = _drive(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram_source.pyindex 4e5364798569..73b33afee9ab 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram_source.py@@ -211,7 +211,7 @@ class TestInstagramSource:assert result.response is not None and result.response.name == "media"assert result.paths == [f"/{pinned_version}/{ACCOUNT_ID}/media"]assert result.requests[0].headers["authorization"] == "Bearer refreshed-token"- assert result.params("since") == ["1717200000"]+ assert result.params("since") == ["1704067200"]def test_a_full_refresh_run_never_forwards_a_watermark(self) -> None:with mock.patch.object(InstagramSource, "_access_token", return_value="tok"):