:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(instagram): add missing media insight metrics and a breakdowns table (#118817)
posthog[bot]b8db5510ed5a
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 6679f0791744..24c431085000 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -6359,7 +6359,7 @@ Note: https://api.insightly.com/v3.1/swagger/docs/v3 (the URL linked from the He## Instagram — gaps-Today (6): `account`, `account_insights`, `media`, `media_comments`, `media_insights`, `stories`+Today (7): `account`, `account_insights`, `media`, `media_comments`, `media_insights`, `media_insights_breakdowns`, `stories`Diffed against: <https://raw.githubusercontent.com/facebook/facebook-python-business-sdk/main/facebook_business/adobjects/igmedia.py>diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.pyindex 60fc93718552..33d50f1bb474 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py@@ -85,13 +85,29 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"media_id": "Identifier of the media the metric belongs to.","media_timestamp": "ISO 8601 creation timestamp of the media, in UTC.","media_product_type": "Surface the media appears on: AD, FEED, STORY or REELS.",- "metric": "Metric name, for example reach, likes, saved, shares, total_interactions or views.",+ "metric": "Metric name, for example reach, likes, saved, shares, total_interactions, views, follows, profile_visits, ig_reels_avg_watch_time or ig_reels_video_view_total_time.","period": "Aggregation period Meta reported the metric over. Media metrics are lifetime totals.","title": "Meta's human-readable title for the metric.","description": "Meta's human-readable description of the metric.","value": "The metric's value.",},},+ "media_insights_breakdowns": {+ "description": "Lifetime post metrics split by a breakdown, in long format: one row per media, metric and breakdown value. Holds profile_activity by action_type for feed posts and stories, and navigation by story_navigation_action_type for stories.",+ "docs_url": _INSIGHTS_DOCS,+ "columns": {+ "media_id": "Identifier of the media the metric belongs to.",+ "media_timestamp": "ISO 8601 creation timestamp of the media, in UTC.",+ "media_product_type": "Surface the media appears on: FEED or STORY.",+ "metric": "Metric name: profile_activity or navigation.",+ "breakdown": "Breakdown the metric is split by: action_type or story_navigation_action_type.",+ "breakdown_value": "Breakdown value, for example bio_link_clicked, email or direction for action_type, or tap_forward, tap_back, tap_exit or swipe_forward for story_navigation_action_type.",+ "period": "Aggregation period Meta reported the metric over. Media metrics are lifetime totals.",+ "title": "Meta's human-readable title for the metric.",+ "description": "Meta's human-readable description of the metric.",+ "value": "The metric's value for that breakdown value.",+ },+ },"account_insights": {"description": "Daily account-level metrics, in long format: one row per metric and day. Meta only serves insights in windows of at most 30 days and retains roughly two years of history.","docs_url": _INSIGHTS_DOCS,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.pyindex a40815c2b77a..45268e5218cc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py@@ -25,6 +25,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.instagram.GRAPH_API_HOST,INSTAGRAM_ENDPOINTS,MAX_INSIGHTS_LOOKBACK_DAYS,+ MEDIA_INSIGHT_BREAKDOWNS,MEDIA_INSIGHT_METRICS,MEDIA_PARENT_FIELDS,PAGE_FIELDS,@@ -436,7 +437,9 @@ def _fanout_rows(# posts whose children were never fetched.return- if config.child_edge == "insights":+ if config.name == "media_insights_breakdowns":+ rows = _media_insight_breakdown_rows(client, media, logger)+ elif config.child_edge == "insights":rows = _media_insight_rows(client, media, logger)else:rows = _media_comment_rows(client, media, config, logger)@@ -546,6 +549,67 @@ def _media_insight_rows(return rows+def _media_insight_breakdown_rows(+ client: InstagramClient,+ media: dict[str, Any],+ logger: FilteringBoundLogger,+) -> list[dict[str, Any]]:+ media_id = str(media["id"])+ product_type = str(media.get("media_product_type") or "")+ media_timestamp = _normalize_timestamp(media.get("timestamp"))++ rows: list[dict[str, Any]] = []+ for metric, breakdown in MEDIA_INSIGHT_BREAKDOWNS.get(product_type, ()):+ if not client.has_request_budget():+ break++ url = client.build_url(f"{media_id}/insights", {"metric": metric, "breakdown": breakdown})+ try:+ body = client.get(url)+ except InstagramBadRequestError as error:+ # Meta refuses some breakdowns on old or deleted posts. Skip this pair only.+ logger.warning(f"Instagram: skipping {metric} by {breakdown} for media {media_id}: {error}")+ continue++ entries = body.get("data")+ for entry in entries if isinstance(entries, list) else []:+ if not isinstance(entry, dict):+ continue+ for breakdown_value, value in _breakdown_points(entry):+ rows.append(+ {+ "media_id": media_id,+ "media_timestamp": media_timestamp,+ "media_product_type": product_type or None,+ "metric": entry.get("name") or metric,+ "breakdown": breakdown,+ "breakdown_value": breakdown_value,+ "period": entry.get("period"),+ "title": entry.get("title"),+ "description": entry.get("description"),+ "value": value,+ }+ )++ return rows+++def _breakdown_points(entry: dict[str, Any]) -> list[tuple[str, Any]]:+ """Flatten `total_value.breakdowns[].results[]` into `(breakdown_value, value)` pairs."""+ total_value = entry.get("total_value")+ breakdowns = total_value.get("breakdowns") if isinstance(total_value, dict) else None++ points: list[tuple[str, Any]] = []+ for breakdown_set in breakdowns if isinstance(breakdowns, list) else []:+ results = breakdown_set.get("results") if isinstance(breakdown_set, dict) else None+ for result in results if isinstance(results, list) else []:+ dimension_values = result.get("dimension_values") if isinstance(result, dict) else None+ # Each request names one breakdown, so each result has one dimension value.+ if isinstance(dimension_values, list) and dimension_values:+ points.append((str(dimension_values[0]), result.get("value")))+ return points++def _insight_windows(since: int, until: int, window_days: int = ACCOUNT_INSIGHTS_WINDOW_DAYS) -> list[tuple[int, int]]:"""Split a range into windows Meta will accept (it rejects spans over 30 days)."""if until <= since:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.pyindex 851607637242..7dc80d798547 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py@@ -66,8 +66,9 @@ ACCOUNT_INSIGHT_TOTAL_VALUE_METRICS: tuple[str, ...] = ("follows_and_unfollows",)-# Per-media metrics, keyed by `media_product_type`. Stories expose a different set from-# feed posts and reels.+# Per-media metrics, keyed by `media_product_type`. Each list must match Meta's media+# insights table exactly: one metric that does not apply to the post fails the whole+# request, and the sync then skips every metric for that post.DEFAULT_MEDIA_INSIGHT_METRICS: tuple[str, ...] = ("reach","likes",@@ -78,10 +79,20 @@ DEFAULT_MEDIA_INSIGHT_METRICS: tuple[str, ...] = ("views",)MEDIA_INSIGHT_METRICS: dict[str, tuple[str, ...]] = {- "FEED": DEFAULT_MEDIA_INSIGHT_METRICS,- "REELS": DEFAULT_MEDIA_INSIGHT_METRICS,+ "FEED": (*DEFAULT_MEDIA_INSIGHT_METRICS, "follows", "profile_visits"),+ "REELS": (*DEFAULT_MEDIA_INSIGHT_METRICS, "ig_reels_avg_watch_time", "ig_reels_video_view_total_time"),"AD": DEFAULT_MEDIA_INSIGHT_METRICS,- "STORY": ("reach", "replies", "views"),+ "STORY": ("reach", "replies", "views", "follows", "profile_visits"),+}++# Per-media `(metric, breakdown)` pairs, keyed by `media_product_type`. Meta rejects a+# breakdown on a metric that does not support it, so each pair is a separate request.+MEDIA_INSIGHT_BREAKDOWNS: dict[str, tuple[tuple[str, str], ...]] = {+ "FEED": (("profile_activity", "action_type"),),+ "STORY": (+ ("profile_activity", "action_type"),+ ("navigation", "story_navigation_action_type"),+ ),}@@ -168,6 +179,18 @@ INSTAGRAM_ENDPOINTS: dict[str, InstagramEndpointConfig] = {sort_mode="desc",timestamp_fields=("media_timestamp",),),+ # One row per (media, metric, breakdown value). Only feed posts and stories have+ # breakdowns, so other media issue no request.+ "media_insights_breakdowns": InstagramEndpointConfig(+ name="media_insights_breakdowns",+ edge="media",+ child_edge="insights",+ primary_keys=["media_id", "metric", "breakdown", "breakdown_value"],+ incremental_fields=[],+ partition_key="media_timestamp",+ sort_mode="desc",+ timestamp_fields=("media_timestamp",),+ ),# Daily account metrics, also long format, walked in 30-day windows."account_insights": InstagramEndpointConfig(name="account_insights",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.pyindex b2ed3a438795..84a68fa38e3d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py@@ -526,6 +526,43 @@ class TestInstagramFanOut:assert result.raised is Noneassert result.rows == []+ def test_media_insight_breakdowns_request_each_pair_alone_and_emit_one_row_per_value(self) -> None:+ def breakdown_body(metric: str, key: str, values: dict[str, int]) -> dict[str, Any]:+ results = [{"dimension_values": [name], "value": value} for name, value in values.items()]+ breakdowns = [{"dimension_keys": [key], "results": results}]+ return {"data": [{"name": metric, "period": "lifetime", "total_value": {"breakdowns": breakdowns}}]}++ result = _drive(+ "media_insights_breakdowns",+ [+ ScriptedResponse(+ json=_page(+ [+ {"id": "r1", "media_product_type": "REELS"},+ {"id": "s1", "media_product_type": "STORY", "timestamp": "2024-03-01T18:10:00+0000"},+ ]+ )+ ),+ ScriptedResponse(status=400, json={"error": {"code": 100, "message": "unsupported breakdown"}}),+ ScriptedResponse(+ json=breakdown_body(+ "navigation", "story_navigation_action_type", {"tap_forward": 19, "tap_exit": 1}+ )+ ),+ ],+ )++ assert result.raised is None+ assert [(request.param("metric"), request.param("breakdown")) for request in result.requests[1:]] == [+ ("profile_activity", "action_type"),+ ("navigation", "story_navigation_action_type"),+ ]+ assert [(row["media_id"], row["metric"], row["breakdown_value"], row["value"]) for row in result.rows] == [+ ("s1", "navigation", "tap_forward", 19),+ ("s1", "navigation", "tap_exit", 1),+ ]+ assert result.rows[0]["media_timestamp"] == "2024-03-01T18:10:00+00:00"+def test_the_parent_page_is_checkpointed_once_its_children_are_drained(self) -> None:result = _drive("media_comments",