Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(instagram): add missing media insight metrics and a breakdowns table (#118817)

posthog[bot]b8db5510ed5a
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 6679f0791744..24c431085000 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -6359,7 +6359,7 @@ Note: https://api.insightly.com/v3.1/swagger/docs/v3 (the URL linked from the He
## Instagram — gaps
-Today (6): `account`, `account_insights`, `media`, `media_comments`, `media_insights`, `stories`
+Today (7): `account`, `account_insights`, `media`, `media_comments`, `media_insights`, `media_insights_breakdowns`, `stories`
Diffed against: <https://raw.githubusercontent.com/facebook/facebook-python-business-sdk/main/facebook_business/adobjects/igmedia.py>
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py
index 60fc93718552..33d50f1bb474 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py
@@ -85,13 +85,29 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"media_id": "Identifier of the media the metric belongs to.",
"media_timestamp": "ISO 8601 creation timestamp of the media, in UTC.",
"media_product_type": "Surface the media appears on: AD, FEED, STORY or REELS.",
- "metric": "Metric name, for example reach, likes, saved, shares, total_interactions or views.",
+ "metric": "Metric name, for example reach, likes, saved, shares, total_interactions, views, follows, profile_visits, ig_reels_avg_watch_time or ig_reels_video_view_total_time.",
"period": "Aggregation period Meta reported the metric over. Media metrics are lifetime totals.",
"title": "Meta's human-readable title for the metric.",
"description": "Meta's human-readable description of the metric.",
"value": "The metric's value.",
},
},
+ "media_insights_breakdowns": {
+ "description": "Lifetime post metrics split by a breakdown, in long format: one row per media, metric and breakdown value. Holds profile_activity by action_type for feed posts and stories, and navigation by story_navigation_action_type for stories.",
+ "docs_url": _INSIGHTS_DOCS,
+ "columns": {
+ "media_id": "Identifier of the media the metric belongs to.",
+ "media_timestamp": "ISO 8601 creation timestamp of the media, in UTC.",
+ "media_product_type": "Surface the media appears on: FEED or STORY.",
+ "metric": "Metric name: profile_activity or navigation.",
+ "breakdown": "Breakdown the metric is split by: action_type or story_navigation_action_type.",
+ "breakdown_value": "Breakdown value, for example bio_link_clicked, email or direction for action_type, or tap_forward, tap_back, tap_exit or swipe_forward for story_navigation_action_type.",
+ "period": "Aggregation period Meta reported the metric over. Media metrics are lifetime totals.",
+ "title": "Meta's human-readable title for the metric.",
+ "description": "Meta's human-readable description of the metric.",
+ "value": "The metric's value for that breakdown value.",
+ },
+ },
"account_insights": {
"description": "Daily account-level metrics, in long format: one row per metric and day. Meta only serves insights in windows of at most 30 days and retains roughly two years of history.",
"docs_url": _INSIGHTS_DOCS,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py
index a40815c2b77a..45268e5218cc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py
@@ -25,6 +25,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.instagram.
GRAPH_API_HOST,
INSTAGRAM_ENDPOINTS,
MAX_INSIGHTS_LOOKBACK_DAYS,
+ MEDIA_INSIGHT_BREAKDOWNS,
MEDIA_INSIGHT_METRICS,
MEDIA_PARENT_FIELDS,
PAGE_FIELDS,
@@ -436,7 +437,9 @@ def _fanout_rows(
# posts whose children were never fetched.
return
- if config.child_edge == "insights":
+ if config.name == "media_insights_breakdowns":
+ rows = _media_insight_breakdown_rows(client, media, logger)
+ elif config.child_edge == "insights":
rows = _media_insight_rows(client, media, logger)
else:
rows = _media_comment_rows(client, media, config, logger)
@@ -546,6 +549,67 @@ def _media_insight_rows(
return rows
+def _media_insight_breakdown_rows(
+ client: InstagramClient,
+ media: dict[str, Any],
+ logger: FilteringBoundLogger,
+) -> list[dict[str, Any]]:
+ media_id = str(media["id"])
+ product_type = str(media.get("media_product_type") or "")
+ media_timestamp = _normalize_timestamp(media.get("timestamp"))
+
+ rows: list[dict[str, Any]] = []
+ for metric, breakdown in MEDIA_INSIGHT_BREAKDOWNS.get(product_type, ()):
+ if not client.has_request_budget():
+ break
+
+ url = client.build_url(f"{media_id}/insights", {"metric": metric, "breakdown": breakdown})
+ try:
+ body = client.get(url)
+ except InstagramBadRequestError as error:
+ # Meta refuses some breakdowns on old or deleted posts. Skip this pair only.
+ logger.warning(f"Instagram: skipping {metric} by {breakdown} for media {media_id}: {error}")
+ continue
+
+ entries = body.get("data")
+ for entry in entries if isinstance(entries, list) else []:
+ if not isinstance(entry, dict):
+ continue
+ for breakdown_value, value in _breakdown_points(entry):
+ rows.append(
+ {
+ "media_id": media_id,
+ "media_timestamp": media_timestamp,
+ "media_product_type": product_type or None,
+ "metric": entry.get("name") or metric,
+ "breakdown": breakdown,
+ "breakdown_value": breakdown_value,
+ "period": entry.get("period"),
+ "title": entry.get("title"),
+ "description": entry.get("description"),
+ "value": value,
+ }
+ )
+
+ return rows
+
+
+def _breakdown_points(entry: dict[str, Any]) -> list[tuple[str, Any]]:
+ """Flatten `total_value.breakdowns[].results[]` into `(breakdown_value, value)` pairs."""
+ total_value = entry.get("total_value")
+ breakdowns = total_value.get("breakdowns") if isinstance(total_value, dict) else None
+
+ points: list[tuple[str, Any]] = []
+ for breakdown_set in breakdowns if isinstance(breakdowns, list) else []:
+ results = breakdown_set.get("results") if isinstance(breakdown_set, dict) else None
+ for result in results if isinstance(results, list) else []:
+ dimension_values = result.get("dimension_values") if isinstance(result, dict) else None
+ # Each request names one breakdown, so each result has one dimension value.
+ if isinstance(dimension_values, list) and dimension_values:
+ points.append((str(dimension_values[0]), result.get("value")))
+ return points
+
+
def _insight_windows(since: int, until: int, window_days: int = ACCOUNT_INSIGHTS_WINDOW_DAYS) -> list[tuple[int, int]]:
"""Split a range into windows Meta will accept (it rejects spans over 30 days)."""
if until <= since:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py
index 851607637242..7dc80d798547 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py
@@ -66,8 +66,9 @@ ACCOUNT_INSIGHT_TOTAL_VALUE_METRICS: tuple[str, ...] = (
"follows_and_unfollows",
)
-# Per-media metrics, keyed by `media_product_type`. Stories expose a different set from
-# feed posts and reels.
+# Per-media metrics, keyed by `media_product_type`. Each list must match Meta's media
+# insights table exactly: one metric that does not apply to the post fails the whole
+# request, and the sync then skips every metric for that post.
DEFAULT_MEDIA_INSIGHT_METRICS: tuple[str, ...] = (
"reach",
"likes",
@@ -78,10 +79,20 @@ DEFAULT_MEDIA_INSIGHT_METRICS: tuple[str, ...] = (
"views",
)
MEDIA_INSIGHT_METRICS: dict[str, tuple[str, ...]] = {
- "FEED": DEFAULT_MEDIA_INSIGHT_METRICS,
- "REELS": DEFAULT_MEDIA_INSIGHT_METRICS,
+ "FEED": (*DEFAULT_MEDIA_INSIGHT_METRICS, "follows", "profile_visits"),
+ "REELS": (*DEFAULT_MEDIA_INSIGHT_METRICS, "ig_reels_avg_watch_time", "ig_reels_video_view_total_time"),
"AD": DEFAULT_MEDIA_INSIGHT_METRICS,
- "STORY": ("reach", "replies", "views"),
+ "STORY": ("reach", "replies", "views", "follows", "profile_visits"),
+}
+
+# Per-media `(metric, breakdown)` pairs, keyed by `media_product_type`. Meta rejects a
+# breakdown on a metric that does not support it, so each pair is a separate request.
+MEDIA_INSIGHT_BREAKDOWNS: dict[str, tuple[tuple[str, str], ...]] = {
+ "FEED": (("profile_activity", "action_type"),),
+ "STORY": (
+ ("profile_activity", "action_type"),
+ ("navigation", "story_navigation_action_type"),
+ ),
}
@@ -168,6 +179,18 @@ INSTAGRAM_ENDPOINTS: dict[str, InstagramEndpointConfig] = {
sort_mode="desc",
timestamp_fields=("media_timestamp",),
),
+ # One row per (media, metric, breakdown value). Only feed posts and stories have
+ # breakdowns, so other media issue no request.
+ "media_insights_breakdowns": InstagramEndpointConfig(
+ name="media_insights_breakdowns",
+ edge="media",
+ child_edge="insights",
+ primary_keys=["media_id", "metric", "breakdown", "breakdown_value"],
+ incremental_fields=[],
+ partition_key="media_timestamp",
+ sort_mode="desc",
+ timestamp_fields=("media_timestamp",),
+ ),
# Daily account metrics, also long format, walked in 30-day windows.
"account_insights": InstagramEndpointConfig(
name="account_insights",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py
index b2ed3a438795..84a68fa38e3d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py
@@ -526,6 +526,43 @@ class TestInstagramFanOut:
assert result.raised is None
assert result.rows == []
+ def test_media_insight_breakdowns_request_each_pair_alone_and_emit_one_row_per_value(self) -> None:
+ def breakdown_body(metric: str, key: str, values: dict[str, int]) -> dict[str, Any]:
+ results = [{"dimension_values": [name], "value": value} for name, value in values.items()]
+ breakdowns = [{"dimension_keys": [key], "results": results}]
+ return {"data": [{"name": metric, "period": "lifetime", "total_value": {"breakdowns": breakdowns}}]}
+
+ result = _drive(
+ "media_insights_breakdowns",
+ [
+ ScriptedResponse(
+ json=_page(
+ [
+ {"id": "r1", "media_product_type": "REELS"},
+ {"id": "s1", "media_product_type": "STORY", "timestamp": "2024-03-01T18:10:00+0000"},
+ ]
+ )
+ ),
+ ScriptedResponse(status=400, json={"error": {"code": 100, "message": "unsupported breakdown"}}),
+ ScriptedResponse(
+ json=breakdown_body(
+ "navigation", "story_navigation_action_type", {"tap_forward": 19, "tap_exit": 1}
+ )
+ ),
+ ],
+ )
+
+ assert result.raised is None
+ assert [(request.param("metric"), request.param("breakdown")) for request in result.requests[1:]] == [
+ ("profile_activity", "action_type"),
+ ("navigation", "story_navigation_action_type"),
+ ]
+ assert [(row["media_id"], row["metric"], row["breakdown_value"], row["value"]) for row in result.rows] == [
+ ("s1", "navigation", "tap_forward", 19),
+ ("s1", "navigation", "tap_exit", 1),
+ ]
+ assert result.rows[0]["media_timestamp"] == "2024-03-01T18:10:00+00:00"
+
def test_the_parent_page_is_checkpointed_once_its_children_are_drained(self) -> None:
result = _drive(
"media_comments",