:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(instagram): add account_insights_breakdowns table (#118820)
posthog[bot]546e567b4243
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.pyindex 33d50f1bb474..604936966d65 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py@@ -121,4 +121,19 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"description": "Meta's human-readable description of the metric.",},},+ "account_insights_breakdowns": {+ "description": "Daily account-level metrics split by a breakdown dimension, in long format: one row per metric, breakdown value and day. Covers follows_and_unfollows by follow_type, likes, comments, shares, saves and total_interactions by media_product_type, and profile_links_taps by contact_button_type.",+ "docs_url": _INSIGHTS_DOCS,+ "columns": {+ "instagram_account_id": "Instagram account the metric belongs to.",+ "metric": "Metric name, for example follows_and_unfollows, likes or profile_links_taps.",+ "breakdown": "Dimension the metric is split by: follow_type, media_product_type or contact_button_type.",+ "breakdown_value": "Value of the breakdown dimension, for example FOLLOWER, REEL or DIRECTION.",+ "period": "Aggregation period. Always day for this table.",+ "date": "ISO 8601 end time of the day the metric covers, in UTC.",+ "value": "The metric's value for that breakdown value and day.",+ "title": "Meta's human-readable title for the metric.",+ "description": "Meta's human-readable description of the metric.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.pyindex 45268e5218cc..550a13d870e2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py@@ -16,8 +16,6 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.httfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.instagram.settings import (- ACCOUNT_INSIGHT_TIME_SERIES_METRICS,- ACCOUNT_INSIGHT_TOTAL_VALUE_METRICS,ACCOUNT_INSIGHTS_WINDOW_DAYS,DEFAULT_INSIGHTS_LOOKBACK_DAYS,DEFAULT_MEDIA_INSIGHT_METRICS,@@ -117,6 +115,8 @@ class _InsightRequest:# End of the one-day span for a `total_value` request, which carries no `end_time` of# its own. None for a time series, whose points each name their day.day_end: Optional[int] = None+ # Dimension Meta splits the `total_value` by. None for an unsplit metric.+ breakdown: Optional[str] = Nonedef _is_permission_code(code: Any) -> bool:@@ -628,6 +628,7 @@ def _insight_windows(since: int, until: int, window_days: int = ACCOUNT_INSIGHTSdef _account_insight_plan(client: InstagramClient,instagram_account_id: str,+ config: InstagramEndpointConfig,since: int,until: int,) -> list[list[_InsightRequest]]:@@ -659,7 +660,7 @@ def _account_insight_plan(},),)- for metric in ACCOUNT_INSIGHT_TIME_SERIES_METRICS+ for metric in config.insight_time_series_metrics]day_start = window_sincewhile day_start + DAY_SECONDS <= window_until:@@ -673,19 +674,67 @@ def _account_insight_plan("metric": metric,"period": "day","metric_type": "total_value",+ **({"breakdown": breakdown} if breakdown else {}),"since": str(day_start),"until": str(day_end),},),day_end=day_end,+ breakdown=breakdown,)- for metric in ACCOUNT_INSIGHT_TOTAL_VALUE_METRICS+ for metric, breakdown in config.insight_total_value_metrics)day_start = day_end- plan.append(window)+ # A trailing window shorter than a day has no requests when there is no time series.+ if window:+ plan.append(window)return plan+def _insight_row(+ instagram_account_id: str, request: _InsightRequest, entry: dict[str, Any], day: str, value: Any, **extra: Any+) -> dict[str, Any]:+ return {+ "instagram_account_id": instagram_account_id,+ "metric": entry.get("name") or request.metric,+ **extra,+ "period": entry.get("period"),+ "date": day,+ "value": value,+ "title": entry.get("title"),+ "description": entry.get("description"),+ }+++def _breakdown_rows(instagram_account_id: str, request: _InsightRequest, entry: dict[str, Any]) -> list[dict[str, Any]]:+ """One row per breakdown value in a `total_value` entry requested with a `breakdown`."""+ total_value = entry.get("total_value")+ if not isinstance(total_value, dict) or request.day_end is None:+ return []+ breakdowns = total_value.get("breakdowns")+ day = datetime.fromtimestamp(request.day_end, UTC).isoformat()++ rows: list[dict[str, Any]] = []+ for breakdown in breakdowns if isinstance(breakdowns, list) else []:+ results = breakdown.get("results") if isinstance(breakdown, dict) else None+ for result in results if isinstance(results, list) else []:+ dimension_values = result.get("dimension_values") if isinstance(result, dict) else None+ if not isinstance(dimension_values, list) or not dimension_values:+ continue+ rows.append(+ _insight_row(+ instagram_account_id,+ request,+ entry,+ day,+ result.get("value"),+ breakdown=request.breakdown,+ breakdown_value=",".join(str(value) for value in dimension_values),+ )+ )+ return rows++def _account_insight_rows(client: InstagramClient,instagram_account_id: str,@@ -709,7 +758,7 @@ def _account_insight_rows(# this table does not have yet.start = start - start % DAY_SECONDS - DAY_SECONDS- plan = _account_insight_plan(client, instagram_account_id, start, until)+ plan = _account_insight_plan(client, instagram_account_id, config, start, until)resume_url = _resume_url(resumable_source_manager, logger, config.name)if resume_url is not None:@@ -718,7 +767,7 @@ def _account_insight_rows(if resume_url in heads:plan = plan[heads.index(resume_url) :]else:- logger.warning("Instagram: saved account_insights checkpoint is outside this run's window, restarting")+ logger.warning(f"Instagram: saved {config.name} checkpoint is outside this run's window, restarting")# A metric Meta rejects once is rejected for every day, so stop asking for it.rejected_metrics: set[str] = set()@@ -747,6 +796,9 @@ def _account_insight_rows(entries = entries if isinstance(entries, list) else []for entry in entries:+ if request.breakdown is not None:+ rows.extend(_breakdown_rows(instagram_account_id, request, entry))+ continuefor point in _insight_points(entry):end_time = point.get("end_time")if end_time is not None:@@ -755,17 +807,7 @@ def _account_insight_rows(day = datetime.fromtimestamp(request.day_end, UTC).isoformat()else:continue- rows.append(- {- "instagram_account_id": instagram_account_id,- "metric": entry.get("name") or request.metric,- "period": entry.get("period"),- "date": day,- "value": point.get("value"),- "title": entry.get("title"),- "description": entry.get("description"),- }- )+ rows.append(_insight_row(instagram_account_id, request, entry, day, point.get("value")))if rows:# Sorted so the batch is genuinely ascending by date, which is what@@ -905,7 +947,7 @@ def get_rows(if endpoint == "account":yield from _account_row(client, instagram_account_id, config)- elif endpoint == "account_insights":+ elif config.edge == "insights":yield from _account_insight_rows(client=client,instagram_account_id=instagram_account_id,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.pyindex 7dc80d798547..2abb8b34e882 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py@@ -1,6 +1,8 @@-from dataclasses import dataclass, field+from dataclasses import fieldfrom typing import Optional+from posthog.dataclasses import frozen+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_fieldfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SortModefrom products.warehouse_sources.backend.types import IncrementalField@@ -65,6 +67,17 @@ ACCOUNT_INSIGHT_TOTAL_VALUE_METRICS: tuple[str, ...] = ("profile_links_taps","follows_and_unfollows",)+# `(metric, breakdown)` pairs for `account_insights_breakdowns`. Each pair is requested+# with `metric_type=total_value` once per day, like the metrics above.+ACCOUNT_INSIGHT_BREAKDOWNS: tuple[tuple[str, str], ...] = (+ ("follows_and_unfollows", "follow_type"),+ ("likes", "media_product_type"),+ ("comments", "media_product_type"),+ ("shares", "media_product_type"),+ ("saves", "media_product_type"),+ ("total_interactions", "media_product_type"),+ ("profile_links_taps", "contact_button_type"),+)# Per-media metrics, keyed by `media_product_type`. Each list must match Meta's media# insights table exactly: one metric that does not apply to the post fails the whole@@ -96,7 +109,7 @@ MEDIA_INSIGHT_BREAKDOWNS: dict[str, tuple[tuple[str, str], ...]] = {}-@dataclass+@frozenclass InstagramEndpointConfig:name: strprimary_keys: list[str]@@ -116,6 +129,10 @@ class InstagramEndpointConfig:supports_time_window: bool = False# Columns carrying Meta's `+0000` offset, normalized to ISO 8601 on the way out.timestamp_fields: tuple[str, ...] = field(default_factory=tuple)+ # Account insight requests for each window: daily time series metrics, then+ # `(metric, breakdown)` pairs asked for as `total_value` once per day.+ insight_time_series_metrics: tuple[str, ...] = field(default_factory=tuple)+ insight_total_value_metrics: tuple[tuple[str, Optional[str]], ...] = field(default_factory=tuple)INSTAGRAM_ENDPOINTS: dict[str, InstagramEndpointConfig] = {@@ -201,6 +218,21 @@ INSTAGRAM_ENDPOINTS: dict[str, InstagramEndpointConfig] = {sort_mode="asc",supports_time_window=True,timestamp_fields=("date",),+ insight_time_series_metrics=ACCOUNT_INSIGHT_TIME_SERIES_METRICS,+ insight_total_value_metrics=tuple((metric, None) for metric in ACCOUNT_INSIGHT_TOTAL_VALUE_METRICS),+ ),+ # Daily account metrics split by a breakdown dimension. A separate table, so the key+ # of `account_insights` stays the same for existing syncs.+ "account_insights_breakdowns": InstagramEndpointConfig(+ name="account_insights_breakdowns",+ edge="insights",+ primary_keys=["instagram_account_id", "metric", "breakdown", "breakdown_value", "date"],+ incremental_fields=[incremental_field("date")],+ partition_key="date",+ sort_mode="asc",+ supports_time_window=True,+ timestamp_fields=("date",),+ insight_total_value_metrics=ACCOUNT_INSIGHT_BREAKDOWNS,),}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.pyindex 84a68fa38e3d..34235a43e537 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py@@ -1,4 +1,5 @@from collections.abc import Iterable+from dataclasses import replacefrom datetime import UTC, datetime, timedeltafrom typing import Any, Optional, castfrom urllib.parse import parse_qs, urlparse@@ -660,7 +661,7 @@ class TestInstagramAccountInsights:with scripted_network([]):client = InstagramClient("tok", "v23.0", LOGGER)- plan = _account_insight_plan(client, ACCOUNT_ID, 0, 45 * 86400 + 3600)+ plan = _account_insight_plan(client, ACCOUNT_ID, INSTAGRAM_ENDPOINTS["account_insights"], 0, 45 * 86400 + 3600)# 45 days is two windows. Each opens with the time series over the whole window,# then asks for every total_value metric once per complete day. The partial@@ -770,6 +771,48 @@ class TestInstagramAccountInsights:# only spends the account's quota.assert [request.param("metric") for request in result.requests].count("follows_and_unfollows") == 1+ def test_breakdown_results_become_one_row_per_breakdown_value_and_day(self) -> None:+ follows = ScriptedResponse(+ json={+ "data": [+ {+ "name": "follows_and_unfollows",+ "period": "day",+ "total_value": {+ "value": 7,+ "breakdowns": [+ {+ "dimension_keys": ["follow_type"],+ "results": [+ {"dimension_values": ["FOLLOWER"], "value": 5},+ {"dimension_values": ["NON_FOLLOWER"], "value": 2},+ ],+ }+ ],+ },+ }+ ]+ }+ )+ # This start date leaves a trailing window shorter than a day, which has no requests.+ result = _drive(+ "account_insights_breakdowns",+ _by_metric({"follows_and_unfollows": follows}),+ start_date=(datetime.now(UTC) - timedelta(days=29)).date().isoformat(),+ )++ assert result.raised is None+ assert all(request.param("breakdown") for request in result.requests)+ today = datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0)+ assert [+ (row["metric"], row["breakdown"], row["breakdown_value"], row["value"])+ for row in result.rows+ if row["date"] == today.isoformat()+ ] == [+ ("follows_and_unfollows", "follow_type", "FOLLOWER", 5),+ ("follows_and_unfollows", "follow_type", "NON_FOLLOWER", 2),+ ]+def test_each_finished_window_checkpoints_the_start_of_the_next(self) -> None:start_date = (datetime.now(UTC) - timedelta(days=75)).date().isoformat()result = _drive("account_insights", always(ScriptedResponse(json={"data": []})), start_date=start_date)@@ -799,7 +842,8 @@ class TestInstagramAccountInsights:# Without the clamp a start date like this fans out into tens of thousands of# empty 30-day windows; Meta only retains ~2 years anyway.# Only the windows matter here, so drop the per-day requests that would fill them.- with mock.patch(f"{MODULE}.ACCOUNT_INSIGHT_TOTAL_VALUE_METRICS", ()):+ without_daily_requests = replace(INSTAGRAM_ENDPOINTS["account_insights"], insight_total_value_metrics=())+ with mock.patch.dict(INSTAGRAM_ENDPOINTS, {"account_insights": without_daily_requests}):result = _drive("account_insights", always(ScriptedResponse(json={"data": []})), start_date="0001-01-01")assert result.raised is None