Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(instagram): add account_insights_breakdowns table (#118820)

posthog[bot]546e567b4243
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py
index 33d50f1bb474..604936966d65 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py
@@ -121,4 +121,19 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"description": "Meta's human-readable description of the metric.",
},
},
+ "account_insights_breakdowns": {
+ "description": "Daily account-level metrics split by a breakdown dimension, in long format: one row per metric, breakdown value and day. Covers follows_and_unfollows by follow_type, likes, comments, shares, saves and total_interactions by media_product_type, and profile_links_taps by contact_button_type.",
+ "docs_url": _INSIGHTS_DOCS,
+ "columns": {
+ "instagram_account_id": "Instagram account the metric belongs to.",
+ "metric": "Metric name, for example follows_and_unfollows, likes or profile_links_taps.",
+ "breakdown": "Dimension the metric is split by: follow_type, media_product_type or contact_button_type.",
+ "breakdown_value": "Value of the breakdown dimension, for example FOLLOWER, REEL or DIRECTION.",
+ "period": "Aggregation period. Always day for this table.",
+ "date": "ISO 8601 end time of the day the metric covers, in UTC.",
+ "value": "The metric's value for that breakdown value and day.",
+ "title": "Meta's human-readable title for the metric.",
+ "description": "Meta's human-readable description of the metric.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py
index 45268e5218cc..550a13d870e2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py
@@ -16,8 +16,6 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.htt
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.instagram.settings import (
- ACCOUNT_INSIGHT_TIME_SERIES_METRICS,
- ACCOUNT_INSIGHT_TOTAL_VALUE_METRICS,
ACCOUNT_INSIGHTS_WINDOW_DAYS,
DEFAULT_INSIGHTS_LOOKBACK_DAYS,
DEFAULT_MEDIA_INSIGHT_METRICS,
@@ -117,6 +115,8 @@ class _InsightRequest:
# End of the one-day span for a `total_value` request, which carries no `end_time` of
# its own. None for a time series, whose points each name their day.
day_end: Optional[int] = None
+ # Dimension Meta splits the `total_value` by. None for an unsplit metric.
+ breakdown: Optional[str] = None
def _is_permission_code(code: Any) -> bool:
@@ -628,6 +628,7 @@ def _insight_windows(since: int, until: int, window_days: int = ACCOUNT_INSIGHTS
def _account_insight_plan(
client: InstagramClient,
instagram_account_id: str,
+ config: InstagramEndpointConfig,
since: int,
until: int,
) -> list[list[_InsightRequest]]:
@@ -659,7 +660,7 @@ def _account_insight_plan(
},
),
)
- for metric in ACCOUNT_INSIGHT_TIME_SERIES_METRICS
+ for metric in config.insight_time_series_metrics
]
day_start = window_since
while day_start + DAY_SECONDS <= window_until:
@@ -673,19 +674,67 @@ def _account_insight_plan(
"metric": metric,
"period": "day",
"metric_type": "total_value",
+ **({"breakdown": breakdown} if breakdown else {}),
"since": str(day_start),
"until": str(day_end),
},
),
day_end=day_end,
+ breakdown=breakdown,
)
- for metric in ACCOUNT_INSIGHT_TOTAL_VALUE_METRICS
+ for metric, breakdown in config.insight_total_value_metrics
)
day_start = day_end
- plan.append(window)
+ # A trailing window shorter than a day has no requests when there is no time series.
+ if window:
+ plan.append(window)
return plan
+def _insight_row(
+ instagram_account_id: str, request: _InsightRequest, entry: dict[str, Any], day: str, value: Any, **extra: Any
+) -> dict[str, Any]:
+ return {
+ "instagram_account_id": instagram_account_id,
+ "metric": entry.get("name") or request.metric,
+ **extra,
+ "period": entry.get("period"),
+ "date": day,
+ "value": value,
+ "title": entry.get("title"),
+ "description": entry.get("description"),
+ }
+
+
+def _breakdown_rows(instagram_account_id: str, request: _InsightRequest, entry: dict[str, Any]) -> list[dict[str, Any]]:
+ """One row per breakdown value in a `total_value` entry requested with a `breakdown`."""
+ total_value = entry.get("total_value")
+ if not isinstance(total_value, dict) or request.day_end is None:
+ return []
+ breakdowns = total_value.get("breakdowns")
+ day = datetime.fromtimestamp(request.day_end, UTC).isoformat()
+
+ rows: list[dict[str, Any]] = []
+ for breakdown in breakdowns if isinstance(breakdowns, list) else []:
+ results = breakdown.get("results") if isinstance(breakdown, dict) else None
+ for result in results if isinstance(results, list) else []:
+ dimension_values = result.get("dimension_values") if isinstance(result, dict) else None
+ if not isinstance(dimension_values, list) or not dimension_values:
+ continue
+ rows.append(
+ _insight_row(
+ instagram_account_id,
+ request,
+ entry,
+ day,
+ result.get("value"),
+ breakdown=request.breakdown,
+ breakdown_value=",".join(str(value) for value in dimension_values),
+ )
+ )
+ return rows
+
+
def _account_insight_rows(
client: InstagramClient,
instagram_account_id: str,
@@ -709,7 +758,7 @@ def _account_insight_rows(
# this table does not have yet.
start = start - start % DAY_SECONDS - DAY_SECONDS
- plan = _account_insight_plan(client, instagram_account_id, start, until)
+ plan = _account_insight_plan(client, instagram_account_id, config, start, until)
resume_url = _resume_url(resumable_source_manager, logger, config.name)
if resume_url is not None:
@@ -718,7 +767,7 @@ def _account_insight_rows(
if resume_url in heads:
plan = plan[heads.index(resume_url) :]
else:
- logger.warning("Instagram: saved account_insights checkpoint is outside this run's window, restarting")
+ logger.warning(f"Instagram: saved {config.name} checkpoint is outside this run's window, restarting")
# A metric Meta rejects once is rejected for every day, so stop asking for it.
rejected_metrics: set[str] = set()
@@ -747,6 +796,9 @@ def _account_insight_rows(
entries = entries if isinstance(entries, list) else []
for entry in entries:
+ if request.breakdown is not None:
+ rows.extend(_breakdown_rows(instagram_account_id, request, entry))
+ continue
for point in _insight_points(entry):
end_time = point.get("end_time")
if end_time is not None:
@@ -755,17 +807,7 @@ def _account_insight_rows(
day = datetime.fromtimestamp(request.day_end, UTC).isoformat()
else:
continue
- rows.append(
- {
- "instagram_account_id": instagram_account_id,
- "metric": entry.get("name") or request.metric,
- "period": entry.get("period"),
- "date": day,
- "value": point.get("value"),
- "title": entry.get("title"),
- "description": entry.get("description"),
- }
- )
+ rows.append(_insight_row(instagram_account_id, request, entry, day, point.get("value")))
if rows:
# Sorted so the batch is genuinely ascending by date, which is what
@@ -905,7 +947,7 @@ def get_rows(
if endpoint == "account":
yield from _account_row(client, instagram_account_id, config)
- elif endpoint == "account_insights":
+ elif config.edge == "insights":
yield from _account_insight_rows(
client=client,
instagram_account_id=instagram_account_id,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py
index 7dc80d798547..2abb8b34e882 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py
@@ -1,6 +1,8 @@
-from dataclasses import dataclass, field
+from dataclasses import field
from typing import Optional
+from posthog.dataclasses import frozen
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_field
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SortMode
from products.warehouse_sources.backend.types import IncrementalField
@@ -65,6 +67,17 @@ ACCOUNT_INSIGHT_TOTAL_VALUE_METRICS: tuple[str, ...] = (
"profile_links_taps",
"follows_and_unfollows",
)
+# `(metric, breakdown)` pairs for `account_insights_breakdowns`. Each pair is requested
+# with `metric_type=total_value` once per day, like the metrics above.
+ACCOUNT_INSIGHT_BREAKDOWNS: tuple[tuple[str, str], ...] = (
+ ("follows_and_unfollows", "follow_type"),
+ ("likes", "media_product_type"),
+ ("comments", "media_product_type"),
+ ("shares", "media_product_type"),
+ ("saves", "media_product_type"),
+ ("total_interactions", "media_product_type"),
+ ("profile_links_taps", "contact_button_type"),
+)
# Per-media metrics, keyed by `media_product_type`. Each list must match Meta's media
# insights table exactly: one metric that does not apply to the post fails the whole
@@ -96,7 +109,7 @@ MEDIA_INSIGHT_BREAKDOWNS: dict[str, tuple[tuple[str, str], ...]] = {
}
-@dataclass
+@frozen
class InstagramEndpointConfig:
name: str
primary_keys: list[str]
@@ -116,6 +129,10 @@ class InstagramEndpointConfig:
supports_time_window: bool = False
# Columns carrying Meta's `+0000` offset, normalized to ISO 8601 on the way out.
timestamp_fields: tuple[str, ...] = field(default_factory=tuple)
+ # Account insight requests for each window: daily time series metrics, then
+ # `(metric, breakdown)` pairs asked for as `total_value` once per day.
+ insight_time_series_metrics: tuple[str, ...] = field(default_factory=tuple)
+ insight_total_value_metrics: tuple[tuple[str, Optional[str]], ...] = field(default_factory=tuple)
INSTAGRAM_ENDPOINTS: dict[str, InstagramEndpointConfig] = {
@@ -201,6 +218,21 @@ INSTAGRAM_ENDPOINTS: dict[str, InstagramEndpointConfig] = {
sort_mode="asc",
supports_time_window=True,
timestamp_fields=("date",),
+ insight_time_series_metrics=ACCOUNT_INSIGHT_TIME_SERIES_METRICS,
+ insight_total_value_metrics=tuple((metric, None) for metric in ACCOUNT_INSIGHT_TOTAL_VALUE_METRICS),
+ ),
+ # Daily account metrics split by a breakdown dimension. A separate table, so the key
+ # of `account_insights` stays the same for existing syncs.
+ "account_insights_breakdowns": InstagramEndpointConfig(
+ name="account_insights_breakdowns",
+ edge="insights",
+ primary_keys=["instagram_account_id", "metric", "breakdown", "breakdown_value", "date"],
+ incremental_fields=[incremental_field("date")],
+ partition_key="date",
+ sort_mode="asc",
+ supports_time_window=True,
+ timestamp_fields=("date",),
+ insight_total_value_metrics=ACCOUNT_INSIGHT_BREAKDOWNS,
),
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py
index 84a68fa38e3d..34235a43e537 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py
@@ -1,4 +1,5 @@
from collections.abc import Iterable
+from dataclasses import replace
from datetime import UTC, datetime, timedelta
from typing import Any, Optional, cast
from urllib.parse import parse_qs, urlparse
@@ -660,7 +661,7 @@ class TestInstagramAccountInsights:
with scripted_network([]):
client = InstagramClient("tok", "v23.0", LOGGER)
- plan = _account_insight_plan(client, ACCOUNT_ID, 0, 45 * 86400 + 3600)
+ plan = _account_insight_plan(client, ACCOUNT_ID, INSTAGRAM_ENDPOINTS["account_insights"], 0, 45 * 86400 + 3600)
# 45 days is two windows. Each opens with the time series over the whole window,
# then asks for every total_value metric once per complete day. The partial
@@ -770,6 +771,48 @@ class TestInstagramAccountInsights:
# only spends the account's quota.
assert [request.param("metric") for request in result.requests].count("follows_and_unfollows") == 1
+ def test_breakdown_results_become_one_row_per_breakdown_value_and_day(self) -> None:
+ follows = ScriptedResponse(
+ json={
+ "data": [
+ {
+ "name": "follows_and_unfollows",
+ "period": "day",
+ "total_value": {
+ "value": 7,
+ "breakdowns": [
+ {
+ "dimension_keys": ["follow_type"],
+ "results": [
+ {"dimension_values": ["FOLLOWER"], "value": 5},
+ {"dimension_values": ["NON_FOLLOWER"], "value": 2},
+ ],
+ }
+ ],
+ },
+ }
+ ]
+ }
+ )
+ # This start date leaves a trailing window shorter than a day, which has no requests.
+ result = _drive(
+ "account_insights_breakdowns",
+ _by_metric({"follows_and_unfollows": follows}),
+ start_date=(datetime.now(UTC) - timedelta(days=29)).date().isoformat(),
+ )
+
+ assert result.raised is None
+ assert all(request.param("breakdown") for request in result.requests)
+ today = datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0)
+ assert [
+ (row["metric"], row["breakdown"], row["breakdown_value"], row["value"])
+ for row in result.rows
+ if row["date"] == today.isoformat()
+ ] == [
+ ("follows_and_unfollows", "follow_type", "FOLLOWER", 5),
+ ("follows_and_unfollows", "follow_type", "NON_FOLLOWER", 2),
+ ]
+
def test_each_finished_window_checkpoints_the_start_of_the_next(self) -> None:
start_date = (datetime.now(UTC) - timedelta(days=75)).date().isoformat()
result = _drive("account_insights", always(ScriptedResponse(json={"data": []})), start_date=start_date)
@@ -799,7 +842,8 @@ class TestInstagramAccountInsights:
# Without the clamp a start date like this fans out into tens of thousands of
# empty 30-day windows; Meta only retains ~2 years anyway.
# Only the windows matter here, so drop the per-day requests that would fill them.
- with mock.patch(f"{MODULE}.ACCOUNT_INSIGHT_TOTAL_VALUE_METRICS", ()):
+ without_daily_requests = replace(INSTAGRAM_ENDPOINTS["account_insights"], insight_total_value_metrics=())
+ with mock.patch.dict(INSTAGRAM_ENDPOINTS, {"account_insights": without_daily_requests}):
result = _drive("account_insights", always(ScriptedResponse(json={"data": []})), start_date="0001-01-01")
assert result.raised is None