:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(instagram): sync daily total_value account metrics (#118818)
posthog[bot]4f2dde0aecbf
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.pyindex 99eabf27ff43..487ac4212fc9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/canonical_descriptions.py@@ -95,7 +95,7 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"docs_url": _INSIGHTS_DOCS,"columns": {"instagram_account_id": "Instagram account the metric belongs to.",- "metric": "Metric name, for example reach or views.",+ "metric": "Metric name, for example reach, views, accounts_engaged, likes or follows_and_unfollows.","period": "Aggregation period. Always day for this table.","date": "ISO 8601 end time of the day the metric covers, in UTC.","value": "The metric's value for that day.",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.pyindex b70799042e39..4e982ec66d31 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/instagram.py@@ -16,7 +16,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.httfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.instagram.settings import (- ACCOUNT_INSIGHT_METRICS,+ ACCOUNT_INSIGHT_TIME_SERIES_METRICS,+ ACCOUNT_INSIGHT_TOTAL_VALUE_METRICS,ACCOUNT_INSIGHTS_WINDOW_DAYS,DEFAULT_INSIGHTS_LOOKBACK_DAYS,DEFAULT_MEDIA_INSIGHT_METRICS,@@ -32,6 +33,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.instagram.)REQUEST_TIMEOUT_SECONDS = 60+DAY_SECONDS = 24 * 60 * 60MAX_RETRY_ATTEMPTS = 5# Instagram account node IDs are plain integers. Anything else spliced into the Graph path@@ -106,6 +108,15 @@ class InstagramResumeConfig:next_url: str[email protected](frozen=True)+class _InsightRequest:+ metric: str+ url: str+ # End of the one-day span for a `total_value` request, which carries no `end_time` of+ # its own. None for a time series, whose points each name their day.+ day_end: Optional[int] = None++def _is_permission_code(code: Any) -> bool:if not isinstance(code, int):return False@@ -507,7 +518,7 @@ def _insight_windows(since: int, until: int, window_days: int = ACCOUNT_INSIGHTSif until <= since:return []- step = window_days * 24 * 60 * 60+ step = window_days * DAY_SECONDSwindows: list[tuple[int, int]] = []start = sincewhile start < until:@@ -522,35 +533,59 @@ def _account_insight_plan(instagram_account_id: str,since: int,until: int,-) -> list[list[tuple[str, str]]]:- """Build the request plan as one `(metric, url)` group per time window.+) -> list[list[_InsightRequest]]:+ """Build the request plan as one group of requests per time window.Window-major, not metric-major: every metric for a window is fetched together so therows for that window can be emitted in date order. Walking metric by metric insteadwould rewind the stream to the start of the range on each new metric, which breaksthe ascending watermark the pipeline checkpoints after every batch.++ A window opens with its time series requests, then one `total_value` request per+ metric for each complete day in the window. A day that has not ended yet is left+ for the next sync."""path = _account_path(instagram_account_id, "insights")- plan: list[list[tuple[str, str]]] = []+ plan: list[list[_InsightRequest]] = []for window_since, window_until in _insight_windows(since, until):- plan.append(- [- (- metric,- client.build_url(+ window = [+ _InsightRequest(+ metric=metric,+ url=client.build_url(+ path,+ {+ "metric": metric,+ "period": "day",+ "metric_type": "time_series",+ "since": str(window_since),+ "until": str(window_until),+ },+ ),+ )+ for metric in ACCOUNT_INSIGHT_TIME_SERIES_METRICS+ ]+ day_start = window_since+ while day_start + DAY_SECONDS <= window_until:+ day_end = day_start + DAY_SECONDS+ window.extend(+ _InsightRequest(+ metric=metric,+ url=client.build_url(path,{"metric": metric,"period": "day",- "metric_type": "time_series",- "since": str(window_since),- "until": str(window_until),+ "metric_type": "total_value",+ "since": str(day_start),+ "until": str(day_end),},),+ day_end=day_end,)- for metric in ACCOUNT_INSIGHT_METRICS- ]- )+ for metric in ACCOUNT_INSIGHT_TOTAL_VALUE_METRICS+ )+ day_start = day_end+ plan.append(window)return plan@@ -572,32 +607,43 @@ def _account_insight_rows(if since is not Noneelse int((now - timedelta(days=DEFAULT_INSIGHTS_LOOKBACK_DAYS)).timestamp()))+ # Start on a UTC day boundary so each `total_value` request covers one whole day. Step+ # back one more day: a `reach` point can move the watermark into a day whose totals+ # this table does not have yet.+ start = start - start % DAY_SECONDS - DAY_SECONDSplan = _account_insight_plan(client, instagram_account_id, start, until)resume_url = _resume_url(resumable_source_manager, logger, config.name)if resume_url is not None:- # A window is identified by the URL of its first metric.- heads = [window[0][1] for window in plan]+ # A window is identified by the URL of its first request.+ heads = [window[0].url for window in plan]if resume_url in heads:plan = plan[heads.index(resume_url) :]else:logger.warning("Instagram: saved account_insights checkpoint is outside this run's window, restarting")- for index, window in enumerate(plan):- if not client.has_request_budget():- # The checkpoint saved by the previous window already points here, so a resumed- # attempt carries on from this window rather than replaying the whole range.- return+ # A metric Meta rejects once is rejected for every day, so stop asking for it.+ rejected_metrics: set[str] = set()+ for index, window in enumerate(plan):rows: list[dict[str, Any]] = []- for metric, url in window:+ for request in window:+ if request.metric in rejected_metrics:+ continue+ if not client.has_request_budget():+ # Drop the part of the window fetched so far: emitting it would move the+ # watermark past days that are missing other metrics. The checkpoint saved+ # by the previous window already points here, so a resumed attempt fetches+ # this window again.+ returntry:- body = client.get(url)+ body = client.get(request.url)except (InstagramBadRequestError, InstagramPermissionError) as error:# `impressions` was retired in v22.0 and Meta rejects the whole request# for a metric the account can't serve. Drop the metric, keep the window.- logger.warning(f"Instagram: skipping account insight metric '{metric}': {error}")+ logger.warning(f"Instagram: skipping account insight metric '{request.metric}': {error}")+ rejected_metrics.add(request.metric)continueentries = body.get("data")@@ -606,14 +652,18 @@ def _account_insight_rows(for entry in entries:for point in _insight_points(entry):end_time = point.get("end_time")- if end_time is None:+ if end_time is not None:+ day = _normalize_timestamp(end_time)+ elif request.day_end is not None:+ day = datetime.fromtimestamp(request.day_end, UTC).isoformat()+ else:continuerows.append({"instagram_account_id": instagram_account_id,- "metric": entry.get("name") or metric,+ "metric": entry.get("name") or request.metric,"period": entry.get("period"),- "date": _normalize_timestamp(end_time),+ "date": day,"value": point.get("value"),"title": entry.get("title"),"description": entry.get("description"),@@ -628,7 +678,7 @@ def _account_insight_rows(next_index = index + 1if next_index < len(plan):- resumable_source_manager.save_state(InstagramResumeConfig(next_url=plan[next_index][0][1]))+ resumable_source_manager.save_state(InstagramResumeConfig(next_url=plan[next_index][0].url))def _resume_url(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.pyindex 89c2a00dcb97..0d40eb19cc88 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/settings.py@@ -42,10 +42,25 @@ DEFAULT_INSIGHTS_LOOKBACK_DAYS = 90# this returns nothing but would still fan out into thousands of empty 30-day windows.MAX_INSIGHTS_LOOKBACK_DAYS = 365 * 2-# Account-level metrics pulled as a daily time series. Meta churns this list per Graph-# version (`impressions` was retired in v22.0 in favour of `views`), so each metric is-# requested on its own and an unsupported one is skipped rather than failing the sync.-ACCOUNT_INSIGHT_METRICS: tuple[str, ...] = ("reach", "views")+# Account-level metrics. Meta churns this list per Graph version (`impressions` was+# retired in v22.0 in favour of `views`), so each metric is requested on its own and an+# unsupported one is skipped rather than failing the sync.+# `reach` is the only one Meta serves as a daily time series.+ACCOUNT_INSIGHT_TIME_SERIES_METRICS: tuple[str, ...] = ("reach",)+# Meta serves these only with `metric_type=total_value`, which sums the whole since/until+# span into one value. Each one is requested once per day to get a daily row.+ACCOUNT_INSIGHT_TOTAL_VALUE_METRICS: tuple[str, ...] = (+ "views",+ "accounts_engaged",+ "total_interactions",+ "likes",+ "comments",+ "shares",+ "saves",+ "replies",+ "profile_links_taps",+ "follows_and_unfollows",+)# Per-media metrics, keyed by `media_product_type`. Stories expose a different set from# feed posts and reels.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.pyindex 4b04abd23287..5ca069260f25 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instagram/tests/test_instagram.py@@ -12,6 +12,7 @@ import structlogfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (DriveResult,+ RecordedRequest,Script,ScriptedResponse,SourceDriver,@@ -45,7 +46,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.instagram.validate_credentials,)from products.warehouse_sources.backend.temporal.data_imports.sources.instagram.settings import (- ACCOUNT_INSIGHT_METRICS,+ ACCOUNT_INSIGHT_TIME_SERIES_METRICS,+ ACCOUNT_INSIGHT_TOTAL_VALUE_METRICS,ACCOUNT_INSIGHTS_WINDOW_DAYS,INSTAGRAM_ENDPOINTS,MAX_INSIGHTS_LOOKBACK_DAYS,@@ -110,6 +112,13 @@ def _page(rows: Iterable[dict[str, Any]], after: Optional[str] = None) -> dict[sreturn body+def _by_metric(answers: dict[str, ScriptedResponse]) -> Script:+ def respond(request: RecordedRequest) -> ScriptedResponse:+ return answers.get(request.param("metric") or "", ScriptedResponse(json={"data": []}))++ return respond++def _drive(endpoint: str,script: Script,@@ -557,11 +566,15 @@ class TestInstagramRequestBudget:assert len(result.requests) == 1def test_account_insight_windows_stop_when_the_budget_runs_out(self) -> None:+ views = ScriptedResponse(json={"data": [{"name": "views", "period": "day", "total_value": {"value": 9}}]})with mock.patch(f"{MODULE}.MAX_REQUESTS_PER_SYNC", 2):- result = _drive("account_insights", always(ScriptedResponse(json={"data": []})))+ result = _drive("account_insights", _by_metric({"views": views}))+ # The budget runs out inside the first window. Its partial rows would move the+ # watermark past days that miss other metrics, so none are emitted.assert result.raised is Noneassert len(result.requests) == 2+ assert result.rows == []def test_the_client_refuses_a_request_once_the_budget_is_spent(self) -> None:with scripted_network([ScriptedResponse(json={"data": []})]) as network:@@ -582,68 +595,82 @@ class TestInstagramAccountInsights:with scripted_network([]):client = InstagramClient("tok", "v23.0", LOGGER)- plan = _account_insight_plan(client, ACCOUNT_ID, 0, 45 * 86400)+ plan = _account_insight_plan(client, ACCOUNT_ID, 0, 45 * 86400 + 3600)- # 45 days is two windows; each carries the full metric set so the window's rows- # can be emitted in date order.- assert [[metric for metric, _ in window] for window in plan] == [- list(ACCOUNT_INSIGHT_METRICS),- list(ACCOUNT_INSIGHT_METRICS),- ]- urls = [url for window in plan for _, url in window]- assert all("metric_type=time_series" in url and "period=day" in url for url in urls)- assert [parse_qs(urlparse(url).query)["since"][0] for url in urls[:2]] == ["0", "0"]- assert parse_qs(urlparse(urls[-1]).query)["since"][0] == str(30 * 86400)+ # 45 days is two windows. Each opens with the time series over the whole window,+ # then asks for every total_value metric once per complete day. The partial+ # 46th day is left for the next sync.+ assert len(plan) == 2+ for window, days in zip(plan, (30, 15)):+ queries = [{key: values[0] for key, values in parse_qs(urlparse(r.url).query).items()} for r in window]+ assert [q["metric"] for q in queries[: len(ACCOUNT_INSIGHT_TIME_SERIES_METRICS)]] == list(+ ACCOUNT_INSIGHT_TIME_SERIES_METRICS+ )+ assert all(q["metric_type"] == "time_series" for q in queries[: len(ACCOUNT_INSIGHT_TIME_SERIES_METRICS)])+ totals = queries[len(ACCOUNT_INSIGHT_TIME_SERIES_METRICS) :]+ assert len(totals) == days * len(ACCOUNT_INSIGHT_TOTAL_VALUE_METRICS)+ assert {q["metric"] for q in totals} == set(ACCOUNT_INSIGHT_TOTAL_VALUE_METRICS)+ assert all(q["metric_type"] == "total_value" and q["period"] == "day" for q in totals)+ assert all(int(q["until"]) - int(q["since"]) == 86400 for q in totals)+ assert parse_qs(urlparse(plan[1][0].url).query)["since"][0] == str(30 * 86400)def test_a_window_is_emitted_as_one_batch_in_date_order(self) -> None:- def _series(name: str, values: list[tuple[int, str]]) -> dict[str, Any]:- return {- "data": [- {- "name": name,- "period": "day",- "time_series": [{"value": value, "end_time": end_time} for value, end_time in values],- }- ]- }-+ # The total_value body carries no end_time, so its row takes the end of the+ # one-day span the request asked for.+ today = datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0)result = _drive("account_insights",- [- ScriptedResponse(json=_series("reach", [(5, "2024-03-02T07:00:00+0000")])),- ScriptedResponse(json=_series("views", [(9, "2024-03-01T07:00:00+0000")])),- ],+ _by_metric(+ {+ "reach": ScriptedResponse(+ json={+ "data": [+ {+ "name": "reach",+ "period": "day",+ "time_series": [{"value": 5, "end_time": "2024-03-02T07:00:00+0000"}],+ }+ ]+ }+ ),+ "views": ScriptedResponse(+ json={"data": [{"name": "views", "period": "day", "total_value": {"value": 9}}]}+ ),+ }+ ),should_use_incremental_field=True,db_incremental_field_last_value=(datetime.now(UTC) - timedelta(days=1)).isoformat(),)assert result.raised is Noneassert len(result.items) == 1- assert [row["date"] for row in result.items[0]] == [- "2024-03-01T07:00:00+00:00",- "2024-03-02T07:00:00+00:00",+ assert [(row["metric"], row["date"], row["value"]) for row in result.items[0]] == [+ ("reach", "2024-03-02T07:00:00+00:00", 5),+ ("views", (today - timedelta(days=1)).isoformat(), 9),+ ("views", today.isoformat(), 9),]def test_daily_points_become_one_row_per_metric_and_day(self) -> None:result = _drive("account_insights",- [- ScriptedResponse(- json={- "data": [- {- "name": "reach",- "period": "day",- "time_series": [- {"value": 5, "end_time": "2024-03-01T07:00:00+0000"},- {"value": 7, "end_time": "2024-03-02T07:00:00+0000"},- ],- }- ]- }- ),- ScriptedResponse(json={"data": []}),- ],+ _by_metric(+ {+ "reach": ScriptedResponse(+ json={+ "data": [+ {+ "name": "reach",+ "period": "day",+ "time_series": [+ {"value": 5, "end_time": "2024-03-01T07:00:00+0000"},+ {"value": 7, "end_time": "2024-03-02T07:00:00+0000"},+ ],+ }+ ]+ }+ )+ }+ ),should_use_incremental_field=True,db_incremental_field_last_value=(datetime.now(UTC) - timedelta(days=1)).isoformat(),)@@ -656,32 +683,31 @@ class TestInstagramAccountInsights:]assert {row["instagram_account_id"] for row in reach_rows} == {ACCOUNT_ID}- def test_a_metric_meta_has_retired_is_dropped_without_failing_the_table(self) -> None:+ def test_a_metric_meta_rejects_is_dropped_for_the_rest_of_the_sync(self) -> None:result = _drive("account_insights",- [- ScriptedResponse(status=400, json={"error": {"code": 100, "message": "metric deprecated"}}),- ScriptedResponse(- json={- "data": [- {- "name": "views",- "period": "day",- "time_series": [{"value": 9, "end_time": "2024-03-01T07:00:00+0000"}],- }- ]- }- ),- ],- should_use_incremental_field=True,- db_incremental_field_last_value=(datetime.now(UTC) - timedelta(days=1)).isoformat(),+ _by_metric(+ {+ "follows_and_unfollows": ScriptedResponse(+ status=400, json={"error": {"code": 100, "message": "metric not available"}}+ ),+ "views": ScriptedResponse(+ json={"data": [{"name": "views", "period": "day", "total_value": {"value": 9}}]}+ ),+ }+ ),+ start_date=(datetime.now(UTC) - timedelta(days=35)).date().isoformat(),)assert result.raised is None- assert [row["metric"] for row in result.rows] == ["views"]+ assert {row["metric"] for row in result.rows} == {"views"}+ # Meta rejects the metric for every day, so asking again on each of the other days+ # only spends the account's quota.+ assert [request.param("metric") for request in result.requests].count("follows_and_unfollows") == 1def test_each_finished_window_checkpoints_the_start_of_the_next(self) -> None:- result = _drive("account_insights", always(ScriptedResponse(json={"data": []})), start_date="2024-01-01")+ start_date = (datetime.now(UTC) - timedelta(days=75)).date().isoformat()+ result = _drive("account_insights", always(ScriptedResponse(json={"data": []})), start_date=start_date)assert result.raised is Noneassert result.saved_states@@ -707,7 +733,9 @@ class TestInstagramAccountInsights:def test_an_ancient_start_date_is_clamped_to_the_retention_horizon(self) -> None:# Without the clamp a start date like this fans out into tens of thousands of# empty 30-day windows; Meta only retains ~2 years anyway.- result = _drive("account_insights", always(ScriptedResponse(json={"data": []})), start_date="0001-01-01")+ # Only the windows matter here, so drop the per-day requests that would fill them.+ with mock.patch(f"{MODULE}.ACCOUNT_INSIGHT_TOTAL_VALUE_METRICS", ()):+ result = _drive("account_insights", always(ScriptedResponse(json={"data": []})), start_date="0001-01-01")assert result.raised is Nonewindows = [url for url in result.urls if "metric=reach" in url]