:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): fix X Ads stats windows for non-UTC accounts (#124924)
posthog[bot]de41a54f757a
diff --git a/products/warehouse_sources/sources/twitter_ads/tests/test_twitter_ads.py b/products/warehouse_sources/sources/twitter_ads/tests/test_twitter_ads.pyindex 65ee54070036..9f39d4cf649f 100644--- a/products/warehouse_sources/sources/twitter_ads/tests/test_twitter_ads.py+++ b/products/warehouse_sources/sources/twitter_ads/tests/test_twitter_ads.py@@ -1,6 +1,6 @@import jsonfrom collections.abc import Iterable-from datetime import UTC, date, datetime, time, timedelta+from datetime import UTC, date, datetime, timedeltafrom typing import Any, castfrom urllib.parse import parse_qs, urlparsefrom zoneinfo import ZoneInfo@@ -129,21 +129,24 @@ def test_stats_limits_daily_rows_currency_and_dst(return response({"data": line_items})if "/stats/" not in path:return response({"data": {"timezone": account_timezone.key, "created_at": "2025-10-25T10:00:00Z"}})- # A bare date (not a full timestamp) lets X resolve midnight-to-midnight in the account's- # own timezone; date.fromisoformat rejects anything else, including a UTC timestamp.- start_date = date.fromisoformat(params["start_time"][0])- end_date = date.fromisoformat(params["end_time"][0])- start = datetime.combine(start_date, time.min, account_timezone)- end = datetime.combine(end_date, time.min, account_timezone)+ # X rejects bare dates for DAY windows on non-UTC accounts, so the bounds must be full UTC+ # timestamps. fromisoformat on a bare date would give a naive value.+ assert params["start_time"][0].endswith("Z") and params["end_time"][0].endswith("Z")+ start = datetime.fromisoformat(params["start_time"][0]).astimezone(account_timezone)+ end = datetime.fromisoformat(params["end_time"][0]).astimezone(account_timezone)ids = params["entity_ids"][0].split(",")assert len(ids) <= 20assert end.astimezone(UTC) - start.astimezone(UTC) <= timedelta(days=7)+ # X rejects a DAY window whose bounds are not midnight in the ad account's own timezone,+ # which means the offset in force on each date rather than today's.+ assert start.hour == end.hour == 0+ assert start.minute == end.minute == 0offsets.update({start.utcoffset(), end.utcoffset()})assert params["granularity"] == ["DAY"]assert params["metric_groups"] == ["ENGAGEMENT,BILLING"]assert params["entity"] == ["CAMPAIGN" if table == "campaign_stats" else "LINE_ITEM"]- days = (end_date - start_date).days- calls.append((start_date, end_date, params["placement"][0], ids))+ days = (end.date() - start.date()).days+ calls.append((start.date(), end.date(), params["placement"][0], ids))return response({"data": [@@ -182,7 +185,7 @@ def test_stats_limits_daily_rows_currency_and_dst(assert max(row["date"] for row in rows) == date(2025, 11, 9)assert {call[2] for call in calls} == set(PLACEMENTS)# The window walked across the November fall-back, so it must have used both offsets. Without- # this the test would still pass on a range that never changes offset.+ # this the midnight assertions above would still pass on a range that never changes offset.assert offsets == {timedelta(hours=-7), timedelta(hours=-8)}assert resource.primary_keys == ["entity_id", "date", "placement"]assert resource.partition_keys == ["date"]@@ -199,12 +202,14 @@ def test_stats_limits_daily_rows_currency_and_dst(assert max(row["date"] for row in resumed) == date(2025, 11, 9)-def test_stats_requests_bare_dates_for_a_half_hour_offset_account(client: TwitterAdsClient, manager: MagicMock) -> None:- # An account on IST (UTC+5:30) never has a UTC midnight; sending our own UTC conversion for- # start_time/end_time would carry ":30" minutes and X rejects that with a 400.+def test_stats_rolls_hours_into_local_days_for_a_half_hour_offset_account(+ client: TwitterAdsClient, manager: MagicMock+) -> None:+ # Local midnight on UTC+5:30 is never a whole UTC hour, and X only takes whole hours, so a DAY+ # window cannot be expressed at all.account_timezone = ZoneInfo("Asia/Kolkata")campaigns = [{"id": "campaign-00", "funding_instrument_id": "funding"}]- requests_sent = []+ windows = []def send(request: requests.PreparedRequest, **kwargs: object) -> requests.Response:url = request_url(request)@@ -215,20 +220,50 @@ def test_stats_requests_bare_dates_for_a_half_hour_offset_account(client: Twittereturn response({"data": campaigns})if "/stats/" not in path:return response({"data": {"timezone": account_timezone.key, "created_at": "2025-10-25T10:00:00Z"}})- requests_sent.append(parse_qs(urlparse(url).query))- return response({"data": []})+ params = parse_qs(urlparse(url).query)+ assert params["granularity"] == ["HOUR"]+ start = datetime.fromisoformat(params["start_time"][0])+ end = datetime.fromisoformat(params["end_time"][0])+ assert start.minute == end.minute == 0+ assert end - start <= timedelta(days=7)+ windows.append((start, end))+ hours = int((end - start) / timedelta(hours=1))+ return response(+ {+ "data": [+ {+ "id": "campaign-00",+ "id_data": [+ {+ "segment": None,+ "metrics": {+ "impressions": list(range(hours)),+ "clicks": [None] * hours,+ "billed_charge_local_micro": None,+ },+ }+ ],+ }+ ]+ }+ )with (time_machine.travel("2025-11-01T20:00:00Z", tick=False),patch.object(client.session, "send", side_effect=send),):resource = twitter_ads_source(client, "account", "campaign_stats", manager, date(2025, 10, 30))- list(sync_items(resource))-- assert requests_sent- for params in requests_sent:- assert date.fromisoformat(params["start_time"][0])- assert date.fromisoformat(params["end_time"][0])+ rows = [row for page in sync_items(resource) for row in page if row["placement"] == PLACEMENTS[0]]++ # Each day starts at 00:30 local, the first whole UTC hour after local midnight.+ assert windows[0] == (datetime(2025, 10, 29, 19, tzinfo=UTC), datetime(2025, 11, 1, 19, tzinfo=UTC))+ assert [(row["date"], row["impressions"]) for row in rows] == [+ (date(2025, 10, 30), sum(range(0, 24))),+ (date(2025, 10, 31), sum(range(24, 48))),+ (date(2025, 11, 1), sum(range(48, 72))),+ ]+ assert {row["clicks"] for row in rows} == {None}+ assert {row["billed_charge_local_micro"] for row in rows} == {None}def test_error_detail_rides_along_with_the_status(client: TwitterAdsClient) -> None:diff --git a/products/warehouse_sources/sources/twitter_ads/twitter_ads.py b/products/warehouse_sources/sources/twitter_ads/twitter_ads.pyindex ed6898c5c369..30dc9d3f110d 100644--- a/products/warehouse_sources/sources/twitter_ads/twitter_ads.py+++ b/products/warehouse_sources/sources/twitter_ads/twitter_ads.py@@ -38,7 +38,7 @@ class TwitterAdsResumeConfig:complete: bool = False-# X rejects a DAY-granularity window wider than this.+# X rejects a stats window wider than this.MAX_STATS_WINDOW = timedelta(days=7)# Cap on how much of X's error body rides along in the exception message.@@ -56,6 +56,49 @@ def _account_midnight(day: date, account_timezone: ZoneInfo) -> datetime:return datetime.combine(day, time.min, account_timezone).astimezone(UTC)+def _hour_ceiling(moment: datetime) -> datetime:+ if moment.minute == moment.second == moment.microsecond == 0:+ return moment+ return moment.replace(minute=0, second=0, microsecond=0) + timedelta(hours=1)+++def _stats_bound(day: date, account_timezone: ZoneInfo, hourly: bool) -> datetime:+ midnight = _account_midnight(day, account_timezone)+ return _hour_ceiling(midnight) if hourly else midnight+++def _utc_param(moment: datetime) -> str:+ return moment.isoformat().replace("+00:00", "Z")+++def _hours_to_days(+ payload: dict[str, Any], start_time: datetime, start: date, end: date, account_timezone: ZoneInfo+) -> dict[str, Any]:+ """Fold an HOUR-granularity stats payload into the per-day arrays `stats_rows` reads.++ Each hour counts toward the account-local date it starts in. On a half-hour offset that day runs+ from 00:30 to 00:30 local time, the closest whole-hour window X will serve.+ """+ days = (end - start).days+ for entity in payload["data"]:+ for segment in entity["id_data"]:+ daily: dict[str, Any] = {}+ for key, values in segment["metrics"].items():+ if not isinstance(values, list):+ daily[key] = values+ continue+ totals: list[Any] = [None] * days+ for hour, value in enumerate(values):+ if value is None:+ continue+ index = ((start_time + timedelta(hours=hour)).astimezone(account_timezone).date() - start).days+ if 0 <= index < days:+ totals[index] = (totals[index] or 0) + value+ daily[key] = totals+ segment["metrics"] = daily+ return payload++def _error_detail(response: requests.Response) -> str:"""X's own explanation for a failed request, for the message a user and an engineer both read.@@ -206,13 +249,16 @@ class TwitterAdsClient:end = date.fromisoformat(state.end_date) if state.end_date else account_now.date()while start < end:window_end = min(start + timedelta(days=7), end)- start_time = _account_midnight(start, account_timezone)- end_time = _account_midnight(window_end, account_timezone)+ # X takes a DAY window only when both bounds are local midnight, sent as whole UTC hours.+ # On a half-hour offset (e.g. UTC+5:30) local midnight never is one, so ask for hours instead.+ hourly = any(_account_midnight(day, account_timezone).minute for day in (start, window_end))+ start_time = _stats_bound(start, account_timezone, hourly)+ end_time = _stats_bound(window_end, account_timezone, hourly)# A week containing a fall-back transition spans 169 hours, one over X's ceiling, so# hand that window a day back rather than letting the request 400.while end_time - start_time > MAX_STATS_WINDOW:window_end -= timedelta(days=1)- end_time = _account_midnight(window_end, account_timezone)+ end_time = _stats_bound(window_end, account_timezone, hourly)for batch_start in range(0, len(entity_ids), 20):for placement in PLACEMENTS:payload = self.get(@@ -220,18 +266,18 @@ class TwitterAdsClient:{"entity": STATS_TABLES[table],"entity_ids": ",".join(entity_ids[batch_start : batch_start + 20]),- # A bare date tells X to resolve midnight-to-midnight in the account's- # own timezone itself. An account on a half-hour-offset timezone (e.g.- # IST) never lands on a whole UTC hour, so sending our own UTC- # conversion here gets a 400 ("whole hours" requirement) that this- # sidesteps entirely.- "start_time": start.isoformat(),- "end_time": window_end.isoformat(),- "granularity": "DAY",+ # X's docs say a bare date means midnight in the account's timezone, but+ # X rejects bare dates for DAY windows on non-UTC accounts with+ # INVALID_TIME_WINDOW. Send local midnight as a UTC timestamp instead.+ "start_time": _utc_param(start_time),+ "end_time": _utc_param(end_time),+ "granularity": "HOUR" if hourly else "DAY","placement": placement,"metric_groups": "ENGAGEMENT,BILLING",},)+ if hourly:+ payload = _hours_to_days(payload, start_time, start, window_end, account_timezone)rows = self.stats_rows(payload, start, window_end, placement, currencies, account_id)if batch_start + 20 >= len(entity_ids) and placement == PLACEMENTS[-1]:manager.save_state(