Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): fix X Ads stats windows for non-UTC accounts (#124924)

posthog[bot]de41a54f757a
diff --git a/products/warehouse_sources/sources/twitter_ads/tests/test_twitter_ads.py b/products/warehouse_sources/sources/twitter_ads/tests/test_twitter_ads.py
index 65ee54070036..9f39d4cf649f 100644
--- a/products/warehouse_sources/sources/twitter_ads/tests/test_twitter_ads.py
+++ b/products/warehouse_sources/sources/twitter_ads/tests/test_twitter_ads.py
@@ -1,6 +1,6 @@
import json
from collections.abc import Iterable
-from datetime import UTC, date, datetime, time, timedelta
+from datetime import UTC, date, datetime, timedelta
from typing import Any, cast
from urllib.parse import parse_qs, urlparse
from zoneinfo import ZoneInfo
@@ -129,21 +129,24 @@ def test_stats_limits_daily_rows_currency_and_dst(
return response({"data": line_items})
if "/stats/" not in path:
return response({"data": {"timezone": account_timezone.key, "created_at": "2025-10-25T10:00:00Z"}})
- # A bare date (not a full timestamp) lets X resolve midnight-to-midnight in the account's
- # own timezone; date.fromisoformat rejects anything else, including a UTC timestamp.
- start_date = date.fromisoformat(params["start_time"][0])
- end_date = date.fromisoformat(params["end_time"][0])
- start = datetime.combine(start_date, time.min, account_timezone)
- end = datetime.combine(end_date, time.min, account_timezone)
+ # X rejects bare dates for DAY windows on non-UTC accounts, so the bounds must be full UTC
+ # timestamps. fromisoformat on a bare date would give a naive value.
+ assert params["start_time"][0].endswith("Z") and params["end_time"][0].endswith("Z")
+ start = datetime.fromisoformat(params["start_time"][0]).astimezone(account_timezone)
+ end = datetime.fromisoformat(params["end_time"][0]).astimezone(account_timezone)
ids = params["entity_ids"][0].split(",")
assert len(ids) <= 20
assert end.astimezone(UTC) - start.astimezone(UTC) <= timedelta(days=7)
+ # X rejects a DAY window whose bounds are not midnight in the ad account's own timezone,
+ # which means the offset in force on each date rather than today's.
+ assert start.hour == end.hour == 0
+ assert start.minute == end.minute == 0
offsets.update({start.utcoffset(), end.utcoffset()})
assert params["granularity"] == ["DAY"]
assert params["metric_groups"] == ["ENGAGEMENT,BILLING"]
assert params["entity"] == ["CAMPAIGN" if table == "campaign_stats" else "LINE_ITEM"]
- days = (end_date - start_date).days
- calls.append((start_date, end_date, params["placement"][0], ids))
+ days = (end.date() - start.date()).days
+ calls.append((start.date(), end.date(), params["placement"][0], ids))
return response(
{
"data": [
@@ -182,7 +185,7 @@ def test_stats_limits_daily_rows_currency_and_dst(
assert max(row["date"] for row in rows) == date(2025, 11, 9)
assert {call[2] for call in calls} == set(PLACEMENTS)
# The window walked across the November fall-back, so it must have used both offsets. Without
- # this the test would still pass on a range that never changes offset.
+ # this the midnight assertions above would still pass on a range that never changes offset.
assert offsets == {timedelta(hours=-7), timedelta(hours=-8)}
assert resource.primary_keys == ["entity_id", "date", "placement"]
assert resource.partition_keys == ["date"]
@@ -199,12 +202,14 @@ def test_stats_limits_daily_rows_currency_and_dst(
assert max(row["date"] for row in resumed) == date(2025, 11, 9)
-def test_stats_requests_bare_dates_for_a_half_hour_offset_account(client: TwitterAdsClient, manager: MagicMock) -> None:
- # An account on IST (UTC+5:30) never has a UTC midnight; sending our own UTC conversion for
- # start_time/end_time would carry ":30" minutes and X rejects that with a 400.
+def test_stats_rolls_hours_into_local_days_for_a_half_hour_offset_account(
+ client: TwitterAdsClient, manager: MagicMock
+) -> None:
+ # Local midnight on UTC+5:30 is never a whole UTC hour, and X only takes whole hours, so a DAY
+ # window cannot be expressed at all.
account_timezone = ZoneInfo("Asia/Kolkata")
campaigns = [{"id": "campaign-00", "funding_instrument_id": "funding"}]
- requests_sent = []
+ windows = []
def send(request: requests.PreparedRequest, **kwargs: object) -> requests.Response:
url = request_url(request)
@@ -215,20 +220,50 @@ def test_stats_requests_bare_dates_for_a_half_hour_offset_account(client: Twitte
return response({"data": campaigns})
if "/stats/" not in path:
return response({"data": {"timezone": account_timezone.key, "created_at": "2025-10-25T10:00:00Z"}})
- requests_sent.append(parse_qs(urlparse(url).query))
- return response({"data": []})
+ params = parse_qs(urlparse(url).query)
+ assert params["granularity"] == ["HOUR"]
+ start = datetime.fromisoformat(params["start_time"][0])
+ end = datetime.fromisoformat(params["end_time"][0])
+ assert start.minute == end.minute == 0
+ assert end - start <= timedelta(days=7)
+ windows.append((start, end))
+ hours = int((end - start) / timedelta(hours=1))
+ return response(
+ {
+ "data": [
+ {
+ "id": "campaign-00",
+ "id_data": [
+ {
+ "segment": None,
+ "metrics": {
+ "impressions": list(range(hours)),
+ "clicks": [None] * hours,
+ "billed_charge_local_micro": None,
+ },
+ }
+ ],
+ }
+ ]
+ }
+ )
with (
time_machine.travel("2025-11-01T20:00:00Z", tick=False),
patch.object(client.session, "send", side_effect=send),
):
resource = twitter_ads_source(client, "account", "campaign_stats", manager, date(2025, 10, 30))
- list(sync_items(resource))
-
- assert requests_sent
- for params in requests_sent:
- assert date.fromisoformat(params["start_time"][0])
- assert date.fromisoformat(params["end_time"][0])
+ rows = [row for page in sync_items(resource) for row in page if row["placement"] == PLACEMENTS[0]]
+
+ # Each day starts at 00:30 local, the first whole UTC hour after local midnight.
+ assert windows[0] == (datetime(2025, 10, 29, 19, tzinfo=UTC), datetime(2025, 11, 1, 19, tzinfo=UTC))
+ assert [(row["date"], row["impressions"]) for row in rows] == [
+ (date(2025, 10, 30), sum(range(0, 24))),
+ (date(2025, 10, 31), sum(range(24, 48))),
+ (date(2025, 11, 1), sum(range(48, 72))),
+ ]
+ assert {row["clicks"] for row in rows} == {None}
+ assert {row["billed_charge_local_micro"] for row in rows} == {None}
def test_error_detail_rides_along_with_the_status(client: TwitterAdsClient) -> None:
diff --git a/products/warehouse_sources/sources/twitter_ads/twitter_ads.py b/products/warehouse_sources/sources/twitter_ads/twitter_ads.py
index ed6898c5c369..30dc9d3f110d 100644
--- a/products/warehouse_sources/sources/twitter_ads/twitter_ads.py
+++ b/products/warehouse_sources/sources/twitter_ads/twitter_ads.py
@@ -38,7 +38,7 @@ class TwitterAdsResumeConfig:
complete: bool = False
-# X rejects a DAY-granularity window wider than this.
+# X rejects a stats window wider than this.
MAX_STATS_WINDOW = timedelta(days=7)
# Cap on how much of X's error body rides along in the exception message.
@@ -56,6 +56,49 @@ def _account_midnight(day: date, account_timezone: ZoneInfo) -> datetime:
return datetime.combine(day, time.min, account_timezone).astimezone(UTC)
+def _hour_ceiling(moment: datetime) -> datetime:
+ if moment.minute == moment.second == moment.microsecond == 0:
+ return moment
+ return moment.replace(minute=0, second=0, microsecond=0) + timedelta(hours=1)
+
+
+def _stats_bound(day: date, account_timezone: ZoneInfo, hourly: bool) -> datetime:
+ midnight = _account_midnight(day, account_timezone)
+ return _hour_ceiling(midnight) if hourly else midnight
+
+
+def _utc_param(moment: datetime) -> str:
+ return moment.isoformat().replace("+00:00", "Z")
+
+
+def _hours_to_days(
+ payload: dict[str, Any], start_time: datetime, start: date, end: date, account_timezone: ZoneInfo
+) -> dict[str, Any]:
+ """Fold an HOUR-granularity stats payload into the per-day arrays `stats_rows` reads.
+
+ Each hour counts toward the account-local date it starts in. On a half-hour offset that day runs
+ from 00:30 to 00:30 local time, the closest whole-hour window X will serve.
+ """
+ days = (end - start).days
+ for entity in payload["data"]:
+ for segment in entity["id_data"]:
+ daily: dict[str, Any] = {}
+ for key, values in segment["metrics"].items():
+ if not isinstance(values, list):
+ daily[key] = values
+ continue
+ totals: list[Any] = [None] * days
+ for hour, value in enumerate(values):
+ if value is None:
+ continue
+ index = ((start_time + timedelta(hours=hour)).astimezone(account_timezone).date() - start).days
+ if 0 <= index < days:
+ totals[index] = (totals[index] or 0) + value
+ daily[key] = totals
+ segment["metrics"] = daily
+ return payload
+
+
def _error_detail(response: requests.Response) -> str:
"""X's own explanation for a failed request, for the message a user and an engineer both read.
@@ -206,13 +249,16 @@ class TwitterAdsClient:
end = date.fromisoformat(state.end_date) if state.end_date else account_now.date()
while start < end:
window_end = min(start + timedelta(days=7), end)
- start_time = _account_midnight(start, account_timezone)
- end_time = _account_midnight(window_end, account_timezone)
+ # X takes a DAY window only when both bounds are local midnight, sent as whole UTC hours.
+ # On a half-hour offset (e.g. UTC+5:30) local midnight never is one, so ask for hours instead.
+ hourly = any(_account_midnight(day, account_timezone).minute for day in (start, window_end))
+ start_time = _stats_bound(start, account_timezone, hourly)
+ end_time = _stats_bound(window_end, account_timezone, hourly)
# A week containing a fall-back transition spans 169 hours, one over X's ceiling, so
# hand that window a day back rather than letting the request 400.
while end_time - start_time > MAX_STATS_WINDOW:
window_end -= timedelta(days=1)
- end_time = _account_midnight(window_end, account_timezone)
+ end_time = _stats_bound(window_end, account_timezone, hourly)
for batch_start in range(0, len(entity_ids), 20):
for placement in PLACEMENTS:
payload = self.get(
@@ -220,18 +266,18 @@ class TwitterAdsClient:
{
"entity": STATS_TABLES[table],
"entity_ids": ",".join(entity_ids[batch_start : batch_start + 20]),
- # A bare date tells X to resolve midnight-to-midnight in the account's
- # own timezone itself. An account on a half-hour-offset timezone (e.g.
- # IST) never lands on a whole UTC hour, so sending our own UTC
- # conversion here gets a 400 ("whole hours" requirement) that this
- # sidesteps entirely.
- "start_time": start.isoformat(),
- "end_time": window_end.isoformat(),
- "granularity": "DAY",
+ # X's docs say a bare date means midnight in the account's timezone, but
+ # X rejects bare dates for DAY windows on non-UTC accounts with
+ # INVALID_TIME_WINDOW. Send local midnight as a UTC timestamp instead.
+ "start_time": _utc_param(start_time),
+ "end_time": _utc_param(end_time),
+ "granularity": "HOUR" if hourly else "DAY",
"placement": placement,
"metric_groups": "ENGAGEMENT,BILLING",
},
)
+ if hourly:
+ payload = _hours_to_days(payload, start_time, start, window_end, account_timezone)
rows = self.stats_rows(payload, start, window_end, placement, currencies, account_id)
if batch_start + 20 >= len(entity_ids) and placement == PLACEMENTS[-1]:
manager.save_state(