Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(openai-ads): read the insights history in 30-day windows (#126208)

Tom Owers20568f5bc9b7
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/openai_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/openai_ads.py
index 66fdc5dae4a4..284181cc7da9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/openai_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/openai_ads.py
@@ -18,6 +18,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
BasePaginator,
SinglePagePaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClient
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
@@ -59,6 +60,11 @@ _T = TypeVar("_T")
# because the API reads dates in the ad account's timezone and this source sends them in UTC.
INSIGHTS_HISTORY_DAYS = 364
+# A first sync or full refresh asks for most of a year of daily buckets. OpenAI Ads can answer that
+# request with a `server_error` on every run, while the few days an incremental run asks for read
+# fine. So a long range is read in windows of at most this many days.
+INSIGHTS_WINDOW_DAYS = 30
+
@dataclasses.dataclass
class OpenAIAdsResumeConfig:
@@ -204,6 +210,18 @@ def _insights_window(db_incremental_field_last_value: Optional[Any]) -> SyncWind
return SyncWindow(start=since.isoformat(), end=until.isoformat())
+def _split_insights_window(window: SyncWindow[str]) -> list[SyncWindow[str]]:
+ start = date.fromisoformat(window.start)
+ end = date.fromisoformat(window.end)
+ windows: list[SyncWindow[str]] = []
+ while True:
+ last = min(start + timedelta(days=INSIGHTS_WINDOW_DAYS - 1), end)
+ windows.append(SyncWindow(start=start.isoformat(), end=last.isoformat()))
+ if last >= end:
+ return windows
+ start = last + timedelta(days=1)
+
+
def _convert_insights_times(row: dict[str, Any]) -> dict[str, Any]:
# Bucket bounds arrive as epoch seconds; real timestamps are needed for the DateTime
# incremental watermark and datetime partitioning.
@@ -319,42 +337,50 @@ def openai_ads_source(
can_resume = resumable_source_manager.can_resume() and not config.single_object
resume = resumable_source_manager.load_state() if can_resume else None
- data_map: Callable[[dict[str, Any]], dict[str, Any]] | None = None
if config.aggregation_level is not None:
- if resume is not None and resume.since and resume.until:
- window: SyncWindow[str] = SyncWindow(start=resume.since, end=resume.until)
- else:
- window = _insights_window(db_incremental_field_last_value)
- params: dict[str, Any] = {
- "aggregation_level": config.aggregation_level,
- "time_granularity": "daily",
- "limit": INSIGHTS_PAGE_SIZE,
- # `requests` encodes the list as one repeated fields[] query param per element.
- "fields[]": list(config.insights_fields),
- # One JSON-encoded time-range object; the explicit timezone keeps daily buckets (and
- # therefore the bucket ids merge dedupes on) stable across runs.
- "time_ranges[]": json.dumps(
- {"type": "date_range", "since": window.start, "until": window.end, "timezone": "UTC"}
- ),
- }
- if config.segment is not None:
- params["segments[]"] = config.segment
- currency = _account_currency(api_key)
+ return _insights_source(
+ config, api_key, team_id, job_id, resumable_source_manager, resume, db_incremental_field_last_value
+ )
- def enrich_insights(row: dict[str, Any]) -> dict[str, Any]:
- return {**_convert_insights_times(_stable_insights_id(row)), "currency_code": currency}
-
- data_map = enrich_insights
- elif config.single_object:
- window = SyncWindow(start="", end="")
- params = {}
+ if config.single_object:
+ params: dict[str, Any] = {}
else:
- window = SyncWindow(start="", end="")
# An explicit stable sort prevents page-boundary skips/duplicates while paginating the
# full campaign list.
params = {"limit": LIST_PAGE_SIZE, "order": "asc"}
- rest_config: RESTAPIConfig = {
+ initial_paginator_state: Optional[dict[str, Any]] = None
+ if resume is not None and resume.cursor:
+ initial_paginator_state = {"cursor": resume.cursor}
+
+ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ # Persist only while a next page remains; the checkpoint is saved AFTER a page is yielded,
+ # pointing at the next page, so a crash resumes from a page whose predecessors were all
+ # yielded — the overlap merge dedupes on the primary key.
+ if state and state.get("cursor"):
+ resumable_source_manager.save_state(OpenAIAdsResumeConfig(cursor=state["cursor"]))
+
+ resource = rest_api_resource(
+ _rest_config(config, api_key, params, None),
+ team_id,
+ job_id,
+ db_incremental_field_last_value,
+ resume_hook=save_checkpoint,
+ initial_paginator_state=initial_paginator_state,
+ )
+
+ return _source_response(
+ config, lambda: _retry_forbidden_server_errors(resource), column_hints=resource.column_hints
+ )
+
+
+def _rest_config(
+ config: OpenAIAdsEndpointConfig,
+ api_key: str,
+ params: dict[str, Any],
+ data_map: Callable[[dict[str, Any]], dict[str, Any]] | None,
+) -> RESTAPIConfig:
+ return {
"client": {
"base_url": OPENAI_ADS_BASE_URL,
"headers": _headers(),
@@ -363,7 +389,7 @@ def openai_ads_source(
"resource_defaults": None,
"resources": [
{
- "name": endpoint,
+ "name": config.name,
"endpoint": {
"path": config.path,
"params": params,
@@ -375,29 +401,82 @@ def openai_ads_source(
],
}
- initial_paginator_state: Optional[dict[str, Any]] = None
- if resume is not None and resume.cursor:
- initial_paginator_state = {"cursor": resume.cursor}
- def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
- # Persist only while a next page remains; the checkpoint is saved AFTER a page is yielded,
- # pointing at the next page, so a crash resumes from a page whose predecessors were all
- # yielded — the overlap merge dedupes on the primary key. Insights pin their window so the
- # resumed cursor pairs with the result set it was issued for.
- if state and state.get("cursor"):
- resumable_source_manager.save_state(
- OpenAIAdsResumeConfig(cursor=state["cursor"], since=window.start or None, until=window.end or None)
- )
+def _insights_source(
+ config: OpenAIAdsEndpointConfig,
+ api_key: str,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[OpenAIAdsResumeConfig],
+ resume: OpenAIAdsResumeConfig | None,
+ db_incremental_field_last_value: Optional[Any],
+) -> SourceResponse:
+ full_window = _insights_window(db_incremental_field_last_value)
+ if resume is not None and resume.since and resume.until:
+ windows = [SyncWindow(start=resume.since, end=resume.until)]
+ next_start = date.fromisoformat(resume.until) + timedelta(days=1)
+ if next_start <= date.fromisoformat(full_window.end):
+ windows += _split_insights_window(SyncWindow(start=next_start.isoformat(), end=full_window.end))
+ else:
+ windows = _split_insights_window(full_window)
- resource = rest_api_resource(
- rest_config,
- team_id,
- job_id,
- db_incremental_field_last_value,
- resume_hook=save_checkpoint,
- initial_paginator_state=initial_paginator_state,
- )
+ currency = _account_currency(api_key)
+
+ def enrich_insights(row: dict[str, Any]) -> dict[str, Any]:
+ return {**_convert_insights_times(_stable_insights_id(row)), "currency_code": currency}
+
+ def window_resource(window: SyncWindow[str], cursor: str | None) -> Resource:
+ params: dict[str, Any] = {
+ "aggregation_level": config.aggregation_level,
+ "time_granularity": "daily",
+ "limit": INSIGHTS_PAGE_SIZE,
+ # `requests` encodes the list as one repeated fields[] query param per element.
+ "fields[]": list(config.insights_fields),
+ # One JSON-encoded time-range object; the explicit timezone keeps daily buckets (and
+ # therefore the bucket ids merge dedupes on) stable across runs.
+ "time_ranges[]": json.dumps(
+ {"type": "date_range", "since": window.start, "until": window.end, "timezone": "UTC"}
+ ),
+ }
+ if config.segment is not None:
+ params["segments[]"] = config.segment
+
+ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ # Persist only while a next page remains; the checkpoint is saved AFTER a page is
+ # yielded, pointing at the next page, so a crash resumes from a page whose predecessors
+ # were all yielded — the overlap merge dedupes on the primary key. The window is pinned
+ # with the cursor so the resumed cursor pairs with the result set it was issued for.
+ if state and state.get("cursor"):
+ resumable_source_manager.save_state(
+ OpenAIAdsResumeConfig(cursor=state["cursor"], since=window.start, until=window.end)
+ )
+
+ return rest_api_resource(
+ _rest_config(config, api_key, params, enrich_insights),
+ team_id,
+ job_id,
+ db_incremental_field_last_value,
+ resume_hook=save_checkpoint,
+ initial_paginator_state={"cursor": cursor} if cursor else None,
+ )
+
+ first_resource = window_resource(windows[0], resume.cursor if resume is not None else None)
+
+ def pages() -> Iterator[Any]:
+ for index, window in enumerate(windows):
+ resource = first_resource
+ if index > 0:
+ # Every row of the earlier windows is already yielded, so a resume can start here.
+ resumable_source_manager.save_state(
+ OpenAIAdsResumeConfig(cursor=None, since=window.start, until=window.end)
+ )
+ resumable_source_manager.safe_point()
+ resource = window_resource(window, None)
+ # This wrapper hides the framework's own safe points, and a window can hold no rows.
+ for page in resource:
+ yield page
+ resumable_source_manager.safe_point()
return _source_response(
- config, lambda: _retry_forbidden_server_errors(resource), column_hints=resource.column_hints
+ config, lambda: _retry_forbidden_server_errors(pages()), column_hints=first_resource.column_hints
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads.py
index 3a27f21e2166..82c17bfb0ca6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads.py
@@ -1,5 +1,6 @@
import json
-from datetime import UTC, datetime
+from collections.abc import Callable
+from datetime import UTC, date, datetime, timedelta
from typing import Any
import time_machine
@@ -8,6 +9,7 @@ from parameterized import parameterized
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import error_message_matches
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+ RecordedRequest,
ScriptedResponse,
SourceDriver,
always,
@@ -34,6 +36,18 @@ def _page(items: list[dict[str, Any]], *, has_more: bool, last_id: str | None =
return ScriptedResponse(json=body)
+def _insights_script(*pages: ScriptedResponse) -> Callable[[RecordedRequest], ScriptedResponse]:
+ # A first sync reads the reporting history in several windows, and the windows after `pages` are empty.
+ remaining = list(pages)
+
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == "/v1/ad_account":
+ return ScriptedResponse(json={"currency_code": "EUR"})
+ return remaining.pop(0) if remaining else _page([], has_more=False)
+
+ return respond
+
+
class TestFanOut:
def test_ads_walk_both_levels_and_carry_full_lineage(self) -> None:
result = _driver().run(
@@ -109,10 +123,7 @@ class TestInsights:
) -> None:
result = _driver().run(
endpoint,
- [
- ScriptedResponse(json={"currency_code": "EUR"}),
- _page([{"id": "b1", "start_time": 1777075200, "end_time": 1777161600}], has_more=False),
- ],
+ _insights_script(_page([{"id": "b1", "start_time": 1777075200, "end_time": 1777161600}], has_more=False)),
)
assert result.raised is None
@@ -139,13 +150,12 @@ class TestInsights:
for plan in ("plan=eJzLSM3JyVcozy/KSQEAGgQEXQ", "plan=eJwrSS0uAQAEXQHB"):
result = _driver().run(
"campaign_insights",
- [
- ScriptedResponse(json={"currency_code": "EUR"}),
+ _insights_script(
_page(
[{"id": id_template.format(plan=plan), "start_time": 1777075200, "end_time": 1777161600}],
has_more=False,
- ),
- ],
+ )
+ ),
)
assert result.raised is None
ids.append(result.rows[0]["id"])
@@ -156,8 +166,6 @@ class TestInsights:
[
("recent_watermark", datetime(2026, 7, 1, tzinfo=UTC), "2026-07-01"),
("watermark_today", datetime(2026, 7, 21, tzinfo=UTC), "2026-07-20"),
- ("first_sync", None, "2025-07-22"),
- ("watermark_older_than_reporting_history", datetime(2024, 3, 1, tzinfo=UTC), "2025-07-22"),
]
)
@time_machine.travel("2026-07-21", tick=False)
@@ -178,21 +186,54 @@ class TestInsights:
assert result.raised is None
assert result.saved_states == [OpenAIAdsResumeConfig(cursor="b1", since=expected_since, until="2026-07-20")]
+ @parameterized.expand(
+ [
+ ("first_sync", None),
+ ("watermark_older_than_reporting_history", datetime(2024, 3, 1, tzinfo=UTC)),
+ ]
+ )
+ @time_machine.travel("2026-07-21", tick=False)
+ def test_long_range_is_read_in_contiguous_short_windows(self, _name: str, watermark: datetime | None) -> None:
+ result = _driver().run(
+ "campaign_insights",
+ _insights_script(),
+ db_incremental_field_last_value=watermark,
+ should_use_incremental_field=True,
+ )
+
+ assert result.raised is None
+ windows = [json.loads(request.param("time_ranges[]") or "null") for request in result.requests[1:]]
+ assert windows[0]["since"] == "2025-07-22"
+ assert windows[-1]["until"] == "2026-07-20"
+ for previous, current in zip(windows, windows[1:]):
+ assert date.fromisoformat(current["since"]) == date.fromisoformat(previous["until"]) + timedelta(days=1)
+ assert all(
+ (date.fromisoformat(window["until"]) - date.fromisoformat(window["since"])).days < 30 for window in windows
+ )
+
@time_machine.travel("2026-07-21", tick=False)
def test_resume_reuses_the_pinned_window_with_the_saved_cursor(self) -> None:
# A cursor is only valid for the result set it was issued for — recomputing `until` as a
# later day on resume would pair it with a different window.
- resume = OpenAIAdsResumeConfig(cursor="b1", since="2026-06-01", until="2026-07-19")
+ resume = OpenAIAdsResumeConfig(cursor="b1", since="2026-06-01", until="2026-07-18")
result = _driver().run(
"campaign_insights",
- [ScriptedResponse(json={"currency_code": "EUR"}), _page([], has_more=False)],
+ [
+ ScriptedResponse(json={"currency_code": "EUR"}),
+ _page([], has_more=False),
+ _page([], has_more=False),
+ ],
resume_state=resume,
)
assert result.raised is None
assert result.requests[1].param("after") == "b1"
- window = json.loads(result.requests[1].param("time_ranges[]") or "null")
- assert (window["since"], window["until"]) == ("2026-06-01", "2026-07-19")
+ windows = [json.loads(request.param("time_ranges[]") or "null") for request in result.requests[1:]]
+ assert [(window["since"], window["until"]) for window in windows] == [
+ ("2026-06-01", "2026-07-18"),
+ ("2026-07-19", "2026-07-20"),
+ ]
+ assert result.requests[2].param("after") is None
def test_missing_account_currency_stops_sync(self) -> None:
result = _driver().run("campaign_insights", [ScriptedResponse(json={"id": "account_example"})])
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads_source.py
index d2f0ab99700a..1b40888829f5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads_source.py
@@ -4,6 +4,7 @@ from parameterized import parameterized
from products.warehouse_sources.backend.facade.source_config import ReleaseStatus, SourceFieldInputConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+ RecordedRequest,
ScriptedResponse,
SourceDriver,
)
@@ -69,8 +70,14 @@ class TestOpenAIAdsSourceForPipeline:
self, endpoint: str, primary_keys: list[str], partition_keys: list[str], sort_mode: str
) -> None:
page = ScriptedResponse(json={"data": [], "has_more": False})
- script = [ScriptedResponse(json={"currency_code": "USD"}), page] if "insights" in endpoint else [page]
- result = SourceDriver(OpenAIAdsSource(), OpenAIAdsSourceConfig(api_key="k")).run(endpoint, script)
+
+ # A first sync of insights reads the currency, then the reporting history in several windows.
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if "insights" in endpoint and request.path == "/v1/ad_account":
+ return ScriptedResponse(json={"currency_code": "USD"})
+ return page
+
+ result = SourceDriver(OpenAIAdsSource(), OpenAIAdsSourceConfig(api_key="k")).run(endpoint, respond)
response = result.response
assert result.raised is None
assert response is not None