:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(openai-ads): read the insights history in 30-day windows (#126208)
Tom Owers20568f5bc9b7
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/openai_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/openai_ads.pyindex 66fdc5dae4a4..284181cc7da9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/openai_ads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/openai_ads.py@@ -18,6 +18,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resBasePaginator,SinglePagePaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClientfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe@@ -59,6 +60,11 @@ _T = TypeVar("_T")# because the API reads dates in the ad account's timezone and this source sends them in UTC.INSIGHTS_HISTORY_DAYS = 364+# A first sync or full refresh asks for most of a year of daily buckets. OpenAI Ads can answer that+# request with a `server_error` on every run, while the few days an incremental run asks for read+# fine. So a long range is read in windows of at most this many days.+INSIGHTS_WINDOW_DAYS = 30+@dataclasses.dataclassclass OpenAIAdsResumeConfig:@@ -204,6 +210,18 @@ def _insights_window(db_incremental_field_last_value: Optional[Any]) -> SyncWindreturn SyncWindow(start=since.isoformat(), end=until.isoformat())+def _split_insights_window(window: SyncWindow[str]) -> list[SyncWindow[str]]:+ start = date.fromisoformat(window.start)+ end = date.fromisoformat(window.end)+ windows: list[SyncWindow[str]] = []+ while True:+ last = min(start + timedelta(days=INSIGHTS_WINDOW_DAYS - 1), end)+ windows.append(SyncWindow(start=start.isoformat(), end=last.isoformat()))+ if last >= end:+ return windows+ start = last + timedelta(days=1)++def _convert_insights_times(row: dict[str, Any]) -> dict[str, Any]:# Bucket bounds arrive as epoch seconds; real timestamps are needed for the DateTime# incremental watermark and datetime partitioning.@@ -319,42 +337,50 @@ def openai_ads_source(can_resume = resumable_source_manager.can_resume() and not config.single_objectresume = resumable_source_manager.load_state() if can_resume else None- data_map: Callable[[dict[str, Any]], dict[str, Any]] | None = Noneif config.aggregation_level is not None:- if resume is not None and resume.since and resume.until:- window: SyncWindow[str] = SyncWindow(start=resume.since, end=resume.until)- else:- window = _insights_window(db_incremental_field_last_value)- params: dict[str, Any] = {- "aggregation_level": config.aggregation_level,- "time_granularity": "daily",- "limit": INSIGHTS_PAGE_SIZE,- # `requests` encodes the list as one repeated fields[] query param per element.- "fields[]": list(config.insights_fields),- # One JSON-encoded time-range object; the explicit timezone keeps daily buckets (and- # therefore the bucket ids merge dedupes on) stable across runs.- "time_ranges[]": json.dumps(- {"type": "date_range", "since": window.start, "until": window.end, "timezone": "UTC"}- ),- }- if config.segment is not None:- params["segments[]"] = config.segment- currency = _account_currency(api_key)+ return _insights_source(+ config, api_key, team_id, job_id, resumable_source_manager, resume, db_incremental_field_last_value+ )- def enrich_insights(row: dict[str, Any]) -> dict[str, Any]:- return {**_convert_insights_times(_stable_insights_id(row)), "currency_code": currency}-- data_map = enrich_insights- elif config.single_object:- window = SyncWindow(start="", end="")- params = {}+ if config.single_object:+ params: dict[str, Any] = {}else:- window = SyncWindow(start="", end="")# An explicit stable sort prevents page-boundary skips/duplicates while paginating the# full campaign list.params = {"limit": LIST_PAGE_SIZE, "order": "asc"}- rest_config: RESTAPIConfig = {+ initial_paginator_state: Optional[dict[str, Any]] = None+ if resume is not None and resume.cursor:+ initial_paginator_state = {"cursor": resume.cursor}++ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:+ # Persist only while a next page remains; the checkpoint is saved AFTER a page is yielded,+ # pointing at the next page, so a crash resumes from a page whose predecessors were all+ # yielded — the overlap merge dedupes on the primary key.+ if state and state.get("cursor"):+ resumable_source_manager.save_state(OpenAIAdsResumeConfig(cursor=state["cursor"]))++ resource = rest_api_resource(+ _rest_config(config, api_key, params, None),+ team_id,+ job_id,+ db_incremental_field_last_value,+ resume_hook=save_checkpoint,+ initial_paginator_state=initial_paginator_state,+ )++ return _source_response(+ config, lambda: _retry_forbidden_server_errors(resource), column_hints=resource.column_hints+ )+++def _rest_config(+ config: OpenAIAdsEndpointConfig,+ api_key: str,+ params: dict[str, Any],+ data_map: Callable[[dict[str, Any]], dict[str, Any]] | None,+) -> RESTAPIConfig:+ return {"client": {"base_url": OPENAI_ADS_BASE_URL,"headers": _headers(),@@ -363,7 +389,7 @@ def openai_ads_source("resource_defaults": None,"resources": [{- "name": endpoint,+ "name": config.name,"endpoint": {"path": config.path,"params": params,@@ -375,29 +401,82 @@ def openai_ads_source(],}- initial_paginator_state: Optional[dict[str, Any]] = None- if resume is not None and resume.cursor:- initial_paginator_state = {"cursor": resume.cursor}- def save_checkpoint(state: Optional[dict[str, Any]]) -> None:- # Persist only while a next page remains; the checkpoint is saved AFTER a page is yielded,- # pointing at the next page, so a crash resumes from a page whose predecessors were all- # yielded — the overlap merge dedupes on the primary key. Insights pin their window so the- # resumed cursor pairs with the result set it was issued for.- if state and state.get("cursor"):- resumable_source_manager.save_state(- OpenAIAdsResumeConfig(cursor=state["cursor"], since=window.start or None, until=window.end or None)- )+def _insights_source(+ config: OpenAIAdsEndpointConfig,+ api_key: str,+ team_id: int,+ job_id: str,+ resumable_source_manager: ResumableSourceManager[OpenAIAdsResumeConfig],+ resume: OpenAIAdsResumeConfig | None,+ db_incremental_field_last_value: Optional[Any],+) -> SourceResponse:+ full_window = _insights_window(db_incremental_field_last_value)+ if resume is not None and resume.since and resume.until:+ windows = [SyncWindow(start=resume.since, end=resume.until)]+ next_start = date.fromisoformat(resume.until) + timedelta(days=1)+ if next_start <= date.fromisoformat(full_window.end):+ windows += _split_insights_window(SyncWindow(start=next_start.isoformat(), end=full_window.end))+ else:+ windows = _split_insights_window(full_window)- resource = rest_api_resource(- rest_config,- team_id,- job_id,- db_incremental_field_last_value,- resume_hook=save_checkpoint,- initial_paginator_state=initial_paginator_state,- )+ currency = _account_currency(api_key)++ def enrich_insights(row: dict[str, Any]) -> dict[str, Any]:+ return {**_convert_insights_times(_stable_insights_id(row)), "currency_code": currency}++ def window_resource(window: SyncWindow[str], cursor: str | None) -> Resource:+ params: dict[str, Any] = {+ "aggregation_level": config.aggregation_level,+ "time_granularity": "daily",+ "limit": INSIGHTS_PAGE_SIZE,+ # `requests` encodes the list as one repeated fields[] query param per element.+ "fields[]": list(config.insights_fields),+ # One JSON-encoded time-range object; the explicit timezone keeps daily buckets (and+ # therefore the bucket ids merge dedupes on) stable across runs.+ "time_ranges[]": json.dumps(+ {"type": "date_range", "since": window.start, "until": window.end, "timezone": "UTC"}+ ),+ }+ if config.segment is not None:+ params["segments[]"] = config.segment++ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:+ # Persist only while a next page remains; the checkpoint is saved AFTER a page is+ # yielded, pointing at the next page, so a crash resumes from a page whose predecessors+ # were all yielded — the overlap merge dedupes on the primary key. The window is pinned+ # with the cursor so the resumed cursor pairs with the result set it was issued for.+ if state and state.get("cursor"):+ resumable_source_manager.save_state(+ OpenAIAdsResumeConfig(cursor=state["cursor"], since=window.start, until=window.end)+ )++ return rest_api_resource(+ _rest_config(config, api_key, params, enrich_insights),+ team_id,+ job_id,+ db_incremental_field_last_value,+ resume_hook=save_checkpoint,+ initial_paginator_state={"cursor": cursor} if cursor else None,+ )++ first_resource = window_resource(windows[0], resume.cursor if resume is not None else None)++ def pages() -> Iterator[Any]:+ for index, window in enumerate(windows):+ resource = first_resource+ if index > 0:+ # Every row of the earlier windows is already yielded, so a resume can start here.+ resumable_source_manager.save_state(+ OpenAIAdsResumeConfig(cursor=None, since=window.start, until=window.end)+ )+ resumable_source_manager.safe_point()+ resource = window_resource(window, None)+ # This wrapper hides the framework's own safe points, and a window can hold no rows.+ for page in resource:+ yield page+ resumable_source_manager.safe_point()return _source_response(- config, lambda: _retry_forbidden_server_errors(resource), column_hints=resource.column_hints+ config, lambda: _retry_forbidden_server_errors(pages()), column_hints=first_resource.column_hints)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads.pyindex 3a27f21e2166..82c17bfb0ca6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads.py@@ -1,5 +1,6 @@import json-from datetime import UTC, datetime+from collections.abc import Callable+from datetime import UTC, date, datetime, timedeltafrom typing import Anyimport time_machine@@ -8,6 +9,7 @@ from parameterized import parameterizedfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.base import error_message_matchesfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (+ RecordedRequest,ScriptedResponse,SourceDriver,always,@@ -34,6 +36,18 @@ def _page(items: list[dict[str, Any]], *, has_more: bool, last_id: str | None =return ScriptedResponse(json=body)+def _insights_script(*pages: ScriptedResponse) -> Callable[[RecordedRequest], ScriptedResponse]:+ # A first sync reads the reporting history in several windows, and the windows after `pages` are empty.+ remaining = list(pages)++ def respond(request: RecordedRequest) -> ScriptedResponse:+ if request.path == "/v1/ad_account":+ return ScriptedResponse(json={"currency_code": "EUR"})+ return remaining.pop(0) if remaining else _page([], has_more=False)++ return respond++class TestFanOut:def test_ads_walk_both_levels_and_carry_full_lineage(self) -> None:result = _driver().run(@@ -109,10 +123,7 @@ class TestInsights:) -> None:result = _driver().run(endpoint,- [- ScriptedResponse(json={"currency_code": "EUR"}),- _page([{"id": "b1", "start_time": 1777075200, "end_time": 1777161600}], has_more=False),- ],+ _insights_script(_page([{"id": "b1", "start_time": 1777075200, "end_time": 1777161600}], has_more=False)),)assert result.raised is None@@ -139,13 +150,12 @@ class TestInsights:for plan in ("plan=eJzLSM3JyVcozy/KSQEAGgQEXQ", "plan=eJwrSS0uAQAEXQHB"):result = _driver().run("campaign_insights",- [- ScriptedResponse(json={"currency_code": "EUR"}),+ _insights_script(_page([{"id": id_template.format(plan=plan), "start_time": 1777075200, "end_time": 1777161600}],has_more=False,- ),- ],+ )+ ),)assert result.raised is Noneids.append(result.rows[0]["id"])@@ -156,8 +166,6 @@ class TestInsights:[("recent_watermark", datetime(2026, 7, 1, tzinfo=UTC), "2026-07-01"),("watermark_today", datetime(2026, 7, 21, tzinfo=UTC), "2026-07-20"),- ("first_sync", None, "2025-07-22"),- ("watermark_older_than_reporting_history", datetime(2024, 3, 1, tzinfo=UTC), "2025-07-22"),])@time_machine.travel("2026-07-21", tick=False)@@ -178,21 +186,54 @@ class TestInsights:assert result.raised is Noneassert result.saved_states == [OpenAIAdsResumeConfig(cursor="b1", since=expected_since, until="2026-07-20")]+ @parameterized.expand(+ [+ ("first_sync", None),+ ("watermark_older_than_reporting_history", datetime(2024, 3, 1, tzinfo=UTC)),+ ]+ )+ @time_machine.travel("2026-07-21", tick=False)+ def test_long_range_is_read_in_contiguous_short_windows(self, _name: str, watermark: datetime | None) -> None:+ result = _driver().run(+ "campaign_insights",+ _insights_script(),+ db_incremental_field_last_value=watermark,+ should_use_incremental_field=True,+ )++ assert result.raised is None+ windows = [json.loads(request.param("time_ranges[]") or "null") for request in result.requests[1:]]+ assert windows[0]["since"] == "2025-07-22"+ assert windows[-1]["until"] == "2026-07-20"+ for previous, current in zip(windows, windows[1:]):+ assert date.fromisoformat(current["since"]) == date.fromisoformat(previous["until"]) + timedelta(days=1)+ assert all(+ (date.fromisoformat(window["until"]) - date.fromisoformat(window["since"])).days < 30 for window in windows+ )+@time_machine.travel("2026-07-21", tick=False)def test_resume_reuses_the_pinned_window_with_the_saved_cursor(self) -> None:# A cursor is only valid for the result set it was issued for — recomputing `until` as a# later day on resume would pair it with a different window.- resume = OpenAIAdsResumeConfig(cursor="b1", since="2026-06-01", until="2026-07-19")+ resume = OpenAIAdsResumeConfig(cursor="b1", since="2026-06-01", until="2026-07-18")result = _driver().run("campaign_insights",- [ScriptedResponse(json={"currency_code": "EUR"}), _page([], has_more=False)],+ [+ ScriptedResponse(json={"currency_code": "EUR"}),+ _page([], has_more=False),+ _page([], has_more=False),+ ],resume_state=resume,)assert result.raised is Noneassert result.requests[1].param("after") == "b1"- window = json.loads(result.requests[1].param("time_ranges[]") or "null")- assert (window["since"], window["until"]) == ("2026-06-01", "2026-07-19")+ windows = [json.loads(request.param("time_ranges[]") or "null") for request in result.requests[1:]]+ assert [(window["since"], window["until"]) for window in windows] == [+ ("2026-06-01", "2026-07-18"),+ ("2026-07-19", "2026-07-20"),+ ]+ assert result.requests[2].param("after") is Nonedef test_missing_account_currency_stops_sync(self) -> None:result = _driver().run("campaign_insights", [ScriptedResponse(json={"id": "account_example"})])diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads_source.pyindex d2f0ab99700a..1b40888829f5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads_source.py@@ -4,6 +4,7 @@ from parameterized import parameterizedfrom products.warehouse_sources.backend.facade.source_config import ReleaseStatus, SourceFieldInputConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (+ RecordedRequest,ScriptedResponse,SourceDriver,)@@ -69,8 +70,14 @@ class TestOpenAIAdsSourceForPipeline:self, endpoint: str, primary_keys: list[str], partition_keys: list[str], sort_mode: str) -> None:page = ScriptedResponse(json={"data": [], "has_more": False})- script = [ScriptedResponse(json={"currency_code": "USD"}), page] if "insights" in endpoint else [page]- result = SourceDriver(OpenAIAdsSource(), OpenAIAdsSourceConfig(api_key="k")).run(endpoint, script)++ # A first sync of insights reads the currency, then the reporting history in several windows.+ def respond(request: RecordedRequest) -> ScriptedResponse:+ if "insights" in endpoint and request.path == "/v1/ad_account":+ return ScriptedResponse(json={"currency_code": "USD"})+ return page++ result = SourceDriver(OpenAIAdsSource(), OpenAIAdsSourceConfig(api_key="k")).run(endpoint, respond)response = result.responseassert result.raised is Noneassert response is not None