:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(openai-ads): start insights history at the oldest campaign (#126760)
Tom Owers496a750bbca4
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/openai_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/openai_ads.pyindex 284181cc7da9..bdccada10182 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/openai_ads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/openai_ads.py@@ -190,19 +190,23 @@ def _to_utc_date(value: Any) -> date:return parser.parse(str(value)).astimezone(UTC).date()-def _insights_window(db_incremental_field_last_value: Optional[Any]) -> SyncWindow[str]:+def _insights_window(+ db_incremental_field_last_value: Optional[Any], history_start: date | None = None+) -> SyncWindow[str]:"""The [since, until] ISO date window for one insights sync.Incremental runs start at the watermark (the pipeline already rewinds it by the configured- lookback); full refreshes start at the oldest day the API still reports. A watermark older than- that day is moved up to it too. `until` is yesterday: the API reads a `date_range` `until` as- inclusive and moves it to the next midnight, and it rejects a bound in the future with a 400. So- `until` set to today asks for a range that ends tomorrow, and every request fails. The trailing- lookback re-reads the day once it is complete.+ lookback); full refreshes start at the oldest day the API still reports, or at `history_start`+ when that is later. A watermark older than that day is moved up to it too. `until` is yesterday:+ the API reads a `date_range` `until` as inclusive and moves it to the next midnight, and it+ rejects a bound in the future with a 400. So `until` set to today asks for a range that ends+ tomorrow, and every request fails. The trailing lookback re-reads the day once it is complete."""today = datetime.now(tz=UTC).date()until = today - timedelta(days=1)earliest = today - timedelta(days=INSIGHTS_HISTORY_DAYS)+ if history_start is not None:+ earliest = max(earliest, history_start)since = earliestif db_incremental_field_last_value is not None:since = max(_to_utc_date(db_incremental_field_last_value), earliest)@@ -265,6 +269,28 @@ def _account_currency(api_key: str) -> str:raise ValueError("OpenAI Ads did not return an account currency. Check the account settings, then sync again.")+def _oldest_campaign_date(api_key: str) -> date | None:+ """The day the account's oldest campaign was created, so a first sync can start there.++ OpenAI Ads answers an insights range from before the account ran any campaign with a+ `server_error`, however short the range is, so a first sync that starts a year back fails on+ its first window every time. An account with no campaign yet has nothing to report, so the+ window starts at today and shrinks to one day. None keeps the full reporting history.+ """+ for page in _retry_forbidden_server_errors(+ _make_client(api_key).paginate(+ path="/v1/campaigns",+ params={"limit": 1, "order": "asc"},+ paginator=SinglePagePaginator(),+ data_selector="data",+ )+ ):+ for campaign in page:+ created_at = campaign.get("created_at")+ return _to_utc_date(created_at) if created_at is not None else None+ return datetime.now(tz=UTC).date()++def _list_pages(client: RESTClient, path: str, params: dict[str, Any]) -> Iterator[list[dict[str, Any]]]:yield from client.paginate(path=path,@@ -411,16 +437,17 @@ def _insights_source(resume: OpenAIAdsResumeConfig | None,db_incremental_field_last_value: Optional[Any],) -> SourceResponse:- full_window = _insights_window(db_incremental_field_last_value)+ currency = _account_currency(api_key)+if resume is not None and resume.since and resume.until:+ full_window = _insights_window(db_incremental_field_last_value)windows = [SyncWindow(start=resume.since, end=resume.until)]next_start = date.fromisoformat(resume.until) + timedelta(days=1)if next_start <= date.fromisoformat(full_window.end):windows += _split_insights_window(SyncWindow(start=next_start.isoformat(), end=full_window.end))else:- windows = _split_insights_window(full_window)-- currency = _account_currency(api_key)+ history_start = _oldest_campaign_date(api_key) if db_incremental_field_last_value is None else None+ windows = _split_insights_window(_insights_window(db_incremental_field_last_value, history_start))def enrich_insights(row: dict[str, Any]) -> dict[str, Any]:return {**_convert_insights_times(_stable_insights_id(row)), "currency_code": currency}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads.pyindex 82c17bfb0ca6..c0bded4e1f64 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads.py@@ -36,18 +36,34 @@ def _page(items: list[dict[str, Any]], *, has_more: bool, last_id: str | None =return ScriptedResponse(json=body)-def _insights_script(*pages: ScriptedResponse) -> Callable[[RecordedRequest], ScriptedResponse]:+# 2024-01-01, older than the reporting history in every test.+_OLD_CAMPAIGN = {"id": "cmpn_1", "created_at": 1704067200}+++def _insights_script(+ *pages: ScriptedResponse, campaigns: list[dict[str, Any]] | None = None+) -> Callable[[RecordedRequest], ScriptedResponse]:# A first sync reads the reporting history in several windows, and the windows after `pages` are empty.remaining = list(pages)def respond(request: RecordedRequest) -> ScriptedResponse:if request.path == "/v1/ad_account":return ScriptedResponse(json={"currency_code": "EUR"})+ if request.path == "/v1/campaigns":+ return _page([_OLD_CAMPAIGN] if campaigns is None else campaigns, has_more=False)return remaining.pop(0) if remaining else _page([], has_more=False)return respond+def _requested_windows(requests: list[RecordedRequest]) -> list[dict[str, Any]]:+ return [+ json.loads(request.param("time_ranges[]") or "null")+ for request in requests+ if request.path == "/v1/ad_account/insights"+ ]++class TestFanOut:def test_ads_walk_both_levels_and_carry_full_lineage(self) -> None:result = _driver().run(@@ -127,7 +143,7 @@ class TestInsights:)assert result.raised is None- insights_request = result.requests[1]+ insights_request = result.requests[2]assert insights_request.path == "/v1/ad_account/insights"assert insights_request.param("aggregation_level") == "campaign"assert insights_request.query.get("segments[]") == ((segment,) if segment else None)@@ -202,7 +218,7 @@ class TestInsights:)assert result.raised is None- windows = [json.loads(request.param("time_ranges[]") or "null") for request in result.requests[1:]]+ windows = _requested_windows(result.requests)assert windows[0]["since"] == "2025-07-22"assert windows[-1]["until"] == "2026-07-20"for previous, current in zip(windows, windows[1:]):@@ -211,6 +227,25 @@ class TestInsights:(date.fromisoformat(window["until"]) - date.fromisoformat(window["since"])).days < 30 for window in windows)+ @parameterized.expand(+ [+ ("campaign_inside_history", [{"id": "cmpn_1", "created_at": 1777075200}], "2026-04-25"),+ ("iso_created_at", [{"id": "cmpn_1", "created_at": "2026-04-25T10:00:00Z"}], "2026-04-25"),+ ("no_campaign_yet", [], "2026-07-20"),+ ("campaign_without_created_at", [{"id": "cmpn_1"}], "2025-07-22"),+ ]+ )+ @time_machine.travel("2026-07-21", tick=False)+ def test_first_sync_starts_at_the_oldest_campaign(+ self, _name: str, campaigns: list[dict[str, Any]], expected_since: str+ ) -> None:+ result = _driver().run("campaign_insights", _insights_script(campaigns=campaigns))++ assert result.raised is None+ windows = _requested_windows(result.requests)+ assert windows[0]["since"] == expected_since+ assert windows[-1]["until"] == "2026-07-20"+@time_machine.travel("2026-07-21", tick=False)def test_resume_reuses_the_pinned_window_with_the_saved_cursor(self) -> None:# A cursor is only valid for the result set it was issued for — recomputing `until` as a