Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(openai-ads): start insights history at the oldest campaign (#126760)

Tom Owers496a750bbca4
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/openai_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/openai_ads.py
index 284181cc7da9..bdccada10182 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/openai_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/openai_ads.py
@@ -190,19 +190,23 @@ def _to_utc_date(value: Any) -> date:
return parser.parse(str(value)).astimezone(UTC).date()
-def _insights_window(db_incremental_field_last_value: Optional[Any]) -> SyncWindow[str]:
+def _insights_window(
+ db_incremental_field_last_value: Optional[Any], history_start: date | None = None
+) -> SyncWindow[str]:
"""The [since, until] ISO date window for one insights sync.
Incremental runs start at the watermark (the pipeline already rewinds it by the configured
- lookback); full refreshes start at the oldest day the API still reports. A watermark older than
- that day is moved up to it too. `until` is yesterday: the API reads a `date_range` `until` as
- inclusive and moves it to the next midnight, and it rejects a bound in the future with a 400. So
- `until` set to today asks for a range that ends tomorrow, and every request fails. The trailing
- lookback re-reads the day once it is complete.
+ lookback); full refreshes start at the oldest day the API still reports, or at `history_start`
+ when that is later. A watermark older than that day is moved up to it too. `until` is yesterday:
+ the API reads a `date_range` `until` as inclusive and moves it to the next midnight, and it
+ rejects a bound in the future with a 400. So `until` set to today asks for a range that ends
+ tomorrow, and every request fails. The trailing lookback re-reads the day once it is complete.
"""
today = datetime.now(tz=UTC).date()
until = today - timedelta(days=1)
earliest = today - timedelta(days=INSIGHTS_HISTORY_DAYS)
+ if history_start is not None:
+ earliest = max(earliest, history_start)
since = earliest
if db_incremental_field_last_value is not None:
since = max(_to_utc_date(db_incremental_field_last_value), earliest)
@@ -265,6 +269,28 @@ def _account_currency(api_key: str) -> str:
raise ValueError("OpenAI Ads did not return an account currency. Check the account settings, then sync again.")
+def _oldest_campaign_date(api_key: str) -> date | None:
+ """The day the account's oldest campaign was created, so a first sync can start there.
+
+ OpenAI Ads answers an insights range from before the account ran any campaign with a
+ `server_error`, however short the range is, so a first sync that starts a year back fails on
+ its first window every time. An account with no campaign yet has nothing to report, so the
+ window starts at today and shrinks to one day. None keeps the full reporting history.
+ """
+ for page in _retry_forbidden_server_errors(
+ _make_client(api_key).paginate(
+ path="/v1/campaigns",
+ params={"limit": 1, "order": "asc"},
+ paginator=SinglePagePaginator(),
+ data_selector="data",
+ )
+ ):
+ for campaign in page:
+ created_at = campaign.get("created_at")
+ return _to_utc_date(created_at) if created_at is not None else None
+ return datetime.now(tz=UTC).date()
+
+
def _list_pages(client: RESTClient, path: str, params: dict[str, Any]) -> Iterator[list[dict[str, Any]]]:
yield from client.paginate(
path=path,
@@ -411,16 +437,17 @@ def _insights_source(
resume: OpenAIAdsResumeConfig | None,
db_incremental_field_last_value: Optional[Any],
) -> SourceResponse:
- full_window = _insights_window(db_incremental_field_last_value)
+ currency = _account_currency(api_key)
+
if resume is not None and resume.since and resume.until:
+ full_window = _insights_window(db_incremental_field_last_value)
windows = [SyncWindow(start=resume.since, end=resume.until)]
next_start = date.fromisoformat(resume.until) + timedelta(days=1)
if next_start <= date.fromisoformat(full_window.end):
windows += _split_insights_window(SyncWindow(start=next_start.isoformat(), end=full_window.end))
else:
- windows = _split_insights_window(full_window)
-
- currency = _account_currency(api_key)
+ history_start = _oldest_campaign_date(api_key) if db_incremental_field_last_value is None else None
+ windows = _split_insights_window(_insights_window(db_incremental_field_last_value, history_start))
def enrich_insights(row: dict[str, Any]) -> dict[str, Any]:
return {**_convert_insights_times(_stable_insights_id(row)), "currency_code": currency}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads.py
index 82c17bfb0ca6..c0bded4e1f64 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads.py
@@ -36,18 +36,34 @@ def _page(items: list[dict[str, Any]], *, has_more: bool, last_id: str | None =
return ScriptedResponse(json=body)
-def _insights_script(*pages: ScriptedResponse) -> Callable[[RecordedRequest], ScriptedResponse]:
+# 2024-01-01, older than the reporting history in every test.
+_OLD_CAMPAIGN = {"id": "cmpn_1", "created_at": 1704067200}
+
+
+def _insights_script(
+ *pages: ScriptedResponse, campaigns: list[dict[str, Any]] | None = None
+) -> Callable[[RecordedRequest], ScriptedResponse]:
# A first sync reads the reporting history in several windows, and the windows after `pages` are empty.
remaining = list(pages)
def respond(request: RecordedRequest) -> ScriptedResponse:
if request.path == "/v1/ad_account":
return ScriptedResponse(json={"currency_code": "EUR"})
+ if request.path == "/v1/campaigns":
+ return _page([_OLD_CAMPAIGN] if campaigns is None else campaigns, has_more=False)
return remaining.pop(0) if remaining else _page([], has_more=False)
return respond
+def _requested_windows(requests: list[RecordedRequest]) -> list[dict[str, Any]]:
+ return [
+ json.loads(request.param("time_ranges[]") or "null")
+ for request in requests
+ if request.path == "/v1/ad_account/insights"
+ ]
+
+
class TestFanOut:
def test_ads_walk_both_levels_and_carry_full_lineage(self) -> None:
result = _driver().run(
@@ -127,7 +143,7 @@ class TestInsights:
)
assert result.raised is None
- insights_request = result.requests[1]
+ insights_request = result.requests[2]
assert insights_request.path == "/v1/ad_account/insights"
assert insights_request.param("aggregation_level") == "campaign"
assert insights_request.query.get("segments[]") == ((segment,) if segment else None)
@@ -202,7 +218,7 @@ class TestInsights:
)
assert result.raised is None
- windows = [json.loads(request.param("time_ranges[]") or "null") for request in result.requests[1:]]
+ windows = _requested_windows(result.requests)
assert windows[0]["since"] == "2025-07-22"
assert windows[-1]["until"] == "2026-07-20"
for previous, current in zip(windows, windows[1:]):
@@ -211,6 +227,25 @@ class TestInsights:
(date.fromisoformat(window["until"]) - date.fromisoformat(window["since"])).days < 30 for window in windows
)
+ @parameterized.expand(
+ [
+ ("campaign_inside_history", [{"id": "cmpn_1", "created_at": 1777075200}], "2026-04-25"),
+ ("iso_created_at", [{"id": "cmpn_1", "created_at": "2026-04-25T10:00:00Z"}], "2026-04-25"),
+ ("no_campaign_yet", [], "2026-07-20"),
+ ("campaign_without_created_at", [{"id": "cmpn_1"}], "2025-07-22"),
+ ]
+ )
+ @time_machine.travel("2026-07-21", tick=False)
+ def test_first_sync_starts_at_the_oldest_campaign(
+ self, _name: str, campaigns: list[dict[str, Any]], expected_since: str
+ ) -> None:
+ result = _driver().run("campaign_insights", _insights_script(campaigns=campaigns))
+
+ assert result.raised is None
+ windows = _requested_windows(result.requests)
+ assert windows[0]["since"] == expected_since
+ assert windows[-1]["until"] == "2026-07-20"
+
@time_machine.travel("2026-07-21", tick=False)
def test_resume_reuses_the_pinned_window_with_the_saved_cursor(self) -> None:
# A cursor is only valid for the result set it was issued for — recomputing `until` as a