:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(impact_partner): narrow Actions date window on a 400 instead of failing the sync (#130465)
Tom Owersdfbcf693122a
diff --git a/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt b/posthog/test/repo_invariants/resume_state_after_yield_baseline.txtindex 172d5fe6c04b..9da8b195acaa 100644--- a/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt+++ b/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt@@ -209,7 +209,7 @@ impact/impact.py::_get_rows_fanoutimpact/impact.py::_get_rows_nestedimpact/impact.py::_get_rows_simpleimpact_partner/impact_partner.py::_get_rows_simple-impact_partner/impact_partner.py::_get_rows_windowed+impact_partner/impact_partner.py::_get_rows_windowed._runinfisical/infisical.py::_get_audit_log_rowsinfisical/infisical.py::_get_offset_paginated_rowsinngest/inngest.py::_get_event_rowsdiff --git a/products/warehouse_sources/sources/impact_partner/impact_partner.py b/products/warehouse_sources/sources/impact_partner/impact_partner.pyindex 622e1c2251e3..8843d766287b 100644--- a/products/warehouse_sources/sources/impact_partner/impact_partner.py+++ b/products/warehouse_sources/sources/impact_partner/impact_partner.py@@ -24,8 +24,10 @@ REQUEST_TIMEOUT_SECONDS = 60class ImpactPartnerResumeConfig:# The last page fetched. Re-fetched on resume (merge dedupes).page: Optional[int] = None- # Actions: ISO start of the date window currently being processed.+ # Actions: ISO bounds of the exact window being processed, narrower than the top-level grid+ # once a 400 has split it. A bookmark missing `window_end` (an older run) restarts instead.window_start: Optional[str] = None+ window_end: Optional[str] = Nonedef _get_session(account_sid: str, auth_token: str, api_version: str) -> requests.Session:@@ -105,6 +107,53 @@ def _iter_windows(start: datetime, end: datetime, max_days: int) -> Iterator[tupcursor = window_end+# Impact caps a date-windowed query at 10 pages, 400ing every page of a denser window regardless+# of which one is requested. Our 45-day span cap doesn't prevent that, so narrow and retry instead.+MIN_SPLIT_WINDOW = timedelta(days=1) # floor so an unrelated, persistent 400 still surfaces++[email protected](frozen=True)+class ActionsWindow:+ start: datetime+ end: datetime++[email protected](frozen=True)+class FetchedActionsPage:+ window: ActionsWindow+ page: int+ rows: list[dict[str, Any]]+++def _split_window(window: ActionsWindow) -> list[ActionsWindow]:+ midpoint = window.start + (window.end - window.start) / 2+ return [ActionsWindow(start=window.start, end=midpoint), ActionsWindow(start=midpoint, end=window.end)]+++def _fetch_actions_window(+ session: requests.Session,+ account_sid: str,+ config: ImpactPartnerEndpointConfig,+ window: ActionsWindow,+ logger: FilteringBoundLogger,+ start_param: str,+ end_param: str,+ start_page: int,+) -> Iterator[FetchedActionsPage]:+ """Yield a page at a time for `window`, narrowing it if Impact rejects it as too large."""+ params: dict[str, Any] = {start_param: _format_datetime(window.start), end_param: _format_datetime(window.end)}+ try:+ for page, rows in _paginate_endpoint(session, account_sid, config, params, logger, start_page=start_page):+ yield FetchedActionsPage(window=window, page=page, rows=rows)+ except requests.HTTPError as e:+ status = e.response.status_code if e.response is not None else None+ if status != 400 or (window.end - window.start) <= MIN_SPLIT_WINDOW:+ raise+ logger.debug(f"Impact Partner: narrowing a {window.end - window.start} Actions window after a 400")+ for half in _split_window(window):+ yield from _fetch_actions_window(session, account_sid, config, half, logger, start_param, end_param, 1)++def _paginate_endpoint(session: requests.Session,account_sid: str,@@ -145,20 +194,25 @@ def _windows_for_actions(return list(_iter_windows(start, now, MAX_WINDOW_DAYS))-def _resume_window_index(+def _resume_target(windows: list[tuple[datetime, datetime]], resume: Optional[ImpactPartnerResumeConfig]-) -> Optional[int]:- """Find which date window a saved bookmark points at.+) -> Optional[tuple[int, ActionsWindow]]:+ """Find the top-level window a saved bookmark falls inside, and the exact slice to resume.- Returns None when there's no bookmark or it no longer matches (window boundaries derive- from the current time, so they shift between runs), in which case the caller starts over- (merge dedupes the re-yielded rows).+ Matches by containment, since a 400 split can leave the bookmark narrower than the top-level+ window. Returns None when there's no bookmark, no `window_end`, or no window contains it+ (boundaries derive from the current time, so they shift between runs); the caller then starts+ over, which merge dedupes."""- if resume is None or resume.window_start is None:+ if resume is None or resume.window_start is None or resume.window_end is None:+ return None+ saved_start = _to_datetime(resume.window_start)+ saved_end = _to_datetime(resume.window_end)+ if saved_start is None or saved_end is None:return Nonefor index, window in enumerate(windows):- if window[0].isoformat() == resume.window_start:- return index+ if window[0] <= saved_start and saved_end <= window[1]:+ return index, ActionsWindow(start=saved_start, end=saved_end)return None@@ -206,29 +260,38 @@ def _get_rows_windowed(# so history is walked in date windows only, with no per-campaign fan-out.windows = _windows_for_actions(should_use_incremental_field, db_incremental_field_last_value)+ def _run(window: ActionsWindow, start_page: int) -> Iterator[list[dict[str, Any]]]:+ for fetched in _fetch_actions_window(+ session, account_sid, config, window, logger, start_param, end_param, start_page+ ):+ if fetched.rows:+ yield fetched.rows+ # Save after yielding: a crash re-fetches this same page and merge dedupes it.+ resumable_source_manager.save_state(+ ImpactPartnerResumeConfig(+ page=fetched.page,+ window_start=fetched.window.start.isoformat(),+ window_end=fetched.window.end.isoformat(),+ )+ )+resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None- resume_index = _resume_window_index(windows, resume)- start_index = resume_index if resume_index is not None else 0+ target = _resume_target(windows, resume)+ start_index = target[0] if target is not None else 0if start_index > 0:logger.debug(f"Impact Partner: resuming Actions from window {start_index}/{len(windows)}")for index in range(start_index, len(windows)):window = windows[index]- # Resume the bookmarked window at its saved page; every other window starts at page 1.- start_page = 1- if resume is not None and index == resume_index and resume.page:- start_page = resume.page- params: dict[str, Any] = {- start_param: _format_datetime(window[0]),- end_param: _format_datetime(window[1]),- }- for page, rows in _paginate_endpoint(session, account_sid, config, params, logger, start_page=start_page):- if rows:- yield rows- # Save after yielding: a crash re-fetches this same page and merge dedupes it.- resumable_source_manager.save_state(- ImpactPartnerResumeConfig(page=page, window_start=window[0].isoformat())- )+ if target is not None and index == target[0]:+ # Resume the exact bookmarked slice, which a 400 may have narrowed below this window.+ resumed_window = target[1]+ yield from _run(resumed_window, resume.page if resume and resume.page else 1)+ if resumed_window.end < window[1]:+ # A split leaves the rest of this window unfetched; get it before moving on.+ yield from _run(ActionsWindow(start=resumed_window.end, end=window[1]), 1)+ else:+ yield from _run(ActionsWindow(start=window[0], end=window[1]), 1)def get_rows(diff --git a/products/warehouse_sources/sources/impact_partner/tests/test_impact_partner.py b/products/warehouse_sources/sources/impact_partner/tests/test_impact_partner.pyindex 4d2a5ecee731..fa9166d7c07f 100644--- a/products/warehouse_sources/sources/impact_partner/tests/test_impact_partner.py+++ b/products/warehouse_sources/sources/impact_partner/tests/test_impact_partner.py@@ -3,6 +3,7 @@ from typing import Optionalimport time_machine+import requestsfrom parameterized import parameterizedfrom products.warehouse_sources.sources.impact_partner._config import ImpactPartnerSourceConfig@@ -110,7 +111,9 @@ class TestGetRowsActions:result = _driver().run("Actions",[ScriptedResponse(json={"Actions": [], "@numpages": "2"})],- resume_state=ImpactPartnerResumeConfig(page=2, window_start=windows[1][0].isoformat()),+ resume_state=ImpactPartnerResumeConfig(+ page=2, window_start=windows[1][0].isoformat(), window_end=windows[1][1].isoformat()+ ),should_use_incremental_field=True,db_incremental_field_last_value=cursor,)@@ -120,12 +123,25 @@ class TestGetRowsActions:assert result.params("ActionDateStart") == ["2026-05-16T00:00:00Z"]assert result.params("Page") == ["2"]+ @parameterized.expand(+ [+ ("no_window_end", ImpactPartnerResumeConfig(page=5, window_start="2026-05-31T00:00:00+00:00")),+ (+ "window_end_fits_no_window",+ ImpactPartnerResumeConfig(+ page=5, window_start="2020-01-01T00:00:00+00:00", window_end="2020-01-02T00:00:00+00:00"+ ),+ ),+ ]+ )@time_machine.travel("2026-06-01", tick=False)- def test_stale_bookmark_restarts_from_the_first_window(self) -> None:+ def test_unplaceable_bookmark_restarts_from_the_first_window(+ self, _name: str, resume_state: ImpactPartnerResumeConfig+ ) -> None:result = _driver().run("Actions",[ScriptedResponse(json={"Actions": [], "@numpages": "1"})],- resume_state=ImpactPartnerResumeConfig(page=5, window_start="2020-01-01T00:00:00+00:00"),+ resume_state=resume_state,should_use_incremental_field=True,db_incremental_field_last_value=datetime(2026, 5, 31, tzinfo=UTC),)@@ -134,6 +150,96 @@ class TestGetRowsActions:assert result.params("Page") == ["1"]+class TestActionsWindowNarrowing:+ @time_machine.travel("2026-06-01", tick=False)+ def test_a_400_narrows_the_window_instead_of_failing_the_sync(self) -> None:+ # A 12-day window: too short to split into more than one level before hitting real rows.+ result = _driver().run(+ "Actions",+ [+ ScriptedResponse(status=400),+ ScriptedResponse(json={"Actions": [{"Id": "a1"}], "@numpages": "1"}),+ ScriptedResponse(json={"Actions": [{"Id": "a2"}], "@numpages": "1"}),+ ],+ should_use_incremental_field=True,+ db_incremental_field_last_value=datetime(2026, 5, 20, tzinfo=UTC),+ )++ assert result.raised is None+ assert result.items == [[{"Id": "a1"}], [{"Id": "a2"}]]+ assert len(result.requests) == 3+ full_start, full_end = result.requests[0].param("ActionDateStart"), result.requests[0].param("ActionDateEnd")+ assert (full_start, full_end) == ("2026-05-20T00:00:00Z", "2026-06-01T00:00:00Z")+ first_half_end = result.requests[1].param("ActionDateEnd")+ assert result.requests[1].param("ActionDateStart") == full_start+ assert result.requests[2].param("ActionDateStart") == first_half_end+ assert result.requests[2].param("ActionDateEnd") == full_end+ assert full_start is not None+ assert first_half_end is not None+ # Resume state bookmarks the narrowed window actually fetched, not the original one.+ assert [state.window_start for state in result.saved_states] == [+ datetime.fromisoformat(full_start.replace("Z", "+00:00")).isoformat(),+ datetime.fromisoformat(first_half_end.replace("Z", "+00:00")).isoformat(),+ ]++ @time_machine.travel("2026-06-01", tick=False)+ def test_a_400_on_a_window_already_at_the_floor_still_raises(self) -> None:+ # A 12-hour window is already below the 1-day floor, so there's nothing left to narrow.+ result = _driver().run(+ "Actions",+ [ScriptedResponse(status=400)],+ should_use_incremental_field=True,+ db_incremental_field_last_value=datetime(2026, 5, 31, 12, tzinfo=UTC),+ )++ assert isinstance(result.raised, requests.HTTPError)+ assert len(result.requests) == 1++ @time_machine.travel("2026-06-01", tick=False)+ def test_resuming_the_second_half_does_not_refetch_the_first(self) -> None:+ # The second half is the one left bookmarked once the first has fully completed.+ bookmark = ImpactPartnerResumeConfig(+ page=1, window_start="2026-05-26T00:00:00+00:00", window_end="2026-06-01T00:00:00+00:00"+ )+ result = _driver().run(+ "Actions",+ [ScriptedResponse(json={"Actions": [{"Id": "a2"}], "@numpages": "1"})],+ resume_state=bookmark,+ should_use_incremental_field=True,+ db_incremental_field_last_value=datetime(2026, 5, 20, tzinfo=UTC),+ )++ assert result.raised is None+ assert len(result.requests) == 1+ assert result.requests[0].param("ActionDateStart") == "2026-05-26T00:00:00Z"+ assert result.requests[0].param("ActionDateEnd") == "2026-06-01T00:00:00Z"+ assert result.requests[0].param("Page") == "1"++ @time_machine.travel("2026-06-01", tick=False)+ def test_resuming_the_first_half_still_fetches_the_second_after_it(self) -> None:+ # Bookmarked mid-pagination, so resuming it must not skip the second half after it.+ bookmark = ImpactPartnerResumeConfig(+ page=1, window_start="2026-05-20T00:00:00+00:00", window_end="2026-05-26T00:00:00+00:00"+ )+ result = _driver().run(+ "Actions",+ [+ ScriptedResponse(json={"Actions": [{"Id": "b1"}], "@numpages": "2"}),+ ScriptedResponse(json={"Actions": [{"Id": "b2"}], "@numpages": "2"}),+ ScriptedResponse(json={"Actions": [{"Id": "b3"}], "@numpages": "1"}),+ ],+ resume_state=bookmark,+ should_use_incremental_field=True,+ db_incremental_field_last_value=datetime(2026, 5, 20, tzinfo=UTC),+ )++ assert result.raised is None+ assert result.items == [[{"Id": "b1"}], [{"Id": "b2"}], [{"Id": "b3"}]]+ assert result.params("ActionDateStart") == ["2026-05-20T00:00:00Z"] * 2 + ["2026-05-26T00:00:00Z"]+ assert result.params("ActionDateEnd") == ["2026-05-26T00:00:00Z"] * 2 + ["2026-06-01T00:00:00Z"]+ assert result.params("Page") == ["1", "2", "1"]++class TestImpactPartnerSourceResponse:@parameterized.expand([