Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(impact_partner): narrow Actions date window on a 400 instead of failing the sync (#130465)

Tom Owersdfbcf693122a
diff --git a/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt b/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt
index 172d5fe6c04b..9da8b195acaa 100644
--- a/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt
+++ b/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt
@@ -209,7 +209,7 @@ impact/impact.py::_get_rows_fanout
impact/impact.py::_get_rows_nested
impact/impact.py::_get_rows_simple
impact_partner/impact_partner.py::_get_rows_simple
-impact_partner/impact_partner.py::_get_rows_windowed
+impact_partner/impact_partner.py::_get_rows_windowed._run
infisical/infisical.py::_get_audit_log_rows
infisical/infisical.py::_get_offset_paginated_rows
inngest/inngest.py::_get_event_rows
diff --git a/products/warehouse_sources/sources/impact_partner/impact_partner.py b/products/warehouse_sources/sources/impact_partner/impact_partner.py
index 622e1c2251e3..8843d766287b 100644
--- a/products/warehouse_sources/sources/impact_partner/impact_partner.py
+++ b/products/warehouse_sources/sources/impact_partner/impact_partner.py
@@ -24,8 +24,10 @@ REQUEST_TIMEOUT_SECONDS = 60
class ImpactPartnerResumeConfig:
# The last page fetched. Re-fetched on resume (merge dedupes).
page: Optional[int] = None
- # Actions: ISO start of the date window currently being processed.
+ # Actions: ISO bounds of the exact window being processed, narrower than the top-level grid
+ # once a 400 has split it. A bookmark missing `window_end` (an older run) restarts instead.
window_start: Optional[str] = None
+ window_end: Optional[str] = None
def _get_session(account_sid: str, auth_token: str, api_version: str) -> requests.Session:
@@ -105,6 +107,53 @@ def _iter_windows(start: datetime, end: datetime, max_days: int) -> Iterator[tup
cursor = window_end
+# Impact caps a date-windowed query at 10 pages, 400ing every page of a denser window regardless
+# of which one is requested. Our 45-day span cap doesn't prevent that, so narrow and retry instead.
+MIN_SPLIT_WINDOW = timedelta(days=1) # floor so an unrelated, persistent 400 still surfaces
+
+
[email protected](frozen=True)
+class ActionsWindow:
+ start: datetime
+ end: datetime
+
+
[email protected](frozen=True)
+class FetchedActionsPage:
+ window: ActionsWindow
+ page: int
+ rows: list[dict[str, Any]]
+
+
+def _split_window(window: ActionsWindow) -> list[ActionsWindow]:
+ midpoint = window.start + (window.end - window.start) / 2
+ return [ActionsWindow(start=window.start, end=midpoint), ActionsWindow(start=midpoint, end=window.end)]
+
+
+def _fetch_actions_window(
+ session: requests.Session,
+ account_sid: str,
+ config: ImpactPartnerEndpointConfig,
+ window: ActionsWindow,
+ logger: FilteringBoundLogger,
+ start_param: str,
+ end_param: str,
+ start_page: int,
+) -> Iterator[FetchedActionsPage]:
+ """Yield a page at a time for `window`, narrowing it if Impact rejects it as too large."""
+ params: dict[str, Any] = {start_param: _format_datetime(window.start), end_param: _format_datetime(window.end)}
+ try:
+ for page, rows in _paginate_endpoint(session, account_sid, config, params, logger, start_page=start_page):
+ yield FetchedActionsPage(window=window, page=page, rows=rows)
+ except requests.HTTPError as e:
+ status = e.response.status_code if e.response is not None else None
+ if status != 400 or (window.end - window.start) <= MIN_SPLIT_WINDOW:
+ raise
+ logger.debug(f"Impact Partner: narrowing a {window.end - window.start} Actions window after a 400")
+ for half in _split_window(window):
+ yield from _fetch_actions_window(session, account_sid, config, half, logger, start_param, end_param, 1)
+
+
def _paginate_endpoint(
session: requests.Session,
account_sid: str,
@@ -145,20 +194,25 @@ def _windows_for_actions(
return list(_iter_windows(start, now, MAX_WINDOW_DAYS))
-def _resume_window_index(
+def _resume_target(
windows: list[tuple[datetime, datetime]], resume: Optional[ImpactPartnerResumeConfig]
-) -> Optional[int]:
- """Find which date window a saved bookmark points at.
+) -> Optional[tuple[int, ActionsWindow]]:
+ """Find the top-level window a saved bookmark falls inside, and the exact slice to resume.
- Returns None when there's no bookmark or it no longer matches (window boundaries derive
- from the current time, so they shift between runs), in which case the caller starts over
- (merge dedupes the re-yielded rows).
+ Matches by containment, since a 400 split can leave the bookmark narrower than the top-level
+ window. Returns None when there's no bookmark, no `window_end`, or no window contains it
+ (boundaries derive from the current time, so they shift between runs); the caller then starts
+ over, which merge dedupes.
"""
- if resume is None or resume.window_start is None:
+ if resume is None or resume.window_start is None or resume.window_end is None:
+ return None
+ saved_start = _to_datetime(resume.window_start)
+ saved_end = _to_datetime(resume.window_end)
+ if saved_start is None or saved_end is None:
return None
for index, window in enumerate(windows):
- if window[0].isoformat() == resume.window_start:
- return index
+ if window[0] <= saved_start and saved_end <= window[1]:
+ return index, ActionsWindow(start=saved_start, end=saved_end)
return None
@@ -206,29 +260,38 @@ def _get_rows_windowed(
# so history is walked in date windows only, with no per-campaign fan-out.
windows = _windows_for_actions(should_use_incremental_field, db_incremental_field_last_value)
+ def _run(window: ActionsWindow, start_page: int) -> Iterator[list[dict[str, Any]]]:
+ for fetched in _fetch_actions_window(
+ session, account_sid, config, window, logger, start_param, end_param, start_page
+ ):
+ if fetched.rows:
+ yield fetched.rows
+ # Save after yielding: a crash re-fetches this same page and merge dedupes it.
+ resumable_source_manager.save_state(
+ ImpactPartnerResumeConfig(
+ page=fetched.page,
+ window_start=fetched.window.start.isoformat(),
+ window_end=fetched.window.end.isoformat(),
+ )
+ )
+
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
- resume_index = _resume_window_index(windows, resume)
- start_index = resume_index if resume_index is not None else 0
+ target = _resume_target(windows, resume)
+ start_index = target[0] if target is not None else 0
if start_index > 0:
logger.debug(f"Impact Partner: resuming Actions from window {start_index}/{len(windows)}")
for index in range(start_index, len(windows)):
window = windows[index]
- # Resume the bookmarked window at its saved page; every other window starts at page 1.
- start_page = 1
- if resume is not None and index == resume_index and resume.page:
- start_page = resume.page
- params: dict[str, Any] = {
- start_param: _format_datetime(window[0]),
- end_param: _format_datetime(window[1]),
- }
- for page, rows in _paginate_endpoint(session, account_sid, config, params, logger, start_page=start_page):
- if rows:
- yield rows
- # Save after yielding: a crash re-fetches this same page and merge dedupes it.
- resumable_source_manager.save_state(
- ImpactPartnerResumeConfig(page=page, window_start=window[0].isoformat())
- )
+ if target is not None and index == target[0]:
+ # Resume the exact bookmarked slice, which a 400 may have narrowed below this window.
+ resumed_window = target[1]
+ yield from _run(resumed_window, resume.page if resume and resume.page else 1)
+ if resumed_window.end < window[1]:
+ # A split leaves the rest of this window unfetched; get it before moving on.
+ yield from _run(ActionsWindow(start=resumed_window.end, end=window[1]), 1)
+ else:
+ yield from _run(ActionsWindow(start=window[0], end=window[1]), 1)
def get_rows(
diff --git a/products/warehouse_sources/sources/impact_partner/tests/test_impact_partner.py b/products/warehouse_sources/sources/impact_partner/tests/test_impact_partner.py
index 4d2a5ecee731..fa9166d7c07f 100644
--- a/products/warehouse_sources/sources/impact_partner/tests/test_impact_partner.py
+++ b/products/warehouse_sources/sources/impact_partner/tests/test_impact_partner.py
@@ -3,6 +3,7 @@ from typing import Optional
import time_machine
+import requests
from parameterized import parameterized
from products.warehouse_sources.sources.impact_partner._config import ImpactPartnerSourceConfig
@@ -110,7 +111,9 @@ class TestGetRowsActions:
result = _driver().run(
"Actions",
[ScriptedResponse(json={"Actions": [], "@numpages": "2"})],
- resume_state=ImpactPartnerResumeConfig(page=2, window_start=windows[1][0].isoformat()),
+ resume_state=ImpactPartnerResumeConfig(
+ page=2, window_start=windows[1][0].isoformat(), window_end=windows[1][1].isoformat()
+ ),
should_use_incremental_field=True,
db_incremental_field_last_value=cursor,
)
@@ -120,12 +123,25 @@ class TestGetRowsActions:
assert result.params("ActionDateStart") == ["2026-05-16T00:00:00Z"]
assert result.params("Page") == ["2"]
+ @parameterized.expand(
+ [
+ ("no_window_end", ImpactPartnerResumeConfig(page=5, window_start="2026-05-31T00:00:00+00:00")),
+ (
+ "window_end_fits_no_window",
+ ImpactPartnerResumeConfig(
+ page=5, window_start="2020-01-01T00:00:00+00:00", window_end="2020-01-02T00:00:00+00:00"
+ ),
+ ),
+ ]
+ )
@time_machine.travel("2026-06-01", tick=False)
- def test_stale_bookmark_restarts_from_the_first_window(self) -> None:
+ def test_unplaceable_bookmark_restarts_from_the_first_window(
+ self, _name: str, resume_state: ImpactPartnerResumeConfig
+ ) -> None:
result = _driver().run(
"Actions",
[ScriptedResponse(json={"Actions": [], "@numpages": "1"})],
- resume_state=ImpactPartnerResumeConfig(page=5, window_start="2020-01-01T00:00:00+00:00"),
+ resume_state=resume_state,
should_use_incremental_field=True,
db_incremental_field_last_value=datetime(2026, 5, 31, tzinfo=UTC),
)
@@ -134,6 +150,96 @@ class TestGetRowsActions:
assert result.params("Page") == ["1"]
+class TestActionsWindowNarrowing:
+ @time_machine.travel("2026-06-01", tick=False)
+ def test_a_400_narrows_the_window_instead_of_failing_the_sync(self) -> None:
+ # A 12-day window: too short to split into more than one level before hitting real rows.
+ result = _driver().run(
+ "Actions",
+ [
+ ScriptedResponse(status=400),
+ ScriptedResponse(json={"Actions": [{"Id": "a1"}], "@numpages": "1"}),
+ ScriptedResponse(json={"Actions": [{"Id": "a2"}], "@numpages": "1"}),
+ ],
+ should_use_incremental_field=True,
+ db_incremental_field_last_value=datetime(2026, 5, 20, tzinfo=UTC),
+ )
+
+ assert result.raised is None
+ assert result.items == [[{"Id": "a1"}], [{"Id": "a2"}]]
+ assert len(result.requests) == 3
+ full_start, full_end = result.requests[0].param("ActionDateStart"), result.requests[0].param("ActionDateEnd")
+ assert (full_start, full_end) == ("2026-05-20T00:00:00Z", "2026-06-01T00:00:00Z")
+ first_half_end = result.requests[1].param("ActionDateEnd")
+ assert result.requests[1].param("ActionDateStart") == full_start
+ assert result.requests[2].param("ActionDateStart") == first_half_end
+ assert result.requests[2].param("ActionDateEnd") == full_end
+ assert full_start is not None
+ assert first_half_end is not None
+ # Resume state bookmarks the narrowed window actually fetched, not the original one.
+ assert [state.window_start for state in result.saved_states] == [
+ datetime.fromisoformat(full_start.replace("Z", "+00:00")).isoformat(),
+ datetime.fromisoformat(first_half_end.replace("Z", "+00:00")).isoformat(),
+ ]
+
+ @time_machine.travel("2026-06-01", tick=False)
+ def test_a_400_on_a_window_already_at_the_floor_still_raises(self) -> None:
+ # A 12-hour window is already below the 1-day floor, so there's nothing left to narrow.
+ result = _driver().run(
+ "Actions",
+ [ScriptedResponse(status=400)],
+ should_use_incremental_field=True,
+ db_incremental_field_last_value=datetime(2026, 5, 31, 12, tzinfo=UTC),
+ )
+
+ assert isinstance(result.raised, requests.HTTPError)
+ assert len(result.requests) == 1
+
+ @time_machine.travel("2026-06-01", tick=False)
+ def test_resuming_the_second_half_does_not_refetch_the_first(self) -> None:
+ # The second half is the one left bookmarked once the first has fully completed.
+ bookmark = ImpactPartnerResumeConfig(
+ page=1, window_start="2026-05-26T00:00:00+00:00", window_end="2026-06-01T00:00:00+00:00"
+ )
+ result = _driver().run(
+ "Actions",
+ [ScriptedResponse(json={"Actions": [{"Id": "a2"}], "@numpages": "1"})],
+ resume_state=bookmark,
+ should_use_incremental_field=True,
+ db_incremental_field_last_value=datetime(2026, 5, 20, tzinfo=UTC),
+ )
+
+ assert result.raised is None
+ assert len(result.requests) == 1
+ assert result.requests[0].param("ActionDateStart") == "2026-05-26T00:00:00Z"
+ assert result.requests[0].param("ActionDateEnd") == "2026-06-01T00:00:00Z"
+ assert result.requests[0].param("Page") == "1"
+
+ @time_machine.travel("2026-06-01", tick=False)
+ def test_resuming_the_first_half_still_fetches_the_second_after_it(self) -> None:
+ # Bookmarked mid-pagination, so resuming it must not skip the second half after it.
+ bookmark = ImpactPartnerResumeConfig(
+ page=1, window_start="2026-05-20T00:00:00+00:00", window_end="2026-05-26T00:00:00+00:00"
+ )
+ result = _driver().run(
+ "Actions",
+ [
+ ScriptedResponse(json={"Actions": [{"Id": "b1"}], "@numpages": "2"}),
+ ScriptedResponse(json={"Actions": [{"Id": "b2"}], "@numpages": "2"}),
+ ScriptedResponse(json={"Actions": [{"Id": "b3"}], "@numpages": "1"}),
+ ],
+ resume_state=bookmark,
+ should_use_incremental_field=True,
+ db_incremental_field_last_value=datetime(2026, 5, 20, tzinfo=UTC),
+ )
+
+ assert result.raised is None
+ assert result.items == [[{"Id": "b1"}], [{"Id": "b2"}], [{"Id": "b3"}]]
+ assert result.params("ActionDateStart") == ["2026-05-20T00:00:00Z"] * 2 + ["2026-05-26T00:00:00Z"]
+ assert result.params("ActionDateEnd") == ["2026-05-26T00:00:00Z"] * 2 + ["2026-06-01T00:00:00Z"]
+ assert result.params("Page") == ["1", "2", "1"]
+
+
class TestImpactPartnerSourceResponse:
@parameterized.expand(
[