Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): resume a fan-out at the saved parent page (#125099)

Daniel RC4a6e5a3495c3
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py
index 853a65adf302..312dc9bedfe9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py
@@ -85,9 +85,11 @@ def get_resources(name: str) -> list[EndpointResource]:
fan_out.query_param: {"type": "resolve", "resource": fan_out.parent, "field": fan_out.parent_field},
"limit": config.page_size,
}
+ child = _resource(name, config, f"{config.path}?{fan_out.query_param}={{{fan_out.query_param}}}", child_params)
+ child["resume_parent_pagination"] = True
return [
_resource(fan_out.parent, parent_config, parent_config.path, {"limit": parent_config.page_size}),
- _resource(name, config, f"{config.path}?{fan_out.query_param}={{{fan_out.query_param}}}", child_params),
+ child,
]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.py
index 089ddf336ee4..1d65138b2bc6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.py
@@ -278,6 +278,7 @@ class TestClerkFilteredEndpoints:
"completed": [f"{config.path}?{config.fan_out.query_param}={parent_id}"],
"current": None,
"child_state": None,
+ "parent": None,
}
assert result.raised is None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py
index 7af0bc712c1e..1892b27d48d0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py
@@ -137,6 +137,77 @@ def rest_api_resources(
return list(resources.values())
+class ParentPosition:
+ """Where the parent listing of a resumable fan-out stands.
+
+ The parent paginator reports the state of its next page before it hands a page on. This keeps the
+ state that fetches the page being fanned out and the page before it. A resume starts one page
+ back, so a parent that a deletion moved into the earlier page is read again rather than skipped.
+ """
+
+ def __init__(self, start: Optional[dict[str, Any]]) -> None:
+ self.start = start
+ self._next: Optional[dict[str, Any]] = start
+ self._current: Optional[dict[str, Any]] = start
+ self._previous: Optional[dict[str, Any]] = start
+ self.tracked = True
+
+ def on_page_state(self, next_state: Optional[dict[str, Any]], has_next_page: bool) -> None:
+ self._previous, self._current, self._next = self._current, self._next, next_state
+ if has_next_page and next_state is None:
+ # The paginator cannot save its place, so the fan-out keeps every finished parent instead.
+ self.tracked = False
+
+ def resume_state(self) -> Optional[dict[str, Any]]:
+ return self._previous
+
+
+class _FinishedParents:
+ """Child paths of the parents a fan-out has finished.
+
+ Without a tracked parent position, a resume reads the parent listing from the start, so every
+ finished path is kept. With one, a resume starts at the page before the current one, so only the
+ paths of those two pages are kept, plus the saved paths this run has not reached yet. Saving
+ every path after each parent costs more with each parent, so a large fan-out would slow down as
+ it runs.
+ """
+
+ def __init__(self, seed: list[str], position: Optional[ParentPosition]) -> None:
+ self._position = position
+ self._seed = set(seed)
+ # A state saved without a parent position resumes from the first parent page. Its paths stay
+ # saved until the listing reaches them, because a parent added at the front makes this run
+ # save a state before it reaches the parents it skips.
+ self._unreached_seed = set(seed)
+ self._everything = set(seed)
+ self._window: list[set[str]] = []
+
+ @property
+ def windowed(self) -> bool:
+ return self._position is not None and self._position.tracked
+
+ def start_parent_page(self) -> None:
+ self._window.append(set())
+ del self._window[:-2]
+
+ def __contains__(self, path: str) -> bool:
+ if self.windowed:
+ return path in self._seed or any(path in page for page in self._window)
+ return path in self._everything
+
+ def add(self, path: str) -> None:
+ self._unreached_seed.discard(path)
+ if self.windowed:
+ self._window[-1].add(path)
+ else:
+ self._everything.add(path)
+
+ def paths(self) -> list[str]:
+ if self.windowed:
+ return sorted(self._unreached_seed.union(*self._window))
+ return sorted(self._everything)
+
+
def _make_paginate_dependent_resource(
*,
client: RESTClient,
@@ -154,27 +225,33 @@ def _make_paginate_dependent_resource(
data_selector_malformed_retryable: bool = False,
on_parent_error: Optional[Callable[[str, Exception], None]] = None,
parent_source: FanoutParentSource = "api",
+ parent_position: Optional[ParentPosition] = None,
) -> Callable[..., Iterator[list[Any]]]:
"""Build the generator for a dependent (child) resource.
When ``resume_hook`` is set the fan-out is resumable: each parent's child pagination is
checkpointed under that parent's resolved child path, so a restart skips parents already fully
- synced and resumes the one that was in progress. Parent pagination itself is not resumed — the
- (usually small) parent list is re-fetched each run and already-completed parents are skipped by
- path. Resume state shape:
- ``{"completed": [child_path, ...], "current": child_path | None, "child_state": {...} | None}``.
+ synced and resumes the one that was in progress. Without ``parent_position`` the parent list is
+ re-fetched from the start each run and already-completed parents are skipped by path. With it,
+ the state also holds ``parent``, the paginator state a resume starts the parent listing from.
+ Resume state shape:
+ ``{"completed": [child_path, ...], "current": child_path | None, "child_state": {...} | None}``,
+ plus ``"parent": {...} | None`` when the parent position is tracked.
"""
# Closure state persists across parent-page invocations within a single run.
parent_rows_consumed = 0
seed: dict[str, Any] = dict(initial_state) if initial_state else {}
- completed: set[str] = set(seed.get("completed") or [])
+ completed = _FinishedParents(seed.get("completed") or [], parent_position)
current_path: Optional[str] = seed.get("current")
current_child_state: Optional[dict[str, Any]] = seed.get("child_state")
- resumed = bool(completed or current_path)
+ resumed = bool(seed.get("completed") or current_path)
def checkpoint(current: Optional[str], child_state: Optional[dict[str, Any]]) -> None:
+ state: dict[str, Any] = {"completed": completed.paths(), "current": current, "child_state": child_state}
+ if parent_position is not None and completed.windowed:
+ state["parent"] = parent_position.resume_state()
# resume_hook is non-None here (only called from the resumable path).
- resume_hook({"completed": sorted(completed), "current": current, "child_state": child_state}) # type: ignore[misc]
+ resume_hook(state) # type: ignore[misc]
def paginate_dependent_resource(
items: list[dict[str, Any]],
@@ -190,6 +267,7 @@ def _make_paginate_dependent_resource(
effective_columns_config = columns_config if columns_config is not None else default_columns_config
page_rows = 0
+ completed.start_parent_page()
if incremental_object:
params, _ = _set_incremental_params(
@@ -205,6 +283,8 @@ def _make_paginate_dependent_resource(
formatted_path, parent_record = process_parent_data_item(path, item, resolved_param, include_from_parent)
if resume_hook is not None and formatted_path in completed:
+ # Kept on this page, so it is still skipped after the resume one page back.
+ completed.add(formatted_path)
continue
page_rows += 1
@@ -311,7 +391,8 @@ def create_resources(
resources: dict[str, Resource] = {}
# Resume is routed to the dependent (child) resource in a fan-out; the parent list is re-fetched
- # each run (see _make_paginate_dependent_resource). So when any resource is dependent, the
+ # each run unless the child sets ``resume_parent_pagination`` (see
+ # _make_paginate_dependent_resource). So when any resource is dependent, the
# non-dependent resources in the same config don't consume the resume hook. With MULTIPLE
# dependent resources (a chained/multi-level fan-out), no resource gets resume — one shared
# hook consumed at several levels would corrupt the saved state; retries re-fetch and the
@@ -321,6 +402,21 @@ def create_resources(
dependent_resume_hook = resume_hook if dependent_count == 1 else None
dependent_initial_state = initial_paginator_state if dependent_count == 1 else None
+ parent_position: Optional[ParentPosition] = None
+ position_parent_name: Optional[str] = None
+ if dependent_resume_hook is not None:
+ child_name, child_params = next((name, rp) for name, rp in resolved_param_map.items() if rp is not None)
+ parent_name = child_params[0].resolve_config["resource"]
+ # A parent skipped through `on_parent_error` is retried only when a resume reads it again, and a
+ # resume that starts at the saved page never reaches the earlier pages.
+ if (
+ endpoint_resource_map[child_name].get("resume_parent_pagination")
+ and endpoint_resource_map[parent_name].get("data_iterator") is None
+ and on_parent_error is None
+ ):
+ parent_position = ParentPosition((dependent_initial_state or {}).get("parent"))
+ position_parent_name = parent_name
+
for resource_name in dependency_graph.static_order():
resource_name = cast(str, resource_name)
endpoint_resource = endpoint_resource_map[resource_name]
@@ -408,6 +504,14 @@ def create_resources(
)
elif resolved_params is None:
+ own_parent_position = parent_position if resource_name == position_parent_name else None
+ own_page_state_hook: Optional[Callable[[Optional[dict[str, Any]], bool], None]]
+ if own_parent_position is not None:
+ own_initial_state = own_parent_position.start
+ own_page_state_hook = own_parent_position.on_page_state
+ else:
+ own_initial_state = None if has_dependent_resource else initial_paginator_state
+ own_page_state_hook = page_checkpoints.defer_page_state if page_checkpoints is not None else None
def paginate_resource(
method: HTTPMethodBasic,
@@ -422,10 +526,8 @@ def create_resources(
incremental_object: Optional[Incremental] = incremental_object,
incremental_param: Optional[IncrementalParam] = incremental_param,
incremental_cursor_transform: Optional[Callable[..., Any]] = incremental_cursor_transform,
- page_checkpoints: Optional[PageCheckpoints] = page_checkpoints,
- initial_paginator_state: Optional[dict[str, Any]] = (
- None if has_dependent_resource else initial_paginator_state
- ),
+ page_state_hook: Optional[Callable[[Optional[dict[str, Any]], bool], None]] = own_page_state_hook,
+ initial_paginator_state: Optional[dict[str, Any]] = own_initial_state,
data_selector_required: bool = bool(endpoint_config.get("data_selector_required")),
data_selector_empty_ok: bool = bool(endpoint_config.get("data_selector_empty_ok")),
data_selector_malformed_retryable: bool = bool(
@@ -450,7 +552,7 @@ def create_resources(
paginator=paginator,
data_selector=data_selector,
hooks=hooks,
- page_state_hook=page_checkpoints.defer_page_state if page_checkpoints is not None else None,
+ page_state_hook=page_state_hook,
initial_paginator_state=initial_paginator_state,
data_selector_required=data_selector_required,
data_selector_empty_ok=data_selector_empty_ok,
@@ -502,6 +604,7 @@ def create_resources(
data_selector_malformed_retryable=bool(endpoint_config.get("data_selector_malformed_retryable")),
on_parent_error=on_parent_error,
parent_source=parent_source,
+ parent_position=parent_position,
)
resources[resource_name] = Resource(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_resume_checkpoints.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_resume_checkpoints.py
index ffdb6b77b345..6b91ccd0c7b4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_resume_checkpoints.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_resume_checkpoints.py
@@ -205,3 +205,65 @@ def test_a_fan_out_resumed_from_the_state_staged_with_a_page_reads_every_row_onc
rows.extend(row["id"] for page in resumed for row in page)
assert rows == ["a1", "a2", "b1"]
+
+
+def _two_parents_per_page(parents: str) -> dict[str, list[list[dict[str, Any]]]]:
+ pages = [[{"id": parent} for parent in parents[index : index + 2]] for index in range(0, len(parents), 2)]
+ return {"/parents": pages, **{f"/parents/{parent}/children": [[{"id": f"{parent}1"}]] for parent in parents}}
+
+
+PARENT_PER_PAGE = {
+ "/parents": [[{"id": parent}] for parent in "abcde"],
+ **{f"/parents/{parent}/children": [[{"id": f"{parent}1"}]] for parent in "abcde"},
+}
+POSITIONED_RESOURCES: list[Any] = ["parents", {**FANOUT_RESOURCES[1], "resume_parent_pagination": True}]
+
+
+ "stop_after_rows,deleted_before_resume",
+ [(1, ""), (3, ""), (5, ""), (4, "abc")],
+ ids=["first_page", "second_page", "third_page", "parents_deleted_before_the_saved_page"],
+)
+def test_a_fan_out_that_saves_the_parent_position_reads_every_remaining_parent_once(
+ stop_after_rows: int, deleted_before_resume: str
+) -> None:
+ states: list[Any] = []
+ first = _resource(paged_config(_two_parents_per_page("abcdefgh"), POSITIONED_RESOURCES), "children", states.append)
+ rows: list[str] = []
+ with activate_safe_point(lambda: None, covers_framework_checkpoints=True):
+ for index, page in enumerate(first, start=1):
+ rows.extend(row["id"] for row in page)
+ if index == stop_after_rows:
+ break
+
+ remaining = "".join(parent for parent in "abcdefgh" if parent not in deleted_before_resume)
+ resumed = _resource(
+ paged_config(_two_parents_per_page(remaining), POSITIONED_RESOURCES),
+ "children",
+ lambda _state: None,
+ states[-1],
+ )
+ rows.extend(row["id"] for page in resumed for row in page)
+
+ assert rows == [f"{parent}1" for parent in "abcdefgh"]
+ assert len(states[-1]["completed"]) <= 4
+
+
+def test_a_saved_state_without_a_parent_position_still_skips_its_finished_parents() -> None:
+ states: list[Any] = []
+ listing = {
+ **PARENT_PER_PAGE,
+ "/parents": [[{"id": "z"}], *PARENT_PER_PAGE["/parents"]],
+ "/parents/z/children": [[{"id": "z1"}]],
+ }
+ legacy = {"completed": ["/parents/a/children", "/parents/b/children"], "current": None, "child_state": None}
+ resumed = _resource(paged_config(listing, POSITIONED_RESOURCES), "children", states.append, legacy)
+
+ with activate_safe_point(lambda: None, covers_framework_checkpoints=True):
+ rows = [row["id"] for page in resumed for row in page]
+
+ assert rows == ["z1", "c1", "d1", "e1"]
+ # A parent added at the front is saved before the run reaches the parents the old state finished.
+ assert states[0]["completed"] == ["/parents/a/children", "/parents/b/children"]
+ assert states[-1]["parent"] == {"page": 4}
+ assert states[-1]["completed"] == ["/parents/d/children", "/parents/e/children"]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py
index ab8d6d0603c0..01769c8e9260 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py
@@ -366,6 +366,11 @@ class EndpointResourceBase(ResourceBase, total=False):
# ``build_dependent_resource`` only after the warehouse table resolves, so it names the parent
# the run actually read rather than the one its config asked for.
parent_source: Optional[FanoutParentSource]
+ # Set on a resumable fan-out child to save where the parent listing stands, so a resume starts
+ # there instead of paging the parent from the start. Only for sources that store the fan-out
+ # state they receive unchanged: the state gains a `parent` key and keeps two pages of `completed`.
+ # Ignored when the source passes `on_parent_error`.
+ resume_parent_pagination: Optional[bool]
class EndpointResource(EndpointResourceBase, total=False):