:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse-sources): resume a fan-out at the saved parent page (#125099)
Daniel RC4a6e5a3495c3
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.pyindex 853a65adf302..312dc9bedfe9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py@@ -85,9 +85,11 @@ def get_resources(name: str) -> list[EndpointResource]:fan_out.query_param: {"type": "resolve", "resource": fan_out.parent, "field": fan_out.parent_field},"limit": config.page_size,}+ child = _resource(name, config, f"{config.path}?{fan_out.query_param}={{{fan_out.query_param}}}", child_params)+ child["resume_parent_pagination"] = Truereturn [_resource(fan_out.parent, parent_config, parent_config.path, {"limit": parent_config.page_size}),- _resource(name, config, f"{config.path}?{fan_out.query_param}={{{fan_out.query_param}}}", child_params),+ child,]diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.pyindex 089ddf336ee4..1d65138b2bc6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.py@@ -278,6 +278,7 @@ class TestClerkFilteredEndpoints:"completed": [f"{config.path}?{config.fan_out.query_param}={parent_id}"],"current": None,"child_state": None,+ "parent": None,}assert result.raised is Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.pyindex 7af0bc712c1e..1892b27d48d0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py@@ -137,6 +137,77 @@ def rest_api_resources(return list(resources.values())+class ParentPosition:+ """Where the parent listing of a resumable fan-out stands.++ The parent paginator reports the state of its next page before it hands a page on. This keeps the+ state that fetches the page being fanned out and the page before it. A resume starts one page+ back, so a parent that a deletion moved into the earlier page is read again rather than skipped.+ """++ def __init__(self, start: Optional[dict[str, Any]]) -> None:+ self.start = start+ self._next: Optional[dict[str, Any]] = start+ self._current: Optional[dict[str, Any]] = start+ self._previous: Optional[dict[str, Any]] = start+ self.tracked = True++ def on_page_state(self, next_state: Optional[dict[str, Any]], has_next_page: bool) -> None:+ self._previous, self._current, self._next = self._current, self._next, next_state+ if has_next_page and next_state is None:+ # The paginator cannot save its place, so the fan-out keeps every finished parent instead.+ self.tracked = False++ def resume_state(self) -> Optional[dict[str, Any]]:+ return self._previous+++class _FinishedParents:+ """Child paths of the parents a fan-out has finished.++ Without a tracked parent position, a resume reads the parent listing from the start, so every+ finished path is kept. With one, a resume starts at the page before the current one, so only the+ paths of those two pages are kept, plus the saved paths this run has not reached yet. Saving+ every path after each parent costs more with each parent, so a large fan-out would slow down as+ it runs.+ """++ def __init__(self, seed: list[str], position: Optional[ParentPosition]) -> None:+ self._position = position+ self._seed = set(seed)+ # A state saved without a parent position resumes from the first parent page. Its paths stay+ # saved until the listing reaches them, because a parent added at the front makes this run+ # save a state before it reaches the parents it skips.+ self._unreached_seed = set(seed)+ self._everything = set(seed)+ self._window: list[set[str]] = []++ @property+ def windowed(self) -> bool:+ return self._position is not None and self._position.tracked++ def start_parent_page(self) -> None:+ self._window.append(set())+ del self._window[:-2]++ def __contains__(self, path: str) -> bool:+ if self.windowed:+ return path in self._seed or any(path in page for page in self._window)+ return path in self._everything++ def add(self, path: str) -> None:+ self._unreached_seed.discard(path)+ if self.windowed:+ self._window[-1].add(path)+ else:+ self._everything.add(path)++ def paths(self) -> list[str]:+ if self.windowed:+ return sorted(self._unreached_seed.union(*self._window))+ return sorted(self._everything)++def _make_paginate_dependent_resource(*,client: RESTClient,@@ -154,27 +225,33 @@ def _make_paginate_dependent_resource(data_selector_malformed_retryable: bool = False,on_parent_error: Optional[Callable[[str, Exception], None]] = None,parent_source: FanoutParentSource = "api",+ parent_position: Optional[ParentPosition] = None,) -> Callable[..., Iterator[list[Any]]]:"""Build the generator for a dependent (child) resource.When ``resume_hook`` is set the fan-out is resumable: each parent's child pagination ischeckpointed under that parent's resolved child path, so a restart skips parents already fully- synced and resumes the one that was in progress. Parent pagination itself is not resumed — the- (usually small) parent list is re-fetched each run and already-completed parents are skipped by- path. Resume state shape:- ``{"completed": [child_path, ...], "current": child_path | None, "child_state": {...} | None}``.+ synced and resumes the one that was in progress. Without ``parent_position`` the parent list is+ re-fetched from the start each run and already-completed parents are skipped by path. With it,+ the state also holds ``parent``, the paginator state a resume starts the parent listing from.+ Resume state shape:+ ``{"completed": [child_path, ...], "current": child_path | None, "child_state": {...} | None}``,+ plus ``"parent": {...} | None`` when the parent position is tracked."""# Closure state persists across parent-page invocations within a single run.parent_rows_consumed = 0seed: dict[str, Any] = dict(initial_state) if initial_state else {}- completed: set[str] = set(seed.get("completed") or [])+ completed = _FinishedParents(seed.get("completed") or [], parent_position)current_path: Optional[str] = seed.get("current")current_child_state: Optional[dict[str, Any]] = seed.get("child_state")- resumed = bool(completed or current_path)+ resumed = bool(seed.get("completed") or current_path)def checkpoint(current: Optional[str], child_state: Optional[dict[str, Any]]) -> None:+ state: dict[str, Any] = {"completed": completed.paths(), "current": current, "child_state": child_state}+ if parent_position is not None and completed.windowed:+ state["parent"] = parent_position.resume_state()# resume_hook is non-None here (only called from the resumable path).- resume_hook({"completed": sorted(completed), "current": current, "child_state": child_state}) # type: ignore[misc]+ resume_hook(state) # type: ignore[misc]def paginate_dependent_resource(items: list[dict[str, Any]],@@ -190,6 +267,7 @@ def _make_paginate_dependent_resource(effective_columns_config = columns_config if columns_config is not None else default_columns_configpage_rows = 0+ completed.start_parent_page()if incremental_object:params, _ = _set_incremental_params(@@ -205,6 +283,8 @@ def _make_paginate_dependent_resource(formatted_path, parent_record = process_parent_data_item(path, item, resolved_param, include_from_parent)if resume_hook is not None and formatted_path in completed:+ # Kept on this page, so it is still skipped after the resume one page back.+ completed.add(formatted_path)continuepage_rows += 1@@ -311,7 +391,8 @@ def create_resources(resources: dict[str, Resource] = {}# Resume is routed to the dependent (child) resource in a fan-out; the parent list is re-fetched- # each run (see _make_paginate_dependent_resource). So when any resource is dependent, the+ # each run unless the child sets ``resume_parent_pagination`` (see+ # _make_paginate_dependent_resource). So when any resource is dependent, the# non-dependent resources in the same config don't consume the resume hook. With MULTIPLE# dependent resources (a chained/multi-level fan-out), no resource gets resume — one shared# hook consumed at several levels would corrupt the saved state; retries re-fetch and the@@ -321,6 +402,21 @@ def create_resources(dependent_resume_hook = resume_hook if dependent_count == 1 else Nonedependent_initial_state = initial_paginator_state if dependent_count == 1 else None+ parent_position: Optional[ParentPosition] = None+ position_parent_name: Optional[str] = None+ if dependent_resume_hook is not None:+ child_name, child_params = next((name, rp) for name, rp in resolved_param_map.items() if rp is not None)+ parent_name = child_params[0].resolve_config["resource"]+ # A parent skipped through `on_parent_error` is retried only when a resume reads it again, and a+ # resume that starts at the saved page never reaches the earlier pages.+ if (+ endpoint_resource_map[child_name].get("resume_parent_pagination")+ and endpoint_resource_map[parent_name].get("data_iterator") is None+ and on_parent_error is None+ ):+ parent_position = ParentPosition((dependent_initial_state or {}).get("parent"))+ position_parent_name = parent_name+for resource_name in dependency_graph.static_order():resource_name = cast(str, resource_name)endpoint_resource = endpoint_resource_map[resource_name]@@ -408,6 +504,14 @@ def create_resources()elif resolved_params is None:+ own_parent_position = parent_position if resource_name == position_parent_name else None+ own_page_state_hook: Optional[Callable[[Optional[dict[str, Any]], bool], None]]+ if own_parent_position is not None:+ own_initial_state = own_parent_position.start+ own_page_state_hook = own_parent_position.on_page_state+ else:+ own_initial_state = None if has_dependent_resource else initial_paginator_state+ own_page_state_hook = page_checkpoints.defer_page_state if page_checkpoints is not None else Nonedef paginate_resource(method: HTTPMethodBasic,@@ -422,10 +526,8 @@ def create_resources(incremental_object: Optional[Incremental] = incremental_object,incremental_param: Optional[IncrementalParam] = incremental_param,incremental_cursor_transform: Optional[Callable[..., Any]] = incremental_cursor_transform,- page_checkpoints: Optional[PageCheckpoints] = page_checkpoints,- initial_paginator_state: Optional[dict[str, Any]] = (- None if has_dependent_resource else initial_paginator_state- ),+ page_state_hook: Optional[Callable[[Optional[dict[str, Any]], bool], None]] = own_page_state_hook,+ initial_paginator_state: Optional[dict[str, Any]] = own_initial_state,data_selector_required: bool = bool(endpoint_config.get("data_selector_required")),data_selector_empty_ok: bool = bool(endpoint_config.get("data_selector_empty_ok")),data_selector_malformed_retryable: bool = bool(@@ -450,7 +552,7 @@ def create_resources(paginator=paginator,data_selector=data_selector,hooks=hooks,- page_state_hook=page_checkpoints.defer_page_state if page_checkpoints is not None else None,+ page_state_hook=page_state_hook,initial_paginator_state=initial_paginator_state,data_selector_required=data_selector_required,data_selector_empty_ok=data_selector_empty_ok,@@ -502,6 +604,7 @@ def create_resources(data_selector_malformed_retryable=bool(endpoint_config.get("data_selector_malformed_retryable")),on_parent_error=on_parent_error,parent_source=parent_source,+ parent_position=parent_position,)resources[resource_name] = Resource(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_resume_checkpoints.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_resume_checkpoints.pyindex ffdb6b77b345..6b91ccd0c7b4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_resume_checkpoints.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/tests/test_resume_checkpoints.py@@ -205,3 +205,65 @@ def test_a_fan_out_resumed_from_the_state_staged_with_a_page_reads_every_row_oncrows.extend(row["id"] for page in resumed for row in page)assert rows == ["a1", "a2", "b1"]+++def _two_parents_per_page(parents: str) -> dict[str, list[list[dict[str, Any]]]]:+ pages = [[{"id": parent} for parent in parents[index : index + 2]] for index in range(0, len(parents), 2)]+ return {"/parents": pages, **{f"/parents/{parent}/children": [[{"id": f"{parent}1"}]] for parent in parents}}+++PARENT_PER_PAGE = {+ "/parents": [[{"id": parent}] for parent in "abcde"],+ **{f"/parents/{parent}/children": [[{"id": f"{parent}1"}]] for parent in "abcde"},+}+POSITIONED_RESOURCES: list[Any] = ["parents", {**FANOUT_RESOURCES[1], "resume_parent_pagination": True}]+++ "stop_after_rows,deleted_before_resume",+ [(1, ""), (3, ""), (5, ""), (4, "abc")],+ ids=["first_page", "second_page", "third_page", "parents_deleted_before_the_saved_page"],+)+def test_a_fan_out_that_saves_the_parent_position_reads_every_remaining_parent_once(+ stop_after_rows: int, deleted_before_resume: str+) -> None:+ states: list[Any] = []+ first = _resource(paged_config(_two_parents_per_page("abcdefgh"), POSITIONED_RESOURCES), "children", states.append)+ rows: list[str] = []+ with activate_safe_point(lambda: None, covers_framework_checkpoints=True):+ for index, page in enumerate(first, start=1):+ rows.extend(row["id"] for row in page)+ if index == stop_after_rows:+ break++ remaining = "".join(parent for parent in "abcdefgh" if parent not in deleted_before_resume)+ resumed = _resource(+ paged_config(_two_parents_per_page(remaining), POSITIONED_RESOURCES),+ "children",+ lambda _state: None,+ states[-1],+ )+ rows.extend(row["id"] for page in resumed for row in page)++ assert rows == [f"{parent}1" for parent in "abcdefgh"]+ assert len(states[-1]["completed"]) <= 4+++def test_a_saved_state_without_a_parent_position_still_skips_its_finished_parents() -> None:+ states: list[Any] = []+ listing = {+ **PARENT_PER_PAGE,+ "/parents": [[{"id": "z"}], *PARENT_PER_PAGE["/parents"]],+ "/parents/z/children": [[{"id": "z1"}]],+ }+ legacy = {"completed": ["/parents/a/children", "/parents/b/children"], "current": None, "child_state": None}+ resumed = _resource(paged_config(listing, POSITIONED_RESOURCES), "children", states.append, legacy)++ with activate_safe_point(lambda: None, covers_framework_checkpoints=True):+ rows = [row["id"] for page in resumed for row in page]++ assert rows == ["z1", "c1", "d1", "e1"]+ # A parent added at the front is saved before the run reaches the parents the old state finished.+ assert states[0]["completed"] == ["/parents/a/children", "/parents/b/children"]+ assert states[-1]["parent"] == {"page": 4}+ assert states[-1]["completed"] == ["/parents/d/children", "/parents/e/children"]diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.pyindex ab8d6d0603c0..01769c8e9260 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py@@ -366,6 +366,11 @@ class EndpointResourceBase(ResourceBase, total=False):# ``build_dependent_resource`` only after the warehouse table resolves, so it names the parent# the run actually read rather than the one its config asked for.parent_source: Optional[FanoutParentSource]+ # Set on a resumable fan-out child to save where the parent listing stands, so a resume starts+ # there instead of paging the parent from the start. Only for sources that store the fan-out+ # state they receive unchanged: the state gains a `parent` key and keeps two pages of `completed`.+ # Ignored when the source passes `on_parent_error`.+ resume_parent_pagination: Optional[bool]class EndpointResource(EndpointResourceBase, total=False):