:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse-sources): retry decagon articles from page 0 (#127928)
jake sciottoc2ce9d3e42f5
diff --git a/products/warehouse_sources/sources/decagon/decagon.py b/products/warehouse_sources/sources/decagon/decagon.pyindex acdac2c98821..0ba5225ba6e0 100644--- a/products/warehouse_sources/sources/decagon/decagon.py+++ b/products/warehouse_sources/sources/decagon/decagon.py@@ -658,6 +658,7 @@ class _RowWalk:def _walk_page(self) -> Iterator[list[dict[str, Any]]]:config = self._configpage = self._resume.page or 1+ page_zero_tried = False# Rows actually kept, not page * page_size: a row that shifted pages mid-walk arrives# twice but counts once toward the server's unique total, so counting raw items could# reach the total a page early and drop the final page. This also stays exact if the@@ -674,9 +675,13 @@ class _RowWalk:batch = self._read(params)total = self._reported_total+ if not page_zero_tried and self._ended_short_of_the_total(batch, rows_walked):+ page_zero_tried = True+ page = 0+ continuerows_walked += len(batch.fresh)page_rows = max(page_rows, len(batch.items))- exhausted = self._page_walk_exhausted(page, rows_walked, page_rows, total, batch)+ exhausted = page_zero_tried or self._page_walk_exhausted(page, rows_walked, page_rows, total, batch)if batch.fresh:yield batch.fresh@@ -687,6 +692,11 @@ class _RowWalk:page += 1+ def _ended_short_of_the_total(self, batch: _Batch, rows_walked: int) -> bool:+ total = _usable_total(self._reported_total)+ listed_no_rows = not batch.items and batch.pagination.get(self._config.data_key) == []+ return listed_no_rows and total is not None and rows_walked < total+def _page_walk_exhausted(self, page: int, rows_walked: int, page_rows: int, reported: Any, batch: _Batch) -> bool:# A page of only already-seen rows does not end the walk: the catalog can shift rows# between pages mid-walk, so a later page can still hold rows this walk has not kept.diff --git a/products/warehouse_sources/sources/decagon/tests/test_decagon.py b/products/warehouse_sources/sources/decagon/tests/test_decagon.pyindex c0fc14836e4c..1337b67022ca 100644--- a/products/warehouse_sources/sources/decagon/tests/test_decagon.py+++ b/products/warehouse_sources/sources/decagon/tests/test_decagon.py@@ -338,6 +338,39 @@ class TestArticleTables:assert "honors the page param" in str(result.raised)assert result.params("page") == [str(n) for n in range(1, max_pages + 1)]+ @parameterized.expand(+ [+ ("zero_based_one_page", 0, [1, 2], 2, ["1", "0"], [1, 2], None),+ ("zero_based_three_pages", 0, [1, 2, 3, 4, 5], 5, ["1", "2", "3", "0"], [3, 4, 5, 1, 2], None),+ ("one_based_with_a_stale_total", 1, [1, 2, 3], 4, ["1", "2", "3", "0"], [1, 2, 3], None),+ ("total_but_no_rows", 0, [], 3, ["1", "0"], [], DecagonContractError),+ ]+ )+ def test_a_walk_that_ends_short_of_the_total_reads_page_zero_once(+ self,+ _name: str,+ first_page: int,+ catalog: list[int],+ total: int,+ pages: list[str],+ ids: list[int],+ expected_error: type[Exception] | None,+ ) -> None:+ cfg = dataclasses.replace(DECAGON_ENDPOINTS["articles"], page_size=2)++ def respond(request: RecordedRequest) -> ScriptedResponse:+ page = int(request.param("page") or 0)+ index = max(page, first_page) - first_page+ rows = [{"id": row_id} for row_id in catalog[index * 2 : index * 2 + 2]]+ return _page({"articles": rows, "page": page, "page_size": 2, "total": total})++ with patch.dict(DECAGON_ENDPOINTS, {"articles": cfg}):+ result = _driver().run("articles", respond)++ assert result.params("page") == pages+ assert [row["id"] for row in result.rows] == ids+ assert (None if result.raised is None else type(result.raised)) is expected_error+@parameterized.expand([("negative", -1), ("boolean", True), ("string", "3"), ("null", None)])def test_malformed_total_falls_back_to_short_page_termination(self, _name: str, total: Any) -> None:# A total that cannot bound the walk must not end it early: a negative or boolean