Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): retry decagon articles from page 0 (#127928)

jake sciottoc2ce9d3e42f5
diff --git a/products/warehouse_sources/sources/decagon/decagon.py b/products/warehouse_sources/sources/decagon/decagon.py
index acdac2c98821..0ba5225ba6e0 100644
--- a/products/warehouse_sources/sources/decagon/decagon.py
+++ b/products/warehouse_sources/sources/decagon/decagon.py
@@ -658,6 +658,7 @@ class _RowWalk:
def _walk_page(self) -> Iterator[list[dict[str, Any]]]:
config = self._config
page = self._resume.page or 1
+ page_zero_tried = False
# Rows actually kept, not page * page_size: a row that shifted pages mid-walk arrives
# twice but counts once toward the server's unique total, so counting raw items could
# reach the total a page early and drop the final page. This also stays exact if the
@@ -674,9 +675,13 @@ class _RowWalk:
batch = self._read(params)
total = self._reported_total
+ if not page_zero_tried and self._ended_short_of_the_total(batch, rows_walked):
+ page_zero_tried = True
+ page = 0
+ continue
rows_walked += len(batch.fresh)
page_rows = max(page_rows, len(batch.items))
- exhausted = self._page_walk_exhausted(page, rows_walked, page_rows, total, batch)
+ exhausted = page_zero_tried or self._page_walk_exhausted(page, rows_walked, page_rows, total, batch)
if batch.fresh:
yield batch.fresh
@@ -687,6 +692,11 @@ class _RowWalk:
page += 1
+ def _ended_short_of_the_total(self, batch: _Batch, rows_walked: int) -> bool:
+ total = _usable_total(self._reported_total)
+ listed_no_rows = not batch.items and batch.pagination.get(self._config.data_key) == []
+ return listed_no_rows and total is not None and rows_walked < total
+
def _page_walk_exhausted(self, page: int, rows_walked: int, page_rows: int, reported: Any, batch: _Batch) -> bool:
# A page of only already-seen rows does not end the walk: the catalog can shift rows
# between pages mid-walk, so a later page can still hold rows this walk has not kept.
diff --git a/products/warehouse_sources/sources/decagon/tests/test_decagon.py b/products/warehouse_sources/sources/decagon/tests/test_decagon.py
index c0fc14836e4c..1337b67022ca 100644
--- a/products/warehouse_sources/sources/decagon/tests/test_decagon.py
+++ b/products/warehouse_sources/sources/decagon/tests/test_decagon.py
@@ -338,6 +338,39 @@ class TestArticleTables:
assert "honors the page param" in str(result.raised)
assert result.params("page") == [str(n) for n in range(1, max_pages + 1)]
+ @parameterized.expand(
+ [
+ ("zero_based_one_page", 0, [1, 2], 2, ["1", "0"], [1, 2], None),
+ ("zero_based_three_pages", 0, [1, 2, 3, 4, 5], 5, ["1", "2", "3", "0"], [3, 4, 5, 1, 2], None),
+ ("one_based_with_a_stale_total", 1, [1, 2, 3], 4, ["1", "2", "3", "0"], [1, 2, 3], None),
+ ("total_but_no_rows", 0, [], 3, ["1", "0"], [], DecagonContractError),
+ ]
+ )
+ def test_a_walk_that_ends_short_of_the_total_reads_page_zero_once(
+ self,
+ _name: str,
+ first_page: int,
+ catalog: list[int],
+ total: int,
+ pages: list[str],
+ ids: list[int],
+ expected_error: type[Exception] | None,
+ ) -> None:
+ cfg = dataclasses.replace(DECAGON_ENDPOINTS["articles"], page_size=2)
+
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ page = int(request.param("page") or 0)
+ index = max(page, first_page) - first_page
+ rows = [{"id": row_id} for row_id in catalog[index * 2 : index * 2 + 2]]
+ return _page({"articles": rows, "page": page, "page_size": 2, "total": total})
+
+ with patch.dict(DECAGON_ENDPOINTS, {"articles": cfg}):
+ result = _driver().run("articles", respond)
+
+ assert result.params("page") == pages
+ assert [row["id"] for row in result.rows] == ids
+ assert (None if result.raised is None else type(result.raised)) is expected_error
+
@parameterized.expand([("negative", -1), ("boolean", True), ("string", "3"), ("null", None)])
def test_malformed_total_falls_back_to_short_page_termination(self, _name: str, total: Any) -> None:
# A total that cannot bound the walk must not end it early: a negative or boolean