Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse_sources): stop the shopify orders page size from swinging (#120390)

Daniel RCe829fa1f5436
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py
index 15ad3a27e1c5..93468210fc0d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py
@@ -432,8 +432,9 @@ def _make_paginated_shopify_request(
vars["pageSize"] = starting_page_size
continue
if size_page_from_cost:
- # The starting size already works, so the cost only ever grows the page from it.
- next_page_size = max(starting_page_size, _page_size_for_cost(vars["pageSize"], payload))
+ # A size Shopify answered is known to fit, so the cost only ever grows the page. The
+ # requested cost is not linear in the page size, and shrinking on it made the size swing.
+ next_page_size = max(vars["pageSize"], _page_size_for_cost(vars["pageSize"], payload))
if next_page_size != vars["pageSize"]:
logger.debug(
f"Changing page size for {graphql_object.name} from {vars['pageSize']} to {next_page_size}. "
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_resumable.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_resumable.py
index 7f17f2781cbf..79f764eb4bce 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_resumable.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_resumable.py
@@ -94,24 +94,29 @@ class TestMakePaginatedShopifyRequest:
assert saved_configs[1] == ShopifyResumeConfig(phase=PHASE_ALL, cursor="cursor-2")
@pytest.mark.parametrize(
- "object_name, cost, expected_page_sizes",
+ "object_name, costs, expected_page_sizes",
[
# 450 points at 20 rows leaves room for 40 rows under the 900 point target.
- (ORDERS, {"requestedQueryCost": 450}, [20, 40]),
- (ORDERS, {"requestedQueryCost": 10}, [20, 250]),
+ (ORDERS, [{"requestedQueryCost": 450}], [20, 40]),
+ (ORDERS, [{"requestedQueryCost": 10}], [20, 250]),
# Half of a 400 point bucket is 200 points, which is 40 rows at 5 points a row.
- (ORDERS, {"requestedQueryCost": 100, "throttleStatus": {"maximumAvailable": 400}}, [20, 40]),
- (ABANDONED_CHECKOUTS, {"requestedQueryCost": 450}, [100, 100]),
+ (ORDERS, [{"requestedQueryCost": 100, "throttleStatus": {"maximumAvailable": 400}}], [20, 40]),
+ # A cost above the target at a size Shopify answered keeps that size.
+ (ORDERS, [{"requestedQueryCost": 782}, {"requestedQueryCost": 938}], [20, 23, 23]),
+ (ABANDONED_CHECKOUTS, [{"requestedQueryCost": 450}], [100, 100]),
],
)
def test_grows_page_from_reported_cost_only_for_cost_limited_objects(
- self, object_name: str, cost: dict[str, Any], expected_page_sizes: list[int]
+ self, object_name: str, costs: list[dict[str, Any]], expected_page_sizes: list[int]
) -> None:
sess = MagicMock()
- first_page = _build_response(object_name, [{"id": "1"}], has_next_page=True, end_cursor="cursor-1")
- first_page.json.return_value["extensions"] = {"cost": cost}
+ pages = []
+ for index, cost in enumerate(costs):
+ page = _build_response(object_name, [{"id": str(index)}], has_next_page=True, end_cursor=f"cursor-{index}")
+ page.json.return_value["extensions"] = {"cost": cost}
+ pages.append(page)
captured = _record_post_vars(
- sess, [first_page, _build_response(object_name, [{"id": "2"}], has_next_page=False, end_cursor=None)]
+ sess, [*pages, _build_response(object_name, [{"id": "last"}], has_next_page=False, end_cursor=None)]
)
list(