:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse_sources): stop the shopify orders page size from swinging (#120390)
Daniel RCe829fa1f5436
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.pyindex 15ad3a27e1c5..93468210fc0d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py@@ -432,8 +432,9 @@ def _make_paginated_shopify_request(vars["pageSize"] = starting_page_sizecontinueif size_page_from_cost:- # The starting size already works, so the cost only ever grows the page from it.- next_page_size = max(starting_page_size, _page_size_for_cost(vars["pageSize"], payload))+ # A size Shopify answered is known to fit, so the cost only ever grows the page. The+ # requested cost is not linear in the page size, and shrinking on it made the size swing.+ next_page_size = max(vars["pageSize"], _page_size_for_cost(vars["pageSize"], payload))if next_page_size != vars["pageSize"]:logger.debug(f"Changing page size for {graphql_object.name} from {vars['pageSize']} to {next_page_size}. "diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_resumable.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_resumable.pyindex 7f17f2781cbf..79f764eb4bce 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_resumable.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_resumable.py@@ -94,24 +94,29 @@ class TestMakePaginatedShopifyRequest:assert saved_configs[1] == ShopifyResumeConfig(phase=PHASE_ALL, cursor="cursor-2")@pytest.mark.parametrize(- "object_name, cost, expected_page_sizes",+ "object_name, costs, expected_page_sizes",[# 450 points at 20 rows leaves room for 40 rows under the 900 point target.- (ORDERS, {"requestedQueryCost": 450}, [20, 40]),- (ORDERS, {"requestedQueryCost": 10}, [20, 250]),+ (ORDERS, [{"requestedQueryCost": 450}], [20, 40]),+ (ORDERS, [{"requestedQueryCost": 10}], [20, 250]),# Half of a 400 point bucket is 200 points, which is 40 rows at 5 points a row.- (ORDERS, {"requestedQueryCost": 100, "throttleStatus": {"maximumAvailable": 400}}, [20, 40]),- (ABANDONED_CHECKOUTS, {"requestedQueryCost": 450}, [100, 100]),+ (ORDERS, [{"requestedQueryCost": 100, "throttleStatus": {"maximumAvailable": 400}}], [20, 40]),+ # A cost above the target at a size Shopify answered keeps that size.+ (ORDERS, [{"requestedQueryCost": 782}, {"requestedQueryCost": 938}], [20, 23, 23]),+ (ABANDONED_CHECKOUTS, [{"requestedQueryCost": 450}], [100, 100]),],)def test_grows_page_from_reported_cost_only_for_cost_limited_objects(- self, object_name: str, cost: dict[str, Any], expected_page_sizes: list[int]+ self, object_name: str, costs: list[dict[str, Any]], expected_page_sizes: list[int]) -> None:sess = MagicMock()- first_page = _build_response(object_name, [{"id": "1"}], has_next_page=True, end_cursor="cursor-1")- first_page.json.return_value["extensions"] = {"cost": cost}+ pages = []+ for index, cost in enumerate(costs):+ page = _build_response(object_name, [{"id": str(index)}], has_next_page=True, end_cursor=f"cursor-{index}")+ page.json.return_value["extensions"] = {"cost": cost}+ pages.append(page)captured = _record_post_vars(- sess, [first_page, _build_response(object_name, [{"id": "2"}], has_next_page=False, end_cursor=None)]+ sess, [*pages, _build_response(object_name, [{"id": "last"}], has_next_page=False, end_cursor=None)])list(