:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add bigcommerce order and customer group tables (#119538)
Tom Owersd4ac292b1301
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 60e2c763104b..788b643d6d52 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -1564,14 +1564,14 @@ Today (5): `brands`, `categories`, `customers`, `orders`, `products`Diffed against: <https://raw.githubusercontent.com/bigcommerce/api-specs/master/reference/orders.v2.oas2.yml>-- [ ] `Order statuses (GET /v2/order_statuses)` — resolve status_id on imported orders to status names and descriptions (high)-- [ ] `Customer groups (GET /v2/customer_groups)` — resolve customer_group_id on imported customers and compare purchasing behavior across customer segments (high)+- [x] `Order statuses (GET /v2/order_statuses)` — resolve status_id on imported orders to status names and descriptions (high)+- [x] `Customer groups (GET /v2/customer_groups)` — resolve customer_group_id on imported customers and compare purchasing behavior across customer segments (high)- [x] `Channels (GET /v3/channels)` — resolve channel_id on orders and channel references on customers to sales channel names and types (high)-- [ ] `Order products (GET /v2/orders/{order_id}/products)` — measure item sales, quantities, discounts, and product revenue from order line details (high)+- [x] `Order products (GET /v2/orders/{order_id}/products)` — measure item sales, quantities, discounts, and product revenue from order line details (high)- [x] `Product variants (GET /v3/catalog/variants)` — analyze SKU prices and stock, and resolve variant IDs in order line records (high)- [x] `Refunds (GET /v3/orders/payment_actions/refunds)` — calculate net sales and analyze refund amounts, reasons, and refunded items (high)- [x] `Order transactions (GET /v3/orders/{order_id}/transactions)` — reconcile payments, captures, refunds, and payment provider outcomes against orders (high)-- [ ] `Order shipments (GET /v2/orders/{order_id}/shipments)` — measure fulfillment and connect shipped quantities, tracking references, and shipping providers to orders (medium)+- [x] `Order shipments (GET /v2/orders/{order_id}/shipments)` — measure fulfillment and connect shipped quantities, tracking references, and shipping providers to orders (medium)- [ ] `Order coupons (GET /v2/orders/{order_id}/coupons)` — attribute discounts and order revenue to specific coupon codes (medium)- [ ] `Product reviews (GET /v3/catalog/products/{product_id}/reviews)` — compare product ratings and review activity with sales and repeat purchases (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/bigcommerce.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/bigcommerce.pyindex a5c0373ca55a..8ae5ad2ae971 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/bigcommerce.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/bigcommerce.py@@ -6,6 +6,7 @@ from typing import Any, Optionalfrom products.warehouse_sources.backend.temporal.data_imports.sources.bigcommerce.settings import (ENDPOINT_PATHS,INCREMENTAL_FILTERS,+ UNPAGINATED_ENDPOINTS,V2_ENDPOINTS,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session@@ -30,15 +31,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resBASE_URL = "https://api.bigcommerce.com"# BigCommerce caps `limit` at 250 on both the V3 catalog/customers collections and the-# legacy V2 orders collection.+# legacy V2 collections.PAGE_SIZE = 250@dataclasses.dataclass(frozen=True)class BigCommerceResumeConfig:- # Next page of a flat listing, or the orders page an order transactions sync is on.+ # Next page of a flat listing, or the orders page a per-order sync is on.page: Optional[int] = None- # Order transactions only: how many orders on `page` are already read.+ # Per-order endpoints only: how many orders on `page` are already read.order_index: int = 0@@ -66,7 +67,7 @@ def get_resource(name: str, should_use_incremental_field: bool) -> EndpointResouis_v2 = name in V2_ENDPOINTSuse_incremental = should_use_incremental_field and name in INCREMENTAL_FILTERS- params: dict[str, Any] = {"limit": PAGE_SIZE}+ params: dict[str, Any] = {} if name in UNPAGINATED_ENDPOINTS else {"limit": PAGE_SIZE}if use_incremental:filter_param, cursor_path = INCREMENTAL_FILTERS[name]params[filter_param] = {@@ -78,18 +79,22 @@ def get_resource(name: str, should_use_incremental_field: bool) -> EndpointResou# V3 reports the page count directly, so pagination stops without an extra empty-page# request. V2 has no such envelope, so it falls back to stopping on a short/empty page.- paginator = PageNumberPaginator(- base_page=1,- page_param="page",- total_path=None if is_v2 else "meta.pagination.total_pages",+ paginator: PageNumberPaginator | SinglePagePaginator = (+ SinglePagePaginator()+ if name in UNPAGINATED_ENDPOINTS+ else PageNumberPaginator(+ base_page=1,+ page_param="page",+ total_path=None if is_v2 else "meta.pagination.total_pages",+ ))endpoint: Endpoint = {"path": path,"params": params,"paginator": paginator,- # V3 wraps rows in {"data": [...], "meta": {...}}; the legacy V2 orders endpoint- # returns a bare JSON array.+ # V3 wraps rows in {"data": [...], "meta": {...}}; the legacy V2 endpoints return a+ # bare JSON array."data_selector": None if is_v2 else "data",}@@ -149,23 +154,56 @@ def bigcommerce_source()-def order_transactions_source(+def _order_children(+ client: RESTClient, endpoint: str, order_id: Any, hooks: Optional[dict[str, Any]]+) -> list[dict[str, Any]]:+ path = ENDPOINT_PATHS[endpoint].format(order_id=order_id)+ if endpoint not in V2_ENDPOINTS:+ return [+ row+ for page_rows in client.paginate(+ path=path, paginator=SinglePagePaginator(), data_selector="data", hooks=hooks+ )+ for row in page_rows+ ]++ # V2 answers an empty page with 204 and no pagination envelope, so a short page is the last one.+ rows: list[dict[str, Any]] = []+ page = 1+ while True:+ page_rows = next(+ client.paginate(+ path=path,+ params={"limit": PAGE_SIZE, "page": page},+ paginator=SinglePagePaginator(),+ hooks=hooks,+ ),+ [],+ )+ rows.extend(page_rows)+ if len(page_rows) < PAGE_SIZE:+ return rows+ page += 1+++def order_fan_out_source(store_hash: str,access_token: str,+ endpoint: str,resumable_source_manager: ResumableSourceManager[BigCommerceResumeConfig],) -> Iterator[list[dict[str, Any]]]:- """Read the transactions of every order, one request per order.+ """Read the rows of a per-order endpoint for every order, one order at a time.- BigCommerce has no collection endpoint for transactions, so this walks the V2 orders listing- page by page. The resume state is the orders page and the position in it, which keeps each- checkpoint the same size however many orders the store has.+ BigCommerce has no collection endpoint for order products, shipments or transactions, so this+ walks the V2 orders listing page by page. The resume state is the orders page and the position+ in it, which keeps each checkpoint the same size however many orders the store has."""client = RESTClient(base_url=f"{BASE_URL}/stores/{store_hash}",headers={"Accept": "application/json"},auth=APIKeyAuth(api_key=access_token, name="X-Auth-Token", location="header"),)- # An order deleted between the orders listing and its transactions request has nothing to read.+ # An order deleted between the orders listing and its child request has nothing to read.skip_deleted_order = create_response_hooks([{"status_code": 404, "action": "ignore"}])page, order_index = 1, 0@@ -187,19 +225,10 @@ def order_transactions_source(returnfor index in range(order_index, len(orders)):- transactions = [- row- for transactions_page in client.paginate(- path=ENDPOINT_PATHS["order_transactions"].format(order_id=orders[index]["id"]),- paginator=SinglePagePaginator(),- data_selector="data",- hooks=skip_deleted_order,- )- for row in transactions_page- ]+ rows = _order_children(client, endpoint, orders[index]["id"], skip_deleted_order)resumable_source_manager.save_state(BigCommerceResumeConfig(page=page, order_index=index + 1))- if transactions:- yield transactions+ if rows:+ yield rowselse:resumable_source_manager.safe_point()diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/canonical_descriptions.pyindex c31241d57f5e..76b84ec1ee53 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/canonical_descriptions.py@@ -160,6 +160,69 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"date_created": "Date and time of the transaction.",},},+ "customer_groups": {+ "description": "A group of customers that shares category access and automatic discounts.",+ "docs_url": "https://developer.bigcommerce.com/docs/rest-management/customers-v2/customer-groups",+ "columns": {+ "id": "Unique identifier for the customer group. Matches `customer_group_id` on customers.",+ "name": "Name of the group.",+ "is_default": "Whether new customers are assigned to this group by default.",+ "category_access": "Catalog categories that members of the group can access.",+ "discount_rules": "Discount rules applied automatically to members of the group.",+ },+ },+ "order_statuses": {+ "description": "An order status the store uses, such as `Pending`, `Shipped` or `Refunded`.",+ "docs_url": "https://developer.bigcommerce.com/docs/rest-management/orders/order-status",+ "columns": {+ "id": "Unique identifier for the order status. Matches `status_id` on orders.",+ "name": "Name of the order status.",+ "system_label": "System name of the order status.",+ "custom_label": "Custom label for the order status, set in the control panel.",+ "system_description": "System description of the order status.",+ },+ },+ "order_products": {+ "description": "A line item on an order, with its quantities, prices, taxes and discounts.",+ "docs_url": "https://developer.bigcommerce.com/docs/rest-management/orders/order-products",+ "columns": {+ "id": "Unique identifier for the order line item.",+ "order_id": "Identifier of the order the line item belongs to.",+ "product_id": "Identifier of the catalog product.",+ "variant_id": "Identifier of the product variant. Not set for custom products.",+ "name": "Name of the product.",+ "sku": "Stock keeping unit (SKU) of the product.",+ "quantity": "Quantity of the product ordered.",+ "base_price": "Price of one item, as entered in the catalog.",+ "price_ex_tax": "Price of one item, excluding tax.",+ "price_inc_tax": "Price of one item, including tax.",+ "total_ex_tax": "Total for the line item, excluding tax.",+ "total_inc_tax": "Total for the line item, including tax.",+ "total_tax": "Total tax for the line item.",+ "applied_discounts": "Discounts applied to the line item.",+ "quantity_shipped": "Quantity of the item shipped.",+ "is_refunded": "Whether the item has been refunded.",+ "quantity_refunded": "Quantity of the item refunded.",+ },+ },+ "order_shipments": {+ "description": "A shipment of items from an order, with its tracking details and shipping provider.",+ "docs_url": "https://developer.bigcommerce.com/docs/rest-management/orders/order-shipments",+ "columns": {+ "id": "Unique identifier for the shipment.",+ "order_id": "Identifier of the order the shipment belongs to.",+ "customer_id": "Identifier of the order's customer.",+ "order_address_id": "Identifier of the order shipping address the shipment goes to.",+ "date_created": "Date and time the shipment was created.",+ "tracking_number": "Tracking number of the shipment.",+ "shipping_method": "Shipping method used for the shipment.",+ "shipping_provider": "BigCommerce shipping provider integration used for the shipment.",+ "tracking_carrier": "Carrier used to build the tracking link.",+ "tracking_link": "Custom tracking link supplied for the shipment.",+ "generated_tracking_link": "Tracking link generated from the tracking number and the provider or carrier.",+ "items": "Order line items in the shipment, with the quantity of each.",+ },+ },"refunds": {"description": "A refund issued against an order, with the refunded items and the payments returned to the shopper.","docs_url": "https://developer.bigcommerce.com/docs/rest-management/orders/payment-actions",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/settings.pyindex 90513c217ff9..e3c09a067dbe 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/settings.py@@ -11,7 +11,11 @@ ENDPOINTS = ("brands","customers","channels",+ "customer_groups","orders",+ "order_statuses",+ "order_products",+ "order_shipments","order_transactions","refunds",)@@ -25,18 +29,28 @@ ENDPOINT_PATHS: dict[str, str] = {"brands": "/v3/catalog/brands","customers": "/v3/customers","channels": "/v3/channels",+ "customer_groups": "/v2/customer_groups","orders": "/v2/orders",+ "order_statuses": "/v2/order_statuses",+ "order_products": "/v2/orders/{order_id}/products",+ "order_shipments": "/v2/orders/{order_id}/shipments","order_transactions": "/v3/orders/{order_id}/transactions","refunds": "/v3/orders/payment_actions/refunds",}# Endpoints still served by the legacy V2 API: no `data` envelope, no `meta.pagination`,# and a different incremental filter param + date format than V3.-V2_ENDPOINTS = frozenset({"orders"})+V2_ENDPOINTS = frozenset({"customer_groups", "orders", "order_statuses", "order_products", "order_shipments"})-# Endpoint -> (server-side "since" filter param, cursor field it filters on). Categories, brands-# and variants carry no modification timestamp, and order transactions have no filter, so those-# are full refresh only.+# Order statuses is a fixed list of BigCommerce's order states with no `page`/`limit` params.+UNPAGINATED_ENDPOINTS = frozenset({"order_statuses"})++# Endpoints read per order, because BigCommerce has no collection endpoint for them.+ORDER_FAN_OUT_ENDPOINTS = frozenset({"order_products", "order_shipments", "order_transactions"})++# Endpoint -> (server-side "since" filter param, cursor field it filters on). Categories, brands,+# variants, customer groups and order statuses carry no modification timestamp, and the per-order+# endpoints have no filter, so those are full refresh only.INCREMENTAL_FILTERS: dict[str, tuple[str, str]] = {"products": ("date_modified:min", "date_modified"),"customers": ("date_modified:min", "date_modified"),@@ -58,19 +72,23 @@ INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {for endpoint, (_, field) in INCREMENTAL_FILTERS.items()}-# Transactions are read per order. The order id joins the key so it stays unique table-wide even-# if a store reuses transaction ids across orders.+# Per-order rows are read one order at a time. The order id joins the key so it stays unique+# table-wide even if a store reuses child ids across orders.PRIMARY_KEYS: dict[str, list[str]] = {+ "order_products": ["order_id", "id"],+ "order_shipments": ["order_id", "id"],"order_transactions": ["order_id", "id"],}-# Stable creation-time field to partition on, per endpoint. Categories, brands and variants-# have no creation timestamp field and are left unpartitioned.+# Stable creation-time field to partition on, per endpoint. Categories, brands, variants,+# customer groups, order statuses and order products have no creation timestamp field and are+# left unpartitioned.PARTITION_FIELDS: dict[str, str] = {"products": "date_created","customers": "date_created","channels": "date_created","orders": "date_created",+ "order_shipments": "date_created","order_transactions": "date_created","refunds": "created",}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/source.pyindex 4d076cd12b83..12911ced024b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/source.py@@ -11,12 +11,13 @@ from products.warehouse_sources.backend.facade.source_config import (from products.warehouse_sources.backend.temporal.data_imports.sources.bigcommerce.bigcommerce import (BigCommerceResumeConfig,bigcommerce_source,- order_transactions_source,+ order_fan_out_source,validate_credentials as validate_bigcommerce_credentials,)from products.warehouse_sources.backend.temporal.data_imports.sources.bigcommerce.settings import (ENDPOINTS,INCREMENTAL_FIELDS,+ ORDER_FAN_OUT_ENDPOINTS,PARTITION_FIELDS,PRIMARY_KEYS,)@@ -129,11 +130,11 @@ class BigCommerceSource(ResumableSource[BigCommerceSourceConfig, BigCommerceResu# incrementally — guard against a stale schema requesting it elsewhere.use_incremental = inputs.should_use_incremental_field and inputs.schema_name in INCREMENTAL_FIELDS- if inputs.schema_name == "order_transactions":+ if inputs.schema_name in ORDER_FAN_OUT_ENDPOINTS:response = SourceResponse(name=inputs.schema_name,- items=lambda: order_transactions_source(- config.store_hash, config.access_token, resumable_source_manager+ items=lambda: order_fan_out_source(+ config.store_hash, config.access_token, inputs.schema_name, resumable_source_manager),primary_keys=PRIMARY_KEYS[inputs.schema_name],sort_mode="asc",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/tests/test_bigcommerce.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/tests/test_bigcommerce.pyindex 34dad0a08b46..c6f9ad7a5140 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/tests/test_bigcommerce.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/tests/test_bigcommerce.py@@ -6,6 +6,7 @@ import pytestfrom parameterized import parameterizedfrom products.warehouse_sources.backend.temporal.data_imports.sources.bigcommerce.bigcommerce import (+ PAGE_SIZE,BigCommerceResumeConfig,_to_v2_timestamp,_to_v3_timestamp,@@ -36,6 +37,7 @@ class TestGetResource:("brands", "/v3/catalog/brands", "data", True),("customers", "/v3/customers", "data", True),("orders", "/v2/orders", None, False),+ ("customer_groups", "/v2/customer_groups", None, False),])def test_path_and_envelope_per_endpoint(@@ -242,6 +244,50 @@ class TestBigCommerceSourceResumeBehavior:assert result.params("page") == ["4", None, "5"]assert result.rows == [{"id": 3, "order_id": 102}]+ @parameterized.expand([("order_products", "products"), ("order_shipments", "shipments")])+ def test_v2_order_children_page_until_short_page(self, endpoint: str, child_path: str) -> None:+ full_page = [{"id": child_id, "order_id": 101} for child_id in range(PAGE_SIZE)]+ result = self._driver().run(+ endpoint,+ [+ ScriptedResponse(json=[{"id": 101}, {"id": 102}, {"id": 103}]),+ ScriptedResponse(json=full_page),+ ScriptedResponse(json=[{"id": PAGE_SIZE, "order_id": 101}]),+ ScriptedResponse(status=204),+ # Order deleted after the order listing was read.+ ScriptedResponse(+ status=404, json=[{"status": 404, "message": "The requested resource was not found."}]+ ),+ ScriptedResponse(json=[]),+ ],+ )++ assert result.raised is None+ assert [request.path for request in result.requests] == [+ "/stores/store123/v2/orders",+ f"/stores/store123/v2/orders/101/{child_path}",+ f"/stores/store123/v2/orders/101/{child_path}",+ f"/stores/store123/v2/orders/102/{child_path}",+ f"/stores/store123/v2/orders/103/{child_path}",+ "/stores/store123/v2/orders",+ ]+ assert result.params("page") == ["1", "1", "2", "1", "1", "2"]+ assert [row["id"] for row in result.rows] == list(range(PAGE_SIZE + 1))+ assert result.saved_states == [BigCommerceResumeConfig(page=1, order_index=i) for i in (1, 2, 3)]++ def test_order_statuses_is_read_in_one_request(self) -> None:+ statuses = [{"id": 0, "name": "Incomplete"}, {"id": 1, "name": "Pending"}]+ result = self._driver().run(+ "order_statuses",+ # The endpoint ignores `page`, so a second request would return the same rows again.+ [ScriptedResponse(json=statuses), ScriptedResponse(json=statuses)],+ )++ assert result.raised is None+ assert [request.path for request in result.requests] == ["/stores/store123/v2/order_statuses"]+ assert result.params("page") == [None]+ assert result.rows == statuses+def test_refunds_stop_on_empty_page_without_pagination_meta(self) -> None:last_value = datetime(2024, 5, 6, 7, 8, 9, tzinfo=UTC)result = self._driver().run(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/tests/test_bigcommerce_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/tests/test_bigcommerce_source.pyindex 19a8dd8c9e8b..41d4c56c48c7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/tests/test_bigcommerce_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/tests/test_bigcommerce_source.py@@ -4,7 +4,9 @@ from parameterized import parameterizedfrom products.warehouse_sources.backend.temporal.data_imports.sources.bigcommerce.settings import (ENDPOINTS,+ ORDER_FAN_OUT_ENDPOINTS,PARTITION_FIELDS,+ V2_ENDPOINTS,)from products.warehouse_sources.backend.temporal.data_imports.sources.bigcommerce.source import BigCommerceSourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (@@ -26,8 +28,8 @@ class TestBigCommerceSource:@staticmethoddef _empty_page(endpoint: str) -> ScriptedResponse:- # Order transactions fan out over the V2 orders listing.- if endpoint in ("orders", "order_transactions"):+ # Per-order endpoints fan out over the V2 orders listing.+ if endpoint in V2_ENDPOINTS | ORDER_FAN_OUT_ENDPOINTS:return ScriptedResponse(json=[])return ScriptedResponse(json={"data": [], "meta": {"pagination": {"current_page": 1, "total_pages": 1}}})