Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add bigcommerce order and customer group tables (#119538)

Tom Owersd4ac292b1301
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 60e2c763104b..788b643d6d52 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -1564,14 +1564,14 @@ Today (5): `brands`, `categories`, `customers`, `orders`, `products`
Diffed against: <https://raw.githubusercontent.com/bigcommerce/api-specs/master/reference/orders.v2.oas2.yml>
-- [ ] `Order statuses (GET /v2/order_statuses)` — resolve status_id on imported orders to status names and descriptions (high)
-- [ ] `Customer groups (GET /v2/customer_groups)` — resolve customer_group_id on imported customers and compare purchasing behavior across customer segments (high)
+- [x] `Order statuses (GET /v2/order_statuses)` — resolve status_id on imported orders to status names and descriptions (high)
+- [x] `Customer groups (GET /v2/customer_groups)` — resolve customer_group_id on imported customers and compare purchasing behavior across customer segments (high)
- [x] `Channels (GET /v3/channels)` — resolve channel_id on orders and channel references on customers to sales channel names and types (high)
-- [ ] `Order products (GET /v2/orders/{order_id}/products)` — measure item sales, quantities, discounts, and product revenue from order line details (high)
+- [x] `Order products (GET /v2/orders/{order_id}/products)` — measure item sales, quantities, discounts, and product revenue from order line details (high)
- [x] `Product variants (GET /v3/catalog/variants)` — analyze SKU prices and stock, and resolve variant IDs in order line records (high)
- [x] `Refunds (GET /v3/orders/payment_actions/refunds)` — calculate net sales and analyze refund amounts, reasons, and refunded items (high)
- [x] `Order transactions (GET /v3/orders/{order_id}/transactions)` — reconcile payments, captures, refunds, and payment provider outcomes against orders (high)
-- [ ] `Order shipments (GET /v2/orders/{order_id}/shipments)` — measure fulfillment and connect shipped quantities, tracking references, and shipping providers to orders (medium)
+- [x] `Order shipments (GET /v2/orders/{order_id}/shipments)` — measure fulfillment and connect shipped quantities, tracking references, and shipping providers to orders (medium)
- [ ] `Order coupons (GET /v2/orders/{order_id}/coupons)` — attribute discounts and order revenue to specific coupon codes (medium)
- [ ] `Product reviews (GET /v3/catalog/products/{product_id}/reviews)` — compare product ratings and review activity with sales and repeat purchases (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/bigcommerce.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/bigcommerce.py
index a5c0373ca55a..8ae5ad2ae971 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/bigcommerce.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/bigcommerce.py
@@ -6,6 +6,7 @@ from typing import Any, Optional
from products.warehouse_sources.backend.temporal.data_imports.sources.bigcommerce.settings import (
ENDPOINT_PATHS,
INCREMENTAL_FILTERS,
+ UNPAGINATED_ENDPOINTS,
V2_ENDPOINTS,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
@@ -30,15 +31,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
BASE_URL = "https://api.bigcommerce.com"
# BigCommerce caps `limit` at 250 on both the V3 catalog/customers collections and the
-# legacy V2 orders collection.
+# legacy V2 collections.
PAGE_SIZE = 250
@dataclasses.dataclass(frozen=True)
class BigCommerceResumeConfig:
- # Next page of a flat listing, or the orders page an order transactions sync is on.
+ # Next page of a flat listing, or the orders page a per-order sync is on.
page: Optional[int] = None
- # Order transactions only: how many orders on `page` are already read.
+ # Per-order endpoints only: how many orders on `page` are already read.
order_index: int = 0
@@ -66,7 +67,7 @@ def get_resource(name: str, should_use_incremental_field: bool) -> EndpointResou
is_v2 = name in V2_ENDPOINTS
use_incremental = should_use_incremental_field and name in INCREMENTAL_FILTERS
- params: dict[str, Any] = {"limit": PAGE_SIZE}
+ params: dict[str, Any] = {} if name in UNPAGINATED_ENDPOINTS else {"limit": PAGE_SIZE}
if use_incremental:
filter_param, cursor_path = INCREMENTAL_FILTERS[name]
params[filter_param] = {
@@ -78,18 +79,22 @@ def get_resource(name: str, should_use_incremental_field: bool) -> EndpointResou
# V3 reports the page count directly, so pagination stops without an extra empty-page
# request. V2 has no such envelope, so it falls back to stopping on a short/empty page.
- paginator = PageNumberPaginator(
- base_page=1,
- page_param="page",
- total_path=None if is_v2 else "meta.pagination.total_pages",
+ paginator: PageNumberPaginator | SinglePagePaginator = (
+ SinglePagePaginator()
+ if name in UNPAGINATED_ENDPOINTS
+ else PageNumberPaginator(
+ base_page=1,
+ page_param="page",
+ total_path=None if is_v2 else "meta.pagination.total_pages",
+ )
)
endpoint: Endpoint = {
"path": path,
"params": params,
"paginator": paginator,
- # V3 wraps rows in {"data": [...], "meta": {...}}; the legacy V2 orders endpoint
- # returns a bare JSON array.
+ # V3 wraps rows in {"data": [...], "meta": {...}}; the legacy V2 endpoints return a
+ # bare JSON array.
"data_selector": None if is_v2 else "data",
}
@@ -149,23 +154,56 @@ def bigcommerce_source(
)
-def order_transactions_source(
+def _order_children(
+ client: RESTClient, endpoint: str, order_id: Any, hooks: Optional[dict[str, Any]]
+) -> list[dict[str, Any]]:
+ path = ENDPOINT_PATHS[endpoint].format(order_id=order_id)
+ if endpoint not in V2_ENDPOINTS:
+ return [
+ row
+ for page_rows in client.paginate(
+ path=path, paginator=SinglePagePaginator(), data_selector="data", hooks=hooks
+ )
+ for row in page_rows
+ ]
+
+ # V2 answers an empty page with 204 and no pagination envelope, so a short page is the last one.
+ rows: list[dict[str, Any]] = []
+ page = 1
+ while True:
+ page_rows = next(
+ client.paginate(
+ path=path,
+ params={"limit": PAGE_SIZE, "page": page},
+ paginator=SinglePagePaginator(),
+ hooks=hooks,
+ ),
+ [],
+ )
+ rows.extend(page_rows)
+ if len(page_rows) < PAGE_SIZE:
+ return rows
+ page += 1
+
+
+def order_fan_out_source(
store_hash: str,
access_token: str,
+ endpoint: str,
resumable_source_manager: ResumableSourceManager[BigCommerceResumeConfig],
) -> Iterator[list[dict[str, Any]]]:
- """Read the transactions of every order, one request per order.
+ """Read the rows of a per-order endpoint for every order, one order at a time.
- BigCommerce has no collection endpoint for transactions, so this walks the V2 orders listing
- page by page. The resume state is the orders page and the position in it, which keeps each
- checkpoint the same size however many orders the store has.
+ BigCommerce has no collection endpoint for order products, shipments or transactions, so this
+ walks the V2 orders listing page by page. The resume state is the orders page and the position
+ in it, which keeps each checkpoint the same size however many orders the store has.
"""
client = RESTClient(
base_url=f"{BASE_URL}/stores/{store_hash}",
headers={"Accept": "application/json"},
auth=APIKeyAuth(api_key=access_token, name="X-Auth-Token", location="header"),
)
- # An order deleted between the orders listing and its transactions request has nothing to read.
+ # An order deleted between the orders listing and its child request has nothing to read.
skip_deleted_order = create_response_hooks([{"status_code": 404, "action": "ignore"}])
page, order_index = 1, 0
@@ -187,19 +225,10 @@ def order_transactions_source(
return
for index in range(order_index, len(orders)):
- transactions = [
- row
- for transactions_page in client.paginate(
- path=ENDPOINT_PATHS["order_transactions"].format(order_id=orders[index]["id"]),
- paginator=SinglePagePaginator(),
- data_selector="data",
- hooks=skip_deleted_order,
- )
- for row in transactions_page
- ]
+ rows = _order_children(client, endpoint, orders[index]["id"], skip_deleted_order)
resumable_source_manager.save_state(BigCommerceResumeConfig(page=page, order_index=index + 1))
- if transactions:
- yield transactions
+ if rows:
+ yield rows
else:
resumable_source_manager.safe_point()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/canonical_descriptions.py
index c31241d57f5e..76b84ec1ee53 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/canonical_descriptions.py
@@ -160,6 +160,69 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"date_created": "Date and time of the transaction.",
},
},
+ "customer_groups": {
+ "description": "A group of customers that shares category access and automatic discounts.",
+ "docs_url": "https://developer.bigcommerce.com/docs/rest-management/customers-v2/customer-groups",
+ "columns": {
+ "id": "Unique identifier for the customer group. Matches `customer_group_id` on customers.",
+ "name": "Name of the group.",
+ "is_default": "Whether new customers are assigned to this group by default.",
+ "category_access": "Catalog categories that members of the group can access.",
+ "discount_rules": "Discount rules applied automatically to members of the group.",
+ },
+ },
+ "order_statuses": {
+ "description": "An order status the store uses, such as `Pending`, `Shipped` or `Refunded`.",
+ "docs_url": "https://developer.bigcommerce.com/docs/rest-management/orders/order-status",
+ "columns": {
+ "id": "Unique identifier for the order status. Matches `status_id` on orders.",
+ "name": "Name of the order status.",
+ "system_label": "System name of the order status.",
+ "custom_label": "Custom label for the order status, set in the control panel.",
+ "system_description": "System description of the order status.",
+ },
+ },
+ "order_products": {
+ "description": "A line item on an order, with its quantities, prices, taxes and discounts.",
+ "docs_url": "https://developer.bigcommerce.com/docs/rest-management/orders/order-products",
+ "columns": {
+ "id": "Unique identifier for the order line item.",
+ "order_id": "Identifier of the order the line item belongs to.",
+ "product_id": "Identifier of the catalog product.",
+ "variant_id": "Identifier of the product variant. Not set for custom products.",
+ "name": "Name of the product.",
+ "sku": "Stock keeping unit (SKU) of the product.",
+ "quantity": "Quantity of the product ordered.",
+ "base_price": "Price of one item, as entered in the catalog.",
+ "price_ex_tax": "Price of one item, excluding tax.",
+ "price_inc_tax": "Price of one item, including tax.",
+ "total_ex_tax": "Total for the line item, excluding tax.",
+ "total_inc_tax": "Total for the line item, including tax.",
+ "total_tax": "Total tax for the line item.",
+ "applied_discounts": "Discounts applied to the line item.",
+ "quantity_shipped": "Quantity of the item shipped.",
+ "is_refunded": "Whether the item has been refunded.",
+ "quantity_refunded": "Quantity of the item refunded.",
+ },
+ },
+ "order_shipments": {
+ "description": "A shipment of items from an order, with its tracking details and shipping provider.",
+ "docs_url": "https://developer.bigcommerce.com/docs/rest-management/orders/order-shipments",
+ "columns": {
+ "id": "Unique identifier for the shipment.",
+ "order_id": "Identifier of the order the shipment belongs to.",
+ "customer_id": "Identifier of the order's customer.",
+ "order_address_id": "Identifier of the order shipping address the shipment goes to.",
+ "date_created": "Date and time the shipment was created.",
+ "tracking_number": "Tracking number of the shipment.",
+ "shipping_method": "Shipping method used for the shipment.",
+ "shipping_provider": "BigCommerce shipping provider integration used for the shipment.",
+ "tracking_carrier": "Carrier used to build the tracking link.",
+ "tracking_link": "Custom tracking link supplied for the shipment.",
+ "generated_tracking_link": "Tracking link generated from the tracking number and the provider or carrier.",
+ "items": "Order line items in the shipment, with the quantity of each.",
+ },
+ },
"refunds": {
"description": "A refund issued against an order, with the refunded items and the payments returned to the shopper.",
"docs_url": "https://developer.bigcommerce.com/docs/rest-management/orders/payment-actions",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/settings.py
index 90513c217ff9..e3c09a067dbe 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/settings.py
@@ -11,7 +11,11 @@ ENDPOINTS = (
"brands",
"customers",
"channels",
+ "customer_groups",
"orders",
+ "order_statuses",
+ "order_products",
+ "order_shipments",
"order_transactions",
"refunds",
)
@@ -25,18 +29,28 @@ ENDPOINT_PATHS: dict[str, str] = {
"brands": "/v3/catalog/brands",
"customers": "/v3/customers",
"channels": "/v3/channels",
+ "customer_groups": "/v2/customer_groups",
"orders": "/v2/orders",
+ "order_statuses": "/v2/order_statuses",
+ "order_products": "/v2/orders/{order_id}/products",
+ "order_shipments": "/v2/orders/{order_id}/shipments",
"order_transactions": "/v3/orders/{order_id}/transactions",
"refunds": "/v3/orders/payment_actions/refunds",
}
# Endpoints still served by the legacy V2 API: no `data` envelope, no `meta.pagination`,
# and a different incremental filter param + date format than V3.
-V2_ENDPOINTS = frozenset({"orders"})
+V2_ENDPOINTS = frozenset({"customer_groups", "orders", "order_statuses", "order_products", "order_shipments"})
-# Endpoint -> (server-side "since" filter param, cursor field it filters on). Categories, brands
-# and variants carry no modification timestamp, and order transactions have no filter, so those
-# are full refresh only.
+# Order statuses is a fixed list of BigCommerce's order states with no `page`/`limit` params.
+UNPAGINATED_ENDPOINTS = frozenset({"order_statuses"})
+
+# Endpoints read per order, because BigCommerce has no collection endpoint for them.
+ORDER_FAN_OUT_ENDPOINTS = frozenset({"order_products", "order_shipments", "order_transactions"})
+
+# Endpoint -> (server-side "since" filter param, cursor field it filters on). Categories, brands,
+# variants, customer groups and order statuses carry no modification timestamp, and the per-order
+# endpoints have no filter, so those are full refresh only.
INCREMENTAL_FILTERS: dict[str, tuple[str, str]] = {
"products": ("date_modified:min", "date_modified"),
"customers": ("date_modified:min", "date_modified"),
@@ -58,19 +72,23 @@ INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
for endpoint, (_, field) in INCREMENTAL_FILTERS.items()
}
-# Transactions are read per order. The order id joins the key so it stays unique table-wide even
-# if a store reuses transaction ids across orders.
+# Per-order rows are read one order at a time. The order id joins the key so it stays unique
+# table-wide even if a store reuses child ids across orders.
PRIMARY_KEYS: dict[str, list[str]] = {
+ "order_products": ["order_id", "id"],
+ "order_shipments": ["order_id", "id"],
"order_transactions": ["order_id", "id"],
}
-# Stable creation-time field to partition on, per endpoint. Categories, brands and variants
-# have no creation timestamp field and are left unpartitioned.
+# Stable creation-time field to partition on, per endpoint. Categories, brands, variants,
+# customer groups, order statuses and order products have no creation timestamp field and are
+# left unpartitioned.
PARTITION_FIELDS: dict[str, str] = {
"products": "date_created",
"customers": "date_created",
"channels": "date_created",
"orders": "date_created",
+ "order_shipments": "date_created",
"order_transactions": "date_created",
"refunds": "created",
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/source.py
index 4d076cd12b83..12911ced024b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/source.py
@@ -11,12 +11,13 @@ from products.warehouse_sources.backend.facade.source_config import (
from products.warehouse_sources.backend.temporal.data_imports.sources.bigcommerce.bigcommerce import (
BigCommerceResumeConfig,
bigcommerce_source,
- order_transactions_source,
+ order_fan_out_source,
validate_credentials as validate_bigcommerce_credentials,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.bigcommerce.settings import (
ENDPOINTS,
INCREMENTAL_FIELDS,
+ ORDER_FAN_OUT_ENDPOINTS,
PARTITION_FIELDS,
PRIMARY_KEYS,
)
@@ -129,11 +130,11 @@ class BigCommerceSource(ResumableSource[BigCommerceSourceConfig, BigCommerceResu
# incrementally — guard against a stale schema requesting it elsewhere.
use_incremental = inputs.should_use_incremental_field and inputs.schema_name in INCREMENTAL_FIELDS
- if inputs.schema_name == "order_transactions":
+ if inputs.schema_name in ORDER_FAN_OUT_ENDPOINTS:
response = SourceResponse(
name=inputs.schema_name,
- items=lambda: order_transactions_source(
- config.store_hash, config.access_token, resumable_source_manager
+ items=lambda: order_fan_out_source(
+ config.store_hash, config.access_token, inputs.schema_name, resumable_source_manager
),
primary_keys=PRIMARY_KEYS[inputs.schema_name],
sort_mode="asc",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/tests/test_bigcommerce.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/tests/test_bigcommerce.py
index 34dad0a08b46..c6f9ad7a5140 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/tests/test_bigcommerce.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/tests/test_bigcommerce.py
@@ -6,6 +6,7 @@ import pytest
from parameterized import parameterized
from products.warehouse_sources.backend.temporal.data_imports.sources.bigcommerce.bigcommerce import (
+ PAGE_SIZE,
BigCommerceResumeConfig,
_to_v2_timestamp,
_to_v3_timestamp,
@@ -36,6 +37,7 @@ class TestGetResource:
("brands", "/v3/catalog/brands", "data", True),
("customers", "/v3/customers", "data", True),
("orders", "/v2/orders", None, False),
+ ("customer_groups", "/v2/customer_groups", None, False),
]
)
def test_path_and_envelope_per_endpoint(
@@ -242,6 +244,50 @@ class TestBigCommerceSourceResumeBehavior:
assert result.params("page") == ["4", None, "5"]
assert result.rows == [{"id": 3, "order_id": 102}]
+ @parameterized.expand([("order_products", "products"), ("order_shipments", "shipments")])
+ def test_v2_order_children_page_until_short_page(self, endpoint: str, child_path: str) -> None:
+ full_page = [{"id": child_id, "order_id": 101} for child_id in range(PAGE_SIZE)]
+ result = self._driver().run(
+ endpoint,
+ [
+ ScriptedResponse(json=[{"id": 101}, {"id": 102}, {"id": 103}]),
+ ScriptedResponse(json=full_page),
+ ScriptedResponse(json=[{"id": PAGE_SIZE, "order_id": 101}]),
+ ScriptedResponse(status=204),
+ # Order deleted after the order listing was read.
+ ScriptedResponse(
+ status=404, json=[{"status": 404, "message": "The requested resource was not found."}]
+ ),
+ ScriptedResponse(json=[]),
+ ],
+ )
+
+ assert result.raised is None
+ assert [request.path for request in result.requests] == [
+ "/stores/store123/v2/orders",
+ f"/stores/store123/v2/orders/101/{child_path}",
+ f"/stores/store123/v2/orders/101/{child_path}",
+ f"/stores/store123/v2/orders/102/{child_path}",
+ f"/stores/store123/v2/orders/103/{child_path}",
+ "/stores/store123/v2/orders",
+ ]
+ assert result.params("page") == ["1", "1", "2", "1", "1", "2"]
+ assert [row["id"] for row in result.rows] == list(range(PAGE_SIZE + 1))
+ assert result.saved_states == [BigCommerceResumeConfig(page=1, order_index=i) for i in (1, 2, 3)]
+
+ def test_order_statuses_is_read_in_one_request(self) -> None:
+ statuses = [{"id": 0, "name": "Incomplete"}, {"id": 1, "name": "Pending"}]
+ result = self._driver().run(
+ "order_statuses",
+ # The endpoint ignores `page`, so a second request would return the same rows again.
+ [ScriptedResponse(json=statuses), ScriptedResponse(json=statuses)],
+ )
+
+ assert result.raised is None
+ assert [request.path for request in result.requests] == ["/stores/store123/v2/order_statuses"]
+ assert result.params("page") == [None]
+ assert result.rows == statuses
+
def test_refunds_stop_on_empty_page_without_pagination_meta(self) -> None:
last_value = datetime(2024, 5, 6, 7, 8, 9, tzinfo=UTC)
result = self._driver().run(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/tests/test_bigcommerce_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/tests/test_bigcommerce_source.py
index 19a8dd8c9e8b..41d4c56c48c7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/tests/test_bigcommerce_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bigcommerce/tests/test_bigcommerce_source.py
@@ -4,7 +4,9 @@ from parameterized import parameterized
from products.warehouse_sources.backend.temporal.data_imports.sources.bigcommerce.settings import (
ENDPOINTS,
+ ORDER_FAN_OUT_ENDPOINTS,
PARTITION_FIELDS,
+ V2_ENDPOINTS,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.bigcommerce.source import BigCommerceSource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
@@ -26,8 +28,8 @@ class TestBigCommerceSource:
@staticmethod
def _empty_page(endpoint: str) -> ScriptedResponse:
- # Order transactions fan out over the V2 orders listing.
- if endpoint in ("orders", "order_transactions"):
+ # Per-order endpoints fan out over the V2 orders listing.
+ if endpoint in V2_ENDPOINTS | ORDER_FAN_OUT_ENDPOINTS:
return ScriptedResponse(json=[])
return ScriptedResponse(json={"data": [], "meta": {"pagination": {"current_page": 1, "total_pages": 1}}})