Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 5 missing tables to the LightspeedRetail source (#121002)

Tom Owersdfcb72fbced1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 4888c976d54c..119410d83bdb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -7326,13 +7326,13 @@ Diffed against: <https://x-series-api.lightspeedhq.com/reference/listcustomers>
- [x] `suppliers (GET /api/2.0/suppliers)` — lookup resolving the supplier_id carried on products and consignments (high)
- [x] `payment_types (GET /api/2.0/payment_types)` — lookup resolving payment type IDs on the sale payments we already sync (high)
- [x] `product_categories (GET /api/2.0/product_categories)` — lookup for category IDs on products - required for any category-level sales breakdown (high)
-- [ ] `brands (GET /api/2.0/brands)` — lookup resolving brand_id on products (medium)
-- [ ] `customer_groups (+ /customer_groups/{id}/customers)` — customer segment membership table for cohort and loyalty analysis (medium)
+- [x] `brands (GET /api/2.0/brands)` — lookup resolving brand_id on products (medium)
+- [x] `customer_groups (+ /customer_groups/{id}/customers)` — customer segment membership table for cohort and loyalty analysis (medium)
- [x] `stock_adjustments (GET /api/2.0/stock_adjustments)` — inventory movement/shrinkage records that explain changes in the inventory table we sync (medium)
- [x] `promotions (+ /promotions/{id}/products, promo codes)` — discount campaigns and their product scope; needed to attribute discounted revenue (medium)
- [x] `gift_cards (+ gift card transactions)` — outstanding liability and gift-card redemption transactions (medium)
-- [ ] `store_credits (+ transactions and balances)` — store credit liability and per-customer transaction ledger (medium)
-- [ ] `shifts (GET /api/2.0/shifts)` — register shift open/close records for labor and till-reconciliation analysis (medium)
+- [x] `store_credits (+ transactions and balances)` — store credit liability and per-customer transaction ledger (medium)
+- [x] `shifts (GET /api/2.0/shifts)` — register shift open/close records for labor and till-reconciliation analysis (medium)
Note: The X-Series docs are a ReadMe.io site with no downloadable OpenAPI; the endpoint inventory was recovered from the 228 operation slugs embedded in the reference page payload. Also unsynced but lower value: price_books/price_book_products, product_types, product_images, serial_numbers, service_orders, quotes, fulfillments, audit log events, custom fields, variant attributes, customer_addresses, channels and retailers.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/canonical_descriptions.py
index f38a160839e2..d9df069355e0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/canonical_descriptions.py
@@ -298,4 +298,66 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"created_at": "Time at which the transaction was created, in UTC.",
},
},
+ "brands": {
+ "description": "A product brand in Lightspeed Retail. Products reference it through brand_id.",
+ "docs_url": "https://x-series-api.lightspeedhq.com/reference/listbrands",
+ "columns": _columns(
+ name="The brand name.",
+ description="The brand description.",
+ ),
+ },
+ "customer_groups": {
+ "description": "A customer group in Lightspeed Retail. Customers reference it through customer_group_id.",
+ "docs_url": "https://x-series-api.lightspeedhq.com/reference/listcustomergroups",
+ "columns": _columns(
+ name="The customer group name.",
+ group_id="The customer group identifier.",
+ retailer_id="ID of the retailer the group belongs to.",
+ created_at="Time at which the customer group was created, in UTC.",
+ updated_at="Time at which the customer group was last updated, in UTC.",
+ ),
+ },
+ "shifts": {
+ "description": "A staff shift in Lightspeed Retail: the time a user clocked in and out.",
+ "docs_url": "https://x-series-api.lightspeedhq.com/reference/listshifts",
+ "columns": {
+ "id": _COMMON_COLUMNS["id"],
+ "version": _COMMON_COLUMNS["version"],
+ "user_id": "ID of the user who started the shift.",
+ "retailer_id": "ID of the retailer where the shift was started.",
+ "time_in": "Time the shift started.",
+ "time_out": "Time the shift ended.",
+ "updated_by": "ID of the user who last updated the shift.",
+ "created_at": "Time at which the shift record was created, in UTC.",
+ "updated_at": "Time at which the shift record was last updated, in UTC.",
+ },
+ },
+ "store_credits": {
+ "description": "The store credit account of a Lightspeed Retail customer, with its balance and transactions.",
+ "docs_url": "https://x-series-api.lightspeedhq.com/reference/liststorecredit",
+ "columns": {
+ "id": "Internal ID of the store credit customer.",
+ "customer_id": "ID of the customer the store credit belongs to.",
+ "balance": "The customer's store credit balance.",
+ "total_credit_issued": "Total amount of store credit issued to the customer.",
+ "total_credit_redeemed": "Total amount of store credit the customer spent.",
+ "store_credit_transactions": "The store credit transactions of the customer.",
+ "created_at": "Time at which the store credit account was created, in UTC.",
+ },
+ },
+ "store_credit_transactions": {
+ "description": "A store credit transaction of a Lightspeed Retail customer, such as an issue or a redemption.",
+ "docs_url": "https://x-series-api.lightspeedhq.com/reference/liststorecredit",
+ "columns": {
+ "id": "Unique identifier for the transaction (base64 encoded).",
+ "customer_id": "ID of the customer whose store credit the transaction changed.",
+ "type": "Transaction type: ISSUE, REDEMPTION, REVERSE or HOLD.",
+ "amount": "Transaction amount. Redemptions are negative.",
+ "client_id": "Idempotency identifier from the client system. A transaction and its reversal share it.",
+ "notes": "Notes attached to the transaction.",
+ "sale_id": "ID of the sale the transaction belongs to. Null for transactions made outside Lightspeed Retail.",
+ "user_id": "ID of the user who made the transaction, such as a cashier or an administrator.",
+ "created_at": "Time at which the transaction was created.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/lightspeed_retail.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/lightspeed_retail.py
index 5d4995341e44..aaf0903ee5fc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/lightspeed_retail.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/lightspeed_retail.py
@@ -189,9 +189,11 @@ def _endpoint_params(config: LightspeedRetailEndpointConfig) -> dict[str, Any]:
return {} if config.pagination == "single_page" else {"page_size": config.page_size}
-def _nested_rows(field_name: str, parent_key: str) -> Callable[[dict[str, Any]], list[dict[str, Any]]]:
+def _nested_rows(
+ field_name: str, parent_key: str, parent_id_field: str = "id"
+) -> Callable[[dict[str, Any]], list[dict[str, Any]]]:
def _explode(item: dict[str, Any]) -> list[dict[str, Any]]:
- return [{**row, parent_key: item.get("id")} for row in item.get(field_name) or []]
+ return [{**row, parent_key: item.get(parent_id_field)} for row in item.get(field_name) or []]
return _explode
@@ -295,7 +297,9 @@ def lightspeed_retail_source(
initial_paginator_state=initial_paginator_state,
)
if config.nested_rows_field and config.nested_parent_key:
- resource.add_map(_nested_rows(config.nested_rows_field, config.nested_parent_key))
+ resource.add_map(
+ _nested_rows(config.nested_rows_field, config.nested_parent_key, config.nested_parent_id_field)
+ )
return _source_response(endpoint, lambda: resource, resource.column_hints)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/settings.py
index 12daa8be0b31..7ac672b65996 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/settings.py
@@ -48,6 +48,7 @@ class LightspeedRetailEndpointConfig:
# `nested_parent_key`, instead of one row per record.
nested_rows_field: Optional[str] = None
nested_parent_key: Optional[str] = None
+ nested_parent_id_field: str = "id"
LIGHTSPEED_RETAIL_ENDPOINTS: dict[str, LightspeedRetailEndpointConfig] = {
@@ -169,6 +170,41 @@ LIGHTSPEED_RETAIL_ENDPOINTS: dict[str, LightspeedRetailEndpointConfig] = {
nested_rows_field="gift_card_transactions",
nested_parent_key="gift_card_id",
),
+ "brands": LightspeedRetailEndpointConfig(
+ name="brands",
+ path="/brands",
+ ),
+ "customer_groups": LightspeedRetailEndpointConfig(
+ name="customer_groups",
+ path="/customer_groups",
+ partition_key="created_at",
+ ),
+ "shifts": LightspeedRetailEndpointConfig(
+ name="shifts",
+ path="/shifts",
+ partition_key="created_at",
+ ),
+ "store_credits": LightspeedRetailEndpointConfig(
+ name="store_credits",
+ path="/store_credits",
+ # One row per customer with store credit; `id` is optional in the schema, `customer_id` is not.
+ primary_key=["customer_id"],
+ incremental_fields=[],
+ partition_key="created_at",
+ # `page_size` is the only paging param and caps the list, so it is not sent.
+ pagination="single_page",
+ ),
+ "store_credit_transactions": LightspeedRetailEndpointConfig(
+ name="store_credit_transactions",
+ path="/store_credits",
+ primary_key=["customer_id", "id"],
+ incremental_fields=[],
+ partition_key="created_at",
+ pagination="single_page",
+ nested_rows_field="store_credit_transactions",
+ nested_parent_key="customer_id",
+ nested_parent_id_field="customer_id",
+ ),
}
ENDPOINTS = tuple(LIGHTSPEED_RETAIL_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/tests/test_lightspeed_retail.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/tests/test_lightspeed_retail.py
index 33865a9992bb..2a5e47dcc24b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/tests/test_lightspeed_retail.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/tests/test_lightspeed_retail.py
@@ -252,6 +252,29 @@ class TestUnversionedEndpoints:
],
[{"id": "code1", "promotion_id": "pr1"}, {"id": "code2", "promotion_id": "pr2"}],
),
+ (
+ "store_credit_transactions",
+ [
+ {
+ "data": [
+ {
+ "id": "internal-1",
+ "customer_id": "cust-1",
+ "store_credit_transactions": [{"id": "tx1"}, {"id": "tx2"}],
+ },
+ {"id": "internal-2", "customer_id": "cust-2", "store_credit_transactions": None},
+ {"customer_id": "cust-3", "store_credit_transactions": [{"id": "tx3"}]},
+ ]
+ }
+ ],
+ None,
+ [("/store_credits", {})],
+ [
+ {"id": "tx1", "customer_id": "cust-1"},
+ {"id": "tx2", "customer_id": "cust-1"},
+ {"id": "tx3", "customer_id": "cust-3"},
+ ],
+ ),
],
)
def test_pages_without_record_versions(