:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 5 missing tables to the LightspeedRetail source (#121002)
Tom Owersdfcb72fbced1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 4888c976d54c..119410d83bdb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -7326,13 +7326,13 @@ Diffed against: <https://x-series-api.lightspeedhq.com/reference/listcustomers>- [x] `suppliers (GET /api/2.0/suppliers)` — lookup resolving the supplier_id carried on products and consignments (high)- [x] `payment_types (GET /api/2.0/payment_types)` — lookup resolving payment type IDs on the sale payments we already sync (high)- [x] `product_categories (GET /api/2.0/product_categories)` — lookup for category IDs on products - required for any category-level sales breakdown (high)-- [ ] `brands (GET /api/2.0/brands)` — lookup resolving brand_id on products (medium)-- [ ] `customer_groups (+ /customer_groups/{id}/customers)` — customer segment membership table for cohort and loyalty analysis (medium)+- [x] `brands (GET /api/2.0/brands)` — lookup resolving brand_id on products (medium)+- [x] `customer_groups (+ /customer_groups/{id}/customers)` — customer segment membership table for cohort and loyalty analysis (medium)- [x] `stock_adjustments (GET /api/2.0/stock_adjustments)` — inventory movement/shrinkage records that explain changes in the inventory table we sync (medium)- [x] `promotions (+ /promotions/{id}/products, promo codes)` — discount campaigns and their product scope; needed to attribute discounted revenue (medium)- [x] `gift_cards (+ gift card transactions)` — outstanding liability and gift-card redemption transactions (medium)-- [ ] `store_credits (+ transactions and balances)` — store credit liability and per-customer transaction ledger (medium)-- [ ] `shifts (GET /api/2.0/shifts)` — register shift open/close records for labor and till-reconciliation analysis (medium)+- [x] `store_credits (+ transactions and balances)` — store credit liability and per-customer transaction ledger (medium)+- [x] `shifts (GET /api/2.0/shifts)` — register shift open/close records for labor and till-reconciliation analysis (medium)Note: The X-Series docs are a ReadMe.io site with no downloadable OpenAPI; the endpoint inventory was recovered from the 228 operation slugs embedded in the reference page payload. Also unsynced but lower value: price_books/price_book_products, product_types, product_images, serial_numbers, service_orders, quotes, fulfillments, audit log events, custom fields, variant attributes, customer_addresses, channels and retailers.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/canonical_descriptions.pyindex f38a160839e2..d9df069355e0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/canonical_descriptions.py@@ -298,4 +298,66 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"created_at": "Time at which the transaction was created, in UTC.",},},+ "brands": {+ "description": "A product brand in Lightspeed Retail. Products reference it through brand_id.",+ "docs_url": "https://x-series-api.lightspeedhq.com/reference/listbrands",+ "columns": _columns(+ name="The brand name.",+ description="The brand description.",+ ),+ },+ "customer_groups": {+ "description": "A customer group in Lightspeed Retail. Customers reference it through customer_group_id.",+ "docs_url": "https://x-series-api.lightspeedhq.com/reference/listcustomergroups",+ "columns": _columns(+ name="The customer group name.",+ group_id="The customer group identifier.",+ retailer_id="ID of the retailer the group belongs to.",+ created_at="Time at which the customer group was created, in UTC.",+ updated_at="Time at which the customer group was last updated, in UTC.",+ ),+ },+ "shifts": {+ "description": "A staff shift in Lightspeed Retail: the time a user clocked in and out.",+ "docs_url": "https://x-series-api.lightspeedhq.com/reference/listshifts",+ "columns": {+ "id": _COMMON_COLUMNS["id"],+ "version": _COMMON_COLUMNS["version"],+ "user_id": "ID of the user who started the shift.",+ "retailer_id": "ID of the retailer where the shift was started.",+ "time_in": "Time the shift started.",+ "time_out": "Time the shift ended.",+ "updated_by": "ID of the user who last updated the shift.",+ "created_at": "Time at which the shift record was created, in UTC.",+ "updated_at": "Time at which the shift record was last updated, in UTC.",+ },+ },+ "store_credits": {+ "description": "The store credit account of a Lightspeed Retail customer, with its balance and transactions.",+ "docs_url": "https://x-series-api.lightspeedhq.com/reference/liststorecredit",+ "columns": {+ "id": "Internal ID of the store credit customer.",+ "customer_id": "ID of the customer the store credit belongs to.",+ "balance": "The customer's store credit balance.",+ "total_credit_issued": "Total amount of store credit issued to the customer.",+ "total_credit_redeemed": "Total amount of store credit the customer spent.",+ "store_credit_transactions": "The store credit transactions of the customer.",+ "created_at": "Time at which the store credit account was created, in UTC.",+ },+ },+ "store_credit_transactions": {+ "description": "A store credit transaction of a Lightspeed Retail customer, such as an issue or a redemption.",+ "docs_url": "https://x-series-api.lightspeedhq.com/reference/liststorecredit",+ "columns": {+ "id": "Unique identifier for the transaction (base64 encoded).",+ "customer_id": "ID of the customer whose store credit the transaction changed.",+ "type": "Transaction type: ISSUE, REDEMPTION, REVERSE or HOLD.",+ "amount": "Transaction amount. Redemptions are negative.",+ "client_id": "Idempotency identifier from the client system. A transaction and its reversal share it.",+ "notes": "Notes attached to the transaction.",+ "sale_id": "ID of the sale the transaction belongs to. Null for transactions made outside Lightspeed Retail.",+ "user_id": "ID of the user who made the transaction, such as a cashier or an administrator.",+ "created_at": "Time at which the transaction was created.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/lightspeed_retail.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/lightspeed_retail.pyindex 5d4995341e44..aaf0903ee5fc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/lightspeed_retail.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/lightspeed_retail.py@@ -189,9 +189,11 @@ def _endpoint_params(config: LightspeedRetailEndpointConfig) -> dict[str, Any]:return {} if config.pagination == "single_page" else {"page_size": config.page_size}-def _nested_rows(field_name: str, parent_key: str) -> Callable[[dict[str, Any]], list[dict[str, Any]]]:+def _nested_rows(+ field_name: str, parent_key: str, parent_id_field: str = "id"+) -> Callable[[dict[str, Any]], list[dict[str, Any]]]:def _explode(item: dict[str, Any]) -> list[dict[str, Any]]:- return [{**row, parent_key: item.get("id")} for row in item.get(field_name) or []]+ return [{**row, parent_key: item.get(parent_id_field)} for row in item.get(field_name) or []]return _explode@@ -295,7 +297,9 @@ def lightspeed_retail_source(initial_paginator_state=initial_paginator_state,)if config.nested_rows_field and config.nested_parent_key:- resource.add_map(_nested_rows(config.nested_rows_field, config.nested_parent_key))+ resource.add_map(+ _nested_rows(config.nested_rows_field, config.nested_parent_key, config.nested_parent_id_field)+ )return _source_response(endpoint, lambda: resource, resource.column_hints)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/settings.pyindex 12daa8be0b31..7ac672b65996 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/settings.py@@ -48,6 +48,7 @@ class LightspeedRetailEndpointConfig:# `nested_parent_key`, instead of one row per record.nested_rows_field: Optional[str] = Nonenested_parent_key: Optional[str] = None+ nested_parent_id_field: str = "id"LIGHTSPEED_RETAIL_ENDPOINTS: dict[str, LightspeedRetailEndpointConfig] = {@@ -169,6 +170,41 @@ LIGHTSPEED_RETAIL_ENDPOINTS: dict[str, LightspeedRetailEndpointConfig] = {nested_rows_field="gift_card_transactions",nested_parent_key="gift_card_id",),+ "brands": LightspeedRetailEndpointConfig(+ name="brands",+ path="/brands",+ ),+ "customer_groups": LightspeedRetailEndpointConfig(+ name="customer_groups",+ path="/customer_groups",+ partition_key="created_at",+ ),+ "shifts": LightspeedRetailEndpointConfig(+ name="shifts",+ path="/shifts",+ partition_key="created_at",+ ),+ "store_credits": LightspeedRetailEndpointConfig(+ name="store_credits",+ path="/store_credits",+ # One row per customer with store credit; `id` is optional in the schema, `customer_id` is not.+ primary_key=["customer_id"],+ incremental_fields=[],+ partition_key="created_at",+ # `page_size` is the only paging param and caps the list, so it is not sent.+ pagination="single_page",+ ),+ "store_credit_transactions": LightspeedRetailEndpointConfig(+ name="store_credit_transactions",+ path="/store_credits",+ primary_key=["customer_id", "id"],+ incremental_fields=[],+ partition_key="created_at",+ pagination="single_page",+ nested_rows_field="store_credit_transactions",+ nested_parent_key="customer_id",+ nested_parent_id_field="customer_id",+ ),}ENDPOINTS = tuple(LIGHTSPEED_RETAIL_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/tests/test_lightspeed_retail.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/tests/test_lightspeed_retail.pyindex 33865a9992bb..2a5e47dcc24b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/tests/test_lightspeed_retail.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lightspeed_retail/tests/test_lightspeed_retail.py@@ -252,6 +252,29 @@ class TestUnversionedEndpoints:],[{"id": "code1", "promotion_id": "pr1"}, {"id": "code2", "promotion_id": "pr2"}],),+ (+ "store_credit_transactions",+ [+ {+ "data": [+ {+ "id": "internal-1",+ "customer_id": "cust-1",+ "store_credit_transactions": [{"id": "tx1"}, {"id": "tx2"}],+ },+ {"id": "internal-2", "customer_id": "cust-2", "store_credit_transactions": None},+ {"customer_id": "cust-3", "store_credit_transactions": [{"id": "tx3"}]},+ ]+ }+ ],+ None,+ [("/store_credits", {})],+ [+ {"id": "tx1", "customer_id": "cust-1"},+ {"id": "tx2", "customer_id": "cust-1"},+ {"id": "tx3", "customer_id": "cust-3"},+ ],+ ),],)def test_pages_without_record_versions(