:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 4 missing tables to the Picqer source (#120875)
Tom Owers0da632379266
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 65be4ef925a4..0b2218d753b8 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -887,8 +887,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/pganalyze/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/phyllo.py1 products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/settings.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/picqer/picqer.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/picqer/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/pingdom.py1 products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/pinterest_ads/pinterest_ads.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex da726d789ee4..30488d25f18e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -9394,10 +9394,10 @@ Today (13): `customers`, `locations`, `orders`, `picklists`, `products`, `purchaDiffed against: <https://picqer.com/en/api>-- [ ] `/stockhistory` — every stock movement in and out of the warehouse with old/new levels, reason and change_type - the core inventory event stream (high)-- [ ] `/shipments (also /picklists/{idpicklist}/shipments)` — actual outbound shipments with carrier and tracking; picklists alone can't answer ship-rate or carrier-mix questions (high)-- [ ] `/warehouses/{idwarehouse}/stock and /products/{idproduct}/stock` — current stock level per product per warehouse; the products table has no per-warehouse quantities (high)-- [ ] `/return_reasons` — lookup resolving the reason ID carried on the returns we already sync (high)+- [x] `/stockhistory` — every stock movement in and out of the warehouse with old/new levels, reason and change_type - the core inventory event stream (high)+- [x] `/shipments (also /picklists/{idpicklist}/shipments)` — actual outbound shipments with carrier and tracking; picklists alone can't answer ship-rate or carrier-mix questions (high)+- [x] `/warehouses/{idwarehouse}/stock and /products/{idproduct}/stock` — current stock level per product per warehouse; the products table has no per-warehouse quantities (high)+- [x] `/return_reasons` — lookup resolving the reason ID carried on the returns we already sync (high)- [ ] `/return_statuses` — lookup resolving the status ID on synced returns (high)- [ ] `/backorders` — unfulfillable order lines - the standard stockout/backorder analysis (medium)- [ ] `/picklists/batches` — batch-picking grouping over picklists; needed for picker throughput and batch efficiency (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/canonical_descriptions.pyindex c8a778047fa9..8e6a6b0f43de 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/canonical_descriptions.py@@ -74,6 +74,60 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"updated_at": "Datetime the return was last changed.",},},+ "shipments": {+ "description": "A shipment created for a picklist or return, with the shipping provider and per-parcel tracking.",+ "docs_url": f"{_DOCS}/picklist-shipments",+ "columns": {+ "idshipment": "Unique Picqer reference for the shipment.",+ "idpicklist": "Picklist this shipment is for.",+ "idorder": "Order this shipment is for.",+ "idreturn": "Return this shipment is for.",+ "idpackaging": "Packaging used for this shipment.",+ "idcompany_shippingprovider": "Shipping provider used to create this shipment.",+ "provider": "Name of the shipping provider.",+ "providername": "User-defined name for this provider.",+ "public_providername": "Name of the service or provider that is known to the public.",+ "carrier_key": "Identifier of the carrier used for this shipment.",+ "weight": "Weight in grams communicated to the shipping provider.",+ "cancelled": "Whether this shipment is cancelled.",+ "idfulfilment_customer": "Fulfilment customer the shipment belongs to (Picqer Fulfilment only).",+ "parcels": "The parcels of the shipment, each with its weight, tracking code, tracking URL and documents.",+ "created": "Datetime the shipment was created.",+ "updated": "Datetime the shipment was last changed.",+ },+ },+ "stockhistory": {+ "description": "One change of the stock level of a product in a warehouse. History items cannot be changed.",+ "docs_url": f"{_DOCS}/stockhistory",+ "columns": {+ "idproduct_stock_history": "Unique Picqer reference for the stock history item.",+ "idproduct": "Product whose stock changed.",+ "idwarehouse": "Warehouse whose stock level changed.",+ "iduser": "User who made the change, if any.",+ "idlocation": "Location the change applies to, if any.",+ "old_stock": "Stock level in the warehouse before the change.",+ "stock_change": "Quantity added (positive) or removed (negative).",+ "new_stock": "Stock level in the warehouse after the change.",+ "reason": "Reason given for the change.",+ "change_type": "Category of the change (e.g. api, manual, picking, receipt, returned, stockcount).",+ "changed_at": "Datetime of the change.",+ },+ },+ "warehouse_stock": {+ "description": "The current stock of one product in one warehouse, for all products with stock or reservations.",+ "docs_url": f"{_DOCS}/warehouses",+ "columns": {+ "idwarehouse": "Warehouse the stock level is for.",+ "idproduct": "Product the stock level is for.",+ "productcode": "Product code of the product.",+ "stock": "Total physical stock in the warehouse, over all locations.",+ "reserved": "Total quantity reserved.",+ "reservedbackorders": "Quantity reserved for backorders.",+ "reservedpicklists": "Quantity reserved for picklists.",+ "reservedallocations": "Quantity reserved for allocations.",+ "freestock": "Stock that is free to sell (stock minus reserved).",+ },+ },"products": {"description": "A product in the Picqer catalog, including its stock and supplier information.","docs_url": f"{_DOCS}/products",@@ -156,4 +210,16 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"color": "The tag's display color.",},},+ "return_reasons": {+ "description": "A reason a customer can give for returning a product. Resolves `idreturn_reason` on returned products.",+ "docs_url": f"{_DOCS}/returns",+ "columns": {+ "idreturn_reason": "Unique Picqer reference for the return reason.",+ "name": "Name of the return reason.",+ "default": "Whether this is the default return reason.",+ "created_at": "Datetime the return reason was created.",+ "updated_at": "Datetime the return reason was last changed.",+ "deleted_at": "Datetime the return reason was deleted, or null.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/picqer.py b/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/picqer.pyindex c8069ec12f6d..57fb97b2e2a0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/picqer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/picqer.py@@ -9,11 +9,15 @@ from requests.auth import HTTPBasicAuthfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_sessionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (RESTAPIConfig,- rest_api_resource,+ rest_api_resources,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (OffsetPaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (+ Endpoint,+ EndpointResource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse@@ -33,10 +37,13 @@ PICQER_USER_AGENT = "PostHog (https://posthog.com - [email protected])"_ACCOUNT_RE = re.compile(r"^[A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?$")[email protected](frozen=True)class PicqerResumeConfig:# Next offset to fetch. None means "start from offset 0".offset: int | None = None+ # Framework fan-out resume state for fan-out endpoints:+ # {"completed": [child_path, ...], "current": child_path | None, "child_state": {"offset": ...} | None}.+ fanout_state: dict | None = Noneclass PicqerOffsetPaginator(OffsetPaginator):@@ -112,6 +119,48 @@ def _build_params(return params+def flatten_warehouse_stock_row(row: dict[str, Any]) -> dict[str, Any]:+ """Lift the nested per-warehouse `stock` object to the row root.++ Picqer returns `{"idproduct", "productcode", "stock": {"idwarehouse", "stock", ...}}`, so the+ `idwarehouse` half of the primary key only exists inside the nested object. The parent+ warehouse id injected by the fan-out is the fallback when the nested object omits it.+ """+ parent_idwarehouse = row.pop("_warehouses_idwarehouse", None)+ stock = row.pop("stock", None)+ if isinstance(stock, dict):+ row.update(stock)+ if row.get("idwarehouse") is None:+ row["idwarehouse"] = parent_idwarehouse+ return row+++_ROW_MAPPERS = {"warehouse_stock": flatten_warehouse_stock_row}+++def _resources(config: PicqerEndpointConfig, params: dict[str, Any]) -> list[str | EndpointResource]:+ endpoint: Endpoint = {"path": config.path, "params": params}+ if not config.paginated:+ endpoint["paginator"] = "single_page"+ resource: EndpointResource = {"name": config.name, "endpoint": endpoint}+ if config.name in _ROW_MAPPERS:+ resource["data_map"] = _ROW_MAPPERS[config.name]++ fanout = config.fanout+ if fanout is None:+ return [resource]++ parent = PICQER_ENDPOINTS[fanout.parent_name]+ endpoint["params"] = {+ **params,+ fanout.resolve_field: {"type": "resolve", "resource": parent.name, "field": fanout.resolve_field},+ }+ # A parent deleted between the parent listing and the child fetch 404s; skip it.+ endpoint["response_actions"] = [{"status_code": 404, "action": "ignore"}]+ resource["include_from_parent"] = [fanout.resolve_field]+ return [{"name": parent.name, "endpoint": {"path": parent.path, "params": {}}}, resource]++def picqer_source(account: str,api_key: str,@@ -141,30 +190,28 @@ def picqer_source("allowed_hosts": [],},"resource_defaults": {},- "resources": [- {- "name": endpoint,- "endpoint": {- "path": config.path,- "params": params,- },- }- ],+ "resources": _resources(config, params),}+ is_fanout = config.fanout is not Noneinitial_paginator_state: Optional[dict[str, Any]] = Noneif resumable_source_manager.can_resume():resume = resumable_source_manager.load_state()if resume is not None:- initial_paginator_state = {"offset": resume.offset}+ if is_fanout:+ initial_paginator_state = resume.fanout_state or None+ elif resume.offset is not None:+ initial_paginator_state = {"offset": resume.offset}def save_checkpoint(state: Optional[dict[str, Any]]) -> None:# Persist only when a next page remains; save AFTER a page is yielded so a crash re-yields# the last page (merge dedupes on the primary key) rather than skipping it.- if state and state.get("offset") is not None:+ if is_fanout:+ resumable_source_manager.save_state(PicqerResumeConfig(fanout_state=state))+ elif state and state.get("offset") is not None:resumable_source_manager.save_state(PicqerResumeConfig(offset=int(state["offset"])))- resource = rest_api_resource(+ resources = rest_api_resources(rest_config,team_id,job_id,@@ -172,6 +219,7 @@ def picqer_source(resume_hook=save_checkpoint,initial_paginator_state=initial_paginator_state,)+ resource = next(r for r in resources if r.name == endpoint)return SourceResponse(name=endpoint,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/settings.pyindex 3e1fd8141d99..1ecd068b5334 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/settings.py@@ -22,15 +22,23 @@ def _incremental_fields(cursor_field: str) -> list[IncrementalField]:]-@dataclass+@dataclass(frozen=True)+class PicqerFanoutConfig:+ # Parent endpoint whose rows drive the child requests, and the parent field substituted into+ # the `{<resolve_field>}` placeholder of the child path.+ parent_name: str+ resolve_field: str+++@dataclass(frozen=True)class PicqerEndpointConfig:name: strpath: str # Path under /api/v1, e.g. "/orders"primary_keys: list[str]# Query param that filters the list by modification time (e.g. `updated_after`). Only set when# the API documents a genuine server-side UPDATE-based filter — creation-only filters- # (`sincedate`) are deliberately left off, since using them as an incremental cursor would miss- # updates to mutable records. None => full refresh.+ # (`sincedate`) are deliberately left off for mutable records, since using them as an+ # incremental cursor would miss updates. Immutable records may use them. None => full refresh.incremental_filter_param: Optional[str] = None# Response field the `incremental_filter_param` compares against, and the cursor the pipeline# tracks. Must be present in every row so the watermark can advance. None => full refresh.@@ -39,6 +47,10 @@ class PicqerEndpointConfig:# sync). None when the resource exposes no reliable creation timestamp.partition_key: Optional[str] = Noneshould_sync_default: bool = True+ # False for list endpoints the docs do not mark as paginated. An offset loop on an endpoint that+ # ignores `offset` would fetch the same full page forever.+ paginated: bool = True+ fanout: Optional[PicqerFanoutConfig] = None@propertydef supports_incremental(self) -> bool:@@ -91,6 +103,31 @@ PICQER_ENDPOINTS: dict[str, PicqerEndpointConfig] = {incremental_cursor_field="updated_at",partition_key="created_at",),+ # Shipments can be cancelled after creation and only expose a creation-date filter+ # (`sincedate`), so they sync full refresh like orders and picklists.+ "shipments": PicqerEndpointConfig(+ name="shipments",+ path="/shipments",+ primary_keys=["idshipment"],+ partition_key="created",+ ),+ # Stock history rows can never be changed, so the creation-date filter is a safe cursor.+ "stockhistory": PicqerEndpointConfig(+ name="stockhistory",+ path="/stockhistory",+ primary_keys=["idproduct_stock_history"],+ incremental_filter_param="sincedate",+ incremental_cursor_field="changed_at",+ partition_key="changed_at",+ ),+ # Current stock per product per warehouse. Fanned out over warehouses rather than products,+ # because a warehouse returns all of its products in pages of 100.+ "warehouse_stock": PicqerEndpointConfig(+ name="warehouse_stock",+ path="/warehouses/{idwarehouse}/stock",+ primary_keys=["idwarehouse", "idproduct"],+ fanout=PicqerFanoutConfig(parent_name="warehouses", resolve_field="idwarehouse"),+ ),# Catalog / reference resources. Products carry a `created` field but expose no time filter."products": PicqerEndpointConfig(name="products",@@ -133,6 +170,12 @@ PICQER_ENDPOINTS: dict[str, PicqerEndpointConfig] = {path="/tags",primary_keys=["idtag"],),+ "return_reasons": PicqerEndpointConfig(+ name="return_reasons",+ path="/return_reasons",+ primary_keys=["idreturn_reason"],+ paginated=False,+ ),}ENDPOINTS = tuple(PICQER_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/tests/test_picqer.py b/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/tests/test_picqer.pyindex c613d9e35d05..e42a0b2f931a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/tests/test_picqer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/tests/test_picqer.py@@ -7,6 +7,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tesScriptedResponse,SourceDriver,always,+ route,scripted_network,)from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.picqer import PicqerSourceConfig@@ -85,17 +86,114 @@ class TestPagination:assert len(result.rows) == 2assert result.params("offset") == [str(PAGE_SIZE)]- def test_incremental_filter_present_on_every_page(self) -> None:+ @pytest.mark.parametrize(+ "endpoint,incremental_field,filter_param",+ [+ ("purchaseorders", "updated", "updated_after"),+ ("stockhistory", "changed_at", "sincedate"),+ ],+ )+ def test_incremental_filter_present_on_every_page(+ self, endpoint: str, incremental_field: str, filter_param: str+ ) -> None:result = _driver().run(- "purchaseorders",+ endpoint,[_page(PAGE_SIZE), _page(1)],- incremental_field="updated",+ incremental_field=incremental_field,db_incremental_field_last_value=datetime(2020, 1, 2, 3, 4, 5),)# The filter must stay on every paginated request so pagination walks only the filtered set.assert result.raised is None- assert result.params("updated_after") == ["2020-01-02 03:04:05"] * 2+ assert result.params(filter_param) == ["2020-01-02 03:04:05"] * 2+ assert result.params("offset") == ["0", str(PAGE_SIZE)]++ def test_unpaginated_endpoint_stops_after_one_full_page(self) -> None:+ # return_reasons is not documented as paginated; an offset loop would refetch the same full+ # page forever if the endpoint ignores `offset`.+ reasons = [{"idreturn_reason": i, "name": f"Reason {i}"} for i in range(PAGE_SIZE)]+ result = _driver().run("return_reasons", route({"/return_reasons": [ScriptedResponse(json=reasons)]}))++ assert result.raised is None+ assert len(result.rows) == PAGE_SIZE+ assert result.paths == ["/api/v1/return_reasons"]+ assert result.params("offset") == [None]+++def _stock_row(idproduct: int, idwarehouse: int | None) -> dict[str, Any]:+ stock: dict[str, Any] = {"stock": 5, "reserved": 1, "freestock": 4}+ if idwarehouse is not None:+ stock["idwarehouse"] = idwarehouse+ return {"idproduct": idproduct, "productcode": f"SKU-{idproduct}", "stock": stock}+++def _warehouse_stock_script(warehouse_ids: list[int], stock_pages: dict[int, list[ScriptedResponse]]) -> Any:+ table: dict[str, list[ScriptedResponse]] = {+ "/warehouses": [ScriptedResponse(json=[{"idwarehouse": i, "name": f"Warehouse {i}"} for i in warehouse_ids])]+ }+ for idwarehouse, pages in stock_pages.items():+ table[f"/warehouses/{idwarehouse}/stock"] = pages+ return route(table)+++class TestWarehouseStock:+ def test_fans_out_over_warehouses_and_flattens_stock(self) -> None:+ script = _warehouse_stock_script(+ [1, 2, 3],+ {+ 1: [+ ScriptedResponse(json=[_stock_row(i, 1) for i in range(PAGE_SIZE)]),+ ScriptedResponse(json=[_stock_row(PAGE_SIZE, 1)]),+ ],+ # Deleted between the warehouse listing and the stock fetch.+ 2: [ScriptedResponse(status=404, json={"error": "not found"})],+ # Nested object without its own idwarehouse: the parent id fills the key.+ 3: [ScriptedResponse(json=[_stock_row(7, None)])],+ },+ )++ result = _driver().run("warehouse_stock", script)++ assert result.raised is None+ assert len(result.rows) == PAGE_SIZE + 2+ keys = {(row["idwarehouse"], row["idproduct"]) for row in result.rows}+ assert len(keys) == len(result.rows)+ assert (3, 7) in keys+ assert result.rows[-1] == {+ "idproduct": 7,+ "productcode": "SKU-7",+ "stock": 5,+ "reserved": 1,+ "freestock": 4,+ "idwarehouse": 3,+ }+ assert result.paths == [+ "/api/v1/warehouses",+ "/api/v1/warehouses/1/stock",+ "/api/v1/warehouses/1/stock",+ "/api/v1/warehouses/2/stock",+ "/api/v1/warehouses/3/stock",+ ]+ assert result.params("offset")[1:3] == ["0", str(PAGE_SIZE)]++ def test_resumes_mid_warehouse_and_skips_completed_ones(self) -> None:+ script = _warehouse_stock_script([1, 2], {2: [ScriptedResponse(json=[_stock_row(9, 2)])]})++ result = _driver().run(+ "warehouse_stock",+ script,+ resume_state=PicqerResumeConfig(+ fanout_state={+ "completed": ["/warehouses/1/stock"],+ "current": "/warehouses/2/stock",+ "child_state": {"offset": PAGE_SIZE},+ }+ ),+ )++ assert result.raised is None+ assert [(row["idwarehouse"], row["idproduct"]) for row in result.rows] == [(2, 9)]+ assert result.paths == ["/api/v1/warehouses", "/api/v1/warehouses/2/stock"]assert result.params("offset") == ["0", str(PAGE_SIZE)]