Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the Picqer source (#120875)

Tom Owers0da632379266
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 65be4ef925a4..0b2218d753b8 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -887,8 +887,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/pganalyze/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/phyllo.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/picqer/picqer.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/picqer/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/pingdom.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pinterest_ads/pinterest_ads.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index da726d789ee4..30488d25f18e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9394,10 +9394,10 @@ Today (13): `customers`, `locations`, `orders`, `picklists`, `products`, `purcha
Diffed against: <https://picqer.com/en/api>
-- [ ] `/stockhistory` — every stock movement in and out of the warehouse with old/new levels, reason and change_type - the core inventory event stream (high)
-- [ ] `/shipments (also /picklists/{idpicklist}/shipments)` — actual outbound shipments with carrier and tracking; picklists alone can't answer ship-rate or carrier-mix questions (high)
-- [ ] `/warehouses/{idwarehouse}/stock and /products/{idproduct}/stock` — current stock level per product per warehouse; the products table has no per-warehouse quantities (high)
-- [ ] `/return_reasons` — lookup resolving the reason ID carried on the returns we already sync (high)
+- [x] `/stockhistory` — every stock movement in and out of the warehouse with old/new levels, reason and change_type - the core inventory event stream (high)
+- [x] `/shipments (also /picklists/{idpicklist}/shipments)` — actual outbound shipments with carrier and tracking; picklists alone can't answer ship-rate or carrier-mix questions (high)
+- [x] `/warehouses/{idwarehouse}/stock and /products/{idproduct}/stock` — current stock level per product per warehouse; the products table has no per-warehouse quantities (high)
+- [x] `/return_reasons` — lookup resolving the reason ID carried on the returns we already sync (high)
- [ ] `/return_statuses` — lookup resolving the status ID on synced returns (high)
- [ ] `/backorders` — unfulfillable order lines - the standard stockout/backorder analysis (medium)
- [ ] `/picklists/batches` — batch-picking grouping over picklists; needed for picker throughput and batch efficiency (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/canonical_descriptions.py
index c8a778047fa9..8e6a6b0f43de 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/canonical_descriptions.py
@@ -74,6 +74,60 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"updated_at": "Datetime the return was last changed.",
},
},
+ "shipments": {
+ "description": "A shipment created for a picklist or return, with the shipping provider and per-parcel tracking.",
+ "docs_url": f"{_DOCS}/picklist-shipments",
+ "columns": {
+ "idshipment": "Unique Picqer reference for the shipment.",
+ "idpicklist": "Picklist this shipment is for.",
+ "idorder": "Order this shipment is for.",
+ "idreturn": "Return this shipment is for.",
+ "idpackaging": "Packaging used for this shipment.",
+ "idcompany_shippingprovider": "Shipping provider used to create this shipment.",
+ "provider": "Name of the shipping provider.",
+ "providername": "User-defined name for this provider.",
+ "public_providername": "Name of the service or provider that is known to the public.",
+ "carrier_key": "Identifier of the carrier used for this shipment.",
+ "weight": "Weight in grams communicated to the shipping provider.",
+ "cancelled": "Whether this shipment is cancelled.",
+ "idfulfilment_customer": "Fulfilment customer the shipment belongs to (Picqer Fulfilment only).",
+ "parcels": "The parcels of the shipment, each with its weight, tracking code, tracking URL and documents.",
+ "created": "Datetime the shipment was created.",
+ "updated": "Datetime the shipment was last changed.",
+ },
+ },
+ "stockhistory": {
+ "description": "One change of the stock level of a product in a warehouse. History items cannot be changed.",
+ "docs_url": f"{_DOCS}/stockhistory",
+ "columns": {
+ "idproduct_stock_history": "Unique Picqer reference for the stock history item.",
+ "idproduct": "Product whose stock changed.",
+ "idwarehouse": "Warehouse whose stock level changed.",
+ "iduser": "User who made the change, if any.",
+ "idlocation": "Location the change applies to, if any.",
+ "old_stock": "Stock level in the warehouse before the change.",
+ "stock_change": "Quantity added (positive) or removed (negative).",
+ "new_stock": "Stock level in the warehouse after the change.",
+ "reason": "Reason given for the change.",
+ "change_type": "Category of the change (e.g. api, manual, picking, receipt, returned, stockcount).",
+ "changed_at": "Datetime of the change.",
+ },
+ },
+ "warehouse_stock": {
+ "description": "The current stock of one product in one warehouse, for all products with stock or reservations.",
+ "docs_url": f"{_DOCS}/warehouses",
+ "columns": {
+ "idwarehouse": "Warehouse the stock level is for.",
+ "idproduct": "Product the stock level is for.",
+ "productcode": "Product code of the product.",
+ "stock": "Total physical stock in the warehouse, over all locations.",
+ "reserved": "Total quantity reserved.",
+ "reservedbackorders": "Quantity reserved for backorders.",
+ "reservedpicklists": "Quantity reserved for picklists.",
+ "reservedallocations": "Quantity reserved for allocations.",
+ "freestock": "Stock that is free to sell (stock minus reserved).",
+ },
+ },
"products": {
"description": "A product in the Picqer catalog, including its stock and supplier information.",
"docs_url": f"{_DOCS}/products",
@@ -156,4 +210,16 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"color": "The tag's display color.",
},
},
+ "return_reasons": {
+ "description": "A reason a customer can give for returning a product. Resolves `idreturn_reason` on returned products.",
+ "docs_url": f"{_DOCS}/returns",
+ "columns": {
+ "idreturn_reason": "Unique Picqer reference for the return reason.",
+ "name": "Name of the return reason.",
+ "default": "Whether this is the default return reason.",
+ "created_at": "Datetime the return reason was created.",
+ "updated_at": "Datetime the return reason was last changed.",
+ "deleted_at": "Datetime the return reason was deleted, or null.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/picqer.py b/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/picqer.py
index c8069ec12f6d..57fb97b2e2a0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/picqer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/picqer.py
@@ -9,11 +9,15 @@ from requests.auth import HTTPBasicAuth
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
- rest_api_resource,
+ rest_api_resources,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
OffsetPaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
+ Endpoint,
+ EndpointResource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -33,10 +37,13 @@ PICQER_USER_AGENT = "PostHog (https://posthog.com - [email protected])"
_ACCOUNT_RE = re.compile(r"^[A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?$")
[email protected](frozen=True)
class PicqerResumeConfig:
# Next offset to fetch. None means "start from offset 0".
offset: int | None = None
+ # Framework fan-out resume state for fan-out endpoints:
+ # {"completed": [child_path, ...], "current": child_path | None, "child_state": {"offset": ...} | None}.
+ fanout_state: dict | None = None
class PicqerOffsetPaginator(OffsetPaginator):
@@ -112,6 +119,48 @@ def _build_params(
return params
+def flatten_warehouse_stock_row(row: dict[str, Any]) -> dict[str, Any]:
+ """Lift the nested per-warehouse `stock` object to the row root.
+
+ Picqer returns `{"idproduct", "productcode", "stock": {"idwarehouse", "stock", ...}}`, so the
+ `idwarehouse` half of the primary key only exists inside the nested object. The parent
+ warehouse id injected by the fan-out is the fallback when the nested object omits it.
+ """
+ parent_idwarehouse = row.pop("_warehouses_idwarehouse", None)
+ stock = row.pop("stock", None)
+ if isinstance(stock, dict):
+ row.update(stock)
+ if row.get("idwarehouse") is None:
+ row["idwarehouse"] = parent_idwarehouse
+ return row
+
+
+_ROW_MAPPERS = {"warehouse_stock": flatten_warehouse_stock_row}
+
+
+def _resources(config: PicqerEndpointConfig, params: dict[str, Any]) -> list[str | EndpointResource]:
+ endpoint: Endpoint = {"path": config.path, "params": params}
+ if not config.paginated:
+ endpoint["paginator"] = "single_page"
+ resource: EndpointResource = {"name": config.name, "endpoint": endpoint}
+ if config.name in _ROW_MAPPERS:
+ resource["data_map"] = _ROW_MAPPERS[config.name]
+
+ fanout = config.fanout
+ if fanout is None:
+ return [resource]
+
+ parent = PICQER_ENDPOINTS[fanout.parent_name]
+ endpoint["params"] = {
+ **params,
+ fanout.resolve_field: {"type": "resolve", "resource": parent.name, "field": fanout.resolve_field},
+ }
+ # A parent deleted between the parent listing and the child fetch 404s; skip it.
+ endpoint["response_actions"] = [{"status_code": 404, "action": "ignore"}]
+ resource["include_from_parent"] = [fanout.resolve_field]
+ return [{"name": parent.name, "endpoint": {"path": parent.path, "params": {}}}, resource]
+
+
def picqer_source(
account: str,
api_key: str,
@@ -141,30 +190,28 @@ def picqer_source(
"allowed_hosts": [],
},
"resource_defaults": {},
- "resources": [
- {
- "name": endpoint,
- "endpoint": {
- "path": config.path,
- "params": params,
- },
- }
- ],
+ "resources": _resources(config, params),
}
+ is_fanout = config.fanout is not None
initial_paginator_state: Optional[dict[str, Any]] = None
if resumable_source_manager.can_resume():
resume = resumable_source_manager.load_state()
if resume is not None:
- initial_paginator_state = {"offset": resume.offset}
+ if is_fanout:
+ initial_paginator_state = resume.fanout_state or None
+ elif resume.offset is not None:
+ initial_paginator_state = {"offset": resume.offset}
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
# Persist only when a next page remains; save AFTER a page is yielded so a crash re-yields
# the last page (merge dedupes on the primary key) rather than skipping it.
- if state and state.get("offset") is not None:
+ if is_fanout:
+ resumable_source_manager.save_state(PicqerResumeConfig(fanout_state=state))
+ elif state and state.get("offset") is not None:
resumable_source_manager.save_state(PicqerResumeConfig(offset=int(state["offset"])))
- resource = rest_api_resource(
+ resources = rest_api_resources(
rest_config,
team_id,
job_id,
@@ -172,6 +219,7 @@ def picqer_source(
resume_hook=save_checkpoint,
initial_paginator_state=initial_paginator_state,
)
+ resource = next(r for r in resources if r.name == endpoint)
return SourceResponse(
name=endpoint,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/settings.py
index 3e1fd8141d99..1ecd068b5334 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/settings.py
@@ -22,15 +22,23 @@ def _incremental_fields(cursor_field: str) -> list[IncrementalField]:
]
-@dataclass
+@dataclass(frozen=True)
+class PicqerFanoutConfig:
+ # Parent endpoint whose rows drive the child requests, and the parent field substituted into
+ # the `{<resolve_field>}` placeholder of the child path.
+ parent_name: str
+ resolve_field: str
+
+
+@dataclass(frozen=True)
class PicqerEndpointConfig:
name: str
path: str # Path under /api/v1, e.g. "/orders"
primary_keys: list[str]
# Query param that filters the list by modification time (e.g. `updated_after`). Only set when
# the API documents a genuine server-side UPDATE-based filter — creation-only filters
- # (`sincedate`) are deliberately left off, since using them as an incremental cursor would miss
- # updates to mutable records. None => full refresh.
+ # (`sincedate`) are deliberately left off for mutable records, since using them as an
+ # incremental cursor would miss updates. Immutable records may use them. None => full refresh.
incremental_filter_param: Optional[str] = None
# Response field the `incremental_filter_param` compares against, and the cursor the pipeline
# tracks. Must be present in every row so the watermark can advance. None => full refresh.
@@ -39,6 +47,10 @@ class PicqerEndpointConfig:
# sync). None when the resource exposes no reliable creation timestamp.
partition_key: Optional[str] = None
should_sync_default: bool = True
+ # False for list endpoints the docs do not mark as paginated. An offset loop on an endpoint that
+ # ignores `offset` would fetch the same full page forever.
+ paginated: bool = True
+ fanout: Optional[PicqerFanoutConfig] = None
@property
def supports_incremental(self) -> bool:
@@ -91,6 +103,31 @@ PICQER_ENDPOINTS: dict[str, PicqerEndpointConfig] = {
incremental_cursor_field="updated_at",
partition_key="created_at",
),
+ # Shipments can be cancelled after creation and only expose a creation-date filter
+ # (`sincedate`), so they sync full refresh like orders and picklists.
+ "shipments": PicqerEndpointConfig(
+ name="shipments",
+ path="/shipments",
+ primary_keys=["idshipment"],
+ partition_key="created",
+ ),
+ # Stock history rows can never be changed, so the creation-date filter is a safe cursor.
+ "stockhistory": PicqerEndpointConfig(
+ name="stockhistory",
+ path="/stockhistory",
+ primary_keys=["idproduct_stock_history"],
+ incremental_filter_param="sincedate",
+ incremental_cursor_field="changed_at",
+ partition_key="changed_at",
+ ),
+ # Current stock per product per warehouse. Fanned out over warehouses rather than products,
+ # because a warehouse returns all of its products in pages of 100.
+ "warehouse_stock": PicqerEndpointConfig(
+ name="warehouse_stock",
+ path="/warehouses/{idwarehouse}/stock",
+ primary_keys=["idwarehouse", "idproduct"],
+ fanout=PicqerFanoutConfig(parent_name="warehouses", resolve_field="idwarehouse"),
+ ),
# Catalog / reference resources. Products carry a `created` field but expose no time filter.
"products": PicqerEndpointConfig(
name="products",
@@ -133,6 +170,12 @@ PICQER_ENDPOINTS: dict[str, PicqerEndpointConfig] = {
path="/tags",
primary_keys=["idtag"],
),
+ "return_reasons": PicqerEndpointConfig(
+ name="return_reasons",
+ path="/return_reasons",
+ primary_keys=["idreturn_reason"],
+ paginated=False,
+ ),
}
ENDPOINTS = tuple(PICQER_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/tests/test_picqer.py b/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/tests/test_picqer.py
index c613d9e35d05..e42a0b2f931a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/tests/test_picqer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/picqer/tests/test_picqer.py
@@ -7,6 +7,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
ScriptedResponse,
SourceDriver,
always,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.picqer import PicqerSourceConfig
@@ -85,17 +86,114 @@ class TestPagination:
assert len(result.rows) == 2
assert result.params("offset") == [str(PAGE_SIZE)]
- def test_incremental_filter_present_on_every_page(self) -> None:
+ @pytest.mark.parametrize(
+ "endpoint,incremental_field,filter_param",
+ [
+ ("purchaseorders", "updated", "updated_after"),
+ ("stockhistory", "changed_at", "sincedate"),
+ ],
+ )
+ def test_incremental_filter_present_on_every_page(
+ self, endpoint: str, incremental_field: str, filter_param: str
+ ) -> None:
result = _driver().run(
- "purchaseorders",
+ endpoint,
[_page(PAGE_SIZE), _page(1)],
- incremental_field="updated",
+ incremental_field=incremental_field,
db_incremental_field_last_value=datetime(2020, 1, 2, 3, 4, 5),
)
# The filter must stay on every paginated request so pagination walks only the filtered set.
assert result.raised is None
- assert result.params("updated_after") == ["2020-01-02 03:04:05"] * 2
+ assert result.params(filter_param) == ["2020-01-02 03:04:05"] * 2
+ assert result.params("offset") == ["0", str(PAGE_SIZE)]
+
+ def test_unpaginated_endpoint_stops_after_one_full_page(self) -> None:
+ # return_reasons is not documented as paginated; an offset loop would refetch the same full
+ # page forever if the endpoint ignores `offset`.
+ reasons = [{"idreturn_reason": i, "name": f"Reason {i}"} for i in range(PAGE_SIZE)]
+ result = _driver().run("return_reasons", route({"/return_reasons": [ScriptedResponse(json=reasons)]}))
+
+ assert result.raised is None
+ assert len(result.rows) == PAGE_SIZE
+ assert result.paths == ["/api/v1/return_reasons"]
+ assert result.params("offset") == [None]
+
+
+def _stock_row(idproduct: int, idwarehouse: int | None) -> dict[str, Any]:
+ stock: dict[str, Any] = {"stock": 5, "reserved": 1, "freestock": 4}
+ if idwarehouse is not None:
+ stock["idwarehouse"] = idwarehouse
+ return {"idproduct": idproduct, "productcode": f"SKU-{idproduct}", "stock": stock}
+
+
+def _warehouse_stock_script(warehouse_ids: list[int], stock_pages: dict[int, list[ScriptedResponse]]) -> Any:
+ table: dict[str, list[ScriptedResponse]] = {
+ "/warehouses": [ScriptedResponse(json=[{"idwarehouse": i, "name": f"Warehouse {i}"} for i in warehouse_ids])]
+ }
+ for idwarehouse, pages in stock_pages.items():
+ table[f"/warehouses/{idwarehouse}/stock"] = pages
+ return route(table)
+
+
+class TestWarehouseStock:
+ def test_fans_out_over_warehouses_and_flattens_stock(self) -> None:
+ script = _warehouse_stock_script(
+ [1, 2, 3],
+ {
+ 1: [
+ ScriptedResponse(json=[_stock_row(i, 1) for i in range(PAGE_SIZE)]),
+ ScriptedResponse(json=[_stock_row(PAGE_SIZE, 1)]),
+ ],
+ # Deleted between the warehouse listing and the stock fetch.
+ 2: [ScriptedResponse(status=404, json={"error": "not found"})],
+ # Nested object without its own idwarehouse: the parent id fills the key.
+ 3: [ScriptedResponse(json=[_stock_row(7, None)])],
+ },
+ )
+
+ result = _driver().run("warehouse_stock", script)
+
+ assert result.raised is None
+ assert len(result.rows) == PAGE_SIZE + 2
+ keys = {(row["idwarehouse"], row["idproduct"]) for row in result.rows}
+ assert len(keys) == len(result.rows)
+ assert (3, 7) in keys
+ assert result.rows[-1] == {
+ "idproduct": 7,
+ "productcode": "SKU-7",
+ "stock": 5,
+ "reserved": 1,
+ "freestock": 4,
+ "idwarehouse": 3,
+ }
+ assert result.paths == [
+ "/api/v1/warehouses",
+ "/api/v1/warehouses/1/stock",
+ "/api/v1/warehouses/1/stock",
+ "/api/v1/warehouses/2/stock",
+ "/api/v1/warehouses/3/stock",
+ ]
+ assert result.params("offset")[1:3] == ["0", str(PAGE_SIZE)]
+
+ def test_resumes_mid_warehouse_and_skips_completed_ones(self) -> None:
+ script = _warehouse_stock_script([1, 2], {2: [ScriptedResponse(json=[_stock_row(9, 2)])]})
+
+ result = _driver().run(
+ "warehouse_stock",
+ script,
+ resume_state=PicqerResumeConfig(
+ fanout_state={
+ "completed": ["/warehouses/1/stock"],
+ "current": "/warehouses/2/stock",
+ "child_state": {"offset": PAGE_SIZE},
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert [(row["idwarehouse"], row["idproduct"]) for row in result.rows] == [(2, 9)]
+ assert result.paths == ["/api/v1/warehouses", "/api/v1/warehouses/2/stock"]
assert result.params("offset") == ["0", str(PAGE_SIZE)]