Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 5 missing tables to the Katana source (#124402)

Tom Owers3eb3d575c04d
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index c74d904e7d80..628a0361db5d 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -695,7 +695,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/k6_cloud/k6_cloud.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/kandji/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/katana/katana.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/katana/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/kernel/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/klaus/klaus.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/klaus/settings.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 8875fa0ac1e7..5e657c88ee4e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -6819,10 +6819,10 @@ Diffed against: <https://api.katanamrp.com/v1/openapi.json>
- [x] `sales_order_fulfillments` — shipment/fulfillment records — the transition from order to delivered (high)
- [x] `sales_return_rows` — line items behind sales_returns, needed to attribute returns to products (medium)
- [x] `manufacturing_order_operation_rows` — per-operation timings and assigned operators, for throughput and labor cost (medium)
-- [ ] `price_list_rows` — priced items per price list; price_lists alone is only the header (medium)
-- [ ] `stocktake_rows` — counted vs expected quantity per stocktake line — the actual shrinkage numbers (medium)
-- [ ] `bin_locations and bin_inventory` — bin-level stock lookup and on-hand quantities below the location grain we already sync (medium)
-- [ ] `operators` — lookup table resolving operator IDs referenced by manufacturing order operations (medium)
+- [x] `price_list_rows` — priced items per price list; price_lists alone is only the header (medium)
+- [x] `stocktake_rows` — counted vs expected quantity per stocktake line — the actual shrinkage numbers (medium)
+- [x] `bin_locations and bin_inventory` — bin-level stock lookup and on-hand quantities below the location grain we already sync (medium)
+- [x] `operators` — lookup table resolving operator IDs referenced by manufacturing order operations (medium)
Note: The docs site (developer.katanamrp.com) is a ReadMe app, but the live gateway serves a full OpenAPI 3.0 document at https://api.katanamrp.com/v1/openapi.json — 120 paths. PostHog's 18 tables cover almost every header-level object but essentially none of the row/line-item children, which is where most manufacturing analytics lives. Also unsynced but lower value: serial_numbers, batch_stocks, additional_costs, po_additional_cost_rows, demand_forecasts, negative_stock, custom_field_definitions, customer_addresses, supplier_addresses.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/katana/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/katana/canonical_descriptions.py
index ae5557f7f8e8..c06799d16a35 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/katana/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/katana/canonical_descriptions.py
@@ -330,6 +330,24 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"updated_at": "Timestamp when the stocktake was last updated.",
},
},
+ "stocktake_rows": {
+ "description": "Line items of a stocktake: the expected and counted quantity of each variant, and the difference between them.",
+ "docs_url": f"{_DOCS}/findstocktakerows",
+ "columns": {
+ "id": "Unique identifier for the stocktake row.",
+ "stocktake_id": "Stocktake the row belongs to.",
+ "variant_id": "Variant being counted.",
+ "batch_id": "Batch being counted, when the variant is batch-tracked.",
+ "bin_location_id": "Bin location being counted, when the location uses bins.",
+ "in_stock_quantity": "Quantity Katana expected to be in stock when the count started.",
+ "counted_quantity": "Quantity counted. Null until the row is counted.",
+ "discrepancy_quantity": "Difference between the counted and the expected quantity.",
+ "notes": "Notes on the counted row.",
+ "created_at": "Timestamp when the stocktake row was created.",
+ "updated_at": "Timestamp when the stocktake row was last updated.",
+ "deleted_at": "Timestamp when the stocktake row was deleted, if it was.",
+ },
+ },
"inventory_movements": {
"description": "Append-only log of every inventory movement created by Katana resources.",
"docs_url": f"{_DOCS}/list-all-inventory-movements",
@@ -362,6 +380,19 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"is_active": "Whether the price list is active.",
},
},
+ "price_list_rows": {
+ "description": "Variant prices of a price list. Full refresh only.",
+ "docs_url": f"{_DOCS}/getallpricelistrows",
+ "columns": {
+ "id": "Unique identifier for the price list row.",
+ "price_list_id": "Price list the row belongs to.",
+ "variant_id": "Variant the price applies to.",
+ "adjustment_method": "How the amount sets the price: fixed, percentage or markup.",
+ "amount": "Price, or the percentage applied, depending on the adjustment method.",
+ "created_at": "Timestamp when the price list row was created.",
+ "updated_at": "Timestamp when the price list row was last updated.",
+ },
+ },
"locations": {
"description": "Warehouses and factories where stock is held and orders are fulfilled.",
"docs_url": f"{_DOCS}/list-all-locations",
@@ -383,4 +414,36 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"is_default_purchases": "Whether this is the default purchases tax rate.",
},
},
+ "bin_locations": {
+ "description": "Storage bins inside a location. Full refresh only.",
+ "docs_url": f"{_DOCS}/getallbinlocations",
+ "columns": {
+ "id": "Unique identifier for the bin location.",
+ "name": "Bin name.",
+ "location_id": "Location the bin is in.",
+ "created_at": "Timestamp when the bin location was created.",
+ "updated_at": "Timestamp when the bin location was last updated.",
+ "deleted_at": "Timestamp when the bin location was deleted, if it was.",
+ },
+ },
+ "bin_inventory": {
+ "description": "Current in-stock, committed and expected stock per location, variant and bin. Full refresh only.",
+ "docs_url": f"{_DOCS}/getbininventory",
+ "columns": {
+ "location_id": "Location the stock balance is for.",
+ "variant_id": "Variant the stock balance is for.",
+ "bin_location_id": "Bin the stock is in. Null for stock with no bin assigned.",
+ "quantity_in_stock": "Quantity currently in stock, as a decimal string.",
+ "quantity_committed": "Quantity committed to orders, as a decimal string.",
+ "quantity_expected": "Quantity expected from incoming orders, as a decimal string.",
+ },
+ },
+ "operators": {
+ "description": "Shop floor and warehouse operators that work on manufacturing order operations. Full refresh only.",
+ "docs_url": f"{_DOCS}/getoperators",
+ "columns": {
+ "id": "Unique identifier for the operator.",
+ "operator_name": "Operator name.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/katana/katana.py b/products/warehouse_sources/backend/temporal/data_imports/sources/katana/katana.py
index b61aeb3169b7..8af7ad644a9c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/katana/katana.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/katana/katana.py
@@ -131,11 +131,10 @@ def katana_source(
"endpoint": {
"path": config.path,
"params": params,
- "data_selector": "data",
- # Every Katana list endpoint wraps results as {"data": [...]}. A 2xx body missing the
- # `data` list (maintenance page, changed envelope) would otherwise read as an empty
- # page and silently end the sync mid-run, so treat it as retryable rather than lose
- # data.
+ "data_selector": config.data_selector,
+ # A 2xx body without the expected row list (maintenance page, changed envelope) would
+ # otherwise read as an empty page and silently end the sync mid-run, so treat it as
+ # retryable rather than lose data.
"data_selector_malformed_retryable": True,
},
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/katana/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/katana/settings.py
index 8318dd7124d9..a73b4794392a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/katana/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/katana/settings.py
@@ -1,4 +1,6 @@
-from dataclasses import dataclass, field
+from dataclasses import field
+
+from posthog.dataclasses import frozen
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
@@ -19,7 +21,7 @@ _UPDATED_AND_CREATED: list[IncrementalField] = [_timestamp_field("updated_at"),
_CREATED_ONLY: list[IncrementalField] = [_timestamp_field("created_at")]
-@dataclass
+@frozen
class KatanaEndpointConfig:
name: str
path: str
@@ -30,6 +32,8 @@ class KatanaEndpointConfig:
# Stable creation-time column to partition on. Never updated_at — partitions must not move.
partition_key: str | None = "created_at"
default_incremental_field: str = "updated_at"
+ # JSON path to the row list. /operators returns a bare array instead of the {"data": [...]} envelope.
+ data_selector: str | None = "data"
KATANA_ENDPOINTS: dict[str, KatanaEndpointConfig] = {
@@ -91,6 +95,9 @@ KATANA_ENDPOINTS: dict[str, KatanaEndpointConfig] = {
name="stock_transfers", path="/stock_transfers", incremental_fields=_UPDATED_AND_CREATED
),
"stocktakes": KatanaEndpointConfig(name="stocktakes", path="/stocktakes", incremental_fields=_UPDATED_AND_CREATED),
+ "stocktake_rows": KatanaEndpointConfig(
+ name="stocktake_rows", path="/stocktake_rows", incremental_fields=_UPDATED_AND_CREATED
+ ),
"inventory_movements": KatanaEndpointConfig(
name="inventory_movements",
path="/inventory_movements",
@@ -111,7 +118,24 @@ KATANA_ENDPOINTS: dict[str, KatanaEndpointConfig] = {
"price_lists": KatanaEndpointConfig(
name="price_lists", path="/price_lists", incremental_fields=[], partition_key=None
),
+ # /price_list_rows has no timestamp filter either; it is one row per (price list, variant).
+ "price_list_rows": KatanaEndpointConfig(name="price_list_rows", path="/price_list_rows", incremental_fields=[]),
"locations": KatanaEndpointConfig(name="locations", path="/locations", incremental_fields=[], partition_key=None),
+ "bin_locations": KatanaEndpointConfig(
+ name="bin_locations", path="/bin_locations", incremental_fields=[], partition_key=None
+ ),
+ # Live per-(location, variant, bin) balance at the default VARIANT granularity. bin_location_id is
+ # null for stock with no bin assigned, and Katana drops a row once all its quantities reach zero.
+ "bin_inventory": KatanaEndpointConfig(
+ name="bin_inventory",
+ path="/bin_inventory",
+ primary_keys=["location_id", "variant_id", "bin_location_id"],
+ incremental_fields=[],
+ partition_key=None,
+ ),
+ "operators": KatanaEndpointConfig(
+ name="operators", path="/operators", incremental_fields=[], partition_key=None, data_selector=None
+ ),
"tax_rates": KatanaEndpointConfig(name="tax_rates", path="/tax_rates", incremental_fields=[], partition_key=None),
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/katana/tests/test_katana.py b/products/warehouse_sources/backend/temporal/data_imports/sources/katana/tests/test_katana.py
index 6e36e43da4dd..6b70ab65bb88 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/katana/tests/test_katana.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/katana/tests/test_katana.py
@@ -104,11 +104,19 @@ class TestErrorHandling:
assert _SECRET_KEY not in message
assert message.startswith(f"{status} Client Error: {reason} for url: https://api.katanamrp.com/v1/customers")
- @parameterized.expand([("missing_data_key", {"items": []}), ("not_a_dict", [1, 2, 3])])
- def test_malformed_2xx_envelope_is_retried(self, _name: str, body: Any) -> None:
- # A 200 body without a list `data` key must NOT silently end the sync as an empty page — it's
- # reissued (retryable), then the valid page is read.
- result = _driver().run("customers", [ScriptedResponse(json=body), _page([{"id": 7}])])
+ @parameterized.expand(
+ [
+ ("missing_data_key", "customers", {"items": []}, {"data": [{"id": 7}]}),
+ ("not_a_dict", "customers", [1, 2, 3], {"data": [{"id": 7}]}),
+ # /operators returns a bare array, so the enveloped shape is the malformed one there.
+ ("bare_list_endpoint_gets_envelope", "operators", {"data": [{"id": 1}]}, [{"id": 7}]),
+ ("bare_list_endpoint_gets_error_object", "operators", {"statusCode": 500}, [{"id": 7}]),
+ ]
+ )
+ def test_malformed_2xx_envelope_is_retried(self, _name: str, endpoint: str, body: Any, valid_body: Any) -> None:
+ # A 200 body without the expected row list must NOT silently end the sync as an empty page or
+ # land as a stray row — it's reissued (retryable), then the valid page is read.
+ result = _driver().run(endpoint, [ScriptedResponse(json=body), ScriptedResponse(json=valid_body)])
assert result.raised is None
assert [row["id"] for row in result.rows] == [7]