:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add inventory and custom field tables to buildium (#119196)
Tom Owers62acd628fd26
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 100bf2a45d67..dd430ffa0302 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -1996,6 +1996,10 @@ Diffed against: <https://developer.buildium.com/>- [ ] `Task history (GET /v1/tasks/{taskId}/history)` — measure request activity, status changes, and time to resolution (medium)- [ ] `Applications (GET /v1/applicants/{applicantId}/applications)` — analyze rental applications and conversion from applicant to tenant (medium)- [ ] `Associations, units, and ownership accounts (GET /v1/associations; GET /v1/associations/units; GET /v1/associations/ownershipaccounts)` — analyze managed community associations and ownership relationships beyond rental properties (medium)+- [x] `Inventory assets (GET /v1/inventoryassets)` — track equipment and supplies by property, unit, and storage location (medium)+- [x] `Inventory storages (GET /v1/inventorystorages)` — resolve InventoryStorageId in inventory assets (medium)+- [x] `Asset categories (GET /v1/assetcategories)` — resolve CategoryId in inventory assets (medium)+- [x] `Custom field definitions (GET /v1/customfields/entityType/{entityType}/definitions)` — identify the custom fields configured for each entity type (medium)Note: The official reference embeds OpenAPI, which was parsed to compare every GET collection. OperatingBankAccountId and work-order task IDs identify missing parent records. Payments, lease transactions, task history, and applications need calls for each parent. Association coverage needs a separate resource family; settings, files, and write operations are excluded.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/buildium.py b/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/buildium.pyindex 03e756755305..3937be4e4e77 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/buildium.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/buildium.py@@ -7,6 +7,7 @@ from posthog.dataclasses import frozenfrom products.warehouse_sources.backend.temporal.data_imports.sources.buildium.settings import (BASE_URL,CREDENTIAL_PROBE_PATHS,+ CUSTOM_FIELD_ENTITY_TYPES,ENDPOINTS,EXCLUDED_COLUMNS,GL_ACCOUNT_BATCH_SIZE,@@ -18,6 +19,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.buildium.sORDER_BY,PAGE_SIZE,PRIMARY_KEYS,+ UNPAGINATED_ENDPOINTS,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_sessionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (@@ -89,6 +91,8 @@ def _list_resource(initial_offset: int | None = None,) -> Resource:endpoint_config: Endpoint = {"path": path, "data_selector": "$", "params": params}+ if endpoint in UNPAGINATED_ENDPOINTS:+ endpoint_config["paginator"] = "single_page"resource_config: EndpointResource = {"name": endpoint,"endpoint": endpoint_config,@@ -146,6 +150,16 @@ def _lease_transactions_resource(client: ClientConfig, team_id: int, job_id: strreturn next(resource for resource in resources if resource.name == "lease_transactions")+def _custom_field_definitions_resource(client: ClientConfig, entity_type: str, team_id: int, job_id: str) -> Resource:+ path = ENDPOINTS["custom_field_definitions"].replace("{entityType}", entity_type)+ return _list_resource(client, "custom_field_definitions", path, {}, team_id, job_id)+++def _iter_custom_field_definitions(client: ClientConfig, team_id: int, job_id: str) -> Iterator[list[dict[str, Any]]]:+ for entity_type in CUSTOM_FIELD_ENTITY_TYPES:+ yield from _custom_field_definitions_resource(client, entity_type, team_id, job_id)++def _gl_account_ids(client: ClientConfig, team_id: int, job_id: str) -> list[int]:accounts = _list_resource(client, "general_ledger_accounts", GL_ACCOUNTS_PATH, {"orderby": "Id asc"}, team_id, job_id@@ -216,7 +230,7 @@ def buildium_source(# Keep offset pagination stable when an object is updated during extraction. Ordering by the# mutable update timestamp can move an already-read object to a later page and skip another one.- params: dict[str, Any] = {"orderby": ORDER_BY.get(endpoint, "Id asc")}+ params: dict[str, Any] = {} if endpoint in UNPAGINATED_ENDPOINTS else {"orderby": ORDER_BY.get(endpoint, "Id asc")}if updated_from is not None:params["lastupdatedfrom"] = updated_from@@ -227,7 +241,12 @@ def buildium_source()resource: Resource- if endpoint == "lease_transactions":+ if endpoint == "custom_field_definitions":+ template = _custom_field_definitions_resource(client, CUSTOM_FIELD_ENTITY_TYPES[0], team_id, job_id)+ resource = Resource(+ _iter_custom_field_definitions, name=endpoint, hints=template._hints, args=(client, team_id, job_id)+ )+ elif endpoint == "lease_transactions":# Fan-out resources cannot resume, so this table restarts from the first lease after a hand-off.resource = _lease_transactions_resource(client, team_id, job_id)elif endpoint == "general_ledger_transactions":diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/canonical_descriptions.pyindex 4c95b8f99cd9..0e701d3b4007 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/canonical_descriptions.py@@ -123,4 +123,43 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"BalanceOver90Days": "Outstanding balance due from over 90 days ago.",},},+ "inventory_assets": {+ "description": "Inventory items and assets tracked for properties, units, and the company.",+ "docs_url": "https://developer.buildium.com/#operation/ExternalApiInventoryAsset_GetInventoryAssets",+ "columns": {+ "Id": "Identifier for this inventory asset.",+ "CategoryId": "Asset category of the item.",+ "PropertyId": "Property associated with the asset, where applicable.",+ "UnitId": "Unit associated with the asset, where applicable.",+ "InventoryStorageId": "Inventory storage location of the asset, where applicable.",+ "WarrantyEndDate": "Date the warranty expires, where applicable.",+ "Quantity": "Quantity of the asset.",+ },+ },+ "inventory_storages": {+ "description": "Storage locations that hold inventory assets.",+ "docs_url": "https://developer.buildium.com/#operation/ExternalApiInventoryStorage_GetInventoryStorages",+ "columns": {+ "Id": "Identifier for this inventory storage.",+ "Name": "Name of the inventory storage.",+ },+ },+ "asset_categories": {+ "description": "Categories used to group inventory assets.",+ "docs_url": "https://developer.buildium.com/#operation/ExternalApiAssetCategory_GetAllAssetCategories",+ "columns": {+ "Id": "Identifier for this asset category.",+ "Name": "Name of the asset category.",+ },+ },+ "custom_field_definitions": {+ "description": "Custom fields defined for each Buildium entity type, such as rentals, leases, and vendors.",+ "docs_url": "https://developer.buildium.com/#operation/ExternalApiCustomFieldsDefinitions_GetDefinitionsByEntityType",+ "columns": {+ "Id": "Identifier for this custom field definition.",+ "GroupId": "Custom field group that contains the definition.",+ "EntityTypeName": "Entity type the custom field applies to.",+ "Configuration": "Field type and settings of the custom field.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/settings.pyindex 850d87195a8a..7ab3e1939795 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/settings.py@@ -17,6 +17,10 @@ ENDPOINTS = {"general_ledger_transactions": "generalledger/transactions","lease_transactions": "leases/{leaseId}/transactions","lease_outstanding_balances": "leases/outstandingbalances",+ "inventory_assets": "inventoryassets",+ "inventory_storages": "inventorystorages",+ "asset_categories": "assetcategories",+ "custom_field_definitions": "customfields/entityType/{entityType}/definitions",}# Other list responses omit update timestamps, even when their endpoints accept update filters.INCREMENTAL_FIELDS = {@@ -30,7 +34,33 @@ ORDER_BY = {"lease_outstanding_balances": "LeaseId asc"}# The full account number is not needed for analysis, and the masked AccountNumber column stays.EXCLUDED_COLUMNS = {"bank_accounts": ("AccountNumberUnmasked",)}# Endpoints that need a parent id or required filters are probed through the list they depend on.-CREDENTIAL_PROBE_PATHS = {"lease_transactions": "leases", "general_ledger_transactions": "glaccounts"}+CREDENTIAL_PROBE_PATHS = {+ "lease_transactions": "leases",+ "general_ledger_transactions": "glaccounts",+ "custom_field_definitions": "customfields/entityType/Rentals/definitions",+}+# These lists return every row in one response and accept no paging or sort options.+UNPAGINATED_ENDPOINTS = ("inventory_storages", "asset_categories", "custom_field_definitions")+# Custom field definitions are listed per entity type, so the table reads each type in turn.+CUSTOM_FIELD_ENTITY_TYPES = (+ "Rentals",+ "Associations",+ "RentalUnits",+ "AssociationUnits",+ "RentalTenants",+ "AssociationTenants",+ "AssociationOwners",+ "Vendors",+ "Leases",+ "RentalOwners",+ "Tasks",+ "WorkOrders",+ "Projects",+ "OwnershipAccounts",+ "Prospects",+ "Applicants",+ "UnitListings",+)LEASE_TRANSACTIONS_PARENT = "leases"GL_ACCOUNTS_PATH = "glaccounts"diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/tests/test_buildium.py b/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/tests/test_buildium.pyindex 0ab3ab2dffcb..37616a8c302d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/tests/test_buildium.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/tests/test_buildium.py@@ -9,6 +9,7 @@ from requests.exceptions import HTTPErrorfrom products.warehouse_sources.backend.temporal.data_imports.sources.buildium.buildium import BuildiumResumeConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.buildium.settings import (AUTH_ERROR,+ CUSTOM_FIELD_ENTITY_TYPES,PERMISSION_ERROR,)from products.warehouse_sources.backend.temporal.data_imports.sources.buildium.source import BuildiumSource@@ -16,6 +17,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tesRecordedRequest,ScriptedResponse,SourceDriver,+ always,route,)from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.buildium import (@@ -258,3 +260,39 @@ def test_top_level_row_shaping_and_order(assert result.raised is Noneassert result.rows == [expected_row]assert result.params("orderby") == [expected_orderby]+++ ("endpoint", "path"), [("inventory_storages", "/v1/inventorystorages"), ("asset_categories", "/v1/assetcategories")]+)+def test_unpaginated_lists_read_one_response(endpoint: str, path: str) -> None:+ rows = [{"Id": index, "Name": f"Item {index}"} for index in range(3)]+ with patch("products.warehouse_sources.backend.temporal.data_imports.sources.buildium.buildium.PAGE_SIZE", 2):+ result = driver().run(endpoint, always(ScriptedResponse(json=rows)))++ assert result.raised is None+ assert result.rows == rows+ assert result.paths == [path]+ assert result.queries == [{}]+ assert result.saved_states == []+++def test_custom_field_definitions_read_every_entity_type() -> None:+ definitions = {+ "Rentals": [{"Id": 1, "EntityTypeName": "Rentals", "Configuration": {"Type": "ShortText"}}],+ "Vendors": [{"Id": 2, "EntityTypeName": "Vendors"}, {"Id": 3, "EntityTypeName": "Vendors"}],+ }++ def respond(request: RecordedRequest) -> ScriptedResponse:+ entity_type = request.path.split("/")[4]+ return ScriptedResponse(json=definitions.get(entity_type, []))++ result = driver().run("custom_field_definitions", respond)++ assert result.raised is None+ assert [row["Id"] for row in result.rows] == [1, 2, 3]+ assert result.rows[0]["Configuration"] == {"Type": "ShortText"}+ assert result.paths == [+ f"/v1/customfields/entityType/{entity_type}/definitions" for entity_type in CUSTOM_FIELD_ENTITY_TYPES+ ]+ assert all(params == {} for params in result.queries)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/tests/test_buildium_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/tests/test_buildium_source.pyindex c1c9801053e3..9bdfdb47f0c9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/tests/test_buildium_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/buildium/tests/test_buildium_source.py@@ -24,6 +24,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_("bills", "/v1/bills"),("lease_transactions", "/v1/leases"),("general_ledger_transactions", "/v1/glaccounts"),+ ("custom_field_definitions", "/v1/customfields/entityType/Rentals/definitions"),],)@pytest.mark.parametrize(