:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): add billomat payments, contacts, confirmations and recurring invoices (#118508)
Tom Owers18d47a6a7752
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 4d0af1127925..5bbf008159b8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -1626,10 +1626,10 @@ Today (6): `Clients`, `CreditNotes`, `Estimates`, `Incomings`, `Invoices`, `SuppDiffed against: <https://www.billomat.com/en/api/>-- [ ] `Invoice payments (GET /api/invoice-payments)` — analyze payment timing, payment methods, and cash received beyond invoice totals (high)-- [ ] `Client contacts (GET /api/contacts?client_id={client_id})` — resolve contact_id in invoices and identify the billing contact for each client (high)-- [ ] `Order confirmations (GET /api/confirmations)` — resolve confirmation_id in invoices and connect confirmed orders to invoiced revenue (high)-- [ ] `Recurring invoices (GET /api/recurrings)` — resolve recurring_id in invoices and analyze scheduled recurring billing (high)+- [x] `Invoice payments (GET /api/invoice-payments)` — analyze payment timing, payment methods, and cash received beyond invoice totals (high)+- [x] `Client contacts (GET /api/contacts?client_id={client_id})` — resolve contact_id in invoices and identify the billing contact for each client (high)+- [x] `Order confirmations (GET /api/confirmations)` — resolve confirmation_id in invoices and connect confirmed orders to invoiced revenue (high)+- [x] `Recurring invoices (GET /api/recurrings)` — resolve recurring_id in invoices and analyze scheduled recurring billing (high)- [ ] `Payment reminders (GET /api/reminders)` — measure collection activity, overdue amounts, and results across reminder levels (medium)- [ ] `Articles (GET /api/articles)` — analyze the product and service catalog, prices, and supplier relationships (medium)- [ ] `Delivery notes (GET /api/delivery-notes)` — connect deliveries to the sales process and analyze fulfillment status (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/billomat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/billomat.pyindex 499d51638b85..0696c76abac4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/billomat.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/billomat.py@@ -8,12 +8,16 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.httfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (RESTAPIConfig,rest_api_resource,+ rest_api_resources,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (PageNumberPaginator,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import EndpointResource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (+ ClientConfig,+ EndpointResource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager# Billomat's documented maximum; requesting the largest page size keeps the per-15-minute@@ -23,7 +27,12 @@ _PER_PAGE = 1000@dataclasses.dataclass(frozen=False)class BillomatResumeConfig:- next_page: int+ next_page: int | None = None+ # ClientContacts fans out over clients and resumes by parent — see+ # `common.rest_source.__init__._make_paginate_dependent_resource` for the state shape.+ completed: list[str] | None = None+ current: str | None = None+ child_state: dict[str, Any] | None = Nonedef _extract_total_count(response: Response) -> Optional[int]:@@ -177,6 +186,50 @@ def get_resource(name: str, should_use_incremental_field: bool) -> EndpointResou},"table_format": "delta",},+ "InvoicePayments": {+ "name": "InvoicePayments",+ "table_name": "invoice_payments",+ "write_disposition": write_disposition,+ "endpoint": {+ "data_selector": '"invoice-payments"."invoice-payment"',+ "path": "/invoice-payments",+ "params": {+ "per_page": _PER_PAGE,+ "order_by": "date ASC",+ "from": _incremental_param() if should_use_incremental_field else None,+ },+ },+ "table_format": "delta",+ },+ "Confirmations": {+ "name": "Confirmations",+ "table_name": "confirmations",+ "write_disposition": write_disposition,+ "endpoint": {+ "data_selector": '"confirmations"."confirmation"',+ "path": "/confirmations",+ "params": {+ "per_page": _PER_PAGE,+ "order_by": "date ASC",+ "from": _incremental_param() if should_use_incremental_field else None,+ },+ },+ "table_format": "delta",+ },+ "Recurrings": {+ "name": "Recurrings",+ "table_name": "recurrings",+ "write_disposition": "replace",+ "endpoint": {+ "data_selector": '"recurrings"."recurring"',+ "path": "/recurrings",+ "params": {+ "per_page": _PER_PAGE,+ "order_by": "id ASC",+ },+ },+ "table_format": "delta",+ },"Incomings": {"name": "Incomings","table_name": "incomings",@@ -196,6 +249,29 @@ def get_resource(name: str, should_use_incremental_field: bool) -> EndpointResoureturn resources[name]+def _client_contacts_resources() -> list[str | EndpointResource]:+ return [+ get_resource("Clients", should_use_incremental_field=False),+ {+ "name": "ClientContacts",+ "table_name": "client_contacts",+ "write_disposition": "replace",+ "endpoint": {+ "data_selector": '"contacts"."contact"',+ # The contacts list only answers for one client at a time (client_id is mandatory),+ # and resolve binds into paths only, so the placeholder rides in the query string.+ "path": "/contacts?client_id={client_id}",+ "params": {+ "client_id": {"type": "resolve", "resource": "Clients", "field": "id"},+ "per_page": _PER_PAGE,+ "order_by": "id ASC",+ },+ },+ "table_format": "delta",+ },+ ]++def billomat_source(api_key: str,billomat_id: str,@@ -216,22 +292,52 @@ def billomat_source(headers=headers, redact_values=tuple(v for v in (api_key, app_secret) if v), allow_redirects=False)+ resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ client: ClientConfig = {+ "base_url": f"https://{billomat_id}.billomat.net/api",+ "session": session,+ "paginator": BillomatPaginator(per_page=_PER_PAGE, base_page=1, page=1, page_param="page"),+ }++ if endpoint == "ClientContacts":+ fanout_state: Optional[dict[str, Any]] = None+ if resume_config is not None and (resume_config.completed or resume_config.current):+ fanout_state = {+ "completed": resume_config.completed or [],+ "current": resume_config.current,+ "child_state": resume_config.child_state,+ }++ def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:+ if state is not None:+ resumable_source_manager.save_state(+ BillomatResumeConfig(+ completed=state.get("completed"),+ current=state.get("current"),+ child_state=state.get("child_state"),+ )+ )++ resources = rest_api_resources(+ {"client": client, "resources": _client_contacts_resources()},+ team_id,+ job_id,+ None,+ resume_hook=save_fanout_checkpoint,+ initial_paginator_state=fanout_state,+ )+ return next(resource for resource in resources if resource.name == "ClientContacts")+initial_paginator_state: Optional[dict[str, Any]] = None- if resumable_source_manager.can_resume():- resume_config = resumable_source_manager.load_state()- if resume_config is not None:- initial_paginator_state = {"page": resume_config.next_page}+ if resume_config is not None and resume_config.next_page is not None:+ initial_paginator_state = {"page": resume_config.next_page}def save_checkpoint(state: Optional[dict[str, Any]]) -> None:if state and state.get("page") is not None:resumable_source_manager.save_state(BillomatResumeConfig(next_page=int(state["page"])))config: RESTAPIConfig = {- "client": {- "base_url": f"https://{billomat_id}.billomat.net/api",- "session": session,- "paginator": BillomatPaginator(per_page=_PER_PAGE, base_page=1, page=1, page_param="page"),- },+ "client": client,"resources": [get_resource(endpoint, should_use_incremental_field)],}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/canonical_descriptions.pyindex 9d1065ce3e21..9153b14921b0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/canonical_descriptions.py@@ -125,4 +125,84 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"category": "Expense category assigned to the bill.",},},+ "InvoicePayments": {+ "description": "A payment booked against an invoice.",+ "docs_url": "https://www.billomat.com/en/api/invoices/payments/",+ "columns": {+ "id": "Unique identifier for the payment.",+ "created": "Time the payment was recorded.",+ "invoice_id": "Invoice the payment was booked against.",+ "user_id": "User who recorded the payment.",+ "date": "Date of the payment.",+ "amount": "Paid amount.",+ "comment": "Comment on the payment.",+ "transaction_purpose": "Transaction purpose text from the bank transfer.",+ "type": "Payment type, e.g. BANK_TRANSFER, CASH, PAYPAL, CREDIT_CARD or CREDIT_NOTE.",+ },+ },+ "ClientContacts": {+ "description": "A contact person stored on a client.",+ "docs_url": "https://www.billomat.com/en/api/clients/contacts/",+ "columns": {+ "id": "Unique identifier for the contact.",+ "client_id": "Client the contact belongs to.",+ "created": "Time the contact was created.",+ "label": "Label of the contact.",+ "name": "Company name.",+ "salutation": "Salutation.",+ "first_name": "First name.",+ "last_name": "Last name.",+ "street": "Street address.",+ "zip": "Postal code.",+ "city": "City.",+ "state": "State, county, district or region.",+ "country_code": "ISO 3166 alpha-2 country code.",+ "phone": "Phone number.",+ "fax": "Fax number.",+ "mobile": "Mobile number.",+ "email": "Email address.",+ "www": "Website URL.",+ },+ },+ "Confirmations": {+ "description": "An order confirmation sent to a client.",+ "docs_url": "https://www.billomat.com/en/api/confirmations/",+ "columns": {+ "id": "Unique identifier for the confirmation.",+ "client_id": "Client the confirmation is addressed to.",+ "contact_id": "Contact the confirmation is addressed to, if any.",+ "created": "Time the confirmation was created.",+ "confirmation_number": "Confirmation number shown on the document.",+ "status": "Confirmation status: DRAFT, COMPLETED, CANCELED or CLEARED.",+ "date": "Confirmation date.",+ "label": "Label text describing the project.",+ "total_gross": "Total amount including tax.",+ "total_net": "Total amount excluding tax.",+ "currency_code": "ISO currency code.",+ "offer_id": "Estimate the confirmation was created from, if any.",+ },+ },+ "Recurrings": {+ "description": "A recurring invoice template that creates invoices on a schedule.",+ "docs_url": "https://www.billomat.com/en/api/recurrings/",+ "columns": {+ "id": "Unique identifier for the recurring invoice.",+ "created": "Time the recurring invoice was created.",+ "client_id": "Client the created invoices are addressed to.",+ "contact_id": "Contact the created invoices are addressed to, if any.",+ "name": "Name of the recurring invoice.",+ "cycle": "Interval unit: DAILY, WEEKLY, MONTHLY or YEARLY.",+ "cycle_number": "Number of interval units between invoices.",+ "action": "What happens on each run: CREATE, COMPLETE, EMAIL or MAIL.",+ "start_date": "Date of the first run.",+ "end_date": "Date after which no more invoices are created.",+ "last_creation_date": "Date the last invoice was created.",+ "next_creation_date": "Date the next invoice will be created.",+ "total_gross": "Total amount including tax per invoice.",+ "total_net": "Total amount excluding tax per invoice.",+ "currency_code": "ISO currency code.",+ "offer_id": "Estimate the recurring invoice was created from, if any.",+ "confirmation_id": "Confirmation the recurring invoice was created from, if any.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/settings.pyindex 7f3acc360ed0..613cbe4a06ab 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/settings.py@@ -7,12 +7,16 @@ ENDPOINTS = ("Estimates","CreditNotes","Incomings",+ "InvoicePayments",+ "ClientContacts",+ "Confirmations",+ "Recurrings",)# Billomat's `from`/`to` list filters apply to the document `date`, not a last-modified# timestamp, so this only catches new documents within a synced date window, not edits to-# existing ones (the API has no `updated_since`-style filter at all). Clients and Suppliers-# have no date filter, so they stay full-refresh only.+# existing ones (the API has no `updated_since`-style filter at all). Clients, Suppliers,+# ClientContacts and Recurrings have no date filter, so they stay full-refresh only.INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {"Invoices": [{@@ -46,4 +50,20 @@ INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {"field_type": IncrementalFieldType.Date,},],+ "InvoicePayments": [+ {+ "label": "date",+ "type": IncrementalFieldType.Date,+ "field": "date",+ "field_type": IncrementalFieldType.Date,+ },+ ],+ "Confirmations": [+ {+ "label": "date",+ "type": IncrementalFieldType.Date,+ "field": "date",+ "field_type": IncrementalFieldType.Date,+ },+ ],}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/source.pyindex 883369640db7..5aa63d47e857 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/source.py@@ -157,7 +157,7 @@ class BillomatSource(ResumableSource[BillomatSourceConfig, BillomatResumeConfig]name=ExternalDataSourceType.BILLOMAT,category=DataWarehouseSourceCategory.FINANCE___ACCOUNTING,label="Billomat",- caption="Sync clients, suppliers, invoices, estimates, credit notes and incoming supplier bills from Billomat.",+ caption="Sync clients, contacts, suppliers, invoices, payments, estimates, confirmations, recurring invoices, credit notes and incoming supplier bills from Billomat.",docsUrl="https://posthog.com/docs/cdp/sources/billomat",iconPath="/static/services/billomat.png",releaseStatus=ReleaseStatus.ALPHA,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/tests/test_billomat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/tests/test_billomat.pyindex 832fd4740f69..c7cb4b01e9dc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/tests/test_billomat.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/tests/test_billomat.py@@ -114,16 +114,68 @@ class TestBillomatSourceResumeBehavior:assert result.raised is Noneassert result.params("page") == ["1"]- def test_incremental_run_sends_formatted_from_param(self) -> None:+ @pytest.mark.parametrize(+ ("schema", "path", "wrapper", "item"),+ [+ ("Invoices", "/api/invoices", "invoices", "invoice"),+ ("InvoicePayments", "/api/invoice-payments", "invoice-payments", "invoice-payment"),+ ("Confirmations", "/api/confirmations", "confirmations", "confirmation"),+ ],+ )+ def test_incremental_run_sends_formatted_from_param(self, schema: str, path: str, wrapper: str, item: str) -> None:result = self._driver().run(- "Invoices",- [self._page("invoices", "invoice", {"id": "1"}, "1")],+ schema,+ [self._page(wrapper, item, {"id": "1"}, "1")],should_use_incremental_field=True,db_incremental_field_last_value=date(2024, 5, 1),)assert result.raised is None+ assert result.paths == [path]assert result.requests[0].param("from") == "2024-05-01"+ assert result.requests[0].param("order_by") == "date ASC"+ assert result.rows == [{"id": "1"}]+++class TestBillomatClientContactsFanout:+ @staticmethod+ def _driver() -> SourceDriver:+ return SourceDriver(BillomatSource(), BillomatSourceConfig(billomat_id="acme", api_key="test-key"))++ @staticmethod+ def _clients(*ids: str) -> ScriptedResponse:+ return ScriptedResponse(json={"clients": {"client": [{"id": i} for i in ids], "total": str(len(ids))}})++ @staticmethod+ def _contacts(client_id: str, *ids: str) -> ScriptedResponse:+ rows = [{"id": i, "client_id": client_id} for i in ids]+ return ScriptedResponse(json={"contacts": {"contact": rows, "total": str(len(ids))}})++ def test_fetches_contacts_for_each_client(self) -> None:+ result = self._driver().run(+ "ClientContacts",+ [self._clients("10", "20"), self._contacts("10", "1", "2"), self._contacts("20", "3")],+ )++ assert result.raised is None+ assert result.paths == ["/api/clients", "/api/contacts", "/api/contacts"]+ assert result.params("client_id") == [None, "10", "20"]+ assert result.rows == [+ {"id": "1", "client_id": "10"},+ {"id": "2", "client_id": "10"},+ {"id": "3", "client_id": "20"},+ ]++ def test_resume_skips_clients_already_synced(self) -> None:+ result = self._driver().run(+ "ClientContacts",+ [self._clients("10", "20"), self._contacts("20", "3")],+ resume_state=BillomatResumeConfig(completed=["/contacts?client_id=10"]),+ )++ assert result.raised is None+ assert result.params("client_id") == [None, "20"]+ assert result.rows == [{"id": "3", "client_id": "20"}]class TestValidateCredentials: