:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 4 missing tables to the Billomat source (#128675)
Tom Owers12bd021c4fb7
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex bb059bd10f15..11dca2133093 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -1648,10 +1648,10 @@ Diffed against: <https://www.billomat.com/en/api/>- [x] `Client contacts (GET /api/contacts?client_id={client_id})` — resolve contact_id in invoices and identify the billing contact for each client (high)- [x] `Order confirmations (GET /api/confirmations)` — resolve confirmation_id in invoices and connect confirmed orders to invoiced revenue (high)- [x] `Recurring invoices (GET /api/recurrings)` — resolve recurring_id in invoices and analyze scheduled recurring billing (high)-- [ ] `Payment reminders (GET /api/reminders)` — measure collection activity, overdue amounts, and results across reminder levels (medium)-- [ ] `Articles (GET /api/articles)` — analyze the product and service catalog, prices, and supplier relationships (medium)-- [ ] `Delivery notes (GET /api/delivery-notes)` — connect deliveries to the sales process and analyze fulfillment status (medium)-- [ ] `Activity feed (GET /api/activity-feed)` — track document actions and user activity over time (medium)+- [x] `Payment reminders (GET /api/reminders)` — measure collection activity, overdue amounts, and results across reminder levels (medium)+- [x] `Articles (GET /api/articles)` — analyze the product and service catalog, prices, and supplier relationships (medium)+- [x] `Delivery notes (GET /api/delivery-notes)` — connect deliveries to the sales process and analyze fulfillment status (medium)+- [x] `Activity feed (GET /api/activity-feed)` — track document actions and user activity over time (medium)- [ ] `Invoice comments (GET /api/invoice-comments?invoice_id={invoice_id})` — analyze invoice discussions and system actions during billing and collection (medium)Note: The official API index and individual reference pages supplied explicit GET examples for all listed resources. No usable specification was found at the index. Contacts and comments require calls for each client or invoice. Estimates correctly map to /offers; settings, webhooks, PDF downloads, and write operations are excluded.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/billomat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/billomat.pyindex 0696c76abac4..d2d2ec7d8963 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/billomat.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/billomat.py@@ -245,6 +245,65 @@ def get_resource(name: str, should_use_incremental_field: bool) -> EndpointResou},"table_format": "delta",},+ "Reminders": {+ "name": "Reminders",+ "table_name": "reminders",+ "write_disposition": write_disposition,+ "endpoint": {+ "data_selector": '"reminders"."reminder"',+ "path": "/reminders",+ "params": {+ "per_page": _PER_PAGE,+ "order_by": "date ASC",+ "from": _incremental_param() if should_use_incremental_field else None,+ },+ },+ "table_format": "delta",+ },+ "Articles": {+ "name": "Articles",+ "table_name": "articles",+ "write_disposition": "replace",+ "endpoint": {+ "data_selector": '"articles"."article"',+ "path": "/articles",+ "params": {+ "per_page": _PER_PAGE,+ "order_by": "id ASC",+ },+ },+ "table_format": "delta",+ },+ "DeliveryNotes": {+ "name": "DeliveryNotes",+ "table_name": "delivery_notes",+ "write_disposition": write_disposition,+ "endpoint": {+ "data_selector": '"delivery-notes"."delivery-note"',+ "path": "/delivery-notes",+ "params": {+ "per_page": _PER_PAGE,+ "order_by": "date ASC",+ "from": _incremental_param() if should_use_incremental_field else None,+ },+ },+ "table_format": "delta",+ },+ "ActivityFeed": {+ "name": "ActivityFeed",+ "table_name": "activity_feed",+ "write_disposition": "replace",+ "endpoint": {+ # The feed documents only `page`, with no `total` or sort option, so paging ends+ # on the first empty page.+ "data_selector": '"activity-feed"."activity"',+ "path": "/activity-feed",+ "params": {+ "per_page": _PER_PAGE,+ },+ },+ "table_format": "delta",+ },}return resources[name]diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/canonical_descriptions.pyindex 9153b14921b0..2a2b2bb7eee5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/canonical_descriptions.py@@ -205,4 +205,77 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"confirmation_id": "Confirmation the recurring invoice was created from, if any.",},},+ "Reminders": {+ "description": "A payment reminder (dunning letter) for an overdue invoice.",+ "docs_url": "https://www.billomat.com/en/api/reminders/",+ "columns": {+ "id": "Unique identifier for the reminder.",+ "created": "Time the reminder was created.",+ "status": "Reminder status: DRAFT, OPEN, OVERDUE, PAID or CANCELED.",+ "invoice_id": "Overdue invoice the reminder is for.",+ "contact_id": "Contact the reminder is addressed to, if any.",+ "reminder_level": "Dunning level of the reminder.",+ "reminder_level_name": "Name of the dunning level.",+ "date": "Reminder date.",+ "label": "Label text describing the project.",+ "subject": "Subject of the reminder.",+ "due_date": "Date by which the reminder must be paid.",+ "due_days": "Number of days until the reminder is due.",+ "total_gross": "Total amount including tax.",+ "paid_amount": "Amount already paid.",+ "open_amount": "Amount still open.",+ },+ },+ "Articles": {+ "description": "A product or service in the article catalog.",+ "docs_url": "https://www.billomat.com/en/api/articles/",+ "columns": {+ "id": "Unique identifier for the article.",+ "created": "Time the article was created.",+ "article_number": "Article number shown on documents.",+ "title": "Title of the article.",+ "description": "Description of the article.",+ "sales_price": "Default sales price.",+ "sales_price2": "Sales price for clients in price group 2.",+ "sales_price3": "Sales price for clients in price group 3.",+ "sales_price4": "Sales price for clients in price group 4.",+ "sales_price5": "Sales price for clients in price group 5.",+ "currency_code": "ISO currency code.",+ "unit_id": "Unit of the article.",+ "tax_id": "Tax rate of the article.",+ "purchase_price": "Purchase price.",+ "purchase_price_net_gross": "Whether the purchase price is NET or GROSS.",+ "supplier_id": "Supplier of the article, if any.",+ "type": "Article type: PRODUCT or SERVICE.",+ },+ },+ "DeliveryNotes": {+ "description": "A delivery note sent to a client with delivered goods.",+ "docs_url": "https://www.billomat.com/en/api/delivery-notes/",+ "columns": {+ "id": "Unique identifier for the delivery note.",+ "client_id": "Client the delivery note is addressed to.",+ "contact_id": "Contact the delivery note is addressed to, if any.",+ "created": "Time the delivery note was created.",+ "delivery_note_number": "Delivery note number shown on the document.",+ "status": "Delivery note status: DRAFT or CREATED.",+ "date": "Delivery date.",+ "label": "Label text describing the project.",+ "offer_id": "Estimate the delivery note was created from, if any.",+ "confirmation_id": "Confirmation the delivery note was created from, if any.",+ "invoice_id": "Invoice the delivery note was created from, if any.",+ },+ },+ "ActivityFeed": {+ "description": "An action on a document or a system message in the account activity feed.",+ "docs_url": "https://www.billomat.com/en/api/activity-feed/",+ "columns": {+ "resource": "Resource type of the document, such as invoices or delivery-notes.",+ "id": "ID of the document the activity belongs to.",+ "date": "Time of the activity.",+ "title": "Title of the activity.",+ "text": "Activity text.",+ "user_id": "User who performed the activity. Empty for system messages.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/settings.pyindex 613cbe4a06ab..c1780a880a2a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/settings.py@@ -11,12 +11,23 @@ ENDPOINTS = ("ClientContacts","Confirmations","Recurrings",+ "Reminders",+ "Articles",+ "DeliveryNotes",+ "ActivityFeed",)+# An activity's `id` is the ID of the document it happened to (an invoice can have many+# activities), so no single field is unique across the feed.+PRIMARY_KEYS: dict[str, list[str]] = {+ "ActivityFeed": ["resource", "id", "date", "text"],+}+# Billomat's `from`/`to` list filters apply to the document `date`, not a last-modified# timestamp, so this only catches new documents within a synced date window, not edits to# existing ones (the API has no `updated_since`-style filter at all). Clients, Suppliers,-# ClientContacts and Recurrings have no date filter, so they stay full-refresh only.+# ClientContacts, Recurrings, Articles and ActivityFeed have no date filter, so they stay+# full-refresh only.INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {"Invoices": [{@@ -66,4 +77,20 @@ INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {"field_type": IncrementalFieldType.Date,},],+ "Reminders": [+ {+ "label": "date",+ "type": IncrementalFieldType.Date,+ "field": "date",+ "field_type": IncrementalFieldType.Date,+ },+ ],+ "DeliveryNotes": [+ {+ "label": "date",+ "type": IncrementalFieldType.Date,+ "field": "date",+ "field_type": IncrementalFieldType.Date,+ },+ ],}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/source.pyindex 605d4b5f70bf..0dde9b6af418 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/source.py@@ -21,6 +21,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.billomat.bfrom products.warehouse_sources.backend.temporal.data_imports.sources.billomat.settings import (ENDPOINTS,INCREMENTAL_FIELDS,+ PRIMARY_KEYS,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (@@ -147,7 +148,7 @@ class BillomatSource(ResumableSource[BillomatSourceConfig, BillomatResumeConfig]return SourceResponse(name=resource.name,items=lambda: resource,- primary_keys=["id"],+ primary_keys=PRIMARY_KEYS.get(inputs.schema_name, ["id"]),column_hints=resource.column_hints,)@@ -157,7 +158,7 @@ class BillomatSource(ResumableSource[BillomatSourceConfig, BillomatResumeConfig]name=ExternalDataSourceType.BILLOMAT,category=DataWarehouseSourceCategory.FINANCE___ACCOUNTING,label="Billomat",- caption="Sync clients, contacts, suppliers, invoices, payments, estimates, confirmations, recurring invoices, credit notes and incoming supplier bills from Billomat.",+ caption="Sync clients, contacts, suppliers, articles, invoices, payments, reminders, estimates, confirmations, delivery notes, recurring invoices, credit notes, incoming supplier bills and the activity feed from Billomat.",docsUrl="https://posthog.com/docs/cdp/sources/billomat",iconPath="/static/services/billomat.png",releaseStatus=ReleaseStatus.BETA,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/tests/test_billomat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/tests/test_billomat.pyindex c7cb4b01e9dc..cc92cf44ddd6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/tests/test_billomat.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/billomat/tests/test_billomat.py@@ -120,6 +120,8 @@ class TestBillomatSourceResumeBehavior:("Invoices", "/api/invoices", "invoices", "invoice"),("InvoicePayments", "/api/invoice-payments", "invoice-payments", "invoice-payment"),("Confirmations", "/api/confirmations", "confirmations", "confirmation"),+ ("Reminders", "/api/reminders", "reminders", "reminder"),+ ("DeliveryNotes", "/api/delivery-notes", "delivery-notes", "delivery-note"),],)def test_incremental_run_sends_formatted_from_param(self, schema: str, path: str, wrapper: str, item: str) -> None:@@ -136,6 +138,46 @@ class TestBillomatSourceResumeBehavior:assert result.requests[0].param("order_by") == "date ASC"assert result.rows == [{"id": "1"}]+ @pytest.mark.parametrize(+ ("schema", "path", "pages", "expected_rows"),+ [+ (+ "Articles",+ "/api/articles",+ [{"articles": {"article": [{"id": "7", "title": "Webdesign"}], "total": "1"}}],+ [{"id": "7", "title": "Webdesign"}],+ ),+ (+ # The feed carries no `total`, so paging only stops on an empty page.+ "ActivityFeed",+ "/api/activity-feed",+ [+ {"activity-feed": {"activity": [{"resource": "invoices", "id": "5", "text": "Invoice created."}]}},+ {"activity-feed": {"activity": [{"resource": "offers", "id": "6", "text": "Estimate sent."}]}},+ {"activity-feed": {"activity": []}},+ ],+ [+ {"resource": "invoices", "id": "5", "text": "Invoice created."},+ {"resource": "offers", "id": "6", "text": "Estimate sent."},+ ],+ ),+ ],+ )+ def test_full_refresh_endpoint_walks_pages_without_date_filter(+ self, schema: str, path: str, pages: list[dict[str, Any]], expected_rows: list[dict[str, Any]]+ ) -> None:+ result = self._driver().run(+ schema,+ [ScriptedResponse(json=page) for page in pages],+ should_use_incremental_field=True,+ db_incremental_field_last_value=date(2024, 5, 1),+ )++ assert result.raised is None+ assert result.paths == [path] * len(pages)+ assert result.params("from") == [None] * len(pages)+ assert result.rows == expected_rows+class TestBillomatClientContactsFanout:@staticmethod