:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): add pretix transactions table (#126778)
Tom Owers89dda39e4132
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex ba8ccf86df2f..c707c7ea22d0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -9744,10 +9744,10 @@ Today (14): `categories`, `checkin_lists`, `customers`, `events`, `gift_cards`,Diffed against: <https://docs.pretix.eu/dev/api/resources/index.html>-- [ ] `/organizers/{org}/transactions/` — pretix's financial ledger — the canonical per-line revenue record and the intended basis for revenue reporting (high)-- [ ] `/organizers/{org}/events/{event}/orders/{code}/payments/` — payment records with provider, amount and state; not embedded in the order JSON (high)-- [ ] `/organizers/{org}/events/{event}/orders/{code}/refunds/` — refund records, the other half of the money story and also not embedded in orders (high)-- [ ] `/organizers/{org}/events/{event}/items/{id}/variations/` — lookup resolving the variation IDs carried on order positions and quotas (high)+- [x] `/organizers/{org}/transactions/` — pretix's financial ledger — the canonical per-line revenue record and the intended basis for revenue reporting (high)+- [ ] `/organizers/{org}/events/{event}/orders/{code}/payments/` — payment records with provider, amount and state; not embedded in the order JSON (high). Skipped: the order resource embeds a `payments` array with the same fields, so the `orders` table already carries them, and this per-order endpoint would cost one request per order+- [ ] `/organizers/{org}/events/{event}/orders/{code}/refunds/` — refund records, the other half of the money story and also not embedded in orders (high). Skipped: the order resource embeds a `refunds` array with the same fields, so the `orders` table already carries them, and this per-order endpoint would cost one request per order+- [ ] `/organizers/{org}/events/{event}/items/{id}/variations/` — lookup resolving the variation IDs carried on order positions and quotas (high). Skipped: the item resource embeds a `variations` array with the same fields, so the `items` table already carries them, and this per-item endpoint would need a second fan-out level- [ ] `/organizers/{org}/events/{event}/checkins/` — check-in history — the attendance event stream behind the checkin_lists already synced (high)- [ ] `/organizers/ (list)` — top-level lookup; every other table is keyed by organizer slug and there is no organizer table (high)- [ ] `/organizers/{org}/orderpositions/` — flat ticket-level line items across all events, far cheaper to query than unnesting the positions array embedded in orders (medium)diff --git a/products/warehouse_sources/sources/pretix/canonical_descriptions.py b/products/warehouse_sources/sources/pretix/canonical_descriptions.pyindex 5db6c86196e5..8e316c781f06 100644--- a/products/warehouse_sources/sources/pretix/canonical_descriptions.py+++ b/products/warehouse_sources/sources/pretix/canonical_descriptions.py@@ -103,6 +103,29 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"item_price_overrides": "Prices that differ from the base event's item prices.",},},+ "transactions": {+ "description": "A line in the organizer's financial ledger. Changes to an order's positions or fees add rows, and cancellations add rows with a negative count.",+ "docs_url": "https://docs.pretix.eu/dev/api/resources/transactions.html",+ "columns": {+ "id": "Internal ID of the transaction.",+ "order": "The code of the order the transaction belongs to.",+ "event": "The slug of the event the transaction belongs to.",+ "created": "The date and time the transaction was recorded.",+ "datetime": "The date and time the transaction is financially relevant.",+ "positionid": "The number of the order position within its order, or null for fees.",+ "count": "The number of items, negative for cancellations.",+ "item": "The ID of the item sold, or null for fees.",+ "variation": "The ID of the item variation sold, or null.",+ "subevent": "The ID of the event series date, or null.",+ "price": "The gross price of the transaction.",+ "tax_rate": "The tax rate applied.",+ "tax_rule": "The ID of the tax rule applied, or null.",+ "tax_code": "The selected tax code, or null.",+ "tax_value": "The computed tax value.",+ "fee_type": "The type of fee, or null for products.",+ "internal_type": "An additional classification of the fee, or null for products.",+ },+ },"items": {"description": "A product that can be sold for an event, such as a ticket type or merchandise, including its variations.","docs_url": "https://docs.pretix.eu/en/latest/api/resources/items.html",diff --git a/products/warehouse_sources/sources/pretix/pretix.py b/products/warehouse_sources/sources/pretix/pretix.pyindex 188256cff968..cd40ffc2ed2e 100644--- a/products/warehouse_sources/sources/pretix/pretix.py+++ b/products/warehouse_sources/sources/pretix/pretix.py@@ -11,8 +11,8 @@ envelope where ``next`` is the full URL of the following page. Orders and invoicorganizer-level list endpoints spanning all events; the remaining event-scoped resources fan outover the organizer's events.-Only ``orders`` documents a server-side timestamp filter (``modified_since``) together with a-``last_modified`` ordering key, so it is the only incremental stream. Everything else is full+Only ``orders`` (``modified_since``) and ``transactions`` (``created_since``) document a+server-side timestamp filter, so they are the only incremental streams. Everything else is fullrefresh — pretix's other list endpoints only support ``If-Modified-Since`` conditional fetching(all-or-nothing 304s), which is not a per-row cursor.@@ -128,8 +128,8 @@ def _get_headers(api_token: str) -> dict[str, str]:return {"Authorization": f"Token {api_token}", "Accept": "application/json"}-def _format_modified_since(value: Any) -> str:- """Format an incremental value for pretix's ``modified_since`` filter (ISO 8601 UTC, Z suffix)."""+def _format_since(value: Any) -> str:+ """Format an incremental value for pretix's ``*_since`` filters (ISO 8601 UTC, Z suffix)."""if isinstance(value, datetime):utc_dt = value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)return utc_dt.strftime("%Y-%m-%dT%H:%M:%SZ")@@ -169,11 +169,11 @@ def _client_config(api_token: str, base_url: str) -> ClientConfig:}-def _modified_since_incremental() -> IncrementalConfig:+def _since_incremental(since_field: str, since_param: str) -> IncrementalConfig:return {- "cursor_path": "last_modified",- "start_param": "modified_since",- "convert": _format_modified_since,+ "cursor_path": since_field,+ "start_param": since_param,+ "convert": _format_since,}@@ -183,14 +183,14 @@ def _should_apply_incremental(db_incremental_field_last_value: Any,incremental_field: Optional[str],) -> bool:- # Mirror the hand-rolled gate: only narrow with the server-side `modified_since` filter when the- # endpoint supports it and the user's chosen cursor is the field that filter targets- # (`last_modified`). Honors the selected incremental_field rather than assuming it.+ # Only narrow with the server-side filter when the endpoint supports it and the user's chosen+ # cursor is the field that filter targets. Honors the selected incremental_field rather than+ # assuming it.return (should_use_incremental_field- and config.modified_since_field is not None+ and config.since_field is not Noneand db_incremental_field_last_value is not None- and incremental_field in (None, config.modified_since_field)+ and incremental_field in (None, config.since_field))@@ -228,10 +228,10 @@ def pretix_source(quoted_organizer = _quote_organizer(organizer)incremental: Optional[IncrementalConfig] = None- if _should_apply_incremental(+ if config.since_field is not None and _should_apply_incremental(config, should_use_incremental_field, db_incremental_field_last_value, incremental_field):- incremental = _modified_since_incremental()+ incremental = _since_incremental(config.since_field, config.since_param)resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else Noneinitial_paginator_state: Optional[dict[str, Any]] = Nonediff --git a/products/warehouse_sources/sources/pretix/settings.py b/products/warehouse_sources/sources/pretix/settings.pyindex e255e11f9334..2e431c1f1bff 100644--- a/products/warehouse_sources/sources/pretix/settings.py+++ b/products/warehouse_sources/sources/pretix/settings.py@@ -17,7 +17,7 @@ class EndpointScope(Enum):EVENT = "event"-@dataclass(frozen=False)+@dataclass(frozen=True)class PretixEndpointConfig:name: strpath: str@@ -26,20 +26,24 @@ class PretixEndpointConfig:incremental_fields: list[IncrementalField] = field(default_factory=list)# Stable created-style datetime field for partitioning. Never a last-modified field.partition_key: Optional[str] = None- # When set, the endpoint supports pretix's server-side `modified_since` filter targeting- # this resource field (true incremental sync); otherwise the stream is full refresh only.- modified_since_field: Optional[str] = None+ # When set, the endpoint supports a server-side `since_param` filter targeting this resource+ # field (true incremental sync); otherwise the stream is full refresh only.+ since_field: Optional[str] = None+ since_param: str = "modified_since"+ # Append sync re-appends the rows at the watermark, because pretix's `*_since` filters are+ # inclusive. Turn it off where those duplicates would be wrong.+ supports_append: bool = True# Explicit `ordering` value to request. Only set where the docs list it as a valid# ordering field for that endpoint — pretix rejects unknown ordering values.ordering: Optional[str] = None-def _last_modified_incremental_field() -> list[IncrementalField]:+def _datetime_incremental_field(field_name: str) -> list[IncrementalField]:return [{- "label": "last_modified",+ "label": field_name,"type": IncrementalFieldType.DateTime,- "field": "last_modified",+ "field": field_name,"field_type": IncrementalFieldType.DateTime,}]@@ -53,12 +57,15 @@ EVENT_SLUG_KEY = "event_slug"# Stream list cross-referenced against the Airbyte `source-pretix` connector plus the official API-# docs. `orders` is the only stream with a documented server-side timestamp filter-# (`modified_since`, orderable by `last_modified`), so it is the only incremental stream — every-# other list endpoint is full refresh. `orders` and `invoices` use the organizer-level endpoints+# docs. `orders` (`modified_since`) and `transactions` (`created_since`) are the only streams with a+# documented server-side timestamp filter, so they are the only incremental streams — every other+# list endpoint is full refresh. `orders` and `invoices` use the organizer-level endpoints# (added in pretix 2023.8) that span all events and stamp each row with its parent `event` slug,# avoiding a per-event fan-out for the two largest tables. Order positions arrive embedded in each-# order's `positions` array, so they don't need a dedicated stream.+# order's `positions` array, so they don't need a dedicated stream. The same holds for payments and+# refunds (each order's `payments`/`refunds` arrays) and item variations (each item's `variations`+# array): their dedicated endpoints are per-order or per-item, so a table would cost a request per+# parent row for data the parent stream already carries.PRETIX_ENDPOINTS: dict[str, PretixEndpointConfig] = {"events": PretixEndpointConfig(name="events",@@ -73,9 +80,9 @@ PRETIX_ENDPOINTS: dict[str, PretixEndpointConfig] = {scope=EndpointScope.ORGANIZER,# Order codes are only documented unique within their event; rows carry the event slug.primary_keys=["event", "code"],- incremental_fields=_last_modified_incremental_field(),+ incremental_fields=_datetime_incremental_field("last_modified"),partition_key="datetime",- modified_since_field="last_modified",+ since_field="last_modified",ordering="last_modified",),"invoices": PretixEndpointConfig(@@ -85,6 +92,21 @@ PRETIX_ENDPOINTS: dict[str, PretixEndpointConfig] = {# Invoice numbers are prefixed per event; rows carry the event slug.primary_keys=["event", "number"],),+ # Organizer-level financial ledger (pretix 2025.7+). Rows are append-only: a cancellation adds a+ # row with a negative count instead of changing an existing one, so `created` is a safe cursor.+ "transactions": PretixEndpointConfig(+ name="transactions",+ path="/organizers/{organizer}/transactions/",+ scope=EndpointScope.ORGANIZER,+ incremental_fields=_datetime_incremental_field("created"),+ partition_key="created",+ since_field="created",+ since_param="created_since",+ # Duplicate ledger rows would inflate revenue sums, so only merge (deduped on `id`) is offered.+ supports_append=False,+ # `id` is unique, so page boundaries stay stable when many rows share a `created` value.+ ordering="id",+ ),"customers": PretixEndpointConfig(name="customers",path="/organizers/{organizer}/customers/",@@ -162,4 +184,8 @@ INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {name: config.incremental_fields for name, config in PRETIX_ENDPOINTS.items()}-INCREMENTAL_ENDPOINTS = tuple(name for name, config in PRETIX_ENDPOINTS.items() if config.modified_since_field)+INCREMENTAL_ENDPOINTS = tuple(name for name, config in PRETIX_ENDPOINTS.items() if config.since_field)++APPEND_ENDPOINTS = tuple(+ name for name, config in PRETIX_ENDPOINTS.items() if config.since_field and config.supports_append+)diff --git a/products/warehouse_sources/sources/pretix/source.py b/products/warehouse_sources/sources/pretix/source.pyindex 5b593a70ab69..da704182b874 100644--- a/products/warehouse_sources/sources/pretix/source.py+++ b/products/warehouse_sources/sources/pretix/source.py@@ -9,6 +9,7 @@ from products.warehouse_sources.sources.pretix.pretix import (validate_credentials as validate_pretix_credentials,)from products.warehouse_sources.sources.pretix.settings import (+ APPEND_ENDPOINTS,ENDPOINTS,INCREMENTAL_ENDPOINTS,INCREMENTAL_FIELDS,@@ -125,7 +126,7 @@ Self-hosted users should set the API URL to their own pretix host (for example `SourceSchema(name=endpoint,supports_incremental=endpoint in INCREMENTAL_ENDPOINTS,- supports_append=endpoint in INCREMENTAL_ENDPOINTS,+ supports_append=endpoint in APPEND_ENDPOINTS,incremental_fields=INCREMENTAL_FIELDS.get(endpoint, []),)for endpoint in ENDPOINTSdiff --git a/products/warehouse_sources/sources/pretix/tests/test_pretix.py b/products/warehouse_sources/sources/pretix/tests/test_pretix.pyindex 63e0a01a98a6..355fca2bc4e9 100644--- a/products/warehouse_sources/sources/pretix/tests/test_pretix.py+++ b/products/warehouse_sources/sources/pretix/tests/test_pretix.py@@ -13,7 +13,7 @@ from products.warehouse_sources.sources.pretix.pretix import (INVALID_ORGANIZER_ERROR,PretixHostNotAllowedError,PretixResumeConfig,- _format_modified_since,+ _format_since,_quote_organizer,normalize_base_url,validate_credentials,@@ -49,7 +49,7 @@ class TestNormalizeBaseUrl:assert normalize_base_url(raw) == expected-class TestFormatModifiedSince:+class TestFormatSince:@parameterized.expand([(datetime(2026, 3, 4, 2, 58, 14, tzinfo=UTC), "2026-03-04T02:58:14Z"),@@ -58,8 +58,8 @@ class TestFormatModifiedSince:("2026-03-04T02:58:14Z", "2026-03-04T02:58:14Z"),])- def test_format_modified_since(self, value: object, expected: str) -> None:- assert _format_modified_since(value) == expected+ def test_format_since(self, value: object, expected: str) -> None:+ assert _format_since(value) == expectedclass TestQuoteOrganizer:@@ -81,16 +81,24 @@ class TestQuoteOrganizer:class TestOrganizerScopePagination:- def test_orders_incremental_url_has_filter_and_stable_ordering(self) -> None:+ @parameterized.expand(+ [+ ("orders", "last_modified", "modified_since", "last_modified"),+ ("transactions", "created", "created_since", "id"),+ ]+ )+ def test_incremental_url_has_filter_and_stable_ordering(+ self, endpoint: str, incremental_field: str, since_param: str, ordering: str+ ) -> None:result = _driver().run(- "orders",- [_envelope([{"code": "A1", "event": "conf"}])],- incremental_field="last_modified",+ endpoint,+ [_envelope([{"id": 1, "code": "A1", "event": "conf"}])],+ incremental_field=incremental_field,db_incremental_field_last_value=datetime(2026, 1, 1, tzinfo=UTC),)- assert result.requests[0].param("modified_since") == "2026-01-01T00:00:00Z"- assert result.requests[0].param("ordering") == "last_modified"+ assert result.requests[0].param(since_param) == "2026-01-01T00:00:00Z"+ assert result.requests[0].param("ordering") == ordering@parameterized.expand([diff --git a/products/warehouse_sources/sources/pretix/tests/test_pretix_source.py b/products/warehouse_sources/sources/pretix/tests/test_pretix_source.pyindex 35f4ae3c6893..106bb21a38d2 100644--- a/products/warehouse_sources/sources/pretix/tests/test_pretix_source.py+++ b/products/warehouse_sources/sources/pretix/tests/test_pretix_source.py@@ -16,10 +16,19 @@ class TestPretixSource:# the preserved token at a host they control.assert self.source.connection_host_fields == ["base_url"]- def test_orders_is_the_only_incremental_endpoint(self):- # Only `orders` has a documented server-side `modified_since` filter; advertising incremental- # on another endpoint would silently full-refresh under an incremental label.- assert set(INCREMENTAL_ENDPOINTS) == {"orders"}+ def test_only_endpoints_with_a_server_side_filter_are_incremental(self):+ # Only `orders` (`modified_since`) and `transactions` (`created_since`) have a documented+ # server-side timestamp filter; advertising incremental on another endpoint would silently+ # full-refresh under an incremental label.+ assert set(INCREMENTAL_ENDPOINTS) == {"orders", "transactions"}++ def test_transactions_does_not_offer_append_sync(self):+ # `created_since` is inclusive, so append sync would duplicate ledger rows at the watermark.+ schemas = {s.name: s for s in self.source.get_schemas(self.config, self.team_id)}++ assert schemas["transactions"].supports_incremental is True+ assert schemas["transactions"].supports_append is False+ assert schemas["orders"].supports_append is True@mock.patch("products.warehouse_sources.sources.pretix.source.pretix_source")def test_source_for_pipeline_plumbs_arguments(self, mock_pretix_source):