Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add pretix transactions table (#126778)

Tom Owers89dda39e4132
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index ba8ccf86df2f..c707c7ea22d0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9744,10 +9744,10 @@ Today (14): `categories`, `checkin_lists`, `customers`, `events`, `gift_cards`,
Diffed against: <https://docs.pretix.eu/dev/api/resources/index.html>
-- [ ] `/organizers/{org}/transactions/` — pretix's financial ledger — the canonical per-line revenue record and the intended basis for revenue reporting (high)
-- [ ] `/organizers/{org}/events/{event}/orders/{code}/payments/` — payment records with provider, amount and state; not embedded in the order JSON (high)
-- [ ] `/organizers/{org}/events/{event}/orders/{code}/refunds/` — refund records, the other half of the money story and also not embedded in orders (high)
-- [ ] `/organizers/{org}/events/{event}/items/{id}/variations/` — lookup resolving the variation IDs carried on order positions and quotas (high)
+- [x] `/organizers/{org}/transactions/` — pretix's financial ledger — the canonical per-line revenue record and the intended basis for revenue reporting (high)
+- [ ] `/organizers/{org}/events/{event}/orders/{code}/payments/` — payment records with provider, amount and state; not embedded in the order JSON (high). Skipped: the order resource embeds a `payments` array with the same fields, so the `orders` table already carries them, and this per-order endpoint would cost one request per order
+- [ ] `/organizers/{org}/events/{event}/orders/{code}/refunds/` — refund records, the other half of the money story and also not embedded in orders (high). Skipped: the order resource embeds a `refunds` array with the same fields, so the `orders` table already carries them, and this per-order endpoint would cost one request per order
+- [ ] `/organizers/{org}/events/{event}/items/{id}/variations/` — lookup resolving the variation IDs carried on order positions and quotas (high). Skipped: the item resource embeds a `variations` array with the same fields, so the `items` table already carries them, and this per-item endpoint would need a second fan-out level
- [ ] `/organizers/{org}/events/{event}/checkins/` — check-in history — the attendance event stream behind the checkin_lists already synced (high)
- [ ] `/organizers/ (list)` — top-level lookup; every other table is keyed by organizer slug and there is no organizer table (high)
- [ ] `/organizers/{org}/orderpositions/` — flat ticket-level line items across all events, far cheaper to query than unnesting the positions array embedded in orders (medium)
diff --git a/products/warehouse_sources/sources/pretix/canonical_descriptions.py b/products/warehouse_sources/sources/pretix/canonical_descriptions.py
index 5db6c86196e5..8e316c781f06 100644
--- a/products/warehouse_sources/sources/pretix/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/pretix/canonical_descriptions.py
@@ -103,6 +103,29 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"item_price_overrides": "Prices that differ from the base event's item prices.",
},
},
+ "transactions": {
+ "description": "A line in the organizer's financial ledger. Changes to an order's positions or fees add rows, and cancellations add rows with a negative count.",
+ "docs_url": "https://docs.pretix.eu/dev/api/resources/transactions.html",
+ "columns": {
+ "id": "Internal ID of the transaction.",
+ "order": "The code of the order the transaction belongs to.",
+ "event": "The slug of the event the transaction belongs to.",
+ "created": "The date and time the transaction was recorded.",
+ "datetime": "The date and time the transaction is financially relevant.",
+ "positionid": "The number of the order position within its order, or null for fees.",
+ "count": "The number of items, negative for cancellations.",
+ "item": "The ID of the item sold, or null for fees.",
+ "variation": "The ID of the item variation sold, or null.",
+ "subevent": "The ID of the event series date, or null.",
+ "price": "The gross price of the transaction.",
+ "tax_rate": "The tax rate applied.",
+ "tax_rule": "The ID of the tax rule applied, or null.",
+ "tax_code": "The selected tax code, or null.",
+ "tax_value": "The computed tax value.",
+ "fee_type": "The type of fee, or null for products.",
+ "internal_type": "An additional classification of the fee, or null for products.",
+ },
+ },
"items": {
"description": "A product that can be sold for an event, such as a ticket type or merchandise, including its variations.",
"docs_url": "https://docs.pretix.eu/en/latest/api/resources/items.html",
diff --git a/products/warehouse_sources/sources/pretix/pretix.py b/products/warehouse_sources/sources/pretix/pretix.py
index 188256cff968..cd40ffc2ed2e 100644
--- a/products/warehouse_sources/sources/pretix/pretix.py
+++ b/products/warehouse_sources/sources/pretix/pretix.py
@@ -11,8 +11,8 @@ envelope where ``next`` is the full URL of the following page. Orders and invoic
organizer-level list endpoints spanning all events; the remaining event-scoped resources fan out
over the organizer's events.
-Only ``orders`` documents a server-side timestamp filter (``modified_since``) together with a
-``last_modified`` ordering key, so it is the only incremental stream. Everything else is full
+Only ``orders`` (``modified_since``) and ``transactions`` (``created_since``) document a
+server-side timestamp filter, so they are the only incremental streams. Everything else is full
refresh — pretix's other list endpoints only support ``If-Modified-Since`` conditional fetching
(all-or-nothing 304s), which is not a per-row cursor.
@@ -128,8 +128,8 @@ def _get_headers(api_token: str) -> dict[str, str]:
return {"Authorization": f"Token {api_token}", "Accept": "application/json"}
-def _format_modified_since(value: Any) -> str:
- """Format an incremental value for pretix's ``modified_since`` filter (ISO 8601 UTC, Z suffix)."""
+def _format_since(value: Any) -> str:
+ """Format an incremental value for pretix's ``*_since`` filters (ISO 8601 UTC, Z suffix)."""
if isinstance(value, datetime):
utc_dt = value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
return utc_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -169,11 +169,11 @@ def _client_config(api_token: str, base_url: str) -> ClientConfig:
}
-def _modified_since_incremental() -> IncrementalConfig:
+def _since_incremental(since_field: str, since_param: str) -> IncrementalConfig:
return {
- "cursor_path": "last_modified",
- "start_param": "modified_since",
- "convert": _format_modified_since,
+ "cursor_path": since_field,
+ "start_param": since_param,
+ "convert": _format_since,
}
@@ -183,14 +183,14 @@ def _should_apply_incremental(
db_incremental_field_last_value: Any,
incremental_field: Optional[str],
) -> bool:
- # Mirror the hand-rolled gate: only narrow with the server-side `modified_since` filter when the
- # endpoint supports it and the user's chosen cursor is the field that filter targets
- # (`last_modified`). Honors the selected incremental_field rather than assuming it.
+ # Only narrow with the server-side filter when the endpoint supports it and the user's chosen
+ # cursor is the field that filter targets. Honors the selected incremental_field rather than
+ # assuming it.
return (
should_use_incremental_field
- and config.modified_since_field is not None
+ and config.since_field is not None
and db_incremental_field_last_value is not None
- and incremental_field in (None, config.modified_since_field)
+ and incremental_field in (None, config.since_field)
)
@@ -228,10 +228,10 @@ def pretix_source(
quoted_organizer = _quote_organizer(organizer)
incremental: Optional[IncrementalConfig] = None
- if _should_apply_incremental(
+ if config.since_field is not None and _should_apply_incremental(
config, should_use_incremental_field, db_incremental_field_last_value, incremental_field
):
- incremental = _modified_since_incremental()
+ incremental = _since_incremental(config.since_field, config.since_param)
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
initial_paginator_state: Optional[dict[str, Any]] = None
diff --git a/products/warehouse_sources/sources/pretix/settings.py b/products/warehouse_sources/sources/pretix/settings.py
index e255e11f9334..2e431c1f1bff 100644
--- a/products/warehouse_sources/sources/pretix/settings.py
+++ b/products/warehouse_sources/sources/pretix/settings.py
@@ -17,7 +17,7 @@ class EndpointScope(Enum):
EVENT = "event"
-@dataclass(frozen=False)
+@dataclass(frozen=True)
class PretixEndpointConfig:
name: str
path: str
@@ -26,20 +26,24 @@ class PretixEndpointConfig:
incremental_fields: list[IncrementalField] = field(default_factory=list)
# Stable created-style datetime field for partitioning. Never a last-modified field.
partition_key: Optional[str] = None
- # When set, the endpoint supports pretix's server-side `modified_since` filter targeting
- # this resource field (true incremental sync); otherwise the stream is full refresh only.
- modified_since_field: Optional[str] = None
+ # When set, the endpoint supports a server-side `since_param` filter targeting this resource
+ # field (true incremental sync); otherwise the stream is full refresh only.
+ since_field: Optional[str] = None
+ since_param: str = "modified_since"
+ # Append sync re-appends the rows at the watermark, because pretix's `*_since` filters are
+ # inclusive. Turn it off where those duplicates would be wrong.
+ supports_append: bool = True
# Explicit `ordering` value to request. Only set where the docs list it as a valid
# ordering field for that endpoint — pretix rejects unknown ordering values.
ordering: Optional[str] = None
-def _last_modified_incremental_field() -> list[IncrementalField]:
+def _datetime_incremental_field(field_name: str) -> list[IncrementalField]:
return [
{
- "label": "last_modified",
+ "label": field_name,
"type": IncrementalFieldType.DateTime,
- "field": "last_modified",
+ "field": field_name,
"field_type": IncrementalFieldType.DateTime,
}
]
@@ -53,12 +57,15 @@ EVENT_SLUG_KEY = "event_slug"
# Stream list cross-referenced against the Airbyte `source-pretix` connector plus the official API
-# docs. `orders` is the only stream with a documented server-side timestamp filter
-# (`modified_since`, orderable by `last_modified`), so it is the only incremental stream — every
-# other list endpoint is full refresh. `orders` and `invoices` use the organizer-level endpoints
+# docs. `orders` (`modified_since`) and `transactions` (`created_since`) are the only streams with a
+# documented server-side timestamp filter, so they are the only incremental streams — every other
+# list endpoint is full refresh. `orders` and `invoices` use the organizer-level endpoints
# (added in pretix 2023.8) that span all events and stamp each row with its parent `event` slug,
# avoiding a per-event fan-out for the two largest tables. Order positions arrive embedded in each
-# order's `positions` array, so they don't need a dedicated stream.
+# order's `positions` array, so they don't need a dedicated stream. The same holds for payments and
+# refunds (each order's `payments`/`refunds` arrays) and item variations (each item's `variations`
+# array): their dedicated endpoints are per-order or per-item, so a table would cost a request per
+# parent row for data the parent stream already carries.
PRETIX_ENDPOINTS: dict[str, PretixEndpointConfig] = {
"events": PretixEndpointConfig(
name="events",
@@ -73,9 +80,9 @@ PRETIX_ENDPOINTS: dict[str, PretixEndpointConfig] = {
scope=EndpointScope.ORGANIZER,
# Order codes are only documented unique within their event; rows carry the event slug.
primary_keys=["event", "code"],
- incremental_fields=_last_modified_incremental_field(),
+ incremental_fields=_datetime_incremental_field("last_modified"),
partition_key="datetime",
- modified_since_field="last_modified",
+ since_field="last_modified",
ordering="last_modified",
),
"invoices": PretixEndpointConfig(
@@ -85,6 +92,21 @@ PRETIX_ENDPOINTS: dict[str, PretixEndpointConfig] = {
# Invoice numbers are prefixed per event; rows carry the event slug.
primary_keys=["event", "number"],
),
+ # Organizer-level financial ledger (pretix 2025.7+). Rows are append-only: a cancellation adds a
+ # row with a negative count instead of changing an existing one, so `created` is a safe cursor.
+ "transactions": PretixEndpointConfig(
+ name="transactions",
+ path="/organizers/{organizer}/transactions/",
+ scope=EndpointScope.ORGANIZER,
+ incremental_fields=_datetime_incremental_field("created"),
+ partition_key="created",
+ since_field="created",
+ since_param="created_since",
+ # Duplicate ledger rows would inflate revenue sums, so only merge (deduped on `id`) is offered.
+ supports_append=False,
+ # `id` is unique, so page boundaries stay stable when many rows share a `created` value.
+ ordering="id",
+ ),
"customers": PretixEndpointConfig(
name="customers",
path="/organizers/{organizer}/customers/",
@@ -162,4 +184,8 @@ INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
name: config.incremental_fields for name, config in PRETIX_ENDPOINTS.items()
}
-INCREMENTAL_ENDPOINTS = tuple(name for name, config in PRETIX_ENDPOINTS.items() if config.modified_since_field)
+INCREMENTAL_ENDPOINTS = tuple(name for name, config in PRETIX_ENDPOINTS.items() if config.since_field)
+
+APPEND_ENDPOINTS = tuple(
+ name for name, config in PRETIX_ENDPOINTS.items() if config.since_field and config.supports_append
+)
diff --git a/products/warehouse_sources/sources/pretix/source.py b/products/warehouse_sources/sources/pretix/source.py
index 5b593a70ab69..da704182b874 100644
--- a/products/warehouse_sources/sources/pretix/source.py
+++ b/products/warehouse_sources/sources/pretix/source.py
@@ -9,6 +9,7 @@ from products.warehouse_sources.sources.pretix.pretix import (
validate_credentials as validate_pretix_credentials,
)
from products.warehouse_sources.sources.pretix.settings import (
+ APPEND_ENDPOINTS,
ENDPOINTS,
INCREMENTAL_ENDPOINTS,
INCREMENTAL_FIELDS,
@@ -125,7 +126,7 @@ Self-hosted users should set the API URL to their own pretix host (for example `
SourceSchema(
name=endpoint,
supports_incremental=endpoint in INCREMENTAL_ENDPOINTS,
- supports_append=endpoint in INCREMENTAL_ENDPOINTS,
+ supports_append=endpoint in APPEND_ENDPOINTS,
incremental_fields=INCREMENTAL_FIELDS.get(endpoint, []),
)
for endpoint in ENDPOINTS
diff --git a/products/warehouse_sources/sources/pretix/tests/test_pretix.py b/products/warehouse_sources/sources/pretix/tests/test_pretix.py
index 63e0a01a98a6..355fca2bc4e9 100644
--- a/products/warehouse_sources/sources/pretix/tests/test_pretix.py
+++ b/products/warehouse_sources/sources/pretix/tests/test_pretix.py
@@ -13,7 +13,7 @@ from products.warehouse_sources.sources.pretix.pretix import (
INVALID_ORGANIZER_ERROR,
PretixHostNotAllowedError,
PretixResumeConfig,
- _format_modified_since,
+ _format_since,
_quote_organizer,
normalize_base_url,
validate_credentials,
@@ -49,7 +49,7 @@ class TestNormalizeBaseUrl:
assert normalize_base_url(raw) == expected
-class TestFormatModifiedSince:
+class TestFormatSince:
@parameterized.expand(
[
(datetime(2026, 3, 4, 2, 58, 14, tzinfo=UTC), "2026-03-04T02:58:14Z"),
@@ -58,8 +58,8 @@ class TestFormatModifiedSince:
("2026-03-04T02:58:14Z", "2026-03-04T02:58:14Z"),
]
)
- def test_format_modified_since(self, value: object, expected: str) -> None:
- assert _format_modified_since(value) == expected
+ def test_format_since(self, value: object, expected: str) -> None:
+ assert _format_since(value) == expected
class TestQuoteOrganizer:
@@ -81,16 +81,24 @@ class TestQuoteOrganizer:
class TestOrganizerScopePagination:
- def test_orders_incremental_url_has_filter_and_stable_ordering(self) -> None:
+ @parameterized.expand(
+ [
+ ("orders", "last_modified", "modified_since", "last_modified"),
+ ("transactions", "created", "created_since", "id"),
+ ]
+ )
+ def test_incremental_url_has_filter_and_stable_ordering(
+ self, endpoint: str, incremental_field: str, since_param: str, ordering: str
+ ) -> None:
result = _driver().run(
- "orders",
- [_envelope([{"code": "A1", "event": "conf"}])],
- incremental_field="last_modified",
+ endpoint,
+ [_envelope([{"id": 1, "code": "A1", "event": "conf"}])],
+ incremental_field=incremental_field,
db_incremental_field_last_value=datetime(2026, 1, 1, tzinfo=UTC),
)
- assert result.requests[0].param("modified_since") == "2026-01-01T00:00:00Z"
- assert result.requests[0].param("ordering") == "last_modified"
+ assert result.requests[0].param(since_param) == "2026-01-01T00:00:00Z"
+ assert result.requests[0].param("ordering") == ordering
@parameterized.expand(
[
diff --git a/products/warehouse_sources/sources/pretix/tests/test_pretix_source.py b/products/warehouse_sources/sources/pretix/tests/test_pretix_source.py
index 35f4ae3c6893..106bb21a38d2 100644
--- a/products/warehouse_sources/sources/pretix/tests/test_pretix_source.py
+++ b/products/warehouse_sources/sources/pretix/tests/test_pretix_source.py
@@ -16,10 +16,19 @@ class TestPretixSource:
# the preserved token at a host they control.
assert self.source.connection_host_fields == ["base_url"]
- def test_orders_is_the_only_incremental_endpoint(self):
- # Only `orders` has a documented server-side `modified_since` filter; advertising incremental
- # on another endpoint would silently full-refresh under an incremental label.
- assert set(INCREMENTAL_ENDPOINTS) == {"orders"}
+ def test_only_endpoints_with_a_server_side_filter_are_incremental(self):
+ # Only `orders` (`modified_since`) and `transactions` (`created_since`) have a documented
+ # server-side timestamp filter; advertising incremental on another endpoint would silently
+ # full-refresh under an incremental label.
+ assert set(INCREMENTAL_ENDPOINTS) == {"orders", "transactions"}
+
+ def test_transactions_does_not_offer_append_sync(self):
+ # `created_since` is inclusive, so append sync would duplicate ledger rows at the watermark.
+ schemas = {s.name: s for s in self.source.get_schemas(self.config, self.team_id)}
+
+ assert schemas["transactions"].supports_incremental is True
+ assert schemas["transactions"].supports_append is False
+ assert schemas["orders"].supports_append is True
@mock.patch("products.warehouse_sources.sources.pretix.source.pretix_source")
def test_source_for_pipeline_plumbs_arguments(self, mock_pretix_source):