:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 4 missing tables to the Tremendous source (#124233)
Tom Owerscff2450c802b
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex ee44caedfa94..2e613da51391 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -1111,7 +1111,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/travis_ci/travis_ci.py1 products/warehouse_sources/backend/temporal/data_imports/sources/trello/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/trello/trello.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/tremendous.py1 products/warehouse_sources/backend/temporal/data_imports/sources/trigger_dev/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/trigger_dev/trigger_dev.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex b2773427cc13..6de86a6f3447 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -12517,10 +12517,10 @@ Today (8): `balance_transactions`, `campaigns`, `funding_sources`, `invoices`, `Diffed against: <https://raw.githubusercontent.com/tremendous-rewards/tremendous-python/HEAD/tremendous/api/tremendous_api.py>- [x] `balance_transactions` — the account ledger; every debit/credit behind orders and funding sources (high)-- [ ] `roles` — lookup table resolving the role ids on members (high)-- [ ] `topups` — funding top-up transactions that explain balance movements (medium)-- [ ] `fields` — custom field definitions that decode the custom_fields carried on orders and rewards (medium)-- [ ] `organizations` — org roster for multi-org accounts; resolves the organization id on orders, members and invoices (medium)+- [x] `roles` — lookup table resolving the role ids on members (high)+- [x] `topups` — funding top-up transactions that explain balance movements (medium)+- [x] `fields` — custom field definitions that decode the custom_fields carried on orders and rewards (medium)+- [x] `organizations` — org roster for multi-org accounts; resolves the organization id on orders, members and invoices (medium)- [ ] `connected_organizations` — platform-partner child orgs, needed to attribute spend for resellers (medium)- [ ] `connected_organization_members` — member roster per connected org (low)- [ ] `fraud_reviews` — orders held for fraud review and their disposition (low)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/canonical_descriptions.pyindex c76c17203468..49df14ff2996 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/canonical_descriptions.py@@ -121,4 +121,57 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"order": "Details of the order associated with the transaction, if any (ID, external ID, and payment breakdown).",},},+ "topups": {+ "description": "Requests to add funds to the organization's Tremendous balance from a funding source.",+ "docs_url": "https://developers.tremendous.com/reference/list-topups",+ "columns": {+ "id": "Unique identifier for the topup request.",+ "amount": "Amount to add to the organization's balance, denominated in currency_code.",+ "currency_code": "Currency of the topup amount. Always matches the organization's currency.",+ "processing_fee": "Processing fee for the topup (typically for credit card topups).",+ "funding_source_id": "ID of the funding source used for this topup.",+ "status": "Status of the topup (created, partially_credited, fully_credited, reversed, rejected).",+ "created_at": "Timestamp when the topup was requested.",+ "fully_credited_at": "Timestamp when the topup amount was fully credited to the balance.",+ "rejected_at": "Timestamp when the topup was rejected.",+ "reversed_at": "Timestamp when the topup was reversed.",+ "reversed_reason": "Explanation of why the topup was reversed.",+ "idempotency_key": "Idempotency key supplied when the topup was created.",+ "instant_credit_amount": "Amount credited to the balance immediately. Can be 0 if nothing was credited instantly.",+ "settled_amount": "Amount that becomes available once the settlement period elapses. 0 if nothing is settling.",+ "expected_settlement_at": "Date when the pending amount will be credited. Null if the topup was fully credited immediately.",+ },+ },+ "roles": {+ "description": "Roles available in the organization. Each member is assigned a role that defines their permissions.",+ "docs_url": "https://developers.tremendous.com/reference/list-roles",+ "columns": {+ "id": "Tremendous ID of the role.",+ "title": "Name of the role (e.g. Admin).",+ "description": "Description of the permissions the role grants.",+ },+ },+ "fields": {+ "description": "Custom field definitions that can be attached to rewards for reporting and analytics.",+ "docs_url": "https://developers.tremendous.com/reference/list-fields",+ "columns": {+ "id": "Tremendous ID of the custom field.",+ "label": "Label of the field.",+ "data_type": "Type of the field's values (e.g. Text, Number, Date, Dropdown, List, Checkbox).",+ "data": "Extra configuration for Dropdown and List fields, such as the valid options.",+ "required": "Whether the field is required (true) or optional (false).",+ "scope": "Type of object the field is associated with (e.g. REWARD).",+ },+ },+ "organizations": {+ "description": "The organization that the API key belongs to. The list contains only that organization.",+ "docs_url": "https://developers.tremendous.com/reference/list-organizations",+ "columns": {+ "id": "Tremendous ID of the organization.",+ "name": "Name of the organization.",+ "website": "URL of the organization's website.",+ "currency_code": "Currency used for the organization's balances, orders, and transactions.",+ "status": "Approval status of the organization (PENDING, APPROVED, REJECTED).",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/settings.pyindex 502313ad1b7f..6e28276a372c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/settings.py@@ -4,7 +4,7 @@ from typing import Optionalfrom products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType-@dataclass+@dataclass(frozen=True)class TremendousEndpointConfig:path: str# Top-level key the list of records is nested under in the response body (e.g. {"orders": [...]}).@@ -12,6 +12,8 @@ class TremendousEndpointConfig:# Endpoints without offset/limit params return the whole collection in one response.paginated: bool = Falsepage_size: int = 500+ # Paginated by `offset` alone: the endpoint takes no `limit` and picks its own page size.+ offset_only: bool = False# Stable creation timestamp used for datetime partitioning. Never an updated_at-style field.partition_key: Optional[str] = Noneincremental_fields: list[IncrementalField] = field(default_factory=list)@@ -84,6 +86,18 @@ TREMENDOUS_ENDPOINTS: dict[str, TremendousEndpointConfig] = {# the API docs but unconfirmed against live accounts, so the table starts opt-in.should_sync_default=False,),+ # Funding requests that add money to the balance. Full refresh: there is no timestamp filter.+ "topups": TremendousEndpointConfig(+ path="/topups",+ data_key="topups",+ paginated=True,+ offset_only=True,+ partition_key="created_at",+ ),+ "roles": TremendousEndpointConfig(path="/roles", data_key="roles"),+ "fields": TremendousEndpointConfig(path="/fields", data_key="fields"),+ # Lists only the organization the API key belongs to.+ "organizations": TremendousEndpointConfig(path="/organizations", data_key="organizations"),}ENDPOINTS = tuple(TREMENDOUS_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/tests/test_tremendous.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/tests/test_tremendous.pyindex aa14b19ba482..df8904dce697 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/tests/test_tremendous.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/tests/test_tremendous.py@@ -46,12 +46,44 @@ def _run(class TestPagination:- def test_resumes_from_saved_offset(self) -> None:- result = _run("orders", [_response([{"id": "X"}])], resume_state=TremendousResumeConfig(offset=1000))+ @parameterized.expand([("orders",), ("topups",)])+ def test_resumes_from_saved_offset(self, endpoint: str) -> None:+ data_key = TREMENDOUS_ENDPOINTS[endpoint].data_key+ result = _run(+ endpoint, [_response([{"id": "X"}], data_key=data_key)], resume_state=TremendousResumeConfig(offset=1000)+ )# The initial (offset=0) page must never be re-fetched on resume.assert result.params("offset") == ["1000"]+ @parameterized.expand(+ [+ # The server picks the page size, so a page shorter than any limit must not end the walk.+ ("stops_at_total_count", [3, 2], 5, ["0", "3"]),+ ("total_count_missing_stops_on_empty_page", [3, 2, 0], None, ["0", "3", "5"]),+ ("empty_first_page", [0], 0, ["0"]),+ ]+ )+ def test_topups_advance_offset_by_rows_received(+ self, _name: str, page_sizes: list[int], total_count: int | None, expected_offsets: list[str]+ ) -> None:+ pages: list[ScriptedResponse] = []+ next_id = 0+ for size in page_sizes:+ rows = [{"id": f"TOPUP{next_id + i}", "created_at": "2026-01-02T03:04:05Z"} for i in range(size)]+ next_id += size+ body: dict[str, Any] = {"topups": rows}+ if total_count is not None:+ body["total_count"] = total_count+ pages.append(ScriptedResponse(json=body))++ result = _run("topups", pages)++ assert result.raised is None+ assert result.params("offset") == expected_offsets+ assert result.params("limit") == [None] * len(expected_offsets)+ assert [row["id"] for row in result.rows] == [f"TOPUP{i}" for i in range(sum(page_sizes))]+@parameterized.expand([("orders",), ("balance_transactions",)])def test_incremental_watermark_sent_as_created_at_gte_on_every_page(self, endpoint: str) -> None:config = TREMENDOUS_ENDPOINTS[endpoint]diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/tremendous.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/tremendous.pyindex 10a949cdba5e..07f2a3ba1673 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/tremendous.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/tremendous.py@@ -4,6 +4,8 @@ from collections.abc import Callablefrom datetime import UTC, date, datetimefrom typing import Any, Optional+from requests import Request, Response+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_sessionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (Endpoint,@@ -41,6 +43,34 @@ class TremendousResumeConfig:offset: int = 0+class TremendousOffsetOnlyPaginator(OffsetPaginator):+ """Pagination for endpoints that accept `offset` but no `limit` (e.g. /topups).++ The server picks the page size and does not document it, so the offset advances by the rows+ actually received instead of a fixed limit. A short page therefore never ends the walk early;+ `total_count` or an empty page does.+ """++ def __init__(self, offset: int = 0) -> None:+ super().__init__(limit=0, offset=offset, total_path=None)++ def init_request(self, request: Request) -> None:+ # The base update_request sends only the offset, never the limit.+ self.update_request(request)++ def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:+ if not data:+ self._has_next_page = False+ return++ self.offset += len(data)+ try:+ total = response.json().get("total_count")+ except (ValueError, AttributeError):+ total = None+ self._has_next_page = not (isinstance(total, int) and self.offset >= total)++def base_url_for_environment(environment: str) -> str:return TREMENDOUS_BASE_URLS.get(environment, TREMENDOUS_BASE_URLS["production"])@@ -119,11 +149,14 @@ def tremendous_source("data_selector_malformed_retryable": True,}- if config.paginated:+ if config.offset_only:+ endpoint_config["paginator"] = TremendousOffsetOnlyPaginator()+ elif config.paginated:# Termination is a short/empty page (OffsetPaginator default); resume by row offset.endpoint_config["paginator"] = OffsetPaginator(limit=config.page_size, total_path=None)else:- # Members, campaigns, products, and funding sources return the whole collection at once.+ # Members, campaigns, products, funding sources, roles, fields, and organizations return the+ # whole collection at once.endpoint_config["paginator"] = SinglePagePaginator()incremental_value: Any = None