Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the Tremendous source (#124233)

Tom Owerscff2450c802b
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index ee44caedfa94..2e613da51391 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -1111,7 +1111,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/travis_ci/travis_ci.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/trello/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/trello/trello.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/tremendous.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/trigger_dev/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/trigger_dev/trigger_dev.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index b2773427cc13..6de86a6f3447 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -12517,10 +12517,10 @@ Today (8): `balance_transactions`, `campaigns`, `funding_sources`, `invoices`, `
Diffed against: <https://raw.githubusercontent.com/tremendous-rewards/tremendous-python/HEAD/tremendous/api/tremendous_api.py>
- [x] `balance_transactions` — the account ledger; every debit/credit behind orders and funding sources (high)
-- [ ] `roles` — lookup table resolving the role ids on members (high)
-- [ ] `topups` — funding top-up transactions that explain balance movements (medium)
-- [ ] `fields` — custom field definitions that decode the custom_fields carried on orders and rewards (medium)
-- [ ] `organizations` — org roster for multi-org accounts; resolves the organization id on orders, members and invoices (medium)
+- [x] `roles` — lookup table resolving the role ids on members (high)
+- [x] `topups` — funding top-up transactions that explain balance movements (medium)
+- [x] `fields` — custom field definitions that decode the custom_fields carried on orders and rewards (medium)
+- [x] `organizations` — org roster for multi-org accounts; resolves the organization id on orders, members and invoices (medium)
- [ ] `connected_organizations` — platform-partner child orgs, needed to attribute spend for resellers (medium)
- [ ] `connected_organization_members` — member roster per connected org (low)
- [ ] `fraud_reviews` — orders held for fraud review and their disposition (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/canonical_descriptions.py
index c76c17203468..49df14ff2996 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/canonical_descriptions.py
@@ -121,4 +121,57 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"order": "Details of the order associated with the transaction, if any (ID, external ID, and payment breakdown).",
},
},
+ "topups": {
+ "description": "Requests to add funds to the organization's Tremendous balance from a funding source.",
+ "docs_url": "https://developers.tremendous.com/reference/list-topups",
+ "columns": {
+ "id": "Unique identifier for the topup request.",
+ "amount": "Amount to add to the organization's balance, denominated in currency_code.",
+ "currency_code": "Currency of the topup amount. Always matches the organization's currency.",
+ "processing_fee": "Processing fee for the topup (typically for credit card topups).",
+ "funding_source_id": "ID of the funding source used for this topup.",
+ "status": "Status of the topup (created, partially_credited, fully_credited, reversed, rejected).",
+ "created_at": "Timestamp when the topup was requested.",
+ "fully_credited_at": "Timestamp when the topup amount was fully credited to the balance.",
+ "rejected_at": "Timestamp when the topup was rejected.",
+ "reversed_at": "Timestamp when the topup was reversed.",
+ "reversed_reason": "Explanation of why the topup was reversed.",
+ "idempotency_key": "Idempotency key supplied when the topup was created.",
+ "instant_credit_amount": "Amount credited to the balance immediately. Can be 0 if nothing was credited instantly.",
+ "settled_amount": "Amount that becomes available once the settlement period elapses. 0 if nothing is settling.",
+ "expected_settlement_at": "Date when the pending amount will be credited. Null if the topup was fully credited immediately.",
+ },
+ },
+ "roles": {
+ "description": "Roles available in the organization. Each member is assigned a role that defines their permissions.",
+ "docs_url": "https://developers.tremendous.com/reference/list-roles",
+ "columns": {
+ "id": "Tremendous ID of the role.",
+ "title": "Name of the role (e.g. Admin).",
+ "description": "Description of the permissions the role grants.",
+ },
+ },
+ "fields": {
+ "description": "Custom field definitions that can be attached to rewards for reporting and analytics.",
+ "docs_url": "https://developers.tremendous.com/reference/list-fields",
+ "columns": {
+ "id": "Tremendous ID of the custom field.",
+ "label": "Label of the field.",
+ "data_type": "Type of the field's values (e.g. Text, Number, Date, Dropdown, List, Checkbox).",
+ "data": "Extra configuration for Dropdown and List fields, such as the valid options.",
+ "required": "Whether the field is required (true) or optional (false).",
+ "scope": "Type of object the field is associated with (e.g. REWARD).",
+ },
+ },
+ "organizations": {
+ "description": "The organization that the API key belongs to. The list contains only that organization.",
+ "docs_url": "https://developers.tremendous.com/reference/list-organizations",
+ "columns": {
+ "id": "Tremendous ID of the organization.",
+ "name": "Name of the organization.",
+ "website": "URL of the organization's website.",
+ "currency_code": "Currency used for the organization's balances, orders, and transactions.",
+ "status": "Approval status of the organization (PENDING, APPROVED, REJECTED).",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/settings.py
index 502313ad1b7f..6e28276a372c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/settings.py
@@ -4,7 +4,7 @@ from typing import Optional
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
-@dataclass
+@dataclass(frozen=True)
class TremendousEndpointConfig:
path: str
# Top-level key the list of records is nested under in the response body (e.g. {"orders": [...]}).
@@ -12,6 +12,8 @@ class TremendousEndpointConfig:
# Endpoints without offset/limit params return the whole collection in one response.
paginated: bool = False
page_size: int = 500
+ # Paginated by `offset` alone: the endpoint takes no `limit` and picks its own page size.
+ offset_only: bool = False
# Stable creation timestamp used for datetime partitioning. Never an updated_at-style field.
partition_key: Optional[str] = None
incremental_fields: list[IncrementalField] = field(default_factory=list)
@@ -84,6 +86,18 @@ TREMENDOUS_ENDPOINTS: dict[str, TremendousEndpointConfig] = {
# the API docs but unconfirmed against live accounts, so the table starts opt-in.
should_sync_default=False,
),
+ # Funding requests that add money to the balance. Full refresh: there is no timestamp filter.
+ "topups": TremendousEndpointConfig(
+ path="/topups",
+ data_key="topups",
+ paginated=True,
+ offset_only=True,
+ partition_key="created_at",
+ ),
+ "roles": TremendousEndpointConfig(path="/roles", data_key="roles"),
+ "fields": TremendousEndpointConfig(path="/fields", data_key="fields"),
+ # Lists only the organization the API key belongs to.
+ "organizations": TremendousEndpointConfig(path="/organizations", data_key="organizations"),
}
ENDPOINTS = tuple(TREMENDOUS_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/tests/test_tremendous.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/tests/test_tremendous.py
index aa14b19ba482..df8904dce697 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/tests/test_tremendous.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/tests/test_tremendous.py
@@ -46,12 +46,44 @@ def _run(
class TestPagination:
- def test_resumes_from_saved_offset(self) -> None:
- result = _run("orders", [_response([{"id": "X"}])], resume_state=TremendousResumeConfig(offset=1000))
+ @parameterized.expand([("orders",), ("topups",)])
+ def test_resumes_from_saved_offset(self, endpoint: str) -> None:
+ data_key = TREMENDOUS_ENDPOINTS[endpoint].data_key
+ result = _run(
+ endpoint, [_response([{"id": "X"}], data_key=data_key)], resume_state=TremendousResumeConfig(offset=1000)
+ )
# The initial (offset=0) page must never be re-fetched on resume.
assert result.params("offset") == ["1000"]
+ @parameterized.expand(
+ [
+ # The server picks the page size, so a page shorter than any limit must not end the walk.
+ ("stops_at_total_count", [3, 2], 5, ["0", "3"]),
+ ("total_count_missing_stops_on_empty_page", [3, 2, 0], None, ["0", "3", "5"]),
+ ("empty_first_page", [0], 0, ["0"]),
+ ]
+ )
+ def test_topups_advance_offset_by_rows_received(
+ self, _name: str, page_sizes: list[int], total_count: int | None, expected_offsets: list[str]
+ ) -> None:
+ pages: list[ScriptedResponse] = []
+ next_id = 0
+ for size in page_sizes:
+ rows = [{"id": f"TOPUP{next_id + i}", "created_at": "2026-01-02T03:04:05Z"} for i in range(size)]
+ next_id += size
+ body: dict[str, Any] = {"topups": rows}
+ if total_count is not None:
+ body["total_count"] = total_count
+ pages.append(ScriptedResponse(json=body))
+
+ result = _run("topups", pages)
+
+ assert result.raised is None
+ assert result.params("offset") == expected_offsets
+ assert result.params("limit") == [None] * len(expected_offsets)
+ assert [row["id"] for row in result.rows] == [f"TOPUP{i}" for i in range(sum(page_sizes))]
+
@parameterized.expand([("orders",), ("balance_transactions",)])
def test_incremental_watermark_sent_as_created_at_gte_on_every_page(self, endpoint: str) -> None:
config = TREMENDOUS_ENDPOINTS[endpoint]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/tremendous.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/tremendous.py
index 10a949cdba5e..07f2a3ba1673 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/tremendous.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tremendous/tremendous.py
@@ -4,6 +4,8 @@ from collections.abc import Callable
from datetime import UTC, date, datetime
from typing import Any, Optional
+from requests import Request, Response
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
Endpoint,
@@ -41,6 +43,34 @@ class TremendousResumeConfig:
offset: int = 0
+class TremendousOffsetOnlyPaginator(OffsetPaginator):
+ """Pagination for endpoints that accept `offset` but no `limit` (e.g. /topups).
+
+ The server picks the page size and does not document it, so the offset advances by the rows
+ actually received instead of a fixed limit. A short page therefore never ends the walk early;
+ `total_count` or an empty page does.
+ """
+
+ def __init__(self, offset: int = 0) -> None:
+ super().__init__(limit=0, offset=offset, total_path=None)
+
+ def init_request(self, request: Request) -> None:
+ # The base update_request sends only the offset, never the limit.
+ self.update_request(request)
+
+ def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:
+ if not data:
+ self._has_next_page = False
+ return
+
+ self.offset += len(data)
+ try:
+ total = response.json().get("total_count")
+ except (ValueError, AttributeError):
+ total = None
+ self._has_next_page = not (isinstance(total, int) and self.offset >= total)
+
+
def base_url_for_environment(environment: str) -> str:
return TREMENDOUS_BASE_URLS.get(environment, TREMENDOUS_BASE_URLS["production"])
@@ -119,11 +149,14 @@ def tremendous_source(
"data_selector_malformed_retryable": True,
}
- if config.paginated:
+ if config.offset_only:
+ endpoint_config["paginator"] = TremendousOffsetOnlyPaginator()
+ elif config.paginated:
# Termination is a short/empty page (OffsetPaginator default); resume by row offset.
endpoint_config["paginator"] = OffsetPaginator(limit=config.page_size, total_path=None)
else:
- # Members, campaigns, products, and funding sources return the whole collection at once.
+ # Members, campaigns, products, funding sources, roles, fields, and organizations return the
+ # whole collection at once.
endpoint_config["paginator"] = SinglePagePaginator()
incremental_value: Any = None