:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): add orb costs, credit blocks and schedule tables (#124781)
Tom Owersc8ff8013e759
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 70475f4bdba1..8cc8a6bede2b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -8949,12 +8949,12 @@ Diffed against: <https://raw.githubusercontent.com/orbcorp/orb-node/main/api.md>- [x] `GET /metrics (billable metrics)` — lookup resolving the billable-metric IDs that prices reference; without it usage-based charges are unexplainable (high)- [x] `GET /customers/{customer_id}/credits/ledger` — credit grant/decrement transaction log — the ledger a warehouse user needs for prepaid-balance analysis (high)- [x] `GET /customers/{customer_id}/balance_transactions` — customer account balance movements over time, joined to synced customers and invoices (high)-- [ ] `GET /customers/{customer_id}/costs` — per-customer cost breakdown by price/metric over a timeframe — Orb's headline revenue metric (high)+- [x] `GET /customers/{customer_id}/costs` — per-customer cost breakdown by price/metric over a timeframe — Orb's headline revenue metric (high)- [ ] `GET /subscriptions/{subscription_id}/usage` — usage quantities per billable metric per subscription, the raw driver of every invoice line (medium)- [ ] `GET /subscription_changes` — state-transition history for the subscriptions table already synced (upgrades, downgrades, cancellations) (medium)-- [ ] `GET /subscriptions/{subscription_id}/schedule` — plan-over-time schedule per subscription — resolves which plan applied in which period (medium)-- [ ] `GET /customers/{customer_id}/credits` — outstanding credit block balances per customer (medium)-- [ ] `GET /subscriptions/{subscription_id}/costs` — cost breakdown at subscription grain, complementing the customer-level view (medium)+- [x] `GET /subscriptions/{subscription_id}/schedule` — plan-over-time schedule per subscription — resolves which plan applied in which period (medium)+- [x] `GET /customers/{customer_id}/credits` — outstanding credit block balances per customer (medium)+- [x] `GET /subscriptions/{subscription_id}/costs` — cost breakdown at subscription grain, complementing the customer-level view (medium)- [ ] `GET /events/volume` — ingested event volume over time — useful for reconciling usage against billing (low)- [ ] `GET /dimensional_price_groups` — lookup for dimensional pricing groups referenced by prices (low)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/orb/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/orb/canonical_descriptions.pyindex d61047582a1d..706d46417456 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/orb/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/orb/canonical_descriptions.py@@ -185,4 +185,65 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"created_at": "The creation time of the transaction.",},},+ "CreditBlocks": {+ "description": "The prepaid credit blocks of each customer in Orb's default credits ledger, including "+ "expired and depleted blocks, with their remaining balance. Blocks in a real-world currency ledger are "+ "not included. Synced per customer as a full refresh.",+ "docs_url": "https://docs.withorb.com/api-reference/credit/fetch-customer-credit-balance",+ "columns": {+ "id": "The Orb-assigned unique identifier for the credit block.",+ "customer_id": "The Orb ID of the customer that owns the credit block.",+ "balance": "The number of credits left in the block.",+ "maximum_initial_balance": "The number of credits that the block started with.",+ "credit_block_source": "How the block was created: allocation, top_up, commitment or manual.",+ "status": "Whether the block is active or waits for payment (active, pending_payment).",+ "effective_date": "The time at which the credits in the block become usable.",+ "expiry_date": "The time at which the remaining credits in the block expire.",+ "per_unit_cost_basis": "The cost basis of one credit in the block.",+ "filters": "The prices that can draw down credits from the block.",+ "metadata": "User-defined key/value pairs attached to the credit block.",+ },+ },+ "CustomerCosts": {+ "description": "A day-by-day series of each customer's costs over the current billing period (or the "+ "last active one), broken down by price. Totals are cumulative from the start of the billing period. "+ "Synced per customer as a full refresh.",+ "docs_url": "https://docs.withorb.com/api-reference/customer/fetch-customer-costs",+ "columns": {+ "customer_id": "The Orb ID of the customer that the costs belong to.",+ "timeframe_start": "The start of the window, inclusive. This is the start of the billing period.",+ "timeframe_end": "The end of the window, exclusive. Each data point moves it one day forward.",+ "subtotal": "Total costs for the window, excluding minimums and discounts.",+ "total": "Total costs for the window, including minimums and discounts.",+ "per_price_costs": "The costs for the window broken down by price, with each price's quantity, "+ "subtotal and total.",+ },+ },+ "SubscriptionCosts": {+ "description": "A day-by-day series of each subscription's costs over its current billing period (or "+ "the last active one), broken down by price. Totals are cumulative from the start of the billing "+ "period. Synced per subscription as a full refresh.",+ "docs_url": "https://docs.withorb.com/api-reference/subscription/fetch-subscription-costs",+ "columns": {+ "subscription_id": "The Orb ID of the subscription that the costs belong to.",+ "timeframe_start": "The start of the window, inclusive. This is the start of the billing period.",+ "timeframe_end": "The end of the window, exclusive. Each data point moves it one day forward.",+ "subtotal": "Total costs for the window, excluding minimums and discounts.",+ "total": "Total costs for the window, including minimums and discounts.",+ "per_price_costs": "The costs for the window broken down by price, with each price's quantity, "+ "subtotal and total.",+ },+ },+ "SubscriptionSchedules": {+ "description": "The plans of each subscription over time: the initial plan and every past and future "+ "plan change, with start and end dates. Synced per subscription as a full refresh.",+ "docs_url": "https://docs.withorb.com/api-reference/subscription/fetch-subscription-schedule",+ "columns": {+ "subscription_id": "The Orb ID of the subscription.",+ "plan": "The plan that applies in this period (id, external_plan_id and name).",+ "start_date": "The time at which the plan starts to apply to the subscription.",+ "end_date": "The time at which the plan stops to apply. Empty when the plan has no end.",+ "created_at": "The time at which the schedule entry was created.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/orb/orb.py b/products/warehouse_sources/backend/temporal/data_imports/sources/orb/orb.pyindex ab40e609ebe8..0fa7d5221428 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/orb/orb.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/orb/orb.py@@ -158,7 +158,10 @@ def orb_source(job_id=job_id,db_incremental_field_last_value=None,parent_endpoint_extra={"data_selector": "data"},- child_endpoint_extra={"data_selector": "data"},+ child_endpoint_extra={"data_selector": "data"}+ if cfg.paginated+ else {"data_selector": "data", "paginator": "single_page"},+ page_size_param="limit" if cfg.paginated else None,resume_hook=save_fanout_checkpoint,initial_paginator_state=resume.fanout_state if resume else None,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/orb/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/orb/settings.pyindex c28014897941..730d34ace643 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/orb/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/orb/settings.py@@ -37,7 +37,9 @@ class OrbEndpointConfig:partition_key: Optional[str] = Noneprimary_keys: list[str] = field(default_factory=lambda: ["id"])page_size: int = DEFAULT_PAGE_SIZE- # Set for per-customer child endpoints, which fan out over the Customers listing.+ # False for endpoints that return their whole result in one response and take no `limit`.+ paginated: bool = True+ # Set for per-customer and per-subscription child endpoints, which fan out over the parent listing.fanout: Optional[DependentEndpointConfig] = None@property@@ -49,23 +51,36 @@ class OrbEndpointConfig:return self.incremental_field-def _customer_fanout() -> DependentEndpointConfig:+def _fanout(+ parent_name: str, resolve_param: str, child_params: Optional[dict[str, str]] = None+) -> DependentEndpointConfig:return DependentEndpointConfig(- parent_name="Customers",- resolve_param="customer_id",+ parent_name=parent_name,+ resolve_param=resolve_param,resolve_field="id",include_from_parent=["id"],- parent_field_renames={"id": "customer_id"},- # A customer deleted between the parent listing and its child fetch 404s; skip it rather+ parent_field_renames={"id": resolve_param},+ # Set here as well as through page_size_param, because unpaginated children send no page size.+ parent_params={"limit": DEFAULT_PAGE_SIZE},+ child_params=child_params or {},+ # A parent deleted between the parent listing and its child fetch 404s; skip it rather# than failing the whole fan-out.child_response_actions=[{"status_code": 404, "action": "ignore"}],)+def _customer_fanout(child_params: Optional[dict[str, str]] = None) -> DependentEndpointConfig:+ return _fanout("Customers", "customer_id", child_params)+++def _subscription_fanout() -> DependentEndpointConfig:+ return _fanout("Subscriptions", "subscription_id")++# Orb list endpoints. All return `{"data": [...], "pagination_metadata": {...}}`, are# cursor-paginated, and sort newest-first by creation time. Incremental endpoints expose a# server-side `<field>[gt]` timestamp filter; the rest are full-refresh only. Endpoints with a-# `fanout` are per-customer and run once for every row of the Customers listing.+# `fanout` are per-customer or per-subscription and run once for every row of the parent listing.## Invoices is the odd one out: it has no `created_at` filter, only `invoice_date[gt]`, so its# incremental cursor is `invoice_date` while it still partitions on the stable `created_at`.@@ -154,6 +169,47 @@ ORB_ENDPOINTS: dict[str, OrbEndpointConfig] = {primary_keys=["customer_id", "id"],fanout=_customer_fanout(),),+ # Credit balances change as usage draws them down, so this is full refresh. Without+ # include_all_blocks Orb returns only unexpired, non-zero blocks. Like CreditLedgerEntries, this+ # reads Orb's default `credits` ledger: Orb has no endpoint that lists a customer's other ledger+ # currencies to fan out over.+ "CreditBlocks": OrbEndpointConfig(+ name="CreditBlocks",+ path="/customers/{customer_id}/credits",+ table_name="credit_blocks",+ primary_keys=["customer_id", "id"],+ fanout=_customer_fanout({"include_all_blocks": "true"}),+ ),+ # Costs are a cumulative day-by-day series over the current billing period (or the last active+ # one), returned in one unpaginated response. Each timeframe_end is one daily data point.+ "CustomerCosts": OrbEndpointConfig(+ name="CustomerCosts",+ path="/customers/{customer_id}/costs",+ table_name="customer_costs",+ partition_key="timeframe_start",+ primary_keys=["customer_id", "timeframe_start", "timeframe_end"],+ paginated=False,+ fanout=_customer_fanout(),+ ),+ "SubscriptionCosts": OrbEndpointConfig(+ name="SubscriptionCosts",+ path="/subscriptions/{subscription_id}/costs",+ table_name="subscription_costs",+ partition_key="timeframe_start",+ primary_keys=["subscription_id", "timeframe_start", "timeframe_end"],+ paginated=False,+ fanout=_subscription_fanout(),+ ),+ # Schedule entries carry no id. Full refresh, because a scheduled plan change sets the+ # end_date of the entry before it.+ "SubscriptionSchedules": OrbEndpointConfig(+ name="SubscriptionSchedules",+ path="/subscriptions/{subscription_id}/schedule",+ table_name="subscription_schedules",+ partition_key="created_at",+ primary_keys=["subscription_id", "start_date"],+ fanout=_subscription_fanout(),+ ),}ENDPOINTS = tuple(ORB_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/orb/tests/test_orb.py b/products/warehouse_sources/backend/temporal/data_imports/sources/orb/tests/test_orb.pyindex 510ea75d0efd..5ae6c8a1d9e1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/orb/tests/test_orb.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/orb/tests/test_orb.py@@ -167,18 +167,27 @@ class TestOrbSource:@parameterized.expand([- ("CreditLedgerEntries", "credits/ledger"),- ("BalanceTransactions", "balance_transactions"),+ ("CreditLedgerEntries", "customers", "credits/ledger", "customer_id", None),+ ("BalanceTransactions", "customers", "balance_transactions", "customer_id", None),+ ("CreditBlocks", "customers", "credits", "customer_id", "true"),+ ("SubscriptionSchedules", "subscriptions", "schedule", "subscription_id", None),])- def test_fanout_walks_every_customer_and_tags_rows(self, endpoint: str, child_suffix: str) -> None:+ def test_fanout_walks_every_parent_and_tags_rows(+ self,+ endpoint: str,+ parent: str,+ child_suffix: str,+ parent_key: str,+ include_all_blocks: str | None,+ ) -> None:result = self._driver().run(endpoint,[- self._page([{"id": "cus_1"}, {"id": "cus_gone"}, {"id": "cus_2"}]),+ self._page([{"id": "p_1"}, {"id": "p_gone"}, {"id": "p_2"}]),self._page([{"id": "entry-1"}], next_cursor="child-next"),self._page([{"id": "entry-2"}]),- # A customer deleted after the parent listing must not fail the fan-out.+ # A parent deleted after the parent listing must not fail the fan-out.ScriptedResponse(status=404, json={"title": "Not found"}),self._page([{"id": "entry-3"}]),],@@ -186,17 +195,47 @@ class TestOrbSource:assert result.raised is Noneassert result.paths == [- "/v1/customers",- f"/v1/customers/cus_1/{child_suffix}",- f"/v1/customers/cus_1/{child_suffix}",- f"/v1/customers/cus_gone/{child_suffix}",- f"/v1/customers/cus_2/{child_suffix}",+ f"/v1/{parent}",+ f"/v1/{parent}/p_1/{child_suffix}",+ f"/v1/{parent}/p_1/{child_suffix}",+ f"/v1/{parent}/p_gone/{child_suffix}",+ f"/v1/{parent}/p_2/{child_suffix}",]assert result.params("cursor") == [None, None, "child-next", None, None]+ assert result.params("limit") == ["100"] * 5+ assert result.params("include_all_blocks") == [None] + [include_all_blocks] * 4assert result.rows == [- {"id": "entry-1", "customer_id": "cus_1"},- {"id": "entry-2", "customer_id": "cus_1"},- {"id": "entry-3", "customer_id": "cus_2"},+ {"id": "entry-1", parent_key: "p_1"},+ {"id": "entry-2", parent_key: "p_1"},+ {"id": "entry-3", parent_key: "p_2"},+ ]++ @parameterized.expand(+ [+ ("CustomerCosts", "customers", "customer_id"),+ ("SubscriptionCosts", "subscriptions", "subscription_id"),+ ]+ )+ def test_costs_fanout_fetches_one_unpaginated_response_per_parent(+ self, endpoint: str, parent: str, parent_key: str+ ) -> None:+ cost = {"timeframe_start": "2026-10-01T00:00:00Z", "subtotal": "1.00", "total": "1.00", "per_price_costs": []}+ result = self._driver().run(+ endpoint,+ [+ self._page([{"id": "p_1"}, {"id": "p_2"}]),+ ScriptedResponse(json={"data": [{**cost, "timeframe_end": "2026-10-02T00:00:00Z"}]}),+ ScriptedResponse(json={"data": [{**cost, "timeframe_end": "2026-10-03T00:00:00Z"}]}),+ ],+ )++ assert result.raised is None+ assert result.paths == [f"/v1/{parent}", f"/v1/{parent}/p_1/costs", f"/v1/{parent}/p_2/costs"]+ # The parent listing still pages at the maximum size; the costs endpoint takes no page size.+ assert result.params("limit") == ["100", None, None]+ assert [(row[parent_key], row["timeframe_end"]) for row in result.rows] == [+ ("p_1", "2026-10-02T00:00:00Z"),+ ("p_2", "2026-10-03T00:00:00Z"),]def test_fanout_resume_skips_completed_customers(self) -> None: