Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add orb prices, metrics, credit ledger and balance tables (#123996)

Tom Owerscb9e68960a38
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 4b82075f66e9..c43de21b2f19 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -8944,10 +8944,10 @@ Today (7): `Coupons`, `CreditNotes`, `Customers`, `Invoices`, `Items`, `Plans`,
Diffed against: <https://raw.githubusercontent.com/orbcorp/orb-node/main/api.md>
-- [ ] `GET /prices` — lookup resolving the price IDs carried on every synced plan, subscription and invoice line item (high)
-- [ ] `GET /metrics (billable metrics)` — lookup resolving the billable-metric IDs that prices reference; without it usage-based charges are unexplainable (high)
-- [ ] `GET /customers/{customer_id}/credits/ledger` — credit grant/decrement transaction log — the ledger a warehouse user needs for prepaid-balance analysis (high)
-- [ ] `GET /customers/{customer_id}/balance_transactions` — customer account balance movements over time, joined to synced customers and invoices (high)
+- [x] `GET /prices` — lookup resolving the price IDs carried on every synced plan, subscription and invoice line item (high)
+- [x] `GET /metrics (billable metrics)` — lookup resolving the billable-metric IDs that prices reference; without it usage-based charges are unexplainable (high)
+- [x] `GET /customers/{customer_id}/credits/ledger` — credit grant/decrement transaction log — the ledger a warehouse user needs for prepaid-balance analysis (high)
+- [x] `GET /customers/{customer_id}/balance_transactions` — customer account balance movements over time, joined to synced customers and invoices (high)
- [ ] `GET /customers/{customer_id}/costs` — per-customer cost breakdown by price/metric over a timeframe — Orb's headline revenue metric (high)
- [ ] `GET /subscriptions/{subscription_id}/usage` — usage quantities per billable metric per subscription, the raw driver of every invoice line (medium)
- [ ] `GET /subscription_changes` — state-transition history for the subscriptions table already synced (upgrades, downgrades, cancellations) (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/orb/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/orb/canonical_descriptions.py
index 2e7b7d31f942..d61047582a1d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/orb/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/orb/canonical_descriptions.py
@@ -108,4 +108,81 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"archived_at": "The time the coupon was archived, if it has been.",
},
},
+ "Prices": {
+ "description": "Prices created through Orb's price creation endpoint (e.g. add-on prices). "
+ "Prices defined inside a plan are not listed here; they are in the `prices` column of the plans table. "
+ "Full refresh only, because the list endpoint exposes no server-side timestamp filter.",
+ "docs_url": "https://docs.withorb.com/api-reference/price/list-prices",
+ "columns": {
+ "id": "The Orb-assigned unique identifier for the price.",
+ "external_price_id": "Your own alias for the price.",
+ "name": "The name of the price.",
+ "model_type": "The pricing model (e.g. unit, tiered, bulk, package, matrix).",
+ "price_type": "Whether the price is a usage price, a fixed price or a composite price.",
+ "cadence": "How often the price is billed (one_time, monthly, quarterly, semi_annual, annual or custom).",
+ "billing_mode": "Whether the price is billed in advance or in arrears.",
+ "currency": "The currency of the price.",
+ "billable_metric": "The billable metric that usage prices are computed from, if any.",
+ "item": "The item this price sells.",
+ "fixed_price_quantity": "The quantity of a fixed price that is billed.",
+ "plan_phase_order": "The order of the plan phase that the price belongs to, if any.",
+ "replaces_price_id": "The price that this price replaces in plan version migrations.",
+ "metadata": "User-defined key/value pairs attached to the price.",
+ "created_at": "The time at which the price was created.",
+ },
+ },
+ "BillableMetrics": {
+ "description": "Billable metrics: the usage queries that usage-based prices are computed from. "
+ "Full refresh only, because metric objects carry no timestamp to track.",
+ "docs_url": "https://docs.withorb.com/api-reference/metric/list-metrics",
+ "columns": {
+ "id": "The Orb-assigned unique identifier for the billable metric.",
+ "name": "The name of the billable metric.",
+ "description": "A description of the billable metric.",
+ "status": "The status of the metric: active, draft or archived.",
+ "sql": "The SQL definition of the metric.",
+ "item": "The item that the metric is associated with.",
+ "metadata": "User-defined key/value pairs attached to the metric.",
+ },
+ },
+ "CreditLedgerEntries": {
+ "description": "The prepaid credit ledger for each customer: every increment, decrement, expiry, void and "
+ "amendment that changed a credit balance. Synced per customer as a full refresh.",
+ "docs_url": "https://docs.withorb.com/api-reference/credit/fetch-customer-credits-ledger",
+ "columns": {
+ "id": "The Orb-assigned unique identifier for the ledger entry.",
+ "customer_id": "The Orb ID of the customer that owns the ledger entry.",
+ "customer": "The customer that owns the ledger entry.",
+ "entry_type": "The kind of change: increment, decrement, expiration_change, credit_block_expiry, "
+ "void, void_initiated or amendment.",
+ "entry_status": "Whether the entry is committed or pending.",
+ "amount": "The number of credits that the entry changes.",
+ "starting_balance": "The customer's credit balance before the entry.",
+ "ending_balance": "The customer's credit balance after the entry.",
+ "currency": "The currency or custom pricing unit of the ledger.",
+ "credit_block": "The credit block that the entry affects.",
+ "ledger_sequence_number": "The position of the entry in the customer's ledger.",
+ "description": "An optional description of the entry.",
+ "metadata": "User-defined key/value pairs attached to the entry.",
+ "created_at": "The time at which the entry was added to the ledger.",
+ },
+ },
+ "BalanceTransactions": {
+ "description": "Changes to each customer's account balance, such as balance applied to invoices, "
+ "credit notes and manual adjustments. Synced per customer as a full refresh.",
+ "docs_url": "https://docs.withorb.com/api-reference/customer/list-balance-transactions",
+ "columns": {
+ "id": "A unique ID for the transaction.",
+ "customer_id": "The Orb ID of the customer whose balance changed.",
+ "action": "The reason for the change (e.g. applied_to_invoice, manual_adjustment, credit_note_applied).",
+ "type": "Whether the transaction increments or decrements the balance.",
+ "amount": "The amount by which the balance changed.",
+ "starting_balance": "The customer's balance before the transaction, in the customer's currency.",
+ "ending_balance": "The customer's balance after the transaction, in the customer's currency.",
+ "invoice": "The invoice that the transaction relates to, if any.",
+ "credit_note": "The credit note that the transaction relates to, if any.",
+ "description": "An optional description for manual balance adjustments.",
+ "created_at": "The creation time of the transaction.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/orb/orb.py b/products/warehouse_sources/backend/temporal/data_imports/sources/orb/orb.py
index 14dd817cc9c9..ab40e609ebe8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/orb/orb.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/orb/orb.py
@@ -1,4 +1,5 @@
import dataclasses
+from collections.abc import Iterable
from datetime import UTC, date, datetime
from typing import Any, Optional
@@ -9,10 +10,16 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
JSONResponseCursorPaginator,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import EndpointResource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
+ ClientConfig,
+ EndpointResource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.orb.settings import (
@@ -22,9 +29,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.orb.settin
)
[email protected](frozen=True)
class OrbResumeConfig:
- next_cursor: str
+ next_cursor: Optional[str] = None
+ # Per-customer resume state of a fan-out endpoint, as built by the rest_source framework.
+ fanout_state: Optional[dict[str, Any]] = None
def _format_incremental_value(value: Any) -> Optional[str]:
@@ -121,48 +130,69 @@ def orb_source(
) -> SourceResponse:
cfg = ORB_ENDPOINTS[endpoint]
- config: RESTAPIConfig = {
- "client": {
- "base_url": ORB_API_BASE_URL,
- "auth": {
- "type": "bearer",
- "token": api_key,
- },
- "paginator": OrbCursorPaginator(),
+ client_config: ClientConfig = {
+ "base_url": ORB_API_BASE_URL,
+ "auth": {
+ "type": "bearer",
+ "token": api_key,
},
- "resource_defaults": None,
- "resources": [get_resource(endpoint, should_use_incremental_field)],
+ "paginator": OrbCursorPaginator(),
}
- initial_paginator_state: Optional[dict[str, Any]] = None
- if resumable_source_manager.can_resume():
- resume_config = resumable_source_manager.load_state()
- if resume_config is not None:
- initial_paginator_state = {"next_cursor": resume_config.next_cursor}
-
- def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
- # Persist only when a next page exists; the Redis TTL cleans up on completion. Saving
- # after each yielded batch means a crash re-yields the last page (merge dedupes on the
- # primary key) rather than skipping it.
- if state and state.get("next_cursor"):
- resumable_source_manager.save_state(OrbResumeConfig(next_cursor=str(state["next_cursor"])))
-
- resource = rest_api_resource(
- config,
- team_id,
- job_id,
- db_incremental_field_last_value,
- resume_hook=save_checkpoint,
- initial_paginator_state=initial_paginator_state,
- )
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+
+ resource: Iterable[Any]
+ if cfg.fanout is not None:
+
+ def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if state:
+ resumable_source_manager.save_state(OrbResumeConfig(fanout_state=state))
+
+ resource = build_dependent_resource(
+ endpoint_configs=ORB_ENDPOINTS,
+ child_endpoint=endpoint,
+ fanout=cfg.fanout,
+ client_config=client_config,
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ parent_endpoint_extra={"data_selector": "data"},
+ child_endpoint_extra={"data_selector": "data"},
+ resume_hook=save_fanout_checkpoint,
+ initial_paginator_state=resume.fanout_state if resume else None,
+ )
+ else:
+ config: RESTAPIConfig = {
+ "client": client_config,
+ "resource_defaults": None,
+ "resources": [get_resource(endpoint, should_use_incremental_field)],
+ }
+
+ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ # Persist only when a next page exists; the Redis TTL cleans up on completion. Saving
+ # after each yielded batch means a crash re-yields the last page (merge dedupes on the
+ # primary key) rather than skipping it.
+ if state and state.get("next_cursor"):
+ resumable_source_manager.save_state(OrbResumeConfig(next_cursor=str(state["next_cursor"])))
+
+ resource = rest_api_resource(
+ config,
+ team_id,
+ job_id,
+ db_incremental_field_last_value,
+ resume_hook=save_checkpoint,
+ initial_paginator_state={"next_cursor": resume.next_cursor} if resume and resume.next_cursor else None,
+ )
return SourceResponse(
- name=resource.name,
+ name=endpoint,
items=lambda: resource,
primary_keys=cfg.primary_keys,
- column_hints=resource.column_hints,
- # Orb has no sort param — list endpoints always return newest-created first.
- sort_mode="desc",
+ column_hints=getattr(resource, "column_hints", None),
+ # Orb has no sort param — list endpoints always return newest-created first. A fan-out
+ # restarts that order for every customer, so its rows are not one ordered series.
+ sort_mode=None if cfg.fanout is not None else "desc",
partition_count=1,
partition_size=1,
partition_mode="datetime" if cfg.partition_key else None,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/orb/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/orb/settings.py
index f1b1d9933b68..c28014897941 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/orb/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/orb/settings.py
@@ -1,6 +1,9 @@
from dataclasses import dataclass, field
from typing import Optional
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
ORB_API_BASE_URL = "https://api.withorb.com/v1"
@@ -18,7 +21,7 @@ def _datetime_field(name: str) -> IncrementalField:
}
-@dataclass
+@dataclass(frozen=True)
class OrbEndpointConfig:
name: str
path: str
@@ -33,11 +36,36 @@ class OrbEndpointConfig:
# creation-style field (created_at), never a mutable one.
partition_key: Optional[str] = None
primary_keys: list[str] = field(default_factory=lambda: ["id"])
+ page_size: int = DEFAULT_PAGE_SIZE
+ # Set for per-customer child endpoints, which fan out over the Customers listing.
+ fanout: Optional[DependentEndpointConfig] = None
+ @property
+ def incremental_fields(self) -> list[IncrementalField]:
+ return [_datetime_field(self.incremental_field)] if self.incremental_field else []
-# Top-level Orb list endpoints. All return `{"data": [...], "pagination_metadata": {...}}`, are
+ @property
+ def default_incremental_field(self) -> Optional[str]:
+ return self.incremental_field
+
+
+def _customer_fanout() -> DependentEndpointConfig:
+ return DependentEndpointConfig(
+ parent_name="Customers",
+ resolve_param="customer_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "customer_id"},
+ # A customer deleted between the parent listing and its child fetch 404s; skip it rather
+ # than failing the whole fan-out.
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ )
+
+
+# Orb list endpoints. All return `{"data": [...], "pagination_metadata": {...}}`, are
# cursor-paginated, and sort newest-first by creation time. Incremental endpoints expose a
-# server-side `<field>[gt]` timestamp filter; the rest are full-refresh only.
+# server-side `<field>[gt]` timestamp filter; the rest are full-refresh only. Endpoints with a
+# `fanout` are per-customer and run once for every row of the Customers listing.
#
# Invoices is the odd one out: it has no `created_at` filter, only `invoice_date[gt]`, so its
# incremental cursor is `invoice_date` while it still partitions on the stable `created_at`.
@@ -93,12 +121,43 @@ ORB_ENDPOINTS: dict[str, OrbEndpointConfig] = {
path="/coupons",
table_name="coupons",
),
+ "Prices": OrbEndpointConfig(
+ name="Prices",
+ path="/prices",
+ table_name="prices",
+ partition_key="created_at",
+ ),
+ # Billable metric objects carry no created_at, so there is neither a cursor nor a stable
+ # datetime to partition on.
+ "BillableMetrics": OrbEndpointConfig(
+ name="BillableMetrics",
+ path="/metrics",
+ table_name="billable_metrics",
+ ),
+ # Ledger entries are full refresh: decrements are grouped per day and pending entries can
+ # later commit, so an entry can change after its created_at has passed the watermark.
+ "CreditLedgerEntries": OrbEndpointConfig(
+ name="CreditLedgerEntries",
+ path="/customers/{customer_id}/credits/ledger",
+ table_name="credit_ledger_entries",
+ partition_key="created_at",
+ primary_keys=["customer_id", "id"],
+ fanout=_customer_fanout(),
+ ),
+ # The list filter is on `operation_time`, which the returned rows do not carry, so there is no
+ # field to track a watermark on.
+ "BalanceTransactions": OrbEndpointConfig(
+ name="BalanceTransactions",
+ path="/customers/{customer_id}/balance_transactions",
+ table_name="balance_transactions",
+ partition_key="created_at",
+ primary_keys=["customer_id", "id"],
+ fanout=_customer_fanout(),
+ ),
}
ENDPOINTS = tuple(ORB_ENDPOINTS.keys())
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
- name: [_datetime_field(cfg.incremental_field)]
- for name, cfg in ORB_ENDPOINTS.items()
- if cfg.incremental_field is not None
+ name: cfg.incremental_fields for name, cfg in ORB_ENDPOINTS.items() if cfg.incremental_fields
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/orb/tests/test_orb.py b/products/warehouse_sources/backend/temporal/data_imports/sources/orb/tests/test_orb.py
index d55303b8b41a..510ea75d0efd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/orb/tests/test_orb.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/orb/tests/test_orb.py
@@ -24,6 +24,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.orb.orb im
from products.warehouse_sources.backend.temporal.data_imports.sources.orb.settings import ENDPOINTS, ORB_ENDPOINTS
from products.warehouse_sources.backend.temporal.data_imports.sources.orb.source import OrbSource
+TOP_LEVEL_ENDPOINTS = [e for e in ENDPOINTS if ORB_ENDPOINTS[e].fanout is None]
+FANOUT_ENDPOINTS = [e for e in ENDPOINTS if ORB_ENDPOINTS[e].fanout is not None]
+
def _response(body: dict[str, Any]) -> MagicMock:
response = MagicMock()
@@ -82,7 +85,7 @@ class TestGetResource:
endpoint = cast(dict[str, Any], resource["endpoint"])
return cast(dict[str, Any], endpoint["params"])
- @parameterized.expand(list(ENDPOINTS))
+ @parameterized.expand(TOP_LEVEL_ENDPOINTS)
def test_resource_shape(self, endpoint: str) -> None:
cfg = ORB_ENDPOINTS[endpoint]
resource = get_resource(endpoint, should_use_incremental_field=False)
@@ -112,7 +115,7 @@ class TestGetResource:
assert param["cursor_path"] == cfg.incremental_field
assert param["convert"] is _format_incremental_value
- @parameterized.expand([e for e in ENDPOINTS if ORB_ENDPOINTS[e].incremental_param is None])
+ @parameterized.expand([e for e in TOP_LEVEL_ENDPOINTS if ORB_ENDPOINTS[e].incremental_param is None])
def test_full_refresh_endpoint_ignores_incremental_flag(self, endpoint: str) -> None:
# Items / Coupons have no server-side time filter: even with incremental requested they
# stay full-refresh (replace) so we never silently emit a bogus filter param.
@@ -162,6 +165,61 @@ class TestOrbSource:
assert result.saved_states == [OrbResumeConfig(next_cursor="next-1")]
assert result.committed_states == [OrbResumeConfig(next_cursor="next-1")]
+ @parameterized.expand(
+ [
+ ("CreditLedgerEntries", "credits/ledger"),
+ ("BalanceTransactions", "balance_transactions"),
+ ]
+ )
+ def test_fanout_walks_every_customer_and_tags_rows(self, endpoint: str, child_suffix: str) -> None:
+ result = self._driver().run(
+ endpoint,
+ [
+ self._page([{"id": "cus_1"}, {"id": "cus_gone"}, {"id": "cus_2"}]),
+ self._page([{"id": "entry-1"}], next_cursor="child-next"),
+ self._page([{"id": "entry-2"}]),
+ # A customer deleted after the parent listing must not fail the fan-out.
+ ScriptedResponse(status=404, json={"title": "Not found"}),
+ self._page([{"id": "entry-3"}]),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.paths == [
+ "/v1/customers",
+ f"/v1/customers/cus_1/{child_suffix}",
+ f"/v1/customers/cus_1/{child_suffix}",
+ f"/v1/customers/cus_gone/{child_suffix}",
+ f"/v1/customers/cus_2/{child_suffix}",
+ ]
+ assert result.params("cursor") == [None, None, "child-next", None, None]
+ assert result.rows == [
+ {"id": "entry-1", "customer_id": "cus_1"},
+ {"id": "entry-2", "customer_id": "cus_1"},
+ {"id": "entry-3", "customer_id": "cus_2"},
+ ]
+
+ def test_fanout_resume_skips_completed_customers(self) -> None:
+ result = self._driver().run(
+ "CreditLedgerEntries",
+ [
+ self._page([{"id": "cus_1"}, {"id": "cus_2"}]),
+ self._page([{"id": "entry-3"}]),
+ ],
+ resume_state=OrbResumeConfig(
+ fanout_state={
+ "completed": ["/customers/cus_1/credits/ledger"],
+ "current": "/customers/cus_2/credits/ledger",
+ "child_state": {"next_cursor": "saved-cursor"},
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/v1/customers", "/v1/customers/cus_2/credits/ledger"]
+ assert result.params("cursor") == [None, "saved-cursor"]
+ assert result.rows == [{"id": "entry-3", "customer_id": "cus_2"}]
+
class TestValidateCredentials:
@parameterized.expand(