Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the Metronome source (#119853)

Tom Owers2dca40709260
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index f03420aa3c0d..6b255c037158 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -7899,9 +7899,9 @@ Today (13): `audit_logs`, `billable_metrics`, `contracts`, `customers`, `invoice
Diffed against: <https://docs.metronome.com/openapi.json>
-- [ ] `Rate schedules (POST /v1/contract-pricing/rate-cards/getRateSchedule; POST /v1/contracts/getContractRateSchedule)` — resolve rate card IDs into prices and customer overrides to explain billed amounts (high)
-- [ ] `List balances (POST /v1/contracts/customerBalances/list)` — measure remaining commitments and credits, including balances and transaction ledgers across contracts (high)
-- [ ] `List invoice breakdowns (GET /v1/customers/{customer_id}/invoices/breakdowns)` — reconcile daily or hourly billed costs with usage, beyond invoice totals and unpriced usage aggregates (high)
+- [x] `Rate schedules (POST /v1/contract-pricing/rate-cards/getRateSchedule; POST /v1/contracts/getContractRateSchedule)` — resolve rate card IDs into prices and customer overrides to explain billed amounts (high)
+- [x] `List balances (POST /v1/contracts/customerBalances/list)` — measure remaining commitments and credits, including balances and transaction ledgers across contracts (high)
+- [x] `List invoice breakdowns (GET /v1/customers/{customer_id}/invoices/breakdowns)` — reconcile daily or hourly billed costs with usage, beyond invoice totals and unpriced usage aggregates (high)
- [ ] `List historical balances (POST /v1/contracts/historicalBalances/list)` — analyze retired commitments, credit consumption, and historical ledger transactions (medium)
- [ ] `Get subscription quantity history (POST /v1/contracts/getSubscriptionQuantityHistory)` — measure changes in recurring quantities and prices for seat-based revenue analysis (medium)
- [ ] `Get subscription seats history (POST /v1/contracts/getSubscriptionSeatsHistory)` — track seat assignments and quantities over time for each subscription (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/canonical_descriptions.py
index 3757f8c4a955..2d7aeb357d5f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/canonical_descriptions.py
@@ -215,4 +215,88 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"description": "A readable summary of the action.",
},
},
+ "balances": {
+ "description": "Every commit and credit a customer can draw down, at customer and contract level, with its current balance and its ledger of drawdowns, rollovers and manual adjustments. Fetched per customer.",
+ "docs_url": "https://docs.metronome.com/api-reference/credits-and-commits/list-balances",
+ "columns": {
+ "id": "The commit's or credit's id.",
+ "customer_id": "The customer the balance belongs to.",
+ "contract": "The contract the commit or credit sits on. Null for a customer-level balance.",
+ "type": "`PREPAID` or `POSTPAID` for a commit, `CREDIT` for a credit.",
+ "rate_type": "Whether usage draws the balance down at the commit rate or the list rate.",
+ "name": "The commit's or credit's display name.",
+ "priority": "Ordering used when several balances could apply. The lower priority applies first.",
+ "product": "The product the commit or credit is billed as.",
+ "access_schedule": "The amounts the customer can draw down, and the dates each amount is available.",
+ "invoice_schedule": "When the customer is invoiced for a commit.",
+ "amount": "The commit's total amount.",
+ "balance": "The amount still available to draw down.",
+ "ledger": "Every entry that changed the balance, in order: segment starts, drawdowns, rollovers, expirations and manual adjustments.",
+ "applicable_product_ids": "Products whose usage can draw down this balance. Empty means all products.",
+ "applicable_product_tags": "Product tags whose usage can draw down this balance.",
+ "custom_fields": "Arbitrary key/value pairs set on the commit or credit by your account.",
+ "created_at": "When the commit or credit was created.",
+ "archived_at": "When the commit was archived. Null while it is active.",
+ },
+ },
+ "invoice_breakdowns": {
+ "description": "Each invoice split into daily windows, with the quantity and cost of every line item in each day. Fetched per customer.",
+ "docs_url": "https://docs.metronome.com/api-reference/invoices/list-invoice-breakdowns",
+ "columns": {
+ "id": "The invoice's id. One invoice has one row per day.",
+ "customer_id": "The customer the invoice was issued to.",
+ "contract_id": "The contract the invoice bills against, if any.",
+ "breakdown_start_timestamp": "Start of the day this row covers.",
+ "breakdown_end_timestamp": "End of the day this row covers.",
+ "start_timestamp": "Start of the invoice's whole billing period.",
+ "end_timestamp": "End of the invoice's whole billing period.",
+ "type": "The kind of invoice, such as `USAGE` or `SCHEDULED`.",
+ "status": "The invoice's status, such as `DRAFT` or `FINALIZED`.",
+ "credit_type": "The pricing unit the amounts are in.",
+ "line_items": "The invoice's line items, with the quantity and cost for this day only.",
+ "subtotal": "The day's total before credits and commits are applied.",
+ "total": "The day's total.",
+ },
+ },
+ "rate_card_rate_schedules": {
+ "description": "The prices on each rate card, one row per product, pricing group, billing frequency and schedule segment, including past and scheduled changes.",
+ "docs_url": "https://docs.metronome.com/api-reference/rate-cards/get-a-rate-schedule",
+ "columns": {
+ "rate_card_id": "The rate card the rate belongs to.",
+ "product_id": "The product the rate prices.",
+ "product_name": "The product's name.",
+ "product_tags": "The product's tags.",
+ "product_custom_fields": "Arbitrary key/value pairs set on the product.",
+ "pricing_group_values": "The pricing group the rate applies to, as dimension/value pairs.",
+ "pricing_group_key": "`pricing_group_values` as a sorted JSON string, so the pricing group can be part of the row's key.",
+ "billing_frequency": "The billing frequency of a subscription rate.",
+ "starting_at": "When this rate takes effect.",
+ "ending_before": "When this rate stops applying. Null when no later change is scheduled.",
+ "entitled": "Whether customers on the rate card can use the product by default.",
+ "rate": "The price, with its rate type and any tiers.",
+ "commit_rate": "The price that applies when usage draws down a commit, if different.",
+ },
+ },
+ "contract_rate_schedules": {
+ "description": "The rates in effect on each active contract at the time of the sync, with the contract's overrides applied. Archived contracts are skipped.",
+ "docs_url": "https://docs.metronome.com/api-reference/contracts/get-the-rate-schedule-for-a-contract",
+ "columns": {
+ "contract_id": "The contract the rate applies to.",
+ "customer_id": "The customer the contract belongs to.",
+ "rate_card_id": "The rate card the list rate comes from.",
+ "product_id": "The product the rate prices.",
+ "product_name": "The product's name.",
+ "product_tags": "The product's tags.",
+ "product_custom_fields": "Arbitrary key/value pairs set on the product.",
+ "pricing_group_values": "The pricing group the rate applies to, as dimension/value pairs.",
+ "pricing_group_key": "`pricing_group_values` as a sorted JSON string, so the pricing group can be part of the row's key.",
+ "billing_frequency": "The billing frequency of a subscription rate.",
+ "starting_at": "When this rate segment takes effect.",
+ "ending_before": "When this rate segment stops applying. Null when no later change is scheduled.",
+ "entitled": "Whether the contract can use the product.",
+ "list_rate": "The rate card's price.",
+ "override_rate": "The contract's override of the list price, if any.",
+ "commit_rate": "The price that applies when usage draws down a commit, if different.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/metronome.py b/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/metronome.py
index 25e6edf983cd..8f1425d30652 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/metronome.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/metronome.py
@@ -1,10 +1,11 @@
+import json
import threading
import dataclasses
from collections import deque
from collections.abc import Callable, Generator, Iterable, Iterator
from concurrent.futures import Future, ThreadPoolExecutor, wait
from datetime import UTC, datetime, timedelta
-from typing import Any, Optional, cast
+from typing import Any, Literal, Optional, cast
from requests import PreparedRequest, Request, Response, Session
@@ -47,6 +48,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.metronome.
CURSOR_PARAM,
CURSOR_PATH,
DATA_SELECTOR,
+ EPOCH_RFC_3339,
METRONOME_BASE_URL,
METRONOME_ENDPOINTS,
USAGE_HISTORY,
@@ -58,9 +60,6 @@ REQUEST_TIMEOUT_SECONDS = 30.0
# Metronome accepts RFC 3339 timestamps and returns them in this shape.
RFC_3339_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
-# Lower bound for a first incremental run — earlier than any Metronome account.
-EPOCH_RFC_3339 = "1970-01-01T00:00:00Z"
-
# `POST /v1/usage` documents `ending_before` as at least one day after `starting_on`.
MIN_USAGE_WINDOW = timedelta(days=1)
@@ -128,8 +127,10 @@ class MetronomeCursorPaginator(JSONResponseCursorPaginator):
window only rides the request that has no cursor on it.
"""
- def __init__(self, first_page_only_params: tuple[str, ...] = ()) -> None:
- super().__init__(cursor_path=CURSOR_PATH, cursor_param=CURSOR_PARAM)
+ def __init__(
+ self, first_page_only_params: tuple[str, ...] = (), param_location: Literal["query", "json"] = "query"
+ ) -> None:
+ super().__init__(cursor_path=CURSOR_PATH, cursor_param=CURSOR_PARAM, param_location=param_location)
self._first_page_only_params = first_page_only_params
def _drop_first_page_only_params(self, request: Request) -> None:
@@ -158,7 +159,7 @@ def _paginator_for(config: MetronomeEndpointConfig) -> BasePaginator:
if not config.paginated:
return SinglePagePaginator()
first_page_only: tuple[str, ...] = (config.incremental_start_param,) if config.incremental_start_param else ()
- return MetronomeCursorPaginator(first_page_only_params=first_page_only)
+ return MetronomeCursorPaginator(first_page_only_params=first_page_only, param_location=config.page_param_location)
def _format_rfc3339(value: Any) -> str:
@@ -247,6 +248,19 @@ def _resolve_window_start(
return _format_rfc3339(_align_to_utc_midnight(start))
+def _query_window(history: timedelta, db_incremental_field_last_value: Any) -> dict[str, str]:
+ """The `starting_on`/`ending_before` query window of a fan-out child that requires one.
+
+ Both bounds sit on UTC midnight for the same reason as a usage window: a daily breakdown's start
+ is part of its primary key, so asking from mid-day would return a partial day as a second row.
+ """
+ now = datetime.now(UTC)
+ ending_before = _format_rfc3339(_usage_window_end(now))
+ start = parse_datetime_value(db_incremental_field_last_value) or now - history
+ starting_on = _clamp_window_start(_format_rfc3339(_align_to_utc_midnight(start)), ending_before)
+ return {"starting_on": starting_on, "ending_before": ending_before}
+
+
@frozen
class MetronomeWalkStart:
"""Where one walk of an endpoint begins: the request window, and the cursor to resume from."""
@@ -348,14 +362,25 @@ def _rest_client(api_key: str) -> RESTClient:
)
+def _sends_page_size(config: MetronomeEndpointConfig) -> bool:
+ return config.paginated and config.accepts_page_size
+
+
def _list_params(config: MetronomeEndpointConfig) -> dict[str, Any]:
params: dict[str, Any] = {}
- if config.paginated and config.accepts_page_size:
+ if _sends_page_size(config) and config.page_param_location == "query":
params["limit"] = config.page_size
params.update(config.extra_params)
return params
+def _list_body(config: MetronomeEndpointConfig) -> dict[str, Any]:
+ body = dict(config.json_body)
+ if _sends_page_size(config) and config.page_param_location == "json":
+ body["limit"] = config.page_size
+ return body
+
+
def _retry_after_seconds(response: Response) -> float | None:
raw = response.headers.get("Retry-After")
if not raw:
@@ -704,35 +729,68 @@ def get_resource(
return resource
-def _body_fanout_pages(client: RESTClient, config: MetronomeEndpointConfig) -> Iterator[list[dict[str, Any]]]:
- """Walk the parent list and call the child once per parent row, binding the id into the body."""
- fanout = config.body_fanout
- if fanout is None:
- raise ValueError(f"'{config.name}' has no body fan-out configured")
+def _pricing_group_key(pricing_group_values: Any) -> str:
+ """A stable string for a rate's pricing group, so the group can be part of the primary key.
+
+ `pricing_group_values` is an object, and a rate with none is a different rate from one scoped
+ to a group, so the absent case gets its own value rather than a null.
+ """
+ return json.dumps(pricing_group_values or {}, sort_keys=True, separators=(",", ":"))
- parent = METRONOME_ENDPOINTS[fanout.parent_name]
- for parent_page in client.paginate(
+def _shape_fanout_row(
+ config: MetronomeEndpointConfig, row: dict[str, Any], parent_values: dict[str, Any]
+) -> dict[str, Any]:
+ for name, value in parent_values.items():
+ if row.get(name) is None:
+ row[name] = value
+ if config.keys_pricing_group:
+ row["pricing_group_key"] = _pricing_group_key(row.get("pricing_group_values"))
+ return row
+
+
+def _parent_pages(client: RESTClient, parent: MetronomeEndpointConfig) -> Iterator[list[dict[str, Any]]]:
+ if parent.body_fanout is not None:
+ yield from _body_fanout_pages(client, parent)
+ return
+ yield from client.paginate(
parent.path,
+ method=parent.method,
params=_list_params(parent),
+ # The POST lists expect a JSON document even when they take no filters.
+ json=_list_body(parent) if parent.method == "post" else None,
data_selector=DATA_SELECTOR,
data_selector_required=True,
paginator=_paginator_for(parent),
- ):
+ )
+
+
+def _body_fanout_pages(client: RESTClient, config: MetronomeEndpointConfig) -> Iterator[list[dict[str, Any]]]:
+ """Walk the parent list and call the child once per parent row, binding its ids into the body."""
+ fanout = config.body_fanout
+ if fanout is None:
+ raise ValueError(f"'{config.name}' has no body fan-out configured")
+
+ parent = METRONOME_ENDPOINTS[fanout.parent_name]
+
+ for parent_page in _parent_pages(client, parent):
for parent_row in parent_page:
- parent_id = parent_row.get(fanout.resolve_field)
- if not parent_id:
+ if fanout.skip_parents_with and parent_row.get(fanout.skip_parents_with):
+ continue
+ parent_values = {param: parent_row.get(field) for param, field in fanout.body_params.items()}
+ if not all(parent_values.values()):
continue
for page in client.paginate(
config.path,
method=config.method,
- json={**config.json_body, fanout.body_param: parent_id},
+ params=_list_params(config),
+ json={**_list_body(config), **parent_values},
data_selector=DATA_SELECTOR,
data_selector_required=True,
paginator=_paginator_for(config),
):
if page:
- yield page
+ yield [_shape_fanout_row(config, row, parent_values) for row in page]
def _make_source_response(
@@ -779,6 +837,9 @@ def metronome_source(
if endpoint_config.fanout:
parent_config = METRONOME_ENDPOINTS[endpoint_config.fanout.parent_name]
+ child_params = dict(endpoint_config.extra_params)
+ if endpoint_config.query_window_history is not None:
+ child_params.update(_query_window(endpoint_config.query_window_history, db_incremental_field_last_value))
# Dependent resources don't support resume in the rest_source framework, so the manager is
# intentionally not threaded into this path.
dependent_resource = cast(
@@ -786,7 +847,7 @@ def metronome_source(
build_dependent_resource(
endpoint_configs=METRONOME_ENDPOINTS,
child_endpoint=endpoint,
- fanout=dataclasses.replace(endpoint_config.fanout, child_params=dict(endpoint_config.extra_params)),
+ fanout=dataclasses.replace(endpoint_config.fanout, child_params=child_params),
client_config=_rest_api_client_config(api_key),
path_format_values={},
team_id=team_id,
@@ -794,6 +855,9 @@ def metronome_source(
db_incremental_field_last_value=db_incremental_field_last_value,
should_use_incremental_field=should_use_incremental_field,
incremental_field=incremental_field,
+ # The window rides `child_params`, so there is no cursor param for the framework to
+ # inject. The child still merges on its primary key.
+ incremental_config_factory=lambda _field: None,
parent_endpoint_extra={
"paginator": _paginator_for(parent_config),
"data_selector": DATA_SELECTOR,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/settings.py
index 6318726573bc..94360f268963 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/settings.py
@@ -23,6 +23,19 @@ CURSOR_PARAM = "next_page"
# `GET /v1/auditLogs` is the only endpoint with a server-side lower bound on record time.
AUDIT_LOG_START_PARAM = "starting_on"
+# `POST /v1/contracts/customerBalances/list` caps `limit` at 25.
+BALANCES_PAGE_SIZE = 25
+
+# Lower bound for a request that asks for a whole schedule — earlier than any Metronome account.
+EPOCH_RFC_3339 = "1970-01-01T00:00:00Z"
+
+# How far back a first sync of `invoice_breakdowns` reaches. It holds one row per invoice and day, so
+# like the daily usage table it reaches back a year rather than to the start of the account.
+INVOICE_BREAKDOWNS_HISTORY = timedelta(days=365)
+# Backdated usage changes a breakdown after its day has passed, and a draft invoice keeps changing
+# until it finalizes, so each incremental run re-reads this much of what it already covered.
+INVOICE_BREAKDOWNS_LOOKBACK_SECONDS = 7 * 24 * 60 * 60
+
# How `POST /v1/usage` aggregates the period it is asked for: one row per hour, one row per day, or
# one row for the whole period.
WindowSize = Literal["none", "hour", "day"]
@@ -75,16 +88,19 @@ USAGE_HOURLY_LOOKBACK_SECONDS = 24 * 60 * 60
@dataclass(frozen=True)
class BodyFanoutConfig:
- """Fan-out whose parent id belongs in the child's JSON body.
+ """Fan-out whose parent ids belong in the child's JSON body.
The declarative fan-out helper binds a resolved parent field into the child's *path* only, so
a child like `POST /v2/contracts/list` — which takes `customer_id` in its body — walks the
- parent itself instead.
+ parent itself instead. The parent can itself be a body fan-out, which chains the walk.
"""
parent_name: str
- resolve_field: str
- body_param: str
+ # Child body param -> the parent row field it is read from. Each one is also copied onto the
+ # child rows that lack it, because several children never echo back the ids they were asked for.
+ body_params: dict[str, str]
+ # Parent rows that have this field set are skipped.
+ skip_parents_with: str | None = None
# frozen=False: MetronomeEndpointConfig is passed as `endpoint_configs: Mapping[str, FanoutEndpointLike]`,
@@ -102,6 +118,9 @@ class MetronomeEndpointConfig:
extra_params: dict[str, Any] = field(default_factory=dict)
# Endpoints whose response carries a `next_page` cursor; the rest return the whole collection.
paginated: bool = True
+ # Where `limit` and `next_page` go. `POST /v1/contracts/customerBalances/list` reads both from
+ # the body and ignores them in the query string.
+ page_param_location: Literal["query", "json"] = "query"
# Whether the endpoint takes a `limit` page-size parameter. `POST /v1/usage` follows
# `next_page` but rejects `limit`, and answers the whole request with a 400 when it is
# present. Following the cursor and setting a page size are separate capabilities, so
@@ -129,6 +148,11 @@ class MetronomeEndpointConfig:
sort_mode: SortMode = "asc"
should_sync_default: bool = True
default_incremental_lookback_seconds: int | None = None
+ # A fan-out child whose query needs a `starting_on`/`ending_before` window. This is how far back
+ # a run with no watermark reaches. None means the endpoint sends no window.
+ query_window_history: timedelta | None = None
+ # Derives `pricing_group_key` on rate schedule rows, whose identity includes an object column.
+ keys_pricing_group: bool = False
# Customer-scoped children bind the parent's `id` into their path as `customer_id`. Every child
@@ -162,6 +186,25 @@ METRONOME_ENDPOINTS: dict[str, MetronomeEndpointConfig] = {
page_size=25,
fanout=_CUSTOMER_PATH_FANOUT,
),
+ # One row per invoice and day. Starts off, because a first sync is one walk per customer over a
+ # year of days.
+ "invoice_breakdowns": MetronomeEndpointConfig(
+ name="invoice_breakdowns",
+ path="/v1/customers/{customer_id}/invoices/breakdowns",
+ primary_key=["id", "breakdown_start_timestamp"],
+ partition_key="breakdown_start_timestamp",
+ page_size=25,
+ # `sort` orders the invoices by issue date, so rows still arrive grouped by customer rather
+ # than by period, and the watermark is held back until the walk ends, like `usage_daily`.
+ extra_params={"window_size": "DAY", "sort": "date_asc"},
+ fanout=_CUSTOMER_PATH_FANOUT,
+ incremental_fields=[incremental_field("breakdown_start_timestamp")],
+ default_incremental_field="breakdown_start_timestamp",
+ default_incremental_lookback_seconds=INVOICE_BREAKDOWNS_LOOKBACK_SECONDS,
+ query_window_history=INVOICE_BREAKDOWNS_HISTORY,
+ sort_mode="desc",
+ should_sync_default=False,
+ ),
"contracts": MetronomeEndpointConfig(
name="contracts",
path="/v2/contracts/list",
@@ -171,7 +214,42 @@ METRONOME_ENDPOINTS: dict[str, MetronomeEndpointConfig] = {
# Returns every contract for the customer in one response — no cursor, no `limit`.
paginated=False,
json_body={"include_archived": True},
- body_fanout=BodyFanoutConfig(parent_name="customers", resolve_field="id", body_param="customer_id"),
+ body_fanout=BodyFanoutConfig(parent_name="customers", body_params={"customer_id": "id"}),
+ ),
+ "balances": MetronomeEndpointConfig(
+ name="balances",
+ path="/v1/contracts/customerBalances/list",
+ method="post",
+ # A commit or credit id is a UUID, but the rows come from one walk per customer, so the
+ # customer joins the key rather than relying on that.
+ primary_key=["customer_id", "id"],
+ page_size=BALANCES_PAGE_SIZE,
+ page_param_location="json",
+ # Ledgers and balances are off by default. Both are what this table exists for.
+ json_body={
+ "include_archived": True,
+ "include_balance": True,
+ "include_contract_balances": True,
+ "include_ledgers": True,
+ },
+ body_fanout=BodyFanoutConfig(parent_name="customers", body_params={"customer_id": "id"}),
+ ),
+ # The rates in effect now on each live contract, with the contract's overrides applied. The
+ # endpoint answers for one point in time only, so the table is a current snapshot. It takes one
+ # walk per contract, so it starts off.
+ "contract_rate_schedules": MetronomeEndpointConfig(
+ name="contract_rate_schedules",
+ path="/v1/contracts/getContractRateSchedule",
+ method="post",
+ primary_key=["contract_id", "product_id", "pricing_group_key", "billing_frequency", "starting_at"],
+ body_fanout=BodyFanoutConfig(
+ parent_name="contracts",
+ body_params={"customer_id": "customer_id", "contract_id": "id"},
+ # An archived contract bills nothing, so it has no rates in effect to report.
+ skip_parents_with="archived_at",
+ ),
+ keys_pricing_group=True,
+ should_sync_default=False,
),
"products": MetronomeEndpointConfig(
name="products",
@@ -188,6 +266,18 @@ METRONOME_ENDPOINTS: dict[str, MetronomeEndpointConfig] = {
primary_key=["id"],
partition_key="created_at",
),
+ "rate_card_rate_schedules": MetronomeEndpointConfig(
+ name="rate_card_rate_schedules",
+ path="/v1/contract-pricing/rate-cards/getRateSchedule",
+ method="post",
+ # A rate carries no id. It is one product's price for one pricing group and billing
+ # frequency, from one point in time.
+ primary_key=["rate_card_id", "product_id", "pricing_group_key", "billing_frequency", "starting_at"],
+ # `starting_at` is required. The epoch asks for every segment, past and scheduled.
+ json_body={"starting_at": EPOCH_RFC_3339},
+ body_fanout=BodyFanoutConfig(parent_name="rate_cards", body_params={"rate_card_id": "id"}),
+ keys_pricing_group=True,
+ ),
"packages": MetronomeEndpointConfig(
name="packages",
path="/v1/packages/list",
@@ -280,6 +370,9 @@ USAGE_HISTORY: dict[str, timedelta] = {
"usage_hourly": USAGE_HOURLY_HISTORY,
}
+# Tables whose incremental runs re-read periods they already hold, so they merge and never append.
+MERGE_ONLY = (*USAGE_HISTORY, "invoice_breakdowns")
+
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
name: config.incremental_fields for name, config in METRONOME_ENDPOINTS.items()
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/source.py
index 7095b71a8a91..bb8dbff57ae5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/source.py
@@ -32,8 +32,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.metronome.
DEFAULT_USAGE_HOURLY_HISTORY_DAYS,
ENDPOINTS,
INCREMENTAL_FIELDS,
+ MERGE_ONLY,
METRONOME_ENDPOINTS,
- USAGE_HISTORY,
usage_history_window,
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
@@ -90,9 +90,9 @@ class MetronomeSource(ResumableSource[MetronomeSourceConfig, MetronomeResumeConf
ENDPOINTS,
INCREMENTAL_FIELDS,
names,
- # Appending would duplicate every period the lookback re-reads, so a bucketed usage
+ # Appending would duplicate every period the lookback re-reads, so a windowed
# table has to merge on its primary key.
- merge_only=tuple(USAGE_HISTORY),
+ merge_only=MERGE_ONLY,
should_sync_default={name: METRONOME_ENDPOINTS[name].should_sync_default for name in ENDPOINTS},
)
for schema in schemas:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/tests/test_metronome.py b/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/tests/test_metronome.py
index 629eb24f1042..bd00c41d5f23 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/tests/test_metronome.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/tests/test_metronome.py
@@ -32,6 +32,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.metronome.
validate_credentials,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.metronome.settings import (
+ INVOICE_BREAKDOWNS_LOOKBACK_SECONDS,
METRONOME_ENDPOINTS,
USAGE_DAILY_LOOKBACK_SECONDS,
USAGE_HOURLY_LOOKBACK_SECONDS,
@@ -666,6 +667,157 @@ class TestMetronomeBodyFanout:
assert client.paginate.call_count == 1
+class TestMetronomeNewFanoutTables:
+ def test_contract_rate_schedules_walk_each_live_contract_with_both_ids_in_the_body(self) -> None:
+ result = _driver().run(
+ "contract_rate_schedules",
+ [
+ ScriptedResponse(json={"data": [{"id": "cust_1"}], "next_page": None}),
+ ScriptedResponse(
+ json={
+ "data": [
+ {"id": "contract_1", "customer_id": "cust_1"},
+ # An archived contract bills nothing, so it is never asked for rates.
+ {"id": "contract_2", "customer_id": "cust_1", "archived_at": "2026-01-01T00:00:00Z"},
+ ]
+ }
+ ),
+ ScriptedResponse(
+ json={
+ "data": [
+ {"rate_card_id": "rc_1", "product_id": "p1", "starting_at": "2026-01-01T00:00:00Z"},
+ ],
+ "next_page": "cursor-2",
+ }
+ ),
+ ScriptedResponse(
+ json={
+ "data": [
+ {
+ "rate_card_id": "rc_1",
+ "product_id": "p1",
+ "starting_at": "2026-01-01T00:00:00Z",
+ "pricing_group_values": {"region": "eu", "cloud": "aws"},
+ },
+ ],
+ "next_page": None,
+ }
+ ),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.paths == [
+ "/v1/customers",
+ "/v2/contracts/list",
+ "/v1/contracts/getContractRateSchedule",
+ "/v1/contracts/getContractRateSchedule",
+ ]
+ assert [request.json() for request in result.requests[2:]] == [
+ {"customer_id": "cust_1", "contract_id": "contract_1"},
+ {"customer_id": "cust_1", "contract_id": "contract_1"},
+ ]
+ assert result.params("next_page")[2:] == [None, "cursor-2"]
+ # The rows never echo the contract they were asked for, and the pricing group is an object,
+ # so both have to be added for the primary key to identify a row.
+ assert [(row["contract_id"], row["customer_id"], row["pricing_group_key"]) for row in result.rows] == [
+ ("contract_1", "cust_1", "{}"),
+ ("contract_1", "cust_1", '{"cloud":"aws","region":"eu"}'),
+ ]
+
+ def test_rate_card_rate_schedules_ask_for_the_whole_schedule_of_each_rate_card(self) -> None:
+ result = _driver().run(
+ "rate_card_rate_schedules",
+ [
+ ScriptedResponse(json={"data": [{"id": "rc_1"}, {"id": "rc_2"}], "next_page": None}),
+ ScriptedResponse(json={"data": [{"product_id": "p1", "starting_at": "2026-01-01T00:00:00Z"}]}),
+ ScriptedResponse(json={"data": []}),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.paths == [
+ "/v1/contract-pricing/rate-cards/list",
+ "/v1/contract-pricing/rate-cards/getRateSchedule",
+ "/v1/contract-pricing/rate-cards/getRateSchedule",
+ ]
+ # The rate card list is a POST that still expects a JSON document.
+ assert result.requests[0].json() == {}
+ assert [request.json() for request in result.requests[1:]] == [
+ {"starting_at": "1970-01-01T00:00:00Z", "rate_card_id": "rc_1"},
+ {"starting_at": "1970-01-01T00:00:00Z", "rate_card_id": "rc_2"},
+ ]
+ assert result.params("limit")[1:] == ["100", "100"]
+ assert [(row["rate_card_id"], row["product_id"]) for row in result.rows] == [("rc_1", "p1")]
+
+ def test_balances_page_through_the_body_and_keep_the_customer(self) -> None:
+ # The endpoint reads `limit` and `next_page` from the body only. A cursor sent in the query
+ # string would be ignored, and the walk would ask for the first page again forever.
+ result = _driver().run(
+ "balances",
+ [
+ ScriptedResponse(json={"data": [{"id": "cust_1"}], "next_page": None}),
+ ScriptedResponse(json={"data": [{"id": "commit_1", "type": "PREPAID"}], "next_page": "cursor-2"}),
+ ScriptedResponse(json={"data": [{"id": "credit_1", "type": "CREDIT"}], "next_page": None}),
+ ],
+ )
+
+ assert result.raised is None
+ bodies = [request.json() for request in result.requests[1:]]
+ assert [body.get("next_page") for body in bodies] == [None, "cursor-2"]
+ assert {body["limit"] for body in bodies} == {25}
+ assert {body["customer_id"] for body in bodies} == {"cust_1"}
+ assert all(body["include_ledgers"] and body["include_balance"] for body in bodies)
+ assert result.params("limit")[1:] == [None, None]
+ assert result.params("next_page")[1:] == [None, None]
+ assert [(row["customer_id"], row["id"]) for row in result.rows] == [
+ ("cust_1", "commit_1"),
+ ("cust_1", "credit_1"),
+ ]
+
+ @parameterized.expand(
+ [
+ # A first sync reaches back a year, from the UTC midnight before it.
+ ("first_sync", {}, "2025-09-03T00:00:00Z"),
+ # The watermark arrives already moved back by the lookback, and lands mid-day. Asking
+ # from mid-day would return a partial day as a second row under a different key.
+ (
+ "incremental",
+ {
+ "should_use_incremental_field": True,
+ "incremental_field": "breakdown_start_timestamp",
+ "db_incremental_field_last_value": "2026-08-27T15:00:00Z",
+ },
+ "2026-08-27T00:00:00Z",
+ ),
+ ]
+ )
+ def test_invoice_breakdowns_ask_for_whole_days_from_the_watermark(self, _name, inputs, starting_on) -> None:
+ with time_machine.travel(NOW, tick=False):
+ result = _driver().run(
+ "invoice_breakdowns",
+ [
+ ScriptedResponse(json={"data": [{"id": "cust_1"}], "next_page": None}),
+ ScriptedResponse(
+ json={
+ "data": [{"id": "inv_1", "breakdown_start_timestamp": "2026-09-02T00:00:00Z"}],
+ "next_page": None,
+ }
+ ),
+ ],
+ **inputs,
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/v1/customers", "/v1/customers/cust_1/invoices/breakdowns"]
+ breakdown_query = result.queries[1]
+ assert breakdown_query["starting_on"] == (starting_on,)
+ # The day in progress is included, so the newest day is never a full period behind.
+ assert breakdown_query["ending_before"] == ("2026-09-04T00:00:00Z",)
+ assert breakdown_query["window_size"] == ("DAY",)
+ assert [row["id"] for row in result.rows] == ["inv_1"]
+
+
class TestMetronomeCredentials:
@parameterized.expand(
[
@@ -711,17 +863,18 @@ class TestMetronomeSchemas:
@parameterized.expand(
[
- ("usage_daily", USAGE_DAILY_LOOKBACK_SECONDS),
- ("usage_hourly", USAGE_HOURLY_LOOKBACK_SECONDS),
+ ("usage_daily", USAGE_DAILY_LOOKBACK_SECONDS, "start_timestamp"),
+ ("usage_hourly", USAGE_HOURLY_LOOKBACK_SECONDS, "start_timestamp"),
+ ("invoice_breakdowns", INVOICE_BREAKDOWNS_LOOKBACK_SECONDS, "breakdown_start_timestamp"),
]
)
- def test_bucketed_usage_merges_incrementally_and_starts_off(self, name, lookback_seconds) -> None:
+ def test_windowed_tables_merge_incrementally_and_start_off(self, name, lookback_seconds, cursor_field) -> None:
schema = self._schema(name)
assert schema.supports_incremental is True
# Appending would duplicate every period the lookback re-reads.
assert schema.supports_append is False
- assert [field["field"] for field in schema.incremental_fields] == ["start_timestamp"]
+ assert [field["field"] for field in schema.incremental_fields] == [cursor_field]
assert schema.should_sync_default is False
assert schema.default_incremental_lookback_seconds == lookback_seconds