:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 4 missing tables to the Metronome source (#119853)
Tom Owers2dca40709260
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex f03420aa3c0d..6b255c037158 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -7899,9 +7899,9 @@ Today (13): `audit_logs`, `billable_metrics`, `contracts`, `customers`, `invoiceDiffed against: <https://docs.metronome.com/openapi.json>-- [ ] `Rate schedules (POST /v1/contract-pricing/rate-cards/getRateSchedule; POST /v1/contracts/getContractRateSchedule)` — resolve rate card IDs into prices and customer overrides to explain billed amounts (high)-- [ ] `List balances (POST /v1/contracts/customerBalances/list)` — measure remaining commitments and credits, including balances and transaction ledgers across contracts (high)-- [ ] `List invoice breakdowns (GET /v1/customers/{customer_id}/invoices/breakdowns)` — reconcile daily or hourly billed costs with usage, beyond invoice totals and unpriced usage aggregates (high)+- [x] `Rate schedules (POST /v1/contract-pricing/rate-cards/getRateSchedule; POST /v1/contracts/getContractRateSchedule)` — resolve rate card IDs into prices and customer overrides to explain billed amounts (high)+- [x] `List balances (POST /v1/contracts/customerBalances/list)` — measure remaining commitments and credits, including balances and transaction ledgers across contracts (high)+- [x] `List invoice breakdowns (GET /v1/customers/{customer_id}/invoices/breakdowns)` — reconcile daily or hourly billed costs with usage, beyond invoice totals and unpriced usage aggregates (high)- [ ] `List historical balances (POST /v1/contracts/historicalBalances/list)` — analyze retired commitments, credit consumption, and historical ledger transactions (medium)- [ ] `Get subscription quantity history (POST /v1/contracts/getSubscriptionQuantityHistory)` — measure changes in recurring quantities and prices for seat-based revenue analysis (medium)- [ ] `Get subscription seats history (POST /v1/contracts/getSubscriptionSeatsHistory)` — track seat assignments and quantities over time for each subscription (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/canonical_descriptions.pyindex 3757f8c4a955..2d7aeb357d5f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/canonical_descriptions.py@@ -215,4 +215,88 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"description": "A readable summary of the action.",},},+ "balances": {+ "description": "Every commit and credit a customer can draw down, at customer and contract level, with its current balance and its ledger of drawdowns, rollovers and manual adjustments. Fetched per customer.",+ "docs_url": "https://docs.metronome.com/api-reference/credits-and-commits/list-balances",+ "columns": {+ "id": "The commit's or credit's id.",+ "customer_id": "The customer the balance belongs to.",+ "contract": "The contract the commit or credit sits on. Null for a customer-level balance.",+ "type": "`PREPAID` or `POSTPAID` for a commit, `CREDIT` for a credit.",+ "rate_type": "Whether usage draws the balance down at the commit rate or the list rate.",+ "name": "The commit's or credit's display name.",+ "priority": "Ordering used when several balances could apply. The lower priority applies first.",+ "product": "The product the commit or credit is billed as.",+ "access_schedule": "The amounts the customer can draw down, and the dates each amount is available.",+ "invoice_schedule": "When the customer is invoiced for a commit.",+ "amount": "The commit's total amount.",+ "balance": "The amount still available to draw down.",+ "ledger": "Every entry that changed the balance, in order: segment starts, drawdowns, rollovers, expirations and manual adjustments.",+ "applicable_product_ids": "Products whose usage can draw down this balance. Empty means all products.",+ "applicable_product_tags": "Product tags whose usage can draw down this balance.",+ "custom_fields": "Arbitrary key/value pairs set on the commit or credit by your account.",+ "created_at": "When the commit or credit was created.",+ "archived_at": "When the commit was archived. Null while it is active.",+ },+ },+ "invoice_breakdowns": {+ "description": "Each invoice split into daily windows, with the quantity and cost of every line item in each day. Fetched per customer.",+ "docs_url": "https://docs.metronome.com/api-reference/invoices/list-invoice-breakdowns",+ "columns": {+ "id": "The invoice's id. One invoice has one row per day.",+ "customer_id": "The customer the invoice was issued to.",+ "contract_id": "The contract the invoice bills against, if any.",+ "breakdown_start_timestamp": "Start of the day this row covers.",+ "breakdown_end_timestamp": "End of the day this row covers.",+ "start_timestamp": "Start of the invoice's whole billing period.",+ "end_timestamp": "End of the invoice's whole billing period.",+ "type": "The kind of invoice, such as `USAGE` or `SCHEDULED`.",+ "status": "The invoice's status, such as `DRAFT` or `FINALIZED`.",+ "credit_type": "The pricing unit the amounts are in.",+ "line_items": "The invoice's line items, with the quantity and cost for this day only.",+ "subtotal": "The day's total before credits and commits are applied.",+ "total": "The day's total.",+ },+ },+ "rate_card_rate_schedules": {+ "description": "The prices on each rate card, one row per product, pricing group, billing frequency and schedule segment, including past and scheduled changes.",+ "docs_url": "https://docs.metronome.com/api-reference/rate-cards/get-a-rate-schedule",+ "columns": {+ "rate_card_id": "The rate card the rate belongs to.",+ "product_id": "The product the rate prices.",+ "product_name": "The product's name.",+ "product_tags": "The product's tags.",+ "product_custom_fields": "Arbitrary key/value pairs set on the product.",+ "pricing_group_values": "The pricing group the rate applies to, as dimension/value pairs.",+ "pricing_group_key": "`pricing_group_values` as a sorted JSON string, so the pricing group can be part of the row's key.",+ "billing_frequency": "The billing frequency of a subscription rate.",+ "starting_at": "When this rate takes effect.",+ "ending_before": "When this rate stops applying. Null when no later change is scheduled.",+ "entitled": "Whether customers on the rate card can use the product by default.",+ "rate": "The price, with its rate type and any tiers.",+ "commit_rate": "The price that applies when usage draws down a commit, if different.",+ },+ },+ "contract_rate_schedules": {+ "description": "The rates in effect on each active contract at the time of the sync, with the contract's overrides applied. Archived contracts are skipped.",+ "docs_url": "https://docs.metronome.com/api-reference/contracts/get-the-rate-schedule-for-a-contract",+ "columns": {+ "contract_id": "The contract the rate applies to.",+ "customer_id": "The customer the contract belongs to.",+ "rate_card_id": "The rate card the list rate comes from.",+ "product_id": "The product the rate prices.",+ "product_name": "The product's name.",+ "product_tags": "The product's tags.",+ "product_custom_fields": "Arbitrary key/value pairs set on the product.",+ "pricing_group_values": "The pricing group the rate applies to, as dimension/value pairs.",+ "pricing_group_key": "`pricing_group_values` as a sorted JSON string, so the pricing group can be part of the row's key.",+ "billing_frequency": "The billing frequency of a subscription rate.",+ "starting_at": "When this rate segment takes effect.",+ "ending_before": "When this rate segment stops applying. Null when no later change is scheduled.",+ "entitled": "Whether the contract can use the product.",+ "list_rate": "The rate card's price.",+ "override_rate": "The contract's override of the list price, if any.",+ "commit_rate": "The price that applies when usage draws down a commit, if different.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/metronome.py b/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/metronome.pyindex 25e6edf983cd..8f1425d30652 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/metronome.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/metronome.py@@ -1,10 +1,11 @@+import jsonimport threadingimport dataclassesfrom collections import dequefrom collections.abc import Callable, Generator, Iterable, Iteratorfrom concurrent.futures import Future, ThreadPoolExecutor, waitfrom datetime import UTC, datetime, timedelta-from typing import Any, Optional, cast+from typing import Any, Literal, Optional, castfrom requests import PreparedRequest, Request, Response, Session@@ -47,6 +48,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.metronome.CURSOR_PARAM,CURSOR_PATH,DATA_SELECTOR,+ EPOCH_RFC_3339,METRONOME_BASE_URL,METRONOME_ENDPOINTS,USAGE_HISTORY,@@ -58,9 +60,6 @@ REQUEST_TIMEOUT_SECONDS = 30.0# Metronome accepts RFC 3339 timestamps and returns them in this shape.RFC_3339_FORMAT = "%Y-%m-%dT%H:%M:%SZ"-# Lower bound for a first incremental run — earlier than any Metronome account.-EPOCH_RFC_3339 = "1970-01-01T00:00:00Z"-# `POST /v1/usage` documents `ending_before` as at least one day after `starting_on`.MIN_USAGE_WINDOW = timedelta(days=1)@@ -128,8 +127,10 @@ class MetronomeCursorPaginator(JSONResponseCursorPaginator):window only rides the request that has no cursor on it."""- def __init__(self, first_page_only_params: tuple[str, ...] = ()) -> None:- super().__init__(cursor_path=CURSOR_PATH, cursor_param=CURSOR_PARAM)+ def __init__(+ self, first_page_only_params: tuple[str, ...] = (), param_location: Literal["query", "json"] = "query"+ ) -> None:+ super().__init__(cursor_path=CURSOR_PATH, cursor_param=CURSOR_PARAM, param_location=param_location)self._first_page_only_params = first_page_only_paramsdef _drop_first_page_only_params(self, request: Request) -> None:@@ -158,7 +159,7 @@ def _paginator_for(config: MetronomeEndpointConfig) -> BasePaginator:if not config.paginated:return SinglePagePaginator()first_page_only: tuple[str, ...] = (config.incremental_start_param,) if config.incremental_start_param else ()- return MetronomeCursorPaginator(first_page_only_params=first_page_only)+ return MetronomeCursorPaginator(first_page_only_params=first_page_only, param_location=config.page_param_location)def _format_rfc3339(value: Any) -> str:@@ -247,6 +248,19 @@ def _resolve_window_start(return _format_rfc3339(_align_to_utc_midnight(start))+def _query_window(history: timedelta, db_incremental_field_last_value: Any) -> dict[str, str]:+ """The `starting_on`/`ending_before` query window of a fan-out child that requires one.++ Both bounds sit on UTC midnight for the same reason as a usage window: a daily breakdown's start+ is part of its primary key, so asking from mid-day would return a partial day as a second row.+ """+ now = datetime.now(UTC)+ ending_before = _format_rfc3339(_usage_window_end(now))+ start = parse_datetime_value(db_incremental_field_last_value) or now - history+ starting_on = _clamp_window_start(_format_rfc3339(_align_to_utc_midnight(start)), ending_before)+ return {"starting_on": starting_on, "ending_before": ending_before}++@frozenclass MetronomeWalkStart:"""Where one walk of an endpoint begins: the request window, and the cursor to resume from."""@@ -348,14 +362,25 @@ def _rest_client(api_key: str) -> RESTClient:)+def _sends_page_size(config: MetronomeEndpointConfig) -> bool:+ return config.paginated and config.accepts_page_size++def _list_params(config: MetronomeEndpointConfig) -> dict[str, Any]:params: dict[str, Any] = {}- if config.paginated and config.accepts_page_size:+ if _sends_page_size(config) and config.page_param_location == "query":params["limit"] = config.page_sizeparams.update(config.extra_params)return params+def _list_body(config: MetronomeEndpointConfig) -> dict[str, Any]:+ body = dict(config.json_body)+ if _sends_page_size(config) and config.page_param_location == "json":+ body["limit"] = config.page_size+ return body++def _retry_after_seconds(response: Response) -> float | None:raw = response.headers.get("Retry-After")if not raw:@@ -704,35 +729,68 @@ def get_resource(return resource-def _body_fanout_pages(client: RESTClient, config: MetronomeEndpointConfig) -> Iterator[list[dict[str, Any]]]:- """Walk the parent list and call the child once per parent row, binding the id into the body."""- fanout = config.body_fanout- if fanout is None:- raise ValueError(f"'{config.name}' has no body fan-out configured")+def _pricing_group_key(pricing_group_values: Any) -> str:+ """A stable string for a rate's pricing group, so the group can be part of the primary key.++ `pricing_group_values` is an object, and a rate with none is a different rate from one scoped+ to a group, so the absent case gets its own value rather than a null.+ """+ return json.dumps(pricing_group_values or {}, sort_keys=True, separators=(",", ":"))- parent = METRONOME_ENDPOINTS[fanout.parent_name]- for parent_page in client.paginate(+def _shape_fanout_row(+ config: MetronomeEndpointConfig, row: dict[str, Any], parent_values: dict[str, Any]+) -> dict[str, Any]:+ for name, value in parent_values.items():+ if row.get(name) is None:+ row[name] = value+ if config.keys_pricing_group:+ row["pricing_group_key"] = _pricing_group_key(row.get("pricing_group_values"))+ return row+++def _parent_pages(client: RESTClient, parent: MetronomeEndpointConfig) -> Iterator[list[dict[str, Any]]]:+ if parent.body_fanout is not None:+ yield from _body_fanout_pages(client, parent)+ return+ yield from client.paginate(parent.path,+ method=parent.method,params=_list_params(parent),+ # The POST lists expect a JSON document even when they take no filters.+ json=_list_body(parent) if parent.method == "post" else None,data_selector=DATA_SELECTOR,data_selector_required=True,paginator=_paginator_for(parent),- ):+ )+++def _body_fanout_pages(client: RESTClient, config: MetronomeEndpointConfig) -> Iterator[list[dict[str, Any]]]:+ """Walk the parent list and call the child once per parent row, binding its ids into the body."""+ fanout = config.body_fanout+ if fanout is None:+ raise ValueError(f"'{config.name}' has no body fan-out configured")++ parent = METRONOME_ENDPOINTS[fanout.parent_name]++ for parent_page in _parent_pages(client, parent):for parent_row in parent_page:- parent_id = parent_row.get(fanout.resolve_field)- if not parent_id:+ if fanout.skip_parents_with and parent_row.get(fanout.skip_parents_with):+ continue+ parent_values = {param: parent_row.get(field) for param, field in fanout.body_params.items()}+ if not all(parent_values.values()):continuefor page in client.paginate(config.path,method=config.method,- json={**config.json_body, fanout.body_param: parent_id},+ params=_list_params(config),+ json={**_list_body(config), **parent_values},data_selector=DATA_SELECTOR,data_selector_required=True,paginator=_paginator_for(config),):if page:- yield page+ yield [_shape_fanout_row(config, row, parent_values) for row in page]def _make_source_response(@@ -779,6 +837,9 @@ def metronome_source(if endpoint_config.fanout:parent_config = METRONOME_ENDPOINTS[endpoint_config.fanout.parent_name]+ child_params = dict(endpoint_config.extra_params)+ if endpoint_config.query_window_history is not None:+ child_params.update(_query_window(endpoint_config.query_window_history, db_incremental_field_last_value))# Dependent resources don't support resume in the rest_source framework, so the manager is# intentionally not threaded into this path.dependent_resource = cast(@@ -786,7 +847,7 @@ def metronome_source(build_dependent_resource(endpoint_configs=METRONOME_ENDPOINTS,child_endpoint=endpoint,- fanout=dataclasses.replace(endpoint_config.fanout, child_params=dict(endpoint_config.extra_params)),+ fanout=dataclasses.replace(endpoint_config.fanout, child_params=child_params),client_config=_rest_api_client_config(api_key),path_format_values={},team_id=team_id,@@ -794,6 +855,9 @@ def metronome_source(db_incremental_field_last_value=db_incremental_field_last_value,should_use_incremental_field=should_use_incremental_field,incremental_field=incremental_field,+ # The window rides `child_params`, so there is no cursor param for the framework to+ # inject. The child still merges on its primary key.+ incremental_config_factory=lambda _field: None,parent_endpoint_extra={"paginator": _paginator_for(parent_config),"data_selector": DATA_SELECTOR,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/settings.pyindex 6318726573bc..94360f268963 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/settings.py@@ -23,6 +23,19 @@ CURSOR_PARAM = "next_page"# `GET /v1/auditLogs` is the only endpoint with a server-side lower bound on record time.AUDIT_LOG_START_PARAM = "starting_on"+# `POST /v1/contracts/customerBalances/list` caps `limit` at 25.+BALANCES_PAGE_SIZE = 25++# Lower bound for a request that asks for a whole schedule — earlier than any Metronome account.+EPOCH_RFC_3339 = "1970-01-01T00:00:00Z"++# How far back a first sync of `invoice_breakdowns` reaches. It holds one row per invoice and day, so+# like the daily usage table it reaches back a year rather than to the start of the account.+INVOICE_BREAKDOWNS_HISTORY = timedelta(days=365)+# Backdated usage changes a breakdown after its day has passed, and a draft invoice keeps changing+# until it finalizes, so each incremental run re-reads this much of what it already covered.+INVOICE_BREAKDOWNS_LOOKBACK_SECONDS = 7 * 24 * 60 * 60+# How `POST /v1/usage` aggregates the period it is asked for: one row per hour, one row per day, or# one row for the whole period.WindowSize = Literal["none", "hour", "day"]@@ -75,16 +88,19 @@ USAGE_HOURLY_LOOKBACK_SECONDS = 24 * 60 * 60@dataclass(frozen=True)class BodyFanoutConfig:- """Fan-out whose parent id belongs in the child's JSON body.+ """Fan-out whose parent ids belong in the child's JSON body.The declarative fan-out helper binds a resolved parent field into the child's *path* only, soa child like `POST /v2/contracts/list` — which takes `customer_id` in its body — walks the- parent itself instead.+ parent itself instead. The parent can itself be a body fan-out, which chains the walk."""parent_name: str- resolve_field: str- body_param: str+ # Child body param -> the parent row field it is read from. Each one is also copied onto the+ # child rows that lack it, because several children never echo back the ids they were asked for.+ body_params: dict[str, str]+ # Parent rows that have this field set are skipped.+ skip_parents_with: str | None = None# frozen=False: MetronomeEndpointConfig is passed as `endpoint_configs: Mapping[str, FanoutEndpointLike]`,@@ -102,6 +118,9 @@ class MetronomeEndpointConfig:extra_params: dict[str, Any] = field(default_factory=dict)# Endpoints whose response carries a `next_page` cursor; the rest return the whole collection.paginated: bool = True+ # Where `limit` and `next_page` go. `POST /v1/contracts/customerBalances/list` reads both from+ # the body and ignores them in the query string.+ page_param_location: Literal["query", "json"] = "query"# Whether the endpoint takes a `limit` page-size parameter. `POST /v1/usage` follows# `next_page` but rejects `limit`, and answers the whole request with a 400 when it is# present. Following the cursor and setting a page size are separate capabilities, so@@ -129,6 +148,11 @@ class MetronomeEndpointConfig:sort_mode: SortMode = "asc"should_sync_default: bool = Truedefault_incremental_lookback_seconds: int | None = None+ # A fan-out child whose query needs a `starting_on`/`ending_before` window. This is how far back+ # a run with no watermark reaches. None means the endpoint sends no window.+ query_window_history: timedelta | None = None+ # Derives `pricing_group_key` on rate schedule rows, whose identity includes an object column.+ keys_pricing_group: bool = False# Customer-scoped children bind the parent's `id` into their path as `customer_id`. Every child@@ -162,6 +186,25 @@ METRONOME_ENDPOINTS: dict[str, MetronomeEndpointConfig] = {page_size=25,fanout=_CUSTOMER_PATH_FANOUT,),+ # One row per invoice and day. Starts off, because a first sync is one walk per customer over a+ # year of days.+ "invoice_breakdowns": MetronomeEndpointConfig(+ name="invoice_breakdowns",+ path="/v1/customers/{customer_id}/invoices/breakdowns",+ primary_key=["id", "breakdown_start_timestamp"],+ partition_key="breakdown_start_timestamp",+ page_size=25,+ # `sort` orders the invoices by issue date, so rows still arrive grouped by customer rather+ # than by period, and the watermark is held back until the walk ends, like `usage_daily`.+ extra_params={"window_size": "DAY", "sort": "date_asc"},+ fanout=_CUSTOMER_PATH_FANOUT,+ incremental_fields=[incremental_field("breakdown_start_timestamp")],+ default_incremental_field="breakdown_start_timestamp",+ default_incremental_lookback_seconds=INVOICE_BREAKDOWNS_LOOKBACK_SECONDS,+ query_window_history=INVOICE_BREAKDOWNS_HISTORY,+ sort_mode="desc",+ should_sync_default=False,+ ),"contracts": MetronomeEndpointConfig(name="contracts",path="/v2/contracts/list",@@ -171,7 +214,42 @@ METRONOME_ENDPOINTS: dict[str, MetronomeEndpointConfig] = {# Returns every contract for the customer in one response — no cursor, no `limit`.paginated=False,json_body={"include_archived": True},- body_fanout=BodyFanoutConfig(parent_name="customers", resolve_field="id", body_param="customer_id"),+ body_fanout=BodyFanoutConfig(parent_name="customers", body_params={"customer_id": "id"}),+ ),+ "balances": MetronomeEndpointConfig(+ name="balances",+ path="/v1/contracts/customerBalances/list",+ method="post",+ # A commit or credit id is a UUID, but the rows come from one walk per customer, so the+ # customer joins the key rather than relying on that.+ primary_key=["customer_id", "id"],+ page_size=BALANCES_PAGE_SIZE,+ page_param_location="json",+ # Ledgers and balances are off by default. Both are what this table exists for.+ json_body={+ "include_archived": True,+ "include_balance": True,+ "include_contract_balances": True,+ "include_ledgers": True,+ },+ body_fanout=BodyFanoutConfig(parent_name="customers", body_params={"customer_id": "id"}),+ ),+ # The rates in effect now on each live contract, with the contract's overrides applied. The+ # endpoint answers for one point in time only, so the table is a current snapshot. It takes one+ # walk per contract, so it starts off.+ "contract_rate_schedules": MetronomeEndpointConfig(+ name="contract_rate_schedules",+ path="/v1/contracts/getContractRateSchedule",+ method="post",+ primary_key=["contract_id", "product_id", "pricing_group_key", "billing_frequency", "starting_at"],+ body_fanout=BodyFanoutConfig(+ parent_name="contracts",+ body_params={"customer_id": "customer_id", "contract_id": "id"},+ # An archived contract bills nothing, so it has no rates in effect to report.+ skip_parents_with="archived_at",+ ),+ keys_pricing_group=True,+ should_sync_default=False,),"products": MetronomeEndpointConfig(name="products",@@ -188,6 +266,18 @@ METRONOME_ENDPOINTS: dict[str, MetronomeEndpointConfig] = {primary_key=["id"],partition_key="created_at",),+ "rate_card_rate_schedules": MetronomeEndpointConfig(+ name="rate_card_rate_schedules",+ path="/v1/contract-pricing/rate-cards/getRateSchedule",+ method="post",+ # A rate carries no id. It is one product's price for one pricing group and billing+ # frequency, from one point in time.+ primary_key=["rate_card_id", "product_id", "pricing_group_key", "billing_frequency", "starting_at"],+ # `starting_at` is required. The epoch asks for every segment, past and scheduled.+ json_body={"starting_at": EPOCH_RFC_3339},+ body_fanout=BodyFanoutConfig(parent_name="rate_cards", body_params={"rate_card_id": "id"}),+ keys_pricing_group=True,+ ),"packages": MetronomeEndpointConfig(name="packages",path="/v1/packages/list",@@ -280,6 +370,9 @@ USAGE_HISTORY: dict[str, timedelta] = {"usage_hourly": USAGE_HOURLY_HISTORY,}+# Tables whose incremental runs re-read periods they already hold, so they merge and never append.+MERGE_ONLY = (*USAGE_HISTORY, "invoice_breakdowns")+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {name: config.incremental_fields for name, config in METRONOME_ENDPOINTS.items()}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/source.pyindex 7095b71a8a91..bb8dbff57ae5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/source.py@@ -32,8 +32,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.metronome.DEFAULT_USAGE_HOURLY_HISTORY_DAYS,ENDPOINTS,INCREMENTAL_FIELDS,+ MERGE_ONLY,METRONOME_ENDPOINTS,- USAGE_HISTORY,usage_history_window,)from products.warehouse_sources.backend.types import ExternalDataSourceType@@ -90,9 +90,9 @@ class MetronomeSource(ResumableSource[MetronomeSourceConfig, MetronomeResumeConfENDPOINTS,INCREMENTAL_FIELDS,names,- # Appending would duplicate every period the lookback re-reads, so a bucketed usage+ # Appending would duplicate every period the lookback re-reads, so a windowed# table has to merge on its primary key.- merge_only=tuple(USAGE_HISTORY),+ merge_only=MERGE_ONLY,should_sync_default={name: METRONOME_ENDPOINTS[name].should_sync_default for name in ENDPOINTS},)for schema in schemas:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/tests/test_metronome.py b/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/tests/test_metronome.pyindex 629eb24f1042..bd00c41d5f23 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/tests/test_metronome.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/metronome/tests/test_metronome.py@@ -32,6 +32,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.metronome.validate_credentials,)from products.warehouse_sources.backend.temporal.data_imports.sources.metronome.settings import (+ INVOICE_BREAKDOWNS_LOOKBACK_SECONDS,METRONOME_ENDPOINTS,USAGE_DAILY_LOOKBACK_SECONDS,USAGE_HOURLY_LOOKBACK_SECONDS,@@ -666,6 +667,157 @@ class TestMetronomeBodyFanout:assert client.paginate.call_count == 1+class TestMetronomeNewFanoutTables:+ def test_contract_rate_schedules_walk_each_live_contract_with_both_ids_in_the_body(self) -> None:+ result = _driver().run(+ "contract_rate_schedules",+ [+ ScriptedResponse(json={"data": [{"id": "cust_1"}], "next_page": None}),+ ScriptedResponse(+ json={+ "data": [+ {"id": "contract_1", "customer_id": "cust_1"},+ # An archived contract bills nothing, so it is never asked for rates.+ {"id": "contract_2", "customer_id": "cust_1", "archived_at": "2026-01-01T00:00:00Z"},+ ]+ }+ ),+ ScriptedResponse(+ json={+ "data": [+ {"rate_card_id": "rc_1", "product_id": "p1", "starting_at": "2026-01-01T00:00:00Z"},+ ],+ "next_page": "cursor-2",+ }+ ),+ ScriptedResponse(+ json={+ "data": [+ {+ "rate_card_id": "rc_1",+ "product_id": "p1",+ "starting_at": "2026-01-01T00:00:00Z",+ "pricing_group_values": {"region": "eu", "cloud": "aws"},+ },+ ],+ "next_page": None,+ }+ ),+ ],+ )++ assert result.raised is None+ assert result.paths == [+ "/v1/customers",+ "/v2/contracts/list",+ "/v1/contracts/getContractRateSchedule",+ "/v1/contracts/getContractRateSchedule",+ ]+ assert [request.json() for request in result.requests[2:]] == [+ {"customer_id": "cust_1", "contract_id": "contract_1"},+ {"customer_id": "cust_1", "contract_id": "contract_1"},+ ]+ assert result.params("next_page")[2:] == [None, "cursor-2"]+ # The rows never echo the contract they were asked for, and the pricing group is an object,+ # so both have to be added for the primary key to identify a row.+ assert [(row["contract_id"], row["customer_id"], row["pricing_group_key"]) for row in result.rows] == [+ ("contract_1", "cust_1", "{}"),+ ("contract_1", "cust_1", '{"cloud":"aws","region":"eu"}'),+ ]++ def test_rate_card_rate_schedules_ask_for_the_whole_schedule_of_each_rate_card(self) -> None:+ result = _driver().run(+ "rate_card_rate_schedules",+ [+ ScriptedResponse(json={"data": [{"id": "rc_1"}, {"id": "rc_2"}], "next_page": None}),+ ScriptedResponse(json={"data": [{"product_id": "p1", "starting_at": "2026-01-01T00:00:00Z"}]}),+ ScriptedResponse(json={"data": []}),+ ],+ )++ assert result.raised is None+ assert result.paths == [+ "/v1/contract-pricing/rate-cards/list",+ "/v1/contract-pricing/rate-cards/getRateSchedule",+ "/v1/contract-pricing/rate-cards/getRateSchedule",+ ]+ # The rate card list is a POST that still expects a JSON document.+ assert result.requests[0].json() == {}+ assert [request.json() for request in result.requests[1:]] == [+ {"starting_at": "1970-01-01T00:00:00Z", "rate_card_id": "rc_1"},+ {"starting_at": "1970-01-01T00:00:00Z", "rate_card_id": "rc_2"},+ ]+ assert result.params("limit")[1:] == ["100", "100"]+ assert [(row["rate_card_id"], row["product_id"]) for row in result.rows] == [("rc_1", "p1")]++ def test_balances_page_through_the_body_and_keep_the_customer(self) -> None:+ # The endpoint reads `limit` and `next_page` from the body only. A cursor sent in the query+ # string would be ignored, and the walk would ask for the first page again forever.+ result = _driver().run(+ "balances",+ [+ ScriptedResponse(json={"data": [{"id": "cust_1"}], "next_page": None}),+ ScriptedResponse(json={"data": [{"id": "commit_1", "type": "PREPAID"}], "next_page": "cursor-2"}),+ ScriptedResponse(json={"data": [{"id": "credit_1", "type": "CREDIT"}], "next_page": None}),+ ],+ )++ assert result.raised is None+ bodies = [request.json() for request in result.requests[1:]]+ assert [body.get("next_page") for body in bodies] == [None, "cursor-2"]+ assert {body["limit"] for body in bodies} == {25}+ assert {body["customer_id"] for body in bodies} == {"cust_1"}+ assert all(body["include_ledgers"] and body["include_balance"] for body in bodies)+ assert result.params("limit")[1:] == [None, None]+ assert result.params("next_page")[1:] == [None, None]+ assert [(row["customer_id"], row["id"]) for row in result.rows] == [+ ("cust_1", "commit_1"),+ ("cust_1", "credit_1"),+ ]++ @parameterized.expand(+ [+ # A first sync reaches back a year, from the UTC midnight before it.+ ("first_sync", {}, "2025-09-03T00:00:00Z"),+ # The watermark arrives already moved back by the lookback, and lands mid-day. Asking+ # from mid-day would return a partial day as a second row under a different key.+ (+ "incremental",+ {+ "should_use_incremental_field": True,+ "incremental_field": "breakdown_start_timestamp",+ "db_incremental_field_last_value": "2026-08-27T15:00:00Z",+ },+ "2026-08-27T00:00:00Z",+ ),+ ]+ )+ def test_invoice_breakdowns_ask_for_whole_days_from_the_watermark(self, _name, inputs, starting_on) -> None:+ with time_machine.travel(NOW, tick=False):+ result = _driver().run(+ "invoice_breakdowns",+ [+ ScriptedResponse(json={"data": [{"id": "cust_1"}], "next_page": None}),+ ScriptedResponse(+ json={+ "data": [{"id": "inv_1", "breakdown_start_timestamp": "2026-09-02T00:00:00Z"}],+ "next_page": None,+ }+ ),+ ],+ **inputs,+ )++ assert result.raised is None+ assert result.paths == ["/v1/customers", "/v1/customers/cust_1/invoices/breakdowns"]+ breakdown_query = result.queries[1]+ assert breakdown_query["starting_on"] == (starting_on,)+ # The day in progress is included, so the newest day is never a full period behind.+ assert breakdown_query["ending_before"] == ("2026-09-04T00:00:00Z",)+ assert breakdown_query["window_size"] == ("DAY",)+ assert [row["id"] for row in result.rows] == ["inv_1"]++class TestMetronomeCredentials:@parameterized.expand([@@ -711,17 +863,18 @@ class TestMetronomeSchemas:@parameterized.expand([- ("usage_daily", USAGE_DAILY_LOOKBACK_SECONDS),- ("usage_hourly", USAGE_HOURLY_LOOKBACK_SECONDS),+ ("usage_daily", USAGE_DAILY_LOOKBACK_SECONDS, "start_timestamp"),+ ("usage_hourly", USAGE_HOURLY_LOOKBACK_SECONDS, "start_timestamp"),+ ("invoice_breakdowns", INVOICE_BREAKDOWNS_LOOKBACK_SECONDS, "breakdown_start_timestamp"),])- def test_bucketed_usage_merges_incrementally_and_starts_off(self, name, lookback_seconds) -> None:+ def test_windowed_tables_merge_incrementally_and_start_off(self, name, lookback_seconds, cursor_field) -> None:schema = self._schema(name)assert schema.supports_incremental is True# Appending would duplicate every period the lookback re-reads.assert schema.supports_append is False- assert [field["field"] for field in schema.incremental_fields] == ["start_timestamp"]+ assert [field["field"] for field in schema.incremental_fields] == [cursor_field]assert schema.should_sync_default is Falseassert schema.default_incremental_lookback_seconds == lookback_seconds