Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add plivo numbers, multiparty call and usage tables (#126280)

Tom Owers36bb17511de3
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 7221d647014c..dca0c5c37a07 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9594,10 +9594,10 @@ Today (4): `applications`, `calls`, `messages`, `recordings`
Diffed against: <https://www.plivo.com/docs/numbers/phone-numbers/>
-- [ ] `GET /v1/Account/{auth_id}/PhoneNumber/` — lookup resolving the rented numbers that appear as from/to on every synced message and call (high)
-- [ ] `GET /v1/Account/{auth_id}/UsageSummary/` — billed usage and spend per period, the headline cost metric (high)
-- [ ] `GET /v1/Account/{auth_id}/MultiPartyCall/` — conference-style calls that never appear in the flat calls table (high)
-- [ ] `GET /v1/Account/{auth_id}/MultiPartyCall/name_{mpc_name}/Participant/` — per-leg participant records — join, exit and duration per member (high)
+- [x] `GET /v1/Account/{auth_id}/PhoneNumber/` (synced as `numbers` from `GET /v1/Account/{auth_id}/Number/`; `PhoneNumber/` searches numbers available to buy) — lookup resolving the rented numbers that appear as from/to on every synced message and call (high)
+- [x] `GET /v1/Account/{auth_id}/UsageSummary/` — billed usage and spend per period, the headline cost metric (high)
+- [x] `GET /v1/Account/{auth_id}/MultiPartyCall/` — conference-style calls that never appear in the flat calls table (high)
+- [x] `GET /v1/Account/{auth_id}/MultiPartyCall/name_{mpc_name}/Participant/` (fetched via `uuid_{mpc_uuid}`, which also serves ended calls) — per-leg participant records — join, exit and duration per member (high)
- [ ] `GET /v1/Account/{auth_id}/Subaccount/` — lookup for attributing message and call traffic to the right subaccount (medium)
- [ ] `GET /v1/Account/{auth_id}/Verify/Session/` — OTP verification sessions, needed to measure verification funnel completion (medium)
- [ ] `GET /v1/Account/{auth_id}/Zentrunk/Call/` — SIP trunking CDRs, a call stream entirely absent from the calls table (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/canonical_descriptions.py
index 2fdcbcc9eb34..577fc1a3a2e9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/canonical_descriptions.py
@@ -82,4 +82,83 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"resource_uri": "URI of the application resource on the Plivo API.",
},
},
+ "numbers": {
+ "description": "A phone number rented on your Plivo account.",
+ "docs_url": "https://www.plivo.com/docs/numbers/account-phone-numbers",
+ "columns": {
+ "number": "The phone number in E.164 format, without the leading +.",
+ "alias": "Friendly name given to the number.",
+ "sub_account": "Subaccount the number is assigned to, if any.",
+ "added_on": "Date on which the number was rented.",
+ "application": "URI of the application linked to the number.",
+ "carrier": "Carrier that provides the number.",
+ "region": "Geographic region of the number.",
+ "number_type": "Type of the number: local, mobile, fixed, national, or tollfree.",
+ "monthly_rental_rate": "Monthly rental charged for the number, in your account currency.",
+ "renewal_date": "Date on which the number's rental next renews.",
+ "sms_enabled": "Whether the number can send and receive SMS.",
+ "mms_enabled": "Whether the number can send and receive MMS.",
+ "voice_enabled": "Whether the number can make and receive voice calls.",
+ "resource_uri": "URI of the number resource on the Plivo API.",
+ },
+ },
+ "multi_party_calls": {
+ "description": "A multiparty call (MPC): a conference-style call that several participants join.",
+ "docs_url": "https://www.plivo.com/docs/voice/api/multiparty-calls",
+ "columns": {
+ "mpc_uuid": "Unique identifier of the multiparty call.",
+ "friendly_name": "Name given to the multiparty call.",
+ "status": "Status of the multiparty call: initialized, active, or ended.",
+ "creation_time": "Time at which the multiparty call was created.",
+ "start_time": "Time at which the multiparty call started.",
+ "end_time": "Time at which the multiparty call ended.",
+ "duration": "Duration of the multiparty call, in seconds.",
+ "billed_duration": "Duration billed for the multiparty call, in seconds.",
+ "billed_amount": "Amount charged for the multiparty call, in your account currency.",
+ "termination_cause": "Reason the multiparty call ended.",
+ "termination_cause_code": "Numeric code of the reason the multiparty call ended.",
+ "sub_account": "Subaccount the multiparty call belongs to, if any.",
+ },
+ },
+ "multi_party_call_participants": {
+ "description": "A participant (call leg) in a multiparty call.",
+ "docs_url": "https://www.plivo.com/docs/voice/api/multiparty-calls",
+ "columns": {
+ "mpc_uuid": "Identifier of the multiparty call the participant belongs to.",
+ "member_id": "Identifier of the participant, unique within its multiparty call.",
+ "call_uuid": "Identifier of the participant's call leg.",
+ "role": "Role of the participant: agent, customer, or supervisor.",
+ "coach_mode": "Whether the participant is a supervisor in coach mode.",
+ "hold": "Whether the participant is on hold.",
+ "mute": "Whether the participant is muted.",
+ "join_time": "Time at which the participant joined the multiparty call.",
+ "exit_time": "Time at which the participant left the multiparty call.",
+ "exit_cause": "Reason the participant left the multiparty call.",
+ "duration": "Time the participant spent in the multiparty call, in seconds.",
+ "billed_amount": "Amount charged for the participant's leg, in your account currency.",
+ "mpc_end_time": "Time at which the parent multiparty call ended. Used as the incremental sync cursor.",
+ },
+ },
+ "usage_summary": {
+ "description": "Billed usage and other charges for one day, with usage broken down by product, type, country and direction.",
+ "docs_url": "https://www.plivo.com/docs/account/api/usage-summary",
+ "columns": {
+ "id": "Identifier PostHog derives from the row's dimensions (date, product, type, country, direction and so on).",
+ "from_date": "Start of the day the usage covers (inclusive).",
+ "to_date": "End of the day the usage covers (exclusive).",
+ "product": "Plivo product the usage is for: message, voice, zentrunk, or transcription.",
+ "type": "Usage type within the product.",
+ "country": "ISO 3166-1 alpha-2 code of the country the usage is attributed to.",
+ "direction": "Direction of the traffic: inbound or outbound.",
+ "subaccount_auth": "Auth ID of the subaccount the usage belongs to, if any.",
+ "total_units": "Number of billed units (messages, calls, and so on).",
+ "duration_seconds": "Total billed duration in seconds, for voice, SIP trunking and transcription usage.",
+ "total_amount": "Total amount charged, in your account currency.",
+ "surcharge": "Surcharge included in the total amount, if any.",
+ "error_code": "Messaging delivery error code the usage is grouped by, if any.",
+ "error_reason": "Description of the messaging delivery error code, if any.",
+ "charge_type": "Kind of row: usage for traffic usage, or other_charge for rentals, fees, taxes and adjustments.",
+ "description": "Description of the charge, for other_charge rows.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/plivo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/plivo.py
index 5b518a6959ba..a2c651280a2b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/plivo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/plivo.py
@@ -1,3 +1,5 @@
+import json
+import hashlib
import dataclasses
from collections.abc import Callable, Iterator
from datetime import UTC, date, datetime, timedelta
@@ -7,8 +9,13 @@ from dateutil import parser as dateutil_parser
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import HttpBasicAuth
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.config_setup import (
+ create_response_hooks,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
+ JSONResponseCursorPaginator,
OffsetPaginator,
+ SinglePagePaginator,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClient
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
@@ -16,15 +23,22 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.sou
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.plivo.settings import (
MAX_QUERY_RANGE_DAYS,
+ MULTI_PARTY_CALL_PARTICIPANTS,
+ MULTI_PARTY_CALLS,
PAGE_SIZE,
PLIVO_BASE_URL,
PLIVO_ENDPOINTS,
RETENTION_DAYS,
+ USAGE_BACKFILL_DAYS,
+ USAGE_MAX_QUERY_RANGE_DAYS,
+ USAGE_METRIC_FIELDS,
+ USAGE_PAGE_SIZE,
+ USAGE_SUMMARY,
PlivoEndpointConfig,
)
[email protected](frozen=True)
class PlivoResumeConfig:
# ISO start/end of the <=30-day window being paged. Both None for non-windowed endpoints.
# window_end pins the resumed window's upper bound so re-deriving it against a later "now"
@@ -33,6 +47,10 @@ class PlivoResumeConfig:
window_end: str | None = None
# Next page offset within the current window/listing.
offset: int = 0
+ # Next page cursor within the current window, for cursor-paginated endpoints (UsageSummary).
+ page_token: str | None = None
+ # Index of the next parent row to fan out within the parent page at `offset`.
+ parent_index: int = 0
def _coerce_datetime(value: Any) -> datetime:
@@ -75,6 +93,13 @@ def _normalize_row(row: dict[str, Any], config: PlivoEndpointConfig) -> dict[str
return row
+def _usage_row_id(row: dict[str, Any]) -> str:
+ """Deterministic id for a usage row. UsageSummary rows have no id, and optional dimensions
+ (subaccount, error code) can be absent, so a composite primary key could hold nulls."""
+ dimensions = {key: value for key, value in row.items() if key not in USAGE_METRIC_FIELDS}
+ return hashlib.sha256(json.dumps(dimensions, sort_keys=True, default=str).encode()).hexdigest()
+
+
def _make_client(auth_id: str, auth_token: str) -> RESTClient:
return RESTClient(base_url=PLIVO_BASE_URL, auth=HttpBasicAuth(auth_id, auth_token))
@@ -95,7 +120,7 @@ def _paginate(
paginator = OffsetPaginator(
limit=PAGE_SIZE,
offset=initial_offset,
- total_path="meta.total_count",
+ total_path=config.total_path,
)
def resume_hook(state: Optional[dict[str, Any]]) -> None:
@@ -111,6 +136,133 @@ def _paginate(
yield [_normalize_row(row, config) for row in page]
+def _get_participant_rows(
+ client: RESTClient,
+ auth_id: str,
+ parent_params: dict[str, Any],
+ resume: PlivoResumeConfig | None,
+ resumable_source_manager: ResumableSourceManager[PlivoResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ parent = PLIVO_ENDPOINTS[MULTI_PARTY_CALLS]
+ child = PLIVO_ENDPOINTS[MULTI_PARTY_CALL_PARTICIPANTS]
+ # An MPC listed by the parent can have no participant listing any more; skip it rather than fail the run.
+ child_hooks = create_response_hooks(
+ [{"status_code": 404, "action": "ignore"}], resource_name=MULTI_PARTY_CALL_PARTICIPANTS
+ )
+
+ parent_offset = resume.offset if resume else 0
+ skip_parents = resume.parent_index if resume else 0
+ parent_pages = client.paginate(
+ path=f"Account/{auth_id}/{parent.path}",
+ params=parent_params,
+ paginator=OffsetPaginator(limit=PAGE_SIZE, offset=parent_offset, total_path=parent.total_path),
+ data_selector="objects",
+ )
+ for mpcs in parent_pages:
+ for index, raw_mpc in enumerate(mpcs):
+ if index < skip_parents:
+ continue
+ mpc = _normalize_row(raw_mpc, parent)
+ mpc_uuid = mpc["mpc_uuid"]
+ # Stage the position after this MPC before yielding its participants, so a resume
+ # starts at the next MPC and never writes these participants twice.
+ resumable_source_manager.save_state(PlivoResumeConfig(offset=parent_offset, parent_index=index + 1))
+ child_path = f"Account/{auth_id}/{child.path}".replace("{mpc_uuid}", mpc_uuid)
+ participants = [
+ {**participant, "mpc_uuid": mpc_uuid, "mpc_end_time": mpc.get("end_time")}
+ for page in client.paginate(
+ path=child_path,
+ # The participant listing takes no limit or offset and returns every participant.
+ paginator=SinglePagePaginator(),
+ data_selector="objects",
+ hooks=child_hooks,
+ )
+ for participant in page
+ ]
+ if participants:
+ yield [_normalize_row(participant, child) for participant in participants]
+ else:
+ resumable_source_manager.safe_point()
+ skip_parents = 0
+ parent_offset += PAGE_SIZE
+
+
+def _get_usage_rows(
+ client: RESTClient,
+ path: str,
+ resume: PlivoResumeConfig | None,
+ resumable_source_manager: ResumableSourceManager[PlivoResumeConfig],
+ last_value: Any,
+ now: datetime,
+) -> Iterator[list[dict[str, Any]]]:
+ config = PLIVO_ENDPOINTS[USAGE_SUMMARY]
+ # to_date is exclusive, so ending at today's date syncs complete days only.
+ end = datetime.combine(now.date(), datetime.min.time(), tzinfo=UTC)
+ if resume is not None and resume.window_start:
+ start = _coerce_datetime(resume.window_start)
+ elif last_value is not None:
+ # from_date is inclusive: re-read the watermark day so a late restatement merges in place.
+ watermark = _coerce_datetime(last_value)
+ start = datetime.combine(watermark.date(), datetime.min.time(), tzinfo=UTC)
+ else:
+ start = end - timedelta(days=USAGE_BACKFILL_DAYS)
+
+ # Pin the resumed window's saved upper bound so its page token stays valid.
+ resumed_end = _coerce_datetime(resume.window_end) if resume is not None and resume.window_end else None
+ window_start = start
+ index = 0
+ while window_start < end:
+ if index == 0 and resumed_end is not None:
+ window_end = resumed_end
+ else:
+ window_end = min(window_start + timedelta(days=USAGE_MAX_QUERY_RANGE_DAYS), end)
+ params = {
+ "granularity": "day",
+ "from_date": window_start.strftime("%Y-%m-%d"),
+ "to_date": window_end.strftime("%Y-%m-%d"),
+ "page_size": USAGE_PAGE_SIZE,
+ }
+ resumed_token = resume.page_token if (index == 0 and resume is not None and resume.window_start) else None
+
+ window_start_iso = window_start.isoformat()
+ window_end_iso = window_end.isoformat()
+
+ def window_checkpoint(
+ state: Optional[dict[str, Any]],
+ _has_next_page: bool,
+ window_start_iso: str = window_start_iso,
+ window_end_iso: str = window_end_iso,
+ ) -> None:
+ if state:
+ resumable_source_manager.save_state(
+ PlivoResumeConfig(
+ window_start=window_start_iso, window_end=window_end_iso, page_token=state["cursor"]
+ )
+ )
+ else:
+ resumable_source_manager.save_state(PlivoResumeConfig(window_start=window_end_iso))
+
+ for page in client.paginate(
+ path=path,
+ params=params,
+ paginator=JSONResponseCursorPaginator(
+ cursor_path="meta.next_page_token", cursor_param="page_token", raise_on_repeated_cursor=True
+ ),
+ # Staged before the page is yielded, so a hand-off after the write resumes past it.
+ page_state_hook=window_checkpoint,
+ initial_paginator_state={"cursor": resumed_token} if resumed_token else None,
+ ):
+ rows: list[dict[str, Any]] = []
+ for body in page:
+ rows.extend({**row, "charge_type": "usage"} for row in body.get("usage") or [])
+ # Rentals, fees, taxes and adjustments arrive only on a window's first page.
+ rows.extend({**charge, "charge_type": "other_charge"} for charge in body.get("other_charges") or [])
+ yield [_normalize_row({**row, "id": _usage_row_id(row)}, config) for row in rows]
+
+ window_start = window_end
+ index += 1
+
+
def get_rows(
auth_id: str,
auth_token: str,
@@ -129,6 +281,20 @@ def get_rows(
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ if endpoint == USAGE_SUMMARY:
+ last_value = db_incremental_field_last_value if should_use_incremental_field else None
+ yield from _get_usage_rows(client, path, resume, resumable_source_manager, last_value, now)
+ return
+
+ if endpoint == MULTI_PARTY_CALL_PARTICIPANTS:
+ parent_params: dict[str, Any] = {}
+ if should_use_incremental_field and db_incremental_field_last_value is not None and config.time_filter_field:
+ parent_params[f"{config.time_filter_field}__gt"] = _format_filter_value(
+ _coerce_datetime(db_incremental_field_last_value)
+ )
+ yield from _get_participant_rows(client, auth_id, parent_params, resume, resumable_source_manager)
+ return
+
if not config.windowed:
params: dict[str, Any] = {}
if should_use_incremental_field and db_incremental_field_last_value is not None and filter_field:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/settings.py
index 17c93813fe41..e36b79f0aa4b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/settings.py
@@ -14,8 +14,20 @@ PAGE_SIZE = 20
RETENTION_DAYS = 90
MAX_QUERY_RANGE_DAYS = 30
+MULTI_PARTY_CALLS = "multi_party_calls"
+MULTI_PARTY_CALL_PARTICIPANTS = "multi_party_call_participants"
+USAGE_SUMMARY = "usage_summary"
-@dataclass
+# UsageSummary pages with a cursor and accepts up to 1000 rows per page. With `day` granularity a
+# single request may span at most 92 days, so the backfill is chunked into windows of that size.
+USAGE_PAGE_SIZE = 1000
+USAGE_MAX_QUERY_RANGE_DAYS = 92
+USAGE_BACKFILL_DAYS = 365
+# Usage figures, not dimensions: a restated day keeps its row id so merge updates it in place.
+USAGE_METRIC_FIELDS = frozenset({"total_units", "duration_seconds", "total_amount", "surcharge"})
+
+
+@dataclass(frozen=True)
class PlivoEndpointConfig:
name: str
# Path relative to /v1/Account/{auth_id}/, e.g. "Message/".
@@ -30,6 +42,9 @@ class PlivoEndpointConfig:
datetime_fields: tuple[str, ...] = ()
# Whether requests must be chunked into MAX_QUERY_RANGE_DAYS windows over RETENTION_DAYS.
windowed: bool = False
+ # Response path of the grand total that ends offset pagination. Listings whose `meta.count`
+ # meaning is undocumented set None and stop on a short page instead.
+ total_path: str | None = "meta.total_count"
PLIVO_ENDPOINTS: dict[str, PlivoEndpointConfig] = {
@@ -73,6 +88,46 @@ PLIVO_ENDPOINTS: dict[str, PlivoEndpointConfig] = {
primary_keys=["app_id"],
# The application list exposes no server-side time filter — full refresh only.
),
+ "numbers": PlivoEndpointConfig(
+ name="numbers",
+ # Rented numbers. `PhoneNumber/` is the search for numbers available to buy, not this list.
+ path="Number/",
+ primary_keys=["number"],
+ # The number list only filters by renewal_date, which changes on every renewal — full refresh only.
+ ),
+ MULTI_PARTY_CALLS: PlivoEndpointConfig(
+ name=MULTI_PARTY_CALLS,
+ path="MultiPartyCall/",
+ primary_keys=["mpc_uuid"],
+ incremental_fields=[incremental_field("end_time")],
+ time_filter_field="end_time",
+ partition_key="creation_time",
+ datetime_fields=("creation_time", "start_time", "end_time"),
+ total_path=None,
+ ),
+ MULTI_PARTY_CALL_PARTICIPANTS: PlivoEndpointConfig(
+ name=MULTI_PARTY_CALL_PARTICIPANTS,
+ # Fanned out per MPC. Plivo serves participants by MPC name only while the call is ongoing,
+ # so the listing is addressed by UUID, which also works for ended calls.
+ path="MultiPartyCall/uuid_{mpc_uuid}/Participant/",
+ # member_id is only unique within its MPC.
+ primary_keys=["mpc_uuid", "member_id"],
+ # The participant listing has no time filter, so incremental syncs filter the parent MPC
+ # list by end_time and stamp it onto each participant as mpc_end_time.
+ incremental_fields=[incremental_field("mpc_end_time")],
+ time_filter_field="end_time",
+ datetime_fields=("join_time", "exit_time"),
+ total_path=None,
+ ),
+ USAGE_SUMMARY: PlivoEndpointConfig(
+ name=USAGE_SUMMARY,
+ path="UsageSummary/",
+ primary_keys=["id"],
+ incremental_fields=[incremental_field("from_date")],
+ time_filter_field="from_date",
+ partition_key="from_date",
+ datetime_fields=("from_date", "to_date"),
+ ),
}
ENDPOINTS = tuple(PLIVO_ENDPOINTS.keys())
@@ -92,4 +147,11 @@ ENDPOINT_DESCRIPTIONS: dict[str, str] = {
),
"recordings": "Call and conference recordings stored on your Plivo account.",
"applications": "Plivo applications that control how calls and messages to your numbers are handled.",
+ "numbers": "Phone numbers rented on your Plivo account, with their capabilities, rates and linked application.",
+ MULTI_PARTY_CALLS: "Multiparty calls (MPCs): conference-style calls with several participants.",
+ MULTI_PARTY_CALL_PARTICIPANTS: "Participants of each multiparty call, with their role and join and exit details.",
+ USAGE_SUMMARY: (
+ "Daily billed usage per product, type, country and direction, plus other charges such as number rentals. "
+ "The first sync backfills the last 365 days."
+ ),
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/source.py
index 390971d85d6f..c10c5ea380e3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/source.py
@@ -30,6 +30,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.plivo.sett
ENDPOINTS,
INCREMENTAL_FIELDS,
PLIVO_BASE_URL,
+ USAGE_SUMMARY,
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
@@ -129,6 +130,8 @@ class PlivoSource(ResumableSource[PlivoSourceConfig, PlivoResumeConfig]):
INCREMENTAL_FIELDS,
names,
descriptions=ENDPOINT_DESCRIPTIONS,
+ # Each sync re-reads the watermark day, which append would write twice.
+ merge_only=(USAGE_SUMMARY,),
)
def validate_credentials(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/tests/test_plivo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/tests/test_plivo.py
index 23cc343409b3..685a9a802afe 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/tests/test_plivo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/tests/test_plivo.py
@@ -31,6 +31,38 @@ def _page(objects: list[dict[str, Any]], total_count: int) -> ScriptedResponse:
return ScriptedResponse(json={"meta": {"limit": PAGE_SIZE, "total_count": total_count}, "objects": objects})
+def _count_page(objects: list[dict[str, Any]]) -> ScriptedResponse:
+ return ScriptedResponse(json={"meta": {"count": len(objects), "limit": PAGE_SIZE, "offset": 0}, "objects": objects})
+
+
+def _usage_page(
+ usage: list[dict[str, Any]],
+ next_page_token: str | None = None,
+ other_charges: list[dict[str, Any]] | None = None,
+) -> ScriptedResponse:
+ meta: dict[str, Any] = {"granularity": "day", "page_size": 1000}
+ if next_page_token:
+ meta["next_page_token"] = next_page_token
+ body: dict[str, Any] = {"meta": meta, "usage": usage}
+ if other_charges is not None:
+ body["other_charges"] = other_charges
+ return ScriptedResponse(json=body)
+
+
+def _usage_row(**overrides: Any) -> dict[str, Any]:
+ return {
+ "from_date": "2026-07-19T00:00:00Z",
+ "to_date": "2026-07-20T00:00:00Z",
+ "product": "message",
+ "type": "sms",
+ "country": "US",
+ "direction": "outbound",
+ "total_units": 10,
+ "total_amount": 0.05,
+ **overrides,
+ }
+
+
class TestWindowedEndpoints:
@pytest.fixture(autouse=True)
def _frozen_clock(self) -> Iterator[None]:
@@ -132,6 +164,145 @@ class TestNonWindowedEndpoints:
assert result.items[0][0]["add_time"] == ""
+class TestUsageSummary:
+ @pytest.fixture(autouse=True)
+ def _frozen_clock(self) -> Iterator[None]:
+ with time_machine.travel(FROZEN_NOW, tick=False):
+ yield
+
+ @pytest.mark.parametrize(
+ "inputs, expected_windows",
+ [
+ (
+ {},
+ [
+ ("2025-07-21", "2025-10-21"),
+ ("2025-10-21", "2026-01-21"),
+ ("2026-01-21", "2026-04-23"),
+ ("2026-04-23", "2026-07-21"),
+ ],
+ ),
+ (
+ # The watermark day is re-read so a restated total merges in place; today is excluded
+ # because to_date is exclusive and today's usage is still incomplete.
+ {
+ "incremental_field": "from_date",
+ "db_incremental_field_last_value": datetime(2026, 7, 19, tzinfo=UTC),
+ },
+ [("2026-07-19", "2026-07-21")],
+ ),
+ ],
+ )
+ def test_windows_requests_by_day(self, inputs: dict[str, Any], expected_windows: list[tuple[str, str]]) -> None:
+ result = _driver().run("usage_summary", [_usage_page([]) for _ in expected_windows], **inputs)
+
+ assert result.raised is None
+ assert result.paths[0] == "/v1/Account/MA123/UsageSummary/"
+ assert list(zip(result.params("from_date"), result.params("to_date"), strict=True)) == expected_windows
+ assert set(result.params("granularity")) == {"day"}
+
+ def test_follows_page_token_within_window(self) -> None:
+ rental = {
+ "from_date": "2026-07-19T00:00:00Z",
+ "to_date": "2026-07-20T00:00:00Z",
+ "description": "Number rental",
+ "total_units": 1,
+ "total_amount": 0.8,
+ }
+ result = _driver().run(
+ "usage_summary",
+ [
+ _usage_page([_usage_row()], next_page_token="tok-2", other_charges=[rental]),
+ _usage_page([_usage_row(type="mms")]),
+ ],
+ incremental_field="from_date",
+ db_incremental_field_last_value=datetime(2026, 7, 19, tzinfo=UTC),
+ )
+
+ assert result.raised is None
+ assert result.params("page_token") == [None, "tok-2"]
+ assert result.params("from_date") == ["2026-07-19", "2026-07-19"]
+ assert [(row["charge_type"], row.get("type"), row.get("description")) for row in result.rows] == [
+ ("usage", "sms", None),
+ ("other_charge", None, "Number rental"),
+ ("usage", "mms", None),
+ ]
+ assert len({row["id"] for row in result.rows}) == 3
+ assert result.rows[0]["from_date"] == datetime(2026, 7, 19, tzinfo=UTC)
+
+ def test_row_id_tracks_dimensions_not_metrics(self) -> None:
+ result = _driver().run(
+ "usage_summary",
+ [
+ _usage_page(
+ [
+ _usage_row(),
+ _usage_row(total_units=12, total_amount=0.06),
+ _usage_row(subaccount_auth="SA1"),
+ _usage_row(direction="inbound"),
+ ]
+ )
+ ],
+ incremental_field="from_date",
+ db_incremental_field_last_value=datetime(2026, 7, 19, tzinfo=UTC),
+ )
+
+ ids = [row["id"] for row in result.rows]
+ # A restated metric keeps the row's id so merge updates it rather than duplicating it.
+ assert ids[0] == ids[1]
+ assert len({ids[0], ids[2], ids[3]}) == 3
+
+
+class TestMultiPartyCallParticipants:
+ def test_fans_out_per_mpc_by_uuid(self) -> None:
+ result = _driver().run(
+ "multi_party_call_participants",
+ [
+ _count_page(
+ [
+ {"mpc_uuid": "mpc-1", "end_time": "2026-07-15 10:00:00+00:00"},
+ {"mpc_uuid": "mpc-gone", "end_time": "2026-07-15 11:00:00+00:00"},
+ ]
+ ),
+ _count_page([{"member_id": "1", "join_time": "2026-07-15 09:00:00+00:00"}, {"member_id": "2"}]),
+ ScriptedResponse(status=404, json={"error": "not found"}),
+ ],
+ incremental_field="mpc_end_time",
+ db_incremental_field_last_value=datetime(2026, 7, 11, 12, tzinfo=UTC),
+ )
+
+ assert result.raised is None
+ assert result.paths == [
+ "/v1/Account/MA123/MultiPartyCall/",
+ "/v1/Account/MA123/MultiPartyCall/uuid_mpc-1/Participant/",
+ "/v1/Account/MA123/MultiPartyCall/uuid_mpc-gone/Participant/",
+ ]
+ # The participant listing is fetched once per MPC, without offset pagination.
+ assert result.params("offset") == ["0", None, None]
+ # The participant list has no time filter, so the incremental cursor narrows the parent MPCs.
+ assert result.params("end_time__gt") == ["2026-07-11 12:00:00", None, None]
+ assert [(row["mpc_uuid"], row["member_id"]) for row in result.rows] == [("mpc-1", "1"), ("mpc-1", "2")]
+ assert {row["mpc_end_time"] for row in result.rows} == {datetime(2026, 7, 15, 10, tzinfo=UTC)}
+ assert result.rows[0]["join_time"] == datetime(2026, 7, 15, 9, tzinfo=UTC)
+
+ def test_resume_skips_mpcs_already_fanned_out(self) -> None:
+ result = _driver().run(
+ "multi_party_call_participants",
+ [
+ _count_page([{"mpc_uuid": "mpc-done"}, {"mpc_uuid": "mpc-next"}]),
+ _count_page([{"member_id": "1"}]),
+ ],
+ resume_state=PlivoResumeConfig(offset=PAGE_SIZE, parent_index=1),
+ )
+
+ assert result.raised is None
+ assert result.params("offset")[0] == str(PAGE_SIZE)
+ assert result.paths[1:] == ["/v1/Account/MA123/MultiPartyCall/uuid_mpc-next/Participant/"]
+ # The position after an MPC is staged before its participants are yielded, so a later
+ # resume never writes them twice.
+ assert result.committed_states == [PlivoResumeConfig(offset=PAGE_SIZE, parent_index=2)]
+
+
class TestValidateCredentials:
@pytest.mark.parametrize(
"status_code, expected",
@@ -154,13 +325,16 @@ class TestPlivoSourceResponse:
("calls", ["call_uuid"], ["end_time"]),
("recordings", ["recording_id"], ["add_time"]),
("applications", ["app_id"], None),
+ ("numbers", ["number"], None),
+ ("multi_party_calls", ["mpc_uuid"], ["creation_time"]),
+ ("usage_summary", ["id"], ["from_date"]),
],
)
def test_source_response_shape(
self, endpoint: str, primary_keys: list[str], partition_keys: list[str] | None
) -> None:
with time_machine.travel(FROZEN_NOW, tick=False):
- result = _driver().run(endpoint, [_page([], 0) for _ in range(3)])
+ result = _driver().run(endpoint, [_page([], 0) for _ in range(4)])
assert result.raised is None
response = result.response
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/tests/test_plivo_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/tests/test_plivo_source.py
index 886c3b2ab531..1707e3b84d70 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/tests/test_plivo_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/tests/test_plivo_source.py
@@ -41,3 +41,10 @@ class TestPlivoSource:
# A stale cursor must not narrow a full refresh to a partial window.
assert mock_plivo_source.call_args.kwargs["db_incremental_field_last_value"] is None
+
+ def test_usage_summary_is_merge_only(self):
+ schemas = {schema.name: schema for schema in self.source.get_schemas(self.config, self.team_id)}
+
+ assert schemas["usage_summary"].supports_incremental
+ assert not schemas["usage_summary"].supports_append
+ assert schemas["messages"].supports_append