:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add plivo numbers, multiparty call and usage tables (#126280)
Tom Owers36bb17511de3
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 7221d647014c..dca0c5c37a07 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -9594,10 +9594,10 @@ Today (4): `applications`, `calls`, `messages`, `recordings`Diffed against: <https://www.plivo.com/docs/numbers/phone-numbers/>-- [ ] `GET /v1/Account/{auth_id}/PhoneNumber/` — lookup resolving the rented numbers that appear as from/to on every synced message and call (high)-- [ ] `GET /v1/Account/{auth_id}/UsageSummary/` — billed usage and spend per period, the headline cost metric (high)-- [ ] `GET /v1/Account/{auth_id}/MultiPartyCall/` — conference-style calls that never appear in the flat calls table (high)-- [ ] `GET /v1/Account/{auth_id}/MultiPartyCall/name_{mpc_name}/Participant/` — per-leg participant records — join, exit and duration per member (high)+- [x] `GET /v1/Account/{auth_id}/PhoneNumber/` (synced as `numbers` from `GET /v1/Account/{auth_id}/Number/`; `PhoneNumber/` searches numbers available to buy) — lookup resolving the rented numbers that appear as from/to on every synced message and call (high)+- [x] `GET /v1/Account/{auth_id}/UsageSummary/` — billed usage and spend per period, the headline cost metric (high)+- [x] `GET /v1/Account/{auth_id}/MultiPartyCall/` — conference-style calls that never appear in the flat calls table (high)+- [x] `GET /v1/Account/{auth_id}/MultiPartyCall/name_{mpc_name}/Participant/` (fetched via `uuid_{mpc_uuid}`, which also serves ended calls) — per-leg participant records — join, exit and duration per member (high)- [ ] `GET /v1/Account/{auth_id}/Subaccount/` — lookup for attributing message and call traffic to the right subaccount (medium)- [ ] `GET /v1/Account/{auth_id}/Verify/Session/` — OTP verification sessions, needed to measure verification funnel completion (medium)- [ ] `GET /v1/Account/{auth_id}/Zentrunk/Call/` — SIP trunking CDRs, a call stream entirely absent from the calls table (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/canonical_descriptions.pyindex 2fdcbcc9eb34..577fc1a3a2e9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/canonical_descriptions.py@@ -82,4 +82,83 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"resource_uri": "URI of the application resource on the Plivo API.",},},+ "numbers": {+ "description": "A phone number rented on your Plivo account.",+ "docs_url": "https://www.plivo.com/docs/numbers/account-phone-numbers",+ "columns": {+ "number": "The phone number in E.164 format, without the leading +.",+ "alias": "Friendly name given to the number.",+ "sub_account": "Subaccount the number is assigned to, if any.",+ "added_on": "Date on which the number was rented.",+ "application": "URI of the application linked to the number.",+ "carrier": "Carrier that provides the number.",+ "region": "Geographic region of the number.",+ "number_type": "Type of the number: local, mobile, fixed, national, or tollfree.",+ "monthly_rental_rate": "Monthly rental charged for the number, in your account currency.",+ "renewal_date": "Date on which the number's rental next renews.",+ "sms_enabled": "Whether the number can send and receive SMS.",+ "mms_enabled": "Whether the number can send and receive MMS.",+ "voice_enabled": "Whether the number can make and receive voice calls.",+ "resource_uri": "URI of the number resource on the Plivo API.",+ },+ },+ "multi_party_calls": {+ "description": "A multiparty call (MPC): a conference-style call that several participants join.",+ "docs_url": "https://www.plivo.com/docs/voice/api/multiparty-calls",+ "columns": {+ "mpc_uuid": "Unique identifier of the multiparty call.",+ "friendly_name": "Name given to the multiparty call.",+ "status": "Status of the multiparty call: initialized, active, or ended.",+ "creation_time": "Time at which the multiparty call was created.",+ "start_time": "Time at which the multiparty call started.",+ "end_time": "Time at which the multiparty call ended.",+ "duration": "Duration of the multiparty call, in seconds.",+ "billed_duration": "Duration billed for the multiparty call, in seconds.",+ "billed_amount": "Amount charged for the multiparty call, in your account currency.",+ "termination_cause": "Reason the multiparty call ended.",+ "termination_cause_code": "Numeric code of the reason the multiparty call ended.",+ "sub_account": "Subaccount the multiparty call belongs to, if any.",+ },+ },+ "multi_party_call_participants": {+ "description": "A participant (call leg) in a multiparty call.",+ "docs_url": "https://www.plivo.com/docs/voice/api/multiparty-calls",+ "columns": {+ "mpc_uuid": "Identifier of the multiparty call the participant belongs to.",+ "member_id": "Identifier of the participant, unique within its multiparty call.",+ "call_uuid": "Identifier of the participant's call leg.",+ "role": "Role of the participant: agent, customer, or supervisor.",+ "coach_mode": "Whether the participant is a supervisor in coach mode.",+ "hold": "Whether the participant is on hold.",+ "mute": "Whether the participant is muted.",+ "join_time": "Time at which the participant joined the multiparty call.",+ "exit_time": "Time at which the participant left the multiparty call.",+ "exit_cause": "Reason the participant left the multiparty call.",+ "duration": "Time the participant spent in the multiparty call, in seconds.",+ "billed_amount": "Amount charged for the participant's leg, in your account currency.",+ "mpc_end_time": "Time at which the parent multiparty call ended. Used as the incremental sync cursor.",+ },+ },+ "usage_summary": {+ "description": "Billed usage and other charges for one day, with usage broken down by product, type, country and direction.",+ "docs_url": "https://www.plivo.com/docs/account/api/usage-summary",+ "columns": {+ "id": "Identifier PostHog derives from the row's dimensions (date, product, type, country, direction and so on).",+ "from_date": "Start of the day the usage covers (inclusive).",+ "to_date": "End of the day the usage covers (exclusive).",+ "product": "Plivo product the usage is for: message, voice, zentrunk, or transcription.",+ "type": "Usage type within the product.",+ "country": "ISO 3166-1 alpha-2 code of the country the usage is attributed to.",+ "direction": "Direction of the traffic: inbound or outbound.",+ "subaccount_auth": "Auth ID of the subaccount the usage belongs to, if any.",+ "total_units": "Number of billed units (messages, calls, and so on).",+ "duration_seconds": "Total billed duration in seconds, for voice, SIP trunking and transcription usage.",+ "total_amount": "Total amount charged, in your account currency.",+ "surcharge": "Surcharge included in the total amount, if any.",+ "error_code": "Messaging delivery error code the usage is grouped by, if any.",+ "error_reason": "Description of the messaging delivery error code, if any.",+ "charge_type": "Kind of row: usage for traffic usage, or other_charge for rentals, fees, taxes and adjustments.",+ "description": "Description of the charge, for other_charge rows.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/plivo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/plivo.pyindex 5b518a6959ba..a2c651280a2b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/plivo.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/plivo.py@@ -1,3 +1,5 @@+import json+import hashlibimport dataclassesfrom collections.abc import Callable, Iteratorfrom datetime import UTC, date, datetime, timedelta@@ -7,8 +9,13 @@ from dateutil import parser as dateutil_parserfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_sessionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import HttpBasicAuth+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.config_setup import (+ create_response_hooks,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (+ JSONResponseCursorPaginator,OffsetPaginator,+ SinglePagePaginator,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClientfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager@@ -16,15 +23,22 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.soufrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.plivo.settings import (MAX_QUERY_RANGE_DAYS,+ MULTI_PARTY_CALL_PARTICIPANTS,+ MULTI_PARTY_CALLS,PAGE_SIZE,PLIVO_BASE_URL,PLIVO_ENDPOINTS,RETENTION_DAYS,+ USAGE_BACKFILL_DAYS,+ USAGE_MAX_QUERY_RANGE_DAYS,+ USAGE_METRIC_FIELDS,+ USAGE_PAGE_SIZE,+ USAGE_SUMMARY,PlivoEndpointConfig,)[email protected](frozen=True)class PlivoResumeConfig:# ISO start/end of the <=30-day window being paged. Both None for non-windowed endpoints.# window_end pins the resumed window's upper bound so re-deriving it against a later "now"@@ -33,6 +47,10 @@ class PlivoResumeConfig:window_end: str | None = None# Next page offset within the current window/listing.offset: int = 0+ # Next page cursor within the current window, for cursor-paginated endpoints (UsageSummary).+ page_token: str | None = None+ # Index of the next parent row to fan out within the parent page at `offset`.+ parent_index: int = 0def _coerce_datetime(value: Any) -> datetime:@@ -75,6 +93,13 @@ def _normalize_row(row: dict[str, Any], config: PlivoEndpointConfig) -> dict[strreturn row+def _usage_row_id(row: dict[str, Any]) -> str:+ """Deterministic id for a usage row. UsageSummary rows have no id, and optional dimensions+ (subaccount, error code) can be absent, so a composite primary key could hold nulls."""+ dimensions = {key: value for key, value in row.items() if key not in USAGE_METRIC_FIELDS}+ return hashlib.sha256(json.dumps(dimensions, sort_keys=True, default=str).encode()).hexdigest()++def _make_client(auth_id: str, auth_token: str) -> RESTClient:return RESTClient(base_url=PLIVO_BASE_URL, auth=HttpBasicAuth(auth_id, auth_token))@@ -95,7 +120,7 @@ def _paginate(paginator = OffsetPaginator(limit=PAGE_SIZE,offset=initial_offset,- total_path="meta.total_count",+ total_path=config.total_path,)def resume_hook(state: Optional[dict[str, Any]]) -> None:@@ -111,6 +136,133 @@ def _paginate(yield [_normalize_row(row, config) for row in page]+def _get_participant_rows(+ client: RESTClient,+ auth_id: str,+ parent_params: dict[str, Any],+ resume: PlivoResumeConfig | None,+ resumable_source_manager: ResumableSourceManager[PlivoResumeConfig],+) -> Iterator[list[dict[str, Any]]]:+ parent = PLIVO_ENDPOINTS[MULTI_PARTY_CALLS]+ child = PLIVO_ENDPOINTS[MULTI_PARTY_CALL_PARTICIPANTS]+ # An MPC listed by the parent can have no participant listing any more; skip it rather than fail the run.+ child_hooks = create_response_hooks(+ [{"status_code": 404, "action": "ignore"}], resource_name=MULTI_PARTY_CALL_PARTICIPANTS+ )++ parent_offset = resume.offset if resume else 0+ skip_parents = resume.parent_index if resume else 0+ parent_pages = client.paginate(+ path=f"Account/{auth_id}/{parent.path}",+ params=parent_params,+ paginator=OffsetPaginator(limit=PAGE_SIZE, offset=parent_offset, total_path=parent.total_path),+ data_selector="objects",+ )+ for mpcs in parent_pages:+ for index, raw_mpc in enumerate(mpcs):+ if index < skip_parents:+ continue+ mpc = _normalize_row(raw_mpc, parent)+ mpc_uuid = mpc["mpc_uuid"]+ # Stage the position after this MPC before yielding its participants, so a resume+ # starts at the next MPC and never writes these participants twice.+ resumable_source_manager.save_state(PlivoResumeConfig(offset=parent_offset, parent_index=index + 1))+ child_path = f"Account/{auth_id}/{child.path}".replace("{mpc_uuid}", mpc_uuid)+ participants = [+ {**participant, "mpc_uuid": mpc_uuid, "mpc_end_time": mpc.get("end_time")}+ for page in client.paginate(+ path=child_path,+ # The participant listing takes no limit or offset and returns every participant.+ paginator=SinglePagePaginator(),+ data_selector="objects",+ hooks=child_hooks,+ )+ for participant in page+ ]+ if participants:+ yield [_normalize_row(participant, child) for participant in participants]+ else:+ resumable_source_manager.safe_point()+ skip_parents = 0+ parent_offset += PAGE_SIZE+++def _get_usage_rows(+ client: RESTClient,+ path: str,+ resume: PlivoResumeConfig | None,+ resumable_source_manager: ResumableSourceManager[PlivoResumeConfig],+ last_value: Any,+ now: datetime,+) -> Iterator[list[dict[str, Any]]]:+ config = PLIVO_ENDPOINTS[USAGE_SUMMARY]+ # to_date is exclusive, so ending at today's date syncs complete days only.+ end = datetime.combine(now.date(), datetime.min.time(), tzinfo=UTC)+ if resume is not None and resume.window_start:+ start = _coerce_datetime(resume.window_start)+ elif last_value is not None:+ # from_date is inclusive: re-read the watermark day so a late restatement merges in place.+ watermark = _coerce_datetime(last_value)+ start = datetime.combine(watermark.date(), datetime.min.time(), tzinfo=UTC)+ else:+ start = end - timedelta(days=USAGE_BACKFILL_DAYS)++ # Pin the resumed window's saved upper bound so its page token stays valid.+ resumed_end = _coerce_datetime(resume.window_end) if resume is not None and resume.window_end else None+ window_start = start+ index = 0+ while window_start < end:+ if index == 0 and resumed_end is not None:+ window_end = resumed_end+ else:+ window_end = min(window_start + timedelta(days=USAGE_MAX_QUERY_RANGE_DAYS), end)+ params = {+ "granularity": "day",+ "from_date": window_start.strftime("%Y-%m-%d"),+ "to_date": window_end.strftime("%Y-%m-%d"),+ "page_size": USAGE_PAGE_SIZE,+ }+ resumed_token = resume.page_token if (index == 0 and resume is not None and resume.window_start) else None++ window_start_iso = window_start.isoformat()+ window_end_iso = window_end.isoformat()++ def window_checkpoint(+ state: Optional[dict[str, Any]],+ _has_next_page: bool,+ window_start_iso: str = window_start_iso,+ window_end_iso: str = window_end_iso,+ ) -> None:+ if state:+ resumable_source_manager.save_state(+ PlivoResumeConfig(+ window_start=window_start_iso, window_end=window_end_iso, page_token=state["cursor"]+ )+ )+ else:+ resumable_source_manager.save_state(PlivoResumeConfig(window_start=window_end_iso))++ for page in client.paginate(+ path=path,+ params=params,+ paginator=JSONResponseCursorPaginator(+ cursor_path="meta.next_page_token", cursor_param="page_token", raise_on_repeated_cursor=True+ ),+ # Staged before the page is yielded, so a hand-off after the write resumes past it.+ page_state_hook=window_checkpoint,+ initial_paginator_state={"cursor": resumed_token} if resumed_token else None,+ ):+ rows: list[dict[str, Any]] = []+ for body in page:+ rows.extend({**row, "charge_type": "usage"} for row in body.get("usage") or [])+ # Rentals, fees, taxes and adjustments arrive only on a window's first page.+ rows.extend({**charge, "charge_type": "other_charge"} for charge in body.get("other_charges") or [])+ yield [_normalize_row({**row, "id": _usage_row_id(row)}, config) for row in rows]++ window_start = window_end+ index += 1++def get_rows(auth_id: str,auth_token: str,@@ -129,6 +281,20 @@ def get_rows(resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ if endpoint == USAGE_SUMMARY:+ last_value = db_incremental_field_last_value if should_use_incremental_field else None+ yield from _get_usage_rows(client, path, resume, resumable_source_manager, last_value, now)+ return++ if endpoint == MULTI_PARTY_CALL_PARTICIPANTS:+ parent_params: dict[str, Any] = {}+ if should_use_incremental_field and db_incremental_field_last_value is not None and config.time_filter_field:+ parent_params[f"{config.time_filter_field}__gt"] = _format_filter_value(+ _coerce_datetime(db_incremental_field_last_value)+ )+ yield from _get_participant_rows(client, auth_id, parent_params, resume, resumable_source_manager)+ return+if not config.windowed:params: dict[str, Any] = {}if should_use_incremental_field and db_incremental_field_last_value is not None and filter_field:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/settings.pyindex 17c93813fe41..e36b79f0aa4b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/settings.py@@ -14,8 +14,20 @@ PAGE_SIZE = 20RETENTION_DAYS = 90MAX_QUERY_RANGE_DAYS = 30+MULTI_PARTY_CALLS = "multi_party_calls"+MULTI_PARTY_CALL_PARTICIPANTS = "multi_party_call_participants"+USAGE_SUMMARY = "usage_summary"-@dataclass+# UsageSummary pages with a cursor and accepts up to 1000 rows per page. With `day` granularity a+# single request may span at most 92 days, so the backfill is chunked into windows of that size.+USAGE_PAGE_SIZE = 1000+USAGE_MAX_QUERY_RANGE_DAYS = 92+USAGE_BACKFILL_DAYS = 365+# Usage figures, not dimensions: a restated day keeps its row id so merge updates it in place.+USAGE_METRIC_FIELDS = frozenset({"total_units", "duration_seconds", "total_amount", "surcharge"})+++@dataclass(frozen=True)class PlivoEndpointConfig:name: str# Path relative to /v1/Account/{auth_id}/, e.g. "Message/".@@ -30,6 +42,9 @@ class PlivoEndpointConfig:datetime_fields: tuple[str, ...] = ()# Whether requests must be chunked into MAX_QUERY_RANGE_DAYS windows over RETENTION_DAYS.windowed: bool = False+ # Response path of the grand total that ends offset pagination. Listings whose `meta.count`+ # meaning is undocumented set None and stop on a short page instead.+ total_path: str | None = "meta.total_count"PLIVO_ENDPOINTS: dict[str, PlivoEndpointConfig] = {@@ -73,6 +88,46 @@ PLIVO_ENDPOINTS: dict[str, PlivoEndpointConfig] = {primary_keys=["app_id"],# The application list exposes no server-side time filter — full refresh only.),+ "numbers": PlivoEndpointConfig(+ name="numbers",+ # Rented numbers. `PhoneNumber/` is the search for numbers available to buy, not this list.+ path="Number/",+ primary_keys=["number"],+ # The number list only filters by renewal_date, which changes on every renewal — full refresh only.+ ),+ MULTI_PARTY_CALLS: PlivoEndpointConfig(+ name=MULTI_PARTY_CALLS,+ path="MultiPartyCall/",+ primary_keys=["mpc_uuid"],+ incremental_fields=[incremental_field("end_time")],+ time_filter_field="end_time",+ partition_key="creation_time",+ datetime_fields=("creation_time", "start_time", "end_time"),+ total_path=None,+ ),+ MULTI_PARTY_CALL_PARTICIPANTS: PlivoEndpointConfig(+ name=MULTI_PARTY_CALL_PARTICIPANTS,+ # Fanned out per MPC. Plivo serves participants by MPC name only while the call is ongoing,+ # so the listing is addressed by UUID, which also works for ended calls.+ path="MultiPartyCall/uuid_{mpc_uuid}/Participant/",+ # member_id is only unique within its MPC.+ primary_keys=["mpc_uuid", "member_id"],+ # The participant listing has no time filter, so incremental syncs filter the parent MPC+ # list by end_time and stamp it onto each participant as mpc_end_time.+ incremental_fields=[incremental_field("mpc_end_time")],+ time_filter_field="end_time",+ datetime_fields=("join_time", "exit_time"),+ total_path=None,+ ),+ USAGE_SUMMARY: PlivoEndpointConfig(+ name=USAGE_SUMMARY,+ path="UsageSummary/",+ primary_keys=["id"],+ incremental_fields=[incremental_field("from_date")],+ time_filter_field="from_date",+ partition_key="from_date",+ datetime_fields=("from_date", "to_date"),+ ),}ENDPOINTS = tuple(PLIVO_ENDPOINTS.keys())@@ -92,4 +147,11 @@ ENDPOINT_DESCRIPTIONS: dict[str, str] = {),"recordings": "Call and conference recordings stored on your Plivo account.","applications": "Plivo applications that control how calls and messages to your numbers are handled.",+ "numbers": "Phone numbers rented on your Plivo account, with their capabilities, rates and linked application.",+ MULTI_PARTY_CALLS: "Multiparty calls (MPCs): conference-style calls with several participants.",+ MULTI_PARTY_CALL_PARTICIPANTS: "Participants of each multiparty call, with their role and join and exit details.",+ USAGE_SUMMARY: (+ "Daily billed usage per product, type, country and direction, plus other charges such as number rentals. "+ "The first sync backfills the last 365 days."+ ),}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/source.pyindex 390971d85d6f..c10c5ea380e3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/source.py@@ -30,6 +30,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.plivo.settENDPOINTS,INCREMENTAL_FIELDS,PLIVO_BASE_URL,+ USAGE_SUMMARY,)from products.warehouse_sources.backend.types import ExternalDataSourceType@@ -129,6 +130,8 @@ class PlivoSource(ResumableSource[PlivoSourceConfig, PlivoResumeConfig]):INCREMENTAL_FIELDS,names,descriptions=ENDPOINT_DESCRIPTIONS,+ # Each sync re-reads the watermark day, which append would write twice.+ merge_only=(USAGE_SUMMARY,),)def validate_credentials(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/tests/test_plivo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/tests/test_plivo.pyindex 23cc343409b3..685a9a802afe 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/tests/test_plivo.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/tests/test_plivo.py@@ -31,6 +31,38 @@ def _page(objects: list[dict[str, Any]], total_count: int) -> ScriptedResponse:return ScriptedResponse(json={"meta": {"limit": PAGE_SIZE, "total_count": total_count}, "objects": objects})+def _count_page(objects: list[dict[str, Any]]) -> ScriptedResponse:+ return ScriptedResponse(json={"meta": {"count": len(objects), "limit": PAGE_SIZE, "offset": 0}, "objects": objects})+++def _usage_page(+ usage: list[dict[str, Any]],+ next_page_token: str | None = None,+ other_charges: list[dict[str, Any]] | None = None,+) -> ScriptedResponse:+ meta: dict[str, Any] = {"granularity": "day", "page_size": 1000}+ if next_page_token:+ meta["next_page_token"] = next_page_token+ body: dict[str, Any] = {"meta": meta, "usage": usage}+ if other_charges is not None:+ body["other_charges"] = other_charges+ return ScriptedResponse(json=body)+++def _usage_row(**overrides: Any) -> dict[str, Any]:+ return {+ "from_date": "2026-07-19T00:00:00Z",+ "to_date": "2026-07-20T00:00:00Z",+ "product": "message",+ "type": "sms",+ "country": "US",+ "direction": "outbound",+ "total_units": 10,+ "total_amount": 0.05,+ **overrides,+ }++class TestWindowedEndpoints:@pytest.fixture(autouse=True)def _frozen_clock(self) -> Iterator[None]:@@ -132,6 +164,145 @@ class TestNonWindowedEndpoints:assert result.items[0][0]["add_time"] == ""+class TestUsageSummary:+ @pytest.fixture(autouse=True)+ def _frozen_clock(self) -> Iterator[None]:+ with time_machine.travel(FROZEN_NOW, tick=False):+ yield++ @pytest.mark.parametrize(+ "inputs, expected_windows",+ [+ (+ {},+ [+ ("2025-07-21", "2025-10-21"),+ ("2025-10-21", "2026-01-21"),+ ("2026-01-21", "2026-04-23"),+ ("2026-04-23", "2026-07-21"),+ ],+ ),+ (+ # The watermark day is re-read so a restated total merges in place; today is excluded+ # because to_date is exclusive and today's usage is still incomplete.+ {+ "incremental_field": "from_date",+ "db_incremental_field_last_value": datetime(2026, 7, 19, tzinfo=UTC),+ },+ [("2026-07-19", "2026-07-21")],+ ),+ ],+ )+ def test_windows_requests_by_day(self, inputs: dict[str, Any], expected_windows: list[tuple[str, str]]) -> None:+ result = _driver().run("usage_summary", [_usage_page([]) for _ in expected_windows], **inputs)++ assert result.raised is None+ assert result.paths[0] == "/v1/Account/MA123/UsageSummary/"+ assert list(zip(result.params("from_date"), result.params("to_date"), strict=True)) == expected_windows+ assert set(result.params("granularity")) == {"day"}++ def test_follows_page_token_within_window(self) -> None:+ rental = {+ "from_date": "2026-07-19T00:00:00Z",+ "to_date": "2026-07-20T00:00:00Z",+ "description": "Number rental",+ "total_units": 1,+ "total_amount": 0.8,+ }+ result = _driver().run(+ "usage_summary",+ [+ _usage_page([_usage_row()], next_page_token="tok-2", other_charges=[rental]),+ _usage_page([_usage_row(type="mms")]),+ ],+ incremental_field="from_date",+ db_incremental_field_last_value=datetime(2026, 7, 19, tzinfo=UTC),+ )++ assert result.raised is None+ assert result.params("page_token") == [None, "tok-2"]+ assert result.params("from_date") == ["2026-07-19", "2026-07-19"]+ assert [(row["charge_type"], row.get("type"), row.get("description")) for row in result.rows] == [+ ("usage", "sms", None),+ ("other_charge", None, "Number rental"),+ ("usage", "mms", None),+ ]+ assert len({row["id"] for row in result.rows}) == 3+ assert result.rows[0]["from_date"] == datetime(2026, 7, 19, tzinfo=UTC)++ def test_row_id_tracks_dimensions_not_metrics(self) -> None:+ result = _driver().run(+ "usage_summary",+ [+ _usage_page(+ [+ _usage_row(),+ _usage_row(total_units=12, total_amount=0.06),+ _usage_row(subaccount_auth="SA1"),+ _usage_row(direction="inbound"),+ ]+ )+ ],+ incremental_field="from_date",+ db_incremental_field_last_value=datetime(2026, 7, 19, tzinfo=UTC),+ )++ ids = [row["id"] for row in result.rows]+ # A restated metric keeps the row's id so merge updates it rather than duplicating it.+ assert ids[0] == ids[1]+ assert len({ids[0], ids[2], ids[3]}) == 3+++class TestMultiPartyCallParticipants:+ def test_fans_out_per_mpc_by_uuid(self) -> None:+ result = _driver().run(+ "multi_party_call_participants",+ [+ _count_page(+ [+ {"mpc_uuid": "mpc-1", "end_time": "2026-07-15 10:00:00+00:00"},+ {"mpc_uuid": "mpc-gone", "end_time": "2026-07-15 11:00:00+00:00"},+ ]+ ),+ _count_page([{"member_id": "1", "join_time": "2026-07-15 09:00:00+00:00"}, {"member_id": "2"}]),+ ScriptedResponse(status=404, json={"error": "not found"}),+ ],+ incremental_field="mpc_end_time",+ db_incremental_field_last_value=datetime(2026, 7, 11, 12, tzinfo=UTC),+ )++ assert result.raised is None+ assert result.paths == [+ "/v1/Account/MA123/MultiPartyCall/",+ "/v1/Account/MA123/MultiPartyCall/uuid_mpc-1/Participant/",+ "/v1/Account/MA123/MultiPartyCall/uuid_mpc-gone/Participant/",+ ]+ # The participant listing is fetched once per MPC, without offset pagination.+ assert result.params("offset") == ["0", None, None]+ # The participant list has no time filter, so the incremental cursor narrows the parent MPCs.+ assert result.params("end_time__gt") == ["2026-07-11 12:00:00", None, None]+ assert [(row["mpc_uuid"], row["member_id"]) for row in result.rows] == [("mpc-1", "1"), ("mpc-1", "2")]+ assert {row["mpc_end_time"] for row in result.rows} == {datetime(2026, 7, 15, 10, tzinfo=UTC)}+ assert result.rows[0]["join_time"] == datetime(2026, 7, 15, 9, tzinfo=UTC)++ def test_resume_skips_mpcs_already_fanned_out(self) -> None:+ result = _driver().run(+ "multi_party_call_participants",+ [+ _count_page([{"mpc_uuid": "mpc-done"}, {"mpc_uuid": "mpc-next"}]),+ _count_page([{"member_id": "1"}]),+ ],+ resume_state=PlivoResumeConfig(offset=PAGE_SIZE, parent_index=1),+ )++ assert result.raised is None+ assert result.params("offset")[0] == str(PAGE_SIZE)+ assert result.paths[1:] == ["/v1/Account/MA123/MultiPartyCall/uuid_mpc-next/Participant/"]+ # The position after an MPC is staged before its participants are yielded, so a later+ # resume never writes them twice.+ assert result.committed_states == [PlivoResumeConfig(offset=PAGE_SIZE, parent_index=2)]++class TestValidateCredentials:@pytest.mark.parametrize("status_code, expected",@@ -154,13 +325,16 @@ class TestPlivoSourceResponse:("calls", ["call_uuid"], ["end_time"]),("recordings", ["recording_id"], ["add_time"]),("applications", ["app_id"], None),+ ("numbers", ["number"], None),+ ("multi_party_calls", ["mpc_uuid"], ["creation_time"]),+ ("usage_summary", ["id"], ["from_date"]),],)def test_source_response_shape(self, endpoint: str, primary_keys: list[str], partition_keys: list[str] | None) -> None:with time_machine.travel(FROZEN_NOW, tick=False):- result = _driver().run(endpoint, [_page([], 0) for _ in range(3)])+ result = _driver().run(endpoint, [_page([], 0) for _ in range(4)])assert result.raised is Noneresponse = result.responsediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/tests/test_plivo_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/tests/test_plivo_source.pyindex 886c3b2ab531..1707e3b84d70 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/tests/test_plivo_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plivo/tests/test_plivo_source.py@@ -41,3 +41,10 @@ class TestPlivoSource:# A stale cursor must not narrow a full refresh to a partial window.assert mock_plivo_source.call_args.kwargs["db_incremental_field_last_value"] is None++ def test_usage_summary_is_merge_only(self):+ schemas = {schema.name: schema for schema in self.source.get_schemas(self.config, self.team_id)}++ assert schemas["usage_summary"].supports_incremental+ assert not schemas["usage_summary"].supports_append+ assert schemas["messages"].supports_append