:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 5 missing tables to the CallRail source (#119996)
Tom Owers61865731e7cc
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex c5324702a9dc..7e2e72f89bfb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -2139,7 +2139,7 @@ Note: developer.calendly.com is a Gatsby shell with no fetchable spec; the real## CallRail — gaps-Today (11): `accounts`, `calls`, `companies`, `form_submissions`, `lead_timelines`, `leads`, `page_views`, `tags`, `text_messages`, `trackers`, `users`+Today (16): `accounts`, `calls`, `calls_summary`, `calls_timeseries`, `companies`, `form_submissions`, `form_submissions_summary`, `lead_timelines`, `leads`, `outbound_caller_ids`, `page_views`, `sms_threads`, `tags`, `text_messages`, `trackers`, `users`Diffed against: <https://apidocs.callrail.com/>@@ -2147,10 +2147,10 @@ Diffed against: <https://apidocs.callrail.com/>- [x] `accounts (/a.json)` — lookup resolving the account_id every other resource is scoped under; required for multi-account (agency) reporting (high)- [x] `page_views (/a/{account_id}/calls/{call_id}/page_views.json)` — per-call visitor page-view journey — the attribution path behind each tracked call (high)- [x] `lead_timelines (/a/{account_id}/leads/{id}/timeline.json)` — state/transition history for a lead across calls, texts, and forms (medium)-- [ ] `sms_threads (/a/{account_id}/sms_threads.json)` — thread-level SMS conversations, the parent grain the text_messages table hangs off (medium)-- [ ] `calls summary and timeseries (/a/{account_id}/calls/summary.json, /calls/timeseries.json)` — vendor-computed call volume breakdowns by source and period, matching what the CallRail UI reports (medium)-- [ ] `form_submissions summary (/a/{account_id}/form_submissions/summary.json)` — vendor-computed form conversion aggregates aligned with the call summary (low)-- [ ] `outbound_caller_ids (/a/{account_id}/caller_ids.json)` — lookup resolving the caller ID numbers appearing on outbound calls (low)+- [x] `sms_threads (/a/{account_id}/sms_threads.json)` — thread-level SMS conversations, the parent grain the text_messages table hangs off (medium)+- [x] `calls summary and timeseries (/a/{account_id}/calls/summary.json, /calls/timeseries.json)` — vendor-computed call volume breakdowns by source and period, matching what the CallRail UI reports (medium)+- [x] `form_submissions summary (/a/{account_id}/form_submissions/summary.json)` — vendor-computed form conversion aggregates aligned with the call summary (low)+- [x] `outbound_caller_ids (/a/{account_id}/caller_ids.json)` — lookup resolving the caller ID numbers appearing on outbound calls (low)Note: apidocs.callrail.com serves the entire v3 reference as one ~890 KB HTML page; I parsed its heading tree for the resource list rather than using the summarizer.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/callrail.py b/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/callrail.pyindex efddcc514878..b21f4c09ae30 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/callrail.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/callrail.py@@ -1,6 +1,6 @@import dataclassesfrom collections.abc import Iterable, Iterator-from datetime import UTC, date, datetime+from datetime import UTC, date, datetime, timedeltafrom typing import Any, Optional, castfrom products.warehouse_sources.backend.temporal.data_imports.sources.callrail.settings import (@@ -35,6 +35,10 @@ CALLRAIL_BASE_URL = "https://api.callrail.com/v3"# forever. 250 rows/page * this cap bounds a single endpoint sync.MAX_PAGES = 100_000+# CallRail deletes communication records after 25 months and rejects a date range that reaches past+# that, so date windows start at the first whole month inside it.+RETENTION_MONTHS = 24+@dataclasses.dataclass(frozen=True)class CallRailResumeConfig:@@ -47,6 +51,8 @@ class CallRailResumeConfig:# Fan-out progress for an endpoint reached once per parent row: which parents are done, which# one was in flight, and that parent's page cursor. Shape owned by the shared fan-out helper.fanout_state: Optional[dict[str, Any]] = None+ # First day (YYYY-MM-DD) of the next monthly window to fetch, for a report endpoint.+ window_start: Optional[str] = Nonedef _get_headers(api_key: str) -> dict[str, str]:@@ -116,6 +122,40 @@ def _format_start_date(value: Any) -> str | None:return None+def _parse_cursor_date(value: Any) -> date | None:+ formatted = _format_start_date(value)+ return date.fromisoformat(formatted) if formatted else None+++def _retention_start(today: date) -> date:+ month_index = today.year * 12 + today.month - 1 - RETENTION_MONTHS+ return date(month_index // 12, month_index % 12 + 1, 1)+++def _reporting_today() -> date:+ """Today's date in the earliest timezone on Earth (UTC-12).++ Reports use the account's timezone, so UTC's date can be a day ahead of the account's. Asking+ for that day returns a zero-filled row that would move the watermark past a still-partial month.+ """+ return (datetime.now(UTC) - timedelta(hours=12)).date()++[email protected](frozen=True)+class _MonthWindow:+ first_day: date+ last_day: date+++def _month_windows(start: date, today: date) -> Iterator[_MonthWindow]:+ """Yield each calendar month from `start` up to and including today, clipped at today."""+ window_start = start.replace(day=1)+ while window_start <= today:+ next_start = (window_start + timedelta(days=32)).replace(day=1)+ yield _MonthWindow(first_day=window_start, last_day=min(next_start - timedelta(days=1), today))+ window_start = next_start++def resolve_account_id(api_key: str, team_id: int, job_id: str, account_id: str | None = None) -> str:"""Return the account id to scope data requests to.@@ -209,6 +249,70 @@ def _fanout_rows()+def _report_rows(+ api_key: str,+ endpoint: str,+ team_id: int,+ job_id: str,+ resumable_source_manager: ResumableSourceManager[CallRailResumeConfig],+ resolved_account_id: str,+ resume_window_start: Optional[str],+ should_use_incremental_field: bool,+ db_incremental_field_last_value: Any,+) -> Iterator[list[dict[str, Any]]]:+ """Fetch a vendor-computed report one calendar month per request, oldest month first.++ A timeseries report holds at most 200 data points and a summary report covers its whole date+ range, so a month is the unit that keeps both complete and gives summary rows a period. An+ incremental sync starts again at the watermark's month, because the newest month is still+ partial when it is read. The merge replaces its rows on the primary key.+ """+ config = CALLRAIL_ENDPOINTS[endpoint]+ today = _reporting_today()+ start = _retention_start(today)+ if resume_window_start is not None:+ start = max(start, date.fromisoformat(resume_window_start))+ elif should_use_incremental_field:+ watermark = _parse_cursor_date(db_incremental_field_last_value)+ if watermark is not None:+ start = max(start, watermark.replace(day=1))++ for window in _month_windows(start, today):+ window_start, window_end = window.first_day, window.last_day+ rest_config: RESTAPIConfig = {+ "client": _client_config(api_key),+ "resources": [+ {+ "name": endpoint,+ "endpoint": {+ "path": config.path.replace("{account_id}", resolved_account_id),+ "params": {+ **config.extra_params,+ "start_date": window_start.isoformat(),+ "end_date": window_end.isoformat(),+ },+ "data_selector": config.response_key,+ "paginator": SinglePagePaginator(),+ },+ }+ ],+ }+ rows = [row for page in rest_api_resource(rest_config, team_id, job_id, None) for row in page]+ if config.report == "summary":+ period = {"period_start": window_start.isoformat(), "period_end": window_end.isoformat()}+ rows = [{**row, **period} for row in rows]++ next_start = window_end + timedelta(days=1)+ if next_start <= today:+ resumable_source_manager.save_state(+ CallRailResumeConfig(account_id=resolved_account_id, window_start=next_start.isoformat())+ )+ if rows:+ yield rows+ else:+ resumable_source_manager.safe_point()++def get_rows(api_key: str,endpoint: str,@@ -243,13 +347,33 @@ def get_rows()return+ if config.report is not None:+ yield from _report_rows(+ api_key=api_key,+ endpoint=endpoint,+ team_id=team_id,+ job_id=job_id,+ resumable_source_manager=resumable_source_manager,+ resolved_account_id=resolved_account_id,+ resume_window_start=resume.window_start if resume is not None else None,+ should_use_incremental_field=should_use_incremental_field,+ db_incremental_field_last_value=db_incremental_field_last_value,+ )+ return+initial_paginator_state: Optional[dict[str, Any]] = Noneif resume is not None and resume.page is not None:initial_paginator_state = {"page": resume.page}+ params: dict[str, Any] = {"per_page": PER_PAGE, **_sort_params(config), **config.extra_params}+ if config.retention_window:+ today = datetime.now(UTC).date()+ params["start_date"] = _retention_start(today).isoformat()+ params["end_date"] = today.isoformat()+endpoint_config: Endpoint = {"path": config.path.replace("{account_id}", resolved_account_id),- "params": {"per_page": PER_PAGE, **_sort_params(config)},+ "params": params,# Key the list lives under in the JSON envelope; a missing key reads as an empty page and# ends pagination, matching the API's "no more data" signal."data_selector": config.response_key,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/canonical_descriptions.pyindex 378a556d3f18..ac076068cd71 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/canonical_descriptions.py@@ -40,6 +40,36 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"company_id": "Identifier of the company the call belongs to.",},},+ "calls_summary": {+ "description": "Call counts per marketing source for each calendar month, as computed by CallRail.",+ "docs_url": "https://apidocs.callrail.com/#summarizing-call-data",+ "columns": {+ "period_start": "First day of the calendar month the row summarizes.",+ "period_end": "Last day of the month the row summarizes, or the sync date for the current month.",+ "key": "The marketing source the calls are attributed to.",+ "total_calls": "Total number of calls in the period.",+ "missed_calls": "Number of calls in the period that were missed.",+ "answered_calls": "Number of calls in the period that were answered.",+ "first_time_callers": "Number of calls in the period from first-time callers.",+ "average_duration": "Average call duration in the period, in seconds.",+ "formatted_average_duration": "Average call duration in the period, formatted for display.",+ "leads": "Number of calls in the period marked as leads.",+ },+ },+ "calls_timeseries": {+ "description": "Daily call counts for the account, as computed by CallRail.",+ "docs_url": "https://apidocs.callrail.com/#summarizing-call-data-by-time-series",+ "columns": {+ "date": "The day the row summarizes (YYYY-MM-DD), in the account time zone.",+ "total_calls": "Total number of calls in the period.",+ "missed_calls": "Number of calls in the period that were missed.",+ "answered_calls": "Number of calls in the period that were answered.",+ "first_time_callers": "Number of calls in the period from first-time callers.",+ "average_duration": "Average call duration in the period, in seconds.",+ "formatted_average_duration": "Average call duration in the period, formatted for display.",+ "leads": "Number of calls in the period marked as leads.",+ },+ },"companies": {"description": "Companies configured in the CallRail account. A company groups trackers, calls, and form submissions.","docs_url": "https://apidocs.callrail.com/#companies",@@ -67,6 +97,18 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"first_form": "Whether this was the person's first form submission.",},},+ "form_submissions_summary": {+ "description": "Form submission counts per marketing source for each calendar month, as computed by CallRail.",+ "docs_url": "https://apidocs.callrail.com/#summarizing-form-data",+ "columns": {+ "period_start": "First day of the calendar month the row summarizes.",+ "period_end": "Last day of the month the row summarizes, or the sync date for the current month.",+ "key": "The marketing source the form submissions are attributed to.",+ "total_forms": "Total number of form submissions in the period.",+ "first_forms": "Number of form submissions in the period that were first-time submissions.",+ "leads": "Number of form submissions in the period marked as leads.",+ },+ },"leads": {"description": "Leads in the CallRail account: the people derived from tracked calls, texts, and form submissions.","docs_url": "https://apidocs.callrail.com/#leads",@@ -107,6 +149,19 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"touchpoint_type": "Attribution touchpoint behind the event, e.g. organic_search or paid_search.",},},+ "outbound_caller_ids": {+ "description": "Phone numbers registered to show on the recipient's caller ID for outbound calls placed through CallRail.",+ "docs_url": "https://apidocs.callrail.com/#outbound-caller-ids",+ "columns": {+ "id": "Unique identifier for the outbound caller ID.",+ "company_id": "Identifier of the company the caller ID belongs to.",+ "phone_number": "The phone number, in E.164 format.",+ "name": "Descriptive name of the caller ID.",+ "verified": "Whether the phone number has been verified in CallRail.",+ "created_at": "When the outbound caller ID was created.",+ "validation_code": "Code used to verify ownership of the phone number.",+ },+ },"page_views": {"description": "Browsing history behind a tracked call: the pages a visitor viewed before dialing, newest first. Only recorded for calls placed to a session tracker, and synced once per call.","docs_url": "https://apidocs.callrail.com/#page-views",@@ -117,6 +172,26 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"created_at": "When the page view was recorded (ISO 8601).",},},+ "sms_threads": {+ "description": "SMS conversation threads between customers and the business, without the individual messages.",+ "docs_url": "https://apidocs.callrail.com/#sms-threads",+ "columns": {+ "id": "Unique identifier for the SMS thread.",+ "company_id": "Identifier of the company the thread belongs to.",+ "customer_name": "Name of the customer.",+ "customer_phone_number": "Phone number of the customer, in E.164 format.",+ "initial_tracker_id": "Tracker whose number started the conversation.",+ "current_tracker_id": "Tracker whose number was used most recently in the conversation.",+ "initial_tracking_number": "Tracking number that started the conversation.",+ "current_tracking_number": "Tracking number used most recently in the conversation.",+ "lead_qualification": "Lead status of the thread: good_lead, not_a_lead, or empty.",+ "value": "Value assigned to the thread.",+ "tags": "Names of the tags applied to the thread.",+ "notes": "Notes on the thread.",+ "state": "Whether the thread is active or archived.",+ "last_message_at": "When the most recent message in the thread was sent or received.",+ },+ },"text_messages": {"description": "SMS conversations between callers and the business, captured by CallRail.","docs_url": "https://apidocs.callrail.com/#text-messages",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/settings.pyindex d6a70881161b..a6d35964c186 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/settings.py@@ -1,5 +1,5 @@from dataclasses import dataclass, field-from typing import Optional+from typing import Literal, Optionalfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (DependentEndpointConfig,@@ -16,6 +16,10 @@ PER_PAGE = 250# exist for calls placed to a session tracker._CHILD_404_IGNORE: list[ResponseAction] = [{"status_code": 404, "action": "ignore"}]+_CALL_SUMMARY_FIELDS = (+ "total_calls,missed_calls,answered_calls,first_time_callers,average_duration,formatted_average_duration,leads"+)+@dataclass(frozen=True)class CallRailEndpointConfig:@@ -40,6 +44,15 @@ class CallRailEndpointConfig:should_sync_default: bool = True# Set for endpoints reached once per parent row rather than once per account.fanout: Optional[DependentEndpointConfig] = None+ # Static query params, such as `fields=` for columns the API only returns on request.+ extra_params: dict[str, str] = field(default_factory=dict)+ # Set for listings that fall back to CallRail's `recent` window (the last 7 days) when sent no+ # dates, so they get an explicit range that covers the retention period instead.+ retention_window: bool = False+ # Set for the vendor-computed report endpoints, which take one calendar-month date window per+ # request instead of pages. A "summary" report carries no date on its rows, so each row gets the+ # window it covers.+ report: Optional[Literal["timeseries", "summary"]] = None@propertydef requires_account(self) -> bool:@@ -65,6 +78,15 @@ _PAGE_VIEWS_FANOUT = DependentEndpointConfig(child_response_actions=_CHILD_404_IGNORE,)+_CALLER_IDS_FANOUT = DependentEndpointConfig(+ parent_name="companies",+ resolve_param="company_id",+ resolve_field="id",+ include_from_parent=["id"],+ parent_field_renames={"id": "company_id"},+ child_response_actions=_CHILD_404_IGNORE,+)+_LEAD_TIMELINE_FANOUT = DependentEndpointConfig(parent_name="leads",resolve_param="lead_id",@@ -82,6 +104,8 @@ _LEAD_TIMELINE_FANOUT = DependentEndpointConfig(# `start_date` date filter that narrows on a stable timestamp (Calls -> start_time,# Form submissions -> submitted_at). For those we advertise exactly that one field as the# incremental cursor so the cursor, the `sort=` field, and the `start_date` filter all agree.+# The report endpoints (calls and form summaries, calls timeseries) take the same date filter and+# advance on the date of the window they cover.# The remaining resources are mutable configuration objects or lack a usable server-side date# filter, so they ship full refresh only.CALLRAIL_ENDPOINTS: dict[str, CallRailEndpointConfig] = {@@ -110,6 +134,40 @@ CALLRAIL_ENDPOINTS: dict[str, CallRailEndpointConfig] = {},],),+ "calls_summary": CallRailEndpointConfig(+ name="calls_summary",+ path="/a/{account_id}/calls/summary.json",+ response_key="grouped_results",+ primary_keys=["period_start", "key"],+ supports_incremental=True,+ report="summary",+ extra_params={"group_by": "source", "fields": _CALL_SUMMARY_FIELDS},+ incremental_fields=[+ {+ "label": "period_start",+ "type": IncrementalFieldType.Date,+ "field": "period_start",+ "field_type": IncrementalFieldType.Date,+ },+ ],+ ),+ "calls_timeseries": CallRailEndpointConfig(+ name="calls_timeseries",+ path="/a/{account_id}/calls/timeseries.json",+ response_key="data",+ primary_keys=["date"],+ supports_incremental=True,+ report="timeseries",+ extra_params={"interval": "day", "fields": _CALL_SUMMARY_FIELDS},+ incremental_fields=[+ {+ "label": "date",+ "type": IncrementalFieldType.Date,+ "field": "date",+ "field_type": IncrementalFieldType.Date,+ },+ ],+ ),"companies": CallRailEndpointConfig(name="companies",path="/a/{account_id}/companies.json",@@ -133,6 +191,23 @@ CALLRAIL_ENDPOINTS: dict[str, CallRailEndpointConfig] = {},],),+ "form_submissions_summary": CallRailEndpointConfig(+ name="form_submissions_summary",+ path="/a/{account_id}/forms/summary.json",+ response_key="grouped_results",+ primary_keys=["period_start", "key"],+ supports_incremental=True,+ report="summary",+ extra_params={"group_by": "source", "fields": "total_forms,first_forms,leads"},+ incremental_fields=[+ {+ "label": "period_start",+ "type": IncrementalFieldType.Date,+ "field": "period_start",+ "field_type": IncrementalFieldType.Date,+ },+ ],+ ),"leads": CallRailEndpointConfig(name="leads",path="/a/{account_id}/leads.json",@@ -170,6 +245,20 @@ CALLRAIL_ENDPOINTS: dict[str, CallRailEndpointConfig] = {},],),+ "outbound_caller_ids": CallRailEndpointConfig(+ name="outbound_caller_ids",+ # `company_id` is a required query param, bound per company row by the fan-out.+ path="/a/{account_id}/caller_ids.json?company_id={company_id}",+ response_key="caller_ids",+ partition_key="created_at",+ primary_keys=["company_id", "id"],+ fanout=_CALLER_IDS_FANOUT,+ # One request per company against a 1,000/hour account-wide budget, so leave it to the user+ # to opt in rather than enabling it on every new connection.+ should_sync_default=False,+ # The parent companies listing is not windowed, so a full refresh sees every caller ID.+ incremental_fields=[],+ ),"page_views": CallRailEndpointConfig(name="page_views",path="/a/{account_id}/calls/{call_id}/page_views.json",@@ -194,6 +283,16 @@ CALLRAIL_ENDPOINTS: dict[str, CallRailEndpointConfig] = {},],),+ "sms_threads": CallRailEndpointConfig(+ name="sms_threads",+ path="/a/{account_id}/sms-threads.json",+ response_key="sms_threads",+ # The listing takes no sort param and returns the thread with the newest message first. Its+ # date filter is not documented as narrowing on a stable field, so this is full refresh.+ retention_window=True,+ extra_params={"fields": "last_message_at"},+ incremental_fields=[],+ ),"text_messages": CallRailEndpointConfig(name="text_messages",# Returns SMS conversations under the "conversations" key.@@ -229,6 +328,8 @@ INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {SHOULD_SYNC_DEFAULT: dict[str, bool] = {name: config.should_sync_default for name, config in CALLRAIL_ENDPOINTS.items()}-# A fan-out child accumulates one row per parent row per sync under append, so these tables offer-# incremental merge only.-MERGE_ONLY = tuple(name for name, config in CALLRAIL_ENDPOINTS.items() if config.fanout is not None)+# A fan-out child accumulates one row per parent row per sync under append, and a report re-reads+# the current month on every sync, so these tables offer incremental merge only.+MERGE_ONLY = tuple(+ name for name, config in CALLRAIL_ENDPOINTS.items() if config.fanout is not None or config.report is not None+)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/tests/test_callrail.py b/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/tests/test_callrail.pyindex ed978dc9cc0f..c9227bb71e2f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/tests/test_callrail.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/tests/test_callrail.py@@ -2,6 +2,7 @@ from datetime import UTC, date, datetimefrom typing import Anyimport pytest+import time_machinefrom products.warehouse_sources.backend.temporal.data_imports.sources.callrail.callrail import (CallRailResumeConfig,@@ -17,6 +18,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.callrail.sfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (ScriptedResponse,SourceDriver,+ always,scripted_network,)from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.callrail import (@@ -139,7 +141,7 @@ class TestCallRailSourceResponse:@pytest.mark.parametrize("endpoint", list(ENDPOINTS))def test_response_metadata_per_endpoint(self, endpoint: str) -> None:config = CALLRAIL_ENDPOINTS[endpoint]- result = _driver().run(endpoint, [ScriptedResponse(json={"total_pages": 1})])+ result = _driver().run(endpoint, always(ScriptedResponse(json={"total_pages": 1})))response = result.responseassert result.raised is None@@ -163,7 +165,7 @@ class TestCallRailSourceResponse:@pytest.mark.parametrize("config", list(CALLRAIL_ENDPOINTS.values()))def test_incremental_endpoints_have_a_sort_field(self, config: Any) -> None:# An incremental endpoint must sort ascending on its cursor so the watermark advances.- if config.supports_incremental:+ if config.supports_incremental and config.report is None:assert config.sort_field is not Noneassert config.incremental_fields@@ -244,3 +246,162 @@ class TestFanoutEndpoints:assert result.raised is Noneassert result.requests[1].param("start_date") is Noneassert result.requests[1].param("created_at") is None+++class TestOutboundCallerIdsEndpoint:+ def test_binds_each_company_as_the_required_query_param(self) -> None:+ result = _driver().run(+ "outbound_caller_ids",+ [+ _page("companies", [{"id": "COM1"}, {"id": "COM2"}], total_pages=1),+ _page("caller_ids", [{"id": 13, "created_at": "2026-01-01T00:00:00Z"}], total_pages=1),+ _page("caller_ids", [{"id": 13, "created_at": "2026-01-02T00:00:00Z"}], total_pages=1),+ ],+ )++ assert result.raised is None+ assert result.paths[1:] == ["/v3/a/ACC/caller_ids.json"] * 2+ assert result.params("company_id")[1:] == ["COM1", "COM2"]+ assert result.requests[1].param("per_page") == "250"+ # Caller ID ids are not documented as unique across companies, so company_id joins the key.+ assert [(row["company_id"], row["id"]) for row in result.rows] == [("COM1", 13), ("COM2", 13)]+ assert not any(key.startswith("_companies_") for row in result.rows for key in row)+++_NOW = datetime(2026, 3, 15, 12, 0, tzinfo=UTC)+++def frozen_now():+ with time_machine.travel(_NOW, tick=False):+ yield++[email protected]("frozen_now")+class TestSmsThreadsEndpoint:+ def test_sends_an_explicit_range_covering_the_retention_period(self) -> None:+ result = _driver().run(+ "sms_threads",+ [+ _page("sms_threads", [{"id": "T1"}], total_pages=2),+ _page("sms_threads", [{"id": "T2"}], total_pages=2),+ ],+ )++ assert result.raised is None+ assert [row["id"] for row in result.rows] == ["T1", "T2"]+ assert result.paths == ["/v3/a/ACC/sms-threads.json"] * 2+ # Without dates the listing falls back to the last 7 days only.+ assert result.params("start_date") == ["2024-03-01", "2024-03-01"]+ assert result.params("end_date") == ["2026-03-15", "2026-03-15"]+ assert result.params("fields") == ["last_message_at", "last_message_at"]+ assert result.params("sort") == [None, None]+++def _timeseries(days: list[str]) -> ScriptedResponse:+ return ScriptedResponse(+ json={"total_results": {"total_calls": 1}, "data": [{"date": d, "total_calls": 1} for d in days]}+ )+++def _summary(keys: list[str]) -> ScriptedResponse:+ return ScriptedResponse(+ json={"grouped_by": "source", "grouped_results": [{"key": key, "total_calls": 2} for key in keys]}+ )++[email protected]("frozen_now")+class TestReportEndpoints:+ @pytest.mark.parametrize(+ "inputs, resume_state, expected_starts",+ [+ # An incremental sync re-reads the watermark's month, which may still have been partial.+ (+ {"should_use_incremental_field": True, "db_incremental_field_last_value": date(2026, 2, 10)},+ None,+ ["2026-02-01", "2026-03-01"],+ ),+ (+ {"should_use_incremental_field": True, "db_incremental_field_last_value": "2026-02-10"},+ None,+ ["2026-02-01", "2026-03-01"],+ ),+ # A resumed run starts at the saved window, not at the watermark.+ (+ {"should_use_incremental_field": True, "db_incremental_field_last_value": date(2025, 1, 1)},+ CallRailResumeConfig(account_id="ACC", window_start="2026-03-01"),+ ["2026-03-01"],+ ),+ ],+ )+ def test_window_start(+ self, inputs: dict[str, Any], resume_state: CallRailResumeConfig | None, expected_starts: list[str]+ ) -> None:+ result = _driver().run(+ "calls_timeseries", always(_timeseries(["2026-03-01"])), resume_state=resume_state, **inputs+ )++ assert result.raised is None+ assert result.params("start_date") == expected_starts++ @pytest.mark.parametrize(+ "inputs",+ [+ {},+ # A watermark older than CallRail's retention would make the API reject the range.+ {"should_use_incremental_field": True, "db_incremental_field_last_value": date(2020, 1, 1)},+ ],+ )+ def test_full_walk_covers_whole_months_inside_retention_up_to_today(self, inputs: dict[str, Any]) -> None:+ result = _driver().run("calls_timeseries", always(_timeseries(["2024-03-01"])), **inputs)++ assert result.raised is None+ starts = result.params("start_date")+ ends = result.params("end_date")+ assert len(starts) == 25+ assert (starts[0], ends[0]) == ("2024-03-01", "2024-03-31")+ assert (starts[11], ends[11]) == ("2025-02-01", "2025-02-28")+ assert (starts[-1], ends[-1]) == ("2026-03-01", "2026-03-15")+ assert result.params("interval")[0] == "day"+ assert result.params("page")[0] is None++ def test_window_never_reaches_past_the_date_in_the_earliest_timezone(self) -> None:+ # 00:30 UTC on March 1 is still February 28 for an account in Los Angeles.+ with time_machine.travel(datetime(2026, 3, 1, 0, 30, tzinfo=UTC), tick=False):+ result = _driver().run(+ "calls_timeseries",+ always(_timeseries(["2026-02-28"])),+ resume_state=CallRailResumeConfig(account_id="ACC", window_start="2026-02-01"),+ )++ assert result.raised is None+ assert result.params("start_date") == ["2026-02-01"]+ assert result.params("end_date") == ["2026-02-28"]++ @pytest.mark.parametrize(+ "endpoint, path",+ [+ ("calls_summary", "/v3/a/ACC/calls/summary.json"),+ ("form_submissions_summary", "/v3/a/ACC/forms/summary.json"),+ ],+ )+ def test_summary_stamps_each_row_with_its_month_and_skips_empty_months(self, endpoint: str, path: str) -> None:+ result = _driver().run(+ endpoint,+ [_summary(["Google Paid", "Direct"]), _summary([]), _summary(["Direct"])],+ resume_state=CallRailResumeConfig(account_id="ACC", window_start="2026-01-01"),+ )++ assert result.raised is None+ assert result.paths == [path] * 3+ assert result.params("group_by") == ["source"] * 3+ assert [(row["period_start"], row["period_end"], row["key"]) for row in result.rows] == [+ ("2026-01-01", "2026-01-31", "Google Paid"),+ ("2026-01-01", "2026-01-31", "Direct"),+ ("2026-03-01", "2026-03-15", "Direct"),+ ]+ # Each cursor points at the next month, and the current month leaves none to resume from.+ assert result.saved_states == [+ CallRailResumeConfig(account_id="ACC", window_start="2026-02-01"),+ CallRailResumeConfig(account_id="ACC", window_start="2026-03-01"),+ ]