Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 5 missing tables to the CallRail source (#119996)

Tom Owers61865731e7cc
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index c5324702a9dc..7e2e72f89bfb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -2139,7 +2139,7 @@ Note: developer.calendly.com is a Gatsby shell with no fetchable spec; the real
## CallRail — gaps
-Today (11): `accounts`, `calls`, `companies`, `form_submissions`, `lead_timelines`, `leads`, `page_views`, `tags`, `text_messages`, `trackers`, `users`
+Today (16): `accounts`, `calls`, `calls_summary`, `calls_timeseries`, `companies`, `form_submissions`, `form_submissions_summary`, `lead_timelines`, `leads`, `outbound_caller_ids`, `page_views`, `sms_threads`, `tags`, `text_messages`, `trackers`, `users`
Diffed against: <https://apidocs.callrail.com/>
@@ -2147,10 +2147,10 @@ Diffed against: <https://apidocs.callrail.com/>
- [x] `accounts (/a.json)` — lookup resolving the account_id every other resource is scoped under; required for multi-account (agency) reporting (high)
- [x] `page_views (/a/{account_id}/calls/{call_id}/page_views.json)` — per-call visitor page-view journey — the attribution path behind each tracked call (high)
- [x] `lead_timelines (/a/{account_id}/leads/{id}/timeline.json)` — state/transition history for a lead across calls, texts, and forms (medium)
-- [ ] `sms_threads (/a/{account_id}/sms_threads.json)` — thread-level SMS conversations, the parent grain the text_messages table hangs off (medium)
-- [ ] `calls summary and timeseries (/a/{account_id}/calls/summary.json, /calls/timeseries.json)` — vendor-computed call volume breakdowns by source and period, matching what the CallRail UI reports (medium)
-- [ ] `form_submissions summary (/a/{account_id}/form_submissions/summary.json)` — vendor-computed form conversion aggregates aligned with the call summary (low)
-- [ ] `outbound_caller_ids (/a/{account_id}/caller_ids.json)` — lookup resolving the caller ID numbers appearing on outbound calls (low)
+- [x] `sms_threads (/a/{account_id}/sms_threads.json)` — thread-level SMS conversations, the parent grain the text_messages table hangs off (medium)
+- [x] `calls summary and timeseries (/a/{account_id}/calls/summary.json, /calls/timeseries.json)` — vendor-computed call volume breakdowns by source and period, matching what the CallRail UI reports (medium)
+- [x] `form_submissions summary (/a/{account_id}/form_submissions/summary.json)` — vendor-computed form conversion aggregates aligned with the call summary (low)
+- [x] `outbound_caller_ids (/a/{account_id}/caller_ids.json)` — lookup resolving the caller ID numbers appearing on outbound calls (low)
Note: apidocs.callrail.com serves the entire v3 reference as one ~890 KB HTML page; I parsed its heading tree for the resource list rather than using the summarizer.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/callrail.py b/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/callrail.py
index efddcc514878..b21f4c09ae30 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/callrail.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/callrail.py
@@ -1,6 +1,6 @@
import dataclasses
from collections.abc import Iterable, Iterator
-from datetime import UTC, date, datetime
+from datetime import UTC, date, datetime, timedelta
from typing import Any, Optional, cast
from products.warehouse_sources.backend.temporal.data_imports.sources.callrail.settings import (
@@ -35,6 +35,10 @@ CALLRAIL_BASE_URL = "https://api.callrail.com/v3"
# forever. 250 rows/page * this cap bounds a single endpoint sync.
MAX_PAGES = 100_000
+# CallRail deletes communication records after 25 months and rejects a date range that reaches past
+# that, so date windows start at the first whole month inside it.
+RETENTION_MONTHS = 24
+
@dataclasses.dataclass(frozen=True)
class CallRailResumeConfig:
@@ -47,6 +51,8 @@ class CallRailResumeConfig:
# Fan-out progress for an endpoint reached once per parent row: which parents are done, which
# one was in flight, and that parent's page cursor. Shape owned by the shared fan-out helper.
fanout_state: Optional[dict[str, Any]] = None
+ # First day (YYYY-MM-DD) of the next monthly window to fetch, for a report endpoint.
+ window_start: Optional[str] = None
def _get_headers(api_key: str) -> dict[str, str]:
@@ -116,6 +122,40 @@ def _format_start_date(value: Any) -> str | None:
return None
+def _parse_cursor_date(value: Any) -> date | None:
+ formatted = _format_start_date(value)
+ return date.fromisoformat(formatted) if formatted else None
+
+
+def _retention_start(today: date) -> date:
+ month_index = today.year * 12 + today.month - 1 - RETENTION_MONTHS
+ return date(month_index // 12, month_index % 12 + 1, 1)
+
+
+def _reporting_today() -> date:
+ """Today's date in the earliest timezone on Earth (UTC-12).
+
+ Reports use the account's timezone, so UTC's date can be a day ahead of the account's. Asking
+ for that day returns a zero-filled row that would move the watermark past a still-partial month.
+ """
+ return (datetime.now(UTC) - timedelta(hours=12)).date()
+
+
[email protected](frozen=True)
+class _MonthWindow:
+ first_day: date
+ last_day: date
+
+
+def _month_windows(start: date, today: date) -> Iterator[_MonthWindow]:
+ """Yield each calendar month from `start` up to and including today, clipped at today."""
+ window_start = start.replace(day=1)
+ while window_start <= today:
+ next_start = (window_start + timedelta(days=32)).replace(day=1)
+ yield _MonthWindow(first_day=window_start, last_day=min(next_start - timedelta(days=1), today))
+ window_start = next_start
+
+
def resolve_account_id(api_key: str, team_id: int, job_id: str, account_id: str | None = None) -> str:
"""Return the account id to scope data requests to.
@@ -209,6 +249,70 @@ def _fanout_rows(
)
+def _report_rows(
+ api_key: str,
+ endpoint: str,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[CallRailResumeConfig],
+ resolved_account_id: str,
+ resume_window_start: Optional[str],
+ should_use_incremental_field: bool,
+ db_incremental_field_last_value: Any,
+) -> Iterator[list[dict[str, Any]]]:
+ """Fetch a vendor-computed report one calendar month per request, oldest month first.
+
+ A timeseries report holds at most 200 data points and a summary report covers its whole date
+ range, so a month is the unit that keeps both complete and gives summary rows a period. An
+ incremental sync starts again at the watermark's month, because the newest month is still
+ partial when it is read. The merge replaces its rows on the primary key.
+ """
+ config = CALLRAIL_ENDPOINTS[endpoint]
+ today = _reporting_today()
+ start = _retention_start(today)
+ if resume_window_start is not None:
+ start = max(start, date.fromisoformat(resume_window_start))
+ elif should_use_incremental_field:
+ watermark = _parse_cursor_date(db_incremental_field_last_value)
+ if watermark is not None:
+ start = max(start, watermark.replace(day=1))
+
+ for window in _month_windows(start, today):
+ window_start, window_end = window.first_day, window.last_day
+ rest_config: RESTAPIConfig = {
+ "client": _client_config(api_key),
+ "resources": [
+ {
+ "name": endpoint,
+ "endpoint": {
+ "path": config.path.replace("{account_id}", resolved_account_id),
+ "params": {
+ **config.extra_params,
+ "start_date": window_start.isoformat(),
+ "end_date": window_end.isoformat(),
+ },
+ "data_selector": config.response_key,
+ "paginator": SinglePagePaginator(),
+ },
+ }
+ ],
+ }
+ rows = [row for page in rest_api_resource(rest_config, team_id, job_id, None) for row in page]
+ if config.report == "summary":
+ period = {"period_start": window_start.isoformat(), "period_end": window_end.isoformat()}
+ rows = [{**row, **period} for row in rows]
+
+ next_start = window_end + timedelta(days=1)
+ if next_start <= today:
+ resumable_source_manager.save_state(
+ CallRailResumeConfig(account_id=resolved_account_id, window_start=next_start.isoformat())
+ )
+ if rows:
+ yield rows
+ else:
+ resumable_source_manager.safe_point()
+
+
def get_rows(
api_key: str,
endpoint: str,
@@ -243,13 +347,33 @@ def get_rows(
)
return
+ if config.report is not None:
+ yield from _report_rows(
+ api_key=api_key,
+ endpoint=endpoint,
+ team_id=team_id,
+ job_id=job_id,
+ resumable_source_manager=resumable_source_manager,
+ resolved_account_id=resolved_account_id,
+ resume_window_start=resume.window_start if resume is not None else None,
+ should_use_incremental_field=should_use_incremental_field,
+ db_incremental_field_last_value=db_incremental_field_last_value,
+ )
+ return
+
initial_paginator_state: Optional[dict[str, Any]] = None
if resume is not None and resume.page is not None:
initial_paginator_state = {"page": resume.page}
+ params: dict[str, Any] = {"per_page": PER_PAGE, **_sort_params(config), **config.extra_params}
+ if config.retention_window:
+ today = datetime.now(UTC).date()
+ params["start_date"] = _retention_start(today).isoformat()
+ params["end_date"] = today.isoformat()
+
endpoint_config: Endpoint = {
"path": config.path.replace("{account_id}", resolved_account_id),
- "params": {"per_page": PER_PAGE, **_sort_params(config)},
+ "params": params,
# Key the list lives under in the JSON envelope; a missing key reads as an empty page and
# ends pagination, matching the API's "no more data" signal.
"data_selector": config.response_key,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/canonical_descriptions.py
index 378a556d3f18..ac076068cd71 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/canonical_descriptions.py
@@ -40,6 +40,36 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"company_id": "Identifier of the company the call belongs to.",
},
},
+ "calls_summary": {
+ "description": "Call counts per marketing source for each calendar month, as computed by CallRail.",
+ "docs_url": "https://apidocs.callrail.com/#summarizing-call-data",
+ "columns": {
+ "period_start": "First day of the calendar month the row summarizes.",
+ "period_end": "Last day of the month the row summarizes, or the sync date for the current month.",
+ "key": "The marketing source the calls are attributed to.",
+ "total_calls": "Total number of calls in the period.",
+ "missed_calls": "Number of calls in the period that were missed.",
+ "answered_calls": "Number of calls in the period that were answered.",
+ "first_time_callers": "Number of calls in the period from first-time callers.",
+ "average_duration": "Average call duration in the period, in seconds.",
+ "formatted_average_duration": "Average call duration in the period, formatted for display.",
+ "leads": "Number of calls in the period marked as leads.",
+ },
+ },
+ "calls_timeseries": {
+ "description": "Daily call counts for the account, as computed by CallRail.",
+ "docs_url": "https://apidocs.callrail.com/#summarizing-call-data-by-time-series",
+ "columns": {
+ "date": "The day the row summarizes (YYYY-MM-DD), in the account time zone.",
+ "total_calls": "Total number of calls in the period.",
+ "missed_calls": "Number of calls in the period that were missed.",
+ "answered_calls": "Number of calls in the period that were answered.",
+ "first_time_callers": "Number of calls in the period from first-time callers.",
+ "average_duration": "Average call duration in the period, in seconds.",
+ "formatted_average_duration": "Average call duration in the period, formatted for display.",
+ "leads": "Number of calls in the period marked as leads.",
+ },
+ },
"companies": {
"description": "Companies configured in the CallRail account. A company groups trackers, calls, and form submissions.",
"docs_url": "https://apidocs.callrail.com/#companies",
@@ -67,6 +97,18 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"first_form": "Whether this was the person's first form submission.",
},
},
+ "form_submissions_summary": {
+ "description": "Form submission counts per marketing source for each calendar month, as computed by CallRail.",
+ "docs_url": "https://apidocs.callrail.com/#summarizing-form-data",
+ "columns": {
+ "period_start": "First day of the calendar month the row summarizes.",
+ "period_end": "Last day of the month the row summarizes, or the sync date for the current month.",
+ "key": "The marketing source the form submissions are attributed to.",
+ "total_forms": "Total number of form submissions in the period.",
+ "first_forms": "Number of form submissions in the period that were first-time submissions.",
+ "leads": "Number of form submissions in the period marked as leads.",
+ },
+ },
"leads": {
"description": "Leads in the CallRail account: the people derived from tracked calls, texts, and form submissions.",
"docs_url": "https://apidocs.callrail.com/#leads",
@@ -107,6 +149,19 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"touchpoint_type": "Attribution touchpoint behind the event, e.g. organic_search or paid_search.",
},
},
+ "outbound_caller_ids": {
+ "description": "Phone numbers registered to show on the recipient's caller ID for outbound calls placed through CallRail.",
+ "docs_url": "https://apidocs.callrail.com/#outbound-caller-ids",
+ "columns": {
+ "id": "Unique identifier for the outbound caller ID.",
+ "company_id": "Identifier of the company the caller ID belongs to.",
+ "phone_number": "The phone number, in E.164 format.",
+ "name": "Descriptive name of the caller ID.",
+ "verified": "Whether the phone number has been verified in CallRail.",
+ "created_at": "When the outbound caller ID was created.",
+ "validation_code": "Code used to verify ownership of the phone number.",
+ },
+ },
"page_views": {
"description": "Browsing history behind a tracked call: the pages a visitor viewed before dialing, newest first. Only recorded for calls placed to a session tracker, and synced once per call.",
"docs_url": "https://apidocs.callrail.com/#page-views",
@@ -117,6 +172,26 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"created_at": "When the page view was recorded (ISO 8601).",
},
},
+ "sms_threads": {
+ "description": "SMS conversation threads between customers and the business, without the individual messages.",
+ "docs_url": "https://apidocs.callrail.com/#sms-threads",
+ "columns": {
+ "id": "Unique identifier for the SMS thread.",
+ "company_id": "Identifier of the company the thread belongs to.",
+ "customer_name": "Name of the customer.",
+ "customer_phone_number": "Phone number of the customer, in E.164 format.",
+ "initial_tracker_id": "Tracker whose number started the conversation.",
+ "current_tracker_id": "Tracker whose number was used most recently in the conversation.",
+ "initial_tracking_number": "Tracking number that started the conversation.",
+ "current_tracking_number": "Tracking number used most recently in the conversation.",
+ "lead_qualification": "Lead status of the thread: good_lead, not_a_lead, or empty.",
+ "value": "Value assigned to the thread.",
+ "tags": "Names of the tags applied to the thread.",
+ "notes": "Notes on the thread.",
+ "state": "Whether the thread is active or archived.",
+ "last_message_at": "When the most recent message in the thread was sent or received.",
+ },
+ },
"text_messages": {
"description": "SMS conversations between callers and the business, captured by CallRail.",
"docs_url": "https://apidocs.callrail.com/#text-messages",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/settings.py
index d6a70881161b..a6d35964c186 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/settings.py
@@ -1,5 +1,5 @@
from dataclasses import dataclass, field
-from typing import Optional
+from typing import Literal, Optional
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
DependentEndpointConfig,
@@ -16,6 +16,10 @@ PER_PAGE = 250
# exist for calls placed to a session tracker.
_CHILD_404_IGNORE: list[ResponseAction] = [{"status_code": 404, "action": "ignore"}]
+_CALL_SUMMARY_FIELDS = (
+ "total_calls,missed_calls,answered_calls,first_time_callers,average_duration,formatted_average_duration,leads"
+)
+
@dataclass(frozen=True)
class CallRailEndpointConfig:
@@ -40,6 +44,15 @@ class CallRailEndpointConfig:
should_sync_default: bool = True
# Set for endpoints reached once per parent row rather than once per account.
fanout: Optional[DependentEndpointConfig] = None
+ # Static query params, such as `fields=` for columns the API only returns on request.
+ extra_params: dict[str, str] = field(default_factory=dict)
+ # Set for listings that fall back to CallRail's `recent` window (the last 7 days) when sent no
+ # dates, so they get an explicit range that covers the retention period instead.
+ retention_window: bool = False
+ # Set for the vendor-computed report endpoints, which take one calendar-month date window per
+ # request instead of pages. A "summary" report carries no date on its rows, so each row gets the
+ # window it covers.
+ report: Optional[Literal["timeseries", "summary"]] = None
@property
def requires_account(self) -> bool:
@@ -65,6 +78,15 @@ _PAGE_VIEWS_FANOUT = DependentEndpointConfig(
child_response_actions=_CHILD_404_IGNORE,
)
+_CALLER_IDS_FANOUT = DependentEndpointConfig(
+ parent_name="companies",
+ resolve_param="company_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "company_id"},
+ child_response_actions=_CHILD_404_IGNORE,
+)
+
_LEAD_TIMELINE_FANOUT = DependentEndpointConfig(
parent_name="leads",
resolve_param="lead_id",
@@ -82,6 +104,8 @@ _LEAD_TIMELINE_FANOUT = DependentEndpointConfig(
# `start_date` date filter that narrows on a stable timestamp (Calls -> start_time,
# Form submissions -> submitted_at). For those we advertise exactly that one field as the
# incremental cursor so the cursor, the `sort=` field, and the `start_date` filter all agree.
+# The report endpoints (calls and form summaries, calls timeseries) take the same date filter and
+# advance on the date of the window they cover.
# The remaining resources are mutable configuration objects or lack a usable server-side date
# filter, so they ship full refresh only.
CALLRAIL_ENDPOINTS: dict[str, CallRailEndpointConfig] = {
@@ -110,6 +134,40 @@ CALLRAIL_ENDPOINTS: dict[str, CallRailEndpointConfig] = {
},
],
),
+ "calls_summary": CallRailEndpointConfig(
+ name="calls_summary",
+ path="/a/{account_id}/calls/summary.json",
+ response_key="grouped_results",
+ primary_keys=["period_start", "key"],
+ supports_incremental=True,
+ report="summary",
+ extra_params={"group_by": "source", "fields": _CALL_SUMMARY_FIELDS},
+ incremental_fields=[
+ {
+ "label": "period_start",
+ "type": IncrementalFieldType.Date,
+ "field": "period_start",
+ "field_type": IncrementalFieldType.Date,
+ },
+ ],
+ ),
+ "calls_timeseries": CallRailEndpointConfig(
+ name="calls_timeseries",
+ path="/a/{account_id}/calls/timeseries.json",
+ response_key="data",
+ primary_keys=["date"],
+ supports_incremental=True,
+ report="timeseries",
+ extra_params={"interval": "day", "fields": _CALL_SUMMARY_FIELDS},
+ incremental_fields=[
+ {
+ "label": "date",
+ "type": IncrementalFieldType.Date,
+ "field": "date",
+ "field_type": IncrementalFieldType.Date,
+ },
+ ],
+ ),
"companies": CallRailEndpointConfig(
name="companies",
path="/a/{account_id}/companies.json",
@@ -133,6 +191,23 @@ CALLRAIL_ENDPOINTS: dict[str, CallRailEndpointConfig] = {
},
],
),
+ "form_submissions_summary": CallRailEndpointConfig(
+ name="form_submissions_summary",
+ path="/a/{account_id}/forms/summary.json",
+ response_key="grouped_results",
+ primary_keys=["period_start", "key"],
+ supports_incremental=True,
+ report="summary",
+ extra_params={"group_by": "source", "fields": "total_forms,first_forms,leads"},
+ incremental_fields=[
+ {
+ "label": "period_start",
+ "type": IncrementalFieldType.Date,
+ "field": "period_start",
+ "field_type": IncrementalFieldType.Date,
+ },
+ ],
+ ),
"leads": CallRailEndpointConfig(
name="leads",
path="/a/{account_id}/leads.json",
@@ -170,6 +245,20 @@ CALLRAIL_ENDPOINTS: dict[str, CallRailEndpointConfig] = {
},
],
),
+ "outbound_caller_ids": CallRailEndpointConfig(
+ name="outbound_caller_ids",
+ # `company_id` is a required query param, bound per company row by the fan-out.
+ path="/a/{account_id}/caller_ids.json?company_id={company_id}",
+ response_key="caller_ids",
+ partition_key="created_at",
+ primary_keys=["company_id", "id"],
+ fanout=_CALLER_IDS_FANOUT,
+ # One request per company against a 1,000/hour account-wide budget, so leave it to the user
+ # to opt in rather than enabling it on every new connection.
+ should_sync_default=False,
+ # The parent companies listing is not windowed, so a full refresh sees every caller ID.
+ incremental_fields=[],
+ ),
"page_views": CallRailEndpointConfig(
name="page_views",
path="/a/{account_id}/calls/{call_id}/page_views.json",
@@ -194,6 +283,16 @@ CALLRAIL_ENDPOINTS: dict[str, CallRailEndpointConfig] = {
},
],
),
+ "sms_threads": CallRailEndpointConfig(
+ name="sms_threads",
+ path="/a/{account_id}/sms-threads.json",
+ response_key="sms_threads",
+ # The listing takes no sort param and returns the thread with the newest message first. Its
+ # date filter is not documented as narrowing on a stable field, so this is full refresh.
+ retention_window=True,
+ extra_params={"fields": "last_message_at"},
+ incremental_fields=[],
+ ),
"text_messages": CallRailEndpointConfig(
name="text_messages",
# Returns SMS conversations under the "conversations" key.
@@ -229,6 +328,8 @@ INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
SHOULD_SYNC_DEFAULT: dict[str, bool] = {name: config.should_sync_default for name, config in CALLRAIL_ENDPOINTS.items()}
-# A fan-out child accumulates one row per parent row per sync under append, so these tables offer
-# incremental merge only.
-MERGE_ONLY = tuple(name for name, config in CALLRAIL_ENDPOINTS.items() if config.fanout is not None)
+# A fan-out child accumulates one row per parent row per sync under append, and a report re-reads
+# the current month on every sync, so these tables offer incremental merge only.
+MERGE_ONLY = tuple(
+ name for name, config in CALLRAIL_ENDPOINTS.items() if config.fanout is not None or config.report is not None
+)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/tests/test_callrail.py b/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/tests/test_callrail.py
index ed978dc9cc0f..c9227bb71e2f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/tests/test_callrail.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/callrail/tests/test_callrail.py
@@ -2,6 +2,7 @@ from datetime import UTC, date, datetime
from typing import Any
import pytest
+import time_machine
from products.warehouse_sources.backend.temporal.data_imports.sources.callrail.callrail import (
CallRailResumeConfig,
@@ -17,6 +18,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.callrail.s
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
ScriptedResponse,
SourceDriver,
+ always,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.callrail import (
@@ -139,7 +141,7 @@ class TestCallRailSourceResponse:
@pytest.mark.parametrize("endpoint", list(ENDPOINTS))
def test_response_metadata_per_endpoint(self, endpoint: str) -> None:
config = CALLRAIL_ENDPOINTS[endpoint]
- result = _driver().run(endpoint, [ScriptedResponse(json={"total_pages": 1})])
+ result = _driver().run(endpoint, always(ScriptedResponse(json={"total_pages": 1})))
response = result.response
assert result.raised is None
@@ -163,7 +165,7 @@ class TestCallRailSourceResponse:
@pytest.mark.parametrize("config", list(CALLRAIL_ENDPOINTS.values()))
def test_incremental_endpoints_have_a_sort_field(self, config: Any) -> None:
# An incremental endpoint must sort ascending on its cursor so the watermark advances.
- if config.supports_incremental:
+ if config.supports_incremental and config.report is None:
assert config.sort_field is not None
assert config.incremental_fields
@@ -244,3 +246,162 @@ class TestFanoutEndpoints:
assert result.raised is None
assert result.requests[1].param("start_date") is None
assert result.requests[1].param("created_at") is None
+
+
+class TestOutboundCallerIdsEndpoint:
+ def test_binds_each_company_as_the_required_query_param(self) -> None:
+ result = _driver().run(
+ "outbound_caller_ids",
+ [
+ _page("companies", [{"id": "COM1"}, {"id": "COM2"}], total_pages=1),
+ _page("caller_ids", [{"id": 13, "created_at": "2026-01-01T00:00:00Z"}], total_pages=1),
+ _page("caller_ids", [{"id": 13, "created_at": "2026-01-02T00:00:00Z"}], total_pages=1),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.paths[1:] == ["/v3/a/ACC/caller_ids.json"] * 2
+ assert result.params("company_id")[1:] == ["COM1", "COM2"]
+ assert result.requests[1].param("per_page") == "250"
+ # Caller ID ids are not documented as unique across companies, so company_id joins the key.
+ assert [(row["company_id"], row["id"]) for row in result.rows] == [("COM1", 13), ("COM2", 13)]
+ assert not any(key.startswith("_companies_") for row in result.rows for key in row)
+
+
+_NOW = datetime(2026, 3, 15, 12, 0, tzinfo=UTC)
+
+
+def frozen_now():
+ with time_machine.travel(_NOW, tick=False):
+ yield
+
+
[email protected]("frozen_now")
+class TestSmsThreadsEndpoint:
+ def test_sends_an_explicit_range_covering_the_retention_period(self) -> None:
+ result = _driver().run(
+ "sms_threads",
+ [
+ _page("sms_threads", [{"id": "T1"}], total_pages=2),
+ _page("sms_threads", [{"id": "T2"}], total_pages=2),
+ ],
+ )
+
+ assert result.raised is None
+ assert [row["id"] for row in result.rows] == ["T1", "T2"]
+ assert result.paths == ["/v3/a/ACC/sms-threads.json"] * 2
+ # Without dates the listing falls back to the last 7 days only.
+ assert result.params("start_date") == ["2024-03-01", "2024-03-01"]
+ assert result.params("end_date") == ["2026-03-15", "2026-03-15"]
+ assert result.params("fields") == ["last_message_at", "last_message_at"]
+ assert result.params("sort") == [None, None]
+
+
+def _timeseries(days: list[str]) -> ScriptedResponse:
+ return ScriptedResponse(
+ json={"total_results": {"total_calls": 1}, "data": [{"date": d, "total_calls": 1} for d in days]}
+ )
+
+
+def _summary(keys: list[str]) -> ScriptedResponse:
+ return ScriptedResponse(
+ json={"grouped_by": "source", "grouped_results": [{"key": key, "total_calls": 2} for key in keys]}
+ )
+
+
[email protected]("frozen_now")
+class TestReportEndpoints:
+ @pytest.mark.parametrize(
+ "inputs, resume_state, expected_starts",
+ [
+ # An incremental sync re-reads the watermark's month, which may still have been partial.
+ (
+ {"should_use_incremental_field": True, "db_incremental_field_last_value": date(2026, 2, 10)},
+ None,
+ ["2026-02-01", "2026-03-01"],
+ ),
+ (
+ {"should_use_incremental_field": True, "db_incremental_field_last_value": "2026-02-10"},
+ None,
+ ["2026-02-01", "2026-03-01"],
+ ),
+ # A resumed run starts at the saved window, not at the watermark.
+ (
+ {"should_use_incremental_field": True, "db_incremental_field_last_value": date(2025, 1, 1)},
+ CallRailResumeConfig(account_id="ACC", window_start="2026-03-01"),
+ ["2026-03-01"],
+ ),
+ ],
+ )
+ def test_window_start(
+ self, inputs: dict[str, Any], resume_state: CallRailResumeConfig | None, expected_starts: list[str]
+ ) -> None:
+ result = _driver().run(
+ "calls_timeseries", always(_timeseries(["2026-03-01"])), resume_state=resume_state, **inputs
+ )
+
+ assert result.raised is None
+ assert result.params("start_date") == expected_starts
+
+ @pytest.mark.parametrize(
+ "inputs",
+ [
+ {},
+ # A watermark older than CallRail's retention would make the API reject the range.
+ {"should_use_incremental_field": True, "db_incremental_field_last_value": date(2020, 1, 1)},
+ ],
+ )
+ def test_full_walk_covers_whole_months_inside_retention_up_to_today(self, inputs: dict[str, Any]) -> None:
+ result = _driver().run("calls_timeseries", always(_timeseries(["2024-03-01"])), **inputs)
+
+ assert result.raised is None
+ starts = result.params("start_date")
+ ends = result.params("end_date")
+ assert len(starts) == 25
+ assert (starts[0], ends[0]) == ("2024-03-01", "2024-03-31")
+ assert (starts[11], ends[11]) == ("2025-02-01", "2025-02-28")
+ assert (starts[-1], ends[-1]) == ("2026-03-01", "2026-03-15")
+ assert result.params("interval")[0] == "day"
+ assert result.params("page")[0] is None
+
+ def test_window_never_reaches_past_the_date_in_the_earliest_timezone(self) -> None:
+ # 00:30 UTC on March 1 is still February 28 for an account in Los Angeles.
+ with time_machine.travel(datetime(2026, 3, 1, 0, 30, tzinfo=UTC), tick=False):
+ result = _driver().run(
+ "calls_timeseries",
+ always(_timeseries(["2026-02-28"])),
+ resume_state=CallRailResumeConfig(account_id="ACC", window_start="2026-02-01"),
+ )
+
+ assert result.raised is None
+ assert result.params("start_date") == ["2026-02-01"]
+ assert result.params("end_date") == ["2026-02-28"]
+
+ @pytest.mark.parametrize(
+ "endpoint, path",
+ [
+ ("calls_summary", "/v3/a/ACC/calls/summary.json"),
+ ("form_submissions_summary", "/v3/a/ACC/forms/summary.json"),
+ ],
+ )
+ def test_summary_stamps_each_row_with_its_month_and_skips_empty_months(self, endpoint: str, path: str) -> None:
+ result = _driver().run(
+ endpoint,
+ [_summary(["Google Paid", "Direct"]), _summary([]), _summary(["Direct"])],
+ resume_state=CallRailResumeConfig(account_id="ACC", window_start="2026-01-01"),
+ )
+
+ assert result.raised is None
+ assert result.paths == [path] * 3
+ assert result.params("group_by") == ["source"] * 3
+ assert [(row["period_start"], row["period_end"], row["key"]) for row in result.rows] == [
+ ("2026-01-01", "2026-01-31", "Google Paid"),
+ ("2026-01-01", "2026-01-31", "Direct"),
+ ("2026-03-01", "2026-03-15", "Direct"),
+ ]
+ # Each cursor points at the next month, and the current month leaves none to resume from.
+ assert result.saved_states == [
+ CallRailResumeConfig(account_id="ACC", window_start="2026-02-01"),
+ CallRailResumeConfig(account_id="ACC", window_start="2026-03-01"),
+ ]