:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): add aircall campaign and tracker tables (#126938)
Tom Owersd134c170374f
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 6d23f4e499f2..763d73aa3cab 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -324,7 +324,7 @@ Note: Airbrake's readable v4 surface is small and coverage is close to complete:## Aircall — gaps-Today (10): `call_evaluations`, `call_sentiments`, `call_topics`, `call_transcriptions`, `calls`, `contacts`, `numbers`, `tags`, `teams`, `users`+Today (15): `call_evaluations`, `call_sentiments`, `call_topics`, `call_transcriptions`, `calls`, `campaign_call_outcomes`, `campaign_outcomes`, `campaign_stats`, `campaigns`, `contacts`, `numbers`, `tags`, `teams`, `trackers`, `users`Diffed against: <https://developer.aircall.io/api-references/>@@ -338,6 +338,11 @@ Diffed against: <https://developer.aircall.io/api-references/>- [ ] `user availabilities (GET /v1/users/availabilities)` — agent availability state, needed for staffing and utilization analysis (medium)- [ ] `analytics report export (POST/GET /v1/analytics/report/export)` — Aircall's own aggregated analytics reports (volumes, service level) as a ready-made fact table (medium)- [ ] `company (GET /v1/company)` — account-level lookup row giving the tenant these calls belong to (low)+- [x] `campaigns (GET /v1/campaigns)` — outbound Campaigns, the dimension every Campaign call outcome hangs off (medium)+- [x] `campaign stats (GET /v1/campaigns/{id}/stats)` — per-Campaign progress snapshot (contacts done, connected, average duration) (medium)+- [x] `campaign call outcomes (GET /v1/campaigns/{id}/call_outcomes)` — disposition log per Campaign, joinable to calls (medium)+- [x] `campaign outcomes (GET /v1/campaign_outcomes)` — lookup decoding custom outcome ids on Campaign call outcomes (low)+- [x] `trackers (GET /v1/trackers)` — Conversation Intelligence keyword trackers configured for the company (low)Note: The Conversation Intelligence endpoints are all per-call sub-resources, so they require fanning out over the already-synced calls table rather than a top-level list call. Aircall also has Users V2 alongside V1 and a Messages/WhatsApp surface that is send-only (inbound arrives via webhooks), so there is no listable conversations table to sync.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aircall/aircall.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aircall/aircall.pyindex 3235fa8f605a..41d0623f61d1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aircall/aircall.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aircall/aircall.py@@ -103,7 +103,7 @@ def _reaches_record_cap(url: str) -> bool:def _build_params(config: AircallEndpointConfig, from_value: Optional[int]) -> dict[str, Any]:- params: dict[str, Any] = {"per_page": PAGE_SIZE}+ params: dict[str, Any] = {"per_page": PAGE_SIZE} if config.paginated else {}# Ascending creation order keeps already-fetched pages stable and lets the incremental# watermark advance monotonically. `from` filters on the resource creation date.if config.incremental_fields or config.reanchor_field:@@ -254,18 +254,21 @@ def _fanout_source(team_id: int,job_id: str,) -> SourceResponse:- """Fan a per-call Conversation Intelligence endpoint out over the calls list.+ """Fan a per-call or per-campaign sub-resource out over its parent list.- The parent calls list is re-fetched each sync (there is no incremental filter on the child- sub-resource), so the parent keeps the calls paginator to page past the 10k cap and reach- every call. The child returns one flat object per call, so it needs a single-page paginator- and the whole body as the row — the parent and child therefore run different paginators,- set per endpoint rather than on the shared client.+ The parent list is re-fetched each sync (there is no incremental filter on the child+ sub-resources), so the parent keeps its own paginator. For calls that paginator pages past the+ 10k cap. A child that returns one object per parent needs a single-page paginator, while a+ paginated child follows its own next-page links. The parent and child therefore run different+ paginators, set per endpoint rather than on the shared client."""config = AIRCALL_ENDPOINTS[endpoint]assert config.fanout is not None- calls_config = AIRCALL_ENDPOINTS[config.fanout.parent_name]- parent_paginator = AircallPaginator(calls_config, calls_config.reanchor_field, from_value=None)+ parent_config = AIRCALL_ENDPOINTS[config.fanout.parent_name]+ parent_paginator = AircallPaginator(parent_config, parent_config.reanchor_field, from_value=None)+ child_paginator: BasePaginator = (+ AircallPaginator(config, cursor_field=None, from_value=None) if config.paginated else SinglePagePaginator()+ )dependent_resource = cast(Iterable[Any],@@ -281,8 +284,8 @@ def _fanout_source(db_incremental_field_last_value=None,should_use_incremental_field=False,page_size_param="per_page",- parent_endpoint_extra={"paginator": parent_paginator, "data_selector": calls_config.data_key},- child_endpoint_extra={"paginator": SinglePagePaginator(), "data_selector": config.data_key},+ parent_endpoint_extra={"paginator": parent_paginator, "data_selector": parent_config.data_key},+ child_endpoint_extra={"paginator": child_paginator, "data_selector": config.data_key},),)@@ -290,8 +293,9 @@ def _fanout_source(name=endpoint,items=lambda: dependent_resource,primary_keys=[config.primary_key],- # Child rows arrive in the parent's ascending-by-started_at order.- sort_mode="asc",+ # Child rows arrive in the parent's order, which is ascending only for a parent walked+ # by its re-anchor field (calls by started_at). The campaigns list has no documented order.+ sort_mode="asc" if parent_config.reanchor_field else None,partition_count=1,partition_size=1,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aircall/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aircall/canonical_descriptions.pyindex b21a6e2fbd94..9078a4ef4c6e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aircall/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aircall/canonical_descriptions.py@@ -143,4 +143,88 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"updated_at": "Time the evaluation was last updated, as an ISO 8601 timestamp.",},},+ "campaigns": {+ "description": "An outbound calling Campaign that dials a list of contacts with human agents or an AI Voice Agent.",+ "docs_url": "https://developer.aircall.io/api-references/#campaign",+ "columns": {+ "id": "Unique identifier for the Campaign (UUID).",+ "direct_link": "Direct API URL of the Campaign.",+ "name": "The Campaign's name.",+ "description": "The Campaign's description.",+ "status": "Lifecycle status: draft, in_progress, paused, completed, or archived.",+ "run_mode": "one_time Campaigns complete once every contact is resolved; continuous ones never auto-complete.",+ "caller_id_strategy": "How the caller ID is chosen for outbound calls: agent_number_pool or specific_number_pool.",+ "local_presence": "Whether the Campaign matches the contact's area code when dialling where possible.",+ "call_attempts": "Maximum number of times a contact is called before being marked unreachable.",+ "min_interval_between_attempts": "Minimum time, in seconds, between two attempts to the same contact.",+ "external_ringing_timeout": "How long, in seconds, a call rings before being considered unanswered.",+ "campaign_outcome_time": "How long, in seconds, an agent has to record a call outcome after a call ends.",+ "end_date": "Date the Campaign is set to stop, in UTC.",+ "outcome_ids": "Ids of the Campaign Outcomes agents can pick from when wrapping up a call.",+ "agent_type": "Who places the calls: human or aiva (AI Voice Agent).",+ "aiva_agent_id": "The AI Voice Agent placing the calls, null for human agent Campaigns.",+ "calling_hours": "The weekly window an AI Voice Agent Campaign may dial in, with its timezone and slots.",+ "created_by": "Aircall user id of whoever created the Campaign, null when created through the API.",+ "created_at": "Campaign's creation date, in UTC.",+ "updated_at": "Campaign's last update date, in UTC.",+ },+ },+ "campaign_outcomes": {+ "description": "The company-wide catalogue of custom call outcomes agents can record on Campaign calls.",+ "docs_url": "https://developer.aircall.io/api-references/#campaign-outcome",+ "columns": {+ "id": "Unique identifier for the Campaign Outcome (UUID).",+ "name": "The outcome's display name.",+ "color": "Display colour chosen in the dashboard, as a hex string.",+ "created_at": "Campaign Outcome's creation date, in UTC.",+ "updated_at": "Campaign Outcome's last update date, in UTC.",+ },+ },+ "campaign_stats": {+ "description": "Progress statistics for a Campaign, computed when the table syncs, one row per Campaign.",+ "docs_url": "https://developer.aircall.io/api-references/#retrieve-campaign-statistics",+ "columns": {+ "campaign_id": "Identifier of the Campaign these statistics belong to.",+ "total_contacts": "Contacts in the Campaign, queued or not.",+ "completed_percentage": "Share of contacts that have gone through the Campaign, from 0 to 100.",+ "connected_contacts": "Contacts who answered at least one call.",+ "average_call_duration": "Average duration, in seconds, of calls that connected.",+ "active_time": "Total time, in seconds, the Campaign has spent in the in_progress status.",+ "active_at": "When the Campaign most recently entered in_progress, in UTC.",+ "active_calls_count": "Calls this Campaign had in progress when the table synced.",+ },+ },+ "campaign_call_outcomes": {+ "description": "The disposition log of a Campaign: the latest call attempt per contact and the outcome recorded for it.",+ "docs_url": "https://developer.aircall.io/api-references/#list-campaign-call-outcomes",+ "columns": {+ "id": "Unique identifier for this outcome entry (UUID).",+ "campaign_id": "Identifier of the Campaign the attempt belongs to.",+ "call_id": "The Aircall call id, joinable to the calls table. Null before a call is placed.",+ "phone_number": "The number dialled.",+ "attempt_number": "Which attempt this is for this contact, starting at 1.",+ "max_attempts": "The Campaign's call_attempts setting at the time of this attempt.",+ "is_final": "Whether this attempt ended this contact's retries.",+ "status": "The attempt's status.",+ "duration": "Call duration in seconds, null if the call never connected.",+ "outcome": "The recorded outcome: a predefined code, or a custom outcome with its id, name, and color.",+ "occurred_at": "When this attempt happened, in UTC.",+ "answered_at": "When the call was answered, in UTC.",+ "ended_at": "When the call ended, in UTC.",+ },+ },+ "trackers": {+ "description": "Conversation Intelligence keyword trackers that flag calls matching a set of terms. Requires the AI Assist add-on.",+ "docs_url": "https://developer.aircall.io/api-references/#trackers",+ "columns": {+ "id": "Unique identifier for the Tracker.",+ "title": "Name of the Tracker.",+ "type": "Type of the Tracker. Only keywords is supported.",+ "filters": "Matching criteria: query terms, excluded terms, call direction, speaker type, numbers, and users.",+ "created_at": "Time the Tracker was created, in UTC.",+ "created_by": "Identifier of the user who created the Tracker.",+ "updated_at": "Time the Tracker was last updated, in UTC.",+ "updated_by": "Identifier of the user who last updated the Tracker.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aircall/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aircall/settings.pyindex 994c0361ec28..5355178a76db 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aircall/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aircall/settings.py@@ -31,6 +31,9 @@ class AircallEndpointConfig:# applies to.reanchor_field: Optional[str] = Nonepage_size: int = PAGE_SIZE+ # False for an endpoint that answers with a single object or an unpaginated list, which+ # takes no `per_page` param and needs no next-page handling.+ paginated: bool = True# Unused by the full-refresh CI children; kept so the config satisfies FanoutEndpointLike.default_incremental_field: Optional[str] = Nonefanout: Optional[DependentEndpointConfig] = None@@ -56,6 +59,18 @@ _CALL_CI_FANOUT = DependentEndpointConfig()+# The Campaign sub-resources are fetched per campaign, so they fan out over the campaigns list.+# A campaign deleted between the parent listing and this fetch answers 404, which is skipped.+_CAMPAIGN_FANOUT = DependentEndpointConfig(+ parent_name="campaigns",+ resolve_param="campaign_id",+ resolve_field="id",+ include_from_parent=["id"],+ parent_field_renames={"id": "campaign_id"},+ child_response_actions=_CI_404_IGNORE,+)++# Aircall timestamps are UNIX epoch seconds, so candidate incremental fields are stored as# integers even though the UI presents them as datetimes. The `from`/`to` list filters take# UNIX timestamps and filter on the resource's creation date, so incremental sync is only@@ -123,6 +138,7 @@ AIRCALL_ENDPOINTS: dict[str, AircallEndpointConfig] = {path="/calls/{call_id}/transcription",data_key="$",primary_key="call_id",+ paginated=False,fanout=_CALL_CI_FANOUT,),"call_sentiments": AircallEndpointConfig(@@ -130,6 +146,7 @@ AIRCALL_ENDPOINTS: dict[str, AircallEndpointConfig] = {path="/calls/{call_id}/sentiments",data_key="$",primary_key="call_id",+ paginated=False,fanout=_CALL_CI_FANOUT,),"call_topics": AircallEndpointConfig(@@ -137,6 +154,7 @@ AIRCALL_ENDPOINTS: dict[str, AircallEndpointConfig] = {path="/calls/{call_id}/topics",data_key="$",primary_key="call_id",+ paginated=False,fanout=_CALL_CI_FANOUT,),"call_evaluations": AircallEndpointConfig(@@ -144,8 +162,45 @@ AIRCALL_ENDPOINTS: dict[str, AircallEndpointConfig] = {path="/calls/{call_id}/evaluations",data_key="$",primary_key="call_id",+ paginated=False,fanout=_CALL_CI_FANOUT,),+ # Outbound Campaigns. Campaign ids are UUIDs and the list pages with a forward-only cursor+ # carried in `meta.next_page_link`. There is no `from` filter, so these are full refresh.+ "campaigns": AircallEndpointConfig(+ name="campaigns",+ path="/campaigns",+ data_key="campaigns",+ ),+ "campaign_outcomes": AircallEndpointConfig(+ name="campaign_outcomes",+ path="/campaign_outcomes",+ data_key="campaign_outcomes",+ ),+ # A point-in-time progress snapshot computed on read, one row per campaign.+ "campaign_stats": AircallEndpointConfig(+ name="campaign_stats",+ path="/campaigns/{campaign_id}/stats",+ data_key="stats",+ primary_key="campaign_id",+ paginated=False,+ fanout=_CAMPAIGN_FANOUT,+ ),+ # The latest attempt per contact, keyed by the entry's UUID.+ "campaign_call_outcomes": AircallEndpointConfig(+ name="campaign_call_outcomes",+ path="/campaigns/{campaign_id}/call_outcomes",+ data_key="call_outcomes",+ fanout=_CAMPAIGN_FANOUT,+ ),+ # Conversation Intelligence keyword trackers. The list is unpaginated and needs the AI Assist+ # add-on, like the per-call CI tables.+ "trackers": AircallEndpointConfig(+ name="trackers",+ path="/trackers",+ data_key="trackers",+ paginated=False,+ ),}ENDPOINTS = tuple(AIRCALL_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aircall/tests/test_aircall.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aircall/tests/test_aircall.pyindex 50819f747e8a..8fa364ae8ff8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aircall/tests/test_aircall.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aircall/tests/test_aircall.py@@ -95,6 +95,13 @@ class TestPagination:# State is saved only while a next page exists.assert result.saved_states == [AircallResumeConfig(next_url="https://api.aircall.io/v1/users?page=2")]+ def test_unpaginated_endpoint_sends_no_page_size(self) -> None:+ result = _driver().run("trackers", [ScriptedResponse(json={"trackers": [{"id": 1}, {"id": 2}]})])++ assert result.raised is None+ assert [row["id"] for row in result.rows] == [1, 2]+ assert result.urls == ["https://api.aircall.io/v1/trackers"]+def test_resumes_from_saved_state(self) -> None:resume_url = "https://api.aircall.io/v1/users?page=5"result = _driver().run(@@ -215,11 +222,47 @@ class TestConversationIntelligenceFanout:assert [row["call_id"] for row in result.rows] == [2]+class TestCampaignFanout:+ def test_call_outcomes_follow_each_campaigns_cursor(self) -> None:+ # Unlike the per-call CI children, a campaign's call outcomes are a cursor-paginated list,+ # so each campaign's chain is walked to its end before the next campaign starts.+ result = _driver().run(+ "campaign_call_outcomes",+ [+ _page("campaigns", [{"id": "c1"}, {"id": "c2"}], None),+ _page("call_outcomes", [{"id": "o1"}], "https://api.aircall.io/v1/campaigns/c1/call_outcomes?cursor=x"),+ _page("call_outcomes", [{"id": "o2"}], None),+ _page("call_outcomes", [{"id": "o3"}], None),+ ],+ )++ assert result.raised is None+ assert [(row["campaign_id"], row["id"]) for row in result.rows] == [("c1", "o1"), ("c1", "o2"), ("c2", "o3")]+ assert result.urls[1:] == [+ "https://api.aircall.io/v1/campaigns/c1/call_outcomes?per_page=50",+ "https://api.aircall.io/v1/campaigns/c1/call_outcomes?cursor=x",+ "https://api.aircall.io/v1/campaigns/c2/call_outcomes?per_page=50",+ ]++ def test_stats_unwraps_object_and_skips_deleted_campaigns(self) -> None:+ result = _driver().run(+ "campaign_stats",+ [+ _page("campaigns", [{"id": "c1"}, {"id": "c2"}], None),+ ScriptedResponse(status=404, json={"error": "not found"}),+ ScriptedResponse(json={"stats": {"total_contacts": 500, "connected_contacts": 130}}),+ ],+ )++ assert result.raised is None+ assert result.rows == [{"total_contacts": 500, "connected_contacts": 130, "campaign_id": "c2"}]++class TestAircallSourceResponse:@pytest.mark.parametrize("endpoint", list(ENDPOINTS))def test_response_metadata_per_endpoint(self, endpoint: str) -> None:config = AIRCALL_ENDPOINTS[endpoint]- schema_name = "calls" if config.fanout else endpoint+ schema_name = config.fanout.parent_name if config.fanout else endpointresult = _driver().run(endpoint, [_page(schema_name, [], None)])response = result.response@@ -227,7 +270,7 @@ class TestAircallSourceResponse:assert response is not Noneassert response.name == endpointassert response.primary_keys == [config.primary_key]- assert response.sort_mode == "asc"+ assert response.sort_mode == (None if config.fanout and config.fanout.parent_name == "campaigns" else "asc")if config.partition_key:assert response.partition_mode == "datetime"assert response.partition_keys == [config.partition_key]