Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 3 missing tables to the Instantly source (#125281)

Tom Owersb5841f5f11f6
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index dfb8c7bf6af3..54f55c4728a6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -6406,9 +6406,9 @@ Diffed against: <https://developer.instantly.ai/api-reference/openapi.json>
- [x] `POST /api/v2/accounts/warmup-analytics` — read-shaped per-account warmup health scores (medium)
- [x] `GET /api/v2/inbox-placement-analytics + /inbox-placement-reports` — inbox vs spam placement results per deliverability test (medium)
- [x] `GET /api/v2/audit-logs` — workspace change history for attribution and compliance (medium)
-- [ ] `GET /api/v2/campaigns/analytics/overview` — aggregate campaign KPI rollup (low)
-- [ ] `GET /api/v2/workspace-members` — membership lookup for user IDs on campaigns and leads (low)
-- [ ] `GET /api/v2/lead-lists/{id}/verification-stats` — email verification quality per lead list (low)
+- [x] `GET /api/v2/campaigns/analytics/overview` — aggregate campaign KPI rollup (low). Added as `campaign_analytics_overview`, one row per campaign.
+- [x] `GET /api/v2/workspace-members` — membership lookup for user IDs on campaigns and leads (low). Added as `workspace_members`.
+- [x] `GET /api/v2/lead-lists/{id}/verification-stats` — email verification quality per lead list (low). Added as `lead_list_verification_stats`.
Note: Full OpenAPI 3.1 spec is public and unauthenticated. Note /leads and /leads/list are POST-only search endpoints (already used by the source); /charges-style write endpoints excluded.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/canonical_descriptions.py
index 9f615a0f2b02..1d8c8454d958 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/canonical_descriptions.py
@@ -317,6 +317,63 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"audit_metadata": "Additional details about the activity.",
},
},
+ "campaign_analytics_overview": {
+ "description": "Analytics overview of one campaign: sends, opens, replies, clicks, bounces, opportunities, and lead interest status totals.",
+ "docs_url": "https://developer.instantly.ai/api-reference/campaign/get-campaigns-analytics-overview",
+ "columns": {
+ "campaign_id": "Campaign the overview covers (UUID). Joins to campaigns.id.",
+ "open_count": "Total email opens, including repeat opens.",
+ "open_count_unique": "Emails opened at least once.",
+ "open_count_unique_by_step": "Unique opens, counted once per lead and step.",
+ "link_click_count": "Total tracked link clicks.",
+ "link_click_count_unique": "Links clicked at least once.",
+ "link_click_count_unique_by_step": "Unique link clicks, counted once per lead and step.",
+ "reply_count": "Total replies received, counting each reply from a lead.",
+ "reply_count_unique": "Unique replies (first reply per lead). Excludes automatic replies.",
+ "reply_count_unique_by_step": "Unique replies per step (first reply per lead and step). Excludes automatic replies.",
+ "reply_count_automatic": "Total automatic replies, such as out-of-office.",
+ "reply_count_automatic_unique": "Unique leads that sent an automatic reply.",
+ "reply_count_automatic_unique_by_step": "Unique automatic replies, counted once per lead and step.",
+ "bounced_count": "Leads whose email bounced.",
+ "unsubscribed_count": "Leads that unsubscribed.",
+ "completed_count": "Leads that completed the campaign sequence.",
+ "emails_sent_count": "Total emails sent.",
+ "contacted_count": "Unique leads contacted.",
+ "new_leads_contacted_count": "Leads contacted for the first time.",
+ "total_opportunities": "Unique opportunities created.",
+ "total_opportunity_value": "Total value of the opportunities created.",
+ "total_interested": "Leads marked as interested (first occurrence per lead).",
+ "total_meeting_booked": "Leads marked as meeting booked (first occurrence per lead).",
+ "total_meeting_completed": "Leads marked as meeting completed (first occurrence per lead).",
+ "total_closed": "Leads marked as closed (first occurrence per lead).",
+ },
+ },
+ "workspace_members": {
+ "description": "A member of the workspace, with the user's role, permissions, and invitation status.",
+ "docs_url": "https://developer.instantly.ai/api-reference/workspacemember/list-workspace-member",
+ "columns": {
+ "id": "Unique identifier of the workspace member (UUID).",
+ "email": "Email address of the workspace member.",
+ "user_id": "User ID of the workspace member (UUID).",
+ "user_email": "Email address of the user account.",
+ "name": "First and last name of the member.",
+ "role": "Access level of the member: owner, admin, editor, view, or client.",
+ "timestamp_created": "When the member was added to the workspace.",
+ "workspace_id": "Workspace the member belongs to.",
+ "accepted": "Whether the member accepted the workspace invitation.",
+ "issuer_id": "User who added the member to the workspace.",
+ "permissions": "Permissions of the member, used in the app to restrict access to certain sections.",
+ },
+ },
+ "lead_list_verification_stats": {
+ "description": "Email verification results of one lead list: lead counts per verification status.",
+ "docs_url": "https://developer.instantly.ai/api-reference/leadlist/get-verification-statistics-for-a-lead-list",
+ "columns": {
+ "lead_list_id": "Lead list the statistics cover (UUID). Joins to lead_lists.id.",
+ "stats": "Lead counts per verification status: verified, invalid, risky, catch_all, job_change, and pending verification jobs.",
+ "total_leads": "Total number of leads in the list.",
+ },
+ },
"webhook_events": {
"description": "Raw Instantly webhook event stream: sends, opens, replies, bounces, and lead status changes as they happen.",
"docs_url": "https://developer.instantly.ai/guides/webhook-events",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/instantly.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/instantly.py
index d31908036a68..677d20fd3364 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/instantly.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/instantly.py
@@ -5,6 +5,7 @@ import dataclasses
from collections.abc import Callable, Iterable, Iterator
from datetime import UTC, date, datetime, timedelta
from typing import Any, Optional
+from urllib.parse import quote
import orjson
import pyarrow as pa
@@ -299,7 +300,7 @@ def _list_request(config: InstantlyEndpointConfig) -> tuple[dict[str, Any], Base
def _fanout_pages(client: RESTClient, config: InstantlyEndpointConfig) -> Iterator[list[dict[str, Any]]]:
- """Walk every parent (campaign or inbox placement test) and fetch the endpoint once per parent.
+ """Walk every parent (campaign, lead list, or inbox placement test) and fetch the endpoint once per parent.
Fan-out endpoints don't resume: a retry re-walks the parents from scratch.
"""
@@ -316,9 +317,14 @@ def _fanout_pages(client: RESTClient, config: InstantlyEndpointConfig) -> Iterat
if not parent_id:
continue
params, paginator = _list_request(config)
+ path = config.path
+ if fanout.in_path:
+ path = path.replace(f"{{{fanout.param}}}", quote(str(parent_id), safe=""))
+ else:
+ params = {**params, fanout.param: parent_id}
for page in client.paginate(
- path=f"/api/v2{config.path}",
- params={**params, fanout.param: parent_id},
+ path=f"/api/v2{path}",
+ params=params,
paginator=paginator,
data_selector=config.data_selector,
):
@@ -429,6 +435,11 @@ def _probe_endpoint(session: requests.Session, endpoint: str) -> requests.Respon
if config.account_email_batches:
# Same accounts:read scope as /accounts, which needs no email list in the request.
return session.get(f"{BASE_URL}/api/v2/accounts", params={"limit": 1}, timeout=REQUEST_TIMEOUT_SECONDS)
+ if config.fanout is not None and config.fanout.in_path:
+ # The path needs a real parent id; the parent list checks the same scope.
+ return session.get(
+ f"{BASE_URL}/api/v2{config.fanout.parent_path}", params={"limit": 1}, timeout=REQUEST_TIMEOUT_SECONDS
+ )
url = f"{BASE_URL}/api/v2{config.path}"
if config.method == "POST":
return session.post(url, json={"limit": 1}, timeout=REQUEST_TIMEOUT_SECONDS)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/settings.py
index cb4cf3e1f5ac..84b55374a7ee 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/settings.py
@@ -39,6 +39,8 @@ class ParentFanout:
stamp_field: Optional[str] = None
# Cursor-paginated list endpoint the parent ids come from.
parent_path: str = "/campaigns"
+ # The parent id fills the `{param}` placeholder in the path instead of a query param.
+ in_path: bool = False
@dataclass(frozen=True)
@@ -235,6 +237,33 @@ INSTANTLY_ENDPOINTS: dict[str, InstantlyEndpointConfig] = {
sort_mode="desc",
description="Workspace activity history: who changed what, when, and from where.",
),
+ "campaign_analytics_overview": InstantlyEndpointConfig(
+ name="campaign_analytics_overview",
+ path="/campaigns/analytics/overview",
+ pagination="single",
+ data_selector=None,
+ # Without an id the API returns one rollup of every campaign; one request per campaign
+ # keeps the rows joinable to campaigns.
+ fanout=ParentFanout(param="id", stamp_field="campaign_id"),
+ primary_keys=["campaign_id"],
+ partition_key=None,
+ description="Analytics overview per campaign: sends, opens, replies, clicks, bounces, opportunities, and lead interest status totals.",
+ ),
+ "workspace_members": InstantlyEndpointConfig(
+ name="workspace_members",
+ path="/workspace-members",
+ description="Members of the workspace with their roles, permissions, and invitation status.",
+ ),
+ "lead_list_verification_stats": InstantlyEndpointConfig(
+ name="lead_list_verification_stats",
+ path="/lead-lists/{id}/verification-stats",
+ pagination="single",
+ data_selector=None,
+ fanout=ParentFanout(param="id", stamp_field="lead_list_id", parent_path="/lead-lists", in_path=True),
+ primary_keys=["lead_list_id"],
+ partition_key=None,
+ description="Email verification results per lead list: counts of verified, invalid, risky, catch-all, and pending leads.",
+ ),
}
ENDPOINTS = tuple(INSTANTLY_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/tests/test_instantly.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/tests/test_instantly.py
index 028ef0bc7488..c3185ada1dd9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/tests/test_instantly.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/tests/test_instantly.py
@@ -241,6 +241,15 @@ class TestInstantly:
{"limit": 100, "test_id": "c-1"},
[{"id": "r-1", "test_id": "c-1", "spam_assassin_score": 1.5}],
),
+ (
+ "lead_list_verification_stats",
+ "/api/v2/lead-lists",
+ # The parent id goes into the path, not the query string.
+ "/api/v2/lead-lists/c-1/verification-stats",
+ [{"stats": {"verified": 7, "invalid": 2}, "total_leads": 9}],
+ {},
+ [{"stats": {"verified": 7, "invalid": 2}, "total_leads": 9, "lead_list_id": "c-1"}],
+ ),
],
)
def test_fanout_scopes_each_request_to_one_parent(
@@ -378,6 +387,8 @@ class TestInstantly:
),
# Warmup analytics needs account emails in a POST body; /accounts checks the same scope.
("account_warmup_analytics", "https://api.instantly.ai/api/v2/accounts", {"limit": 1}),
+ # Verification stats need a real lead list id in the path; /lead-lists checks the same scope.
+ ("lead_list_verification_stats", "https://api.instantly.ai/api/v2/lead-lists", {"limit": 1}),
],
)
def test_probe_sends_required_params(self, endpoint, expected_url, expected_params):