:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 3 missing tables to the Instantly source (#125281)
Tom Owersb5841f5f11f6
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex dfb8c7bf6af3..54f55c4728a6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -6406,9 +6406,9 @@ Diffed against: <https://developer.instantly.ai/api-reference/openapi.json>- [x] `POST /api/v2/accounts/warmup-analytics` — read-shaped per-account warmup health scores (medium)- [x] `GET /api/v2/inbox-placement-analytics + /inbox-placement-reports` — inbox vs spam placement results per deliverability test (medium)- [x] `GET /api/v2/audit-logs` — workspace change history for attribution and compliance (medium)-- [ ] `GET /api/v2/campaigns/analytics/overview` — aggregate campaign KPI rollup (low)-- [ ] `GET /api/v2/workspace-members` — membership lookup for user IDs on campaigns and leads (low)-- [ ] `GET /api/v2/lead-lists/{id}/verification-stats` — email verification quality per lead list (low)+- [x] `GET /api/v2/campaigns/analytics/overview` — aggregate campaign KPI rollup (low). Added as `campaign_analytics_overview`, one row per campaign.+- [x] `GET /api/v2/workspace-members` — membership lookup for user IDs on campaigns and leads (low). Added as `workspace_members`.+- [x] `GET /api/v2/lead-lists/{id}/verification-stats` — email verification quality per lead list (low). Added as `lead_list_verification_stats`.Note: Full OpenAPI 3.1 spec is public and unauthenticated. Note /leads and /leads/list are POST-only search endpoints (already used by the source); /charges-style write endpoints excluded.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/canonical_descriptions.pyindex 9f615a0f2b02..1d8c8454d958 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/canonical_descriptions.py@@ -317,6 +317,63 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"audit_metadata": "Additional details about the activity.",},},+ "campaign_analytics_overview": {+ "description": "Analytics overview of one campaign: sends, opens, replies, clicks, bounces, opportunities, and lead interest status totals.",+ "docs_url": "https://developer.instantly.ai/api-reference/campaign/get-campaigns-analytics-overview",+ "columns": {+ "campaign_id": "Campaign the overview covers (UUID). Joins to campaigns.id.",+ "open_count": "Total email opens, including repeat opens.",+ "open_count_unique": "Emails opened at least once.",+ "open_count_unique_by_step": "Unique opens, counted once per lead and step.",+ "link_click_count": "Total tracked link clicks.",+ "link_click_count_unique": "Links clicked at least once.",+ "link_click_count_unique_by_step": "Unique link clicks, counted once per lead and step.",+ "reply_count": "Total replies received, counting each reply from a lead.",+ "reply_count_unique": "Unique replies (first reply per lead). Excludes automatic replies.",+ "reply_count_unique_by_step": "Unique replies per step (first reply per lead and step). Excludes automatic replies.",+ "reply_count_automatic": "Total automatic replies, such as out-of-office.",+ "reply_count_automatic_unique": "Unique leads that sent an automatic reply.",+ "reply_count_automatic_unique_by_step": "Unique automatic replies, counted once per lead and step.",+ "bounced_count": "Leads whose email bounced.",+ "unsubscribed_count": "Leads that unsubscribed.",+ "completed_count": "Leads that completed the campaign sequence.",+ "emails_sent_count": "Total emails sent.",+ "contacted_count": "Unique leads contacted.",+ "new_leads_contacted_count": "Leads contacted for the first time.",+ "total_opportunities": "Unique opportunities created.",+ "total_opportunity_value": "Total value of the opportunities created.",+ "total_interested": "Leads marked as interested (first occurrence per lead).",+ "total_meeting_booked": "Leads marked as meeting booked (first occurrence per lead).",+ "total_meeting_completed": "Leads marked as meeting completed (first occurrence per lead).",+ "total_closed": "Leads marked as closed (first occurrence per lead).",+ },+ },+ "workspace_members": {+ "description": "A member of the workspace, with the user's role, permissions, and invitation status.",+ "docs_url": "https://developer.instantly.ai/api-reference/workspacemember/list-workspace-member",+ "columns": {+ "id": "Unique identifier of the workspace member (UUID).",+ "email": "Email address of the workspace member.",+ "user_id": "User ID of the workspace member (UUID).",+ "user_email": "Email address of the user account.",+ "name": "First and last name of the member.",+ "role": "Access level of the member: owner, admin, editor, view, or client.",+ "timestamp_created": "When the member was added to the workspace.",+ "workspace_id": "Workspace the member belongs to.",+ "accepted": "Whether the member accepted the workspace invitation.",+ "issuer_id": "User who added the member to the workspace.",+ "permissions": "Permissions of the member, used in the app to restrict access to certain sections.",+ },+ },+ "lead_list_verification_stats": {+ "description": "Email verification results of one lead list: lead counts per verification status.",+ "docs_url": "https://developer.instantly.ai/api-reference/leadlist/get-verification-statistics-for-a-lead-list",+ "columns": {+ "lead_list_id": "Lead list the statistics cover (UUID). Joins to lead_lists.id.",+ "stats": "Lead counts per verification status: verified, invalid, risky, catch_all, job_change, and pending verification jobs.",+ "total_leads": "Total number of leads in the list.",+ },+ },"webhook_events": {"description": "Raw Instantly webhook event stream: sends, opens, replies, bounces, and lead status changes as they happen.","docs_url": "https://developer.instantly.ai/guides/webhook-events",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/instantly.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/instantly.pyindex d31908036a68..677d20fd3364 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/instantly.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/instantly.py@@ -5,6 +5,7 @@ import dataclassesfrom collections.abc import Callable, Iterable, Iteratorfrom datetime import UTC, date, datetime, timedeltafrom typing import Any, Optional+from urllib.parse import quoteimport orjsonimport pyarrow as pa@@ -299,7 +300,7 @@ def _list_request(config: InstantlyEndpointConfig) -> tuple[dict[str, Any], Basedef _fanout_pages(client: RESTClient, config: InstantlyEndpointConfig) -> Iterator[list[dict[str, Any]]]:- """Walk every parent (campaign or inbox placement test) and fetch the endpoint once per parent.+ """Walk every parent (campaign, lead list, or inbox placement test) and fetch the endpoint once per parent.Fan-out endpoints don't resume: a retry re-walks the parents from scratch."""@@ -316,9 +317,14 @@ def _fanout_pages(client: RESTClient, config: InstantlyEndpointConfig) -> Iteratif not parent_id:continueparams, paginator = _list_request(config)+ path = config.path+ if fanout.in_path:+ path = path.replace(f"{{{fanout.param}}}", quote(str(parent_id), safe=""))+ else:+ params = {**params, fanout.param: parent_id}for page in client.paginate(- path=f"/api/v2{config.path}",- params={**params, fanout.param: parent_id},+ path=f"/api/v2{path}",+ params=params,paginator=paginator,data_selector=config.data_selector,):@@ -429,6 +435,11 @@ def _probe_endpoint(session: requests.Session, endpoint: str) -> requests.Responif config.account_email_batches:# Same accounts:read scope as /accounts, which needs no email list in the request.return session.get(f"{BASE_URL}/api/v2/accounts", params={"limit": 1}, timeout=REQUEST_TIMEOUT_SECONDS)+ if config.fanout is not None and config.fanout.in_path:+ # The path needs a real parent id; the parent list checks the same scope.+ return session.get(+ f"{BASE_URL}/api/v2{config.fanout.parent_path}", params={"limit": 1}, timeout=REQUEST_TIMEOUT_SECONDS+ )url = f"{BASE_URL}/api/v2{config.path}"if config.method == "POST":return session.post(url, json={"limit": 1}, timeout=REQUEST_TIMEOUT_SECONDS)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/settings.pyindex cb4cf3e1f5ac..84b55374a7ee 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/settings.py@@ -39,6 +39,8 @@ class ParentFanout:stamp_field: Optional[str] = None# Cursor-paginated list endpoint the parent ids come from.parent_path: str = "/campaigns"+ # The parent id fills the `{param}` placeholder in the path instead of a query param.+ in_path: bool = False@dataclass(frozen=True)@@ -235,6 +237,33 @@ INSTANTLY_ENDPOINTS: dict[str, InstantlyEndpointConfig] = {sort_mode="desc",description="Workspace activity history: who changed what, when, and from where.",),+ "campaign_analytics_overview": InstantlyEndpointConfig(+ name="campaign_analytics_overview",+ path="/campaigns/analytics/overview",+ pagination="single",+ data_selector=None,+ # Without an id the API returns one rollup of every campaign; one request per campaign+ # keeps the rows joinable to campaigns.+ fanout=ParentFanout(param="id", stamp_field="campaign_id"),+ primary_keys=["campaign_id"],+ partition_key=None,+ description="Analytics overview per campaign: sends, opens, replies, clicks, bounces, opportunities, and lead interest status totals.",+ ),+ "workspace_members": InstantlyEndpointConfig(+ name="workspace_members",+ path="/workspace-members",+ description="Members of the workspace with their roles, permissions, and invitation status.",+ ),+ "lead_list_verification_stats": InstantlyEndpointConfig(+ name="lead_list_verification_stats",+ path="/lead-lists/{id}/verification-stats",+ pagination="single",+ data_selector=None,+ fanout=ParentFanout(param="id", stamp_field="lead_list_id", parent_path="/lead-lists", in_path=True),+ primary_keys=["lead_list_id"],+ partition_key=None,+ description="Email verification results per lead list: counts of verified, invalid, risky, catch-all, and pending leads.",+ ),}ENDPOINTS = tuple(INSTANTLY_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/tests/test_instantly.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/tests/test_instantly.pyindex 028ef0bc7488..c3185ada1dd9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/tests/test_instantly.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/tests/test_instantly.py@@ -241,6 +241,15 @@ class TestInstantly:{"limit": 100, "test_id": "c-1"},[{"id": "r-1", "test_id": "c-1", "spam_assassin_score": 1.5}],),+ (+ "lead_list_verification_stats",+ "/api/v2/lead-lists",+ # The parent id goes into the path, not the query string.+ "/api/v2/lead-lists/c-1/verification-stats",+ [{"stats": {"verified": 7, "invalid": 2}, "total_leads": 9}],+ {},+ [{"stats": {"verified": 7, "invalid": 2}, "total_leads": 9, "lead_list_id": "c-1"}],+ ),],)def test_fanout_scopes_each_request_to_one_parent(@@ -378,6 +387,8 @@ class TestInstantly:),# Warmup analytics needs account emails in a POST body; /accounts checks the same scope.("account_warmup_analytics", "https://api.instantly.ai/api/v2/accounts", {"limit": 1}),+ # Verification stats need a real lead list id in the path; /lead-lists checks the same scope.+ ("lead_list_verification_stats", "https://api.instantly.ai/api/v2/lead-lists", {"limit": 1}),],)def test_probe_sends_required_params(self, endpoint, expected_url, expected_params):