:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 3 missing tables to the Smartlead source (#123848)
Tom Owers9ad1ce0f781e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 539e87b49a32..476707a530f5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -11253,10 +11253,10 @@ Today (7): `campaign_analytics`, `campaign_leads`, `campaign_sequences`, `campaiDiffed against: <https://api.smartlead.ai/llms-full.txt>-- [ ] `Get Lead Categories (GET /api/v1/leads/fetch-categories)` — resolve lead_category_id in campaign leads to meaningful sales outcome labels (high)+- [x] `Get Lead Categories (GET /api/v1/leads/fetch-categories)` — resolve lead_category_id in campaign leads to meaningful sales outcome labels (high)- [ ] `Get Lead Message History (GET /api/v1/campaigns/{campaign_id}/leads/{lead_id}/message-history)` — analyze complete conversations, including replies absent from sent email statistics (high)-- [ ] `Get All Leads Activities (GET /api/v1/campaigns/all-leads-activities)` — analyze engagement events over time rather than the current state of each sent email (high)-- [ ] `Get Warmup Statistics (GET /api/v1/email-accounts/{email_account_id}/warmup-stats)` — measure mailbox reputation, spam placement, and warmup performance (medium)+- [x] `Get All Leads Activities (GET /api/v1/campaigns/all-leads-activities)` — analyze engagement events over time rather than the current state of each sent email (high)+- [x] `Get Warmup Statistics (GET /api/v1/email-accounts/{email_account_id}/warmup-stats)` — measure mailbox reputation, spam placement, and warmup performance (medium)- [ ] `Get Campaign Email Accounts (GET /api/v1/campaigns/{campaign_id}/email-accounts)` — join campaigns to rotating sender accounts for mailbox performance analysis (medium)- [ ] `Get Day-wise Overall Stats (GET /api/v1/analytics/day-wise-overall-stats)` — track daily engagement totals and compare outreach performance over time (medium)- [ ] `Get Lead Notes (GET /api/v1/crm/leads/notes/{lead_id})` — analyze sales context and follow-up information attached to prospects (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/canonical_descriptions.pyindex 48969545b93b..c9e4f1c26a2f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/canonical_descriptions.py@@ -124,4 +124,40 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"created_at": "ISO 8601 timestamp when the client was created.",},},+ "lead_categories": {+ "description": "A lead category, either global or custom, used to label the sales outcome of a lead.",+ "docs_url": "https://api.smartlead.ai/api-reference/leads/categories",+ "columns": {+ "id": "Unique category identifier, referenced by lead_category_id on campaign leads.",+ "name": "Category name, e.g. Interested, Not Interested, or Meeting Booked.",+ "sentiment_type": "Sentiment classification: positive, negative, or null for neutral.",+ "created_at": "ISO 8601 timestamp when the category was created.",+ },+ },+ "all_leads_activities": {+ "description": "A lead in a campaign across all campaigns, with the timeline of emails sent to it and its opens, clicks, and replies.",+ "docs_url": "https://api.smartlead.ai/api-reference/leads/activities",+ "columns": {+ "lead_id": "Unique lead identifier.",+ "email_lead_map_id": "Identifier of the lead-to-campaign mapping.",+ "campaign_id": "ID of the campaign the lead is in.",+ "campaign_name": "Name of the campaign.",+ "campaign_status": "Current status of the campaign.",+ "status": "Status of the lead in the campaign.",+ "current_seq_num": "Sequence step the lead is currently on.",+ "created_at": "ISO 8601 timestamp when the lead was added to the campaign.",+ "activities": "Emails sent to the lead, each with open and click counts, click details, reply details, and thread replies.",+ },+ },+ "warmup_stats": {+ "description": "Warmup performance of an email account over the last 7 days, with a daily breakdown.",+ "docs_url": "https://api.smartlead.ai/api-reference/email-accounts/warmup-stats",+ "columns": {+ "email_account_id": "ID of the email account these warmup statistics belong to.",+ "total_sent": "Warmup emails sent in the last 7 days.",+ "spam_count": "Warmup emails that landed in spam in the last 7 days.",+ "reputation_score": "Warmup reputation score of the account (0-100).",+ "daily_stats": "Per-day warmup counts for the last 7 days: date, sent, spam, delivered, opened, and replied.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/settings.pyindex a58e46006745..0fe68e9ad0b2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/settings.py@@ -49,6 +49,9 @@ class SmartleadEndpointConfig:# Set for per-campaign resources that hang off the account-level campaign list.fanout: Optional[DependentEndpointConfig] = Nonepage_size: int = PAGE_SIZE+ # Retry attempts per request when the endpoint has a tighter rate limit than the account-wide one.+ # None uses the framework default.+ max_retries: Optional[int] = Nonedef _campaign_fanout() -> DependentEndpointConfig:@@ -63,6 +66,17 @@ def _campaign_fanout() -> DependentEndpointConfig:)+def _email_account_fanout() -> DependentEndpointConfig:+ # Warmup stats bodies carry no account id, so inject the parent's `id` as `email_account_id`.+ return DependentEndpointConfig(+ parent_name="email_accounts",+ resolve_param="email_account_id",+ resolve_field="id",+ include_from_parent=["id"],+ parent_field_renames={"id": "email_account_id"},+ )++SMARTLEAD_ENDPOINTS: dict[str, SmartleadEndpointConfig] = {# Returns the full campaign list as a bare array, newest first, with no pagination params."campaigns": SmartleadEndpointConfig(@@ -123,6 +137,36 @@ SMARTLEAD_ENDPOINTS: dict[str, SmartleadEndpointConfig] = {name="clients",path="/client/",),+ # Global and custom lead categories as a bare array. Resolves `lead_category_id` on leads.+ "lead_categories": SmartleadEndpointConfig(+ name="lead_categories",+ path="/leads/fetch-categories",+ ),+ # One row per lead-to-campaign mapping across every campaign, with that lead's sent emails,+ # opens, clicks, replies and thread replies nested in `activities`. The `event_time_from`+ # filter applies to activity events, not to any row-level field, so there is no watermark+ # column to bind it to and the table syncs as full refresh.+ "all_leads_activities": SmartleadEndpointConfig(+ name="all_leads_activities",+ path="/campaigns/all-leads-activities",+ pagination="offset",+ data_selector="data",+ primary_keys=["campaign_id", "email_lead_map_id"],+ # The endpoint allows 10 requests per minute and answers 429 with no `Retry-After`. The+ # default 5 attempts back off for 15 seconds in total, which is shorter than the window;+ # 8 attempts back off for about 2 minutes.+ max_retries=8,+ ),+ # One aggregate object per email account: warmup totals, reputation score and a `daily_stats`+ # array that covers only the last 7 days. A full refresh replaces it with the current window.+ "warmup_stats": SmartleadEndpointConfig(+ name="warmup_stats",+ path="/email-accounts/{email_account_id}/warmup-stats",+ expects_list=False,+ primary_keys=["email_account_id"],+ partition_key=None,+ fanout=_email_account_fanout(),+ ),}ENDPOINTS = tuple(SMARTLEAD_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/smartlead.py b/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/smartlead.pyindex 296d7d8dd7d8..d3f062e2e922 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/smartlead.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/smartlead.py@@ -51,8 +51,8 @@ def _paginator(config: SmartleadEndpointConfig) -> BasePaginator:return SinglePagePaginator()-def _client_config(api_key: str, paginator: BasePaginator) -> ClientConfig:- return {+def _client_config(api_key: str, paginator: BasePaginator, config: SmartleadEndpointConfig) -> ClientConfig:+ client_config: ClientConfig = {"base_url": BASE_URL,"headers": {"Accept": "application/json"},# Smartlead's only auth method is the `api_key` query param. The framework auth config@@ -60,6 +60,9 @@ def _client_config(api_key: str, paginator: BasePaginator) -> ClientConfig:"auth": {"type": "api_key", "name": "api_key", "api_key": api_key, "location": "query"},"paginator": paginator,}+ if config.max_retries is not None:+ client_config["max_retries"] = config.max_retries+ return client_configdef _endpoint_extra(config: SmartleadEndpointConfig) -> Endpoint:@@ -115,7 +118,7 @@ def _top_level_source(resource["data_map"] = _redact_fields_mapper(config.redact_fields)rest_config: RESTAPIConfig = {- "client": _client_config(api_key, _paginator(config)),+ "client": _client_config(api_key, _paginator(config), config),"resource_defaults": {},"resources": [cast(Any, resource)],}@@ -184,19 +187,19 @@ def _fanout_source(endpoint_configs=SMARTLEAD_ENDPOINTS,child_endpoint=endpoint,fanout=config.fanout,- client_config=_client_config(api_key, _paginator(config)),+ client_config=_client_config(api_key, _paginator(config), config),path_format_values={},team_id=team_id,job_id=job_id,db_incremental_field_last_value=db_incremental_field_last_value,should_use_incremental_field=should_use_incremental_field,incremental_field=incremental_field,- # The campaign parent list is unpaginated while children page with offset/limit, so- # each side carries its own paginator instead of sharing the client's.+ # Parents and children paginate differently (the campaign list is unpaginated, email+ # accounts page with offset/limit, warmup stats is one object), so each side carries+ # its own paginator instead of sharing the client's.parent_endpoint_extra=_endpoint_extra(parent_config),child_endpoint_extra=_endpoint_extra(config),- # The parent takes no page-size param (unpaginated); the children's OffsetPaginator- # injects `offset` and `limit` itself.+ # Every OffsetPaginator injects `offset` and `limit` itself.page_size_param=None,resume_hook=save_checkpoint,initial_paginator_state=initial_state,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/tests/test_smartlead.py b/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/tests/test_smartlead.pyindex 7b9b46a0a0c1..42a3fd4e89bc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/tests/test_smartlead.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/tests/test_smartlead.py@@ -62,6 +62,26 @@ class TestSmartleadSource:assert isinstance(result.raised, ValueError)assert "Required a list response body" in str(result.raised)+ def test_all_leads_activities_retries_through_the_endpoint_rate_limit_window(self) -> None:+ # The endpoint allows 10 requests a minute and its 429 carries no `Retry-After`, so only+ # the raised attempt budget lets the backoff outlast the 60-second window.+ rate_limited = ScriptedResponse(status=429, body=b"You have exceeded the 10 requests in 60 sec limit")+ activity = {"campaign_id": 7, "lead_id": 1, "activities": [{"stats_id": 1, "open_count": 2}]}+ result = _run(+ "all_leads_activities",+ [+ _json({"data": [{**activity, "email_lead_map_id": i} for i in range(PAGE_SIZE)], "hasMore": True}),+ *[rate_limited] * 6,+ _json({"data": [{**activity, "email_lead_map_id": PAGE_SIZE}], "hasMore": False}),+ ],+ )++ assert result.raised is None+ assert len(result.rows) == PAGE_SIZE + 1+ assert result.rows[-1]["activities"] == [{"stats_id": 1, "open_count": 2}]+ assert set(result.paths) == {"/api/v1/campaigns/all-leads-activities"}+ assert result.params("offset")[-1] == str(PAGE_SIZE)+def test_resumes_from_saved_offset(self) -> None:result = _run("email_accounts",@@ -111,6 +131,28 @@ class TestSmartleadFanout:assert result.rows == [{"campaign_name": "Q1", "total_sent": 10, "campaign_id": 11}]+ def test_warmup_stats_fans_out_over_every_page_of_email_accounts(self) -> None:+ # Unlike the campaign list, the email account parent pages with offset/limit, so a parent+ # walk that stops after one page would drop every account past the first hundred.+ def script(request: RecordedRequest) -> ScriptedResponse:+ if request.path == "/api/v1/email-accounts/":+ offset = int(request.param("offset") or 0)+ return _json(_accounts(PAGE_SIZE if offset == 0 else 1, start=offset))+ return _json({"total_sent": 14, "reputation_score": 98, "daily_stats": [{"date": "2026-01-02", "sent": 2}]})++ result = _run("warmup_stats", script)++ parent_requests = [r for r in result.requests if r.path == "/api/v1/email-accounts/"]+ assert [r.param("offset") for r in parent_requests] == ["0", str(PAGE_SIZE)]+ assert len(result.rows) == PAGE_SIZE + 1+ assert result.requests[1].path == "/api/v1/email-accounts/0/warmup-stats"+ assert result.rows[-1] == {+ "total_sent": 14,+ "reputation_score": 98,+ "daily_stats": [{"date": "2026-01-02", "sent": 2}],+ "email_account_id": PAGE_SIZE,+ }+def test_fan_out_resumes_completed_and_current_campaign(self) -> None:def script(request: RecordedRequest) -> ScriptedResponse:if request.path.endswith("/campaigns/"):