Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 3 missing tables to the Smartlead source (#123848)

Tom Owers9ad1ce0f781e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 539e87b49a32..476707a530f5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -11253,10 +11253,10 @@ Today (7): `campaign_analytics`, `campaign_leads`, `campaign_sequences`, `campai
Diffed against: <https://api.smartlead.ai/llms-full.txt>
-- [ ] `Get Lead Categories (GET /api/v1/leads/fetch-categories)` — resolve lead_category_id in campaign leads to meaningful sales outcome labels (high)
+- [x] `Get Lead Categories (GET /api/v1/leads/fetch-categories)` — resolve lead_category_id in campaign leads to meaningful sales outcome labels (high)
- [ ] `Get Lead Message History (GET /api/v1/campaigns/{campaign_id}/leads/{lead_id}/message-history)` — analyze complete conversations, including replies absent from sent email statistics (high)
-- [ ] `Get All Leads Activities (GET /api/v1/campaigns/all-leads-activities)` — analyze engagement events over time rather than the current state of each sent email (high)
-- [ ] `Get Warmup Statistics (GET /api/v1/email-accounts/{email_account_id}/warmup-stats)` — measure mailbox reputation, spam placement, and warmup performance (medium)
+- [x] `Get All Leads Activities (GET /api/v1/campaigns/all-leads-activities)` — analyze engagement events over time rather than the current state of each sent email (high)
+- [x] `Get Warmup Statistics (GET /api/v1/email-accounts/{email_account_id}/warmup-stats)` — measure mailbox reputation, spam placement, and warmup performance (medium)
- [ ] `Get Campaign Email Accounts (GET /api/v1/campaigns/{campaign_id}/email-accounts)` — join campaigns to rotating sender accounts for mailbox performance analysis (medium)
- [ ] `Get Day-wise Overall Stats (GET /api/v1/analytics/day-wise-overall-stats)` — track daily engagement totals and compare outreach performance over time (medium)
- [ ] `Get Lead Notes (GET /api/v1/crm/leads/notes/{lead_id})` — analyze sales context and follow-up information attached to prospects (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/canonical_descriptions.py
index 48969545b93b..c9e4f1c26a2f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/canonical_descriptions.py
@@ -124,4 +124,40 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"created_at": "ISO 8601 timestamp when the client was created.",
},
},
+ "lead_categories": {
+ "description": "A lead category, either global or custom, used to label the sales outcome of a lead.",
+ "docs_url": "https://api.smartlead.ai/api-reference/leads/categories",
+ "columns": {
+ "id": "Unique category identifier, referenced by lead_category_id on campaign leads.",
+ "name": "Category name, e.g. Interested, Not Interested, or Meeting Booked.",
+ "sentiment_type": "Sentiment classification: positive, negative, or null for neutral.",
+ "created_at": "ISO 8601 timestamp when the category was created.",
+ },
+ },
+ "all_leads_activities": {
+ "description": "A lead in a campaign across all campaigns, with the timeline of emails sent to it and its opens, clicks, and replies.",
+ "docs_url": "https://api.smartlead.ai/api-reference/leads/activities",
+ "columns": {
+ "lead_id": "Unique lead identifier.",
+ "email_lead_map_id": "Identifier of the lead-to-campaign mapping.",
+ "campaign_id": "ID of the campaign the lead is in.",
+ "campaign_name": "Name of the campaign.",
+ "campaign_status": "Current status of the campaign.",
+ "status": "Status of the lead in the campaign.",
+ "current_seq_num": "Sequence step the lead is currently on.",
+ "created_at": "ISO 8601 timestamp when the lead was added to the campaign.",
+ "activities": "Emails sent to the lead, each with open and click counts, click details, reply details, and thread replies.",
+ },
+ },
+ "warmup_stats": {
+ "description": "Warmup performance of an email account over the last 7 days, with a daily breakdown.",
+ "docs_url": "https://api.smartlead.ai/api-reference/email-accounts/warmup-stats",
+ "columns": {
+ "email_account_id": "ID of the email account these warmup statistics belong to.",
+ "total_sent": "Warmup emails sent in the last 7 days.",
+ "spam_count": "Warmup emails that landed in spam in the last 7 days.",
+ "reputation_score": "Warmup reputation score of the account (0-100).",
+ "daily_stats": "Per-day warmup counts for the last 7 days: date, sent, spam, delivered, opened, and replied.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/settings.py
index a58e46006745..0fe68e9ad0b2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/settings.py
@@ -49,6 +49,9 @@ class SmartleadEndpointConfig:
# Set for per-campaign resources that hang off the account-level campaign list.
fanout: Optional[DependentEndpointConfig] = None
page_size: int = PAGE_SIZE
+ # Retry attempts per request when the endpoint has a tighter rate limit than the account-wide one.
+ # None uses the framework default.
+ max_retries: Optional[int] = None
def _campaign_fanout() -> DependentEndpointConfig:
@@ -63,6 +66,17 @@ def _campaign_fanout() -> DependentEndpointConfig:
)
+def _email_account_fanout() -> DependentEndpointConfig:
+ # Warmup stats bodies carry no account id, so inject the parent's `id` as `email_account_id`.
+ return DependentEndpointConfig(
+ parent_name="email_accounts",
+ resolve_param="email_account_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "email_account_id"},
+ )
+
+
SMARTLEAD_ENDPOINTS: dict[str, SmartleadEndpointConfig] = {
# Returns the full campaign list as a bare array, newest first, with no pagination params.
"campaigns": SmartleadEndpointConfig(
@@ -123,6 +137,36 @@ SMARTLEAD_ENDPOINTS: dict[str, SmartleadEndpointConfig] = {
name="clients",
path="/client/",
),
+ # Global and custom lead categories as a bare array. Resolves `lead_category_id` on leads.
+ "lead_categories": SmartleadEndpointConfig(
+ name="lead_categories",
+ path="/leads/fetch-categories",
+ ),
+ # One row per lead-to-campaign mapping across every campaign, with that lead's sent emails,
+ # opens, clicks, replies and thread replies nested in `activities`. The `event_time_from`
+ # filter applies to activity events, not to any row-level field, so there is no watermark
+ # column to bind it to and the table syncs as full refresh.
+ "all_leads_activities": SmartleadEndpointConfig(
+ name="all_leads_activities",
+ path="/campaigns/all-leads-activities",
+ pagination="offset",
+ data_selector="data",
+ primary_keys=["campaign_id", "email_lead_map_id"],
+ # The endpoint allows 10 requests per minute and answers 429 with no `Retry-After`. The
+ # default 5 attempts back off for 15 seconds in total, which is shorter than the window;
+ # 8 attempts back off for about 2 minutes.
+ max_retries=8,
+ ),
+ # One aggregate object per email account: warmup totals, reputation score and a `daily_stats`
+ # array that covers only the last 7 days. A full refresh replaces it with the current window.
+ "warmup_stats": SmartleadEndpointConfig(
+ name="warmup_stats",
+ path="/email-accounts/{email_account_id}/warmup-stats",
+ expects_list=False,
+ primary_keys=["email_account_id"],
+ partition_key=None,
+ fanout=_email_account_fanout(),
+ ),
}
ENDPOINTS = tuple(SMARTLEAD_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/smartlead.py b/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/smartlead.py
index 296d7d8dd7d8..d3f062e2e922 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/smartlead.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/smartlead.py
@@ -51,8 +51,8 @@ def _paginator(config: SmartleadEndpointConfig) -> BasePaginator:
return SinglePagePaginator()
-def _client_config(api_key: str, paginator: BasePaginator) -> ClientConfig:
- return {
+def _client_config(api_key: str, paginator: BasePaginator, config: SmartleadEndpointConfig) -> ClientConfig:
+ client_config: ClientConfig = {
"base_url": BASE_URL,
"headers": {"Accept": "application/json"},
# Smartlead's only auth method is the `api_key` query param. The framework auth config
@@ -60,6 +60,9 @@ def _client_config(api_key: str, paginator: BasePaginator) -> ClientConfig:
"auth": {"type": "api_key", "name": "api_key", "api_key": api_key, "location": "query"},
"paginator": paginator,
}
+ if config.max_retries is not None:
+ client_config["max_retries"] = config.max_retries
+ return client_config
def _endpoint_extra(config: SmartleadEndpointConfig) -> Endpoint:
@@ -115,7 +118,7 @@ def _top_level_source(
resource["data_map"] = _redact_fields_mapper(config.redact_fields)
rest_config: RESTAPIConfig = {
- "client": _client_config(api_key, _paginator(config)),
+ "client": _client_config(api_key, _paginator(config), config),
"resource_defaults": {},
"resources": [cast(Any, resource)],
}
@@ -184,19 +187,19 @@ def _fanout_source(
endpoint_configs=SMARTLEAD_ENDPOINTS,
child_endpoint=endpoint,
fanout=config.fanout,
- client_config=_client_config(api_key, _paginator(config)),
+ client_config=_client_config(api_key, _paginator(config), config),
path_format_values={},
team_id=team_id,
job_id=job_id,
db_incremental_field_last_value=db_incremental_field_last_value,
should_use_incremental_field=should_use_incremental_field,
incremental_field=incremental_field,
- # The campaign parent list is unpaginated while children page with offset/limit, so
- # each side carries its own paginator instead of sharing the client's.
+ # Parents and children paginate differently (the campaign list is unpaginated, email
+ # accounts page with offset/limit, warmup stats is one object), so each side carries
+ # its own paginator instead of sharing the client's.
parent_endpoint_extra=_endpoint_extra(parent_config),
child_endpoint_extra=_endpoint_extra(config),
- # The parent takes no page-size param (unpaginated); the children's OffsetPaginator
- # injects `offset` and `limit` itself.
+ # Every OffsetPaginator injects `offset` and `limit` itself.
page_size_param=None,
resume_hook=save_checkpoint,
initial_paginator_state=initial_state,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/tests/test_smartlead.py b/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/tests/test_smartlead.py
index 7b9b46a0a0c1..42a3fd4e89bc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/tests/test_smartlead.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/smartlead/tests/test_smartlead.py
@@ -62,6 +62,26 @@ class TestSmartleadSource:
assert isinstance(result.raised, ValueError)
assert "Required a list response body" in str(result.raised)
+ def test_all_leads_activities_retries_through_the_endpoint_rate_limit_window(self) -> None:
+ # The endpoint allows 10 requests a minute and its 429 carries no `Retry-After`, so only
+ # the raised attempt budget lets the backoff outlast the 60-second window.
+ rate_limited = ScriptedResponse(status=429, body=b"You have exceeded the 10 requests in 60 sec limit")
+ activity = {"campaign_id": 7, "lead_id": 1, "activities": [{"stats_id": 1, "open_count": 2}]}
+ result = _run(
+ "all_leads_activities",
+ [
+ _json({"data": [{**activity, "email_lead_map_id": i} for i in range(PAGE_SIZE)], "hasMore": True}),
+ *[rate_limited] * 6,
+ _json({"data": [{**activity, "email_lead_map_id": PAGE_SIZE}], "hasMore": False}),
+ ],
+ )
+
+ assert result.raised is None
+ assert len(result.rows) == PAGE_SIZE + 1
+ assert result.rows[-1]["activities"] == [{"stats_id": 1, "open_count": 2}]
+ assert set(result.paths) == {"/api/v1/campaigns/all-leads-activities"}
+ assert result.params("offset")[-1] == str(PAGE_SIZE)
+
def test_resumes_from_saved_offset(self) -> None:
result = _run(
"email_accounts",
@@ -111,6 +131,28 @@ class TestSmartleadFanout:
assert result.rows == [{"campaign_name": "Q1", "total_sent": 10, "campaign_id": 11}]
+ def test_warmup_stats_fans_out_over_every_page_of_email_accounts(self) -> None:
+ # Unlike the campaign list, the email account parent pages with offset/limit, so a parent
+ # walk that stops after one page would drop every account past the first hundred.
+ def script(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == "/api/v1/email-accounts/":
+ offset = int(request.param("offset") or 0)
+ return _json(_accounts(PAGE_SIZE if offset == 0 else 1, start=offset))
+ return _json({"total_sent": 14, "reputation_score": 98, "daily_stats": [{"date": "2026-01-02", "sent": 2}]})
+
+ result = _run("warmup_stats", script)
+
+ parent_requests = [r for r in result.requests if r.path == "/api/v1/email-accounts/"]
+ assert [r.param("offset") for r in parent_requests] == ["0", str(PAGE_SIZE)]
+ assert len(result.rows) == PAGE_SIZE + 1
+ assert result.requests[1].path == "/api/v1/email-accounts/0/warmup-stats"
+ assert result.rows[-1] == {
+ "total_sent": 14,
+ "reputation_score": 98,
+ "daily_stats": [{"date": "2026-01-02", "sent": 2}],
+ "email_account_id": PAGE_SIZE,
+ }
+
def test_fan_out_resumes_completed_and_current_campaign(self) -> None:
def script(request: RecordedRequest) -> ScriptedResponse:
if request.path.endswith("/campaigns/"):