Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add heyreach campaign stats, company and network tables (#122035)

Tom Owers52187695c31d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 88fc9a2ecb6a..bbf77320d5b4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -5913,13 +5913,13 @@ Note: Parsed every GET operation in the official external API specification and
## HeyReach — gaps
-Today (7): `campaign_leads`, `campaigns`, `conversations`, `linkedin_accounts`, `list_leads`, `lists`, `overall_stats`
+Today (10): `campaign_leads`, `campaign_stats`, `campaigns`, `conversations`, `linkedin_accounts`, `list_companies`, `list_leads`, `lists`, `overall_stats`, `sender_network`
Diffed against: <https://documenter.gw.postman.com/api/collections/23808049/2sA2xb5F75?segregateAuth=true&versionTag=latest>
-- [ ] `GetOverallStatsByCampaign (POST /api/public/stats/GetOverallStatsByCampaign)` — compare daily reply rates, connection acceptance, and messages sent for each campaign instead of workspace totals (high)
-- [ ] `GetCompaniesFromList (POST /api/public/list/GetCompaniesFromList)` — analyze company audiences by industry, size, and location; current list_leads covers only lead lists (medium)
-- [ ] `GetMyNetworkForSender (POST /api/public/MyNetwork/GetMyNetworkForSender)` — measure sender network size and identify connections outside active campaigns and lead lists (medium)
+- [x] `GetOverallStatsByCampaign (POST /api/public/stats/GetOverallStatsByCampaign)` — compare daily reply rates, connection acceptance, and messages sent for each campaign instead of workspace totals (high)
+- [x] `GetCompaniesFromList (POST /api/public/list/GetCompaniesFromList)` — analyze company audiences by industry, size, and location; current list_leads covers only lead lists (medium)
+- [x] `GetMyNetworkForSender (POST /api/public/MyNetwork/GetMyNetworkForSender)` — measure sender network size and identify connections outside active campaigns and lead lists (medium)
Note: Parsed all requests and response examples in the vendor's published Postman collection. Read-only POST endpoints are included because this API uses POST for lists. Conversations already contain messages and tags. Company lists and sender networks need fan-out; organization administration is excluded.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/canonical_descriptions.py
index c088de2e3777..e2c826a49cd9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/canonical_descriptions.py
@@ -18,6 +18,24 @@ _LEAD_PROFILE_COLUMNS = {
"emailAddress": "The lead's email address, when known.",
}
+_DAILY_STATS_COLUMNS = {
+ "date": "Day the stats cover.",
+ "profileViews": "Profile views performed that day.",
+ "postLikes": "Post likes performed that day.",
+ "follows": "Follows performed that day.",
+ "messagesSent": "Messages sent that day.",
+ "totalMessageStarted": "Message threads started that day.",
+ "totalMessageReplies": "Replies received to messages that day.",
+ "inmailMessagesSent": "InMail messages sent that day.",
+ "totalInmailStarted": "InMail threads started that day.",
+ "totalInmailReplies": "Replies received to InMails that day.",
+ "connectionsSent": "Connection requests sent that day.",
+ "connectionsAccepted": "Connection requests accepted that day.",
+ "messageReplyRate": "Replies received divided by message threads started.",
+ "inMailReplyRate": "InMail replies received divided by InMail threads started.",
+ "connectionAcceptanceRate": "Connections accepted divided by connection requests sent.",
+}
+
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"campaigns": {
"description": "An outreach campaign: a sequence of LinkedIn actions sent to a list of leads from one or more sender accounts.",
@@ -107,22 +125,48 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"overall_stats": {
"description": "Daily outreach totals across all sender accounts and campaigns.",
"docs_url": _API_DOCS_URL,
+ "columns": _DAILY_STATS_COLUMNS,
+ },
+ "campaign_stats": {
+ "description": "Daily outreach totals for each campaign, across all sender accounts.",
+ "docs_url": _API_DOCS_URL,
+ "columns": {
+ "campaignId": "Identifier of the campaign the stats cover.",
+ "campaignName": "Name of the campaign.",
+ "isCampaignDeleted": "Whether the campaign has been deleted.",
+ **_DAILY_STATS_COLUMNS,
+ },
+ },
+ "list_companies": {
+ "description": "A company belonging to a company list, with its LinkedIn company page details.",
+ "docs_url": _API_DOCS_URL,
+ "columns": {
+ "listId": "Identifier of the company list the company belongs to.",
+ "name": "Name of the company.",
+ "description": "Description of the company from its LinkedIn page.",
+ "industry": "Industry of the company.",
+ "imageUrl": "URL of the company's logo.",
+ "companySize": "Company size range, as shown on LinkedIn.",
+ "employeesOnLinkedIn": "Number of employees on LinkedIn.",
+ "location": "Location of the company.",
+ "specialities": "Specialities listed on the company's LinkedIn page.",
+ "website": "The company's website.",
+ },
+ },
+ "sender_network": {
+ "description": "A LinkedIn connection in the network of a sender account.",
+ "docs_url": _API_DOCS_URL,
"columns": {
- "date": "Day the stats cover.",
- "profileViews": "Profile views performed that day.",
- "postLikes": "Post likes performed that day.",
- "follows": "Follows performed that day.",
- "messagesSent": "Messages sent that day.",
- "totalMessageStarted": "Message threads started that day.",
- "totalMessageReplies": "Replies received to messages that day.",
- "inmailMessagesSent": "InMail messages sent that day.",
- "totalInmailStarted": "InMail threads started that day.",
- "totalInmailReplies": "Replies received to InMails that day.",
- "connectionsSent": "Connection requests sent that day.",
- "connectionsAccepted": "Connection requests accepted that day.",
- "messageReplyRate": "Replies received divided by message threads started.",
- "inMailReplyRate": "InMail replies received divided by InMail threads started.",
- "connectionAcceptanceRate": "Connections accepted divided by connection requests sent.",
+ "senderId": "Identifier of the LinkedIn sender account the connection belongs to.",
+ "linkedin_id": "LinkedIn member identifier of the connection.",
+ "profileUrl": "URL of the connection's LinkedIn profile.",
+ "firstName": "The connection's first name.",
+ "lastName": "The connection's last name.",
+ "headline": "The connection's LinkedIn headline.",
+ "location": "The connection's location, as shown on LinkedIn.",
+ "companyName": "Name of the company the connection works at.",
+ "position": "The connection's job title at their current company.",
+ "emailAddress": "The connection's email address, when known.",
},
},
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/heyreach.py b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/heyreach.py
index 7f2ce238ed21..c4a0b3646ff7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/heyreach.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/heyreach.py
@@ -1,5 +1,5 @@
from collections.abc import Callable, Iterable, Iterator
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
from functools import partial
from typing import Any, Optional
@@ -14,7 +14,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import APIKeyAuth
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
+ BasePaginator,
OffsetPaginator,
+ PageNumberPaginator,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClient
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
@@ -33,12 +35,20 @@ STATS_START_DATE = "2020-01-01T00:00:00.000Z"
AUTH_HEADER = "X-API-KEY"
+STATS_BATCH_SIZE = 1000
+
@frozen
class HeyReachResumeConfig:
offset: int
+@frozen
+class StatsWindow:
+ start_date: str
+ end_date: str
+
+
def _offset_paginator() -> OffsetPaginator:
# Every HeyReach list endpoint is POST with `offset`/`limit` in the JSON body and a
# `{"totalCount": n, "items": [...]}` envelope.
@@ -51,6 +61,13 @@ def _offset_paginator() -> OffsetPaginator:
)
+def _child_paginator(endpoint_config: HeyReachEndpointConfig) -> BasePaginator:
+ if endpoint_config.page_number_pagination:
+ # The response carries a row count, not a page count, so paging stops at the first empty page.
+ return PageNumberPaginator(base_page=0, page_param="pageNumber", param_location="json")
+ return _offset_paginator()
+
+
def _make_client(api_key: str) -> RESTClient:
return RESTClient(
base_url=HEYREACH_BASE_URL,
@@ -154,7 +171,7 @@ def _fanout_rows(api_key: str, endpoint_config: HeyReachEndpointConfig) -> Itera
path=endpoint_config.path,
method="post",
json={**endpoint_config.request_body, fanout.body_param: parent_id},
- paginator=_offset_paginator(),
+ paginator=_child_paginator(endpoint_config),
data_selector="items",
):
# An API-provided field of the same name wins over the injected parent id.
@@ -163,27 +180,59 @@ def _fanout_rows(api_key: str, endpoint_config: HeyReachEndpointConfig) -> Itera
yield rows
-def _overall_stats_rows(api_key: str, endpoint_config: HeyReachEndpointConfig) -> Iterator[list[dict[str, Any]]]:
+def _stats_windows(window_days: int | None) -> Iterator[StatsWindow]:
+ now = datetime.now(tz=UTC)
+ if window_days is None:
+ yield StatsWindow(start_date=STATS_START_DATE, end_date=now.isoformat())
+ return
+ start = parser.parse(STATS_START_DATE)
+ step = timedelta(days=window_days)
+ while start < now:
+ # endDate is inclusive, so each window stops 1 ms before the next one starts.
+ end = min(start + step - timedelta(milliseconds=1), now)
+ yield StatsWindow(
+ start_date=start.isoformat(timespec="milliseconds"),
+ end_date=end.isoformat(timespec="milliseconds"),
+ )
+ start += step
+
+
+def _daily_stats_rows(
+ api_key: str,
+ endpoint_config: HeyReachEndpointConfig,
+ resumable_source_manager: ResumableSourceManager[HeyReachResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
client = _make_client(api_key)
- body = {
- # Empty arrays mean "all senders" / "all campaigns".
- "accountIds": [],
- "campaignIds": [],
- "startDate": STATS_START_DATE,
- "endDate": datetime.now(tz=UTC).isoformat(),
- }
- # The endpoint returns everything in one response; the selector resolves the day-keyed
- # `byDayStats` object, which paginate wraps as a single-element page.
- for page in client.paginate(
- path=endpoint_config.path,
- method="post",
- json=body,
- data_selector="byDayStats",
- ):
- for by_day in page:
- rows = [{"date": parser.parse(day), **(stats or {})} for day, stats in (by_day or {}).items()]
- if rows:
- yield rows
+ for window in _stats_windows(endpoint_config.stats_window_days):
+ body = {
+ # Empty arrays mean "all senders" / "all campaigns".
+ "accountIds": [],
+ "campaignIds": [],
+ "startDate": window.start_date,
+ "endDate": window.end_date,
+ }
+ # The endpoint returns everything in one response; the selector resolves the day-keyed
+ # `byDayStats` object, which paginate wraps as a single-element page.
+ for page in client.paginate(
+ path=endpoint_config.path,
+ method="post",
+ json=body,
+ data_selector="byDayStats",
+ ):
+ for by_day in page:
+ rows: list[dict[str, Any]] = []
+ for day, stats in (by_day or {}).items():
+ date = parser.parse(day)
+ # The by-campaign breakdown maps each day to a list of per-campaign stats.
+ for entry in stats if isinstance(stats, list) else [stats]:
+ rows.append({"date": date, **(entry or {})})
+ if len(rows) >= STATS_BATCH_SIZE:
+ yield rows
+ rows = []
+ if rows:
+ yield rows
+ # Windows before the account existed return no rows; nothing is buffered or staged here.
+ resumable_source_manager.safe_point()
def _build_response(endpoint_config: HeyReachEndpointConfig, items: Callable[[], Iterable[Any]]) -> SourceResponse:
@@ -209,7 +258,9 @@ def heyreach_source(
raise ValueError(f"HeyReach endpoint does not exist: {endpoint}")
if endpoint_config.daily_stats:
- return _build_response(endpoint_config, partial(_overall_stats_rows, api_key, endpoint_config))
+ return _build_response(
+ endpoint_config, partial(_daily_stats_rows, api_key, endpoint_config, resumable_source_manager)
+ )
if endpoint_config.fanout is not None:
# Fan-out runs restart from the first parent on resume: the parent set can change
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/settings.py
index dfb336e7fe2d..0dd26e9426ef 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/settings.py
@@ -36,8 +36,12 @@ class HeyReachEndpointConfig:
request_body: dict[str, Any] = field(default_factory=dict)
fanout: HeyReachFanoutConfig | None = None
# The endpoint returns one `byDayStats` object instead of the paginated `items` envelope,
- # and is reshaped into one row per day.
+ # and is reshaped into one row per day (per campaign, for the by-campaign breakdown).
daily_stats: bool = False
+ # Request daily stats in windows of this many days, so one response holds a bounded number of rows.
+ stats_window_days: int | None = None
+ # The endpoint pages with `pageNumber`/`pageSize` in the body instead of `offset`/`limit`.
+ page_number_pagination: bool = False
ENDPOINTS: dict[str, HeyReachEndpointConfig] = {
@@ -74,6 +78,17 @@ ENDPOINTS: dict[str, HeyReachEndpointConfig] = {
primary_keys=["id"],
partition_key="creationTime",
),
+ "list_companies": HeyReachEndpointConfig(
+ name="list_companies",
+ path="/list/GetCompaniesFromList",
+ # Companies carry no id; the company name is their identity within a list.
+ primary_keys=["listId", "name"],
+ fanout=HeyReachFanoutConfig(
+ parent="lists",
+ body_param="listId",
+ parent_body={"listType": "COMPANY_LIST"},
+ ),
+ ),
"list_leads": HeyReachEndpointConfig(
name="list_leads",
path="/list/GetLeadsFromList",
@@ -93,6 +108,23 @@ ENDPOINTS: dict[str, HeyReachEndpointConfig] = {
primary_keys=["date"],
daily_stats=True,
),
+ "campaign_stats": HeyReachEndpointConfig(
+ name="campaign_stats",
+ path="/stats/GetOverallStatsByCampaign",
+ primary_keys=["date", "campaignId"],
+ partition_key="date",
+ daily_stats=True,
+ # Rows grow with campaigns x days, so the full history doesn't fit one response.
+ stats_window_days=30,
+ ),
+ "sender_network": HeyReachEndpointConfig(
+ name="sender_network",
+ path="/MyNetwork/GetMyNetworkForSender",
+ primary_keys=["senderId", "profileUrl"],
+ request_body={"pageSize": PAGE_SIZE},
+ fanout=HeyReachFanoutConfig(parent="linkedin_accounts", body_param="senderId"),
+ page_number_pagination=True,
+ ),
}
# No list endpoint accepts a created-after filter we could verify server-side,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/source.py
index 53ce15b52171..2d7e9f708c39 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/source.py
@@ -108,8 +108,8 @@ class HeyReachSource(ResumableSource[HeyReachSourceConfig, HeyReachResumeConfig]
category=DataWarehouseSourceCategory.SALES,
label="HeyReach",
caption=(
- "Import campaigns, leads, LinkedIn sender accounts, inbox conversations, and daily "
- "outreach stats from HeyReach. Create an API key in HeyReach under "
+ "Import campaigns, leads, company lists, LinkedIn sender accounts and their networks, "
+ "inbox conversations, and daily outreach stats from HeyReach. Create an API key in HeyReach under "
"Settings > Integrations > HeyReach API."
),
docsUrl="https://posthog.com/docs/cdp/sources/heyreach",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/tests/test_heyreach.py b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/tests/test_heyreach.py
index e54827d20ae2..9a7966d9e834 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/tests/test_heyreach.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/tests/test_heyreach.py
@@ -1,10 +1,14 @@
+import itertools
+from datetime import UTC, datetime, timedelta
from typing import Any
import pytest
import requests
+from dateutil import parser
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+ RecordedRequest,
ScriptedResponse,
SourceDriver,
always,
@@ -82,9 +86,18 @@ class TestFanoutEndpoints:
{"campaignId": 2, "id": 21, "leadCampaignStatus": "Pending"},
]
- def test_list_leads_only_fans_out_lead_lists_and_skips_parents_without_id(self) -> None:
+ @pytest.mark.parametrize(
+ ("endpoint", "list_type", "child_path"),
+ [
+ ("list_leads", "USER_LIST", "/api/public/list/GetLeadsFromList"),
+ ("list_companies", "COMPANY_LIST", "/api/public/list/GetCompaniesFromList"),
+ ],
+ )
+ def test_list_children_fan_out_matching_list_type_and_skip_parents_without_id(
+ self, endpoint: str, list_type: str, child_path: str
+ ) -> None:
result = _driver().run(
- "list_leads",
+ endpoint,
[
_page([{"id": 5}, {"name": "row without id"}], 2),
_page([], 0),
@@ -92,17 +105,38 @@ class TestFanoutEndpoints:
)
assert result.raised is None
- assert result.paths[0] == "/api/public/list/GetAll"
- # Company lists serve companies via a different endpoint, so the parent listing is
- # restricted to lead lists.
- assert result.requests[0].json()["listType"] == "USER_LIST"
+ # Lead lists and company lists serve their rows through different endpoints, so the
+ # parent listing is restricted to the matching list type.
+ assert result.paths == ["/api/public/list/GetAll", child_path]
+ assert result.requests[0].json()["listType"] == list_type
# Only the parent with an id produces a child request, and an empty child page yields no batch.
- assert len(result.requests) == 2
assert result.requests[1].json()["listId"] == 5
assert result.items == []
+ def test_sender_network_pages_by_page_number_until_empty_page(self) -> None:
+ result = _driver().run(
+ "sender_network",
+ [
+ _page([{"id": 7}], 1),
+ _page([{"profileUrl": "https://www.linkedin.com/in/a"}], 2),
+ _page([{"profileUrl": "https://www.linkedin.com/in/b"}], 2),
+ _page([], 2),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/api/public/li_account/GetAll"] + ["/api/public/MyNetwork/GetMyNetworkForSender"] * 3
+ child_bodies = [request.json() for request in result.requests[1:]]
+ assert [body["pageNumber"] for body in child_bodies] == [0, 1, 2]
+ assert all(body["senderId"] == 7 and body["pageSize"] == 100 for body in child_bodies)
+ assert all("offset" not in body for body in child_bodies)
+ assert result.rows == [
+ {"senderId": 7, "profileUrl": "https://www.linkedin.com/in/a"},
+ {"senderId": 7, "profileUrl": "https://www.linkedin.com/in/b"},
+ ]
+
-class TestOverallStats:
+class TestDailyStats:
@pytest.mark.parametrize(
("payload", "expected_days"),
[
@@ -135,6 +169,39 @@ class TestOverallStats:
assert body["accountIds"] == []
assert body["campaignIds"] == []
+ def test_campaign_stats_requests_contiguous_windows_and_yields_one_row_per_day_and_campaign(self) -> None:
+ payload = {
+ "byDayStats": {
+ "2024-12-17T00:00:00Z": [
+ {"campaignId": 1, "campaignName": "A", "messagesSent": 3},
+ {"campaignId": 2, "campaignName": "B", "messagesSent": 4},
+ ],
+ "2024-12-18T00:00:00Z": [],
+ },
+ "overallStats": [{"campaignId": 1, "messagesSent": 3}],
+ }
+
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ body = request.json()
+ if body["startDate"] <= "2024-12-17" <= body["endDate"]:
+ return ScriptedResponse(json=payload)
+ return ScriptedResponse(json={"byDayStats": {}, "overallStats": []})
+
+ result = _driver().run("campaign_stats", respond)
+
+ assert result.raised is None
+ assert set(result.paths) == {"/api/public/stats/GetOverallStatsByCampaign"}
+ windows = [(parser.parse(r.json()["startDate"]), parser.parse(r.json()["endDate"])) for r in result.requests]
+ assert windows[0][0] == parser.parse(STATS_START_DATE)
+ assert windows[-1][1] > datetime.now(tz=UTC) - timedelta(minutes=5)
+ for (start, end), (next_start, _) in itertools.pairwise(windows):
+ assert end - start < timedelta(days=30)
+ assert next_start - end == timedelta(milliseconds=1)
+ assert [(row["date"].isoformat(), row["campaignId"], row["messagesSent"]) for row in result.rows] == [
+ ("2024-12-17T00:00:00+00:00", 1, 3),
+ ("2024-12-17T00:00:00+00:00", 2, 4),
+ ]
+
class TestValidateCredentials:
@pytest.mark.parametrize(