:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add heyreach campaign stats, company and network tables (#122035)
Tom Owers52187695c31d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 88fc9a2ecb6a..bbf77320d5b4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -5913,13 +5913,13 @@ Note: Parsed every GET operation in the official external API specification and## HeyReach — gaps-Today (7): `campaign_leads`, `campaigns`, `conversations`, `linkedin_accounts`, `list_leads`, `lists`, `overall_stats`+Today (10): `campaign_leads`, `campaign_stats`, `campaigns`, `conversations`, `linkedin_accounts`, `list_companies`, `list_leads`, `lists`, `overall_stats`, `sender_network`Diffed against: <https://documenter.gw.postman.com/api/collections/23808049/2sA2xb5F75?segregateAuth=true&versionTag=latest>-- [ ] `GetOverallStatsByCampaign (POST /api/public/stats/GetOverallStatsByCampaign)` — compare daily reply rates, connection acceptance, and messages sent for each campaign instead of workspace totals (high)-- [ ] `GetCompaniesFromList (POST /api/public/list/GetCompaniesFromList)` — analyze company audiences by industry, size, and location; current list_leads covers only lead lists (medium)-- [ ] `GetMyNetworkForSender (POST /api/public/MyNetwork/GetMyNetworkForSender)` — measure sender network size and identify connections outside active campaigns and lead lists (medium)+- [x] `GetOverallStatsByCampaign (POST /api/public/stats/GetOverallStatsByCampaign)` — compare daily reply rates, connection acceptance, and messages sent for each campaign instead of workspace totals (high)+- [x] `GetCompaniesFromList (POST /api/public/list/GetCompaniesFromList)` — analyze company audiences by industry, size, and location; current list_leads covers only lead lists (medium)+- [x] `GetMyNetworkForSender (POST /api/public/MyNetwork/GetMyNetworkForSender)` — measure sender network size and identify connections outside active campaigns and lead lists (medium)Note: Parsed all requests and response examples in the vendor's published Postman collection. Read-only POST endpoints are included because this API uses POST for lists. Conversations already contain messages and tags. Company lists and sender networks need fan-out; organization administration is excluded.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/canonical_descriptions.pyindex c088de2e3777..e2c826a49cd9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/canonical_descriptions.py@@ -18,6 +18,24 @@ _LEAD_PROFILE_COLUMNS = {"emailAddress": "The lead's email address, when known.",}+_DAILY_STATS_COLUMNS = {+ "date": "Day the stats cover.",+ "profileViews": "Profile views performed that day.",+ "postLikes": "Post likes performed that day.",+ "follows": "Follows performed that day.",+ "messagesSent": "Messages sent that day.",+ "totalMessageStarted": "Message threads started that day.",+ "totalMessageReplies": "Replies received to messages that day.",+ "inmailMessagesSent": "InMail messages sent that day.",+ "totalInmailStarted": "InMail threads started that day.",+ "totalInmailReplies": "Replies received to InMails that day.",+ "connectionsSent": "Connection requests sent that day.",+ "connectionsAccepted": "Connection requests accepted that day.",+ "messageReplyRate": "Replies received divided by message threads started.",+ "inMailReplyRate": "InMail replies received divided by InMail threads started.",+ "connectionAcceptanceRate": "Connections accepted divided by connection requests sent.",+}+CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"campaigns": {"description": "An outreach campaign: a sequence of LinkedIn actions sent to a list of leads from one or more sender accounts.",@@ -107,22 +125,48 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"overall_stats": {"description": "Daily outreach totals across all sender accounts and campaigns.","docs_url": _API_DOCS_URL,+ "columns": _DAILY_STATS_COLUMNS,+ },+ "campaign_stats": {+ "description": "Daily outreach totals for each campaign, across all sender accounts.",+ "docs_url": _API_DOCS_URL,+ "columns": {+ "campaignId": "Identifier of the campaign the stats cover.",+ "campaignName": "Name of the campaign.",+ "isCampaignDeleted": "Whether the campaign has been deleted.",+ **_DAILY_STATS_COLUMNS,+ },+ },+ "list_companies": {+ "description": "A company belonging to a company list, with its LinkedIn company page details.",+ "docs_url": _API_DOCS_URL,+ "columns": {+ "listId": "Identifier of the company list the company belongs to.",+ "name": "Name of the company.",+ "description": "Description of the company from its LinkedIn page.",+ "industry": "Industry of the company.",+ "imageUrl": "URL of the company's logo.",+ "companySize": "Company size range, as shown on LinkedIn.",+ "employeesOnLinkedIn": "Number of employees on LinkedIn.",+ "location": "Location of the company.",+ "specialities": "Specialities listed on the company's LinkedIn page.",+ "website": "The company's website.",+ },+ },+ "sender_network": {+ "description": "A LinkedIn connection in the network of a sender account.",+ "docs_url": _API_DOCS_URL,"columns": {- "date": "Day the stats cover.",- "profileViews": "Profile views performed that day.",- "postLikes": "Post likes performed that day.",- "follows": "Follows performed that day.",- "messagesSent": "Messages sent that day.",- "totalMessageStarted": "Message threads started that day.",- "totalMessageReplies": "Replies received to messages that day.",- "inmailMessagesSent": "InMail messages sent that day.",- "totalInmailStarted": "InMail threads started that day.",- "totalInmailReplies": "Replies received to InMails that day.",- "connectionsSent": "Connection requests sent that day.",- "connectionsAccepted": "Connection requests accepted that day.",- "messageReplyRate": "Replies received divided by message threads started.",- "inMailReplyRate": "InMail replies received divided by InMail threads started.",- "connectionAcceptanceRate": "Connections accepted divided by connection requests sent.",+ "senderId": "Identifier of the LinkedIn sender account the connection belongs to.",+ "linkedin_id": "LinkedIn member identifier of the connection.",+ "profileUrl": "URL of the connection's LinkedIn profile.",+ "firstName": "The connection's first name.",+ "lastName": "The connection's last name.",+ "headline": "The connection's LinkedIn headline.",+ "location": "The connection's location, as shown on LinkedIn.",+ "companyName": "Name of the company the connection works at.",+ "position": "The connection's job title at their current company.",+ "emailAddress": "The connection's email address, when known.",},},}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/heyreach.py b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/heyreach.pyindex 7f2ce238ed21..c4a0b3646ff7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/heyreach.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/heyreach.py@@ -1,5 +1,5 @@from collections.abc import Callable, Iterable, Iterator-from datetime import UTC, datetime+from datetime import UTC, datetime, timedeltafrom functools import partialfrom typing import Any, Optional@@ -14,7 +14,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import APIKeyAuthfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (+ BasePaginator,OffsetPaginator,+ PageNumberPaginator,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClientfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager@@ -33,12 +35,20 @@ STATS_START_DATE = "2020-01-01T00:00:00.000Z"AUTH_HEADER = "X-API-KEY"+STATS_BATCH_SIZE = 1000+@frozenclass HeyReachResumeConfig:offset: int+@frozen+class StatsWindow:+ start_date: str+ end_date: str++def _offset_paginator() -> OffsetPaginator:# Every HeyReach list endpoint is POST with `offset`/`limit` in the JSON body and a# `{"totalCount": n, "items": [...]}` envelope.@@ -51,6 +61,13 @@ def _offset_paginator() -> OffsetPaginator:)+def _child_paginator(endpoint_config: HeyReachEndpointConfig) -> BasePaginator:+ if endpoint_config.page_number_pagination:+ # The response carries a row count, not a page count, so paging stops at the first empty page.+ return PageNumberPaginator(base_page=0, page_param="pageNumber", param_location="json")+ return _offset_paginator()++def _make_client(api_key: str) -> RESTClient:return RESTClient(base_url=HEYREACH_BASE_URL,@@ -154,7 +171,7 @@ def _fanout_rows(api_key: str, endpoint_config: HeyReachEndpointConfig) -> Iterapath=endpoint_config.path,method="post",json={**endpoint_config.request_body, fanout.body_param: parent_id},- paginator=_offset_paginator(),+ paginator=_child_paginator(endpoint_config),data_selector="items",):# An API-provided field of the same name wins over the injected parent id.@@ -163,27 +180,59 @@ def _fanout_rows(api_key: str, endpoint_config: HeyReachEndpointConfig) -> Iterayield rows-def _overall_stats_rows(api_key: str, endpoint_config: HeyReachEndpointConfig) -> Iterator[list[dict[str, Any]]]:+def _stats_windows(window_days: int | None) -> Iterator[StatsWindow]:+ now = datetime.now(tz=UTC)+ if window_days is None:+ yield StatsWindow(start_date=STATS_START_DATE, end_date=now.isoformat())+ return+ start = parser.parse(STATS_START_DATE)+ step = timedelta(days=window_days)+ while start < now:+ # endDate is inclusive, so each window stops 1 ms before the next one starts.+ end = min(start + step - timedelta(milliseconds=1), now)+ yield StatsWindow(+ start_date=start.isoformat(timespec="milliseconds"),+ end_date=end.isoformat(timespec="milliseconds"),+ )+ start += step+++def _daily_stats_rows(+ api_key: str,+ endpoint_config: HeyReachEndpointConfig,+ resumable_source_manager: ResumableSourceManager[HeyReachResumeConfig],+) -> Iterator[list[dict[str, Any]]]:client = _make_client(api_key)- body = {- # Empty arrays mean "all senders" / "all campaigns".- "accountIds": [],- "campaignIds": [],- "startDate": STATS_START_DATE,- "endDate": datetime.now(tz=UTC).isoformat(),- }- # The endpoint returns everything in one response; the selector resolves the day-keyed- # `byDayStats` object, which paginate wraps as a single-element page.- for page in client.paginate(- path=endpoint_config.path,- method="post",- json=body,- data_selector="byDayStats",- ):- for by_day in page:- rows = [{"date": parser.parse(day), **(stats or {})} for day, stats in (by_day or {}).items()]- if rows:- yield rows+ for window in _stats_windows(endpoint_config.stats_window_days):+ body = {+ # Empty arrays mean "all senders" / "all campaigns".+ "accountIds": [],+ "campaignIds": [],+ "startDate": window.start_date,+ "endDate": window.end_date,+ }+ # The endpoint returns everything in one response; the selector resolves the day-keyed+ # `byDayStats` object, which paginate wraps as a single-element page.+ for page in client.paginate(+ path=endpoint_config.path,+ method="post",+ json=body,+ data_selector="byDayStats",+ ):+ for by_day in page:+ rows: list[dict[str, Any]] = []+ for day, stats in (by_day or {}).items():+ date = parser.parse(day)+ # The by-campaign breakdown maps each day to a list of per-campaign stats.+ for entry in stats if isinstance(stats, list) else [stats]:+ rows.append({"date": date, **(entry or {})})+ if len(rows) >= STATS_BATCH_SIZE:+ yield rows+ rows = []+ if rows:+ yield rows+ # Windows before the account existed return no rows; nothing is buffered or staged here.+ resumable_source_manager.safe_point()def _build_response(endpoint_config: HeyReachEndpointConfig, items: Callable[[], Iterable[Any]]) -> SourceResponse:@@ -209,7 +258,9 @@ def heyreach_source(raise ValueError(f"HeyReach endpoint does not exist: {endpoint}")if endpoint_config.daily_stats:- return _build_response(endpoint_config, partial(_overall_stats_rows, api_key, endpoint_config))+ return _build_response(+ endpoint_config, partial(_daily_stats_rows, api_key, endpoint_config, resumable_source_manager)+ )if endpoint_config.fanout is not None:# Fan-out runs restart from the first parent on resume: the parent set can changediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/settings.pyindex dfb336e7fe2d..0dd26e9426ef 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/settings.py@@ -36,8 +36,12 @@ class HeyReachEndpointConfig:request_body: dict[str, Any] = field(default_factory=dict)fanout: HeyReachFanoutConfig | None = None# The endpoint returns one `byDayStats` object instead of the paginated `items` envelope,- # and is reshaped into one row per day.+ # and is reshaped into one row per day (per campaign, for the by-campaign breakdown).daily_stats: bool = False+ # Request daily stats in windows of this many days, so one response holds a bounded number of rows.+ stats_window_days: int | None = None+ # The endpoint pages with `pageNumber`/`pageSize` in the body instead of `offset`/`limit`.+ page_number_pagination: bool = FalseENDPOINTS: dict[str, HeyReachEndpointConfig] = {@@ -74,6 +78,17 @@ ENDPOINTS: dict[str, HeyReachEndpointConfig] = {primary_keys=["id"],partition_key="creationTime",),+ "list_companies": HeyReachEndpointConfig(+ name="list_companies",+ path="/list/GetCompaniesFromList",+ # Companies carry no id; the company name is their identity within a list.+ primary_keys=["listId", "name"],+ fanout=HeyReachFanoutConfig(+ parent="lists",+ body_param="listId",+ parent_body={"listType": "COMPANY_LIST"},+ ),+ ),"list_leads": HeyReachEndpointConfig(name="list_leads",path="/list/GetLeadsFromList",@@ -93,6 +108,23 @@ ENDPOINTS: dict[str, HeyReachEndpointConfig] = {primary_keys=["date"],daily_stats=True,),+ "campaign_stats": HeyReachEndpointConfig(+ name="campaign_stats",+ path="/stats/GetOverallStatsByCampaign",+ primary_keys=["date", "campaignId"],+ partition_key="date",+ daily_stats=True,+ # Rows grow with campaigns x days, so the full history doesn't fit one response.+ stats_window_days=30,+ ),+ "sender_network": HeyReachEndpointConfig(+ name="sender_network",+ path="/MyNetwork/GetMyNetworkForSender",+ primary_keys=["senderId", "profileUrl"],+ request_body={"pageSize": PAGE_SIZE},+ fanout=HeyReachFanoutConfig(parent="linkedin_accounts", body_param="senderId"),+ page_number_pagination=True,+ ),}# No list endpoint accepts a created-after filter we could verify server-side,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/source.pyindex 53ce15b52171..2d7e9f708c39 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/source.py@@ -108,8 +108,8 @@ class HeyReachSource(ResumableSource[HeyReachSourceConfig, HeyReachResumeConfig]category=DataWarehouseSourceCategory.SALES,label="HeyReach",caption=(- "Import campaigns, leads, LinkedIn sender accounts, inbox conversations, and daily "- "outreach stats from HeyReach. Create an API key in HeyReach under "+ "Import campaigns, leads, company lists, LinkedIn sender accounts and their networks, "+ "inbox conversations, and daily outreach stats from HeyReach. Create an API key in HeyReach under ""Settings > Integrations > HeyReach API."),docsUrl="https://posthog.com/docs/cdp/sources/heyreach",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/tests/test_heyreach.py b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/tests/test_heyreach.pyindex e54827d20ae2..9a7966d9e834 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/tests/test_heyreach.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/heyreach/tests/test_heyreach.py@@ -1,10 +1,14 @@+import itertools+from datetime import UTC, datetime, timedeltafrom typing import Anyimport pytestimport requests+from dateutil import parserfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (+ RecordedRequest,ScriptedResponse,SourceDriver,always,@@ -82,9 +86,18 @@ class TestFanoutEndpoints:{"campaignId": 2, "id": 21, "leadCampaignStatus": "Pending"},]- def test_list_leads_only_fans_out_lead_lists_and_skips_parents_without_id(self) -> None:+ @pytest.mark.parametrize(+ ("endpoint", "list_type", "child_path"),+ [+ ("list_leads", "USER_LIST", "/api/public/list/GetLeadsFromList"),+ ("list_companies", "COMPANY_LIST", "/api/public/list/GetCompaniesFromList"),+ ],+ )+ def test_list_children_fan_out_matching_list_type_and_skip_parents_without_id(+ self, endpoint: str, list_type: str, child_path: str+ ) -> None:result = _driver().run(- "list_leads",+ endpoint,[_page([{"id": 5}, {"name": "row without id"}], 2),_page([], 0),@@ -92,17 +105,38 @@ class TestFanoutEndpoints:)assert result.raised is None- assert result.paths[0] == "/api/public/list/GetAll"- # Company lists serve companies via a different endpoint, so the parent listing is- # restricted to lead lists.- assert result.requests[0].json()["listType"] == "USER_LIST"+ # Lead lists and company lists serve their rows through different endpoints, so the+ # parent listing is restricted to the matching list type.+ assert result.paths == ["/api/public/list/GetAll", child_path]+ assert result.requests[0].json()["listType"] == list_type# Only the parent with an id produces a child request, and an empty child page yields no batch.- assert len(result.requests) == 2assert result.requests[1].json()["listId"] == 5assert result.items == []+ def test_sender_network_pages_by_page_number_until_empty_page(self) -> None:+ result = _driver().run(+ "sender_network",+ [+ _page([{"id": 7}], 1),+ _page([{"profileUrl": "https://www.linkedin.com/in/a"}], 2),+ _page([{"profileUrl": "https://www.linkedin.com/in/b"}], 2),+ _page([], 2),+ ],+ )++ assert result.raised is None+ assert result.paths == ["/api/public/li_account/GetAll"] + ["/api/public/MyNetwork/GetMyNetworkForSender"] * 3+ child_bodies = [request.json() for request in result.requests[1:]]+ assert [body["pageNumber"] for body in child_bodies] == [0, 1, 2]+ assert all(body["senderId"] == 7 and body["pageSize"] == 100 for body in child_bodies)+ assert all("offset" not in body for body in child_bodies)+ assert result.rows == [+ {"senderId": 7, "profileUrl": "https://www.linkedin.com/in/a"},+ {"senderId": 7, "profileUrl": "https://www.linkedin.com/in/b"},+ ]+-class TestOverallStats:+class TestDailyStats:@pytest.mark.parametrize(("payload", "expected_days"),[@@ -135,6 +169,39 @@ class TestOverallStats:assert body["accountIds"] == []assert body["campaignIds"] == []+ def test_campaign_stats_requests_contiguous_windows_and_yields_one_row_per_day_and_campaign(self) -> None:+ payload = {+ "byDayStats": {+ "2024-12-17T00:00:00Z": [+ {"campaignId": 1, "campaignName": "A", "messagesSent": 3},+ {"campaignId": 2, "campaignName": "B", "messagesSent": 4},+ ],+ "2024-12-18T00:00:00Z": [],+ },+ "overallStats": [{"campaignId": 1, "messagesSent": 3}],+ }++ def respond(request: RecordedRequest) -> ScriptedResponse:+ body = request.json()+ if body["startDate"] <= "2024-12-17" <= body["endDate"]:+ return ScriptedResponse(json=payload)+ return ScriptedResponse(json={"byDayStats": {}, "overallStats": []})++ result = _driver().run("campaign_stats", respond)++ assert result.raised is None+ assert set(result.paths) == {"/api/public/stats/GetOverallStatsByCampaign"}+ windows = [(parser.parse(r.json()["startDate"]), parser.parse(r.json()["endDate"])) for r in result.requests]+ assert windows[0][0] == parser.parse(STATS_START_DATE)+ assert windows[-1][1] > datetime.now(tz=UTC) - timedelta(minutes=5)+ for (start, end), (next_start, _) in itertools.pairwise(windows):+ assert end - start < timedelta(days=30)+ assert next_start - end == timedelta(milliseconds=1)+ assert [(row["date"].isoformat(), row["campaignId"], row["messagesSent"]) for row in result.rows] == [+ ("2024-12-17T00:00:00+00:00", 1, 3),+ ("2024-12-17T00:00:00+00:00", 2, 4),+ ]+class TestValidateCredentials:@pytest.mark.parametrize(