Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add nocrm lead history, comments and follow-up tables (#123166)

Tom Owers76b23adb2b0e
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index bcc0f7137bfb..e48203001f30 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -789,8 +789,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/news_api/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/newsdata.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/nocrm.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/northpass_lms/northpass_lms.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/northpass_lms/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/notion/notion.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 28bbde2700f8..ee8a198071a3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -8433,12 +8433,12 @@ Today (10): `activities`, `categories`, `client_folders`, `fields`, `leads`, `pi
Diffed against: <https://www.nocrm.io/api>
-- [ ] `leads/{lead_id}/action_histories` — per-lead action and step-change history - the state-transition table needed for pipeline velocity and conversion timing (high)
-- [ ] `follow_ups` — scheduled follow-up tasks/reminders on leads, the core activity object driving rep workload analysis (high)
-- [ ] `leads/{lead_id}/comments` — comment thread per lead, the narrative activity stream alongside activities (medium)
+- [x] `leads/{lead_id}/action_histories` — per-lead action and step-change history - the state-transition table needed for pipeline velocity and conversion timing (high). Added as `lead_action_histories` (fan-out over `leads`).
+- [x] `follow_ups` — scheduled follow-up tasks/reminders on leads, the core activity object driving rep workload analysis (high). Added as `follow_ups` (fan-out over `users`, since the endpoint requires a `lead_id` or `user_id`).
+- [x] `leads/{lead_id}/comments` — comment thread per lead, the narrative activity stream alongside activities (medium). Added as `lead_comments` (fan-out over `leads`).
- [ ] `spreadsheets` — prospecting lists - the top-of-funnel container that leads are created from, with no equivalent table today (medium)
- [ ] `spreadsheets/{spreadsheet_id}/rows` — prospect rows inside prospecting lists, plus their conversion-to-lead linkage (medium)
-- [ ] `follow_ups/{follow_up_id}/tasks` — individual tasks within a follow-up sequence, the completion-level detail (medium)
+- [x] `follow_ups/{follow_up_id}/tasks` — individual tasks within a follow-up sequence, the completion-level detail (medium). Skipped: the API has no list endpoint for tasks, only `PUT /follow_ups/{follow_up_id}/tasks/{task_id}`; `follow_ups` rows carry `tasks_count` and `tasks_done_count`.
- [ ] `spreadsheets/{spreadsheet_id}/comments` — comments on prospecting lists and rows, engagement history before a lead exists (low)
- [ ] `leads/{lead_id}/attachments` — attachment metadata per lead (counts/types, not file bytes) (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/canonical_descriptions.py
index 8b8877d203e2..8211ac4d8e1f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/canonical_descriptions.py
@@ -121,4 +121,54 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"type": "Data type of the custom field.",
},
},
+ "lead_action_histories": {
+ "description": "History of actions on each lead, such as step and status changes, amount updates, assignments, comments and emails.",
+ "docs_url": "https://www.nocrm.io/api",
+ "columns": {
+ "id": "Unique identifier for the action history entry.",
+ "lead_id": "Identifier of the lead the action was made on.",
+ "action_type": "Type of action: client_assigned, step_changed, status_changed, amount_set, user_assigned, lead_created, comment_added or email_sent.",
+ "action_item": "Details of the action, such as the new step, status or amount.",
+ "created_at": "Timestamp the action was made.",
+ "user": "User who made the action.",
+ },
+ },
+ "lead_comments": {
+ "description": "Comments posted on each lead, including logged activities and emails.",
+ "docs_url": "https://www.nocrm.io/api",
+ "columns": {
+ "id": "Unique identifier for the comment.",
+ "lead_id": "Identifier of the lead the comment is on.",
+ "content": "Content of the comment.",
+ "raw_content": "Raw content of the comment.",
+ "commented_item": "Item the comment is attached to.",
+ "activity_id": "Identifier of the activity set on the comment.",
+ "activity": "Activity set on the comment.",
+ "is_pinned": "Whether the comment is pinned on the lead.",
+ "attachments": "Attachments on the comment.",
+ "extended_info": "Extra details for email comments, such as sender, recipients and subject.",
+ "action_item": "Item linked to the comment, such as an email.",
+ "reactions": "Reactions on the comment.",
+ "created_at": "Timestamp the comment was created.",
+ "user": "User who wrote the comment.",
+ },
+ },
+ "follow_ups": {
+ "description": "Post-sales processes (follow-ups) on leads, each a list of tasks assigned to a user.",
+ "docs_url": "https://www.nocrm.io/api",
+ "columns": {
+ "id": "Unique identifier for the post-sales process.",
+ "title": "Title of the post-sales process.",
+ "description": "Description of the post-sales process.",
+ "status": "Status of the post-sales process: todo, standby or done.",
+ "tags": "List of tags applied to the post-sales process.",
+ "tasks_count": "Number of tasks in the post-sales process.",
+ "tasks_done_count": "Number of tasks marked as done.",
+ "next_action_at": "Timestamp of the next scheduled task.",
+ "lead_id": "Identifier of the lead the post-sales process belongs to.",
+ "user_id": "Identifier of the user the post-sales process is assigned to.",
+ "created_at": "Timestamp the post-sales process was created.",
+ "updated_at": "Timestamp the post-sales process was last updated.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/nocrm.py b/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/nocrm.py
index bef635d76597..bea8dffb88cf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/nocrm.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/nocrm.py
@@ -1,7 +1,7 @@
import re
import dataclasses
from datetime import UTC, date, datetime
-from typing import Any, Optional
+from typing import Any, Optional, cast
from urllib3.util.retry import Retry
@@ -10,21 +10,24 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
OffsetPaginator,
+ SinglePagePaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.nocrm.settings import (
NOCRM_ENDPOINTS,
+ PAGE_SIZE,
NoCRMEndpointConfig,
)
-# noCRM caps `/leads` at a default of 100 per page; request the max to minimise round-trips against
-# the low (~2000 req/day) account quota.
-PAGE_SIZE = 100
-
# noCRM reports the grand total (used to stop before an extra empty request) in this response header.
TOTAL_COUNT_HEADER = "X-TOTAL-COUNT"
@@ -124,11 +127,13 @@ def _build_base_params(
return params
[email protected](frozen=True)
class NoCRMResumeConfig:
# Offset (row count already consumed) to resume limit/offset pagination from. The incremental
# window is recomputed from the job's stable last-value, so only the offset needs persisting.
offset: int = 0
+ # `build_dependent_resource` checkpoint for the per-lead / per-user fan-out walks.
+ fanout_state: Optional[dict[str, Any]] = None
class NoCRMOffsetPaginator(OffsetPaginator):
@@ -155,6 +160,78 @@ class NoCRMOffsetPaginator(OffsetPaginator):
super().update_state(response, data)
+def _client_config(api_key: str, subdomain: str) -> ClientConfig:
+ return {
+ "base_url": _base_url(subdomain),
+ # Non-secret header only; the API key rides on the framework auth so it's redacted from
+ # logs, samples, and raised error messages.
+ "headers": {"Accept": "application/json"},
+ "auth": {"type": "api_key", "api_key": api_key, "name": "X-API-KEY", "location": "header"},
+ # Pin every request (including seeded resume pages) to the account's own *.nocrm.io host,
+ # and refuse to follow redirects, so the key can't be steered to another origin.
+ "allowed_hosts": [],
+ "allow_redirects": False,
+ }
+
+
+def _paginator(config: NoCRMEndpointConfig) -> OffsetPaginator | SinglePagePaginator:
+ # List endpoints return a bare JSON array; no data_selector needed. noCRM reports the grand
+ # total in a header, so stop on the header total / short / empty page.
+ if not config.paginated:
+ return SinglePagePaginator()
+ return NoCRMOffsetPaginator(limit=PAGE_SIZE, total_path=None, total_header=TOTAL_COUNT_HEADER)
+
+
+def _fanout_source(
+ api_key: str,
+ subdomain: str,
+ endpoint: str,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[NoCRMResumeConfig],
+) -> SourceResponse:
+ config = NOCRM_ENDPOINTS[endpoint]
+ assert config.fanout is not None
+ parent_config = NOCRM_ENDPOINTS[config.fanout.parent_name]
+
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+
+ def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if state:
+ resumable_source_manager.save_state(NoCRMResumeConfig(fanout_state=state))
+
+ child = cast(
+ Resource,
+ build_dependent_resource(
+ endpoint_configs=NOCRM_ENDPOINTS,
+ child_endpoint=endpoint,
+ fanout=config.fanout,
+ client_config=_client_config(api_key, subdomain),
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ parent_endpoint_extra={"paginator": _paginator(parent_config)},
+ child_endpoint_extra={"paginator": _paginator(config)},
+ # The offset paginator sends `limit` itself, and the unpaginated children take none.
+ page_size_param=None,
+ resume_hook=save_fanout_checkpoint,
+ initial_paginator_state=resume.fanout_state if resume else None,
+ ),
+ )
+
+ return SourceResponse(
+ name=endpoint,
+ items=lambda: child,
+ primary_keys=config.primary_keys,
+ partition_count=1,
+ partition_size=1,
+ partition_mode="datetime" if config.partition_key else None,
+ partition_format="month" if config.partition_key else None,
+ partition_keys=[config.partition_key] if config.partition_key else None,
+ )
+
+
def nocrm_source(
api_key: str,
subdomain: str,
@@ -166,31 +243,20 @@ def nocrm_source(
db_incremental_field_last_value: Optional[Any] = None,
) -> SourceResponse:
config = NOCRM_ENDPOINTS[endpoint]
+ if config.fanout is not None:
+ return _fanout_source(api_key, subdomain, endpoint, team_id, job_id, resumable_source_manager)
+
base_params = _build_base_params(config, should_use_incremental_field, db_incremental_field_last_value)
rest_config: RESTAPIConfig = {
- "client": {
- "base_url": _base_url(subdomain),
- # Non-secret header only; the API key rides on the framework auth so it's redacted from
- # logs, samples, and raised error messages.
- "headers": {"Accept": "application/json"},
- "auth": {"type": "api_key", "api_key": api_key, "name": "X-API-KEY", "location": "header"},
- # Pin every request (including seeded resume pages) to the account's own *.nocrm.io host,
- # and refuse to follow redirects, so the key can't be steered to another origin.
- "allowed_hosts": [],
- "allow_redirects": False,
- },
+ "client": _client_config(api_key, subdomain),
"resources": [
{
"name": endpoint,
"endpoint": {
"path": config.path,
"params": base_params,
- # List endpoints return a bare JSON array; no data_selector needed. noCRM reports
- # the grand total in a header, so stop on the header total / short / empty page.
- "paginator": NoCRMOffsetPaginator(
- limit=PAGE_SIZE, total_path=None, total_header=TOTAL_COUNT_HEADER
- ),
+ "paginator": _paginator(config),
},
}
],
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/settings.py
index dd7dc3e6bfc4..ee5142fbbd04 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/settings.py
@@ -1,10 +1,17 @@
from dataclasses import dataclass, field
from typing import Optional
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+# noCRM caps `/leads` at a default of 100 per page; request the max to minimise round-trips against
+# the low (~2000 req/day) account quota.
+PAGE_SIZE = 100
-@dataclass
+
+@dataclass(frozen=True)
class NoCRMEndpointConfig:
name: str
path: str
@@ -27,6 +34,13 @@ class NoCRMEndpointConfig:
partition_key: Optional[str] = None
primary_keys: list[str] = field(default_factory=lambda: ["id"])
should_sync_default: bool = True
+ # Set for endpoints noCRM only exposes per parent row (per lead / per user).
+ fanout: Optional[DependentEndpointConfig] = None
+ # False for child endpoints that return their whole collection in one response, with no
+ # documented limit/offset params.
+ paginated: bool = True
+ default_incremental_field: Optional[str] = None
+ page_size: int = PAGE_SIZE
def _updated_at_incremental_fields() -> list[IncrementalField]:
@@ -42,6 +56,30 @@ def _updated_at_incremental_fields() -> list[IncrementalField]:
]
+# Lead sub-resources are only listable per lead, so these fan out over `/leads` (one request per lead).
+LEAD_FANOUT = DependentEndpointConfig(
+ parent_name="leads",
+ resolve_param="lead_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "lead_id"},
+ # `/leads` hides unassigned leads unless asked, which would drop their histories and comments.
+ parent_params={"order": "id", "direction": "asc", "include_unassigned": "true"},
+ # A lead deleted between the parent listing and its child fetch 404s; skip it.
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+)
+
+# `/follow_ups` requires a `lead_id` or `user_id`. There are far fewer users than leads, so fan out
+# over users. The default status set is undocumented, so ask for every status explicitly.
+USER_FANOUT = DependentEndpointConfig(
+ parent_name="users",
+ resolve_param="user_id",
+ resolve_field="id",
+ include_from_parent=[],
+ child_params={"status": "todo,standby,done"},
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+)
+
NOCRM_ENDPOINTS: dict[str, NoCRMEndpointConfig] = {
"leads": NoCRMEndpointConfig(
name="leads",
@@ -98,6 +136,35 @@ NOCRM_ENDPOINTS: dict[str, NoCRMEndpointConfig] = {
path="/fields",
incremental_fields=[],
),
+ # Per-lead fan-outs cost one request per lead against the daily quota, so they are opt-in.
+ "lead_action_histories": NoCRMEndpointConfig(
+ name="lead_action_histories",
+ path="/leads/{lead_id}/action_histories",
+ incremental_fields=[],
+ partition_key="created_at",
+ primary_keys=["lead_id", "id"],
+ should_sync_default=False,
+ fanout=LEAD_FANOUT,
+ ),
+ "lead_comments": NoCRMEndpointConfig(
+ name="lead_comments",
+ path="/leads/{lead_id}/comments",
+ incremental_fields=[],
+ partition_key="created_at",
+ primary_keys=["lead_id", "id"],
+ should_sync_default=False,
+ fanout=LEAD_FANOUT,
+ paginated=False,
+ ),
+ "follow_ups": NoCRMEndpointConfig(
+ name="follow_ups",
+ # The framework only binds resolve params in the path, so `user_id` rides in the query string here.
+ path="/follow_ups?user_id={user_id}",
+ incremental_fields=[],
+ partition_key="created_at",
+ fanout=USER_FANOUT,
+ paginated=False,
+ ),
}
ENDPOINTS = tuple(NOCRM_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/tests/test_nocrm.py b/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/tests/test_nocrm.py
index c688b9a4eec8..8df3fb6af24c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/tests/test_nocrm.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/tests/test_nocrm.py
@@ -6,6 +6,7 @@ from parameterized import parameterized
from requests import HTTPError
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+ RecordedRequest,
ScriptedResponse,
SourceDriver,
scripted_network,
@@ -201,3 +202,67 @@ class TestSourceResponse:
assert result.response.primary_keys == ["id"]
assert result.response.partition_mode is None
assert result.response.partition_keys is None
+
+
+_LEADS_QUERY = {"include_unassigned": ("true",)}
+
+
+class TestFanout:
+ @parameterized.expand(
+ [
+ ("lead_action_histories", "/api/v2/leads", _LEADS_QUERY, "/api/v2/leads/{id}/action_histories", {}, True),
+ ("lead_comments", "/api/v2/leads", _LEADS_QUERY, "/api/v2/leads/{id}/comments", {}, False),
+ ("follow_ups", "/api/v2/users", {}, "/api/v2/follow_ups", {"status": ("todo,standby,done",)}, False),
+ ]
+ )
+ def test_child_requests_per_parent(
+ self,
+ endpoint: str,
+ parent_path: str,
+ parent_query: dict[str, tuple[str, ...]],
+ child_path: str,
+ child_query: dict[str, tuple[str, ...]],
+ child_paginated: bool,
+ ) -> None:
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == parent_path:
+ return ScriptedResponse(json=_page(2, 1))
+ if "/leads/2/" in request.path or request.query.get("user_id") == ("2",):
+ # A parent deleted between the listing and the child fetch must not fail the sync.
+ return ScriptedResponse(status=404, json={"error": "record_not_found"})
+ return ScriptedResponse(json=[{"id": 10, "lead_id": 1, "created_at": "2026-01-01T00:00:00.000Z"}])
+
+ result = _driver().run(endpoint, respond)
+
+ assert result.raised is None
+ assert [r["id"] for r in result.rows] == [10]
+ assert result.rows[0]["lead_id"] == 1
+ parent_requests = [r for r in result.requests if r.path == parent_path]
+ for key, value in parent_query.items():
+ assert parent_requests[0].query[key] == value
+ child_requests = [r for r in result.requests if r.path != parent_path]
+ assert len(child_requests) == 2
+ for request in child_requests:
+ for key, value in child_query.items():
+ assert request.query[key] == value
+ # Unpaginated endpoints document no limit/offset, so they must not be sent.
+ assert ("limit" in request.query) is child_paginated
+ if endpoint == "follow_ups":
+ assert [r.query["user_id"] for r in child_requests] == [("1",), ("2",)]
+ else:
+ assert [r.path for r in child_requests] == [child_path.format(id=1), child_path.format(id=2)]
+
+ def test_action_histories_walk_every_page_of_a_lead(self) -> None:
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == "/api/v2/leads":
+ return ScriptedResponse(json=_page(1, 7))
+ offset = int(request.query.get("offset", ("0",))[0])
+ return ScriptedResponse(json=_page(PAGE_SIZE if offset == 0 else 3, offset + 1))
+
+ result = _driver().run("lead_action_histories", respond)
+
+ assert result.raised is None
+ assert len(result.rows) == PAGE_SIZE + 3
+ assert {r["lead_id"] for r in result.rows} == {7}
+ assert result.response is not None
+ assert result.response.primary_keys == ["lead_id", "id"]