:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add nocrm lead history, comments and follow-up tables (#123166)
Tom Owers76b23adb2b0e
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex bcc0f7137bfb..e48203001f30 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -789,8 +789,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/news_api/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/newsdata.py1 products/warehouse_sources/backend/temporal/data_imports/sources/newsdata/settings.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/nocrm.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/northpass_lms/northpass_lms.py1 products/warehouse_sources/backend/temporal/data_imports/sources/northpass_lms/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/notion/notion.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 28bbde2700f8..ee8a198071a3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -8433,12 +8433,12 @@ Today (10): `activities`, `categories`, `client_folders`, `fields`, `leads`, `piDiffed against: <https://www.nocrm.io/api>-- [ ] `leads/{lead_id}/action_histories` — per-lead action and step-change history - the state-transition table needed for pipeline velocity and conversion timing (high)-- [ ] `follow_ups` — scheduled follow-up tasks/reminders on leads, the core activity object driving rep workload analysis (high)-- [ ] `leads/{lead_id}/comments` — comment thread per lead, the narrative activity stream alongside activities (medium)+- [x] `leads/{lead_id}/action_histories` — per-lead action and step-change history - the state-transition table needed for pipeline velocity and conversion timing (high). Added as `lead_action_histories` (fan-out over `leads`).+- [x] `follow_ups` — scheduled follow-up tasks/reminders on leads, the core activity object driving rep workload analysis (high). Added as `follow_ups` (fan-out over `users`, since the endpoint requires a `lead_id` or `user_id`).+- [x] `leads/{lead_id}/comments` — comment thread per lead, the narrative activity stream alongside activities (medium). Added as `lead_comments` (fan-out over `leads`).- [ ] `spreadsheets` — prospecting lists - the top-of-funnel container that leads are created from, with no equivalent table today (medium)- [ ] `spreadsheets/{spreadsheet_id}/rows` — prospect rows inside prospecting lists, plus their conversion-to-lead linkage (medium)-- [ ] `follow_ups/{follow_up_id}/tasks` — individual tasks within a follow-up sequence, the completion-level detail (medium)+- [x] `follow_ups/{follow_up_id}/tasks` — individual tasks within a follow-up sequence, the completion-level detail (medium). Skipped: the API has no list endpoint for tasks, only `PUT /follow_ups/{follow_up_id}/tasks/{task_id}`; `follow_ups` rows carry `tasks_count` and `tasks_done_count`.- [ ] `spreadsheets/{spreadsheet_id}/comments` — comments on prospecting lists and rows, engagement history before a lead exists (low)- [ ] `leads/{lead_id}/attachments` — attachment metadata per lead (counts/types, not file bytes) (low)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/canonical_descriptions.pyindex 8b8877d203e2..8211ac4d8e1f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/canonical_descriptions.py@@ -121,4 +121,54 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"type": "Data type of the custom field.",},},+ "lead_action_histories": {+ "description": "History of actions on each lead, such as step and status changes, amount updates, assignments, comments and emails.",+ "docs_url": "https://www.nocrm.io/api",+ "columns": {+ "id": "Unique identifier for the action history entry.",+ "lead_id": "Identifier of the lead the action was made on.",+ "action_type": "Type of action: client_assigned, step_changed, status_changed, amount_set, user_assigned, lead_created, comment_added or email_sent.",+ "action_item": "Details of the action, such as the new step, status or amount.",+ "created_at": "Timestamp the action was made.",+ "user": "User who made the action.",+ },+ },+ "lead_comments": {+ "description": "Comments posted on each lead, including logged activities and emails.",+ "docs_url": "https://www.nocrm.io/api",+ "columns": {+ "id": "Unique identifier for the comment.",+ "lead_id": "Identifier of the lead the comment is on.",+ "content": "Content of the comment.",+ "raw_content": "Raw content of the comment.",+ "commented_item": "Item the comment is attached to.",+ "activity_id": "Identifier of the activity set on the comment.",+ "activity": "Activity set on the comment.",+ "is_pinned": "Whether the comment is pinned on the lead.",+ "attachments": "Attachments on the comment.",+ "extended_info": "Extra details for email comments, such as sender, recipients and subject.",+ "action_item": "Item linked to the comment, such as an email.",+ "reactions": "Reactions on the comment.",+ "created_at": "Timestamp the comment was created.",+ "user": "User who wrote the comment.",+ },+ },+ "follow_ups": {+ "description": "Post-sales processes (follow-ups) on leads, each a list of tasks assigned to a user.",+ "docs_url": "https://www.nocrm.io/api",+ "columns": {+ "id": "Unique identifier for the post-sales process.",+ "title": "Title of the post-sales process.",+ "description": "Description of the post-sales process.",+ "status": "Status of the post-sales process: todo, standby or done.",+ "tags": "List of tags applied to the post-sales process.",+ "tasks_count": "Number of tasks in the post-sales process.",+ "tasks_done_count": "Number of tasks marked as done.",+ "next_action_at": "Timestamp of the next scheduled task.",+ "lead_id": "Identifier of the lead the post-sales process belongs to.",+ "user_id": "Identifier of the user the post-sales process is assigned to.",+ "created_at": "Timestamp the post-sales process was created.",+ "updated_at": "Timestamp the post-sales process was last updated.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/nocrm.py b/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/nocrm.pyindex bef635d76597..bea8dffb88cf 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/nocrm.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/nocrm.py@@ -1,7 +1,7 @@import reimport dataclassesfrom datetime import UTC, date, datetime-from typing import Any, Optional+from typing import Any, Optional, castfrom urllib3.util.retry import Retry@@ -10,21 +10,24 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resRESTAPIConfig,rest_api_resource,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ build_dependent_resource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (OffsetPaginator,+ SinglePagePaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.nocrm.settings import (NOCRM_ENDPOINTS,+ PAGE_SIZE,NoCRMEndpointConfig,)-# noCRM caps `/leads` at a default of 100 per page; request the max to minimise round-trips against-# the low (~2000 req/day) account quota.-PAGE_SIZE = 100-# noCRM reports the grand total (used to stop before an extra empty request) in this response header.TOTAL_COUNT_HEADER = "X-TOTAL-COUNT"@@ -124,11 +127,13 @@ def _build_base_params(return params[email protected](frozen=True)class NoCRMResumeConfig:# Offset (row count already consumed) to resume limit/offset pagination from. The incremental# window is recomputed from the job's stable last-value, so only the offset needs persisting.offset: int = 0+ # `build_dependent_resource` checkpoint for the per-lead / per-user fan-out walks.+ fanout_state: Optional[dict[str, Any]] = Noneclass NoCRMOffsetPaginator(OffsetPaginator):@@ -155,6 +160,78 @@ class NoCRMOffsetPaginator(OffsetPaginator):super().update_state(response, data)+def _client_config(api_key: str, subdomain: str) -> ClientConfig:+ return {+ "base_url": _base_url(subdomain),+ # Non-secret header only; the API key rides on the framework auth so it's redacted from+ # logs, samples, and raised error messages.+ "headers": {"Accept": "application/json"},+ "auth": {"type": "api_key", "api_key": api_key, "name": "X-API-KEY", "location": "header"},+ # Pin every request (including seeded resume pages) to the account's own *.nocrm.io host,+ # and refuse to follow redirects, so the key can't be steered to another origin.+ "allowed_hosts": [],+ "allow_redirects": False,+ }+++def _paginator(config: NoCRMEndpointConfig) -> OffsetPaginator | SinglePagePaginator:+ # List endpoints return a bare JSON array; no data_selector needed. noCRM reports the grand+ # total in a header, so stop on the header total / short / empty page.+ if not config.paginated:+ return SinglePagePaginator()+ return NoCRMOffsetPaginator(limit=PAGE_SIZE, total_path=None, total_header=TOTAL_COUNT_HEADER)+++def _fanout_source(+ api_key: str,+ subdomain: str,+ endpoint: str,+ team_id: int,+ job_id: str,+ resumable_source_manager: ResumableSourceManager[NoCRMResumeConfig],+) -> SourceResponse:+ config = NOCRM_ENDPOINTS[endpoint]+ assert config.fanout is not None+ parent_config = NOCRM_ENDPOINTS[config.fanout.parent_name]++ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None++ def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:+ if state:+ resumable_source_manager.save_state(NoCRMResumeConfig(fanout_state=state))++ child = cast(+ Resource,+ build_dependent_resource(+ endpoint_configs=NOCRM_ENDPOINTS,+ child_endpoint=endpoint,+ fanout=config.fanout,+ client_config=_client_config(api_key, subdomain),+ path_format_values={},+ team_id=team_id,+ job_id=job_id,+ db_incremental_field_last_value=None,+ parent_endpoint_extra={"paginator": _paginator(parent_config)},+ child_endpoint_extra={"paginator": _paginator(config)},+ # The offset paginator sends `limit` itself, and the unpaginated children take none.+ page_size_param=None,+ resume_hook=save_fanout_checkpoint,+ initial_paginator_state=resume.fanout_state if resume else None,+ ),+ )++ return SourceResponse(+ name=endpoint,+ items=lambda: child,+ primary_keys=config.primary_keys,+ partition_count=1,+ partition_size=1,+ partition_mode="datetime" if config.partition_key else None,+ partition_format="month" if config.partition_key else None,+ partition_keys=[config.partition_key] if config.partition_key else None,+ )++def nocrm_source(api_key: str,subdomain: str,@@ -166,31 +243,20 @@ def nocrm_source(db_incremental_field_last_value: Optional[Any] = None,) -> SourceResponse:config = NOCRM_ENDPOINTS[endpoint]+ if config.fanout is not None:+ return _fanout_source(api_key, subdomain, endpoint, team_id, job_id, resumable_source_manager)+base_params = _build_base_params(config, should_use_incremental_field, db_incremental_field_last_value)rest_config: RESTAPIConfig = {- "client": {- "base_url": _base_url(subdomain),- # Non-secret header only; the API key rides on the framework auth so it's redacted from- # logs, samples, and raised error messages.- "headers": {"Accept": "application/json"},- "auth": {"type": "api_key", "api_key": api_key, "name": "X-API-KEY", "location": "header"},- # Pin every request (including seeded resume pages) to the account's own *.nocrm.io host,- # and refuse to follow redirects, so the key can't be steered to another origin.- "allowed_hosts": [],- "allow_redirects": False,- },+ "client": _client_config(api_key, subdomain),"resources": [{"name": endpoint,"endpoint": {"path": config.path,"params": base_params,- # List endpoints return a bare JSON array; no data_selector needed. noCRM reports- # the grand total in a header, so stop on the header total / short / empty page.- "paginator": NoCRMOffsetPaginator(- limit=PAGE_SIZE, total_path=None, total_header=TOTAL_COUNT_HEADER- ),+ "paginator": _paginator(config),},}],diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/settings.pyindex dd7dc3e6bfc4..ee5142fbbd04 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/settings.py@@ -1,10 +1,17 @@from dataclasses import dataclass, fieldfrom typing import Optional+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ DependentEndpointConfig,+)from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType+# noCRM caps `/leads` at a default of 100 per page; request the max to minimise round-trips against+# the low (~2000 req/day) account quota.+PAGE_SIZE = 100-@dataclass++@dataclass(frozen=True)class NoCRMEndpointConfig:name: strpath: str@@ -27,6 +34,13 @@ class NoCRMEndpointConfig:partition_key: Optional[str] = Noneprimary_keys: list[str] = field(default_factory=lambda: ["id"])should_sync_default: bool = True+ # Set for endpoints noCRM only exposes per parent row (per lead / per user).+ fanout: Optional[DependentEndpointConfig] = None+ # False for child endpoints that return their whole collection in one response, with no+ # documented limit/offset params.+ paginated: bool = True+ default_incremental_field: Optional[str] = None+ page_size: int = PAGE_SIZEdef _updated_at_incremental_fields() -> list[IncrementalField]:@@ -42,6 +56,30 @@ def _updated_at_incremental_fields() -> list[IncrementalField]:]+# Lead sub-resources are only listable per lead, so these fan out over `/leads` (one request per lead).+LEAD_FANOUT = DependentEndpointConfig(+ parent_name="leads",+ resolve_param="lead_id",+ resolve_field="id",+ include_from_parent=["id"],+ parent_field_renames={"id": "lead_id"},+ # `/leads` hides unassigned leads unless asked, which would drop their histories and comments.+ parent_params={"order": "id", "direction": "asc", "include_unassigned": "true"},+ # A lead deleted between the parent listing and its child fetch 404s; skip it.+ child_response_actions=[{"status_code": 404, "action": "ignore"}],+)++# `/follow_ups` requires a `lead_id` or `user_id`. There are far fewer users than leads, so fan out+# over users. The default status set is undocumented, so ask for every status explicitly.+USER_FANOUT = DependentEndpointConfig(+ parent_name="users",+ resolve_param="user_id",+ resolve_field="id",+ include_from_parent=[],+ child_params={"status": "todo,standby,done"},+ child_response_actions=[{"status_code": 404, "action": "ignore"}],+)+NOCRM_ENDPOINTS: dict[str, NoCRMEndpointConfig] = {"leads": NoCRMEndpointConfig(name="leads",@@ -98,6 +136,35 @@ NOCRM_ENDPOINTS: dict[str, NoCRMEndpointConfig] = {path="/fields",incremental_fields=[],),+ # Per-lead fan-outs cost one request per lead against the daily quota, so they are opt-in.+ "lead_action_histories": NoCRMEndpointConfig(+ name="lead_action_histories",+ path="/leads/{lead_id}/action_histories",+ incremental_fields=[],+ partition_key="created_at",+ primary_keys=["lead_id", "id"],+ should_sync_default=False,+ fanout=LEAD_FANOUT,+ ),+ "lead_comments": NoCRMEndpointConfig(+ name="lead_comments",+ path="/leads/{lead_id}/comments",+ incremental_fields=[],+ partition_key="created_at",+ primary_keys=["lead_id", "id"],+ should_sync_default=False,+ fanout=LEAD_FANOUT,+ paginated=False,+ ),+ "follow_ups": NoCRMEndpointConfig(+ name="follow_ups",+ # The framework only binds resolve params in the path, so `user_id` rides in the query string here.+ path="/follow_ups?user_id={user_id}",+ incremental_fields=[],+ partition_key="created_at",+ fanout=USER_FANOUT,+ paginated=False,+ ),}ENDPOINTS = tuple(NOCRM_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/tests/test_nocrm.py b/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/tests/test_nocrm.pyindex c688b9a4eec8..8df3fb6af24c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/tests/test_nocrm.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/nocrm/tests/test_nocrm.py@@ -6,6 +6,7 @@ from parameterized import parameterizedfrom requests import HTTPErrorfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (+ RecordedRequest,ScriptedResponse,SourceDriver,scripted_network,@@ -201,3 +202,67 @@ class TestSourceResponse:assert result.response.primary_keys == ["id"]assert result.response.partition_mode is Noneassert result.response.partition_keys is None+++_LEADS_QUERY = {"include_unassigned": ("true",)}+++class TestFanout:+ @parameterized.expand(+ [+ ("lead_action_histories", "/api/v2/leads", _LEADS_QUERY, "/api/v2/leads/{id}/action_histories", {}, True),+ ("lead_comments", "/api/v2/leads", _LEADS_QUERY, "/api/v2/leads/{id}/comments", {}, False),+ ("follow_ups", "/api/v2/users", {}, "/api/v2/follow_ups", {"status": ("todo,standby,done",)}, False),+ ]+ )+ def test_child_requests_per_parent(+ self,+ endpoint: str,+ parent_path: str,+ parent_query: dict[str, tuple[str, ...]],+ child_path: str,+ child_query: dict[str, tuple[str, ...]],+ child_paginated: bool,+ ) -> None:+ def respond(request: RecordedRequest) -> ScriptedResponse:+ if request.path == parent_path:+ return ScriptedResponse(json=_page(2, 1))+ if "/leads/2/" in request.path or request.query.get("user_id") == ("2",):+ # A parent deleted between the listing and the child fetch must not fail the sync.+ return ScriptedResponse(status=404, json={"error": "record_not_found"})+ return ScriptedResponse(json=[{"id": 10, "lead_id": 1, "created_at": "2026-01-01T00:00:00.000Z"}])++ result = _driver().run(endpoint, respond)++ assert result.raised is None+ assert [r["id"] for r in result.rows] == [10]+ assert result.rows[0]["lead_id"] == 1+ parent_requests = [r for r in result.requests if r.path == parent_path]+ for key, value in parent_query.items():+ assert parent_requests[0].query[key] == value+ child_requests = [r for r in result.requests if r.path != parent_path]+ assert len(child_requests) == 2+ for request in child_requests:+ for key, value in child_query.items():+ assert request.query[key] == value+ # Unpaginated endpoints document no limit/offset, so they must not be sent.+ assert ("limit" in request.query) is child_paginated+ if endpoint == "follow_ups":+ assert [r.query["user_id"] for r in child_requests] == [("1",), ("2",)]+ else:+ assert [r.path for r in child_requests] == [child_path.format(id=1), child_path.format(id=2)]++ def test_action_histories_walk_every_page_of_a_lead(self) -> None:+ def respond(request: RecordedRequest) -> ScriptedResponse:+ if request.path == "/api/v2/leads":+ return ScriptedResponse(json=_page(1, 7))+ offset = int(request.query.get("offset", ("0",))[0])+ return ScriptedResponse(json=_page(PAGE_SIZE if offset == 0 else 3, offset + 1))++ result = _driver().run("lead_action_histories", respond)++ assert result.raised is None+ assert len(result.rows) == PAGE_SIZE + 3+ assert {r["lead_id"] for r in result.rows} == {7}+ assert result.response is not None+ assert result.response.primary_keys == ["lead_id", "id"]