:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): add nuntly threads, contacts and event tables (#123162)
Tom Owersea5041829b5d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 8fafd642ab72..c352ab283d2a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -8525,10 +8525,10 @@ Today (5): `Domains`, `Emails`, `Inboxes`, `Messages`, `Webhooks`Diffed against: <https://nuntly.com/docs/llms-full.txt>-- [ ] `Retrieve Email Events (GET /emails/{id}/events)` — measure delivery, open, click, bounce, and complaint events over time for each synced email (high)-- [ ] `List Inbox Threads (GET /inboxes/{inboxId}/threads)` — resolve threadId on synced messages and analyze conversation activity, unread states, and assigned agents (high)-- [ ] `List Namespaces (GET /namespaces)` — resolve namespaceId on synced inboxes to external customer identifiers for tenant-level reporting (high)-- [ ] `List Contacts (GET /contacts)` — build the audience dimension for recipient, subscription, and broadcast analysis (medium)+- [x] `Retrieve Email Events (GET /emails/{id}/events)` — measure delivery, open, click, bounce, and complaint events over time for each synced email (high)+- [x] `List Inbox Threads (GET /inboxes/{inboxId}/threads)` — resolve threadId on synced messages and analyze conversation activity, unread states, and assigned agents (high)+- [x] `List Namespaces (GET /namespaces)` — resolve namespaceId on synced inboxes to external customer identifiers for tenant-level reporting (high)+- [x] `List Contacts (GET /contacts)` — build the audience dimension for recipient, subscription, and broadcast analysis (medium)- [ ] `List Broadcasts (GET /broadcasts)` — analyze broadcast lifecycle, audience targeting, scheduling, and failures beyond transactional email delivery (medium)- [ ] `List Broadcast Recipients (GET /broadcasts/{broadcastId}/recipients)` — measure recipient-level broadcast delivery outcomes and connect campaigns to contacts and sent emails (medium)- [ ] `List Segments and Segment Members (GET /segments; GET /segments/{segmentId}/members)` — analyze audience membership and compare campaign outcomes across customer segments (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/canonical_descriptions.pyindex e9f9f3a6cb34..43d4e608a003 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/canonical_descriptions.py@@ -80,4 +80,55 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"createdAt": "When the webhook was created, in ISO 8601 format.",},},+ "Namespaces": {+ "description": "Namespaces that group Nuntly inboxes, for example one per customer, team, or AI agent.",+ "docs_url": "https://nuntly.com/docs/api-reference/namespaces/list-namespaces",+ "columns": {+ "id": "Unique identifier for the namespace.",+ "name": "Display name of the namespace.",+ "externalId": "Your own identifier for the namespace, such as a tenant or account ID, if set.",+ "createdAt": "When the namespace was created, in ISO 8601 format.",+ "updatedAt": "When the namespace was last updated, in ISO 8601 format.",+ },+ },+ "Contacts": {+ "description": "Contacts in the organization, the audience for Nuntly broadcasts. The list does not include contact property values.",+ "docs_url": "https://nuntly.com/docs/api-reference/contacts/list-contacts",+ "columns": {+ "id": "Unique identifier for the contact.",+ "email": "Email address of the contact. It is unique in the organization and cannot change.",+ "createdAt": "When the contact was created, in ISO 8601 format.",+ "updatedAt": "When the contact was last updated, in ISO 8601 format.",+ },+ },+ "Threads": {+ "description": "Conversation threads in each inbox, grouping related received and sent messages. Spam and trash threads are not included.",+ "docs_url": "https://nuntly.com/docs/api-reference/threads/list-inbox-threads",+ "columns": {+ "id": "Unique identifier for the thread.",+ "inboxId": "Identifier of the inbox the thread belongs to.",+ "domainId": "Identifier of the domain of the inbox, if any.",+ "domainName": "The domain name of the inbox.",+ "subject": "The original subject line of the thread.",+ "lastMessageAt": "When the most recent message in the thread arrived, in ISO 8601 format.",+ "messageCount": "Number of messages in the thread.",+ "labels": "Labels from all messages in the thread, for example unread.",+ "agentId": "Identifier of the AI agent assigned to the thread, if any.",+ "createdAt": "When the thread was created, in ISO 8601 format.",+ "updatedAt": "When the thread was last updated, in ISO 8601 format.",+ },+ },+ "EmailEvents": {+ "description": "Event history of each sent email, one row per event such as queued, sent, delivered, opened, clicked, or bounced.",+ "docs_url": "https://nuntly.com/docs/api-reference/emails/list-email-events",+ "columns": {+ "id": "Unique identifier for the event.",+ "orgId": "Identifier of the organization that owns the email.",+ "emailId": "Identifier of the email the event belongs to.",+ "eventType": "Type of the event, for example email.delivered, email.opened, email.clicked, or email.bounced.",+ "payload": "Event-specific details.",+ "occurredAt": "When the event happened, in ISO 8601 format.",+ "createdAt": "When the event was recorded, in ISO 8601 format.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/nuntly.py b/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/nuntly.pyindex 4d31e1c18853..11b8837284de 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/nuntly.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/nuntly.py@@ -1,3 +1,4 @@+from collections.abc import Iterablefrom typing import Any, Optionalfrom posthog.dataclasses import frozen@@ -7,10 +8,16 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resRESTAPIConfig,rest_api_resource,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ build_dependent_resource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (JSONResponseCursorPaginator,)-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (+ ClientConfig,+ Endpoint,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse@@ -21,7 +28,10 @@ NUNTLY_BASE_URL = "https://api.nuntly.com"@frozenclass NuntlyResumeConfig:- cursor: str+ cursor: Optional[str] = None+ # Fan-out tables checkpoint the shape `build_dependent_resource` emits: which parents finished+ # and where the current one stopped.+ fanout_state: Optional[dict[str, Any]] = Nonedef _client_config(api_key: str) -> ClientConfig:@@ -41,45 +51,71 @@ def nuntly_source(resumable_source_manager: ResumableSourceManager[NuntlyResumeConfig],) -> SourceResponse:endpoint_config = ENDPOINTS[endpoint]-- rest_config: RESTAPIConfig = {- "client": _client_config(api_key),- "resource_defaults": {"write_disposition": "replace"},- "resources": [- {- "name": endpoint,- "table_name": endpoint_config.table_name,- "endpoint": {- "path": endpoint_config.path,- "params": {"limit": MAX_PAGE_SIZE},- "data_selector": "data",- },- "table_format": "delta",- }- ],- }-- initial_paginator_state: Optional[dict[str, Any]] = None- if resumable_source_manager.can_resume():- resume = resumable_source_manager.load_state()- if resume is not None:- initial_paginator_state = {"cursor": resume.cursor}-- def save_checkpoint(state: Optional[dict[str, Any]]) -> None:- # Only persist when there's a next page to resume to; the Redis TTL cleans up on- # completion. Save AFTER a page is yielded so a crash re-yields the last page (merge- # dedupes on primary key) rather than skipping it.- if state and state.get("cursor"):- resumable_source_manager.save_state(NuntlyResumeConfig(cursor=str(state["cursor"])))-- resource = rest_api_resource(- rest_config,- team_id,- job_id,- None,- resume_hook=save_checkpoint,- initial_paginator_state=initial_paginator_state,- )+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None++ resource: Iterable[Any]+ column_hints = None+ if endpoint_config.fanout is not None:++ def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:+ if state:+ resumable_source_manager.save_state(NuntlyResumeConfig(fanout_state=state))++ child_endpoint_extra: Endpoint = {"data_selector": "data"}+ if not endpoint_config.paginated:+ child_endpoint_extra["paginator"] = "single_page"++ resource = build_dependent_resource(+ endpoint_configs=ENDPOINTS,+ child_endpoint=endpoint,+ fanout=endpoint_config.fanout,+ client_config=_client_config(api_key),+ path_format_values={},+ team_id=team_id,+ job_id=job_id,+ db_incremental_field_last_value=None,+ # Page sizes ride in the fan-out params: the unpaginated child takes no `limit`.+ page_size_param=None,+ parent_endpoint_extra={"data_selector": "data"},+ child_endpoint_extra=child_endpoint_extra,+ resume_hook=save_fanout_checkpoint if endpoint_config.checkpoint_fanout else None,+ initial_paginator_state=resume.fanout_state if resume and endpoint_config.checkpoint_fanout else None,+ )+ else:+ rest_config: RESTAPIConfig = {+ "client": _client_config(api_key),+ "resource_defaults": {"write_disposition": "replace"},+ "resources": [+ {+ "name": endpoint,+ "table_name": endpoint_config.table_name,+ "endpoint": {+ "path": endpoint_config.path,+ "params": {"limit": MAX_PAGE_SIZE},+ "data_selector": "data",+ },+ "table_format": "delta",+ }+ ],+ }++ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:+ # Only persist when there's a next page to resume to; the Redis TTL cleans up on+ # completion. Save AFTER a page is yielded so a crash re-yields the last page (merge+ # dedupes on primary key) rather than skipping it.+ if state and state.get("cursor"):+ resumable_source_manager.save_state(NuntlyResumeConfig(cursor=str(state["cursor"])))++ top_level_resource = rest_api_resource(+ rest_config,+ team_id,+ job_id,+ None,+ resume_hook=save_checkpoint,+ initial_paginator_state={"cursor": resume.cursor} if resume and resume.cursor else None,+ )+ resource = top_level_resource+ column_hints = top_level_resource.column_hintsreturn SourceResponse(name=endpoint,@@ -90,7 +126,8 @@ def nuntly_source(partition_mode="datetime",partition_format="month",partition_keys=[endpoint_config.partition_key],- column_hints=resource.column_hints,+ column_hints=column_hints,+ supports_resume=endpoint_config.fanout is None or endpoint_config.checkpoint_fanout,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/settings.pyindex e2dad93083a5..7967e6722ab4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/settings.py@@ -1,26 +1,84 @@+from dataclasses import field+from posthog.dataclasses import frozen+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ DependentEndpointConfig,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ResponseActionfrom products.warehouse_sources.backend.types import IncrementalField# Nuntly caps every list endpoint at 30 rows per page (no larger tier is documented).MAX_PAGE_SIZE = 30+# A parent deleted (or aged out by data retention) between the parent listing and its child fetch 404s.+_SKIP_MISSING_PARENT: list[ResponseAction] = [{"status_code": 404, "action": "ignore"}]+@frozenclass NuntlyEndpointConfig:+ name: strpath: strtable_name: strprimary_keys: list[str]# `createdAt` is stable across every endpoint below; none expose an `updated_at` equivalent.partition_key: str = "createdAt"+ page_size: int = MAX_PAGE_SIZE+ # Unpaginated endpoints return their whole collection in one `{"data": [...]}` body.+ paginated: bool = True+ incremental_fields: list[IncrementalField] = field(default_factory=list)+ default_incremental_field: str | None = None+ # Set for resources only listed per parent; the parent is re-listed each sync.+ fanout: DependentEndpointConfig | None = None+ # The fan-out checkpoint lists every finished parent and is rewritten after each one, so its+ # cost grows quadratically with the parent count. Turn it off where parents are unbounded.+ checkpoint_fanout: bool = TrueENDPOINTS: dict[str, NuntlyEndpointConfig] = {- "Emails": NuntlyEndpointConfig(path="/emails", table_name="emails", primary_keys=["id"]),- "Messages": NuntlyEndpointConfig(path="/messages", table_name="messages", primary_keys=["id"]),- "Inboxes": NuntlyEndpointConfig(path="/inboxes", table_name="inboxes", primary_keys=["id"]),- "Domains": NuntlyEndpointConfig(path="/domains", table_name="domains", primary_keys=["id"]),- "Webhooks": NuntlyEndpointConfig(path="/webhooks", table_name="webhooks", primary_keys=["id"]),+ "Emails": NuntlyEndpointConfig(name="Emails", path="/emails", table_name="emails", primary_keys=["id"]),+ "Messages": NuntlyEndpointConfig(name="Messages", path="/messages", table_name="messages", primary_keys=["id"]),+ "Inboxes": NuntlyEndpointConfig(name="Inboxes", path="/inboxes", table_name="inboxes", primary_keys=["id"]),+ "Domains": NuntlyEndpointConfig(name="Domains", path="/domains", table_name="domains", primary_keys=["id"]),+ "Webhooks": NuntlyEndpointConfig(name="Webhooks", path="/webhooks", table_name="webhooks", primary_keys=["id"]),+ "Namespaces": NuntlyEndpointConfig(+ name="Namespaces", path="/namespaces", table_name="namespaces", primary_keys=["id"]+ ),+ "Contacts": NuntlyEndpointConfig(name="Contacts", path="/contacts", table_name="contacts", primary_keys=["id"]),+ "Threads": NuntlyEndpointConfig(+ name="Threads",+ path="/inboxes/{inboxId}/threads",+ table_name="threads",+ primary_keys=["inboxId", "id"],+ fanout=DependentEndpointConfig(+ parent_name="Inboxes",+ resolve_param="inboxId",+ resolve_field="id",+ # Thread rows already carry `inboxId`; nothing to copy down.+ include_from_parent=[],+ parent_params={"limit": MAX_PAGE_SIZE},+ child_params={"limit": MAX_PAGE_SIZE},+ child_response_actions=_SKIP_MISSING_PARENT,+ ),+ ),+ "EmailEvents": NuntlyEndpointConfig(+ name="EmailEvents",+ path="/emails/{emailId}/events",+ table_name="email_events",+ primary_keys=["emailId", "id"],+ paginated=False,+ # One parent per sent email, so a resumed run starts over rather than carry that list.+ checkpoint_fanout=False,+ fanout=DependentEndpointConfig(+ parent_name="Emails",+ resolve_param="emailId",+ resolve_field="id",+ # Event rows already carry `emailId`; nothing to copy down.+ include_from_parent=[],+ parent_params={"limit": MAX_PAGE_SIZE},+ child_response_actions=_SKIP_MISSING_PARENT,+ ),+ ),}# Nuntly documents no server-side timestamp filter on any list endpoint (no `since` /diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/tests/test_nuntly.py b/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/tests/test_nuntly.pyindex 3567b283aa81..03f7ece01784 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/tests/test_nuntly.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/tests/test_nuntly.py@@ -28,7 +28,9 @@ class TestNuntlySourceResumeBehavior:def _page(rows: list[dict[str, Any]], next_cursor: str | None) -> ScriptedResponse:return ScriptedResponse(json={"data": rows, "nextCursor": next_cursor})- @parameterized.expand([("Emails",), ("Messages",), ("Inboxes",), ("Domains",), ("Webhooks",)])+ @parameterized.expand(+ [("Emails",), ("Messages",), ("Inboxes",), ("Domains",), ("Webhooks",), ("Namespaces",), ("Contacts",)]+ )def test_fresh_run_walks_pages_and_saves_cursor_after_each_non_terminal_page(self, endpoint: str) -> None:result = self._driver().run(endpoint,@@ -76,6 +78,90 @@ class TestNuntlySourceResumeBehavior:assert result.response.partition_keys == ["createdAt"]+class TestNuntlyFanout:+ @staticmethod+ def _driver() -> SourceDriver:+ return SourceDriver(NuntlySource(), NuntlySourceConfig(api_key="apk_test"))++ @staticmethod+ def _page(rows: list[dict[str, Any]], next_cursor: str | None = None) -> ScriptedResponse:+ return ScriptedResponse(json={"data": rows, "nextCursor": next_cursor})++ def test_threads_walk_every_inbox_page_and_thread_page(self) -> None:+ result = self._driver().run(+ "Threads",+ [+ self._page([{"id": "ibx_1"}], "inbox-cursor"),+ self._page([{"id": "thr_1", "inboxId": "ibx_1"}], "thread-cursor"),+ self._page([{"id": "thr_2", "inboxId": "ibx_1"}]),+ self._page([{"id": "ibx_2"}]),+ self._page([{"id": "thr_3", "inboxId": "ibx_2"}]),+ ],+ )++ assert result.raised is None+ assert result.paths == [+ "/inboxes",+ "/inboxes/ibx_1/threads",+ "/inboxes/ibx_1/threads",+ "/inboxes",+ "/inboxes/ibx_2/threads",+ ]+ assert result.params("cursor") == [None, None, "thread-cursor", "inbox-cursor", None]+ assert result.params("limit") == ["30"] * 5+ assert [(row["inboxId"], row["id"]) for row in result.rows] == [+ ("ibx_1", "thr_1"),+ ("ibx_1", "thr_2"),+ ("ibx_2", "thr_3"),+ ]+ assert result.saved_states and all(+ state.fanout_state is not None and state.cursor is None for state in result.saved_states+ )+ assert result.response is not None and result.response.supports_resume is True++ def test_email_events_fetch_one_unpaginated_page_per_email_and_skip_missing_emails(self) -> None:+ result = self._driver().run(+ "EmailEvents",+ [+ self._page([{"id": "em_1"}, {"id": "em_gone"}, {"id": "em_2"}]),+ # Not paginated: a stray `nextCursor` must not trigger a second request.+ self._page([{"id": "evt_1", "emailId": "em_1"}], "ignored"),+ ScriptedResponse(status=404, json={"error": {"status": 404, "code": "not_found", "title": "x"}}),+ ScriptedResponse(json={"data": [{"id": "evt_2", "emailId": "em_2"}]}),+ ],+ )++ assert result.raised is None+ assert result.paths == ["/emails", "/emails/em_1/events", "/emails/em_gone/events", "/emails/em_2/events"]+ assert result.params("limit") == ["30", None, None, None]+ assert [(row["emailId"], row["id"]) for row in result.rows] == [("em_1", "evt_1"), ("em_2", "evt_2")]+ assert result.saved_states == []+ assert result.response is not None and result.response.supports_resume is False++ def test_fanout_resume_skips_inboxes_whose_threads_already_synced(self) -> None:+ first = self._driver().run(+ "Threads",+ [+ self._page([{"id": "ibx_1"}, {"id": "ibx_2"}]),+ self._page([{"id": "thr_1", "inboxId": "ibx_1"}]),+ self._page([{"id": "thr_2", "inboxId": "ibx_2"}]),+ ],+ )++ resumed = self._driver().run(+ "Threads",+ [+ self._page([{"id": "ibx_1"}, {"id": "ibx_2"}]),+ self._page([{"id": "thr_2", "inboxId": "ibx_2"}]),+ ],+ resume_state=first.saved_states[0],+ )++ assert resumed.raised is None+ assert resumed.paths == ["/inboxes", "/inboxes/ibx_2/threads"]+ assert [row["id"] for row in resumed.rows] == ["thr_2"]++class TestValidateCredentials:@parameterized.expand([@@ -88,10 +174,3 @@ class TestValidateCredentials:assert (is_valid, code) == (expected_valid, status_code)assert network.requests_log[0].path == "/emails"---class TestEndpointSettings:- @parameterized.expand([(name,) for name in ENDPOINTS])- def test_every_declared_endpoint_has_a_settings_entry(self, endpoint: str) -> None:- assert ENDPOINTS[endpoint].path.startswith("/")- assert ENDPOINTS[endpoint].primary_keys == ["id"]