Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add nuntly threads, contacts and event tables (#123162)

Tom Owersea5041829b5d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 8fafd642ab72..c352ab283d2a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -8525,10 +8525,10 @@ Today (5): `Domains`, `Emails`, `Inboxes`, `Messages`, `Webhooks`
Diffed against: <https://nuntly.com/docs/llms-full.txt>
-- [ ] `Retrieve Email Events (GET /emails/{id}/events)` — measure delivery, open, click, bounce, and complaint events over time for each synced email (high)
-- [ ] `List Inbox Threads (GET /inboxes/{inboxId}/threads)` — resolve threadId on synced messages and analyze conversation activity, unread states, and assigned agents (high)
-- [ ] `List Namespaces (GET /namespaces)` — resolve namespaceId on synced inboxes to external customer identifiers for tenant-level reporting (high)
-- [ ] `List Contacts (GET /contacts)` — build the audience dimension for recipient, subscription, and broadcast analysis (medium)
+- [x] `Retrieve Email Events (GET /emails/{id}/events)` — measure delivery, open, click, bounce, and complaint events over time for each synced email (high)
+- [x] `List Inbox Threads (GET /inboxes/{inboxId}/threads)` — resolve threadId on synced messages and analyze conversation activity, unread states, and assigned agents (high)
+- [x] `List Namespaces (GET /namespaces)` — resolve namespaceId on synced inboxes to external customer identifiers for tenant-level reporting (high)
+- [x] `List Contacts (GET /contacts)` — build the audience dimension for recipient, subscription, and broadcast analysis (medium)
- [ ] `List Broadcasts (GET /broadcasts)` — analyze broadcast lifecycle, audience targeting, scheduling, and failures beyond transactional email delivery (medium)
- [ ] `List Broadcast Recipients (GET /broadcasts/{broadcastId}/recipients)` — measure recipient-level broadcast delivery outcomes and connect campaigns to contacts and sent emails (medium)
- [ ] `List Segments and Segment Members (GET /segments; GET /segments/{segmentId}/members)` — analyze audience membership and compare campaign outcomes across customer segments (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/canonical_descriptions.py
index e9f9f3a6cb34..43d4e608a003 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/canonical_descriptions.py
@@ -80,4 +80,55 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"createdAt": "When the webhook was created, in ISO 8601 format.",
},
},
+ "Namespaces": {
+ "description": "Namespaces that group Nuntly inboxes, for example one per customer, team, or AI agent.",
+ "docs_url": "https://nuntly.com/docs/api-reference/namespaces/list-namespaces",
+ "columns": {
+ "id": "Unique identifier for the namespace.",
+ "name": "Display name of the namespace.",
+ "externalId": "Your own identifier for the namespace, such as a tenant or account ID, if set.",
+ "createdAt": "When the namespace was created, in ISO 8601 format.",
+ "updatedAt": "When the namespace was last updated, in ISO 8601 format.",
+ },
+ },
+ "Contacts": {
+ "description": "Contacts in the organization, the audience for Nuntly broadcasts. The list does not include contact property values.",
+ "docs_url": "https://nuntly.com/docs/api-reference/contacts/list-contacts",
+ "columns": {
+ "id": "Unique identifier for the contact.",
+ "email": "Email address of the contact. It is unique in the organization and cannot change.",
+ "createdAt": "When the contact was created, in ISO 8601 format.",
+ "updatedAt": "When the contact was last updated, in ISO 8601 format.",
+ },
+ },
+ "Threads": {
+ "description": "Conversation threads in each inbox, grouping related received and sent messages. Spam and trash threads are not included.",
+ "docs_url": "https://nuntly.com/docs/api-reference/threads/list-inbox-threads",
+ "columns": {
+ "id": "Unique identifier for the thread.",
+ "inboxId": "Identifier of the inbox the thread belongs to.",
+ "domainId": "Identifier of the domain of the inbox, if any.",
+ "domainName": "The domain name of the inbox.",
+ "subject": "The original subject line of the thread.",
+ "lastMessageAt": "When the most recent message in the thread arrived, in ISO 8601 format.",
+ "messageCount": "Number of messages in the thread.",
+ "labels": "Labels from all messages in the thread, for example unread.",
+ "agentId": "Identifier of the AI agent assigned to the thread, if any.",
+ "createdAt": "When the thread was created, in ISO 8601 format.",
+ "updatedAt": "When the thread was last updated, in ISO 8601 format.",
+ },
+ },
+ "EmailEvents": {
+ "description": "Event history of each sent email, one row per event such as queued, sent, delivered, opened, clicked, or bounced.",
+ "docs_url": "https://nuntly.com/docs/api-reference/emails/list-email-events",
+ "columns": {
+ "id": "Unique identifier for the event.",
+ "orgId": "Identifier of the organization that owns the email.",
+ "emailId": "Identifier of the email the event belongs to.",
+ "eventType": "Type of the event, for example email.delivered, email.opened, email.clicked, or email.bounced.",
+ "payload": "Event-specific details.",
+ "occurredAt": "When the event happened, in ISO 8601 format.",
+ "createdAt": "When the event was recorded, in ISO 8601 format.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/nuntly.py b/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/nuntly.py
index 4d31e1c18853..11b8837284de 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/nuntly.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/nuntly.py
@@ -1,3 +1,4 @@
+from collections.abc import Iterable
from typing import Any, Optional
from posthog.dataclasses import frozen
@@ -7,10 +8,16 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
JSONResponseCursorPaginator,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
+ ClientConfig,
+ Endpoint,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -21,7 +28,10 @@ NUNTLY_BASE_URL = "https://api.nuntly.com"
@frozen
class NuntlyResumeConfig:
- cursor: str
+ cursor: Optional[str] = None
+ # Fan-out tables checkpoint the shape `build_dependent_resource` emits: which parents finished
+ # and where the current one stopped.
+ fanout_state: Optional[dict[str, Any]] = None
def _client_config(api_key: str) -> ClientConfig:
@@ -41,45 +51,71 @@ def nuntly_source(
resumable_source_manager: ResumableSourceManager[NuntlyResumeConfig],
) -> SourceResponse:
endpoint_config = ENDPOINTS[endpoint]
-
- rest_config: RESTAPIConfig = {
- "client": _client_config(api_key),
- "resource_defaults": {"write_disposition": "replace"},
- "resources": [
- {
- "name": endpoint,
- "table_name": endpoint_config.table_name,
- "endpoint": {
- "path": endpoint_config.path,
- "params": {"limit": MAX_PAGE_SIZE},
- "data_selector": "data",
- },
- "table_format": "delta",
- }
- ],
- }
-
- initial_paginator_state: Optional[dict[str, Any]] = None
- if resumable_source_manager.can_resume():
- resume = resumable_source_manager.load_state()
- if resume is not None:
- initial_paginator_state = {"cursor": resume.cursor}
-
- def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
- # Only persist when there's a next page to resume to; the Redis TTL cleans up on
- # completion. Save AFTER a page is yielded so a crash re-yields the last page (merge
- # dedupes on primary key) rather than skipping it.
- if state and state.get("cursor"):
- resumable_source_manager.save_state(NuntlyResumeConfig(cursor=str(state["cursor"])))
-
- resource = rest_api_resource(
- rest_config,
- team_id,
- job_id,
- None,
- resume_hook=save_checkpoint,
- initial_paginator_state=initial_paginator_state,
- )
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+
+ resource: Iterable[Any]
+ column_hints = None
+ if endpoint_config.fanout is not None:
+
+ def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if state:
+ resumable_source_manager.save_state(NuntlyResumeConfig(fanout_state=state))
+
+ child_endpoint_extra: Endpoint = {"data_selector": "data"}
+ if not endpoint_config.paginated:
+ child_endpoint_extra["paginator"] = "single_page"
+
+ resource = build_dependent_resource(
+ endpoint_configs=ENDPOINTS,
+ child_endpoint=endpoint,
+ fanout=endpoint_config.fanout,
+ client_config=_client_config(api_key),
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ # Page sizes ride in the fan-out params: the unpaginated child takes no `limit`.
+ page_size_param=None,
+ parent_endpoint_extra={"data_selector": "data"},
+ child_endpoint_extra=child_endpoint_extra,
+ resume_hook=save_fanout_checkpoint if endpoint_config.checkpoint_fanout else None,
+ initial_paginator_state=resume.fanout_state if resume and endpoint_config.checkpoint_fanout else None,
+ )
+ else:
+ rest_config: RESTAPIConfig = {
+ "client": _client_config(api_key),
+ "resource_defaults": {"write_disposition": "replace"},
+ "resources": [
+ {
+ "name": endpoint,
+ "table_name": endpoint_config.table_name,
+ "endpoint": {
+ "path": endpoint_config.path,
+ "params": {"limit": MAX_PAGE_SIZE},
+ "data_selector": "data",
+ },
+ "table_format": "delta",
+ }
+ ],
+ }
+
+ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ # Only persist when there's a next page to resume to; the Redis TTL cleans up on
+ # completion. Save AFTER a page is yielded so a crash re-yields the last page (merge
+ # dedupes on primary key) rather than skipping it.
+ if state and state.get("cursor"):
+ resumable_source_manager.save_state(NuntlyResumeConfig(cursor=str(state["cursor"])))
+
+ top_level_resource = rest_api_resource(
+ rest_config,
+ team_id,
+ job_id,
+ None,
+ resume_hook=save_checkpoint,
+ initial_paginator_state={"cursor": resume.cursor} if resume and resume.cursor else None,
+ )
+ resource = top_level_resource
+ column_hints = top_level_resource.column_hints
return SourceResponse(
name=endpoint,
@@ -90,7 +126,8 @@ def nuntly_source(
partition_mode="datetime",
partition_format="month",
partition_keys=[endpoint_config.partition_key],
- column_hints=resource.column_hints,
+ column_hints=column_hints,
+ supports_resume=endpoint_config.fanout is None or endpoint_config.checkpoint_fanout,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/settings.py
index e2dad93083a5..7967e6722ab4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/settings.py
@@ -1,26 +1,84 @@
+from dataclasses import field
+
from posthog.dataclasses import frozen
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ResponseAction
from products.warehouse_sources.backend.types import IncrementalField
# Nuntly caps every list endpoint at 30 rows per page (no larger tier is documented).
MAX_PAGE_SIZE = 30
+# A parent deleted (or aged out by data retention) between the parent listing and its child fetch 404s.
+_SKIP_MISSING_PARENT: list[ResponseAction] = [{"status_code": 404, "action": "ignore"}]
+
@frozen
class NuntlyEndpointConfig:
+ name: str
path: str
table_name: str
primary_keys: list[str]
# `createdAt` is stable across every endpoint below; none expose an `updated_at` equivalent.
partition_key: str = "createdAt"
+ page_size: int = MAX_PAGE_SIZE
+ # Unpaginated endpoints return their whole collection in one `{"data": [...]}` body.
+ paginated: bool = True
+ incremental_fields: list[IncrementalField] = field(default_factory=list)
+ default_incremental_field: str | None = None
+ # Set for resources only listed per parent; the parent is re-listed each sync.
+ fanout: DependentEndpointConfig | None = None
+ # The fan-out checkpoint lists every finished parent and is rewritten after each one, so its
+ # cost grows quadratically with the parent count. Turn it off where parents are unbounded.
+ checkpoint_fanout: bool = True
ENDPOINTS: dict[str, NuntlyEndpointConfig] = {
- "Emails": NuntlyEndpointConfig(path="/emails", table_name="emails", primary_keys=["id"]),
- "Messages": NuntlyEndpointConfig(path="/messages", table_name="messages", primary_keys=["id"]),
- "Inboxes": NuntlyEndpointConfig(path="/inboxes", table_name="inboxes", primary_keys=["id"]),
- "Domains": NuntlyEndpointConfig(path="/domains", table_name="domains", primary_keys=["id"]),
- "Webhooks": NuntlyEndpointConfig(path="/webhooks", table_name="webhooks", primary_keys=["id"]),
+ "Emails": NuntlyEndpointConfig(name="Emails", path="/emails", table_name="emails", primary_keys=["id"]),
+ "Messages": NuntlyEndpointConfig(name="Messages", path="/messages", table_name="messages", primary_keys=["id"]),
+ "Inboxes": NuntlyEndpointConfig(name="Inboxes", path="/inboxes", table_name="inboxes", primary_keys=["id"]),
+ "Domains": NuntlyEndpointConfig(name="Domains", path="/domains", table_name="domains", primary_keys=["id"]),
+ "Webhooks": NuntlyEndpointConfig(name="Webhooks", path="/webhooks", table_name="webhooks", primary_keys=["id"]),
+ "Namespaces": NuntlyEndpointConfig(
+ name="Namespaces", path="/namespaces", table_name="namespaces", primary_keys=["id"]
+ ),
+ "Contacts": NuntlyEndpointConfig(name="Contacts", path="/contacts", table_name="contacts", primary_keys=["id"]),
+ "Threads": NuntlyEndpointConfig(
+ name="Threads",
+ path="/inboxes/{inboxId}/threads",
+ table_name="threads",
+ primary_keys=["inboxId", "id"],
+ fanout=DependentEndpointConfig(
+ parent_name="Inboxes",
+ resolve_param="inboxId",
+ resolve_field="id",
+ # Thread rows already carry `inboxId`; nothing to copy down.
+ include_from_parent=[],
+ parent_params={"limit": MAX_PAGE_SIZE},
+ child_params={"limit": MAX_PAGE_SIZE},
+ child_response_actions=_SKIP_MISSING_PARENT,
+ ),
+ ),
+ "EmailEvents": NuntlyEndpointConfig(
+ name="EmailEvents",
+ path="/emails/{emailId}/events",
+ table_name="email_events",
+ primary_keys=["emailId", "id"],
+ paginated=False,
+ # One parent per sent email, so a resumed run starts over rather than carry that list.
+ checkpoint_fanout=False,
+ fanout=DependentEndpointConfig(
+ parent_name="Emails",
+ resolve_param="emailId",
+ resolve_field="id",
+ # Event rows already carry `emailId`; nothing to copy down.
+ include_from_parent=[],
+ parent_params={"limit": MAX_PAGE_SIZE},
+ child_response_actions=_SKIP_MISSING_PARENT,
+ ),
+ ),
}
# Nuntly documents no server-side timestamp filter on any list endpoint (no `since` /
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/tests/test_nuntly.py b/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/tests/test_nuntly.py
index 3567b283aa81..03f7ece01784 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/tests/test_nuntly.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/nuntly/tests/test_nuntly.py
@@ -28,7 +28,9 @@ class TestNuntlySourceResumeBehavior:
def _page(rows: list[dict[str, Any]], next_cursor: str | None) -> ScriptedResponse:
return ScriptedResponse(json={"data": rows, "nextCursor": next_cursor})
- @parameterized.expand([("Emails",), ("Messages",), ("Inboxes",), ("Domains",), ("Webhooks",)])
+ @parameterized.expand(
+ [("Emails",), ("Messages",), ("Inboxes",), ("Domains",), ("Webhooks",), ("Namespaces",), ("Contacts",)]
+ )
def test_fresh_run_walks_pages_and_saves_cursor_after_each_non_terminal_page(self, endpoint: str) -> None:
result = self._driver().run(
endpoint,
@@ -76,6 +78,90 @@ class TestNuntlySourceResumeBehavior:
assert result.response.partition_keys == ["createdAt"]
+class TestNuntlyFanout:
+ @staticmethod
+ def _driver() -> SourceDriver:
+ return SourceDriver(NuntlySource(), NuntlySourceConfig(api_key="apk_test"))
+
+ @staticmethod
+ def _page(rows: list[dict[str, Any]], next_cursor: str | None = None) -> ScriptedResponse:
+ return ScriptedResponse(json={"data": rows, "nextCursor": next_cursor})
+
+ def test_threads_walk_every_inbox_page_and_thread_page(self) -> None:
+ result = self._driver().run(
+ "Threads",
+ [
+ self._page([{"id": "ibx_1"}], "inbox-cursor"),
+ self._page([{"id": "thr_1", "inboxId": "ibx_1"}], "thread-cursor"),
+ self._page([{"id": "thr_2", "inboxId": "ibx_1"}]),
+ self._page([{"id": "ibx_2"}]),
+ self._page([{"id": "thr_3", "inboxId": "ibx_2"}]),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.paths == [
+ "/inboxes",
+ "/inboxes/ibx_1/threads",
+ "/inboxes/ibx_1/threads",
+ "/inboxes",
+ "/inboxes/ibx_2/threads",
+ ]
+ assert result.params("cursor") == [None, None, "thread-cursor", "inbox-cursor", None]
+ assert result.params("limit") == ["30"] * 5
+ assert [(row["inboxId"], row["id"]) for row in result.rows] == [
+ ("ibx_1", "thr_1"),
+ ("ibx_1", "thr_2"),
+ ("ibx_2", "thr_3"),
+ ]
+ assert result.saved_states and all(
+ state.fanout_state is not None and state.cursor is None for state in result.saved_states
+ )
+ assert result.response is not None and result.response.supports_resume is True
+
+ def test_email_events_fetch_one_unpaginated_page_per_email_and_skip_missing_emails(self) -> None:
+ result = self._driver().run(
+ "EmailEvents",
+ [
+ self._page([{"id": "em_1"}, {"id": "em_gone"}, {"id": "em_2"}]),
+ # Not paginated: a stray `nextCursor` must not trigger a second request.
+ self._page([{"id": "evt_1", "emailId": "em_1"}], "ignored"),
+ ScriptedResponse(status=404, json={"error": {"status": 404, "code": "not_found", "title": "x"}}),
+ ScriptedResponse(json={"data": [{"id": "evt_2", "emailId": "em_2"}]}),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/emails", "/emails/em_1/events", "/emails/em_gone/events", "/emails/em_2/events"]
+ assert result.params("limit") == ["30", None, None, None]
+ assert [(row["emailId"], row["id"]) for row in result.rows] == [("em_1", "evt_1"), ("em_2", "evt_2")]
+ assert result.saved_states == []
+ assert result.response is not None and result.response.supports_resume is False
+
+ def test_fanout_resume_skips_inboxes_whose_threads_already_synced(self) -> None:
+ first = self._driver().run(
+ "Threads",
+ [
+ self._page([{"id": "ibx_1"}, {"id": "ibx_2"}]),
+ self._page([{"id": "thr_1", "inboxId": "ibx_1"}]),
+ self._page([{"id": "thr_2", "inboxId": "ibx_2"}]),
+ ],
+ )
+
+ resumed = self._driver().run(
+ "Threads",
+ [
+ self._page([{"id": "ibx_1"}, {"id": "ibx_2"}]),
+ self._page([{"id": "thr_2", "inboxId": "ibx_2"}]),
+ ],
+ resume_state=first.saved_states[0],
+ )
+
+ assert resumed.raised is None
+ assert resumed.paths == ["/inboxes", "/inboxes/ibx_2/threads"]
+ assert [row["id"] for row in resumed.rows] == ["thr_2"]
+
+
class TestValidateCredentials:
@parameterized.expand(
[
@@ -88,10 +174,3 @@ class TestValidateCredentials:
assert (is_valid, code) == (expected_valid, status_code)
assert network.requests_log[0].path == "/emails"
-
-
-class TestEndpointSettings:
- @parameterized.expand([(name,) for name in ENDPOINTS])
- def test_every_declared_endpoint_has_a_settings_entry(self, endpoint: str) -> None:
- assert ENDPOINTS[endpoint].path.startswith("/")
- assert ENDPOINTS[endpoint].primary_keys == ["id"]