Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add missing persistiq endpoint tables (#125420)

Tom Owersea413ff3bcbf
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index b1607508f397..5cf9684e6a82 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -845,7 +845,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/pendo/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/perigon/perigon.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/persistiq.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/persona/persona.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/persona/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/personio/personio.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index c49b4aac49ad..dc1165705723 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9304,10 +9304,10 @@ Today (3): `campaigns`, `leads`, `users`
Diffed against: <https://web.archive.org/web/20251120081157/https://apidocs.persistiq.com/>
-- [ ] `GET /v1/events` — the sales activity stream (sends, opens, clicks, replies) - the only behavioral data in the API and absent today (high)
-- [ ] `GET /v1/lead_statuses` — lookup resolving the status ID carried on every lead already synced (high)
-- [ ] `GET /v1/lead_fields` — custom field definitions needed to interpret lead custom attributes (medium)
-- [ ] `GET /v1/dnc_domains` — do-not-contact domain list for suppression analysis (low)
+- [x] `GET /v1/events` — the sales activity stream (sends, opens, clicks, replies) - the only behavioral data in the API and absent today (high)
+- [x] `GET /v1/lead_statuses` — lookup resolving the status ID carried on every lead already synced (high)
+- [x] `GET /v1/lead_fields` — custom field definitions needed to interpret lead custom attributes (medium)
+- [x] `GET /v1/dnc_domains` — do-not-contact domain list for suppression analysis (low)
Note: apidocs.persistiq.com is behind a Cloudflare interstitial (403 to curl with any UA), so I read the archived copy of the same Slate single-page doc; it enumerates the full v1 API (base https://api.persistiq.com/v1). Only 4 GET-listable resources are missing, but one of them is the entire activity stream. Note /campaigns/{id}/leads is POST/DELETE only, so campaign membership is not separately syncable.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/canonical_descriptions.py
index 96a08d77bb02..34fc799f2bb1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/canonical_descriptions.py
@@ -38,4 +38,44 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"created_at": "When the campaign was created.",
},
},
+ "events": {
+ "description": "An activity in your PersistIQ account, such as a lead being created, a message being sent, opened, clicked, or replied to, or a campaign change.",
+ "docs_url": "https://apidocs.persistiq.com",
+ "columns": {
+ "id": "The unique ID of the event.",
+ "kind": "The type of event (e.g. lead_created, message_sent, message_opened, message_received).",
+ "event_at": "When the event happened.",
+ "user_id": "The ID of the user who performed the event.",
+ "user": "The user who performed the event (id, name, email).",
+ "campaign": "The campaign the event relates to (id, name), if any.",
+ "lead": "The lead the event relates to (id, email, first and last name), if any.",
+ "params": "The attributes supplied with the event, if any.",
+ "diff": "The changes made by the event, if any.",
+ },
+ },
+ "lead_statuses": {
+ "description": "A lead status defined for your PersistIQ account (e.g. engaged, working, unresponsive).",
+ "docs_url": "https://apidocs.persistiq.com",
+ "columns": {
+ "id": "The unique ID of the lead status.",
+ "name": "The name of the lead status.",
+ },
+ },
+ "lead_fields": {
+ "description": "A lead field defined for your PersistIQ account, including custom fields.",
+ "docs_url": "https://apidocs.persistiq.com",
+ "columns": {
+ "id": "The unique ID of the lead field.",
+ "name": "The key of the field in a lead's data.",
+ "label": "The display label of the field.",
+ },
+ },
+ "dnc_domains": {
+ "description": "A Do Not Contact domain. PersistIQ does not send email to addresses at this domain.",
+ "docs_url": "https://apidocs.persistiq.com",
+ "columns": {
+ "id": "The unique ID of the Do Not Contact domain.",
+ "name": "The domain name.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/persistiq.py b/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/persistiq.py
index 15fb2130a65a..ffe4d7f47107 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/persistiq.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/persistiq.py
@@ -3,6 +3,9 @@ from typing import Any, Optional
from requests import Response
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import (
+ coerce_datetime_to_utc,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
@@ -11,6 +14,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
PageNumberPaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import Endpoint
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -36,6 +40,13 @@ def _headers() -> dict[str, str]:
return {"Accept": "application/json"}
+def _format_datetime(value: Any) -> str:
+ normalized = coerce_datetime_to_utc(value)
+ if normalized is None:
+ return str(value)
+ return normalized.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
class PersistiqPaginator(PageNumberPaginator):
"""PersistIQ paginates its list endpoints by 1-indexed ``page`` and signals whether more pages
remain with a body-level ``has_more`` flag rather than a total-pages count, so termination reads
@@ -68,10 +79,36 @@ def persistiq_source(
team_id: int,
job_id: str,
resumable_source_manager: ResumableSourceManager[PersistiqResumeConfig],
+ should_use_incremental_field: bool = False,
+ incremental_field: Optional[str] = None,
db_incremental_field_last_value: Optional[Any] = None,
) -> SourceResponse:
config = PERSISTIQ_ENDPOINTS[endpoint]
+ endpoint_config: Endpoint = {
+ "path": config.path,
+ "data_selector": config.list_key,
+ # A 200 whose body isn't the expected `{list_key: [...]}` envelope (a bare array,
+ # a missing resource key, or a non-list value) is treated as transient and
+ # retried — the old source raised a retryable error for exactly these shapes.
+ "data_selector_malformed_retryable": True,
+ }
+ if not config.paginated:
+ endpoint_config["paginator"] = "single_page"
+
+ is_incremental = bool(should_use_incremental_field and config.incremental_param)
+ if (
+ is_incremental
+ and config.incremental_param
+ and incremental_field
+ and db_incremental_field_last_value is not None
+ ):
+ endpoint_config["incremental"] = {
+ "start_param": config.incremental_param,
+ "cursor_path": incremental_field,
+ "convert": _format_datetime,
+ }
+
rest_config: RESTAPIConfig = {
"client": {
"base_url": PERSISTIQ_BASE_URL,
@@ -84,14 +121,7 @@ def persistiq_source(
"resources": [
{
"name": endpoint,
- "endpoint": {
- "path": config.path,
- "data_selector": config.list_key,
- # A 200 whose body isn't the expected `{list_key: [...]}` envelope (a bare array,
- # a missing resource key, or a non-list value) is treated as transient and
- # retried — the old source raised a retryable error for exactly these shapes.
- "data_selector_malformed_retryable": True,
- },
+ "endpoint": endpoint_config,
}
],
}
@@ -112,7 +142,7 @@ def persistiq_source(
rest_config,
team_id,
job_id,
- db_incremental_field_last_value,
+ db_incremental_field_last_value if is_incremental else None,
resume_hook=save_checkpoint,
initial_paginator_state=initial_paginator_state,
)
@@ -123,6 +153,9 @@ def persistiq_source(
primary_keys=config.primary_keys,
partition_count=1,
partition_size=1,
+ partition_mode="datetime" if config.partition_key else None,
+ partition_format="month" if config.partition_key else None,
+ partition_keys=[config.partition_key] if config.partition_key else None,
column_hints=resource.column_hints,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/settings.py
index 92b87989864b..e42bf43f1367 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/settings.py
@@ -1,9 +1,10 @@
from dataclasses import dataclass, field
+from typing import Optional
-from products.warehouse_sources.backend.types import IncrementalField
+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
-@dataclass
+@dataclass(frozen=True)
class PersistiqEndpointConfig:
name: str
path: str
@@ -13,17 +14,41 @@ class PersistiqEndpointConfig:
list_key: str
# PersistIQ object IDs are globally unique within an account, so `id` is a safe primary key.
primary_keys: list[str] = field(default_factory=lambda: ["id"])
+ # `lead_statuses` and `lead_fields` return the whole collection in one response with no `has_more`.
+ paginated: bool = True
+ # Query param that filters rows to those after a timestamp; only `events` has one (`start_at`).
+ incremental_param: Optional[str] = None
+ partition_key: Optional[str] = None
-# PersistIQ REST API list endpoints. All are full-refresh only: the list endpoints expose no
-# server-side `updated_after`-style filter, so there is no genuine incremental cursor to advance
-# (a client-side scan of every page would cost the same as a full refresh — see the skill).
+# PersistIQ REST API list endpoints. Only `events` takes a server-side time filter (`start_at`); the
+# others expose no `updated_after`-style filter, so they are full refresh (a client-side scan of every
+# page would cost the same as a full refresh).
PERSISTIQ_ENDPOINTS: dict[str, PersistiqEndpointConfig] = {
"leads": PersistiqEndpointConfig(name="leads", path="/leads", list_key="leads"),
"users": PersistiqEndpointConfig(name="users", path="/users", list_key="users"),
"campaigns": PersistiqEndpointConfig(name="campaigns", path="/campaigns", list_key="campaigns"),
+ "events": PersistiqEndpointConfig(
+ name="events", path="/events", list_key="events", incremental_param="start_at", partition_key="event_at"
+ ),
+ "lead_statuses": PersistiqEndpointConfig(
+ name="lead_statuses", path="/lead_statuses", list_key="lead_statuses", paginated=False
+ ),
+ "lead_fields": PersistiqEndpointConfig(
+ name="lead_fields", path="/lead_fields", list_key="lead_fields", paginated=False
+ ),
+ "dnc_domains": PersistiqEndpointConfig(name="dnc_domains", path="/dnc_domains", list_key="dnc_domains"),
}
ENDPOINTS = tuple(PERSISTIQ_ENDPOINTS.keys())
-INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}
+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
+ "events": [
+ IncrementalField(
+ label="event_at",
+ type=IncrementalFieldType.DateTime,
+ field="event_at",
+ field_type=IncrementalFieldType.DateTime,
+ )
+ ],
+}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/source.py
index ceaa6c443ea5..0877d525c1f7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/source.py
@@ -58,7 +58,7 @@ class PersistIqSource(ResumableSource[PersistIqSourceConfig, PersistiqResumeConf
releaseStatus=ReleaseStatus.BETA,
caption="""Enter your PersistIQ API key to pull your sales engagement data into the PostHog Data warehouse.
-You can find your API key under **Profile → Integrations → PersistIQ API** in [PersistIQ](https://app.persistiq.com). The key grants read access to your leads, users, and campaigns.
+You can find your API key under **Profile → Integrations → PersistIQ API** in [PersistIQ](https://app.persistiq.com). The key grants read access to your leads, users, campaigns, events, and account settings.
""",
iconPath="/static/services/persistiq.png",
docsUrl="https://posthog.com/docs/cdp/sources/persistiq",
@@ -102,8 +102,6 @@ You can find your API key under **Profile → Integrations → PersistIQ API** i
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- # Every endpoint is full refresh only — PersistIQ's list endpoints expose no server-side
- # timestamp filter, so there is no incremental cursor to advance (INCREMENTAL_FIELDS is empty).
return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)
def validate_credentials(
@@ -134,5 +132,7 @@ You can find your API key under **Profile → Integrations → PersistIQ API** i
team_id=inputs.team_id,
job_id=inputs.job_id,
resumable_source_manager=resumable_source_manager,
- db_incremental_field_last_value=None, # every PersistIQ endpoint is full refresh
+ should_use_incremental_field=inputs.should_use_incremental_field,
+ incremental_field=inputs.incremental_field,
+ db_incremental_field_last_value=inputs.db_incremental_field_last_value,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/tests/test_persistiq.py b/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/tests/test_persistiq.py
index 24e7fb152c39..afaf46d8f927 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/tests/test_persistiq.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/tests/test_persistiq.py
@@ -1,3 +1,4 @@
+from datetime import UTC, datetime
from typing import Any
import pytest
@@ -101,6 +102,69 @@ class TestPagination:
assert result.session_options == [{"redact_values": ("secret",)}]
+class TestNewEndpoints:
+ @pytest.mark.parametrize("endpoint", ["lead_statuses", "lead_fields"])
+ def test_unpaginated_endpoint_makes_one_request_without_page_param(self, endpoint: str) -> None:
+ result = _driver().run(endpoint, [ScriptedResponse(json={"status": "success", endpoint: [{"id": "x_1"}]})])
+
+ assert result.raised is None
+ assert result.rows == [{"id": "x_1"}]
+ assert result.paths == [f"/v1/{endpoint}"]
+ assert result.params("page") == [None]
+
+ def test_dnc_domains_follows_pagination(self) -> None:
+ result = _driver().run(
+ "dnc_domains",
+ [
+ _page([{"id": "dom_1"}], has_more=True, list_key="dnc_domains"),
+ _page([{"id": "dom_2"}], has_more=False, list_key="dnc_domains"),
+ ],
+ )
+
+ assert result.raised is None
+ assert [row["id"] for row in result.rows] == ["dom_1", "dom_2"]
+ assert result.params("page") == ["1", "2"]
+
+ @pytest.mark.parametrize(
+ ("inputs", "expected_start_at"),
+ [
+ ({}, None),
+ ({"incremental_field": "event_at"}, None),
+ (
+ {
+ "incremental_field": "event_at",
+ "db_incremental_field_last_value": datetime(2024, 5, 1, 12, 30, tzinfo=UTC),
+ },
+ "2024-05-01T12:30:00Z",
+ ),
+ (
+ {
+ "incremental_field": "event_at",
+ "should_use_incremental_field": False,
+ "db_incremental_field_last_value": datetime(2024, 5, 1, tzinfo=UTC),
+ },
+ None,
+ ),
+ ],
+ )
+ def test_events_start_at_follows_incremental_watermark(
+ self, inputs: dict[str, Any], expected_start_at: str | None
+ ) -> None:
+ result = _driver().run(
+ "events",
+ [
+ _page([{"id": "evt_1", "event_at": "2024-05-02T00:00:00Z"}], has_more=True, list_key="events"),
+ _page([{"id": "evt_2", "event_at": "2024-05-03T00:00:00Z"}], has_more=False, list_key="events"),
+ ],
+ **inputs,
+ )
+
+ assert result.raised is None
+ assert [row["id"] for row in result.rows] == ["evt_1", "evt_2"]
+ assert result.params("page") == ["1", "2"]
+ assert result.params("start_at") == [expected_start_at, expected_start_at]
+
+
class TestMalformedBody:
def test_missing_list_key_is_retried_then_succeeds(self) -> None:
result = _driver().run(