:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): add missing persistiq endpoint tables (#125420)
Tom Owersea413ff3bcbf
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex b1607508f397..5cf9684e6a82 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -845,7 +845,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/pendo/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/perigon/perigon.py1 products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/persistiq.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/persona/persona.py1 products/warehouse_sources/backend/temporal/data_imports/sources/persona/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/personio/personio.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex c49b4aac49ad..dc1165705723 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -9304,10 +9304,10 @@ Today (3): `campaigns`, `leads`, `users`Diffed against: <https://web.archive.org/web/20251120081157/https://apidocs.persistiq.com/>-- [ ] `GET /v1/events` — the sales activity stream (sends, opens, clicks, replies) - the only behavioral data in the API and absent today (high)-- [ ] `GET /v1/lead_statuses` — lookup resolving the status ID carried on every lead already synced (high)-- [ ] `GET /v1/lead_fields` — custom field definitions needed to interpret lead custom attributes (medium)-- [ ] `GET /v1/dnc_domains` — do-not-contact domain list for suppression analysis (low)+- [x] `GET /v1/events` — the sales activity stream (sends, opens, clicks, replies) - the only behavioral data in the API and absent today (high)+- [x] `GET /v1/lead_statuses` — lookup resolving the status ID carried on every lead already synced (high)+- [x] `GET /v1/lead_fields` — custom field definitions needed to interpret lead custom attributes (medium)+- [x] `GET /v1/dnc_domains` — do-not-contact domain list for suppression analysis (low)Note: apidocs.persistiq.com is behind a Cloudflare interstitial (403 to curl with any UA), so I read the archived copy of the same Slate single-page doc; it enumerates the full v1 API (base https://api.persistiq.com/v1). Only 4 GET-listable resources are missing, but one of them is the entire activity stream. Note /campaigns/{id}/leads is POST/DELETE only, so campaign membership is not separately syncable.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/canonical_descriptions.pyindex 96a08d77bb02..34fc799f2bb1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/canonical_descriptions.py@@ -38,4 +38,44 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"created_at": "When the campaign was created.",},},+ "events": {+ "description": "An activity in your PersistIQ account, such as a lead being created, a message being sent, opened, clicked, or replied to, or a campaign change.",+ "docs_url": "https://apidocs.persistiq.com",+ "columns": {+ "id": "The unique ID of the event.",+ "kind": "The type of event (e.g. lead_created, message_sent, message_opened, message_received).",+ "event_at": "When the event happened.",+ "user_id": "The ID of the user who performed the event.",+ "user": "The user who performed the event (id, name, email).",+ "campaign": "The campaign the event relates to (id, name), if any.",+ "lead": "The lead the event relates to (id, email, first and last name), if any.",+ "params": "The attributes supplied with the event, if any.",+ "diff": "The changes made by the event, if any.",+ },+ },+ "lead_statuses": {+ "description": "A lead status defined for your PersistIQ account (e.g. engaged, working, unresponsive).",+ "docs_url": "https://apidocs.persistiq.com",+ "columns": {+ "id": "The unique ID of the lead status.",+ "name": "The name of the lead status.",+ },+ },+ "lead_fields": {+ "description": "A lead field defined for your PersistIQ account, including custom fields.",+ "docs_url": "https://apidocs.persistiq.com",+ "columns": {+ "id": "The unique ID of the lead field.",+ "name": "The key of the field in a lead's data.",+ "label": "The display label of the field.",+ },+ },+ "dnc_domains": {+ "description": "A Do Not Contact domain. PersistIQ does not send email to addresses at this domain.",+ "docs_url": "https://apidocs.persistiq.com",+ "columns": {+ "id": "The unique ID of the Do Not Contact domain.",+ "name": "The domain name.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/persistiq.py b/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/persistiq.pyindex 15fb2130a65a..ffe4d7f47107 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/persistiq.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/persistiq.py@@ -3,6 +3,9 @@ from typing import Any, Optionalfrom requests import Response+from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import (+ coerce_datetime_to_utc,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_sessionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (RESTAPIConfig,@@ -11,6 +14,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (PageNumberPaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import Endpointfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse@@ -36,6 +40,13 @@ def _headers() -> dict[str, str]:return {"Accept": "application/json"}+def _format_datetime(value: Any) -> str:+ normalized = coerce_datetime_to_utc(value)+ if normalized is None:+ return str(value)+ return normalized.strftime("%Y-%m-%dT%H:%M:%SZ")++class PersistiqPaginator(PageNumberPaginator):"""PersistIQ paginates its list endpoints by 1-indexed ``page`` and signals whether more pagesremain with a body-level ``has_more`` flag rather than a total-pages count, so termination reads@@ -68,10 +79,36 @@ def persistiq_source(team_id: int,job_id: str,resumable_source_manager: ResumableSourceManager[PersistiqResumeConfig],+ should_use_incremental_field: bool = False,+ incremental_field: Optional[str] = None,db_incremental_field_last_value: Optional[Any] = None,) -> SourceResponse:config = PERSISTIQ_ENDPOINTS[endpoint]+ endpoint_config: Endpoint = {+ "path": config.path,+ "data_selector": config.list_key,+ # A 200 whose body isn't the expected `{list_key: [...]}` envelope (a bare array,+ # a missing resource key, or a non-list value) is treated as transient and+ # retried — the old source raised a retryable error for exactly these shapes.+ "data_selector_malformed_retryable": True,+ }+ if not config.paginated:+ endpoint_config["paginator"] = "single_page"++ is_incremental = bool(should_use_incremental_field and config.incremental_param)+ if (+ is_incremental+ and config.incremental_param+ and incremental_field+ and db_incremental_field_last_value is not None+ ):+ endpoint_config["incremental"] = {+ "start_param": config.incremental_param,+ "cursor_path": incremental_field,+ "convert": _format_datetime,+ }+rest_config: RESTAPIConfig = {"client": {"base_url": PERSISTIQ_BASE_URL,@@ -84,14 +121,7 @@ def persistiq_source("resources": [{"name": endpoint,- "endpoint": {- "path": config.path,- "data_selector": config.list_key,- # A 200 whose body isn't the expected `{list_key: [...]}` envelope (a bare array,- # a missing resource key, or a non-list value) is treated as transient and- # retried — the old source raised a retryable error for exactly these shapes.- "data_selector_malformed_retryable": True,- },+ "endpoint": endpoint_config,}],}@@ -112,7 +142,7 @@ def persistiq_source(rest_config,team_id,job_id,- db_incremental_field_last_value,+ db_incremental_field_last_value if is_incremental else None,resume_hook=save_checkpoint,initial_paginator_state=initial_paginator_state,)@@ -123,6 +153,9 @@ def persistiq_source(primary_keys=config.primary_keys,partition_count=1,partition_size=1,+ partition_mode="datetime" if config.partition_key else None,+ partition_format="month" if config.partition_key else None,+ partition_keys=[config.partition_key] if config.partition_key else None,column_hints=resource.column_hints,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/settings.pyindex 92b87989864b..e42bf43f1367 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/settings.py@@ -1,9 +1,10 @@from dataclasses import dataclass, field+from typing import Optional-from products.warehouse_sources.backend.types import IncrementalField+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType-@dataclass+@dataclass(frozen=True)class PersistiqEndpointConfig:name: strpath: str@@ -13,17 +14,41 @@ class PersistiqEndpointConfig:list_key: str# PersistIQ object IDs are globally unique within an account, so `id` is a safe primary key.primary_keys: list[str] = field(default_factory=lambda: ["id"])+ # `lead_statuses` and `lead_fields` return the whole collection in one response with no `has_more`.+ paginated: bool = True+ # Query param that filters rows to those after a timestamp; only `events` has one (`start_at`).+ incremental_param: Optional[str] = None+ partition_key: Optional[str] = None-# PersistIQ REST API list endpoints. All are full-refresh only: the list endpoints expose no-# server-side `updated_after`-style filter, so there is no genuine incremental cursor to advance-# (a client-side scan of every page would cost the same as a full refresh — see the skill).+# PersistIQ REST API list endpoints. Only `events` takes a server-side time filter (`start_at`); the+# others expose no `updated_after`-style filter, so they are full refresh (a client-side scan of every+# page would cost the same as a full refresh).PERSISTIQ_ENDPOINTS: dict[str, PersistiqEndpointConfig] = {"leads": PersistiqEndpointConfig(name="leads", path="/leads", list_key="leads"),"users": PersistiqEndpointConfig(name="users", path="/users", list_key="users"),"campaigns": PersistiqEndpointConfig(name="campaigns", path="/campaigns", list_key="campaigns"),+ "events": PersistiqEndpointConfig(+ name="events", path="/events", list_key="events", incremental_param="start_at", partition_key="event_at"+ ),+ "lead_statuses": PersistiqEndpointConfig(+ name="lead_statuses", path="/lead_statuses", list_key="lead_statuses", paginated=False+ ),+ "lead_fields": PersistiqEndpointConfig(+ name="lead_fields", path="/lead_fields", list_key="lead_fields", paginated=False+ ),+ "dnc_domains": PersistiqEndpointConfig(name="dnc_domains", path="/dnc_domains", list_key="dnc_domains"),}ENDPOINTS = tuple(PERSISTIQ_ENDPOINTS.keys())-INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {+ "events": [+ IncrementalField(+ label="event_at",+ type=IncrementalFieldType.DateTime,+ field="event_at",+ field_type=IncrementalFieldType.DateTime,+ )+ ],+}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/source.pyindex ceaa6c443ea5..0877d525c1f7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/source.py@@ -58,7 +58,7 @@ class PersistIqSource(ResumableSource[PersistIqSourceConfig, PersistiqResumeConfreleaseStatus=ReleaseStatus.BETA,caption="""Enter your PersistIQ API key to pull your sales engagement data into the PostHog Data warehouse.-You can find your API key under **Profile → Integrations → PersistIQ API** in [PersistIQ](https://app.persistiq.com). The key grants read access to your leads, users, and campaigns.+You can find your API key under **Profile → Integrations → PersistIQ API** in [PersistIQ](https://app.persistiq.com). The key grants read access to your leads, users, campaigns, events, and account settings.""",iconPath="/static/services/persistiq.png",docsUrl="https://posthog.com/docs/cdp/sources/persistiq",@@ -102,8 +102,6 @@ You can find your API key under **Profile → Integrations → PersistIQ API** iforce_refresh: bool = False,api_version: str | None = None,) -> list[SourceSchema]:- # Every endpoint is full refresh only — PersistIQ's list endpoints expose no server-side- # timestamp filter, so there is no incremental cursor to advance (INCREMENTAL_FIELDS is empty).return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)def validate_credentials(@@ -134,5 +132,7 @@ You can find your API key under **Profile → Integrations → PersistIQ API** iteam_id=inputs.team_id,job_id=inputs.job_id,resumable_source_manager=resumable_source_manager,- db_incremental_field_last_value=None, # every PersistIQ endpoint is full refresh+ should_use_incremental_field=inputs.should_use_incremental_field,+ incremental_field=inputs.incremental_field,+ db_incremental_field_last_value=inputs.db_incremental_field_last_value,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/tests/test_persistiq.py b/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/tests/test_persistiq.pyindex 24e7fb152c39..afaf46d8f927 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/tests/test_persistiq.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/persistiq/tests/test_persistiq.py@@ -1,3 +1,4 @@+from datetime import UTC, datetimefrom typing import Anyimport pytest@@ -101,6 +102,69 @@ class TestPagination:assert result.session_options == [{"redact_values": ("secret",)}]+class TestNewEndpoints:+ @pytest.mark.parametrize("endpoint", ["lead_statuses", "lead_fields"])+ def test_unpaginated_endpoint_makes_one_request_without_page_param(self, endpoint: str) -> None:+ result = _driver().run(endpoint, [ScriptedResponse(json={"status": "success", endpoint: [{"id": "x_1"}]})])++ assert result.raised is None+ assert result.rows == [{"id": "x_1"}]+ assert result.paths == [f"/v1/{endpoint}"]+ assert result.params("page") == [None]++ def test_dnc_domains_follows_pagination(self) -> None:+ result = _driver().run(+ "dnc_domains",+ [+ _page([{"id": "dom_1"}], has_more=True, list_key="dnc_domains"),+ _page([{"id": "dom_2"}], has_more=False, list_key="dnc_domains"),+ ],+ )++ assert result.raised is None+ assert [row["id"] for row in result.rows] == ["dom_1", "dom_2"]+ assert result.params("page") == ["1", "2"]++ @pytest.mark.parametrize(+ ("inputs", "expected_start_at"),+ [+ ({}, None),+ ({"incremental_field": "event_at"}, None),+ (+ {+ "incremental_field": "event_at",+ "db_incremental_field_last_value": datetime(2024, 5, 1, 12, 30, tzinfo=UTC),+ },+ "2024-05-01T12:30:00Z",+ ),+ (+ {+ "incremental_field": "event_at",+ "should_use_incremental_field": False,+ "db_incremental_field_last_value": datetime(2024, 5, 1, tzinfo=UTC),+ },+ None,+ ),+ ],+ )+ def test_events_start_at_follows_incremental_watermark(+ self, inputs: dict[str, Any], expected_start_at: str | None+ ) -> None:+ result = _driver().run(+ "events",+ [+ _page([{"id": "evt_1", "event_at": "2024-05-02T00:00:00Z"}], has_more=True, list_key="events"),+ _page([{"id": "evt_2", "event_at": "2024-05-03T00:00:00Z"}], has_more=False, list_key="events"),+ ],+ **inputs,+ )++ assert result.raised is None+ assert [row["id"] for row in result.rows] == ["evt_1", "evt_2"]+ assert result.params("page") == ["1", "2"]+ assert result.params("start_at") == [expected_start_at, expected_start_at]++class TestMalformedBody:def test_missing_list_key_is_retried_then_succeeds(self) -> None:result = _driver().run(