Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

8.9 KB
Raw Blame
from dataclasses import field
from typing import Optional

from products.warehouse_sources.sources.sdk import (
    DependentEndpointConfig,
    IncrementalField,
    IncrementalFieldType,
    SortMode,
    frozen,
)

DEFAULT_PAGE_SIZE = 1000  # v4 max per_page


@frozen
class ConvertKitEndpointConfig:
    name: str
    path: str
    # Top-level key wrapping the list in the JSON response (e.g. {"subscribers": [...]}).
    data_key: str
    primary_keys: list[str] = field(default_factory=lambda: ["id"])
    # Stable datetime field to partition by. Never use a field that mutates (e.g. updated_at).
    partition_key: Optional[str] = None
    supports_incremental: bool = False
    incremental_fields: list[IncrementalField] = field(default_factory=list)
    # Maps a selected incremental field name to the server-side filter query param it drives,
    # e.g. {"created_at": "created_after"}.
    incremental_param_map: dict[str, str] = field(default_factory=dict)
    # Static query params always sent to the endpoint (e.g. status=all to include every record).
    extra_params: dict[str, str] = field(default_factory=dict)
    page_size: int = DEFAULT_PAGE_SIZE
    # Cleared for a path that returns one object instead of a page, so no paging params are
    # sent to an endpoint that documents none.
    paginated: bool = True
    # Set where the endpoint is only reachable per parent record, so rows are collected by
    # walking the parent listing first.
    fanout: Optional[DependentEndpointConfig] = None
    # "desc" holds the incremental watermark until the sync completes. Fan-out rows arrive
    # grouped by parent rather than in global time order, so a per-batch checkpoint would
    # advance the watermark past parents a crashed run has not reached yet.
    sort_mode: SortMode = "asc"

    @property
    def default_incremental_field(self) -> Optional[str]:
        return self.incremental_fields[0]["field"] if self.incremental_fields else None


def _datetime_field(name: str) -> IncrementalField:
    return {
        "label": name,
        "type": IncrementalFieldType.DateTime,
        "field": name,
        "field_type": IncrementalFieldType.DateTime,
    }


CONVERTKIT_ENDPOINTS: dict[str, ConvertKitEndpointConfig] = {
    # The only list endpoint with server-side timestamp filters (created_after / updated_after).
    "subscribers": ConvertKitEndpointConfig(
        name="subscribers",
        path="/v4/subscribers",
        data_key="subscribers",
        partition_key="created_at",
        supports_incremental=True,
        incremental_fields=[_datetime_field("created_at"), _datetime_field("updated_at")],
        incremental_param_map={"created_at": "created_after", "updated_at": "updated_after"},
        # Default status filter is "active"; "all" pulls every subscriber regardless of state.
        extra_params={"status": "all"},
    ),
    "broadcasts": ConvertKitEndpointConfig(
        name="broadcasts",
        path="/v4/broadcasts",
        data_key="broadcasts",
        partition_key="created_at",
    ),
    "forms": ConvertKitEndpointConfig(
        name="forms",
        path="/v4/forms",
        data_key="forms",
        partition_key="created_at",
        extra_params={"status": "all"},
    ),
    "sequences": ConvertKitEndpointConfig(
        name="sequences",
        path="/v4/sequences",
        data_key="sequences",
        partition_key="created_at",
    ),
    "tags": ConvertKitEndpointConfig(
        name="tags",
        path="/v4/tags",
        data_key="tags",
        partition_key="created_at",
    ),
    "custom_fields": ConvertKitEndpointConfig(
        name="custom_fields",
        path="/v4/custom_fields",
        data_key="custom_fields",
    ),
    "segments": ConvertKitEndpointConfig(
        name="segments",
        path="/v4/segments",
        data_key="segments",
        partition_key="created_at",
    ),
    "purchases": ConvertKitEndpointConfig(
        name="purchases",
        path="/v4/purchases",
        data_key="purchases",
        # Purchases have no created_at; transaction_time is the stable creation timestamp.
        partition_key="transaction_time",
    ),
    "email_templates": ConvertKitEndpointConfig(
        name="email_templates",
        path="/v4/email_templates",
        data_key="email_templates",
    ),
    "broadcast_stats": ConvertKitEndpointConfig(
        name="broadcast_stats",
        path="/v4/broadcasts/stats",
        data_key="broadcasts",
        # Opens and clicks keep accumulating after a broadcast is sent, so the sent_after
        # window would freeze the numbers of every broadcast already synced.
    ),
    "growth_stats": ConvertKitEndpointConfig(
        name="growth_stats",
        path="/v4/account/growth_stats",
        # A single object, not a list: one row covering the window the request asked for.
        data_key="stats",
        # Kit defaults the window to the last 90 days ending today, and reports both bounds in
        # the account's sending time zone rather than UTC.
        primary_keys=["starting", "ending"],
        paginated=False,
    ),
    "sequence_emails": ConvertKitEndpointConfig(
        name="sequence_emails",
        path="/v4/sequences/{sequence_id}/emails",
        data_key="emails",
        # Kit does not document whether an email id is unique account-wide or only within its
        # sequence, so the parent id stays in the key.
        primary_keys=["sequence_id", "id"],
        fanout=DependentEndpointConfig(
            parent_name="sequences",
            resolve_param="sequence_id",
            resolve_field="id",
            include_from_parent=["id"],
            parent_field_renames={"id": "sequence_id"},
        ),
    ),
    "broadcast_clicks": ConvertKitEndpointConfig(
        name="broadcast_clicks",
        path="/v4/broadcasts/{broadcast_id}/clicks",
        # The links sit under the broadcast the request named.
        data_key="broadcast.clicks",
        primary_keys=["broadcast_id", "id"],
        fanout=DependentEndpointConfig(
            parent_name="broadcasts",
            resolve_param="broadcast_id",
            resolve_field="id",
            include_from_parent=["id"],
            parent_field_renames={"id": "broadcast_id"},
            # The parent listing includes drafts and scheduled broadcasts, which were never
            # sent and have no click record to return.
            child_response_actions=[{"status_code": 404, "action": "ignore"}],
        ),
    ),
    "form_subscribers": ConvertKitEndpointConfig(
        name="form_subscribers",
        path="/v4/forms/{form_id}/subscribers",
        data_key="subscribers",
        # A subscriber id is unique per account but repeats across forms.
        primary_keys=["form_id", "id"],
        # added_at is the row's own timestamp, but a subscriber removed and re-added to a form
        # gets a new one, so partition on the subscriber's immutable creation date instead.
        partition_key="created_at",
        supports_incremental=True,
        incremental_fields=[_datetime_field("added_at")],
        incremental_param_map={"added_at": "added_after"},
        extra_params={"status": "all"},
        sort_mode="desc",
        fanout=DependentEndpointConfig(
            parent_name="forms",
            resolve_param="form_id",
            resolve_field="id",
            include_from_parent=["id"],
            parent_field_renames={"id": "form_id"},
            parent_params={"status": "all"},
        ),
    ),
    "tag_subscribers": ConvertKitEndpointConfig(
        name="tag_subscribers",
        path="/v4/tags/{tag_id}/subscribers",
        data_key="subscribers",
        primary_keys=["tag_id", "id"],
        partition_key="created_at",
        # Tags can be removed from subscribers. A full snapshot lets the load replace stale
        # junction rows that an addition-only tagged_after cursor can never emit.
        extra_params={"status": "all"},
        fanout=DependentEndpointConfig(
            parent_name="tags",
            resolve_param="tag_id",
            resolve_field="id",
            include_from_parent=["id"],
            parent_field_renames={"id": "tag_id"},
        ),
    ),
    "sequence_subscribers": ConvertKitEndpointConfig(
        name="sequence_subscribers",
        path="/v4/sequences/{sequence_id}/subscribers",
        data_key="subscribers",
        primary_keys=["sequence_id", "id"],
        partition_key="created_at",
        supports_incremental=True,
        incremental_fields=[_datetime_field("added_at")],
        incremental_param_map={"added_at": "added_after"},
        extra_params={"status": "all"},
        sort_mode="desc",
        fanout=DependentEndpointConfig(
            parent_name="sequences",
            resolve_param="sequence_id",
            resolve_field="id",
            include_from_parent=["id"],
            parent_field_renames={"id": "sequence_id"},
        ),
    ),
}

ENDPOINTS = tuple(CONVERTKIT_ENDPOINTS.keys())

INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
    name: config.incremental_fields for name, config in CONVERTKIT_ENDPOINTS.items()
}