Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

13.2 KB
Raw Blame
from datetime import UTC, date, datetime
from typing import Any, Optional, cast

from requests import Request, Response

from products.warehouse_sources.sources.convertkit.settings import CONVERTKIT_ENDPOINTS, ConvertKitEndpointConfig
from products.warehouse_sources.sources.sdk import (
    BasePaginator,
    ClientConfig,
    IncrementalConfig,
    RESTAPIConfig,
    ResumableSourceManager,
    SourceResponse,
    build_dependent_resource,
    frozen,
    make_tracked_session,
    rest_api_resource,
    validate_via_probe,
)

# Kit (formerly ConvertKit) v4 API. v3 (api.convertkit.com) is deprecated.
CONVERTKIT_BASE_URL = "https://api.kit.com"
REQUEST_TIMEOUT = 60
# Sent as the cursor floor on a fan-out child's first sync, because the framework binds the
# filter param whether or not a watermark exists yet.
EPOCH_CURSOR = "1970-01-01T00:00:00Z"


@frozen
class ConvertKitResumeConfig:
    # Top-level endpoints resume from the `after` cursor of the last fully-yielded page.
    after: Optional[str] = None
    # Fan-out endpoints resume by parent: the child paths already fully synced, the path in
    # progress, and that path's paginator state. See
    # `common.rest_source.__init__._make_paginate_dependent_resource`.
    completed: Optional[list[str]] = None
    current: Optional[str] = None
    child_state: Optional[dict[str, Any]] = None


def _get_headers() -> dict[str, str]:
    # Auth (the X-Kit-Api-Key header) is supplied via the framework auth config so its value is
    # redacted from logs; only the non-secret Accept header is set here.
    return {"Accept": "application/json"}


def _format_incremental_value(value: Any) -> str:
    """Format an incremental cursor value as ISO 8601 with a Z suffix.

    Kit's *_after / *_before filters accept full ISO 8601 timestamps (the docs example
    is ``2023-01-17T11:43:55Z``) even though they describe the format as ``yyyy-mm-dd``.
    """
    if isinstance(value, datetime):
        utc_dt = value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
        return utc_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
    if isinstance(value, date):
        return datetime.combine(value, datetime.min.time(), tzinfo=UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
    return str(value)


class ConvertKitPaginator(BasePaginator):
    """Kit v4 Relay-style cursor pagination.

    The response body carries ``{"pagination": {"has_next_page": bool, "end_cursor": str}}``; the next
    page is fetched with ``after=<end_cursor>``. Kit can return an ``end_cursor`` even on the last page,
    so ``has_next_page`` is the authoritative stop signal — mirror the hand-rolled loop exactly and stop
    when either ``has_next_page`` is falsy or ``end_cursor`` is empty.
    """

    def __init__(self) -> None:
        super().__init__()
        self._next_cursor: Optional[str] = None

    def init_request(self, request: Request) -> None:
        # Honour a seeded resume cursor on the first request.
        if self._next_cursor is not None:
            if request.params is None:
                request.params = {}
            request.params["after"] = self._next_cursor

    def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:
        self._next_cursor = None
        try:
            pagination = response.json().get("pagination", {})
        except Exception:
            pagination = {}

        end_cursor = pagination.get("end_cursor")
        if pagination.get("has_next_page") and end_cursor:
            self._next_cursor = end_cursor
            self._has_next_page = True
        else:
            self._has_next_page = False

    def update_request(self, request: Request) -> None:
        if self._next_cursor is not None:
            if request.params is None:
                request.params = {}
            request.params["after"] = self._next_cursor

    def get_resume_state(self) -> Optional[dict[str, Any]]:
        if self._has_next_page and self._next_cursor is not None:
            return {"after": self._next_cursor}
        return None

    def set_resume_state(self, state: dict[str, Any]) -> None:
        after = state.get("after")
        if after is not None:
            self._next_cursor = str(after)
            self._has_next_page = True


def _build_params(
    config: ConvertKitEndpointConfig,
    should_use_incremental_field: bool,
    db_incremental_field_last_value: Any,
    incremental_field: str | None,
) -> dict[str, Any]:
    params: dict[str, Any] = {"per_page": config.page_size} if config.paginated else {}
    params.update(config.extra_params)

    if (
        config.supports_incremental
        and should_use_incremental_field
        and incremental_field
        and db_incremental_field_last_value is not None
    ):
        filter_param = config.incremental_param_map.get(incremental_field)
        if filter_param:
            # The framework applies `convert` to db_incremental_field_last_value and injects it under
            # `filter_param` (e.g. created_after) — same server-side filter the hand-rolled source sent.
            params[filter_param] = {
                "type": "incremental",
                "cursor_path": incremental_field,
                "convert": _format_incremental_value,
            }

    return params


def _client_config(api_key: str) -> ClientConfig:
    return {
        "base_url": CONVERTKIT_BASE_URL,
        "headers": _get_headers(),
        "auth": {"type": "api_key", "api_key": api_key, "name": "X-Kit-Api-Key", "location": "header"},
        "paginator": ConvertKitPaginator(),
    }


def _incremental_config(config: ConvertKitEndpointConfig, cursor_path: str) -> Optional[IncrementalConfig]:
    filter_param = config.incremental_param_map.get(cursor_path)
    if not filter_param:
        return None
    return {
        "start_param": filter_param,
        "cursor_path": cursor_path,
        "initial_value": EPOCH_CURSOR,
        "convert": _format_incremental_value,
    }


def _top_level_resource(
    config: ConvertKitEndpointConfig,
    api_key: str,
    endpoint: str,
    team_id: int,
    job_id: str,
    resumable_source_manager: ResumableSourceManager[ConvertKitResumeConfig],
    should_use_incremental_field: bool,
    db_incremental_field_last_value: Optional[Any],
    incremental_field: str | None,
) -> Any:
    rest_config: RESTAPIConfig = {
        "client": _client_config(api_key),
        "resources": [
            {
                "name": endpoint,
                "endpoint": {
                    "path": config.path,
                    "params": _build_params(
                        config, should_use_incremental_field, db_incremental_field_last_value, incremental_field
                    ),
                    "data_selector": config.data_key,
                },
            }
        ],
    }

    initial_paginator_state: Optional[dict[str, Any]] = None
    if resumable_source_manager.can_resume():
        resume = resumable_source_manager.load_state()
        if resume is not None and resume.after:
            initial_paginator_state = {"after": resume.after}

    def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
        # Persist only when a next page remains; save AFTER a page is yielded so a crash re-fetches
        # from here (merge dedupes on primary key) rather than skipping the last page.
        if state and state.get("after"):
            resumable_source_manager.save_state(ConvertKitResumeConfig(after=str(state["after"])))

    return rest_api_resource(
        rest_config,
        team_id,
        job_id,
        db_incremental_field_last_value if should_use_incremental_field else None,
        resume_hook=save_checkpoint,
        initial_paginator_state=initial_paginator_state,
    )


def _fanout_resource(
    config: ConvertKitEndpointConfig,
    api_key: str,
    endpoint: str,
    team_id: int,
    job_id: str,
    resumable_source_manager: ResumableSourceManager[ConvertKitResumeConfig],
    should_use_incremental_field: bool,
    db_incremental_field_last_value: Optional[Any],
    incremental_field: str | None,
) -> Any:
    assert config.fanout is not None
    parent_config = CONVERTKIT_ENDPOINTS[config.fanout.parent_name]

    initial_state: Optional[dict[str, Any]] = None
    if resumable_source_manager.can_resume():
        resume = resumable_source_manager.load_state()
        if resume is not None and (resume.completed or resume.current):
            initial_state = {
                "completed": resume.completed or [],
                "current": resume.current,
                "child_state": resume.child_state,
            }

    def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
        if state is not None:
            resumable_source_manager.save_state(
                ConvertKitResumeConfig(
                    completed=state.get("completed"),
                    current=state.get("current"),
                    child_state=state.get("child_state"),
                )
            )

    return cast(
        Any,
        build_dependent_resource(
            endpoint_configs=CONVERTKIT_ENDPOINTS,
            child_endpoint=endpoint,
            fanout=config.fanout,
            client_config=_client_config(api_key),
            path_format_values={},
            team_id=team_id,
            job_id=job_id,
            db_incremental_field_last_value=db_incremental_field_last_value if should_use_incremental_field else None,
            should_use_incremental_field=should_use_incremental_field,
            incremental_field=incremental_field,
            incremental_config_factory=lambda cursor_path: _incremental_config(config, cursor_path),
            parent_endpoint_extra={"data_selector": parent_config.data_key},
            child_endpoint_extra={"data_selector": config.data_key},
            child_params_extra=dict(config.extra_params),
            page_size_param="per_page",
            resume_hook=save_checkpoint,
            initial_paginator_state=initial_state,
        ),
    )


def convertkit_source(
    api_key: str,
    endpoint: str,
    team_id: int,
    job_id: str,
    resumable_source_manager: ResumableSourceManager[ConvertKitResumeConfig],
    should_use_incremental_field: bool = False,
    db_incremental_field_last_value: Optional[Any] = None,
    incremental_field: str | None = None,
) -> SourceResponse:
    config = CONVERTKIT_ENDPOINTS[endpoint]

    resource: Any
    if config.fanout is not None:
        resource = _fanout_resource(
            config,
            api_key,
            endpoint,
            team_id,
            job_id,
            resumable_source_manager,
            should_use_incremental_field,
            db_incremental_field_last_value,
            incremental_field,
        )
    else:
        resource = _top_level_resource(
            config,
            api_key,
            endpoint,
            team_id,
            job_id,
            resumable_source_manager,
            should_use_incremental_field,
            db_incremental_field_last_value,
            incremental_field,
        )

    return SourceResponse(
        name=endpoint,
        items=lambda: resource,
        primary_keys=config.primary_keys,
        partition_count=1,
        partition_size=1,
        partition_mode="datetime" if config.partition_key else None,
        partition_format="month" if config.partition_key else None,
        partition_keys=[config.partition_key] if config.partition_key else None,
        column_hints=resource.column_hints,
        # "asc" endpoints are read in full each sync, so the merge cursor advances to the max
        # incremental value across all pages even though the API guarantees no page order.
        # "desc" holds the watermark until the sync completes; settings.py says why. The
        # earliest-value backfill scrolling that "desc" also enables stays unused, because the
        # resumable manager checkpoints fan-out progress per parent instead.
        sort_mode=config.sort_mode,
    )


def validate_credentials(api_key: str, endpoint: str | None = None) -> tuple[bool, str | None]:
    """Probe the API with the given key. Returns (is_valid, error_message).

    A 403 at source-create time (``endpoint is None``) is treated as valid — the key
    works but may lack scope for a specific endpoint, which the user can grant later.
    """
    if endpoint and endpoint not in CONVERTKIT_ENDPOINTS:
        return False, f"Unknown Kit endpoint: {endpoint}"
    config = CONVERTKIT_ENDPOINTS[endpoint] if endpoint else CONVERTKIT_ENDPOINTS["subscribers"]
    if config.fanout is not None:
        # A fan-out path is only reachable per parent record, so probe the parent listing the
        # child is walked from, which is the call the sync makes first anyway.
        config = CONVERTKIT_ENDPOINTS[config.fanout.parent_name]
    url = f"{CONVERTKIT_BASE_URL}{config.path}?per_page=1"

    ok, status = validate_via_probe(
        lambda: make_tracked_session(redact_values=(api_key,)),
        url,
        headers={"X-Kit-Api-Key": api_key, **_get_headers()},
        timeout=REQUEST_TIMEOUT,
    )

    if ok:
        return True, None
    if status is None:
        return False, "Could not reach the Kit API. Please try again."
    if status == 403 and endpoint is None:
        return True, None
    if status in (401, 403):
        return False, "Invalid or insufficiently scoped Kit API key"
    return False, f"Kit API returned status {status}"