:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
from datetime import UTC, date, datetime
from typing import Any, Optional, cast
from requests import Request, Response
from products.warehouse_sources.sources.convertkit.settings import CONVERTKIT_ENDPOINTS, ConvertKitEndpointConfig
from products.warehouse_sources.sources.sdk import (
BasePaginator,
ClientConfig,
IncrementalConfig,
RESTAPIConfig,
ResumableSourceManager,
SourceResponse,
build_dependent_resource,
frozen,
make_tracked_session,
rest_api_resource,
validate_via_probe,
)
# Kit (formerly ConvertKit) v4 API. v3 (api.convertkit.com) is deprecated.
CONVERTKIT_BASE_URL = "https://api.kit.com"
REQUEST_TIMEOUT = 60
# Sent as the cursor floor on a fan-out child's first sync, because the framework binds the
# filter param whether or not a watermark exists yet.
EPOCH_CURSOR = "1970-01-01T00:00:00Z"
@frozen
class ConvertKitResumeConfig:
# Top-level endpoints resume from the `after` cursor of the last fully-yielded page.
after: Optional[str] = None
# Fan-out endpoints resume by parent: the child paths already fully synced, the path in
# progress, and that path's paginator state. See
# `common.rest_source.__init__._make_paginate_dependent_resource`.
completed: Optional[list[str]] = None
current: Optional[str] = None
child_state: Optional[dict[str, Any]] = None
def _get_headers() -> dict[str, str]:
# Auth (the X-Kit-Api-Key header) is supplied via the framework auth config so its value is
# redacted from logs; only the non-secret Accept header is set here.
return {"Accept": "application/json"}
def _format_incremental_value(value: Any) -> str:
"""Format an incremental cursor value as ISO 8601 with a Z suffix.
Kit's *_after / *_before filters accept full ISO 8601 timestamps (the docs example
is ``2023-01-17T11:43:55Z``) even though they describe the format as ``yyyy-mm-dd``.
"""
if isinstance(value, datetime):
utc_dt = value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
return utc_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
if isinstance(value, date):
return datetime.combine(value, datetime.min.time(), tzinfo=UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
return str(value)
class ConvertKitPaginator(BasePaginator):
"""Kit v4 Relay-style cursor pagination.
The response body carries ``{"pagination": {"has_next_page": bool, "end_cursor": str}}``; the next
page is fetched with ``after=<end_cursor>``. Kit can return an ``end_cursor`` even on the last page,
so ``has_next_page`` is the authoritative stop signal — mirror the hand-rolled loop exactly and stop
when either ``has_next_page`` is falsy or ``end_cursor`` is empty.
"""
def __init__(self) -> None:
super().__init__()
self._next_cursor: Optional[str] = None
def init_request(self, request: Request) -> None:
# Honour a seeded resume cursor on the first request.
if self._next_cursor is not None:
if request.params is None:
request.params = {}
request.params["after"] = self._next_cursor
def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:
self._next_cursor = None
try:
pagination = response.json().get("pagination", {})
except Exception:
pagination = {}
end_cursor = pagination.get("end_cursor")
if pagination.get("has_next_page") and end_cursor:
self._next_cursor = end_cursor
self._has_next_page = True
else:
self._has_next_page = False
def update_request(self, request: Request) -> None:
if self._next_cursor is not None:
if request.params is None:
request.params = {}
request.params["after"] = self._next_cursor
def get_resume_state(self) -> Optional[dict[str, Any]]:
if self._has_next_page and self._next_cursor is not None:
return {"after": self._next_cursor}
return None
def set_resume_state(self, state: dict[str, Any]) -> None:
after = state.get("after")
if after is not None:
self._next_cursor = str(after)
self._has_next_page = True
def _build_params(
config: ConvertKitEndpointConfig,
should_use_incremental_field: bool,
db_incremental_field_last_value: Any,
incremental_field: str | None,
) -> dict[str, Any]:
params: dict[str, Any] = {"per_page": config.page_size} if config.paginated else {}
params.update(config.extra_params)
if (
config.supports_incremental
and should_use_incremental_field
and incremental_field
and db_incremental_field_last_value is not None
):
filter_param = config.incremental_param_map.get(incremental_field)
if filter_param:
# The framework applies `convert` to db_incremental_field_last_value and injects it under
# `filter_param` (e.g. created_after) — same server-side filter the hand-rolled source sent.
params[filter_param] = {
"type": "incremental",
"cursor_path": incremental_field,
"convert": _format_incremental_value,
}
return params
def _client_config(api_key: str) -> ClientConfig:
return {
"base_url": CONVERTKIT_BASE_URL,
"headers": _get_headers(),
"auth": {"type": "api_key", "api_key": api_key, "name": "X-Kit-Api-Key", "location": "header"},
"paginator": ConvertKitPaginator(),
}
def _incremental_config(config: ConvertKitEndpointConfig, cursor_path: str) -> Optional[IncrementalConfig]:
filter_param = config.incremental_param_map.get(cursor_path)
if not filter_param:
return None
return {
"start_param": filter_param,
"cursor_path": cursor_path,
"initial_value": EPOCH_CURSOR,
"convert": _format_incremental_value,
}
def _top_level_resource(
config: ConvertKitEndpointConfig,
api_key: str,
endpoint: str,
team_id: int,
job_id: str,
resumable_source_manager: ResumableSourceManager[ConvertKitResumeConfig],
should_use_incremental_field: bool,
db_incremental_field_last_value: Optional[Any],
incremental_field: str | None,
) -> Any:
rest_config: RESTAPIConfig = {
"client": _client_config(api_key),
"resources": [
{
"name": endpoint,
"endpoint": {
"path": config.path,
"params": _build_params(
config, should_use_incremental_field, db_incremental_field_last_value, incremental_field
),
"data_selector": config.data_key,
},
}
],
}
initial_paginator_state: Optional[dict[str, Any]] = None
if resumable_source_manager.can_resume():
resume = resumable_source_manager.load_state()
if resume is not None and resume.after:
initial_paginator_state = {"after": resume.after}
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
# Persist only when a next page remains; save AFTER a page is yielded so a crash re-fetches
# from here (merge dedupes on primary key) rather than skipping the last page.
if state and state.get("after"):
resumable_source_manager.save_state(ConvertKitResumeConfig(after=str(state["after"])))
return rest_api_resource(
rest_config,
team_id,
job_id,
db_incremental_field_last_value if should_use_incremental_field else None,
resume_hook=save_checkpoint,
initial_paginator_state=initial_paginator_state,
)
def _fanout_resource(
config: ConvertKitEndpointConfig,
api_key: str,
endpoint: str,
team_id: int,
job_id: str,
resumable_source_manager: ResumableSourceManager[ConvertKitResumeConfig],
should_use_incremental_field: bool,
db_incremental_field_last_value: Optional[Any],
incremental_field: str | None,
) -> Any:
assert config.fanout is not None
parent_config = CONVERTKIT_ENDPOINTS[config.fanout.parent_name]
initial_state: Optional[dict[str, Any]] = None
if resumable_source_manager.can_resume():
resume = resumable_source_manager.load_state()
if resume is not None and (resume.completed or resume.current):
initial_state = {
"completed": resume.completed or [],
"current": resume.current,
"child_state": resume.child_state,
}
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
if state is not None:
resumable_source_manager.save_state(
ConvertKitResumeConfig(
completed=state.get("completed"),
current=state.get("current"),
child_state=state.get("child_state"),
)
)
return cast(
Any,
build_dependent_resource(
endpoint_configs=CONVERTKIT_ENDPOINTS,
child_endpoint=endpoint,
fanout=config.fanout,
client_config=_client_config(api_key),
path_format_values={},
team_id=team_id,
job_id=job_id,
db_incremental_field_last_value=db_incremental_field_last_value if should_use_incremental_field else None,
should_use_incremental_field=should_use_incremental_field,
incremental_field=incremental_field,
incremental_config_factory=lambda cursor_path: _incremental_config(config, cursor_path),
parent_endpoint_extra={"data_selector": parent_config.data_key},
child_endpoint_extra={"data_selector": config.data_key},
child_params_extra=dict(config.extra_params),
page_size_param="per_page",
resume_hook=save_checkpoint,
initial_paginator_state=initial_state,
),
)
def convertkit_source(
api_key: str,
endpoint: str,
team_id: int,
job_id: str,
resumable_source_manager: ResumableSourceManager[ConvertKitResumeConfig],
should_use_incremental_field: bool = False,
db_incremental_field_last_value: Optional[Any] = None,
incremental_field: str | None = None,
) -> SourceResponse:
config = CONVERTKIT_ENDPOINTS[endpoint]
resource: Any
if config.fanout is not None:
resource = _fanout_resource(
config,
api_key,
endpoint,
team_id,
job_id,
resumable_source_manager,
should_use_incremental_field,
db_incremental_field_last_value,
incremental_field,
)
else:
resource = _top_level_resource(
config,
api_key,
endpoint,
team_id,
job_id,
resumable_source_manager,
should_use_incremental_field,
db_incremental_field_last_value,
incremental_field,
)
return SourceResponse(
name=endpoint,
items=lambda: resource,
primary_keys=config.primary_keys,
partition_count=1,
partition_size=1,
partition_mode="datetime" if config.partition_key else None,
partition_format="month" if config.partition_key else None,
partition_keys=[config.partition_key] if config.partition_key else None,
column_hints=resource.column_hints,
# "asc" endpoints are read in full each sync, so the merge cursor advances to the max
# incremental value across all pages even though the API guarantees no page order.
# "desc" holds the watermark until the sync completes; settings.py says why. The
# earliest-value backfill scrolling that "desc" also enables stays unused, because the
# resumable manager checkpoints fan-out progress per parent instead.
sort_mode=config.sort_mode,
)
def validate_credentials(api_key: str, endpoint: str | None = None) -> tuple[bool, str | None]:
"""Probe the API with the given key. Returns (is_valid, error_message).
A 403 at source-create time (``endpoint is None``) is treated as valid — the key
works but may lack scope for a specific endpoint, which the user can grant later.
"""
if endpoint and endpoint not in CONVERTKIT_ENDPOINTS:
return False, f"Unknown Kit endpoint: {endpoint}"
config = CONVERTKIT_ENDPOINTS[endpoint] if endpoint else CONVERTKIT_ENDPOINTS["subscribers"]
if config.fanout is not None:
# A fan-out path is only reachable per parent record, so probe the parent listing the
# child is walked from, which is the call the sync makes first anyway.
config = CONVERTKIT_ENDPOINTS[config.fanout.parent_name]
url = f"{CONVERTKIT_BASE_URL}{config.path}?per_page=1"
ok, status = validate_via_probe(
lambda: make_tracked_session(redact_values=(api_key,)),
url,
headers={"X-Kit-Api-Key": api_key, **_get_headers()},
timeout=REQUEST_TIMEOUT,
)
if ok:
return True, None
if status is None:
return False, "Could not reach the Kit API. Please try again."
if status == 403 and endpoint is None:
return True, None
if status in (401, 403):
return False, "Invalid or insufficiently scoped Kit API key"
return False, f"Kit API returned status {status}"