:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import dataclasses
from typing import Any, Optional
from urllib.parse import urlencode
from requests import Request, Response
from products.warehouse_sources.sources.paystack.settings import PAYSTACK_ENDPOINTS, PaystackEndpointConfig
from products.warehouse_sources.sources.sdk import (
BasePaginator,
Endpoint,
EndpointResource,
JSONResponseCursorPaginator,
Resource,
RESTAPIConfig,
ResumableSourceManager,
make_tracked_session,
rename_parent_fields,
rest_api_resource,
rest_api_resources,
)
PAYSTACK_BASE_URL = "https://api.paystack.co"
# Paystack caps list pages at 100 records (default 50).
PAGE_SIZE = 100
@dataclasses.dataclass(frozen=True)
class PaystackResumeConfig:
next_page: Optional[int] = None
next_cursor: Optional[str] = None
# Fan-out tables checkpoint the shape the dependent resource emits: which parents finished
# and where the current one stopped.
fanout_state: Optional[dict[str, Any]] = None
class PaystackPaginator(BasePaginator):
"""Page-number pagination over Paystack list endpoints.
Paystack wraps list responses as ``{"status", "message", "data": [...], "meta": {...}}`` and
paginates with ``page`` / ``perPage``. ``meta.pageCount`` reports the total number of pages, so
we walk ``page`` from 1 until it exceeds ``pageCount``. When ``meta`` is missing (or carries no
usable ``pageCount``) we fall back to stopping on the first empty page, which is Paystack's own
recommended termination signal for offset pagination.
"""
def __init__(self, page: int = 1, page_size_param: str = "perPage") -> None:
super().__init__()
self._page = page
self._page_size_param = page_size_param
def init_request(self, request: Request) -> None:
if request.params is None:
request.params = {}
request.params[self._page_size_param] = PAGE_SIZE
request.params["page"] = self._page
def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:
meta: dict[str, Any] = {}
try:
body = response.json()
if isinstance(body, dict) and isinstance(body.get("meta"), dict):
meta = body["meta"]
except Exception:
meta = {}
page_count = meta.get("pageCount")
if isinstance(page_count, int):
if self._page >= page_count:
self._has_next_page = False
return
elif not data:
self._has_next_page = False
return
self._page += 1
self._has_next_page = True
def update_request(self, request: Request) -> None:
if request.params is None:
request.params = {}
request.params["page"] = self._page
def get_resume_state(self) -> Optional[dict[str, Any]]:
if self._has_next_page:
return {"next_page": self._page}
return None
def set_resume_state(self, state: dict[str, Any]) -> None:
next_page = state.get("next_page")
if next_page is not None:
self._page = int(next_page)
self._has_next_page = True
def _build_paginator(config: PaystackEndpointConfig) -> BasePaginator:
if config.cursor_paginated:
return JSONResponseCursorPaginator(cursor_path="meta.next", cursor_param="cursor")
return PaystackPaginator(page_size_param=config.page_size_param)
def get_resource(name: str) -> EndpointResource:
config = PAYSTACK_ENDPOINTS[name]
endpoint: Endpoint = {
"data_selector": "data",
"path": config.path,
"paginator": _build_paginator(config),
}
resource: EndpointResource = {
"name": name,
"table_name": name.lower(),
# Full refresh: Paystack offers no verified server-side updated-at filter, so we replace
# the table each sync rather than risk a corrupt incremental watermark.
"write_disposition": "replace",
"endpoint": endpoint,
"table_format": "delta",
}
if config.fanout is not None:
endpoint["params"] = {
config.fanout.resolve_param: {
"type": "resolve",
"resource": config.fanout.parent_name,
"field": config.fanout.resolve_field,
}
}
resource["include_from_parent"] = config.fanout.include_from_parent
return resource
def _to_paginator_state(config: PaystackEndpointConfig, resume: PaystackResumeConfig) -> Optional[dict[str, Any]]:
if config.fanout is not None:
return resume.fanout_state
if config.cursor_paginated:
return {"cursor": resume.next_cursor} if resume.next_cursor else None
return {"next_page": resume.next_page} if resume.next_page else None
def _to_resume_config(config: PaystackEndpointConfig, state: dict[str, Any]) -> Optional[PaystackResumeConfig]:
if config.fanout is not None:
return PaystackResumeConfig(fanout_state=state)
if config.cursor_paginated:
return PaystackResumeConfig(next_cursor=str(state["cursor"])) if state.get("cursor") else None
return PaystackResumeConfig(next_page=int(state["next_page"])) if state.get("next_page") else None
def paystack_source(
secret_api_key: str,
endpoint: str,
team_id: int,
job_id: str,
resumable_source_manager: ResumableSourceManager[PaystackResumeConfig],
) -> Resource:
endpoint_config = PAYSTACK_ENDPOINTS[endpoint]
resources: list[str | EndpointResource] = [get_resource(endpoint)]
if endpoint_config.fanout is not None:
resources.insert(0, get_resource(endpoint_config.fanout.parent_name))
config: RESTAPIConfig = {
"client": {
"base_url": PAYSTACK_BASE_URL,
"auth": {
"type": "bearer",
"token": secret_api_key,
},
},
"resource_defaults": {
"write_disposition": "replace",
},
"resources": resources,
}
initial_paginator_state: Optional[dict[str, Any]] = None
if resumable_source_manager.can_resume():
resume_config = resumable_source_manager.load_state()
if resume_config is not None:
initial_paginator_state = _to_paginator_state(endpoint_config, resume_config)
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
# Persist only when there's a next page to resume to; the Redis TTL handles cleanup.
resume_config = _to_resume_config(endpoint_config, state) if state else None
if resume_config is not None:
resumable_source_manager.save_state(resume_config)
if endpoint_config.fanout is None:
return rest_api_resource(
config,
team_id,
job_id,
# Full refresh — no incremental cursor.
None,
resume_hook=save_checkpoint,
initial_paginator_state=initial_paginator_state,
)
fanout = endpoint_config.fanout
child = next(
r
for r in rest_api_resources(
config,
team_id,
job_id,
None,
resume_hook=save_checkpoint,
initial_paginator_state=initial_paginator_state,
)
if r.name == endpoint
)
return child.add_map(rename_parent_fields(fanout.parent_name, fanout.parent_field_renames))
def validate_credentials(secret_api_key: str) -> bool:
"""Cheap authenticated probe. A Paystack secret key is account-wide (no per-resource scopes),
so a 200 on a 1-row transaction list confirms the key is genuine."""
query = urlencode({"perPage": 1})
res = make_tracked_session(redact_values=(secret_api_key,)).get(
f"{PAYSTACK_BASE_URL}/transaction?{query}",
headers={"Authorization": f"Bearer {secret_api_key}"},
timeout=10,
)
return res.status_code == 200