:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import dataclasses
from typing import Any, Optional
from products.warehouse_sources.sources.postscript.settings import (
POSTSCRIPT_BASE_URL,
POSTSCRIPT_ENDPOINTS,
PostscriptEndpointConfig,
)
from products.warehouse_sources.sources.sdk import (
ClientConfig,
Endpoint,
EndpointResource,
IncrementalConfig,
PageNumberPaginator,
RESTAPIConfig,
ResumableSourceManager,
SinglePagePaginator,
SourceResponse,
coerce_datetime_to_utc,
make_tracked_session,
rest_api_resource,
)
@dataclasses.dataclass(frozen=False)
class PostscriptResumeConfig:
page: int
def _format_postscript_datetime(value: Any) -> str:
"""Format the incremental watermark for Postscript's `<field>__gte` filters.
Truncating to whole seconds rounds the lower bound down, and `__gte` is inclusive, so a
resumed sync re-reads a few boundary rows (the merge dedupes them on `id`) instead of
skipping any.
"""
normalized_value = coerce_datetime_to_utc(value)
if normalized_value is None:
return str(value)
return normalized_value.strftime("%Y-%m-%dT%H:%M:%SZ")
def _postscript_incremental_window(cursor_path: str) -> IncrementalConfig:
return {
"cursor_path": cursor_path,
"start_param": f"{cursor_path}__gte",
"initial_value": "1970-01-01T00:00:00Z",
"convert": _format_postscript_datetime,
}
def _rest_api_client_config(api_key: str) -> ClientConfig:
return {
"base_url": POSTSCRIPT_BASE_URL,
"auth": {"type": "bearer", "token": api_key},
"headers": {"Accept": "application/json"},
# Pin every request to the Postscript host and refuse to follow a 3xx, so a redirect
# can never replay the bearer token off-host.
"allowed_hosts": [],
"allow_redirects": False,
}
def _page_paginator() -> PageNumberPaginator:
# Postscript pages are 1-based and the body reports `page_info.total_pages`, so pagination
# stops after the last page instead of paying an extra empty-page request.
return PageNumberPaginator(base_page=1, page_param="page", total_path="page_info.total_pages")
def _resolve_cursor_field(config: PostscriptEndpointConfig, requested: str | None) -> str:
"""Pick the incremental cursor, honoring the user's choice when we advertise it.
A field we never advertised has no matching `__gte` filter or `sort` value, so the API
would 400 on it; fall back to the endpoint's default rather than failing the sync.
"""
advertised = {f["field"] for f in config.incremental_fields}
if requested in advertised:
return str(requested)
return str(config.default_incremental_field)
def get_resource(
endpoint: str,
api_version: str,
should_use_incremental_field: bool,
incremental_field_name: str | None = None,
) -> EndpointResource:
config = POSTSCRIPT_ENDPOINTS[endpoint]
use_incremental = should_use_incremental_field and bool(config.incremental_fields)
params: dict[str, Any] = {}
if config.stable_sort_field is not None:
# The sort field must match the filtered column so rows arrive in watermark order.
sort_field = _resolve_cursor_field(config, incremental_field_name) if use_incremental else None
params["sort"] = f"{sort_field or config.stable_sort_field}__asc"
endpoint_config: Endpoint = {
"path": config.path_template.format(api_version=api_version),
"params": params,
"data_selector": config.data_selector,
# The wrapper key is documented on every list response, so a body without it means the
# API shape changed — fail loud rather than silently syncing 0 rows. A wholly empty
# body still counts as a valid 0-row page: it isn't documented whether a shop with no
# keywords gets `{"keywords": []}` or `{}`, and failing that shop's sync would be worse
# than accepting the empty page.
"data_selector_required": True,
"data_selector_empty_ok": True,
"paginator": _page_paginator() if config.paginated else SinglePagePaginator(),
}
if use_incremental:
endpoint_config["incremental"] = _postscript_incremental_window(
_resolve_cursor_field(config, incremental_field_name)
)
return {
"name": config.name,
"table_name": config.name,
"write_disposition": {"disposition": "merge", "strategy": "upsert"} if use_incremental else "replace",
"endpoint": endpoint_config,
"table_format": "delta",
}
def validate_credentials(api_key: str, api_version: str) -> tuple[bool, str | None]:
response = make_tracked_session(redact_values=(api_key,), allow_redirects=False).get(
f"{POSTSCRIPT_BASE_URL}/api/{api_version}/subscribers",
headers={"Authorization": f"Bearer {api_key}", "Accept": "application/json"},
params={"page": 1},
timeout=10,
)
if response.status_code == 200:
return True, None
if response.status_code == 401:
return False, "Invalid Postscript API key"
if response.status_code == 403:
return (
False,
"Postscript rejected the API key. Use a shop Private API Key from Settings > "
"Integrations > API in your Postscript account. Partner keys are not supported.",
)
return False, f"Postscript API returned an unexpected status code: {response.status_code}"
def postscript_source(
api_key: str,
endpoint: str,
api_version: str,
team_id: int,
job_id: str,
resumable_source_manager: Optional[ResumableSourceManager[PostscriptResumeConfig]] = None,
should_use_incremental_field: bool = False,
db_incremental_field_last_value: Optional[Any] = None,
incremental_field: str | None = None,
) -> SourceResponse:
endpoint_config = POSTSCRIPT_ENDPOINTS[endpoint]
config: RESTAPIConfig = {
"client": _rest_api_client_config(api_key),
"resource_defaults": {},
"resources": [get_resource(endpoint, api_version, should_use_incremental_field, incremental_field)],
}
initial_paginator_state: Optional[dict[str, Any]] = None
if resumable_source_manager is not None and resumable_source_manager.can_resume():
resume_config = resumable_source_manager.load_state()
if resume_config is not None:
initial_paginator_state = {"page": resume_config.page}
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
# Persist only while there is another page to resume to; the Redis TTL cleans up on
# completion.
if resumable_source_manager is None or not state or state.get("page") is None:
return
resumable_source_manager.save_state(PostscriptResumeConfig(page=int(state["page"])))
resource = rest_api_resource(
config,
team_id,
job_id,
db_incremental_field_last_value,
resume_hook=save_checkpoint if resumable_source_manager is not None else None,
initial_paginator_state=initial_paginator_state,
)
return SourceResponse(
name=endpoint_config.name,
items=lambda: resource,
primary_keys=endpoint_config.primary_key,
# Every paginated request pins `sort=<field>__asc`, so rows arrive oldest-first.
sort_mode="asc",
partition_count=1,
partition_size=1,
partition_mode="datetime" if endpoint_config.partition_key else None,
partition_format="month" if endpoint_config.partition_key else None,
partition_keys=[endpoint_config.partition_key] if endpoint_config.partition_key else None,
)