Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

6.6 KB
Raw Blame
import json
from collections.abc import Callable
from datetime import UTC, date, datetime
from typing import Any, Optional, cast

from products.warehouse_sources.sources.sdk import (
    EndpointResource,
    OffsetPaginator,
    Resource,
    RESTAPIConfig,
    ResumableSourceManager,
    frozen,
    make_tracked_session,
    rest_api_resource,
    validate_via_probe,
)
from products.warehouse_sources.sources.zero.settings import ENDPOINT_CONFIGS, ZERO_BASE_URL

PAGE_LIMIT = 100


@frozen
class ZeroResumeConfig:
    offset: int


def _format_zero_datetime(value: Any) -> str:
    """Format an incremental cursor as the ISO 8601 string Zero's `$gt` date operator expects."""
    if isinstance(value, datetime):
        dt = value if value.tzinfo else value.replace(tzinfo=UTC)
        return dt.astimezone(UTC).isoformat()
    if isinstance(value, date):
        return datetime.combine(value, datetime.min.time(), tzinfo=UTC).isoformat()
    return str(value)


def _build_where(workspace_id: Optional[str], date_field: Optional[str]) -> Callable[[Any], str]:
    """Build the `convert` callable for the declarative incremental `where` param.

    Zero's `where` filter is a single JSON-encoded object, so the static workspace scope and the
    dynamic incremental cursor have to be merged into one value here rather than sent as two
    separate query params.
    """

    def convert(last_value: Any) -> str:
        where: dict[str, Any] = {}
        if workspace_id is not None:
            where["workspaceId"] = workspace_id
        if date_field is not None and last_value is not None:
            where[date_field] = {"$gt": _format_zero_datetime(last_value)}
        return json.dumps(where)

    return convert


def get_resource(
    endpoint: str,
    workspace_id: str,
    should_use_incremental_field: bool,
    incremental_field: Optional[str],
) -> EndpointResource:
    config = ENDPOINT_CONFIGS[endpoint]
    is_incremental = should_use_incremental_field and bool(config.incremental_fields)
    scoped_workspace_id = workspace_id if config.scoped_by_workspace else None

    # Stable, ascending sort so offset pagination doesn't skip or duplicate rows as new records
    # are created mid-sync — createdAt never changes, so new rows always land after the current
    # offset. Incremental syncs instead sort on the cursor field, matching SourceResponse.sort_mode.
    sort_field = "createdAt"
    params: dict[str, Any] = {}

    if is_incremental:
        advertised = {incremental["field"] for incremental in config.incremental_fields}
        cursor = (
            incremental_field
            if incremental_field is not None and incremental_field in advertised
            else config.incremental_fields[0]["field"]
        )
        sort_field = cursor
        params["where"] = {
            "type": "incremental",
            "cursor_path": cursor,
            "initial_value": None,
            "convert": _build_where(scoped_workspace_id, cursor),
        }
    elif scoped_workspace_id is not None:
        params["where"] = json.dumps({"workspaceId": scoped_workspace_id})

    params["orderBy"] = json.dumps({sort_field: "asc"})

    return {
        "name": config.name,
        "table_name": config.table_name,
        "primary_key": ["id"],
        "write_disposition": {"disposition": "merge", "strategy": "upsert"} if is_incremental else "replace",
        "endpoint": {
            "path": config.path,
            "data_selector": "data",
            "data_selector_required": True,
            "params": params,
            "paginator": OffsetPaginator(limit=PAGE_LIMIT, total_path="total"),
        },
        "table_format": "delta",
    }


def resolve_workspace_id(api_key: str) -> str:
    """Resolve the workspace this API key belongs to.

    Most Zero resources are workspace-scoped and require an explicit `where={"workspaceId": ...}`
    filter, so the sync resolves the workspace once up front. `GET /api/workspaces` returns every
    workspace the key can see, unfiltered — for a key with access to more than one, taking the
    first result would pick an unintended workspace boundary silently, so this fails loud instead
    and asks for a key scoped to a single workspace.
    """
    session = make_tracked_session(redact_values=(api_key,))
    response = session.get(
        f"{ZERO_BASE_URL}/api/workspaces",
        headers={"Authorization": f"Bearer {api_key}"},
        params={"fields": "id"},
        timeout=10,
    )
    response.raise_for_status()
    data = response.json().get("data") or []
    if not data:
        raise ValueError("This Zero API key isn't a member of any workspace.")
    if len(data) > 1:
        raise ValueError(
            "This Zero API key has access to multiple workspaces. Use an API key scoped to a "
            "single workspace so PostHog imports data from the intended workspace only."
        )
    return cast(str, data[0]["id"])


def validate_credentials(api_key: str) -> bool:
    ok, _status = validate_via_probe(
        lambda: make_tracked_session(redact_values=(api_key,)),
        f"{ZERO_BASE_URL}/api/workspaces",
        headers={"Authorization": f"Bearer {api_key}"},
    )
    return ok


def zero_source(
    api_key: str,
    endpoint: str,
    team_id: int,
    job_id: str,
    resumable_source_manager: ResumableSourceManager[ZeroResumeConfig],
    db_incremental_field_last_value: Optional[Any],
    incremental_field: Optional[str] = None,
    should_use_incremental_field: bool = False,
) -> Resource:
    workspace_id = resolve_workspace_id(api_key)

    config: RESTAPIConfig = {
        "client": {
            "base_url": ZERO_BASE_URL,
            "auth": {"type": "bearer", "token": api_key},
        },
        "resource_defaults": {
            "write_disposition": {"disposition": "merge", "strategy": "upsert"}
            if should_use_incremental_field
            else "replace",
        },
        "resources": [get_resource(endpoint, workspace_id, should_use_incremental_field, incremental_field)],
    }

    initial_paginator_state: Optional[dict[str, Any]] = None
    if resumable_source_manager.can_resume():
        resume_config = resumable_source_manager.load_state()
        if resume_config is not None:
            initial_paginator_state = {"offset": resume_config.offset}

    def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
        if state and "offset" in state:
            resumable_source_manager.save_state(ZeroResumeConfig(offset=int(state["offset"])))

    return rest_api_resource(
        config,
        team_id,
        job_id,
        db_incremental_field_last_value,
        resume_hook=save_checkpoint,
        initial_paginator_state=initial_paginator_state,
    )