Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

15.4 KB
Raw Blame
import dataclasses
from collections.abc import Callable
from datetime import UTC, date, datetime
from typing import Any, Optional, cast

from requests import Request, Response

from products.warehouse_sources.sources.clockify.settings import CLOCKIFY_ENDPOINTS, ClockifyEndpointConfig
from products.warehouse_sources.sources.sdk import (
    BasePaginator,
    Endpoint,
    EndpointResource,
    IncrementalConfig,
    PageNumberPaginator,
    ParamLocation,
    Resource,
    RESTAPIConfig,
    ResumableSourceManager,
    SinglePagePaginator,
    SourceResponse,
    make_tracked_session,
    rename_parent_fields,
    rest_api_resources,
    validate_via_probe,
)

# Global host. Clockify also serves regional hosts (euc1/use2/euw2/apse2); the global host
# resolves to the user's region, so a single base URL works for every key.
CLOCKIFY_BASE_URL = "https://api.clockify.me/api/v1"

# Single-level fan-out endpoints: one request per workspace. Two-level ones (tasks,
# time_entries, invoice_payments) chain a second parent and are handled explicitly below.
_WORKSPACE_CHILD_ENDPOINTS = (
    "users",
    "clients",
    "projects",
    "tags",
    "custom_fields",
    "expenses",
    "expense_categories",
    "invoices",
    "time_off_requests",
    "approval_requests",
    "user_groups",
)


class ClockifyPageNumberPaginator(PageNumberPaginator):
    """Page/`page-size` pagination over Clockify's bare JSON arrays.

    Clockify exposes no total count, so — like the hand-rolled loop this replaces — a page shorter
    than the requested size (or an empty page) is the last page. Stopping on a short page avoids the
    extra empty-page request the plain ``PageNumberPaginator`` would pay.
    """

    def __init__(self, page_size: int, param_location: ParamLocation = "query") -> None:
        super().__init__(base_page=1, page_param="page", param_location=param_location)
        self._page_size = page_size

    def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:
        super().update_state(response, data)
        if self._has_next_page and data is not None and len(data) < self._page_size:
            self._has_next_page = False


class ClockifyTimeOffPaginator(ClockifyPageNumberPaginator):
    """Paging for the time-off listing, whose filters and page fields travel in a POST body.

    A fan-out child request gets no static ``json`` body from the framework, so the whole payload is
    written here. ``statuses`` has to name every state explicitly — Clockify returns only pending
    requests when the filter is left off.
    """

    def __init__(self, page_size: int) -> None:
        super().__init__(page_size, param_location="json")

    def init_request(self, request: Request) -> None:
        if request.json is None:
            request.json = {}
        request.json["pageSize"] = self._page_size
        request.json["statuses"] = ["ALL"]
        super().init_request(request)


@dataclasses.dataclass(frozen=False)
class ClockifyResumeConfig:
    # Opaque paginator/fan-out state handed back by the rest_source framework (paginator page for
    # workspaces, per-parent completed-path progress for single-level fan-out). Retained as a single
    # blob so the shape can evolve without a state-format migration.
    fanout_state: dict[str, Any] | None = None
    # Legacy fields from the hand-rolled resume format. Kept (with defaults) so an old saved state
    # still parses via ``dataclass(**saved)``; a run resumed from one starts fan-out fresh (a re-read
    # the merge dedupes) rather than mis-mapping the old positional scope onto the new state.
    workspace_id: str | None = None
    parent_id: str | None = None
    page: int = 1


def _headers() -> dict[str, str]:
    # Auth (the X-Api-Key header) is supplied via the framework auth config so its value is redacted
    # from logs; only the non-secret Accept header is set here.
    return {"Accept": "application/json"}


def _format_datetime_z(value: Any) -> str:
    """Format a datetime/date as `yyyy-MM-ddThh:mm:ssZ`, the format Clockify's `start` filter wants."""
    if isinstance(value, datetime):
        dt = value
    elif isinstance(value, date):
        dt = datetime.combine(value, datetime.min.time())
    else:
        # Already a string (e.g. an ISO timestamp persisted as the cursor) — pass through.
        return str(value)
    dt = dt.replace(tzinfo=UTC) if dt.tzinfo is None else dt.astimezone(UTC)
    return dt.strftime("%Y-%m-%dT%H:%M:%SZ")


def _clamp_future_value_to_now(value: Any) -> Any:
    """Cap a future datetime cursor at now.

    A future-dated `start` filter would silently match nothing and stall the incremental sync.
    Asking for entries newer than now is a no-op anyway, so capping keeps the sync self-healing.
    """
    now = datetime.now(UTC)
    if isinstance(value, datetime):
        aware = value if value.tzinfo is not None else value.replace(tzinfo=UTC)
        return now if aware > now else value
    if isinstance(value, date):
        return now.date() if value > now.date() else value
    if isinstance(value, str):
        # The cursor can reach us as an ISO string depending on how it was persisted/deserialised.
        # Parse it so a future-dated string is clamped too; a non-ISO string passes through.
        try:
            parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
        except ValueError:
            return value
        aware = parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=UTC)
        return now if aware > now else value
    return value


def _flatten_approval_request(item: dict[str, Any]) -> dict[str, Any]:
    """Surface the nested `approvalRequest` object as top-level columns.

    The row's only identifier lives one level down, so without this the table has no primary key —
    and the owner and approved period, which are what the table is joined and filtered on, would
    need JSON extraction on every query."""
    request = item.get("approvalRequest")
    if not isinstance(request, dict):
        return item
    item["approval_request_id"] = request.get("id")
    status = request.get("status")
    if isinstance(status, dict):
        item["approval_request_state"] = status.get("state")
    owner = request.get("owner")
    if isinstance(owner, dict):
        item["approval_request_owner_user_id"] = owner.get("userId")
    date_range = request.get("dateRange")
    if isinstance(date_range, dict):
        item["approval_request_start"] = date_range.get("start")
        item["approval_request_end"] = date_range.get("end")
    return item


def _flatten_time_entry(item: dict[str, Any]) -> dict[str, Any]:
    """Surface the nested `timeInterval` object as top-level columns so the interval start can be
    used as the incremental cursor and partition key."""
    interval = item.get("timeInterval")
    if isinstance(interval, dict):
        item["time_interval_start"] = interval.get("start")
        item["time_interval_end"] = interval.get("end")
        item["time_interval_duration"] = interval.get("duration")
    return item


# Fan-out injects the parent id under the framework's ``_<parent>_<field>`` naming; rename it back to
# the flat ``workspace_id`` / ``project_id`` / ``user_id`` columns the tables have always exposed.
_rename_workspace = rename_parent_fields("workspaces", {"id": "workspace_id"})
_rename_task_parents = rename_parent_fields("projects", {"workspace_id": "workspace_id", "id": "project_id"})
_rename_time_entry_parents = rename_parent_fields("users", {"workspace_id": "workspace_id", "id": "user_id"})
_rename_invoice_parents = rename_parent_fields("invoices", {"workspace_id": "workspace_id", "id": "invoice_id"})


def _time_entry_map(row: dict[str, Any]) -> dict[str, Any]:
    return _rename_time_entry_parents(_flatten_time_entry(row))


def _approval_request_map(row: dict[str, Any]) -> dict[str, Any] | list[dict[str, Any]]:
    # Clockify documents `approvalRequest` as nullable. Such a row carries no id at all, so writing
    # it would seed a null primary key that every later merge multi-matches; drop it instead.
    if not isinstance(row.get("approvalRequest"), dict):
        return []
    return _rename_workspace(_flatten_approval_request(row))


# Workspace children that need more than the parent-id rename.
_WORKSPACE_CHILD_DATA_MAPS: dict[str, Callable[[dict[str, Any]], dict[str, Any] | list[dict[str, Any]]]] = {
    "approval_requests": _approval_request_map,
}


def _incremental_config(
    should_use_incremental_field: bool, db_incremental_field_last_value: Any
) -> IncrementalConfig | None:
    """Server-side `start` filter for time_entries, only when we have a cursor to filter on.

    Mirrors the old behaviour: no filter on a first (full) sync, and the persisted watermark is
    clamped to now and formatted the way Clockify's `start` expects.
    """
    if not should_use_incremental_field or db_incremental_field_last_value is None:
        return None
    return {
        "start_param": "start",
        "cursor_path": "time_interval_start",
        "convert": lambda value: _format_datetime_z(_clamp_future_value_to_now(value)),
    }


def _endpoint_spec(config: ClockifyEndpointConfig, resolve_params: dict[str, Any]) -> Endpoint:
    params: dict[str, Any] = {**resolve_params, **config.extra_params}
    paginator: BasePaginator
    if config.method == "POST":
        paginator = ClockifyTimeOffPaginator(config.page_size)
    elif config.paginated:
        params["page-size"] = config.page_size
        paginator = ClockifyPageNumberPaginator(config.page_size)
    else:
        paginator = SinglePagePaginator()

    endpoint: Endpoint = {
        "path": config.path,
        "params": params,
        "paginator": paginator,
        "data_selector_required": config.data_selector_required,
    }
    if config.method != "GET":
        endpoint["method"] = config.method
    if config.data_selector is not None:
        endpoint["data_selector"] = config.data_selector
    return endpoint


def _workspaces_resource() -> EndpointResource:
    return {
        "name": "workspaces",
        "endpoint": _endpoint_spec(CLOCKIFY_ENDPOINTS["workspaces"], {}),
    }


def _workspace_child_resource(endpoint: str) -> EndpointResource:
    return {
        "name": endpoint,
        "include_from_parent": ["id"],
        "endpoint": _endpoint_spec(
            CLOCKIFY_ENDPOINTS[endpoint],
            {"workspace_id": {"type": "resolve", "resource": "workspaces", "field": "id"}},
        ),
        "data_map": _WORKSPACE_CHILD_DATA_MAPS.get(endpoint, _rename_workspace),
    }


def _chained_resource(endpoint: str, data_map: Callable[[dict[str, Any]], dict[str, Any]]) -> EndpointResource:
    """A resource fanning out over a workspace child, so the path carries two resolved ids."""
    config = CLOCKIFY_ENDPOINTS[endpoint]
    parent = config.fan_out_parent
    placeholder = config.parent_id_placeholder
    if parent is None or placeholder is None:
        raise ValueError(f"Clockify endpoint {endpoint} declares no fan-out parent to resolve its path against")
    return {
        "name": endpoint,
        "include_from_parent": ["workspace_id", "id"],
        "endpoint": _endpoint_spec(
            config,
            {
                "workspace_id": {"type": "resolve", "resource": parent, "field": "workspace_id"},
                placeholder: {"type": "resolve", "resource": parent, "field": "id"},
            },
        ),
        "data_map": data_map,
    }


def _time_entries_resource(incremental: IncrementalConfig | None) -> EndpointResource:
    resource = _chained_resource("time_entries", _time_entry_map)
    if incremental is not None:
        cast(Endpoint, resource["endpoint"])["incremental"] = incremental
    return resource


def _resources_for(endpoint: str, incremental: IncrementalConfig | None) -> list[str | EndpointResource]:
    if endpoint == "workspaces":
        return [_workspaces_resource()]
    if endpoint in _WORKSPACE_CHILD_ENDPOINTS:
        return [_workspaces_resource(), _workspace_child_resource(endpoint)]
    if endpoint == "tasks":
        return [
            _workspaces_resource(),
            _workspace_child_resource("projects"),
            _chained_resource("tasks", _rename_task_parents),
        ]
    if endpoint == "time_entries":
        return [_workspaces_resource(), _workspace_child_resource("users"), _time_entries_resource(incremental)]
    if endpoint == "invoice_payments":
        return [
            _workspaces_resource(),
            _workspace_child_resource("invoices"),
            _chained_resource("invoice_payments", _rename_invoice_parents),
        ]
    raise ValueError(f"Unknown Clockify endpoint: {endpoint}")


def _build_rest_config(
    api_key: str,
    endpoint: str,
    should_use_incremental_field: bool,
    db_incremental_field_last_value: Any,
) -> RESTAPIConfig:
    incremental = _incremental_config(should_use_incremental_field, db_incremental_field_last_value)
    return {
        "client": {
            "base_url": CLOCKIFY_BASE_URL,
            "headers": _headers(),
            "auth": {"type": "api_key", "api_key": api_key, "name": "X-Api-Key", "location": "header"},
        },
        "resource_defaults": {},
        "resources": _resources_for(endpoint, incremental),
    }


def clockify_source(
    api_key: str,
    endpoint: str,
    team_id: int,
    job_id: str,
    resumable_source_manager: ResumableSourceManager[ClockifyResumeConfig],
    should_use_incremental_field: bool = False,
    db_incremental_field_last_value: Optional[Any] = None,
) -> SourceResponse:
    config = CLOCKIFY_ENDPOINTS[endpoint]
    rest_config = _build_rest_config(api_key, endpoint, should_use_incremental_field, db_incremental_field_last_value)

    initial_paginator_state: dict[str, Any] | None = None
    if resumable_source_manager.can_resume():
        resume = resumable_source_manager.load_state()
        if resume is not None and resume.fanout_state is not None:
            initial_paginator_state = resume.fanout_state

    def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
        # Save AFTER a page is yielded so a crash re-yields the last page (merge dedupes) rather
        # than skipping it. A ``None`` state means no page remains — nothing to persist. Never fires
        # for the two-level fan-outs (tasks/time_entries), where the framework disables resume.
        if state is not None:
            resumable_source_manager.save_state(ClockifyResumeConfig(fanout_state=state))

    resources = rest_api_resources(
        rest_config,
        team_id,
        job_id,
        db_incremental_field_last_value,
        resume_hook=save_checkpoint,
        initial_paginator_state=initial_paginator_state,
    )
    resource: Resource = next(r for r in resources if r.name == endpoint)

    return SourceResponse(
        name=endpoint,
        items=lambda: resource,
        primary_keys=config.primary_keys,
        sort_mode=config.sort_mode,
        partition_count=1,
        partition_size=1,
        partition_mode="datetime" if config.partition_key else None,
        partition_format="week" if config.partition_key else None,
        partition_keys=[config.partition_key] if config.partition_key else None,
    )


def validate_credentials(api_key: str) -> bool:
    """A Clockify API key is user-scoped (no per-endpoint scopes), so one cheap `/user` probe
    confirms the key is genuine for everything it can reach."""
    ok, _status = validate_via_probe(
        lambda: make_tracked_session(redact_values=(api_key,)),
        f"{CLOCKIFY_BASE_URL}/user",
        headers={"X-Api-Key": api_key, **_headers()},
    )
    return ok