:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import dataclasses
from collections.abc import Callable
from datetime import UTC, date, datetime
from typing import Any, Optional, cast
from requests import Request, Response
from products.warehouse_sources.sources.clockify.settings import CLOCKIFY_ENDPOINTS, ClockifyEndpointConfig
from products.warehouse_sources.sources.sdk import (
BasePaginator,
Endpoint,
EndpointResource,
IncrementalConfig,
PageNumberPaginator,
ParamLocation,
Resource,
RESTAPIConfig,
ResumableSourceManager,
SinglePagePaginator,
SourceResponse,
make_tracked_session,
rename_parent_fields,
rest_api_resources,
validate_via_probe,
)
# Global host. Clockify also serves regional hosts (euc1/use2/euw2/apse2); the global host
# resolves to the user's region, so a single base URL works for every key.
CLOCKIFY_BASE_URL = "https://api.clockify.me/api/v1"
# Single-level fan-out endpoints: one request per workspace. Two-level ones (tasks,
# time_entries, invoice_payments) chain a second parent and are handled explicitly below.
_WORKSPACE_CHILD_ENDPOINTS = (
"users",
"clients",
"projects",
"tags",
"custom_fields",
"expenses",
"expense_categories",
"invoices",
"time_off_requests",
"approval_requests",
"user_groups",
)
class ClockifyPageNumberPaginator(PageNumberPaginator):
"""Page/`page-size` pagination over Clockify's bare JSON arrays.
Clockify exposes no total count, so — like the hand-rolled loop this replaces — a page shorter
than the requested size (or an empty page) is the last page. Stopping on a short page avoids the
extra empty-page request the plain ``PageNumberPaginator`` would pay.
"""
def __init__(self, page_size: int, param_location: ParamLocation = "query") -> None:
super().__init__(base_page=1, page_param="page", param_location=param_location)
self._page_size = page_size
def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:
super().update_state(response, data)
if self._has_next_page and data is not None and len(data) < self._page_size:
self._has_next_page = False
class ClockifyTimeOffPaginator(ClockifyPageNumberPaginator):
"""Paging for the time-off listing, whose filters and page fields travel in a POST body.
A fan-out child request gets no static ``json`` body from the framework, so the whole payload is
written here. ``statuses`` has to name every state explicitly — Clockify returns only pending
requests when the filter is left off.
"""
def __init__(self, page_size: int) -> None:
super().__init__(page_size, param_location="json")
def init_request(self, request: Request) -> None:
if request.json is None:
request.json = {}
request.json["pageSize"] = self._page_size
request.json["statuses"] = ["ALL"]
super().init_request(request)
@dataclasses.dataclass(frozen=False)
class ClockifyResumeConfig:
# Opaque paginator/fan-out state handed back by the rest_source framework (paginator page for
# workspaces, per-parent completed-path progress for single-level fan-out). Retained as a single
# blob so the shape can evolve without a state-format migration.
fanout_state: dict[str, Any] | None = None
# Legacy fields from the hand-rolled resume format. Kept (with defaults) so an old saved state
# still parses via ``dataclass(**saved)``; a run resumed from one starts fan-out fresh (a re-read
# the merge dedupes) rather than mis-mapping the old positional scope onto the new state.
workspace_id: str | None = None
parent_id: str | None = None
page: int = 1
def _headers() -> dict[str, str]:
# Auth (the X-Api-Key header) is supplied via the framework auth config so its value is redacted
# from logs; only the non-secret Accept header is set here.
return {"Accept": "application/json"}
def _format_datetime_z(value: Any) -> str:
"""Format a datetime/date as `yyyy-MM-ddThh:mm:ssZ`, the format Clockify's `start` filter wants."""
if isinstance(value, datetime):
dt = value
elif isinstance(value, date):
dt = datetime.combine(value, datetime.min.time())
else:
# Already a string (e.g. an ISO timestamp persisted as the cursor) — pass through.
return str(value)
dt = dt.replace(tzinfo=UTC) if dt.tzinfo is None else dt.astimezone(UTC)
return dt.strftime("%Y-%m-%dT%H:%M:%SZ")
def _clamp_future_value_to_now(value: Any) -> Any:
"""Cap a future datetime cursor at now.
A future-dated `start` filter would silently match nothing and stall the incremental sync.
Asking for entries newer than now is a no-op anyway, so capping keeps the sync self-healing.
"""
now = datetime.now(UTC)
if isinstance(value, datetime):
aware = value if value.tzinfo is not None else value.replace(tzinfo=UTC)
return now if aware > now else value
if isinstance(value, date):
return now.date() if value > now.date() else value
if isinstance(value, str):
# The cursor can reach us as an ISO string depending on how it was persisted/deserialised.
# Parse it so a future-dated string is clamped too; a non-ISO string passes through.
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
return value
aware = parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=UTC)
return now if aware > now else value
return value
def _flatten_approval_request(item: dict[str, Any]) -> dict[str, Any]:
"""Surface the nested `approvalRequest` object as top-level columns.
The row's only identifier lives one level down, so without this the table has no primary key —
and the owner and approved period, which are what the table is joined and filtered on, would
need JSON extraction on every query."""
request = item.get("approvalRequest")
if not isinstance(request, dict):
return item
item["approval_request_id"] = request.get("id")
status = request.get("status")
if isinstance(status, dict):
item["approval_request_state"] = status.get("state")
owner = request.get("owner")
if isinstance(owner, dict):
item["approval_request_owner_user_id"] = owner.get("userId")
date_range = request.get("dateRange")
if isinstance(date_range, dict):
item["approval_request_start"] = date_range.get("start")
item["approval_request_end"] = date_range.get("end")
return item
def _flatten_time_entry(item: dict[str, Any]) -> dict[str, Any]:
"""Surface the nested `timeInterval` object as top-level columns so the interval start can be
used as the incremental cursor and partition key."""
interval = item.get("timeInterval")
if isinstance(interval, dict):
item["time_interval_start"] = interval.get("start")
item["time_interval_end"] = interval.get("end")
item["time_interval_duration"] = interval.get("duration")
return item
# Fan-out injects the parent id under the framework's ``_<parent>_<field>`` naming; rename it back to
# the flat ``workspace_id`` / ``project_id`` / ``user_id`` columns the tables have always exposed.
_rename_workspace = rename_parent_fields("workspaces", {"id": "workspace_id"})
_rename_task_parents = rename_parent_fields("projects", {"workspace_id": "workspace_id", "id": "project_id"})
_rename_time_entry_parents = rename_parent_fields("users", {"workspace_id": "workspace_id", "id": "user_id"})
_rename_invoice_parents = rename_parent_fields("invoices", {"workspace_id": "workspace_id", "id": "invoice_id"})
def _time_entry_map(row: dict[str, Any]) -> dict[str, Any]:
return _rename_time_entry_parents(_flatten_time_entry(row))
def _approval_request_map(row: dict[str, Any]) -> dict[str, Any] | list[dict[str, Any]]:
# Clockify documents `approvalRequest` as nullable. Such a row carries no id at all, so writing
# it would seed a null primary key that every later merge multi-matches; drop it instead.
if not isinstance(row.get("approvalRequest"), dict):
return []
return _rename_workspace(_flatten_approval_request(row))
# Workspace children that need more than the parent-id rename.
_WORKSPACE_CHILD_DATA_MAPS: dict[str, Callable[[dict[str, Any]], dict[str, Any] | list[dict[str, Any]]]] = {
"approval_requests": _approval_request_map,
}
def _incremental_config(
should_use_incremental_field: bool, db_incremental_field_last_value: Any
) -> IncrementalConfig | None:
"""Server-side `start` filter for time_entries, only when we have a cursor to filter on.
Mirrors the old behaviour: no filter on a first (full) sync, and the persisted watermark is
clamped to now and formatted the way Clockify's `start` expects.
"""
if not should_use_incremental_field or db_incremental_field_last_value is None:
return None
return {
"start_param": "start",
"cursor_path": "time_interval_start",
"convert": lambda value: _format_datetime_z(_clamp_future_value_to_now(value)),
}
def _endpoint_spec(config: ClockifyEndpointConfig, resolve_params: dict[str, Any]) -> Endpoint:
params: dict[str, Any] = {**resolve_params, **config.extra_params}
paginator: BasePaginator
if config.method == "POST":
paginator = ClockifyTimeOffPaginator(config.page_size)
elif config.paginated:
params["page-size"] = config.page_size
paginator = ClockifyPageNumberPaginator(config.page_size)
else:
paginator = SinglePagePaginator()
endpoint: Endpoint = {
"path": config.path,
"params": params,
"paginator": paginator,
"data_selector_required": config.data_selector_required,
}
if config.method != "GET":
endpoint["method"] = config.method
if config.data_selector is not None:
endpoint["data_selector"] = config.data_selector
return endpoint
def _workspaces_resource() -> EndpointResource:
return {
"name": "workspaces",
"endpoint": _endpoint_spec(CLOCKIFY_ENDPOINTS["workspaces"], {}),
}
def _workspace_child_resource(endpoint: str) -> EndpointResource:
return {
"name": endpoint,
"include_from_parent": ["id"],
"endpoint": _endpoint_spec(
CLOCKIFY_ENDPOINTS[endpoint],
{"workspace_id": {"type": "resolve", "resource": "workspaces", "field": "id"}},
),
"data_map": _WORKSPACE_CHILD_DATA_MAPS.get(endpoint, _rename_workspace),
}
def _chained_resource(endpoint: str, data_map: Callable[[dict[str, Any]], dict[str, Any]]) -> EndpointResource:
"""A resource fanning out over a workspace child, so the path carries two resolved ids."""
config = CLOCKIFY_ENDPOINTS[endpoint]
parent = config.fan_out_parent
placeholder = config.parent_id_placeholder
if parent is None or placeholder is None:
raise ValueError(f"Clockify endpoint {endpoint} declares no fan-out parent to resolve its path against")
return {
"name": endpoint,
"include_from_parent": ["workspace_id", "id"],
"endpoint": _endpoint_spec(
config,
{
"workspace_id": {"type": "resolve", "resource": parent, "field": "workspace_id"},
placeholder: {"type": "resolve", "resource": parent, "field": "id"},
},
),
"data_map": data_map,
}
def _time_entries_resource(incremental: IncrementalConfig | None) -> EndpointResource:
resource = _chained_resource("time_entries", _time_entry_map)
if incremental is not None:
cast(Endpoint, resource["endpoint"])["incremental"] = incremental
return resource
def _resources_for(endpoint: str, incremental: IncrementalConfig | None) -> list[str | EndpointResource]:
if endpoint == "workspaces":
return [_workspaces_resource()]
if endpoint in _WORKSPACE_CHILD_ENDPOINTS:
return [_workspaces_resource(), _workspace_child_resource(endpoint)]
if endpoint == "tasks":
return [
_workspaces_resource(),
_workspace_child_resource("projects"),
_chained_resource("tasks", _rename_task_parents),
]
if endpoint == "time_entries":
return [_workspaces_resource(), _workspace_child_resource("users"), _time_entries_resource(incremental)]
if endpoint == "invoice_payments":
return [
_workspaces_resource(),
_workspace_child_resource("invoices"),
_chained_resource("invoice_payments", _rename_invoice_parents),
]
raise ValueError(f"Unknown Clockify endpoint: {endpoint}")
def _build_rest_config(
api_key: str,
endpoint: str,
should_use_incremental_field: bool,
db_incremental_field_last_value: Any,
) -> RESTAPIConfig:
incremental = _incremental_config(should_use_incremental_field, db_incremental_field_last_value)
return {
"client": {
"base_url": CLOCKIFY_BASE_URL,
"headers": _headers(),
"auth": {"type": "api_key", "api_key": api_key, "name": "X-Api-Key", "location": "header"},
},
"resource_defaults": {},
"resources": _resources_for(endpoint, incremental),
}
def clockify_source(
api_key: str,
endpoint: str,
team_id: int,
job_id: str,
resumable_source_manager: ResumableSourceManager[ClockifyResumeConfig],
should_use_incremental_field: bool = False,
db_incremental_field_last_value: Optional[Any] = None,
) -> SourceResponse:
config = CLOCKIFY_ENDPOINTS[endpoint]
rest_config = _build_rest_config(api_key, endpoint, should_use_incremental_field, db_incremental_field_last_value)
initial_paginator_state: dict[str, Any] | None = None
if resumable_source_manager.can_resume():
resume = resumable_source_manager.load_state()
if resume is not None and resume.fanout_state is not None:
initial_paginator_state = resume.fanout_state
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
# Save AFTER a page is yielded so a crash re-yields the last page (merge dedupes) rather
# than skipping it. A ``None`` state means no page remains — nothing to persist. Never fires
# for the two-level fan-outs (tasks/time_entries), where the framework disables resume.
if state is not None:
resumable_source_manager.save_state(ClockifyResumeConfig(fanout_state=state))
resources = rest_api_resources(
rest_config,
team_id,
job_id,
db_incremental_field_last_value,
resume_hook=save_checkpoint,
initial_paginator_state=initial_paginator_state,
)
resource: Resource = next(r for r in resources if r.name == endpoint)
return SourceResponse(
name=endpoint,
items=lambda: resource,
primary_keys=config.primary_keys,
sort_mode=config.sort_mode,
partition_count=1,
partition_size=1,
partition_mode="datetime" if config.partition_key else None,
partition_format="week" if config.partition_key else None,
partition_keys=[config.partition_key] if config.partition_key else None,
)
def validate_credentials(api_key: str) -> bool:
"""A Clockify API key is user-scoped (no per-endpoint scopes), so one cheap `/user` probe
confirms the key is genuine for everything it can reach."""
ok, _status = validate_via_probe(
lambda: make_tracked_session(redact_values=(api_key,)),
f"{CLOCKIFY_BASE_URL}/user",
headers={"X-Api-Key": api_key, **_headers()},
)
return ok