Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

14.6 KB
Raw Blame
import re
import dataclasses
from collections.abc import Callable, Iterable
from typing import Any, Optional, cast

from requests import Response

from products.warehouse_sources.sources.cloudsmith.settings import (
    CLOUDSMITH_BASE_URL,
    CLOUDSMITH_ENDPOINTS,
    PAGE_TOTAL_HEADER,
    CloudsmithEndpointConfig,
)
from products.warehouse_sources.sources.sdk import (
    BasePaginator,
    ClientConfig,
    Endpoint,
    EndpointResource,
    IncrementalConfig,
    PageNumberPaginator,
    RESTAPIConfig,
    ResumableSourceManager,
    SinglePagePaginator,
    SourceResponse,
    build_chained_resource,
    build_dependent_resource,
    coerce_datetime_to_utc,
    make_tracked_session,
    rest_api_resource,
)

# Cloudsmith workspace (namespace) slugs, per the API schema's slug pattern. Enforced before the
# slug is interpolated into a request path.
WORKSPACE_SLUG_RE = re.compile(r"^[-a-zA-Z0-9_]+$")

INVALID_WORKSPACE_ERROR = "Enter a valid Cloudsmith workspace slug (letters, numbers, dashes and underscores only)."

# The lower bound sent on a first incremental sync: old enough to select every package, and a
# value the `uploaded` filter parses (a bare `None` would be rejected as an unknown expression).
INCREMENTAL_INITIAL_VALUE = "1970-01-01T00:00:00Z"


@dataclasses.dataclass(frozen=False)
class CloudsmithResumeConfig:
    # Opaque framework checkpoint (the page number for top-level endpoints, per-parent fan-out
    # state for repository-scoped ones), round-tripped into `initial_paginator_state` on resume.
    paginator_state: dict[str, Any]


class CloudsmithPaginator(PageNumberPaginator):
    """Page-number paginator that stops on Cloudsmith's page-count response header.

    Cloudsmith reports the number of pages in `X-Pagination-PageTotal` rather than the response
    body, and answers a page past the last one with a 404 rather than an empty page - so the
    inherited empty-page check would only fire after a request that has already failed.
    """

    def __init__(self, page_size: int, page: int = 1) -> None:
        super().__init__(base_page=1, page=page, page_param="page")
        self.page_size = page_size

    def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:
        if not data:
            self._has_next_page = False
            return

        self.page += 1

        raw_page_total = response.headers.get(PAGE_TOTAL_HEADER)
        if raw_page_total is not None:
            try:
                self._has_next_page = self.page <= int(raw_page_total)
                return
            except ValueError:
                pass

        # Header absent or unparseable: fall back to stopping on a short page, since asking for
        # the page after the last one is an error rather than an empty page.
        self._has_next_page = len(data) >= self.page_size

    def __str__(self) -> str:
        return f"CloudsmithPaginator(page={self.page}, page_size={self.page_size})"


def _format_uploaded_filter(value: Any) -> str:
    """Build the `query` search expression that bounds a package list by upload time.

    Truncates to whole seconds, which rounds the lower bound *down* - so a sync re-reads at most
    a few boundary packages (the merge dedupes them) rather than skipping any package uploaded in
    the same second as the watermark.
    """
    normalized_value = coerce_datetime_to_utc(value)
    formatted = normalized_value.strftime("%Y-%m-%dT%H:%M:%SZ") if normalized_value is not None else str(value)
    return f"uploaded:>={formatted}"


def _cloudsmith_incremental_window(cursor_path: str) -> IncrementalConfig:
    # `query` is a server-side search filter that applies to every page, so pagination stays
    # bounded by the filtered set and terminates at the watermark.
    return {
        "cursor_path": cursor_path,
        "start_param": "query",
        "initial_value": INCREMENTAL_INITIAL_VALUE,
        "convert": _format_uploaded_filter,
    }


def _drop_fields(fields: tuple[str, ...]) -> Callable[[dict[str, Any]], dict[str, Any]]:
    def _mapper(row: dict[str, Any]) -> dict[str, Any]:
        for name in fields:
            row.pop(name, None)
        return row

    return _mapper


def _client_config(api_key: str) -> ClientConfig:
    return {
        "base_url": CLOUDSMITH_BASE_URL,
        "headers": {"Accept": "application/json"},
        # Cloudsmith API keys authenticate via the `X-Api-Key` header (HTTP basic also works, but
        # the header is the documented default and keeps the key out of the URL userinfo).
        "auth": {"type": "api_key", "name": "X-Api-Key", "api_key": api_key, "location": "header"},
        # `capture=False` keeps raw responses out of HTTP sample storage: member emails, audit-log
        # IP addresses and webhook targets ride in these bodies, and the name-based scrubber can't
        # recognise them. Traffic is still metered and logged, with the API key redacted.
        "session": make_tracked_session(redact_values=(api_key,), capture=False),
    }


def validate_credentials(api_key: str, workspace: str, schema_name: Optional[str] = None) -> tuple[bool, str | None]:
    """Confirm the API key is genuine and the workspace is reachable with one cheap probe.

    `GET /namespaces/{slug}/` validates both halves of the config at once: a bad key is a 401 and
    a workspace the key cannot see is a 404.
    """
    if not WORKSPACE_SLUG_RE.match(workspace):
        return False, INVALID_WORKSPACE_ERROR

    response = make_tracked_session(redact_values=(api_key,), capture=False).get(
        f"{CLOUDSMITH_BASE_URL}/namespaces/{workspace}/",
        headers={"X-Api-Key": api_key, "Accept": "application/json"},
        timeout=10,
    )
    if response.status_code == 200:
        return True, None
    if response.status_code == 401:
        return False, "Cloudsmith rejected the API key. Check the key in your Cloudsmith user settings and try again."
    if response.status_code == 403:
        # A Cloudsmith API key inherits its owner's permissions, which vary per repository. Let
        # source creation through so the user can still sync the tables they do have access to,
        # and only fail when a specific table is being checked.
        if schema_name is None:
            return True, None
        return False, "Your Cloudsmith API key does not have permission to read this data."
    if response.status_code == 404:
        return False, f"Cloudsmith workspace '{workspace}' was not found, or this API key cannot see it."
    return False, f"Cloudsmith returned an unexpected response (HTTP {response.status_code})."


def _paginator(config: CloudsmithEndpointConfig) -> BasePaginator:
    return CloudsmithPaginator(page_size=config.page_size) if config.paginated else SinglePagePaginator()


def _endpoint(
    config: CloudsmithEndpointConfig, workspace: str, extra_params: Optional[dict[str, Any]] = None
) -> Endpoint:
    params: dict[str, Any] = {"page_size": config.page_size} if config.paginated else {}
    params.update(config.params)
    params.update(extra_params or {})
    endpoint: Endpoint = {
        "path": config.path.replace("{owner}", workspace),
        "params": params,
        "paginator": _paginator(config),
        # Every Cloudsmith list endpoint returns a bare JSON array or an object with a known rows
        # key; anything else is a shape change we would rather fail on than silently sync as a
        # single row.
        "data_selector_required": True,
    }
    if config.data_selector:
        endpoint["data_selector"] = config.data_selector
    return endpoint


def get_resource(config: CloudsmithEndpointConfig, workspace: str) -> EndpointResource:
    if config.fanout or config.chained_fanout:
        raise ValueError(f"Fan-out endpoint '{config.name}' must use the fan-out path")

    endpoint_config = _endpoint(config, workspace)

    resource: EndpointResource = {
        "name": config.name,
        "table_name": config.name,
        "write_disposition": "replace",
        "endpoint": endpoint_config,
        "table_format": "delta",
    }
    if config.strip_fields:
        resource["data_map"] = _drop_fields(config.strip_fields)
    return resource


def _chained_resource(
    config: CloudsmithEndpointConfig, api_key: str, workspace: str, team_id: int, job_id: str
) -> Iterable[Any]:
    """Build the repositories -> parent -> child chain for a second-level fan-out endpoint."""
    chained = config.chained_fanout
    assert chained is not None
    middle_config = CLOUDSMITH_ENDPOINTS[chained.parent_name]
    root_fanout = middle_config.fanout
    if root_fanout is None:
        raise ValueError(f"'{chained.parent_name}' does not fan out from repositories")
    root_config = CLOUDSMITH_ENDPOINTS[root_fanout.parent_name]

    root_endpoint = _endpoint(root_config, workspace, root_fanout.parent_params)
    middle_endpoint = _endpoint(
        middle_config,
        workspace,
        {
            root_fanout.resolve_param: {
                "type": "resolve",
                "resource": root_config.name,
                "field": root_fanout.resolve_field,
            }
        },
    )
    child_endpoint = _endpoint(
        config,
        workspace,
        {
            param: {"type": "resolve", "resource": middle_config.name, "field": field_name}
            for param, field_name in chained.resolve_fields.items()
        },
    )
    # A package deleted between its packages page and this fetch must not sink the run.
    child_endpoint["response_actions"] = [{"status_code": 404, "action": "ignore"}]

    root_resource: EndpointResource = {
        "name": root_config.name,
        "table_name": root_config.name,
        "write_disposition": "replace",
        "endpoint": root_endpoint,
        "table_format": "delta",
    }
    middle_resource: EndpointResource = {
        "name": middle_config.name,
        "table_name": middle_config.name,
        "write_disposition": "replace",
        "include_from_parent": root_fanout.include_from_parent,
        "endpoint": middle_endpoint,
        "table_format": "delta",
    }
    child_resource: EndpointResource = {
        "name": config.name,
        "table_name": config.name,
        "write_disposition": "replace",
        "include_from_parent": chained.include_from_parent,
        "endpoint": child_endpoint,
        "table_format": "delta",
    }

    return build_chained_resource(
        resources=[root_resource, middle_resource, child_resource],
        child_name=config.name,
        parent_name=middle_config.name,
        parent_field_renames=chained.parent_field_renames,
        client_config=_client_config(api_key),
        team_id=team_id,
        job_id=job_id,
    )


def _make_source_response(
    config: CloudsmithEndpointConfig, items_fn: Callable[[], Any], supports_resume: bool = True
) -> SourceResponse:
    return SourceResponse(
        name=config.name,
        items=items_fn,
        supports_resume=supports_resume,
        primary_keys=config.primary_key if isinstance(config.primary_key, list) else [config.primary_key],
        sort_mode=config.sort_mode,
        partition_count=1,
        partition_size=1,
        partition_mode="datetime" if config.partition_key else None,
        partition_format="month" if config.partition_key else None,
        partition_keys=[config.partition_key] if config.partition_key else None,
    )


def cloudsmith_source(
    api_key: str,
    workspace: str,
    endpoint: str,
    team_id: int,
    job_id: str,
    resumable_source_manager: ResumableSourceManager[CloudsmithResumeConfig],
    should_use_incremental_field: bool = False,
    db_incremental_field_last_value: Optional[Any] = None,
    incremental_field: str | None = None,
) -> SourceResponse:
    if not WORKSPACE_SLUG_RE.match(workspace):
        raise ValueError(INVALID_WORKSPACE_ERROR)

    endpoint_config = CLOUDSMITH_ENDPOINTS[endpoint]

    initial_paginator_state: Optional[dict[str, Any]] = None
    if resumable_source_manager.can_resume():
        resume_config = resumable_source_manager.load_state()
        if resume_config is not None:
            initial_paginator_state = resume_config.paginator_state

    def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
        # Only persist when there's somewhere to resume to; the Redis TTL handles cleanup.
        if state:
            resumable_source_manager.save_state(CloudsmithResumeConfig(paginator_state=dict(state)))

    if endpoint_config.chained_fanout is not None:
        # A two-level chain saves no checkpoint, so a shutdown cannot hand it off mid-walk.
        chained_items = _chained_resource(endpoint_config, api_key, workspace, team_id, job_id)
        return _make_source_response(endpoint_config, lambda: chained_items, supports_resume=False)

    if endpoint_config.fanout is not None:
        parent_config = CLOUDSMITH_ENDPOINTS[endpoint_config.fanout.parent_name]
        dependent_resource: Any = build_dependent_resource(
            endpoint_configs=CLOUDSMITH_ENDPOINTS,
            child_endpoint=endpoint,
            fanout=endpoint_config.fanout,
            client_config=_client_config(api_key),
            path_format_values={"owner": workspace},
            team_id=team_id,
            job_id=job_id,
            db_incremental_field_last_value=db_incremental_field_last_value,
            should_use_incremental_field=should_use_incremental_field,
            incremental_field=incremental_field,
            incremental_config_factory=_cloudsmith_incremental_window,
            page_size_param="page_size",
            parent_endpoint_extra={
                "paginator": CloudsmithPaginator(page_size=parent_config.page_size),
                "data_selector_required": True,
            },
            child_endpoint_extra={
                "paginator": CloudsmithPaginator(page_size=endpoint_config.page_size),
                "data_selector_required": True,
            },
            child_params_extra=dict(endpoint_config.params) if endpoint_config.params else None,
            resume_hook=save_checkpoint,
            initial_paginator_state=initial_paginator_state,
        )
        if endpoint_config.strip_fields:
            dependent_resource = dependent_resource.add_map(_drop_fields(endpoint_config.strip_fields))
        items = cast(Iterable[Any], dependent_resource)
        return _make_source_response(endpoint_config, lambda: items)

    config: RESTAPIConfig = {
        "client": _client_config(api_key),
        "resource_defaults": {"write_disposition": "replace"},
        "resources": [get_resource(endpoint_config, workspace)],
    }

    resource = rest_api_resource(
        config,
        team_id,
        job_id,
        db_incremental_field_last_value,
        resume_hook=save_checkpoint,
        initial_paginator_state=initial_paginator_state,
    )
    return _make_source_response(endpoint_config, lambda: resource)