Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

6.1 KB
Raw Blame
from typing import Any

from requests import Response

from products.warehouse_sources.sources.rudderstack._config import RudderStackSourceConfig
from products.warehouse_sources.sources.rudderstack.settings import (
    BASE_URLS,
    CATALOG_MAX_PAGES,
    ENDPOINTS,
    RudderStackEndpoint,
)
from products.warehouse_sources.sources.sdk import (
    BearerTokenAuth,
    ClientConfig,
    Endpoint,
    PageNumberPaginator,
    RESTAPIConfig,
    ResumableSourceManager,
    SourceInputs,
    SourceResponse,
    build_dependent_resource,
    frozen,
    make_tracked_session,
    rest_api_resource,
    schema_for_resource,
)


@frozen
class RudderStackResumeConfig:
    paginator_state: dict[str, Any]


class RudderStackCatalogLimitError(ValueError):
    pass


class RudderStackCatalogPaginator(PageNumberPaginator):
    def __init__(self) -> None:
        super().__init__(base_page=1)

    def update_state(self, response: Response, data: list[Any] | None = None) -> None:
        payload = response.json()
        total = payload["total"]
        page_size = payload["pageSize"]
        if not isinstance(total, int) or total < 0 or not isinstance(page_size, int) or page_size <= 0:
            raise ValueError("RudderStack returned invalid catalog pagination metadata")
        # The catalog caps requests at 50 pages, so a larger collection cannot produce a complete full refresh.
        if total > page_size * CATALOG_MAX_PAGES:
            raise RudderStackCatalogLimitError("RudderStack catalog exceeds the 50-page API limit")
        super().update_state(response, data)
        if (self.page - 1) * page_size >= total:
            self._has_next_page = False


def endpoint_config(endpoint: RudderStackEndpoint, api_version: str) -> Endpoint:
    return {
        "path": endpoint.path.replace("/v2/", f"/{api_version}/", 1),
        "data_selector": endpoint.data_selector,
        "data_selector_required": True,
        "params": dict(endpoint.params),
        "paginator": RudderStackCatalogPaginator()
        if endpoint.paginator == "catalog"
        else {"type": "json_response", "next_url_path": "paging.next"}
        if endpoint.paginator == "json_response"
        else "single_page",
    }


def endpoint_extra(endpoint: RudderStackEndpoint, api_version: str) -> Endpoint:
    config = endpoint_config(endpoint, api_version)
    config.pop("params", None)
    return config


def token_for_endpoint(config: RudderStackSourceConfig, endpoint: str | None) -> str:
    if endpoint == "audit_logs":
        if not config.organization_access_token:
            raise ValueError("Add an organization Service Access Token to sync audit logs.")
        return config.organization_access_token
    return config.access_token


def probe_credentials(config: RudderStackSourceConfig, schema_name: str | None, api_version: str) -> int:
    endpoint = schema_for_resource(ENDPOINTS, schema_name) if schema_name else None
    if endpoint and endpoint.fanout:
        endpoint = ENDPOINTS[endpoint.fanout.parent_name]
    path = endpoint.path.replace("/v2/", f"/{api_version}/", 1) if endpoint else f"/{api_version}/workspace"
    token = token_for_endpoint(config, schema_name)
    with make_tracked_session(redact_values=(token,)) as session:
        response = session.get(
            f"{BASE_URLS[config.region]}{path}",
            auth=BearerTokenAuth(token),
            headers={"Accept": "application/json"},
            params=endpoint.params if endpoint else None,
            timeout=(10, 30),
            allow_redirects=False,
        )
        return response.status_code


def rudderstack_source(
    config: RudderStackSourceConfig,
    inputs: SourceInputs,
    resumable_source_manager: ResumableSourceManager[RudderStackResumeConfig],
    api_version: str,
) -> SourceResponse:
    endpoint = schema_for_resource(ENDPOINTS, inputs.schema_name)
    client: ClientConfig = {
        "base_url": BASE_URLS[config.region],
        "auth": {"type": "bearer", "token": token_for_endpoint(config, inputs.schema_name)},
        "headers": {"Accept": "application/json"},
        "allowed_hosts": [],
        "allow_redirects": False,
        "request_timeout": (10, 30),
        "retry_budget_seconds": 120,
        "retry_after_max_seconds": 60,
    }
    resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None

    def save_checkpoint(state: dict[str, Any] | None) -> None:
        if state is not None:
            resumable_source_manager.save_state(RudderStackResumeConfig(paginator_state=state))

    if endpoint.fanout:
        resource = build_dependent_resource(
            endpoint_configs=ENDPOINTS,
            child_endpoint=inputs.schema_name,
            fanout=endpoint.fanout,
            client_config=client,
            path_format_values={},
            team_id=inputs.team_id,
            job_id=inputs.job_id,
            db_incremental_field_last_value=None,
            page_size_param=None,
            parent_endpoint_extra=endpoint_extra(ENDPOINTS[endpoint.fanout.parent_name], api_version),
            child_endpoint_extra=endpoint_extra(endpoint, api_version),
            resume_hook=save_checkpoint,
            initial_paginator_state=resume.paginator_state if resume else None,
        )
    else:
        rest_config: RESTAPIConfig = {
            "client": client,
            "resources": [
                {
                    "name": inputs.schema_name,
                    "table_format": "delta",
                    "write_disposition": "replace",
                    "endpoint": endpoint_config(endpoint, api_version),
                }
            ],
        }
        resource = rest_api_resource(
            rest_config,
            inputs.team_id,
            inputs.job_id,
            None,
            resume_hook=save_checkpoint,
            initial_paginator_state=resume.paginator_state if resume else None,
        )
    return SourceResponse(
        name=inputs.schema_name,
        items=lambda: resource,
        primary_keys=list(endpoint.primary_keys),
        partition_mode="datetime",
        partition_format="month",
        partition_keys=[endpoint.partition_key],
        sort_mode=None,
    )