Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

20.7 KB
Raw Blame
import dataclasses
from collections.abc import Iterator
from datetime import UTC, date, datetime, timedelta
from typing import Any, Optional
from urllib.parse import urlencode

import requests
from structlog.types import FilteringBoundLogger
from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter

from products.warehouse_sources.sources.render.settings import RENDER_ENDPOINTS, RenderEndpointConfig
from products.warehouse_sources.sources.sdk import (
    ResumableSourceManager,
    SourceResponse,
    make_tracked_session,
    retry_with_deadline,
    source_helpers,
)

RENDER_BASE_URL = "https://api.render.com/v1"

REQUEST_TIMEOUT_SECONDS = 60

REDACTED_VALUE = "__redacted__"


class RenderRetryableError(Exception):
    pass


@dataclasses.dataclass(frozen=False)
class RenderResumeConfig:
    # Cursor of the last processed item — passed back as `?cursor=` to fetch what follows.
    # None means "start the listing from its first page".
    cursor: str | None = None
    # The fan-out parent currently being processed. A stable parent-ID bookmark (not a
    # positional index) so parents added/removed between a crash and the retry can't resume
    # us into the wrong parent. None for top-level endpoints.
    parent_id: str | None = None


def _get_headers(api_key: str) -> dict[str, str]:
    return {
        "Authorization": f"Bearer {api_key}",
        "Accept": "application/json",
    }


def _format_incremental_value(value: Any) -> str:
    """Format a watermark as an ISO 8601 UTC timestamp, the format Render's time filters document."""
    if isinstance(value, datetime):
        utc_value = value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
        return utc_value.strftime("%Y-%m-%dT%H:%M:%SZ")
    if isinstance(value, date):
        return f"{value.isoformat()}T00:00:00Z"
    return str(value)


def _to_utc_datetime(value: Any) -> datetime | None:
    if isinstance(value, datetime):
        return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
    if isinstance(value, date):
        return datetime(value.year, value.month, value.day, tzinfo=UTC)
    if isinstance(value, str):
        try:
            return _to_utc_datetime(datetime.fromisoformat(value.replace("Z", "+00:00")))
        except ValueError:
            return None
    return None


# Shared with `RenderSource.get_non_retryable_errors` so the same rejection reads the same way
# whether it surfaces while connecting the source or mid-sync.
KEY_REJECTED_MESSAGE = (
    "Your Render API key is invalid or has been revoked. Create a new API key in your Render "
    "account settings, then reconnect."
)
KEY_FORBIDDEN_MESSAGE = (
    "Your Render API key does not have access to this resource. Check the key's workspace access, then reconnect."
)
# `validate_via_probe` reports a transport failure as a `None` status, so anything Render did not
# answer itself leaves the key unjudged. Calling it invalid sends someone off to mint a replacement
# that fails the same way.
PROBE_FAILED_MESSAGE = "PostHog couldn't check your API key with Render. Wait a few minutes and try again."


def validate_credentials(api_key: str) -> tuple[bool, str | None]:
    # /owners is the cheapest authenticated probe: every valid key can list the workspaces
    # it belongs to, regardless of what resources exist.
    ok, status = source_helpers.validate_via_probe(
        make_tracked_session,
        f"{RENDER_BASE_URL}/owners?limit=1",
        headers=_get_headers(api_key),
    )
    if ok:
        return True, None
    if status == 401:
        return False, KEY_REJECTED_MESSAGE
    if status == 403:
        return False, KEY_FORBIDDEN_MESSAGE
    return False, PROBE_FAILED_MESSAGE


@retry_with_deadline(
    retry=retry_if_exception_type(
        (
            RenderRetryableError,
            requests.ReadTimeout,
            requests.ConnectionError,
            requests.exceptions.ChunkedEncodingError,
        )
    ),
    stop=stop_after_attempt(5),
    wait=wait_exponential_jitter(initial=2, max=60),
    reraise=True,
)
def _fetch_page(
    session: requests.Session,
    url: str,
    headers: dict[str, str],
    logger: FilteringBoundLogger,
    response_list_key: str | None = None,
) -> list[dict[str, Any]]:
    response = session.get(url, headers=headers, timeout=REQUEST_TIMEOUT_SECONDS)

    # Render rate-limits per user (400/min for most GETs) and returns 429 with
    # Ratelimit-* headers; exponential backoff comfortably outlasts the window.
    if response.status_code == 429 or response.status_code >= 500:
        raise RenderRetryableError(f"Render API error (retryable): status={response.status_code}, url={url}")

    if not response.ok:
        # 404 (a service deleted mid-sync) and 400 (a service type that can't own the child
        # resource, e.g. custom domains on a cron job) are expected during fan-out and handled
        # by the caller — log them as warnings. Anything else is a genuine error.
        log = logger.warning if response.status_code in (400, 404) else logger.error
        log(f"Render API error: status={response.status_code}, body={response.text}, url={url}")
        response.raise_for_status()

    data = response.json()
    if response_list_key and isinstance(data, dict):
        data = data.get(response_list_key) or []
    if not isinstance(data, list):
        raise ValueError(f"Render API returned a non-list response for url={url}")
    return data


def _unwrap_item(item: dict[str, Any], wrapper_key: str) -> tuple[dict[str, Any], str | None]:
    """Split a list item into (resource row, sibling cursor).

    Render list items wrap the resource next to its pagination cursor:
    `{"service": {...}, "cursor": "..."}`. A couple of endpoints (env groups) are documented
    unwrapped, so fall back to treating the item as the row itself when the key is absent.
    """
    if wrapper_key in item and isinstance(item[wrapper_key], dict):
        return item[wrapper_key], item.get("cursor")
    return item, item.get("cursor") if isinstance(item.get("cursor"), str) else None


def _explode_metric_series(series: dict[str, Any]) -> list[dict[str, Any]]:
    """Turn one metrics time series into a row per data point.

    Labels identify the series (instance, status code, host, quantile, …) and vary by metric,
    so they are folded into a sorted `seriesKey` that makes (service, series, timestamp) unique.
    """
    labels = {
        label["field"]: label.get("value")
        for label in series.get("labels") or []
        if isinstance(label, dict) and "field" in label
    }
    series_key = ",".join(f"{field}={labels[field]}" for field in sorted(labels))
    return [
        {
            "seriesKey": series_key,
            "labels": labels,
            "unit": series.get("unit"),
            "timestamp": point.get("timestamp"),
            "value": point.get("value"),
        }
        for point in series.get("values") or []
        if isinstance(point, dict)
    ]


def _build_params(
    config: RenderEndpointConfig,
    owner_id: str | None,
    should_use_incremental_field: bool,
    db_incremental_field_last_value: Any,
    incremental_field: str | None,
    logger: FilteringBoundLogger,
) -> dict[str, Any]:
    params: dict[str, Any] = {"limit": config.page_size} if config.paginated else {}
    params.update(config.extra_params)

    if config.supports_owner_filter and owner_id:
        params["ownerId"] = owner_id

    lower_bound: Any = None
    if should_use_incremental_field and db_incremental_field_last_value is not None:
        field = incremental_field or config.default_incremental_field
        param = config.incremental_param_by_field.get(field) if field else None
        if param:
            params[param] = _format_incremental_value(db_incremental_field_last_value)
            lower_bound = db_incremental_field_last_value
        else:
            logger.warning(
                f"Render: incremental field {field} has no server-side filter for endpoint "
                f"{config.name}, syncing the full endpoint instead"
            )

    window_param = (
        config.incremental_param_by_field.get(config.default_incremental_field)
        if config.default_incremental_field
        else None
    )
    if config.max_lookback_days and window_param:
        # One day of margin: the server measures the span up to its own "now", which moves on
        # while a fan-out works through its parents.
        floor = datetime.now(UTC) - timedelta(days=config.max_lookback_days - 1)
        lower_bound_dt = _to_utc_datetime(lower_bound) if lower_bound is not None else None
        if lower_bound is None or (lower_bound_dt is not None and lower_bound_dt < floor):
            params[window_param] = _format_incremental_value(floor)

    return params


def _walk_pages(
    session: requests.Session,
    headers: dict[str, str],
    logger: FilteringBoundLogger,
    config: RenderEndpointConfig,
    path: str,
    params: dict[str, Any],
    cursor: str | None,
    max_pages: int | None = None,
) -> Iterator[tuple[list[dict[str, Any]], str | None]]:
    """Yield (rows, next_cursor) per page, following each page's last-item cursor.

    Render doesn't document an explicit end-of-results signal, so pagination only stops on an
    empty page (a short page could in principle still have more results). The next cursor is
    the cursor of the last item in the page.
    """
    pages_fetched = 0
    while True:
        page_params = dict(params)
        if cursor:
            page_params["cursor"] = cursor
        url = f"{RENDER_BASE_URL}{path}?{urlencode(page_params)}"

        items = _fetch_page(session, url, headers, logger, config.response_list_key)
        if not items:
            return

        rows: list[dict[str, Any]] = []
        next_cursor: str | None = None
        for item in items:
            if config.metric_series:
                rows.extend(_explode_metric_series(item))
                continue
            row, item_cursor = _unwrap_item(item, config.wrapper_key)
            rows.append(row)
            if item_cursor:
                next_cursor = item_cursor

        if not rows:
            # Metric series with no data points in the window.
            return

        yield rows, next_cursor

        if not config.paginated:
            return

        if next_cursor is None:
            # Unwrapped responses carry no cursor, so a full page here may mean truncation.
            if len(items) >= config.page_size:
                logger.warning(
                    f"Render: endpoint {config.name} returned a full page with no pagination "
                    f"cursor — results may be truncated at {config.page_size} items"
                )
            return

        pages_fetched += 1
        if max_pages is not None and pages_fetched >= max_pages:
            logger.warning(
                f"Render: page cap of {max_pages} reached for endpoint {config.name} "
                f"(path={path}) — remaining results were not fetched"
            )
            return

        cursor = next_cursor


def _iter_parent_rows(
    session: requests.Session,
    headers: dict[str, str],
    logger: FilteringBoundLogger,
    parent_config: RenderEndpointConfig,
    owner_id: str | None,
) -> Iterator[dict[str, Any]]:
    """Fully enumerate the parent endpoint (no incremental bound: a new child can land on an
    old parent, so every sync must consider every parent)."""
    params: dict[str, Any] = {"limit": parent_config.page_size}
    if parent_config.supports_owner_filter and owner_id:
        params["ownerId"] = owner_id

    for rows, _ in _walk_pages(session, headers, logger, parent_config, parent_config.path, params, cursor=None):
        if parent_config.owner_id_field and owner_id:
            rows = [row for row in rows if row.get(parent_config.owner_id_field) == owner_id]
        yield from rows


def _get_top_level_rows(
    session: requests.Session,
    headers: dict[str, str],
    logger: FilteringBoundLogger,
    resumable_source_manager: ResumableSourceManager[RenderResumeConfig],
    config: RenderEndpointConfig,
    params: dict[str, Any],
) -> Iterator[list[dict[str, Any]]]:
    resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
    cursor = resume.cursor if resume else None
    if cursor:
        logger.debug(f"Render: resuming {config.name} from cursor={cursor}")

    for rows, next_cursor in _walk_pages(session, headers, logger, config, config.path, params, cursor):
        yield rows
        # Save AFTER yielding so a crash re-yields the last page rather than skipping it —
        # merge dedupes on the primary key.
        if next_cursor:
            resumable_source_manager.save_state(RenderResumeConfig(cursor=next_cursor))


def _get_fan_out_rows(
    session: requests.Session,
    headers: dict[str, str],
    logger: FilteringBoundLogger,
    resumable_source_manager: ResumableSourceManager[RenderResumeConfig],
    config: RenderEndpointConfig,
    params: dict[str, Any],
    owner_id: str | None,
) -> Iterator[list[dict[str, Any]]]:
    """Fan out over every parent resource, fetching this child endpoint once per parent.

    Fan-out runs report sort_mode="desc" (see render_source), so the incremental watermark
    persists only after every parent completes — a partial run's max says nothing about
    parents it never reached.
    """
    assert config.parent is not None
    parent_config = RENDER_ENDPOINTS[config.parent]
    parents = [
        (row["id"], row) for row in _iter_parent_rows(session, headers, logger, parent_config, owner_id) if "id" in row
    ]

    # Resolve the saved parent-ID bookmark to the slice of parents still to process. If the
    # bookmarked parent no longer exists (deleted between runs), start over from the first
    # parent — merge dedupes the re-pulled rows on the primary key.
    resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
    remaining = parents
    resume_cursor: str | None = None
    if resume is not None and resume.parent_id is not None:
        parent_ids = [parent_id for parent_id, _ in parents]
        if resume.parent_id in parent_ids:
            remaining = parents[parent_ids.index(resume.parent_id) :]
            resume_cursor = resume.cursor
            logger.debug(f"Render: resuming {config.name} from parent_id={resume.parent_id}, cursor={resume_cursor}")

    window_param = (
        config.incremental_param_by_field.get(config.default_incremental_field)
        if config.window_start_from_parent_field and config.default_incremental_field
        else None
    )

    for index, (parent_id, parent_row) in enumerate(remaining):
        child_params = dict(params)
        # Events default to a one-hour server-side window, so always pass an explicit lower
        # bound: the watermark when incremental, else the parent's creation time.
        if window_param and window_param not in child_params:
            window_start = parent_row.get(config.window_start_from_parent_field or "")
            if window_start:
                child_params[window_param] = window_start

        if config.parent_query_param:
            path = config.path
            child_params[config.parent_query_param] = parent_id
        else:
            path = config.path.format(parent_id=parent_id)

        cursor = resume_cursor
        resume_cursor = None  # only the resumed-into parent uses the saved cursor

        try:
            for rows, next_cursor in _walk_pages(
                session, headers, logger, config, path, child_params, cursor, max_pages=config.max_pages_per_parent
            ):
                if config.inject_parent_key:
                    rows = [{config.inject_parent_key: parent_id, **row} for row in rows]
                yield rows
                if next_cursor:
                    resumable_source_manager.save_state(RenderResumeConfig(cursor=next_cursor, parent_id=parent_id))
        except requests.HTTPError as exc:
            status = exc.response.status_code if exc.response is not None else None
            # A parent deleted between enumeration and this fetch 404s. Skip it rather than
            # failing the whole sync — the resource is genuinely gone.
            if status == 404:
                logger.warning(f"Render: {config.parent} {parent_id} not found while fetching {config.name}, skipping")
            # Render 400s the child endpoint for parents that can't own the resource (custom
            # domains on non-web/static services). Skip them instead of failing the sync.
            elif status == 400 and config.skip_parent_on_bad_request:
                logger.warning(f"Render: {config.parent} {parent_id} cannot have {config.name}, skipping")
            elif status == 403 and config.skip_parent_on_forbidden:
                logger.warning(f"Render: no access to {config.name} for {config.parent} {parent_id}, skipping")
            else:
                raise

        # Advance the bookmark to the next parent so a crash between parents resumes correctly.
        if index + 1 < len(remaining):
            resumable_source_manager.save_state(RenderResumeConfig(cursor=None, parent_id=remaining[index + 1][0]))


def _redact_row(row: dict[str, Any], redact_spec: dict[str, tuple[str, ...]]) -> dict[str, Any]:
    """Strip secret values out of a row before it reaches the warehouse.

    Render's env-group payload carries the actual env var values and secret-file contents; the
    API key is meant to *use* those for sync, not surface them as queryable columns. Replace the
    value-bearing fields with a marker while keeping the surrounding metadata (names/keys/ids).
    """
    for list_field, secret_keys in redact_spec.items():
        items = row.get(list_field)
        if not isinstance(items, list):
            continue
        for item in items:
            if not isinstance(item, dict):
                continue
            for key in secret_keys:
                if key in item:
                    item[key] = REDACTED_VALUE
    return row


def get_rows(
    api_key: str,
    endpoint: str,
    logger: FilteringBoundLogger,
    resumable_source_manager: ResumableSourceManager[RenderResumeConfig],
    owner_id: str | None = None,
    should_use_incremental_field: bool = False,
    db_incremental_field_last_value: Any = None,
    incremental_field: str | None = None,
) -> Iterator[list[dict[str, Any]]]:
    config = RENDER_ENDPOINTS[endpoint]
    headers = _get_headers(api_key)
    # One session reused across every page (and, for fan-out, every parent) so urllib3 keeps
    # the connection alive instead of re-handshaking per request. Sensitive endpoints opt out of
    # HTTP sample capture — capture snapshots the raw response before row-level redaction runs.
    session = make_tracked_session(capture=not config.is_sensitive)

    params = _build_params(
        config, owner_id, should_use_incremental_field, db_incremental_field_last_value, incremental_field, logger
    )

    if config.parent:
        batches = _get_fan_out_rows(session, headers, logger, resumable_source_manager, config, params, owner_id)
    else:
        batches = _get_top_level_rows(session, headers, logger, resumable_source_manager, config, params)

    if config.redact_list_item_fields:
        for batch in batches:
            yield [_redact_row(row, config.redact_list_item_fields) for row in batch]
    else:
        yield from batches


def render_source(
    api_key: str,
    endpoint: str,
    logger: FilteringBoundLogger,
    resumable_source_manager: ResumableSourceManager[RenderResumeConfig],
    owner_id: str | None = None,
    should_use_incremental_field: bool = False,
    db_incremental_field_last_value: Optional[Any] = None,
    incremental_field: str | None = None,
) -> SourceResponse:
    endpoint_config = RENDER_ENDPOINTS[endpoint]

    return SourceResponse(
        name=endpoint,
        items=lambda: get_rows(
            api_key=api_key,
            endpoint=endpoint,
            logger=logger,
            resumable_source_manager=resumable_source_manager,
            owner_id=owner_id,
            should_use_incremental_field=should_use_incremental_field,
            db_incremental_field_last_value=db_incremental_field_last_value,
            incremental_field=incremental_field,
        ),
        primary_keys=endpoint_config.primary_keys,
        # Render doesn't document list ordering and cursor pagination accepts no sort param, so
        # the watermark is only safe to persist once the run completes (desc mode computes the
        # max across all batches and saves it at successful job end).
        sort_mode="desc",
        partition_count=1,
        partition_size=1,
        partition_mode="datetime" if endpoint_config.partition_key else None,
        partition_format="month" if endpoint_config.partition_key else None,
        partition_keys=[endpoint_config.partition_key] if endpoint_config.partition_key else None,
    )