Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

17.8 KB
Raw Blame
import dataclasses
from collections.abc import Iterator
from datetime import UTC, date, datetime
from typing import Any, Optional
from urllib.parse import quote, urlencode, urlsplit, urlunsplit

import requests
from structlog.types import FilteringBoundLogger
from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter

from products.warehouse_sources.sources.replicate.settings import REPLICATE_ENDPOINTS, ReplicateEndpointConfig
from products.warehouse_sources.sources.sdk import (
    ResumableSourceManager,
    SourceResponse,
    frozen,
    make_tracked_session,
    retry_with_deadline,
)

REPLICATE_BASE_URL = "https://api.replicate.com/v1"
REPLICATE_API_HOST = "api.replicate.com"


class ReplicateRetryableError(Exception):
    pass


@dataclasses.dataclass(frozen=True)
class ReplicateResumeConfig:
    # Full next-page URL returned by the API (carries the opaque cursor and any created_after filter).
    # None means "start the endpoint from its first page".
    next_url: str | None = None
    # The `created_after` watermark this cursor was built against. A run whose watermark has since
    # advanced must not resume from this cursor (see `get_rows`), so we can tell the two apart.
    created_after: str | None = None
    # Fan-out endpoints only: the parent (`owner/name` model or collection slug) the cursor belongs to.
    fan_out_parent: str | None = None


def _get_headers(api_key: str) -> dict[str, str]:
    return {
        "Authorization": f"Bearer {api_key}",
        "Accept": "application/json",
    }


def _format_incremental_value(value: Any) -> str:
    """Format an incremental cursor for Replicate's ISO 8601 `created_after` filter.

    Replicate returns and accepts UTC timestamps with a `Z` suffix (e.g.
    `2023-09-08T16:19:34.765994Z`), so we normalize to that rather than the `+00:00`
    offset `isoformat()` emits.
    """
    if isinstance(value, datetime):
        utc_dt = value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
        return utc_dt.isoformat().replace("+00:00", "Z")
    if isinstance(value, date):
        return datetime.combine(value, datetime.min.time(), tzinfo=UTC).isoformat().replace("+00:00", "Z")
    return str(value)


def _parse_datetime(value: Any) -> datetime | None:
    """Parse a Replicate ISO 8601 timestamp into an aware datetime, tolerating the Z suffix."""
    if not isinstance(value, str) or not value:
        return None
    try:
        parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
    except ValueError:
        return None
    return parsed.replace(tzinfo=UTC) if parsed.tzinfo is None else parsed


def _to_cutoff(value: Any) -> datetime | None:
    if isinstance(value, datetime):
        return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
    if isinstance(value, date):
        return datetime.combine(value, datetime.min.time(), tzinfo=UTC)
    return _parse_datetime(value)


def validate_credentials(api_key: str) -> bool:
    try:
        response = make_tracked_session(redact_values=(api_key,)).get(
            f"{REPLICATE_BASE_URL}/account", headers=_get_headers(api_key), timeout=10
        )
        return response.status_code == 200
    except Exception:
        return False


@retry_with_deadline(
    retry=retry_if_exception_type(
        (
            ReplicateRetryableError,
            requests.ReadTimeout,
            requests.ConnectionError,
            requests.exceptions.ChunkedEncodingError,
        )
    ),
    stop=stop_after_attempt(5),
    wait=wait_exponential_jitter(initial=1, max=30),
    reraise=True,
)
def _fetch_page(session: requests.Session, url: str, headers: dict[str, str], logger: FilteringBoundLogger) -> Any:
    response = session.get(url, headers=headers, timeout=60)

    # Replicate rate-limits at 3,000 req/min for read endpoints (429 on excess); 5xx are transient.
    if response.status_code == 429 or response.status_code >= 500:
        raise ReplicateRetryableError(f"Replicate API error (retryable): status={response.status_code}, url={url}")

    if not response.ok:
        logger.error(f"Replicate API error: status={response.status_code}, body={response.text}, url={url}")
        response.raise_for_status()

    return response.json()


def _extract_items(data: Any, config: ReplicateEndpointConfig) -> list[dict[str, Any]]:
    if config.response_shape == "object":
        # A single-object endpoint (e.g. /account) becomes one row.
        return [data] if isinstance(data, dict) else []
    if config.response_shape == "list":
        # A bare JSON array (e.g. /hardware).
        return data if isinstance(data, list) else []
    # Paginated: rows live under "results".
    return data.get("results", []) if isinstance(data, dict) else []


def _sanitize_next_url(raw_url: str | None) -> str | None:
    """Pin a Replicate pagination URL to the fixed HTTPS API origin.

    Replicate's documented pagination `next` (and any cursor loaded from resume state) can come
    back as `http://api.replicate.com/...`, which would send the bearer token over plaintext where
    an on-path attacker could capture it. Only the path and query are trusted; the scheme is forced
    to https and the host must be Replicate's API host, otherwise we stop rather than follow it.
    """
    if not raw_url:
        return None
    parts = urlsplit(raw_url)
    if parts.hostname != REPLICATE_API_HOST:
        return None
    return urlunsplit(("https", REPLICATE_API_HOST, parts.path, parts.query, ""))


def _next_url(data: Any) -> str | None:
    if isinstance(data, dict):
        return _sanitize_next_url(data.get("next"))
    return None


def _build_initial_url(
    config: ReplicateEndpointConfig,
    should_use_incremental_field: bool,
    db_incremental_field_last_value: Any,
) -> str:
    base = f"{REPLICATE_BASE_URL}{config.path}"
    if config.time_filter_param and should_use_incremental_field and db_incremental_field_last_value is not None:
        query = urlencode({config.time_filter_param: _format_incremental_value(db_incremental_field_last_value)})
        return f"{base}?{query}"
    return base


def _page_predates_cutoff(items: list[dict[str, Any]], incremental_field: str, cutoff: datetime) -> bool:
    """True when every dated row in the page is at/older than the cutoff.

    Replicate returns predictions newest-first, so once a whole page predates the watermark there is
    nothing newer left to fetch. This bounds the walk even if the server silently ignores
    `created_after` on paginated requests (unverified against the live API), matching the skill's
    "incremental pagination must terminate at the watermark" rule. Merge dedupes the boundary re-read.
    """
    parsed = [_parse_datetime(item.get(incremental_field)) for item in items]
    dated = [d for d in parsed if d is not None]
    if not dated:
        return False
    return max(dated) <= cutoff


def _walk_pages(
    session: requests.Session, url: str, headers: dict[str, str], logger: FilteringBoundLogger
) -> Iterator[tuple[list[dict[str, Any]], str | None]]:
    """Yield each page's rows with the URL of the page after it."""
    next_url: str | None = url
    while next_url:
        try:
            data = _fetch_page(session, next_url, headers, logger)
        except requests.HTTPError as e:
            # Same 416-as-empty-page handling as the main pagination loop in `get_rows`.
            if e.response is not None and e.response.status_code == 416:
                return
            raise
        next_url = _next_url(data)
        yield (data.get("results", []) if isinstance(data, dict) else []), next_url


def _is_skippable_child_error(error: requests.HTTPError) -> bool:
    # A referenced model can be deleted or made private after a deployment or training used it.
    return error.response is not None and error.response.status_code in (403, 404)


@frozen(order=True)
class ModelRef:
    owner: str
    name: str


def _parse_model_ref(value: Any) -> ModelRef | None:
    """Parse `owner/name` or `owner/name:version_id` into a model reference."""
    if not isinstance(value, str):
        return None
    owner, sep, name = value.split(":", 1)[0].partition("/")
    if not sep or not owner or not name or "/" in name:
        return None
    return ModelRef(owner=owner, name=name)


def _referenced_models(row: dict[str, Any]) -> list[ModelRef]:
    release = row.get("current_release")
    output = row.get("output")
    candidates = [
        release.get("model") if isinstance(release, dict) else None,
        row.get("model"),
        # A finished training's output.version is the destination version, as owner/name:version_id.
        output.get("version") if isinstance(output, dict) else None,
    ]
    return [ref for ref in map(_parse_model_ref, candidates) if ref is not None]


def _get_model_version_rows(
    session: requests.Session,
    headers: dict[str, str],
    logger: FilteringBoundLogger,
    resumable_source_manager: ResumableSourceManager[ReplicateResumeConfig],
) -> Iterator[list[dict[str, Any]]]:
    models: set[ModelRef] = set()
    for parent_path in ("/deployments", "/trainings"):
        for page, _ in _walk_pages(session, f"{REPLICATE_BASE_URL}{parent_path}", headers, logger):
            for item in page:
                models.update(_referenced_models(item))
            resumable_source_manager.safe_point()

    resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
    resume_parent = resume.fan_out_parent if resume is not None else None
    resume_url = _sanitize_next_url(resume.next_url) if resume is not None else None

    # Sorted, so a resumed run can skip every model that sorts before the saved one.
    for model in sorted(models):
        owner, name = model.owner, model.name
        parent = f"{owner}/{name}"
        url = f"{REPLICATE_BASE_URL}/models/{quote(owner, safe='')}/{quote(name, safe='')}/versions"
        if resume_parent is not None and parent <= resume_parent:
            if parent < resume_parent or resume_url is None:
                continue
            url = resume_url
        try:
            for page, next_url in _walk_pages(session, url, headers, logger):
                if not page:
                    resumable_source_manager.safe_point()
                    continue
                resumable_source_manager.save_state(ReplicateResumeConfig(next_url=next_url, fan_out_parent=parent))
                yield [{**version, "model_owner": owner, "model_name": name} for version in page]
        except requests.HTTPError as e:
            if not _is_skippable_child_error(e):
                raise
            logger.info(f"Replicate: skipping versions of {owner}/{name}: {e}")
            resumable_source_manager.safe_point()


def _get_collection_model_rows(
    session: requests.Session,
    headers: dict[str, str],
    logger: FilteringBoundLogger,
    resumable_source_manager: ResumableSourceManager[ReplicateResumeConfig],
) -> Iterator[list[dict[str, Any]]]:
    resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
    resumed_url = _sanitize_next_url(resume.next_url) if resume is not None else None
    page_url = resumed_url or f"{REPLICATE_BASE_URL}/collections"
    # The cursor names the collections page and the last slug on it whose models were yielded.
    skip_through = resume.fan_out_parent if resume is not None else None

    for page, next_url in _walk_pages(session, page_url, headers, logger):
        slugs = [c["slug"] for c in page if isinstance(c.get("slug"), str) and c["slug"]]
        if skip_through in slugs:
            slugs = slugs[slugs.index(skip_through) + 1 :]
        skip_through = None
        for slug in slugs:
            try:
                detail = _fetch_page(
                    session, f"{REPLICATE_BASE_URL}/collections/{quote(slug, safe='')}", headers, logger
                )
            except requests.HTTPError as e:
                if not _is_skippable_child_error(e):
                    raise
                logger.info(f"Replicate: skipping collection {slug}: {e}")
                resumable_source_manager.safe_point()
                continue
            models = detail.get("models") if isinstance(detail, dict) else None
            rows = [{**model, "collection_slug": slug} for model in models or [] if isinstance(model, dict)]
            if rows:
                resumable_source_manager.save_state(ReplicateResumeConfig(next_url=page_url, fan_out_parent=slug))
                yield rows
            else:
                resumable_source_manager.safe_point()
        resumable_source_manager.safe_point()
        if next_url is None:
            break
        page_url = next_url


_FAN_OUT_ITERATORS = {
    "model_versions": _get_model_version_rows,
    "collection_models": _get_collection_model_rows,
}


def get_rows(
    api_key: str,
    endpoint: str,
    logger: FilteringBoundLogger,
    resumable_source_manager: ResumableSourceManager[ReplicateResumeConfig],
    should_use_incremental_field: bool = False,
    db_incremental_field_last_value: Any = None,
    incremental_field: str | None = None,
) -> Iterator[list[dict[str, Any]]]:
    config = REPLICATE_ENDPOINTS[endpoint]
    headers = _get_headers(api_key)
    # One session reused across every page so urllib3 keeps the connection alive. Redact the token so
    # it never lands in tracked HTTP logs, and disable capture entirely: prediction responses carry
    # user-authored model inputs, outputs, and logs that the generic scrubber can't reliably sanitize.
    session = make_tracked_session(redact_values=(api_key,), capture=False)

    fan_out = _FAN_OUT_ITERATORS.get(endpoint)
    if fan_out is not None:
        yield from fan_out(session, headers, logger, resumable_source_manager)
        return

    # Single-request endpoints (bare array / single object) have no pagination or resume state.
    if config.response_shape != "paginated":
        data = _fetch_page(session, f"{REPLICATE_BASE_URL}{config.path}", headers, logger)
        items = _extract_items(data, config)
        if items:
            yield items
        return

    incremental = config.time_filter_param is not None and should_use_incremental_field
    current_watermark = (
        _format_incremental_value(db_incremental_field_last_value)
        if incremental and db_incremental_field_last_value is not None
        else None
    )

    resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
    resumed_url = _sanitize_next_url(resume.next_url) if resume is not None else None
    # Only honor a saved cursor when it was written for the current watermark. If the watermark has
    # advanced, resuming from an older descending page would skip predictions created since the prior
    # run (they live on the first page), so rebuild from the initial URL instead.
    if resumed_url and resume is not None and resume.created_after == current_watermark:
        url = resumed_url
        logger.debug(f"Replicate: resuming {endpoint} from URL: {url}")
    else:
        url = _build_initial_url(config, should_use_incremental_field, db_incremental_field_last_value)

    cutoff = _to_cutoff(db_incremental_field_last_value) if incremental else None
    cursor_field = incremental_field or (config.incremental_fields[0]["field"] if config.incremental_fields else None)

    while True:
        try:
            data = _fetch_page(session, url, headers, logger)
        except requests.HTTPError as e:
            if e.response is not None and e.response.status_code == 416:
                # Replicate returns 416 Range Not Satisfiable instead of an empty page for some
                # requests that match nothing (observed on created_after filters predating the
                # account's data) — treat it like a last page rather than failing the sync.
                break
            raise
        items = _extract_items(data, config)

        if items:
            yield items

        next_url = _next_url(data)

        # Desc + incremental: stop once a whole page predates the watermark, so we don't re-walk the
        # entire history if the server drops the created_after filter on later cursor pages.
        if next_url and cutoff is not None and cursor_field and _page_predates_cutoff(items, cursor_field, cutoff):
            break

        if not next_url:
            break

        # Save AFTER yielding, so a crash re-yields the last page rather than skipping it (merge
        # dedupes on the primary key). Tag it with the watermark so a later run with an advanced
        # watermark rebuilds from the first page instead of trusting this cursor.
        resumable_source_manager.save_state(ReplicateResumeConfig(next_url=next_url, created_after=current_watermark))
        resumable_source_manager.safe_point()
        url = next_url


def replicate_source(
    api_key: str,
    endpoint: str,
    logger: FilteringBoundLogger,
    resumable_source_manager: ResumableSourceManager[ReplicateResumeConfig],
    should_use_incremental_field: bool = False,
    db_incremental_field_last_value: Optional[Any] = None,
    incremental_field: str | None = None,
) -> SourceResponse:
    config = REPLICATE_ENDPOINTS[endpoint]

    return SourceResponse(
        name=endpoint,
        items=lambda: get_rows(
            api_key=api_key,
            endpoint=endpoint,
            logger=logger,
            resumable_source_manager=resumable_source_manager,
            should_use_incremental_field=should_use_incremental_field,
            db_incremental_field_last_value=db_incremental_field_last_value,
            incremental_field=incremental_field,
        ),
        primary_keys=config.primary_keys,
        sort_mode=config.sort_mode,
        partition_count=1,
        partition_size=1,
        partition_mode="datetime" if config.partition_key else None,
        partition_format="month" if config.partition_key else None,
        partition_keys=[config.partition_key] if config.partition_key else None,
    )