Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

24.2 KB
Raw Blame
import dataclasses
from collections.abc import Iterator
from datetime import UTC, date, datetime
from typing import Any, Optional
from urllib.parse import quote, urlencode

import requests
from dateutil import parser as dateutil_parser
from structlog.types import FilteringBoundLogger
from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter

from products.warehouse_sources.sources.codecov.settings import (
    CODECOV_ENDPOINTS,
    CodecovEndpointConfig,
    CodecovParentConfig,
    CodecovResponseShape,
)
from products.warehouse_sources.sources.sdk import (
    ResumableSourceManager,
    SourceResponse,
    make_tracked_session,
    retry_with_deadline,
)

CODECOV_BASE_URL = "https://api.codecov.io/api/v2"
PAGE_SIZE = 500  # Codecov caps page_size at 500


class CodecovRetryableError(Exception):
    pass


@dataclasses.dataclass(frozen=False)
class CodecovResumeConfig:
    # Next page URL to fetch. None means "start the bookmarked repo's endpoint at its first
    # page" — used when the bookmark advances to a repo whose first page URL isn't built yet.
    next_url: str | None = None
    # The repository currently being processed during fan-out. A stable name bookmark (not a
    # positional index) so repos added/removed between a crash and the retry can't resume us
    # into the wrong repo. None for the top-level repos endpoint.
    repo: str | None = None


def _get_headers(api_token: str) -> dict[str, str]:
    # Codecov's documented scheme is the lowercase "bearer" keyword.
    return {"Authorization": f"bearer {api_token}", "Accept": "application/json"}


def _owner_base_url(service: str, owner_username: str) -> str:
    # Both segments are user-supplied; percent-encode them (including "/") so they can't
    # retarget the request path.
    return f"{CODECOV_BASE_URL}/{quote(service, safe='')}/{quote(owner_username, safe='')}"


def _endpoint_url(owner_base_url: str, path: str, params: dict[str, Any], **segments: str) -> str:
    if segments:
        path = path.format(**{name: quote(value, safe="") for name, value in segments.items()})
    url = f"{owner_base_url}{path}"
    return f"{url}?{urlencode(params)}" if params else url


def _ensure_codecov_url(url: str) -> str:
    """Upgrade and pin a pagination/resume URL to the Codecov API origin.

    Codecov's DRF `next` links come back with an http:// scheme, so upgrade them rather than
    send the bearer token in plaintext. Beyond that, `next` is response-controlled and resume
    URLs are replayed from persisted state, so refuse any URL outside the API origin — the
    token must never be sent to a host we didn't choose.
    """
    if url.startswith("http://"):
        url = "https://" + url.removeprefix("http://")
    if not url.startswith(f"{CODECOV_BASE_URL}/"):
        raise ValueError(f"Refusing to follow non-Codecov URL: {url}")
    return url


def _as_utc(dt: datetime) -> datetime:
    return dt.replace(tzinfo=UTC) if dt.tzinfo is None else dt.astimezone(UTC)


def _normalize_cutoff(value: Any) -> datetime | None:
    if isinstance(value, datetime):
        return _as_utc(value)
    if isinstance(value, date):
        return datetime.combine(value, datetime.min.time(), tzinfo=UTC)
    return None


def _is_older_than_cutoff(value: Any, cutoff: datetime) -> bool:
    if value is None:
        return False
    if isinstance(value, str):
        try:
            parsed = dateutil_parser.parse(value)
        except (ValueError, TypeError):
            return False
    elif isinstance(value, datetime):
        parsed = value
    else:
        return False
    return _as_utc(parsed) <= cutoff


def _should_stop_desc(items: list[dict[str, Any]], cutoff_field: str | None, cutoff: datetime | None) -> bool:
    """Newest-first + incremental can stop the moment a page holds a pre-watermark record."""
    if not cutoff_field or cutoff is None or not items:
        return False
    return any(_is_older_than_cutoff(item.get(cutoff_field), cutoff) for item in items if item)


def _format_start_date(cutoff: datetime) -> str:
    # Codecov's coverage report takes calendar dates; the boundary interval is re-returned
    # and deduped by the primary-key merge.
    return cutoff.date().isoformat()


def _totals_rows(data: Any, repo: str) -> list[dict[str, Any]]:
    """One row of headline coverage per repository, with `totals` flattened into the root.

    The per-file breakdown the same response carries is the `report_files` grain, so it is
    dropped here — which also means the row's `files` is the file *count* from `totals`.
    """
    if not data:
        return []
    return [{**(data.get("totals") or {}), "repo": repo, "commit_file_url": data.get("commit_file_url")}]


def _report_file_rows(data: Any, repo: str) -> list[dict[str, Any]]:
    return [{**file, "repo": repo} for file in (data or {}).get("files") or []]


def _tree_rows(nodes: Any, repo: str) -> Iterator[dict[str, Any]]:
    """Flatten the nested directory tree into one row per node, at every depth."""
    for node in nodes or []:
        yield {**{key: value for key, value in node.items() if key != "children"}, "repo": repo}
        yield from _tree_rows(node.get("children"), repo)


def _impacted_file_rows(data: Any, repo: str) -> list[dict[str, Any]]:
    """One row per impacted file, with the commits the comparison ran between.

    `files` stays empty while Codecov is still computing the comparison (`state: "pending"`).
    """
    if not data:
        return []
    comparison = {
        "base_commit": data.get("base_commit"),
        "head_commit": data.get("head_commit"),
        "state": data.get("state"),
    }
    return [{**file, **comparison, "repo": repo} for file in data.get("files") or []]


def _unpaginated_rows(shape: CodecovResponseShape, data: Any, repo: str) -> list[dict[str, Any]]:
    match shape:
        case CodecovResponseShape.BARE_LIST:
            return [{**item, "repo": repo} for item in data or []]
        case CodecovResponseShape.TOTALS:
            return _totals_rows(data, repo)
        case CodecovResponseShape.REPORT_FILES:
            return _report_file_rows(data, repo)
        case CodecovResponseShape.TREE:
            return list(_tree_rows(data, repo))
        case CodecovResponseShape.IMPACTED_FILES:
            return _impacted_file_rows(data, repo)
        case _:
            raise ValueError(f"Endpoint response shape {shape} is paginated")


def parse_repositories(repositories: str | None) -> list[str]:
    """Split the optional comma-separated repository allow-list into clean names."""
    if not repositories:
        return []
    return [name.strip() for name in repositories.split(",") if name.strip()]


@retry_with_deadline(
    retry=retry_if_exception_type(
        (
            CodecovRetryableError,
            requests.ReadTimeout,
            requests.ConnectionError,
            requests.exceptions.ChunkedEncodingError,
        )
    ),
    stop=stop_after_attempt(5),
    wait=wait_exponential_jitter(initial=1, max=30),
    reraise=True,
)
def _fetch_page(session: requests.Session, url: str, headers: dict[str, str], logger: FilteringBoundLogger) -> Any:
    response = session.get(url, headers=headers, timeout=60)

    if response.status_code == 429 or response.status_code >= 500:
        raise CodecovRetryableError(f"Codecov API error (retryable): status={response.status_code}, url={url}")

    if not response.ok:
        # 404 is expected and handled during fan-out (a repo deleted mid-sync).
        log = logger.warning if response.status_code == 404 else logger.error
        log(f"Codecov API error: status={response.status_code}, body={response.text}, url={url}")
        response.raise_for_status()

    return response.json()


def _iter_pages(
    session: requests.Session, url: str, headers: dict[str, str], logger: FilteringBoundLogger
) -> Iterator[tuple[list[dict[str, Any]], str | None]]:
    """Yield (results, next_url) per page of a DRF-paginated Codecov list, following `next`."""
    while True:
        data = _fetch_page(session, url, headers, logger)
        results = data.get("results") or []
        next_url = data.get("next")
        next_url = _ensure_codecov_url(next_url) if next_url else None
        yield results, next_url
        if not next_url:
            return
        url = next_url


def _iter_repo_names(
    session: requests.Session,
    owner_base_url: str,
    headers: dict[str, str],
    logger: FilteringBoundLogger,
    resumable_source_manager: ResumableSourceManager[CodecovResumeConfig],
) -> Iterator[str]:
    """Page through the owner's active repos and yield each name. Inactive repos (never
    received an upload) have no coverage data, so fanning out over them is wasted API cost."""
    url = f"{owner_base_url}/repos?{urlencode({'active': 'true', 'page_size': PAGE_SIZE})}"
    for results, _next_url in _iter_pages(session, url, headers, logger):
        for item in results:
            yield item["name"]
        # The repo list holds parents, not rows, so the walk can stop here and restart.
        resumable_source_manager.safe_point()


def _get_top_level_rows(
    session: requests.Session,
    owner_base_url: str,
    headers: dict[str, str],
    config: CodecovEndpointConfig,
    repositories: list[str],
    logger: FilteringBoundLogger,
    resumable_source_manager: ResumableSourceManager[CodecovResumeConfig],
) -> Iterator[list[dict[str, Any]]]:
    resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
    if resume is not None and resume.next_url:
        url = _ensure_codecov_url(resume.next_url)
        logger.debug(f"Codecov: resuming {config.name} from URL: {url}")
    else:
        url = _endpoint_url(owner_base_url, config.path, {"page_size": PAGE_SIZE, **config.extra_params})

    allowed = set(repositories)
    for results, next_url in _iter_pages(session, url, headers, logger):
        rows = results
        if config.filtered_by_repository_allow_list and allowed:
            # The repos list has no server-side name filter, so the allow-list applies client-side.
            rows = [item for item in results if item.get("name") in allowed]
        if rows:
            yield rows
            # Save AFTER yielding so a crash re-yields the last page rather than skipping it —
            # merge dedupes on the primary key.
            if next_url:
                resumable_source_manager.save_state(CodecovResumeConfig(next_url=next_url))
        resumable_source_manager.safe_point()


def _remaining_repos(
    repo_names: list[str],
    config: CodecovEndpointConfig,
    logger: FilteringBoundLogger,
    resumable_source_manager: ResumableSourceManager[CodecovResumeConfig],
) -> tuple[list[str], str | None]:
    """Resolve the saved repo bookmark to the slice of repos still to process, and the URL the
    first of them resumes from.

    If the bookmarked repo no longer exists (deleted between runs), start over from the first —
    merge dedupes the re-pulled rows on the primary key.
    """
    resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
    if resume is None or resume.repo is None or resume.repo not in repo_names:
        return repo_names, None
    resume_url = _ensure_codecov_url(resume.next_url) if resume.next_url else None
    logger.debug(f"Codecov: resuming {config.name} from repo={resume.repo}, url={resume_url}")
    return repo_names[repo_names.index(resume.repo) :], resume_url


def _get_fan_out_rows(
    session: requests.Session,
    owner_base_url: str,
    headers: dict[str, str],
    config: CodecovEndpointConfig,
    repositories: list[str],
    logger: FilteringBoundLogger,
    resumable_source_manager: ResumableSourceManager[CodecovResumeConfig],
    should_use_incremental_field: bool,
    db_incremental_field_last_value: Any,
    incremental_field: str | None,
) -> Iterator[list[dict[str, Any]]]:
    """Fan out one paginated walk per repository, injecting the repo name onto every row (it
    feeds each endpoint's composite primary key)."""
    repo_names = repositories or list(
        _iter_repo_names(session, owner_base_url, headers, logger, resumable_source_manager)
    )

    cutoff = _normalize_cutoff(db_incremental_field_last_value) if should_use_incremental_field else None
    params: dict[str, Any] = {**config.extra_params}
    if config.response_shape is CodecovResponseShape.PAGINATED_LIST:
        params = {"page_size": PAGE_SIZE, **config.extra_params}
    stop_field: str | None = None
    if cutoff is not None and config.incremental_fields:
        if config.incremental_server_param:
            params[config.incremental_server_param] = _format_start_date(cutoff)
        else:
            stop_field = incremental_field or config.default_incremental_field

    remaining, resume_url = _remaining_repos(repo_names, config, logger, resumable_source_manager)

    for index, repo in enumerate(remaining):
        url = resume_url or _endpoint_url(owner_base_url, config.path, params, repo=repo)
        resume_url = None  # only the resumed-into repo uses the saved URL; the rest start fresh

        try:
            if config.response_shape is not CodecovResponseShape.PAGINATED_LIST:
                # Single-response endpoints: a bare array, or a report object we reshape.
                rows = _unpaginated_rows(config.response_shape, _fetch_page(session, url, headers, logger), repo)
                if rows:
                    yield rows
            else:
                for results, next_url in _iter_pages(session, url, headers, logger):
                    stop_after_this_page = _should_stop_desc(results, stop_field, cutoff)
                    rows = [{**item, "repo": repo} for item in results]
                    if rows:
                        yield rows
                        # Save AFTER yielding (and only when more pages remain) so a crash
                        # re-yields the last page rather than skipping it.
                        if next_url and not stop_after_this_page:
                            resumable_source_manager.save_state(CodecovResumeConfig(next_url=next_url, repo=repo))
                    resumable_source_manager.safe_point()
                    if stop_after_this_page:
                        break
        except requests.HTTPError as exc:
            # A repo deleted (or deactivated) between enumeration and this fetch 404s. Skip it
            # rather than failing the whole sync. Any other HTTP error is re-raised.
            if exc.response is not None and exc.response.status_code == 404:
                logger.warning(f"Codecov: repo {repo} not found while fetching {config.name}, skipping")
            else:
                raise

        # Advance the bookmark to the next repo so a crash between repos resumes correctly.
        if index + 1 < len(remaining):
            resumable_source_manager.save_state(CodecovResumeConfig(next_url=None, repo=remaining[index + 1]))
        resumable_source_manager.safe_point()


def _iter_child_rows(
    session: requests.Session,
    owner_base_url: str,
    headers: dict[str, str],
    config: CodecovEndpointConfig,
    parent: CodecovParentConfig,
    parent_row: dict[str, Any],
    repo: str,
    params: dict[str, Any],
    logger: FilteringBoundLogger,
) -> Iterator[list[dict[str, Any]]]:
    """Fetch the child endpoint for one parent row, yielding each page with the parent columns."""
    key = parent_row.get(parent.key_field)
    if key is None:
        return
    parent_columns = {column: parent_row.get(name) for name, column in parent.include_fields.items()}
    if parent.key_query_param:
        url = _endpoint_url(owner_base_url, config.path, {parent.key_query_param: key, **params}, repo=repo)
    else:
        url = _endpoint_url(owner_base_url, config.path, params, repo=repo, key=str(key))

    try:
        if config.response_shape is CodecovResponseShape.PAGINATED_LIST:
            for results, _next_url in _iter_pages(session, url, headers, logger):
                yield [{**item, **parent_columns, "repo": repo} for item in results]
        else:
            data = _fetch_page(session, url, headers, logger)
            yield [{**row, **parent_columns} for row in _unpaginated_rows(config.response_shape, data, repo)]
    except requests.HTTPError as exc:
        # A pull missing its base or head commit has no comparison, and a flag or commit can go
        # away between listing it and this fetch. Skip that row rather than fail the sync.
        if exc.response is not None and exc.response.status_code == 404:
            logger.warning(f"Codecov: {config.name} not found for repo={repo}, {parent.key_field}={key}, skipping")
        else:
            raise


def _iter_parent_fan_out_batches(
    session: requests.Session,
    owner_base_url: str,
    headers: dict[str, str],
    config: CodecovEndpointConfig,
    parent: CodecovParentConfig,
    repositories: list[str],
    logger: FilteringBoundLogger,
    resumable_source_manager: ResumableSourceManager[CodecovResumeConfig],
    cutoff: datetime | None,
) -> Iterator[tuple[list[dict[str, Any]], CodecovResumeConfig | None]]:
    """Yield (rows, state) per child response. `state` is set, with no rows, once every row of a
    parent page has been yielded: it is the position to resume from after that page."""
    repo_names = repositories or list(
        _iter_repo_names(session, owner_base_url, headers, logger, resumable_source_manager)
    )

    parent_params: dict[str, Any] = {"page_size": PAGE_SIZE}
    child_params: dict[str, Any] = {**config.extra_params}
    if config.response_shape is CodecovResponseShape.PAGINATED_LIST:
        child_params = {"page_size": PAGE_SIZE, **config.extra_params}
    stop_field: str | None = None
    if cutoff is not None:
        if config.incremental_server_param:
            child_params[config.incremental_server_param] = _format_start_date(cutoff)
        if parent.incremental_server_param:
            parent_params[parent.incremental_server_param] = _format_start_date(cutoff)
        stop_field = parent.incremental_stop_field

    remaining, resume_url = _remaining_repos(repo_names, config, logger, resumable_source_manager)

    for index, repo in enumerate(remaining):
        next_repo = (
            CodecovResumeConfig(next_url=None, repo=remaining[index + 1]) if index + 1 < len(remaining) else None
        )
        url = resume_url or _endpoint_url(owner_base_url, parent.path, parent_params, repo=repo)
        resume_url = None

        try:
            for parents, next_url in _iter_pages(session, url, headers, logger):
                for parent_row in parents:
                    for rows in _iter_child_rows(
                        session, owner_base_url, headers, config, parent, parent_row, repo, child_params, logger
                    ):
                        yield rows, None
                if not next_url or _should_stop_desc(parents, stop_field, cutoff):
                    break
                yield [], CodecovResumeConfig(next_url=next_url, repo=repo)
        except requests.HTTPError as exc:
            # The parent list of a repo deleted (or deactivated) since enumeration 404s.
            if exc.response is not None and exc.response.status_code == 404:
                logger.warning(f"Codecov: repo {repo} not found while fetching {config.name}, skipping")
            else:
                raise

        yield [], next_repo


def _get_parent_fan_out_rows(
    session: requests.Session,
    owner_base_url: str,
    headers: dict[str, str],
    config: CodecovEndpointConfig,
    parent: CodecovParentConfig,
    repositories: list[str],
    logger: FilteringBoundLogger,
    resumable_source_manager: ResumableSourceManager[CodecovResumeConfig],
    should_use_incremental_field: bool,
    db_incremental_field_last_value: Any,
) -> Iterator[list[dict[str, Any]]]:
    """Fan out over a list inside each repository (flags, commits or pulls), then fetch the
    child endpoint once per row of that list."""
    cutoff = _normalize_cutoff(db_incremental_field_last_value) if should_use_incremental_field else None
    for rows, state in _iter_parent_fan_out_batches(
        session, owner_base_url, headers, config, parent, repositories, logger, resumable_source_manager, cutoff
    ):
        if state is not None:
            resumable_source_manager.save_state(state)
        if rows:
            yield rows
        else:
            # Many parent rows have no child rows (a commit without uploads, a pull without a
            # comparison). Nothing is buffered here, so a shutdown can hand the run over.
            resumable_source_manager.safe_point()


def get_rows(
    api_token: str,
    service: str,
    owner_username: str,
    repositories: str | None,
    endpoint: str,
    logger: FilteringBoundLogger,
    resumable_source_manager: ResumableSourceManager[CodecovResumeConfig],
    should_use_incremental_field: bool = False,
    db_incremental_field_last_value: Any = None,
    incremental_field: str | None = None,
) -> Iterator[list[dict[str, Any]]]:
    config = CODECOV_ENDPOINTS[endpoint]
    headers = _get_headers(api_token)
    owner_base_url = _owner_base_url(service, owner_username)
    repo_filter = parse_repositories(repositories)
    # One session reused across every page (and repo) so urllib3 keeps the connection alive.
    session = make_tracked_session()

    if config.parent is not None:
        yield from _get_parent_fan_out_rows(
            session,
            owner_base_url,
            headers,
            config,
            config.parent,
            repo_filter,
            logger,
            resumable_source_manager,
            should_use_incremental_field,
            db_incremental_field_last_value,
        )
    elif config.fan_out_over_repos:
        yield from _get_fan_out_rows(
            session,
            owner_base_url,
            headers,
            config,
            repo_filter,
            logger,
            resumable_source_manager,
            should_use_incremental_field,
            db_incremental_field_last_value,
            incremental_field,
        )
    else:
        yield from _get_top_level_rows(
            session, owner_base_url, headers, config, repo_filter, logger, resumable_source_manager
        )


def codecov_source(
    api_token: str,
    service: str,
    owner_username: str,
    repositories: str | None,
    endpoint: str,
    logger: FilteringBoundLogger,
    resumable_source_manager: ResumableSourceManager[CodecovResumeConfig],
    should_use_incremental_field: bool = False,
    db_incremental_field_last_value: Optional[Any] = None,
    incremental_field: str | None = None,
) -> SourceResponse:
    endpoint_config = CODECOV_ENDPOINTS[endpoint]

    return SourceResponse(
        name=endpoint,
        items=lambda: get_rows(
            api_token=api_token,
            service=service,
            owner_username=owner_username,
            repositories=repositories,
            endpoint=endpoint,
            logger=logger,
            resumable_source_manager=resumable_source_manager,
            should_use_incremental_field=should_use_incremental_field,
            db_incremental_field_last_value=db_incremental_field_last_value,
            incremental_field=incremental_field,
        ),
        primary_keys=endpoint_config.primary_keys,
        # Incremental endpoints fan out over repos, so rows are not globally ordered on the
        # cursor (commits are newest-first per repo; the coverage trend restarts per repo).
        # Desc mode persists the watermark only at successful job end, which is the safe
        # behavior for a partial fan-out run.
        sort_mode="desc" if endpoint_config.incremental_fields else "asc",
        partition_count=1,
        partition_size=1,
        partition_mode="datetime" if endpoint_config.partition_key else None,
        partition_format="month" if endpoint_config.partition_key else None,
        partition_keys=[endpoint_config.partition_key] if endpoint_config.partition_key else None,
    )


def validate_credentials(api_token: str, service: str, owner_username: str) -> tuple[bool, int | None]:
    """Probe the owner's repos list to confirm the token/owner pair is genuine.

    Returns ``(ok, status_code)``; ``status_code`` is ``None`` on a transport error.
    """
    url = f"{_owner_base_url(service, owner_username)}/repos?{urlencode({'page_size': 1})}"
    try:
        response = make_tracked_session().get(url, headers=_get_headers(api_token), timeout=10)
    except Exception:
        return False, None
    return response.status_code == 200, response.status_code