:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import dataclasses
from collections.abc import Iterator
from datetime import UTC, date, datetime
from typing import Any, Optional
from urllib.parse import quote, urlencode
import requests
from dateutil import parser as dateutil_parser
from structlog.types import FilteringBoundLogger
from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.sources.codecov.settings import (
CODECOV_ENDPOINTS,
CodecovEndpointConfig,
CodecovParentConfig,
CodecovResponseShape,
)
from products.warehouse_sources.sources.sdk import (
ResumableSourceManager,
SourceResponse,
make_tracked_session,
retry_with_deadline,
)
CODECOV_BASE_URL = "https://api.codecov.io/api/v2"
PAGE_SIZE = 500 # Codecov caps page_size at 500
class CodecovRetryableError(Exception):
pass
@dataclasses.dataclass(frozen=False)
class CodecovResumeConfig:
# Next page URL to fetch. None means "start the bookmarked repo's endpoint at its first
# page" — used when the bookmark advances to a repo whose first page URL isn't built yet.
next_url: str | None = None
# The repository currently being processed during fan-out. A stable name bookmark (not a
# positional index) so repos added/removed between a crash and the retry can't resume us
# into the wrong repo. None for the top-level repos endpoint.
repo: str | None = None
def _get_headers(api_token: str) -> dict[str, str]:
# Codecov's documented scheme is the lowercase "bearer" keyword.
return {"Authorization": f"bearer {api_token}", "Accept": "application/json"}
def _owner_base_url(service: str, owner_username: str) -> str:
# Both segments are user-supplied; percent-encode them (including "/") so they can't
# retarget the request path.
return f"{CODECOV_BASE_URL}/{quote(service, safe='')}/{quote(owner_username, safe='')}"
def _endpoint_url(owner_base_url: str, path: str, params: dict[str, Any], **segments: str) -> str:
if segments:
path = path.format(**{name: quote(value, safe="") for name, value in segments.items()})
url = f"{owner_base_url}{path}"
return f"{url}?{urlencode(params)}" if params else url
def _ensure_codecov_url(url: str) -> str:
"""Upgrade and pin a pagination/resume URL to the Codecov API origin.
Codecov's DRF `next` links come back with an http:// scheme, so upgrade them rather than
send the bearer token in plaintext. Beyond that, `next` is response-controlled and resume
URLs are replayed from persisted state, so refuse any URL outside the API origin — the
token must never be sent to a host we didn't choose.
"""
if url.startswith("http://"):
url = "https://" + url.removeprefix("http://")
if not url.startswith(f"{CODECOV_BASE_URL}/"):
raise ValueError(f"Refusing to follow non-Codecov URL: {url}")
return url
def _as_utc(dt: datetime) -> datetime:
return dt.replace(tzinfo=UTC) if dt.tzinfo is None else dt.astimezone(UTC)
def _normalize_cutoff(value: Any) -> datetime | None:
if isinstance(value, datetime):
return _as_utc(value)
if isinstance(value, date):
return datetime.combine(value, datetime.min.time(), tzinfo=UTC)
return None
def _is_older_than_cutoff(value: Any, cutoff: datetime) -> bool:
if value is None:
return False
if isinstance(value, str):
try:
parsed = dateutil_parser.parse(value)
except (ValueError, TypeError):
return False
elif isinstance(value, datetime):
parsed = value
else:
return False
return _as_utc(parsed) <= cutoff
def _should_stop_desc(items: list[dict[str, Any]], cutoff_field: str | None, cutoff: datetime | None) -> bool:
"""Newest-first + incremental can stop the moment a page holds a pre-watermark record."""
if not cutoff_field or cutoff is None or not items:
return False
return any(_is_older_than_cutoff(item.get(cutoff_field), cutoff) for item in items if item)
def _format_start_date(cutoff: datetime) -> str:
# Codecov's coverage report takes calendar dates; the boundary interval is re-returned
# and deduped by the primary-key merge.
return cutoff.date().isoformat()
def _totals_rows(data: Any, repo: str) -> list[dict[str, Any]]:
"""One row of headline coverage per repository, with `totals` flattened into the root.
The per-file breakdown the same response carries is the `report_files` grain, so it is
dropped here — which also means the row's `files` is the file *count* from `totals`.
"""
if not data:
return []
return [{**(data.get("totals") or {}), "repo": repo, "commit_file_url": data.get("commit_file_url")}]
def _report_file_rows(data: Any, repo: str) -> list[dict[str, Any]]:
return [{**file, "repo": repo} for file in (data or {}).get("files") or []]
def _tree_rows(nodes: Any, repo: str) -> Iterator[dict[str, Any]]:
"""Flatten the nested directory tree into one row per node, at every depth."""
for node in nodes or []:
yield {**{key: value for key, value in node.items() if key != "children"}, "repo": repo}
yield from _tree_rows(node.get("children"), repo)
def _impacted_file_rows(data: Any, repo: str) -> list[dict[str, Any]]:
"""One row per impacted file, with the commits the comparison ran between.
`files` stays empty while Codecov is still computing the comparison (`state: "pending"`).
"""
if not data:
return []
comparison = {
"base_commit": data.get("base_commit"),
"head_commit": data.get("head_commit"),
"state": data.get("state"),
}
return [{**file, **comparison, "repo": repo} for file in data.get("files") or []]
def _unpaginated_rows(shape: CodecovResponseShape, data: Any, repo: str) -> list[dict[str, Any]]:
match shape:
case CodecovResponseShape.BARE_LIST:
return [{**item, "repo": repo} for item in data or []]
case CodecovResponseShape.TOTALS:
return _totals_rows(data, repo)
case CodecovResponseShape.REPORT_FILES:
return _report_file_rows(data, repo)
case CodecovResponseShape.TREE:
return list(_tree_rows(data, repo))
case CodecovResponseShape.IMPACTED_FILES:
return _impacted_file_rows(data, repo)
case _:
raise ValueError(f"Endpoint response shape {shape} is paginated")
def parse_repositories(repositories: str | None) -> list[str]:
"""Split the optional comma-separated repository allow-list into clean names."""
if not repositories:
return []
return [name.strip() for name in repositories.split(",") if name.strip()]
@retry_with_deadline(
retry=retry_if_exception_type(
(
CodecovRetryableError,
requests.ReadTimeout,
requests.ConnectionError,
requests.exceptions.ChunkedEncodingError,
)
),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
reraise=True,
)
def _fetch_page(session: requests.Session, url: str, headers: dict[str, str], logger: FilteringBoundLogger) -> Any:
response = session.get(url, headers=headers, timeout=60)
if response.status_code == 429 or response.status_code >= 500:
raise CodecovRetryableError(f"Codecov API error (retryable): status={response.status_code}, url={url}")
if not response.ok:
# 404 is expected and handled during fan-out (a repo deleted mid-sync).
log = logger.warning if response.status_code == 404 else logger.error
log(f"Codecov API error: status={response.status_code}, body={response.text}, url={url}")
response.raise_for_status()
return response.json()
def _iter_pages(
session: requests.Session, url: str, headers: dict[str, str], logger: FilteringBoundLogger
) -> Iterator[tuple[list[dict[str, Any]], str | None]]:
"""Yield (results, next_url) per page of a DRF-paginated Codecov list, following `next`."""
while True:
data = _fetch_page(session, url, headers, logger)
results = data.get("results") or []
next_url = data.get("next")
next_url = _ensure_codecov_url(next_url) if next_url else None
yield results, next_url
if not next_url:
return
url = next_url
def _iter_repo_names(
session: requests.Session,
owner_base_url: str,
headers: dict[str, str],
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[CodecovResumeConfig],
) -> Iterator[str]:
"""Page through the owner's active repos and yield each name. Inactive repos (never
received an upload) have no coverage data, so fanning out over them is wasted API cost."""
url = f"{owner_base_url}/repos?{urlencode({'active': 'true', 'page_size': PAGE_SIZE})}"
for results, _next_url in _iter_pages(session, url, headers, logger):
for item in results:
yield item["name"]
# The repo list holds parents, not rows, so the walk can stop here and restart.
resumable_source_manager.safe_point()
def _get_top_level_rows(
session: requests.Session,
owner_base_url: str,
headers: dict[str, str],
config: CodecovEndpointConfig,
repositories: list[str],
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[CodecovResumeConfig],
) -> Iterator[list[dict[str, Any]]]:
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
if resume is not None and resume.next_url:
url = _ensure_codecov_url(resume.next_url)
logger.debug(f"Codecov: resuming {config.name} from URL: {url}")
else:
url = _endpoint_url(owner_base_url, config.path, {"page_size": PAGE_SIZE, **config.extra_params})
allowed = set(repositories)
for results, next_url in _iter_pages(session, url, headers, logger):
rows = results
if config.filtered_by_repository_allow_list and allowed:
# The repos list has no server-side name filter, so the allow-list applies client-side.
rows = [item for item in results if item.get("name") in allowed]
if rows:
yield rows
# Save AFTER yielding so a crash re-yields the last page rather than skipping it —
# merge dedupes on the primary key.
if next_url:
resumable_source_manager.save_state(CodecovResumeConfig(next_url=next_url))
resumable_source_manager.safe_point()
def _remaining_repos(
repo_names: list[str],
config: CodecovEndpointConfig,
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[CodecovResumeConfig],
) -> tuple[list[str], str | None]:
"""Resolve the saved repo bookmark to the slice of repos still to process, and the URL the
first of them resumes from.
If the bookmarked repo no longer exists (deleted between runs), start over from the first —
merge dedupes the re-pulled rows on the primary key.
"""
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
if resume is None or resume.repo is None or resume.repo not in repo_names:
return repo_names, None
resume_url = _ensure_codecov_url(resume.next_url) if resume.next_url else None
logger.debug(f"Codecov: resuming {config.name} from repo={resume.repo}, url={resume_url}")
return repo_names[repo_names.index(resume.repo) :], resume_url
def _get_fan_out_rows(
session: requests.Session,
owner_base_url: str,
headers: dict[str, str],
config: CodecovEndpointConfig,
repositories: list[str],
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[CodecovResumeConfig],
should_use_incremental_field: bool,
db_incremental_field_last_value: Any,
incremental_field: str | None,
) -> Iterator[list[dict[str, Any]]]:
"""Fan out one paginated walk per repository, injecting the repo name onto every row (it
feeds each endpoint's composite primary key)."""
repo_names = repositories or list(
_iter_repo_names(session, owner_base_url, headers, logger, resumable_source_manager)
)
cutoff = _normalize_cutoff(db_incremental_field_last_value) if should_use_incremental_field else None
params: dict[str, Any] = {**config.extra_params}
if config.response_shape is CodecovResponseShape.PAGINATED_LIST:
params = {"page_size": PAGE_SIZE, **config.extra_params}
stop_field: str | None = None
if cutoff is not None and config.incremental_fields:
if config.incremental_server_param:
params[config.incremental_server_param] = _format_start_date(cutoff)
else:
stop_field = incremental_field or config.default_incremental_field
remaining, resume_url = _remaining_repos(repo_names, config, logger, resumable_source_manager)
for index, repo in enumerate(remaining):
url = resume_url or _endpoint_url(owner_base_url, config.path, params, repo=repo)
resume_url = None # only the resumed-into repo uses the saved URL; the rest start fresh
try:
if config.response_shape is not CodecovResponseShape.PAGINATED_LIST:
# Single-response endpoints: a bare array, or a report object we reshape.
rows = _unpaginated_rows(config.response_shape, _fetch_page(session, url, headers, logger), repo)
if rows:
yield rows
else:
for results, next_url in _iter_pages(session, url, headers, logger):
stop_after_this_page = _should_stop_desc(results, stop_field, cutoff)
rows = [{**item, "repo": repo} for item in results]
if rows:
yield rows
# Save AFTER yielding (and only when more pages remain) so a crash
# re-yields the last page rather than skipping it.
if next_url and not stop_after_this_page:
resumable_source_manager.save_state(CodecovResumeConfig(next_url=next_url, repo=repo))
resumable_source_manager.safe_point()
if stop_after_this_page:
break
except requests.HTTPError as exc:
# A repo deleted (or deactivated) between enumeration and this fetch 404s. Skip it
# rather than failing the whole sync. Any other HTTP error is re-raised.
if exc.response is not None and exc.response.status_code == 404:
logger.warning(f"Codecov: repo {repo} not found while fetching {config.name}, skipping")
else:
raise
# Advance the bookmark to the next repo so a crash between repos resumes correctly.
if index + 1 < len(remaining):
resumable_source_manager.save_state(CodecovResumeConfig(next_url=None, repo=remaining[index + 1]))
resumable_source_manager.safe_point()
def _iter_child_rows(
session: requests.Session,
owner_base_url: str,
headers: dict[str, str],
config: CodecovEndpointConfig,
parent: CodecovParentConfig,
parent_row: dict[str, Any],
repo: str,
params: dict[str, Any],
logger: FilteringBoundLogger,
) -> Iterator[list[dict[str, Any]]]:
"""Fetch the child endpoint for one parent row, yielding each page with the parent columns."""
key = parent_row.get(parent.key_field)
if key is None:
return
parent_columns = {column: parent_row.get(name) for name, column in parent.include_fields.items()}
if parent.key_query_param:
url = _endpoint_url(owner_base_url, config.path, {parent.key_query_param: key, **params}, repo=repo)
else:
url = _endpoint_url(owner_base_url, config.path, params, repo=repo, key=str(key))
try:
if config.response_shape is CodecovResponseShape.PAGINATED_LIST:
for results, _next_url in _iter_pages(session, url, headers, logger):
yield [{**item, **parent_columns, "repo": repo} for item in results]
else:
data = _fetch_page(session, url, headers, logger)
yield [{**row, **parent_columns} for row in _unpaginated_rows(config.response_shape, data, repo)]
except requests.HTTPError as exc:
# A pull missing its base or head commit has no comparison, and a flag or commit can go
# away between listing it and this fetch. Skip that row rather than fail the sync.
if exc.response is not None and exc.response.status_code == 404:
logger.warning(f"Codecov: {config.name} not found for repo={repo}, {parent.key_field}={key}, skipping")
else:
raise
def _iter_parent_fan_out_batches(
session: requests.Session,
owner_base_url: str,
headers: dict[str, str],
config: CodecovEndpointConfig,
parent: CodecovParentConfig,
repositories: list[str],
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[CodecovResumeConfig],
cutoff: datetime | None,
) -> Iterator[tuple[list[dict[str, Any]], CodecovResumeConfig | None]]:
"""Yield (rows, state) per child response. `state` is set, with no rows, once every row of a
parent page has been yielded: it is the position to resume from after that page."""
repo_names = repositories or list(
_iter_repo_names(session, owner_base_url, headers, logger, resumable_source_manager)
)
parent_params: dict[str, Any] = {"page_size": PAGE_SIZE}
child_params: dict[str, Any] = {**config.extra_params}
if config.response_shape is CodecovResponseShape.PAGINATED_LIST:
child_params = {"page_size": PAGE_SIZE, **config.extra_params}
stop_field: str | None = None
if cutoff is not None:
if config.incremental_server_param:
child_params[config.incremental_server_param] = _format_start_date(cutoff)
if parent.incremental_server_param:
parent_params[parent.incremental_server_param] = _format_start_date(cutoff)
stop_field = parent.incremental_stop_field
remaining, resume_url = _remaining_repos(repo_names, config, logger, resumable_source_manager)
for index, repo in enumerate(remaining):
next_repo = (
CodecovResumeConfig(next_url=None, repo=remaining[index + 1]) if index + 1 < len(remaining) else None
)
url = resume_url or _endpoint_url(owner_base_url, parent.path, parent_params, repo=repo)
resume_url = None
try:
for parents, next_url in _iter_pages(session, url, headers, logger):
for parent_row in parents:
for rows in _iter_child_rows(
session, owner_base_url, headers, config, parent, parent_row, repo, child_params, logger
):
yield rows, None
if not next_url or _should_stop_desc(parents, stop_field, cutoff):
break
yield [], CodecovResumeConfig(next_url=next_url, repo=repo)
except requests.HTTPError as exc:
# The parent list of a repo deleted (or deactivated) since enumeration 404s.
if exc.response is not None and exc.response.status_code == 404:
logger.warning(f"Codecov: repo {repo} not found while fetching {config.name}, skipping")
else:
raise
yield [], next_repo
def _get_parent_fan_out_rows(
session: requests.Session,
owner_base_url: str,
headers: dict[str, str],
config: CodecovEndpointConfig,
parent: CodecovParentConfig,
repositories: list[str],
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[CodecovResumeConfig],
should_use_incremental_field: bool,
db_incremental_field_last_value: Any,
) -> Iterator[list[dict[str, Any]]]:
"""Fan out over a list inside each repository (flags, commits or pulls), then fetch the
child endpoint once per row of that list."""
cutoff = _normalize_cutoff(db_incremental_field_last_value) if should_use_incremental_field else None
for rows, state in _iter_parent_fan_out_batches(
session, owner_base_url, headers, config, parent, repositories, logger, resumable_source_manager, cutoff
):
if state is not None:
resumable_source_manager.save_state(state)
if rows:
yield rows
else:
# Many parent rows have no child rows (a commit without uploads, a pull without a
# comparison). Nothing is buffered here, so a shutdown can hand the run over.
resumable_source_manager.safe_point()
def get_rows(
api_token: str,
service: str,
owner_username: str,
repositories: str | None,
endpoint: str,
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[CodecovResumeConfig],
should_use_incremental_field: bool = False,
db_incremental_field_last_value: Any = None,
incremental_field: str | None = None,
) -> Iterator[list[dict[str, Any]]]:
config = CODECOV_ENDPOINTS[endpoint]
headers = _get_headers(api_token)
owner_base_url = _owner_base_url(service, owner_username)
repo_filter = parse_repositories(repositories)
# One session reused across every page (and repo) so urllib3 keeps the connection alive.
session = make_tracked_session()
if config.parent is not None:
yield from _get_parent_fan_out_rows(
session,
owner_base_url,
headers,
config,
config.parent,
repo_filter,
logger,
resumable_source_manager,
should_use_incremental_field,
db_incremental_field_last_value,
)
elif config.fan_out_over_repos:
yield from _get_fan_out_rows(
session,
owner_base_url,
headers,
config,
repo_filter,
logger,
resumable_source_manager,
should_use_incremental_field,
db_incremental_field_last_value,
incremental_field,
)
else:
yield from _get_top_level_rows(
session, owner_base_url, headers, config, repo_filter, logger, resumable_source_manager
)
def codecov_source(
api_token: str,
service: str,
owner_username: str,
repositories: str | None,
endpoint: str,
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[CodecovResumeConfig],
should_use_incremental_field: bool = False,
db_incremental_field_last_value: Optional[Any] = None,
incremental_field: str | None = None,
) -> SourceResponse:
endpoint_config = CODECOV_ENDPOINTS[endpoint]
return SourceResponse(
name=endpoint,
items=lambda: get_rows(
api_token=api_token,
service=service,
owner_username=owner_username,
repositories=repositories,
endpoint=endpoint,
logger=logger,
resumable_source_manager=resumable_source_manager,
should_use_incremental_field=should_use_incremental_field,
db_incremental_field_last_value=db_incremental_field_last_value,
incremental_field=incremental_field,
),
primary_keys=endpoint_config.primary_keys,
# Incremental endpoints fan out over repos, so rows are not globally ordered on the
# cursor (commits are newest-first per repo; the coverage trend restarts per repo).
# Desc mode persists the watermark only at successful job end, which is the safe
# behavior for a partial fan-out run.
sort_mode="desc" if endpoint_config.incremental_fields else "asc",
partition_count=1,
partition_size=1,
partition_mode="datetime" if endpoint_config.partition_key else None,
partition_format="month" if endpoint_config.partition_key else None,
partition_keys=[endpoint_config.partition_key] if endpoint_config.partition_key else None,
)
def validate_credentials(api_token: str, service: str, owner_username: str) -> tuple[bool, int | None]:
"""Probe the owner's repos list to confirm the token/owner pair is genuine.
Returns ``(ok, status_code)``; ``status_code`` is ``None`` on a transport error.
"""
url = f"{_owner_base_url(service, owner_username)}/repos?{urlencode({'page_size': 1})}"
try:
response = make_tracked_session().get(url, headers=_get_headers(api_token), timeout=10)
except Exception:
return False, None
return response.status_code == 200, response.status_code