:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import dataclasses
from collections.abc import Iterator
from datetime import UTC, date, datetime, timedelta
from typing import Any, Optional
from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.sources.render.settings import RENDER_ENDPOINTS, RenderEndpointConfig
from products.warehouse_sources.sources.sdk import (
ResumableSourceManager,
SourceResponse,
make_tracked_session,
retry_with_deadline,
source_helpers,
)
RENDER_BASE_URL = "https://api.render.com/v1"
REQUEST_TIMEOUT_SECONDS = 60
REDACTED_VALUE = "__redacted__"
class RenderRetryableError(Exception):
pass
@dataclasses.dataclass(frozen=False)
class RenderResumeConfig:
# Cursor of the last processed item — passed back as `?cursor=` to fetch what follows.
# None means "start the listing from its first page".
cursor: str | None = None
# The fan-out parent currently being processed. A stable parent-ID bookmark (not a
# positional index) so parents added/removed between a crash and the retry can't resume
# us into the wrong parent. None for top-level endpoints.
parent_id: str | None = None
def _get_headers(api_key: str) -> dict[str, str]:
return {
"Authorization": f"Bearer {api_key}",
"Accept": "application/json",
}
def _format_incremental_value(value: Any) -> str:
"""Format a watermark as an ISO 8601 UTC timestamp, the format Render's time filters document."""
if isinstance(value, datetime):
utc_value = value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
return utc_value.strftime("%Y-%m-%dT%H:%M:%SZ")
if isinstance(value, date):
return f"{value.isoformat()}T00:00:00Z"
return str(value)
def _to_utc_datetime(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
if isinstance(value, date):
return datetime(value.year, value.month, value.day, tzinfo=UTC)
if isinstance(value, str):
try:
return _to_utc_datetime(datetime.fromisoformat(value.replace("Z", "+00:00")))
except ValueError:
return None
return None
# Shared with `RenderSource.get_non_retryable_errors` so the same rejection reads the same way
# whether it surfaces while connecting the source or mid-sync.
KEY_REJECTED_MESSAGE = (
"Your Render API key is invalid or has been revoked. Create a new API key in your Render "
"account settings, then reconnect."
)
KEY_FORBIDDEN_MESSAGE = (
"Your Render API key does not have access to this resource. Check the key's workspace access, then reconnect."
)
# `validate_via_probe` reports a transport failure as a `None` status, so anything Render did not
# answer itself leaves the key unjudged. Calling it invalid sends someone off to mint a replacement
# that fails the same way.
PROBE_FAILED_MESSAGE = "PostHog couldn't check your API key with Render. Wait a few minutes and try again."
def validate_credentials(api_key: str) -> tuple[bool, str | None]:
# /owners is the cheapest authenticated probe: every valid key can list the workspaces
# it belongs to, regardless of what resources exist.
ok, status = source_helpers.validate_via_probe(
make_tracked_session,
f"{RENDER_BASE_URL}/owners?limit=1",
headers=_get_headers(api_key),
)
if ok:
return True, None
if status == 401:
return False, KEY_REJECTED_MESSAGE
if status == 403:
return False, KEY_FORBIDDEN_MESSAGE
return False, PROBE_FAILED_MESSAGE
@retry_with_deadline(
retry=retry_if_exception_type(
(
RenderRetryableError,
requests.ReadTimeout,
requests.ConnectionError,
requests.exceptions.ChunkedEncodingError,
)
),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=2, max=60),
reraise=True,
)
def _fetch_page(
session: requests.Session,
url: str,
headers: dict[str, str],
logger: FilteringBoundLogger,
response_list_key: str | None = None,
) -> list[dict[str, Any]]:
response = session.get(url, headers=headers, timeout=REQUEST_TIMEOUT_SECONDS)
# Render rate-limits per user (400/min for most GETs) and returns 429 with
# Ratelimit-* headers; exponential backoff comfortably outlasts the window.
if response.status_code == 429 or response.status_code >= 500:
raise RenderRetryableError(f"Render API error (retryable): status={response.status_code}, url={url}")
if not response.ok:
# 404 (a service deleted mid-sync) and 400 (a service type that can't own the child
# resource, e.g. custom domains on a cron job) are expected during fan-out and handled
# by the caller — log them as warnings. Anything else is a genuine error.
log = logger.warning if response.status_code in (400, 404) else logger.error
log(f"Render API error: status={response.status_code}, body={response.text}, url={url}")
response.raise_for_status()
data = response.json()
if response_list_key and isinstance(data, dict):
data = data.get(response_list_key) or []
if not isinstance(data, list):
raise ValueError(f"Render API returned a non-list response for url={url}")
return data
def _unwrap_item(item: dict[str, Any], wrapper_key: str) -> tuple[dict[str, Any], str | None]:
"""Split a list item into (resource row, sibling cursor).
Render list items wrap the resource next to its pagination cursor:
`{"service": {...}, "cursor": "..."}`. A couple of endpoints (env groups) are documented
unwrapped, so fall back to treating the item as the row itself when the key is absent.
"""
if wrapper_key in item and isinstance(item[wrapper_key], dict):
return item[wrapper_key], item.get("cursor")
return item, item.get("cursor") if isinstance(item.get("cursor"), str) else None
def _explode_metric_series(series: dict[str, Any]) -> list[dict[str, Any]]:
"""Turn one metrics time series into a row per data point.
Labels identify the series (instance, status code, host, quantile, …) and vary by metric,
so they are folded into a sorted `seriesKey` that makes (service, series, timestamp) unique.
"""
labels = {
label["field"]: label.get("value")
for label in series.get("labels") or []
if isinstance(label, dict) and "field" in label
}
series_key = ",".join(f"{field}={labels[field]}" for field in sorted(labels))
return [
{
"seriesKey": series_key,
"labels": labels,
"unit": series.get("unit"),
"timestamp": point.get("timestamp"),
"value": point.get("value"),
}
for point in series.get("values") or []
if isinstance(point, dict)
]
def _build_params(
config: RenderEndpointConfig,
owner_id: str | None,
should_use_incremental_field: bool,
db_incremental_field_last_value: Any,
incremental_field: str | None,
logger: FilteringBoundLogger,
) -> dict[str, Any]:
params: dict[str, Any] = {"limit": config.page_size} if config.paginated else {}
params.update(config.extra_params)
if config.supports_owner_filter and owner_id:
params["ownerId"] = owner_id
lower_bound: Any = None
if should_use_incremental_field and db_incremental_field_last_value is not None:
field = incremental_field or config.default_incremental_field
param = config.incremental_param_by_field.get(field) if field else None
if param:
params[param] = _format_incremental_value(db_incremental_field_last_value)
lower_bound = db_incremental_field_last_value
else:
logger.warning(
f"Render: incremental field {field} has no server-side filter for endpoint "
f"{config.name}, syncing the full endpoint instead"
)
window_param = (
config.incremental_param_by_field.get(config.default_incremental_field)
if config.default_incremental_field
else None
)
if config.max_lookback_days and window_param:
# One day of margin: the server measures the span up to its own "now", which moves on
# while a fan-out works through its parents.
floor = datetime.now(UTC) - timedelta(days=config.max_lookback_days - 1)
lower_bound_dt = _to_utc_datetime(lower_bound) if lower_bound is not None else None
if lower_bound is None or (lower_bound_dt is not None and lower_bound_dt < floor):
params[window_param] = _format_incremental_value(floor)
return params
def _walk_pages(
session: requests.Session,
headers: dict[str, str],
logger: FilteringBoundLogger,
config: RenderEndpointConfig,
path: str,
params: dict[str, Any],
cursor: str | None,
max_pages: int | None = None,
) -> Iterator[tuple[list[dict[str, Any]], str | None]]:
"""Yield (rows, next_cursor) per page, following each page's last-item cursor.
Render doesn't document an explicit end-of-results signal, so pagination only stops on an
empty page (a short page could in principle still have more results). The next cursor is
the cursor of the last item in the page.
"""
pages_fetched = 0
while True:
page_params = dict(params)
if cursor:
page_params["cursor"] = cursor
url = f"{RENDER_BASE_URL}{path}?{urlencode(page_params)}"
items = _fetch_page(session, url, headers, logger, config.response_list_key)
if not items:
return
rows: list[dict[str, Any]] = []
next_cursor: str | None = None
for item in items:
if config.metric_series:
rows.extend(_explode_metric_series(item))
continue
row, item_cursor = _unwrap_item(item, config.wrapper_key)
rows.append(row)
if item_cursor:
next_cursor = item_cursor
if not rows:
# Metric series with no data points in the window.
return
yield rows, next_cursor
if not config.paginated:
return
if next_cursor is None:
# Unwrapped responses carry no cursor, so a full page here may mean truncation.
if len(items) >= config.page_size:
logger.warning(
f"Render: endpoint {config.name} returned a full page with no pagination "
f"cursor — results may be truncated at {config.page_size} items"
)
return
pages_fetched += 1
if max_pages is not None and pages_fetched >= max_pages:
logger.warning(
f"Render: page cap of {max_pages} reached for endpoint {config.name} "
f"(path={path}) — remaining results were not fetched"
)
return
cursor = next_cursor
def _iter_parent_rows(
session: requests.Session,
headers: dict[str, str],
logger: FilteringBoundLogger,
parent_config: RenderEndpointConfig,
owner_id: str | None,
) -> Iterator[dict[str, Any]]:
"""Fully enumerate the parent endpoint (no incremental bound: a new child can land on an
old parent, so every sync must consider every parent)."""
params: dict[str, Any] = {"limit": parent_config.page_size}
if parent_config.supports_owner_filter and owner_id:
params["ownerId"] = owner_id
for rows, _ in _walk_pages(session, headers, logger, parent_config, parent_config.path, params, cursor=None):
if parent_config.owner_id_field and owner_id:
rows = [row for row in rows if row.get(parent_config.owner_id_field) == owner_id]
yield from rows
def _get_top_level_rows(
session: requests.Session,
headers: dict[str, str],
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[RenderResumeConfig],
config: RenderEndpointConfig,
params: dict[str, Any],
) -> Iterator[list[dict[str, Any]]]:
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
cursor = resume.cursor if resume else None
if cursor:
logger.debug(f"Render: resuming {config.name} from cursor={cursor}")
for rows, next_cursor in _walk_pages(session, headers, logger, config, config.path, params, cursor):
yield rows
# Save AFTER yielding so a crash re-yields the last page rather than skipping it —
# merge dedupes on the primary key.
if next_cursor:
resumable_source_manager.save_state(RenderResumeConfig(cursor=next_cursor))
def _get_fan_out_rows(
session: requests.Session,
headers: dict[str, str],
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[RenderResumeConfig],
config: RenderEndpointConfig,
params: dict[str, Any],
owner_id: str | None,
) -> Iterator[list[dict[str, Any]]]:
"""Fan out over every parent resource, fetching this child endpoint once per parent.
Fan-out runs report sort_mode="desc" (see render_source), so the incremental watermark
persists only after every parent completes — a partial run's max says nothing about
parents it never reached.
"""
assert config.parent is not None
parent_config = RENDER_ENDPOINTS[config.parent]
parents = [
(row["id"], row) for row in _iter_parent_rows(session, headers, logger, parent_config, owner_id) if "id" in row
]
# Resolve the saved parent-ID bookmark to the slice of parents still to process. If the
# bookmarked parent no longer exists (deleted between runs), start over from the first
# parent — merge dedupes the re-pulled rows on the primary key.
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
remaining = parents
resume_cursor: str | None = None
if resume is not None and resume.parent_id is not None:
parent_ids = [parent_id for parent_id, _ in parents]
if resume.parent_id in parent_ids:
remaining = parents[parent_ids.index(resume.parent_id) :]
resume_cursor = resume.cursor
logger.debug(f"Render: resuming {config.name} from parent_id={resume.parent_id}, cursor={resume_cursor}")
window_param = (
config.incremental_param_by_field.get(config.default_incremental_field)
if config.window_start_from_parent_field and config.default_incremental_field
else None
)
for index, (parent_id, parent_row) in enumerate(remaining):
child_params = dict(params)
# Events default to a one-hour server-side window, so always pass an explicit lower
# bound: the watermark when incremental, else the parent's creation time.
if window_param and window_param not in child_params:
window_start = parent_row.get(config.window_start_from_parent_field or "")
if window_start:
child_params[window_param] = window_start
if config.parent_query_param:
path = config.path
child_params[config.parent_query_param] = parent_id
else:
path = config.path.format(parent_id=parent_id)
cursor = resume_cursor
resume_cursor = None # only the resumed-into parent uses the saved cursor
try:
for rows, next_cursor in _walk_pages(
session, headers, logger, config, path, child_params, cursor, max_pages=config.max_pages_per_parent
):
if config.inject_parent_key:
rows = [{config.inject_parent_key: parent_id, **row} for row in rows]
yield rows
if next_cursor:
resumable_source_manager.save_state(RenderResumeConfig(cursor=next_cursor, parent_id=parent_id))
except requests.HTTPError as exc:
status = exc.response.status_code if exc.response is not None else None
# A parent deleted between enumeration and this fetch 404s. Skip it rather than
# failing the whole sync — the resource is genuinely gone.
if status == 404:
logger.warning(f"Render: {config.parent} {parent_id} not found while fetching {config.name}, skipping")
# Render 400s the child endpoint for parents that can't own the resource (custom
# domains on non-web/static services). Skip them instead of failing the sync.
elif status == 400 and config.skip_parent_on_bad_request:
logger.warning(f"Render: {config.parent} {parent_id} cannot have {config.name}, skipping")
elif status == 403 and config.skip_parent_on_forbidden:
logger.warning(f"Render: no access to {config.name} for {config.parent} {parent_id}, skipping")
else:
raise
# Advance the bookmark to the next parent so a crash between parents resumes correctly.
if index + 1 < len(remaining):
resumable_source_manager.save_state(RenderResumeConfig(cursor=None, parent_id=remaining[index + 1][0]))
def _redact_row(row: dict[str, Any], redact_spec: dict[str, tuple[str, ...]]) -> dict[str, Any]:
"""Strip secret values out of a row before it reaches the warehouse.
Render's env-group payload carries the actual env var values and secret-file contents; the
API key is meant to *use* those for sync, not surface them as queryable columns. Replace the
value-bearing fields with a marker while keeping the surrounding metadata (names/keys/ids).
"""
for list_field, secret_keys in redact_spec.items():
items = row.get(list_field)
if not isinstance(items, list):
continue
for item in items:
if not isinstance(item, dict):
continue
for key in secret_keys:
if key in item:
item[key] = REDACTED_VALUE
return row
def get_rows(
api_key: str,
endpoint: str,
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[RenderResumeConfig],
owner_id: str | None = None,
should_use_incremental_field: bool = False,
db_incremental_field_last_value: Any = None,
incremental_field: str | None = None,
) -> Iterator[list[dict[str, Any]]]:
config = RENDER_ENDPOINTS[endpoint]
headers = _get_headers(api_key)
# One session reused across every page (and, for fan-out, every parent) so urllib3 keeps
# the connection alive instead of re-handshaking per request. Sensitive endpoints opt out of
# HTTP sample capture — capture snapshots the raw response before row-level redaction runs.
session = make_tracked_session(capture=not config.is_sensitive)
params = _build_params(
config, owner_id, should_use_incremental_field, db_incremental_field_last_value, incremental_field, logger
)
if config.parent:
batches = _get_fan_out_rows(session, headers, logger, resumable_source_manager, config, params, owner_id)
else:
batches = _get_top_level_rows(session, headers, logger, resumable_source_manager, config, params)
if config.redact_list_item_fields:
for batch in batches:
yield [_redact_row(row, config.redact_list_item_fields) for row in batch]
else:
yield from batches
def render_source(
api_key: str,
endpoint: str,
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[RenderResumeConfig],
owner_id: str | None = None,
should_use_incremental_field: bool = False,
db_incremental_field_last_value: Optional[Any] = None,
incremental_field: str | None = None,
) -> SourceResponse:
endpoint_config = RENDER_ENDPOINTS[endpoint]
return SourceResponse(
name=endpoint,
items=lambda: get_rows(
api_key=api_key,
endpoint=endpoint,
logger=logger,
resumable_source_manager=resumable_source_manager,
owner_id=owner_id,
should_use_incremental_field=should_use_incremental_field,
db_incremental_field_last_value=db_incremental_field_last_value,
incremental_field=incremental_field,
),
primary_keys=endpoint_config.primary_keys,
# Render doesn't document list ordering and cursor pagination accepts no sort param, so
# the watermark is only safe to persist once the run completes (desc mode computes the
# max across all batches and saves it at successful job end).
sort_mode="desc",
partition_count=1,
partition_size=1,
partition_mode="datetime" if endpoint_config.partition_key else None,
partition_format="month" if endpoint_config.partition_key else None,
partition_keys=[endpoint_config.partition_key] if endpoint_config.partition_key else None,
)