:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
from dataclasses import dataclass, field
from products.warehouse_sources.sources.sdk import IncrementalField, IncrementalFieldType
def _datetime_field(name: str) -> IncrementalField:
return {
"label": name,
"type": IncrementalFieldType.DateTime,
"field": name,
"field_type": IncrementalFieldType.DateTime,
}
@dataclass(frozen=True)
class RenderEndpointConfig:
name: str
# Path relative to the API base. Fan-out children use a `{parent_id}` placeholder
# (or `parent_query_param` when the parent id is passed as a query param instead).
path: str
# Key the resource is nested under in each list item (Render wraps every item as
# `{"<wrapper_key>": {...}, "cursor": "..."}` — the cursor is a sibling of the resource).
wrapper_key: str
# Primary key columns for dedup. Fan-out children include the parent id: Render ids carry
# unique prefixes (dep-, job-, evt-) but global uniqueness isn't documented, and a
# non-unique key would multi-match on every delta merge.
primary_keys: list[str] = field(default_factory=lambda: ["id"])
incremental_fields: list[IncrementalField] = field(default_factory=list)
# Incremental field name -> server-side "after" query param (e.g. updatedAt -> updatedAfter).
# Only fields with a genuine server-side filter are advertised as incremental options.
incremental_param_by_field: dict[str, str] = field(default_factory=dict)
default_incremental_field: str | None = None
# Stable datetime field to partition by (createdAt-style, never updatedAt).
partition_key: str | None = None
# Fan-out: name of the parent endpoint whose rows seed this child's requests.
parent: str | None = None
# Pass the parent id as this query param instead of a `{parent_id}` path placeholder.
parent_query_param: str | None = None
# Column to copy the parent id onto child rows that don't carry it themselves
# (deploys and custom domains omit the service id from their payloads).
inject_parent_key: str | None = None
# Whether the endpoint accepts the `ownerId` filter (used to scope a multi-workspace key).
supports_owner_filter: bool = False
# Rows are immutable (events): append-only is the only incremental-style sync mode.
append_only: bool = False
# Fallback lower bound for endpoints whose time window defaults server-side to a recent
# slice (events default to the last hour) — read from this parent field when no
# incremental watermark applies, so a full refresh still covers the parent's lifetime.
window_start_from_parent_field: str | None = None
# Render caps one audit log export at a 90-day span and only guarantees 90 days of
# retention, so the lower bound sent is never earlier than this many days ago.
max_lookback_days: int | None = None
should_sync_default: bool = True
# False for endpoints that return the whole collection in one response and take no
# `limit`/`cursor` params (instances, members).
paginated: bool = True
page_size: int = 100
# Hard cap on pages fetched per parent in a fan-out, to bound runaway pagination.
# A structured warning is logged if the cap is reached.
max_pages_per_parent: int = 500
# Fan-out children where Render 400s parents that can't own the resource (custom domains
# only exist for web services and static sites, not cron jobs, workers or private
# services). Skip those parents rather than failing the whole sync.
skip_parent_on_bad_request: bool = False
# Fan-out children gated by the parent's plan or the key's role (audit logs need a Pro
# workspace and an admin). Skip a parent that answers 403 instead of failing the sync.
skip_parent_on_forbidden: bool = False
# Set on an endpoint whose rows are owners: when the source is scoped to one workspace,
# fan-out children only visit the parent row whose value in this field matches it.
owner_id_field: str | None = None
# Some endpoints return secrets the API key is only meant to *use* for sync (env var
# values, secret-file contents), not data worth warehousing. Maps a list column to the
# item keys whose values are redacted before a row is yielded, keeping the safe metadata
# (names, keys, ids). Any endpoint with entries here is treated as sensitive: its raw
# responses are also excluded from HTTP sample capture (see `is_sensitive`).
redact_list_item_fields: dict[str, tuple[str, ...]] = field(default_factory=dict)
# Endpoints that answer with an object holding the list under this key instead of a bare
# list (Postgres query insights: `{"topQueries": [...]}`).
response_list_key: str | None = None
# Metrics endpoints return time series (`{labels, values: [{timestamp, value}], unit}`)
# that are exploded into one row per data point.
metric_series: bool = False
# Static query params sent on every request.
extra_params: dict[str, str | int] = field(default_factory=dict)
@property
def is_sensitive(self) -> bool:
return bool(self.redact_list_item_fields)
RENDER_ENDPOINTS: dict[str, RenderEndpointConfig] = {
"owners": RenderEndpointConfig(
name="owners",
path="/owners",
wrapper_key="owner",
# No timestamps on the owner object and no time filters on the endpoint: full refresh only.
owner_id_field="id",
),
"projects": RenderEndpointConfig(
name="projects",
path="/projects",
wrapper_key="project",
incremental_fields=[_datetime_field("updatedAt"), _datetime_field("createdAt")],
incremental_param_by_field={"updatedAt": "updatedAfter", "createdAt": "createdAfter"},
default_incremental_field="updatedAt",
supports_owner_filter=True,
),
"environments": RenderEndpointConfig(
name="environments",
path="/environments",
wrapper_key="environment",
# The environment object carries no timestamps, so there is no watermark column to
# sync incrementally on despite the endpoint's updatedBefore/After filters.
primary_keys=["projectId", "id"],
parent="projects",
parent_query_param="projectId",
),
"services": RenderEndpointConfig(
name="services",
path="/services",
wrapper_key="service",
incremental_fields=[_datetime_field("updatedAt"), _datetime_field("createdAt")],
incremental_param_by_field={"updatedAt": "updatedAfter", "createdAt": "createdAfter"},
default_incremental_field="updatedAt",
partition_key="createdAt",
supports_owner_filter=True,
),
"deploys": RenderEndpointConfig(
name="deploys",
path="/services/{parent_id}/deploys",
wrapper_key="deploy",
primary_keys=["serviceId", "id"],
incremental_fields=[
_datetime_field("finishedAt"),
_datetime_field("updatedAt"),
_datetime_field("createdAt"),
],
incremental_param_by_field={
"finishedAt": "finishedAfter",
"updatedAt": "updatedAfter",
"createdAt": "createdAfter",
},
# finishedAt: in-flight deploys (null finishedAt) are excluded by finishedAfter and
# picked up once they reach a terminal state, so each deploy lands exactly once with
# its final status — the right grain for deploy analytics.
default_incremental_field="finishedAt",
partition_key="createdAt",
parent="services",
inject_parent_key="serviceId",
),
"jobs": RenderEndpointConfig(
name="jobs",
path="/services/{parent_id}/jobs",
wrapper_key="job",
primary_keys=["serviceId", "id"],
incremental_fields=[
_datetime_field("finishedAt"),
_datetime_field("startedAt"),
_datetime_field("createdAt"),
],
incremental_param_by_field={
"finishedAt": "finishedAfter",
"startedAt": "startedAfter",
"createdAt": "createdAfter",
},
default_incremental_field="finishedAt",
partition_key="createdAt",
parent="services",
),
"events": RenderEndpointConfig(
name="events",
path="/services/{parent_id}/events",
wrapper_key="event",
primary_keys=["serviceId", "id"],
incremental_fields=[_datetime_field("timestamp")],
incremental_param_by_field={"timestamp": "startTime"},
default_incremental_field="timestamp",
partition_key="timestamp",
parent="services",
append_only=True,
# The events window defaults to the last hour server-side, so every request must pass
# an explicit startTime; events can't predate their service, so its createdAt is a
# complete lower bound for full refreshes and first syncs.
window_start_from_parent_field="createdAt",
),
"custom_domains": RenderEndpointConfig(
name="custom_domains",
path="/services/{parent_id}/custom-domains",
wrapper_key="customDomain",
primary_keys=["serviceId", "id"],
# Only createdBefore/After filters exist, but verificationStatus mutates after
# creation — a createdAt cursor would freeze it. Tiny table: full refresh only.
parent="services",
inject_parent_key="serviceId",
skip_parent_on_bad_request=True,
),
"env_groups": RenderEndpointConfig(
name="env_groups",
path="/env-groups",
# The docs show env group list items unwrapped (no cursor sibling); the transport
# falls back to treating the item as the row when the wrapper key is absent.
wrapper_key="envGroup",
incremental_fields=[_datetime_field("updatedAt"), _datetime_field("createdAt")],
incremental_param_by_field={"updatedAt": "updatedAfter", "createdAt": "createdAfter"},
default_incremental_field="updatedAt",
supports_owner_filter=True,
# The list response embeds env var values and secret-file contents (database URLs, API
# tokens, …). Redact the value-bearing fields so only names/keys/ids reach the warehouse.
redact_list_item_fields={"envVars": ("value",), "secretFiles": ("content", "contents")},
),
"postgres": RenderEndpointConfig(
name="postgres",
path="/postgres",
wrapper_key="postgres",
incremental_fields=[_datetime_field("updatedAt"), _datetime_field("createdAt")],
incremental_param_by_field={"updatedAt": "updatedAfter", "createdAt": "createdAfter"},
default_incremental_field="updatedAt",
supports_owner_filter=True,
),
"key_value": RenderEndpointConfig(
name="key_value",
path="/key-value",
wrapper_key="keyValue",
incremental_fields=[_datetime_field("updatedAt"), _datetime_field("createdAt")],
incremental_param_by_field={"updatedAt": "updatedAfter", "createdAt": "createdAfter"},
default_incremental_field="updatedAt",
supports_owner_filter=True,
),
"disks": RenderEndpointConfig(
name="disks",
path="/disks",
wrapper_key="disk",
incremental_fields=[_datetime_field("updatedAt"), _datetime_field("createdAt")],
incremental_param_by_field={"updatedAt": "updatedAfter", "createdAt": "createdAfter"},
default_incremental_field="updatedAt",
supports_owner_filter=True,
),
"instances": RenderEndpointConfig(
name="instances",
path="/services/{parent_id}/instances",
wrapper_key="instance",
primary_keys=["serviceId", "id"],
# No time filters on the endpoint: full refresh only.
partition_key="createdAt",
parent="services",
inject_parent_key="serviceId",
paginated=False,
),
"members": RenderEndpointConfig(
name="members",
path="/owners/{parent_id}/members",
wrapper_key="member",
primary_keys=["ownerId", "userId"],
parent="owners",
inject_parent_key="ownerId",
paginated=False,
),
"audit_logs": RenderEndpointConfig(
name="audit_logs",
path="/owners/{parent_id}/audit-logs",
wrapper_key="auditLog",
primary_keys=["ownerId", "id"],
incremental_fields=[_datetime_field("timestamp")],
incremental_param_by_field={"timestamp": "startTime"},
default_incremental_field="timestamp",
partition_key="timestamp",
parent="owners",
inject_parent_key="ownerId",
append_only=True,
max_lookback_days=90,
# Audit logs need a Pro workspace and an admin key. One ineligible workspace must not
# fail the table for a key that spans several workspaces.
skip_parent_on_bad_request=True,
skip_parent_on_forbidden=True,
page_size=1000,
),
"blueprints": RenderEndpointConfig(
name="blueprints",
path="/blueprints",
wrapper_key="blueprint",
# No time filters on the endpoint, and lastSync is the only timestamp: full refresh only.
supports_owner_filter=True,
),
"blueprint_syncs": RenderEndpointConfig(
name="blueprint_syncs",
path="/blueprints/{parent_id}/syncs",
wrapper_key="sync",
primary_keys=["blueprintId", "id"],
# No time filters on the endpoint, and startedAt is optional: full refresh, unpartitioned.
parent="blueprints",
inject_parent_key="blueprintId",
),
"disk_snapshots": RenderEndpointConfig(
name="disk_snapshots",
path="/disks/{parent_id}/snapshots",
wrapper_key="snapshot",
primary_keys=["diskId", "snapshotKey"],
partition_key="createdAt",
parent="disks",
inject_parent_key="diskId",
paginated=False,
),
"postgres_top_queries": RenderEndpointConfig(
name="postgres_top_queries",
path="/postgres/{parent_id}/query/top-queries",
wrapper_key="topQuery",
# A cumulative pg_stat_statements snapshot (top 50 by total time): full refresh only.
primary_keys=["postgresId", "queryId"],
parent="postgres",
inject_parent_key="postgresId",
paginated=False,
response_list_key="topQueries",
# Query insights are unavailable on some instances (e.g. suspended ones). One such
# database must not fail the table for the rest.
skip_parent_on_bad_request=True,
),
"postgres_table_scans": RenderEndpointConfig(
name="postgres_table_scans",
path="/postgres/{parent_id}/query/table-scans",
wrapper_key="tableScan",
primary_keys=["postgresId", "Database", "Schema", "Table"],
parent="postgres",
inject_parent_key="postgresId",
paginated=False,
response_list_key="tableScans",
skip_parent_on_bad_request=True,
),
}
def _service_metric(
name: str, path: str, resolution_seconds: int | None = 300, skip_parent_on_bad_request: bool = False
) -> RenderEndpointConfig:
return RenderEndpointConfig(
name=name,
path=path,
wrapper_key="series",
primary_keys=["serviceId", "seriesKey", "timestamp"],
# startTime filters server-side. Merge (not append) so the still-filling latest bucket
# is overwritten by the next sync instead of duplicated.
incremental_fields=[_datetime_field("timestamp")],
incremental_param_by_field={"timestamp": "startTime"},
default_incremental_field="timestamp",
partition_key="timestamp",
parent="services",
parent_query_param="resource",
inject_parent_key="serviceId",
paginated=False,
metric_series=True,
# The window defaults to the last hour server-side. Render keeps at most 30 days of
# metrics (less on smaller plans), so a full refresh reads from that floor.
max_lookback_days=30,
# 5-minute buckets keep a 30-day window to a few thousand points per series.
extra_params={"resolutionSeconds": resolution_seconds} if resolution_seconds else {},
# High-volume time series: opt-in rather than synced with every new source.
should_sync_default=False,
skip_parent_on_bad_request=skip_parent_on_bad_request,
)
RENDER_ENDPOINTS.update(
{
"metrics_cpu": _service_metric("metrics_cpu", "/metrics/cpu"),
"metrics_memory": _service_metric("metrics_memory", "/metrics/memory"),
"metrics_http_requests": _service_metric("metrics_http_requests", "/metrics/http-requests"),
"metrics_http_latency": _service_metric(
# Render 400s HTTP latency for services in Hobby workspaces. One such service must not
# fail the table for a key that also spans paid workspaces.
"metrics_http_latency",
"/metrics/http-latency",
skip_parent_on_bad_request=True,
),
"metrics_instance_count": _service_metric("metrics_instance_count", "/metrics/instance-count"),
"metrics_bandwidth": _service_metric(
# The bandwidth endpoint takes no resolution param.
"metrics_bandwidth",
"/metrics/bandwidth",
resolution_seconds=None,
),
}
)
ENDPOINTS = tuple(RENDER_ENDPOINTS.keys())
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
name: config.incremental_fields for name, config in RENDER_ENDPOINTS.items()
}