Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

17.1 KB
Raw Blame
from dataclasses import dataclass, field

from products.warehouse_sources.sources.sdk import IncrementalField, IncrementalFieldType


def _datetime_field(name: str) -> IncrementalField:
    return {
        "label": name,
        "type": IncrementalFieldType.DateTime,
        "field": name,
        "field_type": IncrementalFieldType.DateTime,
    }


@dataclass(frozen=True)
class RenderEndpointConfig:
    name: str
    # Path relative to the API base. Fan-out children use a `{parent_id}` placeholder
    # (or `parent_query_param` when the parent id is passed as a query param instead).
    path: str
    # Key the resource is nested under in each list item (Render wraps every item as
    # `{"<wrapper_key>": {...}, "cursor": "..."}` — the cursor is a sibling of the resource).
    wrapper_key: str
    # Primary key columns for dedup. Fan-out children include the parent id: Render ids carry
    # unique prefixes (dep-, job-, evt-) but global uniqueness isn't documented, and a
    # non-unique key would multi-match on every delta merge.
    primary_keys: list[str] = field(default_factory=lambda: ["id"])
    incremental_fields: list[IncrementalField] = field(default_factory=list)
    # Incremental field name -> server-side "after" query param (e.g. updatedAt -> updatedAfter).
    # Only fields with a genuine server-side filter are advertised as incremental options.
    incremental_param_by_field: dict[str, str] = field(default_factory=dict)
    default_incremental_field: str | None = None
    # Stable datetime field to partition by (createdAt-style, never updatedAt).
    partition_key: str | None = None
    # Fan-out: name of the parent endpoint whose rows seed this child's requests.
    parent: str | None = None
    # Pass the parent id as this query param instead of a `{parent_id}` path placeholder.
    parent_query_param: str | None = None
    # Column to copy the parent id onto child rows that don't carry it themselves
    # (deploys and custom domains omit the service id from their payloads).
    inject_parent_key: str | None = None
    # Whether the endpoint accepts the `ownerId` filter (used to scope a multi-workspace key).
    supports_owner_filter: bool = False
    # Rows are immutable (events): append-only is the only incremental-style sync mode.
    append_only: bool = False
    # Fallback lower bound for endpoints whose time window defaults server-side to a recent
    # slice (events default to the last hour) — read from this parent field when no
    # incremental watermark applies, so a full refresh still covers the parent's lifetime.
    window_start_from_parent_field: str | None = None
    # Render caps one audit log export at a 90-day span and only guarantees 90 days of
    # retention, so the lower bound sent is never earlier than this many days ago.
    max_lookback_days: int | None = None
    should_sync_default: bool = True
    # False for endpoints that return the whole collection in one response and take no
    # `limit`/`cursor` params (instances, members).
    paginated: bool = True
    page_size: int = 100
    # Hard cap on pages fetched per parent in a fan-out, to bound runaway pagination.
    # A structured warning is logged if the cap is reached.
    max_pages_per_parent: int = 500
    # Fan-out children where Render 400s parents that can't own the resource (custom domains
    # only exist for web services and static sites, not cron jobs, workers or private
    # services). Skip those parents rather than failing the whole sync.
    skip_parent_on_bad_request: bool = False
    # Fan-out children gated by the parent's plan or the key's role (audit logs need a Pro
    # workspace and an admin). Skip a parent that answers 403 instead of failing the sync.
    skip_parent_on_forbidden: bool = False
    # Set on an endpoint whose rows are owners: when the source is scoped to one workspace,
    # fan-out children only visit the parent row whose value in this field matches it.
    owner_id_field: str | None = None
    # Some endpoints return secrets the API key is only meant to *use* for sync (env var
    # values, secret-file contents), not data worth warehousing. Maps a list column to the
    # item keys whose values are redacted before a row is yielded, keeping the safe metadata
    # (names, keys, ids). Any endpoint with entries here is treated as sensitive: its raw
    # responses are also excluded from HTTP sample capture (see `is_sensitive`).
    redact_list_item_fields: dict[str, tuple[str, ...]] = field(default_factory=dict)
    # Endpoints that answer with an object holding the list under this key instead of a bare
    # list (Postgres query insights: `{"topQueries": [...]}`).
    response_list_key: str | None = None
    # Metrics endpoints return time series (`{labels, values: [{timestamp, value}], unit}`)
    # that are exploded into one row per data point.
    metric_series: bool = False
    # Static query params sent on every request.
    extra_params: dict[str, str | int] = field(default_factory=dict)

    @property
    def is_sensitive(self) -> bool:
        return bool(self.redact_list_item_fields)


RENDER_ENDPOINTS: dict[str, RenderEndpointConfig] = {
    "owners": RenderEndpointConfig(
        name="owners",
        path="/owners",
        wrapper_key="owner",
        # No timestamps on the owner object and no time filters on the endpoint: full refresh only.
        owner_id_field="id",
    ),
    "projects": RenderEndpointConfig(
        name="projects",
        path="/projects",
        wrapper_key="project",
        incremental_fields=[_datetime_field("updatedAt"), _datetime_field("createdAt")],
        incremental_param_by_field={"updatedAt": "updatedAfter", "createdAt": "createdAfter"},
        default_incremental_field="updatedAt",
        supports_owner_filter=True,
    ),
    "environments": RenderEndpointConfig(
        name="environments",
        path="/environments",
        wrapper_key="environment",
        # The environment object carries no timestamps, so there is no watermark column to
        # sync incrementally on despite the endpoint's updatedBefore/After filters.
        primary_keys=["projectId", "id"],
        parent="projects",
        parent_query_param="projectId",
    ),
    "services": RenderEndpointConfig(
        name="services",
        path="/services",
        wrapper_key="service",
        incremental_fields=[_datetime_field("updatedAt"), _datetime_field("createdAt")],
        incremental_param_by_field={"updatedAt": "updatedAfter", "createdAt": "createdAfter"},
        default_incremental_field="updatedAt",
        partition_key="createdAt",
        supports_owner_filter=True,
    ),
    "deploys": RenderEndpointConfig(
        name="deploys",
        path="/services/{parent_id}/deploys",
        wrapper_key="deploy",
        primary_keys=["serviceId", "id"],
        incremental_fields=[
            _datetime_field("finishedAt"),
            _datetime_field("updatedAt"),
            _datetime_field("createdAt"),
        ],
        incremental_param_by_field={
            "finishedAt": "finishedAfter",
            "updatedAt": "updatedAfter",
            "createdAt": "createdAfter",
        },
        # finishedAt: in-flight deploys (null finishedAt) are excluded by finishedAfter and
        # picked up once they reach a terminal state, so each deploy lands exactly once with
        # its final status — the right grain for deploy analytics.
        default_incremental_field="finishedAt",
        partition_key="createdAt",
        parent="services",
        inject_parent_key="serviceId",
    ),
    "jobs": RenderEndpointConfig(
        name="jobs",
        path="/services/{parent_id}/jobs",
        wrapper_key="job",
        primary_keys=["serviceId", "id"],
        incremental_fields=[
            _datetime_field("finishedAt"),
            _datetime_field("startedAt"),
            _datetime_field("createdAt"),
        ],
        incremental_param_by_field={
            "finishedAt": "finishedAfter",
            "startedAt": "startedAfter",
            "createdAt": "createdAfter",
        },
        default_incremental_field="finishedAt",
        partition_key="createdAt",
        parent="services",
    ),
    "events": RenderEndpointConfig(
        name="events",
        path="/services/{parent_id}/events",
        wrapper_key="event",
        primary_keys=["serviceId", "id"],
        incremental_fields=[_datetime_field("timestamp")],
        incremental_param_by_field={"timestamp": "startTime"},
        default_incremental_field="timestamp",
        partition_key="timestamp",
        parent="services",
        append_only=True,
        # The events window defaults to the last hour server-side, so every request must pass
        # an explicit startTime; events can't predate their service, so its createdAt is a
        # complete lower bound for full refreshes and first syncs.
        window_start_from_parent_field="createdAt",
    ),
    "custom_domains": RenderEndpointConfig(
        name="custom_domains",
        path="/services/{parent_id}/custom-domains",
        wrapper_key="customDomain",
        primary_keys=["serviceId", "id"],
        # Only createdBefore/After filters exist, but verificationStatus mutates after
        # creation — a createdAt cursor would freeze it. Tiny table: full refresh only.
        parent="services",
        inject_parent_key="serviceId",
        skip_parent_on_bad_request=True,
    ),
    "env_groups": RenderEndpointConfig(
        name="env_groups",
        path="/env-groups",
        # The docs show env group list items unwrapped (no cursor sibling); the transport
        # falls back to treating the item as the row when the wrapper key is absent.
        wrapper_key="envGroup",
        incremental_fields=[_datetime_field("updatedAt"), _datetime_field("createdAt")],
        incremental_param_by_field={"updatedAt": "updatedAfter", "createdAt": "createdAfter"},
        default_incremental_field="updatedAt",
        supports_owner_filter=True,
        # The list response embeds env var values and secret-file contents (database URLs, API
        # tokens, …). Redact the value-bearing fields so only names/keys/ids reach the warehouse.
        redact_list_item_fields={"envVars": ("value",), "secretFiles": ("content", "contents")},
    ),
    "postgres": RenderEndpointConfig(
        name="postgres",
        path="/postgres",
        wrapper_key="postgres",
        incremental_fields=[_datetime_field("updatedAt"), _datetime_field("createdAt")],
        incremental_param_by_field={"updatedAt": "updatedAfter", "createdAt": "createdAfter"},
        default_incremental_field="updatedAt",
        supports_owner_filter=True,
    ),
    "key_value": RenderEndpointConfig(
        name="key_value",
        path="/key-value",
        wrapper_key="keyValue",
        incremental_fields=[_datetime_field("updatedAt"), _datetime_field("createdAt")],
        incremental_param_by_field={"updatedAt": "updatedAfter", "createdAt": "createdAfter"},
        default_incremental_field="updatedAt",
        supports_owner_filter=True,
    ),
    "disks": RenderEndpointConfig(
        name="disks",
        path="/disks",
        wrapper_key="disk",
        incremental_fields=[_datetime_field("updatedAt"), _datetime_field("createdAt")],
        incremental_param_by_field={"updatedAt": "updatedAfter", "createdAt": "createdAfter"},
        default_incremental_field="updatedAt",
        supports_owner_filter=True,
    ),
    "instances": RenderEndpointConfig(
        name="instances",
        path="/services/{parent_id}/instances",
        wrapper_key="instance",
        primary_keys=["serviceId", "id"],
        # No time filters on the endpoint: full refresh only.
        partition_key="createdAt",
        parent="services",
        inject_parent_key="serviceId",
        paginated=False,
    ),
    "members": RenderEndpointConfig(
        name="members",
        path="/owners/{parent_id}/members",
        wrapper_key="member",
        primary_keys=["ownerId", "userId"],
        parent="owners",
        inject_parent_key="ownerId",
        paginated=False,
    ),
    "audit_logs": RenderEndpointConfig(
        name="audit_logs",
        path="/owners/{parent_id}/audit-logs",
        wrapper_key="auditLog",
        primary_keys=["ownerId", "id"],
        incremental_fields=[_datetime_field("timestamp")],
        incremental_param_by_field={"timestamp": "startTime"},
        default_incremental_field="timestamp",
        partition_key="timestamp",
        parent="owners",
        inject_parent_key="ownerId",
        append_only=True,
        max_lookback_days=90,
        # Audit logs need a Pro workspace and an admin key. One ineligible workspace must not
        # fail the table for a key that spans several workspaces.
        skip_parent_on_bad_request=True,
        skip_parent_on_forbidden=True,
        page_size=1000,
    ),
    "blueprints": RenderEndpointConfig(
        name="blueprints",
        path="/blueprints",
        wrapper_key="blueprint",
        # No time filters on the endpoint, and lastSync is the only timestamp: full refresh only.
        supports_owner_filter=True,
    ),
    "blueprint_syncs": RenderEndpointConfig(
        name="blueprint_syncs",
        path="/blueprints/{parent_id}/syncs",
        wrapper_key="sync",
        primary_keys=["blueprintId", "id"],
        # No time filters on the endpoint, and startedAt is optional: full refresh, unpartitioned.
        parent="blueprints",
        inject_parent_key="blueprintId",
    ),
    "disk_snapshots": RenderEndpointConfig(
        name="disk_snapshots",
        path="/disks/{parent_id}/snapshots",
        wrapper_key="snapshot",
        primary_keys=["diskId", "snapshotKey"],
        partition_key="createdAt",
        parent="disks",
        inject_parent_key="diskId",
        paginated=False,
    ),
    "postgres_top_queries": RenderEndpointConfig(
        name="postgres_top_queries",
        path="/postgres/{parent_id}/query/top-queries",
        wrapper_key="topQuery",
        # A cumulative pg_stat_statements snapshot (top 50 by total time): full refresh only.
        primary_keys=["postgresId", "queryId"],
        parent="postgres",
        inject_parent_key="postgresId",
        paginated=False,
        response_list_key="topQueries",
        # Query insights are unavailable on some instances (e.g. suspended ones). One such
        # database must not fail the table for the rest.
        skip_parent_on_bad_request=True,
    ),
    "postgres_table_scans": RenderEndpointConfig(
        name="postgres_table_scans",
        path="/postgres/{parent_id}/query/table-scans",
        wrapper_key="tableScan",
        primary_keys=["postgresId", "Database", "Schema", "Table"],
        parent="postgres",
        inject_parent_key="postgresId",
        paginated=False,
        response_list_key="tableScans",
        skip_parent_on_bad_request=True,
    ),
}


def _service_metric(
    name: str, path: str, resolution_seconds: int | None = 300, skip_parent_on_bad_request: bool = False
) -> RenderEndpointConfig:
    return RenderEndpointConfig(
        name=name,
        path=path,
        wrapper_key="series",
        primary_keys=["serviceId", "seriesKey", "timestamp"],
        # startTime filters server-side. Merge (not append) so the still-filling latest bucket
        # is overwritten by the next sync instead of duplicated.
        incremental_fields=[_datetime_field("timestamp")],
        incremental_param_by_field={"timestamp": "startTime"},
        default_incremental_field="timestamp",
        partition_key="timestamp",
        parent="services",
        parent_query_param="resource",
        inject_parent_key="serviceId",
        paginated=False,
        metric_series=True,
        # The window defaults to the last hour server-side. Render keeps at most 30 days of
        # metrics (less on smaller plans), so a full refresh reads from that floor.
        max_lookback_days=30,
        # 5-minute buckets keep a 30-day window to a few thousand points per series.
        extra_params={"resolutionSeconds": resolution_seconds} if resolution_seconds else {},
        # High-volume time series: opt-in rather than synced with every new source.
        should_sync_default=False,
        skip_parent_on_bad_request=skip_parent_on_bad_request,
    )


RENDER_ENDPOINTS.update(
    {
        "metrics_cpu": _service_metric("metrics_cpu", "/metrics/cpu"),
        "metrics_memory": _service_metric("metrics_memory", "/metrics/memory"),
        "metrics_http_requests": _service_metric("metrics_http_requests", "/metrics/http-requests"),
        "metrics_http_latency": _service_metric(
            # Render 400s HTTP latency for services in Hobby workspaces. One such service must not
            # fail the table for a key that also spans paid workspaces.
            "metrics_http_latency",
            "/metrics/http-latency",
            skip_parent_on_bad_request=True,
        ),
        "metrics_instance_count": _service_metric("metrics_instance_count", "/metrics/instance-count"),
        "metrics_bandwidth": _service_metric(
            # The bandwidth endpoint takes no resolution param.
            "metrics_bandwidth",
            "/metrics/bandwidth",
            resolution_seconds=None,
        ),
    }
)

ENDPOINTS = tuple(RENDER_ENDPOINTS.keys())

INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
    name: config.incremental_fields for name, config in RENDER_ENDPOINTS.items()
}