Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add render metrics, blueprint and postgres insight tables (#127362)

Tom Owersbc2c083c3658
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 38aea8562ae6..6e2244c4bbc4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10192,10 +10192,10 @@ Diffed against: <https://api-docs.render.com/reference/introduction>
- [x] `/services/{id}/instances (list-instances)` — per-instance rows behind each service - the unit that scaling and cost analysis is done on (high)
- [x] `/owners/{ownerId}/members (retrieve-owner-members)` — membership table linking users to the owners/workspaces already synced (high)
- [x] `/owners/{ownerId}/audit-logs and /organizations/{id}/audit-logs` — who changed what and when - the change history for services, env groups and members (high)
-- [ ] `/metrics/* (http-requests, http-latency, cpu, memory, instance-count, bandwidth)` — the headline observability time series per service; nothing in the current tables carries runtime metrics (high)
-- [ ] `/blueprints and /blueprints/{id}/syncs` — IaC definitions plus their sync history, the deploy path for most multi-service accounts (medium)
-- [ ] `/snapshots (list-snapshots)` — backup/restore point history for Postgres instances already synced (medium)
-- [ ] `/postgres/{id}/top-queries, /table-scans, /processes` — database performance insight rows - the reason most people query Render data at all (medium)
+- [x] `/metrics/* (http-requests, http-latency, cpu, memory, instance-count, bandwidth)` — the headline observability time series per service; nothing in the current tables carries runtime metrics (high)
+- [x] `/blueprints and /blueprints/{id}/syncs` — IaC definitions plus their sync history, the deploy path for most multi-service accounts (medium)
+- [x] `/snapshots (list-snapshots)` — backup/restore point history for Postgres instances already synced (medium) — the operation is `/disks/{diskId}/snapshots`, so this is persistent disk snapshots, synced as `disk_snapshots`
+- [x] `/postgres/{id}/top-queries, /table-scans, /processes` — database performance insight rows - the reason most people query Render data at all (medium) — top queries and table scans added; processes skipped, as it is a live `pg_stat_activity` view with no history
- [ ] `/tasks and /task-runs (listtasks, listtaskruns)` — Render Tasks execution history, analogous to the jobs table already exposed (medium)
- [ ] `/workflows and /workflows/{id}/versions` — workflow definitions and version history for the newer Workflows product (medium)
- [ ] `/redis (list-redis)` — legacy Redis instances are a separate resource from key_value and are invisible today (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/render/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/render/canonical_descriptions.py
index 10bc98f088b5..2a637342431a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/render/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/render/canonical_descriptions.py
@@ -1,5 +1,6 @@
from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
CanonicalDescriptions,
+ CanonicalEndpoint,
)
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
@@ -226,4 +227,103 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"metadata": "Additional context about the event, such as the affected service ID.",
},
},
+ "blueprints": {
+ "description": "Blueprints: infrastructure-as-code definitions (render.yaml) that create and update services from a Git repository.",
+ "docs_url": "https://api-docs.render.com/reference/list-blueprints",
+ "columns": {
+ "id": "Unique identifier for the Blueprint.",
+ "ownerId": "ID of the workspace that owns the Blueprint.",
+ "name": "Name of the Blueprint.",
+ "status": "Sync status of the Blueprint (created, paused, in_sync, syncing, or error). A Blueprint stays created until its first sync is approved.",
+ "autoSync": "Whether the Blueprint re-syncs on each git push to the configured branch.",
+ "repo": "Git repository that holds the Blueprint file.",
+ "branch": "Git branch the Blueprint syncs from.",
+ "path": "Path to the Blueprint file in the repository.",
+ "lastSync": "When the Blueprint last synced.",
+ },
+ },
+ "blueprint_syncs": {
+ "description": "Sync history of each Blueprint: every time Render applied the Blueprint file to the workspace. Syncs that are planned but not yet approved are not included.",
+ "docs_url": "https://api-docs.render.com/reference/list-blueprint-syncs",
+ "columns": {
+ "blueprintId": "ID of the Blueprint the sync belongs to.",
+ "id": "Unique identifier for the sync.",
+ "commit": "The Git commit the sync applied.",
+ "startedAt": "When the sync started.",
+ "completedAt": "When the sync finished.",
+ "state": "State of the sync (pending, running, error, or success).",
+ },
+ },
+ "disk_snapshots": {
+ "description": "Point-in-time snapshots of each persistent disk, which a disk can be restored from.",
+ "docs_url": "https://api-docs.render.com/reference/list-snapshots",
+ "columns": {
+ "diskId": "ID of the disk the snapshot was taken of.",
+ "snapshotKey": "Key that identifies the snapshot, used to restore it.",
+ "instanceId": "Instance the disk was attached to. Each instance of a scaled service has its own disk snapshots.",
+ "createdAt": "When the snapshot was taken.",
+ },
+ },
+ "postgres_top_queries": {
+ "description": "The top 50 queries by total execution time on each Postgres database, from pg_stat_statements. Statistics are cumulative since they were last reset.",
+ "docs_url": "https://api-docs.render.com/reference/list-postgres-top-queries",
+ "columns": {
+ "postgresId": "ID of the Postgres database.",
+ "queryId": "pg_stat_statements identifier of the normalized query.",
+ "query": "Normalized query text, with constants replaced by placeholders.",
+ "calls": "Number of times the query ran.",
+ "totalTimeMs": "Total time spent running the query, in milliseconds.",
+ "minTimeMs": "Fastest run of the query, in milliseconds.",
+ "maxTimeMs": "Slowest run of the query, in milliseconds.",
+ "meanTimeMs": "Mean run time of the query, in milliseconds.",
+ "stddevTimeMs": "Standard deviation of the query run time, in milliseconds.",
+ "rows": "Total number of rows the query retrieved or affected.",
+ "sharedBlocksHit": "Shared buffer cache hits.",
+ "sharedBlocksRead": "Shared blocks read from disk.",
+ "tempBlocksRead": "Temporary blocks read, a sign of sorts or hashes that spill to disk.",
+ "tempBlocksWritten": "Temporary blocks written, a sign of sorts or hashes that spill to disk.",
+ },
+ },
+ "postgres_table_scans": {
+ "description": "Number of sequential scans run against each table on each Postgres database. A high count on a large table often points to a missing index.",
+ "docs_url": "https://api-docs.render.com/reference/list-postgres-table-scans",
+ "columns": {
+ "postgresId": "ID of the Postgres database.",
+ "Database": "Database that holds the table.",
+ "Schema": "Schema that holds the table.",
+ "Table": "Name of the table.",
+ "Scans": "Number of sequential scans run against the table.",
+ },
+ },
}
+
+
+def _metric_description(description: str, operation_id: str) -> CanonicalEndpoint:
+ return {
+ "description": f"{description} One row per service, time series and time bucket. Render keeps 7 to 30 days of metrics, depending on the workspace plan.",
+ "docs_url": f"https://api-docs.render.com/reference/{operation_id}",
+ "columns": {
+ "serviceId": "ID of the service the metric belongs to.",
+ "seriesKey": "The series labels as sorted field=value pairs. Identifies the series within the service.",
+ "labels": "Labels that describe the time series, such as the instance, status code, host, or quantile.",
+ "unit": "Unit of the metric value.",
+ "timestamp": "Start of the time bucket.",
+ "value": "Metric value for the time bucket.",
+ },
+ }
+
+
+CANONICAL_DESCRIPTIONS.update(
+ {
+ "metrics_cpu": _metric_description("CPU usage of each service.", "get-cpu"),
+ "metrics_memory": _metric_description("Memory usage of each service.", "get-memory"),
+ "metrics_http_requests": _metric_description("HTTP request count of each service.", "get-http-requests"),
+ "metrics_http_latency": _metric_description(
+ "HTTP response latency quantiles of each service.", "get-http-latency"
+ ),
+ "metrics_instance_count": _metric_description(
+ "Number of running instances of each service.", "get-instance-count"
+ ),
+ "metrics_bandwidth": _metric_description("Outbound bandwidth usage of each service.", "get-bandwidth"),
+ }
+)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/render/render.py b/products/warehouse_sources/backend/temporal/data_imports/sources/render/render.py
index e6d1cef89b86..11150fbf84d8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/render/render.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/render/render.py
@@ -118,7 +118,11 @@ def validate_credentials(api_key: str) -> tuple[bool, str | None]:
reraise=True,
)
def _fetch_page(
- session: requests.Session, url: str, headers: dict[str, str], logger: FilteringBoundLogger
+ session: requests.Session,
+ url: str,
+ headers: dict[str, str],
+ logger: FilteringBoundLogger,
+ response_list_key: str | None = None,
) -> list[dict[str, Any]]:
response = session.get(url, headers=headers, timeout=REQUEST_TIMEOUT_SECONDS)
@@ -136,6 +140,8 @@ def _fetch_page(
response.raise_for_status()
data = response.json()
+ if response_list_key and isinstance(data, dict):
+ data = data.get(response_list_key) or []
if not isinstance(data, list):
raise ValueError(f"Render API returned a non-list response for url={url}")
return data
@@ -153,6 +159,31 @@ def _unwrap_item(item: dict[str, Any], wrapper_key: str) -> tuple[dict[str, Any]
return item, item.get("cursor") if isinstance(item.get("cursor"), str) else None
+def _explode_metric_series(series: dict[str, Any]) -> list[dict[str, Any]]:
+ """Turn one metrics time series into a row per data point.
+
+ Labels identify the series (instance, status code, host, quantile, …) and vary by metric,
+ so they are folded into a sorted `seriesKey` that makes (service, series, timestamp) unique.
+ """
+ labels = {
+ label["field"]: label.get("value")
+ for label in series.get("labels") or []
+ if isinstance(label, dict) and "field" in label
+ }
+ series_key = ",".join(f"{field}={labels[field]}" for field in sorted(labels))
+ return [
+ {
+ "seriesKey": series_key,
+ "labels": labels,
+ "unit": series.get("unit"),
+ "timestamp": point.get("timestamp"),
+ "value": point.get("value"),
+ }
+ for point in series.get("values") or []
+ if isinstance(point, dict)
+ ]
+
+
def _build_params(
config: RenderEndpointConfig,
owner_id: str | None,
@@ -162,6 +193,7 @@ def _build_params(
logger: FilteringBoundLogger,
) -> dict[str, Any]:
params: dict[str, Any] = {"limit": config.page_size} if config.paginated else {}
+ params.update(config.extra_params)
if config.supports_owner_filter and owner_id:
params["ownerId"] = owner_id
@@ -218,18 +250,25 @@ def _walk_pages(
page_params["cursor"] = cursor
url = f"{RENDER_BASE_URL}{path}?{urlencode(page_params)}"
- items = _fetch_page(session, url, headers, logger)
+ items = _fetch_page(session, url, headers, logger, config.response_list_key)
if not items:
return
rows: list[dict[str, Any]] = []
next_cursor: str | None = None
for item in items:
+ if config.metric_series:
+ rows.extend(_explode_metric_series(item))
+ continue
row, item_cursor = _unwrap_item(item, config.wrapper_key)
rows.append(row)
if item_cursor:
next_cursor = item_cursor
+ if not rows:
+ # Metric series with no data points in the window.
+ return
+
yield rows, next_cursor
if not config.paginated:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/render/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/render/settings.py
index b38bddbe3e84..6adb8e90220f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/render/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/render/settings.py
@@ -74,6 +74,14 @@ class RenderEndpointConfig:
# (names, keys, ids). Any endpoint with entries here is treated as sensitive: its raw
# responses are also excluded from HTTP sample capture (see `is_sensitive`).
redact_list_item_fields: dict[str, tuple[str, ...]] = field(default_factory=dict)
+ # Endpoints that answer with an object holding the list under this key instead of a bare
+ # list (Postgres query insights: `{"topQueries": [...]}`).
+ response_list_key: str | None = None
+ # Metrics endpoints return time series (`{labels, values: [{timestamp, value}], unit}`)
+ # that are exploded into one row per data point.
+ metric_series: bool = False
+ # Static query params sent on every request.
+ extra_params: dict[str, str | int] = field(default_factory=dict)
@property
def is_sensitive(self) -> bool:
@@ -266,8 +274,112 @@ RENDER_ENDPOINTS: dict[str, RenderEndpointConfig] = {
skip_parent_on_forbidden=True,
page_size=1000,
),
+ "blueprints": RenderEndpointConfig(
+ name="blueprints",
+ path="/blueprints",
+ wrapper_key="blueprint",
+ # No time filters on the endpoint, and lastSync is the only timestamp: full refresh only.
+ supports_owner_filter=True,
+ ),
+ "blueprint_syncs": RenderEndpointConfig(
+ name="blueprint_syncs",
+ path="/blueprints/{parent_id}/syncs",
+ wrapper_key="sync",
+ primary_keys=["blueprintId", "id"],
+ # No time filters on the endpoint, and startedAt is optional: full refresh, unpartitioned.
+ parent="blueprints",
+ inject_parent_key="blueprintId",
+ ),
+ "disk_snapshots": RenderEndpointConfig(
+ name="disk_snapshots",
+ path="/disks/{parent_id}/snapshots",
+ wrapper_key="snapshot",
+ primary_keys=["diskId", "snapshotKey"],
+ partition_key="createdAt",
+ parent="disks",
+ inject_parent_key="diskId",
+ paginated=False,
+ ),
+ "postgres_top_queries": RenderEndpointConfig(
+ name="postgres_top_queries",
+ path="/postgres/{parent_id}/query/top-queries",
+ wrapper_key="topQuery",
+ # A cumulative pg_stat_statements snapshot (top 50 by total time): full refresh only.
+ primary_keys=["postgresId", "queryId"],
+ parent="postgres",
+ inject_parent_key="postgresId",
+ paginated=False,
+ response_list_key="topQueries",
+ # Query insights are unavailable on some instances (e.g. suspended ones). One such
+ # database must not fail the table for the rest.
+ skip_parent_on_bad_request=True,
+ ),
+ "postgres_table_scans": RenderEndpointConfig(
+ name="postgres_table_scans",
+ path="/postgres/{parent_id}/query/table-scans",
+ wrapper_key="tableScan",
+ primary_keys=["postgresId", "Database", "Schema", "Table"],
+ parent="postgres",
+ inject_parent_key="postgresId",
+ paginated=False,
+ response_list_key="tableScans",
+ skip_parent_on_bad_request=True,
+ ),
}
+
+def _service_metric(
+ name: str, path: str, resolution_seconds: int | None = 300, skip_parent_on_bad_request: bool = False
+) -> RenderEndpointConfig:
+ return RenderEndpointConfig(
+ name=name,
+ path=path,
+ wrapper_key="series",
+ primary_keys=["serviceId", "seriesKey", "timestamp"],
+ # startTime filters server-side. Merge (not append) so the still-filling latest bucket
+ # is overwritten by the next sync instead of duplicated.
+ incremental_fields=[_datetime_field("timestamp")],
+ incremental_param_by_field={"timestamp": "startTime"},
+ default_incremental_field="timestamp",
+ partition_key="timestamp",
+ parent="services",
+ parent_query_param="resource",
+ inject_parent_key="serviceId",
+ paginated=False,
+ metric_series=True,
+ # The window defaults to the last hour server-side. Render keeps at most 30 days of
+ # metrics (less on smaller plans), so a full refresh reads from that floor.
+ max_lookback_days=30,
+ # 5-minute buckets keep a 30-day window to a few thousand points per series.
+ extra_params={"resolutionSeconds": resolution_seconds} if resolution_seconds else {},
+ # High-volume time series: opt-in rather than synced with every new source.
+ should_sync_default=False,
+ skip_parent_on_bad_request=skip_parent_on_bad_request,
+ )
+
+
+RENDER_ENDPOINTS.update(
+ {
+ "metrics_cpu": _service_metric("metrics_cpu", "/metrics/cpu"),
+ "metrics_memory": _service_metric("metrics_memory", "/metrics/memory"),
+ "metrics_http_requests": _service_metric("metrics_http_requests", "/metrics/http-requests"),
+ "metrics_http_latency": _service_metric(
+ # Render 400s HTTP latency for services in Hobby workspaces. One such service must not
+ # fail the table for a key that also spans paid workspaces.
+ "metrics_http_latency",
+ "/metrics/http-latency",
+ skip_parent_on_bad_request=True,
+ ),
+ "metrics_instance_count": _service_metric("metrics_instance_count", "/metrics/instance-count"),
+ "metrics_bandwidth": _service_metric(
+ # The bandwidth endpoint takes no resolution param.
+ "metrics_bandwidth",
+ "/metrics/bandwidth",
+ resolution_seconds=None,
+ ),
+ }
+)
+
ENDPOINTS = tuple(RENDER_ENDPOINTS.keys())
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/render/tests/test_render.py b/products/warehouse_sources/backend/temporal/data_imports/sources/render/tests/test_render.py
index 2de83f71cb7f..903a7a5f6033 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/render/tests/test_render.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/render/tests/test_render.py
@@ -359,6 +359,82 @@ class TestFanOut:
assert result.requests[1].param("startTime") == "2024-01-01T00:00:00Z"
+ def test_metric_series_explode_into_one_row_per_point(self) -> None:
+ with time_machine.travel(datetime(2026, 6, 1, tzinfo=UTC), tick=False):
+ result = _run(
+ "metrics_http_requests",
+ _pages(
+ self.SERVICES_PAGE,
+ [
+ {
+ "labels": [{"field": "statusCode", "value": "500"}, {"field": "host", "value": "a.io"}],
+ "unit": "count",
+ "values": [
+ {"timestamp": "2026-05-31T00:00:00Z", "value": 3},
+ {"timestamp": "2026-05-31T00:05:00Z", "value": 1},
+ ],
+ },
+ {
+ "labels": [{"field": "statusCode", "value": "200"}, {"field": "host", "value": "a.io"}],
+ "unit": "count",
+ "values": [{"timestamp": "2026-05-31T00:00:00Z", "value": 40}],
+ },
+ ],
+ [{"labels": [], "unit": "count", "values": []}],
+ ),
+ )
+
+ metric_requests = result.requests[1:]
+ assert [request.path for request in metric_requests] == ["/v1/metrics/http-requests"] * 2
+ assert [request.param("resource") for request in metric_requests] == ["srv-1", "srv-2"]
+ # Without an explicit startTime Render returns only the last hour.
+ assert {request.param("startTime") for request in metric_requests} == {"2026-05-03T00:00:00Z"}
+ assert {request.param("resolutionSeconds") for request in metric_requests} == {"300"}
+ # Two series share a timestamp, so the labels must make the (service, series, timestamp) key unique.
+ assert result.rows == [
+ {
+ "serviceId": "srv-1",
+ "seriesKey": "host=a.io,statusCode=500",
+ "labels": {"statusCode": "500", "host": "a.io"},
+ "unit": "count",
+ "timestamp": "2026-05-31T00:00:00Z",
+ "value": 3,
+ },
+ {
+ "serviceId": "srv-1",
+ "seriesKey": "host=a.io,statusCode=500",
+ "labels": {"statusCode": "500", "host": "a.io"},
+ "unit": "count",
+ "timestamp": "2026-05-31T00:05:00Z",
+ "value": 1,
+ },
+ {
+ "serviceId": "srv-1",
+ "seriesKey": "host=a.io,statusCode=200",
+ "labels": {"statusCode": "200", "host": "a.io"},
+ "unit": "count",
+ "timestamp": "2026-05-31T00:00:00Z",
+ "value": 40,
+ },
+ ]
+
+ def test_postgres_insights_read_the_list_out_of_the_response_object(self) -> None:
+ result = _run(
+ "postgres_table_scans",
+ [
+ *_pages([{"postgres": {"id": "dpg-1"}}, {"postgres": {"id": "dpg-2"}}]),
+ # Insights are unavailable on some instances; one must not fail the table.
+ ScriptedResponse(status=400, json={"message": "unavailable"}),
+ *_pages({"tableScans": [{"Database": "app", "Schema": "public", "Table": "users", "Scans": 7}]}),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.paths[1:] == ["/v1/postgres/dpg-1/query/table-scans", "/v1/postgres/dpg-2/query/table-scans"]
+ assert result.rows == [
+ {"postgresId": "dpg-2", "Database": "app", "Schema": "public", "Table": "users", "Scans": 7}
+ ]
+
class TestSensitiveEndpointHandling:
def test_env_group_secret_values_are_redacted_from_rows(self) -> None: