Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 3 missing tables to the Render source (#120812)

Tom Owers5a08e7fcbe51
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 955925b80035..65be4ef925a4 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -949,7 +949,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/recurly/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/reddit_ads/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/render/render.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/render/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/rentcast/rentcast.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/rentcast/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/replicate/replicate.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 3837903af8cd..68e123b4de8b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10157,9 +10157,9 @@ Today (12): `custom_domains`, `deploys`, `disks`, `env_groups`, `environments`,
Diffed against: <https://api-docs.render.com/reference/introduction>
-- [ ] `/services/{id}/instances (list-instances)` — per-instance rows behind each service - the unit that scaling and cost analysis is done on (high)
-- [ ] `/owners/{ownerId}/members (retrieve-owner-members)` — membership table linking users to the owners/workspaces already synced (high)
-- [ ] `/owners/{ownerId}/audit-logs and /organizations/{id}/audit-logs` — who changed what and when - the change history for services, env groups and members (high)
+- [x] `/services/{id}/instances (list-instances)` — per-instance rows behind each service - the unit that scaling and cost analysis is done on (high)
+- [x] `/owners/{ownerId}/members (retrieve-owner-members)` — membership table linking users to the owners/workspaces already synced (high)
+- [x] `/owners/{ownerId}/audit-logs and /organizations/{id}/audit-logs` — who changed what and when - the change history for services, env groups and members (high)
- [ ] `/metrics/* (http-requests, http-latency, cpu, memory, instance-count, bandwidth)` — the headline observability time series per service; nothing in the current tables carries runtime metrics (high)
- [ ] `/blueprints and /blueprints/{id}/syncs` — IaC definitions plus their sync history, the deploy path for most multi-service accounts (medium)
- [ ] `/snapshots (list-snapshots)` — backup/restore point history for Postgres instances already synced (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/render/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/render/canonical_descriptions.py
index 37ad7817b3db..10bc98f088b5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/render/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/render/canonical_descriptions.py
@@ -191,4 +191,39 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"updatedAt": "When the disk was last updated.",
},
},
+ "instances": {
+ "description": "Running instances of each service. A scaled service has one row per instance.",
+ "docs_url": "https://api-docs.render.com/reference/list-instances",
+ "columns": {
+ "serviceId": "ID of the service the instance belongs to.",
+ "id": "Unique identifier for the instance.",
+ "createdAt": "When the instance was created.",
+ },
+ },
+ "members": {
+ "description": "Users who belong to each workspace, with their workspace role and status.",
+ "docs_url": "https://api-docs.render.com/reference/retrieve-owner-members",
+ "columns": {
+ "ownerId": "ID of the workspace the user belongs to.",
+ "userId": "Unique identifier for the user.",
+ "name": "Name of the user.",
+ "email": "Email address of the user.",
+ "status": "Whether the membership is active or inactive.",
+ "role": "The user's role in the workspace, in uppercase (for example ADMIN or DEVELOPER).",
+ "mfaEnabled": "Whether the user has multi-factor authentication turned on.",
+ },
+ },
+ "audit_logs": {
+ "description": "Workspace audit log: material actions performed by members, API keys, and Render itself. Needs a Pro workspace plan or higher and an admin API key.",
+ "docs_url": "https://api-docs.render.com/reference/list-owner-audit-logs",
+ "columns": {
+ "ownerId": "ID of the workspace the event belongs to.",
+ "id": "Unique identifier for the audit log entry.",
+ "timestamp": "When the event occurred.",
+ "event": "The type of event that occurred, for example CreateServerEvent.",
+ "status": "Whether the action succeeded or failed.",
+ "actor": "Who performed the action: the actor type (user, rest_api, or system), and its email and ID if applicable.",
+ "metadata": "Additional context about the event, such as the affected service ID.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/render/render.py b/products/warehouse_sources/backend/temporal/data_imports/sources/render/render.py
index 4cecf6252f71..a42b6f933dcc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/render/render.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/render/render.py
@@ -1,6 +1,6 @@
import dataclasses
from collections.abc import Iterator
-from datetime import UTC, date, datetime
+from datetime import UTC, date, datetime, timedelta
from typing import Any, Optional
from urllib.parse import urlencode
@@ -56,6 +56,19 @@ def _format_incremental_value(value: Any) -> str:
return str(value)
+def _to_utc_datetime(value: Any) -> datetime | None:
+ if isinstance(value, datetime):
+ return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
+ if isinstance(value, date):
+ return datetime(value.year, value.month, value.day, tzinfo=UTC)
+ if isinstance(value, str):
+ try:
+ return _to_utc_datetime(datetime.fromisoformat(value.replace("Z", "+00:00")))
+ except ValueError:
+ return None
+ return None
+
+
# Shared with `RenderSource.get_non_retryable_errors` so the same rejection reads the same way
# whether it surfaces while connecting the source or mid-sync.
KEY_REJECTED_MESSAGE = (
@@ -145,22 +158,37 @@ def _build_params(
incremental_field: str | None,
logger: FilteringBoundLogger,
) -> dict[str, Any]:
- params: dict[str, Any] = {"limit": config.page_size}
+ params: dict[str, Any] = {"limit": config.page_size} if config.paginated else {}
if config.supports_owner_filter and owner_id:
params["ownerId"] = owner_id
+ lower_bound: Any = None
if should_use_incremental_field and db_incremental_field_last_value is not None:
field = incremental_field or config.default_incremental_field
param = config.incremental_param_by_field.get(field) if field else None
if param:
params[param] = _format_incremental_value(db_incremental_field_last_value)
+ lower_bound = db_incremental_field_last_value
else:
logger.warning(
f"Render: incremental field {field} has no server-side filter for endpoint "
f"{config.name}, syncing the full endpoint instead"
)
+ window_param = (
+ config.incremental_param_by_field.get(config.default_incremental_field)
+ if config.default_incremental_field
+ else None
+ )
+ if config.max_lookback_days and window_param:
+ # One day of margin: the server measures the span up to its own "now", which moves on
+ # while a fan-out works through its parents.
+ floor = datetime.now(UTC) - timedelta(days=config.max_lookback_days - 1)
+ lower_bound_dt = _to_utc_datetime(lower_bound) if lower_bound is not None else None
+ if lower_bound is None or (lower_bound_dt is not None and lower_bound_dt < floor):
+ params[window_param] = _format_incremental_value(floor)
+
return params
@@ -201,6 +229,9 @@ def _walk_pages(
yield rows, next_cursor
+ if not config.paginated:
+ return
+
if next_cursor is None:
# Unwrapped responses carry no cursor, so a full page here may mean truncation.
if len(items) >= config.page_size:
@@ -235,6 +266,8 @@ def _iter_parent_rows(
params["ownerId"] = owner_id
for rows, _ in _walk_pages(session, headers, logger, parent_config, parent_config.path, params, cursor=None):
+ if parent_config.owner_id_field and owner_id:
+ rows = [row for row in rows if row.get(parent_config.owner_id_field) == owner_id]
yield from rows
@@ -336,6 +369,8 @@ def _get_fan_out_rows(
# domains on non-web/static services). Skip them instead of failing the sync.
elif status == 400 and config.skip_parent_on_bad_request:
logger.warning(f"Render: {config.parent} {parent_id} cannot have {config.name}, skipping")
+ elif status == 403 and config.skip_parent_on_forbidden:
+ logger.warning(f"Render: no access to {config.name} for {config.parent} {parent_id}, skipping")
else:
raise
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/render/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/render/settings.py
index 175057321e5c..b38bddbe3e84 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/render/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/render/settings.py
@@ -12,7 +12,7 @@ def _datetime_field(name: str) -> IncrementalField:
}
-@dataclass
+@dataclass(frozen=True)
class RenderEndpointConfig:
name: str
# Path relative to the API base. Fan-out children use a `{parent_id}` placeholder
@@ -47,7 +47,13 @@ class RenderEndpointConfig:
# slice (events default to the last hour) — read from this parent field when no
# incremental watermark applies, so a full refresh still covers the parent's lifetime.
window_start_from_parent_field: str | None = None
+ # Render caps one audit log export at a 90-day span and only guarantees 90 days of
+ # retention, so the lower bound sent is never earlier than this many days ago.
+ max_lookback_days: int | None = None
should_sync_default: bool = True
+ # False for endpoints that return the whole collection in one response and take no
+ # `limit`/`cursor` params (instances, members).
+ paginated: bool = True
page_size: int = 100
# Hard cap on pages fetched per parent in a fan-out, to bound runaway pagination.
# A structured warning is logged if the cap is reached.
@@ -56,6 +62,12 @@ class RenderEndpointConfig:
# only exist for web services and static sites, not cron jobs, workers or private
# services). Skip those parents rather than failing the whole sync.
skip_parent_on_bad_request: bool = False
+ # Fan-out children gated by the parent's plan or the key's role (audit logs need a Pro
+ # workspace and an admin). Skip a parent that answers 403 instead of failing the sync.
+ skip_parent_on_forbidden: bool = False
+ # Set on an endpoint whose rows are owners: when the source is scoped to one workspace,
+ # fan-out children only visit the parent row whose value in this field matches it.
+ owner_id_field: str | None = None
# Some endpoints return secrets the API key is only meant to *use* for sync (env var
# values, secret-file contents), not data worth warehousing. Maps a list column to the
# item keys whose values are redacted before a row is yielded, keeping the safe metadata
@@ -74,6 +86,7 @@ RENDER_ENDPOINTS: dict[str, RenderEndpointConfig] = {
path="/owners",
wrapper_key="owner",
# No timestamps on the owner object and no time filters on the endpoint: full refresh only.
+ owner_id_field="id",
),
"projects": RenderEndpointConfig(
name="projects",
@@ -214,6 +227,45 @@ RENDER_ENDPOINTS: dict[str, RenderEndpointConfig] = {
default_incremental_field="updatedAt",
supports_owner_filter=True,
),
+ "instances": RenderEndpointConfig(
+ name="instances",
+ path="/services/{parent_id}/instances",
+ wrapper_key="instance",
+ primary_keys=["serviceId", "id"],
+ # No time filters on the endpoint: full refresh only.
+ partition_key="createdAt",
+ parent="services",
+ inject_parent_key="serviceId",
+ paginated=False,
+ ),
+ "members": RenderEndpointConfig(
+ name="members",
+ path="/owners/{parent_id}/members",
+ wrapper_key="member",
+ primary_keys=["ownerId", "userId"],
+ parent="owners",
+ inject_parent_key="ownerId",
+ paginated=False,
+ ),
+ "audit_logs": RenderEndpointConfig(
+ name="audit_logs",
+ path="/owners/{parent_id}/audit-logs",
+ wrapper_key="auditLog",
+ primary_keys=["ownerId", "id"],
+ incremental_fields=[_datetime_field("timestamp")],
+ incremental_param_by_field={"timestamp": "startTime"},
+ default_incremental_field="timestamp",
+ partition_key="timestamp",
+ parent="owners",
+ inject_parent_key="ownerId",
+ append_only=True,
+ max_lookback_days=90,
+ # Audit logs need a Pro workspace and an admin key. One ineligible workspace must not
+ # fail the table for a key that spans several workspaces.
+ skip_parent_on_bad_request=True,
+ skip_parent_on_forbidden=True,
+ page_size=1000,
+ ),
}
ENDPOINTS = tuple(RENDER_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/render/tests/test_render.py b/products/warehouse_sources/backend/temporal/data_imports/sources/render/tests/test_render.py
index e8230dd833c0..2de83f71cb7f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/render/tests/test_render.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/render/tests/test_render.py
@@ -1,6 +1,7 @@
from datetime import UTC, date, datetime, timedelta, timezone
from typing import Any
+import time_machine
from unittest.mock import MagicMock
import requests
@@ -29,12 +30,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.render.set
from products.warehouse_sources.backend.temporal.data_imports.sources.render.source import RenderSource
-def _driver() -> SourceDriver:
- return SourceDriver(RenderSource(), RenderSourceConfig(api_key="rnd_test", owner_id="tea-123"))
+def _driver(owner_id: str | None = "tea-123") -> SourceDriver:
+ return SourceDriver(RenderSource(), RenderSourceConfig(api_key="rnd_test", owner_id=owner_id))
-def _run(endpoint: str, script: Script, **inputs: Any) -> DriveResult:
- return _driver().run(endpoint, script, **inputs)
+def _run(endpoint: str, script: Script, owner_id: str | None = "tea-123", **inputs: Any) -> DriveResult:
+ return _driver(owner_id).run(endpoint, script, **inputs)
def _pages(*payloads: Any) -> list[ScriptedResponse]:
@@ -124,6 +125,32 @@ class TestBuildParams:
)
assert ("ownerId" in params) == expects_owner
+ @parameterized.expand(
+ [
+ ("full_refresh_starts_at_window_floor", False, None, "2026-03-04T00:00:00Z"),
+ ("stale_watermark_is_clamped_to_window", True, datetime(2025, 1, 1, tzinfo=UTC), "2026-03-04T00:00:00Z"),
+ ("recent_watermark_is_kept", True, datetime(2026, 5, 20, 10, 0, tzinfo=UTC), "2026-05-20T10:00:00Z"),
+ ("recent_string_watermark_is_kept", True, "2026-05-20T10:00:00Z", "2026-05-20T10:00:00Z"),
+ ("stale_string_watermark_is_clamped", True, "2025-01-01T00:00:00Z", "2026-03-04T00:00:00Z"),
+ ]
+ )
+ def test_audit_logs_start_time_stays_inside_lookback_window(
+ self, _name: str, should_use_incremental_field: bool, watermark: Any, expected_start: str
+ ) -> None:
+ # Render caps an audit log query at a 90-day span: a start older than that is rejected,
+ # and with no start at all the server picks its own default window.
+ with time_machine.travel(datetime(2026, 6, 1, tzinfo=UTC), tick=False):
+ params = _build_params(
+ RENDER_ENDPOINTS["audit_logs"],
+ owner_id=None,
+ should_use_incremental_field=should_use_incremental_field,
+ db_incremental_field_last_value=watermark,
+ incremental_field="timestamp",
+ logger=MagicMock(),
+ )
+
+ assert params == {"limit": 1000, "startTime": expected_start}
+
def test_incremental_field_without_server_filter_is_skipped(self) -> None:
config = RenderEndpointConfig(name="x", path="/x", wrapper_key="x", default_incremental_field="updatedAt")
logger = MagicMock()
@@ -198,31 +225,103 @@ class TestFanOut:
assert result.items == [[{"serviceId": "srv-2", "id": "dep-2"}]]
- def test_custom_domains_400_for_unsupported_service_is_skipped(self) -> None:
- # Render 400s /custom-domains for service types that can't have them (cron jobs, etc.);
- # skip that parent instead of failing the whole sync.
+ OWNERS_PAGE = [{"owner": {"id": "tea-1", "type": "team"}}, {"owner": {"id": "tea-2", "type": "team"}}]
+
+ @parameterized.expand(
+ [
+ # Render 400s /custom-domains for service types that can't have them (cron jobs, etc.).
+ ("custom_domains_unsupported_service", "custom_domains", "customDomain", 400, "serviceId", "srv-2"),
+ # Audit logs need a Pro workspace and an admin key: one ineligible workspace must not
+ # fail the table for a key that spans several workspaces.
+ ("audit_logs_bad_request", "audit_logs", "auditLog", 400, "ownerId", "tea-2"),
+ ("audit_logs_forbidden", "audit_logs", "auditLog", 403, "ownerId", "tea-2"),
+ ]
+ )
+ def test_ineligible_parent_is_skipped(
+ self, _name: str, endpoint: str, wrapper_key: str, status: int, parent_key: str, second_parent: str
+ ) -> None:
+ parents_page = self.OWNERS_PAGE if parent_key == "ownerId" else self.SERVICES_PAGE
result = _run(
- "custom_domains",
+ endpoint,
[
- *_pages(self.SERVICES_PAGE),
- ScriptedResponse(status=400, json={"message": "Bad Request"}),
- *_pages([{"customDomain": {"id": "dom-2"}}]),
+ *_pages(parents_page),
+ ScriptedResponse(status=status, json={"message": "rejected"}),
+ *_pages([{wrapper_key: {"id": "row-2"}, "cursor": "c2"}], []),
],
+ owner_id=None,
)
- assert result.items == [[{"serviceId": "srv-2", "id": "dom-2"}]]
+ assert result.raised is None
+ assert result.rows == [{parent_key: second_parent, "id": "row-2"}]
- def test_400_still_fails_for_endpoints_without_skip_flag(self) -> None:
+ @parameterized.expand([("bad_request", 400), ("forbidden", 403)])
+ def test_rejected_parent_still_fails_for_endpoints_without_skip_flag(self, _name: str, status: int) -> None:
result = _run(
"deploys",
[
*_pages(self.SERVICES_PAGE),
- ScriptedResponse(status=400, json={"message": "Bad Request"}),
+ ScriptedResponse(status=status, json={"message": "rejected"}),
],
)
assert isinstance(result.raised, requests.HTTPError)
+ @parameterized.expand(
+ [
+ ("instances", "services", "/v1/services/srv-1/instances", "serviceId", "srv-1"),
+ ("members", "owners", "/v1/owners/tea-123/members", "ownerId", "tea-123"),
+ ]
+ )
+ def test_unpaginated_child_is_fetched_once_per_parent(
+ self, endpoint: str, parent: str, expected_path: str, parent_key: str, parent_id: str
+ ) -> None:
+ # These endpoints return the whole list in one response with no cursor. A long list must
+ # not read as a truncated page, and the request must not carry pagination params.
+ parent_wrapper = "service" if parent == "services" else "owner"
+ children = [{"id": f"row-{i}", "createdAt": "2026-01-01T00:00:00Z"} for i in range(150)]
+ result = _run(endpoint, _pages([{parent_wrapper: {"id": parent_id}}], children))
+
+ child_requests = [request for request in result.requests if request.path == expected_path]
+ assert len(child_requests) == 1
+ assert child_requests[0].param("limit") is None
+ assert len(result.rows) == 150
+ assert result.rows[0] == {parent_key: parent_id, "id": "row-0", "createdAt": "2026-01-01T00:00:00Z"}
+
+ def test_owner_scoped_source_only_fans_out_over_its_workspace(self) -> None:
+ # /owners takes no ownerId filter, so a source scoped to one workspace must drop the
+ # other workspaces itself before it reads their members or audit logs.
+ result = _run(
+ "members",
+ _pages(
+ [{"owner": {"id": "tea-999"}}, {"owner": {"id": "tea-123"}}],
+ [{"userId": "usr-1", "name": "Ada", "role": "ADMIN"}],
+ ),
+ )
+
+ assert result.paths == ["/v1/owners", "/v1/owners/tea-123/members"]
+ assert result.rows == [{"ownerId": "tea-123", "userId": "usr-1", "name": "Ada", "role": "ADMIN"}]
+
+ def test_audit_logs_walk_cursor_pages_per_workspace(self) -> None:
+ with time_machine.travel(datetime(2026, 6, 1, tzinfo=UTC), tick=False):
+ result = _run(
+ "audit_logs",
+ _pages(
+ [{"owner": {"id": "tea-123"}}],
+ [{"auditLog": {"id": "aud-1", "event": "CreateServerEvent"}, "cursor": "c1"}],
+ [{"auditLog": {"id": "aud-2", "event": "LoginEvent"}, "cursor": "c2"}],
+ [],
+ ),
+ )
+
+ audit_requests = result.requests[1:]
+ assert [request.path for request in audit_requests] == ["/v1/owners/tea-123/audit-logs"] * 3
+ assert [request.param("cursor") for request in audit_requests] == [None, "c1", "c2"]
+ assert {request.param("startTime") for request in audit_requests} == {"2026-03-04T00:00:00Z"}
+ assert result.rows == [
+ {"ownerId": "tea-123", "id": "aud-1", "event": "CreateServerEvent"},
+ {"ownerId": "tea-123", "id": "aud-2", "event": "LoginEvent"},
+ ]
+
def test_resume_skips_parents_before_bookmark_and_seeds_cursor(self) -> None:
result = _run(
"deploys",