:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 3 missing tables to the Render source (#120812)
Tom Owers5a08e7fcbe51
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 955925b80035..65be4ef925a4 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -949,7 +949,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/recurly/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/reddit_ads/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/render/render.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/render/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/rentcast/rentcast.py1 products/warehouse_sources/backend/temporal/data_imports/sources/rentcast/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/replicate/replicate.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 3837903af8cd..68e123b4de8b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -10157,9 +10157,9 @@ Today (12): `custom_domains`, `deploys`, `disks`, `env_groups`, `environments`,Diffed against: <https://api-docs.render.com/reference/introduction>-- [ ] `/services/{id}/instances (list-instances)` — per-instance rows behind each service - the unit that scaling and cost analysis is done on (high)-- [ ] `/owners/{ownerId}/members (retrieve-owner-members)` — membership table linking users to the owners/workspaces already synced (high)-- [ ] `/owners/{ownerId}/audit-logs and /organizations/{id}/audit-logs` — who changed what and when - the change history for services, env groups and members (high)+- [x] `/services/{id}/instances (list-instances)` — per-instance rows behind each service - the unit that scaling and cost analysis is done on (high)+- [x] `/owners/{ownerId}/members (retrieve-owner-members)` — membership table linking users to the owners/workspaces already synced (high)+- [x] `/owners/{ownerId}/audit-logs and /organizations/{id}/audit-logs` — who changed what and when - the change history for services, env groups and members (high)- [ ] `/metrics/* (http-requests, http-latency, cpu, memory, instance-count, bandwidth)` — the headline observability time series per service; nothing in the current tables carries runtime metrics (high)- [ ] `/blueprints and /blueprints/{id}/syncs` — IaC definitions plus their sync history, the deploy path for most multi-service accounts (medium)- [ ] `/snapshots (list-snapshots)` — backup/restore point history for Postgres instances already synced (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/render/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/render/canonical_descriptions.pyindex 37ad7817b3db..10bc98f088b5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/render/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/render/canonical_descriptions.py@@ -191,4 +191,39 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"updatedAt": "When the disk was last updated.",},},+ "instances": {+ "description": "Running instances of each service. A scaled service has one row per instance.",+ "docs_url": "https://api-docs.render.com/reference/list-instances",+ "columns": {+ "serviceId": "ID of the service the instance belongs to.",+ "id": "Unique identifier for the instance.",+ "createdAt": "When the instance was created.",+ },+ },+ "members": {+ "description": "Users who belong to each workspace, with their workspace role and status.",+ "docs_url": "https://api-docs.render.com/reference/retrieve-owner-members",+ "columns": {+ "ownerId": "ID of the workspace the user belongs to.",+ "userId": "Unique identifier for the user.",+ "name": "Name of the user.",+ "email": "Email address of the user.",+ "status": "Whether the membership is active or inactive.",+ "role": "The user's role in the workspace, in uppercase (for example ADMIN or DEVELOPER).",+ "mfaEnabled": "Whether the user has multi-factor authentication turned on.",+ },+ },+ "audit_logs": {+ "description": "Workspace audit log: material actions performed by members, API keys, and Render itself. Needs a Pro workspace plan or higher and an admin API key.",+ "docs_url": "https://api-docs.render.com/reference/list-owner-audit-logs",+ "columns": {+ "ownerId": "ID of the workspace the event belongs to.",+ "id": "Unique identifier for the audit log entry.",+ "timestamp": "When the event occurred.",+ "event": "The type of event that occurred, for example CreateServerEvent.",+ "status": "Whether the action succeeded or failed.",+ "actor": "Who performed the action: the actor type (user, rest_api, or system), and its email and ID if applicable.",+ "metadata": "Additional context about the event, such as the affected service ID.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/render/render.py b/products/warehouse_sources/backend/temporal/data_imports/sources/render/render.pyindex 4cecf6252f71..a42b6f933dcc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/render/render.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/render/render.py@@ -1,6 +1,6 @@import dataclassesfrom collections.abc import Iterator-from datetime import UTC, date, datetime+from datetime import UTC, date, datetime, timedeltafrom typing import Any, Optionalfrom urllib.parse import urlencode@@ -56,6 +56,19 @@ def _format_incremental_value(value: Any) -> str:return str(value)+def _to_utc_datetime(value: Any) -> datetime | None:+ if isinstance(value, datetime):+ return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)+ if isinstance(value, date):+ return datetime(value.year, value.month, value.day, tzinfo=UTC)+ if isinstance(value, str):+ try:+ return _to_utc_datetime(datetime.fromisoformat(value.replace("Z", "+00:00")))+ except ValueError:+ return None+ return None++# Shared with `RenderSource.get_non_retryable_errors` so the same rejection reads the same way# whether it surfaces while connecting the source or mid-sync.KEY_REJECTED_MESSAGE = (@@ -145,22 +158,37 @@ def _build_params(incremental_field: str | None,logger: FilteringBoundLogger,) -> dict[str, Any]:- params: dict[str, Any] = {"limit": config.page_size}+ params: dict[str, Any] = {"limit": config.page_size} if config.paginated else {}if config.supports_owner_filter and owner_id:params["ownerId"] = owner_id+ lower_bound: Any = Noneif should_use_incremental_field and db_incremental_field_last_value is not None:field = incremental_field or config.default_incremental_fieldparam = config.incremental_param_by_field.get(field) if field else Noneif param:params[param] = _format_incremental_value(db_incremental_field_last_value)+ lower_bound = db_incremental_field_last_valueelse:logger.warning(f"Render: incremental field {field} has no server-side filter for endpoint "f"{config.name}, syncing the full endpoint instead")+ window_param = (+ config.incremental_param_by_field.get(config.default_incremental_field)+ if config.default_incremental_field+ else None+ )+ if config.max_lookback_days and window_param:+ # One day of margin: the server measures the span up to its own "now", which moves on+ # while a fan-out works through its parents.+ floor = datetime.now(UTC) - timedelta(days=config.max_lookback_days - 1)+ lower_bound_dt = _to_utc_datetime(lower_bound) if lower_bound is not None else None+ if lower_bound is None or (lower_bound_dt is not None and lower_bound_dt < floor):+ params[window_param] = _format_incremental_value(floor)+return params@@ -201,6 +229,9 @@ def _walk_pages(yield rows, next_cursor+ if not config.paginated:+ return+if next_cursor is None:# Unwrapped responses carry no cursor, so a full page here may mean truncation.if len(items) >= config.page_size:@@ -235,6 +266,8 @@ def _iter_parent_rows(params["ownerId"] = owner_idfor rows, _ in _walk_pages(session, headers, logger, parent_config, parent_config.path, params, cursor=None):+ if parent_config.owner_id_field and owner_id:+ rows = [row for row in rows if row.get(parent_config.owner_id_field) == owner_id]yield from rows@@ -336,6 +369,8 @@ def _get_fan_out_rows(# domains on non-web/static services). Skip them instead of failing the sync.elif status == 400 and config.skip_parent_on_bad_request:logger.warning(f"Render: {config.parent} {parent_id} cannot have {config.name}, skipping")+ elif status == 403 and config.skip_parent_on_forbidden:+ logger.warning(f"Render: no access to {config.name} for {config.parent} {parent_id}, skipping")else:raisediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/render/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/render/settings.pyindex 175057321e5c..b38bddbe3e84 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/render/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/render/settings.py@@ -12,7 +12,7 @@ def _datetime_field(name: str) -> IncrementalField:}-@dataclass+@dataclass(frozen=True)class RenderEndpointConfig:name: str# Path relative to the API base. Fan-out children use a `{parent_id}` placeholder@@ -47,7 +47,13 @@ class RenderEndpointConfig:# slice (events default to the last hour) — read from this parent field when no# incremental watermark applies, so a full refresh still covers the parent's lifetime.window_start_from_parent_field: str | None = None+ # Render caps one audit log export at a 90-day span and only guarantees 90 days of+ # retention, so the lower bound sent is never earlier than this many days ago.+ max_lookback_days: int | None = Noneshould_sync_default: bool = True+ # False for endpoints that return the whole collection in one response and take no+ # `limit`/`cursor` params (instances, members).+ paginated: bool = Truepage_size: int = 100# Hard cap on pages fetched per parent in a fan-out, to bound runaway pagination.# A structured warning is logged if the cap is reached.@@ -56,6 +62,12 @@ class RenderEndpointConfig:# only exist for web services and static sites, not cron jobs, workers or private# services). Skip those parents rather than failing the whole sync.skip_parent_on_bad_request: bool = False+ # Fan-out children gated by the parent's plan or the key's role (audit logs need a Pro+ # workspace and an admin). Skip a parent that answers 403 instead of failing the sync.+ skip_parent_on_forbidden: bool = False+ # Set on an endpoint whose rows are owners: when the source is scoped to one workspace,+ # fan-out children only visit the parent row whose value in this field matches it.+ owner_id_field: str | None = None# Some endpoints return secrets the API key is only meant to *use* for sync (env var# values, secret-file contents), not data worth warehousing. Maps a list column to the# item keys whose values are redacted before a row is yielded, keeping the safe metadata@@ -74,6 +86,7 @@ RENDER_ENDPOINTS: dict[str, RenderEndpointConfig] = {path="/owners",wrapper_key="owner",# No timestamps on the owner object and no time filters on the endpoint: full refresh only.+ owner_id_field="id",),"projects": RenderEndpointConfig(name="projects",@@ -214,6 +227,45 @@ RENDER_ENDPOINTS: dict[str, RenderEndpointConfig] = {default_incremental_field="updatedAt",supports_owner_filter=True,),+ "instances": RenderEndpointConfig(+ name="instances",+ path="/services/{parent_id}/instances",+ wrapper_key="instance",+ primary_keys=["serviceId", "id"],+ # No time filters on the endpoint: full refresh only.+ partition_key="createdAt",+ parent="services",+ inject_parent_key="serviceId",+ paginated=False,+ ),+ "members": RenderEndpointConfig(+ name="members",+ path="/owners/{parent_id}/members",+ wrapper_key="member",+ primary_keys=["ownerId", "userId"],+ parent="owners",+ inject_parent_key="ownerId",+ paginated=False,+ ),+ "audit_logs": RenderEndpointConfig(+ name="audit_logs",+ path="/owners/{parent_id}/audit-logs",+ wrapper_key="auditLog",+ primary_keys=["ownerId", "id"],+ incremental_fields=[_datetime_field("timestamp")],+ incremental_param_by_field={"timestamp": "startTime"},+ default_incremental_field="timestamp",+ partition_key="timestamp",+ parent="owners",+ inject_parent_key="ownerId",+ append_only=True,+ max_lookback_days=90,+ # Audit logs need a Pro workspace and an admin key. One ineligible workspace must not+ # fail the table for a key that spans several workspaces.+ skip_parent_on_bad_request=True,+ skip_parent_on_forbidden=True,+ page_size=1000,+ ),}ENDPOINTS = tuple(RENDER_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/render/tests/test_render.py b/products/warehouse_sources/backend/temporal/data_imports/sources/render/tests/test_render.pyindex e8230dd833c0..2de83f71cb7f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/render/tests/test_render.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/render/tests/test_render.py@@ -1,6 +1,7 @@from datetime import UTC, date, datetime, timedelta, timezonefrom typing import Any+import time_machinefrom unittest.mock import MagicMockimport requests@@ -29,12 +30,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.render.setfrom products.warehouse_sources.backend.temporal.data_imports.sources.render.source import RenderSource-def _driver() -> SourceDriver:- return SourceDriver(RenderSource(), RenderSourceConfig(api_key="rnd_test", owner_id="tea-123"))+def _driver(owner_id: str | None = "tea-123") -> SourceDriver:+ return SourceDriver(RenderSource(), RenderSourceConfig(api_key="rnd_test", owner_id=owner_id))-def _run(endpoint: str, script: Script, **inputs: Any) -> DriveResult:- return _driver().run(endpoint, script, **inputs)+def _run(endpoint: str, script: Script, owner_id: str | None = "tea-123", **inputs: Any) -> DriveResult:+ return _driver(owner_id).run(endpoint, script, **inputs)def _pages(*payloads: Any) -> list[ScriptedResponse]:@@ -124,6 +125,32 @@ class TestBuildParams:)assert ("ownerId" in params) == expects_owner+ @parameterized.expand(+ [+ ("full_refresh_starts_at_window_floor", False, None, "2026-03-04T00:00:00Z"),+ ("stale_watermark_is_clamped_to_window", True, datetime(2025, 1, 1, tzinfo=UTC), "2026-03-04T00:00:00Z"),+ ("recent_watermark_is_kept", True, datetime(2026, 5, 20, 10, 0, tzinfo=UTC), "2026-05-20T10:00:00Z"),+ ("recent_string_watermark_is_kept", True, "2026-05-20T10:00:00Z", "2026-05-20T10:00:00Z"),+ ("stale_string_watermark_is_clamped", True, "2025-01-01T00:00:00Z", "2026-03-04T00:00:00Z"),+ ]+ )+ def test_audit_logs_start_time_stays_inside_lookback_window(+ self, _name: str, should_use_incremental_field: bool, watermark: Any, expected_start: str+ ) -> None:+ # Render caps an audit log query at a 90-day span: a start older than that is rejected,+ # and with no start at all the server picks its own default window.+ with time_machine.travel(datetime(2026, 6, 1, tzinfo=UTC), tick=False):+ params = _build_params(+ RENDER_ENDPOINTS["audit_logs"],+ owner_id=None,+ should_use_incremental_field=should_use_incremental_field,+ db_incremental_field_last_value=watermark,+ incremental_field="timestamp",+ logger=MagicMock(),+ )++ assert params == {"limit": 1000, "startTime": expected_start}+def test_incremental_field_without_server_filter_is_skipped(self) -> None:config = RenderEndpointConfig(name="x", path="/x", wrapper_key="x", default_incremental_field="updatedAt")logger = MagicMock()@@ -198,31 +225,103 @@ class TestFanOut:assert result.items == [[{"serviceId": "srv-2", "id": "dep-2"}]]- def test_custom_domains_400_for_unsupported_service_is_skipped(self) -> None:- # Render 400s /custom-domains for service types that can't have them (cron jobs, etc.);- # skip that parent instead of failing the whole sync.+ OWNERS_PAGE = [{"owner": {"id": "tea-1", "type": "team"}}, {"owner": {"id": "tea-2", "type": "team"}}]++ @parameterized.expand(+ [+ # Render 400s /custom-domains for service types that can't have them (cron jobs, etc.).+ ("custom_domains_unsupported_service", "custom_domains", "customDomain", 400, "serviceId", "srv-2"),+ # Audit logs need a Pro workspace and an admin key: one ineligible workspace must not+ # fail the table for a key that spans several workspaces.+ ("audit_logs_bad_request", "audit_logs", "auditLog", 400, "ownerId", "tea-2"),+ ("audit_logs_forbidden", "audit_logs", "auditLog", 403, "ownerId", "tea-2"),+ ]+ )+ def test_ineligible_parent_is_skipped(+ self, _name: str, endpoint: str, wrapper_key: str, status: int, parent_key: str, second_parent: str+ ) -> None:+ parents_page = self.OWNERS_PAGE if parent_key == "ownerId" else self.SERVICES_PAGEresult = _run(- "custom_domains",+ endpoint,[- *_pages(self.SERVICES_PAGE),- ScriptedResponse(status=400, json={"message": "Bad Request"}),- *_pages([{"customDomain": {"id": "dom-2"}}]),+ *_pages(parents_page),+ ScriptedResponse(status=status, json={"message": "rejected"}),+ *_pages([{wrapper_key: {"id": "row-2"}, "cursor": "c2"}], []),],+ owner_id=None,)- assert result.items == [[{"serviceId": "srv-2", "id": "dom-2"}]]+ assert result.raised is None+ assert result.rows == [{parent_key: second_parent, "id": "row-2"}]- def test_400_still_fails_for_endpoints_without_skip_flag(self) -> None:+ @parameterized.expand([("bad_request", 400), ("forbidden", 403)])+ def test_rejected_parent_still_fails_for_endpoints_without_skip_flag(self, _name: str, status: int) -> None:result = _run("deploys",[*_pages(self.SERVICES_PAGE),- ScriptedResponse(status=400, json={"message": "Bad Request"}),+ ScriptedResponse(status=status, json={"message": "rejected"}),],)assert isinstance(result.raised, requests.HTTPError)+ @parameterized.expand(+ [+ ("instances", "services", "/v1/services/srv-1/instances", "serviceId", "srv-1"),+ ("members", "owners", "/v1/owners/tea-123/members", "ownerId", "tea-123"),+ ]+ )+ def test_unpaginated_child_is_fetched_once_per_parent(+ self, endpoint: str, parent: str, expected_path: str, parent_key: str, parent_id: str+ ) -> None:+ # These endpoints return the whole list in one response with no cursor. A long list must+ # not read as a truncated page, and the request must not carry pagination params.+ parent_wrapper = "service" if parent == "services" else "owner"+ children = [{"id": f"row-{i}", "createdAt": "2026-01-01T00:00:00Z"} for i in range(150)]+ result = _run(endpoint, _pages([{parent_wrapper: {"id": parent_id}}], children))++ child_requests = [request for request in result.requests if request.path == expected_path]+ assert len(child_requests) == 1+ assert child_requests[0].param("limit") is None+ assert len(result.rows) == 150+ assert result.rows[0] == {parent_key: parent_id, "id": "row-0", "createdAt": "2026-01-01T00:00:00Z"}++ def test_owner_scoped_source_only_fans_out_over_its_workspace(self) -> None:+ # /owners takes no ownerId filter, so a source scoped to one workspace must drop the+ # other workspaces itself before it reads their members or audit logs.+ result = _run(+ "members",+ _pages(+ [{"owner": {"id": "tea-999"}}, {"owner": {"id": "tea-123"}}],+ [{"userId": "usr-1", "name": "Ada", "role": "ADMIN"}],+ ),+ )++ assert result.paths == ["/v1/owners", "/v1/owners/tea-123/members"]+ assert result.rows == [{"ownerId": "tea-123", "userId": "usr-1", "name": "Ada", "role": "ADMIN"}]++ def test_audit_logs_walk_cursor_pages_per_workspace(self) -> None:+ with time_machine.travel(datetime(2026, 6, 1, tzinfo=UTC), tick=False):+ result = _run(+ "audit_logs",+ _pages(+ [{"owner": {"id": "tea-123"}}],+ [{"auditLog": {"id": "aud-1", "event": "CreateServerEvent"}, "cursor": "c1"}],+ [{"auditLog": {"id": "aud-2", "event": "LoginEvent"}, "cursor": "c2"}],+ [],+ ),+ )++ audit_requests = result.requests[1:]+ assert [request.path for request in audit_requests] == ["/v1/owners/tea-123/audit-logs"] * 3+ assert [request.param("cursor") for request in audit_requests] == [None, "c1", "c2"]+ assert {request.param("startTime") for request in audit_requests} == {"2026-03-04T00:00:00Z"}+ assert result.rows == [+ {"ownerId": "tea-123", "id": "aud-1", "event": "CreateServerEvent"},+ {"ownerId": "tea-123", "id": "aud-2", "event": "LoginEvent"},+ ]+def test_resume_skips_parents_before_bookmark_and_seeds_cursor(self) -> None:result = _run("deploys",