Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse): add netlify dns_records, audit_events and traffic_splits tables (#123048)

Tom Owers11b4524ccc6f
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index eb2f965debbd..975cfc7fc1f4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -8333,16 +8333,16 @@ Note: Compared the official Query API endpoint reference with the source discove
## Netlify — gaps
-Today (8): `accounts`, `builds`, `deploys`, `dns_zones`, `forms`, `members`, `sites`, `submissions`
+Today (11): `accounts`, `audit_events`, `builds`, `deploys`, `dns_records`, `dns_zones`, `forms`, `members`, `sites`, `submissions`, `traffic_splits`
Diffed against: <https://open-api.netlify.com/swagger.json>
-- [ ] `/dns_zones/{zone_id}/dns_records` — the actual records inside the zones already synced — zones alone carry almost no queryable detail (high)
-- [ ] `/accounts/{account_id}/audit` — account audit log: who deployed, changed env vars, or altered site settings and when (high)
+- [x] `/dns_zones/{zone_id}/dns_records` — the actual records inside the zones already synced — zones alone carry almost no queryable detail (high). Added as `dns_records` (fan-out over `dns_zones`).
+- [x] `/accounts/{account_id}/audit` — account audit log: who deployed, changed env vars, or altered site settings and when (high). Added as `audit_events` (fan-out over `accounts`).
- [ ] `/sites/{site_id}/functions` — serverless/edge function inventory per site, needed to attribute build and runtime behavior (medium)
- [ ] `/sites/{site_id}/deployed-branches` — branch-to-deploy mapping for branch-deploy and preview analysis (medium)
-- [ ] `/sites/{site_id}/traffic_splits` — split test definitions and branch weights — the A/B testing objects (medium)
-- [ ] `/accounts/types` — lookup resolving the plan/type id on every account row (medium)
+- [x] `/sites/{site_id}/traffic_splits` — split test definitions and branch weights — the A/B testing objects (medium). Added as `traffic_splits` (fan-out over `sites`).
+- ~~`/accounts/types` — lookup resolving the plan/type id on every account row (medium)~~ — skipped: it lists the plans offered to new teams, not the plan of each existing account, and `accounts` rows already carry `type_id` and `type_name`
- [ ] `/sites/{site_id}/plugin_runs/latest` — build plugin run outcomes per site, complementing build status (low)
- [ ] `/sites/{site_id}/service-instances` — installed add-on instances per site (low)
- [ ] `/sites/{site_id}/ssl/certificates` — certificate state and expiry per site (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/netlify/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/netlify/canonical_descriptions.py
index 913310edbe04..7349a9aa355e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/netlify/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/netlify/canonical_descriptions.py
@@ -101,6 +101,23 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"updated_at": "Time at which the zone was last updated.",
},
},
+ "dns_records": {
+ "description": "A DNS record in a Netlify DNS zone, fanned out one row per record in each zone.",
+ "docs_url": "https://open-api.netlify.com/#tag/dnsZone",
+ "columns": {
+ "id": "Unique identifier for the DNS record.",
+ "dns_zone_id": "Identifier of the DNS zone that holds the record.",
+ "hostname": "The fully qualified hostname the record applies to.",
+ "type": "The record type, such as A, AAAA, CNAME, MX, TXT or NETLIFY.",
+ "value": "The record's value.",
+ "ttl": "Time to live for the record, in seconds.",
+ "priority": "The record's priority, for record types that use one (such as MX).",
+ "site_id": "Identifier of the site the record points to, if any.",
+ "flag": "Flag value for CAA records.",
+ "tag": "Tag value for CAA records.",
+ "managed": "Whether Netlify manages the record automatically.",
+ },
+ },
"accounts": {
"description": "A Netlify account (team) the authenticated user is a member of.",
"docs_url": "https://open-api.netlify.com/#tag/account",
@@ -125,4 +142,28 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"role": "The member's role within the account.",
},
},
+ "audit_events": {
+ "description": "An audit log event for a Netlify account (team), fanned out one row per event in each account.",
+ "docs_url": "https://open-api.netlify.com/#tag/auditLog",
+ "columns": {
+ "id": "Unique identifier for the audit log event.",
+ "account_id": "Identifier of the account the event belongs to.",
+ "payload": "Event details, including the actor (actor_id, actor_name, actor_email), the action, the timestamp and the log_type.",
+ },
+ },
+ "traffic_splits": {
+ "description": "A split test on a Netlify site that divides traffic between branch deploys.",
+ "docs_url": "https://open-api.netlify.com/#tag/splitTest",
+ "columns": {
+ "id": "Unique identifier for the split test.",
+ "site_id": "Identifier of the site the split test runs on.",
+ "name": "The split test's name.",
+ "path": "The path the split test applies to.",
+ "branches": "The branches in the split test and the share of traffic each one gets.",
+ "active": "Whether the split test is running.",
+ "created_at": "Time at which the split test was created.",
+ "updated_at": "Time at which the split test was last updated.",
+ "unpublished_at": "Time at which the split test was unpublished, if it was.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/netlify/netlify.py b/products/warehouse_sources/backend/temporal/data_imports/sources/netlify/netlify.py
index 8887d23e8f01..e4b1b8b05790 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/netlify/netlify.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/netlify/netlify.py
@@ -17,7 +17,8 @@ lists checkpoint the next page URL, and fan-out tables checkpoint the framework'
state so a resumed run skips already-completed parents and re-fans the in-progress one (merge dedupes
on the primary key).
-Site-scoped tables (deploys, builds, forms, submissions) and account-scoped tables (members) are
+Site-scoped tables (deploys, builds, forms, submissions, traffic_splits), account-scoped tables
+(members, audit_events) and zone-scoped tables (dns_records) are
fan-outs via the framework's dependent resources: the parent list seeds a child endpoint per parent,
injecting the parent identifier onto each child row so the composite primary key stays unique
table-wide.
@@ -263,8 +264,9 @@ def _build_fan_out_resource(
# A site-scoped sub-resource answers 404 when the site doesn't serve it at all (a site with
# form detection off has no /forms), and when a site listed by the parent is deleted before
# its children are fetched. Both mean this parent contributes no rows, not that the table
- # can't sync, so skip the parent instead of failing every other site's rows with it.
- "response_actions": [{"status_code": 404, "action": "ignore"}],
+ # can't sync, so skip the parent instead of failing every other site's rows with it. Some
+ # endpoints add other statuses with the same meaning (see `skip_parent_statuses`).
+ "response_actions": [{"status_code": status, "action": "ignore"} for status in config.skip_parent_statuses],
}
child_resource: EndpointResource = {
"name": config.name,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/netlify/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/netlify/settings.py
index dff9c0e14e4f..e093172ecbb9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/netlify/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/netlify/settings.py
@@ -9,7 +9,7 @@ from products.warehouse_sources.backend.types import IncrementalField
# `INCREMENTAL_FIELDS` is therefore empty for all endpoints; see the module docstring in netlify.py.
-@dataclass
+@dataclass(frozen=True)
class NetlifyEndpointConfig:
name: str
# Path template appended to the API base. Fan-out children carry a placeholder filled from the
@@ -42,6 +42,9 @@ class NetlifyEndpointConfig:
# Static query params sent on every request to this endpoint (on top of `per_page` and, for
# fan-out children, the resolve param — which always wins a key collision).
extra_params: dict[str, str] = field(default_factory=dict)
+ # Child statuses that mean "this parent contributes no rows", so the parent is skipped instead of
+ # failing the whole table.
+ skip_parent_statuses: list[int] = field(default_factory=lambda: [404])
# Credential-bearing fields dropped from every row before it's persisted, so they never land in
# a queryable warehouse table. Dotted paths reach nested objects (e.g. `default_hooks_data.access_token`);
# a bare name targets a top-level field. A missing key is a no-op.
@@ -114,6 +117,16 @@ NETLIFY_ENDPOINTS: dict[str, NetlifyEndpointConfig] = {
partition_key="created_at",
page_size=None,
),
+ "dns_records": NetlifyEndpointConfig(
+ name="dns_records",
+ path="/dns_zones/{zone_id}/dns_records",
+ # Record ids are only documented per zone, so key on the zone too. A record has no timestamp.
+ primary_keys=["dns_zone_id", "id"],
+ page_size=None,
+ fan_out_parent="dns_zones",
+ fan_out_path_param="zone_id",
+ fan_out_include_parent_fields={"id": "dns_zone_id"},
+ ),
"accounts": NetlifyEndpointConfig(
name="accounts",
path="/accounts",
@@ -132,6 +145,30 @@ NETLIFY_ENDPOINTS: dict[str, NetlifyEndpointConfig] = {
fan_out_parent_field="slug",
fan_out_include_parent_fields={"slug": "account_slug"},
),
+ "audit_events": NetlifyEndpointConfig(
+ name="audit_events",
+ path="/accounts/{account_id}/audit",
+ # The event timestamp lives inside `payload`, so there is no top-level field to partition by.
+ primary_keys=["account_id", "id"],
+ fan_out_parent="accounts",
+ fan_out_path_param="account_id",
+ fan_out_include_parent_fields={"id": "account_id"},
+ # An active team's audit log runs far longer than a site's deploy list.
+ max_pages_per_parent=1000,
+ # Only account owners on plans with the audit log can read it. Another account's events
+ # must still sync when the token's user can't read one account's log.
+ skip_parent_statuses=[403, 404],
+ ),
+ "traffic_splits": NetlifyEndpointConfig(
+ name="traffic_splits",
+ path="/sites/{site_id}/traffic_splits",
+ primary_keys=["site_id", "id"],
+ partition_key="created_at",
+ page_size=None, # no page/per_page params; a site has few split tests
+ fan_out_parent="sites",
+ fan_out_path_param="site_id",
+ fan_out_include_parent_fields={"id": "site_id"},
+ ),
}
ENDPOINTS = tuple(NETLIFY_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/netlify/tests/test_netlify.py b/products/warehouse_sources/backend/temporal/data_imports/sources/netlify/tests/test_netlify.py
index 3de404c48c71..112f063b34ce 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/netlify/tests/test_netlify.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/netlify/tests/test_netlify.py
@@ -167,6 +167,39 @@ class TestFanOut:
assert result.paths == ["/api/v1/sites", "/api/v1/sites/s2/builds"]
assert result.rows == [{"id": "b2", "site_id": "s2"}]
+ @parameterized.expand(
+ [
+ ("dns_records", "/api/v1/dns_zones", "/api/v1/dns_zones/z1/dns_records", "dns_zone_id"),
+ ("audit_events", "/api/v1/accounts", "/api/v1/accounts/z1/audit", "account_id"),
+ ("traffic_splits", "/api/v1/sites", "/api/v1/sites/z1/traffic_splits", "site_id"),
+ ]
+ )
+ def test_resolves_child_path_from_parent_id(
+ self, endpoint: str, parent_path: str, child_path: str, parent_column: str
+ ) -> None:
+ result = _driver().run(
+ endpoint,
+ [ScriptedResponse(json=[{"id": "z1", "slug": "acme"}]), ScriptedResponse(json=[{"id": "c1"}])],
+ )
+
+ assert result.raised is None
+ assert result.paths == [parent_path, child_path]
+ assert result.rows == [{"id": "c1", parent_column: "z1"}]
+
+ def test_audit_events_skips_account_whose_log_is_forbidden(self) -> None:
+ result = _driver().run(
+ "audit_events",
+ [
+ ScriptedResponse(json=[{"id": "a1"}, {"id": "a2"}]),
+ ScriptedResponse(status=403, json={"code": 403, "message": "Forbidden"}),
+ ScriptedResponse(json=[{"id": "e1"}]),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/api/v1/accounts", "/api/v1/accounts/a1/audit", "/api/v1/accounts/a2/audit"]
+ assert result.rows == [{"id": "e1", "account_id": "a2"}]
+
class TestPageCap:
def test_capped_paginator_raises_when_parent_exceeds_cap(self) -> None: