Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): move cloudflare dns_analytics_report to graphql (#119130)

posthog[bot]b0f81923a8fa
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/canonical_descriptions.py
index daa23347ea57..4ab09fa88bbb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/canonical_descriptions.py
@@ -398,8 +398,8 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
},
},
"dns_analytics_report": {
- "description": "Authoritative DNS query counts for a zone, grouped by query type and response code.",
- "docs_url": "https://developers.cloudflare.com/api/resources/dns/subresources/analytics/",
+ "description": "Authoritative DNS query counts for a zone over the 6 hours before the sync, grouped by query type and response code.",
+ "docs_url": "https://developers.cloudflare.com/analytics/graphql-api/",
"columns": {
"queryType": "The DNS record type that was queried (e.g. A, AAAA, MX).",
"responseCode": "The DNS response code Cloudflare returned (e.g. NOERROR, NXDOMAIN).",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/cloudflare.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/cloudflare.py
index edd0e5f95e2d..31d00bdaf893 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/cloudflare.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/cloudflare.py
@@ -1,7 +1,9 @@
+from collections.abc import Iterator
from datetime import UTC, date, datetime
from typing import Any, Optional
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+import structlog
from requests import Response
from posthog.dataclasses import frozen
@@ -10,8 +12,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.cloudflare
ACCOUNTS_PARENT,
CLOUDFLARE_ENDPOINTS,
CURSOR_PAGINATION,
- DNS_ANALYTICS_DIMENSIONS,
- DNS_ANALYTICS_METRICS,
+ DNS_ANALYTICS_WINDOW,
PAGE_PAGINATION,
SINGLE_PAGE,
ZONES_PARENT,
@@ -23,6 +24,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
rest_api_resource,
rest_api_resources,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import BearerTokenAuth
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.jsonpath_utils import (
find_values,
)
@@ -33,6 +35,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
SinglePagePaginator,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClient
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
ClientConfig,
Endpoint,
@@ -40,6 +43,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
+logger = structlog.get_logger(__name__)
+
CLOUDFLARE_BASE_URL = "https://api.cloudflare.com/client/v4"
# Cloudflare list pages cap at 50 by default; most endpoints allow more.
PAGE_SIZE = 50
@@ -91,13 +96,99 @@ def _to_rfc3339(value: Any) -> Optional[str]:
return str(value)
-def _flatten_dns_analytics_row(row: dict[str, Any]) -> dict[str, Any]:
- """The DNS analytics report returns positional `dimensions`/`metrics` arrays that
- line up with the names we requested, so name them into their own columns."""
- flattened = {key: value for key, value in row.items() if key not in ("dimensions", "metrics")}
- flattened.update(zip(DNS_ANALYTICS_DIMENSIONS, row.get("dimensions") or []))
- flattened.update(zip(DNS_ANALYTICS_METRICS, row.get("metrics") or []))
- return flattened
+# The GraphQL dataset has no uncached-count metric, so the query also groups on
+# `responseCached` and `_dns_analytics_rows` sums the uncached groups.
+_DNS_ANALYTICS_QUERY = """
+query DnsAnalyticsReport($zoneTag: string, $since: Time, $until: Time) {
+ viewer {
+ zones(filter: {zoneTag: $zoneTag}) {
+ dnsAnalyticsAdaptiveGroups(limit: 10000, filter: {datetime_geq: $since, datetime_lt: $until}) {
+ count
+ dimensions {
+ queryType
+ responseCode
+ responseCached
+ }
+ }
+ }
+ }
+}
+"""
+
+DNS_ANALYTICS_ALL_ZONES_FAILED = "Cloudflare refused the DNS analytics query for every zone"
+
+
+def _dns_analytics_rows(zone_id: str, groups: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ rows: dict[tuple[Any, Any], dict[str, Any]] = {}
+ for group in groups:
+ dimensions = group.get("dimensions") or {}
+ query_type, response_code = dimensions.get("queryType"), dimensions.get("responseCode")
+ row = rows.setdefault(
+ (query_type, response_code),
+ {
+ "queryType": query_type,
+ "responseCode": response_code,
+ "queryCount": 0,
+ "uncachedCount": 0,
+ "_zone_id": zone_id,
+ },
+ )
+ count = group.get("count") or 0
+ row["queryCount"] += count
+ if not dimensions.get("responseCached"):
+ row["uncachedCount"] += count
+ return list(rows.values())
+
+
+def _graphql_error(body: Any) -> Optional[str]:
+ errors = body.get("errors") if isinstance(body, dict) else None
+ if not errors:
+ return None
+ first = errors[0]
+ return str(first.get("message") or first) if isinstance(first, dict) else str(first)
+
+
+def _dns_analytics_report(api_token: str) -> Iterator[list[dict[str, Any]]]:
+ client = RESTClient(base_url=CLOUDFLARE_BASE_URL, auth=BearerTokenAuth(token=api_token))
+ until = datetime.now(UTC).replace(microsecond=0)
+ window = {"since": _to_rfc3339(until - DNS_ANALYTICS_WINDOW), "until": _to_rfc3339(until)}
+
+ zones_read = 0
+ first_error: Optional[str] = None
+ for zones in client.paginate(
+ _PARENT_PATHS[ZONES_PARENT],
+ params={"per_page": PAGE_SIZE},
+ paginator=CloudflarePaginator(),
+ data_selector="result",
+ ):
+ for zone in zones:
+ zone_id = zone.get("id")
+ if not zone_id:
+ continue
+ [body] = next(
+ client.paginate(
+ CLOUDFLARE_ENDPOINTS["dns_analytics_report"].path,
+ method="post",
+ json={"query": _DNS_ANALYTICS_QUERY, "variables": {**window, "zoneTag": zone_id}},
+ paginator=SinglePagePaginator(),
+ )
+ )
+ # GraphQL reports a zone the token can't read as an error in a 200 body. Skip that
+ # zone and keep syncing the rest, like the REST fan-out does for a 403.
+ error = _graphql_error(body)
+ if error is not None:
+ logger.info("cloudflare.dns_analytics_zone_skipped", zone_id=zone_id, error=error)
+ first_error = first_error or error
+ continue
+ zones_read += 1
+ zone_results = ((body.get("data") or {}).get("viewer") or {}).get("zones") or []
+ groups = zone_results[0].get("dnsAnalyticsAdaptiveGroups") or [] if zone_results else []
+ rows = _dns_analytics_rows(zone_id, groups)
+ if rows:
+ yield rows
+
+ if first_error is not None and zones_read == 0:
+ raise ValueError(f"{DNS_ANALYTICS_ALL_ZONES_FAILED}: {first_error}")
# A logpush job's `destination_conf` is a URI whose query string can carry the
@@ -187,7 +278,6 @@ def _redact_custom_hostname(row: dict[str, Any]) -> dict[str, Any]:
_DATA_MAPS = {
- "dns_analytics_report": _flatten_dns_analytics_row,
"logpush_jobs": _redact_logpush_destination,
"access_apps": _redact_access_app,
"healthchecks": _redact_healthcheck,
@@ -453,6 +543,15 @@ def cloudflare_source(
) -> SourceResponse:
config = CLOUDFLARE_ENDPOINTS[endpoint]
+ if endpoint == "dns_analytics_report":
+ return SourceResponse(
+ name=endpoint,
+ items=lambda: _dns_analytics_report(api_token),
+ primary_keys=list(config.primary_keys),
+ partition_count=1,
+ partition_size=1,
+ )
+
if config.parent is not None:
resource = _fanout_resource(
api_token,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/settings.py
index fa0f5bf8044f..4a420df79ce1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/settings.py
@@ -1,4 +1,5 @@
from dataclasses import dataclass, field
+from datetime import timedelta
from typing import Any, Optional
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
@@ -13,12 +14,8 @@ PAGE_PAGINATION = "page" # ?page= / ?per_page=, terminated by result_info
CURSOR_PAGINATION = "cursor" # ?cursor=, terminated by an absent next cursor
SINGLE_PAGE = "single" # the whole collection comes back in one response
-# Metrics and dimensions requested from the DNS analytics report. Cloudflare returns
-# each row as positional `metrics`/`dimensions` arrays, so these names are also what
-# the rows are flattened into. Both lists come from the vendor spec's examples for
-# `dns-analytics_metrics` / `dns-analytics_query.dimensions`.
-DNS_ANALYTICS_METRICS = ("queryCount", "uncachedCount")
-DNS_ANALYTICS_DIMENSIONS = ("queryType", "responseCode")
+# Matches the default window of the REST report this replaces.
+DNS_ANALYTICS_WINDOW = timedelta(hours=6)
@dataclass
@@ -49,8 +46,8 @@ class CloudflareEndpointConfig:
# Most Cloudflare v4 REST lists are small configuration tables with no updated-since
# filter, so they sync as full refreshes. `audit_logs` is the exception — it takes a
# server-side `since` filter, so it syncs incrementally.
-# The high-volume traffic datasets live in the separate GraphQL Analytics API, which
-# this REST source has no transport for.
+# The high-volume traffic datasets live in the separate GraphQL Analytics API. Only
+# `dns_analytics_report` reads it, because Cloudflare removes the REST endpoint it used.
CLOUDFLARE_ENDPOINTS: dict[str, CloudflareEndpointConfig] = {
"accounts": CloudflareEndpointConfig(
name="accounts",
@@ -218,16 +215,8 @@ CLOUDFLARE_ENDPOINTS: dict[str, CloudflareEndpointConfig] = {
# --- Zone-scoped analytics ---
"dns_analytics_report": CloudflareEndpointConfig(
name="dns_analytics_report",
- path="/zones/{zone_id}/dns_analytics/report",
- primary_keys=("_zone_id", *DNS_ANALYTICS_DIMENSIONS),
- parent=ZONES_PARENT,
- parent_key="_zone_id",
- pagination=SINGLE_PAGE,
- data_selector="result.data",
- params={
- "metrics": ",".join(DNS_ANALYTICS_METRICS),
- "dimensions": ",".join(DNS_ANALYTICS_DIMENSIONS),
- },
+ path="/graphql",
+ primary_keys=("_zone_id", "queryType", "responseCode"),
),
# --- Account-scoped ---
"audit_logs": CloudflareEndpointConfig(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/source.py
index 199199d8c251..3dcc1d081f0d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/source.py
@@ -9,6 +9,7 @@ from products.warehouse_sources.backend.facade.source_config import (
SourceFieldInputConfigType,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.cloudflare.cloudflare import (
+ DNS_ANALYTICS_ALL_ZONES_FAILED,
cloudflare_source,
validate_credentials as validate_cloudflare_credentials,
)
@@ -192,6 +193,7 @@ class CloudflareSource(SimpleSource[CloudflareSourceConfig]):
return {
"401 Client Error: Unauthorized for url: https://api.cloudflare.com": "Cloudflare authentication failed. Please check your API token.",
"403 Client Error: Forbidden for url: https://api.cloudflare.com": "Cloudflare denied access. Please check that your API token has read permissions for this resource.",
+ DNS_ANALYTICS_ALL_ZONES_FAILED: "Cloudflare refused the DNS analytics query for every zone. Please check that your API token has Analytics read permission for your zones.",
}
@property
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/tests/test_cloudflare_client.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/tests/test_cloudflare_client.py
index c8069ad7039e..705a9777d291 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/tests/test_cloudflare_client.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/tests/test_cloudflare_client.py
@@ -6,6 +6,7 @@ import pytest
import requests
from products.warehouse_sources.backend.temporal.data_imports.sources.cloudflare.cloudflare import (
+ DNS_ANALYTICS_ALL_ZONES_FAILED,
PAGE_SIZE,
TokenCheck,
_redact_access_app,
@@ -256,12 +257,7 @@ class TestSinglePageEndpoints:
# These endpoints document no page params, so Cloudflare ignores `page` and would
# return the same rows forever if the page-number paginator were used.
full_page = [{"id": str(i)} for i in range(PAGE_SIZE)]
- child = (
- _raw_response({"success": True, "result": {"data": full_page}})
- if endpoint == "dns_analytics_report"
- else _response(full_page)
- )
- result = _driver().run(endpoint, [_response([{"id": "z1"}], total_pages=1), child])
+ result = _driver().run(endpoint, [_response([{"id": "z1"}], total_pages=1), _response(full_page)])
assert result.raised is None
assert len(result.requests) == 2
@@ -269,36 +265,70 @@ class TestSinglePageEndpoints:
assert result.requests[1].param("per_page") is None
+def _dns_groups(*groups: tuple[str, str, int, int]) -> ScriptedResponse:
+ return _raw_response(
+ {
+ "data": {
+ "viewer": {
+ "zones": [
+ {
+ "dnsAnalyticsAdaptiveGroups": [
+ {
+ "count": count,
+ "dimensions": {
+ "queryType": query_type,
+ "responseCode": response_code,
+ "responseCached": cached,
+ },
+ }
+ for query_type, response_code, cached, count in groups
+ ]
+ }
+ ]
+ }
+ },
+ "errors": None,
+ }
+ )
+
+
+def _graphql_error(message: str) -> ScriptedResponse:
+ return _raw_response({"data": None, "errors": [{"message": message}]})
+
+
class TestDnsAnalyticsReport:
- def test_positional_report_arrays_become_named_columns(self) -> None:
+ def test_cached_groups_sum_into_report_columns_and_unreadable_zones_are_skipped(self) -> None:
result = _driver().run(
"dns_analytics_report",
[
- _response([{"id": "z1"}], total_pages=1),
- _raw_response(
- {
- "success": True,
- "result": {
- "rows": 1,
- "data": [{"dimensions": ["A", "NOERROR"], "metrics": [10, 4]}],
- },
- }
- ),
+ _response([{"id": "z1"}, {"id": "z2"}], total_pages=1),
+ _graphql_error("not authorized for zone z1"),
+ _dns_groups(("A", "NOERROR", 1, 6), ("A", "NOERROR", 0, 4), ("MX", "NXDOMAIN", 0, 2)),
],
)
assert result.raised is None
assert result.rows == [
- {
- "queryType": "A",
- "responseCode": "NOERROR",
- "queryCount": 10,
- "uncachedCount": 4,
- "_zone_id": "z1",
- }
+ {"queryType": "A", "responseCode": "NOERROR", "queryCount": 10, "uncachedCount": 4, "_zone_id": "z2"},
+ {"queryType": "MX", "responseCode": "NXDOMAIN", "queryCount": 2, "uncachedCount": 2, "_zone_id": "z2"},
]
- assert result.requests[1].param("metrics") == "queryCount,uncachedCount"
- assert result.requests[1].param("dimensions") == "queryType,responseCode"
+ graphql_request = result.requests[2]
+ assert (graphql_request.method, graphql_request.path) == ("POST", "/client/v4/graphql")
+ assert graphql_request.json()["variables"]["zoneTag"] == "z2"
+
+ def test_fails_when_every_zone_is_refused(self) -> None:
+ result = _driver().run(
+ "dns_analytics_report",
+ [
+ _response([{"id": "z1"}, {"id": "z2"}], total_pages=1),
+ _graphql_error("unknown field dnsAnalyticsAdaptiveGroups"),
+ _graphql_error("unknown field dnsAnalyticsAdaptiveGroups"),
+ ],
+ )
+
+ assert isinstance(result.raised, ValueError)
+ assert DNS_ANALYTICS_ALL_ZONES_FAILED in str(result.raised)
+ assert "unknown field dnsAnalyticsAdaptiveGroups" in str(result.raised)
class TestLogpushDestinationRedaction: