:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): move cloudflare dns_analytics_report to graphql (#119130)
posthog[bot]b0f81923a8fa
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/canonical_descriptions.pyindex daa23347ea57..4ab09fa88bbb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/canonical_descriptions.py@@ -398,8 +398,8 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {},},"dns_analytics_report": {- "description": "Authoritative DNS query counts for a zone, grouped by query type and response code.",- "docs_url": "https://developers.cloudflare.com/api/resources/dns/subresources/analytics/",+ "description": "Authoritative DNS query counts for a zone over the 6 hours before the sync, grouped by query type and response code.",+ "docs_url": "https://developers.cloudflare.com/analytics/graphql-api/","columns": {"queryType": "The DNS record type that was queried (e.g. A, AAAA, MX).","responseCode": "The DNS response code Cloudflare returned (e.g. NOERROR, NXDOMAIN).",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/cloudflare.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/cloudflare.pyindex edd0e5f95e2d..31d00bdaf893 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/cloudflare.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/cloudflare.py@@ -1,7 +1,9 @@+from collections.abc import Iteratorfrom datetime import UTC, date, datetimefrom typing import Any, Optionalfrom urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit+import structlogfrom requests import Responsefrom posthog.dataclasses import frozen@@ -10,8 +12,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.cloudflareACCOUNTS_PARENT,CLOUDFLARE_ENDPOINTS,CURSOR_PAGINATION,- DNS_ANALYTICS_DIMENSIONS,- DNS_ANALYTICS_METRICS,+ DNS_ANALYTICS_WINDOW,PAGE_PAGINATION,SINGLE_PAGE,ZONES_PARENT,@@ -23,6 +24,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resrest_api_resource,rest_api_resources,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import BearerTokenAuthfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.jsonpath_utils import (find_values,)@@ -33,6 +35,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resSinglePagePaginator,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClientfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (ClientConfig,Endpoint,@@ -40,6 +43,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res)from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse+logger = structlog.get_logger(__name__)+CLOUDFLARE_BASE_URL = "https://api.cloudflare.com/client/v4"# Cloudflare list pages cap at 50 by default; most endpoints allow more.PAGE_SIZE = 50@@ -91,13 +96,99 @@ def _to_rfc3339(value: Any) -> Optional[str]:return str(value)-def _flatten_dns_analytics_row(row: dict[str, Any]) -> dict[str, Any]:- """The DNS analytics report returns positional `dimensions`/`metrics` arrays that- line up with the names we requested, so name them into their own columns."""- flattened = {key: value for key, value in row.items() if key not in ("dimensions", "metrics")}- flattened.update(zip(DNS_ANALYTICS_DIMENSIONS, row.get("dimensions") or []))- flattened.update(zip(DNS_ANALYTICS_METRICS, row.get("metrics") or []))- return flattened+# The GraphQL dataset has no uncached-count metric, so the query also groups on+# `responseCached` and `_dns_analytics_rows` sums the uncached groups.+_DNS_ANALYTICS_QUERY = """+query DnsAnalyticsReport($zoneTag: string, $since: Time, $until: Time) {+ viewer {+ zones(filter: {zoneTag: $zoneTag}) {+ dnsAnalyticsAdaptiveGroups(limit: 10000, filter: {datetime_geq: $since, datetime_lt: $until}) {+ count+ dimensions {+ queryType+ responseCode+ responseCached+ }+ }+ }+ }+}+"""++DNS_ANALYTICS_ALL_ZONES_FAILED = "Cloudflare refused the DNS analytics query for every zone"+++def _dns_analytics_rows(zone_id: str, groups: list[dict[str, Any]]) -> list[dict[str, Any]]:+ rows: dict[tuple[Any, Any], dict[str, Any]] = {}+ for group in groups:+ dimensions = group.get("dimensions") or {}+ query_type, response_code = dimensions.get("queryType"), dimensions.get("responseCode")+ row = rows.setdefault(+ (query_type, response_code),+ {+ "queryType": query_type,+ "responseCode": response_code,+ "queryCount": 0,+ "uncachedCount": 0,+ "_zone_id": zone_id,+ },+ )+ count = group.get("count") or 0+ row["queryCount"] += count+ if not dimensions.get("responseCached"):+ row["uncachedCount"] += count+ return list(rows.values())+++def _graphql_error(body: Any) -> Optional[str]:+ errors = body.get("errors") if isinstance(body, dict) else None+ if not errors:+ return None+ first = errors[0]+ return str(first.get("message") or first) if isinstance(first, dict) else str(first)+++def _dns_analytics_report(api_token: str) -> Iterator[list[dict[str, Any]]]:+ client = RESTClient(base_url=CLOUDFLARE_BASE_URL, auth=BearerTokenAuth(token=api_token))+ until = datetime.now(UTC).replace(microsecond=0)+ window = {"since": _to_rfc3339(until - DNS_ANALYTICS_WINDOW), "until": _to_rfc3339(until)}++ zones_read = 0+ first_error: Optional[str] = None+ for zones in client.paginate(+ _PARENT_PATHS[ZONES_PARENT],+ params={"per_page": PAGE_SIZE},+ paginator=CloudflarePaginator(),+ data_selector="result",+ ):+ for zone in zones:+ zone_id = zone.get("id")+ if not zone_id:+ continue+ [body] = next(+ client.paginate(+ CLOUDFLARE_ENDPOINTS["dns_analytics_report"].path,+ method="post",+ json={"query": _DNS_ANALYTICS_QUERY, "variables": {**window, "zoneTag": zone_id}},+ paginator=SinglePagePaginator(),+ )+ )+ # GraphQL reports a zone the token can't read as an error in a 200 body. Skip that+ # zone and keep syncing the rest, like the REST fan-out does for a 403.+ error = _graphql_error(body)+ if error is not None:+ logger.info("cloudflare.dns_analytics_zone_skipped", zone_id=zone_id, error=error)+ first_error = first_error or error+ continue+ zones_read += 1+ zone_results = ((body.get("data") or {}).get("viewer") or {}).get("zones") or []+ groups = zone_results[0].get("dnsAnalyticsAdaptiveGroups") or [] if zone_results else []+ rows = _dns_analytics_rows(zone_id, groups)+ if rows:+ yield rows++ if first_error is not None and zones_read == 0:+ raise ValueError(f"{DNS_ANALYTICS_ALL_ZONES_FAILED}: {first_error}")# A logpush job's `destination_conf` is a URI whose query string can carry the@@ -187,7 +278,6 @@ def _redact_custom_hostname(row: dict[str, Any]) -> dict[str, Any]:_DATA_MAPS = {- "dns_analytics_report": _flatten_dns_analytics_row,"logpush_jobs": _redact_logpush_destination,"access_apps": _redact_access_app,"healthchecks": _redact_healthcheck,@@ -453,6 +543,15 @@ def cloudflare_source() -> SourceResponse:config = CLOUDFLARE_ENDPOINTS[endpoint]+ if endpoint == "dns_analytics_report":+ return SourceResponse(+ name=endpoint,+ items=lambda: _dns_analytics_report(api_token),+ primary_keys=list(config.primary_keys),+ partition_count=1,+ partition_size=1,+ )+if config.parent is not None:resource = _fanout_resource(api_token,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/settings.pyindex fa0f5bf8044f..4a420df79ce1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/settings.py@@ -1,4 +1,5 @@from dataclasses import dataclass, field+from datetime import timedeltafrom typing import Any, Optionalfrom products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType@@ -13,12 +14,8 @@ PAGE_PAGINATION = "page" # ?page= / ?per_page=, terminated by result_infoCURSOR_PAGINATION = "cursor" # ?cursor=, terminated by an absent next cursorSINGLE_PAGE = "single" # the whole collection comes back in one response-# Metrics and dimensions requested from the DNS analytics report. Cloudflare returns-# each row as positional `metrics`/`dimensions` arrays, so these names are also what-# the rows are flattened into. Both lists come from the vendor spec's examples for-# `dns-analytics_metrics` / `dns-analytics_query.dimensions`.-DNS_ANALYTICS_METRICS = ("queryCount", "uncachedCount")-DNS_ANALYTICS_DIMENSIONS = ("queryType", "responseCode")+# Matches the default window of the REST report this replaces.+DNS_ANALYTICS_WINDOW = timedelta(hours=6)@dataclass@@ -49,8 +46,8 @@ class CloudflareEndpointConfig:# Most Cloudflare v4 REST lists are small configuration tables with no updated-since# filter, so they sync as full refreshes. `audit_logs` is the exception — it takes a# server-side `since` filter, so it syncs incrementally.-# The high-volume traffic datasets live in the separate GraphQL Analytics API, which-# this REST source has no transport for.+# The high-volume traffic datasets live in the separate GraphQL Analytics API. Only+# `dns_analytics_report` reads it, because Cloudflare removes the REST endpoint it used.CLOUDFLARE_ENDPOINTS: dict[str, CloudflareEndpointConfig] = {"accounts": CloudflareEndpointConfig(name="accounts",@@ -218,16 +215,8 @@ CLOUDFLARE_ENDPOINTS: dict[str, CloudflareEndpointConfig] = {# --- Zone-scoped analytics ---"dns_analytics_report": CloudflareEndpointConfig(name="dns_analytics_report",- path="/zones/{zone_id}/dns_analytics/report",- primary_keys=("_zone_id", *DNS_ANALYTICS_DIMENSIONS),- parent=ZONES_PARENT,- parent_key="_zone_id",- pagination=SINGLE_PAGE,- data_selector="result.data",- params={- "metrics": ",".join(DNS_ANALYTICS_METRICS),- "dimensions": ",".join(DNS_ANALYTICS_DIMENSIONS),- },+ path="/graphql",+ primary_keys=("_zone_id", "queryType", "responseCode"),),# --- Account-scoped ---"audit_logs": CloudflareEndpointConfig(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/source.pyindex 199199d8c251..3dcc1d081f0d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/source.py@@ -9,6 +9,7 @@ from products.warehouse_sources.backend.facade.source_config import (SourceFieldInputConfigType,)from products.warehouse_sources.backend.temporal.data_imports.sources.cloudflare.cloudflare import (+ DNS_ANALYTICS_ALL_ZONES_FAILED,cloudflare_source,validate_credentials as validate_cloudflare_credentials,)@@ -192,6 +193,7 @@ class CloudflareSource(SimpleSource[CloudflareSourceConfig]):return {"401 Client Error: Unauthorized for url: https://api.cloudflare.com": "Cloudflare authentication failed. Please check your API token.","403 Client Error: Forbidden for url: https://api.cloudflare.com": "Cloudflare denied access. Please check that your API token has read permissions for this resource.",+ DNS_ANALYTICS_ALL_ZONES_FAILED: "Cloudflare refused the DNS analytics query for every zone. Please check that your API token has Analytics read permission for your zones.",}@propertydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/tests/test_cloudflare_client.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/tests/test_cloudflare_client.pyindex c8069ad7039e..705a9777d291 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/tests/test_cloudflare_client.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/tests/test_cloudflare_client.py@@ -6,6 +6,7 @@ import pytestimport requestsfrom products.warehouse_sources.backend.temporal.data_imports.sources.cloudflare.cloudflare import (+ DNS_ANALYTICS_ALL_ZONES_FAILED,PAGE_SIZE,TokenCheck,_redact_access_app,@@ -256,12 +257,7 @@ class TestSinglePageEndpoints:# These endpoints document no page params, so Cloudflare ignores `page` and would# return the same rows forever if the page-number paginator were used.full_page = [{"id": str(i)} for i in range(PAGE_SIZE)]- child = (- _raw_response({"success": True, "result": {"data": full_page}})- if endpoint == "dns_analytics_report"- else _response(full_page)- )- result = _driver().run(endpoint, [_response([{"id": "z1"}], total_pages=1), child])+ result = _driver().run(endpoint, [_response([{"id": "z1"}], total_pages=1), _response(full_page)])assert result.raised is Noneassert len(result.requests) == 2@@ -269,36 +265,70 @@ class TestSinglePageEndpoints:assert result.requests[1].param("per_page") is None+def _dns_groups(*groups: tuple[str, str, int, int]) -> ScriptedResponse:+ return _raw_response(+ {+ "data": {+ "viewer": {+ "zones": [+ {+ "dnsAnalyticsAdaptiveGroups": [+ {+ "count": count,+ "dimensions": {+ "queryType": query_type,+ "responseCode": response_code,+ "responseCached": cached,+ },+ }+ for query_type, response_code, cached, count in groups+ ]+ }+ ]+ }+ },+ "errors": None,+ }+ )+++def _graphql_error(message: str) -> ScriptedResponse:+ return _raw_response({"data": None, "errors": [{"message": message}]})++class TestDnsAnalyticsReport:- def test_positional_report_arrays_become_named_columns(self) -> None:+ def test_cached_groups_sum_into_report_columns_and_unreadable_zones_are_skipped(self) -> None:result = _driver().run("dns_analytics_report",[- _response([{"id": "z1"}], total_pages=1),- _raw_response(- {- "success": True,- "result": {- "rows": 1,- "data": [{"dimensions": ["A", "NOERROR"], "metrics": [10, 4]}],- },- }- ),+ _response([{"id": "z1"}, {"id": "z2"}], total_pages=1),+ _graphql_error("not authorized for zone z1"),+ _dns_groups(("A", "NOERROR", 1, 6), ("A", "NOERROR", 0, 4), ("MX", "NXDOMAIN", 0, 2)),],)assert result.raised is Noneassert result.rows == [- {- "queryType": "A",- "responseCode": "NOERROR",- "queryCount": 10,- "uncachedCount": 4,- "_zone_id": "z1",- }+ {"queryType": "A", "responseCode": "NOERROR", "queryCount": 10, "uncachedCount": 4, "_zone_id": "z2"},+ {"queryType": "MX", "responseCode": "NXDOMAIN", "queryCount": 2, "uncachedCount": 2, "_zone_id": "z2"},]- assert result.requests[1].param("metrics") == "queryCount,uncachedCount"- assert result.requests[1].param("dimensions") == "queryType,responseCode"+ graphql_request = result.requests[2]+ assert (graphql_request.method, graphql_request.path) == ("POST", "/client/v4/graphql")+ assert graphql_request.json()["variables"]["zoneTag"] == "z2"++ def test_fails_when_every_zone_is_refused(self) -> None:+ result = _driver().run(+ "dns_analytics_report",+ [+ _response([{"id": "z1"}, {"id": "z2"}], total_pages=1),+ _graphql_error("unknown field dnsAnalyticsAdaptiveGroups"),+ _graphql_error("unknown field dnsAnalyticsAdaptiveGroups"),+ ],+ )++ assert isinstance(result.raised, ValueError)+ assert DNS_ANALYTICS_ALL_ZONES_FAILED in str(result.raised)+ assert "unknown field dnsAnalyticsAdaptiveGroups" in str(result.raised)class TestLogpushDestinationRedaction: