:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add aws waf traffic and managed rule tables (#118539)
Tom Owers46a02590b182
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex de5ddfc4091e..bf575f92954e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -1353,10 +1353,10 @@ Today (4): `ip_sets`, `regex_pattern_sets`, `rule_groups`, `web_acls`Diffed against: <https://raw.githubusercontent.com/boto/botocore/develop/botocore/data/wafv2/2019-07-29/service-2.json>-- [ ] `Managed rule groups (ListAvailableManagedRuleGroups and DescribeManagedRuleGroup, POST /)` — resolve managed rule group references in web ACLs to vendor, rules, labels, and capacity (high)-- [ ] `Protected resources (ListResourcesForWebACL, POST /)` — connect web ACLs to protected application resources and measure protection coverage (high)-- [ ] `WAF request metrics (CloudWatch GetMetricData, POST /)` — measure allowed, blocked, and counted requests over time, which resource inventories cannot provide (high)-- [ ] `Sampled requests (GetSampledRequests, POST /)` — analyze request actions, matching rules, client countries, and attack patterns from recent traffic samples (high)+- [x] `Managed rule groups (ListAvailableManagedRuleGroups and DescribeManagedRuleGroup, POST /)` — resolve managed rule group references in web ACLs to vendor, rules, labels, and capacity (high)+- [x] `Protected resources (ListResourcesForWebACL, POST /)` — connect web ACLs to protected application resources and measure protection coverage (high)+- [x] `WAF request metrics (CloudWatch GetMetricData, POST /)` — measure allowed, blocked, and counted requests over time, which resource inventories cannot provide (high)+- [x] `Sampled requests (GetSampledRequests, POST /)` — analyze request actions, matching rules, client countries, and attack patterns from recent traffic samples (high)- [ ] `Top paths by bot traffic (GetTopPathStatisticsByTraffic, POST /)` — identify application paths and bot categories that account for the most bot traffic (medium)- [ ] `Rate-based rule managed keys (GetRateBasedStatementManagedKeys, POST /)` — track IP addresses currently blocked by rate-based rules and compare rules across web ACLs (medium)- [ ] `Settlement records (ListSettlementRecords, POST /)` — analyze individual monetization payments for protected CloudFront resources (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/aws_waf.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/aws_waf.pyindex 1c44c1c25d98..d343f297ec9c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/aws_waf.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/aws_waf.py@@ -1,6 +1,8 @@import reimport json-from collections.abc import Iterator+import hashlib+from collections.abc import Callable, Iterator+from datetime import UTC, datetimefrom typing import TYPE_CHECKING, Anyimport requests@@ -16,12 +18,29 @@ from posthog.dataclasses import frozenfrom products.warehouse_sources.backend.temporal.data_imports.sources.aws_waf.settings import (ACCESS_DENIED_CODES,AWS_WAF_ENDPOINTS,+ CLOUDWATCH_SERVICE,ERROR_MESSAGES,+ LIST_CALLS,+ MANAGED_RULE_GROUPS,+ METRICS_HISTORY,+ METRICS_NAMESPACE,+ METRICS_PER_REQUEST,+ METRICS_PERIOD_SECONDS,PAGE_SIZE,- TARGET_PREFIX,+ PRIMARY_KEYS,+ PROTECTED_RESOURCE_TYPES,+ REDACTED_HEADERS,+ REQUEST_METRICS,+ SAMPLED_REQUESTS,+ SAMPLED_REQUESTS_MAX_ITEMS,+ SAMPLED_REQUESTS_WINDOW,+ TABLE_PERMISSIONS,WAF_API_VERSION,- WafEndpoint,+ WAFV2_SERVICE,+ WEB_ACL_RESOURCES,+ AwsService,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_valuefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_sessionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.awswaf import AwsWafSourceConfig@@ -78,8 +97,29 @@ def retry_wait(state: RetryCallState) -> float:return max(delay, error.retry_after) if isinstance(error, AwsWafRetryableError) else delay+def utc_now() -> datetime:+ return datetime.now(UTC)+++def snake_keys(data: dict[str, Any]) -> dict[str, Any]:+ return {+ re.sub(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])", "_", key).lower(): value+ for key, value in data.items()+ }+++def from_epoch(value: Any) -> datetime | None:+ return datetime.fromtimestamp(value, UTC) if isinstance(value, int | float) else None++class AwsWafClient:- def __init__(self, config: AwsWafSourceConfig, api_version: str = WAF_API_VERSION) -> None:+ def __init__(+ self,+ config: AwsWafSourceConfig,+ api_version: str = WAF_API_VERSION,+ service: AwsService = WAFV2_SERVICE,+ capture: bool = True,+ ) -> None:if api_version != WAF_API_VERSION:raise ValueError(f"Unsupported AWS WAF API version: {api_version}")if not config.aws_access_key_id or not config.aws_secret_access_key:@@ -91,15 +131,17 @@ class AwsWafClient:self.scope = config.scopeself.region = "us-east-1" if self.scope == "CLOUDFRONT" else config.aws_regionsuffix = "amazonaws.com.cn" if self.region.startswith("cn-") else "amazonaws.com"- self.url = f"https://wafv2.{self.region}.{suffix}/"+ self.service = service+ self.url = f"https://{service.signing_name}.{self.region}.{suffix}/"self.signer = SigV4Auth(Credentials(config.aws_access_key_id, config.aws_secret_access_key, config.aws_session_token or None),- "wafv2",+ service.signing_name,self.region,)# WAF can throttle with HTTP 400, so one retry layer handles both HTTP statuses and AWS error codes.self.session = make_tracked_session(retry=Retry(total=0),+ capture=capture,redact_values=tuple(value for value in (config.aws_secret_access_key, config.aws_session_token) if value),)@@ -118,7 +160,10 @@ class AwsWafClient:method="POST",url=self.url,data=body,- headers={"Content-Type": "application/x-amz-json-1.1", "X-Amz-Target": f"{TARGET_PREFIX}.{operation}"},+ headers={+ "Content-Type": self.service.content_type,+ "X-Amz-Target": f"{self.service.target_prefix}.{operation}",+ },)self.signer.add_auth(request)response = self.session.post(@@ -128,39 +173,86 @@ class AwsWafClient:raise error_for_response(response)return response.json()- def get_resource(self, endpoint: WafEndpoint, summary: dict[str, Any]) -> dict[str, Any]:- result = self.request(- endpoint.get_operation, {"Scope": self.scope, "Id": summary["Id"], "Name": summary["Name"]}- )- resource = result[endpoint.object_key]- row = {- re.sub(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])", "_", key).lower(): value- for key, value in resource.items()- }+ def with_location(self, row: dict[str, Any]) -> dict[str, Any]:row["region"] = self.regionrow["scope"] = self.scopereturn row+ def get_resource(self, endpoint: str, summary: dict[str, Any]) -> dict[str, Any]:+ if endpoint == MANAGED_RULE_GROUPS:+ result = self.request(+ "DescribeManagedRuleGroup",+ {"Scope": self.scope, "VendorName": summary["VendorName"], "Name": summary["Name"]},+ )+ return self.with_location({**snake_keys(summary), **snake_keys(result)})+ waf_endpoint = AWS_WAF_ENDPOINTS[endpoint]+ result = self.request(+ waf_endpoint.get_operation, {"Scope": self.scope, "Id": summary["Id"], "Name": summary["Name"]}+ )+ return self.with_location(snake_keys(result[waf_endpoint.object_key]))++ def get_web_acl(self, summary: dict[str, Any]) -> dict[str, Any] | None:+ try:+ return self.request("GetWebACL", {"Scope": self.scope, "Id": summary["Id"], "Name": summary["Name"]})[+ "WebACL"+ ]+ except AwsWafError as error:+ if error.code != "WAFNonexistentItemException":+ raise+ return None++ def list_resources(self, web_acl_arn: str, resource_type: str) -> list[str]:+ try:+ result = self.request("ListResourcesForWebACL", {"WebACLArn": web_acl_arn, "ResourceType": resource_type})+ except AwsWafError as error:+ # Some resource types are not offered in every region, and AWS rejects them as invalid parameters.+ if error.code not in {"WAFNonexistentItemException", "WAFInvalidParameterException"}:+ raise+ return []+ return result.get("ResourceArns") or []++ def get_sampled_requests(+ self, web_acl_arn: str, metric_name: str, start: datetime, end: datetime, max_items: int+ ) -> dict[str, Any]:+ try:+ return self.request(+ "GetSampledRequests",+ {+ "WebAclArn": web_acl_arn,+ "RuleMetricName": metric_name,+ "Scope": self.scope,+ "TimeWindow": {"StartTime": start.timestamp(), "EndTime": end.timestamp()},+ "MaxItems": max_items,+ },+ )+ except AwsWafError as error:+ if error.code != "WAFNonexistentItemException":+ raise+ return {}+def validate_credentials(config: AwsWafSourceConfig, schema_name: str | None = None, api_version: str = WAF_API_VERSION) -> tuple[bool, str | None]:- endpoint = AWS_WAF_ENDPOINTS.get(schema_name or "web_acls")- if endpoint is None:+ table = schema_name or "web_acls"+ if table not in TABLE_PERMISSIONS:return False, f"Unknown AWS WAF table: {schema_name}"try:- client = AwsWafClient(config, api_version)+ # Sampled requests carry end-user headers, so they stay out of HTTP sample capture.+ client = AwsWafClient(config, api_version, capture=table != SAMPLED_REQUESTS)except ValueError as error:return False, str(error)try:- result = client.request(endpoint.list_operation, {"Scope": client.scope, "Limit": 1})- if schema_name and result[endpoint.list_key]:- client.get_resource(endpoint, result[endpoint.list_key][0])+ if schema_name is None:+ client.request("ListWebACLs", {"Scope": client.scope, "Limit": 1})+ else:+ probe_table(client, config, table, api_version)except AwsWafError as error:if error.code in ACCESS_DENIED_CODES:if schema_name is None:return True, None- return False, f"Grant wafv2:{endpoint.list_operation} and wafv2:{endpoint.get_operation} for this table."+ *others, last = TABLE_PERMISSIONS[table]+ return False, f"Grant {', '.join(others)} and {last} for this table."if error.code in ERROR_MESSAGES:return False, ERROR_MESSAGES[error.code]raise@@ -169,23 +261,90 @@ def validate_credentials(return True, None+def sampled_metric_names(web_acl: dict[str, Any]) -> list[str]:+ rules = [+ *(web_acl.get("PreProcessFirewallManagerRuleGroups") or []),+ *(web_acl.get("Rules") or []),+ *(web_acl.get("PostProcessFirewallManagerRuleGroups") or []),+ ]+ names = [+ rule["VisibilityConfig"]["MetricName"]+ for rule in rules+ if rule.get("VisibilityConfig", {}).get("SampledRequestsEnabled")+ ]+ return list(dict.fromkeys(names))+++def probe_table(client: AwsWafClient, config: AwsWafSourceConfig, table: str, api_version: str) -> None:+ if table == REQUEST_METRICS:+ metrics_client = AwsWafClient(config, api_version, CLOUDWATCH_SERVICE)+ try:+ metrics_client.request("ListMetrics", {"Namespace": METRICS_NAMESPACE})+ finally:+ metrics_client.close()+ return+ list_call = LIST_CALLS.get(table, LIST_CALLS["web_acls"])+ summaries = client.request(list_call.operation, {"Scope": client.scope, "Limit": 1})[list_call.key]+ if not summaries:+ return+ if table == WEB_ACL_RESOURCES:+ if client.scope == "REGIONAL":+ client.list_resources(summaries[0]["ARN"], PROTECTED_RESOURCE_TYPES[0])+ elif table == SAMPLED_REQUESTS:+ web_acl = client.get_web_acl(summaries[0])+ if web_acl and (names := sampled_metric_names(web_acl)):+ end = utc_now()+ client.get_sampled_requests(web_acl["ARN"], names[0], end - SAMPLED_REQUESTS_WINDOW, end, 1)+ else:+ client.get_resource(table, summaries[0])+++def iter_web_acls(+ client: AwsWafClient,+ manager: "ResumableSourceManager[AwsWafResumeConfig]",+ rows_for_web_acl: Callable[[dict[str, Any]], Iterator[list[dict[str, Any]]]],+) -> Iterator[list[dict[str, Any]]]:+ """Yield the rows of every web ACL.++ The marker of the current page stays staged until the next page is read, so a resumed run reads that page again.+ """+ state = manager.load_state() or AwsWafResumeConfig()+ marker = state.next_marker+ while True:+ payload: dict[str, Any] = {"Scope": client.scope, "Limit": PAGE_SIZE}+ if marker:+ payload["NextMarker"] = marker+ result = client.request("ListWebACLs", payload)+ next_marker = result.get("NextMarker") or None+ if next_marker and next_marker == marker:+ raise ValueError("AWS WAF returned a repeated pagination marker.")+ manager.save_state(AwsWafResumeConfig(next_marker=marker))+ for web_acl in result["WebACLs"]:+ yield from rows_for_web_acl(web_acl)+ manager.safe_point()+ if next_marker is None:+ return+ marker = next_marker++def get_rows(config: AwsWafSourceConfig,- endpoint: WafEndpoint,+ endpoint: str,manager: "ResumableSourceManager[AwsWafResumeConfig]",api_version: str,) -> Iterator[list[dict[str, Any]]]:state = manager.load_state() or AwsWafResumeConfig()client = AwsWafClient(config, api_version)marker = state.next_marker+ list_call = LIST_CALLS[endpoint]try:while True:payload: dict[str, Any] = {"Scope": client.scope, "Limit": PAGE_SIZE}if marker:payload["NextMarker"] = marker- result = client.request(endpoint.list_operation, payload)+ result = client.request(list_call.operation, payload)rows = []- for summary in result[endpoint.list_key]:+ for summary in result[list_call.key]:try:rows.append(client.get_resource(endpoint, summary))except AwsWafError as error:@@ -205,18 +364,234 @@ def get_rows(client.close()+def protected_resource_rows(+ client: AwsWafClient, manager: "ResumableSourceManager[AwsWafResumeConfig]", web_acl: dict[str, Any]+) -> Iterator[list[dict[str, Any]]]:+ for resource_type in PROTECTED_RESOURCE_TYPES:+ rows = [+ client.with_location(+ {+ "web_acl_arn": web_acl["ARN"],+ "web_acl_id": web_acl["Id"],+ "web_acl_name": web_acl["Name"],+ "resource_type": resource_type,+ "resource_arn": resource_arn,+ }+ )+ for resource_arn in client.list_resources(web_acl["ARN"], resource_type)+ ]+ if rows:+ yield rows+ else:+ manager.safe_point()+++def get_web_acl_resource_rows(+ config: AwsWafSourceConfig, manager: "ResumableSourceManager[AwsWafResumeConfig]", api_version: str+) -> Iterator[list[dict[str, Any]]]:+ client = AwsWafClient(config, api_version)+ try:+ # CloudFront distributions are listed by the CloudFront API, not by ListResourcesForWebACL.+ if client.scope != "REGIONAL":+ return+ yield from iter_web_acls(client, manager, lambda web_acl: protected_resource_rows(client, manager, web_acl))+ finally:+ client.close()+++def sampled_request_row(+ client: AwsWafClient, web_acl: dict[str, Any], metric_name: str, sample: dict[str, Any], population_size: Any+) -> dict[str, Any]:+ request = dict(sample.get("Request") or {})+ request["Headers"] = [+ {**header, "Value": "[REDACTED]"} if str(header.get("Name", "")).lower() in REDACTED_HEADERS else header+ for header in request.get("Headers") or []+ ]+ sample = {**sample, "Request": request}+ digest = hashlib.sha256(+ json.dumps([web_acl["ARN"], metric_name, sample], sort_keys=True, default=str).encode()+ ).hexdigest()+ row = {+ "id": digest,+ "web_acl_arn": web_acl["ARN"],+ "web_acl_name": web_acl["Name"],+ "rule_metric_name": metric_name,+ **snake_keys(request),+ **snake_keys({key: value for key, value in sample.items() if key != "Request"}),+ "population_size": population_size,+ }+ row["timestamp"] = from_epoch(sample.get("Timestamp"))+ return client.with_location(row)+++def sampled_rows_for_web_acl(+ client: AwsWafClient,+ manager: "ResumableSourceManager[AwsWafResumeConfig]",+ summary: dict[str, Any],+ start: datetime,+ end: datetime,+) -> Iterator[list[dict[str, Any]]]:+ web_acl = client.get_web_acl(summary)+ manager.safe_point()+ if web_acl is None:+ return+ for metric_name in sampled_metric_names(web_acl):+ window_start = start+ while window_start < end:+ result = client.get_sampled_requests(+ web_acl["ARN"], metric_name, window_start, end, SAMPLED_REQUESTS_MAX_ITEMS+ )+ rows = [+ sampled_request_row(client, web_acl, metric_name, sample, result.get("PopulationSize"))+ for sample in result.get("SampledRequests") or []+ ]+ if rows:+ yield rows+ else:+ manager.safe_point()+ # AWS ends the returned window early when the rule saw 5,000 requests before the requested end.+ returned_end = from_epoch((result.get("TimeWindow") or {}).get("EndTime"))+ if returned_end is None or returned_end <= window_start or returned_end >= end:+ break+ window_start = returned_end+++def get_sampled_request_rows(+ config: AwsWafSourceConfig,+ manager: "ResumableSourceManager[AwsWafResumeConfig]",+ api_version: str,+ last_value: Any,+) -> Iterator[list[dict[str, Any]]]:+ end = utc_now()+ start = end - SAMPLED_REQUESTS_WINDOW+ watermark = parse_datetime_value(last_value)+ if watermark and watermark > start:+ start = watermark+ if start >= end:+ return+ client = AwsWafClient(config, api_version, capture=False)+ try:+ yield from iter_web_acls(+ client, manager, lambda summary: sampled_rows_for_web_acl(client, manager, summary, start, end)+ )+ finally:+ client.close()+++def metric_id(metric: dict[str, Any]) -> str:+ dimensions = sorted(f"{dimension['Name']}={dimension['Value']}" for dimension in metric.get("Dimensions") or [])+ return "|".join([metric["MetricName"], *dimensions])+++def get_request_metric_rows(+ config: AwsWafSourceConfig,+ manager: "ResumableSourceManager[AwsWafResumeConfig]",+ api_version: str,+ last_value: Any,+) -> Iterator[list[dict[str, Any]]]:+ end = utc_now()+ start = max(parse_datetime_value(last_value) or end - METRICS_HISTORY, end - METRICS_HISTORY)+ start = start.replace(minute=0, second=0, microsecond=0)+ client = AwsWafClient(config, api_version, CLOUDWATCH_SERVICE)+ try:+ metrics: list[dict[str, Any]] = []+ list_payload: dict[str, Any] = {"Namespace": METRICS_NAMESPACE}+ if client.scope == "REGIONAL":+ list_payload["Dimensions"] = [{"Name": "Region", "Value": client.region}]+ while True:+ result = client.request("ListMetrics", list_payload)+ for metric in result.get("Metrics") or []:+ dimension_names = {dimension["Name"] for dimension in metric.get("Dimensions") or []}+ # CloudFront metrics have no Region dimension, regional metrics always have one.+ if (client.scope == "REGIONAL") == ("Region" in dimension_names):+ metrics.append(metric)+ manager.safe_point()+ next_token = result.get("NextToken")+ if not next_token:+ break+ list_payload["NextToken"] = next_token++ for offset in range(0, len(metrics), METRICS_PER_REQUEST):+ batch = {f"m{index}": metric for index, metric in enumerate(metrics[offset : offset + METRICS_PER_REQUEST])}+ payload: dict[str, Any] = {+ "MetricDataQueries": [+ {+ "Id": query_id,+ "MetricStat": {"Metric": metric, "Period": METRICS_PERIOD_SECONDS, "Stat": "Sum"},+ "ReturnData": True,+ }+ for query_id, metric in batch.items()+ ],+ "StartTime": start.timestamp(),+ "EndTime": end.timestamp(),+ "ScanBy": "TimestampAscending",+ }+ while True:+ result = client.request("GetMetricData", payload)+ rows = []+ for data in result.get("MetricDataResults") or []:+ metric = batch[data["Id"]]+ # A failed series still returns HTTP 200, and skipping it would let the watermark pass its gap.+ if data.get("StatusCode") not in (None, "Complete", "PartialData"):+ raise AwsWafRetryableError(+ str(data["StatusCode"]), f"CloudWatch could not return {metric_id(metric)}."+ )+ dimensions = {dimension["Name"]: dimension["Value"] for dimension in metric.get("Dimensions") or []}+ for timestamp, value in zip(data.get("Timestamps") or [], data.get("Values") or []):+ rows.append(+ client.with_location(+ {+ "metric_id": metric_id(metric),+ "metric_name": metric["MetricName"],+ "dimensions": dimensions,+ "web_acl": dimensions.get("WebACL"),+ "rule": dimensions.get("Rule"),+ "timestamp": from_epoch(timestamp),+ "value": value,+ }+ )+ )+ if rows:+ yield rows+ else:+ manager.safe_point()+ next_token = result.get("NextToken")+ if not next_token:+ break+ payload["NextToken"] = next_token+ finally:+ client.close()+++def iter_rows(+ config: AwsWafSourceConfig,+ endpoint: str,+ manager: "ResumableSourceManager[AwsWafResumeConfig]",+ api_version: str,+ incremental_last_value: Any,+) -> Iterator[list[dict[str, Any]]]:+ if endpoint == WEB_ACL_RESOURCES:+ return get_web_acl_resource_rows(config, manager, api_version)+ if endpoint == SAMPLED_REQUESTS:+ return get_sampled_request_rows(config, manager, api_version, incremental_last_value)+ if endpoint == REQUEST_METRICS:+ return get_request_metric_rows(config, manager, api_version, incremental_last_value)+ return get_rows(config, endpoint, manager, api_version)++def aws_waf_source(config: AwsWafSourceConfig,endpoint: str,manager: "ResumableSourceManager[AwsWafResumeConfig]",api_version: str = WAF_API_VERSION,+ incremental_last_value: Any = None,) -> SourceResponse:- if endpoint not in AWS_WAF_ENDPOINTS:+ if endpoint not in PRIMARY_KEYS:raise ValueError(f"Unknown AWS WAF table: {endpoint}")return SourceResponse(name=endpoint,- items=lambda: get_rows(config, AWS_WAF_ENDPOINTS[endpoint], manager, api_version),- primary_keys=["arn"],+ items=lambda: iter_rows(config, endpoint, manager, api_version, incremental_last_value),+ primary_keys=PRIMARY_KEYS[endpoint],sort_mode=None,on_complete=manager.clear_state,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/canonical_descriptions.pyindex 83df2bfe9a72..140fe9e76393 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/canonical_descriptions.py@@ -53,4 +53,77 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"regular_expression_list": "Regular expressions used to match web request components.",},},+ "managed_rule_groups": {+ "description": "Managed rule groups from AWS and subscribed AWS Marketplace sellers that web ACLs can reference.",+ "docs_url": "https://docs.aws.amazon.com/waf/latest/APIReference/API_DescribeManagedRuleGroup.html",+ "columns": {+ "vendor_name": "Name of the managed rule group vendor, such as AWS.",+ "name": "Name of the managed rule group.",+ "description": "Description of the managed rule group.",+ "versioning_supported": "Whether the managed rule group supports versions.",+ "version_name": "Version of the rule group described in this row.",+ "sns_topic_arn": "Amazon SNS topic that receives notifications about rule group updates.",+ "capacity": "Web ACL capacity units that the rule group uses.",+ "rules": "Rules in the rule group, with their names and actions.",+ "label_namespace": "Label namespace prefix for labels that the rules add.",+ "available_labels": "Labels that rules in this group can add to matching requests.",+ "consumed_labels": "Labels that rules in this group can match.",+ "region": COMMON_COLUMNS["region"],+ "scope": COMMON_COLUMNS["scope"],+ },+ },+ "protected_resources": {+ "description": "Regional AWS resources associated with each web ACL.",+ "docs_url": "https://docs.aws.amazon.com/waf/latest/APIReference/API_ListResourcesForWebACL.html",+ "columns": {+ "web_acl_arn": "Amazon Resource Name of the web ACL.",+ "web_acl_id": "Identifier of the web ACL.",+ "web_acl_name": "Name of the web ACL.",+ "resource_type": "Type of the protected resource, such as APPLICATION_LOAD_BALANCER or API_GATEWAY.",+ "resource_arn": "Amazon Resource Name of the protected resource.",+ "region": COMMON_COLUMNS["region"],+ "scope": COMMON_COLUMNS["scope"],+ },+ },+ "sampled_requests": {+ "description": "Samples of web requests that matched a web ACL rule, selected by AWS WAF from recent traffic.",+ "docs_url": "https://docs.aws.amazon.com/waf/latest/APIReference/API_GetSampledRequests.html",+ "columns": {+ "id": "Hash of the web ACL, rule metric name, and sample, used to identify the row.",+ "web_acl_arn": "Amazon Resource Name of the web ACL.",+ "web_acl_name": "Name of the web ACL.",+ "rule_metric_name": "Metric name of the rule or rule group that the sample was taken for.",+ "client_ip": "IP address of the client that sent the request.",+ "country": "Two-letter country code of the client.",+ "uri": "URI path of the request.",+ "method": "HTTP method of the request.",+ "http_version": "HTTP version of the request.",+ "headers": "Request headers. Credential header values are redacted.",+ "timestamp": "Time at which AWS WAF received the request.",+ "action": "Action that AWS WAF applied to the request.",+ "weight": "Number of requests that this sample represents.",+ "rule_name_within_rule_group": "Name of the matching rule inside a rule group.",+ "labels": "Labels that matching rules added to the request.",+ "response_code_sent": "Response code that AWS WAF sent for a custom response.",+ "overridden_action": "Action that a rule action override replaced.",+ "population_size": "Number of requests that AWS WAF sampled from in the time window.",+ "region": COMMON_COLUMNS["region"],+ "scope": COMMON_COLUMNS["scope"],+ },+ },+ "request_metrics": {+ "description": "Hourly sums of AWS WAF CloudWatch metrics, such as allowed, blocked, and counted requests.",+ "docs_url": "https://docs.aws.amazon.com/waf/latest/developerguide/waf-metrics.html",+ "columns": {+ "metric_id": "Metric name and dimensions that identify one time series.",+ "metric_name": "CloudWatch metric name, such as AllowedRequests or BlockedRequests.",+ "dimensions": "CloudWatch dimensions of the metric, such as WebACL, Rule, and Region.",+ "web_acl": "Web ACL metric name from the WebACL dimension.",+ "rule": "Rule metric name from the Rule dimension.",+ "timestamp": "Start of the hour that the value covers.",+ "value": "Sum of the metric over the hour.",+ "region": COMMON_COLUMNS["region"],+ "scope": COMMON_COLUMNS["scope"],+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/settings.pyindex 04f172afb14c..9b516010d703 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/settings.py@@ -1,5 +1,8 @@+from datetime import timedelta+from posthog.dataclasses import frozen+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_fieldfrom products.warehouse_sources.backend.types import IncrementalFieldWAF_API_VERSION = "2019-07-29"@@ -7,6 +10,19 @@ TARGET_PREFIX = "AWSWAF_20190729"PAGE_SIZE = 100+@frozen+class AwsService:+ signing_name: str+ target_prefix: str+ content_type: str+++WAFV2_SERVICE = AwsService(signing_name="wafv2", target_prefix=TARGET_PREFIX, content_type="application/x-amz-json-1.1")+CLOUDWATCH_SERVICE = AwsService(+ signing_name="monitoring", target_prefix="GraniteServiceVersion20100801", content_type="application/x-amz-json-1.0"+)++@frozenclass WafEndpoint:list_operation: str@@ -47,9 +63,101 @@ AWS_WAF_ENDPOINTS: dict[str, WafEndpoint] = {),}-ENDPOINTS = tuple(AWS_WAF_ENDPOINTS)-ENDPOINT_DESCRIPTIONS = {name: endpoint.description for name, endpoint in AWS_WAF_ENDPOINTS.items()}-INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}+MANAGED_RULE_GROUPS = "managed_rule_groups"+WEB_ACL_RESOURCES = "protected_resources"+SAMPLED_REQUESTS = "sampled_requests"+REQUEST_METRICS = "request_metrics"++TABLE_DESCRIPTIONS = {+ MANAGED_RULE_GROUPS: "Managed rule groups available to the account, with their rules, labels, and capacity.",+ WEB_ACL_RESOURCES: "Regional resources that each web ACL protects, such as load balancers and API stages.",+ SAMPLED_REQUESTS: "Samples of recent web requests that matched web ACL rules, with the action AWS WAF took.",+ REQUEST_METRICS: "Hourly CloudWatch request counts for web ACLs, rules, and labels, such as allowed and blocked requests.",+}+TABLE_PERMISSIONS = {+ **{+ name: (f"wafv2:{endpoint.list_operation}", f"wafv2:{endpoint.get_operation}")+ for name, endpoint in AWS_WAF_ENDPOINTS.items()+ },+ MANAGED_RULE_GROUPS: ("wafv2:ListAvailableManagedRuleGroups", "wafv2:DescribeManagedRuleGroup"),+ WEB_ACL_RESOURCES: ("wafv2:ListWebACLs", "wafv2:ListResourcesForWebACL"),+ SAMPLED_REQUESTS: ("wafv2:ListWebACLs", "wafv2:GetWebACL", "wafv2:GetSampledRequests"),+ REQUEST_METRICS: ("cloudwatch:ListMetrics", "cloudwatch:GetMetricData"),+}+++@frozen+class ListCall:+ operation: str+ key: str+++LIST_CALLS = {+ **{+ name: ListCall(operation=endpoint.list_operation, key=endpoint.list_key)+ for name, endpoint in AWS_WAF_ENDPOINTS.items()+ },+ MANAGED_RULE_GROUPS: ListCall(operation="ListAvailableManagedRuleGroups", key="ManagedRuleGroups"),+}++PRIMARY_KEYS = {+ **dict.fromkeys(AWS_WAF_ENDPOINTS, ["arn"]),+ MANAGED_RULE_GROUPS: ["vendor_name", "name"],+ WEB_ACL_RESOURCES: ["web_acl_arn", "resource_arn"],+ SAMPLED_REQUESTS: ["id"],+ REQUEST_METRICS: ["metric_id", "timestamp"],+}++ENDPOINTS = (*AWS_WAF_ENDPOINTS, *TABLE_DESCRIPTIONS)+ENDPOINT_DESCRIPTIONS = {+ **{name: endpoint.description for name, endpoint in AWS_WAF_ENDPOINTS.items()},+ **TABLE_DESCRIPTIONS,+}+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {+ SAMPLED_REQUESTS: [incremental_field("timestamp")],+ REQUEST_METRICS: [incremental_field("timestamp")],+}+# GetSampledRequests only reaches 3 hours back, longer than the default sync interval, so users opt in and pick a schedule.+SHOULD_SYNC_DEFAULT = {SAMPLED_REQUESTS: False}+# Hourly datapoints can still change while CloudWatch receives late data for the current hour.+INCREMENTAL_LOOKBACK_SECONDS = {REQUEST_METRICS: 3 * 3600}++# ListResourcesForWebACL defaults to load balancers, so each regional resource type is listed explicitly.+PROTECTED_RESOURCE_TYPES = (+ "APPLICATION_LOAD_BALANCER",+ "API_GATEWAY",+ "APPSYNC",+ "COGNITO_USER_POOL",+ "APP_RUNNER_SERVICE",+ "VERIFIED_ACCESS_INSTANCE",+ "AMPLIFY",+ "AGENTCORE_GATEWAY",+)++SAMPLED_REQUESTS_MAX_ITEMS = 500+SAMPLED_REQUESTS_WINDOW = timedelta(hours=3)+# End-user credentials in sampled request headers are not copied into the warehouse.+REDACTED_HEADERS = frozenset(+ {+ "authorization",+ "proxy-authorization",+ "cookie",+ "set-cookie",+ "api-key",+ "x-api-key",+ "x-auth-token",+ "x-session-token",+ "x-csrf-token",+ "x-xsrf-token",+ "x-amz-security-token",+ }+)++METRICS_NAMESPACE = "AWS/WAFV2"+METRICS_PERIOD_SECONDS = 3600+# CloudWatch keeps hourly datapoints for 455 days.+METRICS_HISTORY = timedelta(days=455)+METRICS_PER_REQUEST = 500ACCESS_DENIED_CODES = frozenset({"AccessDenied", "AccessDeniedException", "WAFUnauthorizedOperationException"})ERROR_MESSAGES = {@@ -62,6 +170,7 @@ ERROR_MESSAGES = {"OptInRequired": "Enable AWS WAF for this account before you connect this source.","SubscriptionRequiredException": "Enable the required AWS subscription before you connect this source.",**dict.fromkeys(- ACCESS_DENIED_CODES, "These credentials lack AWS WAF read permissions. Grant the listed wafv2 permissions."+ ACCESS_DENIED_CODES,+ "These credentials lack the AWS read permissions for this table. Grant the permissions listed in the source setup.",),}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/source.pyindex 04a16952e862..3133990f9c97 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/source.py@@ -23,6 +23,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.aws_waf.seENDPOINTS,ERROR_MESSAGES,INCREMENTAL_FIELDS,+ INCREMENTAL_LOOKBACK_SECONDS,+ SHOULD_SYNC_DEFAULT,WAF_API_VERSION,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSource@@ -80,7 +82,14 @@ class AwsWafSource(ResumableSource[AwsWafSourceConfig, AwsWafResumeConfig]):force_refresh: bool = False,api_version: str | None = None,) -> list[SourceSchema]:- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, descriptions=ENDPOINT_DESCRIPTIONS)+ return build_endpoint_schemas(+ ENDPOINTS,+ INCREMENTAL_FIELDS,+ names,+ descriptions=ENDPOINT_DESCRIPTIONS,+ should_sync_default=SHOULD_SYNC_DEFAULT,+ default_incremental_lookback_seconds=INCREMENTAL_LOOKBACK_SECONDS,+ )def validate_credentials(self,@@ -101,7 +110,11 @@ class AwsWafSource(ResumableSource[AwsWafSourceConfig, AwsWafResumeConfig]):inputs: SourceInputs,) -> SourceResponse:return aws_waf_source(- config, inputs.schema_name, resumable_source_manager, self.resolve_api_version(inputs.api_version)+ config,+ inputs.schema_name,+ resumable_source_manager,+ self.resolve_api_version(inputs.api_version),+ inputs.db_incremental_field_last_value if inputs.should_use_incremental_field else None,)@property@@ -116,9 +129,14 @@ Grant `wafv2:ListWebACLs` and `wafv2:GetWebACL` to sync web ACLs.Grant `wafv2:ListRuleGroups` and `wafv2:GetRuleGroup` to sync rule groups.Grant `wafv2:ListIPSets` and `wafv2:GetIPSet` to sync IP sets.Grant `wafv2:ListRegexPatternSets` and `wafv2:GetRegexPatternSet` to sync regex pattern sets.+Grant `wafv2:ListAvailableManagedRuleGroups` and `wafv2:DescribeManagedRuleGroup` to sync managed rule groups.+Grant `wafv2:ListWebACLs` and `wafv2:ListResourcesForWebACL`, plus the read permissions for each protected resource type, to sync protected resources.+Grant `wafv2:ListWebACLs`, `wafv2:GetWebACL`, and `wafv2:GetSampledRequests` to sync sampled requests.+Grant `cloudwatch:ListMetrics` and `cloudwatch:GetMetricData` to sync request metrics.Select a region for regional resources. CloudFront resources use us-east-1.-All tables use full refresh. This source imports configuration data only, without request samples or traffic logs.""",+Sampled requests cover at most the last 3 hours, so sync them at least every 3 hours to keep a continuous history.+Protected resources are only listed for the regional scope.""",iconPath="/static/services/aws_waf.png",docsUrl="https://docs.aws.amazon.com/waf/latest/APIReference/Welcome.html",releaseStatus=ReleaseStatus.ALPHA,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/tests/test_aws_waf.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/tests/test_aws_waf.pyindex 2151a2ccf34b..2a1d3f3bc985 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/tests/test_aws_waf.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/tests/test_aws_waf.py@@ -1,5 +1,6 @@import jsonfrom collections.abc import Generator, Iterable, Iterator+from datetime import UTC, datetime, timedeltafrom typing import Any, castimport pytest@@ -251,16 +252,26 @@ def test_repeated_marker_fails_without_advancing_checkpoint() -> None:assert result.final_state == saved[email protected]("schema", [None, "ip_sets"])+ "schema,expected_reason",+ [+ (None, None),+ ("ip_sets", "Grant wafv2:ListIPSets and wafv2:GetIPSet for this table."),+ (+ "sampled_requests",+ "Grant wafv2:ListWebACLs, wafv2:GetWebACL and wafv2:GetSampledRequests for this table.",+ ),+ ("request_metrics", "Grant cloudwatch:ListMetrics and cloudwatch:GetMetricData for this table."),+ ],+)@pytest.mark.parametrize("code", ["AccessDenied", "AccessDeniedException", "WAFUnauthorizedOperationException"])-def test_missing_permissions_at_creation_or_schema_probe(schema: str | None, code: str) -> None:+def test_missing_permissions_at_creation_or_schema_probe(+ schema: str | None, expected_reason: str | None, code: str+) -> None:with scripted_network([ScriptedResponse(status=400, json={"__type": f"com.amazonaws.wafv2#{code}"})]) as network:valid, reason = validate_credentials(config(), schema)assert valid is (schema is None)- if schema:- assert reason == "Grant wafv2:ListIPSets and wafv2:GetIPSet for this table."- else:- assert reason is None+ assert reason == expected_reasonassert len(network.requests_log) == 1@@ -345,11 +356,13 @@ def test_retries_stop_after_five_attempts() -> None:assert len(network.requests_log) == 5-def test_tracked_transport_redacts_secrets_and_disables_nested_retries() -> None:[email protected]("schema,capture", [(None, True), ("sampled_requests", False)])+def test_tracked_transport_redacts_secrets_and_disables_nested_retries(schema: str | None, capture: bool) -> None:with scripted_network([ScriptedResponse(json={"WebACLs": []})]) as network:- assert validate_credentials(config()) == (True, None)+ assert validate_credentials(config(), schema) == (True, None)assert network.session_options[0]["retry"].total == 0assert set(network.session_options[0]["redact_values"]) == {"example-secret", "example-token"}+ assert network.session_options[0]["capture"] is capturedef test_service_errors_do_not_become_credential_failures(session: MagicMock) -> None:@@ -386,3 +399,244 @@ def test_unknown_schema_and_api_version_fail_before_requests() -> None:assert "Unknown AWS WAF table" in str(result.raised)assert network.requests_log == []assert result.requests == []+++NOW = datetime(2026, 10, 16, 12, 30, tzinfo=UTC)+++def targets(session: MagicMock) -> list[str]:+ return [call.kwargs["headers"]["X-Amz-Target"].split(".")[-1] for call in session.post.call_args_list]+++def payloads(session: MagicMock) -> list[dict[str, Any]]:+ return [json.loads(call.kwargs["data"]) for call in session.post.call_args_list]+++def rows_for(table: str, **kwargs: Any) -> list[dict[str, Any]]:+ with patch(f"{MODULE}.utc_now", return_value=NOW):+ stream = aws_waf_source(config(**kwargs.pop("overrides", {})), table, manager(), **kwargs)+ return [row for batch in cast(Iterable[list[dict[str, Any]]], stream.items()) for row in batch]+++def test_managed_rule_groups_merge_summary_and_description(session: MagicMock) -> None:+ summary = {"VendorName": "AWS", "Name": "AWSManagedRulesCommonRuleSet", "VersioningSupported": True}+ session.post.side_effect = [+ response({"ManagedRuleGroups": [summary]}),+ response({"VersionName": "Version_1.0", "Capacity": 700, "Rules": [{"Name": "NoUserAgent_HEADER"}]}),+ ]+ rows = rows_for("managed_rule_groups")+ assert targets(session) == ["ListAvailableManagedRuleGroups", "DescribeManagedRuleGroup"]+ assert payloads(session)[1] == {"Scope": "REGIONAL", "VendorName": "AWS", "Name": "AWSManagedRulesCommonRuleSet"}+ assert rows == [+ {+ "vendor_name": "AWS",+ "name": "AWSManagedRulesCommonRuleSet",+ "versioning_supported": True,+ "version_name": "Version_1.0",+ "capacity": 700,+ "rules": [{"Name": "NoUserAgent_HEADER"}],+ "region": "eu-west-1",+ "scope": "REGIONAL",+ }+ ]++[email protected]("scope", ["REGIONAL", "CLOUDFRONT"])+def test_protected_resources_cover_every_resource_type(session: MagicMock, scope: str) -> None:+ web_acl = {"ARN": "arn:aws:wafv2:eu-west-1:123456789012:regional/webacl/main/id", "Id": "id", "Name": "main"}+ alb_arn = "arn:aws:elasticloadbalancing:eu-west-1:123456789012:loadbalancer/app/example/1"+ responses = {+ "ListWebACLs": response({"WebACLs": [web_acl]}),+ "APPLICATION_LOAD_BALANCER": response({"ResourceArns": [alb_arn]}),+ "AGENTCORE_GATEWAY": response({"__type": "WAFInvalidParameterException"}, 400),+ }++ def post(*args: Any, **kwargs: Any) -> requests.Response:+ payload = json.loads(kwargs["data"])+ key = payload.get("ResourceType") or kwargs["headers"]["X-Amz-Target"].split(".")[-1]+ return responses.get(key, response({"ResourceArns": []}))++ session.post.side_effect = post+ rows = rows_for("protected_resources", overrides={"scope": scope})+ if scope == "CLOUDFRONT":+ assert rows == []+ session.post.assert_not_called()+ return+ assert [payload.get("ResourceType") for payload in payloads(session)[1:]] == [+ "APPLICATION_LOAD_BALANCER",+ "API_GATEWAY",+ "APPSYNC",+ "COGNITO_USER_POOL",+ "APP_RUNNER_SERVICE",+ "VERIFIED_ACCESS_INSTANCE",+ "AMPLIFY",+ "AGENTCORE_GATEWAY",+ ]+ assert rows == [+ {+ "web_acl_arn": web_acl["ARN"],+ "web_acl_id": "id",+ "web_acl_name": "main",+ "resource_type": "APPLICATION_LOAD_BALANCER",+ "resource_arn": alb_arn,+ "region": "eu-west-1",+ "scope": "REGIONAL",+ }+ ]+++ "last_value,expected_start",+ [+ (None, NOW - timedelta(hours=3)),+ (NOW - timedelta(hours=5), NOW - timedelta(hours=3)),+ ((NOW - timedelta(minutes=20)).isoformat(), NOW - timedelta(minutes=20)),+ ],+)+def test_sampled_requests_window_rules_and_header_redaction(+ session: MagicMock, last_value: Any, expected_start: datetime+) -> None:+ web_acl = {+ "ARN": "arn:aws:wafv2:eu-west-1:123456789012:regional/webacl/main/id",+ "Id": "id",+ "Name": "main",+ "Rules": [+ {"Name": "sampled", "VisibilityConfig": {"MetricName": "sampled-rule", "SampledRequestsEnabled": True}},+ {"Name": "unsampled", "VisibilityConfig": {"MetricName": "quiet-rule", "SampledRequestsEnabled": False}},+ ],+ }+ sample = {+ "Request": {+ "ClientIP": "192.0.2.10",+ "Country": "DE",+ "URI": "/login",+ "Method": "POST",+ "HTTPVersion": "HTTP/2.0",+ "Headers": [{"Name": "Cookie", "Value": "session=fake"}, {"Name": "User-Agent", "Value": "curl"}],+ },+ "Weight": 2,+ "Timestamp": (NOW - timedelta(minutes=5)).timestamp(),+ "Action": "BLOCK",+ }+ session.post.side_effect = [+ response({"WebACLs": [{"Id": "id", "Name": "main", "ARN": web_acl["ARN"]}]}),+ response({"WebACL": web_acl}),+ response(+ {+ "SampledRequests": [sample],+ "PopulationSize": 40,+ "TimeWindow": {+ "StartTime": expected_start.timestamp(),+ "EndTime": (NOW - timedelta(minutes=2)).timestamp(),+ },+ }+ ),+ response({"SampledRequests": [], "TimeWindow": {"EndTime": NOW.timestamp()}}),+ ]+ rows = rows_for("sampled_requests", incremental_last_value=last_value)+ assert targets(session) == ["ListWebACLs", "GetWebACL", "GetSampledRequests", "GetSampledRequests"]+ assert payloads(session)[3]["TimeWindow"] == {+ "StartTime": (NOW - timedelta(minutes=2)).timestamp(),+ "EndTime": NOW.timestamp(),+ }+ assert payloads(session)[2] == {+ "WebAclArn": web_acl["ARN"],+ "RuleMetricName": "sampled-rule",+ "Scope": "REGIONAL",+ "TimeWindow": {"StartTime": expected_start.timestamp(), "EndTime": NOW.timestamp()},+ "MaxItems": 500,+ }+ [row] = rows+ assert row["headers"] == [{"Name": "Cookie", "Value": "[REDACTED]"}, {"Name": "User-Agent", "Value": "curl"}]+ assert "session=fake" not in json.dumps(row, default=str)+ assert {key: row[key] for key in ("client_ip", "uri", "http_version", "action", "weight", "population_size")} == {+ "client_ip": "192.0.2.10",+ "uri": "/login",+ "http_version": "HTTP/2.0",+ "action": "BLOCK",+ "weight": 2,+ "population_size": 40,+ }+ assert row["timestamp"] == NOW - timedelta(minutes=5)+ assert row["rule_metric_name"] == "sampled-rule"+ assert len(row["id"]) == 64++[email protected]("scope,region_filter", [("REGIONAL", True), ("CLOUDFRONT", False)])+def test_request_metrics_from_cloudwatch(session: MagicMock, scope: str, region_filter: bool) -> None:+ regional = {+ "Namespace": "AWS/WAFV2",+ "MetricName": "BlockedRequests",+ "Dimensions": [+ {"Name": "WebACL", "Value": "main"},+ {"Name": "Rule", "Value": "ALL"},+ {"Name": "Region", "Value": "us-east-1"},+ ],+ }+ cloudfront = {+ "Namespace": "AWS/WAFV2",+ "MetricName": "AllowedRequests",+ "Dimensions": [{"Name": "WebACL", "Value": "edge"}, {"Name": "Rule", "Value": "ALL"}],+ }+ hour = NOW.replace(minute=0) - timedelta(hours=1)+ session.post.side_effect = [+ response({"Metrics": [regional], "NextToken": "metrics-2"}),+ response({"Metrics": [cloudfront]}),+ response(+ {+ "MetricDataResults": [{"Id": "m0", "Timestamps": [hour.timestamp()], "Values": [5.0]}],+ "NextToken": "data-2",+ }+ ),+ response(+ {"MetricDataResults": [{"Id": "m0", "Timestamps": [NOW.replace(minute=0).timestamp()], "Values": [7.0]}]}+ ),+ ]+ rows = rows_for(+ "request_metrics",+ overrides={"scope": scope, "aws_region": "us-east-1"},+ incremental_last_value=hour + timedelta(minutes=10),+ )+ calls = session.post.call_args_list+ assert {call.args[0] for call in calls} == {"https://monitoring.us-east-1.amazonaws.com/"}+ assert "/us-east-1/monitoring/aws4_request" in calls[0].kwargs["headers"]["Authorization"]+ assert calls[0].kwargs["headers"]["Content-Type"] == "application/x-amz-json-1.0"+ assert [call.kwargs["headers"]["X-Amz-Target"] for call in calls] == [+ "GraniteServiceVersion20100801.ListMetrics",+ "GraniteServiceVersion20100801.ListMetrics",+ "GraniteServiceVersion20100801.GetMetricData",+ "GraniteServiceVersion20100801.GetMetricData",+ ]+ sent = payloads(session)+ assert ("Dimensions" in sent[0]) is region_filter+ assert sent[1]["NextToken"] == "metrics-2"+ expected_metric = regional if region_filter else cloudfront+ assert sent[2]["MetricDataQueries"] == [+ {"Id": "m0", "MetricStat": {"Metric": expected_metric, "Period": 3600, "Stat": "Sum"}, "ReturnData": True}+ ]+ assert sent[2]["StartTime"] == hour.timestamp()+ assert sent[3]["NextToken"] == "data-2"+ assert [(row["metric_id"], row["timestamp"], row["value"]) for row in rows] == [+ (+ "BlockedRequests|Region=us-east-1|Rule=ALL|WebACL=main"+ if region_filter+ else "AllowedRequests|Rule=ALL|WebACL=edge",+ timestamp,+ value,+ )+ for timestamp, value in [(hour, 5.0), (NOW.replace(minute=0), 7.0)]+ ]+++def test_request_metrics_fail_on_a_failed_series(session: MagicMock) -> None:+ metric = {+ "Namespace": "AWS/WAFV2",+ "MetricName": "BlockedRequests",+ "Dimensions": [{"Name": "Region", "Value": "eu-west-1"}],+ }+ session.post.side_effect = [+ response({"Metrics": [metric]}),+ response({"MetricDataResults": [{"Id": "m0", "StatusCode": "InternalError", "Timestamps": [], "Values": []}]}),+ ]+ with pytest.raises(AwsWafRetryableError, match="InternalError"):+ rows_for("request_metrics")