Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add aws waf traffic and managed rule tables (#118539)

Tom Owers46a02590b182
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index de5ddfc4091e..bf575f92954e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -1353,10 +1353,10 @@ Today (4): `ip_sets`, `regex_pattern_sets`, `rule_groups`, `web_acls`
Diffed against: <https://raw.githubusercontent.com/boto/botocore/develop/botocore/data/wafv2/2019-07-29/service-2.json>
-- [ ] `Managed rule groups (ListAvailableManagedRuleGroups and DescribeManagedRuleGroup, POST /)` — resolve managed rule group references in web ACLs to vendor, rules, labels, and capacity (high)
-- [ ] `Protected resources (ListResourcesForWebACL, POST /)` — connect web ACLs to protected application resources and measure protection coverage (high)
-- [ ] `WAF request metrics (CloudWatch GetMetricData, POST /)` — measure allowed, blocked, and counted requests over time, which resource inventories cannot provide (high)
-- [ ] `Sampled requests (GetSampledRequests, POST /)` — analyze request actions, matching rules, client countries, and attack patterns from recent traffic samples (high)
+- [x] `Managed rule groups (ListAvailableManagedRuleGroups and DescribeManagedRuleGroup, POST /)` — resolve managed rule group references in web ACLs to vendor, rules, labels, and capacity (high)
+- [x] `Protected resources (ListResourcesForWebACL, POST /)` — connect web ACLs to protected application resources and measure protection coverage (high)
+- [x] `WAF request metrics (CloudWatch GetMetricData, POST /)` — measure allowed, blocked, and counted requests over time, which resource inventories cannot provide (high)
+- [x] `Sampled requests (GetSampledRequests, POST /)` — analyze request actions, matching rules, client countries, and attack patterns from recent traffic samples (high)
- [ ] `Top paths by bot traffic (GetTopPathStatisticsByTraffic, POST /)` — identify application paths and bot categories that account for the most bot traffic (medium)
- [ ] `Rate-based rule managed keys (GetRateBasedStatementManagedKeys, POST /)` — track IP addresses currently blocked by rate-based rules and compare rules across web ACLs (medium)
- [ ] `Settlement records (ListSettlementRecords, POST /)` — analyze individual monetization payments for protected CloudFront resources (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/aws_waf.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/aws_waf.py
index 1c44c1c25d98..d343f297ec9c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/aws_waf.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/aws_waf.py
@@ -1,6 +1,8 @@
import re
import json
-from collections.abc import Iterator
+import hashlib
+from collections.abc import Callable, Iterator
+from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any
import requests
@@ -16,12 +18,29 @@ from posthog.dataclasses import frozen
from products.warehouse_sources.backend.temporal.data_imports.sources.aws_waf.settings import (
ACCESS_DENIED_CODES,
AWS_WAF_ENDPOINTS,
+ CLOUDWATCH_SERVICE,
ERROR_MESSAGES,
+ LIST_CALLS,
+ MANAGED_RULE_GROUPS,
+ METRICS_HISTORY,
+ METRICS_NAMESPACE,
+ METRICS_PER_REQUEST,
+ METRICS_PERIOD_SECONDS,
PAGE_SIZE,
- TARGET_PREFIX,
+ PRIMARY_KEYS,
+ PROTECTED_RESOURCE_TYPES,
+ REDACTED_HEADERS,
+ REQUEST_METRICS,
+ SAMPLED_REQUESTS,
+ SAMPLED_REQUESTS_MAX_ITEMS,
+ SAMPLED_REQUESTS_WINDOW,
+ TABLE_PERMISSIONS,
WAF_API_VERSION,
- WafEndpoint,
+ WAFV2_SERVICE,
+ WEB_ACL_RESOURCES,
+ AwsService,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_value
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.awswaf import AwsWafSourceConfig
@@ -78,8 +97,29 @@ def retry_wait(state: RetryCallState) -> float:
return max(delay, error.retry_after) if isinstance(error, AwsWafRetryableError) else delay
+def utc_now() -> datetime:
+ return datetime.now(UTC)
+
+
+def snake_keys(data: dict[str, Any]) -> dict[str, Any]:
+ return {
+ re.sub(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])", "_", key).lower(): value
+ for key, value in data.items()
+ }
+
+
+def from_epoch(value: Any) -> datetime | None:
+ return datetime.fromtimestamp(value, UTC) if isinstance(value, int | float) else None
+
+
class AwsWafClient:
- def __init__(self, config: AwsWafSourceConfig, api_version: str = WAF_API_VERSION) -> None:
+ def __init__(
+ self,
+ config: AwsWafSourceConfig,
+ api_version: str = WAF_API_VERSION,
+ service: AwsService = WAFV2_SERVICE,
+ capture: bool = True,
+ ) -> None:
if api_version != WAF_API_VERSION:
raise ValueError(f"Unsupported AWS WAF API version: {api_version}")
if not config.aws_access_key_id or not config.aws_secret_access_key:
@@ -91,15 +131,17 @@ class AwsWafClient:
self.scope = config.scope
self.region = "us-east-1" if self.scope == "CLOUDFRONT" else config.aws_region
suffix = "amazonaws.com.cn" if self.region.startswith("cn-") else "amazonaws.com"
- self.url = f"https://wafv2.{self.region}.{suffix}/"
+ self.service = service
+ self.url = f"https://{service.signing_name}.{self.region}.{suffix}/"
self.signer = SigV4Auth(
Credentials(config.aws_access_key_id, config.aws_secret_access_key, config.aws_session_token or None),
- "wafv2",
+ service.signing_name,
self.region,
)
# WAF can throttle with HTTP 400, so one retry layer handles both HTTP statuses and AWS error codes.
self.session = make_tracked_session(
retry=Retry(total=0),
+ capture=capture,
redact_values=tuple(value for value in (config.aws_secret_access_key, config.aws_session_token) if value),
)
@@ -118,7 +160,10 @@ class AwsWafClient:
method="POST",
url=self.url,
data=body,
- headers={"Content-Type": "application/x-amz-json-1.1", "X-Amz-Target": f"{TARGET_PREFIX}.{operation}"},
+ headers={
+ "Content-Type": self.service.content_type,
+ "X-Amz-Target": f"{self.service.target_prefix}.{operation}",
+ },
)
self.signer.add_auth(request)
response = self.session.post(
@@ -128,39 +173,86 @@ class AwsWafClient:
raise error_for_response(response)
return response.json()
- def get_resource(self, endpoint: WafEndpoint, summary: dict[str, Any]) -> dict[str, Any]:
- result = self.request(
- endpoint.get_operation, {"Scope": self.scope, "Id": summary["Id"], "Name": summary["Name"]}
- )
- resource = result[endpoint.object_key]
- row = {
- re.sub(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])", "_", key).lower(): value
- for key, value in resource.items()
- }
+ def with_location(self, row: dict[str, Any]) -> dict[str, Any]:
row["region"] = self.region
row["scope"] = self.scope
return row
+ def get_resource(self, endpoint: str, summary: dict[str, Any]) -> dict[str, Any]:
+ if endpoint == MANAGED_RULE_GROUPS:
+ result = self.request(
+ "DescribeManagedRuleGroup",
+ {"Scope": self.scope, "VendorName": summary["VendorName"], "Name": summary["Name"]},
+ )
+ return self.with_location({**snake_keys(summary), **snake_keys(result)})
+ waf_endpoint = AWS_WAF_ENDPOINTS[endpoint]
+ result = self.request(
+ waf_endpoint.get_operation, {"Scope": self.scope, "Id": summary["Id"], "Name": summary["Name"]}
+ )
+ return self.with_location(snake_keys(result[waf_endpoint.object_key]))
+
+ def get_web_acl(self, summary: dict[str, Any]) -> dict[str, Any] | None:
+ try:
+ return self.request("GetWebACL", {"Scope": self.scope, "Id": summary["Id"], "Name": summary["Name"]})[
+ "WebACL"
+ ]
+ except AwsWafError as error:
+ if error.code != "WAFNonexistentItemException":
+ raise
+ return None
+
+ def list_resources(self, web_acl_arn: str, resource_type: str) -> list[str]:
+ try:
+ result = self.request("ListResourcesForWebACL", {"WebACLArn": web_acl_arn, "ResourceType": resource_type})
+ except AwsWafError as error:
+ # Some resource types are not offered in every region, and AWS rejects them as invalid parameters.
+ if error.code not in {"WAFNonexistentItemException", "WAFInvalidParameterException"}:
+ raise
+ return []
+ return result.get("ResourceArns") or []
+
+ def get_sampled_requests(
+ self, web_acl_arn: str, metric_name: str, start: datetime, end: datetime, max_items: int
+ ) -> dict[str, Any]:
+ try:
+ return self.request(
+ "GetSampledRequests",
+ {
+ "WebAclArn": web_acl_arn,
+ "RuleMetricName": metric_name,
+ "Scope": self.scope,
+ "TimeWindow": {"StartTime": start.timestamp(), "EndTime": end.timestamp()},
+ "MaxItems": max_items,
+ },
+ )
+ except AwsWafError as error:
+ if error.code != "WAFNonexistentItemException":
+ raise
+ return {}
+
def validate_credentials(
config: AwsWafSourceConfig, schema_name: str | None = None, api_version: str = WAF_API_VERSION
) -> tuple[bool, str | None]:
- endpoint = AWS_WAF_ENDPOINTS.get(schema_name or "web_acls")
- if endpoint is None:
+ table = schema_name or "web_acls"
+ if table not in TABLE_PERMISSIONS:
return False, f"Unknown AWS WAF table: {schema_name}"
try:
- client = AwsWafClient(config, api_version)
+ # Sampled requests carry end-user headers, so they stay out of HTTP sample capture.
+ client = AwsWafClient(config, api_version, capture=table != SAMPLED_REQUESTS)
except ValueError as error:
return False, str(error)
try:
- result = client.request(endpoint.list_operation, {"Scope": client.scope, "Limit": 1})
- if schema_name and result[endpoint.list_key]:
- client.get_resource(endpoint, result[endpoint.list_key][0])
+ if schema_name is None:
+ client.request("ListWebACLs", {"Scope": client.scope, "Limit": 1})
+ else:
+ probe_table(client, config, table, api_version)
except AwsWafError as error:
if error.code in ACCESS_DENIED_CODES:
if schema_name is None:
return True, None
- return False, f"Grant wafv2:{endpoint.list_operation} and wafv2:{endpoint.get_operation} for this table."
+ *others, last = TABLE_PERMISSIONS[table]
+ return False, f"Grant {', '.join(others)} and {last} for this table."
if error.code in ERROR_MESSAGES:
return False, ERROR_MESSAGES[error.code]
raise
@@ -169,23 +261,90 @@ def validate_credentials(
return True, None
+def sampled_metric_names(web_acl: dict[str, Any]) -> list[str]:
+ rules = [
+ *(web_acl.get("PreProcessFirewallManagerRuleGroups") or []),
+ *(web_acl.get("Rules") or []),
+ *(web_acl.get("PostProcessFirewallManagerRuleGroups") or []),
+ ]
+ names = [
+ rule["VisibilityConfig"]["MetricName"]
+ for rule in rules
+ if rule.get("VisibilityConfig", {}).get("SampledRequestsEnabled")
+ ]
+ return list(dict.fromkeys(names))
+
+
+def probe_table(client: AwsWafClient, config: AwsWafSourceConfig, table: str, api_version: str) -> None:
+ if table == REQUEST_METRICS:
+ metrics_client = AwsWafClient(config, api_version, CLOUDWATCH_SERVICE)
+ try:
+ metrics_client.request("ListMetrics", {"Namespace": METRICS_NAMESPACE})
+ finally:
+ metrics_client.close()
+ return
+ list_call = LIST_CALLS.get(table, LIST_CALLS["web_acls"])
+ summaries = client.request(list_call.operation, {"Scope": client.scope, "Limit": 1})[list_call.key]
+ if not summaries:
+ return
+ if table == WEB_ACL_RESOURCES:
+ if client.scope == "REGIONAL":
+ client.list_resources(summaries[0]["ARN"], PROTECTED_RESOURCE_TYPES[0])
+ elif table == SAMPLED_REQUESTS:
+ web_acl = client.get_web_acl(summaries[0])
+ if web_acl and (names := sampled_metric_names(web_acl)):
+ end = utc_now()
+ client.get_sampled_requests(web_acl["ARN"], names[0], end - SAMPLED_REQUESTS_WINDOW, end, 1)
+ else:
+ client.get_resource(table, summaries[0])
+
+
+def iter_web_acls(
+ client: AwsWafClient,
+ manager: "ResumableSourceManager[AwsWafResumeConfig]",
+ rows_for_web_acl: Callable[[dict[str, Any]], Iterator[list[dict[str, Any]]]],
+) -> Iterator[list[dict[str, Any]]]:
+ """Yield the rows of every web ACL.
+
+ The marker of the current page stays staged until the next page is read, so a resumed run reads that page again.
+ """
+ state = manager.load_state() or AwsWafResumeConfig()
+ marker = state.next_marker
+ while True:
+ payload: dict[str, Any] = {"Scope": client.scope, "Limit": PAGE_SIZE}
+ if marker:
+ payload["NextMarker"] = marker
+ result = client.request("ListWebACLs", payload)
+ next_marker = result.get("NextMarker") or None
+ if next_marker and next_marker == marker:
+ raise ValueError("AWS WAF returned a repeated pagination marker.")
+ manager.save_state(AwsWafResumeConfig(next_marker=marker))
+ for web_acl in result["WebACLs"]:
+ yield from rows_for_web_acl(web_acl)
+ manager.safe_point()
+ if next_marker is None:
+ return
+ marker = next_marker
+
+
def get_rows(
config: AwsWafSourceConfig,
- endpoint: WafEndpoint,
+ endpoint: str,
manager: "ResumableSourceManager[AwsWafResumeConfig]",
api_version: str,
) -> Iterator[list[dict[str, Any]]]:
state = manager.load_state() or AwsWafResumeConfig()
client = AwsWafClient(config, api_version)
marker = state.next_marker
+ list_call = LIST_CALLS[endpoint]
try:
while True:
payload: dict[str, Any] = {"Scope": client.scope, "Limit": PAGE_SIZE}
if marker:
payload["NextMarker"] = marker
- result = client.request(endpoint.list_operation, payload)
+ result = client.request(list_call.operation, payload)
rows = []
- for summary in result[endpoint.list_key]:
+ for summary in result[list_call.key]:
try:
rows.append(client.get_resource(endpoint, summary))
except AwsWafError as error:
@@ -205,18 +364,234 @@ def get_rows(
client.close()
+def protected_resource_rows(
+ client: AwsWafClient, manager: "ResumableSourceManager[AwsWafResumeConfig]", web_acl: dict[str, Any]
+) -> Iterator[list[dict[str, Any]]]:
+ for resource_type in PROTECTED_RESOURCE_TYPES:
+ rows = [
+ client.with_location(
+ {
+ "web_acl_arn": web_acl["ARN"],
+ "web_acl_id": web_acl["Id"],
+ "web_acl_name": web_acl["Name"],
+ "resource_type": resource_type,
+ "resource_arn": resource_arn,
+ }
+ )
+ for resource_arn in client.list_resources(web_acl["ARN"], resource_type)
+ ]
+ if rows:
+ yield rows
+ else:
+ manager.safe_point()
+
+
+def get_web_acl_resource_rows(
+ config: AwsWafSourceConfig, manager: "ResumableSourceManager[AwsWafResumeConfig]", api_version: str
+) -> Iterator[list[dict[str, Any]]]:
+ client = AwsWafClient(config, api_version)
+ try:
+ # CloudFront distributions are listed by the CloudFront API, not by ListResourcesForWebACL.
+ if client.scope != "REGIONAL":
+ return
+ yield from iter_web_acls(client, manager, lambda web_acl: protected_resource_rows(client, manager, web_acl))
+ finally:
+ client.close()
+
+
+def sampled_request_row(
+ client: AwsWafClient, web_acl: dict[str, Any], metric_name: str, sample: dict[str, Any], population_size: Any
+) -> dict[str, Any]:
+ request = dict(sample.get("Request") or {})
+ request["Headers"] = [
+ {**header, "Value": "[REDACTED]"} if str(header.get("Name", "")).lower() in REDACTED_HEADERS else header
+ for header in request.get("Headers") or []
+ ]
+ sample = {**sample, "Request": request}
+ digest = hashlib.sha256(
+ json.dumps([web_acl["ARN"], metric_name, sample], sort_keys=True, default=str).encode()
+ ).hexdigest()
+ row = {
+ "id": digest,
+ "web_acl_arn": web_acl["ARN"],
+ "web_acl_name": web_acl["Name"],
+ "rule_metric_name": metric_name,
+ **snake_keys(request),
+ **snake_keys({key: value for key, value in sample.items() if key != "Request"}),
+ "population_size": population_size,
+ }
+ row["timestamp"] = from_epoch(sample.get("Timestamp"))
+ return client.with_location(row)
+
+
+def sampled_rows_for_web_acl(
+ client: AwsWafClient,
+ manager: "ResumableSourceManager[AwsWafResumeConfig]",
+ summary: dict[str, Any],
+ start: datetime,
+ end: datetime,
+) -> Iterator[list[dict[str, Any]]]:
+ web_acl = client.get_web_acl(summary)
+ manager.safe_point()
+ if web_acl is None:
+ return
+ for metric_name in sampled_metric_names(web_acl):
+ window_start = start
+ while window_start < end:
+ result = client.get_sampled_requests(
+ web_acl["ARN"], metric_name, window_start, end, SAMPLED_REQUESTS_MAX_ITEMS
+ )
+ rows = [
+ sampled_request_row(client, web_acl, metric_name, sample, result.get("PopulationSize"))
+ for sample in result.get("SampledRequests") or []
+ ]
+ if rows:
+ yield rows
+ else:
+ manager.safe_point()
+ # AWS ends the returned window early when the rule saw 5,000 requests before the requested end.
+ returned_end = from_epoch((result.get("TimeWindow") or {}).get("EndTime"))
+ if returned_end is None or returned_end <= window_start or returned_end >= end:
+ break
+ window_start = returned_end
+
+
+def get_sampled_request_rows(
+ config: AwsWafSourceConfig,
+ manager: "ResumableSourceManager[AwsWafResumeConfig]",
+ api_version: str,
+ last_value: Any,
+) -> Iterator[list[dict[str, Any]]]:
+ end = utc_now()
+ start = end - SAMPLED_REQUESTS_WINDOW
+ watermark = parse_datetime_value(last_value)
+ if watermark and watermark > start:
+ start = watermark
+ if start >= end:
+ return
+ client = AwsWafClient(config, api_version, capture=False)
+ try:
+ yield from iter_web_acls(
+ client, manager, lambda summary: sampled_rows_for_web_acl(client, manager, summary, start, end)
+ )
+ finally:
+ client.close()
+
+
+def metric_id(metric: dict[str, Any]) -> str:
+ dimensions = sorted(f"{dimension['Name']}={dimension['Value']}" for dimension in metric.get("Dimensions") or [])
+ return "|".join([metric["MetricName"], *dimensions])
+
+
+def get_request_metric_rows(
+ config: AwsWafSourceConfig,
+ manager: "ResumableSourceManager[AwsWafResumeConfig]",
+ api_version: str,
+ last_value: Any,
+) -> Iterator[list[dict[str, Any]]]:
+ end = utc_now()
+ start = max(parse_datetime_value(last_value) or end - METRICS_HISTORY, end - METRICS_HISTORY)
+ start = start.replace(minute=0, second=0, microsecond=0)
+ client = AwsWafClient(config, api_version, CLOUDWATCH_SERVICE)
+ try:
+ metrics: list[dict[str, Any]] = []
+ list_payload: dict[str, Any] = {"Namespace": METRICS_NAMESPACE}
+ if client.scope == "REGIONAL":
+ list_payload["Dimensions"] = [{"Name": "Region", "Value": client.region}]
+ while True:
+ result = client.request("ListMetrics", list_payload)
+ for metric in result.get("Metrics") or []:
+ dimension_names = {dimension["Name"] for dimension in metric.get("Dimensions") or []}
+ # CloudFront metrics have no Region dimension, regional metrics always have one.
+ if (client.scope == "REGIONAL") == ("Region" in dimension_names):
+ metrics.append(metric)
+ manager.safe_point()
+ next_token = result.get("NextToken")
+ if not next_token:
+ break
+ list_payload["NextToken"] = next_token
+
+ for offset in range(0, len(metrics), METRICS_PER_REQUEST):
+ batch = {f"m{index}": metric for index, metric in enumerate(metrics[offset : offset + METRICS_PER_REQUEST])}
+ payload: dict[str, Any] = {
+ "MetricDataQueries": [
+ {
+ "Id": query_id,
+ "MetricStat": {"Metric": metric, "Period": METRICS_PERIOD_SECONDS, "Stat": "Sum"},
+ "ReturnData": True,
+ }
+ for query_id, metric in batch.items()
+ ],
+ "StartTime": start.timestamp(),
+ "EndTime": end.timestamp(),
+ "ScanBy": "TimestampAscending",
+ }
+ while True:
+ result = client.request("GetMetricData", payload)
+ rows = []
+ for data in result.get("MetricDataResults") or []:
+ metric = batch[data["Id"]]
+ # A failed series still returns HTTP 200, and skipping it would let the watermark pass its gap.
+ if data.get("StatusCode") not in (None, "Complete", "PartialData"):
+ raise AwsWafRetryableError(
+ str(data["StatusCode"]), f"CloudWatch could not return {metric_id(metric)}."
+ )
+ dimensions = {dimension["Name"]: dimension["Value"] for dimension in metric.get("Dimensions") or []}
+ for timestamp, value in zip(data.get("Timestamps") or [], data.get("Values") or []):
+ rows.append(
+ client.with_location(
+ {
+ "metric_id": metric_id(metric),
+ "metric_name": metric["MetricName"],
+ "dimensions": dimensions,
+ "web_acl": dimensions.get("WebACL"),
+ "rule": dimensions.get("Rule"),
+ "timestamp": from_epoch(timestamp),
+ "value": value,
+ }
+ )
+ )
+ if rows:
+ yield rows
+ else:
+ manager.safe_point()
+ next_token = result.get("NextToken")
+ if not next_token:
+ break
+ payload["NextToken"] = next_token
+ finally:
+ client.close()
+
+
+def iter_rows(
+ config: AwsWafSourceConfig,
+ endpoint: str,
+ manager: "ResumableSourceManager[AwsWafResumeConfig]",
+ api_version: str,
+ incremental_last_value: Any,
+) -> Iterator[list[dict[str, Any]]]:
+ if endpoint == WEB_ACL_RESOURCES:
+ return get_web_acl_resource_rows(config, manager, api_version)
+ if endpoint == SAMPLED_REQUESTS:
+ return get_sampled_request_rows(config, manager, api_version, incremental_last_value)
+ if endpoint == REQUEST_METRICS:
+ return get_request_metric_rows(config, manager, api_version, incremental_last_value)
+ return get_rows(config, endpoint, manager, api_version)
+
+
def aws_waf_source(
config: AwsWafSourceConfig,
endpoint: str,
manager: "ResumableSourceManager[AwsWafResumeConfig]",
api_version: str = WAF_API_VERSION,
+ incremental_last_value: Any = None,
) -> SourceResponse:
- if endpoint not in AWS_WAF_ENDPOINTS:
+ if endpoint not in PRIMARY_KEYS:
raise ValueError(f"Unknown AWS WAF table: {endpoint}")
return SourceResponse(
name=endpoint,
- items=lambda: get_rows(config, AWS_WAF_ENDPOINTS[endpoint], manager, api_version),
- primary_keys=["arn"],
+ items=lambda: iter_rows(config, endpoint, manager, api_version, incremental_last_value),
+ primary_keys=PRIMARY_KEYS[endpoint],
sort_mode=None,
on_complete=manager.clear_state,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/canonical_descriptions.py
index 83df2bfe9a72..140fe9e76393 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/canonical_descriptions.py
@@ -53,4 +53,77 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"regular_expression_list": "Regular expressions used to match web request components.",
},
},
+ "managed_rule_groups": {
+ "description": "Managed rule groups from AWS and subscribed AWS Marketplace sellers that web ACLs can reference.",
+ "docs_url": "https://docs.aws.amazon.com/waf/latest/APIReference/API_DescribeManagedRuleGroup.html",
+ "columns": {
+ "vendor_name": "Name of the managed rule group vendor, such as AWS.",
+ "name": "Name of the managed rule group.",
+ "description": "Description of the managed rule group.",
+ "versioning_supported": "Whether the managed rule group supports versions.",
+ "version_name": "Version of the rule group described in this row.",
+ "sns_topic_arn": "Amazon SNS topic that receives notifications about rule group updates.",
+ "capacity": "Web ACL capacity units that the rule group uses.",
+ "rules": "Rules in the rule group, with their names and actions.",
+ "label_namespace": "Label namespace prefix for labels that the rules add.",
+ "available_labels": "Labels that rules in this group can add to matching requests.",
+ "consumed_labels": "Labels that rules in this group can match.",
+ "region": COMMON_COLUMNS["region"],
+ "scope": COMMON_COLUMNS["scope"],
+ },
+ },
+ "protected_resources": {
+ "description": "Regional AWS resources associated with each web ACL.",
+ "docs_url": "https://docs.aws.amazon.com/waf/latest/APIReference/API_ListResourcesForWebACL.html",
+ "columns": {
+ "web_acl_arn": "Amazon Resource Name of the web ACL.",
+ "web_acl_id": "Identifier of the web ACL.",
+ "web_acl_name": "Name of the web ACL.",
+ "resource_type": "Type of the protected resource, such as APPLICATION_LOAD_BALANCER or API_GATEWAY.",
+ "resource_arn": "Amazon Resource Name of the protected resource.",
+ "region": COMMON_COLUMNS["region"],
+ "scope": COMMON_COLUMNS["scope"],
+ },
+ },
+ "sampled_requests": {
+ "description": "Samples of web requests that matched a web ACL rule, selected by AWS WAF from recent traffic.",
+ "docs_url": "https://docs.aws.amazon.com/waf/latest/APIReference/API_GetSampledRequests.html",
+ "columns": {
+ "id": "Hash of the web ACL, rule metric name, and sample, used to identify the row.",
+ "web_acl_arn": "Amazon Resource Name of the web ACL.",
+ "web_acl_name": "Name of the web ACL.",
+ "rule_metric_name": "Metric name of the rule or rule group that the sample was taken for.",
+ "client_ip": "IP address of the client that sent the request.",
+ "country": "Two-letter country code of the client.",
+ "uri": "URI path of the request.",
+ "method": "HTTP method of the request.",
+ "http_version": "HTTP version of the request.",
+ "headers": "Request headers. Credential header values are redacted.",
+ "timestamp": "Time at which AWS WAF received the request.",
+ "action": "Action that AWS WAF applied to the request.",
+ "weight": "Number of requests that this sample represents.",
+ "rule_name_within_rule_group": "Name of the matching rule inside a rule group.",
+ "labels": "Labels that matching rules added to the request.",
+ "response_code_sent": "Response code that AWS WAF sent for a custom response.",
+ "overridden_action": "Action that a rule action override replaced.",
+ "population_size": "Number of requests that AWS WAF sampled from in the time window.",
+ "region": COMMON_COLUMNS["region"],
+ "scope": COMMON_COLUMNS["scope"],
+ },
+ },
+ "request_metrics": {
+ "description": "Hourly sums of AWS WAF CloudWatch metrics, such as allowed, blocked, and counted requests.",
+ "docs_url": "https://docs.aws.amazon.com/waf/latest/developerguide/waf-metrics.html",
+ "columns": {
+ "metric_id": "Metric name and dimensions that identify one time series.",
+ "metric_name": "CloudWatch metric name, such as AllowedRequests or BlockedRequests.",
+ "dimensions": "CloudWatch dimensions of the metric, such as WebACL, Rule, and Region.",
+ "web_acl": "Web ACL metric name from the WebACL dimension.",
+ "rule": "Rule metric name from the Rule dimension.",
+ "timestamp": "Start of the hour that the value covers.",
+ "value": "Sum of the metric over the hour.",
+ "region": COMMON_COLUMNS["region"],
+ "scope": COMMON_COLUMNS["scope"],
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/settings.py
index 04f172afb14c..9b516010d703 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/settings.py
@@ -1,5 +1,8 @@
+from datetime import timedelta
+
from posthog.dataclasses import frozen
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_field
from products.warehouse_sources.backend.types import IncrementalField
WAF_API_VERSION = "2019-07-29"
@@ -7,6 +10,19 @@ TARGET_PREFIX = "AWSWAF_20190729"
PAGE_SIZE = 100
+@frozen
+class AwsService:
+ signing_name: str
+ target_prefix: str
+ content_type: str
+
+
+WAFV2_SERVICE = AwsService(signing_name="wafv2", target_prefix=TARGET_PREFIX, content_type="application/x-amz-json-1.1")
+CLOUDWATCH_SERVICE = AwsService(
+ signing_name="monitoring", target_prefix="GraniteServiceVersion20100801", content_type="application/x-amz-json-1.0"
+)
+
+
@frozen
class WafEndpoint:
list_operation: str
@@ -47,9 +63,101 @@ AWS_WAF_ENDPOINTS: dict[str, WafEndpoint] = {
),
}
-ENDPOINTS = tuple(AWS_WAF_ENDPOINTS)
-ENDPOINT_DESCRIPTIONS = {name: endpoint.description for name, endpoint in AWS_WAF_ENDPOINTS.items()}
-INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}
+MANAGED_RULE_GROUPS = "managed_rule_groups"
+WEB_ACL_RESOURCES = "protected_resources"
+SAMPLED_REQUESTS = "sampled_requests"
+REQUEST_METRICS = "request_metrics"
+
+TABLE_DESCRIPTIONS = {
+ MANAGED_RULE_GROUPS: "Managed rule groups available to the account, with their rules, labels, and capacity.",
+ WEB_ACL_RESOURCES: "Regional resources that each web ACL protects, such as load balancers and API stages.",
+ SAMPLED_REQUESTS: "Samples of recent web requests that matched web ACL rules, with the action AWS WAF took.",
+ REQUEST_METRICS: "Hourly CloudWatch request counts for web ACLs, rules, and labels, such as allowed and blocked requests.",
+}
+TABLE_PERMISSIONS = {
+ **{
+ name: (f"wafv2:{endpoint.list_operation}", f"wafv2:{endpoint.get_operation}")
+ for name, endpoint in AWS_WAF_ENDPOINTS.items()
+ },
+ MANAGED_RULE_GROUPS: ("wafv2:ListAvailableManagedRuleGroups", "wafv2:DescribeManagedRuleGroup"),
+ WEB_ACL_RESOURCES: ("wafv2:ListWebACLs", "wafv2:ListResourcesForWebACL"),
+ SAMPLED_REQUESTS: ("wafv2:ListWebACLs", "wafv2:GetWebACL", "wafv2:GetSampledRequests"),
+ REQUEST_METRICS: ("cloudwatch:ListMetrics", "cloudwatch:GetMetricData"),
+}
+
+
+@frozen
+class ListCall:
+ operation: str
+ key: str
+
+
+LIST_CALLS = {
+ **{
+ name: ListCall(operation=endpoint.list_operation, key=endpoint.list_key)
+ for name, endpoint in AWS_WAF_ENDPOINTS.items()
+ },
+ MANAGED_RULE_GROUPS: ListCall(operation="ListAvailableManagedRuleGroups", key="ManagedRuleGroups"),
+}
+
+PRIMARY_KEYS = {
+ **dict.fromkeys(AWS_WAF_ENDPOINTS, ["arn"]),
+ MANAGED_RULE_GROUPS: ["vendor_name", "name"],
+ WEB_ACL_RESOURCES: ["web_acl_arn", "resource_arn"],
+ SAMPLED_REQUESTS: ["id"],
+ REQUEST_METRICS: ["metric_id", "timestamp"],
+}
+
+ENDPOINTS = (*AWS_WAF_ENDPOINTS, *TABLE_DESCRIPTIONS)
+ENDPOINT_DESCRIPTIONS = {
+ **{name: endpoint.description for name, endpoint in AWS_WAF_ENDPOINTS.items()},
+ **TABLE_DESCRIPTIONS,
+}
+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
+ SAMPLED_REQUESTS: [incremental_field("timestamp")],
+ REQUEST_METRICS: [incremental_field("timestamp")],
+}
+# GetSampledRequests only reaches 3 hours back, longer than the default sync interval, so users opt in and pick a schedule.
+SHOULD_SYNC_DEFAULT = {SAMPLED_REQUESTS: False}
+# Hourly datapoints can still change while CloudWatch receives late data for the current hour.
+INCREMENTAL_LOOKBACK_SECONDS = {REQUEST_METRICS: 3 * 3600}
+
+# ListResourcesForWebACL defaults to load balancers, so each regional resource type is listed explicitly.
+PROTECTED_RESOURCE_TYPES = (
+ "APPLICATION_LOAD_BALANCER",
+ "API_GATEWAY",
+ "APPSYNC",
+ "COGNITO_USER_POOL",
+ "APP_RUNNER_SERVICE",
+ "VERIFIED_ACCESS_INSTANCE",
+ "AMPLIFY",
+ "AGENTCORE_GATEWAY",
+)
+
+SAMPLED_REQUESTS_MAX_ITEMS = 500
+SAMPLED_REQUESTS_WINDOW = timedelta(hours=3)
+# End-user credentials in sampled request headers are not copied into the warehouse.
+REDACTED_HEADERS = frozenset(
+ {
+ "authorization",
+ "proxy-authorization",
+ "cookie",
+ "set-cookie",
+ "api-key",
+ "x-api-key",
+ "x-auth-token",
+ "x-session-token",
+ "x-csrf-token",
+ "x-xsrf-token",
+ "x-amz-security-token",
+ }
+)
+
+METRICS_NAMESPACE = "AWS/WAFV2"
+METRICS_PERIOD_SECONDS = 3600
+# CloudWatch keeps hourly datapoints for 455 days.
+METRICS_HISTORY = timedelta(days=455)
+METRICS_PER_REQUEST = 500
ACCESS_DENIED_CODES = frozenset({"AccessDenied", "AccessDeniedException", "WAFUnauthorizedOperationException"})
ERROR_MESSAGES = {
@@ -62,6 +170,7 @@ ERROR_MESSAGES = {
"OptInRequired": "Enable AWS WAF for this account before you connect this source.",
"SubscriptionRequiredException": "Enable the required AWS subscription before you connect this source.",
**dict.fromkeys(
- ACCESS_DENIED_CODES, "These credentials lack AWS WAF read permissions. Grant the listed wafv2 permissions."
+ ACCESS_DENIED_CODES,
+ "These credentials lack the AWS read permissions for this table. Grant the permissions listed in the source setup.",
),
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/source.py
index 04a16952e862..3133990f9c97 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/source.py
@@ -23,6 +23,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.aws_waf.se
ENDPOINTS,
ERROR_MESSAGES,
INCREMENTAL_FIELDS,
+ INCREMENTAL_LOOKBACK_SECONDS,
+ SHOULD_SYNC_DEFAULT,
WAF_API_VERSION,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSource
@@ -80,7 +82,14 @@ class AwsWafSource(ResumableSource[AwsWafSourceConfig, AwsWafResumeConfig]):
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, descriptions=ENDPOINT_DESCRIPTIONS)
+ return build_endpoint_schemas(
+ ENDPOINTS,
+ INCREMENTAL_FIELDS,
+ names,
+ descriptions=ENDPOINT_DESCRIPTIONS,
+ should_sync_default=SHOULD_SYNC_DEFAULT,
+ default_incremental_lookback_seconds=INCREMENTAL_LOOKBACK_SECONDS,
+ )
def validate_credentials(
self,
@@ -101,7 +110,11 @@ class AwsWafSource(ResumableSource[AwsWafSourceConfig, AwsWafResumeConfig]):
inputs: SourceInputs,
) -> SourceResponse:
return aws_waf_source(
- config, inputs.schema_name, resumable_source_manager, self.resolve_api_version(inputs.api_version)
+ config,
+ inputs.schema_name,
+ resumable_source_manager,
+ self.resolve_api_version(inputs.api_version),
+ inputs.db_incremental_field_last_value if inputs.should_use_incremental_field else None,
)
@property
@@ -116,9 +129,14 @@ Grant `wafv2:ListWebACLs` and `wafv2:GetWebACL` to sync web ACLs.
Grant `wafv2:ListRuleGroups` and `wafv2:GetRuleGroup` to sync rule groups.
Grant `wafv2:ListIPSets` and `wafv2:GetIPSet` to sync IP sets.
Grant `wafv2:ListRegexPatternSets` and `wafv2:GetRegexPatternSet` to sync regex pattern sets.
+Grant `wafv2:ListAvailableManagedRuleGroups` and `wafv2:DescribeManagedRuleGroup` to sync managed rule groups.
+Grant `wafv2:ListWebACLs` and `wafv2:ListResourcesForWebACL`, plus the read permissions for each protected resource type, to sync protected resources.
+Grant `wafv2:ListWebACLs`, `wafv2:GetWebACL`, and `wafv2:GetSampledRequests` to sync sampled requests.
+Grant `cloudwatch:ListMetrics` and `cloudwatch:GetMetricData` to sync request metrics.
Select a region for regional resources. CloudFront resources use us-east-1.
-All tables use full refresh. This source imports configuration data only, without request samples or traffic logs.""",
+Sampled requests cover at most the last 3 hours, so sync them at least every 3 hours to keep a continuous history.
+Protected resources are only listed for the regional scope.""",
iconPath="/static/services/aws_waf.png",
docsUrl="https://docs.aws.amazon.com/waf/latest/APIReference/Welcome.html",
releaseStatus=ReleaseStatus.ALPHA,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/tests/test_aws_waf.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/tests/test_aws_waf.py
index 2151a2ccf34b..2a1d3f3bc985 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/tests/test_aws_waf.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/tests/test_aws_waf.py
@@ -1,5 +1,6 @@
import json
from collections.abc import Generator, Iterable, Iterator
+from datetime import UTC, datetime, timedelta
from typing import Any, cast
import pytest
@@ -251,16 +252,26 @@ def test_repeated_marker_fails_without_advancing_checkpoint() -> None:
assert result.final_state == saved
[email protected]("schema", [None, "ip_sets"])
+ "schema,expected_reason",
+ [
+ (None, None),
+ ("ip_sets", "Grant wafv2:ListIPSets and wafv2:GetIPSet for this table."),
+ (
+ "sampled_requests",
+ "Grant wafv2:ListWebACLs, wafv2:GetWebACL and wafv2:GetSampledRequests for this table.",
+ ),
+ ("request_metrics", "Grant cloudwatch:ListMetrics and cloudwatch:GetMetricData for this table."),
+ ],
+)
@pytest.mark.parametrize("code", ["AccessDenied", "AccessDeniedException", "WAFUnauthorizedOperationException"])
-def test_missing_permissions_at_creation_or_schema_probe(schema: str | None, code: str) -> None:
+def test_missing_permissions_at_creation_or_schema_probe(
+ schema: str | None, expected_reason: str | None, code: str
+) -> None:
with scripted_network([ScriptedResponse(status=400, json={"__type": f"com.amazonaws.wafv2#{code}"})]) as network:
valid, reason = validate_credentials(config(), schema)
assert valid is (schema is None)
- if schema:
- assert reason == "Grant wafv2:ListIPSets and wafv2:GetIPSet for this table."
- else:
- assert reason is None
+ assert reason == expected_reason
assert len(network.requests_log) == 1
@@ -345,11 +356,13 @@ def test_retries_stop_after_five_attempts() -> None:
assert len(network.requests_log) == 5
-def test_tracked_transport_redacts_secrets_and_disables_nested_retries() -> None:
[email protected]("schema,capture", [(None, True), ("sampled_requests", False)])
+def test_tracked_transport_redacts_secrets_and_disables_nested_retries(schema: str | None, capture: bool) -> None:
with scripted_network([ScriptedResponse(json={"WebACLs": []})]) as network:
- assert validate_credentials(config()) == (True, None)
+ assert validate_credentials(config(), schema) == (True, None)
assert network.session_options[0]["retry"].total == 0
assert set(network.session_options[0]["redact_values"]) == {"example-secret", "example-token"}
+ assert network.session_options[0]["capture"] is capture
def test_service_errors_do_not_become_credential_failures(session: MagicMock) -> None:
@@ -386,3 +399,244 @@ def test_unknown_schema_and_api_version_fail_before_requests() -> None:
assert "Unknown AWS WAF table" in str(result.raised)
assert network.requests_log == []
assert result.requests == []
+
+
+NOW = datetime(2026, 10, 16, 12, 30, tzinfo=UTC)
+
+
+def targets(session: MagicMock) -> list[str]:
+ return [call.kwargs["headers"]["X-Amz-Target"].split(".")[-1] for call in session.post.call_args_list]
+
+
+def payloads(session: MagicMock) -> list[dict[str, Any]]:
+ return [json.loads(call.kwargs["data"]) for call in session.post.call_args_list]
+
+
+def rows_for(table: str, **kwargs: Any) -> list[dict[str, Any]]:
+ with patch(f"{MODULE}.utc_now", return_value=NOW):
+ stream = aws_waf_source(config(**kwargs.pop("overrides", {})), table, manager(), **kwargs)
+ return [row for batch in cast(Iterable[list[dict[str, Any]]], stream.items()) for row in batch]
+
+
+def test_managed_rule_groups_merge_summary_and_description(session: MagicMock) -> None:
+ summary = {"VendorName": "AWS", "Name": "AWSManagedRulesCommonRuleSet", "VersioningSupported": True}
+ session.post.side_effect = [
+ response({"ManagedRuleGroups": [summary]}),
+ response({"VersionName": "Version_1.0", "Capacity": 700, "Rules": [{"Name": "NoUserAgent_HEADER"}]}),
+ ]
+ rows = rows_for("managed_rule_groups")
+ assert targets(session) == ["ListAvailableManagedRuleGroups", "DescribeManagedRuleGroup"]
+ assert payloads(session)[1] == {"Scope": "REGIONAL", "VendorName": "AWS", "Name": "AWSManagedRulesCommonRuleSet"}
+ assert rows == [
+ {
+ "vendor_name": "AWS",
+ "name": "AWSManagedRulesCommonRuleSet",
+ "versioning_supported": True,
+ "version_name": "Version_1.0",
+ "capacity": 700,
+ "rules": [{"Name": "NoUserAgent_HEADER"}],
+ "region": "eu-west-1",
+ "scope": "REGIONAL",
+ }
+ ]
+
+
[email protected]("scope", ["REGIONAL", "CLOUDFRONT"])
+def test_protected_resources_cover_every_resource_type(session: MagicMock, scope: str) -> None:
+ web_acl = {"ARN": "arn:aws:wafv2:eu-west-1:123456789012:regional/webacl/main/id", "Id": "id", "Name": "main"}
+ alb_arn = "arn:aws:elasticloadbalancing:eu-west-1:123456789012:loadbalancer/app/example/1"
+ responses = {
+ "ListWebACLs": response({"WebACLs": [web_acl]}),
+ "APPLICATION_LOAD_BALANCER": response({"ResourceArns": [alb_arn]}),
+ "AGENTCORE_GATEWAY": response({"__type": "WAFInvalidParameterException"}, 400),
+ }
+
+ def post(*args: Any, **kwargs: Any) -> requests.Response:
+ payload = json.loads(kwargs["data"])
+ key = payload.get("ResourceType") or kwargs["headers"]["X-Amz-Target"].split(".")[-1]
+ return responses.get(key, response({"ResourceArns": []}))
+
+ session.post.side_effect = post
+ rows = rows_for("protected_resources", overrides={"scope": scope})
+ if scope == "CLOUDFRONT":
+ assert rows == []
+ session.post.assert_not_called()
+ return
+ assert [payload.get("ResourceType") for payload in payloads(session)[1:]] == [
+ "APPLICATION_LOAD_BALANCER",
+ "API_GATEWAY",
+ "APPSYNC",
+ "COGNITO_USER_POOL",
+ "APP_RUNNER_SERVICE",
+ "VERIFIED_ACCESS_INSTANCE",
+ "AMPLIFY",
+ "AGENTCORE_GATEWAY",
+ ]
+ assert rows == [
+ {
+ "web_acl_arn": web_acl["ARN"],
+ "web_acl_id": "id",
+ "web_acl_name": "main",
+ "resource_type": "APPLICATION_LOAD_BALANCER",
+ "resource_arn": alb_arn,
+ "region": "eu-west-1",
+ "scope": "REGIONAL",
+ }
+ ]
+
+
+ "last_value,expected_start",
+ [
+ (None, NOW - timedelta(hours=3)),
+ (NOW - timedelta(hours=5), NOW - timedelta(hours=3)),
+ ((NOW - timedelta(minutes=20)).isoformat(), NOW - timedelta(minutes=20)),
+ ],
+)
+def test_sampled_requests_window_rules_and_header_redaction(
+ session: MagicMock, last_value: Any, expected_start: datetime
+) -> None:
+ web_acl = {
+ "ARN": "arn:aws:wafv2:eu-west-1:123456789012:regional/webacl/main/id",
+ "Id": "id",
+ "Name": "main",
+ "Rules": [
+ {"Name": "sampled", "VisibilityConfig": {"MetricName": "sampled-rule", "SampledRequestsEnabled": True}},
+ {"Name": "unsampled", "VisibilityConfig": {"MetricName": "quiet-rule", "SampledRequestsEnabled": False}},
+ ],
+ }
+ sample = {
+ "Request": {
+ "ClientIP": "192.0.2.10",
+ "Country": "DE",
+ "URI": "/login",
+ "Method": "POST",
+ "HTTPVersion": "HTTP/2.0",
+ "Headers": [{"Name": "Cookie", "Value": "session=fake"}, {"Name": "User-Agent", "Value": "curl"}],
+ },
+ "Weight": 2,
+ "Timestamp": (NOW - timedelta(minutes=5)).timestamp(),
+ "Action": "BLOCK",
+ }
+ session.post.side_effect = [
+ response({"WebACLs": [{"Id": "id", "Name": "main", "ARN": web_acl["ARN"]}]}),
+ response({"WebACL": web_acl}),
+ response(
+ {
+ "SampledRequests": [sample],
+ "PopulationSize": 40,
+ "TimeWindow": {
+ "StartTime": expected_start.timestamp(),
+ "EndTime": (NOW - timedelta(minutes=2)).timestamp(),
+ },
+ }
+ ),
+ response({"SampledRequests": [], "TimeWindow": {"EndTime": NOW.timestamp()}}),
+ ]
+ rows = rows_for("sampled_requests", incremental_last_value=last_value)
+ assert targets(session) == ["ListWebACLs", "GetWebACL", "GetSampledRequests", "GetSampledRequests"]
+ assert payloads(session)[3]["TimeWindow"] == {
+ "StartTime": (NOW - timedelta(minutes=2)).timestamp(),
+ "EndTime": NOW.timestamp(),
+ }
+ assert payloads(session)[2] == {
+ "WebAclArn": web_acl["ARN"],
+ "RuleMetricName": "sampled-rule",
+ "Scope": "REGIONAL",
+ "TimeWindow": {"StartTime": expected_start.timestamp(), "EndTime": NOW.timestamp()},
+ "MaxItems": 500,
+ }
+ [row] = rows
+ assert row["headers"] == [{"Name": "Cookie", "Value": "[REDACTED]"}, {"Name": "User-Agent", "Value": "curl"}]
+ assert "session=fake" not in json.dumps(row, default=str)
+ assert {key: row[key] for key in ("client_ip", "uri", "http_version", "action", "weight", "population_size")} == {
+ "client_ip": "192.0.2.10",
+ "uri": "/login",
+ "http_version": "HTTP/2.0",
+ "action": "BLOCK",
+ "weight": 2,
+ "population_size": 40,
+ }
+ assert row["timestamp"] == NOW - timedelta(minutes=5)
+ assert row["rule_metric_name"] == "sampled-rule"
+ assert len(row["id"]) == 64
+
+
[email protected]("scope,region_filter", [("REGIONAL", True), ("CLOUDFRONT", False)])
+def test_request_metrics_from_cloudwatch(session: MagicMock, scope: str, region_filter: bool) -> None:
+ regional = {
+ "Namespace": "AWS/WAFV2",
+ "MetricName": "BlockedRequests",
+ "Dimensions": [
+ {"Name": "WebACL", "Value": "main"},
+ {"Name": "Rule", "Value": "ALL"},
+ {"Name": "Region", "Value": "us-east-1"},
+ ],
+ }
+ cloudfront = {
+ "Namespace": "AWS/WAFV2",
+ "MetricName": "AllowedRequests",
+ "Dimensions": [{"Name": "WebACL", "Value": "edge"}, {"Name": "Rule", "Value": "ALL"}],
+ }
+ hour = NOW.replace(minute=0) - timedelta(hours=1)
+ session.post.side_effect = [
+ response({"Metrics": [regional], "NextToken": "metrics-2"}),
+ response({"Metrics": [cloudfront]}),
+ response(
+ {
+ "MetricDataResults": [{"Id": "m0", "Timestamps": [hour.timestamp()], "Values": [5.0]}],
+ "NextToken": "data-2",
+ }
+ ),
+ response(
+ {"MetricDataResults": [{"Id": "m0", "Timestamps": [NOW.replace(minute=0).timestamp()], "Values": [7.0]}]}
+ ),
+ ]
+ rows = rows_for(
+ "request_metrics",
+ overrides={"scope": scope, "aws_region": "us-east-1"},
+ incremental_last_value=hour + timedelta(minutes=10),
+ )
+ calls = session.post.call_args_list
+ assert {call.args[0] for call in calls} == {"https://monitoring.us-east-1.amazonaws.com/"}
+ assert "/us-east-1/monitoring/aws4_request" in calls[0].kwargs["headers"]["Authorization"]
+ assert calls[0].kwargs["headers"]["Content-Type"] == "application/x-amz-json-1.0"
+ assert [call.kwargs["headers"]["X-Amz-Target"] for call in calls] == [
+ "GraniteServiceVersion20100801.ListMetrics",
+ "GraniteServiceVersion20100801.ListMetrics",
+ "GraniteServiceVersion20100801.GetMetricData",
+ "GraniteServiceVersion20100801.GetMetricData",
+ ]
+ sent = payloads(session)
+ assert ("Dimensions" in sent[0]) is region_filter
+ assert sent[1]["NextToken"] == "metrics-2"
+ expected_metric = regional if region_filter else cloudfront
+ assert sent[2]["MetricDataQueries"] == [
+ {"Id": "m0", "MetricStat": {"Metric": expected_metric, "Period": 3600, "Stat": "Sum"}, "ReturnData": True}
+ ]
+ assert sent[2]["StartTime"] == hour.timestamp()
+ assert sent[3]["NextToken"] == "data-2"
+ assert [(row["metric_id"], row["timestamp"], row["value"]) for row in rows] == [
+ (
+ "BlockedRequests|Region=us-east-1|Rule=ALL|WebACL=main"
+ if region_filter
+ else "AllowedRequests|Rule=ALL|WebACL=edge",
+ timestamp,
+ value,
+ )
+ for timestamp, value in [(hour, 5.0), (NOW.replace(minute=0), 7.0)]
+ ]
+
+
+def test_request_metrics_fail_on_a_failed_series(session: MagicMock) -> None:
+ metric = {
+ "Namespace": "AWS/WAFV2",
+ "MetricName": "BlockedRequests",
+ "Dimensions": [{"Name": "Region", "Value": "eu-west-1"}],
+ }
+ session.post.side_effect = [
+ response({"Metrics": [metric]}),
+ response({"MetricDataResults": [{"Id": "m0", "StatusCode": "InternalError", "Timestamps": [], "Values": []}]}),
+ ]
+ with pytest.raises(AwsWafRetryableError, match="InternalError"):
+ rows_for("request_metrics")