Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add coverage, malware scan, and usage tables to aws guardduty (#117700)

Tom Owers92204353a404
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 6b255c037158..947d27510941 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -1203,10 +1203,10 @@ Today (3): `detectors`, `findings`, `members`
Diffed against: <https://raw.githubusercontent.com/boto/botocore/develop/botocore/data/guardduty/2017-11-28/service-2.json>
-- [ ] `Runtime Monitoring coverage (ListCoverage, POST /detector/{DetectorId}/coverage)` — identifies protected and unprotected resources, which threat findings alone cannot reveal (high)
-- [ ] `Detector malware scans (DescribeMalwareScans, POST /detector/{DetectorId}/malware-scans)` — tracks EC2 malware scan outcomes, failures, duration, and scanned data volume, including scans without findings (medium)
+- [x] `Runtime Monitoring coverage (ListCoverage, POST /detector/{DetectorId}/coverage)` — identifies protected and unprotected resources, which threat findings alone cannot reveal (high)
+- [x] `Detector malware scans (DescribeMalwareScans, POST /detector/{DetectorId}/malware-scans)` — tracks EC2 malware scan outcomes, failures, duration, and scanned data volume, including scans without findings (medium)
- [ ] `Malware scans (ListMalwareScans, POST /malware-scan)` — tracks scan status and results through the separate resource-oriented malware scan API (medium)
-- [ ] `Usage statistics (GetUsageStatistics, POST /detector/{DetectorId}/usage/statistics)` — reports detector usage and estimated costs by account, resource, or data source (medium)
+- [x] `Usage statistics (GetUsageStatistics, POST /detector/{DetectorId}/usage/statistics)` — reports detector usage and estimated costs by account, resource, or data source (medium)
- [ ] `Investigations (ListInvestigations, POST /detector/{DetectorId}/investigation/list)` — tracks investigation status and links related threat findings into cases (low)
Note: Compared the official botocore operation model with the static endpoint map and detector traversal code. Findings already call GetFindings; detector malware history and the separate malware-scan API remain absent. Scan details require GetMalwareScan hydration; investigations require GetInvestigation hydration and are a regional preview. Detector resources require traversal; filters, threat lists, publishing destinations, and other setup endpoints were excluded.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/aws_guardduty.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/aws_guardduty.py
index d7709a57e0fc..e274a8ba90b8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/aws_guardduty.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/aws_guardduty.py
@@ -20,6 +20,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.aws_guardd
ERROR_MESSAGES,
GUARDDUTY_API_VERSION,
MAX_RESULTS,
+ USAGE_FEATURES,
+ USAGE_STATISTIC_TYPES,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.transport import BoundedRetry
@@ -40,6 +42,7 @@ TRANSPORT_RETRY = BoundedRetry(
_CAMEL_BOUNDARY = re.compile(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])")
_REGION = re.compile(r"(?:us|eu|ap|sa|ca|me|af|il|mx|cn)-(?:gov-)?[a-z]+-\d+")
_SORT_CRITERIA = {"AttributeName": "updatedAt", "OrderBy": "ASC"}
+_TIMESTAMP_COLUMNS = ("created_at", "updated_at", "invited_at", "scan_start_time", "scan_end_time")
@frozen
@@ -47,6 +50,7 @@ class AwsGuarddutyResumeConfig:
detector_ids: list[str] = field(default_factory=list)
next_token: str | None = None
updated_since_ms: int | None = None
+ usage_statistic_types: list[str] = field(default_factory=list)
class AwsGuarddutyError(Exception):
@@ -168,14 +172,35 @@ def normalize_row(item: dict[str, Any], detector_id: str, region: str) -> dict[s
row = {_CAMEL_BOUNDARY.sub("_", key).lower(): value for key, value in item.items()}
row["source_detector_id"] = detector_id
row["region"] = region
- for column in ("created_at", "updated_at", "invited_at"):
+ for column in _TIMESTAMP_COLUMNS:
value = row.get(column)
if isinstance(value, str) and value:
parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
row[column] = parsed if parsed.tzinfo else parsed.replace(tzinfo=dt.UTC)
+ elif isinstance(value, int | float):
+ # Fields typed as timestamps in the service model arrive as epoch seconds.
+ row[column] = dt.datetime.fromtimestamp(value, tz=dt.UTC)
return row
+def request_payload(endpoint: str, detector_id: str, usage_statistic_type: str | None = None) -> dict[str, Any]:
+ payload: dict[str, Any] = {"DetectorId": detector_id}
+ if ENDPOINTS[endpoint].result_key:
+ payload["MaxResults"] = MAX_RESULTS
+ if endpoint == "members":
+ payload["OnlyAssociated"] = "false"
+ elif endpoint == "usage_statistics":
+ payload["UsageStatisticType"] = usage_statistic_type
+ payload["UsageCriteria"] = {"Features": list(USAGE_FEATURES)}
+ return payload
+
+
+def usage_rows(body: dict[str, Any], usage_statistic_type: str) -> list[dict[str, Any]]:
+ list_key, key_field = USAGE_STATISTIC_TYPES[usage_statistic_type]
+ items = (body.get("usageStatistics") or {}).get(list_key) or []
+ return [{**item, "usageStatisticType": usage_statistic_type, "usageKey": item.get(key_field)} for item in items]
+
+
def watermark_milliseconds(value: dt.datetime | str | None) -> int | None:
if value is None:
return None
@@ -200,23 +225,21 @@ def get_rows(
state = AwsGuarddutyResumeConfig(
detector_ids=list(client.detector_ids()),
updated_since_ms=watermark_milliseconds(updated_since) if endpoint == "findings" else None,
+ usage_statistic_types=list(USAGE_STATISTIC_TYPES) if endpoint == "usage_statistics" else [],
)
seen_tokens = {state.next_token} if state.next_token else set()
while state.detector_ids:
detector_id = state.detector_ids[0]
- payload: dict[str, Any] = {"DetectorId": detector_id}
- if endpoint_config.result_key:
- payload["MaxResults"] = MAX_RESULTS
- if state.next_token:
- payload["NextToken"] = state.next_token
+ usage_statistic_type = state.usage_statistic_types[0] if state.usage_statistic_types else None
+ payload = request_payload(endpoint, detector_id, usage_statistic_type)
+ if endpoint_config.result_key and state.next_token:
+ payload["NextToken"] = state.next_token
if endpoint == "findings":
payload["SortCriteria"] = _SORT_CRITERIA
if state.updated_since_ms is not None:
payload["FindingCriteria"] = {
"Criterion": {"updatedAt": {"GreaterThanOrEqual": state.updated_since_ms}}
}
- elif endpoint == "members":
- payload["OnlyAssociated"] = "false"
body = client.request(endpoint_config.operation, payload)
next_token = body.get("nextToken")
if next_token and next_token in seen_tokens:
@@ -234,6 +257,8 @@ def get_rows(
{"DetectorId": detector_id, "FindingIds": finding_ids, "SortCriteria": _SORT_CRITERIA},
)
items = findings.get("findings", [])
+ elif endpoint == "usage_statistics" and usage_statistic_type:
+ items = usage_rows(body, usage_statistic_type)
elif endpoint_config.result_key:
items = body.get(endpoint_config.result_key, [])
else:
@@ -241,11 +266,17 @@ def get_rows(
rows = [normalize_row(item, detector_id, client.region) for item in items]
if rows:
yield rows
- state = replace(
- state,
- detector_ids=state.detector_ids if next_token else state.detector_ids[1:],
- next_token=next_token or None,
- )
+ if next_token:
+ state = replace(state, next_token=next_token)
+ elif len(state.usage_statistic_types) > 1:
+ state = replace(state, next_token=None, usage_statistic_types=state.usage_statistic_types[1:])
+ else:
+ state = replace(
+ state,
+ detector_ids=state.detector_ids[1:],
+ next_token=None,
+ usage_statistic_types=list(USAGE_STATISTIC_TYPES) if endpoint == "usage_statistics" else [],
+ )
manager.save_state(state)
manager.safe_point()
finally:
@@ -277,13 +308,10 @@ def validate_credentials(
detector_id = next(iter(body.get("detectorIds", [])), None)
if detector_id is None:
return False, "Enable GuardDuty in the selected AWS region, then reconnect."
- endpoint = ENDPOINTS[schema_name]
- payload: dict[str, Any] = {"DetectorId": detector_id}
- if endpoint.result_key:
+ payload = request_payload(schema_name, detector_id, next(iter(USAGE_STATISTIC_TYPES)))
+ if "MaxResults" in payload:
payload["MaxResults"] = 1
- if schema_name == "members":
- payload["OnlyAssociated"] = "false"
- body = client.request(endpoint.operation, payload)
+ body = client.request(ENDPOINTS[schema_name].operation, payload)
if schema_name == "findings" and body.get("findingIds"):
client.request("GetFindings", {"DetectorId": detector_id, "FindingIds": body["findingIds"]})
return True, None
@@ -313,8 +341,8 @@ def aws_guardduty_source(
# Findings can change between listing IDs and retrieving details, so global ordering is not guaranteed.
# "desc" saves the watermark once at job end, so unordered rows cannot strand older ones.
sort_mode="desc",
- partition_keys=["created_at"] if endpoint == "findings" else None,
- partition_mode="datetime" if endpoint == "findings" else None,
- partition_format="month" if endpoint == "findings" else None,
+ partition_keys=[endpoint_config.partition_key] if endpoint_config.partition_key else None,
+ partition_mode="datetime" if endpoint_config.partition_key else None,
+ partition_format="month" if endpoint_config.partition_key else None,
on_complete=manager.clear_state,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/canonical_descriptions.py
index 1d145df4193e..3a1731773a33 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/canonical_descriptions.py
@@ -53,4 +53,56 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"relationship_status": "Status of the relationship between the member and administrator accounts.",
},
},
+ "coverage": {
+ "description": "Runtime Monitoring coverage of EKS clusters, ECS clusters, and EC2 instances in the selected region.",
+ "docs_url": "https://docs.aws.amazon.com/guardduty/latest/APIReference/API_ListCoverage.html",
+ "columns": {
+ "source_detector_id": "Detector used to list this coverage entry.",
+ "region": "AWS region that contains the resource.",
+ "resource_id": "Identifier of the covered resource.",
+ "detector_id": "Identifier of the detector that covers the resource.",
+ "account_id": "AWS account that owns the resource.",
+ "resource_details": "Details of the covered resource, including its type and agent status.",
+ "coverage_status": "Coverage status of the resource. One of HEALTHY or UNHEALTHY.",
+ "issue": "Reason the resource is unhealthy, if there is one.",
+ "updated_at": "Time when the coverage details were last updated.",
+ },
+ },
+ "malware_scans": {
+ "description": "Malware Protection for EC2 scans, including scans that found no malware.",
+ "docs_url": "https://docs.aws.amazon.com/guardduty/latest/APIReference/API_DescribeMalwareScans.html",
+ "columns": {
+ "source_detector_id": "Detector used to list this scan.",
+ "region": "AWS region where the scan ran.",
+ "scan_id": "Unique identifier of the malware scan.",
+ "detector_id": "Detector of the account that the scanned resource belongs to.",
+ "admin_detector_id": "Detector of the administrator account that started the scan, if there is one.",
+ "account_id": "AWS account that owns the scanned resource.",
+ "scan_status": "Status of the scan. One of RUNNING, COMPLETED, FAILED, or SKIPPED.",
+ "failure_reason": "Reason the scan failed, if it failed.",
+ "scan_start_time": "Time when the scan started.",
+ "scan_end_time": "Time when the scan ended.",
+ "trigger_details": "Finding or request that started the scan.",
+ "resource_details": "Resource that was scanned.",
+ "scan_result_details": "Result of the scan, such as CLEAN or INFECTED.",
+ "total_bytes": "Total number of bytes scanned.",
+ "file_count": "Number of files scanned.",
+ "attached_volumes": "EBS volumes scanned.",
+ "scan_type": "How the scan started. One of GUARDDUTY_INITIATED or ON_DEMAND.",
+ },
+ },
+ "usage_statistics": {
+ "description": "GuardDuty usage cost over the last 30 days, summed by account, feature, and resource.",
+ "docs_url": "https://docs.aws.amazon.com/guardduty/latest/APIReference/API_GetUsageStatistics.html",
+ "columns": {
+ "source_detector_id": "Detector used to retrieve the usage statistics.",
+ "region": "AWS region of the usage.",
+ "usage_statistic_type": "How the usage is summed. One of SUM_BY_ACCOUNT, SUM_BY_FEATURES, or SUM_BY_RESOURCE.",
+ "usage_key": "Account ID, feature, or resource that the usage is summed by.",
+ "account_id": "AWS account that generated the usage, for SUM_BY_ACCOUNT rows.",
+ "feature": "GuardDuty feature that generated the usage, for SUM_BY_FEATURES rows.",
+ "resource": "AWS resource that generated the usage, for SUM_BY_RESOURCE rows.",
+ "total": "Usage cost amount and its currency unit.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/settings.py
index 55d94f0adf83..276281d51845 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/settings.py
@@ -12,6 +12,7 @@ class GuarddutyEndpoint:
result_key: str | None
primary_keys: tuple[str, ...]
iam_actions: tuple[str, ...]
+ partition_key: str | None = None
ENDPOINTS = {
@@ -20,6 +21,7 @@ ENDPOINTS = {
result_key="findingIds",
primary_keys=("arn",),
iam_actions=("guardduty:ListDetectors", "guardduty:ListFindings", "guardduty:GetFindings"),
+ partition_key="created_at",
),
"detectors": GuarddutyEndpoint(
operation="GetDetector",
@@ -33,13 +35,63 @@ ENDPOINTS = {
primary_keys=("region", "source_detector_id", "account_id"),
iam_actions=("guardduty:ListDetectors", "guardduty:ListMembers"),
),
+ "coverage": GuarddutyEndpoint(
+ operation="ListCoverage",
+ result_key="resources",
+ primary_keys=("region", "source_detector_id", "account_id", "resource_id"),
+ iam_actions=("guardduty:ListDetectors", "guardduty:ListCoverage"),
+ ),
+ "malware_scans": GuarddutyEndpoint(
+ operation="DescribeMalwareScans",
+ result_key="scans",
+ primary_keys=("region", "source_detector_id", "scan_id"),
+ iam_actions=("guardduty:ListDetectors", "guardduty:DescribeMalwareScans"),
+ partition_key="scan_start_time",
+ ),
+ "usage_statistics": GuarddutyEndpoint(
+ operation="GetUsageStatistics",
+ result_key="usageStatistics",
+ primary_keys=("region", "source_detector_id", "usage_statistic_type", "usage_key"),
+ iam_actions=("guardduty:ListDetectors", "guardduty:GetUsageStatistics"),
+ ),
+}
+
+# Statistic type -> (response list key, row field that identifies the usage line).
+USAGE_STATISTIC_TYPES = {
+ "SUM_BY_ACCOUNT": ("sumByAccount", "accountId"),
+ "SUM_BY_FEATURES": ("sumByFeature", "feature"),
+ "SUM_BY_RESOURCE": ("sumByResource", "resource"),
}
+# The API sums only the features named in the request, so list every feature it reports usage for.
+USAGE_FEATURES = (
+ "FLOW_LOGS",
+ "CLOUD_TRAIL",
+ "DNS_LOGS",
+ "S3_DATA_EVENTS",
+ "EKS_AUDIT_LOGS",
+ "EBS_MALWARE_PROTECTION",
+ "RDS_LOGIN_EVENTS",
+ "LAMBDA_NETWORK_LOGS",
+ "EKS_RUNTIME_MONITORING",
+ "EC2_RUNTIME_MONITORING",
+ "FARGATE_RUNTIME_MONITORING",
+ "RDS_DBI_PROTECTION_PROVISIONED",
+ "RDS_DBI_PROTECTION_SERVERLESS",
+ "RDS_DBI_PROTECTION_LIMITLESS",
+ "RDS_DATA_ACTIVITY_DBI_PROTECTION_PROVISIONED",
+ "RDS_DATA_ACTIVITY_DBI_PROTECTION_SERVERLESS",
+ "RDS_DATA_ACTIVITY_DBI_PROTECTION_LIMITLESS",
+ "AI_PROTECTION",
+)
INCREMENTAL_FIELDS = {"findings": [incremental_field("updated_at")]}
ENDPOINT_DESCRIPTIONS = {
"findings": "Threat findings, including severity, affected resources, and detection details.",
"detectors": "Detector status, enabled features, and configuration in the selected region.",
"members": "Member accounts and their relationship with the GuardDuty administrator account.",
+ "coverage": "Runtime Monitoring coverage status for EKS clusters, ECS clusters, and EC2 instances.",
+ "malware_scans": "Malware Protection for EC2 scans, including status, result, trigger, and scanned data volume.",
+ "usage_statistics": "GuardDuty usage cost over the last 30 days, summed by account, feature, and resource.",
}
ERROR_MESSAGES = {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/source.py
index a393942c77b2..71d7331e7158 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/source.py
@@ -113,10 +113,12 @@ class AwsGuarddutySource(ResumableSource[AwsGuarddutySourceConfig, AwsGuarddutyR
label="AWS GuardDuty",
iconPath="/static/services/aws_guardduty.png",
caption=(
- "Sync GuardDuty findings, detectors, and member accounts from one AWS region. "
+ "Sync GuardDuty findings, detectors, member accounts, Runtime Monitoring coverage, "
+ "malware scans, and usage statistics from one AWS region. "
"Enable GuardDuty in that region before syncing. "
"Grant `guardduty:ListDetectors`, `guardduty:GetDetector`, `guardduty:ListFindings`, "
- "`guardduty:GetFindings`, and `guardduty:ListMembers` for all tables. "
+ "`guardduty:GetFindings`, `guardduty:ListMembers`, `guardduty:ListCoverage`, "
+ "`guardduty:DescribeMalwareScans`, and `guardduty:GetUsageStatistics` for all tables. "
"The members table requires a GuardDuty administrator account. "
"Findings use incremental sync through their update time. Other tables use full refresh."
),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/tests/test_aws_guardduty.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/tests/test_aws_guardduty.py
index 0d18a99686ff..83d5fcfe2541 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/tests/test_aws_guardduty.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_guardduty/tests/test_aws_guardduty.py
@@ -142,6 +142,128 @@ def test_detector_details(config: AwsGuarddutySourceConfig) -> None:
assert result.paths[-1].endswith("/detector/detector-example")
+ "endpoint,path,items_key,item,timestamp_column,partition_keys",
+ [
+ (
+ "coverage",
+ "/detector/detector-example/coverage",
+ "resources",
+ {
+ "resourceId": "i-example",
+ "accountId": "111111111111",
+ "coverageStatus": "HEALTHY",
+ "updatedAt": 1735689600.0,
+ },
+ "updated_at",
+ None,
+ ),
+ (
+ "malware_scans",
+ "/detector/detector-example/malware-scans",
+ "scans",
+ {"scanId": "scan-example", "scanStatus": "COMPLETED", "scanStartTime": 1735689600, "totalBytes": 10},
+ "scan_start_time",
+ ["scan_start_time"],
+ ),
+ ],
+)
+def test_detector_list_endpoints_page_and_parse_epoch_timestamps(
+ config: AwsGuarddutySourceConfig,
+ endpoint: str,
+ path: str,
+ items_key: str,
+ item: dict[str, Any],
+ timestamp_column: str,
+ partition_keys: list[str] | None,
+) -> None:
+ result = SourceDriver(AwsGuarddutySource(), config).run(
+ endpoint,
+ [
+ ScriptedResponse(json={"detectorIds": ["detector-example"]}),
+ ScriptedResponse(json={items_key: [item], "nextToken": "page-2"}),
+ ScriptedResponse(json={items_key: [item]}),
+ ],
+ api_version=VERSION,
+ )
+ assert result.raised is None
+ assert len(result.items) == 2
+ row = result.items[0][0]
+ assert row[timestamp_column] == UPDATED_AT
+ assert row["source_detector_id"] == "detector-example"
+ assert all(request.path.endswith(path) for request in result.requests[1:])
+ assert result.requests[1].json() == {"maxResults": 50}
+ assert result.requests[2].json() == {"maxResults": 50, "nextToken": "page-2"}
+ assert result.saved_states[-1].detector_ids == []
+ assert result.response is not None and result.response.partition_keys == partition_keys
+
+
+def test_usage_statistics_walks_every_statistic_type_per_detector(config: AwsGuarddutySourceConfig) -> None:
+ total = {"amount": "1.5", "unit": "USD"}
+ result = SourceDriver(AwsGuarddutySource(), config).run(
+ "usage_statistics",
+ [
+ ScriptedResponse(json={"detectorIds": ["detector-a", "detector-b"]}),
+ ScriptedResponse(
+ json={
+ "usageStatistics": {"sumByAccount": [{"accountId": "111111111111", "total": total}]},
+ "nextToken": "next",
+ }
+ ),
+ ScriptedResponse(
+ json={"usageStatistics": {"sumByAccount": [{"accountId": "222222222222", "total": total}]}}
+ ),
+ ScriptedResponse(json={"usageStatistics": {"sumByFeature": [{"feature": "FLOW_LOGS", "total": total}]}}),
+ ScriptedResponse(json={"usageStatistics": {"sumByResource": []}}),
+ ScriptedResponse(json={"usageStatistics": {}}),
+ ScriptedResponse(json={"usageStatistics": {}}),
+ ScriptedResponse(
+ json={"usageStatistics": {"sumByResource": [{"resource": "bucket-example", "total": total}]}}
+ ),
+ ],
+ api_version=VERSION,
+ )
+ assert result.raised is None
+ rows = [row for batch in result.items for row in batch]
+ assert [(row["source_detector_id"], row["usage_statistic_type"], row["usage_key"]) for row in rows] == [
+ ("detector-a", "SUM_BY_ACCOUNT", "111111111111"),
+ ("detector-a", "SUM_BY_ACCOUNT", "222222222222"),
+ ("detector-a", "SUM_BY_FEATURES", "FLOW_LOGS"),
+ ("detector-b", "SUM_BY_RESOURCE", "bucket-example"),
+ ]
+ assert rows[0]["total"] == total
+ payloads = [request.json() for request in result.requests[1:]]
+ assert [payload["usageStatisticsType"] for payload in payloads] == [
+ "SUM_BY_ACCOUNT",
+ "SUM_BY_ACCOUNT",
+ "SUM_BY_FEATURES",
+ "SUM_BY_RESOURCE",
+ "SUM_BY_ACCOUNT",
+ "SUM_BY_FEATURES",
+ "SUM_BY_RESOURCE",
+ ]
+ assert payloads[1]["nextToken"] == "next"
+ assert "FLOW_LOGS" in payloads[0]["usageCriteria"]["features"]
+ assert result.requests[-1].path.endswith("/detector/detector-b/usage/statistics")
+ assert result.saved_states[-1].detector_ids == []
+
+
+def test_usage_statistics_resumes_at_saved_statistic_type(config: AwsGuarddutySourceConfig) -> None:
+ result = SourceDriver(AwsGuarddutySource(), config).run(
+ "usage_statistics",
+ [ScriptedResponse(json={"usageStatistics": {"sumByResource": [{"resource": "bucket-example"}]}})],
+ api_version=VERSION,
+ resume_state=AwsGuarddutyResumeConfig(
+ detector_ids=["detector-example"], usage_statistic_types=["SUM_BY_RESOURCE"], next_token="resume-token"
+ ),
+ )
+ assert result.raised is None
+ assert result.items[0][0]["usage_key"] == "bucket-example"
+ assert len(result.requests) == 1
+ assert result.requests[0].json()["usageStatisticsType"] == "SUM_BY_RESOURCE"
+ assert result.requests[0].json()["nextToken"] == "resume-token"
+
+
@pytest.mark.parametrize(
"value,expected",
[
@@ -259,7 +381,15 @@ def test_no_detector_validation(config: AwsGuarddutySourceConfig, schema_name: s
@pytest.mark.parametrize(
- "schema_name,path", [("findings", "/findings"), ("members", "/member"), ("detectors", "/detector/detector-example")]
+ "schema_name,path",
+ [
+ ("findings", "/findings"),
+ ("members", "/member"),
+ ("detectors", "/detector/detector-example"),
+ ("coverage", "/coverage"),
+ ("malware_scans", "/malware-scans"),
+ ("usage_statistics", "/usage/statistics"),
+ ],
)
def test_endpoint_validation_calls_its_read_operation(
config: AwsGuarddutySourceConfig, schema_name: str, path: str