Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse): add aws security hub finding history, insight results, and v2 tables (#118514)

Tom Owersc96371e592df
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 9a5cad902cbf..032b5ad439cb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -1298,7 +1298,7 @@ Note: The Savings Plans SDK model was compared with the static table map. The of
## AWS Security Hub — gaps
-Today (8): `enabled_standards`, `findings`, `insights`, `members`, `products`, `security_control_definitions`, `standards`, `standards_controls`
+Today (12): `enabled_standards`, `finding_history`, `findings`, `findings_v2`, `insight_results`, `insights`, `members`, `products`, `resources_v2`, `security_control_definitions`, `standards`, `standards_controls`
Diffed against: <https://raw.githubusercontent.com/boto/botocore/develop/botocore/data/securityhub/2018-10-26/service-2.json>
@@ -1306,10 +1306,10 @@ Diffed against: <https://raw.githubusercontent.com/boto/botocore/develop/botocor
- [x] `Standards controls (GET /standards/controls/{StandardsSubscriptionArn+})` — connect enabled standards to individual controls, enablement status, severity, and disablement reasons (high)
- [x] `Products (GET /products)` — resolve ProductArn in findings to vendor names and product descriptions (high)
- [x] `Member accounts (GET /members)` — resolve finding account IDs to member account details and analyze security coverage across accounts (high)
-- [ ] `Finding history (POST /findingHistory/get)` — measure remediation progress and changes in severity, compliance, and workflow status (high)
-- [ ] `Security Hub V2 findings (GetFindingsV2, POST /findingsv2)` — analyze findings from the newer Security Hub surface that legacy CSPM GetFindings does not provide (high)
-- [ ] `Security Hub V2 resources (GetResourcesV2, POST /resourcesv2)` — analyze resource risk and connect newer findings to a separate resource inventory (medium)
-- [ ] `Insight results (GET /insights/results/{InsightArn+})` — import grouped finding counts for insights instead of only their filter definitions (medium)
+- [x] `Finding history (POST /findingHistory/get)` — measure remediation progress and changes in severity, compliance, and workflow status (high)
+- [x] `Security Hub V2 findings (GetFindingsV2, POST /findingsv2)` — analyze findings from the newer Security Hub surface that legacy CSPM GetFindings does not provide (high)
+- [x] `Security Hub V2 resources (GetResourcesV2, POST /resourcesv2)` — analyze resource risk and connect newer findings to a separate resource inventory (medium)
+- [x] `Insight results (GET /insights/results/{InsightArn+})` — import grouped finding counts for insights instead of only their filter definitions (medium)
- [x] `Remediations (GetRemediationsV2, POST /GetRemediationsV2)` — prioritize the resource changes that resolve or reduce exposure findings (high)
- [x] `Exposures by remediation (ListExposuresByRemediationV2, POST /ListExposuresByRemediationV2)` — connect each remediation target to the exposure findings it resolves or reduces (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_security_hub/aws_security_hub.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_security_hub/aws_security_hub.py
index cf7408c598c6..b94ed143973b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_security_hub/aws_security_hub.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_security_hub/aws_security_hub.py
@@ -20,6 +20,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.aws_securi
ACCESS_DENIED_CODES,
ENDPOINTS,
ERROR_MESSAGES,
+ INCREMENTAL_FIELDS,
SECURITY_HUB_API_VERSION,
SUBSCRIPTION_MESSAGE,
UNMODELED_OPERATION_PATHS,
@@ -43,6 +44,9 @@ TRANSPORT_RETRY = BoundedRetry(
raise_on_status=False,
)
_CAMEL_BOUNDARY = re.compile(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])")
+_UNORDERED_INCREMENTAL = frozenset({"finding_history", "findings_v2"})
+# CreatedAt never changes, so a finding keeps its position while the walk pages through findings.
+_FINDING_HISTORY_PARENT_SORT = [{"Field": "CreatedAt", "SortOrder": "asc"}]
@frozen
@@ -51,6 +55,8 @@ class AwsSecurityHubResumeConfig:
updated_after: str | None = None
finished: bool = False
parent_id: str | None = None
+ parent_token: str | None = None
+ cutoff: str | None = None
class AwsSecurityHubError(Exception):
@@ -144,6 +150,11 @@ def normalize_row(item: dict[str, Any]) -> dict[str, Any]:
"processed_at",
"invited_at",
"control_status_updated_at",
+ "update_time",
+ "created_time",
+ "modified_time",
+ "resource_creation_time_dt",
+ "resource_detail_capture_time_dt",
):
if isinstance(row.get(key), str):
timestamp = dt.datetime.fromisoformat(row[key].replace("Z", "+00:00"))
@@ -151,6 +162,25 @@ def normalize_row(item: dict[str, Any]) -> dict[str, Any]:
return row
+def _prepare(endpoint: str, item: dict[str, Any]) -> dict[str, Any]:
+ if endpoint == "finding_history":
+ record = dict(item)
+ identifier = record.pop("FindingIdentifier", None) or {}
+ return {"ProductArn": identifier.get("ProductArn"), "FindingId": identifier.get("Id"), **record}
+ if endpoint == "findings_v2":
+ # OCSF findings nest their identity and timestamps, so copy them to top-level columns for keys and syncs.
+ info = item.get("finding_info") or {}
+ product = (item.get("metadata") or {}).get("product") or {}
+ return {
+ **item,
+ "product_uid": product.get("uid"),
+ "finding_uid": info.get("uid"),
+ "created_time": info.get("created_time_dt"),
+ "modified_time": info.get("modified_time_dt"),
+ }
+ return item
+
+
def get_rows(
config: AwsSecurityHubSourceConfig,
endpoint: str,
@@ -164,21 +194,34 @@ def get_rows(
if state and state.finished:
return
updated_after = None
- if endpoint == "findings" and should_use_incremental_field and db_incremental_field_last_value is not None:
+ if endpoint in INCREMENTAL_FIELDS and should_use_incremental_field and db_incremental_field_last_value is not None:
value = db_incremental_field_last_value
timestamp = (
value if isinstance(value, dt.datetime) else dt.datetime.fromisoformat(str(value).replace("Z", "+00:00"))
)
updated_after = (timestamp if timestamp.tzinfo else timestamp.replace(tzinfo=dt.UTC)).isoformat()
+ cutoff = None
+ if endpoint in _UNORDERED_INCREMENTAL and should_use_incremental_field:
+ # A row that changes mid-run sits below the run's highest value, so the next run would skip it.
+ # An upper bound keeps every change after the cutoff for the next run.
+ cutoff = dt.datetime.now(dt.UTC).isoformat()
if state:
updated_after = state.updated_after
+ cutoff = state.cutoff
client = AwsSecurityHubClient(config, api_version)
try:
+ if endpoint == "finding_history":
+ yield from _finding_history_rows(client, manager, state, updated_after, cutoff)
+ return
filters: dict[str, Any] = {}
if endpoint == "findings":
filters["SortCriteria"] = [{"Field": "UpdatedAt", "SortOrder": "asc"}]
if updated_after:
filters["Filters"] = {"UpdatedAt": [{"Start": updated_after}]}
+ if endpoint == "findings_v2" and cutoff:
+ window = {"End": cutoff, **({"Start": updated_after} if updated_after else {})}
+ date_filter = {"FieldName": "finding_info.modified_time_dt", "Filter": window}
+ filters["Filters"] = {"CompositeFilters": [{"DateFilters": [date_filter]}]}
parent_ids: list[str | None] = [None]
parent_field = None
if endpoint_config.parent:
@@ -197,11 +240,12 @@ def get_rows(
pages = _pages(client, endpoint_config, start_token, {**filters, **path_params})
start_token = None
for items, token in pages:
- rows = [normalize_row({**path_params, **item}) for item in items]
+ rows = [normalize_row(_prepare(endpoint, {**path_params, **item})) for item in items]
manager.save_state(
AwsSecurityHubResumeConfig(
next_token=token,
updated_after=updated_after,
+ cutoff=cutoff,
finished=token is None and next_parent is None,
parent_id=parent_id if token else next_parent,
)
@@ -213,22 +257,112 @@ def get_rows(
client.session.close()
+def _finding_history_rows(
+ client: AwsSecurityHubClient,
+ manager: ResumableSourceManager[AwsSecurityHubResumeConfig],
+ state: AwsSecurityHubResumeConfig | None,
+ start_time: str | None,
+ cutoff: str | None,
+) -> Iterator[list[dict[str, Any]]]:
+ history = ENDPOINTS["finding_history"]
+ extra = {key: value for key, value in (("StartTime", start_time), ("EndTime", cutoff)) if value}
+ parent_token = state.parent_token if state else None
+ resume_from = state.parent_id if state else None
+ start_token = state.next_token if state else None
+ while True:
+ payload: dict[str, Any] = {"MaxResults": 100, "SortCriteria": _FINDING_HISTORY_PARENT_SORT}
+ if parent_token:
+ payload["NextToken"] = parent_token
+ result = client.request("GetFindings", payload)
+ next_parent_token = result.get("NextToken") or None
+ identifiers = [
+ {"Id": finding["Id"], "ProductArn": finding["ProductArn"]}
+ for finding in result.get("Findings") or []
+ if finding.get("Id") and finding.get("ProductArn")
+ ]
+ keys = [_finding_key(identifier) for identifier in identifiers]
+ if resume_from in keys:
+ identifiers = identifiers[keys.index(resume_from) :]
+ keys = keys[keys.index(resume_from) :]
+ else:
+ start_token = None
+ resume_from = None
+ if not identifiers:
+ manager.save_state(
+ AwsSecurityHubResumeConfig(
+ updated_after=start_time,
+ cutoff=cutoff,
+ parent_token=next_parent_token,
+ finished=next_parent_token is None,
+ )
+ )
+ manager.safe_point()
+ for index, identifier in enumerate(identifiers):
+ next_key = keys[index + 1] if index + 1 < len(keys) else None
+ pages = _pages(client, history, start_token, {"FindingIdentifier": identifier, **extra})
+ start_token = None
+ for records, token in pages:
+ rows = [normalize_row(_prepare("finding_history", record)) for record in records]
+ if token or next_key:
+ next_state = AwsSecurityHubResumeConfig(
+ updated_after=start_time,
+ cutoff=cutoff,
+ parent_token=parent_token,
+ parent_id=keys[index] if token else next_key,
+ next_token=token,
+ )
+ else:
+ next_state = AwsSecurityHubResumeConfig(
+ updated_after=start_time,
+ cutoff=cutoff,
+ parent_token=next_parent_token,
+ finished=next_parent_token is None,
+ )
+ manager.save_state(next_state)
+ if rows:
+ yield rows
+ manager.safe_point()
+ if next_parent_token is None:
+ return
+ parent_token = next_parent_token
+
+
+def _finding_key(identifier: dict[str, str]) -> str:
+ return f"{identifier['ProductArn']}\n{identifier['Id']}"
+
+
+def _items(endpoint: SecurityHubEndpoint, result: dict[str, Any]) -> list[dict[str, Any]]:
+ items = result.get(endpoint.result_key) or []
+ if isinstance(items, dict):
+ # GetInsightResults returns one object with the grouped counts nested inside it.
+ group_by = items.get("GroupByAttribute")
+ return [{"GroupByAttribute": group_by, **value} for value in items.get("ResultValues") or []]
+ return items
+
+
def _pages(
client: AwsSecurityHubClient, endpoint: SecurityHubEndpoint, token: str | None, extra: dict[str, Any]
) -> Iterator[tuple[list[dict[str, Any]], str | None]]:
while True:
- payload: dict[str, Any] = {"MaxResults": endpoint.page_size, **dict(endpoint.params), **extra}
+ payload: dict[str, Any] = {**dict(endpoint.params), **extra}
+ if endpoint.page_size:
+ payload["MaxResults"] = endpoint.page_size
if token:
payload["NextToken"] = token
result = client.request(endpoint.operation, payload)
token = result.get("NextToken") or None
- yield result.get(endpoint.result_key) or [], token
+ yield _items(endpoint, result), token
if token is None:
return
def _probe(client: AwsSecurityHubClient, endpoint: SecurityHubEndpoint) -> None:
path_params: dict[str, Any] = {}
+ if endpoint.operation == "GetFindingHistory":
+ findings = client.request("GetFindings", {"MaxResults": 1}).get("Findings") or []
+ if not findings:
+ return
+ path_params["FindingIdentifier"] = {"Id": findings[0]["Id"], "ProductArn": findings[0]["ProductArn"]}
if endpoint.parent:
parent_name, parent_field = endpoint.parent
parent = ENDPOINTS[parent_name]
@@ -236,7 +370,7 @@ def _probe(client: AwsSecurityHubClient, endpoint: SecurityHubEndpoint) -> None:
if not items:
return
path_params[parent_field] = items[0][parent_field]
- client.request(endpoint.operation, {"MaxResults": 1, **path_params})
+ client.request(endpoint.operation, {**({"MaxResults": 1} if endpoint.page_size else {}), **path_params})
def validate_credentials(
@@ -301,7 +435,9 @@ def aws_security_hub_source(
config, endpoint, api_version, manager, should_use_incremental_field, db_incremental_field_last_value
),
primary_keys=list(endpoint_config.primary_keys),
- sort_mode="asc" if endpoint == "findings" else None,
+ # Finding history arrives grouped by finding, and GetFindingsV2 is not sorted, so the pipeline
+ # takes the highest value of the run as the watermark when the run ends.
+ sort_mode="asc" if endpoint == "findings" else "desc" if endpoint in INCREMENTAL_FIELDS else None,
partition_keys=[endpoint_config.partition_key] if endpoint_config.partition_key else None,
partition_mode="datetime" if endpoint_config.partition_key else None,
partition_format="month" if endpoint_config.partition_key else None,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_security_hub/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_security_hub/canonical_descriptions.py
index ced2023b524e..a17e21187603 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_security_hub/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_security_hub/canonical_descriptions.py
@@ -140,4 +140,60 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"impact": "Whether resolving the remediation target Resolves, Reduces, or leaves Unchanged the exposure finding.",
},
},
+ "insight_results": {
+ "description": "Finding counts for each group of a custom insight.",
+ "docs_url": "https://docs.aws.amazon.com/securityhub/1.0/APIReference/API_InsightResults.html",
+ "columns": {
+ "insight_arn": "The ARN of the insight, which matches insight_arn on insights.",
+ "group_by_attribute": "The attribute that the insight groups findings by.",
+ "group_by_attribute_value": "The value of the grouping attribute for this group.",
+ "count": "The number of findings in this group.",
+ },
+ },
+ "finding_history": {
+ "description": "Events that changed a Security Hub CSPM finding. History excludes changes to top-level timestamp fields.",
+ "docs_url": "https://docs.aws.amazon.com/securityhub/1.0/APIReference/API_FindingHistoryRecord.html",
+ "columns": {
+ "product_arn": "The ARN of the product that generated the finding, which matches product_arn on findings.",
+ "finding_id": "The finding identifier, which matches id on findings.",
+ "update_time": "When Security Hub processed the updated finding record.",
+ "finding_created": "Whether the event created the finding.",
+ "update_source": "The type and identity of the source that changed the finding.",
+ "updates": "The fields that changed, with their old and new values.",
+ },
+ },
+ "findings_v2": {
+ "description": "Security Hub findings in the Open Cybersecurity Schema Framework (OCSF) format.",
+ "docs_url": "https://docs.aws.amazon.com/securityhub/1.0/APIReference/API_GetFindingsV2.html",
+ "columns": {
+ "product_uid": "The product that generated the finding, from metadata.product.uid.",
+ "finding_uid": "The unique identifier of the finding, from finding_info.uid.",
+ "created_time": "When the finding was created, from finding_info.created_time_dt.",
+ "modified_time": "When the finding was last modified, from finding_info.modified_time_dt.",
+ "finding_info": "The title, description, types, and timestamps of the finding.",
+ "metadata": "The OCSF version and the product that reported the finding.",
+ "severity": "The severity of the finding.",
+ "status": "The workflow status of the finding.",
+ "resources": "The resources associated with the finding.",
+ "cloud": "The cloud account and region of the finding.",
+ },
+ },
+ "resources_v2": {
+ "description": "Resources that Security Hub tracks, with configuration, tags, and a summary of their findings.",
+ "docs_url": "https://docs.aws.amazon.com/securityhub/1.0/APIReference/API_ResourceResult.html",
+ "columns": {
+ "resource_guid": "The global identifier of the resource.",
+ "resource_id": "The unique identifier of the resource.",
+ "account_id": "The AWS account that recorded the resource data.",
+ "region": "The AWS region that recorded the resource data.",
+ "resource_type": "The type of the resource.",
+ "resource_name": "The name of the resource.",
+ "resource_provider": "The cloud provider of the resource, such as AWS or Azure.",
+ "resource_creation_time_dt": "When the resource was created.",
+ "resource_detail_capture_time_dt": "When Security Hub captured the resource details.",
+ "resource_config": "The configuration of the resource.",
+ "resource_tags": "The tags on the resource.",
+ "findings_summary": "A summary of the findings for the resource.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_security_hub/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_security_hub/settings.py
index 0ea1d713eb13..df16ff4474ef 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_security_hub/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_security_hub/settings.py
@@ -14,7 +14,8 @@ class SecurityHubEndpoint:
result_key: str
primary_keys: tuple[str, ...]
partition_key: str | None = None
- page_size: int = 100
+ # None for operations that return everything in one response and take no MaxResults.
+ page_size: int | None = 100
params: tuple[tuple[str, Any], ...] = ()
# Endpoint whose rows supply this endpoint's required path parameter, with that row field.
parent: tuple[str, str] | None = None
@@ -66,6 +67,28 @@ ENDPOINTS: dict[str, SecurityHubEndpoint] = {
primary_keys=("target_uid", "metadata_uid"),
parent=("remediations", "TargetUid"),
),
+ "insight_results": SecurityHubEndpoint(
+ operation="GetInsightResults",
+ result_key="InsightResults",
+ primary_keys=("insight_arn", "group_by_attribute_value"),
+ page_size=None,
+ parent=("insights", "InsightArn"),
+ ),
+ # Fans out over every finding. Each request takes the finding's Id and ProductArn.
+ "finding_history": SecurityHubEndpoint(
+ operation="GetFindingHistory",
+ result_key="Records",
+ primary_keys=("product_arn", "finding_id", "update_time"),
+ partition_key="update_time",
+ ),
+ "findings_v2": SecurityHubEndpoint(
+ operation="GetFindingsV2", result_key="Findings", primary_keys=("product_uid", "finding_uid")
+ ),
+ "resources_v2": SecurityHubEndpoint(
+ operation="GetResourcesV2",
+ result_key="Resources",
+ primary_keys=("account_id", "region", "resource_type", "resource_id"),
+ ),
}
# The pinned botocore model predates these operations. Each one is a POST with only JSON body members.
@@ -74,7 +97,11 @@ UNMODELED_OPERATION_PATHS = {
"ListExposuresByRemediationV2": "/ListExposuresByRemediationV2",
}
-INCREMENTAL_FIELDS = {"findings": [incremental_field("updated_at")]}
+INCREMENTAL_FIELDS = {
+ "findings": [incremental_field("updated_at")],
+ "finding_history": [incremental_field("update_time")],
+ "findings_v2": [incremental_field("modified_time")],
+}
ENDPOINT_DESCRIPTIONS = {
"findings": "Security findings with severity, resources, compliance status, and workflow status.",
"standards": "Available security standards and their descriptions.",
@@ -86,6 +113,10 @@ ENDPOINT_DESCRIPTIONS = {
"members": "Member accounts of the Security Hub administrator account and their relationship status.",
"remediations": "Remediation targets for exposure findings, sorted by priority, with their expected outcome and affected resource.",
"remediation_exposures": "Exposure findings linked to each remediation target, with severity before and after the fix.",
+ "insight_results": "Finding counts for each group of every custom insight.",
+ "finding_history": "Changes to each finding over time, with the fields that changed and who changed them.",
+ "findings_v2": "Security Hub findings in the Open Cybersecurity Schema Framework (OCSF) format.",
+ "resources_v2": "Resources that Security Hub tracks, with their configuration, tags, and finding summaries.",
}
ACCESS_DENIED_CODES = frozenset({"AccessDenied", "AccessDeniedException", "InvalidAccessException"})
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_security_hub/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_security_hub/source.py
index 3c57bd68ca1c..0aeb0da43d76 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_security_hub/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_security_hub/source.py
@@ -82,7 +82,11 @@ class AwsSecurityHubSource(ResumableSource[AwsSecurityHubSourceConfig, AwsSecuri
api_version: str | None = None,
) -> list[SourceSchema]:
return build_endpoint_schemas(
- ENDPOINTS, INCREMENTAL_FIELDS, names, merge_only={"findings"}, descriptions=ENDPOINT_DESCRIPTIONS
+ ENDPOINTS,
+ INCREMENTAL_FIELDS,
+ names,
+ merge_only={"findings", "finding_history", "findings_v2"},
+ descriptions=ENDPOINT_DESCRIPTIONS,
)
def validate_credentials(
@@ -127,9 +131,9 @@ class AwsSecurityHubSource(ResumableSource[AwsSecurityHubSourceConfig, AwsSecuri
name=ExternalDataSourceType.AWSSECURITYHUB,
category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,
label="AWS Security Hub",
- caption="""Sync Security Hub CSPM findings, standards, controls, product integrations, member accounts, custom insights, and exposure remediations from the configured AWS region.
+ caption="""Sync Security Hub CSPM findings, finding history, standards, controls, product integrations, member accounts, custom insights, insight results, and exposure remediations from the configured AWS region. You can also sync Security Hub findings and resources in the OCSF format.
-Grant `securityhub:GetFindings`, `securityhub:DescribeStandards`, `securityhub:GetEnabledStandards`, `securityhub:GetInsights`, `securityhub:DescribeStandardsControls`, `securityhub:ListSecurityControlDefinitions`, `securityhub:DescribeProducts`, `securityhub:ListMembers`, `securityhub:GetRemediationsV2`, and `securityhub:ListExposuresByRemediationV2` to the IAM user or role.
+Grant `securityhub:GetFindings`, `securityhub:GetFindingHistory`, `securityhub:DescribeStandards`, `securityhub:GetEnabledStandards`, `securityhub:GetInsights`, `securityhub:GetInsightResults`, `securityhub:DescribeStandardsControls`, `securityhub:ListSecurityControlDefinitions`, `securityhub:DescribeProducts`, `securityhub:ListMembers`, `securityhub:GetResourcesV2`, `securityhub:GetRemediationsV2`, and `securityhub:ListExposuresByRemediationV2` to the IAM user or role.
Enter its access key ID and secret access key. Add a session token for temporary credentials.
Enable Security Hub CSPM in the selected region.
If you select an aggregation region, AWS can also return findings from linked regions.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_security_hub/tests/test_aws_security_hub.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_security_hub/tests/test_aws_security_hub.py
index 0768407f66c5..9d186ec71dd5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_security_hub/tests/test_aws_security_hub.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_security_hub/tests/test_aws_security_hub.py
@@ -4,6 +4,7 @@ from collections.abc import Iterable, Iterator
from typing import Any, cast
import pytest
+import time_machine
from unittest.mock import Mock, patch
import requests
@@ -37,12 +38,19 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
VERSION = "2018-10-26"
SUBSCRIPTION_A = "arn:aws:securityhub:eu-west-1:111111111111:subscription/example-standard/v/1.0.0"
SUBSCRIPTION_B = "arn:aws:securityhub:eu-west-1:111111111111:subscription/other-standard/v/2.0.0"
+INSIGHT_A = "arn:aws:securityhub:eu-west-1:111111111111:insight/111111111111/custom/example-a"
+INSIGHT_B = "arn:aws:securityhub:eu-west-1:111111111111:insight/111111111111/custom/example-b"
+PRODUCT = "arn:aws:securityhub:eu-west-1::product/example/product-one"
def controls_path(subscription_arn: str) -> str:
return "/standards/controls/" + subscription_arn.replace(":", "%3A")
+def insight_results_path(insight_arn: str) -> str:
+ return "/insights/results/" + insight_arn.replace(":", "%3A")
+
+
def config(region: str = "eu-west-1", token: str | None = "fake-session-token") -> AwsSecurityHubSourceConfig:
return AwsSecurityHubSourceConfig(
aws_access_key_id="AKIAEXAMPLE",
@@ -310,6 +318,19 @@ ONE_SUBSCRIPTION = ScriptedResponse(json={"StandardsSubscriptions": [{"Standards
),
("standards_controls", [DENIED], "GetEnabledStandards", ["/standards/get"]),
("standards_controls", [ScriptedResponse(json={"StandardsSubscriptions": []})], None, ["/standards/get"]),
+ (
+ "insight_results",
+ [ScriptedResponse(json={"Insights": [{"InsightArn": INSIGHT_A}]}), DENIED],
+ "GetInsightResults",
+ ["/insights/get", insight_results_path(INSIGHT_A)],
+ ),
+ (
+ "finding_history",
+ [ScriptedResponse(json={"Findings": [{"Id": "finding-a", "ProductArn": PRODUCT}]}), DENIED],
+ "GetFindingHistory",
+ ["/findings", "/findingHistory/get"],
+ ),
+ ("finding_history", [ScriptedResponse(json={"Findings": []})], None, ["/findings"]),
],
)
def test_permission_probe_names_missing_permission(
@@ -434,7 +455,9 @@ def test_standards_controls_resume_skips_finished_subscriptions(next_token: str
assert result.requests[-1].param("NextToken") == next_token
[email protected]("endpoint", ["security_control_definitions", "products", "members"])
+ "endpoint", ["security_control_definitions", "products", "members", "findings_v2", "resources_v2"]
+)
def test_page_size_stays_within_vendor_limit(endpoint: str) -> None:
endpoint_config = ENDPOINTS[endpoint]
result = driver().run(endpoint, [ScriptedResponse(json={endpoint_config.result_key: []})])
@@ -444,7 +467,8 @@ def test_page_size_stays_within_vendor_limit(endpoint: str) -> None:
input_shape = model.operation_model(endpoint_config.operation).input_shape
assert input_shape is not None
limit = input_shape.members["MaxResults"].metadata["max"]
- assert 0 < int(request.param("MaxResults") or 0) <= limit
+ sent = request.param("MaxResults") or (request.json() if request.body else {}).get("MaxResults")
+ assert 0 < int(sent or 0) <= limit
if endpoint == "members":
assert request.param("OnlyAssociated") == "false"
@@ -510,3 +534,188 @@ def test_validation_network_error(session: Mock) -> None:
session.request.side_effect = requests.Timeout()
assert validate_credentials(config(), VERSION) == (False, "Could not reach AWS Security Hub. Try again.")
session.close.assert_called_once()
+
+
+def test_insight_results_fan_out_over_insights_without_page_size() -> None:
+ script = route(
+ {
+ "/insights/get": [
+ ScriptedResponse(json={"Insights": [{"InsightArn": INSIGHT_B}, {"InsightArn": INSIGHT_A}]})
+ ],
+ insight_results_path(INSIGHT_A): [
+ ScriptedResponse(
+ json={
+ "InsightResults": {
+ "InsightArn": INSIGHT_A,
+ "GroupByAttribute": "ResourceId",
+ "ResultValues": [
+ {"GroupByAttributeValue": "resource-1", "Count": 3},
+ {"GroupByAttributeValue": "resource-2", "Count": 1},
+ ],
+ }
+ }
+ )
+ ],
+ insight_results_path(INSIGHT_B): [
+ ScriptedResponse(
+ json={"InsightResults": {"InsightArn": INSIGHT_B, "GroupByAttribute": "Type", "ResultValues": []}}
+ )
+ ],
+ }
+ )
+ result = driver().run("insight_results", script)
+ assert result.raised is None
+ assert result.rows == [
+ {
+ "insight_arn": INSIGHT_A,
+ "group_by_attribute": "ResourceId",
+ "group_by_attribute_value": "resource-1",
+ "count": 3,
+ },
+ {
+ "insight_arn": INSIGHT_A,
+ "group_by_attribute": "ResourceId",
+ "group_by_attribute_value": "resource-2",
+ "count": 1,
+ },
+ ]
+ result_requests = [request for request in result.requests if request.path.startswith("/insights/results/")]
+ assert [request.path for request in result_requests] == [
+ insight_results_path(INSIGHT_A),
+ insight_results_path(INSIGHT_B),
+ ]
+ assert all(request.method == "GET" and request.query == {} for request in result_requests)
+
+
+def history_record(finding_id: str) -> dict[str, Any]:
+ return {
+ "FindingIdentifier": {"Id": finding_id, "ProductArn": PRODUCT},
+ "UpdateTime": "2025-03-02T00:00:00Z",
+ "FindingCreated": False,
+ "Updates": [{"UpdatedField": "Workflow"}],
+ }
+
+
+ "watermark,resume_state,expected_start,expected_end",
+ [
+ (None, None, None, None),
+ ("2025-03-01T00:00:00Z", None, "2025-03-01T00:00:00Z", "2025-06-01T00:00:00Z"),
+ (
+ "2025-04-01T00:00:00Z",
+ AwsSecurityHubResumeConfig(
+ updated_after="2025-03-01T00:00:00+00:00",
+ parent_token="page-2",
+ parent_id=f"{PRODUCT}\nfinding-b",
+ next_token="history-2",
+ cutoff="2025-04-15T00:00:00+00:00",
+ ),
+ "2025-03-01T00:00:00Z",
+ "2025-04-15T00:00:00Z",
+ ),
+ ],
+)
+def test_finding_history_walks_findings_and_history_pages(
+ watermark: str | None,
+ resume_state: AwsSecurityHubResumeConfig | None,
+ expected_start: str | None,
+ expected_end: str | None,
+) -> None:
+ first_page = ScriptedResponse(
+ json={"Findings": [{"Id": "finding-a", "ProductArn": PRODUCT}], "NextToken": "page-2"}
+ )
+ second_page = ScriptedResponse(
+ json={"Findings": [{"Id": "finding-b", "ProductArn": PRODUCT}, {"Id": "no-product"}]}
+ )
+ finding_b_history = [
+ ScriptedResponse(json={"Records": [], "NextToken": "history-2"}),
+ ScriptedResponse(json={"Records": [history_record("finding-b")]}),
+ ]
+ finding_a_history = [ScriptedResponse(json={"Records": [history_record("finding-a")]})]
+ script = route(
+ {
+ "/findings": [second_page] if resume_state else [first_page, second_page],
+ "/findingHistory/get": finding_b_history[1:] if resume_state else finding_a_history + finding_b_history,
+ }
+ )
+ with time_machine.travel(dt.datetime(2025, 6, 1, tzinfo=dt.UTC), tick=False):
+ result = driver().run(
+ "finding_history",
+ script,
+ resume_state=resume_state,
+ should_use_incremental_field=watermark is not None,
+ db_incremental_field_last_value=watermark,
+ )
+ assert result.raised is None
+ walked = ["finding-b"] if resume_state else ["finding-a", "finding-b"]
+ assert [(row["finding_id"], row["product_arn"]) for row in result.rows] == [
+ (finding, PRODUCT) for finding in walked
+ ]
+ assert result.rows[0]["update_time"] == dt.datetime(2025, 3, 2, tzinfo=dt.UTC)
+ assert "finding_identifier" not in result.rows[0]
+ parent_requests = [request.json() for request in result.requests if request.path == "/findings"]
+ assert [payload.get("NextToken") for payload in parent_requests] == ([] if resume_state else [None]) + ["page-2"]
+ assert all(payload["SortCriteria"] == [{"Field": "CreatedAt", "SortOrder": "asc"}] for payload in parent_requests)
+ history_requests = [request.json() for request in result.requests if request.path == "/findingHistory/get"]
+ expected_history = (
+ [("finding-b", "history-2")]
+ if resume_state
+ else [
+ ("finding-a", None),
+ ("finding-b", None),
+ ("finding-b", "history-2"),
+ ]
+ )
+ assert [(payload["FindingIdentifier"]["Id"], payload.get("NextToken")) for payload in history_requests] == (
+ expected_history
+ )
+ assert {(payload.get("StartTime"), payload.get("EndTime")) for payload in history_requests} == {
+ (expected_start, expected_end)
+ }
+ states = [(state.parent_token, state.parent_id, state.next_token, state.finished) for state in result.saved_states]
+ assert states == (
+ [] if resume_state else [("page-2", None, None, False), ("page-2", f"{PRODUCT}\nfinding-b", "history-2", False)]
+ ) + [(None, None, None, True)]
+ assert result.response is not None and result.response.sort_mode == "desc"
+
+
[email protected]("watermark", [None, "2025-05-01T00:00:00Z"])
+def test_findings_v2_filters_on_modified_time_and_lifts_identity(watermark: str | None) -> None:
+ finding = {
+ "finding_info": {
+ "uid": "example-finding",
+ "created_time_dt": "2025-01-01T00:00:00Z",
+ "modified_time_dt": "2025-05-02T00:00:00Z",
+ },
+ "metadata": {"product": {"uid": PRODUCT}},
+ "severity": "High",
+ }
+ with time_machine.travel(dt.datetime(2025, 6, 1, tzinfo=dt.UTC), tick=False):
+ result = driver().run(
+ "findings_v2",
+ [ScriptedResponse(json={"Findings": [finding]})],
+ should_use_incremental_field=watermark is not None,
+ db_incremental_field_last_value=watermark,
+ )
+ assert result.raised is None
+ payload = result.requests[0].json()
+ if watermark:
+ assert payload["Filters"] == {
+ "CompositeFilters": [
+ {
+ "DateFilters": [
+ {
+ "FieldName": "finding_info.modified_time_dt",
+ "Filter": {"End": "2025-06-01T00:00:00+00:00", "Start": "2025-05-01T00:00:00+00:00"},
+ }
+ ]
+ }
+ ]
+ }
+ else:
+ assert "Filters" not in payload
+ assert "SortCriteria" not in payload
+ row = result.rows[0]
+ assert (row["product_uid"], row["finding_uid"]) == (PRODUCT, "example-finding")
+ assert row["modified_time"] == dt.datetime(2025, 5, 2, tzinfo=dt.UTC)
+ assert row["finding_info"] == finding["finding_info"]